1 /* Expand the basic unary and binary arithmetic operations, for GNU compiler.
2 Copyright (C) 1987-2017 Free Software Foundation, Inc.
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 3, or (at your option) any later
11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
23 #include "coretypes.h"
35 #include "diagnostic-core.h"
37 /* Include insn-config.h before expr.h so that HAVE_conditional_move
38 is properly defined. */
39 #include "stor-layout.h"
44 #include "optabs-tree.h"
47 static void prepare_float_lib_cmp (rtx
, rtx
, enum rtx_code
, rtx
*,
49 static rtx
expand_unop_direct (machine_mode
, optab
, rtx
, rtx
, int);
50 static void emit_libcall_block_1 (rtx_insn
*, rtx
, rtx
, rtx
, bool);
52 /* Debug facility for use in GDB. */
53 void debug_optab_libfuncs (void);
55 /* Add a REG_EQUAL note to the last insn in INSNS. TARGET is being set to
56 the result of operation CODE applied to OP0 (and OP1 if it is a binary
59 If the last insn does not set TARGET, don't do anything, but return 1.
61 If the last insn or a previous insn sets TARGET and TARGET is one of OP0
62 or OP1, don't add the REG_EQUAL note but return 0. Our caller can then
63 try again, ensuring that TARGET is not one of the operands. */
66 add_equal_note (rtx_insn
*insns
, rtx target
, enum rtx_code code
, rtx op0
, rtx op1
)
72 gcc_assert (insns
&& INSN_P (insns
) && NEXT_INSN (insns
));
74 if (GET_RTX_CLASS (code
) != RTX_COMM_ARITH
75 && GET_RTX_CLASS (code
) != RTX_BIN_ARITH
76 && GET_RTX_CLASS (code
) != RTX_COMM_COMPARE
77 && GET_RTX_CLASS (code
) != RTX_COMPARE
78 && GET_RTX_CLASS (code
) != RTX_UNARY
)
81 if (GET_CODE (target
) == ZERO_EXTRACT
)
84 for (last_insn
= insns
;
85 NEXT_INSN (last_insn
) != NULL_RTX
;
86 last_insn
= NEXT_INSN (last_insn
))
89 /* If TARGET is in OP0 or OP1, punt. We'd end up with a note referencing
90 a value changing in the insn, so the note would be invalid for CSE. */
91 if (reg_overlap_mentioned_p (target
, op0
)
92 || (op1
&& reg_overlap_mentioned_p (target
, op1
)))
95 && (rtx_equal_p (target
, op0
)
96 || (op1
&& rtx_equal_p (target
, op1
))))
98 /* For MEM target, with MEM = MEM op X, prefer no REG_EQUAL note
99 over expanding it as temp = MEM op X, MEM = temp. If the target
100 supports MEM = MEM op X instructions, it is sometimes too hard
101 to reconstruct that form later, especially if X is also a memory,
102 and due to multiple occurrences of addresses the address might
103 be forced into register unnecessarily.
104 Note that not emitting the REG_EQUIV note might inhibit
105 CSE in some cases. */
106 set
= single_set (last_insn
);
108 && GET_CODE (SET_SRC (set
)) == code
109 && MEM_P (SET_DEST (set
))
110 && (rtx_equal_p (SET_DEST (set
), XEXP (SET_SRC (set
), 0))
111 || (op1
&& rtx_equal_p (SET_DEST (set
),
112 XEXP (SET_SRC (set
), 1)))))
118 set
= set_for_reg_notes (last_insn
);
122 if (! rtx_equal_p (SET_DEST (set
), target
)
123 /* For a STRICT_LOW_PART, the REG_NOTE applies to what is inside it. */
124 && (GET_CODE (SET_DEST (set
)) != STRICT_LOW_PART
125 || ! rtx_equal_p (XEXP (SET_DEST (set
), 0), target
)))
128 if (GET_RTX_CLASS (code
) == RTX_UNARY
)
138 if (GET_MODE (op0
) != VOIDmode
&& GET_MODE (target
) != GET_MODE (op0
))
140 note
= gen_rtx_fmt_e (code
, GET_MODE (op0
), copy_rtx (op0
));
141 if (GET_MODE_UNIT_SIZE (GET_MODE (op0
))
142 > GET_MODE_UNIT_SIZE (GET_MODE (target
)))
143 note
= simplify_gen_unary (TRUNCATE
, GET_MODE (target
),
144 note
, GET_MODE (op0
));
146 note
= simplify_gen_unary (ZERO_EXTEND
, GET_MODE (target
),
147 note
, GET_MODE (op0
));
152 note
= gen_rtx_fmt_e (code
, GET_MODE (target
), copy_rtx (op0
));
156 note
= gen_rtx_fmt_ee (code
, GET_MODE (target
), copy_rtx (op0
), copy_rtx (op1
));
158 set_unique_reg_note (last_insn
, REG_EQUAL
, note
);
163 /* Given two input operands, OP0 and OP1, determine what the correct from_mode
164 for a widening operation would be. In most cases this would be OP0, but if
165 that's a constant it'll be VOIDmode, which isn't useful. */
168 widened_mode (machine_mode to_mode
, rtx op0
, rtx op1
)
170 machine_mode m0
= GET_MODE (op0
);
171 machine_mode m1
= GET_MODE (op1
);
174 if (m0
== VOIDmode
&& m1
== VOIDmode
)
176 else if (m0
== VOIDmode
|| GET_MODE_UNIT_SIZE (m0
) < GET_MODE_UNIT_SIZE (m1
))
181 if (GET_MODE_UNIT_SIZE (result
) > GET_MODE_UNIT_SIZE (to_mode
))
187 /* Widen OP to MODE and return the rtx for the widened operand. UNSIGNEDP
188 says whether OP is signed or unsigned. NO_EXTEND is nonzero if we need
189 not actually do a sign-extend or zero-extend, but can leave the
190 higher-order bits of the result rtx undefined, for example, in the case
191 of logical operations, but not right shifts. */
194 widen_operand (rtx op
, machine_mode mode
, machine_mode oldmode
,
195 int unsignedp
, int no_extend
)
198 scalar_int_mode int_mode
;
200 /* If we don't have to extend and this is a constant, return it. */
201 if (no_extend
&& GET_MODE (op
) == VOIDmode
)
204 /* If we must extend do so. If OP is a SUBREG for a promoted object, also
205 extend since it will be more efficient to do so unless the signedness of
206 a promoted object differs from our extension. */
208 || !is_a
<scalar_int_mode
> (mode
, &int_mode
)
209 || (GET_CODE (op
) == SUBREG
&& SUBREG_PROMOTED_VAR_P (op
)
210 && SUBREG_CHECK_PROMOTED_SIGN (op
, unsignedp
)))
211 return convert_modes (mode
, oldmode
, op
, unsignedp
);
213 /* If MODE is no wider than a single word, we return a lowpart or paradoxical
215 if (GET_MODE_SIZE (int_mode
) <= UNITS_PER_WORD
)
216 return gen_lowpart (int_mode
, force_reg (GET_MODE (op
), op
));
218 /* Otherwise, get an object of MODE, clobber it, and set the low-order
221 result
= gen_reg_rtx (int_mode
);
222 emit_clobber (result
);
223 emit_move_insn (gen_lowpart (GET_MODE (op
), result
), op
);
227 /* Expand vector widening operations.
229 There are two different classes of operations handled here:
230 1) Operations whose result is wider than all the arguments to the operation.
231 Examples: VEC_UNPACK_HI/LO_EXPR, VEC_WIDEN_MULT_HI/LO_EXPR
232 In this case OP0 and optionally OP1 would be initialized,
233 but WIDE_OP wouldn't (not relevant for this case).
234 2) Operations whose result is of the same size as the last argument to the
235 operation, but wider than all the other arguments to the operation.
236 Examples: WIDEN_SUM_EXPR, VEC_DOT_PROD_EXPR.
237 In the case WIDE_OP, OP0 and optionally OP1 would be initialized.
239 E.g, when called to expand the following operations, this is how
240 the arguments will be initialized:
242 widening-sum 2 oprnd0 - oprnd1
243 widening-dot-product 3 oprnd0 oprnd1 oprnd2
244 widening-mult 2 oprnd0 oprnd1 -
245 type-promotion (vec-unpack) 1 oprnd0 - - */
248 expand_widen_pattern_expr (sepops ops
, rtx op0
, rtx op1
, rtx wide_op
,
249 rtx target
, int unsignedp
)
251 struct expand_operand eops
[4];
252 tree oprnd0
, oprnd1
, oprnd2
;
253 machine_mode wmode
= VOIDmode
, tmode0
, tmode1
= VOIDmode
;
254 optab widen_pattern_optab
;
255 enum insn_code icode
;
256 int nops
= TREE_CODE_LENGTH (ops
->code
);
260 tmode0
= TYPE_MODE (TREE_TYPE (oprnd0
));
261 widen_pattern_optab
=
262 optab_for_tree_code (ops
->code
, TREE_TYPE (oprnd0
), optab_default
);
263 if (ops
->code
== WIDEN_MULT_PLUS_EXPR
264 || ops
->code
== WIDEN_MULT_MINUS_EXPR
)
265 icode
= find_widening_optab_handler (widen_pattern_optab
,
266 TYPE_MODE (TREE_TYPE (ops
->op2
)),
269 icode
= optab_handler (widen_pattern_optab
, tmode0
);
270 gcc_assert (icode
!= CODE_FOR_nothing
);
275 tmode1
= TYPE_MODE (TREE_TYPE (oprnd1
));
278 /* The last operand is of a wider mode than the rest of the operands. */
283 gcc_assert (tmode1
== tmode0
);
286 wmode
= TYPE_MODE (TREE_TYPE (oprnd2
));
290 create_output_operand (&eops
[op
++], target
, TYPE_MODE (ops
->type
));
291 create_convert_operand_from (&eops
[op
++], op0
, tmode0
, unsignedp
);
293 create_convert_operand_from (&eops
[op
++], op1
, tmode1
, unsignedp
);
295 create_convert_operand_from (&eops
[op
++], wide_op
, wmode
, unsignedp
);
296 expand_insn (icode
, op
, eops
);
297 return eops
[0].value
;
300 /* Generate code to perform an operation specified by TERNARY_OPTAB
301 on operands OP0, OP1 and OP2, with result having machine-mode MODE.
303 UNSIGNEDP is for the case where we have to widen the operands
304 to perform the operation. It says to use zero-extension.
306 If TARGET is nonzero, the value
307 is generated there, if it is convenient to do so.
308 In all cases an rtx is returned for the locus of the value;
309 this may or may not be TARGET. */
312 expand_ternary_op (machine_mode mode
, optab ternary_optab
, rtx op0
,
313 rtx op1
, rtx op2
, rtx target
, int unsignedp
)
315 struct expand_operand ops
[4];
316 enum insn_code icode
= optab_handler (ternary_optab
, mode
);
318 gcc_assert (optab_handler (ternary_optab
, mode
) != CODE_FOR_nothing
);
320 create_output_operand (&ops
[0], target
, mode
);
321 create_convert_operand_from (&ops
[1], op0
, mode
, unsignedp
);
322 create_convert_operand_from (&ops
[2], op1
, mode
, unsignedp
);
323 create_convert_operand_from (&ops
[3], op2
, mode
, unsignedp
);
324 expand_insn (icode
, 4, ops
);
329 /* Like expand_binop, but return a constant rtx if the result can be
330 calculated at compile time. The arguments and return value are
331 otherwise the same as for expand_binop. */
334 simplify_expand_binop (machine_mode mode
, optab binoptab
,
335 rtx op0
, rtx op1
, rtx target
, int unsignedp
,
336 enum optab_methods methods
)
338 if (CONSTANT_P (op0
) && CONSTANT_P (op1
))
340 rtx x
= simplify_binary_operation (optab_to_code (binoptab
),
346 return expand_binop (mode
, binoptab
, op0
, op1
, target
, unsignedp
, methods
);
349 /* Like simplify_expand_binop, but always put the result in TARGET.
350 Return true if the expansion succeeded. */
353 force_expand_binop (machine_mode mode
, optab binoptab
,
354 rtx op0
, rtx op1
, rtx target
, int unsignedp
,
355 enum optab_methods methods
)
357 rtx x
= simplify_expand_binop (mode
, binoptab
, op0
, op1
,
358 target
, unsignedp
, methods
);
362 emit_move_insn (target
, x
);
366 /* Create a new vector value in VMODE with all elements set to OP. The
367 mode of OP must be the element mode of VMODE. If OP is a constant,
368 then the return value will be a constant. */
371 expand_vector_broadcast (machine_mode vmode
, rtx op
)
373 enum insn_code icode
;
378 gcc_checking_assert (VECTOR_MODE_P (vmode
));
380 if (valid_for_const_vec_duplicate_p (vmode
, op
))
381 return gen_const_vec_duplicate (vmode
, op
);
383 /* ??? If the target doesn't have a vec_init, then we have no easy way
384 of performing this operation. Most of this sort of generic support
385 is hidden away in the vector lowering support in gimple. */
386 icode
= convert_optab_handler (vec_init_optab
, vmode
,
387 GET_MODE_INNER (vmode
));
388 if (icode
== CODE_FOR_nothing
)
391 n
= GET_MODE_NUNITS (vmode
);
392 vec
= rtvec_alloc (n
);
393 for (i
= 0; i
< n
; ++i
)
394 RTVEC_ELT (vec
, i
) = op
;
395 ret
= gen_reg_rtx (vmode
);
396 emit_insn (GEN_FCN (icode
) (ret
, gen_rtx_PARALLEL (vmode
, vec
)));
401 /* This subroutine of expand_doubleword_shift handles the cases in which
402 the effective shift value is >= BITS_PER_WORD. The arguments and return
403 value are the same as for the parent routine, except that SUPERWORD_OP1
404 is the shift count to use when shifting OUTOF_INPUT into INTO_TARGET.
405 INTO_TARGET may be null if the caller has decided to calculate it. */
408 expand_superword_shift (optab binoptab
, rtx outof_input
, rtx superword_op1
,
409 rtx outof_target
, rtx into_target
,
410 int unsignedp
, enum optab_methods methods
)
412 if (into_target
!= 0)
413 if (!force_expand_binop (word_mode
, binoptab
, outof_input
, superword_op1
,
414 into_target
, unsignedp
, methods
))
417 if (outof_target
!= 0)
419 /* For a signed right shift, we must fill OUTOF_TARGET with copies
420 of the sign bit, otherwise we must fill it with zeros. */
421 if (binoptab
!= ashr_optab
)
422 emit_move_insn (outof_target
, CONST0_RTX (word_mode
));
424 if (!force_expand_binop (word_mode
, binoptab
,
425 outof_input
, GEN_INT (BITS_PER_WORD
- 1),
426 outof_target
, unsignedp
, methods
))
432 /* This subroutine of expand_doubleword_shift handles the cases in which
433 the effective shift value is < BITS_PER_WORD. The arguments and return
434 value are the same as for the parent routine. */
437 expand_subword_shift (scalar_int_mode op1_mode
, optab binoptab
,
438 rtx outof_input
, rtx into_input
, rtx op1
,
439 rtx outof_target
, rtx into_target
,
440 int unsignedp
, enum optab_methods methods
,
441 unsigned HOST_WIDE_INT shift_mask
)
443 optab reverse_unsigned_shift
, unsigned_shift
;
446 reverse_unsigned_shift
= (binoptab
== ashl_optab
? lshr_optab
: ashl_optab
);
447 unsigned_shift
= (binoptab
== ashl_optab
? ashl_optab
: lshr_optab
);
449 /* The low OP1 bits of INTO_TARGET come from the high bits of OUTOF_INPUT.
450 We therefore need to shift OUTOF_INPUT by (BITS_PER_WORD - OP1) bits in
451 the opposite direction to BINOPTAB. */
452 if (CONSTANT_P (op1
) || shift_mask
>= BITS_PER_WORD
)
454 carries
= outof_input
;
455 tmp
= immed_wide_int_const (wi::shwi (BITS_PER_WORD
,
456 op1_mode
), op1_mode
);
457 tmp
= simplify_expand_binop (op1_mode
, sub_optab
, tmp
, op1
,
462 /* We must avoid shifting by BITS_PER_WORD bits since that is either
463 the same as a zero shift (if shift_mask == BITS_PER_WORD - 1) or
464 has unknown behavior. Do a single shift first, then shift by the
465 remainder. It's OK to use ~OP1 as the remainder if shift counts
466 are truncated to the mode size. */
467 carries
= expand_binop (word_mode
, reverse_unsigned_shift
,
468 outof_input
, const1_rtx
, 0, unsignedp
, methods
);
469 if (shift_mask
== BITS_PER_WORD
- 1)
471 tmp
= immed_wide_int_const
472 (wi::minus_one (GET_MODE_PRECISION (op1_mode
)), op1_mode
);
473 tmp
= simplify_expand_binop (op1_mode
, xor_optab
, op1
, tmp
,
478 tmp
= immed_wide_int_const (wi::shwi (BITS_PER_WORD
- 1,
479 op1_mode
), op1_mode
);
480 tmp
= simplify_expand_binop (op1_mode
, sub_optab
, tmp
, op1
,
484 if (tmp
== 0 || carries
== 0)
486 carries
= expand_binop (word_mode
, reverse_unsigned_shift
,
487 carries
, tmp
, 0, unsignedp
, methods
);
491 /* Shift INTO_INPUT logically by OP1. This is the last use of INTO_INPUT
492 so the result can go directly into INTO_TARGET if convenient. */
493 tmp
= expand_binop (word_mode
, unsigned_shift
, into_input
, op1
,
494 into_target
, unsignedp
, methods
);
498 /* Now OR in the bits carried over from OUTOF_INPUT. */
499 if (!force_expand_binop (word_mode
, ior_optab
, tmp
, carries
,
500 into_target
, unsignedp
, methods
))
503 /* Use a standard word_mode shift for the out-of half. */
504 if (outof_target
!= 0)
505 if (!force_expand_binop (word_mode
, binoptab
, outof_input
, op1
,
506 outof_target
, unsignedp
, methods
))
513 /* Try implementing expand_doubleword_shift using conditional moves.
514 The shift is by < BITS_PER_WORD if (CMP_CODE CMP1 CMP2) is true,
515 otherwise it is by >= BITS_PER_WORD. SUBWORD_OP1 and SUPERWORD_OP1
516 are the shift counts to use in the former and latter case. All other
517 arguments are the same as the parent routine. */
520 expand_doubleword_shift_condmove (scalar_int_mode op1_mode
, optab binoptab
,
521 enum rtx_code cmp_code
, rtx cmp1
, rtx cmp2
,
522 rtx outof_input
, rtx into_input
,
523 rtx subword_op1
, rtx superword_op1
,
524 rtx outof_target
, rtx into_target
,
525 int unsignedp
, enum optab_methods methods
,
526 unsigned HOST_WIDE_INT shift_mask
)
528 rtx outof_superword
, into_superword
;
530 /* Put the superword version of the output into OUTOF_SUPERWORD and
532 outof_superword
= outof_target
!= 0 ? gen_reg_rtx (word_mode
) : 0;
533 if (outof_target
!= 0 && subword_op1
== superword_op1
)
535 /* The value INTO_TARGET >> SUBWORD_OP1, which we later store in
536 OUTOF_TARGET, is the same as the value of INTO_SUPERWORD. */
537 into_superword
= outof_target
;
538 if (!expand_superword_shift (binoptab
, outof_input
, superword_op1
,
539 outof_superword
, 0, unsignedp
, methods
))
544 into_superword
= gen_reg_rtx (word_mode
);
545 if (!expand_superword_shift (binoptab
, outof_input
, superword_op1
,
546 outof_superword
, into_superword
,
551 /* Put the subword version directly in OUTOF_TARGET and INTO_TARGET. */
552 if (!expand_subword_shift (op1_mode
, binoptab
,
553 outof_input
, into_input
, subword_op1
,
554 outof_target
, into_target
,
555 unsignedp
, methods
, shift_mask
))
558 /* Select between them. Do the INTO half first because INTO_SUPERWORD
559 might be the current value of OUTOF_TARGET. */
560 if (!emit_conditional_move (into_target
, cmp_code
, cmp1
, cmp2
, op1_mode
,
561 into_target
, into_superword
, word_mode
, false))
564 if (outof_target
!= 0)
565 if (!emit_conditional_move (outof_target
, cmp_code
, cmp1
, cmp2
, op1_mode
,
566 outof_target
, outof_superword
,
573 /* Expand a doubleword shift (ashl, ashr or lshr) using word-mode shifts.
574 OUTOF_INPUT and INTO_INPUT are the two word-sized halves of the first
575 input operand; the shift moves bits in the direction OUTOF_INPUT->
576 INTO_TARGET. OUTOF_TARGET and INTO_TARGET are the equivalent words
577 of the target. OP1 is the shift count and OP1_MODE is its mode.
578 If OP1 is constant, it will have been truncated as appropriate
579 and is known to be nonzero.
581 If SHIFT_MASK is zero, the result of word shifts is undefined when the
582 shift count is outside the range [0, BITS_PER_WORD). This routine must
583 avoid generating such shifts for OP1s in the range [0, BITS_PER_WORD * 2).
585 If SHIFT_MASK is nonzero, all word-mode shift counts are effectively
586 masked by it and shifts in the range [BITS_PER_WORD, SHIFT_MASK) will
587 fill with zeros or sign bits as appropriate.
589 If SHIFT_MASK is BITS_PER_WORD - 1, this routine will synthesize
590 a doubleword shift whose equivalent mask is BITS_PER_WORD * 2 - 1.
591 Doing this preserves semantics required by SHIFT_COUNT_TRUNCATED.
592 In all other cases, shifts by values outside [0, BITS_PER_UNIT * 2)
595 BINOPTAB, UNSIGNEDP and METHODS are as for expand_binop. This function
596 may not use INTO_INPUT after modifying INTO_TARGET, and similarly for
597 OUTOF_INPUT and OUTOF_TARGET. OUTOF_TARGET can be null if the parent
598 function wants to calculate it itself.
600 Return true if the shift could be successfully synthesized. */
603 expand_doubleword_shift (scalar_int_mode op1_mode
, optab binoptab
,
604 rtx outof_input
, rtx into_input
, rtx op1
,
605 rtx outof_target
, rtx into_target
,
606 int unsignedp
, enum optab_methods methods
,
607 unsigned HOST_WIDE_INT shift_mask
)
609 rtx superword_op1
, tmp
, cmp1
, cmp2
;
610 enum rtx_code cmp_code
;
612 /* See if word-mode shifts by BITS_PER_WORD...BITS_PER_WORD * 2 - 1 will
613 fill the result with sign or zero bits as appropriate. If so, the value
614 of OUTOF_TARGET will always be (SHIFT OUTOF_INPUT OP1). Recursively call
615 this routine to calculate INTO_TARGET (which depends on both OUTOF_INPUT
616 and INTO_INPUT), then emit code to set up OUTOF_TARGET.
618 This isn't worthwhile for constant shifts since the optimizers will
619 cope better with in-range shift counts. */
620 if (shift_mask
>= BITS_PER_WORD
622 && !CONSTANT_P (op1
))
624 if (!expand_doubleword_shift (op1_mode
, binoptab
,
625 outof_input
, into_input
, op1
,
627 unsignedp
, methods
, shift_mask
))
629 if (!force_expand_binop (word_mode
, binoptab
, outof_input
, op1
,
630 outof_target
, unsignedp
, methods
))
635 /* Set CMP_CODE, CMP1 and CMP2 so that the rtx (CMP_CODE CMP1 CMP2)
636 is true when the effective shift value is less than BITS_PER_WORD.
637 Set SUPERWORD_OP1 to the shift count that should be used to shift
638 OUTOF_INPUT into INTO_TARGET when the condition is false. */
639 tmp
= immed_wide_int_const (wi::shwi (BITS_PER_WORD
, op1_mode
), op1_mode
);
640 if (!CONSTANT_P (op1
) && shift_mask
== BITS_PER_WORD
- 1)
642 /* Set CMP1 to OP1 & BITS_PER_WORD. The result is zero iff OP1
643 is a subword shift count. */
644 cmp1
= simplify_expand_binop (op1_mode
, and_optab
, op1
, tmp
,
646 cmp2
= CONST0_RTX (op1_mode
);
652 /* Set CMP1 to OP1 - BITS_PER_WORD. */
653 cmp1
= simplify_expand_binop (op1_mode
, sub_optab
, op1
, tmp
,
655 cmp2
= CONST0_RTX (op1_mode
);
657 superword_op1
= cmp1
;
662 /* If we can compute the condition at compile time, pick the
663 appropriate subroutine. */
664 tmp
= simplify_relational_operation (cmp_code
, SImode
, op1_mode
, cmp1
, cmp2
);
665 if (tmp
!= 0 && CONST_INT_P (tmp
))
667 if (tmp
== const0_rtx
)
668 return expand_superword_shift (binoptab
, outof_input
, superword_op1
,
669 outof_target
, into_target
,
672 return expand_subword_shift (op1_mode
, binoptab
,
673 outof_input
, into_input
, op1
,
674 outof_target
, into_target
,
675 unsignedp
, methods
, shift_mask
);
678 /* Try using conditional moves to generate straight-line code. */
679 if (HAVE_conditional_move
)
681 rtx_insn
*start
= get_last_insn ();
682 if (expand_doubleword_shift_condmove (op1_mode
, binoptab
,
683 cmp_code
, cmp1
, cmp2
,
684 outof_input
, into_input
,
686 outof_target
, into_target
,
687 unsignedp
, methods
, shift_mask
))
689 delete_insns_since (start
);
692 /* As a last resort, use branches to select the correct alternative. */
693 rtx_code_label
*subword_label
= gen_label_rtx ();
694 rtx_code_label
*done_label
= gen_label_rtx ();
697 do_compare_rtx_and_jump (cmp1
, cmp2
, cmp_code
, false, op1_mode
,
699 profile_probability::uninitialized ());
702 if (!expand_superword_shift (binoptab
, outof_input
, superword_op1
,
703 outof_target
, into_target
,
707 emit_jump_insn (targetm
.gen_jump (done_label
));
709 emit_label (subword_label
);
711 if (!expand_subword_shift (op1_mode
, binoptab
,
712 outof_input
, into_input
, op1
,
713 outof_target
, into_target
,
714 unsignedp
, methods
, shift_mask
))
717 emit_label (done_label
);
721 /* Subroutine of expand_binop. Perform a double word multiplication of
722 operands OP0 and OP1 both of mode MODE, which is exactly twice as wide
723 as the target's word_mode. This function return NULL_RTX if anything
724 goes wrong, in which case it may have already emitted instructions
725 which need to be deleted.
727 If we want to multiply two two-word values and have normal and widening
728 multiplies of single-word values, we can do this with three smaller
731 The multiplication proceeds as follows:
732 _______________________
733 [__op0_high_|__op0_low__]
734 _______________________
735 * [__op1_high_|__op1_low__]
736 _______________________________________________
737 _______________________
738 (1) [__op0_low__*__op1_low__]
739 _______________________
740 (2a) [__op0_low__*__op1_high_]
741 _______________________
742 (2b) [__op0_high_*__op1_low__]
743 _______________________
744 (3) [__op0_high_*__op1_high_]
747 This gives a 4-word result. Since we are only interested in the
748 lower 2 words, partial result (3) and the upper words of (2a) and
749 (2b) don't need to be calculated. Hence (2a) and (2b) can be
750 calculated using non-widening multiplication.
752 (1), however, needs to be calculated with an unsigned widening
753 multiplication. If this operation is not directly supported we
754 try using a signed widening multiplication and adjust the result.
755 This adjustment works as follows:
757 If both operands are positive then no adjustment is needed.
759 If the operands have different signs, for example op0_low < 0 and
760 op1_low >= 0, the instruction treats the most significant bit of
761 op0_low as a sign bit instead of a bit with significance
762 2**(BITS_PER_WORD-1), i.e. the instruction multiplies op1_low
763 with 2**BITS_PER_WORD - op0_low, and two's complements the
764 result. Conclusion: We need to add op1_low * 2**BITS_PER_WORD to
767 Similarly, if both operands are negative, we need to add
768 (op0_low + op1_low) * 2**BITS_PER_WORD.
770 We use a trick to adjust quickly. We logically shift op0_low right
771 (op1_low) BITS_PER_WORD-1 steps to get 0 or 1, and add this to
772 op0_high (op1_high) before it is used to calculate 2b (2a). If no
773 logical shift exists, we do an arithmetic right shift and subtract
777 expand_doubleword_mult (machine_mode mode
, rtx op0
, rtx op1
, rtx target
,
778 bool umulp
, enum optab_methods methods
)
780 int low
= (WORDS_BIG_ENDIAN
? 1 : 0);
781 int high
= (WORDS_BIG_ENDIAN
? 0 : 1);
782 rtx wordm1
= umulp
? NULL_RTX
: GEN_INT (BITS_PER_WORD
- 1);
783 rtx product
, adjust
, product_high
, temp
;
785 rtx op0_high
= operand_subword_force (op0
, high
, mode
);
786 rtx op0_low
= operand_subword_force (op0
, low
, mode
);
787 rtx op1_high
= operand_subword_force (op1
, high
, mode
);
788 rtx op1_low
= operand_subword_force (op1
, low
, mode
);
790 /* If we're using an unsigned multiply to directly compute the product
791 of the low-order words of the operands and perform any required
792 adjustments of the operands, we begin by trying two more multiplications
793 and then computing the appropriate sum.
795 We have checked above that the required addition is provided.
796 Full-word addition will normally always succeed, especially if
797 it is provided at all, so we don't worry about its failure. The
798 multiplication may well fail, however, so we do handle that. */
802 /* ??? This could be done with emit_store_flag where available. */
803 temp
= expand_binop (word_mode
, lshr_optab
, op0_low
, wordm1
,
804 NULL_RTX
, 1, methods
);
806 op0_high
= expand_binop (word_mode
, add_optab
, op0_high
, temp
,
807 NULL_RTX
, 0, OPTAB_DIRECT
);
810 temp
= expand_binop (word_mode
, ashr_optab
, op0_low
, wordm1
,
811 NULL_RTX
, 0, methods
);
814 op0_high
= expand_binop (word_mode
, sub_optab
, op0_high
, temp
,
815 NULL_RTX
, 0, OPTAB_DIRECT
);
822 adjust
= expand_binop (word_mode
, smul_optab
, op0_high
, op1_low
,
823 NULL_RTX
, 0, OPTAB_DIRECT
);
827 /* OP0_HIGH should now be dead. */
831 /* ??? This could be done with emit_store_flag where available. */
832 temp
= expand_binop (word_mode
, lshr_optab
, op1_low
, wordm1
,
833 NULL_RTX
, 1, methods
);
835 op1_high
= expand_binop (word_mode
, add_optab
, op1_high
, temp
,
836 NULL_RTX
, 0, OPTAB_DIRECT
);
839 temp
= expand_binop (word_mode
, ashr_optab
, op1_low
, wordm1
,
840 NULL_RTX
, 0, methods
);
843 op1_high
= expand_binop (word_mode
, sub_optab
, op1_high
, temp
,
844 NULL_RTX
, 0, OPTAB_DIRECT
);
851 temp
= expand_binop (word_mode
, smul_optab
, op1_high
, op0_low
,
852 NULL_RTX
, 0, OPTAB_DIRECT
);
856 /* OP1_HIGH should now be dead. */
858 adjust
= expand_binop (word_mode
, add_optab
, adjust
, temp
,
859 NULL_RTX
, 0, OPTAB_DIRECT
);
861 if (target
&& !REG_P (target
))
865 product
= expand_binop (mode
, umul_widen_optab
, op0_low
, op1_low
,
866 target
, 1, OPTAB_DIRECT
);
868 product
= expand_binop (mode
, smul_widen_optab
, op0_low
, op1_low
,
869 target
, 1, OPTAB_DIRECT
);
874 product_high
= operand_subword (product
, high
, 1, mode
);
875 adjust
= expand_binop (word_mode
, add_optab
, product_high
, adjust
,
876 NULL_RTX
, 0, OPTAB_DIRECT
);
877 emit_move_insn (product_high
, adjust
);
881 /* Wrapper around expand_binop which takes an rtx code to specify
882 the operation to perform, not an optab pointer. All other
883 arguments are the same. */
885 expand_simple_binop (machine_mode mode
, enum rtx_code code
, rtx op0
,
886 rtx op1
, rtx target
, int unsignedp
,
887 enum optab_methods methods
)
889 optab binop
= code_to_optab (code
);
892 return expand_binop (mode
, binop
, op0
, op1
, target
, unsignedp
, methods
);
895 /* Return whether OP0 and OP1 should be swapped when expanding a commutative
896 binop. Order them according to commutative_operand_precedence and, if
897 possible, try to put TARGET or a pseudo first. */
899 swap_commutative_operands_with_target (rtx target
, rtx op0
, rtx op1
)
901 int op0_prec
= commutative_operand_precedence (op0
);
902 int op1_prec
= commutative_operand_precedence (op1
);
904 if (op0_prec
< op1_prec
)
907 if (op0_prec
> op1_prec
)
910 /* With equal precedence, both orders are ok, but it is better if the
911 first operand is TARGET, or if both TARGET and OP0 are pseudos. */
912 if (target
== 0 || REG_P (target
))
913 return (REG_P (op1
) && !REG_P (op0
)) || target
== op1
;
915 return rtx_equal_p (op1
, target
);
918 /* Return true if BINOPTAB implements a shift operation. */
921 shift_optab_p (optab binoptab
)
923 switch (optab_to_code (binoptab
))
939 /* Return true if BINOPTAB implements a commutative binary operation. */
942 commutative_optab_p (optab binoptab
)
944 return (GET_RTX_CLASS (optab_to_code (binoptab
)) == RTX_COMM_ARITH
945 || binoptab
== smul_widen_optab
946 || binoptab
== umul_widen_optab
947 || binoptab
== smul_highpart_optab
948 || binoptab
== umul_highpart_optab
);
951 /* X is to be used in mode MODE as operand OPN to BINOPTAB. If we're
952 optimizing, and if the operand is a constant that costs more than
953 1 instruction, force the constant into a register and return that
954 register. Return X otherwise. UNSIGNEDP says whether X is unsigned. */
957 avoid_expensive_constant (machine_mode mode
, optab binoptab
,
958 int opn
, rtx x
, bool unsignedp
)
960 bool speed
= optimize_insn_for_speed_p ();
965 && (rtx_cost (x
, mode
, optab_to_code (binoptab
), opn
, speed
)
966 > set_src_cost (x
, mode
, speed
)))
970 HOST_WIDE_INT intval
= trunc_int_for_mode (INTVAL (x
), mode
);
971 if (intval
!= INTVAL (x
))
972 x
= GEN_INT (intval
);
975 x
= convert_modes (mode
, VOIDmode
, x
, unsignedp
);
976 x
= force_reg (mode
, x
);
981 /* Helper function for expand_binop: handle the case where there
982 is an insn ICODE that directly implements the indicated operation.
983 Returns null if this is not possible. */
985 expand_binop_directly (enum insn_code icode
, machine_mode mode
, optab binoptab
,
987 rtx target
, int unsignedp
, enum optab_methods methods
,
990 machine_mode xmode0
= insn_data
[(int) icode
].operand
[1].mode
;
991 machine_mode xmode1
= insn_data
[(int) icode
].operand
[2].mode
;
992 machine_mode mode0
, mode1
, tmp_mode
;
993 struct expand_operand ops
[3];
996 rtx xop0
= op0
, xop1
= op1
;
997 bool canonicalize_op1
= false;
999 /* If it is a commutative operator and the modes would match
1000 if we would swap the operands, we can save the conversions. */
1001 commutative_p
= commutative_optab_p (binoptab
);
1003 && GET_MODE (xop0
) != xmode0
&& GET_MODE (xop1
) != xmode1
1004 && GET_MODE (xop0
) == xmode1
&& GET_MODE (xop1
) == xmode1
)
1005 std::swap (xop0
, xop1
);
1007 /* If we are optimizing, force expensive constants into a register. */
1008 xop0
= avoid_expensive_constant (xmode0
, binoptab
, 0, xop0
, unsignedp
);
1009 if (!shift_optab_p (binoptab
))
1010 xop1
= avoid_expensive_constant (xmode1
, binoptab
, 1, xop1
, unsignedp
);
1012 /* Shifts and rotates often use a different mode for op1 from op0;
1013 for VOIDmode constants we don't know the mode, so force it
1014 to be canonicalized using convert_modes. */
1015 canonicalize_op1
= true;
1017 /* In case the insn wants input operands in modes different from
1018 those of the actual operands, convert the operands. It would
1019 seem that we don't need to convert CONST_INTs, but we do, so
1020 that they're properly zero-extended, sign-extended or truncated
1023 mode0
= GET_MODE (xop0
) != VOIDmode
? GET_MODE (xop0
) : mode
;
1024 if (xmode0
!= VOIDmode
&& xmode0
!= mode0
)
1026 xop0
= convert_modes (xmode0
, mode0
, xop0
, unsignedp
);
1030 mode1
= ((GET_MODE (xop1
) != VOIDmode
|| canonicalize_op1
)
1031 ? GET_MODE (xop1
) : mode
);
1032 if (xmode1
!= VOIDmode
&& xmode1
!= mode1
)
1034 xop1
= convert_modes (xmode1
, mode1
, xop1
, unsignedp
);
1038 /* If operation is commutative,
1039 try to make the first operand a register.
1040 Even better, try to make it the same as the target.
1041 Also try to make the last operand a constant. */
1043 && swap_commutative_operands_with_target (target
, xop0
, xop1
))
1044 std::swap (xop0
, xop1
);
1046 /* Now, if insn's predicates don't allow our operands, put them into
1049 if (binoptab
== vec_pack_trunc_optab
1050 || binoptab
== vec_pack_usat_optab
1051 || binoptab
== vec_pack_ssat_optab
1052 || binoptab
== vec_pack_ufix_trunc_optab
1053 || binoptab
== vec_pack_sfix_trunc_optab
)
1055 /* The mode of the result is different then the mode of the
1057 tmp_mode
= insn_data
[(int) icode
].operand
[0].mode
;
1058 if (VECTOR_MODE_P (mode
)
1059 && GET_MODE_NUNITS (tmp_mode
) != 2 * GET_MODE_NUNITS (mode
))
1061 delete_insns_since (last
);
1068 create_output_operand (&ops
[0], target
, tmp_mode
);
1069 create_input_operand (&ops
[1], xop0
, mode0
);
1070 create_input_operand (&ops
[2], xop1
, mode1
);
1071 pat
= maybe_gen_insn (icode
, 3, ops
);
1074 /* If PAT is composed of more than one insn, try to add an appropriate
1075 REG_EQUAL note to it. If we can't because TEMP conflicts with an
1076 operand, call expand_binop again, this time without a target. */
1077 if (INSN_P (pat
) && NEXT_INSN (pat
) != NULL_RTX
1078 && ! add_equal_note (pat
, ops
[0].value
,
1079 optab_to_code (binoptab
),
1080 ops
[1].value
, ops
[2].value
))
1082 delete_insns_since (last
);
1083 return expand_binop (mode
, binoptab
, op0
, op1
, NULL_RTX
,
1084 unsignedp
, methods
);
1088 return ops
[0].value
;
1090 delete_insns_since (last
);
1094 /* Generate code to perform an operation specified by BINOPTAB
1095 on operands OP0 and OP1, with result having machine-mode MODE.
1097 UNSIGNEDP is for the case where we have to widen the operands
1098 to perform the operation. It says to use zero-extension.
1100 If TARGET is nonzero, the value
1101 is generated there, if it is convenient to do so.
1102 In all cases an rtx is returned for the locus of the value;
1103 this may or may not be TARGET. */
1106 expand_binop (machine_mode mode
, optab binoptab
, rtx op0
, rtx op1
,
1107 rtx target
, int unsignedp
, enum optab_methods methods
)
1109 enum optab_methods next_methods
1110 = (methods
== OPTAB_LIB
|| methods
== OPTAB_LIB_WIDEN
1111 ? OPTAB_WIDEN
: methods
);
1112 enum mode_class mclass
;
1113 enum insn_code icode
;
1114 machine_mode wider_mode
;
1115 scalar_int_mode int_mode
;
1118 rtx_insn
*entry_last
= get_last_insn ();
1121 mclass
= GET_MODE_CLASS (mode
);
1123 /* If subtracting an integer constant, convert this into an addition of
1124 the negated constant. */
1126 if (binoptab
== sub_optab
&& CONST_INT_P (op1
))
1128 op1
= negate_rtx (mode
, op1
);
1129 binoptab
= add_optab
;
1131 /* For shifts, constant invalid op1 might be expanded from different
1132 mode than MODE. As those are invalid, force them to a register
1133 to avoid further problems during expansion. */
1134 else if (CONST_INT_P (op1
)
1135 && shift_optab_p (binoptab
)
1136 && UINTVAL (op1
) >= GET_MODE_BITSIZE (GET_MODE_INNER (mode
)))
1138 op1
= gen_int_mode (INTVAL (op1
), GET_MODE_INNER (mode
));
1139 op1
= force_reg (GET_MODE_INNER (mode
), op1
);
1142 /* Record where to delete back to if we backtrack. */
1143 last
= get_last_insn ();
1145 /* If we can do it with a three-operand insn, do so. */
1147 if (methods
!= OPTAB_MUST_WIDEN
)
1149 if (convert_optab_p (binoptab
))
1151 machine_mode from_mode
= widened_mode (mode
, op0
, op1
);
1152 icode
= find_widening_optab_handler (binoptab
, mode
, from_mode
);
1155 icode
= optab_handler (binoptab
, mode
);
1156 if (icode
!= CODE_FOR_nothing
)
1158 temp
= expand_binop_directly (icode
, mode
, binoptab
, op0
, op1
,
1159 target
, unsignedp
, methods
, last
);
1165 /* If we were trying to rotate, and that didn't work, try rotating
1166 the other direction before falling back to shifts and bitwise-or. */
1167 if (((binoptab
== rotl_optab
1168 && (icode
= optab_handler (rotr_optab
, mode
)) != CODE_FOR_nothing
)
1169 || (binoptab
== rotr_optab
1170 && (icode
= optab_handler (rotl_optab
, mode
)) != CODE_FOR_nothing
))
1171 && is_int_mode (mode
, &int_mode
))
1173 optab otheroptab
= (binoptab
== rotl_optab
? rotr_optab
: rotl_optab
);
1175 unsigned int bits
= GET_MODE_PRECISION (int_mode
);
1177 if (CONST_INT_P (op1
))
1178 newop1
= GEN_INT (bits
- INTVAL (op1
));
1179 else if (targetm
.shift_truncation_mask (int_mode
) == bits
- 1)
1180 newop1
= negate_rtx (GET_MODE (op1
), op1
);
1182 newop1
= expand_binop (GET_MODE (op1
), sub_optab
,
1183 gen_int_mode (bits
, GET_MODE (op1
)), op1
,
1184 NULL_RTX
, unsignedp
, OPTAB_DIRECT
);
1186 temp
= expand_binop_directly (icode
, int_mode
, otheroptab
, op0
, newop1
,
1187 target
, unsignedp
, methods
, last
);
1192 /* If this is a multiply, see if we can do a widening operation that
1193 takes operands of this mode and makes a wider mode. */
1195 if (binoptab
== smul_optab
1196 && GET_MODE_2XWIDER_MODE (mode
).exists (&wider_mode
)
1197 && (convert_optab_handler ((unsignedp
1199 : smul_widen_optab
),
1200 wider_mode
, mode
) != CODE_FOR_nothing
))
1202 temp
= expand_binop (wider_mode
,
1203 unsignedp
? umul_widen_optab
: smul_widen_optab
,
1204 op0
, op1
, NULL_RTX
, unsignedp
, OPTAB_DIRECT
);
1208 if (GET_MODE_CLASS (mode
) == MODE_INT
1209 && TRULY_NOOP_TRUNCATION_MODES_P (mode
, GET_MODE (temp
)))
1210 return gen_lowpart (mode
, temp
);
1212 return convert_to_mode (mode
, temp
, unsignedp
);
1216 /* If this is a vector shift by a scalar, see if we can do a vector
1217 shift by a vector. If so, broadcast the scalar into a vector. */
1218 if (mclass
== MODE_VECTOR_INT
)
1220 optab otheroptab
= unknown_optab
;
1222 if (binoptab
== ashl_optab
)
1223 otheroptab
= vashl_optab
;
1224 else if (binoptab
== ashr_optab
)
1225 otheroptab
= vashr_optab
;
1226 else if (binoptab
== lshr_optab
)
1227 otheroptab
= vlshr_optab
;
1228 else if (binoptab
== rotl_optab
)
1229 otheroptab
= vrotl_optab
;
1230 else if (binoptab
== rotr_optab
)
1231 otheroptab
= vrotr_optab
;
1234 && (icode
= optab_handler (otheroptab
, mode
)) != CODE_FOR_nothing
)
1236 /* The scalar may have been extended to be too wide. Truncate
1237 it back to the proper size to fit in the broadcast vector. */
1238 scalar_mode inner_mode
= GET_MODE_INNER (mode
);
1239 if (!CONST_INT_P (op1
)
1240 && (GET_MODE_BITSIZE (as_a
<scalar_int_mode
> (GET_MODE (op1
)))
1241 > GET_MODE_BITSIZE (inner_mode
)))
1242 op1
= force_reg (inner_mode
,
1243 simplify_gen_unary (TRUNCATE
, inner_mode
, op1
,
1245 rtx vop1
= expand_vector_broadcast (mode
, op1
);
1248 temp
= expand_binop_directly (icode
, mode
, otheroptab
, op0
, vop1
,
1249 target
, unsignedp
, methods
, last
);
1256 /* Look for a wider mode of the same class for which we think we
1257 can open-code the operation. Check for a widening multiply at the
1258 wider mode as well. */
1260 if (CLASS_HAS_WIDER_MODES_P (mclass
)
1261 && methods
!= OPTAB_DIRECT
&& methods
!= OPTAB_LIB
)
1262 FOR_EACH_WIDER_MODE (wider_mode
, mode
)
1264 machine_mode next_mode
;
1265 if (optab_handler (binoptab
, wider_mode
) != CODE_FOR_nothing
1266 || (binoptab
== smul_optab
1267 && GET_MODE_WIDER_MODE (wider_mode
).exists (&next_mode
)
1268 && (find_widening_optab_handler ((unsignedp
1270 : smul_widen_optab
),
1272 != CODE_FOR_nothing
)))
1274 rtx xop0
= op0
, xop1
= op1
;
1277 /* For certain integer operations, we need not actually extend
1278 the narrow operands, as long as we will truncate
1279 the results to the same narrowness. */
1281 if ((binoptab
== ior_optab
|| binoptab
== and_optab
1282 || binoptab
== xor_optab
1283 || binoptab
== add_optab
|| binoptab
== sub_optab
1284 || binoptab
== smul_optab
|| binoptab
== ashl_optab
)
1285 && mclass
== MODE_INT
)
1288 xop0
= avoid_expensive_constant (mode
, binoptab
, 0,
1290 if (binoptab
!= ashl_optab
)
1291 xop1
= avoid_expensive_constant (mode
, binoptab
, 1,
1295 xop0
= widen_operand (xop0
, wider_mode
, mode
, unsignedp
, no_extend
);
1297 /* The second operand of a shift must always be extended. */
1298 xop1
= widen_operand (xop1
, wider_mode
, mode
, unsignedp
,
1299 no_extend
&& binoptab
!= ashl_optab
);
1301 temp
= expand_binop (wider_mode
, binoptab
, xop0
, xop1
, NULL_RTX
,
1302 unsignedp
, OPTAB_DIRECT
);
1305 if (mclass
!= MODE_INT
1306 || !TRULY_NOOP_TRUNCATION_MODES_P (mode
, wider_mode
))
1309 target
= gen_reg_rtx (mode
);
1310 convert_move (target
, temp
, 0);
1314 return gen_lowpart (mode
, temp
);
1317 delete_insns_since (last
);
1321 /* If operation is commutative,
1322 try to make the first operand a register.
1323 Even better, try to make it the same as the target.
1324 Also try to make the last operand a constant. */
1325 if (commutative_optab_p (binoptab
)
1326 && swap_commutative_operands_with_target (target
, op0
, op1
))
1327 std::swap (op0
, op1
);
1329 /* These can be done a word at a time. */
1330 if ((binoptab
== and_optab
|| binoptab
== ior_optab
|| binoptab
== xor_optab
)
1331 && is_int_mode (mode
, &int_mode
)
1332 && GET_MODE_SIZE (int_mode
) > UNITS_PER_WORD
1333 && optab_handler (binoptab
, word_mode
) != CODE_FOR_nothing
)
1338 /* If TARGET is the same as one of the operands, the REG_EQUAL note
1339 won't be accurate, so use a new target. */
1343 || !valid_multiword_target_p (target
))
1344 target
= gen_reg_rtx (int_mode
);
1348 /* Do the actual arithmetic. */
1349 for (i
= 0; i
< GET_MODE_BITSIZE (int_mode
) / BITS_PER_WORD
; i
++)
1351 rtx target_piece
= operand_subword (target
, i
, 1, int_mode
);
1352 rtx x
= expand_binop (word_mode
, binoptab
,
1353 operand_subword_force (op0
, i
, int_mode
),
1354 operand_subword_force (op1
, i
, int_mode
),
1355 target_piece
, unsignedp
, next_methods
);
1360 if (target_piece
!= x
)
1361 emit_move_insn (target_piece
, x
);
1364 insns
= get_insns ();
1367 if (i
== GET_MODE_BITSIZE (int_mode
) / BITS_PER_WORD
)
1374 /* Synthesize double word shifts from single word shifts. */
1375 if ((binoptab
== lshr_optab
|| binoptab
== ashl_optab
1376 || binoptab
== ashr_optab
)
1377 && is_int_mode (mode
, &int_mode
)
1378 && (CONST_INT_P (op1
) || optimize_insn_for_speed_p ())
1379 && GET_MODE_SIZE (int_mode
) == 2 * UNITS_PER_WORD
1380 && GET_MODE_PRECISION (int_mode
) == GET_MODE_BITSIZE (int_mode
)
1381 && optab_handler (binoptab
, word_mode
) != CODE_FOR_nothing
1382 && optab_handler (ashl_optab
, word_mode
) != CODE_FOR_nothing
1383 && optab_handler (lshr_optab
, word_mode
) != CODE_FOR_nothing
)
1385 unsigned HOST_WIDE_INT shift_mask
, double_shift_mask
;
1386 scalar_int_mode op1_mode
;
1388 double_shift_mask
= targetm
.shift_truncation_mask (int_mode
);
1389 shift_mask
= targetm
.shift_truncation_mask (word_mode
);
1390 op1_mode
= (GET_MODE (op1
) != VOIDmode
1391 ? as_a
<scalar_int_mode
> (GET_MODE (op1
))
1394 /* Apply the truncation to constant shifts. */
1395 if (double_shift_mask
> 0 && CONST_INT_P (op1
))
1396 op1
= GEN_INT (INTVAL (op1
) & double_shift_mask
);
1398 if (op1
== CONST0_RTX (op1_mode
))
1401 /* Make sure that this is a combination that expand_doubleword_shift
1402 can handle. See the comments there for details. */
1403 if (double_shift_mask
== 0
1404 || (shift_mask
== BITS_PER_WORD
- 1
1405 && double_shift_mask
== BITS_PER_WORD
* 2 - 1))
1408 rtx into_target
, outof_target
;
1409 rtx into_input
, outof_input
;
1410 int left_shift
, outof_word
;
1412 /* If TARGET is the same as one of the operands, the REG_EQUAL note
1413 won't be accurate, so use a new target. */
1417 || !valid_multiword_target_p (target
))
1418 target
= gen_reg_rtx (int_mode
);
1422 /* OUTOF_* is the word we are shifting bits away from, and
1423 INTO_* is the word that we are shifting bits towards, thus
1424 they differ depending on the direction of the shift and
1425 WORDS_BIG_ENDIAN. */
1427 left_shift
= binoptab
== ashl_optab
;
1428 outof_word
= left_shift
^ ! WORDS_BIG_ENDIAN
;
1430 outof_target
= operand_subword (target
, outof_word
, 1, int_mode
);
1431 into_target
= operand_subword (target
, 1 - outof_word
, 1, int_mode
);
1433 outof_input
= operand_subword_force (op0
, outof_word
, int_mode
);
1434 into_input
= operand_subword_force (op0
, 1 - outof_word
, int_mode
);
1436 if (expand_doubleword_shift (op1_mode
, binoptab
,
1437 outof_input
, into_input
, op1
,
1438 outof_target
, into_target
,
1439 unsignedp
, next_methods
, shift_mask
))
1441 insns
= get_insns ();
1451 /* Synthesize double word rotates from single word shifts. */
1452 if ((binoptab
== rotl_optab
|| binoptab
== rotr_optab
)
1453 && is_int_mode (mode
, &int_mode
)
1454 && CONST_INT_P (op1
)
1455 && GET_MODE_PRECISION (int_mode
) == 2 * BITS_PER_WORD
1456 && optab_handler (ashl_optab
, word_mode
) != CODE_FOR_nothing
1457 && optab_handler (lshr_optab
, word_mode
) != CODE_FOR_nothing
)
1460 rtx into_target
, outof_target
;
1461 rtx into_input
, outof_input
;
1463 int shift_count
, left_shift
, outof_word
;
1465 /* If TARGET is the same as one of the operands, the REG_EQUAL note
1466 won't be accurate, so use a new target. Do this also if target is not
1467 a REG, first because having a register instead may open optimization
1468 opportunities, and second because if target and op0 happen to be MEMs
1469 designating the same location, we would risk clobbering it too early
1470 in the code sequence we generate below. */
1475 || !valid_multiword_target_p (target
))
1476 target
= gen_reg_rtx (int_mode
);
1480 shift_count
= INTVAL (op1
);
1482 /* OUTOF_* is the word we are shifting bits away from, and
1483 INTO_* is the word that we are shifting bits towards, thus
1484 they differ depending on the direction of the shift and
1485 WORDS_BIG_ENDIAN. */
1487 left_shift
= (binoptab
== rotl_optab
);
1488 outof_word
= left_shift
^ ! WORDS_BIG_ENDIAN
;
1490 outof_target
= operand_subword (target
, outof_word
, 1, int_mode
);
1491 into_target
= operand_subword (target
, 1 - outof_word
, 1, int_mode
);
1493 outof_input
= operand_subword_force (op0
, outof_word
, int_mode
);
1494 into_input
= operand_subword_force (op0
, 1 - outof_word
, int_mode
);
1496 if (shift_count
== BITS_PER_WORD
)
1498 /* This is just a word swap. */
1499 emit_move_insn (outof_target
, into_input
);
1500 emit_move_insn (into_target
, outof_input
);
1505 rtx into_temp1
, into_temp2
, outof_temp1
, outof_temp2
;
1506 rtx first_shift_count
, second_shift_count
;
1507 optab reverse_unsigned_shift
, unsigned_shift
;
1509 reverse_unsigned_shift
= (left_shift
^ (shift_count
< BITS_PER_WORD
)
1510 ? lshr_optab
: ashl_optab
);
1512 unsigned_shift
= (left_shift
^ (shift_count
< BITS_PER_WORD
)
1513 ? ashl_optab
: lshr_optab
);
1515 if (shift_count
> BITS_PER_WORD
)
1517 first_shift_count
= GEN_INT (shift_count
- BITS_PER_WORD
);
1518 second_shift_count
= GEN_INT (2 * BITS_PER_WORD
- shift_count
);
1522 first_shift_count
= GEN_INT (BITS_PER_WORD
- shift_count
);
1523 second_shift_count
= GEN_INT (shift_count
);
1526 into_temp1
= expand_binop (word_mode
, unsigned_shift
,
1527 outof_input
, first_shift_count
,
1528 NULL_RTX
, unsignedp
, next_methods
);
1529 into_temp2
= expand_binop (word_mode
, reverse_unsigned_shift
,
1530 into_input
, second_shift_count
,
1531 NULL_RTX
, unsignedp
, next_methods
);
1533 if (into_temp1
!= 0 && into_temp2
!= 0)
1534 inter
= expand_binop (word_mode
, ior_optab
, into_temp1
, into_temp2
,
1535 into_target
, unsignedp
, next_methods
);
1539 if (inter
!= 0 && inter
!= into_target
)
1540 emit_move_insn (into_target
, inter
);
1542 outof_temp1
= expand_binop (word_mode
, unsigned_shift
,
1543 into_input
, first_shift_count
,
1544 NULL_RTX
, unsignedp
, next_methods
);
1545 outof_temp2
= expand_binop (word_mode
, reverse_unsigned_shift
,
1546 outof_input
, second_shift_count
,
1547 NULL_RTX
, unsignedp
, next_methods
);
1549 if (inter
!= 0 && outof_temp1
!= 0 && outof_temp2
!= 0)
1550 inter
= expand_binop (word_mode
, ior_optab
,
1551 outof_temp1
, outof_temp2
,
1552 outof_target
, unsignedp
, next_methods
);
1554 if (inter
!= 0 && inter
!= outof_target
)
1555 emit_move_insn (outof_target
, inter
);
1558 insns
= get_insns ();
1568 /* These can be done a word at a time by propagating carries. */
1569 if ((binoptab
== add_optab
|| binoptab
== sub_optab
)
1570 && is_int_mode (mode
, &int_mode
)
1571 && GET_MODE_SIZE (int_mode
) >= 2 * UNITS_PER_WORD
1572 && optab_handler (binoptab
, word_mode
) != CODE_FOR_nothing
)
1575 optab otheroptab
= binoptab
== add_optab
? sub_optab
: add_optab
;
1576 const unsigned int nwords
= GET_MODE_BITSIZE (int_mode
) / BITS_PER_WORD
;
1577 rtx carry_in
= NULL_RTX
, carry_out
= NULL_RTX
;
1578 rtx xop0
, xop1
, xtarget
;
1580 /* We can handle either a 1 or -1 value for the carry. If STORE_FLAG
1581 value is one of those, use it. Otherwise, use 1 since it is the
1582 one easiest to get. */
1583 #if STORE_FLAG_VALUE == 1 || STORE_FLAG_VALUE == -1
1584 int normalizep
= STORE_FLAG_VALUE
;
1589 /* Prepare the operands. */
1590 xop0
= force_reg (int_mode
, op0
);
1591 xop1
= force_reg (int_mode
, op1
);
1593 xtarget
= gen_reg_rtx (int_mode
);
1595 if (target
== 0 || !REG_P (target
) || !valid_multiword_target_p (target
))
1598 /* Indicate for flow that the entire target reg is being set. */
1600 emit_clobber (xtarget
);
1602 /* Do the actual arithmetic. */
1603 for (i
= 0; i
< nwords
; i
++)
1605 int index
= (WORDS_BIG_ENDIAN
? nwords
- i
- 1 : i
);
1606 rtx target_piece
= operand_subword (xtarget
, index
, 1, int_mode
);
1607 rtx op0_piece
= operand_subword_force (xop0
, index
, int_mode
);
1608 rtx op1_piece
= operand_subword_force (xop1
, index
, int_mode
);
1611 /* Main add/subtract of the input operands. */
1612 x
= expand_binop (word_mode
, binoptab
,
1613 op0_piece
, op1_piece
,
1614 target_piece
, unsignedp
, next_methods
);
1620 /* Store carry from main add/subtract. */
1621 carry_out
= gen_reg_rtx (word_mode
);
1622 carry_out
= emit_store_flag_force (carry_out
,
1623 (binoptab
== add_optab
1626 word_mode
, 1, normalizep
);
1633 /* Add/subtract previous carry to main result. */
1634 newx
= expand_binop (word_mode
,
1635 normalizep
== 1 ? binoptab
: otheroptab
,
1637 NULL_RTX
, 1, next_methods
);
1641 /* Get out carry from adding/subtracting carry in. */
1642 rtx carry_tmp
= gen_reg_rtx (word_mode
);
1643 carry_tmp
= emit_store_flag_force (carry_tmp
,
1644 (binoptab
== add_optab
1647 word_mode
, 1, normalizep
);
1649 /* Logical-ior the two poss. carry together. */
1650 carry_out
= expand_binop (word_mode
, ior_optab
,
1651 carry_out
, carry_tmp
,
1652 carry_out
, 0, next_methods
);
1656 emit_move_insn (target_piece
, newx
);
1660 if (x
!= target_piece
)
1661 emit_move_insn (target_piece
, x
);
1664 carry_in
= carry_out
;
1667 if (i
== GET_MODE_BITSIZE (int_mode
) / (unsigned) BITS_PER_WORD
)
1669 if (optab_handler (mov_optab
, int_mode
) != CODE_FOR_nothing
1670 || ! rtx_equal_p (target
, xtarget
))
1672 rtx_insn
*temp
= emit_move_insn (target
, xtarget
);
1674 set_dst_reg_note (temp
, REG_EQUAL
,
1675 gen_rtx_fmt_ee (optab_to_code (binoptab
),
1676 int_mode
, copy_rtx (xop0
),
1687 delete_insns_since (last
);
1690 /* Attempt to synthesize double word multiplies using a sequence of word
1691 mode multiplications. We first attempt to generate a sequence using a
1692 more efficient unsigned widening multiply, and if that fails we then
1693 try using a signed widening multiply. */
1695 if (binoptab
== smul_optab
1696 && is_int_mode (mode
, &int_mode
)
1697 && GET_MODE_SIZE (int_mode
) == 2 * UNITS_PER_WORD
1698 && optab_handler (smul_optab
, word_mode
) != CODE_FOR_nothing
1699 && optab_handler (add_optab
, word_mode
) != CODE_FOR_nothing
)
1701 rtx product
= NULL_RTX
;
1702 if (convert_optab_handler (umul_widen_optab
, int_mode
, word_mode
)
1703 != CODE_FOR_nothing
)
1705 product
= expand_doubleword_mult (int_mode
, op0
, op1
, target
,
1708 delete_insns_since (last
);
1711 if (product
== NULL_RTX
1712 && (convert_optab_handler (smul_widen_optab
, int_mode
, word_mode
)
1713 != CODE_FOR_nothing
))
1715 product
= expand_doubleword_mult (int_mode
, op0
, op1
, target
,
1718 delete_insns_since (last
);
1721 if (product
!= NULL_RTX
)
1723 if (optab_handler (mov_optab
, int_mode
) != CODE_FOR_nothing
)
1725 rtx_insn
*move
= emit_move_insn (target
? target
: product
,
1727 set_dst_reg_note (move
,
1729 gen_rtx_fmt_ee (MULT
, int_mode
,
1732 target
? target
: product
);
1738 /* It can't be open-coded in this mode.
1739 Use a library call if one is available and caller says that's ok. */
1741 libfunc
= optab_libfunc (binoptab
, mode
);
1743 && (methods
== OPTAB_LIB
|| methods
== OPTAB_LIB_WIDEN
))
1747 machine_mode op1_mode
= mode
;
1752 if (shift_optab_p (binoptab
))
1754 op1_mode
= targetm
.libgcc_shift_count_mode ();
1755 /* Specify unsigned here,
1756 since negative shift counts are meaningless. */
1757 op1x
= convert_to_mode (op1_mode
, op1
, 1);
1760 if (GET_MODE (op0
) != VOIDmode
1761 && GET_MODE (op0
) != mode
)
1762 op0
= convert_to_mode (mode
, op0
, unsignedp
);
1764 /* Pass 1 for NO_QUEUE so we don't lose any increments
1765 if the libcall is cse'd or moved. */
1766 value
= emit_library_call_value (libfunc
,
1767 NULL_RTX
, LCT_CONST
, mode
,
1768 op0
, mode
, op1x
, op1_mode
);
1770 insns
= get_insns ();
1773 bool trapv
= trapv_binoptab_p (binoptab
);
1774 target
= gen_reg_rtx (mode
);
1775 emit_libcall_block_1 (insns
, target
, value
,
1777 : gen_rtx_fmt_ee (optab_to_code (binoptab
),
1778 mode
, op0
, op1
), trapv
);
1783 delete_insns_since (last
);
1785 /* It can't be done in this mode. Can we do it in a wider mode? */
1787 if (! (methods
== OPTAB_WIDEN
|| methods
== OPTAB_LIB_WIDEN
1788 || methods
== OPTAB_MUST_WIDEN
))
1790 /* Caller says, don't even try. */
1791 delete_insns_since (entry_last
);
1795 /* Compute the value of METHODS to pass to recursive calls.
1796 Don't allow widening to be tried recursively. */
1798 methods
= (methods
== OPTAB_LIB_WIDEN
? OPTAB_LIB
: OPTAB_DIRECT
);
1800 /* Look for a wider mode of the same class for which it appears we can do
1803 if (CLASS_HAS_WIDER_MODES_P (mclass
))
1805 /* This code doesn't make sense for conversion optabs, since we
1806 wouldn't then want to extend the operands to be the same size
1808 gcc_assert (!convert_optab_p (binoptab
));
1809 FOR_EACH_WIDER_MODE (wider_mode
, mode
)
1811 if (optab_handler (binoptab
, wider_mode
)
1812 || (methods
== OPTAB_LIB
1813 && optab_libfunc (binoptab
, wider_mode
)))
1815 rtx xop0
= op0
, xop1
= op1
;
1818 /* For certain integer operations, we need not actually extend
1819 the narrow operands, as long as we will truncate
1820 the results to the same narrowness. */
1822 if ((binoptab
== ior_optab
|| binoptab
== and_optab
1823 || binoptab
== xor_optab
1824 || binoptab
== add_optab
|| binoptab
== sub_optab
1825 || binoptab
== smul_optab
|| binoptab
== ashl_optab
)
1826 && mclass
== MODE_INT
)
1829 xop0
= widen_operand (xop0
, wider_mode
, mode
,
1830 unsignedp
, no_extend
);
1832 /* The second operand of a shift must always be extended. */
1833 xop1
= widen_operand (xop1
, wider_mode
, mode
, unsignedp
,
1834 no_extend
&& binoptab
!= ashl_optab
);
1836 temp
= expand_binop (wider_mode
, binoptab
, xop0
, xop1
, NULL_RTX
,
1837 unsignedp
, methods
);
1840 if (mclass
!= MODE_INT
1841 || !TRULY_NOOP_TRUNCATION_MODES_P (mode
, wider_mode
))
1844 target
= gen_reg_rtx (mode
);
1845 convert_move (target
, temp
, 0);
1849 return gen_lowpart (mode
, temp
);
1852 delete_insns_since (last
);
1857 delete_insns_since (entry_last
);
1861 /* Expand a binary operator which has both signed and unsigned forms.
1862 UOPTAB is the optab for unsigned operations, and SOPTAB is for
1865 If we widen unsigned operands, we may use a signed wider operation instead
1866 of an unsigned wider operation, since the result would be the same. */
1869 sign_expand_binop (machine_mode mode
, optab uoptab
, optab soptab
,
1870 rtx op0
, rtx op1
, rtx target
, int unsignedp
,
1871 enum optab_methods methods
)
1874 optab direct_optab
= unsignedp
? uoptab
: soptab
;
1877 /* Do it without widening, if possible. */
1878 temp
= expand_binop (mode
, direct_optab
, op0
, op1
, target
,
1879 unsignedp
, OPTAB_DIRECT
);
1880 if (temp
|| methods
== OPTAB_DIRECT
)
1883 /* Try widening to a signed int. Disable any direct use of any
1884 signed insn in the current mode. */
1885 save_enable
= swap_optab_enable (soptab
, mode
, false);
1887 temp
= expand_binop (mode
, soptab
, op0
, op1
, target
,
1888 unsignedp
, OPTAB_WIDEN
);
1890 /* For unsigned operands, try widening to an unsigned int. */
1891 if (!temp
&& unsignedp
)
1892 temp
= expand_binop (mode
, uoptab
, op0
, op1
, target
,
1893 unsignedp
, OPTAB_WIDEN
);
1894 if (temp
|| methods
== OPTAB_WIDEN
)
1897 /* Use the right width libcall if that exists. */
1898 temp
= expand_binop (mode
, direct_optab
, op0
, op1
, target
,
1899 unsignedp
, OPTAB_LIB
);
1900 if (temp
|| methods
== OPTAB_LIB
)
1903 /* Must widen and use a libcall, use either signed or unsigned. */
1904 temp
= expand_binop (mode
, soptab
, op0
, op1
, target
,
1905 unsignedp
, methods
);
1906 if (!temp
&& unsignedp
)
1907 temp
= expand_binop (mode
, uoptab
, op0
, op1
, target
,
1908 unsignedp
, methods
);
1911 /* Undo the fiddling above. */
1913 swap_optab_enable (soptab
, mode
, true);
1917 /* Generate code to perform an operation specified by UNOPPTAB
1918 on operand OP0, with two results to TARG0 and TARG1.
1919 We assume that the order of the operands for the instruction
1920 is TARG0, TARG1, OP0.
1922 Either TARG0 or TARG1 may be zero, but what that means is that
1923 the result is not actually wanted. We will generate it into
1924 a dummy pseudo-reg and discard it. They may not both be zero.
1926 Returns 1 if this operation can be performed; 0 if not. */
1929 expand_twoval_unop (optab unoptab
, rtx op0
, rtx targ0
, rtx targ1
,
1932 machine_mode mode
= GET_MODE (targ0
? targ0
: targ1
);
1933 enum mode_class mclass
;
1934 machine_mode wider_mode
;
1935 rtx_insn
*entry_last
= get_last_insn ();
1938 mclass
= GET_MODE_CLASS (mode
);
1941 targ0
= gen_reg_rtx (mode
);
1943 targ1
= gen_reg_rtx (mode
);
1945 /* Record where to go back to if we fail. */
1946 last
= get_last_insn ();
1948 if (optab_handler (unoptab
, mode
) != CODE_FOR_nothing
)
1950 struct expand_operand ops
[3];
1951 enum insn_code icode
= optab_handler (unoptab
, mode
);
1953 create_fixed_operand (&ops
[0], targ0
);
1954 create_fixed_operand (&ops
[1], targ1
);
1955 create_convert_operand_from (&ops
[2], op0
, mode
, unsignedp
);
1956 if (maybe_expand_insn (icode
, 3, ops
))
1960 /* It can't be done in this mode. Can we do it in a wider mode? */
1962 if (CLASS_HAS_WIDER_MODES_P (mclass
))
1964 FOR_EACH_WIDER_MODE (wider_mode
, mode
)
1966 if (optab_handler (unoptab
, wider_mode
) != CODE_FOR_nothing
)
1968 rtx t0
= gen_reg_rtx (wider_mode
);
1969 rtx t1
= gen_reg_rtx (wider_mode
);
1970 rtx cop0
= convert_modes (wider_mode
, mode
, op0
, unsignedp
);
1972 if (expand_twoval_unop (unoptab
, cop0
, t0
, t1
, unsignedp
))
1974 convert_move (targ0
, t0
, unsignedp
);
1975 convert_move (targ1
, t1
, unsignedp
);
1979 delete_insns_since (last
);
1984 delete_insns_since (entry_last
);
1988 /* Generate code to perform an operation specified by BINOPTAB
1989 on operands OP0 and OP1, with two results to TARG1 and TARG2.
1990 We assume that the order of the operands for the instruction
1991 is TARG0, OP0, OP1, TARG1, which would fit a pattern like
1992 [(set TARG0 (operate OP0 OP1)) (set TARG1 (operate ...))].
1994 Either TARG0 or TARG1 may be zero, but what that means is that
1995 the result is not actually wanted. We will generate it into
1996 a dummy pseudo-reg and discard it. They may not both be zero.
1998 Returns 1 if this operation can be performed; 0 if not. */
2001 expand_twoval_binop (optab binoptab
, rtx op0
, rtx op1
, rtx targ0
, rtx targ1
,
2004 machine_mode mode
= GET_MODE (targ0
? targ0
: targ1
);
2005 enum mode_class mclass
;
2006 machine_mode wider_mode
;
2007 rtx_insn
*entry_last
= get_last_insn ();
2010 mclass
= GET_MODE_CLASS (mode
);
2013 targ0
= gen_reg_rtx (mode
);
2015 targ1
= gen_reg_rtx (mode
);
2017 /* Record where to go back to if we fail. */
2018 last
= get_last_insn ();
2020 if (optab_handler (binoptab
, mode
) != CODE_FOR_nothing
)
2022 struct expand_operand ops
[4];
2023 enum insn_code icode
= optab_handler (binoptab
, mode
);
2024 machine_mode mode0
= insn_data
[icode
].operand
[1].mode
;
2025 machine_mode mode1
= insn_data
[icode
].operand
[2].mode
;
2026 rtx xop0
= op0
, xop1
= op1
;
2028 /* If we are optimizing, force expensive constants into a register. */
2029 xop0
= avoid_expensive_constant (mode0
, binoptab
, 0, xop0
, unsignedp
);
2030 xop1
= avoid_expensive_constant (mode1
, binoptab
, 1, xop1
, unsignedp
);
2032 create_fixed_operand (&ops
[0], targ0
);
2033 create_convert_operand_from (&ops
[1], op0
, mode
, unsignedp
);
2034 create_convert_operand_from (&ops
[2], op1
, mode
, unsignedp
);
2035 create_fixed_operand (&ops
[3], targ1
);
2036 if (maybe_expand_insn (icode
, 4, ops
))
2038 delete_insns_since (last
);
2041 /* It can't be done in this mode. Can we do it in a wider mode? */
2043 if (CLASS_HAS_WIDER_MODES_P (mclass
))
2045 FOR_EACH_WIDER_MODE (wider_mode
, mode
)
2047 if (optab_handler (binoptab
, wider_mode
) != CODE_FOR_nothing
)
2049 rtx t0
= gen_reg_rtx (wider_mode
);
2050 rtx t1
= gen_reg_rtx (wider_mode
);
2051 rtx cop0
= convert_modes (wider_mode
, mode
, op0
, unsignedp
);
2052 rtx cop1
= convert_modes (wider_mode
, mode
, op1
, unsignedp
);
2054 if (expand_twoval_binop (binoptab
, cop0
, cop1
,
2057 convert_move (targ0
, t0
, unsignedp
);
2058 convert_move (targ1
, t1
, unsignedp
);
2062 delete_insns_since (last
);
2067 delete_insns_since (entry_last
);
2071 /* Expand the two-valued library call indicated by BINOPTAB, but
2072 preserve only one of the values. If TARG0 is non-NULL, the first
2073 value is placed into TARG0; otherwise the second value is placed
2074 into TARG1. Exactly one of TARG0 and TARG1 must be non-NULL. The
2075 value stored into TARG0 or TARG1 is equivalent to (CODE OP0 OP1).
2076 This routine assumes that the value returned by the library call is
2077 as if the return value was of an integral mode twice as wide as the
2078 mode of OP0. Returns 1 if the call was successful. */
2081 expand_twoval_binop_libfunc (optab binoptab
, rtx op0
, rtx op1
,
2082 rtx targ0
, rtx targ1
, enum rtx_code code
)
2085 machine_mode libval_mode
;
2090 /* Exactly one of TARG0 or TARG1 should be non-NULL. */
2091 gcc_assert (!targ0
!= !targ1
);
2093 mode
= GET_MODE (op0
);
2094 libfunc
= optab_libfunc (binoptab
, mode
);
2098 /* The value returned by the library function will have twice as
2099 many bits as the nominal MODE. */
2100 libval_mode
= smallest_int_mode_for_size (2 * GET_MODE_BITSIZE (mode
));
2102 libval
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
,
2106 /* Get the part of VAL containing the value that we want. */
2107 libval
= simplify_gen_subreg (mode
, libval
, libval_mode
,
2108 targ0
? 0 : GET_MODE_SIZE (mode
));
2109 insns
= get_insns ();
2111 /* Move the into the desired location. */
2112 emit_libcall_block (insns
, targ0
? targ0
: targ1
, libval
,
2113 gen_rtx_fmt_ee (code
, mode
, op0
, op1
));
2119 /* Wrapper around expand_unop which takes an rtx code to specify
2120 the operation to perform, not an optab pointer. All other
2121 arguments are the same. */
2123 expand_simple_unop (machine_mode mode
, enum rtx_code code
, rtx op0
,
2124 rtx target
, int unsignedp
)
2126 optab unop
= code_to_optab (code
);
2129 return expand_unop (mode
, unop
, op0
, target
, unsignedp
);
2135 (clz:wide (zero_extend:wide x)) - ((width wide) - (width narrow)).
2137 A similar operation can be used for clrsb. UNOPTAB says which operation
2138 we are trying to expand. */
2140 widen_leading (scalar_int_mode mode
, rtx op0
, rtx target
, optab unoptab
)
2142 opt_scalar_int_mode wider_mode_iter
;
2143 FOR_EACH_WIDER_MODE (wider_mode_iter
, mode
)
2145 scalar_int_mode wider_mode
= wider_mode_iter
.require ();
2146 if (optab_handler (unoptab
, wider_mode
) != CODE_FOR_nothing
)
2151 last
= get_last_insn ();
2154 target
= gen_reg_rtx (mode
);
2155 xop0
= widen_operand (op0
, wider_mode
, mode
,
2156 unoptab
!= clrsb_optab
, false);
2157 temp
= expand_unop (wider_mode
, unoptab
, xop0
, NULL_RTX
,
2158 unoptab
!= clrsb_optab
);
2161 (wider_mode
, sub_optab
, temp
,
2162 gen_int_mode (GET_MODE_PRECISION (wider_mode
)
2163 - GET_MODE_PRECISION (mode
),
2165 target
, true, OPTAB_DIRECT
);
2167 delete_insns_since (last
);
2175 /* Try calculating clz of a double-word quantity as two clz's of word-sized
2176 quantities, choosing which based on whether the high word is nonzero. */
2178 expand_doubleword_clz (scalar_int_mode mode
, rtx op0
, rtx target
)
2180 rtx xop0
= force_reg (mode
, op0
);
2181 rtx subhi
= gen_highpart (word_mode
, xop0
);
2182 rtx sublo
= gen_lowpart (word_mode
, xop0
);
2183 rtx_code_label
*hi0_label
= gen_label_rtx ();
2184 rtx_code_label
*after_label
= gen_label_rtx ();
2188 /* If we were not given a target, use a word_mode register, not a
2189 'mode' register. The result will fit, and nobody is expecting
2190 anything bigger (the return type of __builtin_clz* is int). */
2192 target
= gen_reg_rtx (word_mode
);
2194 /* In any case, write to a word_mode scratch in both branches of the
2195 conditional, so we can ensure there is a single move insn setting
2196 'target' to tag a REG_EQUAL note on. */
2197 result
= gen_reg_rtx (word_mode
);
2201 /* If the high word is not equal to zero,
2202 then clz of the full value is clz of the high word. */
2203 emit_cmp_and_jump_insns (subhi
, CONST0_RTX (word_mode
), EQ
, 0,
2204 word_mode
, true, hi0_label
);
2206 temp
= expand_unop_direct (word_mode
, clz_optab
, subhi
, result
, true);
2211 convert_move (result
, temp
, true);
2213 emit_jump_insn (targetm
.gen_jump (after_label
));
2216 /* Else clz of the full value is clz of the low word plus the number
2217 of bits in the high word. */
2218 emit_label (hi0_label
);
2220 temp
= expand_unop_direct (word_mode
, clz_optab
, sublo
, 0, true);
2223 temp
= expand_binop (word_mode
, add_optab
, temp
,
2224 gen_int_mode (GET_MODE_BITSIZE (word_mode
), word_mode
),
2225 result
, true, OPTAB_DIRECT
);
2229 convert_move (result
, temp
, true);
2231 emit_label (after_label
);
2232 convert_move (target
, result
, true);
2237 add_equal_note (seq
, target
, CLZ
, xop0
, 0);
2246 /* Try calculating popcount of a double-word quantity as two popcount's of
2247 word-sized quantities and summing up the results. */
2249 expand_doubleword_popcount (scalar_int_mode mode
, rtx op0
, rtx target
)
2256 t0
= expand_unop_direct (word_mode
, popcount_optab
,
2257 operand_subword_force (op0
, 0, mode
), NULL_RTX
,
2259 t1
= expand_unop_direct (word_mode
, popcount_optab
,
2260 operand_subword_force (op0
, 1, mode
), NULL_RTX
,
2268 /* If we were not given a target, use a word_mode register, not a
2269 'mode' register. The result will fit, and nobody is expecting
2270 anything bigger (the return type of __builtin_popcount* is int). */
2272 target
= gen_reg_rtx (word_mode
);
2274 t
= expand_binop (word_mode
, add_optab
, t0
, t1
, target
, 0, OPTAB_DIRECT
);
2279 add_equal_note (seq
, t
, POPCOUNT
, op0
, 0);
2287 (parity:narrow (low (x) ^ high (x))) */
2289 expand_doubleword_parity (scalar_int_mode mode
, rtx op0
, rtx target
)
2291 rtx t
= expand_binop (word_mode
, xor_optab
,
2292 operand_subword_force (op0
, 0, mode
),
2293 operand_subword_force (op0
, 1, mode
),
2294 NULL_RTX
, 0, OPTAB_DIRECT
);
2295 return expand_unop (word_mode
, parity_optab
, t
, target
, true);
2301 (lshiftrt:wide (bswap:wide x) ((width wide) - (width narrow))). */
2303 widen_bswap (scalar_int_mode mode
, rtx op0
, rtx target
)
2307 opt_scalar_int_mode wider_mode_iter
;
2309 FOR_EACH_WIDER_MODE (wider_mode_iter
, mode
)
2310 if (optab_handler (bswap_optab
, wider_mode_iter
.require ())
2311 != CODE_FOR_nothing
)
2314 if (!wider_mode_iter
.exists ())
2317 scalar_int_mode wider_mode
= wider_mode_iter
.require ();
2318 last
= get_last_insn ();
2320 x
= widen_operand (op0
, wider_mode
, mode
, true, true);
2321 x
= expand_unop (wider_mode
, bswap_optab
, x
, NULL_RTX
, true);
2323 gcc_assert (GET_MODE_PRECISION (wider_mode
) == GET_MODE_BITSIZE (wider_mode
)
2324 && GET_MODE_PRECISION (mode
) == GET_MODE_BITSIZE (mode
));
2326 x
= expand_shift (RSHIFT_EXPR
, wider_mode
, x
,
2327 GET_MODE_BITSIZE (wider_mode
)
2328 - GET_MODE_BITSIZE (mode
),
2334 target
= gen_reg_rtx (mode
);
2335 emit_move_insn (target
, gen_lowpart (mode
, x
));
2338 delete_insns_since (last
);
2343 /* Try calculating bswap as two bswaps of two word-sized operands. */
2346 expand_doubleword_bswap (machine_mode mode
, rtx op
, rtx target
)
2350 t1
= expand_unop (word_mode
, bswap_optab
,
2351 operand_subword_force (op
, 0, mode
), NULL_RTX
, true);
2352 t0
= expand_unop (word_mode
, bswap_optab
,
2353 operand_subword_force (op
, 1, mode
), NULL_RTX
, true);
2355 if (target
== 0 || !valid_multiword_target_p (target
))
2356 target
= gen_reg_rtx (mode
);
2358 emit_clobber (target
);
2359 emit_move_insn (operand_subword (target
, 0, 1, mode
), t0
);
2360 emit_move_insn (operand_subword (target
, 1, 1, mode
), t1
);
2365 /* Try calculating (parity x) as (and (popcount x) 1), where
2366 popcount can also be done in a wider mode. */
2368 expand_parity (scalar_int_mode mode
, rtx op0
, rtx target
)
2370 enum mode_class mclass
= GET_MODE_CLASS (mode
);
2371 opt_scalar_int_mode wider_mode_iter
;
2372 FOR_EACH_MODE_FROM (wider_mode_iter
, mode
)
2374 scalar_int_mode wider_mode
= wider_mode_iter
.require ();
2375 if (optab_handler (popcount_optab
, wider_mode
) != CODE_FOR_nothing
)
2380 last
= get_last_insn ();
2382 if (target
== 0 || GET_MODE (target
) != wider_mode
)
2383 target
= gen_reg_rtx (wider_mode
);
2385 xop0
= widen_operand (op0
, wider_mode
, mode
, true, false);
2386 temp
= expand_unop (wider_mode
, popcount_optab
, xop0
, NULL_RTX
,
2389 temp
= expand_binop (wider_mode
, and_optab
, temp
, const1_rtx
,
2390 target
, true, OPTAB_DIRECT
);
2394 if (mclass
!= MODE_INT
2395 || !TRULY_NOOP_TRUNCATION_MODES_P (mode
, wider_mode
))
2396 return convert_to_mode (mode
, temp
, 0);
2398 return gen_lowpart (mode
, temp
);
2401 delete_insns_since (last
);
2407 /* Try calculating ctz(x) as K - clz(x & -x) ,
2408 where K is GET_MODE_PRECISION(mode) - 1.
2410 Both __builtin_ctz and __builtin_clz are undefined at zero, so we
2411 don't have to worry about what the hardware does in that case. (If
2412 the clz instruction produces the usual value at 0, which is K, the
2413 result of this code sequence will be -1; expand_ffs, below, relies
2414 on this. It might be nice to have it be K instead, for consistency
2415 with the (very few) processors that provide a ctz with a defined
2416 value, but that would take one more instruction, and it would be
2417 less convenient for expand_ffs anyway. */
2420 expand_ctz (scalar_int_mode mode
, rtx op0
, rtx target
)
2425 if (optab_handler (clz_optab
, mode
) == CODE_FOR_nothing
)
2430 temp
= expand_unop_direct (mode
, neg_optab
, op0
, NULL_RTX
, true);
2432 temp
= expand_binop (mode
, and_optab
, op0
, temp
, NULL_RTX
,
2433 true, OPTAB_DIRECT
);
2435 temp
= expand_unop_direct (mode
, clz_optab
, temp
, NULL_RTX
, true);
2437 temp
= expand_binop (mode
, sub_optab
,
2438 gen_int_mode (GET_MODE_PRECISION (mode
) - 1, mode
),
2440 true, OPTAB_DIRECT
);
2450 add_equal_note (seq
, temp
, CTZ
, op0
, 0);
2456 /* Try calculating ffs(x) using ctz(x) if we have that instruction, or
2457 else with the sequence used by expand_clz.
2459 The ffs builtin promises to return zero for a zero value and ctz/clz
2460 may have an undefined value in that case. If they do not give us a
2461 convenient value, we have to generate a test and branch. */
2463 expand_ffs (scalar_int_mode mode
, rtx op0
, rtx target
)
2465 HOST_WIDE_INT val
= 0;
2466 bool defined_at_zero
= false;
2470 if (optab_handler (ctz_optab
, mode
) != CODE_FOR_nothing
)
2474 temp
= expand_unop_direct (mode
, ctz_optab
, op0
, 0, true);
2478 defined_at_zero
= (CTZ_DEFINED_VALUE_AT_ZERO (mode
, val
) == 2);
2480 else if (optab_handler (clz_optab
, mode
) != CODE_FOR_nothing
)
2483 temp
= expand_ctz (mode
, op0
, 0);
2487 if (CLZ_DEFINED_VALUE_AT_ZERO (mode
, val
) == 2)
2489 defined_at_zero
= true;
2490 val
= (GET_MODE_PRECISION (mode
) - 1) - val
;
2496 if (defined_at_zero
&& val
== -1)
2497 /* No correction needed at zero. */;
2500 /* We don't try to do anything clever with the situation found
2501 on some processors (eg Alpha) where ctz(0:mode) ==
2502 bitsize(mode). If someone can think of a way to send N to -1
2503 and leave alone all values in the range 0..N-1 (where N is a
2504 power of two), cheaper than this test-and-branch, please add it.
2506 The test-and-branch is done after the operation itself, in case
2507 the operation sets condition codes that can be recycled for this.
2508 (This is true on i386, for instance.) */
2510 rtx_code_label
*nonzero_label
= gen_label_rtx ();
2511 emit_cmp_and_jump_insns (op0
, CONST0_RTX (mode
), NE
, 0,
2512 mode
, true, nonzero_label
);
2514 convert_move (temp
, GEN_INT (-1), false);
2515 emit_label (nonzero_label
);
2518 /* temp now has a value in the range -1..bitsize-1. ffs is supposed
2519 to produce a value in the range 0..bitsize. */
2520 temp
= expand_binop (mode
, add_optab
, temp
, gen_int_mode (1, mode
),
2521 target
, false, OPTAB_DIRECT
);
2528 add_equal_note (seq
, temp
, FFS
, op0
, 0);
2537 /* Extract the OMODE lowpart from VAL, which has IMODE. Under certain
2538 conditions, VAL may already be a SUBREG against which we cannot generate
2539 a further SUBREG. In this case, we expect forcing the value into a
2540 register will work around the situation. */
2543 lowpart_subreg_maybe_copy (machine_mode omode
, rtx val
,
2547 ret
= lowpart_subreg (omode
, val
, imode
);
2550 val
= force_reg (imode
, val
);
2551 ret
= lowpart_subreg (omode
, val
, imode
);
2552 gcc_assert (ret
!= NULL
);
2557 /* Expand a floating point absolute value or negation operation via a
2558 logical operation on the sign bit. */
2561 expand_absneg_bit (enum rtx_code code
, scalar_float_mode mode
,
2562 rtx op0
, rtx target
)
2564 const struct real_format
*fmt
;
2565 int bitpos
, word
, nwords
, i
;
2566 scalar_int_mode imode
;
2570 /* The format has to have a simple sign bit. */
2571 fmt
= REAL_MODE_FORMAT (mode
);
2575 bitpos
= fmt
->signbit_rw
;
2579 /* Don't create negative zeros if the format doesn't support them. */
2580 if (code
== NEG
&& !fmt
->has_signed_zero
)
2583 if (GET_MODE_SIZE (mode
) <= UNITS_PER_WORD
)
2585 if (!int_mode_for_mode (mode
).exists (&imode
))
2594 if (FLOAT_WORDS_BIG_ENDIAN
)
2595 word
= (GET_MODE_BITSIZE (mode
) - bitpos
) / BITS_PER_WORD
;
2597 word
= bitpos
/ BITS_PER_WORD
;
2598 bitpos
= bitpos
% BITS_PER_WORD
;
2599 nwords
= (GET_MODE_BITSIZE (mode
) + BITS_PER_WORD
- 1) / BITS_PER_WORD
;
2602 wide_int mask
= wi::set_bit_in_zero (bitpos
, GET_MODE_PRECISION (imode
));
2608 || (nwords
> 1 && !valid_multiword_target_p (target
)))
2609 target
= gen_reg_rtx (mode
);
2615 for (i
= 0; i
< nwords
; ++i
)
2617 rtx targ_piece
= operand_subword (target
, i
, 1, mode
);
2618 rtx op0_piece
= operand_subword_force (op0
, i
, mode
);
2622 temp
= expand_binop (imode
, code
== ABS
? and_optab
: xor_optab
,
2624 immed_wide_int_const (mask
, imode
),
2625 targ_piece
, 1, OPTAB_LIB_WIDEN
);
2626 if (temp
!= targ_piece
)
2627 emit_move_insn (targ_piece
, temp
);
2630 emit_move_insn (targ_piece
, op0_piece
);
2633 insns
= get_insns ();
2640 temp
= expand_binop (imode
, code
== ABS
? and_optab
: xor_optab
,
2641 gen_lowpart (imode
, op0
),
2642 immed_wide_int_const (mask
, imode
),
2643 gen_lowpart (imode
, target
), 1, OPTAB_LIB_WIDEN
);
2644 target
= lowpart_subreg_maybe_copy (mode
, temp
, imode
);
2646 set_dst_reg_note (get_last_insn (), REG_EQUAL
,
2647 gen_rtx_fmt_e (code
, mode
, copy_rtx (op0
)),
2654 /* As expand_unop, but will fail rather than attempt the operation in a
2655 different mode or with a libcall. */
2657 expand_unop_direct (machine_mode mode
, optab unoptab
, rtx op0
, rtx target
,
2660 if (optab_handler (unoptab
, mode
) != CODE_FOR_nothing
)
2662 struct expand_operand ops
[2];
2663 enum insn_code icode
= optab_handler (unoptab
, mode
);
2664 rtx_insn
*last
= get_last_insn ();
2667 create_output_operand (&ops
[0], target
, mode
);
2668 create_convert_operand_from (&ops
[1], op0
, mode
, unsignedp
);
2669 pat
= maybe_gen_insn (icode
, 2, ops
);
2672 if (INSN_P (pat
) && NEXT_INSN (pat
) != NULL_RTX
2673 && ! add_equal_note (pat
, ops
[0].value
,
2674 optab_to_code (unoptab
),
2675 ops
[1].value
, NULL_RTX
))
2677 delete_insns_since (last
);
2678 return expand_unop (mode
, unoptab
, op0
, NULL_RTX
, unsignedp
);
2683 return ops
[0].value
;
2689 /* Generate code to perform an operation specified by UNOPTAB
2690 on operand OP0, with result having machine-mode MODE.
2692 UNSIGNEDP is for the case where we have to widen the operands
2693 to perform the operation. It says to use zero-extension.
2695 If TARGET is nonzero, the value
2696 is generated there, if it is convenient to do so.
2697 In all cases an rtx is returned for the locus of the value;
2698 this may or may not be TARGET. */
2701 expand_unop (machine_mode mode
, optab unoptab
, rtx op0
, rtx target
,
2704 enum mode_class mclass
= GET_MODE_CLASS (mode
);
2705 machine_mode wider_mode
;
2706 scalar_int_mode int_mode
;
2707 scalar_float_mode float_mode
;
2711 temp
= expand_unop_direct (mode
, unoptab
, op0
, target
, unsignedp
);
2715 /* It can't be done in this mode. Can we open-code it in a wider mode? */
2717 /* Widening (or narrowing) clz needs special treatment. */
2718 if (unoptab
== clz_optab
)
2720 if (is_a
<scalar_int_mode
> (mode
, &int_mode
))
2722 temp
= widen_leading (int_mode
, op0
, target
, unoptab
);
2726 if (GET_MODE_SIZE (int_mode
) == 2 * UNITS_PER_WORD
2727 && optab_handler (unoptab
, word_mode
) != CODE_FOR_nothing
)
2729 temp
= expand_doubleword_clz (int_mode
, op0
, target
);
2738 if (unoptab
== clrsb_optab
)
2740 if (is_a
<scalar_int_mode
> (mode
, &int_mode
))
2742 temp
= widen_leading (int_mode
, op0
, target
, unoptab
);
2749 if (unoptab
== popcount_optab
2750 && is_a
<scalar_int_mode
> (mode
, &int_mode
)
2751 && GET_MODE_SIZE (int_mode
) == 2 * UNITS_PER_WORD
2752 && optab_handler (unoptab
, word_mode
) != CODE_FOR_nothing
2753 && optimize_insn_for_speed_p ())
2755 temp
= expand_doubleword_popcount (int_mode
, op0
, target
);
2760 if (unoptab
== parity_optab
2761 && is_a
<scalar_int_mode
> (mode
, &int_mode
)
2762 && GET_MODE_SIZE (int_mode
) == 2 * UNITS_PER_WORD
2763 && (optab_handler (unoptab
, word_mode
) != CODE_FOR_nothing
2764 || optab_handler (popcount_optab
, word_mode
) != CODE_FOR_nothing
)
2765 && optimize_insn_for_speed_p ())
2767 temp
= expand_doubleword_parity (int_mode
, op0
, target
);
2772 /* Widening (or narrowing) bswap needs special treatment. */
2773 if (unoptab
== bswap_optab
)
2775 /* HImode is special because in this mode BSWAP is equivalent to ROTATE
2776 or ROTATERT. First try these directly; if this fails, then try the
2777 obvious pair of shifts with allowed widening, as this will probably
2778 be always more efficient than the other fallback methods. */
2784 if (optab_handler (rotl_optab
, mode
) != CODE_FOR_nothing
)
2786 temp
= expand_binop (mode
, rotl_optab
, op0
, GEN_INT (8), target
,
2787 unsignedp
, OPTAB_DIRECT
);
2792 if (optab_handler (rotr_optab
, mode
) != CODE_FOR_nothing
)
2794 temp
= expand_binop (mode
, rotr_optab
, op0
, GEN_INT (8), target
,
2795 unsignedp
, OPTAB_DIRECT
);
2800 last
= get_last_insn ();
2802 temp1
= expand_binop (mode
, ashl_optab
, op0
, GEN_INT (8), NULL_RTX
,
2803 unsignedp
, OPTAB_WIDEN
);
2804 temp2
= expand_binop (mode
, lshr_optab
, op0
, GEN_INT (8), NULL_RTX
,
2805 unsignedp
, OPTAB_WIDEN
);
2808 temp
= expand_binop (mode
, ior_optab
, temp1
, temp2
, target
,
2809 unsignedp
, OPTAB_WIDEN
);
2814 delete_insns_since (last
);
2817 if (is_a
<scalar_int_mode
> (mode
, &int_mode
))
2819 temp
= widen_bswap (int_mode
, op0
, target
);
2823 if (GET_MODE_SIZE (int_mode
) == 2 * UNITS_PER_WORD
2824 && optab_handler (unoptab
, word_mode
) != CODE_FOR_nothing
)
2826 temp
= expand_doubleword_bswap (mode
, op0
, target
);
2835 if (CLASS_HAS_WIDER_MODES_P (mclass
))
2836 FOR_EACH_WIDER_MODE (wider_mode
, mode
)
2838 if (optab_handler (unoptab
, wider_mode
) != CODE_FOR_nothing
)
2841 rtx_insn
*last
= get_last_insn ();
2843 /* For certain operations, we need not actually extend
2844 the narrow operand, as long as we will truncate the
2845 results to the same narrowness. */
2847 xop0
= widen_operand (xop0
, wider_mode
, mode
, unsignedp
,
2848 (unoptab
== neg_optab
2849 || unoptab
== one_cmpl_optab
)
2850 && mclass
== MODE_INT
);
2852 temp
= expand_unop (wider_mode
, unoptab
, xop0
, NULL_RTX
,
2857 if (mclass
!= MODE_INT
2858 || !TRULY_NOOP_TRUNCATION_MODES_P (mode
, wider_mode
))
2861 target
= gen_reg_rtx (mode
);
2862 convert_move (target
, temp
, 0);
2866 return gen_lowpart (mode
, temp
);
2869 delete_insns_since (last
);
2873 /* These can be done a word at a time. */
2874 if (unoptab
== one_cmpl_optab
2875 && is_int_mode (mode
, &int_mode
)
2876 && GET_MODE_SIZE (int_mode
) > UNITS_PER_WORD
2877 && optab_handler (unoptab
, word_mode
) != CODE_FOR_nothing
)
2882 if (target
== 0 || target
== op0
|| !valid_multiword_target_p (target
))
2883 target
= gen_reg_rtx (int_mode
);
2887 /* Do the actual arithmetic. */
2888 for (i
= 0; i
< GET_MODE_BITSIZE (int_mode
) / BITS_PER_WORD
; i
++)
2890 rtx target_piece
= operand_subword (target
, i
, 1, int_mode
);
2891 rtx x
= expand_unop (word_mode
, unoptab
,
2892 operand_subword_force (op0
, i
, int_mode
),
2893 target_piece
, unsignedp
);
2895 if (target_piece
!= x
)
2896 emit_move_insn (target_piece
, x
);
2899 insns
= get_insns ();
2906 if (optab_to_code (unoptab
) == NEG
)
2908 /* Try negating floating point values by flipping the sign bit. */
2909 if (is_a
<scalar_float_mode
> (mode
, &float_mode
))
2911 temp
= expand_absneg_bit (NEG
, float_mode
, op0
, target
);
2916 /* If there is no negation pattern, and we have no negative zero,
2917 try subtracting from zero. */
2918 if (!HONOR_SIGNED_ZEROS (mode
))
2920 temp
= expand_binop (mode
, (unoptab
== negv_optab
2921 ? subv_optab
: sub_optab
),
2922 CONST0_RTX (mode
), op0
, target
,
2923 unsignedp
, OPTAB_DIRECT
);
2929 /* Try calculating parity (x) as popcount (x) % 2. */
2930 if (unoptab
== parity_optab
&& is_a
<scalar_int_mode
> (mode
, &int_mode
))
2932 temp
= expand_parity (int_mode
, op0
, target
);
2937 /* Try implementing ffs (x) in terms of clz (x). */
2938 if (unoptab
== ffs_optab
&& is_a
<scalar_int_mode
> (mode
, &int_mode
))
2940 temp
= expand_ffs (int_mode
, op0
, target
);
2945 /* Try implementing ctz (x) in terms of clz (x). */
2946 if (unoptab
== ctz_optab
&& is_a
<scalar_int_mode
> (mode
, &int_mode
))
2948 temp
= expand_ctz (int_mode
, op0
, target
);
2954 /* Now try a library call in this mode. */
2955 libfunc
= optab_libfunc (unoptab
, mode
);
2961 machine_mode outmode
= mode
;
2963 /* All of these functions return small values. Thus we choose to
2964 have them return something that isn't a double-word. */
2965 if (unoptab
== ffs_optab
|| unoptab
== clz_optab
|| unoptab
== ctz_optab
2966 || unoptab
== clrsb_optab
|| unoptab
== popcount_optab
2967 || unoptab
== parity_optab
)
2969 = GET_MODE (hard_libcall_value (TYPE_MODE (integer_type_node
),
2970 optab_libfunc (unoptab
, mode
)));
2974 /* Pass 1 for NO_QUEUE so we don't lose any increments
2975 if the libcall is cse'd or moved. */
2976 value
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
, outmode
,
2978 insns
= get_insns ();
2981 target
= gen_reg_rtx (outmode
);
2982 bool trapv
= trapv_unoptab_p (unoptab
);
2984 eq_value
= NULL_RTX
;
2987 eq_value
= gen_rtx_fmt_e (optab_to_code (unoptab
), mode
, op0
);
2988 if (GET_MODE_UNIT_SIZE (outmode
) < GET_MODE_UNIT_SIZE (mode
))
2989 eq_value
= simplify_gen_unary (TRUNCATE
, outmode
, eq_value
, mode
);
2990 else if (GET_MODE_UNIT_SIZE (outmode
) > GET_MODE_UNIT_SIZE (mode
))
2991 eq_value
= simplify_gen_unary (ZERO_EXTEND
,
2992 outmode
, eq_value
, mode
);
2994 emit_libcall_block_1 (insns
, target
, value
, eq_value
, trapv
);
2999 /* It can't be done in this mode. Can we do it in a wider mode? */
3001 if (CLASS_HAS_WIDER_MODES_P (mclass
))
3003 FOR_EACH_WIDER_MODE (wider_mode
, mode
)
3005 if (optab_handler (unoptab
, wider_mode
) != CODE_FOR_nothing
3006 || optab_libfunc (unoptab
, wider_mode
))
3009 rtx_insn
*last
= get_last_insn ();
3011 /* For certain operations, we need not actually extend
3012 the narrow operand, as long as we will truncate the
3013 results to the same narrowness. */
3014 xop0
= widen_operand (xop0
, wider_mode
, mode
, unsignedp
,
3015 (unoptab
== neg_optab
3016 || unoptab
== one_cmpl_optab
3017 || unoptab
== bswap_optab
)
3018 && mclass
== MODE_INT
);
3020 temp
= expand_unop (wider_mode
, unoptab
, xop0
, NULL_RTX
,
3023 /* If we are generating clz using wider mode, adjust the
3024 result. Similarly for clrsb. */
3025 if ((unoptab
== clz_optab
|| unoptab
== clrsb_optab
)
3028 scalar_int_mode wider_int_mode
3029 = as_a
<scalar_int_mode
> (wider_mode
);
3030 int_mode
= as_a
<scalar_int_mode
> (mode
);
3032 (wider_mode
, sub_optab
, temp
,
3033 gen_int_mode (GET_MODE_PRECISION (wider_int_mode
)
3034 - GET_MODE_PRECISION (int_mode
),
3036 target
, true, OPTAB_DIRECT
);
3039 /* Likewise for bswap. */
3040 if (unoptab
== bswap_optab
&& temp
!= 0)
3042 scalar_int_mode wider_int_mode
3043 = as_a
<scalar_int_mode
> (wider_mode
);
3044 int_mode
= as_a
<scalar_int_mode
> (mode
);
3045 gcc_assert (GET_MODE_PRECISION (wider_int_mode
)
3046 == GET_MODE_BITSIZE (wider_int_mode
)
3047 && GET_MODE_PRECISION (int_mode
)
3048 == GET_MODE_BITSIZE (int_mode
));
3050 temp
= expand_shift (RSHIFT_EXPR
, wider_int_mode
, temp
,
3051 GET_MODE_BITSIZE (wider_int_mode
)
3052 - GET_MODE_BITSIZE (int_mode
),
3058 if (mclass
!= MODE_INT
)
3061 target
= gen_reg_rtx (mode
);
3062 convert_move (target
, temp
, 0);
3066 return gen_lowpart (mode
, temp
);
3069 delete_insns_since (last
);
3074 /* One final attempt at implementing negation via subtraction,
3075 this time allowing widening of the operand. */
3076 if (optab_to_code (unoptab
) == NEG
&& !HONOR_SIGNED_ZEROS (mode
))
3079 temp
= expand_binop (mode
,
3080 unoptab
== negv_optab
? subv_optab
: sub_optab
,
3081 CONST0_RTX (mode
), op0
,
3082 target
, unsignedp
, OPTAB_LIB_WIDEN
);
3090 /* Emit code to compute the absolute value of OP0, with result to
3091 TARGET if convenient. (TARGET may be 0.) The return value says
3092 where the result actually is to be found.
3094 MODE is the mode of the operand; the mode of the result is
3095 different but can be deduced from MODE.
3100 expand_abs_nojump (machine_mode mode
, rtx op0
, rtx target
,
3101 int result_unsignedp
)
3105 if (GET_MODE_CLASS (mode
) != MODE_INT
3107 result_unsignedp
= 1;
3109 /* First try to do it with a special abs instruction. */
3110 temp
= expand_unop (mode
, result_unsignedp
? abs_optab
: absv_optab
,
3115 /* For floating point modes, try clearing the sign bit. */
3116 scalar_float_mode float_mode
;
3117 if (is_a
<scalar_float_mode
> (mode
, &float_mode
))
3119 temp
= expand_absneg_bit (ABS
, float_mode
, op0
, target
);
3124 /* If we have a MAX insn, we can do this as MAX (x, -x). */
3125 if (optab_handler (smax_optab
, mode
) != CODE_FOR_nothing
3126 && !HONOR_SIGNED_ZEROS (mode
))
3128 rtx_insn
*last
= get_last_insn ();
3130 temp
= expand_unop (mode
, result_unsignedp
? neg_optab
: negv_optab
,
3133 temp
= expand_binop (mode
, smax_optab
, op0
, temp
, target
, 0,
3139 delete_insns_since (last
);
3142 /* If this machine has expensive jumps, we can do integer absolute
3143 value of X as (((signed) x >> (W-1)) ^ x) - ((signed) x >> (W-1)),
3144 where W is the width of MODE. */
3146 scalar_int_mode int_mode
;
3147 if (is_int_mode (mode
, &int_mode
)
3148 && BRANCH_COST (optimize_insn_for_speed_p (),
3151 rtx extended
= expand_shift (RSHIFT_EXPR
, int_mode
, op0
,
3152 GET_MODE_PRECISION (int_mode
) - 1,
3155 temp
= expand_binop (int_mode
, xor_optab
, extended
, op0
, target
, 0,
3158 temp
= expand_binop (int_mode
,
3159 result_unsignedp
? sub_optab
: subv_optab
,
3160 temp
, extended
, target
, 0, OPTAB_LIB_WIDEN
);
3170 expand_abs (machine_mode mode
, rtx op0
, rtx target
,
3171 int result_unsignedp
, int safe
)
3174 rtx_code_label
*op1
;
3176 if (GET_MODE_CLASS (mode
) != MODE_INT
3178 result_unsignedp
= 1;
3180 temp
= expand_abs_nojump (mode
, op0
, target
, result_unsignedp
);
3184 /* If that does not win, use conditional jump and negate. */
3186 /* It is safe to use the target if it is the same
3187 as the source if this is also a pseudo register */
3188 if (op0
== target
&& REG_P (op0
)
3189 && REGNO (op0
) >= FIRST_PSEUDO_REGISTER
)
3192 op1
= gen_label_rtx ();
3193 if (target
== 0 || ! safe
3194 || GET_MODE (target
) != mode
3195 || (MEM_P (target
) && MEM_VOLATILE_P (target
))
3197 && REGNO (target
) < FIRST_PSEUDO_REGISTER
))
3198 target
= gen_reg_rtx (mode
);
3200 emit_move_insn (target
, op0
);
3203 do_compare_rtx_and_jump (target
, CONST0_RTX (mode
), GE
, 0, mode
,
3204 NULL_RTX
, NULL
, op1
,
3205 profile_probability::uninitialized ());
3207 op0
= expand_unop (mode
, result_unsignedp
? neg_optab
: negv_optab
,
3210 emit_move_insn (target
, op0
);
3216 /* Emit code to compute the one's complement absolute value of OP0
3217 (if (OP0 < 0) OP0 = ~OP0), with result to TARGET if convenient.
3218 (TARGET may be NULL_RTX.) The return value says where the result
3219 actually is to be found.
3221 MODE is the mode of the operand; the mode of the result is
3222 different but can be deduced from MODE. */
3225 expand_one_cmpl_abs_nojump (machine_mode mode
, rtx op0
, rtx target
)
3229 /* Not applicable for floating point modes. */
3230 if (FLOAT_MODE_P (mode
))
3233 /* If we have a MAX insn, we can do this as MAX (x, ~x). */
3234 if (optab_handler (smax_optab
, mode
) != CODE_FOR_nothing
)
3236 rtx_insn
*last
= get_last_insn ();
3238 temp
= expand_unop (mode
, one_cmpl_optab
, op0
, NULL_RTX
, 0);
3240 temp
= expand_binop (mode
, smax_optab
, op0
, temp
, target
, 0,
3246 delete_insns_since (last
);
3249 /* If this machine has expensive jumps, we can do one's complement
3250 absolute value of X as (((signed) x >> (W-1)) ^ x). */
3252 scalar_int_mode int_mode
;
3253 if (is_int_mode (mode
, &int_mode
)
3254 && BRANCH_COST (optimize_insn_for_speed_p (),
3257 rtx extended
= expand_shift (RSHIFT_EXPR
, int_mode
, op0
,
3258 GET_MODE_PRECISION (int_mode
) - 1,
3261 temp
= expand_binop (int_mode
, xor_optab
, extended
, op0
, target
, 0,
3271 /* A subroutine of expand_copysign, perform the copysign operation using the
3272 abs and neg primitives advertised to exist on the target. The assumption
3273 is that we have a split register file, and leaving op0 in fp registers,
3274 and not playing with subregs so much, will help the register allocator. */
3277 expand_copysign_absneg (scalar_float_mode mode
, rtx op0
, rtx op1
, rtx target
,
3278 int bitpos
, bool op0_is_abs
)
3280 scalar_int_mode imode
;
3281 enum insn_code icode
;
3283 rtx_code_label
*label
;
3288 /* Check if the back end provides an insn that handles signbit for the
3290 icode
= optab_handler (signbit_optab
, mode
);
3291 if (icode
!= CODE_FOR_nothing
)
3293 imode
= as_a
<scalar_int_mode
> (insn_data
[(int) icode
].operand
[0].mode
);
3294 sign
= gen_reg_rtx (imode
);
3295 emit_unop_insn (icode
, sign
, op1
, UNKNOWN
);
3299 if (GET_MODE_SIZE (mode
) <= UNITS_PER_WORD
)
3301 if (!int_mode_for_mode (mode
).exists (&imode
))
3303 op1
= gen_lowpart (imode
, op1
);
3310 if (FLOAT_WORDS_BIG_ENDIAN
)
3311 word
= (GET_MODE_BITSIZE (mode
) - bitpos
) / BITS_PER_WORD
;
3313 word
= bitpos
/ BITS_PER_WORD
;
3314 bitpos
= bitpos
% BITS_PER_WORD
;
3315 op1
= operand_subword_force (op1
, word
, mode
);
3318 wide_int mask
= wi::set_bit_in_zero (bitpos
, GET_MODE_PRECISION (imode
));
3319 sign
= expand_binop (imode
, and_optab
, op1
,
3320 immed_wide_int_const (mask
, imode
),
3321 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
3326 op0
= expand_unop (mode
, abs_optab
, op0
, target
, 0);
3333 if (target
== NULL_RTX
)
3334 target
= copy_to_reg (op0
);
3336 emit_move_insn (target
, op0
);
3339 label
= gen_label_rtx ();
3340 emit_cmp_and_jump_insns (sign
, const0_rtx
, EQ
, NULL_RTX
, imode
, 1, label
);
3342 if (CONST_DOUBLE_AS_FLOAT_P (op0
))
3343 op0
= simplify_unary_operation (NEG
, mode
, op0
, mode
);
3345 op0
= expand_unop (mode
, neg_optab
, op0
, target
, 0);
3347 emit_move_insn (target
, op0
);
3355 /* A subroutine of expand_copysign, perform the entire copysign operation
3356 with integer bitmasks. BITPOS is the position of the sign bit; OP0_IS_ABS
3357 is true if op0 is known to have its sign bit clear. */
3360 expand_copysign_bit (scalar_float_mode mode
, rtx op0
, rtx op1
, rtx target
,
3361 int bitpos
, bool op0_is_abs
)
3363 scalar_int_mode imode
;
3364 int word
, nwords
, i
;
3368 if (GET_MODE_SIZE (mode
) <= UNITS_PER_WORD
)
3370 if (!int_mode_for_mode (mode
).exists (&imode
))
3379 if (FLOAT_WORDS_BIG_ENDIAN
)
3380 word
= (GET_MODE_BITSIZE (mode
) - bitpos
) / BITS_PER_WORD
;
3382 word
= bitpos
/ BITS_PER_WORD
;
3383 bitpos
= bitpos
% BITS_PER_WORD
;
3384 nwords
= (GET_MODE_BITSIZE (mode
) + BITS_PER_WORD
- 1) / BITS_PER_WORD
;
3387 wide_int mask
= wi::set_bit_in_zero (bitpos
, GET_MODE_PRECISION (imode
));
3392 || (nwords
> 1 && !valid_multiword_target_p (target
)))
3393 target
= gen_reg_rtx (mode
);
3399 for (i
= 0; i
< nwords
; ++i
)
3401 rtx targ_piece
= operand_subword (target
, i
, 1, mode
);
3402 rtx op0_piece
= operand_subword_force (op0
, i
, mode
);
3408 = expand_binop (imode
, and_optab
, op0_piece
,
3409 immed_wide_int_const (~mask
, imode
),
3410 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
3411 op1
= expand_binop (imode
, and_optab
,
3412 operand_subword_force (op1
, i
, mode
),
3413 immed_wide_int_const (mask
, imode
),
3414 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
3416 temp
= expand_binop (imode
, ior_optab
, op0_piece
, op1
,
3417 targ_piece
, 1, OPTAB_LIB_WIDEN
);
3418 if (temp
!= targ_piece
)
3419 emit_move_insn (targ_piece
, temp
);
3422 emit_move_insn (targ_piece
, op0_piece
);
3425 insns
= get_insns ();
3432 op1
= expand_binop (imode
, and_optab
, gen_lowpart (imode
, op1
),
3433 immed_wide_int_const (mask
, imode
),
3434 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
3436 op0
= gen_lowpart (imode
, op0
);
3438 op0
= expand_binop (imode
, and_optab
, op0
,
3439 immed_wide_int_const (~mask
, imode
),
3440 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
3442 temp
= expand_binop (imode
, ior_optab
, op0
, op1
,
3443 gen_lowpart (imode
, target
), 1, OPTAB_LIB_WIDEN
);
3444 target
= lowpart_subreg_maybe_copy (mode
, temp
, imode
);
3450 /* Expand the C99 copysign operation. OP0 and OP1 must be the same
3451 scalar floating point mode. Return NULL if we do not know how to
3452 expand the operation inline. */
3455 expand_copysign (rtx op0
, rtx op1
, rtx target
)
3457 scalar_float_mode mode
;
3458 const struct real_format
*fmt
;
3462 mode
= as_a
<scalar_float_mode
> (GET_MODE (op0
));
3463 gcc_assert (GET_MODE (op1
) == mode
);
3465 /* First try to do it with a special instruction. */
3466 temp
= expand_binop (mode
, copysign_optab
, op0
, op1
,
3467 target
, 0, OPTAB_DIRECT
);
3471 fmt
= REAL_MODE_FORMAT (mode
);
3472 if (fmt
== NULL
|| !fmt
->has_signed_zero
)
3476 if (CONST_DOUBLE_AS_FLOAT_P (op0
))
3478 if (real_isneg (CONST_DOUBLE_REAL_VALUE (op0
)))
3479 op0
= simplify_unary_operation (ABS
, mode
, op0
, mode
);
3483 if (fmt
->signbit_ro
>= 0
3484 && (CONST_DOUBLE_AS_FLOAT_P (op0
)
3485 || (optab_handler (neg_optab
, mode
) != CODE_FOR_nothing
3486 && optab_handler (abs_optab
, mode
) != CODE_FOR_nothing
)))
3488 temp
= expand_copysign_absneg (mode
, op0
, op1
, target
,
3489 fmt
->signbit_ro
, op0_is_abs
);
3494 if (fmt
->signbit_rw
< 0)
3496 return expand_copysign_bit (mode
, op0
, op1
, target
,
3497 fmt
->signbit_rw
, op0_is_abs
);
3500 /* Generate an instruction whose insn-code is INSN_CODE,
3501 with two operands: an output TARGET and an input OP0.
3502 TARGET *must* be nonzero, and the output is always stored there.
3503 CODE is an rtx code such that (CODE OP0) is an rtx that describes
3504 the value that is stored into TARGET.
3506 Return false if expansion failed. */
3509 maybe_emit_unop_insn (enum insn_code icode
, rtx target
, rtx op0
,
3512 struct expand_operand ops
[2];
3515 create_output_operand (&ops
[0], target
, GET_MODE (target
));
3516 create_input_operand (&ops
[1], op0
, GET_MODE (op0
));
3517 pat
= maybe_gen_insn (icode
, 2, ops
);
3521 if (INSN_P (pat
) && NEXT_INSN (pat
) != NULL_RTX
3523 add_equal_note (pat
, ops
[0].value
, code
, ops
[1].value
, NULL_RTX
);
3527 if (ops
[0].value
!= target
)
3528 emit_move_insn (target
, ops
[0].value
);
3531 /* Generate an instruction whose insn-code is INSN_CODE,
3532 with two operands: an output TARGET and an input OP0.
3533 TARGET *must* be nonzero, and the output is always stored there.
3534 CODE is an rtx code such that (CODE OP0) is an rtx that describes
3535 the value that is stored into TARGET. */
3538 emit_unop_insn (enum insn_code icode
, rtx target
, rtx op0
, enum rtx_code code
)
3540 bool ok
= maybe_emit_unop_insn (icode
, target
, op0
, code
);
3544 struct no_conflict_data
3547 rtx_insn
*first
, *insn
;
3551 /* Called via note_stores by emit_libcall_block. Set P->must_stay if
3552 the currently examined clobber / store has to stay in the list of
3553 insns that constitute the actual libcall block. */
3555 no_conflict_move_test (rtx dest
, const_rtx set
, void *p0
)
3557 struct no_conflict_data
*p
= (struct no_conflict_data
*) p0
;
3559 /* If this inns directly contributes to setting the target, it must stay. */
3560 if (reg_overlap_mentioned_p (p
->target
, dest
))
3561 p
->must_stay
= true;
3562 /* If we haven't committed to keeping any other insns in the list yet,
3563 there is nothing more to check. */
3564 else if (p
->insn
== p
->first
)
3566 /* If this insn sets / clobbers a register that feeds one of the insns
3567 already in the list, this insn has to stay too. */
3568 else if (reg_overlap_mentioned_p (dest
, PATTERN (p
->first
))
3569 || (CALL_P (p
->first
) && (find_reg_fusage (p
->first
, USE
, dest
)))
3570 || reg_used_between_p (dest
, p
->first
, p
->insn
)
3571 /* Likewise if this insn depends on a register set by a previous
3572 insn in the list, or if it sets a result (presumably a hard
3573 register) that is set or clobbered by a previous insn.
3574 N.B. the modified_*_p (SET_DEST...) tests applied to a MEM
3575 SET_DEST perform the former check on the address, and the latter
3576 check on the MEM. */
3577 || (GET_CODE (set
) == SET
3578 && (modified_in_p (SET_SRC (set
), p
->first
)
3579 || modified_in_p (SET_DEST (set
), p
->first
)
3580 || modified_between_p (SET_SRC (set
), p
->first
, p
->insn
)
3581 || modified_between_p (SET_DEST (set
), p
->first
, p
->insn
))))
3582 p
->must_stay
= true;
3586 /* Emit code to make a call to a constant function or a library call.
3588 INSNS is a list containing all insns emitted in the call.
3589 These insns leave the result in RESULT. Our block is to copy RESULT
3590 to TARGET, which is logically equivalent to EQUIV.
3592 We first emit any insns that set a pseudo on the assumption that these are
3593 loading constants into registers; doing so allows them to be safely cse'ed
3594 between blocks. Then we emit all the other insns in the block, followed by
3595 an insn to move RESULT to TARGET. This last insn will have a REQ_EQUAL
3596 note with an operand of EQUIV. */
3599 emit_libcall_block_1 (rtx_insn
*insns
, rtx target
, rtx result
, rtx equiv
,
3600 bool equiv_may_trap
)
3602 rtx final_dest
= target
;
3603 rtx_insn
*next
, *last
, *insn
;
3605 /* If this is a reg with REG_USERVAR_P set, then it could possibly turn
3606 into a MEM later. Protect the libcall block from this change. */
3607 if (! REG_P (target
) || REG_USERVAR_P (target
))
3608 target
= gen_reg_rtx (GET_MODE (target
));
3610 /* If we're using non-call exceptions, a libcall corresponding to an
3611 operation that may trap may also trap. */
3612 /* ??? See the comment in front of make_reg_eh_region_note. */
3613 if (cfun
->can_throw_non_call_exceptions
3614 && (equiv_may_trap
|| may_trap_p (equiv
)))
3616 for (insn
= insns
; insn
; insn
= NEXT_INSN (insn
))
3619 rtx note
= find_reg_note (insn
, REG_EH_REGION
, NULL_RTX
);
3622 int lp_nr
= INTVAL (XEXP (note
, 0));
3623 if (lp_nr
== 0 || lp_nr
== INT_MIN
)
3624 remove_note (insn
, note
);
3630 /* Look for any CALL_INSNs in this sequence, and attach a REG_EH_REGION
3631 reg note to indicate that this call cannot throw or execute a nonlocal
3632 goto (unless there is already a REG_EH_REGION note, in which case
3634 for (insn
= insns
; insn
; insn
= NEXT_INSN (insn
))
3636 make_reg_eh_region_note_nothrow_nononlocal (insn
);
3639 /* First emit all insns that set pseudos. Remove them from the list as
3640 we go. Avoid insns that set pseudos which were referenced in previous
3641 insns. These can be generated by move_by_pieces, for example,
3642 to update an address. Similarly, avoid insns that reference things
3643 set in previous insns. */
3645 for (insn
= insns
; insn
; insn
= next
)
3647 rtx set
= single_set (insn
);
3649 next
= NEXT_INSN (insn
);
3651 if (set
!= 0 && REG_P (SET_DEST (set
))
3652 && REGNO (SET_DEST (set
)) >= FIRST_PSEUDO_REGISTER
)
3654 struct no_conflict_data data
;
3656 data
.target
= const0_rtx
;
3660 note_stores (PATTERN (insn
), no_conflict_move_test
, &data
);
3661 if (! data
.must_stay
)
3663 if (PREV_INSN (insn
))
3664 SET_NEXT_INSN (PREV_INSN (insn
)) = next
;
3669 SET_PREV_INSN (next
) = PREV_INSN (insn
);
3675 /* Some ports use a loop to copy large arguments onto the stack.
3676 Don't move anything outside such a loop. */
3681 /* Write the remaining insns followed by the final copy. */
3682 for (insn
= insns
; insn
; insn
= next
)
3684 next
= NEXT_INSN (insn
);
3689 last
= emit_move_insn (target
, result
);
3691 set_dst_reg_note (last
, REG_EQUAL
, copy_rtx (equiv
), target
);
3693 if (final_dest
!= target
)
3694 emit_move_insn (final_dest
, target
);
3698 emit_libcall_block (rtx_insn
*insns
, rtx target
, rtx result
, rtx equiv
)
3700 emit_libcall_block_1 (insns
, target
, result
, equiv
, false);
3703 /* Nonzero if we can perform a comparison of mode MODE straightforwardly.
3704 PURPOSE describes how this comparison will be used. CODE is the rtx
3705 comparison code we will be using.
3707 ??? Actually, CODE is slightly weaker than that. A target is still
3708 required to implement all of the normal bcc operations, but not
3709 required to implement all (or any) of the unordered bcc operations. */
3712 can_compare_p (enum rtx_code code
, machine_mode mode
,
3713 enum can_compare_purpose purpose
)
3716 test
= gen_rtx_fmt_ee (code
, mode
, const0_rtx
, const0_rtx
);
3719 enum insn_code icode
;
3721 if (purpose
== ccp_jump
3722 && (icode
= optab_handler (cbranch_optab
, mode
)) != CODE_FOR_nothing
3723 && insn_operand_matches (icode
, 0, test
))
3725 if (purpose
== ccp_store_flag
3726 && (icode
= optab_handler (cstore_optab
, mode
)) != CODE_FOR_nothing
3727 && insn_operand_matches (icode
, 1, test
))
3729 if (purpose
== ccp_cmov
3730 && optab_handler (cmov_optab
, mode
) != CODE_FOR_nothing
)
3733 mode
= GET_MODE_WIDER_MODE (mode
).else_void ();
3734 PUT_MODE (test
, mode
);
3736 while (mode
!= VOIDmode
);
3741 /* This function is called when we are going to emit a compare instruction that
3742 compares the values found in X and Y, using the rtl operator COMPARISON.
3744 If they have mode BLKmode, then SIZE specifies the size of both operands.
3746 UNSIGNEDP nonzero says that the operands are unsigned;
3747 this matters if they need to be widened (as given by METHODS).
3749 *PTEST is where the resulting comparison RTX is returned or NULL_RTX
3750 if we failed to produce one.
3752 *PMODE is the mode of the inputs (in case they are const_int).
3754 This function performs all the setup necessary so that the caller only has
3755 to emit a single comparison insn. This setup can involve doing a BLKmode
3756 comparison or emitting a library call to perform the comparison if no insn
3757 is available to handle it.
3758 The values which are passed in through pointers can be modified; the caller
3759 should perform the comparison on the modified values. Constant
3760 comparisons must have already been folded. */
3763 prepare_cmp_insn (rtx x
, rtx y
, enum rtx_code comparison
, rtx size
,
3764 int unsignedp
, enum optab_methods methods
,
3765 rtx
*ptest
, machine_mode
*pmode
)
3767 machine_mode mode
= *pmode
;
3769 machine_mode cmp_mode
;
3770 enum mode_class mclass
;
3772 /* The other methods are not needed. */
3773 gcc_assert (methods
== OPTAB_DIRECT
|| methods
== OPTAB_WIDEN
3774 || methods
== OPTAB_LIB_WIDEN
);
3776 /* If we are optimizing, force expensive constants into a register. */
3777 if (CONSTANT_P (x
) && optimize
3778 && (rtx_cost (x
, mode
, COMPARE
, 0, optimize_insn_for_speed_p ())
3779 > COSTS_N_INSNS (1)))
3780 x
= force_reg (mode
, x
);
3782 if (CONSTANT_P (y
) && optimize
3783 && (rtx_cost (y
, mode
, COMPARE
, 1, optimize_insn_for_speed_p ())
3784 > COSTS_N_INSNS (1)))
3785 y
= force_reg (mode
, y
);
3788 /* Make sure if we have a canonical comparison. The RTL
3789 documentation states that canonical comparisons are required only
3790 for targets which have cc0. */
3791 gcc_assert (!CONSTANT_P (x
) || CONSTANT_P (y
));
3794 /* Don't let both operands fail to indicate the mode. */
3795 if (GET_MODE (x
) == VOIDmode
&& GET_MODE (y
) == VOIDmode
)
3796 x
= force_reg (mode
, x
);
3797 if (mode
== VOIDmode
)
3798 mode
= GET_MODE (x
) != VOIDmode
? GET_MODE (x
) : GET_MODE (y
);
3800 /* Handle all BLKmode compares. */
3802 if (mode
== BLKmode
)
3804 machine_mode result_mode
;
3805 enum insn_code cmp_code
;
3808 = GEN_INT (MIN (MEM_ALIGN (x
), MEM_ALIGN (y
)) / BITS_PER_UNIT
);
3812 /* Try to use a memory block compare insn - either cmpstr
3813 or cmpmem will do. */
3814 opt_scalar_int_mode cmp_mode_iter
;
3815 FOR_EACH_MODE_IN_CLASS (cmp_mode_iter
, MODE_INT
)
3817 scalar_int_mode cmp_mode
= cmp_mode_iter
.require ();
3818 cmp_code
= direct_optab_handler (cmpmem_optab
, cmp_mode
);
3819 if (cmp_code
== CODE_FOR_nothing
)
3820 cmp_code
= direct_optab_handler (cmpstr_optab
, cmp_mode
);
3821 if (cmp_code
== CODE_FOR_nothing
)
3822 cmp_code
= direct_optab_handler (cmpstrn_optab
, cmp_mode
);
3823 if (cmp_code
== CODE_FOR_nothing
)
3826 /* Must make sure the size fits the insn's mode. */
3827 if (CONST_INT_P (size
)
3828 ? INTVAL (size
) >= (1 << GET_MODE_BITSIZE (cmp_mode
))
3829 : (GET_MODE_BITSIZE (as_a
<scalar_int_mode
> (GET_MODE (size
)))
3830 > GET_MODE_BITSIZE (cmp_mode
)))
3833 result_mode
= insn_data
[cmp_code
].operand
[0].mode
;
3834 result
= gen_reg_rtx (result_mode
);
3835 size
= convert_to_mode (cmp_mode
, size
, 1);
3836 emit_insn (GEN_FCN (cmp_code
) (result
, x
, y
, size
, opalign
));
3838 *ptest
= gen_rtx_fmt_ee (comparison
, VOIDmode
, result
, const0_rtx
);
3839 *pmode
= result_mode
;
3843 if (methods
!= OPTAB_LIB
&& methods
!= OPTAB_LIB_WIDEN
)
3846 /* Otherwise call a library function. */
3847 result
= emit_block_comp_via_libcall (XEXP (x
, 0), XEXP (y
, 0), size
);
3851 mode
= TYPE_MODE (integer_type_node
);
3852 methods
= OPTAB_LIB_WIDEN
;
3856 /* Don't allow operands to the compare to trap, as that can put the
3857 compare and branch in different basic blocks. */
3858 if (cfun
->can_throw_non_call_exceptions
)
3861 x
= copy_to_reg (x
);
3863 y
= copy_to_reg (y
);
3866 if (GET_MODE_CLASS (mode
) == MODE_CC
)
3868 enum insn_code icode
= optab_handler (cbranch_optab
, CCmode
);
3869 test
= gen_rtx_fmt_ee (comparison
, VOIDmode
, x
, y
);
3870 gcc_assert (icode
!= CODE_FOR_nothing
3871 && insn_operand_matches (icode
, 0, test
));
3876 mclass
= GET_MODE_CLASS (mode
);
3877 test
= gen_rtx_fmt_ee (comparison
, VOIDmode
, x
, y
);
3878 FOR_EACH_MODE_FROM (cmp_mode
, mode
)
3880 enum insn_code icode
;
3881 icode
= optab_handler (cbranch_optab
, cmp_mode
);
3882 if (icode
!= CODE_FOR_nothing
3883 && insn_operand_matches (icode
, 0, test
))
3885 rtx_insn
*last
= get_last_insn ();
3886 rtx op0
= prepare_operand (icode
, x
, 1, mode
, cmp_mode
, unsignedp
);
3887 rtx op1
= prepare_operand (icode
, y
, 2, mode
, cmp_mode
, unsignedp
);
3889 && insn_operand_matches (icode
, 1, op0
)
3890 && insn_operand_matches (icode
, 2, op1
))
3892 XEXP (test
, 0) = op0
;
3893 XEXP (test
, 1) = op1
;
3898 delete_insns_since (last
);
3901 if (methods
== OPTAB_DIRECT
|| !CLASS_HAS_WIDER_MODES_P (mclass
))
3905 if (methods
!= OPTAB_LIB_WIDEN
)
3908 if (!SCALAR_FLOAT_MODE_P (mode
))
3911 machine_mode ret_mode
;
3913 /* Handle a libcall just for the mode we are using. */
3914 libfunc
= optab_libfunc (cmp_optab
, mode
);
3915 gcc_assert (libfunc
);
3917 /* If we want unsigned, and this mode has a distinct unsigned
3918 comparison routine, use that. */
3921 rtx ulibfunc
= optab_libfunc (ucmp_optab
, mode
);
3926 ret_mode
= targetm
.libgcc_cmp_return_mode ();
3927 result
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
,
3928 ret_mode
, x
, mode
, y
, mode
);
3930 /* There are two kinds of comparison routines. Biased routines
3931 return 0/1/2, and unbiased routines return -1/0/1. Other parts
3932 of gcc expect that the comparison operation is equivalent
3933 to the modified comparison. For signed comparisons compare the
3934 result against 1 in the biased case, and zero in the unbiased
3935 case. For unsigned comparisons always compare against 1 after
3936 biasing the unbiased result by adding 1. This gives us a way to
3938 The comparisons in the fixed-point helper library are always
3943 if (!TARGET_LIB_INT_CMP_BIASED
&& !ALL_FIXED_POINT_MODE_P (mode
))
3946 x
= plus_constant (ret_mode
, result
, 1);
3952 prepare_cmp_insn (x
, y
, comparison
, NULL_RTX
, unsignedp
, methods
,
3956 prepare_float_lib_cmp (x
, y
, comparison
, ptest
, pmode
);
3964 /* Before emitting an insn with code ICODE, make sure that X, which is going
3965 to be used for operand OPNUM of the insn, is converted from mode MODE to
3966 WIDER_MODE (UNSIGNEDP determines whether it is an unsigned conversion), and
3967 that it is accepted by the operand predicate. Return the new value. */
3970 prepare_operand (enum insn_code icode
, rtx x
, int opnum
, machine_mode mode
,
3971 machine_mode wider_mode
, int unsignedp
)
3973 if (mode
!= wider_mode
)
3974 x
= convert_modes (wider_mode
, mode
, x
, unsignedp
);
3976 if (!insn_operand_matches (icode
, opnum
, x
))
3978 machine_mode op_mode
= insn_data
[(int) icode
].operand
[opnum
].mode
;
3979 if (reload_completed
)
3981 if (GET_MODE (x
) != op_mode
&& GET_MODE (x
) != VOIDmode
)
3983 x
= copy_to_mode_reg (op_mode
, x
);
3989 /* Subroutine of emit_cmp_and_jump_insns; this function is called when we know
3990 we can do the branch. */
3993 emit_cmp_and_jump_insn_1 (rtx test
, machine_mode mode
, rtx label
,
3994 profile_probability prob
)
3996 machine_mode optab_mode
;
3997 enum mode_class mclass
;
3998 enum insn_code icode
;
4001 mclass
= GET_MODE_CLASS (mode
);
4002 optab_mode
= (mclass
== MODE_CC
) ? CCmode
: mode
;
4003 icode
= optab_handler (cbranch_optab
, optab_mode
);
4005 gcc_assert (icode
!= CODE_FOR_nothing
);
4006 gcc_assert (insn_operand_matches (icode
, 0, test
));
4007 insn
= emit_jump_insn (GEN_FCN (icode
) (test
, XEXP (test
, 0),
4008 XEXP (test
, 1), label
));
4009 if (prob
.initialized_p ()
4010 && profile_status_for_fn (cfun
) != PROFILE_ABSENT
4013 && any_condjump_p (insn
)
4014 && !find_reg_note (insn
, REG_BR_PROB
, 0))
4015 add_reg_br_prob_note (insn
, prob
);
4018 /* Generate code to compare X with Y so that the condition codes are
4019 set and to jump to LABEL if the condition is true. If X is a
4020 constant and Y is not a constant, then the comparison is swapped to
4021 ensure that the comparison RTL has the canonical form.
4023 UNSIGNEDP nonzero says that X and Y are unsigned; this matters if they
4024 need to be widened. UNSIGNEDP is also used to select the proper
4025 branch condition code.
4027 If X and Y have mode BLKmode, then SIZE specifies the size of both X and Y.
4029 MODE is the mode of the inputs (in case they are const_int).
4031 COMPARISON is the rtl operator to compare with (EQ, NE, GT, etc.).
4032 It will be potentially converted into an unsigned variant based on
4033 UNSIGNEDP to select a proper jump instruction.
4035 PROB is the probability of jumping to LABEL. */
4038 emit_cmp_and_jump_insns (rtx x
, rtx y
, enum rtx_code comparison
, rtx size
,
4039 machine_mode mode
, int unsignedp
, rtx label
,
4040 profile_probability prob
)
4042 rtx op0
= x
, op1
= y
;
4045 /* Swap operands and condition to ensure canonical RTL. */
4046 if (swap_commutative_operands_p (x
, y
)
4047 && can_compare_p (swap_condition (comparison
), mode
, ccp_jump
))
4050 comparison
= swap_condition (comparison
);
4053 /* If OP0 is still a constant, then both X and Y must be constants
4054 or the opposite comparison is not supported. Force X into a register
4055 to create canonical RTL. */
4056 if (CONSTANT_P (op0
))
4057 op0
= force_reg (mode
, op0
);
4060 comparison
= unsigned_condition (comparison
);
4062 prepare_cmp_insn (op0
, op1
, comparison
, size
, unsignedp
, OPTAB_LIB_WIDEN
,
4064 emit_cmp_and_jump_insn_1 (test
, mode
, label
, prob
);
4068 /* Emit a library call comparison between floating point X and Y.
4069 COMPARISON is the rtl operator to compare with (EQ, NE, GT, etc.). */
4072 prepare_float_lib_cmp (rtx x
, rtx y
, enum rtx_code comparison
,
4073 rtx
*ptest
, machine_mode
*pmode
)
4075 enum rtx_code swapped
= swap_condition (comparison
);
4076 enum rtx_code reversed
= reverse_condition_maybe_unordered (comparison
);
4077 machine_mode orig_mode
= GET_MODE (x
);
4079 rtx true_rtx
, false_rtx
;
4080 rtx value
, target
, equiv
;
4083 bool reversed_p
= false;
4084 scalar_int_mode cmp_mode
= targetm
.libgcc_cmp_return_mode ();
4086 FOR_EACH_MODE_FROM (mode
, orig_mode
)
4088 if (code_to_optab (comparison
)
4089 && (libfunc
= optab_libfunc (code_to_optab (comparison
), mode
)))
4092 if (code_to_optab (swapped
)
4093 && (libfunc
= optab_libfunc (code_to_optab (swapped
), mode
)))
4096 comparison
= swapped
;
4100 if (code_to_optab (reversed
)
4101 && (libfunc
= optab_libfunc (code_to_optab (reversed
), mode
)))
4103 comparison
= reversed
;
4109 gcc_assert (mode
!= VOIDmode
);
4111 if (mode
!= orig_mode
)
4113 x
= convert_to_mode (mode
, x
, 0);
4114 y
= convert_to_mode (mode
, y
, 0);
4117 /* Attach a REG_EQUAL note describing the semantics of the libcall to
4118 the RTL. The allows the RTL optimizers to delete the libcall if the
4119 condition can be determined at compile-time. */
4120 if (comparison
== UNORDERED
4121 || FLOAT_LIB_COMPARE_RETURNS_BOOL (mode
, comparison
))
4123 true_rtx
= const_true_rtx
;
4124 false_rtx
= const0_rtx
;
4131 true_rtx
= const0_rtx
;
4132 false_rtx
= const_true_rtx
;
4136 true_rtx
= const_true_rtx
;
4137 false_rtx
= const0_rtx
;
4141 true_rtx
= const1_rtx
;
4142 false_rtx
= const0_rtx
;
4146 true_rtx
= const0_rtx
;
4147 false_rtx
= constm1_rtx
;
4151 true_rtx
= constm1_rtx
;
4152 false_rtx
= const0_rtx
;
4156 true_rtx
= const0_rtx
;
4157 false_rtx
= const1_rtx
;
4165 if (comparison
== UNORDERED
)
4167 rtx temp
= simplify_gen_relational (NE
, cmp_mode
, mode
, x
, x
);
4168 equiv
= simplify_gen_relational (NE
, cmp_mode
, mode
, y
, y
);
4169 equiv
= simplify_gen_ternary (IF_THEN_ELSE
, cmp_mode
, cmp_mode
,
4170 temp
, const_true_rtx
, equiv
);
4174 equiv
= simplify_gen_relational (comparison
, cmp_mode
, mode
, x
, y
);
4175 if (! FLOAT_LIB_COMPARE_RETURNS_BOOL (mode
, comparison
))
4176 equiv
= simplify_gen_ternary (IF_THEN_ELSE
, cmp_mode
, cmp_mode
,
4177 equiv
, true_rtx
, false_rtx
);
4181 value
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
,
4182 cmp_mode
, x
, mode
, y
, mode
);
4183 insns
= get_insns ();
4186 target
= gen_reg_rtx (cmp_mode
);
4187 emit_libcall_block (insns
, target
, value
, equiv
);
4189 if (comparison
== UNORDERED
4190 || FLOAT_LIB_COMPARE_RETURNS_BOOL (mode
, comparison
)
4192 *ptest
= gen_rtx_fmt_ee (reversed_p
? EQ
: NE
, VOIDmode
, target
, false_rtx
);
4194 *ptest
= gen_rtx_fmt_ee (comparison
, VOIDmode
, target
, const0_rtx
);
4199 /* Generate code to indirectly jump to a location given in the rtx LOC. */
4202 emit_indirect_jump (rtx loc
)
4204 if (!targetm
.have_indirect_jump ())
4205 sorry ("indirect jumps are not available on this target");
4208 struct expand_operand ops
[1];
4209 create_address_operand (&ops
[0], loc
);
4210 expand_jump_insn (targetm
.code_for_indirect_jump
, 1, ops
);
4216 /* Emit a conditional move instruction if the machine supports one for that
4217 condition and machine mode.
4219 OP0 and OP1 are the operands that should be compared using CODE. CMODE is
4220 the mode to use should they be constants. If it is VOIDmode, they cannot
4223 OP2 should be stored in TARGET if the comparison is true, otherwise OP3
4224 should be stored there. MODE is the mode to use should they be constants.
4225 If it is VOIDmode, they cannot both be constants.
4227 The result is either TARGET (perhaps modified) or NULL_RTX if the operation
4228 is not supported. */
4231 emit_conditional_move (rtx target
, enum rtx_code code
, rtx op0
, rtx op1
,
4232 machine_mode cmode
, rtx op2
, rtx op3
,
4233 machine_mode mode
, int unsignedp
)
4237 enum insn_code icode
;
4238 enum rtx_code reversed
;
4240 /* If the two source operands are identical, that's just a move. */
4242 if (rtx_equal_p (op2
, op3
))
4245 target
= gen_reg_rtx (mode
);
4247 emit_move_insn (target
, op3
);
4251 /* If one operand is constant, make it the second one. Only do this
4252 if the other operand is not constant as well. */
4254 if (swap_commutative_operands_p (op0
, op1
))
4256 std::swap (op0
, op1
);
4257 code
= swap_condition (code
);
4260 /* get_condition will prefer to generate LT and GT even if the old
4261 comparison was against zero, so undo that canonicalization here since
4262 comparisons against zero are cheaper. */
4263 if (code
== LT
&& op1
== const1_rtx
)
4264 code
= LE
, op1
= const0_rtx
;
4265 else if (code
== GT
&& op1
== constm1_rtx
)
4266 code
= GE
, op1
= const0_rtx
;
4268 if (cmode
== VOIDmode
)
4269 cmode
= GET_MODE (op0
);
4271 enum rtx_code orig_code
= code
;
4272 bool swapped
= false;
4273 if (swap_commutative_operands_p (op2
, op3
)
4274 && ((reversed
= reversed_comparison_code_parts (code
, op0
, op1
, NULL
))
4277 std::swap (op2
, op3
);
4282 if (mode
== VOIDmode
)
4283 mode
= GET_MODE (op2
);
4285 icode
= direct_optab_handler (movcc_optab
, mode
);
4287 if (icode
== CODE_FOR_nothing
)
4291 target
= gen_reg_rtx (mode
);
4293 for (int pass
= 0; ; pass
++)
4295 code
= unsignedp
? unsigned_condition (code
) : code
;
4296 comparison
= simplify_gen_relational (code
, VOIDmode
, cmode
, op0
, op1
);
4298 /* We can get const0_rtx or const_true_rtx in some circumstances. Just
4299 punt and let the caller figure out how best to deal with this
4301 if (COMPARISON_P (comparison
))
4303 saved_pending_stack_adjust save
;
4304 save_pending_stack_adjust (&save
);
4305 last
= get_last_insn ();
4306 do_pending_stack_adjust ();
4307 prepare_cmp_insn (XEXP (comparison
, 0), XEXP (comparison
, 1),
4308 GET_CODE (comparison
), NULL_RTX
, unsignedp
,
4309 OPTAB_WIDEN
, &comparison
, &cmode
);
4312 struct expand_operand ops
[4];
4314 create_output_operand (&ops
[0], target
, mode
);
4315 create_fixed_operand (&ops
[1], comparison
);
4316 create_input_operand (&ops
[2], op2
, mode
);
4317 create_input_operand (&ops
[3], op3
, mode
);
4318 if (maybe_expand_insn (icode
, 4, ops
))
4320 if (ops
[0].value
!= target
)
4321 convert_move (target
, ops
[0].value
, false);
4325 delete_insns_since (last
);
4326 restore_pending_stack_adjust (&save
);
4332 /* If the preferred op2/op3 order is not usable, retry with other
4333 operand order, perhaps it will expand successfully. */
4336 else if ((reversed
= reversed_comparison_code_parts (orig_code
, op0
, op1
,
4342 std::swap (op2
, op3
);
4347 /* Emit a conditional negate or bitwise complement using the
4348 negcc or notcc optabs if available. Return NULL_RTX if such operations
4349 are not available. Otherwise return the RTX holding the result.
4350 TARGET is the desired destination of the result. COMP is the comparison
4351 on which to negate. If COND is true move into TARGET the negation
4352 or bitwise complement of OP1. Otherwise move OP2 into TARGET.
4353 CODE is either NEG or NOT. MODE is the machine mode in which the
4354 operation is performed. */
4357 emit_conditional_neg_or_complement (rtx target
, rtx_code code
,
4358 machine_mode mode
, rtx cond
, rtx op1
,
4361 optab op
= unknown_optab
;
4364 else if (code
== NOT
)
4369 insn_code icode
= direct_optab_handler (op
, mode
);
4371 if (icode
== CODE_FOR_nothing
)
4375 target
= gen_reg_rtx (mode
);
4377 rtx_insn
*last
= get_last_insn ();
4378 struct expand_operand ops
[4];
4380 create_output_operand (&ops
[0], target
, mode
);
4381 create_fixed_operand (&ops
[1], cond
);
4382 create_input_operand (&ops
[2], op1
, mode
);
4383 create_input_operand (&ops
[3], op2
, mode
);
4385 if (maybe_expand_insn (icode
, 4, ops
))
4387 if (ops
[0].value
!= target
)
4388 convert_move (target
, ops
[0].value
, false);
4392 delete_insns_since (last
);
4396 /* Emit a conditional addition instruction if the machine supports one for that
4397 condition and machine mode.
4399 OP0 and OP1 are the operands that should be compared using CODE. CMODE is
4400 the mode to use should they be constants. If it is VOIDmode, they cannot
4403 OP2 should be stored in TARGET if the comparison is false, otherwise OP2+OP3
4404 should be stored there. MODE is the mode to use should they be constants.
4405 If it is VOIDmode, they cannot both be constants.
4407 The result is either TARGET (perhaps modified) or NULL_RTX if the operation
4408 is not supported. */
4411 emit_conditional_add (rtx target
, enum rtx_code code
, rtx op0
, rtx op1
,
4412 machine_mode cmode
, rtx op2
, rtx op3
,
4413 machine_mode mode
, int unsignedp
)
4417 enum insn_code icode
;
4419 /* If one operand is constant, make it the second one. Only do this
4420 if the other operand is not constant as well. */
4422 if (swap_commutative_operands_p (op0
, op1
))
4424 std::swap (op0
, op1
);
4425 code
= swap_condition (code
);
4428 /* get_condition will prefer to generate LT and GT even if the old
4429 comparison was against zero, so undo that canonicalization here since
4430 comparisons against zero are cheaper. */
4431 if (code
== LT
&& op1
== const1_rtx
)
4432 code
= LE
, op1
= const0_rtx
;
4433 else if (code
== GT
&& op1
== constm1_rtx
)
4434 code
= GE
, op1
= const0_rtx
;
4436 if (cmode
== VOIDmode
)
4437 cmode
= GET_MODE (op0
);
4439 if (mode
== VOIDmode
)
4440 mode
= GET_MODE (op2
);
4442 icode
= optab_handler (addcc_optab
, mode
);
4444 if (icode
== CODE_FOR_nothing
)
4448 target
= gen_reg_rtx (mode
);
4450 code
= unsignedp
? unsigned_condition (code
) : code
;
4451 comparison
= simplify_gen_relational (code
, VOIDmode
, cmode
, op0
, op1
);
4453 /* We can get const0_rtx or const_true_rtx in some circumstances. Just
4454 return NULL and let the caller figure out how best to deal with this
4456 if (!COMPARISON_P (comparison
))
4459 do_pending_stack_adjust ();
4460 last
= get_last_insn ();
4461 prepare_cmp_insn (XEXP (comparison
, 0), XEXP (comparison
, 1),
4462 GET_CODE (comparison
), NULL_RTX
, unsignedp
, OPTAB_WIDEN
,
4463 &comparison
, &cmode
);
4466 struct expand_operand ops
[4];
4468 create_output_operand (&ops
[0], target
, mode
);
4469 create_fixed_operand (&ops
[1], comparison
);
4470 create_input_operand (&ops
[2], op2
, mode
);
4471 create_input_operand (&ops
[3], op3
, mode
);
4472 if (maybe_expand_insn (icode
, 4, ops
))
4474 if (ops
[0].value
!= target
)
4475 convert_move (target
, ops
[0].value
, false);
4479 delete_insns_since (last
);
4483 /* These functions attempt to generate an insn body, rather than
4484 emitting the insn, but if the gen function already emits them, we
4485 make no attempt to turn them back into naked patterns. */
4487 /* Generate and return an insn body to add Y to X. */
4490 gen_add2_insn (rtx x
, rtx y
)
4492 enum insn_code icode
= optab_handler (add_optab
, GET_MODE (x
));
4494 gcc_assert (insn_operand_matches (icode
, 0, x
));
4495 gcc_assert (insn_operand_matches (icode
, 1, x
));
4496 gcc_assert (insn_operand_matches (icode
, 2, y
));
4498 return GEN_FCN (icode
) (x
, x
, y
);
4501 /* Generate and return an insn body to add r1 and c,
4502 storing the result in r0. */
4505 gen_add3_insn (rtx r0
, rtx r1
, rtx c
)
4507 enum insn_code icode
= optab_handler (add_optab
, GET_MODE (r0
));
4509 if (icode
== CODE_FOR_nothing
4510 || !insn_operand_matches (icode
, 0, r0
)
4511 || !insn_operand_matches (icode
, 1, r1
)
4512 || !insn_operand_matches (icode
, 2, c
))
4515 return GEN_FCN (icode
) (r0
, r1
, c
);
4519 have_add2_insn (rtx x
, rtx y
)
4521 enum insn_code icode
;
4523 gcc_assert (GET_MODE (x
) != VOIDmode
);
4525 icode
= optab_handler (add_optab
, GET_MODE (x
));
4527 if (icode
== CODE_FOR_nothing
)
4530 if (!insn_operand_matches (icode
, 0, x
)
4531 || !insn_operand_matches (icode
, 1, x
)
4532 || !insn_operand_matches (icode
, 2, y
))
4538 /* Generate and return an insn body to add Y to X. */
4541 gen_addptr3_insn (rtx x
, rtx y
, rtx z
)
4543 enum insn_code icode
= optab_handler (addptr3_optab
, GET_MODE (x
));
4545 gcc_assert (insn_operand_matches (icode
, 0, x
));
4546 gcc_assert (insn_operand_matches (icode
, 1, y
));
4547 gcc_assert (insn_operand_matches (icode
, 2, z
));
4549 return GEN_FCN (icode
) (x
, y
, z
);
4552 /* Return true if the target implements an addptr pattern and X, Y,
4553 and Z are valid for the pattern predicates. */
4556 have_addptr3_insn (rtx x
, rtx y
, rtx z
)
4558 enum insn_code icode
;
4560 gcc_assert (GET_MODE (x
) != VOIDmode
);
4562 icode
= optab_handler (addptr3_optab
, GET_MODE (x
));
4564 if (icode
== CODE_FOR_nothing
)
4567 if (!insn_operand_matches (icode
, 0, x
)
4568 || !insn_operand_matches (icode
, 1, y
)
4569 || !insn_operand_matches (icode
, 2, z
))
4575 /* Generate and return an insn body to subtract Y from X. */
4578 gen_sub2_insn (rtx x
, rtx y
)
4580 enum insn_code icode
= optab_handler (sub_optab
, GET_MODE (x
));
4582 gcc_assert (insn_operand_matches (icode
, 0, x
));
4583 gcc_assert (insn_operand_matches (icode
, 1, x
));
4584 gcc_assert (insn_operand_matches (icode
, 2, y
));
4586 return GEN_FCN (icode
) (x
, x
, y
);
4589 /* Generate and return an insn body to subtract r1 and c,
4590 storing the result in r0. */
4593 gen_sub3_insn (rtx r0
, rtx r1
, rtx c
)
4595 enum insn_code icode
= optab_handler (sub_optab
, GET_MODE (r0
));
4597 if (icode
== CODE_FOR_nothing
4598 || !insn_operand_matches (icode
, 0, r0
)
4599 || !insn_operand_matches (icode
, 1, r1
)
4600 || !insn_operand_matches (icode
, 2, c
))
4603 return GEN_FCN (icode
) (r0
, r1
, c
);
4607 have_sub2_insn (rtx x
, rtx y
)
4609 enum insn_code icode
;
4611 gcc_assert (GET_MODE (x
) != VOIDmode
);
4613 icode
= optab_handler (sub_optab
, GET_MODE (x
));
4615 if (icode
== CODE_FOR_nothing
)
4618 if (!insn_operand_matches (icode
, 0, x
)
4619 || !insn_operand_matches (icode
, 1, x
)
4620 || !insn_operand_matches (icode
, 2, y
))
4626 /* Generate the body of an insn to extend Y (with mode MFROM)
4627 into X (with mode MTO). Do zero-extension if UNSIGNEDP is nonzero. */
4630 gen_extend_insn (rtx x
, rtx y
, machine_mode mto
,
4631 machine_mode mfrom
, int unsignedp
)
4633 enum insn_code icode
= can_extend_p (mto
, mfrom
, unsignedp
);
4634 return GEN_FCN (icode
) (x
, y
);
4637 /* Generate code to convert FROM to floating point
4638 and store in TO. FROM must be fixed point and not VOIDmode.
4639 UNSIGNEDP nonzero means regard FROM as unsigned.
4640 Normally this is done by correcting the final value
4641 if it is negative. */
4644 expand_float (rtx to
, rtx from
, int unsignedp
)
4646 enum insn_code icode
;
4648 scalar_mode from_mode
, to_mode
;
4649 machine_mode fmode
, imode
;
4650 bool can_do_signed
= false;
4652 /* Crash now, because we won't be able to decide which mode to use. */
4653 gcc_assert (GET_MODE (from
) != VOIDmode
);
4655 /* Look for an insn to do the conversion. Do it in the specified
4656 modes if possible; otherwise convert either input, output or both to
4657 wider mode. If the integer mode is wider than the mode of FROM,
4658 we can do the conversion signed even if the input is unsigned. */
4660 FOR_EACH_MODE_FROM (fmode
, GET_MODE (to
))
4661 FOR_EACH_MODE_FROM (imode
, GET_MODE (from
))
4663 int doing_unsigned
= unsignedp
;
4665 if (fmode
!= GET_MODE (to
)
4666 && (significand_size (fmode
)
4667 < GET_MODE_UNIT_PRECISION (GET_MODE (from
))))
4670 icode
= can_float_p (fmode
, imode
, unsignedp
);
4671 if (icode
== CODE_FOR_nothing
&& unsignedp
)
4673 enum insn_code scode
= can_float_p (fmode
, imode
, 0);
4674 if (scode
!= CODE_FOR_nothing
)
4675 can_do_signed
= true;
4676 if (imode
!= GET_MODE (from
))
4677 icode
= scode
, doing_unsigned
= 0;
4680 if (icode
!= CODE_FOR_nothing
)
4682 if (imode
!= GET_MODE (from
))
4683 from
= convert_to_mode (imode
, from
, unsignedp
);
4685 if (fmode
!= GET_MODE (to
))
4686 target
= gen_reg_rtx (fmode
);
4688 emit_unop_insn (icode
, target
, from
,
4689 doing_unsigned
? UNSIGNED_FLOAT
: FLOAT
);
4692 convert_move (to
, target
, 0);
4697 /* Unsigned integer, and no way to convert directly. Convert as signed,
4698 then unconditionally adjust the result. */
4701 && is_a
<scalar_mode
> (GET_MODE (to
), &to_mode
)
4702 && is_a
<scalar_mode
> (GET_MODE (from
), &from_mode
))
4704 opt_scalar_mode fmode_iter
;
4705 rtx_code_label
*label
= gen_label_rtx ();
4707 REAL_VALUE_TYPE offset
;
4709 /* Look for a usable floating mode FMODE wider than the source and at
4710 least as wide as the target. Using FMODE will avoid rounding woes
4711 with unsigned values greater than the signed maximum value. */
4713 FOR_EACH_MODE_FROM (fmode_iter
, to_mode
)
4715 scalar_mode fmode
= fmode_iter
.require ();
4716 if (GET_MODE_PRECISION (from_mode
) < GET_MODE_BITSIZE (fmode
)
4717 && can_float_p (fmode
, from_mode
, 0) != CODE_FOR_nothing
)
4721 if (!fmode_iter
.exists (&fmode
))
4723 /* There is no such mode. Pretend the target is wide enough. */
4726 /* Avoid double-rounding when TO is narrower than FROM. */
4727 if ((significand_size (fmode
) + 1)
4728 < GET_MODE_PRECISION (from_mode
))
4731 rtx_code_label
*neglabel
= gen_label_rtx ();
4733 /* Don't use TARGET if it isn't a register, is a hard register,
4734 or is the wrong mode. */
4736 || REGNO (target
) < FIRST_PSEUDO_REGISTER
4737 || GET_MODE (target
) != fmode
)
4738 target
= gen_reg_rtx (fmode
);
4741 do_pending_stack_adjust ();
4743 /* Test whether the sign bit is set. */
4744 emit_cmp_and_jump_insns (from
, const0_rtx
, LT
, NULL_RTX
, imode
,
4747 /* The sign bit is not set. Convert as signed. */
4748 expand_float (target
, from
, 0);
4749 emit_jump_insn (targetm
.gen_jump (label
));
4752 /* The sign bit is set.
4753 Convert to a usable (positive signed) value by shifting right
4754 one bit, while remembering if a nonzero bit was shifted
4755 out; i.e., compute (from & 1) | (from >> 1). */
4757 emit_label (neglabel
);
4758 temp
= expand_binop (imode
, and_optab
, from
, const1_rtx
,
4759 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
4760 temp1
= expand_shift (RSHIFT_EXPR
, imode
, from
, 1, NULL_RTX
, 1);
4761 temp
= expand_binop (imode
, ior_optab
, temp
, temp1
, temp
, 1,
4763 expand_float (target
, temp
, 0);
4765 /* Multiply by 2 to undo the shift above. */
4766 temp
= expand_binop (fmode
, add_optab
, target
, target
,
4767 target
, 0, OPTAB_LIB_WIDEN
);
4769 emit_move_insn (target
, temp
);
4771 do_pending_stack_adjust ();
4777 /* If we are about to do some arithmetic to correct for an
4778 unsigned operand, do it in a pseudo-register. */
4780 if (to_mode
!= fmode
4781 || !REG_P (to
) || REGNO (to
) < FIRST_PSEUDO_REGISTER
)
4782 target
= gen_reg_rtx (fmode
);
4784 /* Convert as signed integer to floating. */
4785 expand_float (target
, from
, 0);
4787 /* If FROM is negative (and therefore TO is negative),
4788 correct its value by 2**bitwidth. */
4790 do_pending_stack_adjust ();
4791 emit_cmp_and_jump_insns (from
, const0_rtx
, GE
, NULL_RTX
, from_mode
,
4795 real_2expN (&offset
, GET_MODE_PRECISION (from_mode
), fmode
);
4796 temp
= expand_binop (fmode
, add_optab
, target
,
4797 const_double_from_real_value (offset
, fmode
),
4798 target
, 0, OPTAB_LIB_WIDEN
);
4800 emit_move_insn (target
, temp
);
4802 do_pending_stack_adjust ();
4807 /* No hardware instruction available; call a library routine. */
4812 convert_optab tab
= unsignedp
? ufloat_optab
: sfloat_optab
;
4814 if (is_narrower_int_mode (GET_MODE (from
), SImode
))
4815 from
= convert_to_mode (SImode
, from
, unsignedp
);
4817 libfunc
= convert_optab_libfunc (tab
, GET_MODE (to
), GET_MODE (from
));
4818 gcc_assert (libfunc
);
4822 value
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
,
4823 GET_MODE (to
), from
, GET_MODE (from
));
4824 insns
= get_insns ();
4827 emit_libcall_block (insns
, target
, value
,
4828 gen_rtx_fmt_e (unsignedp
? UNSIGNED_FLOAT
: FLOAT
,
4829 GET_MODE (to
), from
));
4834 /* Copy result to requested destination
4835 if we have been computing in a temp location. */
4839 if (GET_MODE (target
) == GET_MODE (to
))
4840 emit_move_insn (to
, target
);
4842 convert_move (to
, target
, 0);
4846 /* Generate code to convert FROM to fixed point and store in TO. FROM
4847 must be floating point. */
4850 expand_fix (rtx to
, rtx from
, int unsignedp
)
4852 enum insn_code icode
;
4854 machine_mode fmode
, imode
;
4855 opt_scalar_mode fmode_iter
;
4856 bool must_trunc
= false;
4858 /* We first try to find a pair of modes, one real and one integer, at
4859 least as wide as FROM and TO, respectively, in which we can open-code
4860 this conversion. If the integer mode is wider than the mode of TO,
4861 we can do the conversion either signed or unsigned. */
4863 FOR_EACH_MODE_FROM (fmode
, GET_MODE (from
))
4864 FOR_EACH_MODE_FROM (imode
, GET_MODE (to
))
4866 int doing_unsigned
= unsignedp
;
4868 icode
= can_fix_p (imode
, fmode
, unsignedp
, &must_trunc
);
4869 if (icode
== CODE_FOR_nothing
&& imode
!= GET_MODE (to
) && unsignedp
)
4870 icode
= can_fix_p (imode
, fmode
, 0, &must_trunc
), doing_unsigned
= 0;
4872 if (icode
!= CODE_FOR_nothing
)
4874 rtx_insn
*last
= get_last_insn ();
4875 if (fmode
!= GET_MODE (from
))
4876 from
= convert_to_mode (fmode
, from
, 0);
4880 rtx temp
= gen_reg_rtx (GET_MODE (from
));
4881 from
= expand_unop (GET_MODE (from
), ftrunc_optab
, from
,
4885 if (imode
!= GET_MODE (to
))
4886 target
= gen_reg_rtx (imode
);
4888 if (maybe_emit_unop_insn (icode
, target
, from
,
4889 doing_unsigned
? UNSIGNED_FIX
: FIX
))
4892 convert_move (to
, target
, unsignedp
);
4895 delete_insns_since (last
);
4899 /* For an unsigned conversion, there is one more way to do it.
4900 If we have a signed conversion, we generate code that compares
4901 the real value to the largest representable positive number. If if
4902 is smaller, the conversion is done normally. Otherwise, subtract
4903 one plus the highest signed number, convert, and add it back.
4905 We only need to check all real modes, since we know we didn't find
4906 anything with a wider integer mode.
4908 This code used to extend FP value into mode wider than the destination.
4909 This is needed for decimal float modes which cannot accurately
4910 represent one plus the highest signed number of the same size, but
4911 not for binary modes. Consider, for instance conversion from SFmode
4914 The hot path through the code is dealing with inputs smaller than 2^63
4915 and doing just the conversion, so there is no bits to lose.
4917 In the other path we know the value is positive in the range 2^63..2^64-1
4918 inclusive. (as for other input overflow happens and result is undefined)
4919 So we know that the most important bit set in mantissa corresponds to
4920 2^63. The subtraction of 2^63 should not generate any rounding as it
4921 simply clears out that bit. The rest is trivial. */
4923 scalar_int_mode to_mode
;
4925 && is_a
<scalar_int_mode
> (GET_MODE (to
), &to_mode
)
4926 && HWI_COMPUTABLE_MODE_P (to_mode
))
4927 FOR_EACH_MODE_FROM (fmode_iter
, as_a
<scalar_mode
> (GET_MODE (from
)))
4929 scalar_mode fmode
= fmode_iter
.require ();
4930 if (CODE_FOR_nothing
!= can_fix_p (to_mode
, fmode
,
4932 && (!DECIMAL_FLOAT_MODE_P (fmode
)
4933 || (GET_MODE_BITSIZE (fmode
) > GET_MODE_PRECISION (to_mode
))))
4936 REAL_VALUE_TYPE offset
;
4938 rtx_code_label
*lab1
, *lab2
;
4941 bitsize
= GET_MODE_PRECISION (to_mode
);
4942 real_2expN (&offset
, bitsize
- 1, fmode
);
4943 limit
= const_double_from_real_value (offset
, fmode
);
4944 lab1
= gen_label_rtx ();
4945 lab2
= gen_label_rtx ();
4947 if (fmode
!= GET_MODE (from
))
4948 from
= convert_to_mode (fmode
, from
, 0);
4950 /* See if we need to do the subtraction. */
4951 do_pending_stack_adjust ();
4952 emit_cmp_and_jump_insns (from
, limit
, GE
, NULL_RTX
,
4953 GET_MODE (from
), 0, lab1
);
4955 /* If not, do the signed "fix" and branch around fixup code. */
4956 expand_fix (to
, from
, 0);
4957 emit_jump_insn (targetm
.gen_jump (lab2
));
4960 /* Otherwise, subtract 2**(N-1), convert to signed number,
4961 then add 2**(N-1). Do the addition using XOR since this
4962 will often generate better code. */
4964 target
= expand_binop (GET_MODE (from
), sub_optab
, from
, limit
,
4965 NULL_RTX
, 0, OPTAB_LIB_WIDEN
);
4966 expand_fix (to
, target
, 0);
4967 target
= expand_binop (to_mode
, xor_optab
, to
,
4969 (HOST_WIDE_INT_1
<< (bitsize
- 1),
4971 to
, 1, OPTAB_LIB_WIDEN
);
4974 emit_move_insn (to
, target
);
4978 if (optab_handler (mov_optab
, to_mode
) != CODE_FOR_nothing
)
4980 /* Make a place for a REG_NOTE and add it. */
4981 insn
= emit_move_insn (to
, to
);
4982 set_dst_reg_note (insn
, REG_EQUAL
,
4983 gen_rtx_fmt_e (UNSIGNED_FIX
, to_mode
,
4992 /* We can't do it with an insn, so use a library call. But first ensure
4993 that the mode of TO is at least as wide as SImode, since those are the
4994 only library calls we know about. */
4996 if (is_narrower_int_mode (GET_MODE (to
), SImode
))
4998 target
= gen_reg_rtx (SImode
);
5000 expand_fix (target
, from
, unsignedp
);
5008 convert_optab tab
= unsignedp
? ufix_optab
: sfix_optab
;
5009 libfunc
= convert_optab_libfunc (tab
, GET_MODE (to
), GET_MODE (from
));
5010 gcc_assert (libfunc
);
5014 value
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
,
5015 GET_MODE (to
), from
, GET_MODE (from
));
5016 insns
= get_insns ();
5019 emit_libcall_block (insns
, target
, value
,
5020 gen_rtx_fmt_e (unsignedp
? UNSIGNED_FIX
: FIX
,
5021 GET_MODE (to
), from
));
5026 if (GET_MODE (to
) == GET_MODE (target
))
5027 emit_move_insn (to
, target
);
5029 convert_move (to
, target
, 0);
5034 /* Promote integer arguments for a libcall if necessary.
5035 emit_library_call_value cannot do the promotion because it does not
5036 know if it should do a signed or unsigned promotion. This is because
5037 there are no tree types defined for libcalls. */
5040 prepare_libcall_arg (rtx arg
, int uintp
)
5042 scalar_int_mode mode
;
5043 machine_mode arg_mode
;
5044 if (is_a
<scalar_int_mode
> (GET_MODE (arg
), &mode
))
5046 /* If we need to promote the integer function argument we need to do
5047 it here instead of inside emit_library_call_value because in
5048 emit_library_call_value we don't know if we should do a signed or
5049 unsigned promotion. */
5052 arg_mode
= promote_function_mode (NULL_TREE
, mode
,
5053 &unsigned_p
, NULL_TREE
, 0);
5054 if (arg_mode
!= mode
)
5055 return convert_to_mode (arg_mode
, arg
, uintp
);
5060 /* Generate code to convert FROM or TO a fixed-point.
5061 If UINTP is true, either TO or FROM is an unsigned integer.
5062 If SATP is true, we need to saturate the result. */
5065 expand_fixed_convert (rtx to
, rtx from
, int uintp
, int satp
)
5067 machine_mode to_mode
= GET_MODE (to
);
5068 machine_mode from_mode
= GET_MODE (from
);
5070 enum rtx_code this_code
;
5071 enum insn_code code
;
5076 if (to_mode
== from_mode
)
5078 emit_move_insn (to
, from
);
5084 tab
= satp
? satfractuns_optab
: fractuns_optab
;
5085 this_code
= satp
? UNSIGNED_SAT_FRACT
: UNSIGNED_FRACT_CONVERT
;
5089 tab
= satp
? satfract_optab
: fract_optab
;
5090 this_code
= satp
? SAT_FRACT
: FRACT_CONVERT
;
5092 code
= convert_optab_handler (tab
, to_mode
, from_mode
);
5093 if (code
!= CODE_FOR_nothing
)
5095 emit_unop_insn (code
, to
, from
, this_code
);
5099 libfunc
= convert_optab_libfunc (tab
, to_mode
, from_mode
);
5100 gcc_assert (libfunc
);
5102 from
= prepare_libcall_arg (from
, uintp
);
5103 from_mode
= GET_MODE (from
);
5106 value
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
, to_mode
,
5108 insns
= get_insns ();
5111 emit_libcall_block (insns
, to
, value
,
5112 gen_rtx_fmt_e (optab_to_code (tab
), to_mode
, from
));
5115 /* Generate code to convert FROM to fixed point and store in TO. FROM
5116 must be floating point, TO must be signed. Use the conversion optab
5117 TAB to do the conversion. */
5120 expand_sfix_optab (rtx to
, rtx from
, convert_optab tab
)
5122 enum insn_code icode
;
5124 machine_mode fmode
, imode
;
5126 /* We first try to find a pair of modes, one real and one integer, at
5127 least as wide as FROM and TO, respectively, in which we can open-code
5128 this conversion. If the integer mode is wider than the mode of TO,
5129 we can do the conversion either signed or unsigned. */
5131 FOR_EACH_MODE_FROM (fmode
, GET_MODE (from
))
5132 FOR_EACH_MODE_FROM (imode
, GET_MODE (to
))
5134 icode
= convert_optab_handler (tab
, imode
, fmode
);
5135 if (icode
!= CODE_FOR_nothing
)
5137 rtx_insn
*last
= get_last_insn ();
5138 if (fmode
!= GET_MODE (from
))
5139 from
= convert_to_mode (fmode
, from
, 0);
5141 if (imode
!= GET_MODE (to
))
5142 target
= gen_reg_rtx (imode
);
5144 if (!maybe_emit_unop_insn (icode
, target
, from
, UNKNOWN
))
5146 delete_insns_since (last
);
5150 convert_move (to
, target
, 0);
5158 /* Report whether we have an instruction to perform the operation
5159 specified by CODE on operands of mode MODE. */
5161 have_insn_for (enum rtx_code code
, machine_mode mode
)
5163 return (code_to_optab (code
)
5164 && (optab_handler (code_to_optab (code
), mode
)
5165 != CODE_FOR_nothing
));
5168 /* Print information about the current contents of the optabs on
5172 debug_optab_libfuncs (void)
5176 /* Dump the arithmetic optabs. */
5177 for (i
= FIRST_NORM_OPTAB
; i
<= LAST_NORMLIB_OPTAB
; ++i
)
5178 for (j
= 0; j
< NUM_MACHINE_MODES
; ++j
)
5180 rtx l
= optab_libfunc ((optab
) i
, (machine_mode
) j
);
5183 gcc_assert (GET_CODE (l
) == SYMBOL_REF
);
5184 fprintf (stderr
, "%s\t%s:\t%s\n",
5185 GET_RTX_NAME (optab_to_code ((optab
) i
)),
5191 /* Dump the conversion optabs. */
5192 for (i
= FIRST_CONV_OPTAB
; i
<= LAST_CONVLIB_OPTAB
; ++i
)
5193 for (j
= 0; j
< NUM_MACHINE_MODES
; ++j
)
5194 for (k
= 0; k
< NUM_MACHINE_MODES
; ++k
)
5196 rtx l
= convert_optab_libfunc ((optab
) i
, (machine_mode
) j
,
5200 gcc_assert (GET_CODE (l
) == SYMBOL_REF
);
5201 fprintf (stderr
, "%s\t%s\t%s:\t%s\n",
5202 GET_RTX_NAME (optab_to_code ((optab
) i
)),
5210 /* Generate insns to trap with code TCODE if OP1 and OP2 satisfy condition
5211 CODE. Return 0 on failure. */
5214 gen_cond_trap (enum rtx_code code
, rtx op1
, rtx op2
, rtx tcode
)
5216 machine_mode mode
= GET_MODE (op1
);
5217 enum insn_code icode
;
5221 if (mode
== VOIDmode
)
5224 icode
= optab_handler (ctrap_optab
, mode
);
5225 if (icode
== CODE_FOR_nothing
)
5228 /* Some targets only accept a zero trap code. */
5229 if (!insn_operand_matches (icode
, 3, tcode
))
5232 do_pending_stack_adjust ();
5234 prepare_cmp_insn (op1
, op2
, code
, NULL_RTX
, false, OPTAB_DIRECT
,
5239 insn
= GEN_FCN (icode
) (trap_rtx
, XEXP (trap_rtx
, 0), XEXP (trap_rtx
, 1),
5242 /* If that failed, then give up. */
5250 insn
= get_insns ();
5255 /* Return rtx code for TCODE. Use UNSIGNEDP to select signed
5256 or unsigned operation code. */
5259 get_rtx_code (enum tree_code tcode
, bool unsignedp
)
5271 code
= unsignedp
? LTU
: LT
;
5274 code
= unsignedp
? LEU
: LE
;
5277 code
= unsignedp
? GTU
: GT
;
5280 code
= unsignedp
? GEU
: GE
;
5283 case UNORDERED_EXPR
:
5322 /* Return a comparison rtx of mode CMP_MODE for COND. Use UNSIGNEDP to
5323 select signed or unsigned operators. OPNO holds the index of the
5324 first comparison operand for insn ICODE. Do not generate the
5325 compare instruction itself. */
5328 vector_compare_rtx (machine_mode cmp_mode
, enum tree_code tcode
,
5329 tree t_op0
, tree t_op1
, bool unsignedp
,
5330 enum insn_code icode
, unsigned int opno
)
5332 struct expand_operand ops
[2];
5333 rtx rtx_op0
, rtx_op1
;
5334 machine_mode m0
, m1
;
5335 enum rtx_code rcode
= get_rtx_code (tcode
, unsignedp
);
5337 gcc_assert (TREE_CODE_CLASS (tcode
) == tcc_comparison
);
5339 /* Expand operands. For vector types with scalar modes, e.g. where int64x1_t
5340 has mode DImode, this can produce a constant RTX of mode VOIDmode; in such
5341 cases, use the original mode. */
5342 rtx_op0
= expand_expr (t_op0
, NULL_RTX
, TYPE_MODE (TREE_TYPE (t_op0
)),
5344 m0
= GET_MODE (rtx_op0
);
5346 m0
= TYPE_MODE (TREE_TYPE (t_op0
));
5348 rtx_op1
= expand_expr (t_op1
, NULL_RTX
, TYPE_MODE (TREE_TYPE (t_op1
)),
5350 m1
= GET_MODE (rtx_op1
);
5352 m1
= TYPE_MODE (TREE_TYPE (t_op1
));
5354 create_input_operand (&ops
[0], rtx_op0
, m0
);
5355 create_input_operand (&ops
[1], rtx_op1
, m1
);
5356 if (!maybe_legitimize_operands (icode
, opno
, 2, ops
))
5358 return gen_rtx_fmt_ee (rcode
, cmp_mode
, ops
[0].value
, ops
[1].value
);
5361 /* Checks if vec_perm mask SEL is a constant equivalent to a shift of the first
5362 vec_perm operand, assuming the second operand is a constant vector of zeroes.
5363 Return the shift distance in bits if so, or NULL_RTX if the vec_perm is not a
5366 shift_amt_for_vec_perm_mask (rtx sel
)
5368 unsigned int i
, first
, nelt
= GET_MODE_NUNITS (GET_MODE (sel
));
5369 unsigned int bitsize
= GET_MODE_UNIT_BITSIZE (GET_MODE (sel
));
5371 if (GET_CODE (sel
) != CONST_VECTOR
)
5374 first
= INTVAL (CONST_VECTOR_ELT (sel
, 0));
5377 for (i
= 1; i
< nelt
; i
++)
5379 int idx
= INTVAL (CONST_VECTOR_ELT (sel
, i
));
5380 unsigned int expected
= i
+ first
;
5381 /* Indices into the second vector are all equivalent. */
5382 if (idx
< 0 || (MIN (nelt
, (unsigned) idx
) != MIN (nelt
, expected
)))
5386 return GEN_INT (first
* bitsize
);
5389 /* A subroutine of expand_vec_perm for expanding one vec_perm insn. */
5392 expand_vec_perm_1 (enum insn_code icode
, rtx target
,
5393 rtx v0
, rtx v1
, rtx sel
)
5395 machine_mode tmode
= GET_MODE (target
);
5396 machine_mode smode
= GET_MODE (sel
);
5397 struct expand_operand ops
[4];
5399 create_output_operand (&ops
[0], target
, tmode
);
5400 create_input_operand (&ops
[3], sel
, smode
);
5402 /* Make an effort to preserve v0 == v1. The target expander is able to
5403 rely on this to determine if we're permuting a single input operand. */
5404 if (rtx_equal_p (v0
, v1
))
5406 if (!insn_operand_matches (icode
, 1, v0
))
5407 v0
= force_reg (tmode
, v0
);
5408 gcc_checking_assert (insn_operand_matches (icode
, 1, v0
));
5409 gcc_checking_assert (insn_operand_matches (icode
, 2, v0
));
5411 create_fixed_operand (&ops
[1], v0
);
5412 create_fixed_operand (&ops
[2], v0
);
5416 create_input_operand (&ops
[1], v0
, tmode
);
5417 create_input_operand (&ops
[2], v1
, tmode
);
5420 if (maybe_expand_insn (icode
, 4, ops
))
5421 return ops
[0].value
;
5425 /* Generate instructions for vec_perm optab given its mode
5426 and three operands. */
5429 expand_vec_perm (machine_mode mode
, rtx v0
, rtx v1
, rtx sel
, rtx target
)
5431 enum insn_code icode
;
5432 machine_mode qimode
;
5433 unsigned int i
, w
, e
, u
;
5434 rtx tmp
, sel_qi
= NULL
;
5437 if (!target
|| GET_MODE (target
) != mode
)
5438 target
= gen_reg_rtx (mode
);
5440 w
= GET_MODE_SIZE (mode
);
5441 e
= GET_MODE_NUNITS (mode
);
5442 u
= GET_MODE_UNIT_SIZE (mode
);
5444 /* Set QIMODE to a different vector mode with byte elements.
5445 If no such mode, or if MODE already has byte elements, use VOIDmode. */
5446 if (GET_MODE_INNER (mode
) == QImode
5447 || !mode_for_vector (QImode
, w
).exists (&qimode
)
5448 || !VECTOR_MODE_P (qimode
))
5451 /* If the input is a constant, expand it specially. */
5452 gcc_assert (GET_MODE_CLASS (GET_MODE (sel
)) == MODE_VECTOR_INT
);
5453 if (GET_CODE (sel
) == CONST_VECTOR
)
5455 /* See if this can be handled with a vec_shr. We only do this if the
5456 second vector is all zeroes. */
5457 enum insn_code shift_code
= optab_handler (vec_shr_optab
, mode
);
5458 enum insn_code shift_code_qi
= ((qimode
!= VOIDmode
&& qimode
!= mode
)
5459 ? optab_handler (vec_shr_optab
, qimode
)
5460 : CODE_FOR_nothing
);
5461 rtx shift_amt
= NULL_RTX
;
5462 if (v1
== CONST0_RTX (GET_MODE (v1
))
5463 && (shift_code
!= CODE_FOR_nothing
5464 || shift_code_qi
!= CODE_FOR_nothing
))
5466 shift_amt
= shift_amt_for_vec_perm_mask (sel
);
5469 struct expand_operand ops
[3];
5470 if (shift_code
!= CODE_FOR_nothing
)
5472 create_output_operand (&ops
[0], target
, mode
);
5473 create_input_operand (&ops
[1], v0
, mode
);
5474 create_convert_operand_from_type (&ops
[2], shift_amt
,
5476 if (maybe_expand_insn (shift_code
, 3, ops
))
5477 return ops
[0].value
;
5479 if (shift_code_qi
!= CODE_FOR_nothing
)
5481 tmp
= gen_reg_rtx (qimode
);
5482 create_output_operand (&ops
[0], tmp
, qimode
);
5483 create_input_operand (&ops
[1], gen_lowpart (qimode
, v0
),
5485 create_convert_operand_from_type (&ops
[2], shift_amt
,
5487 if (maybe_expand_insn (shift_code_qi
, 3, ops
))
5488 return gen_lowpart (mode
, ops
[0].value
);
5493 icode
= direct_optab_handler (vec_perm_const_optab
, mode
);
5494 if (icode
!= CODE_FOR_nothing
)
5496 tmp
= expand_vec_perm_1 (icode
, target
, v0
, v1
, sel
);
5501 /* Fall back to a constant byte-based permutation. */
5502 if (qimode
!= VOIDmode
)
5504 vec
= rtvec_alloc (w
);
5505 for (i
= 0; i
< e
; ++i
)
5507 unsigned int j
, this_e
;
5509 this_e
= INTVAL (CONST_VECTOR_ELT (sel
, i
));
5510 this_e
&= 2 * e
- 1;
5513 for (j
= 0; j
< u
; ++j
)
5514 RTVEC_ELT (vec
, i
* u
+ j
) = GEN_INT (this_e
+ j
);
5516 sel_qi
= gen_rtx_CONST_VECTOR (qimode
, vec
);
5518 icode
= direct_optab_handler (vec_perm_const_optab
, qimode
);
5519 if (icode
!= CODE_FOR_nothing
)
5521 tmp
= mode
!= qimode
? gen_reg_rtx (qimode
) : target
;
5522 tmp
= expand_vec_perm_1 (icode
, tmp
, gen_lowpart (qimode
, v0
),
5523 gen_lowpart (qimode
, v1
), sel_qi
);
5525 return gen_lowpart (mode
, tmp
);
5530 /* Otherwise expand as a fully variable permuation. */
5531 icode
= direct_optab_handler (vec_perm_optab
, mode
);
5532 if (icode
!= CODE_FOR_nothing
)
5534 tmp
= expand_vec_perm_1 (icode
, target
, v0
, v1
, sel
);
5539 /* As a special case to aid several targets, lower the element-based
5540 permutation to a byte-based permutation and try again. */
5541 if (qimode
== VOIDmode
)
5543 icode
= direct_optab_handler (vec_perm_optab
, qimode
);
5544 if (icode
== CODE_FOR_nothing
)
5549 /* Multiply each element by its byte size. */
5550 machine_mode selmode
= GET_MODE (sel
);
5552 sel
= expand_simple_binop (selmode
, PLUS
, sel
, sel
,
5553 NULL
, 0, OPTAB_DIRECT
);
5555 sel
= expand_simple_binop (selmode
, ASHIFT
, sel
,
5556 GEN_INT (exact_log2 (u
)),
5557 NULL
, 0, OPTAB_DIRECT
);
5558 gcc_assert (sel
!= NULL
);
5560 /* Broadcast the low byte each element into each of its bytes. */
5561 vec
= rtvec_alloc (w
);
5562 for (i
= 0; i
< w
; ++i
)
5564 int this_e
= i
/ u
* u
;
5565 if (BYTES_BIG_ENDIAN
)
5567 RTVEC_ELT (vec
, i
) = GEN_INT (this_e
);
5569 tmp
= gen_rtx_CONST_VECTOR (qimode
, vec
);
5570 sel
= gen_lowpart (qimode
, sel
);
5571 sel
= expand_vec_perm (qimode
, sel
, sel
, tmp
, NULL
);
5572 gcc_assert (sel
!= NULL
);
5574 /* Add the byte offset to each byte element. */
5575 /* Note that the definition of the indicies here is memory ordering,
5576 so there should be no difference between big and little endian. */
5577 vec
= rtvec_alloc (w
);
5578 for (i
= 0; i
< w
; ++i
)
5579 RTVEC_ELT (vec
, i
) = GEN_INT (i
% u
);
5580 tmp
= gen_rtx_CONST_VECTOR (qimode
, vec
);
5581 sel_qi
= expand_simple_binop (qimode
, PLUS
, sel
, tmp
,
5582 sel
, 0, OPTAB_DIRECT
);
5583 gcc_assert (sel_qi
!= NULL
);
5586 tmp
= mode
!= qimode
? gen_reg_rtx (qimode
) : target
;
5587 tmp
= expand_vec_perm_1 (icode
, tmp
, gen_lowpart (qimode
, v0
),
5588 gen_lowpart (qimode
, v1
), sel_qi
);
5590 tmp
= gen_lowpart (mode
, tmp
);
5594 /* Generate insns for a VEC_COND_EXPR with mask, given its TYPE and its
5598 expand_vec_cond_mask_expr (tree vec_cond_type
, tree op0
, tree op1
, tree op2
,
5601 struct expand_operand ops
[4];
5602 machine_mode mode
= TYPE_MODE (vec_cond_type
);
5603 machine_mode mask_mode
= TYPE_MODE (TREE_TYPE (op0
));
5604 enum insn_code icode
= get_vcond_mask_icode (mode
, mask_mode
);
5605 rtx mask
, rtx_op1
, rtx_op2
;
5607 if (icode
== CODE_FOR_nothing
)
5610 mask
= expand_normal (op0
);
5611 rtx_op1
= expand_normal (op1
);
5612 rtx_op2
= expand_normal (op2
);
5614 mask
= force_reg (mask_mode
, mask
);
5615 rtx_op1
= force_reg (GET_MODE (rtx_op1
), rtx_op1
);
5617 create_output_operand (&ops
[0], target
, mode
);
5618 create_input_operand (&ops
[1], rtx_op1
, mode
);
5619 create_input_operand (&ops
[2], rtx_op2
, mode
);
5620 create_input_operand (&ops
[3], mask
, mask_mode
);
5621 expand_insn (icode
, 4, ops
);
5623 return ops
[0].value
;
5626 /* Generate insns for a VEC_COND_EXPR, given its TYPE and its
5630 expand_vec_cond_expr (tree vec_cond_type
, tree op0
, tree op1
, tree op2
,
5633 struct expand_operand ops
[6];
5634 enum insn_code icode
;
5635 rtx comparison
, rtx_op1
, rtx_op2
;
5636 machine_mode mode
= TYPE_MODE (vec_cond_type
);
5637 machine_mode cmp_op_mode
;
5640 enum tree_code tcode
;
5642 if (COMPARISON_CLASS_P (op0
))
5644 op0a
= TREE_OPERAND (op0
, 0);
5645 op0b
= TREE_OPERAND (op0
, 1);
5646 tcode
= TREE_CODE (op0
);
5650 gcc_assert (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (op0
)));
5651 if (get_vcond_mask_icode (mode
, TYPE_MODE (TREE_TYPE (op0
)))
5652 != CODE_FOR_nothing
)
5653 return expand_vec_cond_mask_expr (vec_cond_type
, op0
, op1
,
5658 gcc_assert (GET_MODE_CLASS (TYPE_MODE (TREE_TYPE (op0
)))
5659 == MODE_VECTOR_INT
);
5661 op0b
= build_zero_cst (TREE_TYPE (op0
));
5665 cmp_op_mode
= TYPE_MODE (TREE_TYPE (op0a
));
5666 unsignedp
= TYPE_UNSIGNED (TREE_TYPE (op0a
));
5669 gcc_assert (GET_MODE_SIZE (mode
) == GET_MODE_SIZE (cmp_op_mode
)
5670 && GET_MODE_NUNITS (mode
) == GET_MODE_NUNITS (cmp_op_mode
));
5672 icode
= get_vcond_icode (mode
, cmp_op_mode
, unsignedp
);
5673 if (icode
== CODE_FOR_nothing
)
5675 if (tcode
== EQ_EXPR
|| tcode
== NE_EXPR
)
5676 icode
= get_vcond_eq_icode (mode
, cmp_op_mode
);
5677 if (icode
== CODE_FOR_nothing
)
5681 comparison
= vector_compare_rtx (VOIDmode
, tcode
, op0a
, op0b
, unsignedp
,
5683 rtx_op1
= expand_normal (op1
);
5684 rtx_op2
= expand_normal (op2
);
5686 create_output_operand (&ops
[0], target
, mode
);
5687 create_input_operand (&ops
[1], rtx_op1
, mode
);
5688 create_input_operand (&ops
[2], rtx_op2
, mode
);
5689 create_fixed_operand (&ops
[3], comparison
);
5690 create_fixed_operand (&ops
[4], XEXP (comparison
, 0));
5691 create_fixed_operand (&ops
[5], XEXP (comparison
, 1));
5692 expand_insn (icode
, 6, ops
);
5693 return ops
[0].value
;
5696 /* Generate insns for a vector comparison into a mask. */
5699 expand_vec_cmp_expr (tree type
, tree exp
, rtx target
)
5701 struct expand_operand ops
[4];
5702 enum insn_code icode
;
5704 machine_mode mask_mode
= TYPE_MODE (type
);
5708 enum tree_code tcode
;
5710 op0a
= TREE_OPERAND (exp
, 0);
5711 op0b
= TREE_OPERAND (exp
, 1);
5712 tcode
= TREE_CODE (exp
);
5714 unsignedp
= TYPE_UNSIGNED (TREE_TYPE (op0a
));
5715 vmode
= TYPE_MODE (TREE_TYPE (op0a
));
5717 icode
= get_vec_cmp_icode (vmode
, mask_mode
, unsignedp
);
5718 if (icode
== CODE_FOR_nothing
)
5720 if (tcode
== EQ_EXPR
|| tcode
== NE_EXPR
)
5721 icode
= get_vec_cmp_eq_icode (vmode
, mask_mode
);
5722 if (icode
== CODE_FOR_nothing
)
5726 comparison
= vector_compare_rtx (mask_mode
, tcode
, op0a
, op0b
,
5727 unsignedp
, icode
, 2);
5728 create_output_operand (&ops
[0], target
, mask_mode
);
5729 create_fixed_operand (&ops
[1], comparison
);
5730 create_fixed_operand (&ops
[2], XEXP (comparison
, 0));
5731 create_fixed_operand (&ops
[3], XEXP (comparison
, 1));
5732 expand_insn (icode
, 4, ops
);
5733 return ops
[0].value
;
5736 /* Expand a highpart multiply. */
5739 expand_mult_highpart (machine_mode mode
, rtx op0
, rtx op1
,
5740 rtx target
, bool uns_p
)
5742 struct expand_operand eops
[3];
5743 enum insn_code icode
;
5744 int method
, i
, nunits
;
5750 method
= can_mult_highpart_p (mode
, uns_p
);
5756 tab1
= uns_p
? umul_highpart_optab
: smul_highpart_optab
;
5757 return expand_binop (mode
, tab1
, op0
, op1
, target
, uns_p
,
5760 tab1
= uns_p
? vec_widen_umult_even_optab
: vec_widen_smult_even_optab
;
5761 tab2
= uns_p
? vec_widen_umult_odd_optab
: vec_widen_smult_odd_optab
;
5764 tab1
= uns_p
? vec_widen_umult_lo_optab
: vec_widen_smult_lo_optab
;
5765 tab2
= uns_p
? vec_widen_umult_hi_optab
: vec_widen_smult_hi_optab
;
5766 if (BYTES_BIG_ENDIAN
)
5767 std::swap (tab1
, tab2
);
5773 icode
= optab_handler (tab1
, mode
);
5774 nunits
= GET_MODE_NUNITS (mode
);
5775 wmode
= insn_data
[icode
].operand
[0].mode
;
5776 gcc_checking_assert (2 * GET_MODE_NUNITS (wmode
) == nunits
);
5777 gcc_checking_assert (GET_MODE_SIZE (wmode
) == GET_MODE_SIZE (mode
));
5779 create_output_operand (&eops
[0], gen_reg_rtx (wmode
), wmode
);
5780 create_input_operand (&eops
[1], op0
, mode
);
5781 create_input_operand (&eops
[2], op1
, mode
);
5782 expand_insn (icode
, 3, eops
);
5783 m1
= gen_lowpart (mode
, eops
[0].value
);
5785 create_output_operand (&eops
[0], gen_reg_rtx (wmode
), wmode
);
5786 create_input_operand (&eops
[1], op0
, mode
);
5787 create_input_operand (&eops
[2], op1
, mode
);
5788 expand_insn (optab_handler (tab2
, mode
), 3, eops
);
5789 m2
= gen_lowpart (mode
, eops
[0].value
);
5791 v
= rtvec_alloc (nunits
);
5794 for (i
= 0; i
< nunits
; ++i
)
5795 RTVEC_ELT (v
, i
) = GEN_INT (!BYTES_BIG_ENDIAN
+ (i
& ~1)
5796 + ((i
& 1) ? nunits
: 0));
5797 perm
= gen_rtx_CONST_VECTOR (mode
, v
);
5801 int base
= BYTES_BIG_ENDIAN
? 0 : 1;
5802 perm
= gen_const_vec_series (mode
, GEN_INT (base
), GEN_INT (2));
5805 return expand_vec_perm (mode
, m1
, m2
, perm
, target
);
5808 /* Helper function to find the MODE_CC set in a sync_compare_and_swap
5812 find_cc_set (rtx x
, const_rtx pat
, void *data
)
5814 if (REG_P (x
) && GET_MODE_CLASS (GET_MODE (x
)) == MODE_CC
5815 && GET_CODE (pat
) == SET
)
5817 rtx
*p_cc_reg
= (rtx
*) data
;
5818 gcc_assert (!*p_cc_reg
);
5823 /* This is a helper function for the other atomic operations. This function
5824 emits a loop that contains SEQ that iterates until a compare-and-swap
5825 operation at the end succeeds. MEM is the memory to be modified. SEQ is
5826 a set of instructions that takes a value from OLD_REG as an input and
5827 produces a value in NEW_REG as an output. Before SEQ, OLD_REG will be
5828 set to the current contents of MEM. After SEQ, a compare-and-swap will
5829 attempt to update MEM with NEW_REG. The function returns true when the
5830 loop was generated successfully. */
5833 expand_compare_and_swap_loop (rtx mem
, rtx old_reg
, rtx new_reg
, rtx seq
)
5835 machine_mode mode
= GET_MODE (mem
);
5836 rtx_code_label
*label
;
5837 rtx cmp_reg
, success
, oldval
;
5839 /* The loop we want to generate looks like
5845 (success, cmp_reg) = compare-and-swap(mem, old_reg, new_reg)
5849 Note that we only do the plain load from memory once. Subsequent
5850 iterations use the value loaded by the compare-and-swap pattern. */
5852 label
= gen_label_rtx ();
5853 cmp_reg
= gen_reg_rtx (mode
);
5855 emit_move_insn (cmp_reg
, mem
);
5857 emit_move_insn (old_reg
, cmp_reg
);
5863 if (!expand_atomic_compare_and_swap (&success
, &oldval
, mem
, old_reg
,
5864 new_reg
, false, MEMMODEL_SYNC_SEQ_CST
,
5868 if (oldval
!= cmp_reg
)
5869 emit_move_insn (cmp_reg
, oldval
);
5871 /* Mark this jump predicted not taken. */
5872 emit_cmp_and_jump_insns (success
, const0_rtx
, EQ
, const0_rtx
,
5873 GET_MODE (success
), 1, label
,
5874 profile_probability::guessed_never ());
5879 /* This function tries to emit an atomic_exchange intruction. VAL is written
5880 to *MEM using memory model MODEL. The previous contents of *MEM are returned,
5881 using TARGET if possible. */
5884 maybe_emit_atomic_exchange (rtx target
, rtx mem
, rtx val
, enum memmodel model
)
5886 machine_mode mode
= GET_MODE (mem
);
5887 enum insn_code icode
;
5889 /* If the target supports the exchange directly, great. */
5890 icode
= direct_optab_handler (atomic_exchange_optab
, mode
);
5891 if (icode
!= CODE_FOR_nothing
)
5893 struct expand_operand ops
[4];
5895 create_output_operand (&ops
[0], target
, mode
);
5896 create_fixed_operand (&ops
[1], mem
);
5897 create_input_operand (&ops
[2], val
, mode
);
5898 create_integer_operand (&ops
[3], model
);
5899 if (maybe_expand_insn (icode
, 4, ops
))
5900 return ops
[0].value
;
5906 /* This function tries to implement an atomic exchange operation using
5907 __sync_lock_test_and_set. VAL is written to *MEM using memory model MODEL.
5908 The previous contents of *MEM are returned, using TARGET if possible.
5909 Since this instructionn is an acquire barrier only, stronger memory
5910 models may require additional barriers to be emitted. */
5913 maybe_emit_sync_lock_test_and_set (rtx target
, rtx mem
, rtx val
,
5914 enum memmodel model
)
5916 machine_mode mode
= GET_MODE (mem
);
5917 enum insn_code icode
;
5918 rtx_insn
*last_insn
= get_last_insn ();
5920 icode
= optab_handler (sync_lock_test_and_set_optab
, mode
);
5922 /* Legacy sync_lock_test_and_set is an acquire barrier. If the pattern
5923 exists, and the memory model is stronger than acquire, add a release
5924 barrier before the instruction. */
5926 if (is_mm_seq_cst (model
) || is_mm_release (model
) || is_mm_acq_rel (model
))
5927 expand_mem_thread_fence (model
);
5929 if (icode
!= CODE_FOR_nothing
)
5931 struct expand_operand ops
[3];
5932 create_output_operand (&ops
[0], target
, mode
);
5933 create_fixed_operand (&ops
[1], mem
);
5934 create_input_operand (&ops
[2], val
, mode
);
5935 if (maybe_expand_insn (icode
, 3, ops
))
5936 return ops
[0].value
;
5939 /* If an external test-and-set libcall is provided, use that instead of
5940 any external compare-and-swap that we might get from the compare-and-
5941 swap-loop expansion later. */
5942 if (!can_compare_and_swap_p (mode
, false))
5944 rtx libfunc
= optab_libfunc (sync_lock_test_and_set_optab
, mode
);
5945 if (libfunc
!= NULL
)
5949 addr
= convert_memory_address (ptr_mode
, XEXP (mem
, 0));
5950 return emit_library_call_value (libfunc
, NULL_RTX
, LCT_NORMAL
,
5951 mode
, addr
, ptr_mode
,
5956 /* If the test_and_set can't be emitted, eliminate any barrier that might
5957 have been emitted. */
5958 delete_insns_since (last_insn
);
5962 /* This function tries to implement an atomic exchange operation using a
5963 compare_and_swap loop. VAL is written to *MEM. The previous contents of
5964 *MEM are returned, using TARGET if possible. No memory model is required
5965 since a compare_and_swap loop is seq-cst. */
5968 maybe_emit_compare_and_swap_exchange_loop (rtx target
, rtx mem
, rtx val
)
5970 machine_mode mode
= GET_MODE (mem
);
5972 if (can_compare_and_swap_p (mode
, true))
5974 if (!target
|| !register_operand (target
, mode
))
5975 target
= gen_reg_rtx (mode
);
5976 if (expand_compare_and_swap_loop (mem
, target
, val
, NULL_RTX
))
5983 /* This function tries to implement an atomic test-and-set operation
5984 using the atomic_test_and_set instruction pattern. A boolean value
5985 is returned from the operation, using TARGET if possible. */
5988 maybe_emit_atomic_test_and_set (rtx target
, rtx mem
, enum memmodel model
)
5990 machine_mode pat_bool_mode
;
5991 struct expand_operand ops
[3];
5993 if (!targetm
.have_atomic_test_and_set ())
5996 /* While we always get QImode from __atomic_test_and_set, we get
5997 other memory modes from __sync_lock_test_and_set. Note that we
5998 use no endian adjustment here. This matches the 4.6 behavior
5999 in the Sparc backend. */
6000 enum insn_code icode
= targetm
.code_for_atomic_test_and_set
;
6001 gcc_checking_assert (insn_data
[icode
].operand
[1].mode
== QImode
);
6002 if (GET_MODE (mem
) != QImode
)
6003 mem
= adjust_address_nv (mem
, QImode
, 0);
6005 pat_bool_mode
= insn_data
[icode
].operand
[0].mode
;
6006 create_output_operand (&ops
[0], target
, pat_bool_mode
);
6007 create_fixed_operand (&ops
[1], mem
);
6008 create_integer_operand (&ops
[2], model
);
6010 if (maybe_expand_insn (icode
, 3, ops
))
6011 return ops
[0].value
;
6015 /* This function expands the legacy _sync_lock test_and_set operation which is
6016 generally an atomic exchange. Some limited targets only allow the
6017 constant 1 to be stored. This is an ACQUIRE operation.
6019 TARGET is an optional place to stick the return value.
6020 MEM is where VAL is stored. */
6023 expand_sync_lock_test_and_set (rtx target
, rtx mem
, rtx val
)
6027 /* Try an atomic_exchange first. */
6028 ret
= maybe_emit_atomic_exchange (target
, mem
, val
, MEMMODEL_SYNC_ACQUIRE
);
6032 ret
= maybe_emit_sync_lock_test_and_set (target
, mem
, val
,
6033 MEMMODEL_SYNC_ACQUIRE
);
6037 ret
= maybe_emit_compare_and_swap_exchange_loop (target
, mem
, val
);
6041 /* If there are no other options, try atomic_test_and_set if the value
6042 being stored is 1. */
6043 if (val
== const1_rtx
)
6044 ret
= maybe_emit_atomic_test_and_set (target
, mem
, MEMMODEL_SYNC_ACQUIRE
);
6049 /* This function expands the atomic test_and_set operation:
6050 atomically store a boolean TRUE into MEM and return the previous value.
6052 MEMMODEL is the memory model variant to use.
6053 TARGET is an optional place to stick the return value. */
6056 expand_atomic_test_and_set (rtx target
, rtx mem
, enum memmodel model
)
6058 machine_mode mode
= GET_MODE (mem
);
6059 rtx ret
, trueval
, subtarget
;
6061 ret
= maybe_emit_atomic_test_and_set (target
, mem
, model
);
6065 /* Be binary compatible with non-default settings of trueval, and different
6066 cpu revisions. E.g. one revision may have atomic-test-and-set, but
6067 another only has atomic-exchange. */
6068 if (targetm
.atomic_test_and_set_trueval
== 1)
6070 trueval
= const1_rtx
;
6071 subtarget
= target
? target
: gen_reg_rtx (mode
);
6075 trueval
= gen_int_mode (targetm
.atomic_test_and_set_trueval
, mode
);
6076 subtarget
= gen_reg_rtx (mode
);
6079 /* Try the atomic-exchange optab... */
6080 ret
= maybe_emit_atomic_exchange (subtarget
, mem
, trueval
, model
);
6082 /* ... then an atomic-compare-and-swap loop ... */
6084 ret
= maybe_emit_compare_and_swap_exchange_loop (subtarget
, mem
, trueval
);
6086 /* ... before trying the vaguely defined legacy lock_test_and_set. */
6088 ret
= maybe_emit_sync_lock_test_and_set (subtarget
, mem
, trueval
, model
);
6090 /* Recall that the legacy lock_test_and_set optab was allowed to do magic
6091 things with the value 1. Thus we try again without trueval. */
6092 if (!ret
&& targetm
.atomic_test_and_set_trueval
!= 1)
6093 ret
= maybe_emit_sync_lock_test_and_set (subtarget
, mem
, const1_rtx
, model
);
6095 /* Failing all else, assume a single threaded environment and simply
6096 perform the operation. */
6099 /* If the result is ignored skip the move to target. */
6100 if (subtarget
!= const0_rtx
)
6101 emit_move_insn (subtarget
, mem
);
6103 emit_move_insn (mem
, trueval
);
6107 /* Recall that have to return a boolean value; rectify if trueval
6108 is not exactly one. */
6109 if (targetm
.atomic_test_and_set_trueval
!= 1)
6110 ret
= emit_store_flag_force (target
, NE
, ret
, const0_rtx
, mode
, 0, 1);
6115 /* This function expands the atomic exchange operation:
6116 atomically store VAL in MEM and return the previous value in MEM.
6118 MEMMODEL is the memory model variant to use.
6119 TARGET is an optional place to stick the return value. */
6122 expand_atomic_exchange (rtx target
, rtx mem
, rtx val
, enum memmodel model
)
6124 machine_mode mode
= GET_MODE (mem
);
6127 /* If loads are not atomic for the required size and we are not called to
6128 provide a __sync builtin, do not do anything so that we stay consistent
6129 with atomic loads of the same size. */
6130 if (!can_atomic_load_p (mode
) && !is_mm_sync (model
))
6133 ret
= maybe_emit_atomic_exchange (target
, mem
, val
, model
);
6135 /* Next try a compare-and-swap loop for the exchange. */
6137 ret
= maybe_emit_compare_and_swap_exchange_loop (target
, mem
, val
);
6142 /* This function expands the atomic compare exchange operation:
6144 *PTARGET_BOOL is an optional place to store the boolean success/failure.
6145 *PTARGET_OVAL is an optional place to store the old value from memory.
6146 Both target parameters may be NULL or const0_rtx to indicate that we do
6147 not care about that return value. Both target parameters are updated on
6148 success to the actual location of the corresponding result.
6150 MEMMODEL is the memory model variant to use.
6152 The return value of the function is true for success. */
6155 expand_atomic_compare_and_swap (rtx
*ptarget_bool
, rtx
*ptarget_oval
,
6156 rtx mem
, rtx expected
, rtx desired
,
6157 bool is_weak
, enum memmodel succ_model
,
6158 enum memmodel fail_model
)
6160 machine_mode mode
= GET_MODE (mem
);
6161 struct expand_operand ops
[8];
6162 enum insn_code icode
;
6163 rtx target_oval
, target_bool
= NULL_RTX
;
6166 /* If loads are not atomic for the required size and we are not called to
6167 provide a __sync builtin, do not do anything so that we stay consistent
6168 with atomic loads of the same size. */
6169 if (!can_atomic_load_p (mode
) && !is_mm_sync (succ_model
))
6172 /* Load expected into a register for the compare and swap. */
6173 if (MEM_P (expected
))
6174 expected
= copy_to_reg (expected
);
6176 /* Make sure we always have some place to put the return oldval.
6177 Further, make sure that place is distinct from the input expected,
6178 just in case we need that path down below. */
6179 if (ptarget_oval
&& *ptarget_oval
== const0_rtx
)
6180 ptarget_oval
= NULL
;
6182 if (ptarget_oval
== NULL
6183 || (target_oval
= *ptarget_oval
) == NULL
6184 || reg_overlap_mentioned_p (expected
, target_oval
))
6185 target_oval
= gen_reg_rtx (mode
);
6187 icode
= direct_optab_handler (atomic_compare_and_swap_optab
, mode
);
6188 if (icode
!= CODE_FOR_nothing
)
6190 machine_mode bool_mode
= insn_data
[icode
].operand
[0].mode
;
6192 if (ptarget_bool
&& *ptarget_bool
== const0_rtx
)
6193 ptarget_bool
= NULL
;
6195 /* Make sure we always have a place for the bool operand. */
6196 if (ptarget_bool
== NULL
6197 || (target_bool
= *ptarget_bool
) == NULL
6198 || GET_MODE (target_bool
) != bool_mode
)
6199 target_bool
= gen_reg_rtx (bool_mode
);
6201 /* Emit the compare_and_swap. */
6202 create_output_operand (&ops
[0], target_bool
, bool_mode
);
6203 create_output_operand (&ops
[1], target_oval
, mode
);
6204 create_fixed_operand (&ops
[2], mem
);
6205 create_input_operand (&ops
[3], expected
, mode
);
6206 create_input_operand (&ops
[4], desired
, mode
);
6207 create_integer_operand (&ops
[5], is_weak
);
6208 create_integer_operand (&ops
[6], succ_model
);
6209 create_integer_operand (&ops
[7], fail_model
);
6210 if (maybe_expand_insn (icode
, 8, ops
))
6212 /* Return success/failure. */
6213 target_bool
= ops
[0].value
;
6214 target_oval
= ops
[1].value
;
6219 /* Otherwise fall back to the original __sync_val_compare_and_swap
6220 which is always seq-cst. */
6221 icode
= optab_handler (sync_compare_and_swap_optab
, mode
);
6222 if (icode
!= CODE_FOR_nothing
)
6226 create_output_operand (&ops
[0], target_oval
, mode
);
6227 create_fixed_operand (&ops
[1], mem
);
6228 create_input_operand (&ops
[2], expected
, mode
);
6229 create_input_operand (&ops
[3], desired
, mode
);
6230 if (!maybe_expand_insn (icode
, 4, ops
))
6233 target_oval
= ops
[0].value
;
6235 /* If the caller isn't interested in the boolean return value,
6236 skip the computation of it. */
6237 if (ptarget_bool
== NULL
)
6240 /* Otherwise, work out if the compare-and-swap succeeded. */
6242 if (have_insn_for (COMPARE
, CCmode
))
6243 note_stores (PATTERN (get_last_insn ()), find_cc_set
, &cc_reg
);
6246 target_bool
= emit_store_flag_force (target_bool
, EQ
, cc_reg
,
6247 const0_rtx
, VOIDmode
, 0, 1);
6250 goto success_bool_from_val
;
6253 /* Also check for library support for __sync_val_compare_and_swap. */
6254 libfunc
= optab_libfunc (sync_compare_and_swap_optab
, mode
);
6255 if (libfunc
!= NULL
)
6257 rtx addr
= convert_memory_address (ptr_mode
, XEXP (mem
, 0));
6258 rtx target
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_NORMAL
,
6259 mode
, addr
, ptr_mode
,
6260 expected
, mode
, desired
, mode
);
6261 emit_move_insn (target_oval
, target
);
6263 /* Compute the boolean return value only if requested. */
6265 goto success_bool_from_val
;
6273 success_bool_from_val
:
6274 target_bool
= emit_store_flag_force (target_bool
, EQ
, target_oval
,
6275 expected
, VOIDmode
, 1, 1);
6277 /* Make sure that the oval output winds up where the caller asked. */
6279 *ptarget_oval
= target_oval
;
6281 *ptarget_bool
= target_bool
;
6285 /* Generate asm volatile("" : : : "memory") as the memory blockage. */
6288 expand_asm_memory_blockage (void)
6292 asm_op
= gen_rtx_ASM_OPERANDS (VOIDmode
, "", "", 0,
6293 rtvec_alloc (0), rtvec_alloc (0),
6294 rtvec_alloc (0), UNKNOWN_LOCATION
);
6295 MEM_VOLATILE_P (asm_op
) = 1;
6297 clob
= gen_rtx_SCRATCH (VOIDmode
);
6298 clob
= gen_rtx_MEM (BLKmode
, clob
);
6299 clob
= gen_rtx_CLOBBER (VOIDmode
, clob
);
6301 emit_insn (gen_rtx_PARALLEL (VOIDmode
, gen_rtvec (2, asm_op
, clob
)));
6304 /* Do not propagate memory accesses across this point. */
6307 expand_memory_blockage (void)
6309 if (targetm
.have_memory_blockage ())
6310 emit_insn (targetm
.gen_memory_blockage ());
6312 expand_asm_memory_blockage ();
6315 /* This routine will either emit the mem_thread_fence pattern or issue a
6316 sync_synchronize to generate a fence for memory model MEMMODEL. */
6319 expand_mem_thread_fence (enum memmodel model
)
6321 if (is_mm_relaxed (model
))
6323 if (targetm
.have_mem_thread_fence ())
6325 emit_insn (targetm
.gen_mem_thread_fence (GEN_INT (model
)));
6326 expand_memory_blockage ();
6328 else if (targetm
.have_memory_barrier ())
6329 emit_insn (targetm
.gen_memory_barrier ());
6330 else if (synchronize_libfunc
!= NULL_RTX
)
6331 emit_library_call (synchronize_libfunc
, LCT_NORMAL
, VOIDmode
);
6333 expand_memory_blockage ();
6336 /* Emit a signal fence with given memory model. */
6339 expand_mem_signal_fence (enum memmodel model
)
6341 /* No machine barrier is required to implement a signal fence, but
6342 a compiler memory barrier must be issued, except for relaxed MM. */
6343 if (!is_mm_relaxed (model
))
6344 expand_memory_blockage ();
6347 /* This function expands the atomic load operation:
6348 return the atomically loaded value in MEM.
6350 MEMMODEL is the memory model variant to use.
6351 TARGET is an option place to stick the return value. */
6354 expand_atomic_load (rtx target
, rtx mem
, enum memmodel model
)
6356 machine_mode mode
= GET_MODE (mem
);
6357 enum insn_code icode
;
6359 /* If the target supports the load directly, great. */
6360 icode
= direct_optab_handler (atomic_load_optab
, mode
);
6361 if (icode
!= CODE_FOR_nothing
)
6363 struct expand_operand ops
[3];
6364 rtx_insn
*last
= get_last_insn ();
6365 if (is_mm_seq_cst (model
))
6366 expand_memory_blockage ();
6368 create_output_operand (&ops
[0], target
, mode
);
6369 create_fixed_operand (&ops
[1], mem
);
6370 create_integer_operand (&ops
[2], model
);
6371 if (maybe_expand_insn (icode
, 3, ops
))
6373 if (!is_mm_relaxed (model
))
6374 expand_memory_blockage ();
6375 return ops
[0].value
;
6377 delete_insns_since (last
);
6380 /* If the size of the object is greater than word size on this target,
6381 then we assume that a load will not be atomic. We could try to
6382 emulate a load with a compare-and-swap operation, but the store that
6383 doing this could result in would be incorrect if this is a volatile
6384 atomic load or targetting read-only-mapped memory. */
6385 if (GET_MODE_PRECISION (mode
) > BITS_PER_WORD
)
6386 /* If there is no atomic load, leave the library call. */
6389 /* Otherwise assume loads are atomic, and emit the proper barriers. */
6390 if (!target
|| target
== const0_rtx
)
6391 target
= gen_reg_rtx (mode
);
6393 /* For SEQ_CST, emit a barrier before the load. */
6394 if (is_mm_seq_cst (model
))
6395 expand_mem_thread_fence (model
);
6397 emit_move_insn (target
, mem
);
6399 /* Emit the appropriate barrier after the load. */
6400 expand_mem_thread_fence (model
);
6405 /* This function expands the atomic store operation:
6406 Atomically store VAL in MEM.
6407 MEMMODEL is the memory model variant to use.
6408 USE_RELEASE is true if __sync_lock_release can be used as a fall back.
6409 function returns const0_rtx if a pattern was emitted. */
6412 expand_atomic_store (rtx mem
, rtx val
, enum memmodel model
, bool use_release
)
6414 machine_mode mode
= GET_MODE (mem
);
6415 enum insn_code icode
;
6416 struct expand_operand ops
[3];
6418 /* If the target supports the store directly, great. */
6419 icode
= direct_optab_handler (atomic_store_optab
, mode
);
6420 if (icode
!= CODE_FOR_nothing
)
6422 rtx_insn
*last
= get_last_insn ();
6423 if (!is_mm_relaxed (model
))
6424 expand_memory_blockage ();
6425 create_fixed_operand (&ops
[0], mem
);
6426 create_input_operand (&ops
[1], val
, mode
);
6427 create_integer_operand (&ops
[2], model
);
6428 if (maybe_expand_insn (icode
, 3, ops
))
6430 if (is_mm_seq_cst (model
))
6431 expand_memory_blockage ();
6434 delete_insns_since (last
);
6437 /* If using __sync_lock_release is a viable alternative, try it.
6438 Note that this will not be set to true if we are expanding a generic
6439 __atomic_store_n. */
6442 icode
= direct_optab_handler (sync_lock_release_optab
, mode
);
6443 if (icode
!= CODE_FOR_nothing
)
6445 create_fixed_operand (&ops
[0], mem
);
6446 create_input_operand (&ops
[1], const0_rtx
, mode
);
6447 if (maybe_expand_insn (icode
, 2, ops
))
6449 /* lock_release is only a release barrier. */
6450 if (is_mm_seq_cst (model
))
6451 expand_mem_thread_fence (model
);
6457 /* If the size of the object is greater than word size on this target,
6458 a default store will not be atomic. */
6459 if (GET_MODE_PRECISION (mode
) > BITS_PER_WORD
)
6461 /* If loads are atomic or we are called to provide a __sync builtin,
6462 we can try a atomic_exchange and throw away the result. Otherwise,
6463 don't do anything so that we do not create an inconsistency between
6464 loads and stores. */
6465 if (can_atomic_load_p (mode
) || is_mm_sync (model
))
6467 rtx target
= maybe_emit_atomic_exchange (NULL_RTX
, mem
, val
, model
);
6469 target
= maybe_emit_compare_and_swap_exchange_loop (NULL_RTX
, mem
,
6477 /* Otherwise assume stores are atomic, and emit the proper barriers. */
6478 expand_mem_thread_fence (model
);
6480 emit_move_insn (mem
, val
);
6482 /* For SEQ_CST, also emit a barrier after the store. */
6483 if (is_mm_seq_cst (model
))
6484 expand_mem_thread_fence (model
);
6490 /* Structure containing the pointers and values required to process the
6491 various forms of the atomic_fetch_op and atomic_op_fetch builtins. */
6493 struct atomic_op_functions
6495 direct_optab mem_fetch_before
;
6496 direct_optab mem_fetch_after
;
6497 direct_optab mem_no_result
;
6500 direct_optab no_result
;
6501 enum rtx_code reverse_code
;
6505 /* Fill in structure pointed to by OP with the various optab entries for an
6506 operation of type CODE. */
6509 get_atomic_op_for_code (struct atomic_op_functions
*op
, enum rtx_code code
)
6511 gcc_assert (op
!= NULL
);
6513 /* If SWITCHABLE_TARGET is defined, then subtargets can be switched
6514 in the source code during compilation, and the optab entries are not
6515 computable until runtime. Fill in the values at runtime. */
6519 op
->mem_fetch_before
= atomic_fetch_add_optab
;
6520 op
->mem_fetch_after
= atomic_add_fetch_optab
;
6521 op
->mem_no_result
= atomic_add_optab
;
6522 op
->fetch_before
= sync_old_add_optab
;
6523 op
->fetch_after
= sync_new_add_optab
;
6524 op
->no_result
= sync_add_optab
;
6525 op
->reverse_code
= MINUS
;
6528 op
->mem_fetch_before
= atomic_fetch_sub_optab
;
6529 op
->mem_fetch_after
= atomic_sub_fetch_optab
;
6530 op
->mem_no_result
= atomic_sub_optab
;
6531 op
->fetch_before
= sync_old_sub_optab
;
6532 op
->fetch_after
= sync_new_sub_optab
;
6533 op
->no_result
= sync_sub_optab
;
6534 op
->reverse_code
= PLUS
;
6537 op
->mem_fetch_before
= atomic_fetch_xor_optab
;
6538 op
->mem_fetch_after
= atomic_xor_fetch_optab
;
6539 op
->mem_no_result
= atomic_xor_optab
;
6540 op
->fetch_before
= sync_old_xor_optab
;
6541 op
->fetch_after
= sync_new_xor_optab
;
6542 op
->no_result
= sync_xor_optab
;
6543 op
->reverse_code
= XOR
;
6546 op
->mem_fetch_before
= atomic_fetch_and_optab
;
6547 op
->mem_fetch_after
= atomic_and_fetch_optab
;
6548 op
->mem_no_result
= atomic_and_optab
;
6549 op
->fetch_before
= sync_old_and_optab
;
6550 op
->fetch_after
= sync_new_and_optab
;
6551 op
->no_result
= sync_and_optab
;
6552 op
->reverse_code
= UNKNOWN
;
6555 op
->mem_fetch_before
= atomic_fetch_or_optab
;
6556 op
->mem_fetch_after
= atomic_or_fetch_optab
;
6557 op
->mem_no_result
= atomic_or_optab
;
6558 op
->fetch_before
= sync_old_ior_optab
;
6559 op
->fetch_after
= sync_new_ior_optab
;
6560 op
->no_result
= sync_ior_optab
;
6561 op
->reverse_code
= UNKNOWN
;
6564 op
->mem_fetch_before
= atomic_fetch_nand_optab
;
6565 op
->mem_fetch_after
= atomic_nand_fetch_optab
;
6566 op
->mem_no_result
= atomic_nand_optab
;
6567 op
->fetch_before
= sync_old_nand_optab
;
6568 op
->fetch_after
= sync_new_nand_optab
;
6569 op
->no_result
= sync_nand_optab
;
6570 op
->reverse_code
= UNKNOWN
;
6577 /* See if there is a more optimal way to implement the operation "*MEM CODE VAL"
6578 using memory order MODEL. If AFTER is true the operation needs to return
6579 the value of *MEM after the operation, otherwise the previous value.
6580 TARGET is an optional place to place the result. The result is unused if
6582 Return the result if there is a better sequence, otherwise NULL_RTX. */
6585 maybe_optimize_fetch_op (rtx target
, rtx mem
, rtx val
, enum rtx_code code
,
6586 enum memmodel model
, bool after
)
6588 /* If the value is prefetched, or not used, it may be possible to replace
6589 the sequence with a native exchange operation. */
6590 if (!after
|| target
== const0_rtx
)
6592 /* fetch_and (&x, 0, m) can be replaced with exchange (&x, 0, m). */
6593 if (code
== AND
&& val
== const0_rtx
)
6595 if (target
== const0_rtx
)
6596 target
= gen_reg_rtx (GET_MODE (mem
));
6597 return maybe_emit_atomic_exchange (target
, mem
, val
, model
);
6600 /* fetch_or (&x, -1, m) can be replaced with exchange (&x, -1, m). */
6601 if (code
== IOR
&& val
== constm1_rtx
)
6603 if (target
== const0_rtx
)
6604 target
= gen_reg_rtx (GET_MODE (mem
));
6605 return maybe_emit_atomic_exchange (target
, mem
, val
, model
);
6612 /* Try to emit an instruction for a specific operation varaition.
6613 OPTAB contains the OP functions.
6614 TARGET is an optional place to return the result. const0_rtx means unused.
6615 MEM is the memory location to operate on.
6616 VAL is the value to use in the operation.
6617 USE_MEMMODEL is TRUE if the variation with a memory model should be tried.
6618 MODEL is the memory model, if used.
6619 AFTER is true if the returned result is the value after the operation. */
6622 maybe_emit_op (const struct atomic_op_functions
*optab
, rtx target
, rtx mem
,
6623 rtx val
, bool use_memmodel
, enum memmodel model
, bool after
)
6625 machine_mode mode
= GET_MODE (mem
);
6626 struct expand_operand ops
[4];
6627 enum insn_code icode
;
6631 /* Check to see if there is a result returned. */
6632 if (target
== const0_rtx
)
6636 icode
= direct_optab_handler (optab
->mem_no_result
, mode
);
6637 create_integer_operand (&ops
[2], model
);
6642 icode
= direct_optab_handler (optab
->no_result
, mode
);
6646 /* Otherwise, we need to generate a result. */
6651 icode
= direct_optab_handler (after
? optab
->mem_fetch_after
6652 : optab
->mem_fetch_before
, mode
);
6653 create_integer_operand (&ops
[3], model
);
6658 icode
= optab_handler (after
? optab
->fetch_after
6659 : optab
->fetch_before
, mode
);
6662 create_output_operand (&ops
[op_counter
++], target
, mode
);
6664 if (icode
== CODE_FOR_nothing
)
6667 create_fixed_operand (&ops
[op_counter
++], mem
);
6668 /* VAL may have been promoted to a wider mode. Shrink it if so. */
6669 create_convert_operand_to (&ops
[op_counter
++], val
, mode
, true);
6671 if (maybe_expand_insn (icode
, num_ops
, ops
))
6672 return (target
== const0_rtx
? const0_rtx
: ops
[0].value
);
6678 /* This function expands an atomic fetch_OP or OP_fetch operation:
6679 TARGET is an option place to stick the return value. const0_rtx indicates
6680 the result is unused.
6681 atomically fetch MEM, perform the operation with VAL and return it to MEM.
6682 CODE is the operation being performed (OP)
6683 MEMMODEL is the memory model variant to use.
6684 AFTER is true to return the result of the operation (OP_fetch).
6685 AFTER is false to return the value before the operation (fetch_OP).
6687 This function will *only* generate instructions if there is a direct
6688 optab. No compare and swap loops or libcalls will be generated. */
6691 expand_atomic_fetch_op_no_fallback (rtx target
, rtx mem
, rtx val
,
6692 enum rtx_code code
, enum memmodel model
,
6695 machine_mode mode
= GET_MODE (mem
);
6696 struct atomic_op_functions optab
;
6698 bool unused_result
= (target
== const0_rtx
);
6700 get_atomic_op_for_code (&optab
, code
);
6702 /* Check to see if there are any better instructions. */
6703 result
= maybe_optimize_fetch_op (target
, mem
, val
, code
, model
, after
);
6707 /* Check for the case where the result isn't used and try those patterns. */
6710 /* Try the memory model variant first. */
6711 result
= maybe_emit_op (&optab
, target
, mem
, val
, true, model
, true);
6715 /* Next try the old style withuot a memory model. */
6716 result
= maybe_emit_op (&optab
, target
, mem
, val
, false, model
, true);
6720 /* There is no no-result pattern, so try patterns with a result. */
6724 /* Try the __atomic version. */
6725 result
= maybe_emit_op (&optab
, target
, mem
, val
, true, model
, after
);
6729 /* Try the older __sync version. */
6730 result
= maybe_emit_op (&optab
, target
, mem
, val
, false, model
, after
);
6734 /* If the fetch value can be calculated from the other variation of fetch,
6735 try that operation. */
6736 if (after
|| unused_result
|| optab
.reverse_code
!= UNKNOWN
)
6738 /* Try the __atomic version, then the older __sync version. */
6739 result
= maybe_emit_op (&optab
, target
, mem
, val
, true, model
, !after
);
6741 result
= maybe_emit_op (&optab
, target
, mem
, val
, false, model
, !after
);
6745 /* If the result isn't used, no need to do compensation code. */
6749 /* Issue compensation code. Fetch_after == fetch_before OP val.
6750 Fetch_before == after REVERSE_OP val. */
6752 code
= optab
.reverse_code
;
6755 result
= expand_simple_binop (mode
, AND
, result
, val
, NULL_RTX
,
6756 true, OPTAB_LIB_WIDEN
);
6757 result
= expand_simple_unop (mode
, NOT
, result
, target
, true);
6760 result
= expand_simple_binop (mode
, code
, result
, val
, target
,
6761 true, OPTAB_LIB_WIDEN
);
6766 /* No direct opcode can be generated. */
6772 /* This function expands an atomic fetch_OP or OP_fetch operation:
6773 TARGET is an option place to stick the return value. const0_rtx indicates
6774 the result is unused.
6775 atomically fetch MEM, perform the operation with VAL and return it to MEM.
6776 CODE is the operation being performed (OP)
6777 MEMMODEL is the memory model variant to use.
6778 AFTER is true to return the result of the operation (OP_fetch).
6779 AFTER is false to return the value before the operation (fetch_OP). */
6781 expand_atomic_fetch_op (rtx target
, rtx mem
, rtx val
, enum rtx_code code
,
6782 enum memmodel model
, bool after
)
6784 machine_mode mode
= GET_MODE (mem
);
6786 bool unused_result
= (target
== const0_rtx
);
6788 /* If loads are not atomic for the required size and we are not called to
6789 provide a __sync builtin, do not do anything so that we stay consistent
6790 with atomic loads of the same size. */
6791 if (!can_atomic_load_p (mode
) && !is_mm_sync (model
))
6794 result
= expand_atomic_fetch_op_no_fallback (target
, mem
, val
, code
, model
,
6800 /* Add/sub can be implemented by doing the reverse operation with -(val). */
6801 if (code
== PLUS
|| code
== MINUS
)
6804 enum rtx_code reverse
= (code
== PLUS
? MINUS
: PLUS
);
6807 tmp
= expand_simple_unop (mode
, NEG
, val
, NULL_RTX
, true);
6808 result
= expand_atomic_fetch_op_no_fallback (target
, mem
, tmp
, reverse
,
6812 /* PLUS worked so emit the insns and return. */
6819 /* PLUS did not work, so throw away the negation code and continue. */
6823 /* Try the __sync libcalls only if we can't do compare-and-swap inline. */
6824 if (!can_compare_and_swap_p (mode
, false))
6828 enum rtx_code orig_code
= code
;
6829 struct atomic_op_functions optab
;
6831 get_atomic_op_for_code (&optab
, code
);
6832 libfunc
= optab_libfunc (after
? optab
.fetch_after
6833 : optab
.fetch_before
, mode
);
6835 && (after
|| unused_result
|| optab
.reverse_code
!= UNKNOWN
))
6839 code
= optab
.reverse_code
;
6840 libfunc
= optab_libfunc (after
? optab
.fetch_before
6841 : optab
.fetch_after
, mode
);
6843 if (libfunc
!= NULL
)
6845 rtx addr
= convert_memory_address (ptr_mode
, XEXP (mem
, 0));
6846 result
= emit_library_call_value (libfunc
, NULL
, LCT_NORMAL
, mode
,
6847 addr
, ptr_mode
, val
, mode
);
6849 if (!unused_result
&& fixup
)
6850 result
= expand_simple_binop (mode
, code
, result
, val
, target
,
6851 true, OPTAB_LIB_WIDEN
);
6855 /* We need the original code for any further attempts. */
6859 /* If nothing else has succeeded, default to a compare and swap loop. */
6860 if (can_compare_and_swap_p (mode
, true))
6863 rtx t0
= gen_reg_rtx (mode
), t1
;
6867 /* If the result is used, get a register for it. */
6870 if (!target
|| !register_operand (target
, mode
))
6871 target
= gen_reg_rtx (mode
);
6872 /* If fetch_before, copy the value now. */
6874 emit_move_insn (target
, t0
);
6877 target
= const0_rtx
;
6882 t1
= expand_simple_binop (mode
, AND
, t1
, val
, NULL_RTX
,
6883 true, OPTAB_LIB_WIDEN
);
6884 t1
= expand_simple_unop (mode
, code
, t1
, NULL_RTX
, true);
6887 t1
= expand_simple_binop (mode
, code
, t1
, val
, NULL_RTX
, true,
6890 /* For after, copy the value now. */
6891 if (!unused_result
&& after
)
6892 emit_move_insn (target
, t1
);
6893 insn
= get_insns ();
6896 if (t1
!= NULL
&& expand_compare_and_swap_loop (mem
, t0
, t1
, insn
))
6903 /* Return true if OPERAND is suitable for operand number OPNO of
6904 instruction ICODE. */
6907 insn_operand_matches (enum insn_code icode
, unsigned int opno
, rtx operand
)
6909 return (!insn_data
[(int) icode
].operand
[opno
].predicate
6910 || (insn_data
[(int) icode
].operand
[opno
].predicate
6911 (operand
, insn_data
[(int) icode
].operand
[opno
].mode
)));
6914 /* TARGET is a target of a multiword operation that we are going to
6915 implement as a series of word-mode operations. Return true if
6916 TARGET is suitable for this purpose. */
6919 valid_multiword_target_p (rtx target
)
6924 mode
= GET_MODE (target
);
6925 for (i
= 0; i
< GET_MODE_SIZE (mode
); i
+= UNITS_PER_WORD
)
6926 if (!validate_subreg (word_mode
, mode
, target
, i
))
6931 /* Like maybe_legitimize_operand, but do not change the code of the
6932 current rtx value. */
6935 maybe_legitimize_operand_same_code (enum insn_code icode
, unsigned int opno
,
6936 struct expand_operand
*op
)
6938 /* See if the operand matches in its current form. */
6939 if (insn_operand_matches (icode
, opno
, op
->value
))
6942 /* If the operand is a memory whose address has no side effects,
6943 try forcing the address into a non-virtual pseudo register.
6944 The check for side effects is important because copy_to_mode_reg
6945 cannot handle things like auto-modified addresses. */
6946 if (insn_data
[(int) icode
].operand
[opno
].allows_mem
&& MEM_P (op
->value
))
6951 addr
= XEXP (mem
, 0);
6952 if (!(REG_P (addr
) && REGNO (addr
) > LAST_VIRTUAL_REGISTER
)
6953 && !side_effects_p (addr
))
6958 last
= get_last_insn ();
6959 mode
= get_address_mode (mem
);
6960 mem
= replace_equiv_address (mem
, copy_to_mode_reg (mode
, addr
));
6961 if (insn_operand_matches (icode
, opno
, mem
))
6966 delete_insns_since (last
);
6973 /* Try to make OP match operand OPNO of instruction ICODE. Return true
6974 on success, storing the new operand value back in OP. */
6977 maybe_legitimize_operand (enum insn_code icode
, unsigned int opno
,
6978 struct expand_operand
*op
)
6980 machine_mode mode
, imode
;
6981 bool old_volatile_ok
, result
;
6987 old_volatile_ok
= volatile_ok
;
6989 result
= maybe_legitimize_operand_same_code (icode
, opno
, op
);
6990 volatile_ok
= old_volatile_ok
;
6994 gcc_assert (mode
!= VOIDmode
);
6996 && op
->value
!= const0_rtx
6997 && GET_MODE (op
->value
) == mode
6998 && maybe_legitimize_operand_same_code (icode
, opno
, op
))
7001 op
->value
= gen_reg_rtx (mode
);
7007 gcc_assert (mode
!= VOIDmode
);
7008 gcc_assert (GET_MODE (op
->value
) == VOIDmode
7009 || GET_MODE (op
->value
) == mode
);
7010 if (maybe_legitimize_operand_same_code (icode
, opno
, op
))
7013 op
->value
= copy_to_mode_reg (mode
, op
->value
);
7016 case EXPAND_CONVERT_TO
:
7017 gcc_assert (mode
!= VOIDmode
);
7018 op
->value
= convert_to_mode (mode
, op
->value
, op
->unsigned_p
);
7021 case EXPAND_CONVERT_FROM
:
7022 if (GET_MODE (op
->value
) != VOIDmode
)
7023 mode
= GET_MODE (op
->value
);
7025 /* The caller must tell us what mode this value has. */
7026 gcc_assert (mode
!= VOIDmode
);
7028 imode
= insn_data
[(int) icode
].operand
[opno
].mode
;
7029 if (imode
!= VOIDmode
&& imode
!= mode
)
7031 op
->value
= convert_modes (imode
, mode
, op
->value
, op
->unsigned_p
);
7036 case EXPAND_ADDRESS
:
7037 op
->value
= convert_memory_address (as_a
<scalar_int_mode
> (mode
),
7041 case EXPAND_INTEGER
:
7042 mode
= insn_data
[(int) icode
].operand
[opno
].mode
;
7043 if (mode
!= VOIDmode
&& const_int_operand (op
->value
, mode
))
7047 return insn_operand_matches (icode
, opno
, op
->value
);
7050 /* Make OP describe an input operand that should have the same value
7051 as VALUE, after any mode conversion that the target might request.
7052 TYPE is the type of VALUE. */
7055 create_convert_operand_from_type (struct expand_operand
*op
,
7056 rtx value
, tree type
)
7058 create_convert_operand_from (op
, value
, TYPE_MODE (type
),
7059 TYPE_UNSIGNED (type
));
7062 /* Try to make operands [OPS, OPS + NOPS) match operands [OPNO, OPNO + NOPS)
7063 of instruction ICODE. Return true on success, leaving the new operand
7064 values in the OPS themselves. Emit no code on failure. */
7067 maybe_legitimize_operands (enum insn_code icode
, unsigned int opno
,
7068 unsigned int nops
, struct expand_operand
*ops
)
7073 last
= get_last_insn ();
7074 for (i
= 0; i
< nops
; i
++)
7075 if (!maybe_legitimize_operand (icode
, opno
+ i
, &ops
[i
]))
7077 delete_insns_since (last
);
7083 /* Try to generate instruction ICODE, using operands [OPS, OPS + NOPS)
7084 as its operands. Return the instruction pattern on success,
7085 and emit any necessary set-up code. Return null and emit no
7089 maybe_gen_insn (enum insn_code icode
, unsigned int nops
,
7090 struct expand_operand
*ops
)
7092 gcc_assert (nops
== (unsigned int) insn_data
[(int) icode
].n_generator_args
);
7093 if (!maybe_legitimize_operands (icode
, 0, nops
, ops
))
7099 return GEN_FCN (icode
) (ops
[0].value
);
7101 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
);
7103 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
);
7105 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
7108 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
7109 ops
[3].value
, ops
[4].value
);
7111 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
7112 ops
[3].value
, ops
[4].value
, ops
[5].value
);
7114 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
7115 ops
[3].value
, ops
[4].value
, ops
[5].value
,
7118 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
7119 ops
[3].value
, ops
[4].value
, ops
[5].value
,
7120 ops
[6].value
, ops
[7].value
);
7122 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
7123 ops
[3].value
, ops
[4].value
, ops
[5].value
,
7124 ops
[6].value
, ops
[7].value
, ops
[8].value
);
7129 /* Try to emit instruction ICODE, using operands [OPS, OPS + NOPS)
7130 as its operands. Return true on success and emit no code on failure. */
7133 maybe_expand_insn (enum insn_code icode
, unsigned int nops
,
7134 struct expand_operand
*ops
)
7136 rtx_insn
*pat
= maybe_gen_insn (icode
, nops
, ops
);
7145 /* Like maybe_expand_insn, but for jumps. */
7148 maybe_expand_jump_insn (enum insn_code icode
, unsigned int nops
,
7149 struct expand_operand
*ops
)
7151 rtx_insn
*pat
= maybe_gen_insn (icode
, nops
, ops
);
7154 emit_jump_insn (pat
);
7160 /* Emit instruction ICODE, using operands [OPS, OPS + NOPS)
7164 expand_insn (enum insn_code icode
, unsigned int nops
,
7165 struct expand_operand
*ops
)
7167 if (!maybe_expand_insn (icode
, nops
, ops
))
7171 /* Like expand_insn, but for jumps. */
7174 expand_jump_insn (enum insn_code icode
, unsigned int nops
,
7175 struct expand_operand
*ops
)
7177 if (!maybe_expand_jump_insn (icode
, nops
, ops
))