1 /* Expand the basic unary and binary arithmetic operations, for GNU compiler.
2 Copyright (C) 1987-2017 Free Software Foundation, Inc.
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 3, or (at your option) any later
11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
23 #include "coretypes.h"
35 #include "diagnostic-core.h"
37 /* Include insn-config.h before expr.h so that HAVE_conditional_move
38 is properly defined. */
39 #include "stor-layout.h"
44 #include "optabs-tree.h"
47 static void prepare_float_lib_cmp (rtx
, rtx
, enum rtx_code
, rtx
*,
49 static rtx
expand_unop_direct (machine_mode
, optab
, rtx
, rtx
, int);
50 static void emit_libcall_block_1 (rtx_insn
*, rtx
, rtx
, rtx
, bool);
52 /* Debug facility for use in GDB. */
53 void debug_optab_libfuncs (void);
55 /* Add a REG_EQUAL note to the last insn in INSNS. TARGET is being set to
56 the result of operation CODE applied to OP0 (and OP1 if it is a binary
59 If the last insn does not set TARGET, don't do anything, but return 1.
61 If the last insn or a previous insn sets TARGET and TARGET is one of OP0
62 or OP1, don't add the REG_EQUAL note but return 0. Our caller can then
63 try again, ensuring that TARGET is not one of the operands. */
66 add_equal_note (rtx_insn
*insns
, rtx target
, enum rtx_code code
, rtx op0
, rtx op1
)
72 gcc_assert (insns
&& INSN_P (insns
) && NEXT_INSN (insns
));
74 if (GET_RTX_CLASS (code
) != RTX_COMM_ARITH
75 && GET_RTX_CLASS (code
) != RTX_BIN_ARITH
76 && GET_RTX_CLASS (code
) != RTX_COMM_COMPARE
77 && GET_RTX_CLASS (code
) != RTX_COMPARE
78 && GET_RTX_CLASS (code
) != RTX_UNARY
)
81 if (GET_CODE (target
) == ZERO_EXTRACT
)
84 for (last_insn
= insns
;
85 NEXT_INSN (last_insn
) != NULL_RTX
;
86 last_insn
= NEXT_INSN (last_insn
))
89 /* If TARGET is in OP0 or OP1, punt. We'd end up with a note referencing
90 a value changing in the insn, so the note would be invalid for CSE. */
91 if (reg_overlap_mentioned_p (target
, op0
)
92 || (op1
&& reg_overlap_mentioned_p (target
, op1
)))
95 && (rtx_equal_p (target
, op0
)
96 || (op1
&& rtx_equal_p (target
, op1
))))
98 /* For MEM target, with MEM = MEM op X, prefer no REG_EQUAL note
99 over expanding it as temp = MEM op X, MEM = temp. If the target
100 supports MEM = MEM op X instructions, it is sometimes too hard
101 to reconstruct that form later, especially if X is also a memory,
102 and due to multiple occurrences of addresses the address might
103 be forced into register unnecessarily.
104 Note that not emitting the REG_EQUIV note might inhibit
105 CSE in some cases. */
106 set
= single_set (last_insn
);
108 && GET_CODE (SET_SRC (set
)) == code
109 && MEM_P (SET_DEST (set
))
110 && (rtx_equal_p (SET_DEST (set
), XEXP (SET_SRC (set
), 0))
111 || (op1
&& rtx_equal_p (SET_DEST (set
),
112 XEXP (SET_SRC (set
), 1)))))
118 set
= set_for_reg_notes (last_insn
);
122 if (! rtx_equal_p (SET_DEST (set
), target
)
123 /* For a STRICT_LOW_PART, the REG_NOTE applies to what is inside it. */
124 && (GET_CODE (SET_DEST (set
)) != STRICT_LOW_PART
125 || ! rtx_equal_p (XEXP (SET_DEST (set
), 0), target
)))
128 if (GET_RTX_CLASS (code
) == RTX_UNARY
)
138 if (GET_MODE (op0
) != VOIDmode
&& GET_MODE (target
) != GET_MODE (op0
))
140 note
= gen_rtx_fmt_e (code
, GET_MODE (op0
), copy_rtx (op0
));
141 if (GET_MODE_UNIT_SIZE (GET_MODE (op0
))
142 > GET_MODE_UNIT_SIZE (GET_MODE (target
)))
143 note
= simplify_gen_unary (TRUNCATE
, GET_MODE (target
),
144 note
, GET_MODE (op0
));
146 note
= simplify_gen_unary (ZERO_EXTEND
, GET_MODE (target
),
147 note
, GET_MODE (op0
));
152 note
= gen_rtx_fmt_e (code
, GET_MODE (target
), copy_rtx (op0
));
156 note
= gen_rtx_fmt_ee (code
, GET_MODE (target
), copy_rtx (op0
), copy_rtx (op1
));
158 set_unique_reg_note (last_insn
, REG_EQUAL
, note
);
163 /* Given two input operands, OP0 and OP1, determine what the correct from_mode
164 for a widening operation would be. In most cases this would be OP0, but if
165 that's a constant it'll be VOIDmode, which isn't useful. */
168 widened_mode (machine_mode to_mode
, rtx op0
, rtx op1
)
170 machine_mode m0
= GET_MODE (op0
);
171 machine_mode m1
= GET_MODE (op1
);
174 if (m0
== VOIDmode
&& m1
== VOIDmode
)
176 else if (m0
== VOIDmode
|| GET_MODE_UNIT_SIZE (m0
) < GET_MODE_UNIT_SIZE (m1
))
181 if (GET_MODE_UNIT_SIZE (result
) > GET_MODE_UNIT_SIZE (to_mode
))
187 /* Widen OP to MODE and return the rtx for the widened operand. UNSIGNEDP
188 says whether OP is signed or unsigned. NO_EXTEND is nonzero if we need
189 not actually do a sign-extend or zero-extend, but can leave the
190 higher-order bits of the result rtx undefined, for example, in the case
191 of logical operations, but not right shifts. */
194 widen_operand (rtx op
, machine_mode mode
, machine_mode oldmode
,
195 int unsignedp
, int no_extend
)
198 scalar_int_mode int_mode
;
200 /* If we don't have to extend and this is a constant, return it. */
201 if (no_extend
&& GET_MODE (op
) == VOIDmode
)
204 /* If we must extend do so. If OP is a SUBREG for a promoted object, also
205 extend since it will be more efficient to do so unless the signedness of
206 a promoted object differs from our extension. */
208 || !is_a
<scalar_int_mode
> (mode
, &int_mode
)
209 || (GET_CODE (op
) == SUBREG
&& SUBREG_PROMOTED_VAR_P (op
)
210 && SUBREG_CHECK_PROMOTED_SIGN (op
, unsignedp
)))
211 return convert_modes (mode
, oldmode
, op
, unsignedp
);
213 /* If MODE is no wider than a single word, we return a lowpart or paradoxical
215 if (GET_MODE_SIZE (int_mode
) <= UNITS_PER_WORD
)
216 return gen_lowpart (int_mode
, force_reg (GET_MODE (op
), op
));
218 /* Otherwise, get an object of MODE, clobber it, and set the low-order
221 result
= gen_reg_rtx (int_mode
);
222 emit_clobber (result
);
223 emit_move_insn (gen_lowpart (GET_MODE (op
), result
), op
);
227 /* Expand vector widening operations.
229 There are two different classes of operations handled here:
230 1) Operations whose result is wider than all the arguments to the operation.
231 Examples: VEC_UNPACK_HI/LO_EXPR, VEC_WIDEN_MULT_HI/LO_EXPR
232 In this case OP0 and optionally OP1 would be initialized,
233 but WIDE_OP wouldn't (not relevant for this case).
234 2) Operations whose result is of the same size as the last argument to the
235 operation, but wider than all the other arguments to the operation.
236 Examples: WIDEN_SUM_EXPR, VEC_DOT_PROD_EXPR.
237 In the case WIDE_OP, OP0 and optionally OP1 would be initialized.
239 E.g, when called to expand the following operations, this is how
240 the arguments will be initialized:
242 widening-sum 2 oprnd0 - oprnd1
243 widening-dot-product 3 oprnd0 oprnd1 oprnd2
244 widening-mult 2 oprnd0 oprnd1 -
245 type-promotion (vec-unpack) 1 oprnd0 - - */
248 expand_widen_pattern_expr (sepops ops
, rtx op0
, rtx op1
, rtx wide_op
,
249 rtx target
, int unsignedp
)
251 struct expand_operand eops
[4];
252 tree oprnd0
, oprnd1
, oprnd2
;
253 machine_mode wmode
= VOIDmode
, tmode0
, tmode1
= VOIDmode
;
254 optab widen_pattern_optab
;
255 enum insn_code icode
;
256 int nops
= TREE_CODE_LENGTH (ops
->code
);
260 tmode0
= TYPE_MODE (TREE_TYPE (oprnd0
));
261 widen_pattern_optab
=
262 optab_for_tree_code (ops
->code
, TREE_TYPE (oprnd0
), optab_default
);
263 if (ops
->code
== WIDEN_MULT_PLUS_EXPR
264 || ops
->code
== WIDEN_MULT_MINUS_EXPR
)
265 icode
= find_widening_optab_handler (widen_pattern_optab
,
266 TYPE_MODE (TREE_TYPE (ops
->op2
)),
269 icode
= optab_handler (widen_pattern_optab
, tmode0
);
270 gcc_assert (icode
!= CODE_FOR_nothing
);
275 tmode1
= TYPE_MODE (TREE_TYPE (oprnd1
));
278 /* The last operand is of a wider mode than the rest of the operands. */
283 gcc_assert (tmode1
== tmode0
);
286 wmode
= TYPE_MODE (TREE_TYPE (oprnd2
));
290 create_output_operand (&eops
[op
++], target
, TYPE_MODE (ops
->type
));
291 create_convert_operand_from (&eops
[op
++], op0
, tmode0
, unsignedp
);
293 create_convert_operand_from (&eops
[op
++], op1
, tmode1
, unsignedp
);
295 create_convert_operand_from (&eops
[op
++], wide_op
, wmode
, unsignedp
);
296 expand_insn (icode
, op
, eops
);
297 return eops
[0].value
;
300 /* Generate code to perform an operation specified by TERNARY_OPTAB
301 on operands OP0, OP1 and OP2, with result having machine-mode MODE.
303 UNSIGNEDP is for the case where we have to widen the operands
304 to perform the operation. It says to use zero-extension.
306 If TARGET is nonzero, the value
307 is generated there, if it is convenient to do so.
308 In all cases an rtx is returned for the locus of the value;
309 this may or may not be TARGET. */
312 expand_ternary_op (machine_mode mode
, optab ternary_optab
, rtx op0
,
313 rtx op1
, rtx op2
, rtx target
, int unsignedp
)
315 struct expand_operand ops
[4];
316 enum insn_code icode
= optab_handler (ternary_optab
, mode
);
318 gcc_assert (optab_handler (ternary_optab
, mode
) != CODE_FOR_nothing
);
320 create_output_operand (&ops
[0], target
, mode
);
321 create_convert_operand_from (&ops
[1], op0
, mode
, unsignedp
);
322 create_convert_operand_from (&ops
[2], op1
, mode
, unsignedp
);
323 create_convert_operand_from (&ops
[3], op2
, mode
, unsignedp
);
324 expand_insn (icode
, 4, ops
);
329 /* Like expand_binop, but return a constant rtx if the result can be
330 calculated at compile time. The arguments and return value are
331 otherwise the same as for expand_binop. */
334 simplify_expand_binop (machine_mode mode
, optab binoptab
,
335 rtx op0
, rtx op1
, rtx target
, int unsignedp
,
336 enum optab_methods methods
)
338 if (CONSTANT_P (op0
) && CONSTANT_P (op1
))
340 rtx x
= simplify_binary_operation (optab_to_code (binoptab
),
346 return expand_binop (mode
, binoptab
, op0
, op1
, target
, unsignedp
, methods
);
349 /* Like simplify_expand_binop, but always put the result in TARGET.
350 Return true if the expansion succeeded. */
353 force_expand_binop (machine_mode mode
, optab binoptab
,
354 rtx op0
, rtx op1
, rtx target
, int unsignedp
,
355 enum optab_methods methods
)
357 rtx x
= simplify_expand_binop (mode
, binoptab
, op0
, op1
,
358 target
, unsignedp
, methods
);
362 emit_move_insn (target
, x
);
366 /* Create a new vector value in VMODE with all elements set to OP. The
367 mode of OP must be the element mode of VMODE. If OP is a constant,
368 then the return value will be a constant. */
371 expand_vector_broadcast (machine_mode vmode
, rtx op
)
373 enum insn_code icode
;
378 gcc_checking_assert (VECTOR_MODE_P (vmode
));
380 n
= GET_MODE_NUNITS (vmode
);
381 vec
= rtvec_alloc (n
);
382 for (i
= 0; i
< n
; ++i
)
383 RTVEC_ELT (vec
, i
) = op
;
386 return gen_rtx_CONST_VECTOR (vmode
, vec
);
388 /* ??? If the target doesn't have a vec_init, then we have no easy way
389 of performing this operation. Most of this sort of generic support
390 is hidden away in the vector lowering support in gimple. */
391 icode
= convert_optab_handler (vec_init_optab
, vmode
,
392 GET_MODE_INNER (vmode
));
393 if (icode
== CODE_FOR_nothing
)
396 ret
= gen_reg_rtx (vmode
);
397 emit_insn (GEN_FCN (icode
) (ret
, gen_rtx_PARALLEL (vmode
, vec
)));
402 /* This subroutine of expand_doubleword_shift handles the cases in which
403 the effective shift value is >= BITS_PER_WORD. The arguments and return
404 value are the same as for the parent routine, except that SUPERWORD_OP1
405 is the shift count to use when shifting OUTOF_INPUT into INTO_TARGET.
406 INTO_TARGET may be null if the caller has decided to calculate it. */
409 expand_superword_shift (optab binoptab
, rtx outof_input
, rtx superword_op1
,
410 rtx outof_target
, rtx into_target
,
411 int unsignedp
, enum optab_methods methods
)
413 if (into_target
!= 0)
414 if (!force_expand_binop (word_mode
, binoptab
, outof_input
, superword_op1
,
415 into_target
, unsignedp
, methods
))
418 if (outof_target
!= 0)
420 /* For a signed right shift, we must fill OUTOF_TARGET with copies
421 of the sign bit, otherwise we must fill it with zeros. */
422 if (binoptab
!= ashr_optab
)
423 emit_move_insn (outof_target
, CONST0_RTX (word_mode
));
425 if (!force_expand_binop (word_mode
, binoptab
,
426 outof_input
, GEN_INT (BITS_PER_WORD
- 1),
427 outof_target
, unsignedp
, methods
))
433 /* This subroutine of expand_doubleword_shift handles the cases in which
434 the effective shift value is < BITS_PER_WORD. The arguments and return
435 value are the same as for the parent routine. */
438 expand_subword_shift (scalar_int_mode op1_mode
, optab binoptab
,
439 rtx outof_input
, rtx into_input
, rtx op1
,
440 rtx outof_target
, rtx into_target
,
441 int unsignedp
, enum optab_methods methods
,
442 unsigned HOST_WIDE_INT shift_mask
)
444 optab reverse_unsigned_shift
, unsigned_shift
;
447 reverse_unsigned_shift
= (binoptab
== ashl_optab
? lshr_optab
: ashl_optab
);
448 unsigned_shift
= (binoptab
== ashl_optab
? ashl_optab
: lshr_optab
);
450 /* The low OP1 bits of INTO_TARGET come from the high bits of OUTOF_INPUT.
451 We therefore need to shift OUTOF_INPUT by (BITS_PER_WORD - OP1) bits in
452 the opposite direction to BINOPTAB. */
453 if (CONSTANT_P (op1
) || shift_mask
>= BITS_PER_WORD
)
455 carries
= outof_input
;
456 tmp
= immed_wide_int_const (wi::shwi (BITS_PER_WORD
,
457 op1_mode
), op1_mode
);
458 tmp
= simplify_expand_binop (op1_mode
, sub_optab
, tmp
, op1
,
463 /* We must avoid shifting by BITS_PER_WORD bits since that is either
464 the same as a zero shift (if shift_mask == BITS_PER_WORD - 1) or
465 has unknown behavior. Do a single shift first, then shift by the
466 remainder. It's OK to use ~OP1 as the remainder if shift counts
467 are truncated to the mode size. */
468 carries
= expand_binop (word_mode
, reverse_unsigned_shift
,
469 outof_input
, const1_rtx
, 0, unsignedp
, methods
);
470 if (shift_mask
== BITS_PER_WORD
- 1)
472 tmp
= immed_wide_int_const
473 (wi::minus_one (GET_MODE_PRECISION (op1_mode
)), op1_mode
);
474 tmp
= simplify_expand_binop (op1_mode
, xor_optab
, op1
, tmp
,
479 tmp
= immed_wide_int_const (wi::shwi (BITS_PER_WORD
- 1,
480 op1_mode
), op1_mode
);
481 tmp
= simplify_expand_binop (op1_mode
, sub_optab
, tmp
, op1
,
485 if (tmp
== 0 || carries
== 0)
487 carries
= expand_binop (word_mode
, reverse_unsigned_shift
,
488 carries
, tmp
, 0, unsignedp
, methods
);
492 /* Shift INTO_INPUT logically by OP1. This is the last use of INTO_INPUT
493 so the result can go directly into INTO_TARGET if convenient. */
494 tmp
= expand_binop (word_mode
, unsigned_shift
, into_input
, op1
,
495 into_target
, unsignedp
, methods
);
499 /* Now OR in the bits carried over from OUTOF_INPUT. */
500 if (!force_expand_binop (word_mode
, ior_optab
, tmp
, carries
,
501 into_target
, unsignedp
, methods
))
504 /* Use a standard word_mode shift for the out-of half. */
505 if (outof_target
!= 0)
506 if (!force_expand_binop (word_mode
, binoptab
, outof_input
, op1
,
507 outof_target
, unsignedp
, methods
))
514 /* Try implementing expand_doubleword_shift using conditional moves.
515 The shift is by < BITS_PER_WORD if (CMP_CODE CMP1 CMP2) is true,
516 otherwise it is by >= BITS_PER_WORD. SUBWORD_OP1 and SUPERWORD_OP1
517 are the shift counts to use in the former and latter case. All other
518 arguments are the same as the parent routine. */
521 expand_doubleword_shift_condmove (scalar_int_mode op1_mode
, optab binoptab
,
522 enum rtx_code cmp_code
, rtx cmp1
, rtx cmp2
,
523 rtx outof_input
, rtx into_input
,
524 rtx subword_op1
, rtx superword_op1
,
525 rtx outof_target
, rtx into_target
,
526 int unsignedp
, enum optab_methods methods
,
527 unsigned HOST_WIDE_INT shift_mask
)
529 rtx outof_superword
, into_superword
;
531 /* Put the superword version of the output into OUTOF_SUPERWORD and
533 outof_superword
= outof_target
!= 0 ? gen_reg_rtx (word_mode
) : 0;
534 if (outof_target
!= 0 && subword_op1
== superword_op1
)
536 /* The value INTO_TARGET >> SUBWORD_OP1, which we later store in
537 OUTOF_TARGET, is the same as the value of INTO_SUPERWORD. */
538 into_superword
= outof_target
;
539 if (!expand_superword_shift (binoptab
, outof_input
, superword_op1
,
540 outof_superword
, 0, unsignedp
, methods
))
545 into_superword
= gen_reg_rtx (word_mode
);
546 if (!expand_superword_shift (binoptab
, outof_input
, superword_op1
,
547 outof_superword
, into_superword
,
552 /* Put the subword version directly in OUTOF_TARGET and INTO_TARGET. */
553 if (!expand_subword_shift (op1_mode
, binoptab
,
554 outof_input
, into_input
, subword_op1
,
555 outof_target
, into_target
,
556 unsignedp
, methods
, shift_mask
))
559 /* Select between them. Do the INTO half first because INTO_SUPERWORD
560 might be the current value of OUTOF_TARGET. */
561 if (!emit_conditional_move (into_target
, cmp_code
, cmp1
, cmp2
, op1_mode
,
562 into_target
, into_superword
, word_mode
, false))
565 if (outof_target
!= 0)
566 if (!emit_conditional_move (outof_target
, cmp_code
, cmp1
, cmp2
, op1_mode
,
567 outof_target
, outof_superword
,
574 /* Expand a doubleword shift (ashl, ashr or lshr) using word-mode shifts.
575 OUTOF_INPUT and INTO_INPUT are the two word-sized halves of the first
576 input operand; the shift moves bits in the direction OUTOF_INPUT->
577 INTO_TARGET. OUTOF_TARGET and INTO_TARGET are the equivalent words
578 of the target. OP1 is the shift count and OP1_MODE is its mode.
579 If OP1 is constant, it will have been truncated as appropriate
580 and is known to be nonzero.
582 If SHIFT_MASK is zero, the result of word shifts is undefined when the
583 shift count is outside the range [0, BITS_PER_WORD). This routine must
584 avoid generating such shifts for OP1s in the range [0, BITS_PER_WORD * 2).
586 If SHIFT_MASK is nonzero, all word-mode shift counts are effectively
587 masked by it and shifts in the range [BITS_PER_WORD, SHIFT_MASK) will
588 fill with zeros or sign bits as appropriate.
590 If SHIFT_MASK is BITS_PER_WORD - 1, this routine will synthesize
591 a doubleword shift whose equivalent mask is BITS_PER_WORD * 2 - 1.
592 Doing this preserves semantics required by SHIFT_COUNT_TRUNCATED.
593 In all other cases, shifts by values outside [0, BITS_PER_UNIT * 2)
596 BINOPTAB, UNSIGNEDP and METHODS are as for expand_binop. This function
597 may not use INTO_INPUT after modifying INTO_TARGET, and similarly for
598 OUTOF_INPUT and OUTOF_TARGET. OUTOF_TARGET can be null if the parent
599 function wants to calculate it itself.
601 Return true if the shift could be successfully synthesized. */
604 expand_doubleword_shift (scalar_int_mode op1_mode
, optab binoptab
,
605 rtx outof_input
, rtx into_input
, rtx op1
,
606 rtx outof_target
, rtx into_target
,
607 int unsignedp
, enum optab_methods methods
,
608 unsigned HOST_WIDE_INT shift_mask
)
610 rtx superword_op1
, tmp
, cmp1
, cmp2
;
611 enum rtx_code cmp_code
;
613 /* See if word-mode shifts by BITS_PER_WORD...BITS_PER_WORD * 2 - 1 will
614 fill the result with sign or zero bits as appropriate. If so, the value
615 of OUTOF_TARGET will always be (SHIFT OUTOF_INPUT OP1). Recursively call
616 this routine to calculate INTO_TARGET (which depends on both OUTOF_INPUT
617 and INTO_INPUT), then emit code to set up OUTOF_TARGET.
619 This isn't worthwhile for constant shifts since the optimizers will
620 cope better with in-range shift counts. */
621 if (shift_mask
>= BITS_PER_WORD
623 && !CONSTANT_P (op1
))
625 if (!expand_doubleword_shift (op1_mode
, binoptab
,
626 outof_input
, into_input
, op1
,
628 unsignedp
, methods
, shift_mask
))
630 if (!force_expand_binop (word_mode
, binoptab
, outof_input
, op1
,
631 outof_target
, unsignedp
, methods
))
636 /* Set CMP_CODE, CMP1 and CMP2 so that the rtx (CMP_CODE CMP1 CMP2)
637 is true when the effective shift value is less than BITS_PER_WORD.
638 Set SUPERWORD_OP1 to the shift count that should be used to shift
639 OUTOF_INPUT into INTO_TARGET when the condition is false. */
640 tmp
= immed_wide_int_const (wi::shwi (BITS_PER_WORD
, op1_mode
), op1_mode
);
641 if (!CONSTANT_P (op1
) && shift_mask
== BITS_PER_WORD
- 1)
643 /* Set CMP1 to OP1 & BITS_PER_WORD. The result is zero iff OP1
644 is a subword shift count. */
645 cmp1
= simplify_expand_binop (op1_mode
, and_optab
, op1
, tmp
,
647 cmp2
= CONST0_RTX (op1_mode
);
653 /* Set CMP1 to OP1 - BITS_PER_WORD. */
654 cmp1
= simplify_expand_binop (op1_mode
, sub_optab
, op1
, tmp
,
656 cmp2
= CONST0_RTX (op1_mode
);
658 superword_op1
= cmp1
;
663 /* If we can compute the condition at compile time, pick the
664 appropriate subroutine. */
665 tmp
= simplify_relational_operation (cmp_code
, SImode
, op1_mode
, cmp1
, cmp2
);
666 if (tmp
!= 0 && CONST_INT_P (tmp
))
668 if (tmp
== const0_rtx
)
669 return expand_superword_shift (binoptab
, outof_input
, superword_op1
,
670 outof_target
, into_target
,
673 return expand_subword_shift (op1_mode
, binoptab
,
674 outof_input
, into_input
, op1
,
675 outof_target
, into_target
,
676 unsignedp
, methods
, shift_mask
);
679 /* Try using conditional moves to generate straight-line code. */
680 if (HAVE_conditional_move
)
682 rtx_insn
*start
= get_last_insn ();
683 if (expand_doubleword_shift_condmove (op1_mode
, binoptab
,
684 cmp_code
, cmp1
, cmp2
,
685 outof_input
, into_input
,
687 outof_target
, into_target
,
688 unsignedp
, methods
, shift_mask
))
690 delete_insns_since (start
);
693 /* As a last resort, use branches to select the correct alternative. */
694 rtx_code_label
*subword_label
= gen_label_rtx ();
695 rtx_code_label
*done_label
= gen_label_rtx ();
698 do_compare_rtx_and_jump (cmp1
, cmp2
, cmp_code
, false, op1_mode
,
700 profile_probability::uninitialized ());
703 if (!expand_superword_shift (binoptab
, outof_input
, superword_op1
,
704 outof_target
, into_target
,
708 emit_jump_insn (targetm
.gen_jump (done_label
));
710 emit_label (subword_label
);
712 if (!expand_subword_shift (op1_mode
, binoptab
,
713 outof_input
, into_input
, op1
,
714 outof_target
, into_target
,
715 unsignedp
, methods
, shift_mask
))
718 emit_label (done_label
);
722 /* Subroutine of expand_binop. Perform a double word multiplication of
723 operands OP0 and OP1 both of mode MODE, which is exactly twice as wide
724 as the target's word_mode. This function return NULL_RTX if anything
725 goes wrong, in which case it may have already emitted instructions
726 which need to be deleted.
728 If we want to multiply two two-word values and have normal and widening
729 multiplies of single-word values, we can do this with three smaller
732 The multiplication proceeds as follows:
733 _______________________
734 [__op0_high_|__op0_low__]
735 _______________________
736 * [__op1_high_|__op1_low__]
737 _______________________________________________
738 _______________________
739 (1) [__op0_low__*__op1_low__]
740 _______________________
741 (2a) [__op0_low__*__op1_high_]
742 _______________________
743 (2b) [__op0_high_*__op1_low__]
744 _______________________
745 (3) [__op0_high_*__op1_high_]
748 This gives a 4-word result. Since we are only interested in the
749 lower 2 words, partial result (3) and the upper words of (2a) and
750 (2b) don't need to be calculated. Hence (2a) and (2b) can be
751 calculated using non-widening multiplication.
753 (1), however, needs to be calculated with an unsigned widening
754 multiplication. If this operation is not directly supported we
755 try using a signed widening multiplication and adjust the result.
756 This adjustment works as follows:
758 If both operands are positive then no adjustment is needed.
760 If the operands have different signs, for example op0_low < 0 and
761 op1_low >= 0, the instruction treats the most significant bit of
762 op0_low as a sign bit instead of a bit with significance
763 2**(BITS_PER_WORD-1), i.e. the instruction multiplies op1_low
764 with 2**BITS_PER_WORD - op0_low, and two's complements the
765 result. Conclusion: We need to add op1_low * 2**BITS_PER_WORD to
768 Similarly, if both operands are negative, we need to add
769 (op0_low + op1_low) * 2**BITS_PER_WORD.
771 We use a trick to adjust quickly. We logically shift op0_low right
772 (op1_low) BITS_PER_WORD-1 steps to get 0 or 1, and add this to
773 op0_high (op1_high) before it is used to calculate 2b (2a). If no
774 logical shift exists, we do an arithmetic right shift and subtract
778 expand_doubleword_mult (machine_mode mode
, rtx op0
, rtx op1
, rtx target
,
779 bool umulp
, enum optab_methods methods
)
781 int low
= (WORDS_BIG_ENDIAN
? 1 : 0);
782 int high
= (WORDS_BIG_ENDIAN
? 0 : 1);
783 rtx wordm1
= umulp
? NULL_RTX
: GEN_INT (BITS_PER_WORD
- 1);
784 rtx product
, adjust
, product_high
, temp
;
786 rtx op0_high
= operand_subword_force (op0
, high
, mode
);
787 rtx op0_low
= operand_subword_force (op0
, low
, mode
);
788 rtx op1_high
= operand_subword_force (op1
, high
, mode
);
789 rtx op1_low
= operand_subword_force (op1
, low
, mode
);
791 /* If we're using an unsigned multiply to directly compute the product
792 of the low-order words of the operands and perform any required
793 adjustments of the operands, we begin by trying two more multiplications
794 and then computing the appropriate sum.
796 We have checked above that the required addition is provided.
797 Full-word addition will normally always succeed, especially if
798 it is provided at all, so we don't worry about its failure. The
799 multiplication may well fail, however, so we do handle that. */
803 /* ??? This could be done with emit_store_flag where available. */
804 temp
= expand_binop (word_mode
, lshr_optab
, op0_low
, wordm1
,
805 NULL_RTX
, 1, methods
);
807 op0_high
= expand_binop (word_mode
, add_optab
, op0_high
, temp
,
808 NULL_RTX
, 0, OPTAB_DIRECT
);
811 temp
= expand_binop (word_mode
, ashr_optab
, op0_low
, wordm1
,
812 NULL_RTX
, 0, methods
);
815 op0_high
= expand_binop (word_mode
, sub_optab
, op0_high
, temp
,
816 NULL_RTX
, 0, OPTAB_DIRECT
);
823 adjust
= expand_binop (word_mode
, smul_optab
, op0_high
, op1_low
,
824 NULL_RTX
, 0, OPTAB_DIRECT
);
828 /* OP0_HIGH should now be dead. */
832 /* ??? This could be done with emit_store_flag where available. */
833 temp
= expand_binop (word_mode
, lshr_optab
, op1_low
, wordm1
,
834 NULL_RTX
, 1, methods
);
836 op1_high
= expand_binop (word_mode
, add_optab
, op1_high
, temp
,
837 NULL_RTX
, 0, OPTAB_DIRECT
);
840 temp
= expand_binop (word_mode
, ashr_optab
, op1_low
, wordm1
,
841 NULL_RTX
, 0, methods
);
844 op1_high
= expand_binop (word_mode
, sub_optab
, op1_high
, temp
,
845 NULL_RTX
, 0, OPTAB_DIRECT
);
852 temp
= expand_binop (word_mode
, smul_optab
, op1_high
, op0_low
,
853 NULL_RTX
, 0, OPTAB_DIRECT
);
857 /* OP1_HIGH should now be dead. */
859 adjust
= expand_binop (word_mode
, add_optab
, adjust
, temp
,
860 NULL_RTX
, 0, OPTAB_DIRECT
);
862 if (target
&& !REG_P (target
))
866 product
= expand_binop (mode
, umul_widen_optab
, op0_low
, op1_low
,
867 target
, 1, OPTAB_DIRECT
);
869 product
= expand_binop (mode
, smul_widen_optab
, op0_low
, op1_low
,
870 target
, 1, OPTAB_DIRECT
);
875 product_high
= operand_subword (product
, high
, 1, mode
);
876 adjust
= expand_binop (word_mode
, add_optab
, product_high
, adjust
,
877 NULL_RTX
, 0, OPTAB_DIRECT
);
878 emit_move_insn (product_high
, adjust
);
882 /* Wrapper around expand_binop which takes an rtx code to specify
883 the operation to perform, not an optab pointer. All other
884 arguments are the same. */
886 expand_simple_binop (machine_mode mode
, enum rtx_code code
, rtx op0
,
887 rtx op1
, rtx target
, int unsignedp
,
888 enum optab_methods methods
)
890 optab binop
= code_to_optab (code
);
893 return expand_binop (mode
, binop
, op0
, op1
, target
, unsignedp
, methods
);
896 /* Return whether OP0 and OP1 should be swapped when expanding a commutative
897 binop. Order them according to commutative_operand_precedence and, if
898 possible, try to put TARGET or a pseudo first. */
900 swap_commutative_operands_with_target (rtx target
, rtx op0
, rtx op1
)
902 int op0_prec
= commutative_operand_precedence (op0
);
903 int op1_prec
= commutative_operand_precedence (op1
);
905 if (op0_prec
< op1_prec
)
908 if (op0_prec
> op1_prec
)
911 /* With equal precedence, both orders are ok, but it is better if the
912 first operand is TARGET, or if both TARGET and OP0 are pseudos. */
913 if (target
== 0 || REG_P (target
))
914 return (REG_P (op1
) && !REG_P (op0
)) || target
== op1
;
916 return rtx_equal_p (op1
, target
);
919 /* Return true if BINOPTAB implements a shift operation. */
922 shift_optab_p (optab binoptab
)
924 switch (optab_to_code (binoptab
))
940 /* Return true if BINOPTAB implements a commutative binary operation. */
943 commutative_optab_p (optab binoptab
)
945 return (GET_RTX_CLASS (optab_to_code (binoptab
)) == RTX_COMM_ARITH
946 || binoptab
== smul_widen_optab
947 || binoptab
== umul_widen_optab
948 || binoptab
== smul_highpart_optab
949 || binoptab
== umul_highpart_optab
);
952 /* X is to be used in mode MODE as operand OPN to BINOPTAB. If we're
953 optimizing, and if the operand is a constant that costs more than
954 1 instruction, force the constant into a register and return that
955 register. Return X otherwise. UNSIGNEDP says whether X is unsigned. */
958 avoid_expensive_constant (machine_mode mode
, optab binoptab
,
959 int opn
, rtx x
, bool unsignedp
)
961 bool speed
= optimize_insn_for_speed_p ();
966 && (rtx_cost (x
, mode
, optab_to_code (binoptab
), opn
, speed
)
967 > set_src_cost (x
, mode
, speed
)))
971 HOST_WIDE_INT intval
= trunc_int_for_mode (INTVAL (x
), mode
);
972 if (intval
!= INTVAL (x
))
973 x
= GEN_INT (intval
);
976 x
= convert_modes (mode
, VOIDmode
, x
, unsignedp
);
977 x
= force_reg (mode
, x
);
982 /* Helper function for expand_binop: handle the case where there
983 is an insn that directly implements the indicated operation.
984 Returns null if this is not possible. */
986 expand_binop_directly (machine_mode mode
, optab binoptab
,
988 rtx target
, int unsignedp
, enum optab_methods methods
,
991 machine_mode from_mode
= widened_mode (mode
, op0
, op1
);
992 enum insn_code icode
= find_widening_optab_handler (binoptab
, mode
,
994 machine_mode xmode0
= insn_data
[(int) icode
].operand
[1].mode
;
995 machine_mode xmode1
= insn_data
[(int) icode
].operand
[2].mode
;
996 machine_mode mode0
, mode1
, tmp_mode
;
997 struct expand_operand ops
[3];
1000 rtx xop0
= op0
, xop1
= op1
;
1001 bool canonicalize_op1
= false;
1003 /* If it is a commutative operator and the modes would match
1004 if we would swap the operands, we can save the conversions. */
1005 commutative_p
= commutative_optab_p (binoptab
);
1007 && GET_MODE (xop0
) != xmode0
&& GET_MODE (xop1
) != xmode1
1008 && GET_MODE (xop0
) == xmode1
&& GET_MODE (xop1
) == xmode1
)
1009 std::swap (xop0
, xop1
);
1011 /* If we are optimizing, force expensive constants into a register. */
1012 xop0
= avoid_expensive_constant (xmode0
, binoptab
, 0, xop0
, unsignedp
);
1013 if (!shift_optab_p (binoptab
))
1014 xop1
= avoid_expensive_constant (xmode1
, binoptab
, 1, xop1
, unsignedp
);
1016 /* Shifts and rotates often use a different mode for op1 from op0;
1017 for VOIDmode constants we don't know the mode, so force it
1018 to be canonicalized using convert_modes. */
1019 canonicalize_op1
= true;
1021 /* In case the insn wants input operands in modes different from
1022 those of the actual operands, convert the operands. It would
1023 seem that we don't need to convert CONST_INTs, but we do, so
1024 that they're properly zero-extended, sign-extended or truncated
1027 mode0
= GET_MODE (xop0
) != VOIDmode
? GET_MODE (xop0
) : mode
;
1028 if (xmode0
!= VOIDmode
&& xmode0
!= mode0
)
1030 xop0
= convert_modes (xmode0
, mode0
, xop0
, unsignedp
);
1034 mode1
= ((GET_MODE (xop1
) != VOIDmode
|| canonicalize_op1
)
1035 ? GET_MODE (xop1
) : mode
);
1036 if (xmode1
!= VOIDmode
&& xmode1
!= mode1
)
1038 xop1
= convert_modes (xmode1
, mode1
, xop1
, unsignedp
);
1042 /* If operation is commutative,
1043 try to make the first operand a register.
1044 Even better, try to make it the same as the target.
1045 Also try to make the last operand a constant. */
1047 && swap_commutative_operands_with_target (target
, xop0
, xop1
))
1048 std::swap (xop0
, xop1
);
1050 /* Now, if insn's predicates don't allow our operands, put them into
1053 if (binoptab
== vec_pack_trunc_optab
1054 || binoptab
== vec_pack_usat_optab
1055 || binoptab
== vec_pack_ssat_optab
1056 || binoptab
== vec_pack_ufix_trunc_optab
1057 || binoptab
== vec_pack_sfix_trunc_optab
)
1059 /* The mode of the result is different then the mode of the
1061 tmp_mode
= insn_data
[(int) icode
].operand
[0].mode
;
1062 if (VECTOR_MODE_P (mode
)
1063 && GET_MODE_NUNITS (tmp_mode
) != 2 * GET_MODE_NUNITS (mode
))
1065 delete_insns_since (last
);
1072 create_output_operand (&ops
[0], target
, tmp_mode
);
1073 create_input_operand (&ops
[1], xop0
, mode0
);
1074 create_input_operand (&ops
[2], xop1
, mode1
);
1075 pat
= maybe_gen_insn (icode
, 3, ops
);
1078 /* If PAT is composed of more than one insn, try to add an appropriate
1079 REG_EQUAL note to it. If we can't because TEMP conflicts with an
1080 operand, call expand_binop again, this time without a target. */
1081 if (INSN_P (pat
) && NEXT_INSN (pat
) != NULL_RTX
1082 && ! add_equal_note (pat
, ops
[0].value
,
1083 optab_to_code (binoptab
),
1084 ops
[1].value
, ops
[2].value
))
1086 delete_insns_since (last
);
1087 return expand_binop (mode
, binoptab
, op0
, op1
, NULL_RTX
,
1088 unsignedp
, methods
);
1092 return ops
[0].value
;
1094 delete_insns_since (last
);
1098 /* Generate code to perform an operation specified by BINOPTAB
1099 on operands OP0 and OP1, with result having machine-mode MODE.
1101 UNSIGNEDP is for the case where we have to widen the operands
1102 to perform the operation. It says to use zero-extension.
1104 If TARGET is nonzero, the value
1105 is generated there, if it is convenient to do so.
1106 In all cases an rtx is returned for the locus of the value;
1107 this may or may not be TARGET. */
1110 expand_binop (machine_mode mode
, optab binoptab
, rtx op0
, rtx op1
,
1111 rtx target
, int unsignedp
, enum optab_methods methods
)
1113 enum optab_methods next_methods
1114 = (methods
== OPTAB_LIB
|| methods
== OPTAB_LIB_WIDEN
1115 ? OPTAB_WIDEN
: methods
);
1116 enum mode_class mclass
;
1117 machine_mode wider_mode
;
1118 scalar_int_mode int_mode
;
1121 rtx_insn
*entry_last
= get_last_insn ();
1124 mclass
= GET_MODE_CLASS (mode
);
1126 /* If subtracting an integer constant, convert this into an addition of
1127 the negated constant. */
1129 if (binoptab
== sub_optab
&& CONST_INT_P (op1
))
1131 op1
= negate_rtx (mode
, op1
);
1132 binoptab
= add_optab
;
1134 /* For shifts, constant invalid op1 might be expanded from different
1135 mode than MODE. As those are invalid, force them to a register
1136 to avoid further problems during expansion. */
1137 else if (CONST_INT_P (op1
)
1138 && shift_optab_p (binoptab
)
1139 && UINTVAL (op1
) >= GET_MODE_BITSIZE (GET_MODE_INNER (mode
)))
1141 op1
= gen_int_mode (INTVAL (op1
), GET_MODE_INNER (mode
));
1142 op1
= force_reg (GET_MODE_INNER (mode
), op1
);
1145 /* Record where to delete back to if we backtrack. */
1146 last
= get_last_insn ();
1148 /* If we can do it with a three-operand insn, do so. */
1150 if (methods
!= OPTAB_MUST_WIDEN
1151 && find_widening_optab_handler (binoptab
, mode
,
1152 widened_mode (mode
, op0
, op1
), 1)
1153 != CODE_FOR_nothing
)
1155 temp
= expand_binop_directly (mode
, binoptab
, op0
, op1
, target
,
1156 unsignedp
, methods
, last
);
1161 /* If we were trying to rotate, and that didn't work, try rotating
1162 the other direction before falling back to shifts and bitwise-or. */
1163 if (((binoptab
== rotl_optab
1164 && optab_handler (rotr_optab
, mode
) != CODE_FOR_nothing
)
1165 || (binoptab
== rotr_optab
1166 && optab_handler (rotl_optab
, mode
) != CODE_FOR_nothing
))
1167 && is_int_mode (mode
, &int_mode
))
1169 optab otheroptab
= (binoptab
== rotl_optab
? rotr_optab
: rotl_optab
);
1171 unsigned int bits
= GET_MODE_PRECISION (int_mode
);
1173 if (CONST_INT_P (op1
))
1174 newop1
= GEN_INT (bits
- INTVAL (op1
));
1175 else if (targetm
.shift_truncation_mask (int_mode
) == bits
- 1)
1176 newop1
= negate_rtx (GET_MODE (op1
), op1
);
1178 newop1
= expand_binop (GET_MODE (op1
), sub_optab
,
1179 gen_int_mode (bits
, GET_MODE (op1
)), op1
,
1180 NULL_RTX
, unsignedp
, OPTAB_DIRECT
);
1182 temp
= expand_binop_directly (int_mode
, otheroptab
, op0
, newop1
,
1183 target
, unsignedp
, methods
, last
);
1188 /* If this is a multiply, see if we can do a widening operation that
1189 takes operands of this mode and makes a wider mode. */
1191 if (binoptab
== smul_optab
1192 && GET_MODE_2XWIDER_MODE (mode
).exists (&wider_mode
)
1193 && (convert_optab_handler ((unsignedp
1195 : smul_widen_optab
),
1196 wider_mode
, mode
) != CODE_FOR_nothing
))
1198 temp
= expand_binop (wider_mode
,
1199 unsignedp
? umul_widen_optab
: smul_widen_optab
,
1200 op0
, op1
, NULL_RTX
, unsignedp
, OPTAB_DIRECT
);
1204 if (GET_MODE_CLASS (mode
) == MODE_INT
1205 && TRULY_NOOP_TRUNCATION_MODES_P (mode
, GET_MODE (temp
)))
1206 return gen_lowpart (mode
, temp
);
1208 return convert_to_mode (mode
, temp
, unsignedp
);
1212 /* If this is a vector shift by a scalar, see if we can do a vector
1213 shift by a vector. If so, broadcast the scalar into a vector. */
1214 if (mclass
== MODE_VECTOR_INT
)
1216 optab otheroptab
= unknown_optab
;
1218 if (binoptab
== ashl_optab
)
1219 otheroptab
= vashl_optab
;
1220 else if (binoptab
== ashr_optab
)
1221 otheroptab
= vashr_optab
;
1222 else if (binoptab
== lshr_optab
)
1223 otheroptab
= vlshr_optab
;
1224 else if (binoptab
== rotl_optab
)
1225 otheroptab
= vrotl_optab
;
1226 else if (binoptab
== rotr_optab
)
1227 otheroptab
= vrotr_optab
;
1229 if (otheroptab
&& optab_handler (otheroptab
, mode
) != CODE_FOR_nothing
)
1231 /* The scalar may have been extended to be too wide. Truncate
1232 it back to the proper size to fit in the broadcast vector. */
1233 scalar_mode inner_mode
= GET_MODE_INNER (mode
);
1234 if (!CONST_INT_P (op1
)
1235 && (GET_MODE_BITSIZE (as_a
<scalar_int_mode
> (GET_MODE (op1
)))
1236 > GET_MODE_BITSIZE (inner_mode
)))
1237 op1
= force_reg (inner_mode
,
1238 simplify_gen_unary (TRUNCATE
, inner_mode
, op1
,
1240 rtx vop1
= expand_vector_broadcast (mode
, op1
);
1243 temp
= expand_binop_directly (mode
, otheroptab
, op0
, vop1
,
1244 target
, unsignedp
, methods
, last
);
1251 /* Look for a wider mode of the same class for which we think we
1252 can open-code the operation. Check for a widening multiply at the
1253 wider mode as well. */
1255 if (CLASS_HAS_WIDER_MODES_P (mclass
)
1256 && methods
!= OPTAB_DIRECT
&& methods
!= OPTAB_LIB
)
1257 FOR_EACH_WIDER_MODE (wider_mode
, mode
)
1259 machine_mode next_mode
;
1260 if (optab_handler (binoptab
, wider_mode
) != CODE_FOR_nothing
1261 || (binoptab
== smul_optab
1262 && GET_MODE_WIDER_MODE (wider_mode
).exists (&next_mode
)
1263 && (find_widening_optab_handler ((unsignedp
1265 : smul_widen_optab
),
1267 != CODE_FOR_nothing
)))
1269 rtx xop0
= op0
, xop1
= op1
;
1272 /* For certain integer operations, we need not actually extend
1273 the narrow operands, as long as we will truncate
1274 the results to the same narrowness. */
1276 if ((binoptab
== ior_optab
|| binoptab
== and_optab
1277 || binoptab
== xor_optab
1278 || binoptab
== add_optab
|| binoptab
== sub_optab
1279 || binoptab
== smul_optab
|| binoptab
== ashl_optab
)
1280 && mclass
== MODE_INT
)
1283 xop0
= avoid_expensive_constant (mode
, binoptab
, 0,
1285 if (binoptab
!= ashl_optab
)
1286 xop1
= avoid_expensive_constant (mode
, binoptab
, 1,
1290 xop0
= widen_operand (xop0
, wider_mode
, mode
, unsignedp
, no_extend
);
1292 /* The second operand of a shift must always be extended. */
1293 xop1
= widen_operand (xop1
, wider_mode
, mode
, unsignedp
,
1294 no_extend
&& binoptab
!= ashl_optab
);
1296 temp
= expand_binop (wider_mode
, binoptab
, xop0
, xop1
, NULL_RTX
,
1297 unsignedp
, OPTAB_DIRECT
);
1300 if (mclass
!= MODE_INT
1301 || !TRULY_NOOP_TRUNCATION_MODES_P (mode
, wider_mode
))
1304 target
= gen_reg_rtx (mode
);
1305 convert_move (target
, temp
, 0);
1309 return gen_lowpart (mode
, temp
);
1312 delete_insns_since (last
);
1316 /* If operation is commutative,
1317 try to make the first operand a register.
1318 Even better, try to make it the same as the target.
1319 Also try to make the last operand a constant. */
1320 if (commutative_optab_p (binoptab
)
1321 && swap_commutative_operands_with_target (target
, op0
, op1
))
1322 std::swap (op0
, op1
);
1324 /* These can be done a word at a time. */
1325 if ((binoptab
== and_optab
|| binoptab
== ior_optab
|| binoptab
== xor_optab
)
1326 && is_int_mode (mode
, &int_mode
)
1327 && GET_MODE_SIZE (int_mode
) > UNITS_PER_WORD
1328 && optab_handler (binoptab
, word_mode
) != CODE_FOR_nothing
)
1333 /* If TARGET is the same as one of the operands, the REG_EQUAL note
1334 won't be accurate, so use a new target. */
1338 || !valid_multiword_target_p (target
))
1339 target
= gen_reg_rtx (int_mode
);
1343 /* Do the actual arithmetic. */
1344 for (i
= 0; i
< GET_MODE_BITSIZE (int_mode
) / BITS_PER_WORD
; i
++)
1346 rtx target_piece
= operand_subword (target
, i
, 1, int_mode
);
1347 rtx x
= expand_binop (word_mode
, binoptab
,
1348 operand_subword_force (op0
, i
, int_mode
),
1349 operand_subword_force (op1
, i
, int_mode
),
1350 target_piece
, unsignedp
, next_methods
);
1355 if (target_piece
!= x
)
1356 emit_move_insn (target_piece
, x
);
1359 insns
= get_insns ();
1362 if (i
== GET_MODE_BITSIZE (int_mode
) / BITS_PER_WORD
)
1369 /* Synthesize double word shifts from single word shifts. */
1370 if ((binoptab
== lshr_optab
|| binoptab
== ashl_optab
1371 || binoptab
== ashr_optab
)
1372 && is_int_mode (mode
, &int_mode
)
1373 && (CONST_INT_P (op1
) || optimize_insn_for_speed_p ())
1374 && GET_MODE_SIZE (int_mode
) == 2 * UNITS_PER_WORD
1375 && GET_MODE_PRECISION (int_mode
) == GET_MODE_BITSIZE (int_mode
)
1376 && optab_handler (binoptab
, word_mode
) != CODE_FOR_nothing
1377 && optab_handler (ashl_optab
, word_mode
) != CODE_FOR_nothing
1378 && optab_handler (lshr_optab
, word_mode
) != CODE_FOR_nothing
)
1380 unsigned HOST_WIDE_INT shift_mask
, double_shift_mask
;
1381 scalar_int_mode op1_mode
;
1383 double_shift_mask
= targetm
.shift_truncation_mask (int_mode
);
1384 shift_mask
= targetm
.shift_truncation_mask (word_mode
);
1385 op1_mode
= (GET_MODE (op1
) != VOIDmode
1386 ? as_a
<scalar_int_mode
> (GET_MODE (op1
))
1389 /* Apply the truncation to constant shifts. */
1390 if (double_shift_mask
> 0 && CONST_INT_P (op1
))
1391 op1
= GEN_INT (INTVAL (op1
) & double_shift_mask
);
1393 if (op1
== CONST0_RTX (op1_mode
))
1396 /* Make sure that this is a combination that expand_doubleword_shift
1397 can handle. See the comments there for details. */
1398 if (double_shift_mask
== 0
1399 || (shift_mask
== BITS_PER_WORD
- 1
1400 && double_shift_mask
== BITS_PER_WORD
* 2 - 1))
1403 rtx into_target
, outof_target
;
1404 rtx into_input
, outof_input
;
1405 int left_shift
, outof_word
;
1407 /* If TARGET is the same as one of the operands, the REG_EQUAL note
1408 won't be accurate, so use a new target. */
1412 || !valid_multiword_target_p (target
))
1413 target
= gen_reg_rtx (int_mode
);
1417 /* OUTOF_* is the word we are shifting bits away from, and
1418 INTO_* is the word that we are shifting bits towards, thus
1419 they differ depending on the direction of the shift and
1420 WORDS_BIG_ENDIAN. */
1422 left_shift
= binoptab
== ashl_optab
;
1423 outof_word
= left_shift
^ ! WORDS_BIG_ENDIAN
;
1425 outof_target
= operand_subword (target
, outof_word
, 1, int_mode
);
1426 into_target
= operand_subword (target
, 1 - outof_word
, 1, int_mode
);
1428 outof_input
= operand_subword_force (op0
, outof_word
, int_mode
);
1429 into_input
= operand_subword_force (op0
, 1 - outof_word
, int_mode
);
1431 if (expand_doubleword_shift (op1_mode
, binoptab
,
1432 outof_input
, into_input
, op1
,
1433 outof_target
, into_target
,
1434 unsignedp
, next_methods
, shift_mask
))
1436 insns
= get_insns ();
1446 /* Synthesize double word rotates from single word shifts. */
1447 if ((binoptab
== rotl_optab
|| binoptab
== rotr_optab
)
1448 && is_int_mode (mode
, &int_mode
)
1449 && CONST_INT_P (op1
)
1450 && GET_MODE_PRECISION (int_mode
) == 2 * BITS_PER_WORD
1451 && optab_handler (ashl_optab
, word_mode
) != CODE_FOR_nothing
1452 && optab_handler (lshr_optab
, word_mode
) != CODE_FOR_nothing
)
1455 rtx into_target
, outof_target
;
1456 rtx into_input
, outof_input
;
1458 int shift_count
, left_shift
, outof_word
;
1460 /* If TARGET is the same as one of the operands, the REG_EQUAL note
1461 won't be accurate, so use a new target. Do this also if target is not
1462 a REG, first because having a register instead may open optimization
1463 opportunities, and second because if target and op0 happen to be MEMs
1464 designating the same location, we would risk clobbering it too early
1465 in the code sequence we generate below. */
1470 || !valid_multiword_target_p (target
))
1471 target
= gen_reg_rtx (int_mode
);
1475 shift_count
= INTVAL (op1
);
1477 /* OUTOF_* is the word we are shifting bits away from, and
1478 INTO_* is the word that we are shifting bits towards, thus
1479 they differ depending on the direction of the shift and
1480 WORDS_BIG_ENDIAN. */
1482 left_shift
= (binoptab
== rotl_optab
);
1483 outof_word
= left_shift
^ ! WORDS_BIG_ENDIAN
;
1485 outof_target
= operand_subword (target
, outof_word
, 1, int_mode
);
1486 into_target
= operand_subword (target
, 1 - outof_word
, 1, int_mode
);
1488 outof_input
= operand_subword_force (op0
, outof_word
, int_mode
);
1489 into_input
= operand_subword_force (op0
, 1 - outof_word
, int_mode
);
1491 if (shift_count
== BITS_PER_WORD
)
1493 /* This is just a word swap. */
1494 emit_move_insn (outof_target
, into_input
);
1495 emit_move_insn (into_target
, outof_input
);
1500 rtx into_temp1
, into_temp2
, outof_temp1
, outof_temp2
;
1501 rtx first_shift_count
, second_shift_count
;
1502 optab reverse_unsigned_shift
, unsigned_shift
;
1504 reverse_unsigned_shift
= (left_shift
^ (shift_count
< BITS_PER_WORD
)
1505 ? lshr_optab
: ashl_optab
);
1507 unsigned_shift
= (left_shift
^ (shift_count
< BITS_PER_WORD
)
1508 ? ashl_optab
: lshr_optab
);
1510 if (shift_count
> BITS_PER_WORD
)
1512 first_shift_count
= GEN_INT (shift_count
- BITS_PER_WORD
);
1513 second_shift_count
= GEN_INT (2 * BITS_PER_WORD
- shift_count
);
1517 first_shift_count
= GEN_INT (BITS_PER_WORD
- shift_count
);
1518 second_shift_count
= GEN_INT (shift_count
);
1521 into_temp1
= expand_binop (word_mode
, unsigned_shift
,
1522 outof_input
, first_shift_count
,
1523 NULL_RTX
, unsignedp
, next_methods
);
1524 into_temp2
= expand_binop (word_mode
, reverse_unsigned_shift
,
1525 into_input
, second_shift_count
,
1526 NULL_RTX
, unsignedp
, next_methods
);
1528 if (into_temp1
!= 0 && into_temp2
!= 0)
1529 inter
= expand_binop (word_mode
, ior_optab
, into_temp1
, into_temp2
,
1530 into_target
, unsignedp
, next_methods
);
1534 if (inter
!= 0 && inter
!= into_target
)
1535 emit_move_insn (into_target
, inter
);
1537 outof_temp1
= expand_binop (word_mode
, unsigned_shift
,
1538 into_input
, first_shift_count
,
1539 NULL_RTX
, unsignedp
, next_methods
);
1540 outof_temp2
= expand_binop (word_mode
, reverse_unsigned_shift
,
1541 outof_input
, second_shift_count
,
1542 NULL_RTX
, unsignedp
, next_methods
);
1544 if (inter
!= 0 && outof_temp1
!= 0 && outof_temp2
!= 0)
1545 inter
= expand_binop (word_mode
, ior_optab
,
1546 outof_temp1
, outof_temp2
,
1547 outof_target
, unsignedp
, next_methods
);
1549 if (inter
!= 0 && inter
!= outof_target
)
1550 emit_move_insn (outof_target
, inter
);
1553 insns
= get_insns ();
1563 /* These can be done a word at a time by propagating carries. */
1564 if ((binoptab
== add_optab
|| binoptab
== sub_optab
)
1565 && is_int_mode (mode
, &int_mode
)
1566 && GET_MODE_SIZE (int_mode
) >= 2 * UNITS_PER_WORD
1567 && optab_handler (binoptab
, word_mode
) != CODE_FOR_nothing
)
1570 optab otheroptab
= binoptab
== add_optab
? sub_optab
: add_optab
;
1571 const unsigned int nwords
= GET_MODE_BITSIZE (int_mode
) / BITS_PER_WORD
;
1572 rtx carry_in
= NULL_RTX
, carry_out
= NULL_RTX
;
1573 rtx xop0
, xop1
, xtarget
;
1575 /* We can handle either a 1 or -1 value for the carry. If STORE_FLAG
1576 value is one of those, use it. Otherwise, use 1 since it is the
1577 one easiest to get. */
1578 #if STORE_FLAG_VALUE == 1 || STORE_FLAG_VALUE == -1
1579 int normalizep
= STORE_FLAG_VALUE
;
1584 /* Prepare the operands. */
1585 xop0
= force_reg (int_mode
, op0
);
1586 xop1
= force_reg (int_mode
, op1
);
1588 xtarget
= gen_reg_rtx (int_mode
);
1590 if (target
== 0 || !REG_P (target
) || !valid_multiword_target_p (target
))
1593 /* Indicate for flow that the entire target reg is being set. */
1595 emit_clobber (xtarget
);
1597 /* Do the actual arithmetic. */
1598 for (i
= 0; i
< nwords
; i
++)
1600 int index
= (WORDS_BIG_ENDIAN
? nwords
- i
- 1 : i
);
1601 rtx target_piece
= operand_subword (xtarget
, index
, 1, int_mode
);
1602 rtx op0_piece
= operand_subword_force (xop0
, index
, int_mode
);
1603 rtx op1_piece
= operand_subword_force (xop1
, index
, int_mode
);
1606 /* Main add/subtract of the input operands. */
1607 x
= expand_binop (word_mode
, binoptab
,
1608 op0_piece
, op1_piece
,
1609 target_piece
, unsignedp
, next_methods
);
1615 /* Store carry from main add/subtract. */
1616 carry_out
= gen_reg_rtx (word_mode
);
1617 carry_out
= emit_store_flag_force (carry_out
,
1618 (binoptab
== add_optab
1621 word_mode
, 1, normalizep
);
1628 /* Add/subtract previous carry to main result. */
1629 newx
= expand_binop (word_mode
,
1630 normalizep
== 1 ? binoptab
: otheroptab
,
1632 NULL_RTX
, 1, next_methods
);
1636 /* Get out carry from adding/subtracting carry in. */
1637 rtx carry_tmp
= gen_reg_rtx (word_mode
);
1638 carry_tmp
= emit_store_flag_force (carry_tmp
,
1639 (binoptab
== add_optab
1642 word_mode
, 1, normalizep
);
1644 /* Logical-ior the two poss. carry together. */
1645 carry_out
= expand_binop (word_mode
, ior_optab
,
1646 carry_out
, carry_tmp
,
1647 carry_out
, 0, next_methods
);
1651 emit_move_insn (target_piece
, newx
);
1655 if (x
!= target_piece
)
1656 emit_move_insn (target_piece
, x
);
1659 carry_in
= carry_out
;
1662 if (i
== GET_MODE_BITSIZE (int_mode
) / (unsigned) BITS_PER_WORD
)
1664 if (optab_handler (mov_optab
, int_mode
) != CODE_FOR_nothing
1665 || ! rtx_equal_p (target
, xtarget
))
1667 rtx_insn
*temp
= emit_move_insn (target
, xtarget
);
1669 set_dst_reg_note (temp
, REG_EQUAL
,
1670 gen_rtx_fmt_ee (optab_to_code (binoptab
),
1671 int_mode
, copy_rtx (xop0
),
1682 delete_insns_since (last
);
1685 /* Attempt to synthesize double word multiplies using a sequence of word
1686 mode multiplications. We first attempt to generate a sequence using a
1687 more efficient unsigned widening multiply, and if that fails we then
1688 try using a signed widening multiply. */
1690 if (binoptab
== smul_optab
1691 && is_int_mode (mode
, &int_mode
)
1692 && GET_MODE_SIZE (int_mode
) == 2 * UNITS_PER_WORD
1693 && optab_handler (smul_optab
, word_mode
) != CODE_FOR_nothing
1694 && optab_handler (add_optab
, word_mode
) != CODE_FOR_nothing
)
1696 rtx product
= NULL_RTX
;
1697 if (widening_optab_handler (umul_widen_optab
, int_mode
, word_mode
)
1698 != CODE_FOR_nothing
)
1700 product
= expand_doubleword_mult (int_mode
, op0
, op1
, target
,
1703 delete_insns_since (last
);
1706 if (product
== NULL_RTX
1707 && (widening_optab_handler (smul_widen_optab
, int_mode
, word_mode
)
1708 != CODE_FOR_nothing
))
1710 product
= expand_doubleword_mult (int_mode
, op0
, op1
, target
,
1713 delete_insns_since (last
);
1716 if (product
!= NULL_RTX
)
1718 if (optab_handler (mov_optab
, int_mode
) != CODE_FOR_nothing
)
1720 rtx_insn
*move
= emit_move_insn (target
? target
: product
,
1722 set_dst_reg_note (move
,
1724 gen_rtx_fmt_ee (MULT
, int_mode
,
1727 target
? target
: product
);
1733 /* It can't be open-coded in this mode.
1734 Use a library call if one is available and caller says that's ok. */
1736 libfunc
= optab_libfunc (binoptab
, mode
);
1738 && (methods
== OPTAB_LIB
|| methods
== OPTAB_LIB_WIDEN
))
1742 machine_mode op1_mode
= mode
;
1747 if (shift_optab_p (binoptab
))
1749 op1_mode
= targetm
.libgcc_shift_count_mode ();
1750 /* Specify unsigned here,
1751 since negative shift counts are meaningless. */
1752 op1x
= convert_to_mode (op1_mode
, op1
, 1);
1755 if (GET_MODE (op0
) != VOIDmode
1756 && GET_MODE (op0
) != mode
)
1757 op0
= convert_to_mode (mode
, op0
, unsignedp
);
1759 /* Pass 1 for NO_QUEUE so we don't lose any increments
1760 if the libcall is cse'd or moved. */
1761 value
= emit_library_call_value (libfunc
,
1762 NULL_RTX
, LCT_CONST
, mode
,
1763 op0
, mode
, op1x
, op1_mode
);
1765 insns
= get_insns ();
1768 bool trapv
= trapv_binoptab_p (binoptab
);
1769 target
= gen_reg_rtx (mode
);
1770 emit_libcall_block_1 (insns
, target
, value
,
1772 : gen_rtx_fmt_ee (optab_to_code (binoptab
),
1773 mode
, op0
, op1
), trapv
);
1778 delete_insns_since (last
);
1780 /* It can't be done in this mode. Can we do it in a wider mode? */
1782 if (! (methods
== OPTAB_WIDEN
|| methods
== OPTAB_LIB_WIDEN
1783 || methods
== OPTAB_MUST_WIDEN
))
1785 /* Caller says, don't even try. */
1786 delete_insns_since (entry_last
);
1790 /* Compute the value of METHODS to pass to recursive calls.
1791 Don't allow widening to be tried recursively. */
1793 methods
= (methods
== OPTAB_LIB_WIDEN
? OPTAB_LIB
: OPTAB_DIRECT
);
1795 /* Look for a wider mode of the same class for which it appears we can do
1798 if (CLASS_HAS_WIDER_MODES_P (mclass
))
1800 FOR_EACH_WIDER_MODE (wider_mode
, mode
)
1802 if (find_widening_optab_handler (binoptab
, wider_mode
, mode
, 1)
1804 || (methods
== OPTAB_LIB
1805 && optab_libfunc (binoptab
, wider_mode
)))
1807 rtx xop0
= op0
, xop1
= op1
;
1810 /* For certain integer operations, we need not actually extend
1811 the narrow operands, as long as we will truncate
1812 the results to the same narrowness. */
1814 if ((binoptab
== ior_optab
|| binoptab
== and_optab
1815 || binoptab
== xor_optab
1816 || binoptab
== add_optab
|| binoptab
== sub_optab
1817 || binoptab
== smul_optab
|| binoptab
== ashl_optab
)
1818 && mclass
== MODE_INT
)
1821 xop0
= widen_operand (xop0
, wider_mode
, mode
,
1822 unsignedp
, no_extend
);
1824 /* The second operand of a shift must always be extended. */
1825 xop1
= widen_operand (xop1
, wider_mode
, mode
, unsignedp
,
1826 no_extend
&& binoptab
!= ashl_optab
);
1828 temp
= expand_binop (wider_mode
, binoptab
, xop0
, xop1
, NULL_RTX
,
1829 unsignedp
, methods
);
1832 if (mclass
!= MODE_INT
1833 || !TRULY_NOOP_TRUNCATION_MODES_P (mode
, wider_mode
))
1836 target
= gen_reg_rtx (mode
);
1837 convert_move (target
, temp
, 0);
1841 return gen_lowpart (mode
, temp
);
1844 delete_insns_since (last
);
1849 delete_insns_since (entry_last
);
1853 /* Expand a binary operator which has both signed and unsigned forms.
1854 UOPTAB is the optab for unsigned operations, and SOPTAB is for
1857 If we widen unsigned operands, we may use a signed wider operation instead
1858 of an unsigned wider operation, since the result would be the same. */
1861 sign_expand_binop (machine_mode mode
, optab uoptab
, optab soptab
,
1862 rtx op0
, rtx op1
, rtx target
, int unsignedp
,
1863 enum optab_methods methods
)
1866 optab direct_optab
= unsignedp
? uoptab
: soptab
;
1869 /* Do it without widening, if possible. */
1870 temp
= expand_binop (mode
, direct_optab
, op0
, op1
, target
,
1871 unsignedp
, OPTAB_DIRECT
);
1872 if (temp
|| methods
== OPTAB_DIRECT
)
1875 /* Try widening to a signed int. Disable any direct use of any
1876 signed insn in the current mode. */
1877 save_enable
= swap_optab_enable (soptab
, mode
, false);
1879 temp
= expand_binop (mode
, soptab
, op0
, op1
, target
,
1880 unsignedp
, OPTAB_WIDEN
);
1882 /* For unsigned operands, try widening to an unsigned int. */
1883 if (!temp
&& unsignedp
)
1884 temp
= expand_binop (mode
, uoptab
, op0
, op1
, target
,
1885 unsignedp
, OPTAB_WIDEN
);
1886 if (temp
|| methods
== OPTAB_WIDEN
)
1889 /* Use the right width libcall if that exists. */
1890 temp
= expand_binop (mode
, direct_optab
, op0
, op1
, target
,
1891 unsignedp
, OPTAB_LIB
);
1892 if (temp
|| methods
== OPTAB_LIB
)
1895 /* Must widen and use a libcall, use either signed or unsigned. */
1896 temp
= expand_binop (mode
, soptab
, op0
, op1
, target
,
1897 unsignedp
, methods
);
1898 if (!temp
&& unsignedp
)
1899 temp
= expand_binop (mode
, uoptab
, op0
, op1
, target
,
1900 unsignedp
, methods
);
1903 /* Undo the fiddling above. */
1905 swap_optab_enable (soptab
, mode
, true);
1909 /* Generate code to perform an operation specified by UNOPPTAB
1910 on operand OP0, with two results to TARG0 and TARG1.
1911 We assume that the order of the operands for the instruction
1912 is TARG0, TARG1, OP0.
1914 Either TARG0 or TARG1 may be zero, but what that means is that
1915 the result is not actually wanted. We will generate it into
1916 a dummy pseudo-reg and discard it. They may not both be zero.
1918 Returns 1 if this operation can be performed; 0 if not. */
1921 expand_twoval_unop (optab unoptab
, rtx op0
, rtx targ0
, rtx targ1
,
1924 machine_mode mode
= GET_MODE (targ0
? targ0
: targ1
);
1925 enum mode_class mclass
;
1926 machine_mode wider_mode
;
1927 rtx_insn
*entry_last
= get_last_insn ();
1930 mclass
= GET_MODE_CLASS (mode
);
1933 targ0
= gen_reg_rtx (mode
);
1935 targ1
= gen_reg_rtx (mode
);
1937 /* Record where to go back to if we fail. */
1938 last
= get_last_insn ();
1940 if (optab_handler (unoptab
, mode
) != CODE_FOR_nothing
)
1942 struct expand_operand ops
[3];
1943 enum insn_code icode
= optab_handler (unoptab
, mode
);
1945 create_fixed_operand (&ops
[0], targ0
);
1946 create_fixed_operand (&ops
[1], targ1
);
1947 create_convert_operand_from (&ops
[2], op0
, mode
, unsignedp
);
1948 if (maybe_expand_insn (icode
, 3, ops
))
1952 /* It can't be done in this mode. Can we do it in a wider mode? */
1954 if (CLASS_HAS_WIDER_MODES_P (mclass
))
1956 FOR_EACH_WIDER_MODE (wider_mode
, mode
)
1958 if (optab_handler (unoptab
, wider_mode
) != CODE_FOR_nothing
)
1960 rtx t0
= gen_reg_rtx (wider_mode
);
1961 rtx t1
= gen_reg_rtx (wider_mode
);
1962 rtx cop0
= convert_modes (wider_mode
, mode
, op0
, unsignedp
);
1964 if (expand_twoval_unop (unoptab
, cop0
, t0
, t1
, unsignedp
))
1966 convert_move (targ0
, t0
, unsignedp
);
1967 convert_move (targ1
, t1
, unsignedp
);
1971 delete_insns_since (last
);
1976 delete_insns_since (entry_last
);
1980 /* Generate code to perform an operation specified by BINOPTAB
1981 on operands OP0 and OP1, with two results to TARG1 and TARG2.
1982 We assume that the order of the operands for the instruction
1983 is TARG0, OP0, OP1, TARG1, which would fit a pattern like
1984 [(set TARG0 (operate OP0 OP1)) (set TARG1 (operate ...))].
1986 Either TARG0 or TARG1 may be zero, but what that means is that
1987 the result is not actually wanted. We will generate it into
1988 a dummy pseudo-reg and discard it. They may not both be zero.
1990 Returns 1 if this operation can be performed; 0 if not. */
1993 expand_twoval_binop (optab binoptab
, rtx op0
, rtx op1
, rtx targ0
, rtx targ1
,
1996 machine_mode mode
= GET_MODE (targ0
? targ0
: targ1
);
1997 enum mode_class mclass
;
1998 machine_mode wider_mode
;
1999 rtx_insn
*entry_last
= get_last_insn ();
2002 mclass
= GET_MODE_CLASS (mode
);
2005 targ0
= gen_reg_rtx (mode
);
2007 targ1
= gen_reg_rtx (mode
);
2009 /* Record where to go back to if we fail. */
2010 last
= get_last_insn ();
2012 if (optab_handler (binoptab
, mode
) != CODE_FOR_nothing
)
2014 struct expand_operand ops
[4];
2015 enum insn_code icode
= optab_handler (binoptab
, mode
);
2016 machine_mode mode0
= insn_data
[icode
].operand
[1].mode
;
2017 machine_mode mode1
= insn_data
[icode
].operand
[2].mode
;
2018 rtx xop0
= op0
, xop1
= op1
;
2020 /* If we are optimizing, force expensive constants into a register. */
2021 xop0
= avoid_expensive_constant (mode0
, binoptab
, 0, xop0
, unsignedp
);
2022 xop1
= avoid_expensive_constant (mode1
, binoptab
, 1, xop1
, unsignedp
);
2024 create_fixed_operand (&ops
[0], targ0
);
2025 create_convert_operand_from (&ops
[1], op0
, mode
, unsignedp
);
2026 create_convert_operand_from (&ops
[2], op1
, mode
, unsignedp
);
2027 create_fixed_operand (&ops
[3], targ1
);
2028 if (maybe_expand_insn (icode
, 4, ops
))
2030 delete_insns_since (last
);
2033 /* It can't be done in this mode. Can we do it in a wider mode? */
2035 if (CLASS_HAS_WIDER_MODES_P (mclass
))
2037 FOR_EACH_WIDER_MODE (wider_mode
, mode
)
2039 if (optab_handler (binoptab
, wider_mode
) != CODE_FOR_nothing
)
2041 rtx t0
= gen_reg_rtx (wider_mode
);
2042 rtx t1
= gen_reg_rtx (wider_mode
);
2043 rtx cop0
= convert_modes (wider_mode
, mode
, op0
, unsignedp
);
2044 rtx cop1
= convert_modes (wider_mode
, mode
, op1
, unsignedp
);
2046 if (expand_twoval_binop (binoptab
, cop0
, cop1
,
2049 convert_move (targ0
, t0
, unsignedp
);
2050 convert_move (targ1
, t1
, unsignedp
);
2054 delete_insns_since (last
);
2059 delete_insns_since (entry_last
);
2063 /* Expand the two-valued library call indicated by BINOPTAB, but
2064 preserve only one of the values. If TARG0 is non-NULL, the first
2065 value is placed into TARG0; otherwise the second value is placed
2066 into TARG1. Exactly one of TARG0 and TARG1 must be non-NULL. The
2067 value stored into TARG0 or TARG1 is equivalent to (CODE OP0 OP1).
2068 This routine assumes that the value returned by the library call is
2069 as if the return value was of an integral mode twice as wide as the
2070 mode of OP0. Returns 1 if the call was successful. */
2073 expand_twoval_binop_libfunc (optab binoptab
, rtx op0
, rtx op1
,
2074 rtx targ0
, rtx targ1
, enum rtx_code code
)
2077 machine_mode libval_mode
;
2082 /* Exactly one of TARG0 or TARG1 should be non-NULL. */
2083 gcc_assert (!targ0
!= !targ1
);
2085 mode
= GET_MODE (op0
);
2086 libfunc
= optab_libfunc (binoptab
, mode
);
2090 /* The value returned by the library function will have twice as
2091 many bits as the nominal MODE. */
2092 libval_mode
= smallest_int_mode_for_size (2 * GET_MODE_BITSIZE (mode
));
2094 libval
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
,
2098 /* Get the part of VAL containing the value that we want. */
2099 libval
= simplify_gen_subreg (mode
, libval
, libval_mode
,
2100 targ0
? 0 : GET_MODE_SIZE (mode
));
2101 insns
= get_insns ();
2103 /* Move the into the desired location. */
2104 emit_libcall_block (insns
, targ0
? targ0
: targ1
, libval
,
2105 gen_rtx_fmt_ee (code
, mode
, op0
, op1
));
2111 /* Wrapper around expand_unop which takes an rtx code to specify
2112 the operation to perform, not an optab pointer. All other
2113 arguments are the same. */
2115 expand_simple_unop (machine_mode mode
, enum rtx_code code
, rtx op0
,
2116 rtx target
, int unsignedp
)
2118 optab unop
= code_to_optab (code
);
2121 return expand_unop (mode
, unop
, op0
, target
, unsignedp
);
2127 (clz:wide (zero_extend:wide x)) - ((width wide) - (width narrow)).
2129 A similar operation can be used for clrsb. UNOPTAB says which operation
2130 we are trying to expand. */
2132 widen_leading (scalar_int_mode mode
, rtx op0
, rtx target
, optab unoptab
)
2134 opt_scalar_int_mode wider_mode_iter
;
2135 FOR_EACH_WIDER_MODE (wider_mode_iter
, mode
)
2137 scalar_int_mode wider_mode
= wider_mode_iter
.require ();
2138 if (optab_handler (unoptab
, wider_mode
) != CODE_FOR_nothing
)
2143 last
= get_last_insn ();
2146 target
= gen_reg_rtx (mode
);
2147 xop0
= widen_operand (op0
, wider_mode
, mode
,
2148 unoptab
!= clrsb_optab
, false);
2149 temp
= expand_unop (wider_mode
, unoptab
, xop0
, NULL_RTX
,
2150 unoptab
!= clrsb_optab
);
2153 (wider_mode
, sub_optab
, temp
,
2154 gen_int_mode (GET_MODE_PRECISION (wider_mode
)
2155 - GET_MODE_PRECISION (mode
),
2157 target
, true, OPTAB_DIRECT
);
2159 delete_insns_since (last
);
2167 /* Try calculating clz of a double-word quantity as two clz's of word-sized
2168 quantities, choosing which based on whether the high word is nonzero. */
2170 expand_doubleword_clz (scalar_int_mode mode
, rtx op0
, rtx target
)
2172 rtx xop0
= force_reg (mode
, op0
);
2173 rtx subhi
= gen_highpart (word_mode
, xop0
);
2174 rtx sublo
= gen_lowpart (word_mode
, xop0
);
2175 rtx_code_label
*hi0_label
= gen_label_rtx ();
2176 rtx_code_label
*after_label
= gen_label_rtx ();
2180 /* If we were not given a target, use a word_mode register, not a
2181 'mode' register. The result will fit, and nobody is expecting
2182 anything bigger (the return type of __builtin_clz* is int). */
2184 target
= gen_reg_rtx (word_mode
);
2186 /* In any case, write to a word_mode scratch in both branches of the
2187 conditional, so we can ensure there is a single move insn setting
2188 'target' to tag a REG_EQUAL note on. */
2189 result
= gen_reg_rtx (word_mode
);
2193 /* If the high word is not equal to zero,
2194 then clz of the full value is clz of the high word. */
2195 emit_cmp_and_jump_insns (subhi
, CONST0_RTX (word_mode
), EQ
, 0,
2196 word_mode
, true, hi0_label
);
2198 temp
= expand_unop_direct (word_mode
, clz_optab
, subhi
, result
, true);
2203 convert_move (result
, temp
, true);
2205 emit_jump_insn (targetm
.gen_jump (after_label
));
2208 /* Else clz of the full value is clz of the low word plus the number
2209 of bits in the high word. */
2210 emit_label (hi0_label
);
2212 temp
= expand_unop_direct (word_mode
, clz_optab
, sublo
, 0, true);
2215 temp
= expand_binop (word_mode
, add_optab
, temp
,
2216 gen_int_mode (GET_MODE_BITSIZE (word_mode
), word_mode
),
2217 result
, true, OPTAB_DIRECT
);
2221 convert_move (result
, temp
, true);
2223 emit_label (after_label
);
2224 convert_move (target
, result
, true);
2229 add_equal_note (seq
, target
, CLZ
, xop0
, 0);
2238 /* Try calculating popcount of a double-word quantity as two popcount's of
2239 word-sized quantities and summing up the results. */
2241 expand_doubleword_popcount (scalar_int_mode mode
, rtx op0
, rtx target
)
2248 t0
= expand_unop_direct (word_mode
, popcount_optab
,
2249 operand_subword_force (op0
, 0, mode
), NULL_RTX
,
2251 t1
= expand_unop_direct (word_mode
, popcount_optab
,
2252 operand_subword_force (op0
, 1, mode
), NULL_RTX
,
2260 /* If we were not given a target, use a word_mode register, not a
2261 'mode' register. The result will fit, and nobody is expecting
2262 anything bigger (the return type of __builtin_popcount* is int). */
2264 target
= gen_reg_rtx (word_mode
);
2266 t
= expand_binop (word_mode
, add_optab
, t0
, t1
, target
, 0, OPTAB_DIRECT
);
2271 add_equal_note (seq
, t
, POPCOUNT
, op0
, 0);
2279 (parity:narrow (low (x) ^ high (x))) */
2281 expand_doubleword_parity (scalar_int_mode mode
, rtx op0
, rtx target
)
2283 rtx t
= expand_binop (word_mode
, xor_optab
,
2284 operand_subword_force (op0
, 0, mode
),
2285 operand_subword_force (op0
, 1, mode
),
2286 NULL_RTX
, 0, OPTAB_DIRECT
);
2287 return expand_unop (word_mode
, parity_optab
, t
, target
, true);
2293 (lshiftrt:wide (bswap:wide x) ((width wide) - (width narrow))). */
2295 widen_bswap (scalar_int_mode mode
, rtx op0
, rtx target
)
2299 opt_scalar_int_mode wider_mode_iter
;
2301 FOR_EACH_WIDER_MODE (wider_mode_iter
, mode
)
2302 if (optab_handler (bswap_optab
, wider_mode_iter
.require ())
2303 != CODE_FOR_nothing
)
2306 if (!wider_mode_iter
.exists ())
2309 scalar_int_mode wider_mode
= wider_mode_iter
.require ();
2310 last
= get_last_insn ();
2312 x
= widen_operand (op0
, wider_mode
, mode
, true, true);
2313 x
= expand_unop (wider_mode
, bswap_optab
, x
, NULL_RTX
, true);
2315 gcc_assert (GET_MODE_PRECISION (wider_mode
) == GET_MODE_BITSIZE (wider_mode
)
2316 && GET_MODE_PRECISION (mode
) == GET_MODE_BITSIZE (mode
));
2318 x
= expand_shift (RSHIFT_EXPR
, wider_mode
, x
,
2319 GET_MODE_BITSIZE (wider_mode
)
2320 - GET_MODE_BITSIZE (mode
),
2326 target
= gen_reg_rtx (mode
);
2327 emit_move_insn (target
, gen_lowpart (mode
, x
));
2330 delete_insns_since (last
);
2335 /* Try calculating bswap as two bswaps of two word-sized operands. */
2338 expand_doubleword_bswap (machine_mode mode
, rtx op
, rtx target
)
2342 t1
= expand_unop (word_mode
, bswap_optab
,
2343 operand_subword_force (op
, 0, mode
), NULL_RTX
, true);
2344 t0
= expand_unop (word_mode
, bswap_optab
,
2345 operand_subword_force (op
, 1, mode
), NULL_RTX
, true);
2347 if (target
== 0 || !valid_multiword_target_p (target
))
2348 target
= gen_reg_rtx (mode
);
2350 emit_clobber (target
);
2351 emit_move_insn (operand_subword (target
, 0, 1, mode
), t0
);
2352 emit_move_insn (operand_subword (target
, 1, 1, mode
), t1
);
2357 /* Try calculating (parity x) as (and (popcount x) 1), where
2358 popcount can also be done in a wider mode. */
2360 expand_parity (scalar_int_mode mode
, rtx op0
, rtx target
)
2362 enum mode_class mclass
= GET_MODE_CLASS (mode
);
2363 opt_scalar_int_mode wider_mode_iter
;
2364 FOR_EACH_MODE_FROM (wider_mode_iter
, mode
)
2366 scalar_int_mode wider_mode
= wider_mode_iter
.require ();
2367 if (optab_handler (popcount_optab
, wider_mode
) != CODE_FOR_nothing
)
2372 last
= get_last_insn ();
2374 if (target
== 0 || GET_MODE (target
) != wider_mode
)
2375 target
= gen_reg_rtx (wider_mode
);
2377 xop0
= widen_operand (op0
, wider_mode
, mode
, true, false);
2378 temp
= expand_unop (wider_mode
, popcount_optab
, xop0
, NULL_RTX
,
2381 temp
= expand_binop (wider_mode
, and_optab
, temp
, const1_rtx
,
2382 target
, true, OPTAB_DIRECT
);
2386 if (mclass
!= MODE_INT
2387 || !TRULY_NOOP_TRUNCATION_MODES_P (mode
, wider_mode
))
2388 return convert_to_mode (mode
, temp
, 0);
2390 return gen_lowpart (mode
, temp
);
2393 delete_insns_since (last
);
2399 /* Try calculating ctz(x) as K - clz(x & -x) ,
2400 where K is GET_MODE_PRECISION(mode) - 1.
2402 Both __builtin_ctz and __builtin_clz are undefined at zero, so we
2403 don't have to worry about what the hardware does in that case. (If
2404 the clz instruction produces the usual value at 0, which is K, the
2405 result of this code sequence will be -1; expand_ffs, below, relies
2406 on this. It might be nice to have it be K instead, for consistency
2407 with the (very few) processors that provide a ctz with a defined
2408 value, but that would take one more instruction, and it would be
2409 less convenient for expand_ffs anyway. */
2412 expand_ctz (scalar_int_mode mode
, rtx op0
, rtx target
)
2417 if (optab_handler (clz_optab
, mode
) == CODE_FOR_nothing
)
2422 temp
= expand_unop_direct (mode
, neg_optab
, op0
, NULL_RTX
, true);
2424 temp
= expand_binop (mode
, and_optab
, op0
, temp
, NULL_RTX
,
2425 true, OPTAB_DIRECT
);
2427 temp
= expand_unop_direct (mode
, clz_optab
, temp
, NULL_RTX
, true);
2429 temp
= expand_binop (mode
, sub_optab
,
2430 gen_int_mode (GET_MODE_PRECISION (mode
) - 1, mode
),
2432 true, OPTAB_DIRECT
);
2442 add_equal_note (seq
, temp
, CTZ
, op0
, 0);
2448 /* Try calculating ffs(x) using ctz(x) if we have that instruction, or
2449 else with the sequence used by expand_clz.
2451 The ffs builtin promises to return zero for a zero value and ctz/clz
2452 may have an undefined value in that case. If they do not give us a
2453 convenient value, we have to generate a test and branch. */
2455 expand_ffs (scalar_int_mode mode
, rtx op0
, rtx target
)
2457 HOST_WIDE_INT val
= 0;
2458 bool defined_at_zero
= false;
2462 if (optab_handler (ctz_optab
, mode
) != CODE_FOR_nothing
)
2466 temp
= expand_unop_direct (mode
, ctz_optab
, op0
, 0, true);
2470 defined_at_zero
= (CTZ_DEFINED_VALUE_AT_ZERO (mode
, val
) == 2);
2472 else if (optab_handler (clz_optab
, mode
) != CODE_FOR_nothing
)
2475 temp
= expand_ctz (mode
, op0
, 0);
2479 if (CLZ_DEFINED_VALUE_AT_ZERO (mode
, val
) == 2)
2481 defined_at_zero
= true;
2482 val
= (GET_MODE_PRECISION (mode
) - 1) - val
;
2488 if (defined_at_zero
&& val
== -1)
2489 /* No correction needed at zero. */;
2492 /* We don't try to do anything clever with the situation found
2493 on some processors (eg Alpha) where ctz(0:mode) ==
2494 bitsize(mode). If someone can think of a way to send N to -1
2495 and leave alone all values in the range 0..N-1 (where N is a
2496 power of two), cheaper than this test-and-branch, please add it.
2498 The test-and-branch is done after the operation itself, in case
2499 the operation sets condition codes that can be recycled for this.
2500 (This is true on i386, for instance.) */
2502 rtx_code_label
*nonzero_label
= gen_label_rtx ();
2503 emit_cmp_and_jump_insns (op0
, CONST0_RTX (mode
), NE
, 0,
2504 mode
, true, nonzero_label
);
2506 convert_move (temp
, GEN_INT (-1), false);
2507 emit_label (nonzero_label
);
2510 /* temp now has a value in the range -1..bitsize-1. ffs is supposed
2511 to produce a value in the range 0..bitsize. */
2512 temp
= expand_binop (mode
, add_optab
, temp
, gen_int_mode (1, mode
),
2513 target
, false, OPTAB_DIRECT
);
2520 add_equal_note (seq
, temp
, FFS
, op0
, 0);
2529 /* Extract the OMODE lowpart from VAL, which has IMODE. Under certain
2530 conditions, VAL may already be a SUBREG against which we cannot generate
2531 a further SUBREG. In this case, we expect forcing the value into a
2532 register will work around the situation. */
2535 lowpart_subreg_maybe_copy (machine_mode omode
, rtx val
,
2539 ret
= lowpart_subreg (omode
, val
, imode
);
2542 val
= force_reg (imode
, val
);
2543 ret
= lowpart_subreg (omode
, val
, imode
);
2544 gcc_assert (ret
!= NULL
);
2549 /* Expand a floating point absolute value or negation operation via a
2550 logical operation on the sign bit. */
2553 expand_absneg_bit (enum rtx_code code
, scalar_float_mode mode
,
2554 rtx op0
, rtx target
)
2556 const struct real_format
*fmt
;
2557 int bitpos
, word
, nwords
, i
;
2558 scalar_int_mode imode
;
2562 /* The format has to have a simple sign bit. */
2563 fmt
= REAL_MODE_FORMAT (mode
);
2567 bitpos
= fmt
->signbit_rw
;
2571 /* Don't create negative zeros if the format doesn't support them. */
2572 if (code
== NEG
&& !fmt
->has_signed_zero
)
2575 if (GET_MODE_SIZE (mode
) <= UNITS_PER_WORD
)
2577 if (!int_mode_for_mode (mode
).exists (&imode
))
2586 if (FLOAT_WORDS_BIG_ENDIAN
)
2587 word
= (GET_MODE_BITSIZE (mode
) - bitpos
) / BITS_PER_WORD
;
2589 word
= bitpos
/ BITS_PER_WORD
;
2590 bitpos
= bitpos
% BITS_PER_WORD
;
2591 nwords
= (GET_MODE_BITSIZE (mode
) + BITS_PER_WORD
- 1) / BITS_PER_WORD
;
2594 wide_int mask
= wi::set_bit_in_zero (bitpos
, GET_MODE_PRECISION (imode
));
2600 || (nwords
> 1 && !valid_multiword_target_p (target
)))
2601 target
= gen_reg_rtx (mode
);
2607 for (i
= 0; i
< nwords
; ++i
)
2609 rtx targ_piece
= operand_subword (target
, i
, 1, mode
);
2610 rtx op0_piece
= operand_subword_force (op0
, i
, mode
);
2614 temp
= expand_binop (imode
, code
== ABS
? and_optab
: xor_optab
,
2616 immed_wide_int_const (mask
, imode
),
2617 targ_piece
, 1, OPTAB_LIB_WIDEN
);
2618 if (temp
!= targ_piece
)
2619 emit_move_insn (targ_piece
, temp
);
2622 emit_move_insn (targ_piece
, op0_piece
);
2625 insns
= get_insns ();
2632 temp
= expand_binop (imode
, code
== ABS
? and_optab
: xor_optab
,
2633 gen_lowpart (imode
, op0
),
2634 immed_wide_int_const (mask
, imode
),
2635 gen_lowpart (imode
, target
), 1, OPTAB_LIB_WIDEN
);
2636 target
= lowpart_subreg_maybe_copy (mode
, temp
, imode
);
2638 set_dst_reg_note (get_last_insn (), REG_EQUAL
,
2639 gen_rtx_fmt_e (code
, mode
, copy_rtx (op0
)),
2646 /* As expand_unop, but will fail rather than attempt the operation in a
2647 different mode or with a libcall. */
2649 expand_unop_direct (machine_mode mode
, optab unoptab
, rtx op0
, rtx target
,
2652 if (optab_handler (unoptab
, mode
) != CODE_FOR_nothing
)
2654 struct expand_operand ops
[2];
2655 enum insn_code icode
= optab_handler (unoptab
, mode
);
2656 rtx_insn
*last
= get_last_insn ();
2659 create_output_operand (&ops
[0], target
, mode
);
2660 create_convert_operand_from (&ops
[1], op0
, mode
, unsignedp
);
2661 pat
= maybe_gen_insn (icode
, 2, ops
);
2664 if (INSN_P (pat
) && NEXT_INSN (pat
) != NULL_RTX
2665 && ! add_equal_note (pat
, ops
[0].value
,
2666 optab_to_code (unoptab
),
2667 ops
[1].value
, NULL_RTX
))
2669 delete_insns_since (last
);
2670 return expand_unop (mode
, unoptab
, op0
, NULL_RTX
, unsignedp
);
2675 return ops
[0].value
;
2681 /* Generate code to perform an operation specified by UNOPTAB
2682 on operand OP0, with result having machine-mode MODE.
2684 UNSIGNEDP is for the case where we have to widen the operands
2685 to perform the operation. It says to use zero-extension.
2687 If TARGET is nonzero, the value
2688 is generated there, if it is convenient to do so.
2689 In all cases an rtx is returned for the locus of the value;
2690 this may or may not be TARGET. */
2693 expand_unop (machine_mode mode
, optab unoptab
, rtx op0
, rtx target
,
2696 enum mode_class mclass
= GET_MODE_CLASS (mode
);
2697 machine_mode wider_mode
;
2698 scalar_int_mode int_mode
;
2699 scalar_float_mode float_mode
;
2703 temp
= expand_unop_direct (mode
, unoptab
, op0
, target
, unsignedp
);
2707 /* It can't be done in this mode. Can we open-code it in a wider mode? */
2709 /* Widening (or narrowing) clz needs special treatment. */
2710 if (unoptab
== clz_optab
)
2712 if (is_a
<scalar_int_mode
> (mode
, &int_mode
))
2714 temp
= widen_leading (int_mode
, op0
, target
, unoptab
);
2718 if (GET_MODE_SIZE (int_mode
) == 2 * UNITS_PER_WORD
2719 && optab_handler (unoptab
, word_mode
) != CODE_FOR_nothing
)
2721 temp
= expand_doubleword_clz (int_mode
, op0
, target
);
2730 if (unoptab
== clrsb_optab
)
2732 if (is_a
<scalar_int_mode
> (mode
, &int_mode
))
2734 temp
= widen_leading (int_mode
, op0
, target
, unoptab
);
2741 if (unoptab
== popcount_optab
2742 && is_a
<scalar_int_mode
> (mode
, &int_mode
)
2743 && GET_MODE_SIZE (int_mode
) == 2 * UNITS_PER_WORD
2744 && optab_handler (unoptab
, word_mode
) != CODE_FOR_nothing
2745 && optimize_insn_for_speed_p ())
2747 temp
= expand_doubleword_popcount (int_mode
, op0
, target
);
2752 if (unoptab
== parity_optab
2753 && is_a
<scalar_int_mode
> (mode
, &int_mode
)
2754 && GET_MODE_SIZE (int_mode
) == 2 * UNITS_PER_WORD
2755 && (optab_handler (unoptab
, word_mode
) != CODE_FOR_nothing
2756 || optab_handler (popcount_optab
, word_mode
) != CODE_FOR_nothing
)
2757 && optimize_insn_for_speed_p ())
2759 temp
= expand_doubleword_parity (int_mode
, op0
, target
);
2764 /* Widening (or narrowing) bswap needs special treatment. */
2765 if (unoptab
== bswap_optab
)
2767 /* HImode is special because in this mode BSWAP is equivalent to ROTATE
2768 or ROTATERT. First try these directly; if this fails, then try the
2769 obvious pair of shifts with allowed widening, as this will probably
2770 be always more efficient than the other fallback methods. */
2776 if (optab_handler (rotl_optab
, mode
) != CODE_FOR_nothing
)
2778 temp
= expand_binop (mode
, rotl_optab
, op0
, GEN_INT (8), target
,
2779 unsignedp
, OPTAB_DIRECT
);
2784 if (optab_handler (rotr_optab
, mode
) != CODE_FOR_nothing
)
2786 temp
= expand_binop (mode
, rotr_optab
, op0
, GEN_INT (8), target
,
2787 unsignedp
, OPTAB_DIRECT
);
2792 last
= get_last_insn ();
2794 temp1
= expand_binop (mode
, ashl_optab
, op0
, GEN_INT (8), NULL_RTX
,
2795 unsignedp
, OPTAB_WIDEN
);
2796 temp2
= expand_binop (mode
, lshr_optab
, op0
, GEN_INT (8), NULL_RTX
,
2797 unsignedp
, OPTAB_WIDEN
);
2800 temp
= expand_binop (mode
, ior_optab
, temp1
, temp2
, target
,
2801 unsignedp
, OPTAB_WIDEN
);
2806 delete_insns_since (last
);
2809 if (is_a
<scalar_int_mode
> (mode
, &int_mode
))
2811 temp
= widen_bswap (int_mode
, op0
, target
);
2815 if (GET_MODE_SIZE (int_mode
) == 2 * UNITS_PER_WORD
2816 && optab_handler (unoptab
, word_mode
) != CODE_FOR_nothing
)
2818 temp
= expand_doubleword_bswap (mode
, op0
, target
);
2827 if (CLASS_HAS_WIDER_MODES_P (mclass
))
2828 FOR_EACH_WIDER_MODE (wider_mode
, mode
)
2830 if (optab_handler (unoptab
, wider_mode
) != CODE_FOR_nothing
)
2833 rtx_insn
*last
= get_last_insn ();
2835 /* For certain operations, we need not actually extend
2836 the narrow operand, as long as we will truncate the
2837 results to the same narrowness. */
2839 xop0
= widen_operand (xop0
, wider_mode
, mode
, unsignedp
,
2840 (unoptab
== neg_optab
2841 || unoptab
== one_cmpl_optab
)
2842 && mclass
== MODE_INT
);
2844 temp
= expand_unop (wider_mode
, unoptab
, xop0
, NULL_RTX
,
2849 if (mclass
!= MODE_INT
2850 || !TRULY_NOOP_TRUNCATION_MODES_P (mode
, wider_mode
))
2853 target
= gen_reg_rtx (mode
);
2854 convert_move (target
, temp
, 0);
2858 return gen_lowpart (mode
, temp
);
2861 delete_insns_since (last
);
2865 /* These can be done a word at a time. */
2866 if (unoptab
== one_cmpl_optab
2867 && is_int_mode (mode
, &int_mode
)
2868 && GET_MODE_SIZE (int_mode
) > UNITS_PER_WORD
2869 && optab_handler (unoptab
, word_mode
) != CODE_FOR_nothing
)
2874 if (target
== 0 || target
== op0
|| !valid_multiword_target_p (target
))
2875 target
= gen_reg_rtx (int_mode
);
2879 /* Do the actual arithmetic. */
2880 for (i
= 0; i
< GET_MODE_BITSIZE (int_mode
) / BITS_PER_WORD
; i
++)
2882 rtx target_piece
= operand_subword (target
, i
, 1, int_mode
);
2883 rtx x
= expand_unop (word_mode
, unoptab
,
2884 operand_subword_force (op0
, i
, int_mode
),
2885 target_piece
, unsignedp
);
2887 if (target_piece
!= x
)
2888 emit_move_insn (target_piece
, x
);
2891 insns
= get_insns ();
2898 if (optab_to_code (unoptab
) == NEG
)
2900 /* Try negating floating point values by flipping the sign bit. */
2901 if (is_a
<scalar_float_mode
> (mode
, &float_mode
))
2903 temp
= expand_absneg_bit (NEG
, float_mode
, op0
, target
);
2908 /* If there is no negation pattern, and we have no negative zero,
2909 try subtracting from zero. */
2910 if (!HONOR_SIGNED_ZEROS (mode
))
2912 temp
= expand_binop (mode
, (unoptab
== negv_optab
2913 ? subv_optab
: sub_optab
),
2914 CONST0_RTX (mode
), op0
, target
,
2915 unsignedp
, OPTAB_DIRECT
);
2921 /* Try calculating parity (x) as popcount (x) % 2. */
2922 if (unoptab
== parity_optab
&& is_a
<scalar_int_mode
> (mode
, &int_mode
))
2924 temp
= expand_parity (int_mode
, op0
, target
);
2929 /* Try implementing ffs (x) in terms of clz (x). */
2930 if (unoptab
== ffs_optab
&& is_a
<scalar_int_mode
> (mode
, &int_mode
))
2932 temp
= expand_ffs (int_mode
, op0
, target
);
2937 /* Try implementing ctz (x) in terms of clz (x). */
2938 if (unoptab
== ctz_optab
&& is_a
<scalar_int_mode
> (mode
, &int_mode
))
2940 temp
= expand_ctz (int_mode
, op0
, target
);
2946 /* Now try a library call in this mode. */
2947 libfunc
= optab_libfunc (unoptab
, mode
);
2953 machine_mode outmode
= mode
;
2955 /* All of these functions return small values. Thus we choose to
2956 have them return something that isn't a double-word. */
2957 if (unoptab
== ffs_optab
|| unoptab
== clz_optab
|| unoptab
== ctz_optab
2958 || unoptab
== clrsb_optab
|| unoptab
== popcount_optab
2959 || unoptab
== parity_optab
)
2961 = GET_MODE (hard_libcall_value (TYPE_MODE (integer_type_node
),
2962 optab_libfunc (unoptab
, mode
)));
2966 /* Pass 1 for NO_QUEUE so we don't lose any increments
2967 if the libcall is cse'd or moved. */
2968 value
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
, outmode
,
2970 insns
= get_insns ();
2973 target
= gen_reg_rtx (outmode
);
2974 bool trapv
= trapv_unoptab_p (unoptab
);
2976 eq_value
= NULL_RTX
;
2979 eq_value
= gen_rtx_fmt_e (optab_to_code (unoptab
), mode
, op0
);
2980 if (GET_MODE_UNIT_SIZE (outmode
) < GET_MODE_UNIT_SIZE (mode
))
2981 eq_value
= simplify_gen_unary (TRUNCATE
, outmode
, eq_value
, mode
);
2982 else if (GET_MODE_UNIT_SIZE (outmode
) > GET_MODE_UNIT_SIZE (mode
))
2983 eq_value
= simplify_gen_unary (ZERO_EXTEND
,
2984 outmode
, eq_value
, mode
);
2986 emit_libcall_block_1 (insns
, target
, value
, eq_value
, trapv
);
2991 /* It can't be done in this mode. Can we do it in a wider mode? */
2993 if (CLASS_HAS_WIDER_MODES_P (mclass
))
2995 FOR_EACH_WIDER_MODE (wider_mode
, mode
)
2997 if (optab_handler (unoptab
, wider_mode
) != CODE_FOR_nothing
2998 || optab_libfunc (unoptab
, wider_mode
))
3001 rtx_insn
*last
= get_last_insn ();
3003 /* For certain operations, we need not actually extend
3004 the narrow operand, as long as we will truncate the
3005 results to the same narrowness. */
3006 xop0
= widen_operand (xop0
, wider_mode
, mode
, unsignedp
,
3007 (unoptab
== neg_optab
3008 || unoptab
== one_cmpl_optab
3009 || unoptab
== bswap_optab
)
3010 && mclass
== MODE_INT
);
3012 temp
= expand_unop (wider_mode
, unoptab
, xop0
, NULL_RTX
,
3015 /* If we are generating clz using wider mode, adjust the
3016 result. Similarly for clrsb. */
3017 if ((unoptab
== clz_optab
|| unoptab
== clrsb_optab
)
3020 scalar_int_mode wider_int_mode
3021 = as_a
<scalar_int_mode
> (wider_mode
);
3022 int_mode
= as_a
<scalar_int_mode
> (mode
);
3024 (wider_mode
, sub_optab
, temp
,
3025 gen_int_mode (GET_MODE_PRECISION (wider_int_mode
)
3026 - GET_MODE_PRECISION (int_mode
),
3028 target
, true, OPTAB_DIRECT
);
3031 /* Likewise for bswap. */
3032 if (unoptab
== bswap_optab
&& temp
!= 0)
3034 scalar_int_mode wider_int_mode
3035 = as_a
<scalar_int_mode
> (wider_mode
);
3036 int_mode
= as_a
<scalar_int_mode
> (mode
);
3037 gcc_assert (GET_MODE_PRECISION (wider_int_mode
)
3038 == GET_MODE_BITSIZE (wider_int_mode
)
3039 && GET_MODE_PRECISION (int_mode
)
3040 == GET_MODE_BITSIZE (int_mode
));
3042 temp
= expand_shift (RSHIFT_EXPR
, wider_int_mode
, temp
,
3043 GET_MODE_BITSIZE (wider_int_mode
)
3044 - GET_MODE_BITSIZE (int_mode
),
3050 if (mclass
!= MODE_INT
)
3053 target
= gen_reg_rtx (mode
);
3054 convert_move (target
, temp
, 0);
3058 return gen_lowpart (mode
, temp
);
3061 delete_insns_since (last
);
3066 /* One final attempt at implementing negation via subtraction,
3067 this time allowing widening of the operand. */
3068 if (optab_to_code (unoptab
) == NEG
&& !HONOR_SIGNED_ZEROS (mode
))
3071 temp
= expand_binop (mode
,
3072 unoptab
== negv_optab
? subv_optab
: sub_optab
,
3073 CONST0_RTX (mode
), op0
,
3074 target
, unsignedp
, OPTAB_LIB_WIDEN
);
3082 /* Emit code to compute the absolute value of OP0, with result to
3083 TARGET if convenient. (TARGET may be 0.) The return value says
3084 where the result actually is to be found.
3086 MODE is the mode of the operand; the mode of the result is
3087 different but can be deduced from MODE.
3092 expand_abs_nojump (machine_mode mode
, rtx op0
, rtx target
,
3093 int result_unsignedp
)
3097 if (GET_MODE_CLASS (mode
) != MODE_INT
3099 result_unsignedp
= 1;
3101 /* First try to do it with a special abs instruction. */
3102 temp
= expand_unop (mode
, result_unsignedp
? abs_optab
: absv_optab
,
3107 /* For floating point modes, try clearing the sign bit. */
3108 scalar_float_mode float_mode
;
3109 if (is_a
<scalar_float_mode
> (mode
, &float_mode
))
3111 temp
= expand_absneg_bit (ABS
, float_mode
, op0
, target
);
3116 /* If we have a MAX insn, we can do this as MAX (x, -x). */
3117 if (optab_handler (smax_optab
, mode
) != CODE_FOR_nothing
3118 && !HONOR_SIGNED_ZEROS (mode
))
3120 rtx_insn
*last
= get_last_insn ();
3122 temp
= expand_unop (mode
, result_unsignedp
? neg_optab
: negv_optab
,
3125 temp
= expand_binop (mode
, smax_optab
, op0
, temp
, target
, 0,
3131 delete_insns_since (last
);
3134 /* If this machine has expensive jumps, we can do integer absolute
3135 value of X as (((signed) x >> (W-1)) ^ x) - ((signed) x >> (W-1)),
3136 where W is the width of MODE. */
3138 scalar_int_mode int_mode
;
3139 if (is_int_mode (mode
, &int_mode
)
3140 && BRANCH_COST (optimize_insn_for_speed_p (),
3143 rtx extended
= expand_shift (RSHIFT_EXPR
, int_mode
, op0
,
3144 GET_MODE_PRECISION (int_mode
) - 1,
3147 temp
= expand_binop (int_mode
, xor_optab
, extended
, op0
, target
, 0,
3150 temp
= expand_binop (int_mode
,
3151 result_unsignedp
? sub_optab
: subv_optab
,
3152 temp
, extended
, target
, 0, OPTAB_LIB_WIDEN
);
3162 expand_abs (machine_mode mode
, rtx op0
, rtx target
,
3163 int result_unsignedp
, int safe
)
3166 rtx_code_label
*op1
;
3168 if (GET_MODE_CLASS (mode
) != MODE_INT
3170 result_unsignedp
= 1;
3172 temp
= expand_abs_nojump (mode
, op0
, target
, result_unsignedp
);
3176 /* If that does not win, use conditional jump and negate. */
3178 /* It is safe to use the target if it is the same
3179 as the source if this is also a pseudo register */
3180 if (op0
== target
&& REG_P (op0
)
3181 && REGNO (op0
) >= FIRST_PSEUDO_REGISTER
)
3184 op1
= gen_label_rtx ();
3185 if (target
== 0 || ! safe
3186 || GET_MODE (target
) != mode
3187 || (MEM_P (target
) && MEM_VOLATILE_P (target
))
3189 && REGNO (target
) < FIRST_PSEUDO_REGISTER
))
3190 target
= gen_reg_rtx (mode
);
3192 emit_move_insn (target
, op0
);
3195 do_compare_rtx_and_jump (target
, CONST0_RTX (mode
), GE
, 0, mode
,
3196 NULL_RTX
, NULL
, op1
,
3197 profile_probability::uninitialized ());
3199 op0
= expand_unop (mode
, result_unsignedp
? neg_optab
: negv_optab
,
3202 emit_move_insn (target
, op0
);
3208 /* Emit code to compute the one's complement absolute value of OP0
3209 (if (OP0 < 0) OP0 = ~OP0), with result to TARGET if convenient.
3210 (TARGET may be NULL_RTX.) The return value says where the result
3211 actually is to be found.
3213 MODE is the mode of the operand; the mode of the result is
3214 different but can be deduced from MODE. */
3217 expand_one_cmpl_abs_nojump (machine_mode mode
, rtx op0
, rtx target
)
3221 /* Not applicable for floating point modes. */
3222 if (FLOAT_MODE_P (mode
))
3225 /* If we have a MAX insn, we can do this as MAX (x, ~x). */
3226 if (optab_handler (smax_optab
, mode
) != CODE_FOR_nothing
)
3228 rtx_insn
*last
= get_last_insn ();
3230 temp
= expand_unop (mode
, one_cmpl_optab
, op0
, NULL_RTX
, 0);
3232 temp
= expand_binop (mode
, smax_optab
, op0
, temp
, target
, 0,
3238 delete_insns_since (last
);
3241 /* If this machine has expensive jumps, we can do one's complement
3242 absolute value of X as (((signed) x >> (W-1)) ^ x). */
3244 scalar_int_mode int_mode
;
3245 if (is_int_mode (mode
, &int_mode
)
3246 && BRANCH_COST (optimize_insn_for_speed_p (),
3249 rtx extended
= expand_shift (RSHIFT_EXPR
, int_mode
, op0
,
3250 GET_MODE_PRECISION (int_mode
) - 1,
3253 temp
= expand_binop (int_mode
, xor_optab
, extended
, op0
, target
, 0,
3263 /* A subroutine of expand_copysign, perform the copysign operation using the
3264 abs and neg primitives advertised to exist on the target. The assumption
3265 is that we have a split register file, and leaving op0 in fp registers,
3266 and not playing with subregs so much, will help the register allocator. */
3269 expand_copysign_absneg (scalar_float_mode mode
, rtx op0
, rtx op1
, rtx target
,
3270 int bitpos
, bool op0_is_abs
)
3272 scalar_int_mode imode
;
3273 enum insn_code icode
;
3275 rtx_code_label
*label
;
3280 /* Check if the back end provides an insn that handles signbit for the
3282 icode
= optab_handler (signbit_optab
, mode
);
3283 if (icode
!= CODE_FOR_nothing
)
3285 imode
= as_a
<scalar_int_mode
> (insn_data
[(int) icode
].operand
[0].mode
);
3286 sign
= gen_reg_rtx (imode
);
3287 emit_unop_insn (icode
, sign
, op1
, UNKNOWN
);
3291 if (GET_MODE_SIZE (mode
) <= UNITS_PER_WORD
)
3293 if (!int_mode_for_mode (mode
).exists (&imode
))
3295 op1
= gen_lowpart (imode
, op1
);
3302 if (FLOAT_WORDS_BIG_ENDIAN
)
3303 word
= (GET_MODE_BITSIZE (mode
) - bitpos
) / BITS_PER_WORD
;
3305 word
= bitpos
/ BITS_PER_WORD
;
3306 bitpos
= bitpos
% BITS_PER_WORD
;
3307 op1
= operand_subword_force (op1
, word
, mode
);
3310 wide_int mask
= wi::set_bit_in_zero (bitpos
, GET_MODE_PRECISION (imode
));
3311 sign
= expand_binop (imode
, and_optab
, op1
,
3312 immed_wide_int_const (mask
, imode
),
3313 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
3318 op0
= expand_unop (mode
, abs_optab
, op0
, target
, 0);
3325 if (target
== NULL_RTX
)
3326 target
= copy_to_reg (op0
);
3328 emit_move_insn (target
, op0
);
3331 label
= gen_label_rtx ();
3332 emit_cmp_and_jump_insns (sign
, const0_rtx
, EQ
, NULL_RTX
, imode
, 1, label
);
3334 if (CONST_DOUBLE_AS_FLOAT_P (op0
))
3335 op0
= simplify_unary_operation (NEG
, mode
, op0
, mode
);
3337 op0
= expand_unop (mode
, neg_optab
, op0
, target
, 0);
3339 emit_move_insn (target
, op0
);
3347 /* A subroutine of expand_copysign, perform the entire copysign operation
3348 with integer bitmasks. BITPOS is the position of the sign bit; OP0_IS_ABS
3349 is true if op0 is known to have its sign bit clear. */
3352 expand_copysign_bit (scalar_float_mode mode
, rtx op0
, rtx op1
, rtx target
,
3353 int bitpos
, bool op0_is_abs
)
3355 scalar_int_mode imode
;
3356 int word
, nwords
, i
;
3360 if (GET_MODE_SIZE (mode
) <= UNITS_PER_WORD
)
3362 if (!int_mode_for_mode (mode
).exists (&imode
))
3371 if (FLOAT_WORDS_BIG_ENDIAN
)
3372 word
= (GET_MODE_BITSIZE (mode
) - bitpos
) / BITS_PER_WORD
;
3374 word
= bitpos
/ BITS_PER_WORD
;
3375 bitpos
= bitpos
% BITS_PER_WORD
;
3376 nwords
= (GET_MODE_BITSIZE (mode
) + BITS_PER_WORD
- 1) / BITS_PER_WORD
;
3379 wide_int mask
= wi::set_bit_in_zero (bitpos
, GET_MODE_PRECISION (imode
));
3384 || (nwords
> 1 && !valid_multiword_target_p (target
)))
3385 target
= gen_reg_rtx (mode
);
3391 for (i
= 0; i
< nwords
; ++i
)
3393 rtx targ_piece
= operand_subword (target
, i
, 1, mode
);
3394 rtx op0_piece
= operand_subword_force (op0
, i
, mode
);
3400 = expand_binop (imode
, and_optab
, op0_piece
,
3401 immed_wide_int_const (~mask
, imode
),
3402 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
3403 op1
= expand_binop (imode
, and_optab
,
3404 operand_subword_force (op1
, i
, mode
),
3405 immed_wide_int_const (mask
, imode
),
3406 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
3408 temp
= expand_binop (imode
, ior_optab
, op0_piece
, op1
,
3409 targ_piece
, 1, OPTAB_LIB_WIDEN
);
3410 if (temp
!= targ_piece
)
3411 emit_move_insn (targ_piece
, temp
);
3414 emit_move_insn (targ_piece
, op0_piece
);
3417 insns
= get_insns ();
3424 op1
= expand_binop (imode
, and_optab
, gen_lowpart (imode
, op1
),
3425 immed_wide_int_const (mask
, imode
),
3426 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
3428 op0
= gen_lowpart (imode
, op0
);
3430 op0
= expand_binop (imode
, and_optab
, op0
,
3431 immed_wide_int_const (~mask
, imode
),
3432 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
3434 temp
= expand_binop (imode
, ior_optab
, op0
, op1
,
3435 gen_lowpart (imode
, target
), 1, OPTAB_LIB_WIDEN
);
3436 target
= lowpart_subreg_maybe_copy (mode
, temp
, imode
);
3442 /* Expand the C99 copysign operation. OP0 and OP1 must be the same
3443 scalar floating point mode. Return NULL if we do not know how to
3444 expand the operation inline. */
3447 expand_copysign (rtx op0
, rtx op1
, rtx target
)
3449 scalar_float_mode mode
;
3450 const struct real_format
*fmt
;
3454 mode
= as_a
<scalar_float_mode
> (GET_MODE (op0
));
3455 gcc_assert (GET_MODE (op1
) == mode
);
3457 /* First try to do it with a special instruction. */
3458 temp
= expand_binop (mode
, copysign_optab
, op0
, op1
,
3459 target
, 0, OPTAB_DIRECT
);
3463 fmt
= REAL_MODE_FORMAT (mode
);
3464 if (fmt
== NULL
|| !fmt
->has_signed_zero
)
3468 if (CONST_DOUBLE_AS_FLOAT_P (op0
))
3470 if (real_isneg (CONST_DOUBLE_REAL_VALUE (op0
)))
3471 op0
= simplify_unary_operation (ABS
, mode
, op0
, mode
);
3475 if (fmt
->signbit_ro
>= 0
3476 && (CONST_DOUBLE_AS_FLOAT_P (op0
)
3477 || (optab_handler (neg_optab
, mode
) != CODE_FOR_nothing
3478 && optab_handler (abs_optab
, mode
) != CODE_FOR_nothing
)))
3480 temp
= expand_copysign_absneg (mode
, op0
, op1
, target
,
3481 fmt
->signbit_ro
, op0_is_abs
);
3486 if (fmt
->signbit_rw
< 0)
3488 return expand_copysign_bit (mode
, op0
, op1
, target
,
3489 fmt
->signbit_rw
, op0_is_abs
);
3492 /* Generate an instruction whose insn-code is INSN_CODE,
3493 with two operands: an output TARGET and an input OP0.
3494 TARGET *must* be nonzero, and the output is always stored there.
3495 CODE is an rtx code such that (CODE OP0) is an rtx that describes
3496 the value that is stored into TARGET.
3498 Return false if expansion failed. */
3501 maybe_emit_unop_insn (enum insn_code icode
, rtx target
, rtx op0
,
3504 struct expand_operand ops
[2];
3507 create_output_operand (&ops
[0], target
, GET_MODE (target
));
3508 create_input_operand (&ops
[1], op0
, GET_MODE (op0
));
3509 pat
= maybe_gen_insn (icode
, 2, ops
);
3513 if (INSN_P (pat
) && NEXT_INSN (pat
) != NULL_RTX
3515 add_equal_note (pat
, ops
[0].value
, code
, ops
[1].value
, NULL_RTX
);
3519 if (ops
[0].value
!= target
)
3520 emit_move_insn (target
, ops
[0].value
);
3523 /* Generate an instruction whose insn-code is INSN_CODE,
3524 with two operands: an output TARGET and an input OP0.
3525 TARGET *must* be nonzero, and the output is always stored there.
3526 CODE is an rtx code such that (CODE OP0) is an rtx that describes
3527 the value that is stored into TARGET. */
3530 emit_unop_insn (enum insn_code icode
, rtx target
, rtx op0
, enum rtx_code code
)
3532 bool ok
= maybe_emit_unop_insn (icode
, target
, op0
, code
);
3536 struct no_conflict_data
3539 rtx_insn
*first
, *insn
;
3543 /* Called via note_stores by emit_libcall_block. Set P->must_stay if
3544 the currently examined clobber / store has to stay in the list of
3545 insns that constitute the actual libcall block. */
3547 no_conflict_move_test (rtx dest
, const_rtx set
, void *p0
)
3549 struct no_conflict_data
*p
= (struct no_conflict_data
*) p0
;
3551 /* If this inns directly contributes to setting the target, it must stay. */
3552 if (reg_overlap_mentioned_p (p
->target
, dest
))
3553 p
->must_stay
= true;
3554 /* If we haven't committed to keeping any other insns in the list yet,
3555 there is nothing more to check. */
3556 else if (p
->insn
== p
->first
)
3558 /* If this insn sets / clobbers a register that feeds one of the insns
3559 already in the list, this insn has to stay too. */
3560 else if (reg_overlap_mentioned_p (dest
, PATTERN (p
->first
))
3561 || (CALL_P (p
->first
) && (find_reg_fusage (p
->first
, USE
, dest
)))
3562 || reg_used_between_p (dest
, p
->first
, p
->insn
)
3563 /* Likewise if this insn depends on a register set by a previous
3564 insn in the list, or if it sets a result (presumably a hard
3565 register) that is set or clobbered by a previous insn.
3566 N.B. the modified_*_p (SET_DEST...) tests applied to a MEM
3567 SET_DEST perform the former check on the address, and the latter
3568 check on the MEM. */
3569 || (GET_CODE (set
) == SET
3570 && (modified_in_p (SET_SRC (set
), p
->first
)
3571 || modified_in_p (SET_DEST (set
), p
->first
)
3572 || modified_between_p (SET_SRC (set
), p
->first
, p
->insn
)
3573 || modified_between_p (SET_DEST (set
), p
->first
, p
->insn
))))
3574 p
->must_stay
= true;
3578 /* Emit code to make a call to a constant function or a library call.
3580 INSNS is a list containing all insns emitted in the call.
3581 These insns leave the result in RESULT. Our block is to copy RESULT
3582 to TARGET, which is logically equivalent to EQUIV.
3584 We first emit any insns that set a pseudo on the assumption that these are
3585 loading constants into registers; doing so allows them to be safely cse'ed
3586 between blocks. Then we emit all the other insns in the block, followed by
3587 an insn to move RESULT to TARGET. This last insn will have a REQ_EQUAL
3588 note with an operand of EQUIV. */
3591 emit_libcall_block_1 (rtx_insn
*insns
, rtx target
, rtx result
, rtx equiv
,
3592 bool equiv_may_trap
)
3594 rtx final_dest
= target
;
3595 rtx_insn
*next
, *last
, *insn
;
3597 /* If this is a reg with REG_USERVAR_P set, then it could possibly turn
3598 into a MEM later. Protect the libcall block from this change. */
3599 if (! REG_P (target
) || REG_USERVAR_P (target
))
3600 target
= gen_reg_rtx (GET_MODE (target
));
3602 /* If we're using non-call exceptions, a libcall corresponding to an
3603 operation that may trap may also trap. */
3604 /* ??? See the comment in front of make_reg_eh_region_note. */
3605 if (cfun
->can_throw_non_call_exceptions
3606 && (equiv_may_trap
|| may_trap_p (equiv
)))
3608 for (insn
= insns
; insn
; insn
= NEXT_INSN (insn
))
3611 rtx note
= find_reg_note (insn
, REG_EH_REGION
, NULL_RTX
);
3614 int lp_nr
= INTVAL (XEXP (note
, 0));
3615 if (lp_nr
== 0 || lp_nr
== INT_MIN
)
3616 remove_note (insn
, note
);
3622 /* Look for any CALL_INSNs in this sequence, and attach a REG_EH_REGION
3623 reg note to indicate that this call cannot throw or execute a nonlocal
3624 goto (unless there is already a REG_EH_REGION note, in which case
3626 for (insn
= insns
; insn
; insn
= NEXT_INSN (insn
))
3628 make_reg_eh_region_note_nothrow_nononlocal (insn
);
3631 /* First emit all insns that set pseudos. Remove them from the list as
3632 we go. Avoid insns that set pseudos which were referenced in previous
3633 insns. These can be generated by move_by_pieces, for example,
3634 to update an address. Similarly, avoid insns that reference things
3635 set in previous insns. */
3637 for (insn
= insns
; insn
; insn
= next
)
3639 rtx set
= single_set (insn
);
3641 next
= NEXT_INSN (insn
);
3643 if (set
!= 0 && REG_P (SET_DEST (set
))
3644 && REGNO (SET_DEST (set
)) >= FIRST_PSEUDO_REGISTER
)
3646 struct no_conflict_data data
;
3648 data
.target
= const0_rtx
;
3652 note_stores (PATTERN (insn
), no_conflict_move_test
, &data
);
3653 if (! data
.must_stay
)
3655 if (PREV_INSN (insn
))
3656 SET_NEXT_INSN (PREV_INSN (insn
)) = next
;
3661 SET_PREV_INSN (next
) = PREV_INSN (insn
);
3667 /* Some ports use a loop to copy large arguments onto the stack.
3668 Don't move anything outside such a loop. */
3673 /* Write the remaining insns followed by the final copy. */
3674 for (insn
= insns
; insn
; insn
= next
)
3676 next
= NEXT_INSN (insn
);
3681 last
= emit_move_insn (target
, result
);
3683 set_dst_reg_note (last
, REG_EQUAL
, copy_rtx (equiv
), target
);
3685 if (final_dest
!= target
)
3686 emit_move_insn (final_dest
, target
);
3690 emit_libcall_block (rtx_insn
*insns
, rtx target
, rtx result
, rtx equiv
)
3692 emit_libcall_block_1 (insns
, target
, result
, equiv
, false);
3695 /* Nonzero if we can perform a comparison of mode MODE straightforwardly.
3696 PURPOSE describes how this comparison will be used. CODE is the rtx
3697 comparison code we will be using.
3699 ??? Actually, CODE is slightly weaker than that. A target is still
3700 required to implement all of the normal bcc operations, but not
3701 required to implement all (or any) of the unordered bcc operations. */
3704 can_compare_p (enum rtx_code code
, machine_mode mode
,
3705 enum can_compare_purpose purpose
)
3708 test
= gen_rtx_fmt_ee (code
, mode
, const0_rtx
, const0_rtx
);
3711 enum insn_code icode
;
3713 if (purpose
== ccp_jump
3714 && (icode
= optab_handler (cbranch_optab
, mode
)) != CODE_FOR_nothing
3715 && insn_operand_matches (icode
, 0, test
))
3717 if (purpose
== ccp_store_flag
3718 && (icode
= optab_handler (cstore_optab
, mode
)) != CODE_FOR_nothing
3719 && insn_operand_matches (icode
, 1, test
))
3721 if (purpose
== ccp_cmov
3722 && optab_handler (cmov_optab
, mode
) != CODE_FOR_nothing
)
3725 mode
= GET_MODE_WIDER_MODE (mode
).else_void ();
3726 PUT_MODE (test
, mode
);
3728 while (mode
!= VOIDmode
);
3733 /* This function is called when we are going to emit a compare instruction that
3734 compares the values found in X and Y, using the rtl operator COMPARISON.
3736 If they have mode BLKmode, then SIZE specifies the size of both operands.
3738 UNSIGNEDP nonzero says that the operands are unsigned;
3739 this matters if they need to be widened (as given by METHODS).
3741 *PTEST is where the resulting comparison RTX is returned or NULL_RTX
3742 if we failed to produce one.
3744 *PMODE is the mode of the inputs (in case they are const_int).
3746 This function performs all the setup necessary so that the caller only has
3747 to emit a single comparison insn. This setup can involve doing a BLKmode
3748 comparison or emitting a library call to perform the comparison if no insn
3749 is available to handle it.
3750 The values which are passed in through pointers can be modified; the caller
3751 should perform the comparison on the modified values. Constant
3752 comparisons must have already been folded. */
3755 prepare_cmp_insn (rtx x
, rtx y
, enum rtx_code comparison
, rtx size
,
3756 int unsignedp
, enum optab_methods methods
,
3757 rtx
*ptest
, machine_mode
*pmode
)
3759 machine_mode mode
= *pmode
;
3761 machine_mode cmp_mode
;
3762 enum mode_class mclass
;
3764 /* The other methods are not needed. */
3765 gcc_assert (methods
== OPTAB_DIRECT
|| methods
== OPTAB_WIDEN
3766 || methods
== OPTAB_LIB_WIDEN
);
3768 /* If we are optimizing, force expensive constants into a register. */
3769 if (CONSTANT_P (x
) && optimize
3770 && (rtx_cost (x
, mode
, COMPARE
, 0, optimize_insn_for_speed_p ())
3771 > COSTS_N_INSNS (1)))
3772 x
= force_reg (mode
, x
);
3774 if (CONSTANT_P (y
) && optimize
3775 && (rtx_cost (y
, mode
, COMPARE
, 1, optimize_insn_for_speed_p ())
3776 > COSTS_N_INSNS (1)))
3777 y
= force_reg (mode
, y
);
3780 /* Make sure if we have a canonical comparison. The RTL
3781 documentation states that canonical comparisons are required only
3782 for targets which have cc0. */
3783 gcc_assert (!CONSTANT_P (x
) || CONSTANT_P (y
));
3786 /* Don't let both operands fail to indicate the mode. */
3787 if (GET_MODE (x
) == VOIDmode
&& GET_MODE (y
) == VOIDmode
)
3788 x
= force_reg (mode
, x
);
3789 if (mode
== VOIDmode
)
3790 mode
= GET_MODE (x
) != VOIDmode
? GET_MODE (x
) : GET_MODE (y
);
3792 /* Handle all BLKmode compares. */
3794 if (mode
== BLKmode
)
3796 machine_mode result_mode
;
3797 enum insn_code cmp_code
;
3800 = GEN_INT (MIN (MEM_ALIGN (x
), MEM_ALIGN (y
)) / BITS_PER_UNIT
);
3804 /* Try to use a memory block compare insn - either cmpstr
3805 or cmpmem will do. */
3806 opt_scalar_int_mode cmp_mode_iter
;
3807 FOR_EACH_MODE_IN_CLASS (cmp_mode_iter
, MODE_INT
)
3809 scalar_int_mode cmp_mode
= cmp_mode_iter
.require ();
3810 cmp_code
= direct_optab_handler (cmpmem_optab
, cmp_mode
);
3811 if (cmp_code
== CODE_FOR_nothing
)
3812 cmp_code
= direct_optab_handler (cmpstr_optab
, cmp_mode
);
3813 if (cmp_code
== CODE_FOR_nothing
)
3814 cmp_code
= direct_optab_handler (cmpstrn_optab
, cmp_mode
);
3815 if (cmp_code
== CODE_FOR_nothing
)
3818 /* Must make sure the size fits the insn's mode. */
3819 if (CONST_INT_P (size
)
3820 ? INTVAL (size
) >= (1 << GET_MODE_BITSIZE (cmp_mode
))
3821 : (GET_MODE_BITSIZE (as_a
<scalar_int_mode
> (GET_MODE (size
)))
3822 > GET_MODE_BITSIZE (cmp_mode
)))
3825 result_mode
= insn_data
[cmp_code
].operand
[0].mode
;
3826 result
= gen_reg_rtx (result_mode
);
3827 size
= convert_to_mode (cmp_mode
, size
, 1);
3828 emit_insn (GEN_FCN (cmp_code
) (result
, x
, y
, size
, opalign
));
3830 *ptest
= gen_rtx_fmt_ee (comparison
, VOIDmode
, result
, const0_rtx
);
3831 *pmode
= result_mode
;
3835 if (methods
!= OPTAB_LIB
&& methods
!= OPTAB_LIB_WIDEN
)
3838 /* Otherwise call a library function. */
3839 result
= emit_block_comp_via_libcall (XEXP (x
, 0), XEXP (y
, 0), size
);
3843 mode
= TYPE_MODE (integer_type_node
);
3844 methods
= OPTAB_LIB_WIDEN
;
3848 /* Don't allow operands to the compare to trap, as that can put the
3849 compare and branch in different basic blocks. */
3850 if (cfun
->can_throw_non_call_exceptions
)
3853 x
= copy_to_reg (x
);
3855 y
= copy_to_reg (y
);
3858 if (GET_MODE_CLASS (mode
) == MODE_CC
)
3860 enum insn_code icode
= optab_handler (cbranch_optab
, CCmode
);
3861 test
= gen_rtx_fmt_ee (comparison
, VOIDmode
, x
, y
);
3862 gcc_assert (icode
!= CODE_FOR_nothing
3863 && insn_operand_matches (icode
, 0, test
));
3868 mclass
= GET_MODE_CLASS (mode
);
3869 test
= gen_rtx_fmt_ee (comparison
, VOIDmode
, x
, y
);
3870 FOR_EACH_MODE_FROM (cmp_mode
, mode
)
3872 enum insn_code icode
;
3873 icode
= optab_handler (cbranch_optab
, cmp_mode
);
3874 if (icode
!= CODE_FOR_nothing
3875 && insn_operand_matches (icode
, 0, test
))
3877 rtx_insn
*last
= get_last_insn ();
3878 rtx op0
= prepare_operand (icode
, x
, 1, mode
, cmp_mode
, unsignedp
);
3879 rtx op1
= prepare_operand (icode
, y
, 2, mode
, cmp_mode
, unsignedp
);
3881 && insn_operand_matches (icode
, 1, op0
)
3882 && insn_operand_matches (icode
, 2, op1
))
3884 XEXP (test
, 0) = op0
;
3885 XEXP (test
, 1) = op1
;
3890 delete_insns_since (last
);
3893 if (methods
== OPTAB_DIRECT
|| !CLASS_HAS_WIDER_MODES_P (mclass
))
3897 if (methods
!= OPTAB_LIB_WIDEN
)
3900 if (!SCALAR_FLOAT_MODE_P (mode
))
3903 machine_mode ret_mode
;
3905 /* Handle a libcall just for the mode we are using. */
3906 libfunc
= optab_libfunc (cmp_optab
, mode
);
3907 gcc_assert (libfunc
);
3909 /* If we want unsigned, and this mode has a distinct unsigned
3910 comparison routine, use that. */
3913 rtx ulibfunc
= optab_libfunc (ucmp_optab
, mode
);
3918 ret_mode
= targetm
.libgcc_cmp_return_mode ();
3919 result
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
,
3920 ret_mode
, x
, mode
, y
, mode
);
3922 /* There are two kinds of comparison routines. Biased routines
3923 return 0/1/2, and unbiased routines return -1/0/1. Other parts
3924 of gcc expect that the comparison operation is equivalent
3925 to the modified comparison. For signed comparisons compare the
3926 result against 1 in the biased case, and zero in the unbiased
3927 case. For unsigned comparisons always compare against 1 after
3928 biasing the unbiased result by adding 1. This gives us a way to
3930 The comparisons in the fixed-point helper library are always
3935 if (!TARGET_LIB_INT_CMP_BIASED
&& !ALL_FIXED_POINT_MODE_P (mode
))
3938 x
= plus_constant (ret_mode
, result
, 1);
3944 prepare_cmp_insn (x
, y
, comparison
, NULL_RTX
, unsignedp
, methods
,
3948 prepare_float_lib_cmp (x
, y
, comparison
, ptest
, pmode
);
3956 /* Before emitting an insn with code ICODE, make sure that X, which is going
3957 to be used for operand OPNUM of the insn, is converted from mode MODE to
3958 WIDER_MODE (UNSIGNEDP determines whether it is an unsigned conversion), and
3959 that it is accepted by the operand predicate. Return the new value. */
3962 prepare_operand (enum insn_code icode
, rtx x
, int opnum
, machine_mode mode
,
3963 machine_mode wider_mode
, int unsignedp
)
3965 if (mode
!= wider_mode
)
3966 x
= convert_modes (wider_mode
, mode
, x
, unsignedp
);
3968 if (!insn_operand_matches (icode
, opnum
, x
))
3970 machine_mode op_mode
= insn_data
[(int) icode
].operand
[opnum
].mode
;
3971 if (reload_completed
)
3973 if (GET_MODE (x
) != op_mode
&& GET_MODE (x
) != VOIDmode
)
3975 x
= copy_to_mode_reg (op_mode
, x
);
3981 /* Subroutine of emit_cmp_and_jump_insns; this function is called when we know
3982 we can do the branch. */
3985 emit_cmp_and_jump_insn_1 (rtx test
, machine_mode mode
, rtx label
,
3986 profile_probability prob
)
3988 machine_mode optab_mode
;
3989 enum mode_class mclass
;
3990 enum insn_code icode
;
3993 mclass
= GET_MODE_CLASS (mode
);
3994 optab_mode
= (mclass
== MODE_CC
) ? CCmode
: mode
;
3995 icode
= optab_handler (cbranch_optab
, optab_mode
);
3997 gcc_assert (icode
!= CODE_FOR_nothing
);
3998 gcc_assert (insn_operand_matches (icode
, 0, test
));
3999 insn
= emit_jump_insn (GEN_FCN (icode
) (test
, XEXP (test
, 0),
4000 XEXP (test
, 1), label
));
4001 if (prob
.initialized_p ()
4002 && profile_status_for_fn (cfun
) != PROFILE_ABSENT
4005 && any_condjump_p (insn
)
4006 && !find_reg_note (insn
, REG_BR_PROB
, 0))
4007 add_reg_br_prob_note (insn
, prob
);
4010 /* Generate code to compare X with Y so that the condition codes are
4011 set and to jump to LABEL if the condition is true. If X is a
4012 constant and Y is not a constant, then the comparison is swapped to
4013 ensure that the comparison RTL has the canonical form.
4015 UNSIGNEDP nonzero says that X and Y are unsigned; this matters if they
4016 need to be widened. UNSIGNEDP is also used to select the proper
4017 branch condition code.
4019 If X and Y have mode BLKmode, then SIZE specifies the size of both X and Y.
4021 MODE is the mode of the inputs (in case they are const_int).
4023 COMPARISON is the rtl operator to compare with (EQ, NE, GT, etc.).
4024 It will be potentially converted into an unsigned variant based on
4025 UNSIGNEDP to select a proper jump instruction.
4027 PROB is the probability of jumping to LABEL. */
4030 emit_cmp_and_jump_insns (rtx x
, rtx y
, enum rtx_code comparison
, rtx size
,
4031 machine_mode mode
, int unsignedp
, rtx label
,
4032 profile_probability prob
)
4034 rtx op0
= x
, op1
= y
;
4037 /* Swap operands and condition to ensure canonical RTL. */
4038 if (swap_commutative_operands_p (x
, y
)
4039 && can_compare_p (swap_condition (comparison
), mode
, ccp_jump
))
4042 comparison
= swap_condition (comparison
);
4045 /* If OP0 is still a constant, then both X and Y must be constants
4046 or the opposite comparison is not supported. Force X into a register
4047 to create canonical RTL. */
4048 if (CONSTANT_P (op0
))
4049 op0
= force_reg (mode
, op0
);
4052 comparison
= unsigned_condition (comparison
);
4054 prepare_cmp_insn (op0
, op1
, comparison
, size
, unsignedp
, OPTAB_LIB_WIDEN
,
4056 emit_cmp_and_jump_insn_1 (test
, mode
, label
, prob
);
4060 /* Emit a library call comparison between floating point X and Y.
4061 COMPARISON is the rtl operator to compare with (EQ, NE, GT, etc.). */
4064 prepare_float_lib_cmp (rtx x
, rtx y
, enum rtx_code comparison
,
4065 rtx
*ptest
, machine_mode
*pmode
)
4067 enum rtx_code swapped
= swap_condition (comparison
);
4068 enum rtx_code reversed
= reverse_condition_maybe_unordered (comparison
);
4069 machine_mode orig_mode
= GET_MODE (x
);
4071 rtx true_rtx
, false_rtx
;
4072 rtx value
, target
, equiv
;
4075 bool reversed_p
= false;
4076 scalar_int_mode cmp_mode
= targetm
.libgcc_cmp_return_mode ();
4078 FOR_EACH_MODE_FROM (mode
, orig_mode
)
4080 if (code_to_optab (comparison
)
4081 && (libfunc
= optab_libfunc (code_to_optab (comparison
), mode
)))
4084 if (code_to_optab (swapped
)
4085 && (libfunc
= optab_libfunc (code_to_optab (swapped
), mode
)))
4088 comparison
= swapped
;
4092 if (code_to_optab (reversed
)
4093 && (libfunc
= optab_libfunc (code_to_optab (reversed
), mode
)))
4095 comparison
= reversed
;
4101 gcc_assert (mode
!= VOIDmode
);
4103 if (mode
!= orig_mode
)
4105 x
= convert_to_mode (mode
, x
, 0);
4106 y
= convert_to_mode (mode
, y
, 0);
4109 /* Attach a REG_EQUAL note describing the semantics of the libcall to
4110 the RTL. The allows the RTL optimizers to delete the libcall if the
4111 condition can be determined at compile-time. */
4112 if (comparison
== UNORDERED
4113 || FLOAT_LIB_COMPARE_RETURNS_BOOL (mode
, comparison
))
4115 true_rtx
= const_true_rtx
;
4116 false_rtx
= const0_rtx
;
4123 true_rtx
= const0_rtx
;
4124 false_rtx
= const_true_rtx
;
4128 true_rtx
= const_true_rtx
;
4129 false_rtx
= const0_rtx
;
4133 true_rtx
= const1_rtx
;
4134 false_rtx
= const0_rtx
;
4138 true_rtx
= const0_rtx
;
4139 false_rtx
= constm1_rtx
;
4143 true_rtx
= constm1_rtx
;
4144 false_rtx
= const0_rtx
;
4148 true_rtx
= const0_rtx
;
4149 false_rtx
= const1_rtx
;
4157 if (comparison
== UNORDERED
)
4159 rtx temp
= simplify_gen_relational (NE
, cmp_mode
, mode
, x
, x
);
4160 equiv
= simplify_gen_relational (NE
, cmp_mode
, mode
, y
, y
);
4161 equiv
= simplify_gen_ternary (IF_THEN_ELSE
, cmp_mode
, cmp_mode
,
4162 temp
, const_true_rtx
, equiv
);
4166 equiv
= simplify_gen_relational (comparison
, cmp_mode
, mode
, x
, y
);
4167 if (! FLOAT_LIB_COMPARE_RETURNS_BOOL (mode
, comparison
))
4168 equiv
= simplify_gen_ternary (IF_THEN_ELSE
, cmp_mode
, cmp_mode
,
4169 equiv
, true_rtx
, false_rtx
);
4173 value
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
,
4174 cmp_mode
, x
, mode
, y
, mode
);
4175 insns
= get_insns ();
4178 target
= gen_reg_rtx (cmp_mode
);
4179 emit_libcall_block (insns
, target
, value
, equiv
);
4181 if (comparison
== UNORDERED
4182 || FLOAT_LIB_COMPARE_RETURNS_BOOL (mode
, comparison
)
4184 *ptest
= gen_rtx_fmt_ee (reversed_p
? EQ
: NE
, VOIDmode
, target
, false_rtx
);
4186 *ptest
= gen_rtx_fmt_ee (comparison
, VOIDmode
, target
, const0_rtx
);
4191 /* Generate code to indirectly jump to a location given in the rtx LOC. */
4194 emit_indirect_jump (rtx loc
)
4196 if (!targetm
.have_indirect_jump ())
4197 sorry ("indirect jumps are not available on this target");
4200 struct expand_operand ops
[1];
4201 create_address_operand (&ops
[0], loc
);
4202 expand_jump_insn (targetm
.code_for_indirect_jump
, 1, ops
);
4208 /* Emit a conditional move instruction if the machine supports one for that
4209 condition and machine mode.
4211 OP0 and OP1 are the operands that should be compared using CODE. CMODE is
4212 the mode to use should they be constants. If it is VOIDmode, they cannot
4215 OP2 should be stored in TARGET if the comparison is true, otherwise OP3
4216 should be stored there. MODE is the mode to use should they be constants.
4217 If it is VOIDmode, they cannot both be constants.
4219 The result is either TARGET (perhaps modified) or NULL_RTX if the operation
4220 is not supported. */
4223 emit_conditional_move (rtx target
, enum rtx_code code
, rtx op0
, rtx op1
,
4224 machine_mode cmode
, rtx op2
, rtx op3
,
4225 machine_mode mode
, int unsignedp
)
4229 enum insn_code icode
;
4230 enum rtx_code reversed
;
4232 /* If the two source operands are identical, that's just a move. */
4234 if (rtx_equal_p (op2
, op3
))
4237 target
= gen_reg_rtx (mode
);
4239 emit_move_insn (target
, op3
);
4243 /* If one operand is constant, make it the second one. Only do this
4244 if the other operand is not constant as well. */
4246 if (swap_commutative_operands_p (op0
, op1
))
4248 std::swap (op0
, op1
);
4249 code
= swap_condition (code
);
4252 /* get_condition will prefer to generate LT and GT even if the old
4253 comparison was against zero, so undo that canonicalization here since
4254 comparisons against zero are cheaper. */
4255 if (code
== LT
&& op1
== const1_rtx
)
4256 code
= LE
, op1
= const0_rtx
;
4257 else if (code
== GT
&& op1
== constm1_rtx
)
4258 code
= GE
, op1
= const0_rtx
;
4260 if (cmode
== VOIDmode
)
4261 cmode
= GET_MODE (op0
);
4263 enum rtx_code orig_code
= code
;
4264 bool swapped
= false;
4265 if (swap_commutative_operands_p (op2
, op3
)
4266 && ((reversed
= reversed_comparison_code_parts (code
, op0
, op1
, NULL
))
4269 std::swap (op2
, op3
);
4274 if (mode
== VOIDmode
)
4275 mode
= GET_MODE (op2
);
4277 icode
= direct_optab_handler (movcc_optab
, mode
);
4279 if (icode
== CODE_FOR_nothing
)
4283 target
= gen_reg_rtx (mode
);
4285 for (int pass
= 0; ; pass
++)
4287 code
= unsignedp
? unsigned_condition (code
) : code
;
4288 comparison
= simplify_gen_relational (code
, VOIDmode
, cmode
, op0
, op1
);
4290 /* We can get const0_rtx or const_true_rtx in some circumstances. Just
4291 punt and let the caller figure out how best to deal with this
4293 if (COMPARISON_P (comparison
))
4295 saved_pending_stack_adjust save
;
4296 save_pending_stack_adjust (&save
);
4297 last
= get_last_insn ();
4298 do_pending_stack_adjust ();
4299 prepare_cmp_insn (XEXP (comparison
, 0), XEXP (comparison
, 1),
4300 GET_CODE (comparison
), NULL_RTX
, unsignedp
,
4301 OPTAB_WIDEN
, &comparison
, &cmode
);
4304 struct expand_operand ops
[4];
4306 create_output_operand (&ops
[0], target
, mode
);
4307 create_fixed_operand (&ops
[1], comparison
);
4308 create_input_operand (&ops
[2], op2
, mode
);
4309 create_input_operand (&ops
[3], op3
, mode
);
4310 if (maybe_expand_insn (icode
, 4, ops
))
4312 if (ops
[0].value
!= target
)
4313 convert_move (target
, ops
[0].value
, false);
4317 delete_insns_since (last
);
4318 restore_pending_stack_adjust (&save
);
4324 /* If the preferred op2/op3 order is not usable, retry with other
4325 operand order, perhaps it will expand successfully. */
4328 else if ((reversed
= reversed_comparison_code_parts (orig_code
, op0
, op1
,
4334 std::swap (op2
, op3
);
4339 /* Emit a conditional negate or bitwise complement using the
4340 negcc or notcc optabs if available. Return NULL_RTX if such operations
4341 are not available. Otherwise return the RTX holding the result.
4342 TARGET is the desired destination of the result. COMP is the comparison
4343 on which to negate. If COND is true move into TARGET the negation
4344 or bitwise complement of OP1. Otherwise move OP2 into TARGET.
4345 CODE is either NEG or NOT. MODE is the machine mode in which the
4346 operation is performed. */
4349 emit_conditional_neg_or_complement (rtx target
, rtx_code code
,
4350 machine_mode mode
, rtx cond
, rtx op1
,
4353 optab op
= unknown_optab
;
4356 else if (code
== NOT
)
4361 insn_code icode
= direct_optab_handler (op
, mode
);
4363 if (icode
== CODE_FOR_nothing
)
4367 target
= gen_reg_rtx (mode
);
4369 rtx_insn
*last
= get_last_insn ();
4370 struct expand_operand ops
[4];
4372 create_output_operand (&ops
[0], target
, mode
);
4373 create_fixed_operand (&ops
[1], cond
);
4374 create_input_operand (&ops
[2], op1
, mode
);
4375 create_input_operand (&ops
[3], op2
, mode
);
4377 if (maybe_expand_insn (icode
, 4, ops
))
4379 if (ops
[0].value
!= target
)
4380 convert_move (target
, ops
[0].value
, false);
4384 delete_insns_since (last
);
4388 /* Emit a conditional addition instruction if the machine supports one for that
4389 condition and machine mode.
4391 OP0 and OP1 are the operands that should be compared using CODE. CMODE is
4392 the mode to use should they be constants. If it is VOIDmode, they cannot
4395 OP2 should be stored in TARGET if the comparison is false, otherwise OP2+OP3
4396 should be stored there. MODE is the mode to use should they be constants.
4397 If it is VOIDmode, they cannot both be constants.
4399 The result is either TARGET (perhaps modified) or NULL_RTX if the operation
4400 is not supported. */
4403 emit_conditional_add (rtx target
, enum rtx_code code
, rtx op0
, rtx op1
,
4404 machine_mode cmode
, rtx op2
, rtx op3
,
4405 machine_mode mode
, int unsignedp
)
4409 enum insn_code icode
;
4411 /* If one operand is constant, make it the second one. Only do this
4412 if the other operand is not constant as well. */
4414 if (swap_commutative_operands_p (op0
, op1
))
4416 std::swap (op0
, op1
);
4417 code
= swap_condition (code
);
4420 /* get_condition will prefer to generate LT and GT even if the old
4421 comparison was against zero, so undo that canonicalization here since
4422 comparisons against zero are cheaper. */
4423 if (code
== LT
&& op1
== const1_rtx
)
4424 code
= LE
, op1
= const0_rtx
;
4425 else if (code
== GT
&& op1
== constm1_rtx
)
4426 code
= GE
, op1
= const0_rtx
;
4428 if (cmode
== VOIDmode
)
4429 cmode
= GET_MODE (op0
);
4431 if (mode
== VOIDmode
)
4432 mode
= GET_MODE (op2
);
4434 icode
= optab_handler (addcc_optab
, mode
);
4436 if (icode
== CODE_FOR_nothing
)
4440 target
= gen_reg_rtx (mode
);
4442 code
= unsignedp
? unsigned_condition (code
) : code
;
4443 comparison
= simplify_gen_relational (code
, VOIDmode
, cmode
, op0
, op1
);
4445 /* We can get const0_rtx or const_true_rtx in some circumstances. Just
4446 return NULL and let the caller figure out how best to deal with this
4448 if (!COMPARISON_P (comparison
))
4451 do_pending_stack_adjust ();
4452 last
= get_last_insn ();
4453 prepare_cmp_insn (XEXP (comparison
, 0), XEXP (comparison
, 1),
4454 GET_CODE (comparison
), NULL_RTX
, unsignedp
, OPTAB_WIDEN
,
4455 &comparison
, &cmode
);
4458 struct expand_operand ops
[4];
4460 create_output_operand (&ops
[0], target
, mode
);
4461 create_fixed_operand (&ops
[1], comparison
);
4462 create_input_operand (&ops
[2], op2
, mode
);
4463 create_input_operand (&ops
[3], op3
, mode
);
4464 if (maybe_expand_insn (icode
, 4, ops
))
4466 if (ops
[0].value
!= target
)
4467 convert_move (target
, ops
[0].value
, false);
4471 delete_insns_since (last
);
4475 /* These functions attempt to generate an insn body, rather than
4476 emitting the insn, but if the gen function already emits them, we
4477 make no attempt to turn them back into naked patterns. */
4479 /* Generate and return an insn body to add Y to X. */
4482 gen_add2_insn (rtx x
, rtx y
)
4484 enum insn_code icode
= optab_handler (add_optab
, GET_MODE (x
));
4486 gcc_assert (insn_operand_matches (icode
, 0, x
));
4487 gcc_assert (insn_operand_matches (icode
, 1, x
));
4488 gcc_assert (insn_operand_matches (icode
, 2, y
));
4490 return GEN_FCN (icode
) (x
, x
, y
);
4493 /* Generate and return an insn body to add r1 and c,
4494 storing the result in r0. */
4497 gen_add3_insn (rtx r0
, rtx r1
, rtx c
)
4499 enum insn_code icode
= optab_handler (add_optab
, GET_MODE (r0
));
4501 if (icode
== CODE_FOR_nothing
4502 || !insn_operand_matches (icode
, 0, r0
)
4503 || !insn_operand_matches (icode
, 1, r1
)
4504 || !insn_operand_matches (icode
, 2, c
))
4507 return GEN_FCN (icode
) (r0
, r1
, c
);
4511 have_add2_insn (rtx x
, rtx y
)
4513 enum insn_code icode
;
4515 gcc_assert (GET_MODE (x
) != VOIDmode
);
4517 icode
= optab_handler (add_optab
, GET_MODE (x
));
4519 if (icode
== CODE_FOR_nothing
)
4522 if (!insn_operand_matches (icode
, 0, x
)
4523 || !insn_operand_matches (icode
, 1, x
)
4524 || !insn_operand_matches (icode
, 2, y
))
4530 /* Generate and return an insn body to add Y to X. */
4533 gen_addptr3_insn (rtx x
, rtx y
, rtx z
)
4535 enum insn_code icode
= optab_handler (addptr3_optab
, GET_MODE (x
));
4537 gcc_assert (insn_operand_matches (icode
, 0, x
));
4538 gcc_assert (insn_operand_matches (icode
, 1, y
));
4539 gcc_assert (insn_operand_matches (icode
, 2, z
));
4541 return GEN_FCN (icode
) (x
, y
, z
);
4544 /* Return true if the target implements an addptr pattern and X, Y,
4545 and Z are valid for the pattern predicates. */
4548 have_addptr3_insn (rtx x
, rtx y
, rtx z
)
4550 enum insn_code icode
;
4552 gcc_assert (GET_MODE (x
) != VOIDmode
);
4554 icode
= optab_handler (addptr3_optab
, GET_MODE (x
));
4556 if (icode
== CODE_FOR_nothing
)
4559 if (!insn_operand_matches (icode
, 0, x
)
4560 || !insn_operand_matches (icode
, 1, y
)
4561 || !insn_operand_matches (icode
, 2, z
))
4567 /* Generate and return an insn body to subtract Y from X. */
4570 gen_sub2_insn (rtx x
, rtx y
)
4572 enum insn_code icode
= optab_handler (sub_optab
, GET_MODE (x
));
4574 gcc_assert (insn_operand_matches (icode
, 0, x
));
4575 gcc_assert (insn_operand_matches (icode
, 1, x
));
4576 gcc_assert (insn_operand_matches (icode
, 2, y
));
4578 return GEN_FCN (icode
) (x
, x
, y
);
4581 /* Generate and return an insn body to subtract r1 and c,
4582 storing the result in r0. */
4585 gen_sub3_insn (rtx r0
, rtx r1
, rtx c
)
4587 enum insn_code icode
= optab_handler (sub_optab
, GET_MODE (r0
));
4589 if (icode
== CODE_FOR_nothing
4590 || !insn_operand_matches (icode
, 0, r0
)
4591 || !insn_operand_matches (icode
, 1, r1
)
4592 || !insn_operand_matches (icode
, 2, c
))
4595 return GEN_FCN (icode
) (r0
, r1
, c
);
4599 have_sub2_insn (rtx x
, rtx y
)
4601 enum insn_code icode
;
4603 gcc_assert (GET_MODE (x
) != VOIDmode
);
4605 icode
= optab_handler (sub_optab
, GET_MODE (x
));
4607 if (icode
== CODE_FOR_nothing
)
4610 if (!insn_operand_matches (icode
, 0, x
)
4611 || !insn_operand_matches (icode
, 1, x
)
4612 || !insn_operand_matches (icode
, 2, y
))
4618 /* Generate the body of an insn to extend Y (with mode MFROM)
4619 into X (with mode MTO). Do zero-extension if UNSIGNEDP is nonzero. */
4622 gen_extend_insn (rtx x
, rtx y
, machine_mode mto
,
4623 machine_mode mfrom
, int unsignedp
)
4625 enum insn_code icode
= can_extend_p (mto
, mfrom
, unsignedp
);
4626 return GEN_FCN (icode
) (x
, y
);
4629 /* Generate code to convert FROM to floating point
4630 and store in TO. FROM must be fixed point and not VOIDmode.
4631 UNSIGNEDP nonzero means regard FROM as unsigned.
4632 Normally this is done by correcting the final value
4633 if it is negative. */
4636 expand_float (rtx to
, rtx from
, int unsignedp
)
4638 enum insn_code icode
;
4640 scalar_mode from_mode
, to_mode
;
4641 machine_mode fmode
, imode
;
4642 bool can_do_signed
= false;
4644 /* Crash now, because we won't be able to decide which mode to use. */
4645 gcc_assert (GET_MODE (from
) != VOIDmode
);
4647 /* Look for an insn to do the conversion. Do it in the specified
4648 modes if possible; otherwise convert either input, output or both to
4649 wider mode. If the integer mode is wider than the mode of FROM,
4650 we can do the conversion signed even if the input is unsigned. */
4652 FOR_EACH_MODE_FROM (fmode
, GET_MODE (to
))
4653 FOR_EACH_MODE_FROM (imode
, GET_MODE (from
))
4655 int doing_unsigned
= unsignedp
;
4657 if (fmode
!= GET_MODE (to
)
4658 && (significand_size (fmode
)
4659 < GET_MODE_UNIT_PRECISION (GET_MODE (from
))))
4662 icode
= can_float_p (fmode
, imode
, unsignedp
);
4663 if (icode
== CODE_FOR_nothing
&& unsignedp
)
4665 enum insn_code scode
= can_float_p (fmode
, imode
, 0);
4666 if (scode
!= CODE_FOR_nothing
)
4667 can_do_signed
= true;
4668 if (imode
!= GET_MODE (from
))
4669 icode
= scode
, doing_unsigned
= 0;
4672 if (icode
!= CODE_FOR_nothing
)
4674 if (imode
!= GET_MODE (from
))
4675 from
= convert_to_mode (imode
, from
, unsignedp
);
4677 if (fmode
!= GET_MODE (to
))
4678 target
= gen_reg_rtx (fmode
);
4680 emit_unop_insn (icode
, target
, from
,
4681 doing_unsigned
? UNSIGNED_FLOAT
: FLOAT
);
4684 convert_move (to
, target
, 0);
4689 /* Unsigned integer, and no way to convert directly. Convert as signed,
4690 then unconditionally adjust the result. */
4693 && is_a
<scalar_mode
> (GET_MODE (to
), &to_mode
)
4694 && is_a
<scalar_mode
> (GET_MODE (from
), &from_mode
))
4696 opt_scalar_mode fmode_iter
;
4697 rtx_code_label
*label
= gen_label_rtx ();
4699 REAL_VALUE_TYPE offset
;
4701 /* Look for a usable floating mode FMODE wider than the source and at
4702 least as wide as the target. Using FMODE will avoid rounding woes
4703 with unsigned values greater than the signed maximum value. */
4705 FOR_EACH_MODE_FROM (fmode_iter
, to_mode
)
4707 scalar_mode fmode
= fmode_iter
.require ();
4708 if (GET_MODE_PRECISION (from_mode
) < GET_MODE_BITSIZE (fmode
)
4709 && can_float_p (fmode
, from_mode
, 0) != CODE_FOR_nothing
)
4713 if (!fmode_iter
.exists (&fmode
))
4715 /* There is no such mode. Pretend the target is wide enough. */
4718 /* Avoid double-rounding when TO is narrower than FROM. */
4719 if ((significand_size (fmode
) + 1)
4720 < GET_MODE_PRECISION (from_mode
))
4723 rtx_code_label
*neglabel
= gen_label_rtx ();
4725 /* Don't use TARGET if it isn't a register, is a hard register,
4726 or is the wrong mode. */
4728 || REGNO (target
) < FIRST_PSEUDO_REGISTER
4729 || GET_MODE (target
) != fmode
)
4730 target
= gen_reg_rtx (fmode
);
4733 do_pending_stack_adjust ();
4735 /* Test whether the sign bit is set. */
4736 emit_cmp_and_jump_insns (from
, const0_rtx
, LT
, NULL_RTX
, imode
,
4739 /* The sign bit is not set. Convert as signed. */
4740 expand_float (target
, from
, 0);
4741 emit_jump_insn (targetm
.gen_jump (label
));
4744 /* The sign bit is set.
4745 Convert to a usable (positive signed) value by shifting right
4746 one bit, while remembering if a nonzero bit was shifted
4747 out; i.e., compute (from & 1) | (from >> 1). */
4749 emit_label (neglabel
);
4750 temp
= expand_binop (imode
, and_optab
, from
, const1_rtx
,
4751 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
4752 temp1
= expand_shift (RSHIFT_EXPR
, imode
, from
, 1, NULL_RTX
, 1);
4753 temp
= expand_binop (imode
, ior_optab
, temp
, temp1
, temp
, 1,
4755 expand_float (target
, temp
, 0);
4757 /* Multiply by 2 to undo the shift above. */
4758 temp
= expand_binop (fmode
, add_optab
, target
, target
,
4759 target
, 0, OPTAB_LIB_WIDEN
);
4761 emit_move_insn (target
, temp
);
4763 do_pending_stack_adjust ();
4769 /* If we are about to do some arithmetic to correct for an
4770 unsigned operand, do it in a pseudo-register. */
4772 if (to_mode
!= fmode
4773 || !REG_P (to
) || REGNO (to
) < FIRST_PSEUDO_REGISTER
)
4774 target
= gen_reg_rtx (fmode
);
4776 /* Convert as signed integer to floating. */
4777 expand_float (target
, from
, 0);
4779 /* If FROM is negative (and therefore TO is negative),
4780 correct its value by 2**bitwidth. */
4782 do_pending_stack_adjust ();
4783 emit_cmp_and_jump_insns (from
, const0_rtx
, GE
, NULL_RTX
, from_mode
,
4787 real_2expN (&offset
, GET_MODE_PRECISION (from_mode
), fmode
);
4788 temp
= expand_binop (fmode
, add_optab
, target
,
4789 const_double_from_real_value (offset
, fmode
),
4790 target
, 0, OPTAB_LIB_WIDEN
);
4792 emit_move_insn (target
, temp
);
4794 do_pending_stack_adjust ();
4799 /* No hardware instruction available; call a library routine. */
4804 convert_optab tab
= unsignedp
? ufloat_optab
: sfloat_optab
;
4806 if (GET_MODE_PRECISION (GET_MODE (from
)) < GET_MODE_PRECISION (SImode
))
4807 from
= convert_to_mode (SImode
, from
, unsignedp
);
4809 libfunc
= convert_optab_libfunc (tab
, GET_MODE (to
), GET_MODE (from
));
4810 gcc_assert (libfunc
);
4814 value
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
,
4815 GET_MODE (to
), from
, GET_MODE (from
));
4816 insns
= get_insns ();
4819 emit_libcall_block (insns
, target
, value
,
4820 gen_rtx_fmt_e (unsignedp
? UNSIGNED_FLOAT
: FLOAT
,
4821 GET_MODE (to
), from
));
4826 /* Copy result to requested destination
4827 if we have been computing in a temp location. */
4831 if (GET_MODE (target
) == GET_MODE (to
))
4832 emit_move_insn (to
, target
);
4834 convert_move (to
, target
, 0);
4838 /* Generate code to convert FROM to fixed point and store in TO. FROM
4839 must be floating point. */
4842 expand_fix (rtx to
, rtx from
, int unsignedp
)
4844 enum insn_code icode
;
4846 machine_mode fmode
, imode
;
4847 opt_scalar_mode fmode_iter
;
4848 bool must_trunc
= false;
4850 /* We first try to find a pair of modes, one real and one integer, at
4851 least as wide as FROM and TO, respectively, in which we can open-code
4852 this conversion. If the integer mode is wider than the mode of TO,
4853 we can do the conversion either signed or unsigned. */
4855 FOR_EACH_MODE_FROM (fmode
, GET_MODE (from
))
4856 FOR_EACH_MODE_FROM (imode
, GET_MODE (to
))
4858 int doing_unsigned
= unsignedp
;
4860 icode
= can_fix_p (imode
, fmode
, unsignedp
, &must_trunc
);
4861 if (icode
== CODE_FOR_nothing
&& imode
!= GET_MODE (to
) && unsignedp
)
4862 icode
= can_fix_p (imode
, fmode
, 0, &must_trunc
), doing_unsigned
= 0;
4864 if (icode
!= CODE_FOR_nothing
)
4866 rtx_insn
*last
= get_last_insn ();
4867 if (fmode
!= GET_MODE (from
))
4868 from
= convert_to_mode (fmode
, from
, 0);
4872 rtx temp
= gen_reg_rtx (GET_MODE (from
));
4873 from
= expand_unop (GET_MODE (from
), ftrunc_optab
, from
,
4877 if (imode
!= GET_MODE (to
))
4878 target
= gen_reg_rtx (imode
);
4880 if (maybe_emit_unop_insn (icode
, target
, from
,
4881 doing_unsigned
? UNSIGNED_FIX
: FIX
))
4884 convert_move (to
, target
, unsignedp
);
4887 delete_insns_since (last
);
4891 /* For an unsigned conversion, there is one more way to do it.
4892 If we have a signed conversion, we generate code that compares
4893 the real value to the largest representable positive number. If if
4894 is smaller, the conversion is done normally. Otherwise, subtract
4895 one plus the highest signed number, convert, and add it back.
4897 We only need to check all real modes, since we know we didn't find
4898 anything with a wider integer mode.
4900 This code used to extend FP value into mode wider than the destination.
4901 This is needed for decimal float modes which cannot accurately
4902 represent one plus the highest signed number of the same size, but
4903 not for binary modes. Consider, for instance conversion from SFmode
4906 The hot path through the code is dealing with inputs smaller than 2^63
4907 and doing just the conversion, so there is no bits to lose.
4909 In the other path we know the value is positive in the range 2^63..2^64-1
4910 inclusive. (as for other input overflow happens and result is undefined)
4911 So we know that the most important bit set in mantissa corresponds to
4912 2^63. The subtraction of 2^63 should not generate any rounding as it
4913 simply clears out that bit. The rest is trivial. */
4915 scalar_int_mode to_mode
;
4917 && is_a
<scalar_int_mode
> (GET_MODE (to
), &to_mode
)
4918 && HWI_COMPUTABLE_MODE_P (to_mode
))
4919 FOR_EACH_MODE_FROM (fmode_iter
, as_a
<scalar_mode
> (GET_MODE (from
)))
4921 scalar_mode fmode
= fmode_iter
.require ();
4922 if (CODE_FOR_nothing
!= can_fix_p (to_mode
, fmode
,
4924 && (!DECIMAL_FLOAT_MODE_P (fmode
)
4925 || (GET_MODE_BITSIZE (fmode
) > GET_MODE_PRECISION (to_mode
))))
4928 REAL_VALUE_TYPE offset
;
4930 rtx_code_label
*lab1
, *lab2
;
4933 bitsize
= GET_MODE_PRECISION (to_mode
);
4934 real_2expN (&offset
, bitsize
- 1, fmode
);
4935 limit
= const_double_from_real_value (offset
, fmode
);
4936 lab1
= gen_label_rtx ();
4937 lab2
= gen_label_rtx ();
4939 if (fmode
!= GET_MODE (from
))
4940 from
= convert_to_mode (fmode
, from
, 0);
4942 /* See if we need to do the subtraction. */
4943 do_pending_stack_adjust ();
4944 emit_cmp_and_jump_insns (from
, limit
, GE
, NULL_RTX
,
4945 GET_MODE (from
), 0, lab1
);
4947 /* If not, do the signed "fix" and branch around fixup code. */
4948 expand_fix (to
, from
, 0);
4949 emit_jump_insn (targetm
.gen_jump (lab2
));
4952 /* Otherwise, subtract 2**(N-1), convert to signed number,
4953 then add 2**(N-1). Do the addition using XOR since this
4954 will often generate better code. */
4956 target
= expand_binop (GET_MODE (from
), sub_optab
, from
, limit
,
4957 NULL_RTX
, 0, OPTAB_LIB_WIDEN
);
4958 expand_fix (to
, target
, 0);
4959 target
= expand_binop (to_mode
, xor_optab
, to
,
4961 (HOST_WIDE_INT_1
<< (bitsize
- 1),
4963 to
, 1, OPTAB_LIB_WIDEN
);
4966 emit_move_insn (to
, target
);
4970 if (optab_handler (mov_optab
, to_mode
) != CODE_FOR_nothing
)
4972 /* Make a place for a REG_NOTE and add it. */
4973 insn
= emit_move_insn (to
, to
);
4974 set_dst_reg_note (insn
, REG_EQUAL
,
4975 gen_rtx_fmt_e (UNSIGNED_FIX
, to_mode
,
4984 /* We can't do it with an insn, so use a library call. But first ensure
4985 that the mode of TO is at least as wide as SImode, since those are the
4986 only library calls we know about. */
4988 if (GET_MODE_PRECISION (GET_MODE (to
)) < GET_MODE_PRECISION (SImode
))
4990 target
= gen_reg_rtx (SImode
);
4992 expand_fix (target
, from
, unsignedp
);
5000 convert_optab tab
= unsignedp
? ufix_optab
: sfix_optab
;
5001 libfunc
= convert_optab_libfunc (tab
, GET_MODE (to
), GET_MODE (from
));
5002 gcc_assert (libfunc
);
5006 value
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
,
5007 GET_MODE (to
), from
, GET_MODE (from
));
5008 insns
= get_insns ();
5011 emit_libcall_block (insns
, target
, value
,
5012 gen_rtx_fmt_e (unsignedp
? UNSIGNED_FIX
: FIX
,
5013 GET_MODE (to
), from
));
5018 if (GET_MODE (to
) == GET_MODE (target
))
5019 emit_move_insn (to
, target
);
5021 convert_move (to
, target
, 0);
5026 /* Promote integer arguments for a libcall if necessary.
5027 emit_library_call_value cannot do the promotion because it does not
5028 know if it should do a signed or unsigned promotion. This is because
5029 there are no tree types defined for libcalls. */
5032 prepare_libcall_arg (rtx arg
, int uintp
)
5034 scalar_int_mode mode
;
5035 machine_mode arg_mode
;
5036 if (is_a
<scalar_int_mode
> (GET_MODE (arg
), &mode
))
5038 /* If we need to promote the integer function argument we need to do
5039 it here instead of inside emit_library_call_value because in
5040 emit_library_call_value we don't know if we should do a signed or
5041 unsigned promotion. */
5044 arg_mode
= promote_function_mode (NULL_TREE
, mode
,
5045 &unsigned_p
, NULL_TREE
, 0);
5046 if (arg_mode
!= mode
)
5047 return convert_to_mode (arg_mode
, arg
, uintp
);
5052 /* Generate code to convert FROM or TO a fixed-point.
5053 If UINTP is true, either TO or FROM is an unsigned integer.
5054 If SATP is true, we need to saturate the result. */
5057 expand_fixed_convert (rtx to
, rtx from
, int uintp
, int satp
)
5059 machine_mode to_mode
= GET_MODE (to
);
5060 machine_mode from_mode
= GET_MODE (from
);
5062 enum rtx_code this_code
;
5063 enum insn_code code
;
5068 if (to_mode
== from_mode
)
5070 emit_move_insn (to
, from
);
5076 tab
= satp
? satfractuns_optab
: fractuns_optab
;
5077 this_code
= satp
? UNSIGNED_SAT_FRACT
: UNSIGNED_FRACT_CONVERT
;
5081 tab
= satp
? satfract_optab
: fract_optab
;
5082 this_code
= satp
? SAT_FRACT
: FRACT_CONVERT
;
5084 code
= convert_optab_handler (tab
, to_mode
, from_mode
);
5085 if (code
!= CODE_FOR_nothing
)
5087 emit_unop_insn (code
, to
, from
, this_code
);
5091 libfunc
= convert_optab_libfunc (tab
, to_mode
, from_mode
);
5092 gcc_assert (libfunc
);
5094 from
= prepare_libcall_arg (from
, uintp
);
5095 from_mode
= GET_MODE (from
);
5098 value
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
, to_mode
,
5100 insns
= get_insns ();
5103 emit_libcall_block (insns
, to
, value
,
5104 gen_rtx_fmt_e (optab_to_code (tab
), to_mode
, from
));
5107 /* Generate code to convert FROM to fixed point and store in TO. FROM
5108 must be floating point, TO must be signed. Use the conversion optab
5109 TAB to do the conversion. */
5112 expand_sfix_optab (rtx to
, rtx from
, convert_optab tab
)
5114 enum insn_code icode
;
5116 machine_mode fmode
, imode
;
5118 /* We first try to find a pair of modes, one real and one integer, at
5119 least as wide as FROM and TO, respectively, in which we can open-code
5120 this conversion. If the integer mode is wider than the mode of TO,
5121 we can do the conversion either signed or unsigned. */
5123 FOR_EACH_MODE_FROM (fmode
, GET_MODE (from
))
5124 FOR_EACH_MODE_FROM (imode
, GET_MODE (to
))
5126 icode
= convert_optab_handler (tab
, imode
, fmode
);
5127 if (icode
!= CODE_FOR_nothing
)
5129 rtx_insn
*last
= get_last_insn ();
5130 if (fmode
!= GET_MODE (from
))
5131 from
= convert_to_mode (fmode
, from
, 0);
5133 if (imode
!= GET_MODE (to
))
5134 target
= gen_reg_rtx (imode
);
5136 if (!maybe_emit_unop_insn (icode
, target
, from
, UNKNOWN
))
5138 delete_insns_since (last
);
5142 convert_move (to
, target
, 0);
5150 /* Report whether we have an instruction to perform the operation
5151 specified by CODE on operands of mode MODE. */
5153 have_insn_for (enum rtx_code code
, machine_mode mode
)
5155 return (code_to_optab (code
)
5156 && (optab_handler (code_to_optab (code
), mode
)
5157 != CODE_FOR_nothing
));
5160 /* Print information about the current contents of the optabs on
5164 debug_optab_libfuncs (void)
5168 /* Dump the arithmetic optabs. */
5169 for (i
= FIRST_NORM_OPTAB
; i
<= LAST_NORMLIB_OPTAB
; ++i
)
5170 for (j
= 0; j
< NUM_MACHINE_MODES
; ++j
)
5172 rtx l
= optab_libfunc ((optab
) i
, (machine_mode
) j
);
5175 gcc_assert (GET_CODE (l
) == SYMBOL_REF
);
5176 fprintf (stderr
, "%s\t%s:\t%s\n",
5177 GET_RTX_NAME (optab_to_code ((optab
) i
)),
5183 /* Dump the conversion optabs. */
5184 for (i
= FIRST_CONV_OPTAB
; i
<= LAST_CONVLIB_OPTAB
; ++i
)
5185 for (j
= 0; j
< NUM_MACHINE_MODES
; ++j
)
5186 for (k
= 0; k
< NUM_MACHINE_MODES
; ++k
)
5188 rtx l
= convert_optab_libfunc ((optab
) i
, (machine_mode
) j
,
5192 gcc_assert (GET_CODE (l
) == SYMBOL_REF
);
5193 fprintf (stderr
, "%s\t%s\t%s:\t%s\n",
5194 GET_RTX_NAME (optab_to_code ((optab
) i
)),
5202 /* Generate insns to trap with code TCODE if OP1 and OP2 satisfy condition
5203 CODE. Return 0 on failure. */
5206 gen_cond_trap (enum rtx_code code
, rtx op1
, rtx op2
, rtx tcode
)
5208 machine_mode mode
= GET_MODE (op1
);
5209 enum insn_code icode
;
5213 if (mode
== VOIDmode
)
5216 icode
= optab_handler (ctrap_optab
, mode
);
5217 if (icode
== CODE_FOR_nothing
)
5220 /* Some targets only accept a zero trap code. */
5221 if (!insn_operand_matches (icode
, 3, tcode
))
5224 do_pending_stack_adjust ();
5226 prepare_cmp_insn (op1
, op2
, code
, NULL_RTX
, false, OPTAB_DIRECT
,
5231 insn
= GEN_FCN (icode
) (trap_rtx
, XEXP (trap_rtx
, 0), XEXP (trap_rtx
, 1),
5234 /* If that failed, then give up. */
5242 insn
= get_insns ();
5247 /* Return rtx code for TCODE. Use UNSIGNEDP to select signed
5248 or unsigned operation code. */
5251 get_rtx_code (enum tree_code tcode
, bool unsignedp
)
5263 code
= unsignedp
? LTU
: LT
;
5266 code
= unsignedp
? LEU
: LE
;
5269 code
= unsignedp
? GTU
: GT
;
5272 code
= unsignedp
? GEU
: GE
;
5275 case UNORDERED_EXPR
:
5314 /* Return a comparison rtx of mode CMP_MODE for COND. Use UNSIGNEDP to
5315 select signed or unsigned operators. OPNO holds the index of the
5316 first comparison operand for insn ICODE. Do not generate the
5317 compare instruction itself. */
5320 vector_compare_rtx (machine_mode cmp_mode
, enum tree_code tcode
,
5321 tree t_op0
, tree t_op1
, bool unsignedp
,
5322 enum insn_code icode
, unsigned int opno
)
5324 struct expand_operand ops
[2];
5325 rtx rtx_op0
, rtx_op1
;
5326 machine_mode m0
, m1
;
5327 enum rtx_code rcode
= get_rtx_code (tcode
, unsignedp
);
5329 gcc_assert (TREE_CODE_CLASS (tcode
) == tcc_comparison
);
5331 /* Expand operands. For vector types with scalar modes, e.g. where int64x1_t
5332 has mode DImode, this can produce a constant RTX of mode VOIDmode; in such
5333 cases, use the original mode. */
5334 rtx_op0
= expand_expr (t_op0
, NULL_RTX
, TYPE_MODE (TREE_TYPE (t_op0
)),
5336 m0
= GET_MODE (rtx_op0
);
5338 m0
= TYPE_MODE (TREE_TYPE (t_op0
));
5340 rtx_op1
= expand_expr (t_op1
, NULL_RTX
, TYPE_MODE (TREE_TYPE (t_op1
)),
5342 m1
= GET_MODE (rtx_op1
);
5344 m1
= TYPE_MODE (TREE_TYPE (t_op1
));
5346 create_input_operand (&ops
[0], rtx_op0
, m0
);
5347 create_input_operand (&ops
[1], rtx_op1
, m1
);
5348 if (!maybe_legitimize_operands (icode
, opno
, 2, ops
))
5350 return gen_rtx_fmt_ee (rcode
, cmp_mode
, ops
[0].value
, ops
[1].value
);
5353 /* Checks if vec_perm mask SEL is a constant equivalent to a shift of the first
5354 vec_perm operand, assuming the second operand is a constant vector of zeroes.
5355 Return the shift distance in bits if so, or NULL_RTX if the vec_perm is not a
5358 shift_amt_for_vec_perm_mask (rtx sel
)
5360 unsigned int i
, first
, nelt
= GET_MODE_NUNITS (GET_MODE (sel
));
5361 unsigned int bitsize
= GET_MODE_UNIT_BITSIZE (GET_MODE (sel
));
5363 if (GET_CODE (sel
) != CONST_VECTOR
)
5366 first
= INTVAL (CONST_VECTOR_ELT (sel
, 0));
5369 for (i
= 1; i
< nelt
; i
++)
5371 int idx
= INTVAL (CONST_VECTOR_ELT (sel
, i
));
5372 unsigned int expected
= i
+ first
;
5373 /* Indices into the second vector are all equivalent. */
5374 if (idx
< 0 || (MIN (nelt
, (unsigned) idx
) != MIN (nelt
, expected
)))
5378 return GEN_INT (first
* bitsize
);
5381 /* A subroutine of expand_vec_perm for expanding one vec_perm insn. */
5384 expand_vec_perm_1 (enum insn_code icode
, rtx target
,
5385 rtx v0
, rtx v1
, rtx sel
)
5387 machine_mode tmode
= GET_MODE (target
);
5388 machine_mode smode
= GET_MODE (sel
);
5389 struct expand_operand ops
[4];
5391 create_output_operand (&ops
[0], target
, tmode
);
5392 create_input_operand (&ops
[3], sel
, smode
);
5394 /* Make an effort to preserve v0 == v1. The target expander is able to
5395 rely on this to determine if we're permuting a single input operand. */
5396 if (rtx_equal_p (v0
, v1
))
5398 if (!insn_operand_matches (icode
, 1, v0
))
5399 v0
= force_reg (tmode
, v0
);
5400 gcc_checking_assert (insn_operand_matches (icode
, 1, v0
));
5401 gcc_checking_assert (insn_operand_matches (icode
, 2, v0
));
5403 create_fixed_operand (&ops
[1], v0
);
5404 create_fixed_operand (&ops
[2], v0
);
5408 create_input_operand (&ops
[1], v0
, tmode
);
5409 create_input_operand (&ops
[2], v1
, tmode
);
5412 if (maybe_expand_insn (icode
, 4, ops
))
5413 return ops
[0].value
;
5417 /* Generate instructions for vec_perm optab given its mode
5418 and three operands. */
5421 expand_vec_perm (machine_mode mode
, rtx v0
, rtx v1
, rtx sel
, rtx target
)
5423 enum insn_code icode
;
5424 machine_mode qimode
;
5425 unsigned int i
, w
, e
, u
;
5426 rtx tmp
, sel_qi
= NULL
;
5429 if (!target
|| GET_MODE (target
) != mode
)
5430 target
= gen_reg_rtx (mode
);
5432 w
= GET_MODE_SIZE (mode
);
5433 e
= GET_MODE_NUNITS (mode
);
5434 u
= GET_MODE_UNIT_SIZE (mode
);
5436 /* Set QIMODE to a different vector mode with byte elements.
5437 If no such mode, or if MODE already has byte elements, use VOIDmode. */
5438 if (GET_MODE_INNER (mode
) == QImode
5439 || !mode_for_vector (QImode
, w
).exists (&qimode
)
5440 || !VECTOR_MODE_P (qimode
))
5443 /* If the input is a constant, expand it specially. */
5444 gcc_assert (GET_MODE_CLASS (GET_MODE (sel
)) == MODE_VECTOR_INT
);
5445 if (GET_CODE (sel
) == CONST_VECTOR
)
5447 /* See if this can be handled with a vec_shr. We only do this if the
5448 second vector is all zeroes. */
5449 enum insn_code shift_code
= optab_handler (vec_shr_optab
, mode
);
5450 enum insn_code shift_code_qi
= ((qimode
!= VOIDmode
&& qimode
!= mode
)
5451 ? optab_handler (vec_shr_optab
, qimode
)
5452 : CODE_FOR_nothing
);
5453 rtx shift_amt
= NULL_RTX
;
5454 if (v1
== CONST0_RTX (GET_MODE (v1
))
5455 && (shift_code
!= CODE_FOR_nothing
5456 || shift_code_qi
!= CODE_FOR_nothing
))
5458 shift_amt
= shift_amt_for_vec_perm_mask (sel
);
5461 struct expand_operand ops
[3];
5462 if (shift_code
!= CODE_FOR_nothing
)
5464 create_output_operand (&ops
[0], target
, mode
);
5465 create_input_operand (&ops
[1], v0
, mode
);
5466 create_convert_operand_from_type (&ops
[2], shift_amt
,
5468 if (maybe_expand_insn (shift_code
, 3, ops
))
5469 return ops
[0].value
;
5471 if (shift_code_qi
!= CODE_FOR_nothing
)
5473 tmp
= gen_reg_rtx (qimode
);
5474 create_output_operand (&ops
[0], tmp
, qimode
);
5475 create_input_operand (&ops
[1], gen_lowpart (qimode
, v0
),
5477 create_convert_operand_from_type (&ops
[2], shift_amt
,
5479 if (maybe_expand_insn (shift_code_qi
, 3, ops
))
5480 return gen_lowpart (mode
, ops
[0].value
);
5485 icode
= direct_optab_handler (vec_perm_const_optab
, mode
);
5486 if (icode
!= CODE_FOR_nothing
)
5488 tmp
= expand_vec_perm_1 (icode
, target
, v0
, v1
, sel
);
5493 /* Fall back to a constant byte-based permutation. */
5494 if (qimode
!= VOIDmode
)
5496 vec
= rtvec_alloc (w
);
5497 for (i
= 0; i
< e
; ++i
)
5499 unsigned int j
, this_e
;
5501 this_e
= INTVAL (CONST_VECTOR_ELT (sel
, i
));
5502 this_e
&= 2 * e
- 1;
5505 for (j
= 0; j
< u
; ++j
)
5506 RTVEC_ELT (vec
, i
* u
+ j
) = GEN_INT (this_e
+ j
);
5508 sel_qi
= gen_rtx_CONST_VECTOR (qimode
, vec
);
5510 icode
= direct_optab_handler (vec_perm_const_optab
, qimode
);
5511 if (icode
!= CODE_FOR_nothing
)
5513 tmp
= mode
!= qimode
? gen_reg_rtx (qimode
) : target
;
5514 tmp
= expand_vec_perm_1 (icode
, tmp
, gen_lowpart (qimode
, v0
),
5515 gen_lowpart (qimode
, v1
), sel_qi
);
5517 return gen_lowpart (mode
, tmp
);
5522 /* Otherwise expand as a fully variable permuation. */
5523 icode
= direct_optab_handler (vec_perm_optab
, mode
);
5524 if (icode
!= CODE_FOR_nothing
)
5526 tmp
= expand_vec_perm_1 (icode
, target
, v0
, v1
, sel
);
5531 /* As a special case to aid several targets, lower the element-based
5532 permutation to a byte-based permutation and try again. */
5533 if (qimode
== VOIDmode
)
5535 icode
= direct_optab_handler (vec_perm_optab
, qimode
);
5536 if (icode
== CODE_FOR_nothing
)
5541 /* Multiply each element by its byte size. */
5542 machine_mode selmode
= GET_MODE (sel
);
5544 sel
= expand_simple_binop (selmode
, PLUS
, sel
, sel
,
5545 NULL
, 0, OPTAB_DIRECT
);
5547 sel
= expand_simple_binop (selmode
, ASHIFT
, sel
,
5548 GEN_INT (exact_log2 (u
)),
5549 NULL
, 0, OPTAB_DIRECT
);
5550 gcc_assert (sel
!= NULL
);
5552 /* Broadcast the low byte each element into each of its bytes. */
5553 vec
= rtvec_alloc (w
);
5554 for (i
= 0; i
< w
; ++i
)
5556 int this_e
= i
/ u
* u
;
5557 if (BYTES_BIG_ENDIAN
)
5559 RTVEC_ELT (vec
, i
) = GEN_INT (this_e
);
5561 tmp
= gen_rtx_CONST_VECTOR (qimode
, vec
);
5562 sel
= gen_lowpart (qimode
, sel
);
5563 sel
= expand_vec_perm (qimode
, sel
, sel
, tmp
, NULL
);
5564 gcc_assert (sel
!= NULL
);
5566 /* Add the byte offset to each byte element. */
5567 /* Note that the definition of the indicies here is memory ordering,
5568 so there should be no difference between big and little endian. */
5569 vec
= rtvec_alloc (w
);
5570 for (i
= 0; i
< w
; ++i
)
5571 RTVEC_ELT (vec
, i
) = GEN_INT (i
% u
);
5572 tmp
= gen_rtx_CONST_VECTOR (qimode
, vec
);
5573 sel_qi
= expand_simple_binop (qimode
, PLUS
, sel
, tmp
,
5574 sel
, 0, OPTAB_DIRECT
);
5575 gcc_assert (sel_qi
!= NULL
);
5578 tmp
= mode
!= qimode
? gen_reg_rtx (qimode
) : target
;
5579 tmp
= expand_vec_perm_1 (icode
, tmp
, gen_lowpart (qimode
, v0
),
5580 gen_lowpart (qimode
, v1
), sel_qi
);
5582 tmp
= gen_lowpart (mode
, tmp
);
5586 /* Generate insns for a VEC_COND_EXPR with mask, given its TYPE and its
5590 expand_vec_cond_mask_expr (tree vec_cond_type
, tree op0
, tree op1
, tree op2
,
5593 struct expand_operand ops
[4];
5594 machine_mode mode
= TYPE_MODE (vec_cond_type
);
5595 machine_mode mask_mode
= TYPE_MODE (TREE_TYPE (op0
));
5596 enum insn_code icode
= get_vcond_mask_icode (mode
, mask_mode
);
5597 rtx mask
, rtx_op1
, rtx_op2
;
5599 if (icode
== CODE_FOR_nothing
)
5602 mask
= expand_normal (op0
);
5603 rtx_op1
= expand_normal (op1
);
5604 rtx_op2
= expand_normal (op2
);
5606 mask
= force_reg (mask_mode
, mask
);
5607 rtx_op1
= force_reg (GET_MODE (rtx_op1
), rtx_op1
);
5609 create_output_operand (&ops
[0], target
, mode
);
5610 create_input_operand (&ops
[1], rtx_op1
, mode
);
5611 create_input_operand (&ops
[2], rtx_op2
, mode
);
5612 create_input_operand (&ops
[3], mask
, mask_mode
);
5613 expand_insn (icode
, 4, ops
);
5615 return ops
[0].value
;
5618 /* Generate insns for a VEC_COND_EXPR, given its TYPE and its
5622 expand_vec_cond_expr (tree vec_cond_type
, tree op0
, tree op1
, tree op2
,
5625 struct expand_operand ops
[6];
5626 enum insn_code icode
;
5627 rtx comparison
, rtx_op1
, rtx_op2
;
5628 machine_mode mode
= TYPE_MODE (vec_cond_type
);
5629 machine_mode cmp_op_mode
;
5632 enum tree_code tcode
;
5634 if (COMPARISON_CLASS_P (op0
))
5636 op0a
= TREE_OPERAND (op0
, 0);
5637 op0b
= TREE_OPERAND (op0
, 1);
5638 tcode
= TREE_CODE (op0
);
5642 gcc_assert (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (op0
)));
5643 if (get_vcond_mask_icode (mode
, TYPE_MODE (TREE_TYPE (op0
)))
5644 != CODE_FOR_nothing
)
5645 return expand_vec_cond_mask_expr (vec_cond_type
, op0
, op1
,
5650 gcc_assert (GET_MODE_CLASS (TYPE_MODE (TREE_TYPE (op0
)))
5651 == MODE_VECTOR_INT
);
5653 op0b
= build_zero_cst (TREE_TYPE (op0
));
5657 cmp_op_mode
= TYPE_MODE (TREE_TYPE (op0a
));
5658 unsignedp
= TYPE_UNSIGNED (TREE_TYPE (op0a
));
5661 gcc_assert (GET_MODE_SIZE (mode
) == GET_MODE_SIZE (cmp_op_mode
)
5662 && GET_MODE_NUNITS (mode
) == GET_MODE_NUNITS (cmp_op_mode
));
5664 icode
= get_vcond_icode (mode
, cmp_op_mode
, unsignedp
);
5665 if (icode
== CODE_FOR_nothing
)
5667 if (tcode
== EQ_EXPR
|| tcode
== NE_EXPR
)
5668 icode
= get_vcond_eq_icode (mode
, cmp_op_mode
);
5669 if (icode
== CODE_FOR_nothing
)
5673 comparison
= vector_compare_rtx (VOIDmode
, tcode
, op0a
, op0b
, unsignedp
,
5675 rtx_op1
= expand_normal (op1
);
5676 rtx_op2
= expand_normal (op2
);
5678 create_output_operand (&ops
[0], target
, mode
);
5679 create_input_operand (&ops
[1], rtx_op1
, mode
);
5680 create_input_operand (&ops
[2], rtx_op2
, mode
);
5681 create_fixed_operand (&ops
[3], comparison
);
5682 create_fixed_operand (&ops
[4], XEXP (comparison
, 0));
5683 create_fixed_operand (&ops
[5], XEXP (comparison
, 1));
5684 expand_insn (icode
, 6, ops
);
5685 return ops
[0].value
;
5688 /* Generate insns for a vector comparison into a mask. */
5691 expand_vec_cmp_expr (tree type
, tree exp
, rtx target
)
5693 struct expand_operand ops
[4];
5694 enum insn_code icode
;
5696 machine_mode mask_mode
= TYPE_MODE (type
);
5700 enum tree_code tcode
;
5702 op0a
= TREE_OPERAND (exp
, 0);
5703 op0b
= TREE_OPERAND (exp
, 1);
5704 tcode
= TREE_CODE (exp
);
5706 unsignedp
= TYPE_UNSIGNED (TREE_TYPE (op0a
));
5707 vmode
= TYPE_MODE (TREE_TYPE (op0a
));
5709 icode
= get_vec_cmp_icode (vmode
, mask_mode
, unsignedp
);
5710 if (icode
== CODE_FOR_nothing
)
5712 if (tcode
== EQ_EXPR
|| tcode
== NE_EXPR
)
5713 icode
= get_vec_cmp_eq_icode (vmode
, mask_mode
);
5714 if (icode
== CODE_FOR_nothing
)
5718 comparison
= vector_compare_rtx (mask_mode
, tcode
, op0a
, op0b
,
5719 unsignedp
, icode
, 2);
5720 create_output_operand (&ops
[0], target
, mask_mode
);
5721 create_fixed_operand (&ops
[1], comparison
);
5722 create_fixed_operand (&ops
[2], XEXP (comparison
, 0));
5723 create_fixed_operand (&ops
[3], XEXP (comparison
, 1));
5724 expand_insn (icode
, 4, ops
);
5725 return ops
[0].value
;
5728 /* Expand a highpart multiply. */
5731 expand_mult_highpart (machine_mode mode
, rtx op0
, rtx op1
,
5732 rtx target
, bool uns_p
)
5734 struct expand_operand eops
[3];
5735 enum insn_code icode
;
5736 int method
, i
, nunits
;
5742 method
= can_mult_highpart_p (mode
, uns_p
);
5748 tab1
= uns_p
? umul_highpart_optab
: smul_highpart_optab
;
5749 return expand_binop (mode
, tab1
, op0
, op1
, target
, uns_p
,
5752 tab1
= uns_p
? vec_widen_umult_even_optab
: vec_widen_smult_even_optab
;
5753 tab2
= uns_p
? vec_widen_umult_odd_optab
: vec_widen_smult_odd_optab
;
5756 tab1
= uns_p
? vec_widen_umult_lo_optab
: vec_widen_smult_lo_optab
;
5757 tab2
= uns_p
? vec_widen_umult_hi_optab
: vec_widen_smult_hi_optab
;
5758 if (BYTES_BIG_ENDIAN
)
5759 std::swap (tab1
, tab2
);
5765 icode
= optab_handler (tab1
, mode
);
5766 nunits
= GET_MODE_NUNITS (mode
);
5767 wmode
= insn_data
[icode
].operand
[0].mode
;
5768 gcc_checking_assert (2 * GET_MODE_NUNITS (wmode
) == nunits
);
5769 gcc_checking_assert (GET_MODE_SIZE (wmode
) == GET_MODE_SIZE (mode
));
5771 create_output_operand (&eops
[0], gen_reg_rtx (wmode
), wmode
);
5772 create_input_operand (&eops
[1], op0
, mode
);
5773 create_input_operand (&eops
[2], op1
, mode
);
5774 expand_insn (icode
, 3, eops
);
5775 m1
= gen_lowpart (mode
, eops
[0].value
);
5777 create_output_operand (&eops
[0], gen_reg_rtx (wmode
), wmode
);
5778 create_input_operand (&eops
[1], op0
, mode
);
5779 create_input_operand (&eops
[2], op1
, mode
);
5780 expand_insn (optab_handler (tab2
, mode
), 3, eops
);
5781 m2
= gen_lowpart (mode
, eops
[0].value
);
5783 v
= rtvec_alloc (nunits
);
5786 for (i
= 0; i
< nunits
; ++i
)
5787 RTVEC_ELT (v
, i
) = GEN_INT (!BYTES_BIG_ENDIAN
+ (i
& ~1)
5788 + ((i
& 1) ? nunits
: 0));
5792 for (i
= 0; i
< nunits
; ++i
)
5793 RTVEC_ELT (v
, i
) = GEN_INT (2 * i
+ (BYTES_BIG_ENDIAN
? 0 : 1));
5795 perm
= gen_rtx_CONST_VECTOR (mode
, v
);
5797 return expand_vec_perm (mode
, m1
, m2
, perm
, target
);
5800 /* Helper function to find the MODE_CC set in a sync_compare_and_swap
5804 find_cc_set (rtx x
, const_rtx pat
, void *data
)
5806 if (REG_P (x
) && GET_MODE_CLASS (GET_MODE (x
)) == MODE_CC
5807 && GET_CODE (pat
) == SET
)
5809 rtx
*p_cc_reg
= (rtx
*) data
;
5810 gcc_assert (!*p_cc_reg
);
5815 /* This is a helper function for the other atomic operations. This function
5816 emits a loop that contains SEQ that iterates until a compare-and-swap
5817 operation at the end succeeds. MEM is the memory to be modified. SEQ is
5818 a set of instructions that takes a value from OLD_REG as an input and
5819 produces a value in NEW_REG as an output. Before SEQ, OLD_REG will be
5820 set to the current contents of MEM. After SEQ, a compare-and-swap will
5821 attempt to update MEM with NEW_REG. The function returns true when the
5822 loop was generated successfully. */
5825 expand_compare_and_swap_loop (rtx mem
, rtx old_reg
, rtx new_reg
, rtx seq
)
5827 machine_mode mode
= GET_MODE (mem
);
5828 rtx_code_label
*label
;
5829 rtx cmp_reg
, success
, oldval
;
5831 /* The loop we want to generate looks like
5837 (success, cmp_reg) = compare-and-swap(mem, old_reg, new_reg)
5841 Note that we only do the plain load from memory once. Subsequent
5842 iterations use the value loaded by the compare-and-swap pattern. */
5844 label
= gen_label_rtx ();
5845 cmp_reg
= gen_reg_rtx (mode
);
5847 emit_move_insn (cmp_reg
, mem
);
5849 emit_move_insn (old_reg
, cmp_reg
);
5855 if (!expand_atomic_compare_and_swap (&success
, &oldval
, mem
, old_reg
,
5856 new_reg
, false, MEMMODEL_SYNC_SEQ_CST
,
5860 if (oldval
!= cmp_reg
)
5861 emit_move_insn (cmp_reg
, oldval
);
5863 /* Mark this jump predicted not taken. */
5864 emit_cmp_and_jump_insns (success
, const0_rtx
, EQ
, const0_rtx
,
5865 GET_MODE (success
), 1, label
,
5866 profile_probability::guessed_never ());
5871 /* This function tries to emit an atomic_exchange intruction. VAL is written
5872 to *MEM using memory model MODEL. The previous contents of *MEM are returned,
5873 using TARGET if possible. */
5876 maybe_emit_atomic_exchange (rtx target
, rtx mem
, rtx val
, enum memmodel model
)
5878 machine_mode mode
= GET_MODE (mem
);
5879 enum insn_code icode
;
5881 /* If the target supports the exchange directly, great. */
5882 icode
= direct_optab_handler (atomic_exchange_optab
, mode
);
5883 if (icode
!= CODE_FOR_nothing
)
5885 struct expand_operand ops
[4];
5887 create_output_operand (&ops
[0], target
, mode
);
5888 create_fixed_operand (&ops
[1], mem
);
5889 create_input_operand (&ops
[2], val
, mode
);
5890 create_integer_operand (&ops
[3], model
);
5891 if (maybe_expand_insn (icode
, 4, ops
))
5892 return ops
[0].value
;
5898 /* This function tries to implement an atomic exchange operation using
5899 __sync_lock_test_and_set. VAL is written to *MEM using memory model MODEL.
5900 The previous contents of *MEM are returned, using TARGET if possible.
5901 Since this instructionn is an acquire barrier only, stronger memory
5902 models may require additional barriers to be emitted. */
5905 maybe_emit_sync_lock_test_and_set (rtx target
, rtx mem
, rtx val
,
5906 enum memmodel model
)
5908 machine_mode mode
= GET_MODE (mem
);
5909 enum insn_code icode
;
5910 rtx_insn
*last_insn
= get_last_insn ();
5912 icode
= optab_handler (sync_lock_test_and_set_optab
, mode
);
5914 /* Legacy sync_lock_test_and_set is an acquire barrier. If the pattern
5915 exists, and the memory model is stronger than acquire, add a release
5916 barrier before the instruction. */
5918 if (is_mm_seq_cst (model
) || is_mm_release (model
) || is_mm_acq_rel (model
))
5919 expand_mem_thread_fence (model
);
5921 if (icode
!= CODE_FOR_nothing
)
5923 struct expand_operand ops
[3];
5924 create_output_operand (&ops
[0], target
, mode
);
5925 create_fixed_operand (&ops
[1], mem
);
5926 create_input_operand (&ops
[2], val
, mode
);
5927 if (maybe_expand_insn (icode
, 3, ops
))
5928 return ops
[0].value
;
5931 /* If an external test-and-set libcall is provided, use that instead of
5932 any external compare-and-swap that we might get from the compare-and-
5933 swap-loop expansion later. */
5934 if (!can_compare_and_swap_p (mode
, false))
5936 rtx libfunc
= optab_libfunc (sync_lock_test_and_set_optab
, mode
);
5937 if (libfunc
!= NULL
)
5941 addr
= convert_memory_address (ptr_mode
, XEXP (mem
, 0));
5942 return emit_library_call_value (libfunc
, NULL_RTX
, LCT_NORMAL
,
5943 mode
, addr
, ptr_mode
,
5948 /* If the test_and_set can't be emitted, eliminate any barrier that might
5949 have been emitted. */
5950 delete_insns_since (last_insn
);
5954 /* This function tries to implement an atomic exchange operation using a
5955 compare_and_swap loop. VAL is written to *MEM. The previous contents of
5956 *MEM are returned, using TARGET if possible. No memory model is required
5957 since a compare_and_swap loop is seq-cst. */
5960 maybe_emit_compare_and_swap_exchange_loop (rtx target
, rtx mem
, rtx val
)
5962 machine_mode mode
= GET_MODE (mem
);
5964 if (can_compare_and_swap_p (mode
, true))
5966 if (!target
|| !register_operand (target
, mode
))
5967 target
= gen_reg_rtx (mode
);
5968 if (expand_compare_and_swap_loop (mem
, target
, val
, NULL_RTX
))
5975 /* This function tries to implement an atomic test-and-set operation
5976 using the atomic_test_and_set instruction pattern. A boolean value
5977 is returned from the operation, using TARGET if possible. */
5980 maybe_emit_atomic_test_and_set (rtx target
, rtx mem
, enum memmodel model
)
5982 machine_mode pat_bool_mode
;
5983 struct expand_operand ops
[3];
5985 if (!targetm
.have_atomic_test_and_set ())
5988 /* While we always get QImode from __atomic_test_and_set, we get
5989 other memory modes from __sync_lock_test_and_set. Note that we
5990 use no endian adjustment here. This matches the 4.6 behavior
5991 in the Sparc backend. */
5992 enum insn_code icode
= targetm
.code_for_atomic_test_and_set
;
5993 gcc_checking_assert (insn_data
[icode
].operand
[1].mode
== QImode
);
5994 if (GET_MODE (mem
) != QImode
)
5995 mem
= adjust_address_nv (mem
, QImode
, 0);
5997 pat_bool_mode
= insn_data
[icode
].operand
[0].mode
;
5998 create_output_operand (&ops
[0], target
, pat_bool_mode
);
5999 create_fixed_operand (&ops
[1], mem
);
6000 create_integer_operand (&ops
[2], model
);
6002 if (maybe_expand_insn (icode
, 3, ops
))
6003 return ops
[0].value
;
6007 /* This function expands the legacy _sync_lock test_and_set operation which is
6008 generally an atomic exchange. Some limited targets only allow the
6009 constant 1 to be stored. This is an ACQUIRE operation.
6011 TARGET is an optional place to stick the return value.
6012 MEM is where VAL is stored. */
6015 expand_sync_lock_test_and_set (rtx target
, rtx mem
, rtx val
)
6019 /* Try an atomic_exchange first. */
6020 ret
= maybe_emit_atomic_exchange (target
, mem
, val
, MEMMODEL_SYNC_ACQUIRE
);
6024 ret
= maybe_emit_sync_lock_test_and_set (target
, mem
, val
,
6025 MEMMODEL_SYNC_ACQUIRE
);
6029 ret
= maybe_emit_compare_and_swap_exchange_loop (target
, mem
, val
);
6033 /* If there are no other options, try atomic_test_and_set if the value
6034 being stored is 1. */
6035 if (val
== const1_rtx
)
6036 ret
= maybe_emit_atomic_test_and_set (target
, mem
, MEMMODEL_SYNC_ACQUIRE
);
6041 /* This function expands the atomic test_and_set operation:
6042 atomically store a boolean TRUE into MEM and return the previous value.
6044 MEMMODEL is the memory model variant to use.
6045 TARGET is an optional place to stick the return value. */
6048 expand_atomic_test_and_set (rtx target
, rtx mem
, enum memmodel model
)
6050 machine_mode mode
= GET_MODE (mem
);
6051 rtx ret
, trueval
, subtarget
;
6053 ret
= maybe_emit_atomic_test_and_set (target
, mem
, model
);
6057 /* Be binary compatible with non-default settings of trueval, and different
6058 cpu revisions. E.g. one revision may have atomic-test-and-set, but
6059 another only has atomic-exchange. */
6060 if (targetm
.atomic_test_and_set_trueval
== 1)
6062 trueval
= const1_rtx
;
6063 subtarget
= target
? target
: gen_reg_rtx (mode
);
6067 trueval
= gen_int_mode (targetm
.atomic_test_and_set_trueval
, mode
);
6068 subtarget
= gen_reg_rtx (mode
);
6071 /* Try the atomic-exchange optab... */
6072 ret
= maybe_emit_atomic_exchange (subtarget
, mem
, trueval
, model
);
6074 /* ... then an atomic-compare-and-swap loop ... */
6076 ret
= maybe_emit_compare_and_swap_exchange_loop (subtarget
, mem
, trueval
);
6078 /* ... before trying the vaguely defined legacy lock_test_and_set. */
6080 ret
= maybe_emit_sync_lock_test_and_set (subtarget
, mem
, trueval
, model
);
6082 /* Recall that the legacy lock_test_and_set optab was allowed to do magic
6083 things with the value 1. Thus we try again without trueval. */
6084 if (!ret
&& targetm
.atomic_test_and_set_trueval
!= 1)
6085 ret
= maybe_emit_sync_lock_test_and_set (subtarget
, mem
, const1_rtx
, model
);
6087 /* Failing all else, assume a single threaded environment and simply
6088 perform the operation. */
6091 /* If the result is ignored skip the move to target. */
6092 if (subtarget
!= const0_rtx
)
6093 emit_move_insn (subtarget
, mem
);
6095 emit_move_insn (mem
, trueval
);
6099 /* Recall that have to return a boolean value; rectify if trueval
6100 is not exactly one. */
6101 if (targetm
.atomic_test_and_set_trueval
!= 1)
6102 ret
= emit_store_flag_force (target
, NE
, ret
, const0_rtx
, mode
, 0, 1);
6107 /* This function expands the atomic exchange operation:
6108 atomically store VAL in MEM and return the previous value in MEM.
6110 MEMMODEL is the memory model variant to use.
6111 TARGET is an optional place to stick the return value. */
6114 expand_atomic_exchange (rtx target
, rtx mem
, rtx val
, enum memmodel model
)
6116 machine_mode mode
= GET_MODE (mem
);
6119 /* If loads are not atomic for the required size and we are not called to
6120 provide a __sync builtin, do not do anything so that we stay consistent
6121 with atomic loads of the same size. */
6122 if (!can_atomic_load_p (mode
) && !is_mm_sync (model
))
6125 ret
= maybe_emit_atomic_exchange (target
, mem
, val
, model
);
6127 /* Next try a compare-and-swap loop for the exchange. */
6129 ret
= maybe_emit_compare_and_swap_exchange_loop (target
, mem
, val
);
6134 /* This function expands the atomic compare exchange operation:
6136 *PTARGET_BOOL is an optional place to store the boolean success/failure.
6137 *PTARGET_OVAL is an optional place to store the old value from memory.
6138 Both target parameters may be NULL or const0_rtx to indicate that we do
6139 not care about that return value. Both target parameters are updated on
6140 success to the actual location of the corresponding result.
6142 MEMMODEL is the memory model variant to use.
6144 The return value of the function is true for success. */
6147 expand_atomic_compare_and_swap (rtx
*ptarget_bool
, rtx
*ptarget_oval
,
6148 rtx mem
, rtx expected
, rtx desired
,
6149 bool is_weak
, enum memmodel succ_model
,
6150 enum memmodel fail_model
)
6152 machine_mode mode
= GET_MODE (mem
);
6153 struct expand_operand ops
[8];
6154 enum insn_code icode
;
6155 rtx target_oval
, target_bool
= NULL_RTX
;
6158 /* If loads are not atomic for the required size and we are not called to
6159 provide a __sync builtin, do not do anything so that we stay consistent
6160 with atomic loads of the same size. */
6161 if (!can_atomic_load_p (mode
) && !is_mm_sync (succ_model
))
6164 /* Load expected into a register for the compare and swap. */
6165 if (MEM_P (expected
))
6166 expected
= copy_to_reg (expected
);
6168 /* Make sure we always have some place to put the return oldval.
6169 Further, make sure that place is distinct from the input expected,
6170 just in case we need that path down below. */
6171 if (ptarget_oval
&& *ptarget_oval
== const0_rtx
)
6172 ptarget_oval
= NULL
;
6174 if (ptarget_oval
== NULL
6175 || (target_oval
= *ptarget_oval
) == NULL
6176 || reg_overlap_mentioned_p (expected
, target_oval
))
6177 target_oval
= gen_reg_rtx (mode
);
6179 icode
= direct_optab_handler (atomic_compare_and_swap_optab
, mode
);
6180 if (icode
!= CODE_FOR_nothing
)
6182 machine_mode bool_mode
= insn_data
[icode
].operand
[0].mode
;
6184 if (ptarget_bool
&& *ptarget_bool
== const0_rtx
)
6185 ptarget_bool
= NULL
;
6187 /* Make sure we always have a place for the bool operand. */
6188 if (ptarget_bool
== NULL
6189 || (target_bool
= *ptarget_bool
) == NULL
6190 || GET_MODE (target_bool
) != bool_mode
)
6191 target_bool
= gen_reg_rtx (bool_mode
);
6193 /* Emit the compare_and_swap. */
6194 create_output_operand (&ops
[0], target_bool
, bool_mode
);
6195 create_output_operand (&ops
[1], target_oval
, mode
);
6196 create_fixed_operand (&ops
[2], mem
);
6197 create_input_operand (&ops
[3], expected
, mode
);
6198 create_input_operand (&ops
[4], desired
, mode
);
6199 create_integer_operand (&ops
[5], is_weak
);
6200 create_integer_operand (&ops
[6], succ_model
);
6201 create_integer_operand (&ops
[7], fail_model
);
6202 if (maybe_expand_insn (icode
, 8, ops
))
6204 /* Return success/failure. */
6205 target_bool
= ops
[0].value
;
6206 target_oval
= ops
[1].value
;
6211 /* Otherwise fall back to the original __sync_val_compare_and_swap
6212 which is always seq-cst. */
6213 icode
= optab_handler (sync_compare_and_swap_optab
, mode
);
6214 if (icode
!= CODE_FOR_nothing
)
6218 create_output_operand (&ops
[0], target_oval
, mode
);
6219 create_fixed_operand (&ops
[1], mem
);
6220 create_input_operand (&ops
[2], expected
, mode
);
6221 create_input_operand (&ops
[3], desired
, mode
);
6222 if (!maybe_expand_insn (icode
, 4, ops
))
6225 target_oval
= ops
[0].value
;
6227 /* If the caller isn't interested in the boolean return value,
6228 skip the computation of it. */
6229 if (ptarget_bool
== NULL
)
6232 /* Otherwise, work out if the compare-and-swap succeeded. */
6234 if (have_insn_for (COMPARE
, CCmode
))
6235 note_stores (PATTERN (get_last_insn ()), find_cc_set
, &cc_reg
);
6238 target_bool
= emit_store_flag_force (target_bool
, EQ
, cc_reg
,
6239 const0_rtx
, VOIDmode
, 0, 1);
6242 goto success_bool_from_val
;
6245 /* Also check for library support for __sync_val_compare_and_swap. */
6246 libfunc
= optab_libfunc (sync_compare_and_swap_optab
, mode
);
6247 if (libfunc
!= NULL
)
6249 rtx addr
= convert_memory_address (ptr_mode
, XEXP (mem
, 0));
6250 rtx target
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_NORMAL
,
6251 mode
, addr
, ptr_mode
,
6252 expected
, mode
, desired
, mode
);
6253 emit_move_insn (target_oval
, target
);
6255 /* Compute the boolean return value only if requested. */
6257 goto success_bool_from_val
;
6265 success_bool_from_val
:
6266 target_bool
= emit_store_flag_force (target_bool
, EQ
, target_oval
,
6267 expected
, VOIDmode
, 1, 1);
6269 /* Make sure that the oval output winds up where the caller asked. */
6271 *ptarget_oval
= target_oval
;
6273 *ptarget_bool
= target_bool
;
6277 /* Generate asm volatile("" : : : "memory") as the memory blockage. */
6280 expand_asm_memory_blockage (void)
6284 asm_op
= gen_rtx_ASM_OPERANDS (VOIDmode
, "", "", 0,
6285 rtvec_alloc (0), rtvec_alloc (0),
6286 rtvec_alloc (0), UNKNOWN_LOCATION
);
6287 MEM_VOLATILE_P (asm_op
) = 1;
6289 clob
= gen_rtx_SCRATCH (VOIDmode
);
6290 clob
= gen_rtx_MEM (BLKmode
, clob
);
6291 clob
= gen_rtx_CLOBBER (VOIDmode
, clob
);
6293 emit_insn (gen_rtx_PARALLEL (VOIDmode
, gen_rtvec (2, asm_op
, clob
)));
6296 /* Do not propagate memory accesses across this point. */
6299 expand_memory_blockage (void)
6301 if (targetm
.have_memory_blockage ())
6302 emit_insn (targetm
.gen_memory_blockage ());
6304 expand_asm_memory_blockage ();
6307 /* This routine will either emit the mem_thread_fence pattern or issue a
6308 sync_synchronize to generate a fence for memory model MEMMODEL. */
6311 expand_mem_thread_fence (enum memmodel model
)
6313 if (is_mm_relaxed (model
))
6315 if (targetm
.have_mem_thread_fence ())
6317 emit_insn (targetm
.gen_mem_thread_fence (GEN_INT (model
)));
6318 expand_memory_blockage ();
6320 else if (targetm
.have_memory_barrier ())
6321 emit_insn (targetm
.gen_memory_barrier ());
6322 else if (synchronize_libfunc
!= NULL_RTX
)
6323 emit_library_call (synchronize_libfunc
, LCT_NORMAL
, VOIDmode
);
6325 expand_memory_blockage ();
6328 /* Emit a signal fence with given memory model. */
6331 expand_mem_signal_fence (enum memmodel model
)
6333 /* No machine barrier is required to implement a signal fence, but
6334 a compiler memory barrier must be issued, except for relaxed MM. */
6335 if (!is_mm_relaxed (model
))
6336 expand_memory_blockage ();
6339 /* This function expands the atomic load operation:
6340 return the atomically loaded value in MEM.
6342 MEMMODEL is the memory model variant to use.
6343 TARGET is an option place to stick the return value. */
6346 expand_atomic_load (rtx target
, rtx mem
, enum memmodel model
)
6348 machine_mode mode
= GET_MODE (mem
);
6349 enum insn_code icode
;
6351 /* If the target supports the load directly, great. */
6352 icode
= direct_optab_handler (atomic_load_optab
, mode
);
6353 if (icode
!= CODE_FOR_nothing
)
6355 struct expand_operand ops
[3];
6356 rtx_insn
*last
= get_last_insn ();
6357 if (is_mm_seq_cst (model
))
6358 expand_memory_blockage ();
6360 create_output_operand (&ops
[0], target
, mode
);
6361 create_fixed_operand (&ops
[1], mem
);
6362 create_integer_operand (&ops
[2], model
);
6363 if (maybe_expand_insn (icode
, 3, ops
))
6365 if (!is_mm_relaxed (model
))
6366 expand_memory_blockage ();
6367 return ops
[0].value
;
6369 delete_insns_since (last
);
6372 /* If the size of the object is greater than word size on this target,
6373 then we assume that a load will not be atomic. We could try to
6374 emulate a load with a compare-and-swap operation, but the store that
6375 doing this could result in would be incorrect if this is a volatile
6376 atomic load or targetting read-only-mapped memory. */
6377 if (GET_MODE_PRECISION (mode
) > BITS_PER_WORD
)
6378 /* If there is no atomic load, leave the library call. */
6381 /* Otherwise assume loads are atomic, and emit the proper barriers. */
6382 if (!target
|| target
== const0_rtx
)
6383 target
= gen_reg_rtx (mode
);
6385 /* For SEQ_CST, emit a barrier before the load. */
6386 if (is_mm_seq_cst (model
))
6387 expand_mem_thread_fence (model
);
6389 emit_move_insn (target
, mem
);
6391 /* Emit the appropriate barrier after the load. */
6392 expand_mem_thread_fence (model
);
6397 /* This function expands the atomic store operation:
6398 Atomically store VAL in MEM.
6399 MEMMODEL is the memory model variant to use.
6400 USE_RELEASE is true if __sync_lock_release can be used as a fall back.
6401 function returns const0_rtx if a pattern was emitted. */
6404 expand_atomic_store (rtx mem
, rtx val
, enum memmodel model
, bool use_release
)
6406 machine_mode mode
= GET_MODE (mem
);
6407 enum insn_code icode
;
6408 struct expand_operand ops
[3];
6410 /* If the target supports the store directly, great. */
6411 icode
= direct_optab_handler (atomic_store_optab
, mode
);
6412 if (icode
!= CODE_FOR_nothing
)
6414 rtx_insn
*last
= get_last_insn ();
6415 if (!is_mm_relaxed (model
))
6416 expand_memory_blockage ();
6417 create_fixed_operand (&ops
[0], mem
);
6418 create_input_operand (&ops
[1], val
, mode
);
6419 create_integer_operand (&ops
[2], model
);
6420 if (maybe_expand_insn (icode
, 3, ops
))
6422 if (is_mm_seq_cst (model
))
6423 expand_memory_blockage ();
6426 delete_insns_since (last
);
6429 /* If using __sync_lock_release is a viable alternative, try it.
6430 Note that this will not be set to true if we are expanding a generic
6431 __atomic_store_n. */
6434 icode
= direct_optab_handler (sync_lock_release_optab
, mode
);
6435 if (icode
!= CODE_FOR_nothing
)
6437 create_fixed_operand (&ops
[0], mem
);
6438 create_input_operand (&ops
[1], const0_rtx
, mode
);
6439 if (maybe_expand_insn (icode
, 2, ops
))
6441 /* lock_release is only a release barrier. */
6442 if (is_mm_seq_cst (model
))
6443 expand_mem_thread_fence (model
);
6449 /* If the size of the object is greater than word size on this target,
6450 a default store will not be atomic. */
6451 if (GET_MODE_PRECISION (mode
) > BITS_PER_WORD
)
6453 /* If loads are atomic or we are called to provide a __sync builtin,
6454 we can try a atomic_exchange and throw away the result. Otherwise,
6455 don't do anything so that we do not create an inconsistency between
6456 loads and stores. */
6457 if (can_atomic_load_p (mode
) || is_mm_sync (model
))
6459 rtx target
= maybe_emit_atomic_exchange (NULL_RTX
, mem
, val
, model
);
6461 target
= maybe_emit_compare_and_swap_exchange_loop (NULL_RTX
, mem
,
6469 /* Otherwise assume stores are atomic, and emit the proper barriers. */
6470 expand_mem_thread_fence (model
);
6472 emit_move_insn (mem
, val
);
6474 /* For SEQ_CST, also emit a barrier after the store. */
6475 if (is_mm_seq_cst (model
))
6476 expand_mem_thread_fence (model
);
6482 /* Structure containing the pointers and values required to process the
6483 various forms of the atomic_fetch_op and atomic_op_fetch builtins. */
6485 struct atomic_op_functions
6487 direct_optab mem_fetch_before
;
6488 direct_optab mem_fetch_after
;
6489 direct_optab mem_no_result
;
6492 direct_optab no_result
;
6493 enum rtx_code reverse_code
;
6497 /* Fill in structure pointed to by OP with the various optab entries for an
6498 operation of type CODE. */
6501 get_atomic_op_for_code (struct atomic_op_functions
*op
, enum rtx_code code
)
6503 gcc_assert (op
!= NULL
);
6505 /* If SWITCHABLE_TARGET is defined, then subtargets can be switched
6506 in the source code during compilation, and the optab entries are not
6507 computable until runtime. Fill in the values at runtime. */
6511 op
->mem_fetch_before
= atomic_fetch_add_optab
;
6512 op
->mem_fetch_after
= atomic_add_fetch_optab
;
6513 op
->mem_no_result
= atomic_add_optab
;
6514 op
->fetch_before
= sync_old_add_optab
;
6515 op
->fetch_after
= sync_new_add_optab
;
6516 op
->no_result
= sync_add_optab
;
6517 op
->reverse_code
= MINUS
;
6520 op
->mem_fetch_before
= atomic_fetch_sub_optab
;
6521 op
->mem_fetch_after
= atomic_sub_fetch_optab
;
6522 op
->mem_no_result
= atomic_sub_optab
;
6523 op
->fetch_before
= sync_old_sub_optab
;
6524 op
->fetch_after
= sync_new_sub_optab
;
6525 op
->no_result
= sync_sub_optab
;
6526 op
->reverse_code
= PLUS
;
6529 op
->mem_fetch_before
= atomic_fetch_xor_optab
;
6530 op
->mem_fetch_after
= atomic_xor_fetch_optab
;
6531 op
->mem_no_result
= atomic_xor_optab
;
6532 op
->fetch_before
= sync_old_xor_optab
;
6533 op
->fetch_after
= sync_new_xor_optab
;
6534 op
->no_result
= sync_xor_optab
;
6535 op
->reverse_code
= XOR
;
6538 op
->mem_fetch_before
= atomic_fetch_and_optab
;
6539 op
->mem_fetch_after
= atomic_and_fetch_optab
;
6540 op
->mem_no_result
= atomic_and_optab
;
6541 op
->fetch_before
= sync_old_and_optab
;
6542 op
->fetch_after
= sync_new_and_optab
;
6543 op
->no_result
= sync_and_optab
;
6544 op
->reverse_code
= UNKNOWN
;
6547 op
->mem_fetch_before
= atomic_fetch_or_optab
;
6548 op
->mem_fetch_after
= atomic_or_fetch_optab
;
6549 op
->mem_no_result
= atomic_or_optab
;
6550 op
->fetch_before
= sync_old_ior_optab
;
6551 op
->fetch_after
= sync_new_ior_optab
;
6552 op
->no_result
= sync_ior_optab
;
6553 op
->reverse_code
= UNKNOWN
;
6556 op
->mem_fetch_before
= atomic_fetch_nand_optab
;
6557 op
->mem_fetch_after
= atomic_nand_fetch_optab
;
6558 op
->mem_no_result
= atomic_nand_optab
;
6559 op
->fetch_before
= sync_old_nand_optab
;
6560 op
->fetch_after
= sync_new_nand_optab
;
6561 op
->no_result
= sync_nand_optab
;
6562 op
->reverse_code
= UNKNOWN
;
6569 /* See if there is a more optimal way to implement the operation "*MEM CODE VAL"
6570 using memory order MODEL. If AFTER is true the operation needs to return
6571 the value of *MEM after the operation, otherwise the previous value.
6572 TARGET is an optional place to place the result. The result is unused if
6574 Return the result if there is a better sequence, otherwise NULL_RTX. */
6577 maybe_optimize_fetch_op (rtx target
, rtx mem
, rtx val
, enum rtx_code code
,
6578 enum memmodel model
, bool after
)
6580 /* If the value is prefetched, or not used, it may be possible to replace
6581 the sequence with a native exchange operation. */
6582 if (!after
|| target
== const0_rtx
)
6584 /* fetch_and (&x, 0, m) can be replaced with exchange (&x, 0, m). */
6585 if (code
== AND
&& val
== const0_rtx
)
6587 if (target
== const0_rtx
)
6588 target
= gen_reg_rtx (GET_MODE (mem
));
6589 return maybe_emit_atomic_exchange (target
, mem
, val
, model
);
6592 /* fetch_or (&x, -1, m) can be replaced with exchange (&x, -1, m). */
6593 if (code
== IOR
&& val
== constm1_rtx
)
6595 if (target
== const0_rtx
)
6596 target
= gen_reg_rtx (GET_MODE (mem
));
6597 return maybe_emit_atomic_exchange (target
, mem
, val
, model
);
6604 /* Try to emit an instruction for a specific operation varaition.
6605 OPTAB contains the OP functions.
6606 TARGET is an optional place to return the result. const0_rtx means unused.
6607 MEM is the memory location to operate on.
6608 VAL is the value to use in the operation.
6609 USE_MEMMODEL is TRUE if the variation with a memory model should be tried.
6610 MODEL is the memory model, if used.
6611 AFTER is true if the returned result is the value after the operation. */
6614 maybe_emit_op (const struct atomic_op_functions
*optab
, rtx target
, rtx mem
,
6615 rtx val
, bool use_memmodel
, enum memmodel model
, bool after
)
6617 machine_mode mode
= GET_MODE (mem
);
6618 struct expand_operand ops
[4];
6619 enum insn_code icode
;
6623 /* Check to see if there is a result returned. */
6624 if (target
== const0_rtx
)
6628 icode
= direct_optab_handler (optab
->mem_no_result
, mode
);
6629 create_integer_operand (&ops
[2], model
);
6634 icode
= direct_optab_handler (optab
->no_result
, mode
);
6638 /* Otherwise, we need to generate a result. */
6643 icode
= direct_optab_handler (after
? optab
->mem_fetch_after
6644 : optab
->mem_fetch_before
, mode
);
6645 create_integer_operand (&ops
[3], model
);
6650 icode
= optab_handler (after
? optab
->fetch_after
6651 : optab
->fetch_before
, mode
);
6654 create_output_operand (&ops
[op_counter
++], target
, mode
);
6656 if (icode
== CODE_FOR_nothing
)
6659 create_fixed_operand (&ops
[op_counter
++], mem
);
6660 /* VAL may have been promoted to a wider mode. Shrink it if so. */
6661 create_convert_operand_to (&ops
[op_counter
++], val
, mode
, true);
6663 if (maybe_expand_insn (icode
, num_ops
, ops
))
6664 return (target
== const0_rtx
? const0_rtx
: ops
[0].value
);
6670 /* This function expands an atomic fetch_OP or OP_fetch operation:
6671 TARGET is an option place to stick the return value. const0_rtx indicates
6672 the result is unused.
6673 atomically fetch MEM, perform the operation with VAL and return it to MEM.
6674 CODE is the operation being performed (OP)
6675 MEMMODEL is the memory model variant to use.
6676 AFTER is true to return the result of the operation (OP_fetch).
6677 AFTER is false to return the value before the operation (fetch_OP).
6679 This function will *only* generate instructions if there is a direct
6680 optab. No compare and swap loops or libcalls will be generated. */
6683 expand_atomic_fetch_op_no_fallback (rtx target
, rtx mem
, rtx val
,
6684 enum rtx_code code
, enum memmodel model
,
6687 machine_mode mode
= GET_MODE (mem
);
6688 struct atomic_op_functions optab
;
6690 bool unused_result
= (target
== const0_rtx
);
6692 get_atomic_op_for_code (&optab
, code
);
6694 /* Check to see if there are any better instructions. */
6695 result
= maybe_optimize_fetch_op (target
, mem
, val
, code
, model
, after
);
6699 /* Check for the case where the result isn't used and try those patterns. */
6702 /* Try the memory model variant first. */
6703 result
= maybe_emit_op (&optab
, target
, mem
, val
, true, model
, true);
6707 /* Next try the old style withuot a memory model. */
6708 result
= maybe_emit_op (&optab
, target
, mem
, val
, false, model
, true);
6712 /* There is no no-result pattern, so try patterns with a result. */
6716 /* Try the __atomic version. */
6717 result
= maybe_emit_op (&optab
, target
, mem
, val
, true, model
, after
);
6721 /* Try the older __sync version. */
6722 result
= maybe_emit_op (&optab
, target
, mem
, val
, false, model
, after
);
6726 /* If the fetch value can be calculated from the other variation of fetch,
6727 try that operation. */
6728 if (after
|| unused_result
|| optab
.reverse_code
!= UNKNOWN
)
6730 /* Try the __atomic version, then the older __sync version. */
6731 result
= maybe_emit_op (&optab
, target
, mem
, val
, true, model
, !after
);
6733 result
= maybe_emit_op (&optab
, target
, mem
, val
, false, model
, !after
);
6737 /* If the result isn't used, no need to do compensation code. */
6741 /* Issue compensation code. Fetch_after == fetch_before OP val.
6742 Fetch_before == after REVERSE_OP val. */
6744 code
= optab
.reverse_code
;
6747 result
= expand_simple_binop (mode
, AND
, result
, val
, NULL_RTX
,
6748 true, OPTAB_LIB_WIDEN
);
6749 result
= expand_simple_unop (mode
, NOT
, result
, target
, true);
6752 result
= expand_simple_binop (mode
, code
, result
, val
, target
,
6753 true, OPTAB_LIB_WIDEN
);
6758 /* No direct opcode can be generated. */
6764 /* This function expands an atomic fetch_OP or OP_fetch operation:
6765 TARGET is an option place to stick the return value. const0_rtx indicates
6766 the result is unused.
6767 atomically fetch MEM, perform the operation with VAL and return it to MEM.
6768 CODE is the operation being performed (OP)
6769 MEMMODEL is the memory model variant to use.
6770 AFTER is true to return the result of the operation (OP_fetch).
6771 AFTER is false to return the value before the operation (fetch_OP). */
6773 expand_atomic_fetch_op (rtx target
, rtx mem
, rtx val
, enum rtx_code code
,
6774 enum memmodel model
, bool after
)
6776 machine_mode mode
= GET_MODE (mem
);
6778 bool unused_result
= (target
== const0_rtx
);
6780 /* If loads are not atomic for the required size and we are not called to
6781 provide a __sync builtin, do not do anything so that we stay consistent
6782 with atomic loads of the same size. */
6783 if (!can_atomic_load_p (mode
) && !is_mm_sync (model
))
6786 result
= expand_atomic_fetch_op_no_fallback (target
, mem
, val
, code
, model
,
6792 /* Add/sub can be implemented by doing the reverse operation with -(val). */
6793 if (code
== PLUS
|| code
== MINUS
)
6796 enum rtx_code reverse
= (code
== PLUS
? MINUS
: PLUS
);
6799 tmp
= expand_simple_unop (mode
, NEG
, val
, NULL_RTX
, true);
6800 result
= expand_atomic_fetch_op_no_fallback (target
, mem
, tmp
, reverse
,
6804 /* PLUS worked so emit the insns and return. */
6811 /* PLUS did not work, so throw away the negation code and continue. */
6815 /* Try the __sync libcalls only if we can't do compare-and-swap inline. */
6816 if (!can_compare_and_swap_p (mode
, false))
6820 enum rtx_code orig_code
= code
;
6821 struct atomic_op_functions optab
;
6823 get_atomic_op_for_code (&optab
, code
);
6824 libfunc
= optab_libfunc (after
? optab
.fetch_after
6825 : optab
.fetch_before
, mode
);
6827 && (after
|| unused_result
|| optab
.reverse_code
!= UNKNOWN
))
6831 code
= optab
.reverse_code
;
6832 libfunc
= optab_libfunc (after
? optab
.fetch_before
6833 : optab
.fetch_after
, mode
);
6835 if (libfunc
!= NULL
)
6837 rtx addr
= convert_memory_address (ptr_mode
, XEXP (mem
, 0));
6838 result
= emit_library_call_value (libfunc
, NULL
, LCT_NORMAL
, mode
,
6839 addr
, ptr_mode
, val
, mode
);
6841 if (!unused_result
&& fixup
)
6842 result
= expand_simple_binop (mode
, code
, result
, val
, target
,
6843 true, OPTAB_LIB_WIDEN
);
6847 /* We need the original code for any further attempts. */
6851 /* If nothing else has succeeded, default to a compare and swap loop. */
6852 if (can_compare_and_swap_p (mode
, true))
6855 rtx t0
= gen_reg_rtx (mode
), t1
;
6859 /* If the result is used, get a register for it. */
6862 if (!target
|| !register_operand (target
, mode
))
6863 target
= gen_reg_rtx (mode
);
6864 /* If fetch_before, copy the value now. */
6866 emit_move_insn (target
, t0
);
6869 target
= const0_rtx
;
6874 t1
= expand_simple_binop (mode
, AND
, t1
, val
, NULL_RTX
,
6875 true, OPTAB_LIB_WIDEN
);
6876 t1
= expand_simple_unop (mode
, code
, t1
, NULL_RTX
, true);
6879 t1
= expand_simple_binop (mode
, code
, t1
, val
, NULL_RTX
, true,
6882 /* For after, copy the value now. */
6883 if (!unused_result
&& after
)
6884 emit_move_insn (target
, t1
);
6885 insn
= get_insns ();
6888 if (t1
!= NULL
&& expand_compare_and_swap_loop (mem
, t0
, t1
, insn
))
6895 /* Return true if OPERAND is suitable for operand number OPNO of
6896 instruction ICODE. */
6899 insn_operand_matches (enum insn_code icode
, unsigned int opno
, rtx operand
)
6901 return (!insn_data
[(int) icode
].operand
[opno
].predicate
6902 || (insn_data
[(int) icode
].operand
[opno
].predicate
6903 (operand
, insn_data
[(int) icode
].operand
[opno
].mode
)));
6906 /* TARGET is a target of a multiword operation that we are going to
6907 implement as a series of word-mode operations. Return true if
6908 TARGET is suitable for this purpose. */
6911 valid_multiword_target_p (rtx target
)
6916 mode
= GET_MODE (target
);
6917 for (i
= 0; i
< GET_MODE_SIZE (mode
); i
+= UNITS_PER_WORD
)
6918 if (!validate_subreg (word_mode
, mode
, target
, i
))
6923 /* Like maybe_legitimize_operand, but do not change the code of the
6924 current rtx value. */
6927 maybe_legitimize_operand_same_code (enum insn_code icode
, unsigned int opno
,
6928 struct expand_operand
*op
)
6930 /* See if the operand matches in its current form. */
6931 if (insn_operand_matches (icode
, opno
, op
->value
))
6934 /* If the operand is a memory whose address has no side effects,
6935 try forcing the address into a non-virtual pseudo register.
6936 The check for side effects is important because copy_to_mode_reg
6937 cannot handle things like auto-modified addresses. */
6938 if (insn_data
[(int) icode
].operand
[opno
].allows_mem
&& MEM_P (op
->value
))
6943 addr
= XEXP (mem
, 0);
6944 if (!(REG_P (addr
) && REGNO (addr
) > LAST_VIRTUAL_REGISTER
)
6945 && !side_effects_p (addr
))
6950 last
= get_last_insn ();
6951 mode
= get_address_mode (mem
);
6952 mem
= replace_equiv_address (mem
, copy_to_mode_reg (mode
, addr
));
6953 if (insn_operand_matches (icode
, opno
, mem
))
6958 delete_insns_since (last
);
6965 /* Try to make OP match operand OPNO of instruction ICODE. Return true
6966 on success, storing the new operand value back in OP. */
6969 maybe_legitimize_operand (enum insn_code icode
, unsigned int opno
,
6970 struct expand_operand
*op
)
6972 machine_mode mode
, imode
;
6973 bool old_volatile_ok
, result
;
6979 old_volatile_ok
= volatile_ok
;
6981 result
= maybe_legitimize_operand_same_code (icode
, opno
, op
);
6982 volatile_ok
= old_volatile_ok
;
6986 gcc_assert (mode
!= VOIDmode
);
6988 && op
->value
!= const0_rtx
6989 && GET_MODE (op
->value
) == mode
6990 && maybe_legitimize_operand_same_code (icode
, opno
, op
))
6993 op
->value
= gen_reg_rtx (mode
);
6999 gcc_assert (mode
!= VOIDmode
);
7000 gcc_assert (GET_MODE (op
->value
) == VOIDmode
7001 || GET_MODE (op
->value
) == mode
);
7002 if (maybe_legitimize_operand_same_code (icode
, opno
, op
))
7005 op
->value
= copy_to_mode_reg (mode
, op
->value
);
7008 case EXPAND_CONVERT_TO
:
7009 gcc_assert (mode
!= VOIDmode
);
7010 op
->value
= convert_to_mode (mode
, op
->value
, op
->unsigned_p
);
7013 case EXPAND_CONVERT_FROM
:
7014 if (GET_MODE (op
->value
) != VOIDmode
)
7015 mode
= GET_MODE (op
->value
);
7017 /* The caller must tell us what mode this value has. */
7018 gcc_assert (mode
!= VOIDmode
);
7020 imode
= insn_data
[(int) icode
].operand
[opno
].mode
;
7021 if (imode
!= VOIDmode
&& imode
!= mode
)
7023 op
->value
= convert_modes (imode
, mode
, op
->value
, op
->unsigned_p
);
7028 case EXPAND_ADDRESS
:
7029 op
->value
= convert_memory_address (as_a
<scalar_int_mode
> (mode
),
7033 case EXPAND_INTEGER
:
7034 mode
= insn_data
[(int) icode
].operand
[opno
].mode
;
7035 if (mode
!= VOIDmode
&& const_int_operand (op
->value
, mode
))
7039 return insn_operand_matches (icode
, opno
, op
->value
);
7042 /* Make OP describe an input operand that should have the same value
7043 as VALUE, after any mode conversion that the target might request.
7044 TYPE is the type of VALUE. */
7047 create_convert_operand_from_type (struct expand_operand
*op
,
7048 rtx value
, tree type
)
7050 create_convert_operand_from (op
, value
, TYPE_MODE (type
),
7051 TYPE_UNSIGNED (type
));
7054 /* Try to make operands [OPS, OPS + NOPS) match operands [OPNO, OPNO + NOPS)
7055 of instruction ICODE. Return true on success, leaving the new operand
7056 values in the OPS themselves. Emit no code on failure. */
7059 maybe_legitimize_operands (enum insn_code icode
, unsigned int opno
,
7060 unsigned int nops
, struct expand_operand
*ops
)
7065 last
= get_last_insn ();
7066 for (i
= 0; i
< nops
; i
++)
7067 if (!maybe_legitimize_operand (icode
, opno
+ i
, &ops
[i
]))
7069 delete_insns_since (last
);
7075 /* Try to generate instruction ICODE, using operands [OPS, OPS + NOPS)
7076 as its operands. Return the instruction pattern on success,
7077 and emit any necessary set-up code. Return null and emit no
7081 maybe_gen_insn (enum insn_code icode
, unsigned int nops
,
7082 struct expand_operand
*ops
)
7084 gcc_assert (nops
== (unsigned int) insn_data
[(int) icode
].n_generator_args
);
7085 if (!maybe_legitimize_operands (icode
, 0, nops
, ops
))
7091 return GEN_FCN (icode
) (ops
[0].value
);
7093 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
);
7095 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
);
7097 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
7100 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
7101 ops
[3].value
, ops
[4].value
);
7103 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
7104 ops
[3].value
, ops
[4].value
, ops
[5].value
);
7106 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
7107 ops
[3].value
, ops
[4].value
, ops
[5].value
,
7110 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
7111 ops
[3].value
, ops
[4].value
, ops
[5].value
,
7112 ops
[6].value
, ops
[7].value
);
7114 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
7115 ops
[3].value
, ops
[4].value
, ops
[5].value
,
7116 ops
[6].value
, ops
[7].value
, ops
[8].value
);
7121 /* Try to emit instruction ICODE, using operands [OPS, OPS + NOPS)
7122 as its operands. Return true on success and emit no code on failure. */
7125 maybe_expand_insn (enum insn_code icode
, unsigned int nops
,
7126 struct expand_operand
*ops
)
7128 rtx_insn
*pat
= maybe_gen_insn (icode
, nops
, ops
);
7137 /* Like maybe_expand_insn, but for jumps. */
7140 maybe_expand_jump_insn (enum insn_code icode
, unsigned int nops
,
7141 struct expand_operand
*ops
)
7143 rtx_insn
*pat
= maybe_gen_insn (icode
, nops
, ops
);
7146 emit_jump_insn (pat
);
7152 /* Emit instruction ICODE, using operands [OPS, OPS + NOPS)
7156 expand_insn (enum insn_code icode
, unsigned int nops
,
7157 struct expand_operand
*ops
)
7159 if (!maybe_expand_insn (icode
, nops
, ops
))
7163 /* Like expand_insn, but for jumps. */
7166 expand_jump_insn (enum insn_code icode
, unsigned int nops
,
7167 struct expand_operand
*ops
)
7169 if (!maybe_expand_jump_insn (icode
, nops
, ops
))