1 /* Expand the basic unary and binary arithmetic operations, for GNU compiler.
2 Copyright (C) 1987-2018 Free Software Foundation, Inc.
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 3, or (at your option) any later
11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
23 #include "coretypes.h"
35 #include "diagnostic-core.h"
36 #include "rtx-vector-builder.h"
38 /* Include insn-config.h before expr.h so that HAVE_conditional_move
39 is properly defined. */
40 #include "stor-layout.h"
45 #include "optabs-tree.h"
48 static void prepare_float_lib_cmp (rtx
, rtx
, enum rtx_code
, rtx
*,
50 static rtx
expand_unop_direct (machine_mode
, optab
, rtx
, rtx
, int);
51 static void emit_libcall_block_1 (rtx_insn
*, rtx
, rtx
, rtx
, bool);
53 /* Debug facility for use in GDB. */
54 void debug_optab_libfuncs (void);
56 /* Add a REG_EQUAL note to the last insn in INSNS. TARGET is being set to
57 the result of operation CODE applied to OP0 (and OP1 if it is a binary
60 If the last insn does not set TARGET, don't do anything, but return 1.
62 If the last insn or a previous insn sets TARGET and TARGET is one of OP0
63 or OP1, don't add the REG_EQUAL note but return 0. Our caller can then
64 try again, ensuring that TARGET is not one of the operands. */
67 add_equal_note (rtx_insn
*insns
, rtx target
, enum rtx_code code
, rtx op0
, rtx op1
)
73 gcc_assert (insns
&& INSN_P (insns
) && NEXT_INSN (insns
));
75 if (GET_RTX_CLASS (code
) != RTX_COMM_ARITH
76 && GET_RTX_CLASS (code
) != RTX_BIN_ARITH
77 && GET_RTX_CLASS (code
) != RTX_COMM_COMPARE
78 && GET_RTX_CLASS (code
) != RTX_COMPARE
79 && GET_RTX_CLASS (code
) != RTX_UNARY
)
82 if (GET_CODE (target
) == ZERO_EXTRACT
)
85 for (last_insn
= insns
;
86 NEXT_INSN (last_insn
) != NULL_RTX
;
87 last_insn
= NEXT_INSN (last_insn
))
90 /* If TARGET is in OP0 or OP1, punt. We'd end up with a note referencing
91 a value changing in the insn, so the note would be invalid for CSE. */
92 if (reg_overlap_mentioned_p (target
, op0
)
93 || (op1
&& reg_overlap_mentioned_p (target
, op1
)))
96 && (rtx_equal_p (target
, op0
)
97 || (op1
&& rtx_equal_p (target
, op1
))))
99 /* For MEM target, with MEM = MEM op X, prefer no REG_EQUAL note
100 over expanding it as temp = MEM op X, MEM = temp. If the target
101 supports MEM = MEM op X instructions, it is sometimes too hard
102 to reconstruct that form later, especially if X is also a memory,
103 and due to multiple occurrences of addresses the address might
104 be forced into register unnecessarily.
105 Note that not emitting the REG_EQUIV note might inhibit
106 CSE in some cases. */
107 set
= single_set (last_insn
);
109 && GET_CODE (SET_SRC (set
)) == code
110 && MEM_P (SET_DEST (set
))
111 && (rtx_equal_p (SET_DEST (set
), XEXP (SET_SRC (set
), 0))
112 || (op1
&& rtx_equal_p (SET_DEST (set
),
113 XEXP (SET_SRC (set
), 1)))))
119 set
= set_for_reg_notes (last_insn
);
123 if (! rtx_equal_p (SET_DEST (set
), target
)
124 /* For a STRICT_LOW_PART, the REG_NOTE applies to what is inside it. */
125 && (GET_CODE (SET_DEST (set
)) != STRICT_LOW_PART
126 || ! rtx_equal_p (XEXP (SET_DEST (set
), 0), target
)))
129 if (GET_RTX_CLASS (code
) == RTX_UNARY
)
139 if (GET_MODE (op0
) != VOIDmode
&& GET_MODE (target
) != GET_MODE (op0
))
141 note
= gen_rtx_fmt_e (code
, GET_MODE (op0
), copy_rtx (op0
));
142 if (GET_MODE_UNIT_SIZE (GET_MODE (op0
))
143 > GET_MODE_UNIT_SIZE (GET_MODE (target
)))
144 note
= simplify_gen_unary (TRUNCATE
, GET_MODE (target
),
145 note
, GET_MODE (op0
));
147 note
= simplify_gen_unary (ZERO_EXTEND
, GET_MODE (target
),
148 note
, GET_MODE (op0
));
153 note
= gen_rtx_fmt_e (code
, GET_MODE (target
), copy_rtx (op0
));
157 note
= gen_rtx_fmt_ee (code
, GET_MODE (target
), copy_rtx (op0
), copy_rtx (op1
));
159 set_unique_reg_note (last_insn
, REG_EQUAL
, note
);
164 /* Given two input operands, OP0 and OP1, determine what the correct from_mode
165 for a widening operation would be. In most cases this would be OP0, but if
166 that's a constant it'll be VOIDmode, which isn't useful. */
169 widened_mode (machine_mode to_mode
, rtx op0
, rtx op1
)
171 machine_mode m0
= GET_MODE (op0
);
172 machine_mode m1
= GET_MODE (op1
);
175 if (m0
== VOIDmode
&& m1
== VOIDmode
)
177 else if (m0
== VOIDmode
|| GET_MODE_UNIT_SIZE (m0
) < GET_MODE_UNIT_SIZE (m1
))
182 if (GET_MODE_UNIT_SIZE (result
) > GET_MODE_UNIT_SIZE (to_mode
))
188 /* Widen OP to MODE and return the rtx for the widened operand. UNSIGNEDP
189 says whether OP is signed or unsigned. NO_EXTEND is nonzero if we need
190 not actually do a sign-extend or zero-extend, but can leave the
191 higher-order bits of the result rtx undefined, for example, in the case
192 of logical operations, but not right shifts. */
195 widen_operand (rtx op
, machine_mode mode
, machine_mode oldmode
,
196 int unsignedp
, int no_extend
)
199 scalar_int_mode int_mode
;
201 /* If we don't have to extend and this is a constant, return it. */
202 if (no_extend
&& GET_MODE (op
) == VOIDmode
)
205 /* If we must extend do so. If OP is a SUBREG for a promoted object, also
206 extend since it will be more efficient to do so unless the signedness of
207 a promoted object differs from our extension. */
209 || !is_a
<scalar_int_mode
> (mode
, &int_mode
)
210 || (GET_CODE (op
) == SUBREG
&& SUBREG_PROMOTED_VAR_P (op
)
211 && SUBREG_CHECK_PROMOTED_SIGN (op
, unsignedp
)))
212 return convert_modes (mode
, oldmode
, op
, unsignedp
);
214 /* If MODE is no wider than a single word, we return a lowpart or paradoxical
216 if (GET_MODE_SIZE (int_mode
) <= UNITS_PER_WORD
)
217 return gen_lowpart (int_mode
, force_reg (GET_MODE (op
), op
));
219 /* Otherwise, get an object of MODE, clobber it, and set the low-order
222 result
= gen_reg_rtx (int_mode
);
223 emit_clobber (result
);
224 emit_move_insn (gen_lowpart (GET_MODE (op
), result
), op
);
228 /* Expand vector widening operations.
230 There are two different classes of operations handled here:
231 1) Operations whose result is wider than all the arguments to the operation.
232 Examples: VEC_UNPACK_HI/LO_EXPR, VEC_WIDEN_MULT_HI/LO_EXPR
233 In this case OP0 and optionally OP1 would be initialized,
234 but WIDE_OP wouldn't (not relevant for this case).
235 2) Operations whose result is of the same size as the last argument to the
236 operation, but wider than all the other arguments to the operation.
237 Examples: WIDEN_SUM_EXPR, VEC_DOT_PROD_EXPR.
238 In the case WIDE_OP, OP0 and optionally OP1 would be initialized.
240 E.g, when called to expand the following operations, this is how
241 the arguments will be initialized:
243 widening-sum 2 oprnd0 - oprnd1
244 widening-dot-product 3 oprnd0 oprnd1 oprnd2
245 widening-mult 2 oprnd0 oprnd1 -
246 type-promotion (vec-unpack) 1 oprnd0 - - */
249 expand_widen_pattern_expr (sepops ops
, rtx op0
, rtx op1
, rtx wide_op
,
250 rtx target
, int unsignedp
)
252 struct expand_operand eops
[4];
253 tree oprnd0
, oprnd1
, oprnd2
;
254 machine_mode wmode
= VOIDmode
, tmode0
, tmode1
= VOIDmode
;
255 optab widen_pattern_optab
;
256 enum insn_code icode
;
257 int nops
= TREE_CODE_LENGTH (ops
->code
);
261 tmode0
= TYPE_MODE (TREE_TYPE (oprnd0
));
262 if (ops
->code
== VEC_UNPACK_FIX_TRUNC_HI_EXPR
263 || ops
->code
== VEC_UNPACK_FIX_TRUNC_LO_EXPR
)
264 /* The sign is from the result type rather than operand's type
267 = optab_for_tree_code (ops
->code
, ops
->type
, optab_default
);
270 = optab_for_tree_code (ops
->code
, TREE_TYPE (oprnd0
), optab_default
);
271 if (ops
->code
== WIDEN_MULT_PLUS_EXPR
272 || ops
->code
== WIDEN_MULT_MINUS_EXPR
)
273 icode
= find_widening_optab_handler (widen_pattern_optab
,
274 TYPE_MODE (TREE_TYPE (ops
->op2
)),
277 icode
= optab_handler (widen_pattern_optab
, tmode0
);
278 gcc_assert (icode
!= CODE_FOR_nothing
);
283 tmode1
= TYPE_MODE (TREE_TYPE (oprnd1
));
286 /* The last operand is of a wider mode than the rest of the operands. */
291 gcc_assert (tmode1
== tmode0
);
294 wmode
= TYPE_MODE (TREE_TYPE (oprnd2
));
298 create_output_operand (&eops
[op
++], target
, TYPE_MODE (ops
->type
));
299 create_convert_operand_from (&eops
[op
++], op0
, tmode0
, unsignedp
);
301 create_convert_operand_from (&eops
[op
++], op1
, tmode1
, unsignedp
);
303 create_convert_operand_from (&eops
[op
++], wide_op
, wmode
, unsignedp
);
304 expand_insn (icode
, op
, eops
);
305 return eops
[0].value
;
308 /* Generate code to perform an operation specified by TERNARY_OPTAB
309 on operands OP0, OP1 and OP2, with result having machine-mode MODE.
311 UNSIGNEDP is for the case where we have to widen the operands
312 to perform the operation. It says to use zero-extension.
314 If TARGET is nonzero, the value
315 is generated there, if it is convenient to do so.
316 In all cases an rtx is returned for the locus of the value;
317 this may or may not be TARGET. */
320 expand_ternary_op (machine_mode mode
, optab ternary_optab
, rtx op0
,
321 rtx op1
, rtx op2
, rtx target
, int unsignedp
)
323 struct expand_operand ops
[4];
324 enum insn_code icode
= optab_handler (ternary_optab
, mode
);
326 gcc_assert (optab_handler (ternary_optab
, mode
) != CODE_FOR_nothing
);
328 create_output_operand (&ops
[0], target
, mode
);
329 create_convert_operand_from (&ops
[1], op0
, mode
, unsignedp
);
330 create_convert_operand_from (&ops
[2], op1
, mode
, unsignedp
);
331 create_convert_operand_from (&ops
[3], op2
, mode
, unsignedp
);
332 expand_insn (icode
, 4, ops
);
337 /* Like expand_binop, but return a constant rtx if the result can be
338 calculated at compile time. The arguments and return value are
339 otherwise the same as for expand_binop. */
342 simplify_expand_binop (machine_mode mode
, optab binoptab
,
343 rtx op0
, rtx op1
, rtx target
, int unsignedp
,
344 enum optab_methods methods
)
346 if (CONSTANT_P (op0
) && CONSTANT_P (op1
))
348 rtx x
= simplify_binary_operation (optab_to_code (binoptab
),
354 return expand_binop (mode
, binoptab
, op0
, op1
, target
, unsignedp
, methods
);
357 /* Like simplify_expand_binop, but always put the result in TARGET.
358 Return true if the expansion succeeded. */
361 force_expand_binop (machine_mode mode
, optab binoptab
,
362 rtx op0
, rtx op1
, rtx target
, int unsignedp
,
363 enum optab_methods methods
)
365 rtx x
= simplify_expand_binop (mode
, binoptab
, op0
, op1
,
366 target
, unsignedp
, methods
);
370 emit_move_insn (target
, x
);
374 /* Create a new vector value in VMODE with all elements set to OP. The
375 mode of OP must be the element mode of VMODE. If OP is a constant,
376 then the return value will be a constant. */
379 expand_vector_broadcast (machine_mode vmode
, rtx op
)
384 gcc_checking_assert (VECTOR_MODE_P (vmode
));
386 if (valid_for_const_vector_p (vmode
, op
))
387 return gen_const_vec_duplicate (vmode
, op
);
389 insn_code icode
= optab_handler (vec_duplicate_optab
, vmode
);
390 if (icode
!= CODE_FOR_nothing
)
392 struct expand_operand ops
[2];
393 create_output_operand (&ops
[0], NULL_RTX
, vmode
);
394 create_input_operand (&ops
[1], op
, GET_MODE (op
));
395 expand_insn (icode
, 2, ops
);
399 if (!GET_MODE_NUNITS (vmode
).is_constant (&n
))
402 /* ??? If the target doesn't have a vec_init, then we have no easy way
403 of performing this operation. Most of this sort of generic support
404 is hidden away in the vector lowering support in gimple. */
405 icode
= convert_optab_handler (vec_init_optab
, vmode
,
406 GET_MODE_INNER (vmode
));
407 if (icode
== CODE_FOR_nothing
)
410 vec
= rtvec_alloc (n
);
411 for (int i
= 0; i
< n
; ++i
)
412 RTVEC_ELT (vec
, i
) = op
;
413 rtx ret
= gen_reg_rtx (vmode
);
414 emit_insn (GEN_FCN (icode
) (ret
, gen_rtx_PARALLEL (vmode
, vec
)));
419 /* This subroutine of expand_doubleword_shift handles the cases in which
420 the effective shift value is >= BITS_PER_WORD. The arguments and return
421 value are the same as for the parent routine, except that SUPERWORD_OP1
422 is the shift count to use when shifting OUTOF_INPUT into INTO_TARGET.
423 INTO_TARGET may be null if the caller has decided to calculate it. */
426 expand_superword_shift (optab binoptab
, rtx outof_input
, rtx superword_op1
,
427 rtx outof_target
, rtx into_target
,
428 int unsignedp
, enum optab_methods methods
)
430 if (into_target
!= 0)
431 if (!force_expand_binop (word_mode
, binoptab
, outof_input
, superword_op1
,
432 into_target
, unsignedp
, methods
))
435 if (outof_target
!= 0)
437 /* For a signed right shift, we must fill OUTOF_TARGET with copies
438 of the sign bit, otherwise we must fill it with zeros. */
439 if (binoptab
!= ashr_optab
)
440 emit_move_insn (outof_target
, CONST0_RTX (word_mode
));
442 if (!force_expand_binop (word_mode
, binoptab
, outof_input
,
443 gen_int_shift_amount (word_mode
,
445 outof_target
, unsignedp
, methods
))
451 /* This subroutine of expand_doubleword_shift handles the cases in which
452 the effective shift value is < BITS_PER_WORD. The arguments and return
453 value are the same as for the parent routine. */
456 expand_subword_shift (scalar_int_mode op1_mode
, optab binoptab
,
457 rtx outof_input
, rtx into_input
, rtx op1
,
458 rtx outof_target
, rtx into_target
,
459 int unsignedp
, enum optab_methods methods
,
460 unsigned HOST_WIDE_INT shift_mask
)
462 optab reverse_unsigned_shift
, unsigned_shift
;
465 reverse_unsigned_shift
= (binoptab
== ashl_optab
? lshr_optab
: ashl_optab
);
466 unsigned_shift
= (binoptab
== ashl_optab
? ashl_optab
: lshr_optab
);
468 /* The low OP1 bits of INTO_TARGET come from the high bits of OUTOF_INPUT.
469 We therefore need to shift OUTOF_INPUT by (BITS_PER_WORD - OP1) bits in
470 the opposite direction to BINOPTAB. */
471 if (CONSTANT_P (op1
) || shift_mask
>= BITS_PER_WORD
)
473 carries
= outof_input
;
474 tmp
= immed_wide_int_const (wi::shwi (BITS_PER_WORD
,
475 op1_mode
), op1_mode
);
476 tmp
= simplify_expand_binop (op1_mode
, sub_optab
, tmp
, op1
,
481 /* We must avoid shifting by BITS_PER_WORD bits since that is either
482 the same as a zero shift (if shift_mask == BITS_PER_WORD - 1) or
483 has unknown behavior. Do a single shift first, then shift by the
484 remainder. It's OK to use ~OP1 as the remainder if shift counts
485 are truncated to the mode size. */
486 carries
= expand_binop (word_mode
, reverse_unsigned_shift
,
487 outof_input
, const1_rtx
, 0, unsignedp
, methods
);
488 if (shift_mask
== BITS_PER_WORD
- 1)
490 tmp
= immed_wide_int_const
491 (wi::minus_one (GET_MODE_PRECISION (op1_mode
)), op1_mode
);
492 tmp
= simplify_expand_binop (op1_mode
, xor_optab
, op1
, tmp
,
497 tmp
= immed_wide_int_const (wi::shwi (BITS_PER_WORD
- 1,
498 op1_mode
), op1_mode
);
499 tmp
= simplify_expand_binop (op1_mode
, sub_optab
, tmp
, op1
,
503 if (tmp
== 0 || carries
== 0)
505 carries
= expand_binop (word_mode
, reverse_unsigned_shift
,
506 carries
, tmp
, 0, unsignedp
, methods
);
510 /* Shift INTO_INPUT logically by OP1. This is the last use of INTO_INPUT
511 so the result can go directly into INTO_TARGET if convenient. */
512 tmp
= expand_binop (word_mode
, unsigned_shift
, into_input
, op1
,
513 into_target
, unsignedp
, methods
);
517 /* Now OR in the bits carried over from OUTOF_INPUT. */
518 if (!force_expand_binop (word_mode
, ior_optab
, tmp
, carries
,
519 into_target
, unsignedp
, methods
))
522 /* Use a standard word_mode shift for the out-of half. */
523 if (outof_target
!= 0)
524 if (!force_expand_binop (word_mode
, binoptab
, outof_input
, op1
,
525 outof_target
, unsignedp
, methods
))
532 /* Try implementing expand_doubleword_shift using conditional moves.
533 The shift is by < BITS_PER_WORD if (CMP_CODE CMP1 CMP2) is true,
534 otherwise it is by >= BITS_PER_WORD. SUBWORD_OP1 and SUPERWORD_OP1
535 are the shift counts to use in the former and latter case. All other
536 arguments are the same as the parent routine. */
539 expand_doubleword_shift_condmove (scalar_int_mode op1_mode
, optab binoptab
,
540 enum rtx_code cmp_code
, rtx cmp1
, rtx cmp2
,
541 rtx outof_input
, rtx into_input
,
542 rtx subword_op1
, rtx superword_op1
,
543 rtx outof_target
, rtx into_target
,
544 int unsignedp
, enum optab_methods methods
,
545 unsigned HOST_WIDE_INT shift_mask
)
547 rtx outof_superword
, into_superword
;
549 /* Put the superword version of the output into OUTOF_SUPERWORD and
551 outof_superword
= outof_target
!= 0 ? gen_reg_rtx (word_mode
) : 0;
552 if (outof_target
!= 0 && subword_op1
== superword_op1
)
554 /* The value INTO_TARGET >> SUBWORD_OP1, which we later store in
555 OUTOF_TARGET, is the same as the value of INTO_SUPERWORD. */
556 into_superword
= outof_target
;
557 if (!expand_superword_shift (binoptab
, outof_input
, superword_op1
,
558 outof_superword
, 0, unsignedp
, methods
))
563 into_superword
= gen_reg_rtx (word_mode
);
564 if (!expand_superword_shift (binoptab
, outof_input
, superword_op1
,
565 outof_superword
, into_superword
,
570 /* Put the subword version directly in OUTOF_TARGET and INTO_TARGET. */
571 if (!expand_subword_shift (op1_mode
, binoptab
,
572 outof_input
, into_input
, subword_op1
,
573 outof_target
, into_target
,
574 unsignedp
, methods
, shift_mask
))
577 /* Select between them. Do the INTO half first because INTO_SUPERWORD
578 might be the current value of OUTOF_TARGET. */
579 if (!emit_conditional_move (into_target
, cmp_code
, cmp1
, cmp2
, op1_mode
,
580 into_target
, into_superword
, word_mode
, false))
583 if (outof_target
!= 0)
584 if (!emit_conditional_move (outof_target
, cmp_code
, cmp1
, cmp2
, op1_mode
,
585 outof_target
, outof_superword
,
592 /* Expand a doubleword shift (ashl, ashr or lshr) using word-mode shifts.
593 OUTOF_INPUT and INTO_INPUT are the two word-sized halves of the first
594 input operand; the shift moves bits in the direction OUTOF_INPUT->
595 INTO_TARGET. OUTOF_TARGET and INTO_TARGET are the equivalent words
596 of the target. OP1 is the shift count and OP1_MODE is its mode.
597 If OP1 is constant, it will have been truncated as appropriate
598 and is known to be nonzero.
600 If SHIFT_MASK is zero, the result of word shifts is undefined when the
601 shift count is outside the range [0, BITS_PER_WORD). This routine must
602 avoid generating such shifts for OP1s in the range [0, BITS_PER_WORD * 2).
604 If SHIFT_MASK is nonzero, all word-mode shift counts are effectively
605 masked by it and shifts in the range [BITS_PER_WORD, SHIFT_MASK) will
606 fill with zeros or sign bits as appropriate.
608 If SHIFT_MASK is BITS_PER_WORD - 1, this routine will synthesize
609 a doubleword shift whose equivalent mask is BITS_PER_WORD * 2 - 1.
610 Doing this preserves semantics required by SHIFT_COUNT_TRUNCATED.
611 In all other cases, shifts by values outside [0, BITS_PER_UNIT * 2)
614 BINOPTAB, UNSIGNEDP and METHODS are as for expand_binop. This function
615 may not use INTO_INPUT after modifying INTO_TARGET, and similarly for
616 OUTOF_INPUT and OUTOF_TARGET. OUTOF_TARGET can be null if the parent
617 function wants to calculate it itself.
619 Return true if the shift could be successfully synthesized. */
622 expand_doubleword_shift (scalar_int_mode op1_mode
, optab binoptab
,
623 rtx outof_input
, rtx into_input
, rtx op1
,
624 rtx outof_target
, rtx into_target
,
625 int unsignedp
, enum optab_methods methods
,
626 unsigned HOST_WIDE_INT shift_mask
)
628 rtx superword_op1
, tmp
, cmp1
, cmp2
;
629 enum rtx_code cmp_code
;
631 /* See if word-mode shifts by BITS_PER_WORD...BITS_PER_WORD * 2 - 1 will
632 fill the result with sign or zero bits as appropriate. If so, the value
633 of OUTOF_TARGET will always be (SHIFT OUTOF_INPUT OP1). Recursively call
634 this routine to calculate INTO_TARGET (which depends on both OUTOF_INPUT
635 and INTO_INPUT), then emit code to set up OUTOF_TARGET.
637 This isn't worthwhile for constant shifts since the optimizers will
638 cope better with in-range shift counts. */
639 if (shift_mask
>= BITS_PER_WORD
641 && !CONSTANT_P (op1
))
643 if (!expand_doubleword_shift (op1_mode
, binoptab
,
644 outof_input
, into_input
, op1
,
646 unsignedp
, methods
, shift_mask
))
648 if (!force_expand_binop (word_mode
, binoptab
, outof_input
, op1
,
649 outof_target
, unsignedp
, methods
))
654 /* Set CMP_CODE, CMP1 and CMP2 so that the rtx (CMP_CODE CMP1 CMP2)
655 is true when the effective shift value is less than BITS_PER_WORD.
656 Set SUPERWORD_OP1 to the shift count that should be used to shift
657 OUTOF_INPUT into INTO_TARGET when the condition is false. */
658 tmp
= immed_wide_int_const (wi::shwi (BITS_PER_WORD
, op1_mode
), op1_mode
);
659 if (!CONSTANT_P (op1
) && shift_mask
== BITS_PER_WORD
- 1)
661 /* Set CMP1 to OP1 & BITS_PER_WORD. The result is zero iff OP1
662 is a subword shift count. */
663 cmp1
= simplify_expand_binop (op1_mode
, and_optab
, op1
, tmp
,
665 cmp2
= CONST0_RTX (op1_mode
);
671 /* Set CMP1 to OP1 - BITS_PER_WORD. */
672 cmp1
= simplify_expand_binop (op1_mode
, sub_optab
, op1
, tmp
,
674 cmp2
= CONST0_RTX (op1_mode
);
676 superword_op1
= cmp1
;
681 /* If we can compute the condition at compile time, pick the
682 appropriate subroutine. */
683 tmp
= simplify_relational_operation (cmp_code
, SImode
, op1_mode
, cmp1
, cmp2
);
684 if (tmp
!= 0 && CONST_INT_P (tmp
))
686 if (tmp
== const0_rtx
)
687 return expand_superword_shift (binoptab
, outof_input
, superword_op1
,
688 outof_target
, into_target
,
691 return expand_subword_shift (op1_mode
, binoptab
,
692 outof_input
, into_input
, op1
,
693 outof_target
, into_target
,
694 unsignedp
, methods
, shift_mask
);
697 /* Try using conditional moves to generate straight-line code. */
698 if (HAVE_conditional_move
)
700 rtx_insn
*start
= get_last_insn ();
701 if (expand_doubleword_shift_condmove (op1_mode
, binoptab
,
702 cmp_code
, cmp1
, cmp2
,
703 outof_input
, into_input
,
705 outof_target
, into_target
,
706 unsignedp
, methods
, shift_mask
))
708 delete_insns_since (start
);
711 /* As a last resort, use branches to select the correct alternative. */
712 rtx_code_label
*subword_label
= gen_label_rtx ();
713 rtx_code_label
*done_label
= gen_label_rtx ();
716 do_compare_rtx_and_jump (cmp1
, cmp2
, cmp_code
, false, op1_mode
,
718 profile_probability::uninitialized ());
721 if (!expand_superword_shift (binoptab
, outof_input
, superword_op1
,
722 outof_target
, into_target
,
726 emit_jump_insn (targetm
.gen_jump (done_label
));
728 emit_label (subword_label
);
730 if (!expand_subword_shift (op1_mode
, binoptab
,
731 outof_input
, into_input
, op1
,
732 outof_target
, into_target
,
733 unsignedp
, methods
, shift_mask
))
736 emit_label (done_label
);
740 /* Subroutine of expand_binop. Perform a double word multiplication of
741 operands OP0 and OP1 both of mode MODE, which is exactly twice as wide
742 as the target's word_mode. This function return NULL_RTX if anything
743 goes wrong, in which case it may have already emitted instructions
744 which need to be deleted.
746 If we want to multiply two two-word values and have normal and widening
747 multiplies of single-word values, we can do this with three smaller
750 The multiplication proceeds as follows:
751 _______________________
752 [__op0_high_|__op0_low__]
753 _______________________
754 * [__op1_high_|__op1_low__]
755 _______________________________________________
756 _______________________
757 (1) [__op0_low__*__op1_low__]
758 _______________________
759 (2a) [__op0_low__*__op1_high_]
760 _______________________
761 (2b) [__op0_high_*__op1_low__]
762 _______________________
763 (3) [__op0_high_*__op1_high_]
766 This gives a 4-word result. Since we are only interested in the
767 lower 2 words, partial result (3) and the upper words of (2a) and
768 (2b) don't need to be calculated. Hence (2a) and (2b) can be
769 calculated using non-widening multiplication.
771 (1), however, needs to be calculated with an unsigned widening
772 multiplication. If this operation is not directly supported we
773 try using a signed widening multiplication and adjust the result.
774 This adjustment works as follows:
776 If both operands are positive then no adjustment is needed.
778 If the operands have different signs, for example op0_low < 0 and
779 op1_low >= 0, the instruction treats the most significant bit of
780 op0_low as a sign bit instead of a bit with significance
781 2**(BITS_PER_WORD-1), i.e. the instruction multiplies op1_low
782 with 2**BITS_PER_WORD - op0_low, and two's complements the
783 result. Conclusion: We need to add op1_low * 2**BITS_PER_WORD to
786 Similarly, if both operands are negative, we need to add
787 (op0_low + op1_low) * 2**BITS_PER_WORD.
789 We use a trick to adjust quickly. We logically shift op0_low right
790 (op1_low) BITS_PER_WORD-1 steps to get 0 or 1, and add this to
791 op0_high (op1_high) before it is used to calculate 2b (2a). If no
792 logical shift exists, we do an arithmetic right shift and subtract
796 expand_doubleword_mult (machine_mode mode
, rtx op0
, rtx op1
, rtx target
,
797 bool umulp
, enum optab_methods methods
)
799 int low
= (WORDS_BIG_ENDIAN
? 1 : 0);
800 int high
= (WORDS_BIG_ENDIAN
? 0 : 1);
801 rtx wordm1
= (umulp
? NULL_RTX
802 : gen_int_shift_amount (word_mode
, BITS_PER_WORD
- 1));
803 rtx product
, adjust
, product_high
, temp
;
805 rtx op0_high
= operand_subword_force (op0
, high
, mode
);
806 rtx op0_low
= operand_subword_force (op0
, low
, mode
);
807 rtx op1_high
= operand_subword_force (op1
, high
, mode
);
808 rtx op1_low
= operand_subword_force (op1
, low
, mode
);
810 /* If we're using an unsigned multiply to directly compute the product
811 of the low-order words of the operands and perform any required
812 adjustments of the operands, we begin by trying two more multiplications
813 and then computing the appropriate sum.
815 We have checked above that the required addition is provided.
816 Full-word addition will normally always succeed, especially if
817 it is provided at all, so we don't worry about its failure. The
818 multiplication may well fail, however, so we do handle that. */
822 /* ??? This could be done with emit_store_flag where available. */
823 temp
= expand_binop (word_mode
, lshr_optab
, op0_low
, wordm1
,
824 NULL_RTX
, 1, methods
);
826 op0_high
= expand_binop (word_mode
, add_optab
, op0_high
, temp
,
827 NULL_RTX
, 0, OPTAB_DIRECT
);
830 temp
= expand_binop (word_mode
, ashr_optab
, op0_low
, wordm1
,
831 NULL_RTX
, 0, methods
);
834 op0_high
= expand_binop (word_mode
, sub_optab
, op0_high
, temp
,
835 NULL_RTX
, 0, OPTAB_DIRECT
);
842 adjust
= expand_binop (word_mode
, smul_optab
, op0_high
, op1_low
,
843 NULL_RTX
, 0, OPTAB_DIRECT
);
847 /* OP0_HIGH should now be dead. */
851 /* ??? This could be done with emit_store_flag where available. */
852 temp
= expand_binop (word_mode
, lshr_optab
, op1_low
, wordm1
,
853 NULL_RTX
, 1, methods
);
855 op1_high
= expand_binop (word_mode
, add_optab
, op1_high
, temp
,
856 NULL_RTX
, 0, OPTAB_DIRECT
);
859 temp
= expand_binop (word_mode
, ashr_optab
, op1_low
, wordm1
,
860 NULL_RTX
, 0, methods
);
863 op1_high
= expand_binop (word_mode
, sub_optab
, op1_high
, temp
,
864 NULL_RTX
, 0, OPTAB_DIRECT
);
871 temp
= expand_binop (word_mode
, smul_optab
, op1_high
, op0_low
,
872 NULL_RTX
, 0, OPTAB_DIRECT
);
876 /* OP1_HIGH should now be dead. */
878 adjust
= expand_binop (word_mode
, add_optab
, adjust
, temp
,
879 NULL_RTX
, 0, OPTAB_DIRECT
);
881 if (target
&& !REG_P (target
))
884 /* *_widen_optab needs to determine operand mode, make sure at least
885 one operand has non-VOID mode. */
886 if (GET_MODE (op0_low
) == VOIDmode
&& GET_MODE (op1_low
) == VOIDmode
)
887 op0_low
= force_reg (word_mode
, op0_low
);
890 product
= expand_binop (mode
, umul_widen_optab
, op0_low
, op1_low
,
891 target
, 1, OPTAB_DIRECT
);
893 product
= expand_binop (mode
, smul_widen_optab
, op0_low
, op1_low
,
894 target
, 1, OPTAB_DIRECT
);
899 product_high
= operand_subword (product
, high
, 1, mode
);
900 adjust
= expand_binop (word_mode
, add_optab
, product_high
, adjust
,
901 NULL_RTX
, 0, OPTAB_DIRECT
);
902 emit_move_insn (product_high
, adjust
);
906 /* Wrapper around expand_binop which takes an rtx code to specify
907 the operation to perform, not an optab pointer. All other
908 arguments are the same. */
910 expand_simple_binop (machine_mode mode
, enum rtx_code code
, rtx op0
,
911 rtx op1
, rtx target
, int unsignedp
,
912 enum optab_methods methods
)
914 optab binop
= code_to_optab (code
);
917 return expand_binop (mode
, binop
, op0
, op1
, target
, unsignedp
, methods
);
920 /* Return whether OP0 and OP1 should be swapped when expanding a commutative
921 binop. Order them according to commutative_operand_precedence and, if
922 possible, try to put TARGET or a pseudo first. */
924 swap_commutative_operands_with_target (rtx target
, rtx op0
, rtx op1
)
926 int op0_prec
= commutative_operand_precedence (op0
);
927 int op1_prec
= commutative_operand_precedence (op1
);
929 if (op0_prec
< op1_prec
)
932 if (op0_prec
> op1_prec
)
935 /* With equal precedence, both orders are ok, but it is better if the
936 first operand is TARGET, or if both TARGET and OP0 are pseudos. */
937 if (target
== 0 || REG_P (target
))
938 return (REG_P (op1
) && !REG_P (op0
)) || target
== op1
;
940 return rtx_equal_p (op1
, target
);
943 /* Return true if BINOPTAB implements a shift operation. */
946 shift_optab_p (optab binoptab
)
948 switch (optab_to_code (binoptab
))
964 /* Return true if BINOPTAB implements a commutative binary operation. */
967 commutative_optab_p (optab binoptab
)
969 return (GET_RTX_CLASS (optab_to_code (binoptab
)) == RTX_COMM_ARITH
970 || binoptab
== smul_widen_optab
971 || binoptab
== umul_widen_optab
972 || binoptab
== smul_highpart_optab
973 || binoptab
== umul_highpart_optab
);
976 /* X is to be used in mode MODE as operand OPN to BINOPTAB. If we're
977 optimizing, and if the operand is a constant that costs more than
978 1 instruction, force the constant into a register and return that
979 register. Return X otherwise. UNSIGNEDP says whether X is unsigned. */
982 avoid_expensive_constant (machine_mode mode
, optab binoptab
,
983 int opn
, rtx x
, bool unsignedp
)
985 bool speed
= optimize_insn_for_speed_p ();
990 && (rtx_cost (x
, mode
, optab_to_code (binoptab
), opn
, speed
)
991 > set_src_cost (x
, mode
, speed
)))
995 HOST_WIDE_INT intval
= trunc_int_for_mode (INTVAL (x
), mode
);
996 if (intval
!= INTVAL (x
))
997 x
= GEN_INT (intval
);
1000 x
= convert_modes (mode
, VOIDmode
, x
, unsignedp
);
1001 x
= force_reg (mode
, x
);
1006 /* Helper function for expand_binop: handle the case where there
1007 is an insn ICODE that directly implements the indicated operation.
1008 Returns null if this is not possible. */
1010 expand_binop_directly (enum insn_code icode
, machine_mode mode
, optab binoptab
,
1012 rtx target
, int unsignedp
, enum optab_methods methods
,
1015 machine_mode xmode0
= insn_data
[(int) icode
].operand
[1].mode
;
1016 machine_mode xmode1
= insn_data
[(int) icode
].operand
[2].mode
;
1017 machine_mode mode0
, mode1
, tmp_mode
;
1018 struct expand_operand ops
[3];
1021 rtx xop0
= op0
, xop1
= op1
;
1022 bool canonicalize_op1
= false;
1024 /* If it is a commutative operator and the modes would match
1025 if we would swap the operands, we can save the conversions. */
1026 commutative_p
= commutative_optab_p (binoptab
);
1028 && GET_MODE (xop0
) != xmode0
&& GET_MODE (xop1
) != xmode1
1029 && GET_MODE (xop0
) == xmode1
&& GET_MODE (xop1
) == xmode1
)
1030 std::swap (xop0
, xop1
);
1032 /* If we are optimizing, force expensive constants into a register. */
1033 xop0
= avoid_expensive_constant (xmode0
, binoptab
, 0, xop0
, unsignedp
);
1034 if (!shift_optab_p (binoptab
))
1035 xop1
= avoid_expensive_constant (xmode1
, binoptab
, 1, xop1
, unsignedp
);
1037 /* Shifts and rotates often use a different mode for op1 from op0;
1038 for VOIDmode constants we don't know the mode, so force it
1039 to be canonicalized using convert_modes. */
1040 canonicalize_op1
= true;
1042 /* In case the insn wants input operands in modes different from
1043 those of the actual operands, convert the operands. It would
1044 seem that we don't need to convert CONST_INTs, but we do, so
1045 that they're properly zero-extended, sign-extended or truncated
1048 mode0
= GET_MODE (xop0
) != VOIDmode
? GET_MODE (xop0
) : mode
;
1049 if (xmode0
!= VOIDmode
&& xmode0
!= mode0
)
1051 xop0
= convert_modes (xmode0
, mode0
, xop0
, unsignedp
);
1055 mode1
= ((GET_MODE (xop1
) != VOIDmode
|| canonicalize_op1
)
1056 ? GET_MODE (xop1
) : mode
);
1057 if (xmode1
!= VOIDmode
&& xmode1
!= mode1
)
1059 xop1
= convert_modes (xmode1
, mode1
, xop1
, unsignedp
);
1063 /* If operation is commutative,
1064 try to make the first operand a register.
1065 Even better, try to make it the same as the target.
1066 Also try to make the last operand a constant. */
1068 && swap_commutative_operands_with_target (target
, xop0
, xop1
))
1069 std::swap (xop0
, xop1
);
1071 /* Now, if insn's predicates don't allow our operands, put them into
1074 if (binoptab
== vec_pack_trunc_optab
1075 || binoptab
== vec_pack_usat_optab
1076 || binoptab
== vec_pack_ssat_optab
1077 || binoptab
== vec_pack_ufix_trunc_optab
1078 || binoptab
== vec_pack_sfix_trunc_optab
1079 || binoptab
== vec_packu_float_optab
1080 || binoptab
== vec_packs_float_optab
)
1082 /* The mode of the result is different then the mode of the
1084 tmp_mode
= insn_data
[(int) icode
].operand
[0].mode
;
1085 if (VECTOR_MODE_P (mode
)
1086 && maybe_ne (GET_MODE_NUNITS (tmp_mode
), 2 * GET_MODE_NUNITS (mode
)))
1088 delete_insns_since (last
);
1095 create_output_operand (&ops
[0], target
, tmp_mode
);
1096 create_input_operand (&ops
[1], xop0
, mode0
);
1097 create_input_operand (&ops
[2], xop1
, mode1
);
1098 pat
= maybe_gen_insn (icode
, 3, ops
);
1101 /* If PAT is composed of more than one insn, try to add an appropriate
1102 REG_EQUAL note to it. If we can't because TEMP conflicts with an
1103 operand, call expand_binop again, this time without a target. */
1104 if (INSN_P (pat
) && NEXT_INSN (pat
) != NULL_RTX
1105 && ! add_equal_note (pat
, ops
[0].value
,
1106 optab_to_code (binoptab
),
1107 ops
[1].value
, ops
[2].value
))
1109 delete_insns_since (last
);
1110 return expand_binop (mode
, binoptab
, op0
, op1
, NULL_RTX
,
1111 unsignedp
, methods
);
1115 return ops
[0].value
;
1117 delete_insns_since (last
);
1121 /* Generate code to perform an operation specified by BINOPTAB
1122 on operands OP0 and OP1, with result having machine-mode MODE.
1124 UNSIGNEDP is for the case where we have to widen the operands
1125 to perform the operation. It says to use zero-extension.
1127 If TARGET is nonzero, the value
1128 is generated there, if it is convenient to do so.
1129 In all cases an rtx is returned for the locus of the value;
1130 this may or may not be TARGET. */
1133 expand_binop (machine_mode mode
, optab binoptab
, rtx op0
, rtx op1
,
1134 rtx target
, int unsignedp
, enum optab_methods methods
)
1136 enum optab_methods next_methods
1137 = (methods
== OPTAB_LIB
|| methods
== OPTAB_LIB_WIDEN
1138 ? OPTAB_WIDEN
: methods
);
1139 enum mode_class mclass
;
1140 enum insn_code icode
;
1141 machine_mode wider_mode
;
1142 scalar_int_mode int_mode
;
1145 rtx_insn
*entry_last
= get_last_insn ();
1148 mclass
= GET_MODE_CLASS (mode
);
1150 /* If subtracting an integer constant, convert this into an addition of
1151 the negated constant. */
1153 if (binoptab
== sub_optab
&& CONST_INT_P (op1
))
1155 op1
= negate_rtx (mode
, op1
);
1156 binoptab
= add_optab
;
1158 /* For shifts, constant invalid op1 might be expanded from different
1159 mode than MODE. As those are invalid, force them to a register
1160 to avoid further problems during expansion. */
1161 else if (CONST_INT_P (op1
)
1162 && shift_optab_p (binoptab
)
1163 && UINTVAL (op1
) >= GET_MODE_BITSIZE (GET_MODE_INNER (mode
)))
1165 op1
= gen_int_mode (INTVAL (op1
), GET_MODE_INNER (mode
));
1166 op1
= force_reg (GET_MODE_INNER (mode
), op1
);
1169 /* Record where to delete back to if we backtrack. */
1170 last
= get_last_insn ();
1172 /* If we can do it with a three-operand insn, do so. */
1174 if (methods
!= OPTAB_MUST_WIDEN
)
1176 if (convert_optab_p (binoptab
))
1178 machine_mode from_mode
= widened_mode (mode
, op0
, op1
);
1179 icode
= find_widening_optab_handler (binoptab
, mode
, from_mode
);
1182 icode
= optab_handler (binoptab
, mode
);
1183 if (icode
!= CODE_FOR_nothing
)
1185 temp
= expand_binop_directly (icode
, mode
, binoptab
, op0
, op1
,
1186 target
, unsignedp
, methods
, last
);
1192 /* If we were trying to rotate, and that didn't work, try rotating
1193 the other direction before falling back to shifts and bitwise-or. */
1194 if (((binoptab
== rotl_optab
1195 && (icode
= optab_handler (rotr_optab
, mode
)) != CODE_FOR_nothing
)
1196 || (binoptab
== rotr_optab
1197 && (icode
= optab_handler (rotl_optab
, mode
)) != CODE_FOR_nothing
))
1198 && is_int_mode (mode
, &int_mode
))
1200 optab otheroptab
= (binoptab
== rotl_optab
? rotr_optab
: rotl_optab
);
1202 unsigned int bits
= GET_MODE_PRECISION (int_mode
);
1204 if (CONST_INT_P (op1
))
1205 newop1
= gen_int_shift_amount (int_mode
, bits
- INTVAL (op1
));
1206 else if (targetm
.shift_truncation_mask (int_mode
) == bits
- 1)
1207 newop1
= negate_rtx (GET_MODE (op1
), op1
);
1209 newop1
= expand_binop (GET_MODE (op1
), sub_optab
,
1210 gen_int_mode (bits
, GET_MODE (op1
)), op1
,
1211 NULL_RTX
, unsignedp
, OPTAB_DIRECT
);
1213 temp
= expand_binop_directly (icode
, int_mode
, otheroptab
, op0
, newop1
,
1214 target
, unsignedp
, methods
, last
);
1219 /* If this is a multiply, see if we can do a widening operation that
1220 takes operands of this mode and makes a wider mode. */
1222 if (binoptab
== smul_optab
1223 && GET_MODE_2XWIDER_MODE (mode
).exists (&wider_mode
)
1224 && (convert_optab_handler ((unsignedp
1226 : smul_widen_optab
),
1227 wider_mode
, mode
) != CODE_FOR_nothing
))
1229 /* *_widen_optab needs to determine operand mode, make sure at least
1230 one operand has non-VOID mode. */
1231 if (GET_MODE (op0
) == VOIDmode
&& GET_MODE (op1
) == VOIDmode
)
1232 op0
= force_reg (mode
, op0
);
1233 temp
= expand_binop (wider_mode
,
1234 unsignedp
? umul_widen_optab
: smul_widen_optab
,
1235 op0
, op1
, NULL_RTX
, unsignedp
, OPTAB_DIRECT
);
1239 if (GET_MODE_CLASS (mode
) == MODE_INT
1240 && TRULY_NOOP_TRUNCATION_MODES_P (mode
, GET_MODE (temp
)))
1241 return gen_lowpart (mode
, temp
);
1243 return convert_to_mode (mode
, temp
, unsignedp
);
1247 /* If this is a vector shift by a scalar, see if we can do a vector
1248 shift by a vector. If so, broadcast the scalar into a vector. */
1249 if (mclass
== MODE_VECTOR_INT
)
1251 optab otheroptab
= unknown_optab
;
1253 if (binoptab
== ashl_optab
)
1254 otheroptab
= vashl_optab
;
1255 else if (binoptab
== ashr_optab
)
1256 otheroptab
= vashr_optab
;
1257 else if (binoptab
== lshr_optab
)
1258 otheroptab
= vlshr_optab
;
1259 else if (binoptab
== rotl_optab
)
1260 otheroptab
= vrotl_optab
;
1261 else if (binoptab
== rotr_optab
)
1262 otheroptab
= vrotr_optab
;
1265 && (icode
= optab_handler (otheroptab
, mode
)) != CODE_FOR_nothing
)
1267 /* The scalar may have been extended to be too wide. Truncate
1268 it back to the proper size to fit in the broadcast vector. */
1269 scalar_mode inner_mode
= GET_MODE_INNER (mode
);
1270 if (!CONST_INT_P (op1
)
1271 && (GET_MODE_BITSIZE (as_a
<scalar_int_mode
> (GET_MODE (op1
)))
1272 > GET_MODE_BITSIZE (inner_mode
)))
1273 op1
= force_reg (inner_mode
,
1274 simplify_gen_unary (TRUNCATE
, inner_mode
, op1
,
1276 rtx vop1
= expand_vector_broadcast (mode
, op1
);
1279 temp
= expand_binop_directly (icode
, mode
, otheroptab
, op0
, vop1
,
1280 target
, unsignedp
, methods
, last
);
1287 /* Look for a wider mode of the same class for which we think we
1288 can open-code the operation. Check for a widening multiply at the
1289 wider mode as well. */
1291 if (CLASS_HAS_WIDER_MODES_P (mclass
)
1292 && methods
!= OPTAB_DIRECT
&& methods
!= OPTAB_LIB
)
1293 FOR_EACH_WIDER_MODE (wider_mode
, mode
)
1295 machine_mode next_mode
;
1296 if (optab_handler (binoptab
, wider_mode
) != CODE_FOR_nothing
1297 || (binoptab
== smul_optab
1298 && GET_MODE_WIDER_MODE (wider_mode
).exists (&next_mode
)
1299 && (find_widening_optab_handler ((unsignedp
1301 : smul_widen_optab
),
1303 != CODE_FOR_nothing
)))
1305 rtx xop0
= op0
, xop1
= op1
;
1308 /* For certain integer operations, we need not actually extend
1309 the narrow operands, as long as we will truncate
1310 the results to the same narrowness. */
1312 if ((binoptab
== ior_optab
|| binoptab
== and_optab
1313 || binoptab
== xor_optab
1314 || binoptab
== add_optab
|| binoptab
== sub_optab
1315 || binoptab
== smul_optab
|| binoptab
== ashl_optab
)
1316 && mclass
== MODE_INT
)
1319 xop0
= avoid_expensive_constant (mode
, binoptab
, 0,
1321 if (binoptab
!= ashl_optab
)
1322 xop1
= avoid_expensive_constant (mode
, binoptab
, 1,
1326 xop0
= widen_operand (xop0
, wider_mode
, mode
, unsignedp
, no_extend
);
1328 /* The second operand of a shift must always be extended. */
1329 xop1
= widen_operand (xop1
, wider_mode
, mode
, unsignedp
,
1330 no_extend
&& binoptab
!= ashl_optab
);
1332 temp
= expand_binop (wider_mode
, binoptab
, xop0
, xop1
, NULL_RTX
,
1333 unsignedp
, OPTAB_DIRECT
);
1336 if (mclass
!= MODE_INT
1337 || !TRULY_NOOP_TRUNCATION_MODES_P (mode
, wider_mode
))
1340 target
= gen_reg_rtx (mode
);
1341 convert_move (target
, temp
, 0);
1345 return gen_lowpart (mode
, temp
);
1348 delete_insns_since (last
);
1352 /* If operation is commutative,
1353 try to make the first operand a register.
1354 Even better, try to make it the same as the target.
1355 Also try to make the last operand a constant. */
1356 if (commutative_optab_p (binoptab
)
1357 && swap_commutative_operands_with_target (target
, op0
, op1
))
1358 std::swap (op0
, op1
);
1360 /* These can be done a word at a time. */
1361 if ((binoptab
== and_optab
|| binoptab
== ior_optab
|| binoptab
== xor_optab
)
1362 && is_int_mode (mode
, &int_mode
)
1363 && GET_MODE_SIZE (int_mode
) > UNITS_PER_WORD
1364 && optab_handler (binoptab
, word_mode
) != CODE_FOR_nothing
)
1369 /* If TARGET is the same as one of the operands, the REG_EQUAL note
1370 won't be accurate, so use a new target. */
1374 || !valid_multiword_target_p (target
))
1375 target
= gen_reg_rtx (int_mode
);
1379 /* Do the actual arithmetic. */
1380 enum machine_mode op0_mode
= CONSTANT_P (op0
) ? int_mode
: VOIDmode
;
1381 enum machine_mode op1_mode
= CONSTANT_P (op1
) ? int_mode
: VOIDmode
;
1382 for (i
= 0; i
< GET_MODE_BITSIZE (int_mode
) / BITS_PER_WORD
; i
++)
1384 rtx target_piece
= operand_subword (target
, i
, 1, int_mode
);
1385 rtx x
= expand_binop (word_mode
, binoptab
,
1386 operand_subword_force (op0
, i
, op0_mode
),
1387 operand_subword_force (op1
, i
, op1_mode
),
1388 target_piece
, unsignedp
, next_methods
);
1393 if (target_piece
!= x
)
1394 emit_move_insn (target_piece
, x
);
1397 insns
= get_insns ();
1400 if (i
== GET_MODE_BITSIZE (int_mode
) / BITS_PER_WORD
)
1407 /* Synthesize double word shifts from single word shifts. */
1408 if ((binoptab
== lshr_optab
|| binoptab
== ashl_optab
1409 || binoptab
== ashr_optab
)
1410 && is_int_mode (mode
, &int_mode
)
1411 && (CONST_INT_P (op1
) || optimize_insn_for_speed_p ())
1412 && GET_MODE_SIZE (int_mode
) == 2 * UNITS_PER_WORD
1413 && GET_MODE_PRECISION (int_mode
) == GET_MODE_BITSIZE (int_mode
)
1414 && optab_handler (binoptab
, word_mode
) != CODE_FOR_nothing
1415 && optab_handler (ashl_optab
, word_mode
) != CODE_FOR_nothing
1416 && optab_handler (lshr_optab
, word_mode
) != CODE_FOR_nothing
)
1418 unsigned HOST_WIDE_INT shift_mask
, double_shift_mask
;
1419 scalar_int_mode op1_mode
;
1421 double_shift_mask
= targetm
.shift_truncation_mask (int_mode
);
1422 shift_mask
= targetm
.shift_truncation_mask (word_mode
);
1423 op1_mode
= (GET_MODE (op1
) != VOIDmode
1424 ? as_a
<scalar_int_mode
> (GET_MODE (op1
))
1427 /* Apply the truncation to constant shifts. */
1428 if (double_shift_mask
> 0 && CONST_INT_P (op1
))
1429 op1
= gen_int_mode (INTVAL (op1
) & double_shift_mask
, op1_mode
);
1431 if (op1
== CONST0_RTX (op1_mode
))
1434 /* Make sure that this is a combination that expand_doubleword_shift
1435 can handle. See the comments there for details. */
1436 if (double_shift_mask
== 0
1437 || (shift_mask
== BITS_PER_WORD
- 1
1438 && double_shift_mask
== BITS_PER_WORD
* 2 - 1))
1441 rtx into_target
, outof_target
;
1442 rtx into_input
, outof_input
;
1443 int left_shift
, outof_word
;
1445 /* If TARGET is the same as one of the operands, the REG_EQUAL note
1446 won't be accurate, so use a new target. */
1450 || !valid_multiword_target_p (target
))
1451 target
= gen_reg_rtx (int_mode
);
1455 /* OUTOF_* is the word we are shifting bits away from, and
1456 INTO_* is the word that we are shifting bits towards, thus
1457 they differ depending on the direction of the shift and
1458 WORDS_BIG_ENDIAN. */
1460 left_shift
= binoptab
== ashl_optab
;
1461 outof_word
= left_shift
^ ! WORDS_BIG_ENDIAN
;
1463 outof_target
= operand_subword (target
, outof_word
, 1, int_mode
);
1464 into_target
= operand_subword (target
, 1 - outof_word
, 1, int_mode
);
1466 outof_input
= operand_subword_force (op0
, outof_word
, int_mode
);
1467 into_input
= operand_subword_force (op0
, 1 - outof_word
, int_mode
);
1469 if (expand_doubleword_shift (op1_mode
, binoptab
,
1470 outof_input
, into_input
, op1
,
1471 outof_target
, into_target
,
1472 unsignedp
, next_methods
, shift_mask
))
1474 insns
= get_insns ();
1484 /* Synthesize double word rotates from single word shifts. */
1485 if ((binoptab
== rotl_optab
|| binoptab
== rotr_optab
)
1486 && is_int_mode (mode
, &int_mode
)
1487 && CONST_INT_P (op1
)
1488 && GET_MODE_PRECISION (int_mode
) == 2 * BITS_PER_WORD
1489 && optab_handler (ashl_optab
, word_mode
) != CODE_FOR_nothing
1490 && optab_handler (lshr_optab
, word_mode
) != CODE_FOR_nothing
)
1493 rtx into_target
, outof_target
;
1494 rtx into_input
, outof_input
;
1496 int shift_count
, left_shift
, outof_word
;
1498 /* If TARGET is the same as one of the operands, the REG_EQUAL note
1499 won't be accurate, so use a new target. Do this also if target is not
1500 a REG, first because having a register instead may open optimization
1501 opportunities, and second because if target and op0 happen to be MEMs
1502 designating the same location, we would risk clobbering it too early
1503 in the code sequence we generate below. */
1508 || !valid_multiword_target_p (target
))
1509 target
= gen_reg_rtx (int_mode
);
1513 shift_count
= INTVAL (op1
);
1515 /* OUTOF_* is the word we are shifting bits away from, and
1516 INTO_* is the word that we are shifting bits towards, thus
1517 they differ depending on the direction of the shift and
1518 WORDS_BIG_ENDIAN. */
1520 left_shift
= (binoptab
== rotl_optab
);
1521 outof_word
= left_shift
^ ! WORDS_BIG_ENDIAN
;
1523 outof_target
= operand_subword (target
, outof_word
, 1, int_mode
);
1524 into_target
= operand_subword (target
, 1 - outof_word
, 1, int_mode
);
1526 outof_input
= operand_subword_force (op0
, outof_word
, int_mode
);
1527 into_input
= operand_subword_force (op0
, 1 - outof_word
, int_mode
);
1529 if (shift_count
== BITS_PER_WORD
)
1531 /* This is just a word swap. */
1532 emit_move_insn (outof_target
, into_input
);
1533 emit_move_insn (into_target
, outof_input
);
1538 rtx into_temp1
, into_temp2
, outof_temp1
, outof_temp2
;
1539 HOST_WIDE_INT first_shift_count
, second_shift_count
;
1540 optab reverse_unsigned_shift
, unsigned_shift
;
1542 reverse_unsigned_shift
= (left_shift
^ (shift_count
< BITS_PER_WORD
)
1543 ? lshr_optab
: ashl_optab
);
1545 unsigned_shift
= (left_shift
^ (shift_count
< BITS_PER_WORD
)
1546 ? ashl_optab
: lshr_optab
);
1548 if (shift_count
> BITS_PER_WORD
)
1550 first_shift_count
= shift_count
- BITS_PER_WORD
;
1551 second_shift_count
= 2 * BITS_PER_WORD
- shift_count
;
1555 first_shift_count
= BITS_PER_WORD
- shift_count
;
1556 second_shift_count
= shift_count
;
1558 rtx first_shift_count_rtx
1559 = gen_int_shift_amount (word_mode
, first_shift_count
);
1560 rtx second_shift_count_rtx
1561 = gen_int_shift_amount (word_mode
, second_shift_count
);
1563 into_temp1
= expand_binop (word_mode
, unsigned_shift
,
1564 outof_input
, first_shift_count_rtx
,
1565 NULL_RTX
, unsignedp
, next_methods
);
1566 into_temp2
= expand_binop (word_mode
, reverse_unsigned_shift
,
1567 into_input
, second_shift_count_rtx
,
1568 NULL_RTX
, unsignedp
, next_methods
);
1570 if (into_temp1
!= 0 && into_temp2
!= 0)
1571 inter
= expand_binop (word_mode
, ior_optab
, into_temp1
, into_temp2
,
1572 into_target
, unsignedp
, next_methods
);
1576 if (inter
!= 0 && inter
!= into_target
)
1577 emit_move_insn (into_target
, inter
);
1579 outof_temp1
= expand_binop (word_mode
, unsigned_shift
,
1580 into_input
, first_shift_count_rtx
,
1581 NULL_RTX
, unsignedp
, next_methods
);
1582 outof_temp2
= expand_binop (word_mode
, reverse_unsigned_shift
,
1583 outof_input
, second_shift_count_rtx
,
1584 NULL_RTX
, unsignedp
, next_methods
);
1586 if (inter
!= 0 && outof_temp1
!= 0 && outof_temp2
!= 0)
1587 inter
= expand_binop (word_mode
, ior_optab
,
1588 outof_temp1
, outof_temp2
,
1589 outof_target
, unsignedp
, next_methods
);
1591 if (inter
!= 0 && inter
!= outof_target
)
1592 emit_move_insn (outof_target
, inter
);
1595 insns
= get_insns ();
1605 /* These can be done a word at a time by propagating carries. */
1606 if ((binoptab
== add_optab
|| binoptab
== sub_optab
)
1607 && is_int_mode (mode
, &int_mode
)
1608 && GET_MODE_SIZE (int_mode
) >= 2 * UNITS_PER_WORD
1609 && optab_handler (binoptab
, word_mode
) != CODE_FOR_nothing
)
1612 optab otheroptab
= binoptab
== add_optab
? sub_optab
: add_optab
;
1613 const unsigned int nwords
= GET_MODE_BITSIZE (int_mode
) / BITS_PER_WORD
;
1614 rtx carry_in
= NULL_RTX
, carry_out
= NULL_RTX
;
1615 rtx xop0
, xop1
, xtarget
;
1617 /* We can handle either a 1 or -1 value for the carry. If STORE_FLAG
1618 value is one of those, use it. Otherwise, use 1 since it is the
1619 one easiest to get. */
1620 #if STORE_FLAG_VALUE == 1 || STORE_FLAG_VALUE == -1
1621 int normalizep
= STORE_FLAG_VALUE
;
1626 /* Prepare the operands. */
1627 xop0
= force_reg (int_mode
, op0
);
1628 xop1
= force_reg (int_mode
, op1
);
1630 xtarget
= gen_reg_rtx (int_mode
);
1632 if (target
== 0 || !REG_P (target
) || !valid_multiword_target_p (target
))
1635 /* Indicate for flow that the entire target reg is being set. */
1637 emit_clobber (xtarget
);
1639 /* Do the actual arithmetic. */
1640 for (i
= 0; i
< nwords
; i
++)
1642 int index
= (WORDS_BIG_ENDIAN
? nwords
- i
- 1 : i
);
1643 rtx target_piece
= operand_subword (xtarget
, index
, 1, int_mode
);
1644 rtx op0_piece
= operand_subword_force (xop0
, index
, int_mode
);
1645 rtx op1_piece
= operand_subword_force (xop1
, index
, int_mode
);
1648 /* Main add/subtract of the input operands. */
1649 x
= expand_binop (word_mode
, binoptab
,
1650 op0_piece
, op1_piece
,
1651 target_piece
, unsignedp
, next_methods
);
1657 /* Store carry from main add/subtract. */
1658 carry_out
= gen_reg_rtx (word_mode
);
1659 carry_out
= emit_store_flag_force (carry_out
,
1660 (binoptab
== add_optab
1663 word_mode
, 1, normalizep
);
1670 /* Add/subtract previous carry to main result. */
1671 newx
= expand_binop (word_mode
,
1672 normalizep
== 1 ? binoptab
: otheroptab
,
1674 NULL_RTX
, 1, next_methods
);
1678 /* Get out carry from adding/subtracting carry in. */
1679 rtx carry_tmp
= gen_reg_rtx (word_mode
);
1680 carry_tmp
= emit_store_flag_force (carry_tmp
,
1681 (binoptab
== add_optab
1684 word_mode
, 1, normalizep
);
1686 /* Logical-ior the two poss. carry together. */
1687 carry_out
= expand_binop (word_mode
, ior_optab
,
1688 carry_out
, carry_tmp
,
1689 carry_out
, 0, next_methods
);
1693 emit_move_insn (target_piece
, newx
);
1697 if (x
!= target_piece
)
1698 emit_move_insn (target_piece
, x
);
1701 carry_in
= carry_out
;
1704 if (i
== GET_MODE_BITSIZE (int_mode
) / (unsigned) BITS_PER_WORD
)
1706 if (optab_handler (mov_optab
, int_mode
) != CODE_FOR_nothing
1707 || ! rtx_equal_p (target
, xtarget
))
1709 rtx_insn
*temp
= emit_move_insn (target
, xtarget
);
1711 set_dst_reg_note (temp
, REG_EQUAL
,
1712 gen_rtx_fmt_ee (optab_to_code (binoptab
),
1713 int_mode
, copy_rtx (xop0
),
1724 delete_insns_since (last
);
1727 /* Attempt to synthesize double word multiplies using a sequence of word
1728 mode multiplications. We first attempt to generate a sequence using a
1729 more efficient unsigned widening multiply, and if that fails we then
1730 try using a signed widening multiply. */
1732 if (binoptab
== smul_optab
1733 && is_int_mode (mode
, &int_mode
)
1734 && GET_MODE_SIZE (int_mode
) == 2 * UNITS_PER_WORD
1735 && optab_handler (smul_optab
, word_mode
) != CODE_FOR_nothing
1736 && optab_handler (add_optab
, word_mode
) != CODE_FOR_nothing
)
1738 rtx product
= NULL_RTX
;
1739 if (convert_optab_handler (umul_widen_optab
, int_mode
, word_mode
)
1740 != CODE_FOR_nothing
)
1742 product
= expand_doubleword_mult (int_mode
, op0
, op1
, target
,
1745 delete_insns_since (last
);
1748 if (product
== NULL_RTX
1749 && (convert_optab_handler (smul_widen_optab
, int_mode
, word_mode
)
1750 != CODE_FOR_nothing
))
1752 product
= expand_doubleword_mult (int_mode
, op0
, op1
, target
,
1755 delete_insns_since (last
);
1758 if (product
!= NULL_RTX
)
1760 if (optab_handler (mov_optab
, int_mode
) != CODE_FOR_nothing
)
1762 rtx_insn
*move
= emit_move_insn (target
? target
: product
,
1764 set_dst_reg_note (move
,
1766 gen_rtx_fmt_ee (MULT
, int_mode
,
1769 target
? target
: product
);
1775 /* It can't be open-coded in this mode.
1776 Use a library call if one is available and caller says that's ok. */
1778 libfunc
= optab_libfunc (binoptab
, mode
);
1780 && (methods
== OPTAB_LIB
|| methods
== OPTAB_LIB_WIDEN
))
1784 machine_mode op1_mode
= mode
;
1789 if (shift_optab_p (binoptab
))
1791 op1_mode
= targetm
.libgcc_shift_count_mode ();
1792 /* Specify unsigned here,
1793 since negative shift counts are meaningless. */
1794 op1x
= convert_to_mode (op1_mode
, op1
, 1);
1797 if (GET_MODE (op0
) != VOIDmode
1798 && GET_MODE (op0
) != mode
)
1799 op0
= convert_to_mode (mode
, op0
, unsignedp
);
1801 /* Pass 1 for NO_QUEUE so we don't lose any increments
1802 if the libcall is cse'd or moved. */
1803 value
= emit_library_call_value (libfunc
,
1804 NULL_RTX
, LCT_CONST
, mode
,
1805 op0
, mode
, op1x
, op1_mode
);
1807 insns
= get_insns ();
1810 bool trapv
= trapv_binoptab_p (binoptab
);
1811 target
= gen_reg_rtx (mode
);
1812 emit_libcall_block_1 (insns
, target
, value
,
1814 : gen_rtx_fmt_ee (optab_to_code (binoptab
),
1815 mode
, op0
, op1
), trapv
);
1820 delete_insns_since (last
);
1822 /* It can't be done in this mode. Can we do it in a wider mode? */
1824 if (! (methods
== OPTAB_WIDEN
|| methods
== OPTAB_LIB_WIDEN
1825 || methods
== OPTAB_MUST_WIDEN
))
1827 /* Caller says, don't even try. */
1828 delete_insns_since (entry_last
);
1832 /* Compute the value of METHODS to pass to recursive calls.
1833 Don't allow widening to be tried recursively. */
1835 methods
= (methods
== OPTAB_LIB_WIDEN
? OPTAB_LIB
: OPTAB_DIRECT
);
1837 /* Look for a wider mode of the same class for which it appears we can do
1840 if (CLASS_HAS_WIDER_MODES_P (mclass
))
1842 /* This code doesn't make sense for conversion optabs, since we
1843 wouldn't then want to extend the operands to be the same size
1845 gcc_assert (!convert_optab_p (binoptab
));
1846 FOR_EACH_WIDER_MODE (wider_mode
, mode
)
1848 if (optab_handler (binoptab
, wider_mode
)
1849 || (methods
== OPTAB_LIB
1850 && optab_libfunc (binoptab
, wider_mode
)))
1852 rtx xop0
= op0
, xop1
= op1
;
1855 /* For certain integer operations, we need not actually extend
1856 the narrow operands, as long as we will truncate
1857 the results to the same narrowness. */
1859 if ((binoptab
== ior_optab
|| binoptab
== and_optab
1860 || binoptab
== xor_optab
1861 || binoptab
== add_optab
|| binoptab
== sub_optab
1862 || binoptab
== smul_optab
|| binoptab
== ashl_optab
)
1863 && mclass
== MODE_INT
)
1866 xop0
= widen_operand (xop0
, wider_mode
, mode
,
1867 unsignedp
, no_extend
);
1869 /* The second operand of a shift must always be extended. */
1870 xop1
= widen_operand (xop1
, wider_mode
, mode
, unsignedp
,
1871 no_extend
&& binoptab
!= ashl_optab
);
1873 temp
= expand_binop (wider_mode
, binoptab
, xop0
, xop1
, NULL_RTX
,
1874 unsignedp
, methods
);
1877 if (mclass
!= MODE_INT
1878 || !TRULY_NOOP_TRUNCATION_MODES_P (mode
, wider_mode
))
1881 target
= gen_reg_rtx (mode
);
1882 convert_move (target
, temp
, 0);
1886 return gen_lowpart (mode
, temp
);
1889 delete_insns_since (last
);
1894 delete_insns_since (entry_last
);
1898 /* Expand a binary operator which has both signed and unsigned forms.
1899 UOPTAB is the optab for unsigned operations, and SOPTAB is for
1902 If we widen unsigned operands, we may use a signed wider operation instead
1903 of an unsigned wider operation, since the result would be the same. */
1906 sign_expand_binop (machine_mode mode
, optab uoptab
, optab soptab
,
1907 rtx op0
, rtx op1
, rtx target
, int unsignedp
,
1908 enum optab_methods methods
)
1911 optab direct_optab
= unsignedp
? uoptab
: soptab
;
1914 /* Do it without widening, if possible. */
1915 temp
= expand_binop (mode
, direct_optab
, op0
, op1
, target
,
1916 unsignedp
, OPTAB_DIRECT
);
1917 if (temp
|| methods
== OPTAB_DIRECT
)
1920 /* Try widening to a signed int. Disable any direct use of any
1921 signed insn in the current mode. */
1922 save_enable
= swap_optab_enable (soptab
, mode
, false);
1924 temp
= expand_binop (mode
, soptab
, op0
, op1
, target
,
1925 unsignedp
, OPTAB_WIDEN
);
1927 /* For unsigned operands, try widening to an unsigned int. */
1928 if (!temp
&& unsignedp
)
1929 temp
= expand_binop (mode
, uoptab
, op0
, op1
, target
,
1930 unsignedp
, OPTAB_WIDEN
);
1931 if (temp
|| methods
== OPTAB_WIDEN
)
1934 /* Use the right width libcall if that exists. */
1935 temp
= expand_binop (mode
, direct_optab
, op0
, op1
, target
,
1936 unsignedp
, OPTAB_LIB
);
1937 if (temp
|| methods
== OPTAB_LIB
)
1940 /* Must widen and use a libcall, use either signed or unsigned. */
1941 temp
= expand_binop (mode
, soptab
, op0
, op1
, target
,
1942 unsignedp
, methods
);
1943 if (!temp
&& unsignedp
)
1944 temp
= expand_binop (mode
, uoptab
, op0
, op1
, target
,
1945 unsignedp
, methods
);
1948 /* Undo the fiddling above. */
1950 swap_optab_enable (soptab
, mode
, true);
1954 /* Generate code to perform an operation specified by UNOPPTAB
1955 on operand OP0, with two results to TARG0 and TARG1.
1956 We assume that the order of the operands for the instruction
1957 is TARG0, TARG1, OP0.
1959 Either TARG0 or TARG1 may be zero, but what that means is that
1960 the result is not actually wanted. We will generate it into
1961 a dummy pseudo-reg and discard it. They may not both be zero.
1963 Returns 1 if this operation can be performed; 0 if not. */
1966 expand_twoval_unop (optab unoptab
, rtx op0
, rtx targ0
, rtx targ1
,
1969 machine_mode mode
= GET_MODE (targ0
? targ0
: targ1
);
1970 enum mode_class mclass
;
1971 machine_mode wider_mode
;
1972 rtx_insn
*entry_last
= get_last_insn ();
1975 mclass
= GET_MODE_CLASS (mode
);
1978 targ0
= gen_reg_rtx (mode
);
1980 targ1
= gen_reg_rtx (mode
);
1982 /* Record where to go back to if we fail. */
1983 last
= get_last_insn ();
1985 if (optab_handler (unoptab
, mode
) != CODE_FOR_nothing
)
1987 struct expand_operand ops
[3];
1988 enum insn_code icode
= optab_handler (unoptab
, mode
);
1990 create_fixed_operand (&ops
[0], targ0
);
1991 create_fixed_operand (&ops
[1], targ1
);
1992 create_convert_operand_from (&ops
[2], op0
, mode
, unsignedp
);
1993 if (maybe_expand_insn (icode
, 3, ops
))
1997 /* It can't be done in this mode. Can we do it in a wider mode? */
1999 if (CLASS_HAS_WIDER_MODES_P (mclass
))
2001 FOR_EACH_WIDER_MODE (wider_mode
, mode
)
2003 if (optab_handler (unoptab
, wider_mode
) != CODE_FOR_nothing
)
2005 rtx t0
= gen_reg_rtx (wider_mode
);
2006 rtx t1
= gen_reg_rtx (wider_mode
);
2007 rtx cop0
= convert_modes (wider_mode
, mode
, op0
, unsignedp
);
2009 if (expand_twoval_unop (unoptab
, cop0
, t0
, t1
, unsignedp
))
2011 convert_move (targ0
, t0
, unsignedp
);
2012 convert_move (targ1
, t1
, unsignedp
);
2016 delete_insns_since (last
);
2021 delete_insns_since (entry_last
);
2025 /* Generate code to perform an operation specified by BINOPTAB
2026 on operands OP0 and OP1, with two results to TARG1 and TARG2.
2027 We assume that the order of the operands for the instruction
2028 is TARG0, OP0, OP1, TARG1, which would fit a pattern like
2029 [(set TARG0 (operate OP0 OP1)) (set TARG1 (operate ...))].
2031 Either TARG0 or TARG1 may be zero, but what that means is that
2032 the result is not actually wanted. We will generate it into
2033 a dummy pseudo-reg and discard it. They may not both be zero.
2035 Returns 1 if this operation can be performed; 0 if not. */
2038 expand_twoval_binop (optab binoptab
, rtx op0
, rtx op1
, rtx targ0
, rtx targ1
,
2041 machine_mode mode
= GET_MODE (targ0
? targ0
: targ1
);
2042 enum mode_class mclass
;
2043 machine_mode wider_mode
;
2044 rtx_insn
*entry_last
= get_last_insn ();
2047 mclass
= GET_MODE_CLASS (mode
);
2050 targ0
= gen_reg_rtx (mode
);
2052 targ1
= gen_reg_rtx (mode
);
2054 /* Record where to go back to if we fail. */
2055 last
= get_last_insn ();
2057 if (optab_handler (binoptab
, mode
) != CODE_FOR_nothing
)
2059 struct expand_operand ops
[4];
2060 enum insn_code icode
= optab_handler (binoptab
, mode
);
2061 machine_mode mode0
= insn_data
[icode
].operand
[1].mode
;
2062 machine_mode mode1
= insn_data
[icode
].operand
[2].mode
;
2063 rtx xop0
= op0
, xop1
= op1
;
2065 /* If we are optimizing, force expensive constants into a register. */
2066 xop0
= avoid_expensive_constant (mode0
, binoptab
, 0, xop0
, unsignedp
);
2067 xop1
= avoid_expensive_constant (mode1
, binoptab
, 1, xop1
, unsignedp
);
2069 create_fixed_operand (&ops
[0], targ0
);
2070 create_convert_operand_from (&ops
[1], op0
, mode
, unsignedp
);
2071 create_convert_operand_from (&ops
[2], op1
, mode
, unsignedp
);
2072 create_fixed_operand (&ops
[3], targ1
);
2073 if (maybe_expand_insn (icode
, 4, ops
))
2075 delete_insns_since (last
);
2078 /* It can't be done in this mode. Can we do it in a wider mode? */
2080 if (CLASS_HAS_WIDER_MODES_P (mclass
))
2082 FOR_EACH_WIDER_MODE (wider_mode
, mode
)
2084 if (optab_handler (binoptab
, wider_mode
) != CODE_FOR_nothing
)
2086 rtx t0
= gen_reg_rtx (wider_mode
);
2087 rtx t1
= gen_reg_rtx (wider_mode
);
2088 rtx cop0
= convert_modes (wider_mode
, mode
, op0
, unsignedp
);
2089 rtx cop1
= convert_modes (wider_mode
, mode
, op1
, unsignedp
);
2091 if (expand_twoval_binop (binoptab
, cop0
, cop1
,
2094 convert_move (targ0
, t0
, unsignedp
);
2095 convert_move (targ1
, t1
, unsignedp
);
2099 delete_insns_since (last
);
2104 delete_insns_since (entry_last
);
2108 /* Expand the two-valued library call indicated by BINOPTAB, but
2109 preserve only one of the values. If TARG0 is non-NULL, the first
2110 value is placed into TARG0; otherwise the second value is placed
2111 into TARG1. Exactly one of TARG0 and TARG1 must be non-NULL. The
2112 value stored into TARG0 or TARG1 is equivalent to (CODE OP0 OP1).
2113 This routine assumes that the value returned by the library call is
2114 as if the return value was of an integral mode twice as wide as the
2115 mode of OP0. Returns 1 if the call was successful. */
2118 expand_twoval_binop_libfunc (optab binoptab
, rtx op0
, rtx op1
,
2119 rtx targ0
, rtx targ1
, enum rtx_code code
)
2122 machine_mode libval_mode
;
2127 /* Exactly one of TARG0 or TARG1 should be non-NULL. */
2128 gcc_assert (!targ0
!= !targ1
);
2130 mode
= GET_MODE (op0
);
2131 libfunc
= optab_libfunc (binoptab
, mode
);
2135 /* The value returned by the library function will have twice as
2136 many bits as the nominal MODE. */
2137 libval_mode
= smallest_int_mode_for_size (2 * GET_MODE_BITSIZE (mode
));
2139 libval
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
,
2143 /* Get the part of VAL containing the value that we want. */
2144 libval
= simplify_gen_subreg (mode
, libval
, libval_mode
,
2145 targ0
? 0 : GET_MODE_SIZE (mode
));
2146 insns
= get_insns ();
2148 /* Move the into the desired location. */
2149 emit_libcall_block (insns
, targ0
? targ0
: targ1
, libval
,
2150 gen_rtx_fmt_ee (code
, mode
, op0
, op1
));
2156 /* Wrapper around expand_unop which takes an rtx code to specify
2157 the operation to perform, not an optab pointer. All other
2158 arguments are the same. */
2160 expand_simple_unop (machine_mode mode
, enum rtx_code code
, rtx op0
,
2161 rtx target
, int unsignedp
)
2163 optab unop
= code_to_optab (code
);
2166 return expand_unop (mode
, unop
, op0
, target
, unsignedp
);
2172 (clz:wide (zero_extend:wide x)) - ((width wide) - (width narrow)).
2174 A similar operation can be used for clrsb. UNOPTAB says which operation
2175 we are trying to expand. */
2177 widen_leading (scalar_int_mode mode
, rtx op0
, rtx target
, optab unoptab
)
2179 opt_scalar_int_mode wider_mode_iter
;
2180 FOR_EACH_WIDER_MODE (wider_mode_iter
, mode
)
2182 scalar_int_mode wider_mode
= wider_mode_iter
.require ();
2183 if (optab_handler (unoptab
, wider_mode
) != CODE_FOR_nothing
)
2188 last
= get_last_insn ();
2191 target
= gen_reg_rtx (mode
);
2192 xop0
= widen_operand (op0
, wider_mode
, mode
,
2193 unoptab
!= clrsb_optab
, false);
2194 temp
= expand_unop (wider_mode
, unoptab
, xop0
, NULL_RTX
,
2195 unoptab
!= clrsb_optab
);
2198 (wider_mode
, sub_optab
, temp
,
2199 gen_int_mode (GET_MODE_PRECISION (wider_mode
)
2200 - GET_MODE_PRECISION (mode
),
2202 target
, true, OPTAB_DIRECT
);
2204 delete_insns_since (last
);
2212 /* Try calculating clz of a double-word quantity as two clz's of word-sized
2213 quantities, choosing which based on whether the high word is nonzero. */
2215 expand_doubleword_clz (scalar_int_mode mode
, rtx op0
, rtx target
)
2217 rtx xop0
= force_reg (mode
, op0
);
2218 rtx subhi
= gen_highpart (word_mode
, xop0
);
2219 rtx sublo
= gen_lowpart (word_mode
, xop0
);
2220 rtx_code_label
*hi0_label
= gen_label_rtx ();
2221 rtx_code_label
*after_label
= gen_label_rtx ();
2225 /* If we were not given a target, use a word_mode register, not a
2226 'mode' register. The result will fit, and nobody is expecting
2227 anything bigger (the return type of __builtin_clz* is int). */
2229 target
= gen_reg_rtx (word_mode
);
2231 /* In any case, write to a word_mode scratch in both branches of the
2232 conditional, so we can ensure there is a single move insn setting
2233 'target' to tag a REG_EQUAL note on. */
2234 result
= gen_reg_rtx (word_mode
);
2238 /* If the high word is not equal to zero,
2239 then clz of the full value is clz of the high word. */
2240 emit_cmp_and_jump_insns (subhi
, CONST0_RTX (word_mode
), EQ
, 0,
2241 word_mode
, true, hi0_label
);
2243 temp
= expand_unop_direct (word_mode
, clz_optab
, subhi
, result
, true);
2248 convert_move (result
, temp
, true);
2250 emit_jump_insn (targetm
.gen_jump (after_label
));
2253 /* Else clz of the full value is clz of the low word plus the number
2254 of bits in the high word. */
2255 emit_label (hi0_label
);
2257 temp
= expand_unop_direct (word_mode
, clz_optab
, sublo
, 0, true);
2260 temp
= expand_binop (word_mode
, add_optab
, temp
,
2261 gen_int_mode (GET_MODE_BITSIZE (word_mode
), word_mode
),
2262 result
, true, OPTAB_DIRECT
);
2266 convert_move (result
, temp
, true);
2268 emit_label (after_label
);
2269 convert_move (target
, result
, true);
2274 add_equal_note (seq
, target
, CLZ
, xop0
, 0);
2283 /* Try calculating popcount of a double-word quantity as two popcount's of
2284 word-sized quantities and summing up the results. */
2286 expand_doubleword_popcount (scalar_int_mode mode
, rtx op0
, rtx target
)
2293 t0
= expand_unop_direct (word_mode
, popcount_optab
,
2294 operand_subword_force (op0
, 0, mode
), NULL_RTX
,
2296 t1
= expand_unop_direct (word_mode
, popcount_optab
,
2297 operand_subword_force (op0
, 1, mode
), NULL_RTX
,
2305 /* If we were not given a target, use a word_mode register, not a
2306 'mode' register. The result will fit, and nobody is expecting
2307 anything bigger (the return type of __builtin_popcount* is int). */
2309 target
= gen_reg_rtx (word_mode
);
2311 t
= expand_binop (word_mode
, add_optab
, t0
, t1
, target
, 0, OPTAB_DIRECT
);
2316 add_equal_note (seq
, t
, POPCOUNT
, op0
, 0);
2324 (parity:narrow (low (x) ^ high (x))) */
2326 expand_doubleword_parity (scalar_int_mode mode
, rtx op0
, rtx target
)
2328 rtx t
= expand_binop (word_mode
, xor_optab
,
2329 operand_subword_force (op0
, 0, mode
),
2330 operand_subword_force (op0
, 1, mode
),
2331 NULL_RTX
, 0, OPTAB_DIRECT
);
2332 return expand_unop (word_mode
, parity_optab
, t
, target
, true);
2338 (lshiftrt:wide (bswap:wide x) ((width wide) - (width narrow))). */
2340 widen_bswap (scalar_int_mode mode
, rtx op0
, rtx target
)
2344 opt_scalar_int_mode wider_mode_iter
;
2346 FOR_EACH_WIDER_MODE (wider_mode_iter
, mode
)
2347 if (optab_handler (bswap_optab
, wider_mode_iter
.require ())
2348 != CODE_FOR_nothing
)
2351 if (!wider_mode_iter
.exists ())
2354 scalar_int_mode wider_mode
= wider_mode_iter
.require ();
2355 last
= get_last_insn ();
2357 x
= widen_operand (op0
, wider_mode
, mode
, true, true);
2358 x
= expand_unop (wider_mode
, bswap_optab
, x
, NULL_RTX
, true);
2360 gcc_assert (GET_MODE_PRECISION (wider_mode
) == GET_MODE_BITSIZE (wider_mode
)
2361 && GET_MODE_PRECISION (mode
) == GET_MODE_BITSIZE (mode
));
2363 x
= expand_shift (RSHIFT_EXPR
, wider_mode
, x
,
2364 GET_MODE_BITSIZE (wider_mode
)
2365 - GET_MODE_BITSIZE (mode
),
2371 target
= gen_reg_rtx (mode
);
2372 emit_move_insn (target
, gen_lowpart (mode
, x
));
2375 delete_insns_since (last
);
2380 /* Try calculating bswap as two bswaps of two word-sized operands. */
2383 expand_doubleword_bswap (machine_mode mode
, rtx op
, rtx target
)
2387 t1
= expand_unop (word_mode
, bswap_optab
,
2388 operand_subword_force (op
, 0, mode
), NULL_RTX
, true);
2389 t0
= expand_unop (word_mode
, bswap_optab
,
2390 operand_subword_force (op
, 1, mode
), NULL_RTX
, true);
2392 if (target
== 0 || !valid_multiword_target_p (target
))
2393 target
= gen_reg_rtx (mode
);
2395 emit_clobber (target
);
2396 emit_move_insn (operand_subword (target
, 0, 1, mode
), t0
);
2397 emit_move_insn (operand_subword (target
, 1, 1, mode
), t1
);
2402 /* Try calculating (parity x) as (and (popcount x) 1), where
2403 popcount can also be done in a wider mode. */
2405 expand_parity (scalar_int_mode mode
, rtx op0
, rtx target
)
2407 enum mode_class mclass
= GET_MODE_CLASS (mode
);
2408 opt_scalar_int_mode wider_mode_iter
;
2409 FOR_EACH_MODE_FROM (wider_mode_iter
, mode
)
2411 scalar_int_mode wider_mode
= wider_mode_iter
.require ();
2412 if (optab_handler (popcount_optab
, wider_mode
) != CODE_FOR_nothing
)
2417 last
= get_last_insn ();
2419 if (target
== 0 || GET_MODE (target
) != wider_mode
)
2420 target
= gen_reg_rtx (wider_mode
);
2422 xop0
= widen_operand (op0
, wider_mode
, mode
, true, false);
2423 temp
= expand_unop (wider_mode
, popcount_optab
, xop0
, NULL_RTX
,
2426 temp
= expand_binop (wider_mode
, and_optab
, temp
, const1_rtx
,
2427 target
, true, OPTAB_DIRECT
);
2431 if (mclass
!= MODE_INT
2432 || !TRULY_NOOP_TRUNCATION_MODES_P (mode
, wider_mode
))
2433 return convert_to_mode (mode
, temp
, 0);
2435 return gen_lowpart (mode
, temp
);
2438 delete_insns_since (last
);
2444 /* Try calculating ctz(x) as K - clz(x & -x) ,
2445 where K is GET_MODE_PRECISION(mode) - 1.
2447 Both __builtin_ctz and __builtin_clz are undefined at zero, so we
2448 don't have to worry about what the hardware does in that case. (If
2449 the clz instruction produces the usual value at 0, which is K, the
2450 result of this code sequence will be -1; expand_ffs, below, relies
2451 on this. It might be nice to have it be K instead, for consistency
2452 with the (very few) processors that provide a ctz with a defined
2453 value, but that would take one more instruction, and it would be
2454 less convenient for expand_ffs anyway. */
2457 expand_ctz (scalar_int_mode mode
, rtx op0
, rtx target
)
2462 if (optab_handler (clz_optab
, mode
) == CODE_FOR_nothing
)
2467 temp
= expand_unop_direct (mode
, neg_optab
, op0
, NULL_RTX
, true);
2469 temp
= expand_binop (mode
, and_optab
, op0
, temp
, NULL_RTX
,
2470 true, OPTAB_DIRECT
);
2472 temp
= expand_unop_direct (mode
, clz_optab
, temp
, NULL_RTX
, true);
2474 temp
= expand_binop (mode
, sub_optab
,
2475 gen_int_mode (GET_MODE_PRECISION (mode
) - 1, mode
),
2477 true, OPTAB_DIRECT
);
2487 add_equal_note (seq
, temp
, CTZ
, op0
, 0);
2493 /* Try calculating ffs(x) using ctz(x) if we have that instruction, or
2494 else with the sequence used by expand_clz.
2496 The ffs builtin promises to return zero for a zero value and ctz/clz
2497 may have an undefined value in that case. If they do not give us a
2498 convenient value, we have to generate a test and branch. */
2500 expand_ffs (scalar_int_mode mode
, rtx op0
, rtx target
)
2502 HOST_WIDE_INT val
= 0;
2503 bool defined_at_zero
= false;
2507 if (optab_handler (ctz_optab
, mode
) != CODE_FOR_nothing
)
2511 temp
= expand_unop_direct (mode
, ctz_optab
, op0
, 0, true);
2515 defined_at_zero
= (CTZ_DEFINED_VALUE_AT_ZERO (mode
, val
) == 2);
2517 else if (optab_handler (clz_optab
, mode
) != CODE_FOR_nothing
)
2520 temp
= expand_ctz (mode
, op0
, 0);
2524 if (CLZ_DEFINED_VALUE_AT_ZERO (mode
, val
) == 2)
2526 defined_at_zero
= true;
2527 val
= (GET_MODE_PRECISION (mode
) - 1) - val
;
2533 if (defined_at_zero
&& val
== -1)
2534 /* No correction needed at zero. */;
2537 /* We don't try to do anything clever with the situation found
2538 on some processors (eg Alpha) where ctz(0:mode) ==
2539 bitsize(mode). If someone can think of a way to send N to -1
2540 and leave alone all values in the range 0..N-1 (where N is a
2541 power of two), cheaper than this test-and-branch, please add it.
2543 The test-and-branch is done after the operation itself, in case
2544 the operation sets condition codes that can be recycled for this.
2545 (This is true on i386, for instance.) */
2547 rtx_code_label
*nonzero_label
= gen_label_rtx ();
2548 emit_cmp_and_jump_insns (op0
, CONST0_RTX (mode
), NE
, 0,
2549 mode
, true, nonzero_label
);
2551 convert_move (temp
, GEN_INT (-1), false);
2552 emit_label (nonzero_label
);
2555 /* temp now has a value in the range -1..bitsize-1. ffs is supposed
2556 to produce a value in the range 0..bitsize. */
2557 temp
= expand_binop (mode
, add_optab
, temp
, gen_int_mode (1, mode
),
2558 target
, false, OPTAB_DIRECT
);
2565 add_equal_note (seq
, temp
, FFS
, op0
, 0);
2574 /* Extract the OMODE lowpart from VAL, which has IMODE. Under certain
2575 conditions, VAL may already be a SUBREG against which we cannot generate
2576 a further SUBREG. In this case, we expect forcing the value into a
2577 register will work around the situation. */
2580 lowpart_subreg_maybe_copy (machine_mode omode
, rtx val
,
2584 ret
= lowpart_subreg (omode
, val
, imode
);
2587 val
= force_reg (imode
, val
);
2588 ret
= lowpart_subreg (omode
, val
, imode
);
2589 gcc_assert (ret
!= NULL
);
2594 /* Expand a floating point absolute value or negation operation via a
2595 logical operation on the sign bit. */
2598 expand_absneg_bit (enum rtx_code code
, scalar_float_mode mode
,
2599 rtx op0
, rtx target
)
2601 const struct real_format
*fmt
;
2602 int bitpos
, word
, nwords
, i
;
2603 scalar_int_mode imode
;
2607 /* The format has to have a simple sign bit. */
2608 fmt
= REAL_MODE_FORMAT (mode
);
2612 bitpos
= fmt
->signbit_rw
;
2616 /* Don't create negative zeros if the format doesn't support them. */
2617 if (code
== NEG
&& !fmt
->has_signed_zero
)
2620 if (GET_MODE_SIZE (mode
) <= UNITS_PER_WORD
)
2622 if (!int_mode_for_mode (mode
).exists (&imode
))
2631 if (FLOAT_WORDS_BIG_ENDIAN
)
2632 word
= (GET_MODE_BITSIZE (mode
) - bitpos
) / BITS_PER_WORD
;
2634 word
= bitpos
/ BITS_PER_WORD
;
2635 bitpos
= bitpos
% BITS_PER_WORD
;
2636 nwords
= (GET_MODE_BITSIZE (mode
) + BITS_PER_WORD
- 1) / BITS_PER_WORD
;
2639 wide_int mask
= wi::set_bit_in_zero (bitpos
, GET_MODE_PRECISION (imode
));
2645 || (nwords
> 1 && !valid_multiword_target_p (target
)))
2646 target
= gen_reg_rtx (mode
);
2652 for (i
= 0; i
< nwords
; ++i
)
2654 rtx targ_piece
= operand_subword (target
, i
, 1, mode
);
2655 rtx op0_piece
= operand_subword_force (op0
, i
, mode
);
2659 temp
= expand_binop (imode
, code
== ABS
? and_optab
: xor_optab
,
2661 immed_wide_int_const (mask
, imode
),
2662 targ_piece
, 1, OPTAB_LIB_WIDEN
);
2663 if (temp
!= targ_piece
)
2664 emit_move_insn (targ_piece
, temp
);
2667 emit_move_insn (targ_piece
, op0_piece
);
2670 insns
= get_insns ();
2677 temp
= expand_binop (imode
, code
== ABS
? and_optab
: xor_optab
,
2678 gen_lowpart (imode
, op0
),
2679 immed_wide_int_const (mask
, imode
),
2680 gen_lowpart (imode
, target
), 1, OPTAB_LIB_WIDEN
);
2681 target
= lowpart_subreg_maybe_copy (mode
, temp
, imode
);
2683 set_dst_reg_note (get_last_insn (), REG_EQUAL
,
2684 gen_rtx_fmt_e (code
, mode
, copy_rtx (op0
)),
2691 /* As expand_unop, but will fail rather than attempt the operation in a
2692 different mode or with a libcall. */
2694 expand_unop_direct (machine_mode mode
, optab unoptab
, rtx op0
, rtx target
,
2697 if (optab_handler (unoptab
, mode
) != CODE_FOR_nothing
)
2699 struct expand_operand ops
[2];
2700 enum insn_code icode
= optab_handler (unoptab
, mode
);
2701 rtx_insn
*last
= get_last_insn ();
2704 create_output_operand (&ops
[0], target
, mode
);
2705 create_convert_operand_from (&ops
[1], op0
, mode
, unsignedp
);
2706 pat
= maybe_gen_insn (icode
, 2, ops
);
2709 if (INSN_P (pat
) && NEXT_INSN (pat
) != NULL_RTX
2710 && ! add_equal_note (pat
, ops
[0].value
,
2711 optab_to_code (unoptab
),
2712 ops
[1].value
, NULL_RTX
))
2714 delete_insns_since (last
);
2715 return expand_unop (mode
, unoptab
, op0
, NULL_RTX
, unsignedp
);
2720 return ops
[0].value
;
2726 /* Generate code to perform an operation specified by UNOPTAB
2727 on operand OP0, with result having machine-mode MODE.
2729 UNSIGNEDP is for the case where we have to widen the operands
2730 to perform the operation. It says to use zero-extension.
2732 If TARGET is nonzero, the value
2733 is generated there, if it is convenient to do so.
2734 In all cases an rtx is returned for the locus of the value;
2735 this may or may not be TARGET. */
2738 expand_unop (machine_mode mode
, optab unoptab
, rtx op0
, rtx target
,
2741 enum mode_class mclass
= GET_MODE_CLASS (mode
);
2742 machine_mode wider_mode
;
2743 scalar_int_mode int_mode
;
2744 scalar_float_mode float_mode
;
2748 temp
= expand_unop_direct (mode
, unoptab
, op0
, target
, unsignedp
);
2752 /* It can't be done in this mode. Can we open-code it in a wider mode? */
2754 /* Widening (or narrowing) clz needs special treatment. */
2755 if (unoptab
== clz_optab
)
2757 if (is_a
<scalar_int_mode
> (mode
, &int_mode
))
2759 temp
= widen_leading (int_mode
, op0
, target
, unoptab
);
2763 if (GET_MODE_SIZE (int_mode
) == 2 * UNITS_PER_WORD
2764 && optab_handler (unoptab
, word_mode
) != CODE_FOR_nothing
)
2766 temp
= expand_doubleword_clz (int_mode
, op0
, target
);
2775 if (unoptab
== clrsb_optab
)
2777 if (is_a
<scalar_int_mode
> (mode
, &int_mode
))
2779 temp
= widen_leading (int_mode
, op0
, target
, unoptab
);
2786 if (unoptab
== popcount_optab
2787 && is_a
<scalar_int_mode
> (mode
, &int_mode
)
2788 && GET_MODE_SIZE (int_mode
) == 2 * UNITS_PER_WORD
2789 && optab_handler (unoptab
, word_mode
) != CODE_FOR_nothing
2790 && optimize_insn_for_speed_p ())
2792 temp
= expand_doubleword_popcount (int_mode
, op0
, target
);
2797 if (unoptab
== parity_optab
2798 && is_a
<scalar_int_mode
> (mode
, &int_mode
)
2799 && GET_MODE_SIZE (int_mode
) == 2 * UNITS_PER_WORD
2800 && (optab_handler (unoptab
, word_mode
) != CODE_FOR_nothing
2801 || optab_handler (popcount_optab
, word_mode
) != CODE_FOR_nothing
)
2802 && optimize_insn_for_speed_p ())
2804 temp
= expand_doubleword_parity (int_mode
, op0
, target
);
2809 /* Widening (or narrowing) bswap needs special treatment. */
2810 if (unoptab
== bswap_optab
)
2812 /* HImode is special because in this mode BSWAP is equivalent to ROTATE
2813 or ROTATERT. First try these directly; if this fails, then try the
2814 obvious pair of shifts with allowed widening, as this will probably
2815 be always more efficient than the other fallback methods. */
2821 if (optab_handler (rotl_optab
, mode
) != CODE_FOR_nothing
)
2823 temp
= expand_binop (mode
, rotl_optab
, op0
,
2824 gen_int_shift_amount (mode
, 8),
2825 target
, unsignedp
, OPTAB_DIRECT
);
2830 if (optab_handler (rotr_optab
, mode
) != CODE_FOR_nothing
)
2832 temp
= expand_binop (mode
, rotr_optab
, op0
,
2833 gen_int_shift_amount (mode
, 8),
2834 target
, unsignedp
, OPTAB_DIRECT
);
2839 last
= get_last_insn ();
2841 temp1
= expand_binop (mode
, ashl_optab
, op0
,
2842 gen_int_shift_amount (mode
, 8), NULL_RTX
,
2843 unsignedp
, OPTAB_WIDEN
);
2844 temp2
= expand_binop (mode
, lshr_optab
, op0
,
2845 gen_int_shift_amount (mode
, 8), NULL_RTX
,
2846 unsignedp
, OPTAB_WIDEN
);
2849 temp
= expand_binop (mode
, ior_optab
, temp1
, temp2
, target
,
2850 unsignedp
, OPTAB_WIDEN
);
2855 delete_insns_since (last
);
2858 if (is_a
<scalar_int_mode
> (mode
, &int_mode
))
2860 temp
= widen_bswap (int_mode
, op0
, target
);
2864 if (GET_MODE_SIZE (int_mode
) == 2 * UNITS_PER_WORD
2865 && optab_handler (unoptab
, word_mode
) != CODE_FOR_nothing
)
2867 temp
= expand_doubleword_bswap (mode
, op0
, target
);
2876 if (CLASS_HAS_WIDER_MODES_P (mclass
))
2877 FOR_EACH_WIDER_MODE (wider_mode
, mode
)
2879 if (optab_handler (unoptab
, wider_mode
) != CODE_FOR_nothing
)
2882 rtx_insn
*last
= get_last_insn ();
2884 /* For certain operations, we need not actually extend
2885 the narrow operand, as long as we will truncate the
2886 results to the same narrowness. */
2888 xop0
= widen_operand (xop0
, wider_mode
, mode
, unsignedp
,
2889 (unoptab
== neg_optab
2890 || unoptab
== one_cmpl_optab
)
2891 && mclass
== MODE_INT
);
2893 temp
= expand_unop (wider_mode
, unoptab
, xop0
, NULL_RTX
,
2898 if (mclass
!= MODE_INT
2899 || !TRULY_NOOP_TRUNCATION_MODES_P (mode
, wider_mode
))
2902 target
= gen_reg_rtx (mode
);
2903 convert_move (target
, temp
, 0);
2907 return gen_lowpart (mode
, temp
);
2910 delete_insns_since (last
);
2914 /* These can be done a word at a time. */
2915 if (unoptab
== one_cmpl_optab
2916 && is_int_mode (mode
, &int_mode
)
2917 && GET_MODE_SIZE (int_mode
) > UNITS_PER_WORD
2918 && optab_handler (unoptab
, word_mode
) != CODE_FOR_nothing
)
2923 if (target
== 0 || target
== op0
|| !valid_multiword_target_p (target
))
2924 target
= gen_reg_rtx (int_mode
);
2928 /* Do the actual arithmetic. */
2929 for (i
= 0; i
< GET_MODE_BITSIZE (int_mode
) / BITS_PER_WORD
; i
++)
2931 rtx target_piece
= operand_subword (target
, i
, 1, int_mode
);
2932 rtx x
= expand_unop (word_mode
, unoptab
,
2933 operand_subword_force (op0
, i
, int_mode
),
2934 target_piece
, unsignedp
);
2936 if (target_piece
!= x
)
2937 emit_move_insn (target_piece
, x
);
2940 insns
= get_insns ();
2947 if (optab_to_code (unoptab
) == NEG
)
2949 /* Try negating floating point values by flipping the sign bit. */
2950 if (is_a
<scalar_float_mode
> (mode
, &float_mode
))
2952 temp
= expand_absneg_bit (NEG
, float_mode
, op0
, target
);
2957 /* If there is no negation pattern, and we have no negative zero,
2958 try subtracting from zero. */
2959 if (!HONOR_SIGNED_ZEROS (mode
))
2961 temp
= expand_binop (mode
, (unoptab
== negv_optab
2962 ? subv_optab
: sub_optab
),
2963 CONST0_RTX (mode
), op0
, target
,
2964 unsignedp
, OPTAB_DIRECT
);
2970 /* Try calculating parity (x) as popcount (x) % 2. */
2971 if (unoptab
== parity_optab
&& is_a
<scalar_int_mode
> (mode
, &int_mode
))
2973 temp
= expand_parity (int_mode
, op0
, target
);
2978 /* Try implementing ffs (x) in terms of clz (x). */
2979 if (unoptab
== ffs_optab
&& is_a
<scalar_int_mode
> (mode
, &int_mode
))
2981 temp
= expand_ffs (int_mode
, op0
, target
);
2986 /* Try implementing ctz (x) in terms of clz (x). */
2987 if (unoptab
== ctz_optab
&& is_a
<scalar_int_mode
> (mode
, &int_mode
))
2989 temp
= expand_ctz (int_mode
, op0
, target
);
2995 /* Now try a library call in this mode. */
2996 libfunc
= optab_libfunc (unoptab
, mode
);
3002 machine_mode outmode
= mode
;
3004 /* All of these functions return small values. Thus we choose to
3005 have them return something that isn't a double-word. */
3006 if (unoptab
== ffs_optab
|| unoptab
== clz_optab
|| unoptab
== ctz_optab
3007 || unoptab
== clrsb_optab
|| unoptab
== popcount_optab
3008 || unoptab
== parity_optab
)
3010 = GET_MODE (hard_libcall_value (TYPE_MODE (integer_type_node
),
3011 optab_libfunc (unoptab
, mode
)));
3015 /* Pass 1 for NO_QUEUE so we don't lose any increments
3016 if the libcall is cse'd or moved. */
3017 value
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
, outmode
,
3019 insns
= get_insns ();
3022 target
= gen_reg_rtx (outmode
);
3023 bool trapv
= trapv_unoptab_p (unoptab
);
3025 eq_value
= NULL_RTX
;
3028 eq_value
= gen_rtx_fmt_e (optab_to_code (unoptab
), mode
, op0
);
3029 if (GET_MODE_UNIT_SIZE (outmode
) < GET_MODE_UNIT_SIZE (mode
))
3030 eq_value
= simplify_gen_unary (TRUNCATE
, outmode
, eq_value
, mode
);
3031 else if (GET_MODE_UNIT_SIZE (outmode
) > GET_MODE_UNIT_SIZE (mode
))
3032 eq_value
= simplify_gen_unary (ZERO_EXTEND
,
3033 outmode
, eq_value
, mode
);
3035 emit_libcall_block_1 (insns
, target
, value
, eq_value
, trapv
);
3040 /* It can't be done in this mode. Can we do it in a wider mode? */
3042 if (CLASS_HAS_WIDER_MODES_P (mclass
))
3044 FOR_EACH_WIDER_MODE (wider_mode
, mode
)
3046 if (optab_handler (unoptab
, wider_mode
) != CODE_FOR_nothing
3047 || optab_libfunc (unoptab
, wider_mode
))
3050 rtx_insn
*last
= get_last_insn ();
3052 /* For certain operations, we need not actually extend
3053 the narrow operand, as long as we will truncate the
3054 results to the same narrowness. */
3055 xop0
= widen_operand (xop0
, wider_mode
, mode
, unsignedp
,
3056 (unoptab
== neg_optab
3057 || unoptab
== one_cmpl_optab
3058 || unoptab
== bswap_optab
)
3059 && mclass
== MODE_INT
);
3061 temp
= expand_unop (wider_mode
, unoptab
, xop0
, NULL_RTX
,
3064 /* If we are generating clz using wider mode, adjust the
3065 result. Similarly for clrsb. */
3066 if ((unoptab
== clz_optab
|| unoptab
== clrsb_optab
)
3069 scalar_int_mode wider_int_mode
3070 = as_a
<scalar_int_mode
> (wider_mode
);
3071 int_mode
= as_a
<scalar_int_mode
> (mode
);
3073 (wider_mode
, sub_optab
, temp
,
3074 gen_int_mode (GET_MODE_PRECISION (wider_int_mode
)
3075 - GET_MODE_PRECISION (int_mode
),
3077 target
, true, OPTAB_DIRECT
);
3080 /* Likewise for bswap. */
3081 if (unoptab
== bswap_optab
&& temp
!= 0)
3083 scalar_int_mode wider_int_mode
3084 = as_a
<scalar_int_mode
> (wider_mode
);
3085 int_mode
= as_a
<scalar_int_mode
> (mode
);
3086 gcc_assert (GET_MODE_PRECISION (wider_int_mode
)
3087 == GET_MODE_BITSIZE (wider_int_mode
)
3088 && GET_MODE_PRECISION (int_mode
)
3089 == GET_MODE_BITSIZE (int_mode
));
3091 temp
= expand_shift (RSHIFT_EXPR
, wider_int_mode
, temp
,
3092 GET_MODE_BITSIZE (wider_int_mode
)
3093 - GET_MODE_BITSIZE (int_mode
),
3099 if (mclass
!= MODE_INT
)
3102 target
= gen_reg_rtx (mode
);
3103 convert_move (target
, temp
, 0);
3107 return gen_lowpart (mode
, temp
);
3110 delete_insns_since (last
);
3115 /* One final attempt at implementing negation via subtraction,
3116 this time allowing widening of the operand. */
3117 if (optab_to_code (unoptab
) == NEG
&& !HONOR_SIGNED_ZEROS (mode
))
3120 temp
= expand_binop (mode
,
3121 unoptab
== negv_optab
? subv_optab
: sub_optab
,
3122 CONST0_RTX (mode
), op0
,
3123 target
, unsignedp
, OPTAB_LIB_WIDEN
);
3131 /* Emit code to compute the absolute value of OP0, with result to
3132 TARGET if convenient. (TARGET may be 0.) The return value says
3133 where the result actually is to be found.
3135 MODE is the mode of the operand; the mode of the result is
3136 different but can be deduced from MODE.
3141 expand_abs_nojump (machine_mode mode
, rtx op0
, rtx target
,
3142 int result_unsignedp
)
3146 if (GET_MODE_CLASS (mode
) != MODE_INT
3148 result_unsignedp
= 1;
3150 /* First try to do it with a special abs instruction. */
3151 temp
= expand_unop (mode
, result_unsignedp
? abs_optab
: absv_optab
,
3156 /* For floating point modes, try clearing the sign bit. */
3157 scalar_float_mode float_mode
;
3158 if (is_a
<scalar_float_mode
> (mode
, &float_mode
))
3160 temp
= expand_absneg_bit (ABS
, float_mode
, op0
, target
);
3165 /* If we have a MAX insn, we can do this as MAX (x, -x). */
3166 if (optab_handler (smax_optab
, mode
) != CODE_FOR_nothing
3167 && !HONOR_SIGNED_ZEROS (mode
))
3169 rtx_insn
*last
= get_last_insn ();
3171 temp
= expand_unop (mode
, result_unsignedp
? neg_optab
: negv_optab
,
3174 temp
= expand_binop (mode
, smax_optab
, op0
, temp
, target
, 0,
3180 delete_insns_since (last
);
3183 /* If this machine has expensive jumps, we can do integer absolute
3184 value of X as (((signed) x >> (W-1)) ^ x) - ((signed) x >> (W-1)),
3185 where W is the width of MODE. */
3187 scalar_int_mode int_mode
;
3188 if (is_int_mode (mode
, &int_mode
)
3189 && BRANCH_COST (optimize_insn_for_speed_p (),
3192 rtx extended
= expand_shift (RSHIFT_EXPR
, int_mode
, op0
,
3193 GET_MODE_PRECISION (int_mode
) - 1,
3196 temp
= expand_binop (int_mode
, xor_optab
, extended
, op0
, target
, 0,
3199 temp
= expand_binop (int_mode
,
3200 result_unsignedp
? sub_optab
: subv_optab
,
3201 temp
, extended
, target
, 0, OPTAB_LIB_WIDEN
);
3211 expand_abs (machine_mode mode
, rtx op0
, rtx target
,
3212 int result_unsignedp
, int safe
)
3215 rtx_code_label
*op1
;
3217 if (GET_MODE_CLASS (mode
) != MODE_INT
3219 result_unsignedp
= 1;
3221 temp
= expand_abs_nojump (mode
, op0
, target
, result_unsignedp
);
3225 /* If that does not win, use conditional jump and negate. */
3227 /* It is safe to use the target if it is the same
3228 as the source if this is also a pseudo register */
3229 if (op0
== target
&& REG_P (op0
)
3230 && REGNO (op0
) >= FIRST_PSEUDO_REGISTER
)
3233 op1
= gen_label_rtx ();
3234 if (target
== 0 || ! safe
3235 || GET_MODE (target
) != mode
3236 || (MEM_P (target
) && MEM_VOLATILE_P (target
))
3238 && REGNO (target
) < FIRST_PSEUDO_REGISTER
))
3239 target
= gen_reg_rtx (mode
);
3241 emit_move_insn (target
, op0
);
3244 do_compare_rtx_and_jump (target
, CONST0_RTX (mode
), GE
, 0, mode
,
3245 NULL_RTX
, NULL
, op1
,
3246 profile_probability::uninitialized ());
3248 op0
= expand_unop (mode
, result_unsignedp
? neg_optab
: negv_optab
,
3251 emit_move_insn (target
, op0
);
3257 /* Emit code to compute the one's complement absolute value of OP0
3258 (if (OP0 < 0) OP0 = ~OP0), with result to TARGET if convenient.
3259 (TARGET may be NULL_RTX.) The return value says where the result
3260 actually is to be found.
3262 MODE is the mode of the operand; the mode of the result is
3263 different but can be deduced from MODE. */
3266 expand_one_cmpl_abs_nojump (machine_mode mode
, rtx op0
, rtx target
)
3270 /* Not applicable for floating point modes. */
3271 if (FLOAT_MODE_P (mode
))
3274 /* If we have a MAX insn, we can do this as MAX (x, ~x). */
3275 if (optab_handler (smax_optab
, mode
) != CODE_FOR_nothing
)
3277 rtx_insn
*last
= get_last_insn ();
3279 temp
= expand_unop (mode
, one_cmpl_optab
, op0
, NULL_RTX
, 0);
3281 temp
= expand_binop (mode
, smax_optab
, op0
, temp
, target
, 0,
3287 delete_insns_since (last
);
3290 /* If this machine has expensive jumps, we can do one's complement
3291 absolute value of X as (((signed) x >> (W-1)) ^ x). */
3293 scalar_int_mode int_mode
;
3294 if (is_int_mode (mode
, &int_mode
)
3295 && BRANCH_COST (optimize_insn_for_speed_p (),
3298 rtx extended
= expand_shift (RSHIFT_EXPR
, int_mode
, op0
,
3299 GET_MODE_PRECISION (int_mode
) - 1,
3302 temp
= expand_binop (int_mode
, xor_optab
, extended
, op0
, target
, 0,
3312 /* A subroutine of expand_copysign, perform the copysign operation using the
3313 abs and neg primitives advertised to exist on the target. The assumption
3314 is that we have a split register file, and leaving op0 in fp registers,
3315 and not playing with subregs so much, will help the register allocator. */
3318 expand_copysign_absneg (scalar_float_mode mode
, rtx op0
, rtx op1
, rtx target
,
3319 int bitpos
, bool op0_is_abs
)
3321 scalar_int_mode imode
;
3322 enum insn_code icode
;
3324 rtx_code_label
*label
;
3329 /* Check if the back end provides an insn that handles signbit for the
3331 icode
= optab_handler (signbit_optab
, mode
);
3332 if (icode
!= CODE_FOR_nothing
)
3334 imode
= as_a
<scalar_int_mode
> (insn_data
[(int) icode
].operand
[0].mode
);
3335 sign
= gen_reg_rtx (imode
);
3336 emit_unop_insn (icode
, sign
, op1
, UNKNOWN
);
3340 if (GET_MODE_SIZE (mode
) <= UNITS_PER_WORD
)
3342 if (!int_mode_for_mode (mode
).exists (&imode
))
3344 op1
= gen_lowpart (imode
, op1
);
3351 if (FLOAT_WORDS_BIG_ENDIAN
)
3352 word
= (GET_MODE_BITSIZE (mode
) - bitpos
) / BITS_PER_WORD
;
3354 word
= bitpos
/ BITS_PER_WORD
;
3355 bitpos
= bitpos
% BITS_PER_WORD
;
3356 op1
= operand_subword_force (op1
, word
, mode
);
3359 wide_int mask
= wi::set_bit_in_zero (bitpos
, GET_MODE_PRECISION (imode
));
3360 sign
= expand_binop (imode
, and_optab
, op1
,
3361 immed_wide_int_const (mask
, imode
),
3362 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
3367 op0
= expand_unop (mode
, abs_optab
, op0
, target
, 0);
3374 if (target
== NULL_RTX
)
3375 target
= copy_to_reg (op0
);
3377 emit_move_insn (target
, op0
);
3380 label
= gen_label_rtx ();
3381 emit_cmp_and_jump_insns (sign
, const0_rtx
, EQ
, NULL_RTX
, imode
, 1, label
);
3383 if (CONST_DOUBLE_AS_FLOAT_P (op0
))
3384 op0
= simplify_unary_operation (NEG
, mode
, op0
, mode
);
3386 op0
= expand_unop (mode
, neg_optab
, op0
, target
, 0);
3388 emit_move_insn (target
, op0
);
3396 /* A subroutine of expand_copysign, perform the entire copysign operation
3397 with integer bitmasks. BITPOS is the position of the sign bit; OP0_IS_ABS
3398 is true if op0 is known to have its sign bit clear. */
3401 expand_copysign_bit (scalar_float_mode mode
, rtx op0
, rtx op1
, rtx target
,
3402 int bitpos
, bool op0_is_abs
)
3404 scalar_int_mode imode
;
3405 int word
, nwords
, i
;
3409 if (GET_MODE_SIZE (mode
) <= UNITS_PER_WORD
)
3411 if (!int_mode_for_mode (mode
).exists (&imode
))
3420 if (FLOAT_WORDS_BIG_ENDIAN
)
3421 word
= (GET_MODE_BITSIZE (mode
) - bitpos
) / BITS_PER_WORD
;
3423 word
= bitpos
/ BITS_PER_WORD
;
3424 bitpos
= bitpos
% BITS_PER_WORD
;
3425 nwords
= (GET_MODE_BITSIZE (mode
) + BITS_PER_WORD
- 1) / BITS_PER_WORD
;
3428 wide_int mask
= wi::set_bit_in_zero (bitpos
, GET_MODE_PRECISION (imode
));
3433 || (nwords
> 1 && !valid_multiword_target_p (target
)))
3434 target
= gen_reg_rtx (mode
);
3440 for (i
= 0; i
< nwords
; ++i
)
3442 rtx targ_piece
= operand_subword (target
, i
, 1, mode
);
3443 rtx op0_piece
= operand_subword_force (op0
, i
, mode
);
3449 = expand_binop (imode
, and_optab
, op0_piece
,
3450 immed_wide_int_const (~mask
, imode
),
3451 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
3452 op1
= expand_binop (imode
, and_optab
,
3453 operand_subword_force (op1
, i
, mode
),
3454 immed_wide_int_const (mask
, imode
),
3455 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
3457 temp
= expand_binop (imode
, ior_optab
, op0_piece
, op1
,
3458 targ_piece
, 1, OPTAB_LIB_WIDEN
);
3459 if (temp
!= targ_piece
)
3460 emit_move_insn (targ_piece
, temp
);
3463 emit_move_insn (targ_piece
, op0_piece
);
3466 insns
= get_insns ();
3473 op1
= expand_binop (imode
, and_optab
, gen_lowpart (imode
, op1
),
3474 immed_wide_int_const (mask
, imode
),
3475 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
3477 op0
= gen_lowpart (imode
, op0
);
3479 op0
= expand_binop (imode
, and_optab
, op0
,
3480 immed_wide_int_const (~mask
, imode
),
3481 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
3483 temp
= expand_binop (imode
, ior_optab
, op0
, op1
,
3484 gen_lowpart (imode
, target
), 1, OPTAB_LIB_WIDEN
);
3485 target
= lowpart_subreg_maybe_copy (mode
, temp
, imode
);
3491 /* Expand the C99 copysign operation. OP0 and OP1 must be the same
3492 scalar floating point mode. Return NULL if we do not know how to
3493 expand the operation inline. */
3496 expand_copysign (rtx op0
, rtx op1
, rtx target
)
3498 scalar_float_mode mode
;
3499 const struct real_format
*fmt
;
3503 mode
= as_a
<scalar_float_mode
> (GET_MODE (op0
));
3504 gcc_assert (GET_MODE (op1
) == mode
);
3506 /* First try to do it with a special instruction. */
3507 temp
= expand_binop (mode
, copysign_optab
, op0
, op1
,
3508 target
, 0, OPTAB_DIRECT
);
3512 fmt
= REAL_MODE_FORMAT (mode
);
3513 if (fmt
== NULL
|| !fmt
->has_signed_zero
)
3517 if (CONST_DOUBLE_AS_FLOAT_P (op0
))
3519 if (real_isneg (CONST_DOUBLE_REAL_VALUE (op0
)))
3520 op0
= simplify_unary_operation (ABS
, mode
, op0
, mode
);
3524 if (fmt
->signbit_ro
>= 0
3525 && (CONST_DOUBLE_AS_FLOAT_P (op0
)
3526 || (optab_handler (neg_optab
, mode
) != CODE_FOR_nothing
3527 && optab_handler (abs_optab
, mode
) != CODE_FOR_nothing
)))
3529 temp
= expand_copysign_absneg (mode
, op0
, op1
, target
,
3530 fmt
->signbit_ro
, op0_is_abs
);
3535 if (fmt
->signbit_rw
< 0)
3537 return expand_copysign_bit (mode
, op0
, op1
, target
,
3538 fmt
->signbit_rw
, op0_is_abs
);
3541 /* Generate an instruction whose insn-code is INSN_CODE,
3542 with two operands: an output TARGET and an input OP0.
3543 TARGET *must* be nonzero, and the output is always stored there.
3544 CODE is an rtx code such that (CODE OP0) is an rtx that describes
3545 the value that is stored into TARGET.
3547 Return false if expansion failed. */
3550 maybe_emit_unop_insn (enum insn_code icode
, rtx target
, rtx op0
,
3553 struct expand_operand ops
[2];
3556 create_output_operand (&ops
[0], target
, GET_MODE (target
));
3557 create_input_operand (&ops
[1], op0
, GET_MODE (op0
));
3558 pat
= maybe_gen_insn (icode
, 2, ops
);
3562 if (INSN_P (pat
) && NEXT_INSN (pat
) != NULL_RTX
3564 add_equal_note (pat
, ops
[0].value
, code
, ops
[1].value
, NULL_RTX
);
3568 if (ops
[0].value
!= target
)
3569 emit_move_insn (target
, ops
[0].value
);
3572 /* Generate an instruction whose insn-code is INSN_CODE,
3573 with two operands: an output TARGET and an input OP0.
3574 TARGET *must* be nonzero, and the output is always stored there.
3575 CODE is an rtx code such that (CODE OP0) is an rtx that describes
3576 the value that is stored into TARGET. */
3579 emit_unop_insn (enum insn_code icode
, rtx target
, rtx op0
, enum rtx_code code
)
3581 bool ok
= maybe_emit_unop_insn (icode
, target
, op0
, code
);
3585 struct no_conflict_data
3588 rtx_insn
*first
, *insn
;
3592 /* Called via note_stores by emit_libcall_block. Set P->must_stay if
3593 the currently examined clobber / store has to stay in the list of
3594 insns that constitute the actual libcall block. */
3596 no_conflict_move_test (rtx dest
, const_rtx set
, void *p0
)
3598 struct no_conflict_data
*p
= (struct no_conflict_data
*) p0
;
3600 /* If this inns directly contributes to setting the target, it must stay. */
3601 if (reg_overlap_mentioned_p (p
->target
, dest
))
3602 p
->must_stay
= true;
3603 /* If we haven't committed to keeping any other insns in the list yet,
3604 there is nothing more to check. */
3605 else if (p
->insn
== p
->first
)
3607 /* If this insn sets / clobbers a register that feeds one of the insns
3608 already in the list, this insn has to stay too. */
3609 else if (reg_overlap_mentioned_p (dest
, PATTERN (p
->first
))
3610 || (CALL_P (p
->first
) && (find_reg_fusage (p
->first
, USE
, dest
)))
3611 || reg_used_between_p (dest
, p
->first
, p
->insn
)
3612 /* Likewise if this insn depends on a register set by a previous
3613 insn in the list, or if it sets a result (presumably a hard
3614 register) that is set or clobbered by a previous insn.
3615 N.B. the modified_*_p (SET_DEST...) tests applied to a MEM
3616 SET_DEST perform the former check on the address, and the latter
3617 check on the MEM. */
3618 || (GET_CODE (set
) == SET
3619 && (modified_in_p (SET_SRC (set
), p
->first
)
3620 || modified_in_p (SET_DEST (set
), p
->first
)
3621 || modified_between_p (SET_SRC (set
), p
->first
, p
->insn
)
3622 || modified_between_p (SET_DEST (set
), p
->first
, p
->insn
))))
3623 p
->must_stay
= true;
3627 /* Emit code to make a call to a constant function or a library call.
3629 INSNS is a list containing all insns emitted in the call.
3630 These insns leave the result in RESULT. Our block is to copy RESULT
3631 to TARGET, which is logically equivalent to EQUIV.
3633 We first emit any insns that set a pseudo on the assumption that these are
3634 loading constants into registers; doing so allows them to be safely cse'ed
3635 between blocks. Then we emit all the other insns in the block, followed by
3636 an insn to move RESULT to TARGET. This last insn will have a REQ_EQUAL
3637 note with an operand of EQUIV. */
3640 emit_libcall_block_1 (rtx_insn
*insns
, rtx target
, rtx result
, rtx equiv
,
3641 bool equiv_may_trap
)
3643 rtx final_dest
= target
;
3644 rtx_insn
*next
, *last
, *insn
;
3646 /* If this is a reg with REG_USERVAR_P set, then it could possibly turn
3647 into a MEM later. Protect the libcall block from this change. */
3648 if (! REG_P (target
) || REG_USERVAR_P (target
))
3649 target
= gen_reg_rtx (GET_MODE (target
));
3651 /* If we're using non-call exceptions, a libcall corresponding to an
3652 operation that may trap may also trap. */
3653 /* ??? See the comment in front of make_reg_eh_region_note. */
3654 if (cfun
->can_throw_non_call_exceptions
3655 && (equiv_may_trap
|| may_trap_p (equiv
)))
3657 for (insn
= insns
; insn
; insn
= NEXT_INSN (insn
))
3660 rtx note
= find_reg_note (insn
, REG_EH_REGION
, NULL_RTX
);
3663 int lp_nr
= INTVAL (XEXP (note
, 0));
3664 if (lp_nr
== 0 || lp_nr
== INT_MIN
)
3665 remove_note (insn
, note
);
3671 /* Look for any CALL_INSNs in this sequence, and attach a REG_EH_REGION
3672 reg note to indicate that this call cannot throw or execute a nonlocal
3673 goto (unless there is already a REG_EH_REGION note, in which case
3675 for (insn
= insns
; insn
; insn
= NEXT_INSN (insn
))
3677 make_reg_eh_region_note_nothrow_nononlocal (insn
);
3680 /* First emit all insns that set pseudos. Remove them from the list as
3681 we go. Avoid insns that set pseudos which were referenced in previous
3682 insns. These can be generated by move_by_pieces, for example,
3683 to update an address. Similarly, avoid insns that reference things
3684 set in previous insns. */
3686 for (insn
= insns
; insn
; insn
= next
)
3688 rtx set
= single_set (insn
);
3690 next
= NEXT_INSN (insn
);
3692 if (set
!= 0 && REG_P (SET_DEST (set
))
3693 && REGNO (SET_DEST (set
)) >= FIRST_PSEUDO_REGISTER
)
3695 struct no_conflict_data data
;
3697 data
.target
= const0_rtx
;
3701 note_stores (PATTERN (insn
), no_conflict_move_test
, &data
);
3702 if (! data
.must_stay
)
3704 if (PREV_INSN (insn
))
3705 SET_NEXT_INSN (PREV_INSN (insn
)) = next
;
3710 SET_PREV_INSN (next
) = PREV_INSN (insn
);
3716 /* Some ports use a loop to copy large arguments onto the stack.
3717 Don't move anything outside such a loop. */
3722 /* Write the remaining insns followed by the final copy. */
3723 for (insn
= insns
; insn
; insn
= next
)
3725 next
= NEXT_INSN (insn
);
3730 last
= emit_move_insn (target
, result
);
3732 set_dst_reg_note (last
, REG_EQUAL
, copy_rtx (equiv
), target
);
3734 if (final_dest
!= target
)
3735 emit_move_insn (final_dest
, target
);
3739 emit_libcall_block (rtx_insn
*insns
, rtx target
, rtx result
, rtx equiv
)
3741 emit_libcall_block_1 (insns
, target
, result
, equiv
, false);
3744 /* Nonzero if we can perform a comparison of mode MODE straightforwardly.
3745 PURPOSE describes how this comparison will be used. CODE is the rtx
3746 comparison code we will be using.
3748 ??? Actually, CODE is slightly weaker than that. A target is still
3749 required to implement all of the normal bcc operations, but not
3750 required to implement all (or any) of the unordered bcc operations. */
3753 can_compare_p (enum rtx_code code
, machine_mode mode
,
3754 enum can_compare_purpose purpose
)
3757 test
= gen_rtx_fmt_ee (code
, mode
, const0_rtx
, const0_rtx
);
3760 enum insn_code icode
;
3762 if (purpose
== ccp_jump
3763 && (icode
= optab_handler (cbranch_optab
, mode
)) != CODE_FOR_nothing
3764 && insn_operand_matches (icode
, 0, test
))
3766 if (purpose
== ccp_store_flag
3767 && (icode
= optab_handler (cstore_optab
, mode
)) != CODE_FOR_nothing
3768 && insn_operand_matches (icode
, 1, test
))
3770 if (purpose
== ccp_cmov
3771 && optab_handler (cmov_optab
, mode
) != CODE_FOR_nothing
)
3774 mode
= GET_MODE_WIDER_MODE (mode
).else_void ();
3775 PUT_MODE (test
, mode
);
3777 while (mode
!= VOIDmode
);
3782 /* This function is called when we are going to emit a compare instruction that
3783 compares the values found in X and Y, using the rtl operator COMPARISON.
3785 If they have mode BLKmode, then SIZE specifies the size of both operands.
3787 UNSIGNEDP nonzero says that the operands are unsigned;
3788 this matters if they need to be widened (as given by METHODS).
3790 *PTEST is where the resulting comparison RTX is returned or NULL_RTX
3791 if we failed to produce one.
3793 *PMODE is the mode of the inputs (in case they are const_int).
3795 This function performs all the setup necessary so that the caller only has
3796 to emit a single comparison insn. This setup can involve doing a BLKmode
3797 comparison or emitting a library call to perform the comparison if no insn
3798 is available to handle it.
3799 The values which are passed in through pointers can be modified; the caller
3800 should perform the comparison on the modified values. Constant
3801 comparisons must have already been folded. */
3804 prepare_cmp_insn (rtx x
, rtx y
, enum rtx_code comparison
, rtx size
,
3805 int unsignedp
, enum optab_methods methods
,
3806 rtx
*ptest
, machine_mode
*pmode
)
3808 machine_mode mode
= *pmode
;
3810 machine_mode cmp_mode
;
3811 enum mode_class mclass
;
3813 /* The other methods are not needed. */
3814 gcc_assert (methods
== OPTAB_DIRECT
|| methods
== OPTAB_WIDEN
3815 || methods
== OPTAB_LIB_WIDEN
);
3817 if (CONST_SCALAR_INT_P (y
))
3818 canonicalize_comparison (mode
, &comparison
, &y
);
3820 /* If we are optimizing, force expensive constants into a register. */
3821 if (CONSTANT_P (x
) && optimize
3822 && (rtx_cost (x
, mode
, COMPARE
, 0, optimize_insn_for_speed_p ())
3823 > COSTS_N_INSNS (1)))
3824 x
= force_reg (mode
, x
);
3826 if (CONSTANT_P (y
) && optimize
3827 && (rtx_cost (y
, mode
, COMPARE
, 1, optimize_insn_for_speed_p ())
3828 > COSTS_N_INSNS (1)))
3829 y
= force_reg (mode
, y
);
3832 /* Make sure if we have a canonical comparison. The RTL
3833 documentation states that canonical comparisons are required only
3834 for targets which have cc0. */
3835 gcc_assert (!CONSTANT_P (x
) || CONSTANT_P (y
));
3838 /* Don't let both operands fail to indicate the mode. */
3839 if (GET_MODE (x
) == VOIDmode
&& GET_MODE (y
) == VOIDmode
)
3840 x
= force_reg (mode
, x
);
3841 if (mode
== VOIDmode
)
3842 mode
= GET_MODE (x
) != VOIDmode
? GET_MODE (x
) : GET_MODE (y
);
3844 /* Handle all BLKmode compares. */
3846 if (mode
== BLKmode
)
3848 machine_mode result_mode
;
3849 enum insn_code cmp_code
;
3852 = GEN_INT (MIN (MEM_ALIGN (x
), MEM_ALIGN (y
)) / BITS_PER_UNIT
);
3856 /* Try to use a memory block compare insn - either cmpstr
3857 or cmpmem will do. */
3858 opt_scalar_int_mode cmp_mode_iter
;
3859 FOR_EACH_MODE_IN_CLASS (cmp_mode_iter
, MODE_INT
)
3861 scalar_int_mode cmp_mode
= cmp_mode_iter
.require ();
3862 cmp_code
= direct_optab_handler (cmpmem_optab
, cmp_mode
);
3863 if (cmp_code
== CODE_FOR_nothing
)
3864 cmp_code
= direct_optab_handler (cmpstr_optab
, cmp_mode
);
3865 if (cmp_code
== CODE_FOR_nothing
)
3866 cmp_code
= direct_optab_handler (cmpstrn_optab
, cmp_mode
);
3867 if (cmp_code
== CODE_FOR_nothing
)
3870 /* Must make sure the size fits the insn's mode. */
3871 if (CONST_INT_P (size
)
3872 ? INTVAL (size
) >= (1 << GET_MODE_BITSIZE (cmp_mode
))
3873 : (GET_MODE_BITSIZE (as_a
<scalar_int_mode
> (GET_MODE (size
)))
3874 > GET_MODE_BITSIZE (cmp_mode
)))
3877 result_mode
= insn_data
[cmp_code
].operand
[0].mode
;
3878 result
= gen_reg_rtx (result_mode
);
3879 size
= convert_to_mode (cmp_mode
, size
, 1);
3880 emit_insn (GEN_FCN (cmp_code
) (result
, x
, y
, size
, opalign
));
3882 *ptest
= gen_rtx_fmt_ee (comparison
, VOIDmode
, result
, const0_rtx
);
3883 *pmode
= result_mode
;
3887 if (methods
!= OPTAB_LIB
&& methods
!= OPTAB_LIB_WIDEN
)
3890 /* Otherwise call a library function. */
3891 result
= emit_block_comp_via_libcall (XEXP (x
, 0), XEXP (y
, 0), size
);
3895 mode
= TYPE_MODE (integer_type_node
);
3896 methods
= OPTAB_LIB_WIDEN
;
3900 /* Don't allow operands to the compare to trap, as that can put the
3901 compare and branch in different basic blocks. */
3902 if (cfun
->can_throw_non_call_exceptions
)
3905 x
= copy_to_reg (x
);
3907 y
= copy_to_reg (y
);
3910 if (GET_MODE_CLASS (mode
) == MODE_CC
)
3912 enum insn_code icode
= optab_handler (cbranch_optab
, CCmode
);
3913 test
= gen_rtx_fmt_ee (comparison
, VOIDmode
, x
, y
);
3914 gcc_assert (icode
!= CODE_FOR_nothing
3915 && insn_operand_matches (icode
, 0, test
));
3920 mclass
= GET_MODE_CLASS (mode
);
3921 test
= gen_rtx_fmt_ee (comparison
, VOIDmode
, x
, y
);
3922 FOR_EACH_MODE_FROM (cmp_mode
, mode
)
3924 enum insn_code icode
;
3925 icode
= optab_handler (cbranch_optab
, cmp_mode
);
3926 if (icode
!= CODE_FOR_nothing
3927 && insn_operand_matches (icode
, 0, test
))
3929 rtx_insn
*last
= get_last_insn ();
3930 rtx op0
= prepare_operand (icode
, x
, 1, mode
, cmp_mode
, unsignedp
);
3931 rtx op1
= prepare_operand (icode
, y
, 2, mode
, cmp_mode
, unsignedp
);
3933 && insn_operand_matches (icode
, 1, op0
)
3934 && insn_operand_matches (icode
, 2, op1
))
3936 XEXP (test
, 0) = op0
;
3937 XEXP (test
, 1) = op1
;
3942 delete_insns_since (last
);
3945 if (methods
== OPTAB_DIRECT
|| !CLASS_HAS_WIDER_MODES_P (mclass
))
3949 if (methods
!= OPTAB_LIB_WIDEN
)
3952 if (SCALAR_FLOAT_MODE_P (mode
))
3954 /* Small trick if UNORDERED isn't implemented by the hardware. */
3955 if (comparison
== UNORDERED
&& rtx_equal_p (x
, y
))
3957 prepare_cmp_insn (x
, y
, UNLT
, NULL_RTX
, unsignedp
, OPTAB_WIDEN
,
3963 prepare_float_lib_cmp (x
, y
, comparison
, ptest
, pmode
);
3968 machine_mode ret_mode
;
3970 /* Handle a libcall just for the mode we are using. */
3971 libfunc
= optab_libfunc (cmp_optab
, mode
);
3972 gcc_assert (libfunc
);
3974 /* If we want unsigned, and this mode has a distinct unsigned
3975 comparison routine, use that. */
3978 rtx ulibfunc
= optab_libfunc (ucmp_optab
, mode
);
3983 ret_mode
= targetm
.libgcc_cmp_return_mode ();
3984 result
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
,
3985 ret_mode
, x
, mode
, y
, mode
);
3987 /* There are two kinds of comparison routines. Biased routines
3988 return 0/1/2, and unbiased routines return -1/0/1. Other parts
3989 of gcc expect that the comparison operation is equivalent
3990 to the modified comparison. For signed comparisons compare the
3991 result against 1 in the biased case, and zero in the unbiased
3992 case. For unsigned comparisons always compare against 1 after
3993 biasing the unbiased result by adding 1. This gives us a way to
3995 The comparisons in the fixed-point helper library are always
4000 if (!TARGET_LIB_INT_CMP_BIASED
&& !ALL_FIXED_POINT_MODE_P (mode
))
4003 x
= plus_constant (ret_mode
, result
, 1);
4009 prepare_cmp_insn (x
, y
, comparison
, NULL_RTX
, unsignedp
, methods
,
4019 /* Before emitting an insn with code ICODE, make sure that X, which is going
4020 to be used for operand OPNUM of the insn, is converted from mode MODE to
4021 WIDER_MODE (UNSIGNEDP determines whether it is an unsigned conversion), and
4022 that it is accepted by the operand predicate. Return the new value. */
4025 prepare_operand (enum insn_code icode
, rtx x
, int opnum
, machine_mode mode
,
4026 machine_mode wider_mode
, int unsignedp
)
4028 if (mode
!= wider_mode
)
4029 x
= convert_modes (wider_mode
, mode
, x
, unsignedp
);
4031 if (!insn_operand_matches (icode
, opnum
, x
))
4033 machine_mode op_mode
= insn_data
[(int) icode
].operand
[opnum
].mode
;
4034 if (reload_completed
)
4036 if (GET_MODE (x
) != op_mode
&& GET_MODE (x
) != VOIDmode
)
4038 x
= copy_to_mode_reg (op_mode
, x
);
4044 /* Subroutine of emit_cmp_and_jump_insns; this function is called when we know
4045 we can do the branch. */
4048 emit_cmp_and_jump_insn_1 (rtx test
, machine_mode mode
, rtx label
,
4049 profile_probability prob
)
4051 machine_mode optab_mode
;
4052 enum mode_class mclass
;
4053 enum insn_code icode
;
4056 mclass
= GET_MODE_CLASS (mode
);
4057 optab_mode
= (mclass
== MODE_CC
) ? CCmode
: mode
;
4058 icode
= optab_handler (cbranch_optab
, optab_mode
);
4060 gcc_assert (icode
!= CODE_FOR_nothing
);
4061 gcc_assert (insn_operand_matches (icode
, 0, test
));
4062 insn
= emit_jump_insn (GEN_FCN (icode
) (test
, XEXP (test
, 0),
4063 XEXP (test
, 1), label
));
4064 if (prob
.initialized_p ()
4065 && profile_status_for_fn (cfun
) != PROFILE_ABSENT
4068 && any_condjump_p (insn
)
4069 && !find_reg_note (insn
, REG_BR_PROB
, 0))
4070 add_reg_br_prob_note (insn
, prob
);
4073 /* Generate code to compare X with Y so that the condition codes are
4074 set and to jump to LABEL if the condition is true. If X is a
4075 constant and Y is not a constant, then the comparison is swapped to
4076 ensure that the comparison RTL has the canonical form.
4078 UNSIGNEDP nonzero says that X and Y are unsigned; this matters if they
4079 need to be widened. UNSIGNEDP is also used to select the proper
4080 branch condition code.
4082 If X and Y have mode BLKmode, then SIZE specifies the size of both X and Y.
4084 MODE is the mode of the inputs (in case they are const_int).
4086 COMPARISON is the rtl operator to compare with (EQ, NE, GT, etc.).
4087 It will be potentially converted into an unsigned variant based on
4088 UNSIGNEDP to select a proper jump instruction.
4090 PROB is the probability of jumping to LABEL. */
4093 emit_cmp_and_jump_insns (rtx x
, rtx y
, enum rtx_code comparison
, rtx size
,
4094 machine_mode mode
, int unsignedp
, rtx label
,
4095 profile_probability prob
)
4097 rtx op0
= x
, op1
= y
;
4100 /* Swap operands and condition to ensure canonical RTL. */
4101 if (swap_commutative_operands_p (x
, y
)
4102 && can_compare_p (swap_condition (comparison
), mode
, ccp_jump
))
4105 comparison
= swap_condition (comparison
);
4108 /* If OP0 is still a constant, then both X and Y must be constants
4109 or the opposite comparison is not supported. Force X into a register
4110 to create canonical RTL. */
4111 if (CONSTANT_P (op0
))
4112 op0
= force_reg (mode
, op0
);
4115 comparison
= unsigned_condition (comparison
);
4117 prepare_cmp_insn (op0
, op1
, comparison
, size
, unsignedp
, OPTAB_LIB_WIDEN
,
4119 emit_cmp_and_jump_insn_1 (test
, mode
, label
, prob
);
4123 /* Emit a library call comparison between floating point X and Y.
4124 COMPARISON is the rtl operator to compare with (EQ, NE, GT, etc.). */
4127 prepare_float_lib_cmp (rtx x
, rtx y
, enum rtx_code comparison
,
4128 rtx
*ptest
, machine_mode
*pmode
)
4130 enum rtx_code swapped
= swap_condition (comparison
);
4131 enum rtx_code reversed
= reverse_condition_maybe_unordered (comparison
);
4132 machine_mode orig_mode
= GET_MODE (x
);
4134 rtx true_rtx
, false_rtx
;
4135 rtx value
, target
, equiv
;
4138 bool reversed_p
= false;
4139 scalar_int_mode cmp_mode
= targetm
.libgcc_cmp_return_mode ();
4141 FOR_EACH_MODE_FROM (mode
, orig_mode
)
4143 if (code_to_optab (comparison
)
4144 && (libfunc
= optab_libfunc (code_to_optab (comparison
), mode
)))
4147 if (code_to_optab (swapped
)
4148 && (libfunc
= optab_libfunc (code_to_optab (swapped
), mode
)))
4151 comparison
= swapped
;
4155 if (code_to_optab (reversed
)
4156 && (libfunc
= optab_libfunc (code_to_optab (reversed
), mode
)))
4158 comparison
= reversed
;
4164 gcc_assert (mode
!= VOIDmode
);
4166 if (mode
!= orig_mode
)
4168 x
= convert_to_mode (mode
, x
, 0);
4169 y
= convert_to_mode (mode
, y
, 0);
4172 /* Attach a REG_EQUAL note describing the semantics of the libcall to
4173 the RTL. The allows the RTL optimizers to delete the libcall if the
4174 condition can be determined at compile-time. */
4175 if (comparison
== UNORDERED
4176 || FLOAT_LIB_COMPARE_RETURNS_BOOL (mode
, comparison
))
4178 true_rtx
= const_true_rtx
;
4179 false_rtx
= const0_rtx
;
4186 true_rtx
= const0_rtx
;
4187 false_rtx
= const_true_rtx
;
4191 true_rtx
= const_true_rtx
;
4192 false_rtx
= const0_rtx
;
4196 true_rtx
= const1_rtx
;
4197 false_rtx
= const0_rtx
;
4201 true_rtx
= const0_rtx
;
4202 false_rtx
= constm1_rtx
;
4206 true_rtx
= constm1_rtx
;
4207 false_rtx
= const0_rtx
;
4211 true_rtx
= const0_rtx
;
4212 false_rtx
= const1_rtx
;
4220 if (comparison
== UNORDERED
)
4222 rtx temp
= simplify_gen_relational (NE
, cmp_mode
, mode
, x
, x
);
4223 equiv
= simplify_gen_relational (NE
, cmp_mode
, mode
, y
, y
);
4224 equiv
= simplify_gen_ternary (IF_THEN_ELSE
, cmp_mode
, cmp_mode
,
4225 temp
, const_true_rtx
, equiv
);
4229 equiv
= simplify_gen_relational (comparison
, cmp_mode
, mode
, x
, y
);
4230 if (! FLOAT_LIB_COMPARE_RETURNS_BOOL (mode
, comparison
))
4231 equiv
= simplify_gen_ternary (IF_THEN_ELSE
, cmp_mode
, cmp_mode
,
4232 equiv
, true_rtx
, false_rtx
);
4236 value
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
,
4237 cmp_mode
, x
, mode
, y
, mode
);
4238 insns
= get_insns ();
4241 target
= gen_reg_rtx (cmp_mode
);
4242 emit_libcall_block (insns
, target
, value
, equiv
);
4244 if (comparison
== UNORDERED
4245 || FLOAT_LIB_COMPARE_RETURNS_BOOL (mode
, comparison
)
4247 *ptest
= gen_rtx_fmt_ee (reversed_p
? EQ
: NE
, VOIDmode
, target
, false_rtx
);
4249 *ptest
= gen_rtx_fmt_ee (comparison
, VOIDmode
, target
, const0_rtx
);
4254 /* Generate code to indirectly jump to a location given in the rtx LOC. */
4257 emit_indirect_jump (rtx loc
)
4259 if (!targetm
.have_indirect_jump ())
4260 sorry ("indirect jumps are not available on this target");
4263 struct expand_operand ops
[1];
4264 create_address_operand (&ops
[0], loc
);
4265 expand_jump_insn (targetm
.code_for_indirect_jump
, 1, ops
);
4271 /* Emit a conditional move instruction if the machine supports one for that
4272 condition and machine mode.
4274 OP0 and OP1 are the operands that should be compared using CODE. CMODE is
4275 the mode to use should they be constants. If it is VOIDmode, they cannot
4278 OP2 should be stored in TARGET if the comparison is true, otherwise OP3
4279 should be stored there. MODE is the mode to use should they be constants.
4280 If it is VOIDmode, they cannot both be constants.
4282 The result is either TARGET (perhaps modified) or NULL_RTX if the operation
4283 is not supported. */
4286 emit_conditional_move (rtx target
, enum rtx_code code
, rtx op0
, rtx op1
,
4287 machine_mode cmode
, rtx op2
, rtx op3
,
4288 machine_mode mode
, int unsignedp
)
4292 enum insn_code icode
;
4293 enum rtx_code reversed
;
4295 /* If the two source operands are identical, that's just a move. */
4297 if (rtx_equal_p (op2
, op3
))
4300 target
= gen_reg_rtx (mode
);
4302 emit_move_insn (target
, op3
);
4306 /* If one operand is constant, make it the second one. Only do this
4307 if the other operand is not constant as well. */
4309 if (swap_commutative_operands_p (op0
, op1
))
4311 std::swap (op0
, op1
);
4312 code
= swap_condition (code
);
4315 /* get_condition will prefer to generate LT and GT even if the old
4316 comparison was against zero, so undo that canonicalization here since
4317 comparisons against zero are cheaper. */
4318 if (code
== LT
&& op1
== const1_rtx
)
4319 code
= LE
, op1
= const0_rtx
;
4320 else if (code
== GT
&& op1
== constm1_rtx
)
4321 code
= GE
, op1
= const0_rtx
;
4323 if (cmode
== VOIDmode
)
4324 cmode
= GET_MODE (op0
);
4326 enum rtx_code orig_code
= code
;
4327 bool swapped
= false;
4328 if (swap_commutative_operands_p (op2
, op3
)
4329 && ((reversed
= reversed_comparison_code_parts (code
, op0
, op1
, NULL
))
4332 std::swap (op2
, op3
);
4337 if (mode
== VOIDmode
)
4338 mode
= GET_MODE (op2
);
4340 icode
= direct_optab_handler (movcc_optab
, mode
);
4342 if (icode
== CODE_FOR_nothing
)
4346 target
= gen_reg_rtx (mode
);
4348 for (int pass
= 0; ; pass
++)
4350 code
= unsignedp
? unsigned_condition (code
) : code
;
4351 comparison
= simplify_gen_relational (code
, VOIDmode
, cmode
, op0
, op1
);
4353 /* We can get const0_rtx or const_true_rtx in some circumstances. Just
4354 punt and let the caller figure out how best to deal with this
4356 if (COMPARISON_P (comparison
))
4358 saved_pending_stack_adjust save
;
4359 save_pending_stack_adjust (&save
);
4360 last
= get_last_insn ();
4361 do_pending_stack_adjust ();
4362 machine_mode cmpmode
= cmode
;
4363 prepare_cmp_insn (XEXP (comparison
, 0), XEXP (comparison
, 1),
4364 GET_CODE (comparison
), NULL_RTX
, unsignedp
,
4365 OPTAB_WIDEN
, &comparison
, &cmpmode
);
4368 struct expand_operand ops
[4];
4370 create_output_operand (&ops
[0], target
, mode
);
4371 create_fixed_operand (&ops
[1], comparison
);
4372 create_input_operand (&ops
[2], op2
, mode
);
4373 create_input_operand (&ops
[3], op3
, mode
);
4374 if (maybe_expand_insn (icode
, 4, ops
))
4376 if (ops
[0].value
!= target
)
4377 convert_move (target
, ops
[0].value
, false);
4381 delete_insns_since (last
);
4382 restore_pending_stack_adjust (&save
);
4388 /* If the preferred op2/op3 order is not usable, retry with other
4389 operand order, perhaps it will expand successfully. */
4392 else if ((reversed
= reversed_comparison_code_parts (orig_code
, op0
, op1
,
4398 std::swap (op2
, op3
);
4403 /* Emit a conditional negate or bitwise complement using the
4404 negcc or notcc optabs if available. Return NULL_RTX if such operations
4405 are not available. Otherwise return the RTX holding the result.
4406 TARGET is the desired destination of the result. COMP is the comparison
4407 on which to negate. If COND is true move into TARGET the negation
4408 or bitwise complement of OP1. Otherwise move OP2 into TARGET.
4409 CODE is either NEG or NOT. MODE is the machine mode in which the
4410 operation is performed. */
4413 emit_conditional_neg_or_complement (rtx target
, rtx_code code
,
4414 machine_mode mode
, rtx cond
, rtx op1
,
4417 optab op
= unknown_optab
;
4420 else if (code
== NOT
)
4425 insn_code icode
= direct_optab_handler (op
, mode
);
4427 if (icode
== CODE_FOR_nothing
)
4431 target
= gen_reg_rtx (mode
);
4433 rtx_insn
*last
= get_last_insn ();
4434 struct expand_operand ops
[4];
4436 create_output_operand (&ops
[0], target
, mode
);
4437 create_fixed_operand (&ops
[1], cond
);
4438 create_input_operand (&ops
[2], op1
, mode
);
4439 create_input_operand (&ops
[3], op2
, mode
);
4441 if (maybe_expand_insn (icode
, 4, ops
))
4443 if (ops
[0].value
!= target
)
4444 convert_move (target
, ops
[0].value
, false);
4448 delete_insns_since (last
);
4452 /* Emit a conditional addition instruction if the machine supports one for that
4453 condition and machine mode.
4455 OP0 and OP1 are the operands that should be compared using CODE. CMODE is
4456 the mode to use should they be constants. If it is VOIDmode, they cannot
4459 OP2 should be stored in TARGET if the comparison is false, otherwise OP2+OP3
4460 should be stored there. MODE is the mode to use should they be constants.
4461 If it is VOIDmode, they cannot both be constants.
4463 The result is either TARGET (perhaps modified) or NULL_RTX if the operation
4464 is not supported. */
4467 emit_conditional_add (rtx target
, enum rtx_code code
, rtx op0
, rtx op1
,
4468 machine_mode cmode
, rtx op2
, rtx op3
,
4469 machine_mode mode
, int unsignedp
)
4473 enum insn_code icode
;
4475 /* If one operand is constant, make it the second one. Only do this
4476 if the other operand is not constant as well. */
4478 if (swap_commutative_operands_p (op0
, op1
))
4480 std::swap (op0
, op1
);
4481 code
= swap_condition (code
);
4484 /* get_condition will prefer to generate LT and GT even if the old
4485 comparison was against zero, so undo that canonicalization here since
4486 comparisons against zero are cheaper. */
4487 if (code
== LT
&& op1
== const1_rtx
)
4488 code
= LE
, op1
= const0_rtx
;
4489 else if (code
== GT
&& op1
== constm1_rtx
)
4490 code
= GE
, op1
= const0_rtx
;
4492 if (cmode
== VOIDmode
)
4493 cmode
= GET_MODE (op0
);
4495 if (mode
== VOIDmode
)
4496 mode
= GET_MODE (op2
);
4498 icode
= optab_handler (addcc_optab
, mode
);
4500 if (icode
== CODE_FOR_nothing
)
4504 target
= gen_reg_rtx (mode
);
4506 code
= unsignedp
? unsigned_condition (code
) : code
;
4507 comparison
= simplify_gen_relational (code
, VOIDmode
, cmode
, op0
, op1
);
4509 /* We can get const0_rtx or const_true_rtx in some circumstances. Just
4510 return NULL and let the caller figure out how best to deal with this
4512 if (!COMPARISON_P (comparison
))
4515 do_pending_stack_adjust ();
4516 last
= get_last_insn ();
4517 prepare_cmp_insn (XEXP (comparison
, 0), XEXP (comparison
, 1),
4518 GET_CODE (comparison
), NULL_RTX
, unsignedp
, OPTAB_WIDEN
,
4519 &comparison
, &cmode
);
4522 struct expand_operand ops
[4];
4524 create_output_operand (&ops
[0], target
, mode
);
4525 create_fixed_operand (&ops
[1], comparison
);
4526 create_input_operand (&ops
[2], op2
, mode
);
4527 create_input_operand (&ops
[3], op3
, mode
);
4528 if (maybe_expand_insn (icode
, 4, ops
))
4530 if (ops
[0].value
!= target
)
4531 convert_move (target
, ops
[0].value
, false);
4535 delete_insns_since (last
);
4539 /* These functions attempt to generate an insn body, rather than
4540 emitting the insn, but if the gen function already emits them, we
4541 make no attempt to turn them back into naked patterns. */
4543 /* Generate and return an insn body to add Y to X. */
4546 gen_add2_insn (rtx x
, rtx y
)
4548 enum insn_code icode
= optab_handler (add_optab
, GET_MODE (x
));
4550 gcc_assert (insn_operand_matches (icode
, 0, x
));
4551 gcc_assert (insn_operand_matches (icode
, 1, x
));
4552 gcc_assert (insn_operand_matches (icode
, 2, y
));
4554 return GEN_FCN (icode
) (x
, x
, y
);
4557 /* Generate and return an insn body to add r1 and c,
4558 storing the result in r0. */
4561 gen_add3_insn (rtx r0
, rtx r1
, rtx c
)
4563 enum insn_code icode
= optab_handler (add_optab
, GET_MODE (r0
));
4565 if (icode
== CODE_FOR_nothing
4566 || !insn_operand_matches (icode
, 0, r0
)
4567 || !insn_operand_matches (icode
, 1, r1
)
4568 || !insn_operand_matches (icode
, 2, c
))
4571 return GEN_FCN (icode
) (r0
, r1
, c
);
4575 have_add2_insn (rtx x
, rtx y
)
4577 enum insn_code icode
;
4579 gcc_assert (GET_MODE (x
) != VOIDmode
);
4581 icode
= optab_handler (add_optab
, GET_MODE (x
));
4583 if (icode
== CODE_FOR_nothing
)
4586 if (!insn_operand_matches (icode
, 0, x
)
4587 || !insn_operand_matches (icode
, 1, x
)
4588 || !insn_operand_matches (icode
, 2, y
))
4594 /* Generate and return an insn body to add Y to X. */
4597 gen_addptr3_insn (rtx x
, rtx y
, rtx z
)
4599 enum insn_code icode
= optab_handler (addptr3_optab
, GET_MODE (x
));
4601 gcc_assert (insn_operand_matches (icode
, 0, x
));
4602 gcc_assert (insn_operand_matches (icode
, 1, y
));
4603 gcc_assert (insn_operand_matches (icode
, 2, z
));
4605 return GEN_FCN (icode
) (x
, y
, z
);
4608 /* Return true if the target implements an addptr pattern and X, Y,
4609 and Z are valid for the pattern predicates. */
4612 have_addptr3_insn (rtx x
, rtx y
, rtx z
)
4614 enum insn_code icode
;
4616 gcc_assert (GET_MODE (x
) != VOIDmode
);
4618 icode
= optab_handler (addptr3_optab
, GET_MODE (x
));
4620 if (icode
== CODE_FOR_nothing
)
4623 if (!insn_operand_matches (icode
, 0, x
)
4624 || !insn_operand_matches (icode
, 1, y
)
4625 || !insn_operand_matches (icode
, 2, z
))
4631 /* Generate and return an insn body to subtract Y from X. */
4634 gen_sub2_insn (rtx x
, rtx y
)
4636 enum insn_code icode
= optab_handler (sub_optab
, GET_MODE (x
));
4638 gcc_assert (insn_operand_matches (icode
, 0, x
));
4639 gcc_assert (insn_operand_matches (icode
, 1, x
));
4640 gcc_assert (insn_operand_matches (icode
, 2, y
));
4642 return GEN_FCN (icode
) (x
, x
, y
);
4645 /* Generate and return an insn body to subtract r1 and c,
4646 storing the result in r0. */
4649 gen_sub3_insn (rtx r0
, rtx r1
, rtx c
)
4651 enum insn_code icode
= optab_handler (sub_optab
, GET_MODE (r0
));
4653 if (icode
== CODE_FOR_nothing
4654 || !insn_operand_matches (icode
, 0, r0
)
4655 || !insn_operand_matches (icode
, 1, r1
)
4656 || !insn_operand_matches (icode
, 2, c
))
4659 return GEN_FCN (icode
) (r0
, r1
, c
);
4663 have_sub2_insn (rtx x
, rtx y
)
4665 enum insn_code icode
;
4667 gcc_assert (GET_MODE (x
) != VOIDmode
);
4669 icode
= optab_handler (sub_optab
, GET_MODE (x
));
4671 if (icode
== CODE_FOR_nothing
)
4674 if (!insn_operand_matches (icode
, 0, x
)
4675 || !insn_operand_matches (icode
, 1, x
)
4676 || !insn_operand_matches (icode
, 2, y
))
4682 /* Generate the body of an insn to extend Y (with mode MFROM)
4683 into X (with mode MTO). Do zero-extension if UNSIGNEDP is nonzero. */
4686 gen_extend_insn (rtx x
, rtx y
, machine_mode mto
,
4687 machine_mode mfrom
, int unsignedp
)
4689 enum insn_code icode
= can_extend_p (mto
, mfrom
, unsignedp
);
4690 return GEN_FCN (icode
) (x
, y
);
4693 /* Generate code to convert FROM to floating point
4694 and store in TO. FROM must be fixed point and not VOIDmode.
4695 UNSIGNEDP nonzero means regard FROM as unsigned.
4696 Normally this is done by correcting the final value
4697 if it is negative. */
4700 expand_float (rtx to
, rtx from
, int unsignedp
)
4702 enum insn_code icode
;
4704 scalar_mode from_mode
, to_mode
;
4705 machine_mode fmode
, imode
;
4706 bool can_do_signed
= false;
4708 /* Crash now, because we won't be able to decide which mode to use. */
4709 gcc_assert (GET_MODE (from
) != VOIDmode
);
4711 /* Look for an insn to do the conversion. Do it in the specified
4712 modes if possible; otherwise convert either input, output or both to
4713 wider mode. If the integer mode is wider than the mode of FROM,
4714 we can do the conversion signed even if the input is unsigned. */
4716 FOR_EACH_MODE_FROM (fmode
, GET_MODE (to
))
4717 FOR_EACH_MODE_FROM (imode
, GET_MODE (from
))
4719 int doing_unsigned
= unsignedp
;
4721 if (fmode
!= GET_MODE (to
)
4722 && (significand_size (fmode
)
4723 < GET_MODE_UNIT_PRECISION (GET_MODE (from
))))
4726 icode
= can_float_p (fmode
, imode
, unsignedp
);
4727 if (icode
== CODE_FOR_nothing
&& unsignedp
)
4729 enum insn_code scode
= can_float_p (fmode
, imode
, 0);
4730 if (scode
!= CODE_FOR_nothing
)
4731 can_do_signed
= true;
4732 if (imode
!= GET_MODE (from
))
4733 icode
= scode
, doing_unsigned
= 0;
4736 if (icode
!= CODE_FOR_nothing
)
4738 if (imode
!= GET_MODE (from
))
4739 from
= convert_to_mode (imode
, from
, unsignedp
);
4741 if (fmode
!= GET_MODE (to
))
4742 target
= gen_reg_rtx (fmode
);
4744 emit_unop_insn (icode
, target
, from
,
4745 doing_unsigned
? UNSIGNED_FLOAT
: FLOAT
);
4748 convert_move (to
, target
, 0);
4753 /* Unsigned integer, and no way to convert directly. Convert as signed,
4754 then unconditionally adjust the result. */
4757 && is_a
<scalar_mode
> (GET_MODE (to
), &to_mode
)
4758 && is_a
<scalar_mode
> (GET_MODE (from
), &from_mode
))
4760 opt_scalar_mode fmode_iter
;
4761 rtx_code_label
*label
= gen_label_rtx ();
4763 REAL_VALUE_TYPE offset
;
4765 /* Look for a usable floating mode FMODE wider than the source and at
4766 least as wide as the target. Using FMODE will avoid rounding woes
4767 with unsigned values greater than the signed maximum value. */
4769 FOR_EACH_MODE_FROM (fmode_iter
, to_mode
)
4771 scalar_mode fmode
= fmode_iter
.require ();
4772 if (GET_MODE_PRECISION (from_mode
) < GET_MODE_BITSIZE (fmode
)
4773 && can_float_p (fmode
, from_mode
, 0) != CODE_FOR_nothing
)
4777 if (!fmode_iter
.exists (&fmode
))
4779 /* There is no such mode. Pretend the target is wide enough. */
4782 /* Avoid double-rounding when TO is narrower than FROM. */
4783 if ((significand_size (fmode
) + 1)
4784 < GET_MODE_PRECISION (from_mode
))
4787 rtx_code_label
*neglabel
= gen_label_rtx ();
4789 /* Don't use TARGET if it isn't a register, is a hard register,
4790 or is the wrong mode. */
4792 || REGNO (target
) < FIRST_PSEUDO_REGISTER
4793 || GET_MODE (target
) != fmode
)
4794 target
= gen_reg_rtx (fmode
);
4797 do_pending_stack_adjust ();
4799 /* Test whether the sign bit is set. */
4800 emit_cmp_and_jump_insns (from
, const0_rtx
, LT
, NULL_RTX
, imode
,
4803 /* The sign bit is not set. Convert as signed. */
4804 expand_float (target
, from
, 0);
4805 emit_jump_insn (targetm
.gen_jump (label
));
4808 /* The sign bit is set.
4809 Convert to a usable (positive signed) value by shifting right
4810 one bit, while remembering if a nonzero bit was shifted
4811 out; i.e., compute (from & 1) | (from >> 1). */
4813 emit_label (neglabel
);
4814 temp
= expand_binop (imode
, and_optab
, from
, const1_rtx
,
4815 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
4816 temp1
= expand_shift (RSHIFT_EXPR
, imode
, from
, 1, NULL_RTX
, 1);
4817 temp
= expand_binop (imode
, ior_optab
, temp
, temp1
, temp
, 1,
4819 expand_float (target
, temp
, 0);
4821 /* Multiply by 2 to undo the shift above. */
4822 temp
= expand_binop (fmode
, add_optab
, target
, target
,
4823 target
, 0, OPTAB_LIB_WIDEN
);
4825 emit_move_insn (target
, temp
);
4827 do_pending_stack_adjust ();
4833 /* If we are about to do some arithmetic to correct for an
4834 unsigned operand, do it in a pseudo-register. */
4836 if (to_mode
!= fmode
4837 || !REG_P (to
) || REGNO (to
) < FIRST_PSEUDO_REGISTER
)
4838 target
= gen_reg_rtx (fmode
);
4840 /* Convert as signed integer to floating. */
4841 expand_float (target
, from
, 0);
4843 /* If FROM is negative (and therefore TO is negative),
4844 correct its value by 2**bitwidth. */
4846 do_pending_stack_adjust ();
4847 emit_cmp_and_jump_insns (from
, const0_rtx
, GE
, NULL_RTX
, from_mode
,
4851 real_2expN (&offset
, GET_MODE_PRECISION (from_mode
), fmode
);
4852 temp
= expand_binop (fmode
, add_optab
, target
,
4853 const_double_from_real_value (offset
, fmode
),
4854 target
, 0, OPTAB_LIB_WIDEN
);
4856 emit_move_insn (target
, temp
);
4858 do_pending_stack_adjust ();
4863 /* No hardware instruction available; call a library routine. */
4868 convert_optab tab
= unsignedp
? ufloat_optab
: sfloat_optab
;
4870 if (is_narrower_int_mode (GET_MODE (from
), SImode
))
4871 from
= convert_to_mode (SImode
, from
, unsignedp
);
4873 libfunc
= convert_optab_libfunc (tab
, GET_MODE (to
), GET_MODE (from
));
4874 gcc_assert (libfunc
);
4878 value
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
,
4879 GET_MODE (to
), from
, GET_MODE (from
));
4880 insns
= get_insns ();
4883 emit_libcall_block (insns
, target
, value
,
4884 gen_rtx_fmt_e (unsignedp
? UNSIGNED_FLOAT
: FLOAT
,
4885 GET_MODE (to
), from
));
4890 /* Copy result to requested destination
4891 if we have been computing in a temp location. */
4895 if (GET_MODE (target
) == GET_MODE (to
))
4896 emit_move_insn (to
, target
);
4898 convert_move (to
, target
, 0);
4902 /* Generate code to convert FROM to fixed point and store in TO. FROM
4903 must be floating point. */
4906 expand_fix (rtx to
, rtx from
, int unsignedp
)
4908 enum insn_code icode
;
4910 machine_mode fmode
, imode
;
4911 opt_scalar_mode fmode_iter
;
4912 bool must_trunc
= false;
4914 /* We first try to find a pair of modes, one real and one integer, at
4915 least as wide as FROM and TO, respectively, in which we can open-code
4916 this conversion. If the integer mode is wider than the mode of TO,
4917 we can do the conversion either signed or unsigned. */
4919 FOR_EACH_MODE_FROM (fmode
, GET_MODE (from
))
4920 FOR_EACH_MODE_FROM (imode
, GET_MODE (to
))
4922 int doing_unsigned
= unsignedp
;
4924 icode
= can_fix_p (imode
, fmode
, unsignedp
, &must_trunc
);
4925 if (icode
== CODE_FOR_nothing
&& imode
!= GET_MODE (to
) && unsignedp
)
4926 icode
= can_fix_p (imode
, fmode
, 0, &must_trunc
), doing_unsigned
= 0;
4928 if (icode
!= CODE_FOR_nothing
)
4930 rtx_insn
*last
= get_last_insn ();
4931 if (fmode
!= GET_MODE (from
))
4932 from
= convert_to_mode (fmode
, from
, 0);
4936 rtx temp
= gen_reg_rtx (GET_MODE (from
));
4937 from
= expand_unop (GET_MODE (from
), ftrunc_optab
, from
,
4941 if (imode
!= GET_MODE (to
))
4942 target
= gen_reg_rtx (imode
);
4944 if (maybe_emit_unop_insn (icode
, target
, from
,
4945 doing_unsigned
? UNSIGNED_FIX
: FIX
))
4948 convert_move (to
, target
, unsignedp
);
4951 delete_insns_since (last
);
4955 /* For an unsigned conversion, there is one more way to do it.
4956 If we have a signed conversion, we generate code that compares
4957 the real value to the largest representable positive number. If if
4958 is smaller, the conversion is done normally. Otherwise, subtract
4959 one plus the highest signed number, convert, and add it back.
4961 We only need to check all real modes, since we know we didn't find
4962 anything with a wider integer mode.
4964 This code used to extend FP value into mode wider than the destination.
4965 This is needed for decimal float modes which cannot accurately
4966 represent one plus the highest signed number of the same size, but
4967 not for binary modes. Consider, for instance conversion from SFmode
4970 The hot path through the code is dealing with inputs smaller than 2^63
4971 and doing just the conversion, so there is no bits to lose.
4973 In the other path we know the value is positive in the range 2^63..2^64-1
4974 inclusive. (as for other input overflow happens and result is undefined)
4975 So we know that the most important bit set in mantissa corresponds to
4976 2^63. The subtraction of 2^63 should not generate any rounding as it
4977 simply clears out that bit. The rest is trivial. */
4979 scalar_int_mode to_mode
;
4981 && is_a
<scalar_int_mode
> (GET_MODE (to
), &to_mode
)
4982 && HWI_COMPUTABLE_MODE_P (to_mode
))
4983 FOR_EACH_MODE_FROM (fmode_iter
, as_a
<scalar_mode
> (GET_MODE (from
)))
4985 scalar_mode fmode
= fmode_iter
.require ();
4986 if (CODE_FOR_nothing
!= can_fix_p (to_mode
, fmode
,
4988 && (!DECIMAL_FLOAT_MODE_P (fmode
)
4989 || (GET_MODE_BITSIZE (fmode
) > GET_MODE_PRECISION (to_mode
))))
4992 REAL_VALUE_TYPE offset
;
4994 rtx_code_label
*lab1
, *lab2
;
4997 bitsize
= GET_MODE_PRECISION (to_mode
);
4998 real_2expN (&offset
, bitsize
- 1, fmode
);
4999 limit
= const_double_from_real_value (offset
, fmode
);
5000 lab1
= gen_label_rtx ();
5001 lab2
= gen_label_rtx ();
5003 if (fmode
!= GET_MODE (from
))
5004 from
= convert_to_mode (fmode
, from
, 0);
5006 /* See if we need to do the subtraction. */
5007 do_pending_stack_adjust ();
5008 emit_cmp_and_jump_insns (from
, limit
, GE
, NULL_RTX
,
5009 GET_MODE (from
), 0, lab1
);
5011 /* If not, do the signed "fix" and branch around fixup code. */
5012 expand_fix (to
, from
, 0);
5013 emit_jump_insn (targetm
.gen_jump (lab2
));
5016 /* Otherwise, subtract 2**(N-1), convert to signed number,
5017 then add 2**(N-1). Do the addition using XOR since this
5018 will often generate better code. */
5020 target
= expand_binop (GET_MODE (from
), sub_optab
, from
, limit
,
5021 NULL_RTX
, 0, OPTAB_LIB_WIDEN
);
5022 expand_fix (to
, target
, 0);
5023 target
= expand_binop (to_mode
, xor_optab
, to
,
5025 (HOST_WIDE_INT_1
<< (bitsize
- 1),
5027 to
, 1, OPTAB_LIB_WIDEN
);
5030 emit_move_insn (to
, target
);
5034 if (optab_handler (mov_optab
, to_mode
) != CODE_FOR_nothing
)
5036 /* Make a place for a REG_NOTE and add it. */
5037 insn
= emit_move_insn (to
, to
);
5038 set_dst_reg_note (insn
, REG_EQUAL
,
5039 gen_rtx_fmt_e (UNSIGNED_FIX
, to_mode
,
5048 /* We can't do it with an insn, so use a library call. But first ensure
5049 that the mode of TO is at least as wide as SImode, since those are the
5050 only library calls we know about. */
5052 if (is_narrower_int_mode (GET_MODE (to
), SImode
))
5054 target
= gen_reg_rtx (SImode
);
5056 expand_fix (target
, from
, unsignedp
);
5064 convert_optab tab
= unsignedp
? ufix_optab
: sfix_optab
;
5065 libfunc
= convert_optab_libfunc (tab
, GET_MODE (to
), GET_MODE (from
));
5066 gcc_assert (libfunc
);
5070 value
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
,
5071 GET_MODE (to
), from
, GET_MODE (from
));
5072 insns
= get_insns ();
5075 emit_libcall_block (insns
, target
, value
,
5076 gen_rtx_fmt_e (unsignedp
? UNSIGNED_FIX
: FIX
,
5077 GET_MODE (to
), from
));
5082 if (GET_MODE (to
) == GET_MODE (target
))
5083 emit_move_insn (to
, target
);
5085 convert_move (to
, target
, 0);
5090 /* Promote integer arguments for a libcall if necessary.
5091 emit_library_call_value cannot do the promotion because it does not
5092 know if it should do a signed or unsigned promotion. This is because
5093 there are no tree types defined for libcalls. */
5096 prepare_libcall_arg (rtx arg
, int uintp
)
5098 scalar_int_mode mode
;
5099 machine_mode arg_mode
;
5100 if (is_a
<scalar_int_mode
> (GET_MODE (arg
), &mode
))
5102 /* If we need to promote the integer function argument we need to do
5103 it here instead of inside emit_library_call_value because in
5104 emit_library_call_value we don't know if we should do a signed or
5105 unsigned promotion. */
5108 arg_mode
= promote_function_mode (NULL_TREE
, mode
,
5109 &unsigned_p
, NULL_TREE
, 0);
5110 if (arg_mode
!= mode
)
5111 return convert_to_mode (arg_mode
, arg
, uintp
);
5116 /* Generate code to convert FROM or TO a fixed-point.
5117 If UINTP is true, either TO or FROM is an unsigned integer.
5118 If SATP is true, we need to saturate the result. */
5121 expand_fixed_convert (rtx to
, rtx from
, int uintp
, int satp
)
5123 machine_mode to_mode
= GET_MODE (to
);
5124 machine_mode from_mode
= GET_MODE (from
);
5126 enum rtx_code this_code
;
5127 enum insn_code code
;
5132 if (to_mode
== from_mode
)
5134 emit_move_insn (to
, from
);
5140 tab
= satp
? satfractuns_optab
: fractuns_optab
;
5141 this_code
= satp
? UNSIGNED_SAT_FRACT
: UNSIGNED_FRACT_CONVERT
;
5145 tab
= satp
? satfract_optab
: fract_optab
;
5146 this_code
= satp
? SAT_FRACT
: FRACT_CONVERT
;
5148 code
= convert_optab_handler (tab
, to_mode
, from_mode
);
5149 if (code
!= CODE_FOR_nothing
)
5151 emit_unop_insn (code
, to
, from
, this_code
);
5155 libfunc
= convert_optab_libfunc (tab
, to_mode
, from_mode
);
5156 gcc_assert (libfunc
);
5158 from
= prepare_libcall_arg (from
, uintp
);
5159 from_mode
= GET_MODE (from
);
5162 value
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
, to_mode
,
5164 insns
= get_insns ();
5167 emit_libcall_block (insns
, to
, value
,
5168 gen_rtx_fmt_e (optab_to_code (tab
), to_mode
, from
));
5171 /* Generate code to convert FROM to fixed point and store in TO. FROM
5172 must be floating point, TO must be signed. Use the conversion optab
5173 TAB to do the conversion. */
5176 expand_sfix_optab (rtx to
, rtx from
, convert_optab tab
)
5178 enum insn_code icode
;
5180 machine_mode fmode
, imode
;
5182 /* We first try to find a pair of modes, one real and one integer, at
5183 least as wide as FROM and TO, respectively, in which we can open-code
5184 this conversion. If the integer mode is wider than the mode of TO,
5185 we can do the conversion either signed or unsigned. */
5187 FOR_EACH_MODE_FROM (fmode
, GET_MODE (from
))
5188 FOR_EACH_MODE_FROM (imode
, GET_MODE (to
))
5190 icode
= convert_optab_handler (tab
, imode
, fmode
);
5191 if (icode
!= CODE_FOR_nothing
)
5193 rtx_insn
*last
= get_last_insn ();
5194 if (fmode
!= GET_MODE (from
))
5195 from
= convert_to_mode (fmode
, from
, 0);
5197 if (imode
!= GET_MODE (to
))
5198 target
= gen_reg_rtx (imode
);
5200 if (!maybe_emit_unop_insn (icode
, target
, from
, UNKNOWN
))
5202 delete_insns_since (last
);
5206 convert_move (to
, target
, 0);
5214 /* Report whether we have an instruction to perform the operation
5215 specified by CODE on operands of mode MODE. */
5217 have_insn_for (enum rtx_code code
, machine_mode mode
)
5219 return (code_to_optab (code
)
5220 && (optab_handler (code_to_optab (code
), mode
)
5221 != CODE_FOR_nothing
));
5224 /* Print information about the current contents of the optabs on
5228 debug_optab_libfuncs (void)
5232 /* Dump the arithmetic optabs. */
5233 for (i
= FIRST_NORM_OPTAB
; i
<= LAST_NORMLIB_OPTAB
; ++i
)
5234 for (j
= 0; j
< NUM_MACHINE_MODES
; ++j
)
5236 rtx l
= optab_libfunc ((optab
) i
, (machine_mode
) j
);
5239 gcc_assert (GET_CODE (l
) == SYMBOL_REF
);
5240 fprintf (stderr
, "%s\t%s:\t%s\n",
5241 GET_RTX_NAME (optab_to_code ((optab
) i
)),
5247 /* Dump the conversion optabs. */
5248 for (i
= FIRST_CONV_OPTAB
; i
<= LAST_CONVLIB_OPTAB
; ++i
)
5249 for (j
= 0; j
< NUM_MACHINE_MODES
; ++j
)
5250 for (k
= 0; k
< NUM_MACHINE_MODES
; ++k
)
5252 rtx l
= convert_optab_libfunc ((optab
) i
, (machine_mode
) j
,
5256 gcc_assert (GET_CODE (l
) == SYMBOL_REF
);
5257 fprintf (stderr
, "%s\t%s\t%s:\t%s\n",
5258 GET_RTX_NAME (optab_to_code ((optab
) i
)),
5266 /* Generate insns to trap with code TCODE if OP1 and OP2 satisfy condition
5267 CODE. Return 0 on failure. */
5270 gen_cond_trap (enum rtx_code code
, rtx op1
, rtx op2
, rtx tcode
)
5272 machine_mode mode
= GET_MODE (op1
);
5273 enum insn_code icode
;
5277 if (mode
== VOIDmode
)
5280 icode
= optab_handler (ctrap_optab
, mode
);
5281 if (icode
== CODE_FOR_nothing
)
5284 /* Some targets only accept a zero trap code. */
5285 if (!insn_operand_matches (icode
, 3, tcode
))
5288 do_pending_stack_adjust ();
5290 prepare_cmp_insn (op1
, op2
, code
, NULL_RTX
, false, OPTAB_DIRECT
,
5295 insn
= GEN_FCN (icode
) (trap_rtx
, XEXP (trap_rtx
, 0), XEXP (trap_rtx
, 1),
5298 /* If that failed, then give up. */
5306 insn
= get_insns ();
5311 /* Return rtx code for TCODE. Use UNSIGNEDP to select signed
5312 or unsigned operation code. */
5315 get_rtx_code (enum tree_code tcode
, bool unsignedp
)
5327 code
= unsignedp
? LTU
: LT
;
5330 code
= unsignedp
? LEU
: LE
;
5333 code
= unsignedp
? GTU
: GT
;
5336 code
= unsignedp
? GEU
: GE
;
5339 case UNORDERED_EXPR
:
5378 /* Return a comparison rtx of mode CMP_MODE for COND. Use UNSIGNEDP to
5379 select signed or unsigned operators. OPNO holds the index of the
5380 first comparison operand for insn ICODE. Do not generate the
5381 compare instruction itself. */
5384 vector_compare_rtx (machine_mode cmp_mode
, enum tree_code tcode
,
5385 tree t_op0
, tree t_op1
, bool unsignedp
,
5386 enum insn_code icode
, unsigned int opno
)
5388 struct expand_operand ops
[2];
5389 rtx rtx_op0
, rtx_op1
;
5390 machine_mode m0
, m1
;
5391 enum rtx_code rcode
= get_rtx_code (tcode
, unsignedp
);
5393 gcc_assert (TREE_CODE_CLASS (tcode
) == tcc_comparison
);
5395 /* Expand operands. For vector types with scalar modes, e.g. where int64x1_t
5396 has mode DImode, this can produce a constant RTX of mode VOIDmode; in such
5397 cases, use the original mode. */
5398 rtx_op0
= expand_expr (t_op0
, NULL_RTX
, TYPE_MODE (TREE_TYPE (t_op0
)),
5400 m0
= GET_MODE (rtx_op0
);
5402 m0
= TYPE_MODE (TREE_TYPE (t_op0
));
5404 rtx_op1
= expand_expr (t_op1
, NULL_RTX
, TYPE_MODE (TREE_TYPE (t_op1
)),
5406 m1
= GET_MODE (rtx_op1
);
5408 m1
= TYPE_MODE (TREE_TYPE (t_op1
));
5410 create_input_operand (&ops
[0], rtx_op0
, m0
);
5411 create_input_operand (&ops
[1], rtx_op1
, m1
);
5412 if (!maybe_legitimize_operands (icode
, opno
, 2, ops
))
5414 return gen_rtx_fmt_ee (rcode
, cmp_mode
, ops
[0].value
, ops
[1].value
);
5417 /* Check if vec_perm mask SEL is a constant equivalent to a shift of
5418 the first vec_perm operand, assuming the second operand is a constant
5419 vector of zeros. Return the shift distance in bits if so, or NULL_RTX
5420 if the vec_perm is not a shift. MODE is the mode of the value being
5423 shift_amt_for_vec_perm_mask (machine_mode mode
, const vec_perm_indices
&sel
)
5425 unsigned int bitsize
= GET_MODE_UNIT_BITSIZE (mode
);
5426 poly_int64 first
= sel
[0];
5427 if (maybe_ge (sel
[0], GET_MODE_NUNITS (mode
)))
5430 if (!sel
.series_p (0, 1, first
, 1))
5433 if (!GET_MODE_NUNITS (mode
).is_constant (&nelt
))
5435 for (unsigned int i
= 1; i
< nelt
; i
++)
5437 poly_int64 expected
= i
+ first
;
5438 /* Indices into the second vector are all equivalent. */
5439 if (maybe_lt (sel
[i
], nelt
)
5440 ? maybe_ne (sel
[i
], expected
)
5441 : maybe_lt (expected
, nelt
))
5446 return gen_int_shift_amount (mode
, first
* bitsize
);
5449 /* A subroutine of expand_vec_perm_var for expanding one vec_perm insn. */
5452 expand_vec_perm_1 (enum insn_code icode
, rtx target
,
5453 rtx v0
, rtx v1
, rtx sel
)
5455 machine_mode tmode
= GET_MODE (target
);
5456 machine_mode smode
= GET_MODE (sel
);
5457 struct expand_operand ops
[4];
5459 gcc_assert (GET_MODE_CLASS (smode
) == MODE_VECTOR_INT
5460 || mode_for_int_vector (tmode
).require () == smode
);
5461 create_output_operand (&ops
[0], target
, tmode
);
5462 create_input_operand (&ops
[3], sel
, smode
);
5464 /* Make an effort to preserve v0 == v1. The target expander is able to
5465 rely on this to determine if we're permuting a single input operand. */
5466 if (rtx_equal_p (v0
, v1
))
5468 if (!insn_operand_matches (icode
, 1, v0
))
5469 v0
= force_reg (tmode
, v0
);
5470 gcc_checking_assert (insn_operand_matches (icode
, 1, v0
));
5471 gcc_checking_assert (insn_operand_matches (icode
, 2, v0
));
5473 create_fixed_operand (&ops
[1], v0
);
5474 create_fixed_operand (&ops
[2], v0
);
5478 create_input_operand (&ops
[1], v0
, tmode
);
5479 create_input_operand (&ops
[2], v1
, tmode
);
5482 if (maybe_expand_insn (icode
, 4, ops
))
5483 return ops
[0].value
;
5487 /* Implement a permutation of vectors v0 and v1 using the permutation
5488 vector in SEL and return the result. Use TARGET to hold the result
5489 if nonnull and convenient.
5491 MODE is the mode of the vectors being permuted (V0 and V1). SEL_MODE
5492 is the TYPE_MODE associated with SEL, or BLKmode if SEL isn't known
5493 to have a particular mode. */
5496 expand_vec_perm_const (machine_mode mode
, rtx v0
, rtx v1
,
5497 const vec_perm_builder
&sel
, machine_mode sel_mode
,
5500 if (!target
|| !register_operand (target
, mode
))
5501 target
= gen_reg_rtx (mode
);
5503 /* Set QIMODE to a different vector mode with byte elements.
5504 If no such mode, or if MODE already has byte elements, use VOIDmode. */
5505 machine_mode qimode
;
5506 if (!qimode_for_vec_perm (mode
).exists (&qimode
))
5509 rtx_insn
*last
= get_last_insn ();
5511 bool single_arg_p
= rtx_equal_p (v0
, v1
);
5512 /* Always specify two input vectors here and leave the target to handle
5513 cases in which the inputs are equal. Not all backends can cope with
5514 the single-input representation when testing for a double-input
5515 target instruction. */
5516 vec_perm_indices
indices (sel
, 2, GET_MODE_NUNITS (mode
));
5518 /* See if this can be handled with a vec_shr. We only do this if the
5519 second vector is all zeroes. */
5520 insn_code shift_code
= optab_handler (vec_shr_optab
, mode
);
5521 insn_code shift_code_qi
= ((qimode
!= VOIDmode
&& qimode
!= mode
)
5522 ? optab_handler (vec_shr_optab
, qimode
)
5523 : CODE_FOR_nothing
);
5525 if (v1
== CONST0_RTX (GET_MODE (v1
))
5526 && (shift_code
!= CODE_FOR_nothing
5527 || shift_code_qi
!= CODE_FOR_nothing
))
5529 rtx shift_amt
= shift_amt_for_vec_perm_mask (mode
, indices
);
5532 struct expand_operand ops
[3];
5533 if (shift_code
!= CODE_FOR_nothing
)
5535 create_output_operand (&ops
[0], target
, mode
);
5536 create_input_operand (&ops
[1], v0
, mode
);
5537 create_convert_operand_from_type (&ops
[2], shift_amt
, sizetype
);
5538 if (maybe_expand_insn (shift_code
, 3, ops
))
5539 return ops
[0].value
;
5541 if (shift_code_qi
!= CODE_FOR_nothing
)
5543 rtx tmp
= gen_reg_rtx (qimode
);
5544 create_output_operand (&ops
[0], tmp
, qimode
);
5545 create_input_operand (&ops
[1], gen_lowpart (qimode
, v0
), qimode
);
5546 create_convert_operand_from_type (&ops
[2], shift_amt
, sizetype
);
5547 if (maybe_expand_insn (shift_code_qi
, 3, ops
))
5548 return gen_lowpart (mode
, ops
[0].value
);
5553 if (targetm
.vectorize
.vec_perm_const
!= NULL
)
5555 v0
= force_reg (mode
, v0
);
5559 v1
= force_reg (mode
, v1
);
5561 if (targetm
.vectorize
.vec_perm_const (mode
, target
, v0
, v1
, indices
))
5565 /* Fall back to a constant byte-based permutation. */
5566 vec_perm_indices qimode_indices
;
5567 rtx target_qi
= NULL_RTX
, v0_qi
= NULL_RTX
, v1_qi
= NULL_RTX
;
5568 if (qimode
!= VOIDmode
)
5570 qimode_indices
.new_expanded_vector (indices
, GET_MODE_UNIT_SIZE (mode
));
5571 target_qi
= gen_reg_rtx (qimode
);
5572 v0_qi
= gen_lowpart (qimode
, v0
);
5573 v1_qi
= gen_lowpart (qimode
, v1
);
5574 if (targetm
.vectorize
.vec_perm_const
!= NULL
5575 && targetm
.vectorize
.vec_perm_const (qimode
, target_qi
, v0_qi
,
5576 v1_qi
, qimode_indices
))
5577 return gen_lowpart (mode
, target_qi
);
5580 /* Otherwise expand as a fully variable permuation. */
5582 /* The optabs are only defined for selectors with the same width
5583 as the values being permuted. */
5584 machine_mode required_sel_mode
;
5585 if (!mode_for_int_vector (mode
).exists (&required_sel_mode
)
5586 || !VECTOR_MODE_P (required_sel_mode
))
5588 delete_insns_since (last
);
5592 /* We know that it is semantically valid to treat SEL as having SEL_MODE.
5593 If that isn't the mode we want then we need to prove that using
5594 REQUIRED_SEL_MODE is OK. */
5595 if (sel_mode
!= required_sel_mode
)
5597 if (!selector_fits_mode_p (required_sel_mode
, indices
))
5599 delete_insns_since (last
);
5602 sel_mode
= required_sel_mode
;
5605 insn_code icode
= direct_optab_handler (vec_perm_optab
, mode
);
5606 if (icode
!= CODE_FOR_nothing
)
5608 rtx sel_rtx
= vec_perm_indices_to_rtx (sel_mode
, indices
);
5609 rtx tmp
= expand_vec_perm_1 (icode
, target
, v0
, v1
, sel_rtx
);
5614 if (qimode
!= VOIDmode
5615 && selector_fits_mode_p (qimode
, qimode_indices
))
5617 icode
= direct_optab_handler (vec_perm_optab
, qimode
);
5618 if (icode
!= CODE_FOR_nothing
)
5620 rtx sel_qi
= vec_perm_indices_to_rtx (qimode
, qimode_indices
);
5621 rtx tmp
= expand_vec_perm_1 (icode
, target_qi
, v0_qi
, v1_qi
, sel_qi
);
5623 return gen_lowpart (mode
, tmp
);
5627 delete_insns_since (last
);
5631 /* Implement a permutation of vectors v0 and v1 using the permutation
5632 vector in SEL and return the result. Use TARGET to hold the result
5633 if nonnull and convenient.
5635 MODE is the mode of the vectors being permuted (V0 and V1).
5636 SEL must have the integer equivalent of MODE and is known to be
5637 unsuitable for permutes with a constant permutation vector. */
5640 expand_vec_perm_var (machine_mode mode
, rtx v0
, rtx v1
, rtx sel
, rtx target
)
5642 enum insn_code icode
;
5646 u
= GET_MODE_UNIT_SIZE (mode
);
5648 if (!target
|| GET_MODE (target
) != mode
)
5649 target
= gen_reg_rtx (mode
);
5651 icode
= direct_optab_handler (vec_perm_optab
, mode
);
5652 if (icode
!= CODE_FOR_nothing
)
5654 tmp
= expand_vec_perm_1 (icode
, target
, v0
, v1
, sel
);
5659 /* As a special case to aid several targets, lower the element-based
5660 permutation to a byte-based permutation and try again. */
5661 machine_mode qimode
;
5662 if (!qimode_for_vec_perm (mode
).exists (&qimode
)
5663 || maybe_gt (GET_MODE_NUNITS (qimode
), GET_MODE_MASK (QImode
) + 1))
5665 icode
= direct_optab_handler (vec_perm_optab
, qimode
);
5666 if (icode
== CODE_FOR_nothing
)
5669 /* Multiply each element by its byte size. */
5670 machine_mode selmode
= GET_MODE (sel
);
5672 sel
= expand_simple_binop (selmode
, PLUS
, sel
, sel
,
5673 NULL
, 0, OPTAB_DIRECT
);
5675 sel
= expand_simple_binop (selmode
, ASHIFT
, sel
,
5676 gen_int_shift_amount (selmode
, exact_log2 (u
)),
5677 NULL
, 0, OPTAB_DIRECT
);
5678 gcc_assert (sel
!= NULL
);
5680 /* Broadcast the low byte each element into each of its bytes.
5681 The encoding has U interleaved stepped patterns, one for each
5682 byte of an element. */
5683 vec_perm_builder
const_sel (GET_MODE_SIZE (mode
), u
, 3);
5684 unsigned int low_byte_in_u
= BYTES_BIG_ENDIAN
? u
- 1 : 0;
5685 for (i
= 0; i
< 3; ++i
)
5686 for (unsigned int j
= 0; j
< u
; ++j
)
5687 const_sel
.quick_push (i
* u
+ low_byte_in_u
);
5688 sel
= gen_lowpart (qimode
, sel
);
5689 sel
= expand_vec_perm_const (qimode
, sel
, sel
, const_sel
, qimode
, NULL
);
5690 gcc_assert (sel
!= NULL
);
5692 /* Add the byte offset to each byte element. */
5693 /* Note that the definition of the indicies here is memory ordering,
5694 so there should be no difference between big and little endian. */
5695 rtx_vector_builder
byte_indices (qimode
, u
, 1);
5696 for (i
= 0; i
< u
; ++i
)
5697 byte_indices
.quick_push (GEN_INT (i
));
5698 tmp
= byte_indices
.build ();
5699 sel_qi
= expand_simple_binop (qimode
, PLUS
, sel
, tmp
,
5700 sel
, 0, OPTAB_DIRECT
);
5701 gcc_assert (sel_qi
!= NULL
);
5703 tmp
= mode
!= qimode
? gen_reg_rtx (qimode
) : target
;
5704 tmp
= expand_vec_perm_1 (icode
, tmp
, gen_lowpart (qimode
, v0
),
5705 gen_lowpart (qimode
, v1
), sel_qi
);
5707 tmp
= gen_lowpart (mode
, tmp
);
5711 /* Generate insns for a VEC_COND_EXPR with mask, given its TYPE and its
5715 expand_vec_cond_mask_expr (tree vec_cond_type
, tree op0
, tree op1
, tree op2
,
5718 struct expand_operand ops
[4];
5719 machine_mode mode
= TYPE_MODE (vec_cond_type
);
5720 machine_mode mask_mode
= TYPE_MODE (TREE_TYPE (op0
));
5721 enum insn_code icode
= get_vcond_mask_icode (mode
, mask_mode
);
5722 rtx mask
, rtx_op1
, rtx_op2
;
5724 if (icode
== CODE_FOR_nothing
)
5727 mask
= expand_normal (op0
);
5728 rtx_op1
= expand_normal (op1
);
5729 rtx_op2
= expand_normal (op2
);
5731 mask
= force_reg (mask_mode
, mask
);
5732 rtx_op1
= force_reg (GET_MODE (rtx_op1
), rtx_op1
);
5734 create_output_operand (&ops
[0], target
, mode
);
5735 create_input_operand (&ops
[1], rtx_op1
, mode
);
5736 create_input_operand (&ops
[2], rtx_op2
, mode
);
5737 create_input_operand (&ops
[3], mask
, mask_mode
);
5738 expand_insn (icode
, 4, ops
);
5740 return ops
[0].value
;
5743 /* Generate insns for a VEC_COND_EXPR, given its TYPE and its
5747 expand_vec_cond_expr (tree vec_cond_type
, tree op0
, tree op1
, tree op2
,
5750 struct expand_operand ops
[6];
5751 enum insn_code icode
;
5752 rtx comparison
, rtx_op1
, rtx_op2
;
5753 machine_mode mode
= TYPE_MODE (vec_cond_type
);
5754 machine_mode cmp_op_mode
;
5757 enum tree_code tcode
;
5759 if (COMPARISON_CLASS_P (op0
))
5761 op0a
= TREE_OPERAND (op0
, 0);
5762 op0b
= TREE_OPERAND (op0
, 1);
5763 tcode
= TREE_CODE (op0
);
5767 gcc_assert (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (op0
)));
5768 if (get_vcond_mask_icode (mode
, TYPE_MODE (TREE_TYPE (op0
)))
5769 != CODE_FOR_nothing
)
5770 return expand_vec_cond_mask_expr (vec_cond_type
, op0
, op1
,
5775 gcc_assert (GET_MODE_CLASS (TYPE_MODE (TREE_TYPE (op0
)))
5776 == MODE_VECTOR_INT
);
5778 op0b
= build_zero_cst (TREE_TYPE (op0
));
5782 cmp_op_mode
= TYPE_MODE (TREE_TYPE (op0a
));
5783 unsignedp
= TYPE_UNSIGNED (TREE_TYPE (op0a
));
5786 gcc_assert (known_eq (GET_MODE_SIZE (mode
), GET_MODE_SIZE (cmp_op_mode
))
5787 && known_eq (GET_MODE_NUNITS (mode
),
5788 GET_MODE_NUNITS (cmp_op_mode
)));
5790 icode
= get_vcond_icode (mode
, cmp_op_mode
, unsignedp
);
5791 if (icode
== CODE_FOR_nothing
)
5793 if (tcode
== EQ_EXPR
|| tcode
== NE_EXPR
)
5794 icode
= get_vcond_eq_icode (mode
, cmp_op_mode
);
5795 if (icode
== CODE_FOR_nothing
)
5799 comparison
= vector_compare_rtx (VOIDmode
, tcode
, op0a
, op0b
, unsignedp
,
5801 rtx_op1
= expand_normal (op1
);
5802 rtx_op2
= expand_normal (op2
);
5804 create_output_operand (&ops
[0], target
, mode
);
5805 create_input_operand (&ops
[1], rtx_op1
, mode
);
5806 create_input_operand (&ops
[2], rtx_op2
, mode
);
5807 create_fixed_operand (&ops
[3], comparison
);
5808 create_fixed_operand (&ops
[4], XEXP (comparison
, 0));
5809 create_fixed_operand (&ops
[5], XEXP (comparison
, 1));
5810 expand_insn (icode
, 6, ops
);
5811 return ops
[0].value
;
5814 /* Generate VEC_SERIES_EXPR <OP0, OP1>, returning a value of mode VMODE.
5815 Use TARGET for the result if nonnull and convenient. */
5818 expand_vec_series_expr (machine_mode vmode
, rtx op0
, rtx op1
, rtx target
)
5820 struct expand_operand ops
[3];
5821 enum insn_code icode
;
5822 machine_mode emode
= GET_MODE_INNER (vmode
);
5824 icode
= direct_optab_handler (vec_series_optab
, vmode
);
5825 gcc_assert (icode
!= CODE_FOR_nothing
);
5827 create_output_operand (&ops
[0], target
, vmode
);
5828 create_input_operand (&ops
[1], op0
, emode
);
5829 create_input_operand (&ops
[2], op1
, emode
);
5831 expand_insn (icode
, 3, ops
);
5832 return ops
[0].value
;
5835 /* Generate insns for a vector comparison into a mask. */
5838 expand_vec_cmp_expr (tree type
, tree exp
, rtx target
)
5840 struct expand_operand ops
[4];
5841 enum insn_code icode
;
5843 machine_mode mask_mode
= TYPE_MODE (type
);
5847 enum tree_code tcode
;
5849 op0a
= TREE_OPERAND (exp
, 0);
5850 op0b
= TREE_OPERAND (exp
, 1);
5851 tcode
= TREE_CODE (exp
);
5853 unsignedp
= TYPE_UNSIGNED (TREE_TYPE (op0a
));
5854 vmode
= TYPE_MODE (TREE_TYPE (op0a
));
5856 icode
= get_vec_cmp_icode (vmode
, mask_mode
, unsignedp
);
5857 if (icode
== CODE_FOR_nothing
)
5859 if (tcode
== EQ_EXPR
|| tcode
== NE_EXPR
)
5860 icode
= get_vec_cmp_eq_icode (vmode
, mask_mode
);
5861 if (icode
== CODE_FOR_nothing
)
5865 comparison
= vector_compare_rtx (mask_mode
, tcode
, op0a
, op0b
,
5866 unsignedp
, icode
, 2);
5867 create_output_operand (&ops
[0], target
, mask_mode
);
5868 create_fixed_operand (&ops
[1], comparison
);
5869 create_fixed_operand (&ops
[2], XEXP (comparison
, 0));
5870 create_fixed_operand (&ops
[3], XEXP (comparison
, 1));
5871 expand_insn (icode
, 4, ops
);
5872 return ops
[0].value
;
5875 /* Expand a highpart multiply. */
5878 expand_mult_highpart (machine_mode mode
, rtx op0
, rtx op1
,
5879 rtx target
, bool uns_p
)
5881 struct expand_operand eops
[3];
5882 enum insn_code icode
;
5888 method
= can_mult_highpart_p (mode
, uns_p
);
5894 tab1
= uns_p
? umul_highpart_optab
: smul_highpart_optab
;
5895 return expand_binop (mode
, tab1
, op0
, op1
, target
, uns_p
,
5898 tab1
= uns_p
? vec_widen_umult_even_optab
: vec_widen_smult_even_optab
;
5899 tab2
= uns_p
? vec_widen_umult_odd_optab
: vec_widen_smult_odd_optab
;
5902 tab1
= uns_p
? vec_widen_umult_lo_optab
: vec_widen_smult_lo_optab
;
5903 tab2
= uns_p
? vec_widen_umult_hi_optab
: vec_widen_smult_hi_optab
;
5904 if (BYTES_BIG_ENDIAN
)
5905 std::swap (tab1
, tab2
);
5911 icode
= optab_handler (tab1
, mode
);
5912 wmode
= insn_data
[icode
].operand
[0].mode
;
5913 gcc_checking_assert (known_eq (2 * GET_MODE_NUNITS (wmode
),
5914 GET_MODE_NUNITS (mode
)));
5915 gcc_checking_assert (known_eq (GET_MODE_SIZE (wmode
), GET_MODE_SIZE (mode
)));
5917 create_output_operand (&eops
[0], gen_reg_rtx (wmode
), wmode
);
5918 create_input_operand (&eops
[1], op0
, mode
);
5919 create_input_operand (&eops
[2], op1
, mode
);
5920 expand_insn (icode
, 3, eops
);
5921 m1
= gen_lowpart (mode
, eops
[0].value
);
5923 create_output_operand (&eops
[0], gen_reg_rtx (wmode
), wmode
);
5924 create_input_operand (&eops
[1], op0
, mode
);
5925 create_input_operand (&eops
[2], op1
, mode
);
5926 expand_insn (optab_handler (tab2
, mode
), 3, eops
);
5927 m2
= gen_lowpart (mode
, eops
[0].value
);
5929 vec_perm_builder sel
;
5932 /* The encoding has 2 interleaved stepped patterns. */
5933 sel
.new_vector (GET_MODE_NUNITS (mode
), 2, 3);
5934 for (i
= 0; i
< 6; ++i
)
5935 sel
.quick_push (!BYTES_BIG_ENDIAN
+ (i
& ~1)
5936 + ((i
& 1) ? GET_MODE_NUNITS (mode
) : 0));
5940 /* The encoding has a single interleaved stepped pattern. */
5941 sel
.new_vector (GET_MODE_NUNITS (mode
), 1, 3);
5942 for (i
= 0; i
< 3; ++i
)
5943 sel
.quick_push (2 * i
+ (BYTES_BIG_ENDIAN
? 0 : 1));
5946 return expand_vec_perm_const (mode
, m1
, m2
, sel
, BLKmode
, target
);
5949 /* Helper function to find the MODE_CC set in a sync_compare_and_swap
5953 find_cc_set (rtx x
, const_rtx pat
, void *data
)
5955 if (REG_P (x
) && GET_MODE_CLASS (GET_MODE (x
)) == MODE_CC
5956 && GET_CODE (pat
) == SET
)
5958 rtx
*p_cc_reg
= (rtx
*) data
;
5959 gcc_assert (!*p_cc_reg
);
5964 /* This is a helper function for the other atomic operations. This function
5965 emits a loop that contains SEQ that iterates until a compare-and-swap
5966 operation at the end succeeds. MEM is the memory to be modified. SEQ is
5967 a set of instructions that takes a value from OLD_REG as an input and
5968 produces a value in NEW_REG as an output. Before SEQ, OLD_REG will be
5969 set to the current contents of MEM. After SEQ, a compare-and-swap will
5970 attempt to update MEM with NEW_REG. The function returns true when the
5971 loop was generated successfully. */
5974 expand_compare_and_swap_loop (rtx mem
, rtx old_reg
, rtx new_reg
, rtx seq
)
5976 machine_mode mode
= GET_MODE (mem
);
5977 rtx_code_label
*label
;
5978 rtx cmp_reg
, success
, oldval
;
5980 /* The loop we want to generate looks like
5986 (success, cmp_reg) = compare-and-swap(mem, old_reg, new_reg)
5990 Note that we only do the plain load from memory once. Subsequent
5991 iterations use the value loaded by the compare-and-swap pattern. */
5993 label
= gen_label_rtx ();
5994 cmp_reg
= gen_reg_rtx (mode
);
5996 emit_move_insn (cmp_reg
, mem
);
5998 emit_move_insn (old_reg
, cmp_reg
);
6004 if (!expand_atomic_compare_and_swap (&success
, &oldval
, mem
, old_reg
,
6005 new_reg
, false, MEMMODEL_SYNC_SEQ_CST
,
6009 if (oldval
!= cmp_reg
)
6010 emit_move_insn (cmp_reg
, oldval
);
6012 /* Mark this jump predicted not taken. */
6013 emit_cmp_and_jump_insns (success
, const0_rtx
, EQ
, const0_rtx
,
6014 GET_MODE (success
), 1, label
,
6015 profile_probability::guessed_never ());
6020 /* This function tries to emit an atomic_exchange intruction. VAL is written
6021 to *MEM using memory model MODEL. The previous contents of *MEM are returned,
6022 using TARGET if possible. */
6025 maybe_emit_atomic_exchange (rtx target
, rtx mem
, rtx val
, enum memmodel model
)
6027 machine_mode mode
= GET_MODE (mem
);
6028 enum insn_code icode
;
6030 /* If the target supports the exchange directly, great. */
6031 icode
= direct_optab_handler (atomic_exchange_optab
, mode
);
6032 if (icode
!= CODE_FOR_nothing
)
6034 struct expand_operand ops
[4];
6036 create_output_operand (&ops
[0], target
, mode
);
6037 create_fixed_operand (&ops
[1], mem
);
6038 create_input_operand (&ops
[2], val
, mode
);
6039 create_integer_operand (&ops
[3], model
);
6040 if (maybe_expand_insn (icode
, 4, ops
))
6041 return ops
[0].value
;
6047 /* This function tries to implement an atomic exchange operation using
6048 __sync_lock_test_and_set. VAL is written to *MEM using memory model MODEL.
6049 The previous contents of *MEM are returned, using TARGET if possible.
6050 Since this instructionn is an acquire barrier only, stronger memory
6051 models may require additional barriers to be emitted. */
6054 maybe_emit_sync_lock_test_and_set (rtx target
, rtx mem
, rtx val
,
6055 enum memmodel model
)
6057 machine_mode mode
= GET_MODE (mem
);
6058 enum insn_code icode
;
6059 rtx_insn
*last_insn
= get_last_insn ();
6061 icode
= optab_handler (sync_lock_test_and_set_optab
, mode
);
6063 /* Legacy sync_lock_test_and_set is an acquire barrier. If the pattern
6064 exists, and the memory model is stronger than acquire, add a release
6065 barrier before the instruction. */
6067 if (is_mm_seq_cst (model
) || is_mm_release (model
) || is_mm_acq_rel (model
))
6068 expand_mem_thread_fence (model
);
6070 if (icode
!= CODE_FOR_nothing
)
6072 struct expand_operand ops
[3];
6073 create_output_operand (&ops
[0], target
, mode
);
6074 create_fixed_operand (&ops
[1], mem
);
6075 create_input_operand (&ops
[2], val
, mode
);
6076 if (maybe_expand_insn (icode
, 3, ops
))
6077 return ops
[0].value
;
6080 /* If an external test-and-set libcall is provided, use that instead of
6081 any external compare-and-swap that we might get from the compare-and-
6082 swap-loop expansion later. */
6083 if (!can_compare_and_swap_p (mode
, false))
6085 rtx libfunc
= optab_libfunc (sync_lock_test_and_set_optab
, mode
);
6086 if (libfunc
!= NULL
)
6090 addr
= convert_memory_address (ptr_mode
, XEXP (mem
, 0));
6091 return emit_library_call_value (libfunc
, NULL_RTX
, LCT_NORMAL
,
6092 mode
, addr
, ptr_mode
,
6097 /* If the test_and_set can't be emitted, eliminate any barrier that might
6098 have been emitted. */
6099 delete_insns_since (last_insn
);
6103 /* This function tries to implement an atomic exchange operation using a
6104 compare_and_swap loop. VAL is written to *MEM. The previous contents of
6105 *MEM are returned, using TARGET if possible. No memory model is required
6106 since a compare_and_swap loop is seq-cst. */
6109 maybe_emit_compare_and_swap_exchange_loop (rtx target
, rtx mem
, rtx val
)
6111 machine_mode mode
= GET_MODE (mem
);
6113 if (can_compare_and_swap_p (mode
, true))
6115 if (!target
|| !register_operand (target
, mode
))
6116 target
= gen_reg_rtx (mode
);
6117 if (expand_compare_and_swap_loop (mem
, target
, val
, NULL_RTX
))
6124 /* This function tries to implement an atomic test-and-set operation
6125 using the atomic_test_and_set instruction pattern. A boolean value
6126 is returned from the operation, using TARGET if possible. */
6129 maybe_emit_atomic_test_and_set (rtx target
, rtx mem
, enum memmodel model
)
6131 machine_mode pat_bool_mode
;
6132 struct expand_operand ops
[3];
6134 if (!targetm
.have_atomic_test_and_set ())
6137 /* While we always get QImode from __atomic_test_and_set, we get
6138 other memory modes from __sync_lock_test_and_set. Note that we
6139 use no endian adjustment here. This matches the 4.6 behavior
6140 in the Sparc backend. */
6141 enum insn_code icode
= targetm
.code_for_atomic_test_and_set
;
6142 gcc_checking_assert (insn_data
[icode
].operand
[1].mode
== QImode
);
6143 if (GET_MODE (mem
) != QImode
)
6144 mem
= adjust_address_nv (mem
, QImode
, 0);
6146 pat_bool_mode
= insn_data
[icode
].operand
[0].mode
;
6147 create_output_operand (&ops
[0], target
, pat_bool_mode
);
6148 create_fixed_operand (&ops
[1], mem
);
6149 create_integer_operand (&ops
[2], model
);
6151 if (maybe_expand_insn (icode
, 3, ops
))
6152 return ops
[0].value
;
6156 /* This function expands the legacy _sync_lock test_and_set operation which is
6157 generally an atomic exchange. Some limited targets only allow the
6158 constant 1 to be stored. This is an ACQUIRE operation.
6160 TARGET is an optional place to stick the return value.
6161 MEM is where VAL is stored. */
6164 expand_sync_lock_test_and_set (rtx target
, rtx mem
, rtx val
)
6168 /* Try an atomic_exchange first. */
6169 ret
= maybe_emit_atomic_exchange (target
, mem
, val
, MEMMODEL_SYNC_ACQUIRE
);
6173 ret
= maybe_emit_sync_lock_test_and_set (target
, mem
, val
,
6174 MEMMODEL_SYNC_ACQUIRE
);
6178 ret
= maybe_emit_compare_and_swap_exchange_loop (target
, mem
, val
);
6182 /* If there are no other options, try atomic_test_and_set if the value
6183 being stored is 1. */
6184 if (val
== const1_rtx
)
6185 ret
= maybe_emit_atomic_test_and_set (target
, mem
, MEMMODEL_SYNC_ACQUIRE
);
6190 /* This function expands the atomic test_and_set operation:
6191 atomically store a boolean TRUE into MEM and return the previous value.
6193 MEMMODEL is the memory model variant to use.
6194 TARGET is an optional place to stick the return value. */
6197 expand_atomic_test_and_set (rtx target
, rtx mem
, enum memmodel model
)
6199 machine_mode mode
= GET_MODE (mem
);
6200 rtx ret
, trueval
, subtarget
;
6202 ret
= maybe_emit_atomic_test_and_set (target
, mem
, model
);
6206 /* Be binary compatible with non-default settings of trueval, and different
6207 cpu revisions. E.g. one revision may have atomic-test-and-set, but
6208 another only has atomic-exchange. */
6209 if (targetm
.atomic_test_and_set_trueval
== 1)
6211 trueval
= const1_rtx
;
6212 subtarget
= target
? target
: gen_reg_rtx (mode
);
6216 trueval
= gen_int_mode (targetm
.atomic_test_and_set_trueval
, mode
);
6217 subtarget
= gen_reg_rtx (mode
);
6220 /* Try the atomic-exchange optab... */
6221 ret
= maybe_emit_atomic_exchange (subtarget
, mem
, trueval
, model
);
6223 /* ... then an atomic-compare-and-swap loop ... */
6225 ret
= maybe_emit_compare_and_swap_exchange_loop (subtarget
, mem
, trueval
);
6227 /* ... before trying the vaguely defined legacy lock_test_and_set. */
6229 ret
= maybe_emit_sync_lock_test_and_set (subtarget
, mem
, trueval
, model
);
6231 /* Recall that the legacy lock_test_and_set optab was allowed to do magic
6232 things with the value 1. Thus we try again without trueval. */
6233 if (!ret
&& targetm
.atomic_test_and_set_trueval
!= 1)
6234 ret
= maybe_emit_sync_lock_test_and_set (subtarget
, mem
, const1_rtx
, model
);
6236 /* Failing all else, assume a single threaded environment and simply
6237 perform the operation. */
6240 /* If the result is ignored skip the move to target. */
6241 if (subtarget
!= const0_rtx
)
6242 emit_move_insn (subtarget
, mem
);
6244 emit_move_insn (mem
, trueval
);
6248 /* Recall that have to return a boolean value; rectify if trueval
6249 is not exactly one. */
6250 if (targetm
.atomic_test_and_set_trueval
!= 1)
6251 ret
= emit_store_flag_force (target
, NE
, ret
, const0_rtx
, mode
, 0, 1);
6256 /* This function expands the atomic exchange operation:
6257 atomically store VAL in MEM and return the previous value in MEM.
6259 MEMMODEL is the memory model variant to use.
6260 TARGET is an optional place to stick the return value. */
6263 expand_atomic_exchange (rtx target
, rtx mem
, rtx val
, enum memmodel model
)
6265 machine_mode mode
= GET_MODE (mem
);
6268 /* If loads are not atomic for the required size and we are not called to
6269 provide a __sync builtin, do not do anything so that we stay consistent
6270 with atomic loads of the same size. */
6271 if (!can_atomic_load_p (mode
) && !is_mm_sync (model
))
6274 ret
= maybe_emit_atomic_exchange (target
, mem
, val
, model
);
6276 /* Next try a compare-and-swap loop for the exchange. */
6278 ret
= maybe_emit_compare_and_swap_exchange_loop (target
, mem
, val
);
6283 /* This function expands the atomic compare exchange operation:
6285 *PTARGET_BOOL is an optional place to store the boolean success/failure.
6286 *PTARGET_OVAL is an optional place to store the old value from memory.
6287 Both target parameters may be NULL or const0_rtx to indicate that we do
6288 not care about that return value. Both target parameters are updated on
6289 success to the actual location of the corresponding result.
6291 MEMMODEL is the memory model variant to use.
6293 The return value of the function is true for success. */
6296 expand_atomic_compare_and_swap (rtx
*ptarget_bool
, rtx
*ptarget_oval
,
6297 rtx mem
, rtx expected
, rtx desired
,
6298 bool is_weak
, enum memmodel succ_model
,
6299 enum memmodel fail_model
)
6301 machine_mode mode
= GET_MODE (mem
);
6302 struct expand_operand ops
[8];
6303 enum insn_code icode
;
6304 rtx target_oval
, target_bool
= NULL_RTX
;
6307 /* If loads are not atomic for the required size and we are not called to
6308 provide a __sync builtin, do not do anything so that we stay consistent
6309 with atomic loads of the same size. */
6310 if (!can_atomic_load_p (mode
) && !is_mm_sync (succ_model
))
6313 /* Load expected into a register for the compare and swap. */
6314 if (MEM_P (expected
))
6315 expected
= copy_to_reg (expected
);
6317 /* Make sure we always have some place to put the return oldval.
6318 Further, make sure that place is distinct from the input expected,
6319 just in case we need that path down below. */
6320 if (ptarget_oval
&& *ptarget_oval
== const0_rtx
)
6321 ptarget_oval
= NULL
;
6323 if (ptarget_oval
== NULL
6324 || (target_oval
= *ptarget_oval
) == NULL
6325 || reg_overlap_mentioned_p (expected
, target_oval
))
6326 target_oval
= gen_reg_rtx (mode
);
6328 icode
= direct_optab_handler (atomic_compare_and_swap_optab
, mode
);
6329 if (icode
!= CODE_FOR_nothing
)
6331 machine_mode bool_mode
= insn_data
[icode
].operand
[0].mode
;
6333 if (ptarget_bool
&& *ptarget_bool
== const0_rtx
)
6334 ptarget_bool
= NULL
;
6336 /* Make sure we always have a place for the bool operand. */
6337 if (ptarget_bool
== NULL
6338 || (target_bool
= *ptarget_bool
) == NULL
6339 || GET_MODE (target_bool
) != bool_mode
)
6340 target_bool
= gen_reg_rtx (bool_mode
);
6342 /* Emit the compare_and_swap. */
6343 create_output_operand (&ops
[0], target_bool
, bool_mode
);
6344 create_output_operand (&ops
[1], target_oval
, mode
);
6345 create_fixed_operand (&ops
[2], mem
);
6346 create_input_operand (&ops
[3], expected
, mode
);
6347 create_input_operand (&ops
[4], desired
, mode
);
6348 create_integer_operand (&ops
[5], is_weak
);
6349 create_integer_operand (&ops
[6], succ_model
);
6350 create_integer_operand (&ops
[7], fail_model
);
6351 if (maybe_expand_insn (icode
, 8, ops
))
6353 /* Return success/failure. */
6354 target_bool
= ops
[0].value
;
6355 target_oval
= ops
[1].value
;
6360 /* Otherwise fall back to the original __sync_val_compare_and_swap
6361 which is always seq-cst. */
6362 icode
= optab_handler (sync_compare_and_swap_optab
, mode
);
6363 if (icode
!= CODE_FOR_nothing
)
6367 create_output_operand (&ops
[0], target_oval
, mode
);
6368 create_fixed_operand (&ops
[1], mem
);
6369 create_input_operand (&ops
[2], expected
, mode
);
6370 create_input_operand (&ops
[3], desired
, mode
);
6371 if (!maybe_expand_insn (icode
, 4, ops
))
6374 target_oval
= ops
[0].value
;
6376 /* If the caller isn't interested in the boolean return value,
6377 skip the computation of it. */
6378 if (ptarget_bool
== NULL
)
6381 /* Otherwise, work out if the compare-and-swap succeeded. */
6383 if (have_insn_for (COMPARE
, CCmode
))
6384 note_stores (PATTERN (get_last_insn ()), find_cc_set
, &cc_reg
);
6387 target_bool
= emit_store_flag_force (target_bool
, EQ
, cc_reg
,
6388 const0_rtx
, VOIDmode
, 0, 1);
6391 goto success_bool_from_val
;
6394 /* Also check for library support for __sync_val_compare_and_swap. */
6395 libfunc
= optab_libfunc (sync_compare_and_swap_optab
, mode
);
6396 if (libfunc
!= NULL
)
6398 rtx addr
= convert_memory_address (ptr_mode
, XEXP (mem
, 0));
6399 rtx target
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_NORMAL
,
6400 mode
, addr
, ptr_mode
,
6401 expected
, mode
, desired
, mode
);
6402 emit_move_insn (target_oval
, target
);
6404 /* Compute the boolean return value only if requested. */
6406 goto success_bool_from_val
;
6414 success_bool_from_val
:
6415 target_bool
= emit_store_flag_force (target_bool
, EQ
, target_oval
,
6416 expected
, VOIDmode
, 1, 1);
6418 /* Make sure that the oval output winds up where the caller asked. */
6420 *ptarget_oval
= target_oval
;
6422 *ptarget_bool
= target_bool
;
6426 /* Generate asm volatile("" : : : "memory") as the memory blockage. */
6429 expand_asm_memory_blockage (void)
6433 asm_op
= gen_rtx_ASM_OPERANDS (VOIDmode
, "", "", 0,
6434 rtvec_alloc (0), rtvec_alloc (0),
6435 rtvec_alloc (0), UNKNOWN_LOCATION
);
6436 MEM_VOLATILE_P (asm_op
) = 1;
6438 clob
= gen_rtx_SCRATCH (VOIDmode
);
6439 clob
= gen_rtx_MEM (BLKmode
, clob
);
6440 clob
= gen_rtx_CLOBBER (VOIDmode
, clob
);
6442 emit_insn (gen_rtx_PARALLEL (VOIDmode
, gen_rtvec (2, asm_op
, clob
)));
6445 /* Do not propagate memory accesses across this point. */
6448 expand_memory_blockage (void)
6450 if (targetm
.have_memory_blockage ())
6451 emit_insn (targetm
.gen_memory_blockage ());
6453 expand_asm_memory_blockage ();
6456 /* This routine will either emit the mem_thread_fence pattern or issue a
6457 sync_synchronize to generate a fence for memory model MEMMODEL. */
6460 expand_mem_thread_fence (enum memmodel model
)
6462 if (is_mm_relaxed (model
))
6464 if (targetm
.have_mem_thread_fence ())
6466 emit_insn (targetm
.gen_mem_thread_fence (GEN_INT (model
)));
6467 expand_memory_blockage ();
6469 else if (targetm
.have_memory_barrier ())
6470 emit_insn (targetm
.gen_memory_barrier ());
6471 else if (synchronize_libfunc
!= NULL_RTX
)
6472 emit_library_call (synchronize_libfunc
, LCT_NORMAL
, VOIDmode
);
6474 expand_memory_blockage ();
6477 /* Emit a signal fence with given memory model. */
6480 expand_mem_signal_fence (enum memmodel model
)
6482 /* No machine barrier is required to implement a signal fence, but
6483 a compiler memory barrier must be issued, except for relaxed MM. */
6484 if (!is_mm_relaxed (model
))
6485 expand_memory_blockage ();
6488 /* This function expands the atomic load operation:
6489 return the atomically loaded value in MEM.
6491 MEMMODEL is the memory model variant to use.
6492 TARGET is an option place to stick the return value. */
6495 expand_atomic_load (rtx target
, rtx mem
, enum memmodel model
)
6497 machine_mode mode
= GET_MODE (mem
);
6498 enum insn_code icode
;
6500 /* If the target supports the load directly, great. */
6501 icode
= direct_optab_handler (atomic_load_optab
, mode
);
6502 if (icode
!= CODE_FOR_nothing
)
6504 struct expand_operand ops
[3];
6505 rtx_insn
*last
= get_last_insn ();
6506 if (is_mm_seq_cst (model
))
6507 expand_memory_blockage ();
6509 create_output_operand (&ops
[0], target
, mode
);
6510 create_fixed_operand (&ops
[1], mem
);
6511 create_integer_operand (&ops
[2], model
);
6512 if (maybe_expand_insn (icode
, 3, ops
))
6514 if (!is_mm_relaxed (model
))
6515 expand_memory_blockage ();
6516 return ops
[0].value
;
6518 delete_insns_since (last
);
6521 /* If the size of the object is greater than word size on this target,
6522 then we assume that a load will not be atomic. We could try to
6523 emulate a load with a compare-and-swap operation, but the store that
6524 doing this could result in would be incorrect if this is a volatile
6525 atomic load or targetting read-only-mapped memory. */
6526 if (maybe_gt (GET_MODE_PRECISION (mode
), BITS_PER_WORD
))
6527 /* If there is no atomic load, leave the library call. */
6530 /* Otherwise assume loads are atomic, and emit the proper barriers. */
6531 if (!target
|| target
== const0_rtx
)
6532 target
= gen_reg_rtx (mode
);
6534 /* For SEQ_CST, emit a barrier before the load. */
6535 if (is_mm_seq_cst (model
))
6536 expand_mem_thread_fence (model
);
6538 emit_move_insn (target
, mem
);
6540 /* Emit the appropriate barrier after the load. */
6541 expand_mem_thread_fence (model
);
6546 /* This function expands the atomic store operation:
6547 Atomically store VAL in MEM.
6548 MEMMODEL is the memory model variant to use.
6549 USE_RELEASE is true if __sync_lock_release can be used as a fall back.
6550 function returns const0_rtx if a pattern was emitted. */
6553 expand_atomic_store (rtx mem
, rtx val
, enum memmodel model
, bool use_release
)
6555 machine_mode mode
= GET_MODE (mem
);
6556 enum insn_code icode
;
6557 struct expand_operand ops
[3];
6559 /* If the target supports the store directly, great. */
6560 icode
= direct_optab_handler (atomic_store_optab
, mode
);
6561 if (icode
!= CODE_FOR_nothing
)
6563 rtx_insn
*last
= get_last_insn ();
6564 if (!is_mm_relaxed (model
))
6565 expand_memory_blockage ();
6566 create_fixed_operand (&ops
[0], mem
);
6567 create_input_operand (&ops
[1], val
, mode
);
6568 create_integer_operand (&ops
[2], model
);
6569 if (maybe_expand_insn (icode
, 3, ops
))
6571 if (is_mm_seq_cst (model
))
6572 expand_memory_blockage ();
6575 delete_insns_since (last
);
6578 /* If using __sync_lock_release is a viable alternative, try it.
6579 Note that this will not be set to true if we are expanding a generic
6580 __atomic_store_n. */
6583 icode
= direct_optab_handler (sync_lock_release_optab
, mode
);
6584 if (icode
!= CODE_FOR_nothing
)
6586 create_fixed_operand (&ops
[0], mem
);
6587 create_input_operand (&ops
[1], const0_rtx
, mode
);
6588 if (maybe_expand_insn (icode
, 2, ops
))
6590 /* lock_release is only a release barrier. */
6591 if (is_mm_seq_cst (model
))
6592 expand_mem_thread_fence (model
);
6598 /* If the size of the object is greater than word size on this target,
6599 a default store will not be atomic. */
6600 if (maybe_gt (GET_MODE_PRECISION (mode
), BITS_PER_WORD
))
6602 /* If loads are atomic or we are called to provide a __sync builtin,
6603 we can try a atomic_exchange and throw away the result. Otherwise,
6604 don't do anything so that we do not create an inconsistency between
6605 loads and stores. */
6606 if (can_atomic_load_p (mode
) || is_mm_sync (model
))
6608 rtx target
= maybe_emit_atomic_exchange (NULL_RTX
, mem
, val
, model
);
6610 target
= maybe_emit_compare_and_swap_exchange_loop (NULL_RTX
, mem
,
6618 /* Otherwise assume stores are atomic, and emit the proper barriers. */
6619 expand_mem_thread_fence (model
);
6621 emit_move_insn (mem
, val
);
6623 /* For SEQ_CST, also emit a barrier after the store. */
6624 if (is_mm_seq_cst (model
))
6625 expand_mem_thread_fence (model
);
6631 /* Structure containing the pointers and values required to process the
6632 various forms of the atomic_fetch_op and atomic_op_fetch builtins. */
6634 struct atomic_op_functions
6636 direct_optab mem_fetch_before
;
6637 direct_optab mem_fetch_after
;
6638 direct_optab mem_no_result
;
6641 direct_optab no_result
;
6642 enum rtx_code reverse_code
;
6646 /* Fill in structure pointed to by OP with the various optab entries for an
6647 operation of type CODE. */
6650 get_atomic_op_for_code (struct atomic_op_functions
*op
, enum rtx_code code
)
6652 gcc_assert (op
!= NULL
);
6654 /* If SWITCHABLE_TARGET is defined, then subtargets can be switched
6655 in the source code during compilation, and the optab entries are not
6656 computable until runtime. Fill in the values at runtime. */
6660 op
->mem_fetch_before
= atomic_fetch_add_optab
;
6661 op
->mem_fetch_after
= atomic_add_fetch_optab
;
6662 op
->mem_no_result
= atomic_add_optab
;
6663 op
->fetch_before
= sync_old_add_optab
;
6664 op
->fetch_after
= sync_new_add_optab
;
6665 op
->no_result
= sync_add_optab
;
6666 op
->reverse_code
= MINUS
;
6669 op
->mem_fetch_before
= atomic_fetch_sub_optab
;
6670 op
->mem_fetch_after
= atomic_sub_fetch_optab
;
6671 op
->mem_no_result
= atomic_sub_optab
;
6672 op
->fetch_before
= sync_old_sub_optab
;
6673 op
->fetch_after
= sync_new_sub_optab
;
6674 op
->no_result
= sync_sub_optab
;
6675 op
->reverse_code
= PLUS
;
6678 op
->mem_fetch_before
= atomic_fetch_xor_optab
;
6679 op
->mem_fetch_after
= atomic_xor_fetch_optab
;
6680 op
->mem_no_result
= atomic_xor_optab
;
6681 op
->fetch_before
= sync_old_xor_optab
;
6682 op
->fetch_after
= sync_new_xor_optab
;
6683 op
->no_result
= sync_xor_optab
;
6684 op
->reverse_code
= XOR
;
6687 op
->mem_fetch_before
= atomic_fetch_and_optab
;
6688 op
->mem_fetch_after
= atomic_and_fetch_optab
;
6689 op
->mem_no_result
= atomic_and_optab
;
6690 op
->fetch_before
= sync_old_and_optab
;
6691 op
->fetch_after
= sync_new_and_optab
;
6692 op
->no_result
= sync_and_optab
;
6693 op
->reverse_code
= UNKNOWN
;
6696 op
->mem_fetch_before
= atomic_fetch_or_optab
;
6697 op
->mem_fetch_after
= atomic_or_fetch_optab
;
6698 op
->mem_no_result
= atomic_or_optab
;
6699 op
->fetch_before
= sync_old_ior_optab
;
6700 op
->fetch_after
= sync_new_ior_optab
;
6701 op
->no_result
= sync_ior_optab
;
6702 op
->reverse_code
= UNKNOWN
;
6705 op
->mem_fetch_before
= atomic_fetch_nand_optab
;
6706 op
->mem_fetch_after
= atomic_nand_fetch_optab
;
6707 op
->mem_no_result
= atomic_nand_optab
;
6708 op
->fetch_before
= sync_old_nand_optab
;
6709 op
->fetch_after
= sync_new_nand_optab
;
6710 op
->no_result
= sync_nand_optab
;
6711 op
->reverse_code
= UNKNOWN
;
6718 /* See if there is a more optimal way to implement the operation "*MEM CODE VAL"
6719 using memory order MODEL. If AFTER is true the operation needs to return
6720 the value of *MEM after the operation, otherwise the previous value.
6721 TARGET is an optional place to place the result. The result is unused if
6723 Return the result if there is a better sequence, otherwise NULL_RTX. */
6726 maybe_optimize_fetch_op (rtx target
, rtx mem
, rtx val
, enum rtx_code code
,
6727 enum memmodel model
, bool after
)
6729 /* If the value is prefetched, or not used, it may be possible to replace
6730 the sequence with a native exchange operation. */
6731 if (!after
|| target
== const0_rtx
)
6733 /* fetch_and (&x, 0, m) can be replaced with exchange (&x, 0, m). */
6734 if (code
== AND
&& val
== const0_rtx
)
6736 if (target
== const0_rtx
)
6737 target
= gen_reg_rtx (GET_MODE (mem
));
6738 return maybe_emit_atomic_exchange (target
, mem
, val
, model
);
6741 /* fetch_or (&x, -1, m) can be replaced with exchange (&x, -1, m). */
6742 if (code
== IOR
&& val
== constm1_rtx
)
6744 if (target
== const0_rtx
)
6745 target
= gen_reg_rtx (GET_MODE (mem
));
6746 return maybe_emit_atomic_exchange (target
, mem
, val
, model
);
6753 /* Try to emit an instruction for a specific operation varaition.
6754 OPTAB contains the OP functions.
6755 TARGET is an optional place to return the result. const0_rtx means unused.
6756 MEM is the memory location to operate on.
6757 VAL is the value to use in the operation.
6758 USE_MEMMODEL is TRUE if the variation with a memory model should be tried.
6759 MODEL is the memory model, if used.
6760 AFTER is true if the returned result is the value after the operation. */
6763 maybe_emit_op (const struct atomic_op_functions
*optab
, rtx target
, rtx mem
,
6764 rtx val
, bool use_memmodel
, enum memmodel model
, bool after
)
6766 machine_mode mode
= GET_MODE (mem
);
6767 struct expand_operand ops
[4];
6768 enum insn_code icode
;
6772 /* Check to see if there is a result returned. */
6773 if (target
== const0_rtx
)
6777 icode
= direct_optab_handler (optab
->mem_no_result
, mode
);
6778 create_integer_operand (&ops
[2], model
);
6783 icode
= direct_optab_handler (optab
->no_result
, mode
);
6787 /* Otherwise, we need to generate a result. */
6792 icode
= direct_optab_handler (after
? optab
->mem_fetch_after
6793 : optab
->mem_fetch_before
, mode
);
6794 create_integer_operand (&ops
[3], model
);
6799 icode
= optab_handler (after
? optab
->fetch_after
6800 : optab
->fetch_before
, mode
);
6803 create_output_operand (&ops
[op_counter
++], target
, mode
);
6805 if (icode
== CODE_FOR_nothing
)
6808 create_fixed_operand (&ops
[op_counter
++], mem
);
6809 /* VAL may have been promoted to a wider mode. Shrink it if so. */
6810 create_convert_operand_to (&ops
[op_counter
++], val
, mode
, true);
6812 if (maybe_expand_insn (icode
, num_ops
, ops
))
6813 return (target
== const0_rtx
? const0_rtx
: ops
[0].value
);
6819 /* This function expands an atomic fetch_OP or OP_fetch operation:
6820 TARGET is an option place to stick the return value. const0_rtx indicates
6821 the result is unused.
6822 atomically fetch MEM, perform the operation with VAL and return it to MEM.
6823 CODE is the operation being performed (OP)
6824 MEMMODEL is the memory model variant to use.
6825 AFTER is true to return the result of the operation (OP_fetch).
6826 AFTER is false to return the value before the operation (fetch_OP).
6828 This function will *only* generate instructions if there is a direct
6829 optab. No compare and swap loops or libcalls will be generated. */
6832 expand_atomic_fetch_op_no_fallback (rtx target
, rtx mem
, rtx val
,
6833 enum rtx_code code
, enum memmodel model
,
6836 machine_mode mode
= GET_MODE (mem
);
6837 struct atomic_op_functions optab
;
6839 bool unused_result
= (target
== const0_rtx
);
6841 get_atomic_op_for_code (&optab
, code
);
6843 /* Check to see if there are any better instructions. */
6844 result
= maybe_optimize_fetch_op (target
, mem
, val
, code
, model
, after
);
6848 /* Check for the case where the result isn't used and try those patterns. */
6851 /* Try the memory model variant first. */
6852 result
= maybe_emit_op (&optab
, target
, mem
, val
, true, model
, true);
6856 /* Next try the old style withuot a memory model. */
6857 result
= maybe_emit_op (&optab
, target
, mem
, val
, false, model
, true);
6861 /* There is no no-result pattern, so try patterns with a result. */
6865 /* Try the __atomic version. */
6866 result
= maybe_emit_op (&optab
, target
, mem
, val
, true, model
, after
);
6870 /* Try the older __sync version. */
6871 result
= maybe_emit_op (&optab
, target
, mem
, val
, false, model
, after
);
6875 /* If the fetch value can be calculated from the other variation of fetch,
6876 try that operation. */
6877 if (after
|| unused_result
|| optab
.reverse_code
!= UNKNOWN
)
6879 /* Try the __atomic version, then the older __sync version. */
6880 result
= maybe_emit_op (&optab
, target
, mem
, val
, true, model
, !after
);
6882 result
= maybe_emit_op (&optab
, target
, mem
, val
, false, model
, !after
);
6886 /* If the result isn't used, no need to do compensation code. */
6890 /* Issue compensation code. Fetch_after == fetch_before OP val.
6891 Fetch_before == after REVERSE_OP val. */
6893 code
= optab
.reverse_code
;
6896 result
= expand_simple_binop (mode
, AND
, result
, val
, NULL_RTX
,
6897 true, OPTAB_LIB_WIDEN
);
6898 result
= expand_simple_unop (mode
, NOT
, result
, target
, true);
6901 result
= expand_simple_binop (mode
, code
, result
, val
, target
,
6902 true, OPTAB_LIB_WIDEN
);
6907 /* No direct opcode can be generated. */
6913 /* This function expands an atomic fetch_OP or OP_fetch operation:
6914 TARGET is an option place to stick the return value. const0_rtx indicates
6915 the result is unused.
6916 atomically fetch MEM, perform the operation with VAL and return it to MEM.
6917 CODE is the operation being performed (OP)
6918 MEMMODEL is the memory model variant to use.
6919 AFTER is true to return the result of the operation (OP_fetch).
6920 AFTER is false to return the value before the operation (fetch_OP). */
6922 expand_atomic_fetch_op (rtx target
, rtx mem
, rtx val
, enum rtx_code code
,
6923 enum memmodel model
, bool after
)
6925 machine_mode mode
= GET_MODE (mem
);
6927 bool unused_result
= (target
== const0_rtx
);
6929 /* If loads are not atomic for the required size and we are not called to
6930 provide a __sync builtin, do not do anything so that we stay consistent
6931 with atomic loads of the same size. */
6932 if (!can_atomic_load_p (mode
) && !is_mm_sync (model
))
6935 result
= expand_atomic_fetch_op_no_fallback (target
, mem
, val
, code
, model
,
6941 /* Add/sub can be implemented by doing the reverse operation with -(val). */
6942 if (code
== PLUS
|| code
== MINUS
)
6945 enum rtx_code reverse
= (code
== PLUS
? MINUS
: PLUS
);
6948 tmp
= expand_simple_unop (mode
, NEG
, val
, NULL_RTX
, true);
6949 result
= expand_atomic_fetch_op_no_fallback (target
, mem
, tmp
, reverse
,
6953 /* PLUS worked so emit the insns and return. */
6960 /* PLUS did not work, so throw away the negation code and continue. */
6964 /* Try the __sync libcalls only if we can't do compare-and-swap inline. */
6965 if (!can_compare_and_swap_p (mode
, false))
6969 enum rtx_code orig_code
= code
;
6970 struct atomic_op_functions optab
;
6972 get_atomic_op_for_code (&optab
, code
);
6973 libfunc
= optab_libfunc (after
? optab
.fetch_after
6974 : optab
.fetch_before
, mode
);
6976 && (after
|| unused_result
|| optab
.reverse_code
!= UNKNOWN
))
6980 code
= optab
.reverse_code
;
6981 libfunc
= optab_libfunc (after
? optab
.fetch_before
6982 : optab
.fetch_after
, mode
);
6984 if (libfunc
!= NULL
)
6986 rtx addr
= convert_memory_address (ptr_mode
, XEXP (mem
, 0));
6987 result
= emit_library_call_value (libfunc
, NULL
, LCT_NORMAL
, mode
,
6988 addr
, ptr_mode
, val
, mode
);
6990 if (!unused_result
&& fixup
)
6991 result
= expand_simple_binop (mode
, code
, result
, val
, target
,
6992 true, OPTAB_LIB_WIDEN
);
6996 /* We need the original code for any further attempts. */
7000 /* If nothing else has succeeded, default to a compare and swap loop. */
7001 if (can_compare_and_swap_p (mode
, true))
7004 rtx t0
= gen_reg_rtx (mode
), t1
;
7008 /* If the result is used, get a register for it. */
7011 if (!target
|| !register_operand (target
, mode
))
7012 target
= gen_reg_rtx (mode
);
7013 /* If fetch_before, copy the value now. */
7015 emit_move_insn (target
, t0
);
7018 target
= const0_rtx
;
7023 t1
= expand_simple_binop (mode
, AND
, t1
, val
, NULL_RTX
,
7024 true, OPTAB_LIB_WIDEN
);
7025 t1
= expand_simple_unop (mode
, code
, t1
, NULL_RTX
, true);
7028 t1
= expand_simple_binop (mode
, code
, t1
, val
, NULL_RTX
, true,
7031 /* For after, copy the value now. */
7032 if (!unused_result
&& after
)
7033 emit_move_insn (target
, t1
);
7034 insn
= get_insns ();
7037 if (t1
!= NULL
&& expand_compare_and_swap_loop (mem
, t0
, t1
, insn
))
7044 /* Return true if OPERAND is suitable for operand number OPNO of
7045 instruction ICODE. */
7048 insn_operand_matches (enum insn_code icode
, unsigned int opno
, rtx operand
)
7050 return (!insn_data
[(int) icode
].operand
[opno
].predicate
7051 || (insn_data
[(int) icode
].operand
[opno
].predicate
7052 (operand
, insn_data
[(int) icode
].operand
[opno
].mode
)));
7055 /* TARGET is a target of a multiword operation that we are going to
7056 implement as a series of word-mode operations. Return true if
7057 TARGET is suitable for this purpose. */
7060 valid_multiword_target_p (rtx target
)
7065 mode
= GET_MODE (target
);
7066 if (!GET_MODE_SIZE (mode
).is_constant (&size
))
7068 for (i
= 0; i
< size
; i
+= UNITS_PER_WORD
)
7069 if (!validate_subreg (word_mode
, mode
, target
, i
))
7074 /* Make OP describe an input operand that has value INTVAL and that has
7075 no inherent mode. This function should only be used for operands that
7076 are always expand-time constants. The backend may request that INTVAL
7077 be copied into a different kind of rtx, but it must specify the mode
7078 of that rtx if so. */
7081 create_integer_operand (struct expand_operand
*op
, poly_int64 intval
)
7083 create_expand_operand (op
, EXPAND_INTEGER
,
7084 gen_int_mode (intval
, MAX_MODE_INT
),
7085 VOIDmode
, false, intval
);
7088 /* Like maybe_legitimize_operand, but do not change the code of the
7089 current rtx value. */
7092 maybe_legitimize_operand_same_code (enum insn_code icode
, unsigned int opno
,
7093 struct expand_operand
*op
)
7095 /* See if the operand matches in its current form. */
7096 if (insn_operand_matches (icode
, opno
, op
->value
))
7099 /* If the operand is a memory whose address has no side effects,
7100 try forcing the address into a non-virtual pseudo register.
7101 The check for side effects is important because copy_to_mode_reg
7102 cannot handle things like auto-modified addresses. */
7103 if (insn_data
[(int) icode
].operand
[opno
].allows_mem
&& MEM_P (op
->value
))
7108 addr
= XEXP (mem
, 0);
7109 if (!(REG_P (addr
) && REGNO (addr
) > LAST_VIRTUAL_REGISTER
)
7110 && !side_effects_p (addr
))
7115 last
= get_last_insn ();
7116 mode
= get_address_mode (mem
);
7117 mem
= replace_equiv_address (mem
, copy_to_mode_reg (mode
, addr
));
7118 if (insn_operand_matches (icode
, opno
, mem
))
7123 delete_insns_since (last
);
7130 /* Try to make OP match operand OPNO of instruction ICODE. Return true
7131 on success, storing the new operand value back in OP. */
7134 maybe_legitimize_operand (enum insn_code icode
, unsigned int opno
,
7135 struct expand_operand
*op
)
7137 machine_mode mode
, imode
;
7138 bool old_volatile_ok
, result
;
7144 old_volatile_ok
= volatile_ok
;
7146 result
= maybe_legitimize_operand_same_code (icode
, opno
, op
);
7147 volatile_ok
= old_volatile_ok
;
7151 gcc_assert (mode
!= VOIDmode
);
7153 && op
->value
!= const0_rtx
7154 && GET_MODE (op
->value
) == mode
7155 && maybe_legitimize_operand_same_code (icode
, opno
, op
))
7158 op
->value
= gen_reg_rtx (mode
);
7164 gcc_assert (mode
!= VOIDmode
);
7165 gcc_assert (GET_MODE (op
->value
) == VOIDmode
7166 || GET_MODE (op
->value
) == mode
);
7167 if (maybe_legitimize_operand_same_code (icode
, opno
, op
))
7170 op
->value
= copy_to_mode_reg (mode
, op
->value
);
7173 case EXPAND_CONVERT_TO
:
7174 gcc_assert (mode
!= VOIDmode
);
7175 op
->value
= convert_to_mode (mode
, op
->value
, op
->unsigned_p
);
7178 case EXPAND_CONVERT_FROM
:
7179 if (GET_MODE (op
->value
) != VOIDmode
)
7180 mode
= GET_MODE (op
->value
);
7182 /* The caller must tell us what mode this value has. */
7183 gcc_assert (mode
!= VOIDmode
);
7185 imode
= insn_data
[(int) icode
].operand
[opno
].mode
;
7186 if (imode
!= VOIDmode
&& imode
!= mode
)
7188 op
->value
= convert_modes (imode
, mode
, op
->value
, op
->unsigned_p
);
7193 case EXPAND_ADDRESS
:
7194 op
->value
= convert_memory_address (as_a
<scalar_int_mode
> (mode
),
7198 case EXPAND_INTEGER
:
7199 mode
= insn_data
[(int) icode
].operand
[opno
].mode
;
7200 if (mode
!= VOIDmode
7201 && known_eq (trunc_int_for_mode (op
->int_value
, mode
),
7204 op
->value
= gen_int_mode (op
->int_value
, mode
);
7209 return insn_operand_matches (icode
, opno
, op
->value
);
7212 /* Make OP describe an input operand that should have the same value
7213 as VALUE, after any mode conversion that the target might request.
7214 TYPE is the type of VALUE. */
7217 create_convert_operand_from_type (struct expand_operand
*op
,
7218 rtx value
, tree type
)
7220 create_convert_operand_from (op
, value
, TYPE_MODE (type
),
7221 TYPE_UNSIGNED (type
));
7224 /* Return true if the requirements on operands OP1 and OP2 of instruction
7225 ICODE are similar enough for the result of legitimizing OP1 to be
7226 reusable for OP2. OPNO1 and OPNO2 are the operand numbers associated
7227 with OP1 and OP2 respectively. */
7230 can_reuse_operands_p (enum insn_code icode
,
7231 unsigned int opno1
, unsigned int opno2
,
7232 const struct expand_operand
*op1
,
7233 const struct expand_operand
*op2
)
7235 /* Check requirements that are common to all types. */
7236 if (op1
->type
!= op2
->type
7237 || op1
->mode
!= op2
->mode
7238 || (insn_data
[(int) icode
].operand
[opno1
].mode
7239 != insn_data
[(int) icode
].operand
[opno2
].mode
))
7242 /* Check the requirements for specific types. */
7246 /* Outputs must remain distinct. */
7251 case EXPAND_ADDRESS
:
7252 case EXPAND_INTEGER
:
7255 case EXPAND_CONVERT_TO
:
7256 case EXPAND_CONVERT_FROM
:
7257 return op1
->unsigned_p
== op2
->unsigned_p
;
7262 /* Try to make operands [OPS, OPS + NOPS) match operands [OPNO, OPNO + NOPS)
7263 of instruction ICODE. Return true on success, leaving the new operand
7264 values in the OPS themselves. Emit no code on failure. */
7267 maybe_legitimize_operands (enum insn_code icode
, unsigned int opno
,
7268 unsigned int nops
, struct expand_operand
*ops
)
7270 rtx_insn
*last
= get_last_insn ();
7271 rtx
*orig_values
= XALLOCAVEC (rtx
, nops
);
7272 for (unsigned int i
= 0; i
< nops
; i
++)
7274 orig_values
[i
] = ops
[i
].value
;
7276 /* First try reusing the result of an earlier legitimization.
7277 This avoids duplicate rtl and ensures that tied operands
7280 This search is linear, but NOPS is bounded at compile time
7281 to a small number (current a single digit). */
7284 if (can_reuse_operands_p (icode
, opno
+ j
, opno
+ i
, &ops
[j
], &ops
[i
])
7285 && rtx_equal_p (orig_values
[j
], orig_values
[i
])
7287 && insn_operand_matches (icode
, opno
+ i
, ops
[j
].value
))
7289 ops
[i
].value
= copy_rtx (ops
[j
].value
);
7293 /* Otherwise try legitimizing the operand on its own. */
7294 if (j
== i
&& !maybe_legitimize_operand (icode
, opno
+ i
, &ops
[i
]))
7296 delete_insns_since (last
);
7303 /* Try to generate instruction ICODE, using operands [OPS, OPS + NOPS)
7304 as its operands. Return the instruction pattern on success,
7305 and emit any necessary set-up code. Return null and emit no
7309 maybe_gen_insn (enum insn_code icode
, unsigned int nops
,
7310 struct expand_operand
*ops
)
7312 gcc_assert (nops
== (unsigned int) insn_data
[(int) icode
].n_generator_args
);
7313 if (!maybe_legitimize_operands (icode
, 0, nops
, ops
))
7319 return GEN_FCN (icode
) (ops
[0].value
);
7321 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
);
7323 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
);
7325 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
7328 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
7329 ops
[3].value
, ops
[4].value
);
7331 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
7332 ops
[3].value
, ops
[4].value
, ops
[5].value
);
7334 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
7335 ops
[3].value
, ops
[4].value
, ops
[5].value
,
7338 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
7339 ops
[3].value
, ops
[4].value
, ops
[5].value
,
7340 ops
[6].value
, ops
[7].value
);
7342 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
7343 ops
[3].value
, ops
[4].value
, ops
[5].value
,
7344 ops
[6].value
, ops
[7].value
, ops
[8].value
);
7349 /* Try to emit instruction ICODE, using operands [OPS, OPS + NOPS)
7350 as its operands. Return true on success and emit no code on failure. */
7353 maybe_expand_insn (enum insn_code icode
, unsigned int nops
,
7354 struct expand_operand
*ops
)
7356 rtx_insn
*pat
= maybe_gen_insn (icode
, nops
, ops
);
7365 /* Like maybe_expand_insn, but for jumps. */
7368 maybe_expand_jump_insn (enum insn_code icode
, unsigned int nops
,
7369 struct expand_operand
*ops
)
7371 rtx_insn
*pat
= maybe_gen_insn (icode
, nops
, ops
);
7374 emit_jump_insn (pat
);
7380 /* Emit instruction ICODE, using operands [OPS, OPS + NOPS)
7384 expand_insn (enum insn_code icode
, unsigned int nops
,
7385 struct expand_operand
*ops
)
7387 if (!maybe_expand_insn (icode
, nops
, ops
))
7391 /* Like expand_insn, but for jumps. */
7394 expand_jump_insn (enum insn_code icode
, unsigned int nops
,
7395 struct expand_operand
*ops
)
7397 if (!maybe_expand_jump_insn (icode
, nops
, ops
))