Fix warnings occured during profiledboostrap on
[official-gcc.git] / gcc / optabs.c
blobe9dc7981c631d0c060fed2e0f1a1142d166bfe41
1 /* Expand the basic unary and binary arithmetic operations, for GNU compiler.
2 Copyright (C) 1987-2015 Free Software Foundation, Inc.
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 3, or (at your option) any later
9 version.
11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 for more details.
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
21 #include "config.h"
22 #include "system.h"
23 #include "coretypes.h"
24 #include "tm.h"
25 #include "diagnostic-core.h"
27 /* Include insn-config.h before expr.h so that HAVE_conditional_move
28 is properly defined. */
29 #include "insn-config.h"
30 #include "rtl.h"
31 #include "hash-set.h"
32 #include "machmode.h"
33 #include "vec.h"
34 #include "double-int.h"
35 #include "input.h"
36 #include "alias.h"
37 #include "symtab.h"
38 #include "wide-int.h"
39 #include "inchash.h"
40 #include "tree.h"
41 #include "tree-hasher.h"
42 #include "stor-layout.h"
43 #include "stringpool.h"
44 #include "varasm.h"
45 #include "tm_p.h"
46 #include "flags.h"
47 #include "hard-reg-set.h"
48 #include "function.h"
49 #include "except.h"
50 #include "hashtab.h"
51 #include "statistics.h"
52 #include "real.h"
53 #include "fixed-value.h"
54 #include "expmed.h"
55 #include "dojump.h"
56 #include "explow.h"
57 #include "calls.h"
58 #include "emit-rtl.h"
59 #include "stmt.h"
60 #include "expr.h"
61 #include "insn-codes.h"
62 #include "optabs.h"
63 #include "libfuncs.h"
64 #include "recog.h"
65 #include "reload.h"
66 #include "ggc.h"
67 #include "predict.h"
68 #include "dominance.h"
69 #include "cfg.h"
70 #include "basic-block.h"
71 #include "target.h"
73 struct target_optabs default_target_optabs;
74 struct target_libfuncs default_target_libfuncs;
75 struct target_optabs *this_fn_optabs = &default_target_optabs;
76 #if SWITCHABLE_TARGET
77 struct target_optabs *this_target_optabs = &default_target_optabs;
78 struct target_libfuncs *this_target_libfuncs = &default_target_libfuncs;
79 #endif
81 #define libfunc_hash \
82 (this_target_libfuncs->x_libfunc_hash)
84 static void prepare_float_lib_cmp (rtx, rtx, enum rtx_code, rtx *,
85 machine_mode *);
86 static rtx expand_unop_direct (machine_mode, optab, rtx, rtx, int);
87 static void emit_libcall_block_1 (rtx_insn *, rtx, rtx, rtx, bool);
89 /* Debug facility for use in GDB. */
90 void debug_optab_libfuncs (void);
92 /* Prefixes for the current version of decimal floating point (BID vs. DPD) */
93 #if ENABLE_DECIMAL_BID_FORMAT
94 #define DECIMAL_PREFIX "bid_"
95 #else
96 #define DECIMAL_PREFIX "dpd_"
97 #endif
99 /* Used for libfunc_hash. */
101 hashval_t
102 libfunc_hasher::hash (libfunc_entry *e)
104 return ((e->mode1 + e->mode2 * NUM_MACHINE_MODES) ^ e->op);
107 /* Used for libfunc_hash. */
109 bool
110 libfunc_hasher::equal (libfunc_entry *e1, libfunc_entry *e2)
112 return e1->op == e2->op && e1->mode1 == e2->mode1 && e1->mode2 == e2->mode2;
115 /* Return libfunc corresponding operation defined by OPTAB converting
116 from MODE2 to MODE1. Trigger lazy initialization if needed, return NULL
117 if no libfunc is available. */
119 convert_optab_libfunc (convert_optab optab, machine_mode mode1,
120 machine_mode mode2)
122 struct libfunc_entry e;
123 struct libfunc_entry **slot;
125 /* ??? This ought to be an assert, but not all of the places
126 that we expand optabs know about the optabs that got moved
127 to being direct. */
128 if (!(optab >= FIRST_CONV_OPTAB && optab <= LAST_CONVLIB_OPTAB))
129 return NULL_RTX;
131 e.op = optab;
132 e.mode1 = mode1;
133 e.mode2 = mode2;
134 slot = libfunc_hash->find_slot (&e, NO_INSERT);
135 if (!slot)
137 const struct convert_optab_libcall_d *d
138 = &convlib_def[optab - FIRST_CONV_OPTAB];
140 if (d->libcall_gen == NULL)
141 return NULL;
143 d->libcall_gen (optab, d->libcall_basename, mode1, mode2);
144 slot = libfunc_hash->find_slot (&e, NO_INSERT);
145 if (!slot)
146 return NULL;
148 return (*slot)->libfunc;
151 /* Return libfunc corresponding operation defined by OPTAB in MODE.
152 Trigger lazy initialization if needed, return NULL if no libfunc is
153 available. */
155 optab_libfunc (optab optab, machine_mode mode)
157 struct libfunc_entry e;
158 struct libfunc_entry **slot;
160 /* ??? This ought to be an assert, but not all of the places
161 that we expand optabs know about the optabs that got moved
162 to being direct. */
163 if (!(optab >= FIRST_NORM_OPTAB && optab <= LAST_NORMLIB_OPTAB))
164 return NULL_RTX;
166 e.op = optab;
167 e.mode1 = mode;
168 e.mode2 = VOIDmode;
169 slot = libfunc_hash->find_slot (&e, NO_INSERT);
170 if (!slot)
172 const struct optab_libcall_d *d
173 = &normlib_def[optab - FIRST_NORM_OPTAB];
175 if (d->libcall_gen == NULL)
176 return NULL;
178 d->libcall_gen (optab, d->libcall_basename, d->libcall_suffix, mode);
179 slot = libfunc_hash->find_slot (&e, NO_INSERT);
180 if (!slot)
181 return NULL;
183 return (*slot)->libfunc;
187 /* Add a REG_EQUAL note to the last insn in INSNS. TARGET is being set to
188 the result of operation CODE applied to OP0 (and OP1 if it is a binary
189 operation).
191 If the last insn does not set TARGET, don't do anything, but return 1.
193 If the last insn or a previous insn sets TARGET and TARGET is one of OP0
194 or OP1, don't add the REG_EQUAL note but return 0. Our caller can then
195 try again, ensuring that TARGET is not one of the operands. */
197 static int
198 add_equal_note (rtx_insn *insns, rtx target, enum rtx_code code, rtx op0, rtx op1)
200 rtx_insn *last_insn;
201 rtx set;
202 rtx note;
204 gcc_assert (insns && INSN_P (insns) && NEXT_INSN (insns));
206 if (GET_RTX_CLASS (code) != RTX_COMM_ARITH
207 && GET_RTX_CLASS (code) != RTX_BIN_ARITH
208 && GET_RTX_CLASS (code) != RTX_COMM_COMPARE
209 && GET_RTX_CLASS (code) != RTX_COMPARE
210 && GET_RTX_CLASS (code) != RTX_UNARY)
211 return 1;
213 if (GET_CODE (target) == ZERO_EXTRACT)
214 return 1;
216 for (last_insn = insns;
217 NEXT_INSN (last_insn) != NULL_RTX;
218 last_insn = NEXT_INSN (last_insn))
221 /* If TARGET is in OP0 or OP1, punt. We'd end up with a note referencing
222 a value changing in the insn, so the note would be invalid for CSE. */
223 if (reg_overlap_mentioned_p (target, op0)
224 || (op1 && reg_overlap_mentioned_p (target, op1)))
226 if (MEM_P (target)
227 && (rtx_equal_p (target, op0)
228 || (op1 && rtx_equal_p (target, op1))))
230 /* For MEM target, with MEM = MEM op X, prefer no REG_EQUAL note
231 over expanding it as temp = MEM op X, MEM = temp. If the target
232 supports MEM = MEM op X instructions, it is sometimes too hard
233 to reconstruct that form later, especially if X is also a memory,
234 and due to multiple occurrences of addresses the address might
235 be forced into register unnecessarily.
236 Note that not emitting the REG_EQUIV note might inhibit
237 CSE in some cases. */
238 set = single_set (last_insn);
239 if (set
240 && GET_CODE (SET_SRC (set)) == code
241 && MEM_P (SET_DEST (set))
242 && (rtx_equal_p (SET_DEST (set), XEXP (SET_SRC (set), 0))
243 || (op1 && rtx_equal_p (SET_DEST (set),
244 XEXP (SET_SRC (set), 1)))))
245 return 1;
247 return 0;
250 set = set_for_reg_notes (last_insn);
251 if (set == NULL_RTX)
252 return 1;
254 if (! rtx_equal_p (SET_DEST (set), target)
255 /* For a STRICT_LOW_PART, the REG_NOTE applies to what is inside it. */
256 && (GET_CODE (SET_DEST (set)) != STRICT_LOW_PART
257 || ! rtx_equal_p (XEXP (SET_DEST (set), 0), target)))
258 return 1;
260 if (GET_RTX_CLASS (code) == RTX_UNARY)
261 switch (code)
263 case FFS:
264 case CLZ:
265 case CTZ:
266 case CLRSB:
267 case POPCOUNT:
268 case PARITY:
269 case BSWAP:
270 if (GET_MODE (op0) != VOIDmode && GET_MODE (target) != GET_MODE (op0))
272 note = gen_rtx_fmt_e (code, GET_MODE (op0), copy_rtx (op0));
273 if (GET_MODE_SIZE (GET_MODE (op0))
274 > GET_MODE_SIZE (GET_MODE (target)))
275 note = simplify_gen_unary (TRUNCATE, GET_MODE (target),
276 note, GET_MODE (op0));
277 else
278 note = simplify_gen_unary (ZERO_EXTEND, GET_MODE (target),
279 note, GET_MODE (op0));
280 break;
282 /* FALLTHRU */
283 default:
284 note = gen_rtx_fmt_e (code, GET_MODE (target), copy_rtx (op0));
285 break;
287 else
288 note = gen_rtx_fmt_ee (code, GET_MODE (target), copy_rtx (op0), copy_rtx (op1));
290 set_unique_reg_note (last_insn, REG_EQUAL, note);
292 return 1;
295 /* Given two input operands, OP0 and OP1, determine what the correct from_mode
296 for a widening operation would be. In most cases this would be OP0, but if
297 that's a constant it'll be VOIDmode, which isn't useful. */
299 static machine_mode
300 widened_mode (machine_mode to_mode, rtx op0, rtx op1)
302 machine_mode m0 = GET_MODE (op0);
303 machine_mode m1 = GET_MODE (op1);
304 machine_mode result;
306 if (m0 == VOIDmode && m1 == VOIDmode)
307 return to_mode;
308 else if (m0 == VOIDmode || GET_MODE_SIZE (m0) < GET_MODE_SIZE (m1))
309 result = m1;
310 else
311 result = m0;
313 if (GET_MODE_SIZE (result) > GET_MODE_SIZE (to_mode))
314 return to_mode;
316 return result;
319 /* Like optab_handler, but for widening_operations that have a
320 TO_MODE and a FROM_MODE. */
322 enum insn_code
323 widening_optab_handler (optab op, machine_mode to_mode,
324 machine_mode from_mode)
326 unsigned scode = (op << 16) | to_mode;
327 if (to_mode != from_mode && from_mode != VOIDmode)
329 /* ??? Why does find_widening_optab_handler_and_mode attempt to
330 widen things that can't be widened? E.g. add_optab... */
331 if (op > LAST_CONV_OPTAB)
332 return CODE_FOR_nothing;
333 scode |= from_mode << 8;
335 return raw_optab_handler (scode);
338 /* Find a widening optab even if it doesn't widen as much as we want.
339 E.g. if from_mode is HImode, and to_mode is DImode, and there is no
340 direct HI->SI insn, then return SI->DI, if that exists.
341 If PERMIT_NON_WIDENING is non-zero then this can be used with
342 non-widening optabs also. */
344 enum insn_code
345 find_widening_optab_handler_and_mode (optab op, machine_mode to_mode,
346 machine_mode from_mode,
347 int permit_non_widening,
348 machine_mode *found_mode)
350 for (; (permit_non_widening || from_mode != to_mode)
351 && GET_MODE_SIZE (from_mode) <= GET_MODE_SIZE (to_mode)
352 && from_mode != VOIDmode;
353 from_mode = GET_MODE_WIDER_MODE (from_mode))
355 enum insn_code handler = widening_optab_handler (op, to_mode,
356 from_mode);
358 if (handler != CODE_FOR_nothing)
360 if (found_mode)
361 *found_mode = from_mode;
362 return handler;
366 return CODE_FOR_nothing;
369 /* Widen OP to MODE and return the rtx for the widened operand. UNSIGNEDP
370 says whether OP is signed or unsigned. NO_EXTEND is nonzero if we need
371 not actually do a sign-extend or zero-extend, but can leave the
372 higher-order bits of the result rtx undefined, for example, in the case
373 of logical operations, but not right shifts. */
375 static rtx
376 widen_operand (rtx op, machine_mode mode, machine_mode oldmode,
377 int unsignedp, int no_extend)
379 rtx result;
381 /* If we don't have to extend and this is a constant, return it. */
382 if (no_extend && GET_MODE (op) == VOIDmode)
383 return op;
385 /* If we must extend do so. If OP is a SUBREG for a promoted object, also
386 extend since it will be more efficient to do so unless the signedness of
387 a promoted object differs from our extension. */
388 if (! no_extend
389 || (GET_CODE (op) == SUBREG && SUBREG_PROMOTED_VAR_P (op)
390 && SUBREG_CHECK_PROMOTED_SIGN (op, unsignedp)))
391 return convert_modes (mode, oldmode, op, unsignedp);
393 /* If MODE is no wider than a single word, we return a lowpart or paradoxical
394 SUBREG. */
395 if (GET_MODE_SIZE (mode) <= UNITS_PER_WORD)
396 return gen_lowpart (mode, force_reg (GET_MODE (op), op));
398 /* Otherwise, get an object of MODE, clobber it, and set the low-order
399 part to OP. */
401 result = gen_reg_rtx (mode);
402 emit_clobber (result);
403 emit_move_insn (gen_lowpart (GET_MODE (op), result), op);
404 return result;
407 /* Return the optab used for computing the operation given by the tree code,
408 CODE and the tree EXP. This function is not always usable (for example, it
409 cannot give complete results for multiplication or division) but probably
410 ought to be relied on more widely throughout the expander. */
411 optab
412 optab_for_tree_code (enum tree_code code, const_tree type,
413 enum optab_subtype subtype)
415 bool trapv;
416 switch (code)
418 case BIT_AND_EXPR:
419 return and_optab;
421 case BIT_IOR_EXPR:
422 return ior_optab;
424 case BIT_NOT_EXPR:
425 return one_cmpl_optab;
427 case BIT_XOR_EXPR:
428 return xor_optab;
430 case MULT_HIGHPART_EXPR:
431 return TYPE_UNSIGNED (type) ? umul_highpart_optab : smul_highpart_optab;
433 case TRUNC_MOD_EXPR:
434 case CEIL_MOD_EXPR:
435 case FLOOR_MOD_EXPR:
436 case ROUND_MOD_EXPR:
437 return TYPE_UNSIGNED (type) ? umod_optab : smod_optab;
439 case RDIV_EXPR:
440 case TRUNC_DIV_EXPR:
441 case CEIL_DIV_EXPR:
442 case FLOOR_DIV_EXPR:
443 case ROUND_DIV_EXPR:
444 case EXACT_DIV_EXPR:
445 if (TYPE_SATURATING (type))
446 return TYPE_UNSIGNED (type) ? usdiv_optab : ssdiv_optab;
447 return TYPE_UNSIGNED (type) ? udiv_optab : sdiv_optab;
449 case LSHIFT_EXPR:
450 if (TREE_CODE (type) == VECTOR_TYPE)
452 if (subtype == optab_vector)
453 return TYPE_SATURATING (type) ? unknown_optab : vashl_optab;
455 gcc_assert (subtype == optab_scalar);
457 if (TYPE_SATURATING (type))
458 return TYPE_UNSIGNED (type) ? usashl_optab : ssashl_optab;
459 return ashl_optab;
461 case RSHIFT_EXPR:
462 if (TREE_CODE (type) == VECTOR_TYPE)
464 if (subtype == optab_vector)
465 return TYPE_UNSIGNED (type) ? vlshr_optab : vashr_optab;
467 gcc_assert (subtype == optab_scalar);
469 return TYPE_UNSIGNED (type) ? lshr_optab : ashr_optab;
471 case LROTATE_EXPR:
472 if (TREE_CODE (type) == VECTOR_TYPE)
474 if (subtype == optab_vector)
475 return vrotl_optab;
477 gcc_assert (subtype == optab_scalar);
479 return rotl_optab;
481 case RROTATE_EXPR:
482 if (TREE_CODE (type) == VECTOR_TYPE)
484 if (subtype == optab_vector)
485 return vrotr_optab;
487 gcc_assert (subtype == optab_scalar);
489 return rotr_optab;
491 case MAX_EXPR:
492 return TYPE_UNSIGNED (type) ? umax_optab : smax_optab;
494 case MIN_EXPR:
495 return TYPE_UNSIGNED (type) ? umin_optab : smin_optab;
497 case REALIGN_LOAD_EXPR:
498 return vec_realign_load_optab;
500 case WIDEN_SUM_EXPR:
501 return TYPE_UNSIGNED (type) ? usum_widen_optab : ssum_widen_optab;
503 case DOT_PROD_EXPR:
504 return TYPE_UNSIGNED (type) ? udot_prod_optab : sdot_prod_optab;
506 case SAD_EXPR:
507 return TYPE_UNSIGNED (type) ? usad_optab : ssad_optab;
509 case WIDEN_MULT_PLUS_EXPR:
510 return (TYPE_UNSIGNED (type)
511 ? (TYPE_SATURATING (type)
512 ? usmadd_widen_optab : umadd_widen_optab)
513 : (TYPE_SATURATING (type)
514 ? ssmadd_widen_optab : smadd_widen_optab));
516 case WIDEN_MULT_MINUS_EXPR:
517 return (TYPE_UNSIGNED (type)
518 ? (TYPE_SATURATING (type)
519 ? usmsub_widen_optab : umsub_widen_optab)
520 : (TYPE_SATURATING (type)
521 ? ssmsub_widen_optab : smsub_widen_optab));
523 case FMA_EXPR:
524 return fma_optab;
526 case REDUC_MAX_EXPR:
527 return TYPE_UNSIGNED (type)
528 ? reduc_umax_scal_optab : reduc_smax_scal_optab;
530 case REDUC_MIN_EXPR:
531 return TYPE_UNSIGNED (type)
532 ? reduc_umin_scal_optab : reduc_smin_scal_optab;
534 case REDUC_PLUS_EXPR:
535 return reduc_plus_scal_optab;
537 case VEC_WIDEN_MULT_HI_EXPR:
538 return TYPE_UNSIGNED (type) ?
539 vec_widen_umult_hi_optab : vec_widen_smult_hi_optab;
541 case VEC_WIDEN_MULT_LO_EXPR:
542 return TYPE_UNSIGNED (type) ?
543 vec_widen_umult_lo_optab : vec_widen_smult_lo_optab;
545 case VEC_WIDEN_MULT_EVEN_EXPR:
546 return TYPE_UNSIGNED (type) ?
547 vec_widen_umult_even_optab : vec_widen_smult_even_optab;
549 case VEC_WIDEN_MULT_ODD_EXPR:
550 return TYPE_UNSIGNED (type) ?
551 vec_widen_umult_odd_optab : vec_widen_smult_odd_optab;
553 case VEC_WIDEN_LSHIFT_HI_EXPR:
554 return TYPE_UNSIGNED (type) ?
555 vec_widen_ushiftl_hi_optab : vec_widen_sshiftl_hi_optab;
557 case VEC_WIDEN_LSHIFT_LO_EXPR:
558 return TYPE_UNSIGNED (type) ?
559 vec_widen_ushiftl_lo_optab : vec_widen_sshiftl_lo_optab;
561 case VEC_UNPACK_HI_EXPR:
562 return TYPE_UNSIGNED (type) ?
563 vec_unpacku_hi_optab : vec_unpacks_hi_optab;
565 case VEC_UNPACK_LO_EXPR:
566 return TYPE_UNSIGNED (type) ?
567 vec_unpacku_lo_optab : vec_unpacks_lo_optab;
569 case VEC_UNPACK_FLOAT_HI_EXPR:
570 /* The signedness is determined from input operand. */
571 return TYPE_UNSIGNED (type) ?
572 vec_unpacku_float_hi_optab : vec_unpacks_float_hi_optab;
574 case VEC_UNPACK_FLOAT_LO_EXPR:
575 /* The signedness is determined from input operand. */
576 return TYPE_UNSIGNED (type) ?
577 vec_unpacku_float_lo_optab : vec_unpacks_float_lo_optab;
579 case VEC_PACK_TRUNC_EXPR:
580 return vec_pack_trunc_optab;
582 case VEC_PACK_SAT_EXPR:
583 return TYPE_UNSIGNED (type) ? vec_pack_usat_optab : vec_pack_ssat_optab;
585 case VEC_PACK_FIX_TRUNC_EXPR:
586 /* The signedness is determined from output operand. */
587 return TYPE_UNSIGNED (type) ?
588 vec_pack_ufix_trunc_optab : vec_pack_sfix_trunc_optab;
590 default:
591 break;
594 trapv = INTEGRAL_TYPE_P (type) && TYPE_OVERFLOW_TRAPS (type);
595 switch (code)
597 case POINTER_PLUS_EXPR:
598 case PLUS_EXPR:
599 if (TYPE_SATURATING (type))
600 return TYPE_UNSIGNED (type) ? usadd_optab : ssadd_optab;
601 return trapv ? addv_optab : add_optab;
603 case MINUS_EXPR:
604 if (TYPE_SATURATING (type))
605 return TYPE_UNSIGNED (type) ? ussub_optab : sssub_optab;
606 return trapv ? subv_optab : sub_optab;
608 case MULT_EXPR:
609 if (TYPE_SATURATING (type))
610 return TYPE_UNSIGNED (type) ? usmul_optab : ssmul_optab;
611 return trapv ? smulv_optab : smul_optab;
613 case NEGATE_EXPR:
614 if (TYPE_SATURATING (type))
615 return TYPE_UNSIGNED (type) ? usneg_optab : ssneg_optab;
616 return trapv ? negv_optab : neg_optab;
618 case ABS_EXPR:
619 return trapv ? absv_optab : abs_optab;
621 default:
622 return unknown_optab;
626 /* Given optab UNOPTAB that reduces a vector to a scalar, find instead the old
627 optab that produces a vector with the reduction result in one element,
628 for a tree with type TYPE. */
630 optab
631 scalar_reduc_to_vector (optab unoptab, const_tree type)
633 switch (unoptab)
635 case reduc_plus_scal_optab:
636 return TYPE_UNSIGNED (type) ? reduc_uplus_optab : reduc_splus_optab;
638 case reduc_smin_scal_optab: return reduc_smin_optab;
639 case reduc_umin_scal_optab: return reduc_umin_optab;
640 case reduc_smax_scal_optab: return reduc_smax_optab;
641 case reduc_umax_scal_optab: return reduc_umax_optab;
642 default: return unknown_optab;
646 /* Expand vector widening operations.
648 There are two different classes of operations handled here:
649 1) Operations whose result is wider than all the arguments to the operation.
650 Examples: VEC_UNPACK_HI/LO_EXPR, VEC_WIDEN_MULT_HI/LO_EXPR
651 In this case OP0 and optionally OP1 would be initialized,
652 but WIDE_OP wouldn't (not relevant for this case).
653 2) Operations whose result is of the same size as the last argument to the
654 operation, but wider than all the other arguments to the operation.
655 Examples: WIDEN_SUM_EXPR, VEC_DOT_PROD_EXPR.
656 In the case WIDE_OP, OP0 and optionally OP1 would be initialized.
658 E.g, when called to expand the following operations, this is how
659 the arguments will be initialized:
660 nops OP0 OP1 WIDE_OP
661 widening-sum 2 oprnd0 - oprnd1
662 widening-dot-product 3 oprnd0 oprnd1 oprnd2
663 widening-mult 2 oprnd0 oprnd1 -
664 type-promotion (vec-unpack) 1 oprnd0 - - */
667 expand_widen_pattern_expr (sepops ops, rtx op0, rtx op1, rtx wide_op,
668 rtx target, int unsignedp)
670 struct expand_operand eops[4];
671 tree oprnd0, oprnd1, oprnd2;
672 machine_mode wmode = VOIDmode, tmode0, tmode1 = VOIDmode;
673 optab widen_pattern_optab;
674 enum insn_code icode;
675 int nops = TREE_CODE_LENGTH (ops->code);
676 int op;
678 oprnd0 = ops->op0;
679 tmode0 = TYPE_MODE (TREE_TYPE (oprnd0));
680 widen_pattern_optab =
681 optab_for_tree_code (ops->code, TREE_TYPE (oprnd0), optab_default);
682 if (ops->code == WIDEN_MULT_PLUS_EXPR
683 || ops->code == WIDEN_MULT_MINUS_EXPR)
684 icode = find_widening_optab_handler (widen_pattern_optab,
685 TYPE_MODE (TREE_TYPE (ops->op2)),
686 tmode0, 0);
687 else
688 icode = optab_handler (widen_pattern_optab, tmode0);
689 gcc_assert (icode != CODE_FOR_nothing);
691 if (nops >= 2)
693 oprnd1 = ops->op1;
694 tmode1 = TYPE_MODE (TREE_TYPE (oprnd1));
697 /* The last operand is of a wider mode than the rest of the operands. */
698 if (nops == 2)
699 wmode = tmode1;
700 else if (nops == 3)
702 gcc_assert (tmode1 == tmode0);
703 gcc_assert (op1);
704 oprnd2 = ops->op2;
705 wmode = TYPE_MODE (TREE_TYPE (oprnd2));
708 op = 0;
709 create_output_operand (&eops[op++], target, TYPE_MODE (ops->type));
710 create_convert_operand_from (&eops[op++], op0, tmode0, unsignedp);
711 if (op1)
712 create_convert_operand_from (&eops[op++], op1, tmode1, unsignedp);
713 if (wide_op)
714 create_convert_operand_from (&eops[op++], wide_op, wmode, unsignedp);
715 expand_insn (icode, op, eops);
716 return eops[0].value;
719 /* Generate code to perform an operation specified by TERNARY_OPTAB
720 on operands OP0, OP1 and OP2, with result having machine-mode MODE.
722 UNSIGNEDP is for the case where we have to widen the operands
723 to perform the operation. It says to use zero-extension.
725 If TARGET is nonzero, the value
726 is generated there, if it is convenient to do so.
727 In all cases an rtx is returned for the locus of the value;
728 this may or may not be TARGET. */
731 expand_ternary_op (machine_mode mode, optab ternary_optab, rtx op0,
732 rtx op1, rtx op2, rtx target, int unsignedp)
734 struct expand_operand ops[4];
735 enum insn_code icode = optab_handler (ternary_optab, mode);
737 gcc_assert (optab_handler (ternary_optab, mode) != CODE_FOR_nothing);
739 create_output_operand (&ops[0], target, mode);
740 create_convert_operand_from (&ops[1], op0, mode, unsignedp);
741 create_convert_operand_from (&ops[2], op1, mode, unsignedp);
742 create_convert_operand_from (&ops[3], op2, mode, unsignedp);
743 expand_insn (icode, 4, ops);
744 return ops[0].value;
748 /* Like expand_binop, but return a constant rtx if the result can be
749 calculated at compile time. The arguments and return value are
750 otherwise the same as for expand_binop. */
753 simplify_expand_binop (machine_mode mode, optab binoptab,
754 rtx op0, rtx op1, rtx target, int unsignedp,
755 enum optab_methods methods)
757 if (CONSTANT_P (op0) && CONSTANT_P (op1))
759 rtx x = simplify_binary_operation (optab_to_code (binoptab),
760 mode, op0, op1);
761 if (x)
762 return x;
765 return expand_binop (mode, binoptab, op0, op1, target, unsignedp, methods);
768 /* Like simplify_expand_binop, but always put the result in TARGET.
769 Return true if the expansion succeeded. */
771 bool
772 force_expand_binop (machine_mode mode, optab binoptab,
773 rtx op0, rtx op1, rtx target, int unsignedp,
774 enum optab_methods methods)
776 rtx x = simplify_expand_binop (mode, binoptab, op0, op1,
777 target, unsignedp, methods);
778 if (x == 0)
779 return false;
780 if (x != target)
781 emit_move_insn (target, x);
782 return true;
785 /* Create a new vector value in VMODE with all elements set to OP. The
786 mode of OP must be the element mode of VMODE. If OP is a constant,
787 then the return value will be a constant. */
789 static rtx
790 expand_vector_broadcast (machine_mode vmode, rtx op)
792 enum insn_code icode;
793 rtvec vec;
794 rtx ret;
795 int i, n;
797 gcc_checking_assert (VECTOR_MODE_P (vmode));
799 n = GET_MODE_NUNITS (vmode);
800 vec = rtvec_alloc (n);
801 for (i = 0; i < n; ++i)
802 RTVEC_ELT (vec, i) = op;
804 if (CONSTANT_P (op))
805 return gen_rtx_CONST_VECTOR (vmode, vec);
807 /* ??? If the target doesn't have a vec_init, then we have no easy way
808 of performing this operation. Most of this sort of generic support
809 is hidden away in the vector lowering support in gimple. */
810 icode = optab_handler (vec_init_optab, vmode);
811 if (icode == CODE_FOR_nothing)
812 return NULL;
814 ret = gen_reg_rtx (vmode);
815 emit_insn (GEN_FCN (icode) (ret, gen_rtx_PARALLEL (vmode, vec)));
817 return ret;
820 /* This subroutine of expand_doubleword_shift handles the cases in which
821 the effective shift value is >= BITS_PER_WORD. The arguments and return
822 value are the same as for the parent routine, except that SUPERWORD_OP1
823 is the shift count to use when shifting OUTOF_INPUT into INTO_TARGET.
824 INTO_TARGET may be null if the caller has decided to calculate it. */
826 static bool
827 expand_superword_shift (optab binoptab, rtx outof_input, rtx superword_op1,
828 rtx outof_target, rtx into_target,
829 int unsignedp, enum optab_methods methods)
831 if (into_target != 0)
832 if (!force_expand_binop (word_mode, binoptab, outof_input, superword_op1,
833 into_target, unsignedp, methods))
834 return false;
836 if (outof_target != 0)
838 /* For a signed right shift, we must fill OUTOF_TARGET with copies
839 of the sign bit, otherwise we must fill it with zeros. */
840 if (binoptab != ashr_optab)
841 emit_move_insn (outof_target, CONST0_RTX (word_mode));
842 else
843 if (!force_expand_binop (word_mode, binoptab,
844 outof_input, GEN_INT (BITS_PER_WORD - 1),
845 outof_target, unsignedp, methods))
846 return false;
848 return true;
851 /* This subroutine of expand_doubleword_shift handles the cases in which
852 the effective shift value is < BITS_PER_WORD. The arguments and return
853 value are the same as for the parent routine. */
855 static bool
856 expand_subword_shift (machine_mode op1_mode, optab binoptab,
857 rtx outof_input, rtx into_input, rtx op1,
858 rtx outof_target, rtx into_target,
859 int unsignedp, enum optab_methods methods,
860 unsigned HOST_WIDE_INT shift_mask)
862 optab reverse_unsigned_shift, unsigned_shift;
863 rtx tmp, carries;
865 reverse_unsigned_shift = (binoptab == ashl_optab ? lshr_optab : ashl_optab);
866 unsigned_shift = (binoptab == ashl_optab ? ashl_optab : lshr_optab);
868 /* The low OP1 bits of INTO_TARGET come from the high bits of OUTOF_INPUT.
869 We therefore need to shift OUTOF_INPUT by (BITS_PER_WORD - OP1) bits in
870 the opposite direction to BINOPTAB. */
871 if (CONSTANT_P (op1) || shift_mask >= BITS_PER_WORD)
873 carries = outof_input;
874 tmp = immed_wide_int_const (wi::shwi (BITS_PER_WORD,
875 op1_mode), op1_mode);
876 tmp = simplify_expand_binop (op1_mode, sub_optab, tmp, op1,
877 0, true, methods);
879 else
881 /* We must avoid shifting by BITS_PER_WORD bits since that is either
882 the same as a zero shift (if shift_mask == BITS_PER_WORD - 1) or
883 has unknown behavior. Do a single shift first, then shift by the
884 remainder. It's OK to use ~OP1 as the remainder if shift counts
885 are truncated to the mode size. */
886 carries = expand_binop (word_mode, reverse_unsigned_shift,
887 outof_input, const1_rtx, 0, unsignedp, methods);
888 if (shift_mask == BITS_PER_WORD - 1)
890 tmp = immed_wide_int_const
891 (wi::minus_one (GET_MODE_PRECISION (op1_mode)), op1_mode);
892 tmp = simplify_expand_binop (op1_mode, xor_optab, op1, tmp,
893 0, true, methods);
895 else
897 tmp = immed_wide_int_const (wi::shwi (BITS_PER_WORD - 1,
898 op1_mode), op1_mode);
899 tmp = simplify_expand_binop (op1_mode, sub_optab, tmp, op1,
900 0, true, methods);
903 if (tmp == 0 || carries == 0)
904 return false;
905 carries = expand_binop (word_mode, reverse_unsigned_shift,
906 carries, tmp, 0, unsignedp, methods);
907 if (carries == 0)
908 return false;
910 /* Shift INTO_INPUT logically by OP1. This is the last use of INTO_INPUT
911 so the result can go directly into INTO_TARGET if convenient. */
912 tmp = expand_binop (word_mode, unsigned_shift, into_input, op1,
913 into_target, unsignedp, methods);
914 if (tmp == 0)
915 return false;
917 /* Now OR in the bits carried over from OUTOF_INPUT. */
918 if (!force_expand_binop (word_mode, ior_optab, tmp, carries,
919 into_target, unsignedp, methods))
920 return false;
922 /* Use a standard word_mode shift for the out-of half. */
923 if (outof_target != 0)
924 if (!force_expand_binop (word_mode, binoptab, outof_input, op1,
925 outof_target, unsignedp, methods))
926 return false;
928 return true;
932 #ifdef HAVE_conditional_move
933 /* Try implementing expand_doubleword_shift using conditional moves.
934 The shift is by < BITS_PER_WORD if (CMP_CODE CMP1 CMP2) is true,
935 otherwise it is by >= BITS_PER_WORD. SUBWORD_OP1 and SUPERWORD_OP1
936 are the shift counts to use in the former and latter case. All other
937 arguments are the same as the parent routine. */
939 static bool
940 expand_doubleword_shift_condmove (machine_mode op1_mode, optab binoptab,
941 enum rtx_code cmp_code, rtx cmp1, rtx cmp2,
942 rtx outof_input, rtx into_input,
943 rtx subword_op1, rtx superword_op1,
944 rtx outof_target, rtx into_target,
945 int unsignedp, enum optab_methods methods,
946 unsigned HOST_WIDE_INT shift_mask)
948 rtx outof_superword, into_superword;
950 /* Put the superword version of the output into OUTOF_SUPERWORD and
951 INTO_SUPERWORD. */
952 outof_superword = outof_target != 0 ? gen_reg_rtx (word_mode) : 0;
953 if (outof_target != 0 && subword_op1 == superword_op1)
955 /* The value INTO_TARGET >> SUBWORD_OP1, which we later store in
956 OUTOF_TARGET, is the same as the value of INTO_SUPERWORD. */
957 into_superword = outof_target;
958 if (!expand_superword_shift (binoptab, outof_input, superword_op1,
959 outof_superword, 0, unsignedp, methods))
960 return false;
962 else
964 into_superword = gen_reg_rtx (word_mode);
965 if (!expand_superword_shift (binoptab, outof_input, superword_op1,
966 outof_superword, into_superword,
967 unsignedp, methods))
968 return false;
971 /* Put the subword version directly in OUTOF_TARGET and INTO_TARGET. */
972 if (!expand_subword_shift (op1_mode, binoptab,
973 outof_input, into_input, subword_op1,
974 outof_target, into_target,
975 unsignedp, methods, shift_mask))
976 return false;
978 /* Select between them. Do the INTO half first because INTO_SUPERWORD
979 might be the current value of OUTOF_TARGET. */
980 if (!emit_conditional_move (into_target, cmp_code, cmp1, cmp2, op1_mode,
981 into_target, into_superword, word_mode, false))
982 return false;
984 if (outof_target != 0)
985 if (!emit_conditional_move (outof_target, cmp_code, cmp1, cmp2, op1_mode,
986 outof_target, outof_superword,
987 word_mode, false))
988 return false;
990 return true;
992 #endif
994 /* Expand a doubleword shift (ashl, ashr or lshr) using word-mode shifts.
995 OUTOF_INPUT and INTO_INPUT are the two word-sized halves of the first
996 input operand; the shift moves bits in the direction OUTOF_INPUT->
997 INTO_TARGET. OUTOF_TARGET and INTO_TARGET are the equivalent words
998 of the target. OP1 is the shift count and OP1_MODE is its mode.
999 If OP1 is constant, it will have been truncated as appropriate
1000 and is known to be nonzero.
1002 If SHIFT_MASK is zero, the result of word shifts is undefined when the
1003 shift count is outside the range [0, BITS_PER_WORD). This routine must
1004 avoid generating such shifts for OP1s in the range [0, BITS_PER_WORD * 2).
1006 If SHIFT_MASK is nonzero, all word-mode shift counts are effectively
1007 masked by it and shifts in the range [BITS_PER_WORD, SHIFT_MASK) will
1008 fill with zeros or sign bits as appropriate.
1010 If SHIFT_MASK is BITS_PER_WORD - 1, this routine will synthesize
1011 a doubleword shift whose equivalent mask is BITS_PER_WORD * 2 - 1.
1012 Doing this preserves semantics required by SHIFT_COUNT_TRUNCATED.
1013 In all other cases, shifts by values outside [0, BITS_PER_UNIT * 2)
1014 are undefined.
1016 BINOPTAB, UNSIGNEDP and METHODS are as for expand_binop. This function
1017 may not use INTO_INPUT after modifying INTO_TARGET, and similarly for
1018 OUTOF_INPUT and OUTOF_TARGET. OUTOF_TARGET can be null if the parent
1019 function wants to calculate it itself.
1021 Return true if the shift could be successfully synthesized. */
1023 static bool
1024 expand_doubleword_shift (machine_mode op1_mode, optab binoptab,
1025 rtx outof_input, rtx into_input, rtx op1,
1026 rtx outof_target, rtx into_target,
1027 int unsignedp, enum optab_methods methods,
1028 unsigned HOST_WIDE_INT shift_mask)
1030 rtx superword_op1, tmp, cmp1, cmp2;
1031 enum rtx_code cmp_code;
1033 /* See if word-mode shifts by BITS_PER_WORD...BITS_PER_WORD * 2 - 1 will
1034 fill the result with sign or zero bits as appropriate. If so, the value
1035 of OUTOF_TARGET will always be (SHIFT OUTOF_INPUT OP1). Recursively call
1036 this routine to calculate INTO_TARGET (which depends on both OUTOF_INPUT
1037 and INTO_INPUT), then emit code to set up OUTOF_TARGET.
1039 This isn't worthwhile for constant shifts since the optimizers will
1040 cope better with in-range shift counts. */
1041 if (shift_mask >= BITS_PER_WORD
1042 && outof_target != 0
1043 && !CONSTANT_P (op1))
1045 if (!expand_doubleword_shift (op1_mode, binoptab,
1046 outof_input, into_input, op1,
1047 0, into_target,
1048 unsignedp, methods, shift_mask))
1049 return false;
1050 if (!force_expand_binop (word_mode, binoptab, outof_input, op1,
1051 outof_target, unsignedp, methods))
1052 return false;
1053 return true;
1056 /* Set CMP_CODE, CMP1 and CMP2 so that the rtx (CMP_CODE CMP1 CMP2)
1057 is true when the effective shift value is less than BITS_PER_WORD.
1058 Set SUPERWORD_OP1 to the shift count that should be used to shift
1059 OUTOF_INPUT into INTO_TARGET when the condition is false. */
1060 tmp = immed_wide_int_const (wi::shwi (BITS_PER_WORD, op1_mode), op1_mode);
1061 if (!CONSTANT_P (op1) && shift_mask == BITS_PER_WORD - 1)
1063 /* Set CMP1 to OP1 & BITS_PER_WORD. The result is zero iff OP1
1064 is a subword shift count. */
1065 cmp1 = simplify_expand_binop (op1_mode, and_optab, op1, tmp,
1066 0, true, methods);
1067 cmp2 = CONST0_RTX (op1_mode);
1068 cmp_code = EQ;
1069 superword_op1 = op1;
1071 else
1073 /* Set CMP1 to OP1 - BITS_PER_WORD. */
1074 cmp1 = simplify_expand_binop (op1_mode, sub_optab, op1, tmp,
1075 0, true, methods);
1076 cmp2 = CONST0_RTX (op1_mode);
1077 cmp_code = LT;
1078 superword_op1 = cmp1;
1080 if (cmp1 == 0)
1081 return false;
1083 /* If we can compute the condition at compile time, pick the
1084 appropriate subroutine. */
1085 tmp = simplify_relational_operation (cmp_code, SImode, op1_mode, cmp1, cmp2);
1086 if (tmp != 0 && CONST_INT_P (tmp))
1088 if (tmp == const0_rtx)
1089 return expand_superword_shift (binoptab, outof_input, superword_op1,
1090 outof_target, into_target,
1091 unsignedp, methods);
1092 else
1093 return expand_subword_shift (op1_mode, binoptab,
1094 outof_input, into_input, op1,
1095 outof_target, into_target,
1096 unsignedp, methods, shift_mask);
1099 #ifdef HAVE_conditional_move
1100 /* Try using conditional moves to generate straight-line code. */
1102 rtx_insn *start = get_last_insn ();
1103 if (expand_doubleword_shift_condmove (op1_mode, binoptab,
1104 cmp_code, cmp1, cmp2,
1105 outof_input, into_input,
1106 op1, superword_op1,
1107 outof_target, into_target,
1108 unsignedp, methods, shift_mask))
1109 return true;
1110 delete_insns_since (start);
1112 #endif
1114 /* As a last resort, use branches to select the correct alternative. */
1115 rtx_code_label *subword_label = gen_label_rtx ();
1116 rtx_code_label *done_label = gen_label_rtx ();
1118 NO_DEFER_POP;
1119 do_compare_rtx_and_jump (cmp1, cmp2, cmp_code, false, op1_mode,
1120 0, 0, subword_label, -1);
1121 OK_DEFER_POP;
1123 if (!expand_superword_shift (binoptab, outof_input, superword_op1,
1124 outof_target, into_target,
1125 unsignedp, methods))
1126 return false;
1128 emit_jump_insn (gen_jump (done_label));
1129 emit_barrier ();
1130 emit_label (subword_label);
1132 if (!expand_subword_shift (op1_mode, binoptab,
1133 outof_input, into_input, op1,
1134 outof_target, into_target,
1135 unsignedp, methods, shift_mask))
1136 return false;
1138 emit_label (done_label);
1139 return true;
1142 /* Subroutine of expand_binop. Perform a double word multiplication of
1143 operands OP0 and OP1 both of mode MODE, which is exactly twice as wide
1144 as the target's word_mode. This function return NULL_RTX if anything
1145 goes wrong, in which case it may have already emitted instructions
1146 which need to be deleted.
1148 If we want to multiply two two-word values and have normal and widening
1149 multiplies of single-word values, we can do this with three smaller
1150 multiplications.
1152 The multiplication proceeds as follows:
1153 _______________________
1154 [__op0_high_|__op0_low__]
1155 _______________________
1156 * [__op1_high_|__op1_low__]
1157 _______________________________________________
1158 _______________________
1159 (1) [__op0_low__*__op1_low__]
1160 _______________________
1161 (2a) [__op0_low__*__op1_high_]
1162 _______________________
1163 (2b) [__op0_high_*__op1_low__]
1164 _______________________
1165 (3) [__op0_high_*__op1_high_]
1168 This gives a 4-word result. Since we are only interested in the
1169 lower 2 words, partial result (3) and the upper words of (2a) and
1170 (2b) don't need to be calculated. Hence (2a) and (2b) can be
1171 calculated using non-widening multiplication.
1173 (1), however, needs to be calculated with an unsigned widening
1174 multiplication. If this operation is not directly supported we
1175 try using a signed widening multiplication and adjust the result.
1176 This adjustment works as follows:
1178 If both operands are positive then no adjustment is needed.
1180 If the operands have different signs, for example op0_low < 0 and
1181 op1_low >= 0, the instruction treats the most significant bit of
1182 op0_low as a sign bit instead of a bit with significance
1183 2**(BITS_PER_WORD-1), i.e. the instruction multiplies op1_low
1184 with 2**BITS_PER_WORD - op0_low, and two's complements the
1185 result. Conclusion: We need to add op1_low * 2**BITS_PER_WORD to
1186 the result.
1188 Similarly, if both operands are negative, we need to add
1189 (op0_low + op1_low) * 2**BITS_PER_WORD.
1191 We use a trick to adjust quickly. We logically shift op0_low right
1192 (op1_low) BITS_PER_WORD-1 steps to get 0 or 1, and add this to
1193 op0_high (op1_high) before it is used to calculate 2b (2a). If no
1194 logical shift exists, we do an arithmetic right shift and subtract
1195 the 0 or -1. */
1197 static rtx
1198 expand_doubleword_mult (machine_mode mode, rtx op0, rtx op1, rtx target,
1199 bool umulp, enum optab_methods methods)
1201 int low = (WORDS_BIG_ENDIAN ? 1 : 0);
1202 int high = (WORDS_BIG_ENDIAN ? 0 : 1);
1203 rtx wordm1 = umulp ? NULL_RTX : GEN_INT (BITS_PER_WORD - 1);
1204 rtx product, adjust, product_high, temp;
1206 rtx op0_high = operand_subword_force (op0, high, mode);
1207 rtx op0_low = operand_subword_force (op0, low, mode);
1208 rtx op1_high = operand_subword_force (op1, high, mode);
1209 rtx op1_low = operand_subword_force (op1, low, mode);
1211 /* If we're using an unsigned multiply to directly compute the product
1212 of the low-order words of the operands and perform any required
1213 adjustments of the operands, we begin by trying two more multiplications
1214 and then computing the appropriate sum.
1216 We have checked above that the required addition is provided.
1217 Full-word addition will normally always succeed, especially if
1218 it is provided at all, so we don't worry about its failure. The
1219 multiplication may well fail, however, so we do handle that. */
1221 if (!umulp)
1223 /* ??? This could be done with emit_store_flag where available. */
1224 temp = expand_binop (word_mode, lshr_optab, op0_low, wordm1,
1225 NULL_RTX, 1, methods);
1226 if (temp)
1227 op0_high = expand_binop (word_mode, add_optab, op0_high, temp,
1228 NULL_RTX, 0, OPTAB_DIRECT);
1229 else
1231 temp = expand_binop (word_mode, ashr_optab, op0_low, wordm1,
1232 NULL_RTX, 0, methods);
1233 if (!temp)
1234 return NULL_RTX;
1235 op0_high = expand_binop (word_mode, sub_optab, op0_high, temp,
1236 NULL_RTX, 0, OPTAB_DIRECT);
1239 if (!op0_high)
1240 return NULL_RTX;
1243 adjust = expand_binop (word_mode, smul_optab, op0_high, op1_low,
1244 NULL_RTX, 0, OPTAB_DIRECT);
1245 if (!adjust)
1246 return NULL_RTX;
1248 /* OP0_HIGH should now be dead. */
1250 if (!umulp)
1252 /* ??? This could be done with emit_store_flag where available. */
1253 temp = expand_binop (word_mode, lshr_optab, op1_low, wordm1,
1254 NULL_RTX, 1, methods);
1255 if (temp)
1256 op1_high = expand_binop (word_mode, add_optab, op1_high, temp,
1257 NULL_RTX, 0, OPTAB_DIRECT);
1258 else
1260 temp = expand_binop (word_mode, ashr_optab, op1_low, wordm1,
1261 NULL_RTX, 0, methods);
1262 if (!temp)
1263 return NULL_RTX;
1264 op1_high = expand_binop (word_mode, sub_optab, op1_high, temp,
1265 NULL_RTX, 0, OPTAB_DIRECT);
1268 if (!op1_high)
1269 return NULL_RTX;
1272 temp = expand_binop (word_mode, smul_optab, op1_high, op0_low,
1273 NULL_RTX, 0, OPTAB_DIRECT);
1274 if (!temp)
1275 return NULL_RTX;
1277 /* OP1_HIGH should now be dead. */
1279 adjust = expand_binop (word_mode, add_optab, adjust, temp,
1280 NULL_RTX, 0, OPTAB_DIRECT);
1282 if (target && !REG_P (target))
1283 target = NULL_RTX;
1285 if (umulp)
1286 product = expand_binop (mode, umul_widen_optab, op0_low, op1_low,
1287 target, 1, OPTAB_DIRECT);
1288 else
1289 product = expand_binop (mode, smul_widen_optab, op0_low, op1_low,
1290 target, 1, OPTAB_DIRECT);
1292 if (!product)
1293 return NULL_RTX;
1295 product_high = operand_subword (product, high, 1, mode);
1296 adjust = expand_binop (word_mode, add_optab, product_high, adjust,
1297 NULL_RTX, 0, OPTAB_DIRECT);
1298 emit_move_insn (product_high, adjust);
1299 return product;
1302 /* Wrapper around expand_binop which takes an rtx code to specify
1303 the operation to perform, not an optab pointer. All other
1304 arguments are the same. */
1306 expand_simple_binop (machine_mode mode, enum rtx_code code, rtx op0,
1307 rtx op1, rtx target, int unsignedp,
1308 enum optab_methods methods)
1310 optab binop = code_to_optab (code);
1311 gcc_assert (binop);
1313 return expand_binop (mode, binop, op0, op1, target, unsignedp, methods);
1316 /* Return whether OP0 and OP1 should be swapped when expanding a commutative
1317 binop. Order them according to commutative_operand_precedence and, if
1318 possible, try to put TARGET or a pseudo first. */
1319 static bool
1320 swap_commutative_operands_with_target (rtx target, rtx op0, rtx op1)
1322 int op0_prec = commutative_operand_precedence (op0);
1323 int op1_prec = commutative_operand_precedence (op1);
1325 if (op0_prec < op1_prec)
1326 return true;
1328 if (op0_prec > op1_prec)
1329 return false;
1331 /* With equal precedence, both orders are ok, but it is better if the
1332 first operand is TARGET, or if both TARGET and OP0 are pseudos. */
1333 if (target == 0 || REG_P (target))
1334 return (REG_P (op1) && !REG_P (op0)) || target == op1;
1335 else
1336 return rtx_equal_p (op1, target);
1339 /* Return true if BINOPTAB implements a shift operation. */
1341 static bool
1342 shift_optab_p (optab binoptab)
1344 switch (optab_to_code (binoptab))
1346 case ASHIFT:
1347 case SS_ASHIFT:
1348 case US_ASHIFT:
1349 case ASHIFTRT:
1350 case LSHIFTRT:
1351 case ROTATE:
1352 case ROTATERT:
1353 return true;
1355 default:
1356 return false;
1360 /* Return true if BINOPTAB implements a commutative binary operation. */
1362 static bool
1363 commutative_optab_p (optab binoptab)
1365 return (GET_RTX_CLASS (optab_to_code (binoptab)) == RTX_COMM_ARITH
1366 || binoptab == smul_widen_optab
1367 || binoptab == umul_widen_optab
1368 || binoptab == smul_highpart_optab
1369 || binoptab == umul_highpart_optab);
1372 /* X is to be used in mode MODE as operand OPN to BINOPTAB. If we're
1373 optimizing, and if the operand is a constant that costs more than
1374 1 instruction, force the constant into a register and return that
1375 register. Return X otherwise. UNSIGNEDP says whether X is unsigned. */
1377 static rtx
1378 avoid_expensive_constant (machine_mode mode, optab binoptab,
1379 int opn, rtx x, bool unsignedp)
1381 bool speed = optimize_insn_for_speed_p ();
1383 if (mode != VOIDmode
1384 && optimize
1385 && CONSTANT_P (x)
1386 && (rtx_cost (x, optab_to_code (binoptab), opn, speed)
1387 > set_src_cost (x, speed)))
1389 if (CONST_INT_P (x))
1391 HOST_WIDE_INT intval = trunc_int_for_mode (INTVAL (x), mode);
1392 if (intval != INTVAL (x))
1393 x = GEN_INT (intval);
1395 else
1396 x = convert_modes (mode, VOIDmode, x, unsignedp);
1397 x = force_reg (mode, x);
1399 return x;
1402 /* Helper function for expand_binop: handle the case where there
1403 is an insn that directly implements the indicated operation.
1404 Returns null if this is not possible. */
1405 static rtx
1406 expand_binop_directly (machine_mode mode, optab binoptab,
1407 rtx op0, rtx op1,
1408 rtx target, int unsignedp, enum optab_methods methods,
1409 rtx_insn *last)
1411 machine_mode from_mode = widened_mode (mode, op0, op1);
1412 enum insn_code icode = find_widening_optab_handler (binoptab, mode,
1413 from_mode, 1);
1414 machine_mode xmode0 = insn_data[(int) icode].operand[1].mode;
1415 machine_mode xmode1 = insn_data[(int) icode].operand[2].mode;
1416 machine_mode mode0, mode1, tmp_mode;
1417 struct expand_operand ops[3];
1418 bool commutative_p;
1419 rtx pat;
1420 rtx xop0 = op0, xop1 = op1;
1421 rtx swap;
1423 /* If it is a commutative operator and the modes would match
1424 if we would swap the operands, we can save the conversions. */
1425 commutative_p = commutative_optab_p (binoptab);
1426 if (commutative_p
1427 && GET_MODE (xop0) != xmode0 && GET_MODE (xop1) != xmode1
1428 && GET_MODE (xop0) == xmode1 && GET_MODE (xop1) == xmode1)
1430 swap = xop0;
1431 xop0 = xop1;
1432 xop1 = swap;
1435 /* If we are optimizing, force expensive constants into a register. */
1436 xop0 = avoid_expensive_constant (xmode0, binoptab, 0, xop0, unsignedp);
1437 if (!shift_optab_p (binoptab))
1438 xop1 = avoid_expensive_constant (xmode1, binoptab, 1, xop1, unsignedp);
1440 /* In case the insn wants input operands in modes different from
1441 those of the actual operands, convert the operands. It would
1442 seem that we don't need to convert CONST_INTs, but we do, so
1443 that they're properly zero-extended, sign-extended or truncated
1444 for their mode. */
1446 mode0 = GET_MODE (xop0) != VOIDmode ? GET_MODE (xop0) : mode;
1447 if (xmode0 != VOIDmode && xmode0 != mode0)
1449 xop0 = convert_modes (xmode0, mode0, xop0, unsignedp);
1450 mode0 = xmode0;
1453 mode1 = GET_MODE (xop1) != VOIDmode ? GET_MODE (xop1) : mode;
1454 if (xmode1 != VOIDmode && xmode1 != mode1)
1456 xop1 = convert_modes (xmode1, mode1, xop1, unsignedp);
1457 mode1 = xmode1;
1460 /* If operation is commutative,
1461 try to make the first operand a register.
1462 Even better, try to make it the same as the target.
1463 Also try to make the last operand a constant. */
1464 if (commutative_p
1465 && swap_commutative_operands_with_target (target, xop0, xop1))
1467 swap = xop1;
1468 xop1 = xop0;
1469 xop0 = swap;
1472 /* Now, if insn's predicates don't allow our operands, put them into
1473 pseudo regs. */
1475 if (binoptab == vec_pack_trunc_optab
1476 || binoptab == vec_pack_usat_optab
1477 || binoptab == vec_pack_ssat_optab
1478 || binoptab == vec_pack_ufix_trunc_optab
1479 || binoptab == vec_pack_sfix_trunc_optab)
1481 /* The mode of the result is different then the mode of the
1482 arguments. */
1483 tmp_mode = insn_data[(int) icode].operand[0].mode;
1484 if (GET_MODE_NUNITS (tmp_mode) != 2 * GET_MODE_NUNITS (mode))
1486 delete_insns_since (last);
1487 return NULL_RTX;
1490 else
1491 tmp_mode = mode;
1493 create_output_operand (&ops[0], target, tmp_mode);
1494 create_input_operand (&ops[1], xop0, mode0);
1495 create_input_operand (&ops[2], xop1, mode1);
1496 pat = maybe_gen_insn (icode, 3, ops);
1497 if (pat)
1499 /* If PAT is composed of more than one insn, try to add an appropriate
1500 REG_EQUAL note to it. If we can't because TEMP conflicts with an
1501 operand, call expand_binop again, this time without a target. */
1502 if (INSN_P (pat) && NEXT_INSN (as_a <rtx_insn *> (pat)) != NULL_RTX
1503 && ! add_equal_note (as_a <rtx_insn *> (pat), ops[0].value,
1504 optab_to_code (binoptab),
1505 ops[1].value, ops[2].value))
1507 delete_insns_since (last);
1508 return expand_binop (mode, binoptab, op0, op1, NULL_RTX,
1509 unsignedp, methods);
1512 emit_insn (pat);
1513 return ops[0].value;
1515 delete_insns_since (last);
1516 return NULL_RTX;
1519 /* Generate code to perform an operation specified by BINOPTAB
1520 on operands OP0 and OP1, with result having machine-mode MODE.
1522 UNSIGNEDP is for the case where we have to widen the operands
1523 to perform the operation. It says to use zero-extension.
1525 If TARGET is nonzero, the value
1526 is generated there, if it is convenient to do so.
1527 In all cases an rtx is returned for the locus of the value;
1528 this may or may not be TARGET. */
1531 expand_binop (machine_mode mode, optab binoptab, rtx op0, rtx op1,
1532 rtx target, int unsignedp, enum optab_methods methods)
1534 enum optab_methods next_methods
1535 = (methods == OPTAB_LIB || methods == OPTAB_LIB_WIDEN
1536 ? OPTAB_WIDEN : methods);
1537 enum mode_class mclass;
1538 machine_mode wider_mode;
1539 rtx libfunc;
1540 rtx temp;
1541 rtx_insn *entry_last = get_last_insn ();
1542 rtx_insn *last;
1544 mclass = GET_MODE_CLASS (mode);
1546 /* If subtracting an integer constant, convert this into an addition of
1547 the negated constant. */
1549 if (binoptab == sub_optab && CONST_INT_P (op1))
1551 op1 = negate_rtx (mode, op1);
1552 binoptab = add_optab;
1555 /* Record where to delete back to if we backtrack. */
1556 last = get_last_insn ();
1558 /* If we can do it with a three-operand insn, do so. */
1560 if (methods != OPTAB_MUST_WIDEN
1561 && find_widening_optab_handler (binoptab, mode,
1562 widened_mode (mode, op0, op1), 1)
1563 != CODE_FOR_nothing)
1565 temp = expand_binop_directly (mode, binoptab, op0, op1, target,
1566 unsignedp, methods, last);
1567 if (temp)
1568 return temp;
1571 /* If we were trying to rotate, and that didn't work, try rotating
1572 the other direction before falling back to shifts and bitwise-or. */
1573 if (((binoptab == rotl_optab
1574 && optab_handler (rotr_optab, mode) != CODE_FOR_nothing)
1575 || (binoptab == rotr_optab
1576 && optab_handler (rotl_optab, mode) != CODE_FOR_nothing))
1577 && mclass == MODE_INT)
1579 optab otheroptab = (binoptab == rotl_optab ? rotr_optab : rotl_optab);
1580 rtx newop1;
1581 unsigned int bits = GET_MODE_PRECISION (mode);
1583 if (CONST_INT_P (op1))
1584 newop1 = GEN_INT (bits - INTVAL (op1));
1585 else if (targetm.shift_truncation_mask (mode) == bits - 1)
1586 newop1 = negate_rtx (GET_MODE (op1), op1);
1587 else
1588 newop1 = expand_binop (GET_MODE (op1), sub_optab,
1589 gen_int_mode (bits, GET_MODE (op1)), op1,
1590 NULL_RTX, unsignedp, OPTAB_DIRECT);
1592 temp = expand_binop_directly (mode, otheroptab, op0, newop1,
1593 target, unsignedp, methods, last);
1594 if (temp)
1595 return temp;
1598 /* If this is a multiply, see if we can do a widening operation that
1599 takes operands of this mode and makes a wider mode. */
1601 if (binoptab == smul_optab
1602 && GET_MODE_2XWIDER_MODE (mode) != VOIDmode
1603 && (widening_optab_handler ((unsignedp ? umul_widen_optab
1604 : smul_widen_optab),
1605 GET_MODE_2XWIDER_MODE (mode), mode)
1606 != CODE_FOR_nothing))
1608 temp = expand_binop (GET_MODE_2XWIDER_MODE (mode),
1609 unsignedp ? umul_widen_optab : smul_widen_optab,
1610 op0, op1, NULL_RTX, unsignedp, OPTAB_DIRECT);
1612 if (temp != 0)
1614 if (GET_MODE_CLASS (mode) == MODE_INT
1615 && TRULY_NOOP_TRUNCATION_MODES_P (mode, GET_MODE (temp)))
1616 return gen_lowpart (mode, temp);
1617 else
1618 return convert_to_mode (mode, temp, unsignedp);
1622 /* If this is a vector shift by a scalar, see if we can do a vector
1623 shift by a vector. If so, broadcast the scalar into a vector. */
1624 if (mclass == MODE_VECTOR_INT)
1626 optab otheroptab = unknown_optab;
1628 if (binoptab == ashl_optab)
1629 otheroptab = vashl_optab;
1630 else if (binoptab == ashr_optab)
1631 otheroptab = vashr_optab;
1632 else if (binoptab == lshr_optab)
1633 otheroptab = vlshr_optab;
1634 else if (binoptab == rotl_optab)
1635 otheroptab = vrotl_optab;
1636 else if (binoptab == rotr_optab)
1637 otheroptab = vrotr_optab;
1639 if (otheroptab && optab_handler (otheroptab, mode) != CODE_FOR_nothing)
1641 rtx vop1 = expand_vector_broadcast (mode, op1);
1642 if (vop1)
1644 temp = expand_binop_directly (mode, otheroptab, op0, vop1,
1645 target, unsignedp, methods, last);
1646 if (temp)
1647 return temp;
1652 /* Look for a wider mode of the same class for which we think we
1653 can open-code the operation. Check for a widening multiply at the
1654 wider mode as well. */
1656 if (CLASS_HAS_WIDER_MODES_P (mclass)
1657 && methods != OPTAB_DIRECT && methods != OPTAB_LIB)
1658 for (wider_mode = GET_MODE_WIDER_MODE (mode);
1659 wider_mode != VOIDmode;
1660 wider_mode = GET_MODE_WIDER_MODE (wider_mode))
1662 if (optab_handler (binoptab, wider_mode) != CODE_FOR_nothing
1663 || (binoptab == smul_optab
1664 && GET_MODE_WIDER_MODE (wider_mode) != VOIDmode
1665 && (find_widening_optab_handler ((unsignedp
1666 ? umul_widen_optab
1667 : smul_widen_optab),
1668 GET_MODE_WIDER_MODE (wider_mode),
1669 mode, 0)
1670 != CODE_FOR_nothing)))
1672 rtx xop0 = op0, xop1 = op1;
1673 int no_extend = 0;
1675 /* For certain integer operations, we need not actually extend
1676 the narrow operands, as long as we will truncate
1677 the results to the same narrowness. */
1679 if ((binoptab == ior_optab || binoptab == and_optab
1680 || binoptab == xor_optab
1681 || binoptab == add_optab || binoptab == sub_optab
1682 || binoptab == smul_optab || binoptab == ashl_optab)
1683 && mclass == MODE_INT)
1685 no_extend = 1;
1686 xop0 = avoid_expensive_constant (mode, binoptab, 0,
1687 xop0, unsignedp);
1688 if (binoptab != ashl_optab)
1689 xop1 = avoid_expensive_constant (mode, binoptab, 1,
1690 xop1, unsignedp);
1693 xop0 = widen_operand (xop0, wider_mode, mode, unsignedp, no_extend);
1695 /* The second operand of a shift must always be extended. */
1696 xop1 = widen_operand (xop1, wider_mode, mode, unsignedp,
1697 no_extend && binoptab != ashl_optab);
1699 temp = expand_binop (wider_mode, binoptab, xop0, xop1, NULL_RTX,
1700 unsignedp, OPTAB_DIRECT);
1701 if (temp)
1703 if (mclass != MODE_INT
1704 || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode))
1706 if (target == 0)
1707 target = gen_reg_rtx (mode);
1708 convert_move (target, temp, 0);
1709 return target;
1711 else
1712 return gen_lowpart (mode, temp);
1714 else
1715 delete_insns_since (last);
1719 /* If operation is commutative,
1720 try to make the first operand a register.
1721 Even better, try to make it the same as the target.
1722 Also try to make the last operand a constant. */
1723 if (commutative_optab_p (binoptab)
1724 && swap_commutative_operands_with_target (target, op0, op1))
1726 temp = op1;
1727 op1 = op0;
1728 op0 = temp;
1731 /* These can be done a word at a time. */
1732 if ((binoptab == and_optab || binoptab == ior_optab || binoptab == xor_optab)
1733 && mclass == MODE_INT
1734 && GET_MODE_SIZE (mode) > UNITS_PER_WORD
1735 && optab_handler (binoptab, word_mode) != CODE_FOR_nothing)
1737 int i;
1738 rtx_insn *insns;
1740 /* If TARGET is the same as one of the operands, the REG_EQUAL note
1741 won't be accurate, so use a new target. */
1742 if (target == 0
1743 || target == op0
1744 || target == op1
1745 || !valid_multiword_target_p (target))
1746 target = gen_reg_rtx (mode);
1748 start_sequence ();
1750 /* Do the actual arithmetic. */
1751 for (i = 0; i < GET_MODE_BITSIZE (mode) / BITS_PER_WORD; i++)
1753 rtx target_piece = operand_subword (target, i, 1, mode);
1754 rtx x = expand_binop (word_mode, binoptab,
1755 operand_subword_force (op0, i, mode),
1756 operand_subword_force (op1, i, mode),
1757 target_piece, unsignedp, next_methods);
1759 if (x == 0)
1760 break;
1762 if (target_piece != x)
1763 emit_move_insn (target_piece, x);
1766 insns = get_insns ();
1767 end_sequence ();
1769 if (i == GET_MODE_BITSIZE (mode) / BITS_PER_WORD)
1771 emit_insn (insns);
1772 return target;
1776 /* Synthesize double word shifts from single word shifts. */
1777 if ((binoptab == lshr_optab || binoptab == ashl_optab
1778 || binoptab == ashr_optab)
1779 && mclass == MODE_INT
1780 && (CONST_INT_P (op1) || optimize_insn_for_speed_p ())
1781 && GET_MODE_SIZE (mode) == 2 * UNITS_PER_WORD
1782 && GET_MODE_PRECISION (mode) == GET_MODE_BITSIZE (mode)
1783 && optab_handler (binoptab, word_mode) != CODE_FOR_nothing
1784 && optab_handler (ashl_optab, word_mode) != CODE_FOR_nothing
1785 && optab_handler (lshr_optab, word_mode) != CODE_FOR_nothing)
1787 unsigned HOST_WIDE_INT shift_mask, double_shift_mask;
1788 machine_mode op1_mode;
1790 double_shift_mask = targetm.shift_truncation_mask (mode);
1791 shift_mask = targetm.shift_truncation_mask (word_mode);
1792 op1_mode = GET_MODE (op1) != VOIDmode ? GET_MODE (op1) : word_mode;
1794 /* Apply the truncation to constant shifts. */
1795 if (double_shift_mask > 0 && CONST_INT_P (op1))
1796 op1 = GEN_INT (INTVAL (op1) & double_shift_mask);
1798 if (op1 == CONST0_RTX (op1_mode))
1799 return op0;
1801 /* Make sure that this is a combination that expand_doubleword_shift
1802 can handle. See the comments there for details. */
1803 if (double_shift_mask == 0
1804 || (shift_mask == BITS_PER_WORD - 1
1805 && double_shift_mask == BITS_PER_WORD * 2 - 1))
1807 rtx_insn *insns;
1808 rtx into_target, outof_target;
1809 rtx into_input, outof_input;
1810 int left_shift, outof_word;
1812 /* If TARGET is the same as one of the operands, the REG_EQUAL note
1813 won't be accurate, so use a new target. */
1814 if (target == 0
1815 || target == op0
1816 || target == op1
1817 || !valid_multiword_target_p (target))
1818 target = gen_reg_rtx (mode);
1820 start_sequence ();
1822 /* OUTOF_* is the word we are shifting bits away from, and
1823 INTO_* is the word that we are shifting bits towards, thus
1824 they differ depending on the direction of the shift and
1825 WORDS_BIG_ENDIAN. */
1827 left_shift = binoptab == ashl_optab;
1828 outof_word = left_shift ^ ! WORDS_BIG_ENDIAN;
1830 outof_target = operand_subword (target, outof_word, 1, mode);
1831 into_target = operand_subword (target, 1 - outof_word, 1, mode);
1833 outof_input = operand_subword_force (op0, outof_word, mode);
1834 into_input = operand_subword_force (op0, 1 - outof_word, mode);
1836 if (expand_doubleword_shift (op1_mode, binoptab,
1837 outof_input, into_input, op1,
1838 outof_target, into_target,
1839 unsignedp, next_methods, shift_mask))
1841 insns = get_insns ();
1842 end_sequence ();
1844 emit_insn (insns);
1845 return target;
1847 end_sequence ();
1851 /* Synthesize double word rotates from single word shifts. */
1852 if ((binoptab == rotl_optab || binoptab == rotr_optab)
1853 && mclass == MODE_INT
1854 && CONST_INT_P (op1)
1855 && GET_MODE_PRECISION (mode) == 2 * BITS_PER_WORD
1856 && optab_handler (ashl_optab, word_mode) != CODE_FOR_nothing
1857 && optab_handler (lshr_optab, word_mode) != CODE_FOR_nothing)
1859 rtx_insn *insns;
1860 rtx into_target, outof_target;
1861 rtx into_input, outof_input;
1862 rtx inter;
1863 int shift_count, left_shift, outof_word;
1865 /* If TARGET is the same as one of the operands, the REG_EQUAL note
1866 won't be accurate, so use a new target. Do this also if target is not
1867 a REG, first because having a register instead may open optimization
1868 opportunities, and second because if target and op0 happen to be MEMs
1869 designating the same location, we would risk clobbering it too early
1870 in the code sequence we generate below. */
1871 if (target == 0
1872 || target == op0
1873 || target == op1
1874 || !REG_P (target)
1875 || !valid_multiword_target_p (target))
1876 target = gen_reg_rtx (mode);
1878 start_sequence ();
1880 shift_count = INTVAL (op1);
1882 /* OUTOF_* is the word we are shifting bits away from, and
1883 INTO_* is the word that we are shifting bits towards, thus
1884 they differ depending on the direction of the shift and
1885 WORDS_BIG_ENDIAN. */
1887 left_shift = (binoptab == rotl_optab);
1888 outof_word = left_shift ^ ! WORDS_BIG_ENDIAN;
1890 outof_target = operand_subword (target, outof_word, 1, mode);
1891 into_target = operand_subword (target, 1 - outof_word, 1, mode);
1893 outof_input = operand_subword_force (op0, outof_word, mode);
1894 into_input = operand_subword_force (op0, 1 - outof_word, mode);
1896 if (shift_count == BITS_PER_WORD)
1898 /* This is just a word swap. */
1899 emit_move_insn (outof_target, into_input);
1900 emit_move_insn (into_target, outof_input);
1901 inter = const0_rtx;
1903 else
1905 rtx into_temp1, into_temp2, outof_temp1, outof_temp2;
1906 rtx first_shift_count, second_shift_count;
1907 optab reverse_unsigned_shift, unsigned_shift;
1909 reverse_unsigned_shift = (left_shift ^ (shift_count < BITS_PER_WORD)
1910 ? lshr_optab : ashl_optab);
1912 unsigned_shift = (left_shift ^ (shift_count < BITS_PER_WORD)
1913 ? ashl_optab : lshr_optab);
1915 if (shift_count > BITS_PER_WORD)
1917 first_shift_count = GEN_INT (shift_count - BITS_PER_WORD);
1918 second_shift_count = GEN_INT (2 * BITS_PER_WORD - shift_count);
1920 else
1922 first_shift_count = GEN_INT (BITS_PER_WORD - shift_count);
1923 second_shift_count = GEN_INT (shift_count);
1926 into_temp1 = expand_binop (word_mode, unsigned_shift,
1927 outof_input, first_shift_count,
1928 NULL_RTX, unsignedp, next_methods);
1929 into_temp2 = expand_binop (word_mode, reverse_unsigned_shift,
1930 into_input, second_shift_count,
1931 NULL_RTX, unsignedp, next_methods);
1933 if (into_temp1 != 0 && into_temp2 != 0)
1934 inter = expand_binop (word_mode, ior_optab, into_temp1, into_temp2,
1935 into_target, unsignedp, next_methods);
1936 else
1937 inter = 0;
1939 if (inter != 0 && inter != into_target)
1940 emit_move_insn (into_target, inter);
1942 outof_temp1 = expand_binop (word_mode, unsigned_shift,
1943 into_input, first_shift_count,
1944 NULL_RTX, unsignedp, next_methods);
1945 outof_temp2 = expand_binop (word_mode, reverse_unsigned_shift,
1946 outof_input, second_shift_count,
1947 NULL_RTX, unsignedp, next_methods);
1949 if (inter != 0 && outof_temp1 != 0 && outof_temp2 != 0)
1950 inter = expand_binop (word_mode, ior_optab,
1951 outof_temp1, outof_temp2,
1952 outof_target, unsignedp, next_methods);
1954 if (inter != 0 && inter != outof_target)
1955 emit_move_insn (outof_target, inter);
1958 insns = get_insns ();
1959 end_sequence ();
1961 if (inter != 0)
1963 emit_insn (insns);
1964 return target;
1968 /* These can be done a word at a time by propagating carries. */
1969 if ((binoptab == add_optab || binoptab == sub_optab)
1970 && mclass == MODE_INT
1971 && GET_MODE_SIZE (mode) >= 2 * UNITS_PER_WORD
1972 && optab_handler (binoptab, word_mode) != CODE_FOR_nothing)
1974 unsigned int i;
1975 optab otheroptab = binoptab == add_optab ? sub_optab : add_optab;
1976 const unsigned int nwords = GET_MODE_BITSIZE (mode) / BITS_PER_WORD;
1977 rtx carry_in = NULL_RTX, carry_out = NULL_RTX;
1978 rtx xop0, xop1, xtarget;
1980 /* We can handle either a 1 or -1 value for the carry. If STORE_FLAG
1981 value is one of those, use it. Otherwise, use 1 since it is the
1982 one easiest to get. */
1983 #if STORE_FLAG_VALUE == 1 || STORE_FLAG_VALUE == -1
1984 int normalizep = STORE_FLAG_VALUE;
1985 #else
1986 int normalizep = 1;
1987 #endif
1989 /* Prepare the operands. */
1990 xop0 = force_reg (mode, op0);
1991 xop1 = force_reg (mode, op1);
1993 xtarget = gen_reg_rtx (mode);
1995 if (target == 0 || !REG_P (target) || !valid_multiword_target_p (target))
1996 target = xtarget;
1998 /* Indicate for flow that the entire target reg is being set. */
1999 if (REG_P (target))
2000 emit_clobber (xtarget);
2002 /* Do the actual arithmetic. */
2003 for (i = 0; i < nwords; i++)
2005 int index = (WORDS_BIG_ENDIAN ? nwords - i - 1 : i);
2006 rtx target_piece = operand_subword (xtarget, index, 1, mode);
2007 rtx op0_piece = operand_subword_force (xop0, index, mode);
2008 rtx op1_piece = operand_subword_force (xop1, index, mode);
2009 rtx x;
2011 /* Main add/subtract of the input operands. */
2012 x = expand_binop (word_mode, binoptab,
2013 op0_piece, op1_piece,
2014 target_piece, unsignedp, next_methods);
2015 if (x == 0)
2016 break;
2018 if (i + 1 < nwords)
2020 /* Store carry from main add/subtract. */
2021 carry_out = gen_reg_rtx (word_mode);
2022 carry_out = emit_store_flag_force (carry_out,
2023 (binoptab == add_optab
2024 ? LT : GT),
2025 x, op0_piece,
2026 word_mode, 1, normalizep);
2029 if (i > 0)
2031 rtx newx;
2033 /* Add/subtract previous carry to main result. */
2034 newx = expand_binop (word_mode,
2035 normalizep == 1 ? binoptab : otheroptab,
2036 x, carry_in,
2037 NULL_RTX, 1, next_methods);
2039 if (i + 1 < nwords)
2041 /* Get out carry from adding/subtracting carry in. */
2042 rtx carry_tmp = gen_reg_rtx (word_mode);
2043 carry_tmp = emit_store_flag_force (carry_tmp,
2044 (binoptab == add_optab
2045 ? LT : GT),
2046 newx, x,
2047 word_mode, 1, normalizep);
2049 /* Logical-ior the two poss. carry together. */
2050 carry_out = expand_binop (word_mode, ior_optab,
2051 carry_out, carry_tmp,
2052 carry_out, 0, next_methods);
2053 if (carry_out == 0)
2054 break;
2056 emit_move_insn (target_piece, newx);
2058 else
2060 if (x != target_piece)
2061 emit_move_insn (target_piece, x);
2064 carry_in = carry_out;
2067 if (i == GET_MODE_BITSIZE (mode) / (unsigned) BITS_PER_WORD)
2069 if (optab_handler (mov_optab, mode) != CODE_FOR_nothing
2070 || ! rtx_equal_p (target, xtarget))
2072 rtx temp = emit_move_insn (target, xtarget);
2074 set_dst_reg_note (temp, REG_EQUAL,
2075 gen_rtx_fmt_ee (optab_to_code (binoptab),
2076 mode, copy_rtx (xop0),
2077 copy_rtx (xop1)),
2078 target);
2080 else
2081 target = xtarget;
2083 return target;
2086 else
2087 delete_insns_since (last);
2090 /* Attempt to synthesize double word multiplies using a sequence of word
2091 mode multiplications. We first attempt to generate a sequence using a
2092 more efficient unsigned widening multiply, and if that fails we then
2093 try using a signed widening multiply. */
2095 if (binoptab == smul_optab
2096 && mclass == MODE_INT
2097 && GET_MODE_SIZE (mode) == 2 * UNITS_PER_WORD
2098 && optab_handler (smul_optab, word_mode) != CODE_FOR_nothing
2099 && optab_handler (add_optab, word_mode) != CODE_FOR_nothing)
2101 rtx product = NULL_RTX;
2102 if (widening_optab_handler (umul_widen_optab, mode, word_mode)
2103 != CODE_FOR_nothing)
2105 product = expand_doubleword_mult (mode, op0, op1, target,
2106 true, methods);
2107 if (!product)
2108 delete_insns_since (last);
2111 if (product == NULL_RTX
2112 && widening_optab_handler (smul_widen_optab, mode, word_mode)
2113 != CODE_FOR_nothing)
2115 product = expand_doubleword_mult (mode, op0, op1, target,
2116 false, methods);
2117 if (!product)
2118 delete_insns_since (last);
2121 if (product != NULL_RTX)
2123 if (optab_handler (mov_optab, mode) != CODE_FOR_nothing)
2125 temp = emit_move_insn (target ? target : product, product);
2126 set_dst_reg_note (temp,
2127 REG_EQUAL,
2128 gen_rtx_fmt_ee (MULT, mode,
2129 copy_rtx (op0),
2130 copy_rtx (op1)),
2131 target ? target : product);
2133 return product;
2137 /* It can't be open-coded in this mode.
2138 Use a library call if one is available and caller says that's ok. */
2140 libfunc = optab_libfunc (binoptab, mode);
2141 if (libfunc
2142 && (methods == OPTAB_LIB || methods == OPTAB_LIB_WIDEN))
2144 rtx_insn *insns;
2145 rtx op1x = op1;
2146 machine_mode op1_mode = mode;
2147 rtx value;
2149 start_sequence ();
2151 if (shift_optab_p (binoptab))
2153 op1_mode = targetm.libgcc_shift_count_mode ();
2154 /* Specify unsigned here,
2155 since negative shift counts are meaningless. */
2156 op1x = convert_to_mode (op1_mode, op1, 1);
2159 if (GET_MODE (op0) != VOIDmode
2160 && GET_MODE (op0) != mode)
2161 op0 = convert_to_mode (mode, op0, unsignedp);
2163 /* Pass 1 for NO_QUEUE so we don't lose any increments
2164 if the libcall is cse'd or moved. */
2165 value = emit_library_call_value (libfunc,
2166 NULL_RTX, LCT_CONST, mode, 2,
2167 op0, mode, op1x, op1_mode);
2169 insns = get_insns ();
2170 end_sequence ();
2172 target = gen_reg_rtx (mode);
2173 emit_libcall_block_1 (insns, target, value,
2174 gen_rtx_fmt_ee (optab_to_code (binoptab),
2175 mode, op0, op1),
2176 trapv_binoptab_p (binoptab));
2178 return target;
2181 delete_insns_since (last);
2183 /* It can't be done in this mode. Can we do it in a wider mode? */
2185 if (! (methods == OPTAB_WIDEN || methods == OPTAB_LIB_WIDEN
2186 || methods == OPTAB_MUST_WIDEN))
2188 /* Caller says, don't even try. */
2189 delete_insns_since (entry_last);
2190 return 0;
2193 /* Compute the value of METHODS to pass to recursive calls.
2194 Don't allow widening to be tried recursively. */
2196 methods = (methods == OPTAB_LIB_WIDEN ? OPTAB_LIB : OPTAB_DIRECT);
2198 /* Look for a wider mode of the same class for which it appears we can do
2199 the operation. */
2201 if (CLASS_HAS_WIDER_MODES_P (mclass))
2203 for (wider_mode = GET_MODE_WIDER_MODE (mode);
2204 wider_mode != VOIDmode;
2205 wider_mode = GET_MODE_WIDER_MODE (wider_mode))
2207 if (find_widening_optab_handler (binoptab, wider_mode, mode, 1)
2208 != CODE_FOR_nothing
2209 || (methods == OPTAB_LIB
2210 && optab_libfunc (binoptab, wider_mode)))
2212 rtx xop0 = op0, xop1 = op1;
2213 int no_extend = 0;
2215 /* For certain integer operations, we need not actually extend
2216 the narrow operands, as long as we will truncate
2217 the results to the same narrowness. */
2219 if ((binoptab == ior_optab || binoptab == and_optab
2220 || binoptab == xor_optab
2221 || binoptab == add_optab || binoptab == sub_optab
2222 || binoptab == smul_optab || binoptab == ashl_optab)
2223 && mclass == MODE_INT)
2224 no_extend = 1;
2226 xop0 = widen_operand (xop0, wider_mode, mode,
2227 unsignedp, no_extend);
2229 /* The second operand of a shift must always be extended. */
2230 xop1 = widen_operand (xop1, wider_mode, mode, unsignedp,
2231 no_extend && binoptab != ashl_optab);
2233 temp = expand_binop (wider_mode, binoptab, xop0, xop1, NULL_RTX,
2234 unsignedp, methods);
2235 if (temp)
2237 if (mclass != MODE_INT
2238 || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode))
2240 if (target == 0)
2241 target = gen_reg_rtx (mode);
2242 convert_move (target, temp, 0);
2243 return target;
2245 else
2246 return gen_lowpart (mode, temp);
2248 else
2249 delete_insns_since (last);
2254 delete_insns_since (entry_last);
2255 return 0;
2258 /* Expand a binary operator which has both signed and unsigned forms.
2259 UOPTAB is the optab for unsigned operations, and SOPTAB is for
2260 signed operations.
2262 If we widen unsigned operands, we may use a signed wider operation instead
2263 of an unsigned wider operation, since the result would be the same. */
2266 sign_expand_binop (machine_mode mode, optab uoptab, optab soptab,
2267 rtx op0, rtx op1, rtx target, int unsignedp,
2268 enum optab_methods methods)
2270 rtx temp;
2271 optab direct_optab = unsignedp ? uoptab : soptab;
2272 bool save_enable;
2274 /* Do it without widening, if possible. */
2275 temp = expand_binop (mode, direct_optab, op0, op1, target,
2276 unsignedp, OPTAB_DIRECT);
2277 if (temp || methods == OPTAB_DIRECT)
2278 return temp;
2280 /* Try widening to a signed int. Disable any direct use of any
2281 signed insn in the current mode. */
2282 save_enable = swap_optab_enable (soptab, mode, false);
2284 temp = expand_binop (mode, soptab, op0, op1, target,
2285 unsignedp, OPTAB_WIDEN);
2287 /* For unsigned operands, try widening to an unsigned int. */
2288 if (!temp && unsignedp)
2289 temp = expand_binop (mode, uoptab, op0, op1, target,
2290 unsignedp, OPTAB_WIDEN);
2291 if (temp || methods == OPTAB_WIDEN)
2292 goto egress;
2294 /* Use the right width libcall if that exists. */
2295 temp = expand_binop (mode, direct_optab, op0, op1, target,
2296 unsignedp, OPTAB_LIB);
2297 if (temp || methods == OPTAB_LIB)
2298 goto egress;
2300 /* Must widen and use a libcall, use either signed or unsigned. */
2301 temp = expand_binop (mode, soptab, op0, op1, target,
2302 unsignedp, methods);
2303 if (!temp && unsignedp)
2304 temp = expand_binop (mode, uoptab, op0, op1, target,
2305 unsignedp, methods);
2307 egress:
2308 /* Undo the fiddling above. */
2309 if (save_enable)
2310 swap_optab_enable (soptab, mode, true);
2311 return temp;
2314 /* Generate code to perform an operation specified by UNOPPTAB
2315 on operand OP0, with two results to TARG0 and TARG1.
2316 We assume that the order of the operands for the instruction
2317 is TARG0, TARG1, OP0.
2319 Either TARG0 or TARG1 may be zero, but what that means is that
2320 the result is not actually wanted. We will generate it into
2321 a dummy pseudo-reg and discard it. They may not both be zero.
2323 Returns 1 if this operation can be performed; 0 if not. */
2326 expand_twoval_unop (optab unoptab, rtx op0, rtx targ0, rtx targ1,
2327 int unsignedp)
2329 machine_mode mode = GET_MODE (targ0 ? targ0 : targ1);
2330 enum mode_class mclass;
2331 machine_mode wider_mode;
2332 rtx_insn *entry_last = get_last_insn ();
2333 rtx_insn *last;
2335 mclass = GET_MODE_CLASS (mode);
2337 if (!targ0)
2338 targ0 = gen_reg_rtx (mode);
2339 if (!targ1)
2340 targ1 = gen_reg_rtx (mode);
2342 /* Record where to go back to if we fail. */
2343 last = get_last_insn ();
2345 if (optab_handler (unoptab, mode) != CODE_FOR_nothing)
2347 struct expand_operand ops[3];
2348 enum insn_code icode = optab_handler (unoptab, mode);
2350 create_fixed_operand (&ops[0], targ0);
2351 create_fixed_operand (&ops[1], targ1);
2352 create_convert_operand_from (&ops[2], op0, mode, unsignedp);
2353 if (maybe_expand_insn (icode, 3, ops))
2354 return 1;
2357 /* It can't be done in this mode. Can we do it in a wider mode? */
2359 if (CLASS_HAS_WIDER_MODES_P (mclass))
2361 for (wider_mode = GET_MODE_WIDER_MODE (mode);
2362 wider_mode != VOIDmode;
2363 wider_mode = GET_MODE_WIDER_MODE (wider_mode))
2365 if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing)
2367 rtx t0 = gen_reg_rtx (wider_mode);
2368 rtx t1 = gen_reg_rtx (wider_mode);
2369 rtx cop0 = convert_modes (wider_mode, mode, op0, unsignedp);
2371 if (expand_twoval_unop (unoptab, cop0, t0, t1, unsignedp))
2373 convert_move (targ0, t0, unsignedp);
2374 convert_move (targ1, t1, unsignedp);
2375 return 1;
2377 else
2378 delete_insns_since (last);
2383 delete_insns_since (entry_last);
2384 return 0;
2387 /* Generate code to perform an operation specified by BINOPTAB
2388 on operands OP0 and OP1, with two results to TARG1 and TARG2.
2389 We assume that the order of the operands for the instruction
2390 is TARG0, OP0, OP1, TARG1, which would fit a pattern like
2391 [(set TARG0 (operate OP0 OP1)) (set TARG1 (operate ...))].
2393 Either TARG0 or TARG1 may be zero, but what that means is that
2394 the result is not actually wanted. We will generate it into
2395 a dummy pseudo-reg and discard it. They may not both be zero.
2397 Returns 1 if this operation can be performed; 0 if not. */
2400 expand_twoval_binop (optab binoptab, rtx op0, rtx op1, rtx targ0, rtx targ1,
2401 int unsignedp)
2403 machine_mode mode = GET_MODE (targ0 ? targ0 : targ1);
2404 enum mode_class mclass;
2405 machine_mode wider_mode;
2406 rtx_insn *entry_last = get_last_insn ();
2407 rtx_insn *last;
2409 mclass = GET_MODE_CLASS (mode);
2411 if (!targ0)
2412 targ0 = gen_reg_rtx (mode);
2413 if (!targ1)
2414 targ1 = gen_reg_rtx (mode);
2416 /* Record where to go back to if we fail. */
2417 last = get_last_insn ();
2419 if (optab_handler (binoptab, mode) != CODE_FOR_nothing)
2421 struct expand_operand ops[4];
2422 enum insn_code icode = optab_handler (binoptab, mode);
2423 machine_mode mode0 = insn_data[icode].operand[1].mode;
2424 machine_mode mode1 = insn_data[icode].operand[2].mode;
2425 rtx xop0 = op0, xop1 = op1;
2427 /* If we are optimizing, force expensive constants into a register. */
2428 xop0 = avoid_expensive_constant (mode0, binoptab, 0, xop0, unsignedp);
2429 xop1 = avoid_expensive_constant (mode1, binoptab, 1, xop1, unsignedp);
2431 create_fixed_operand (&ops[0], targ0);
2432 create_convert_operand_from (&ops[1], op0, mode, unsignedp);
2433 create_convert_operand_from (&ops[2], op1, mode, unsignedp);
2434 create_fixed_operand (&ops[3], targ1);
2435 if (maybe_expand_insn (icode, 4, ops))
2436 return 1;
2437 delete_insns_since (last);
2440 /* It can't be done in this mode. Can we do it in a wider mode? */
2442 if (CLASS_HAS_WIDER_MODES_P (mclass))
2444 for (wider_mode = GET_MODE_WIDER_MODE (mode);
2445 wider_mode != VOIDmode;
2446 wider_mode = GET_MODE_WIDER_MODE (wider_mode))
2448 if (optab_handler (binoptab, wider_mode) != CODE_FOR_nothing)
2450 rtx t0 = gen_reg_rtx (wider_mode);
2451 rtx t1 = gen_reg_rtx (wider_mode);
2452 rtx cop0 = convert_modes (wider_mode, mode, op0, unsignedp);
2453 rtx cop1 = convert_modes (wider_mode, mode, op1, unsignedp);
2455 if (expand_twoval_binop (binoptab, cop0, cop1,
2456 t0, t1, unsignedp))
2458 convert_move (targ0, t0, unsignedp);
2459 convert_move (targ1, t1, unsignedp);
2460 return 1;
2462 else
2463 delete_insns_since (last);
2468 delete_insns_since (entry_last);
2469 return 0;
2472 /* Expand the two-valued library call indicated by BINOPTAB, but
2473 preserve only one of the values. If TARG0 is non-NULL, the first
2474 value is placed into TARG0; otherwise the second value is placed
2475 into TARG1. Exactly one of TARG0 and TARG1 must be non-NULL. The
2476 value stored into TARG0 or TARG1 is equivalent to (CODE OP0 OP1).
2477 This routine assumes that the value returned by the library call is
2478 as if the return value was of an integral mode twice as wide as the
2479 mode of OP0. Returns 1 if the call was successful. */
2481 bool
2482 expand_twoval_binop_libfunc (optab binoptab, rtx op0, rtx op1,
2483 rtx targ0, rtx targ1, enum rtx_code code)
2485 machine_mode mode;
2486 machine_mode libval_mode;
2487 rtx libval;
2488 rtx_insn *insns;
2489 rtx libfunc;
2491 /* Exactly one of TARG0 or TARG1 should be non-NULL. */
2492 gcc_assert (!targ0 != !targ1);
2494 mode = GET_MODE (op0);
2495 libfunc = optab_libfunc (binoptab, mode);
2496 if (!libfunc)
2497 return false;
2499 /* The value returned by the library function will have twice as
2500 many bits as the nominal MODE. */
2501 libval_mode = smallest_mode_for_size (2 * GET_MODE_BITSIZE (mode),
2502 MODE_INT);
2503 start_sequence ();
2504 libval = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
2505 libval_mode, 2,
2506 op0, mode,
2507 op1, mode);
2508 /* Get the part of VAL containing the value that we want. */
2509 libval = simplify_gen_subreg (mode, libval, libval_mode,
2510 targ0 ? 0 : GET_MODE_SIZE (mode));
2511 insns = get_insns ();
2512 end_sequence ();
2513 /* Move the into the desired location. */
2514 emit_libcall_block (insns, targ0 ? targ0 : targ1, libval,
2515 gen_rtx_fmt_ee (code, mode, op0, op1));
2517 return true;
2521 /* Wrapper around expand_unop which takes an rtx code to specify
2522 the operation to perform, not an optab pointer. All other
2523 arguments are the same. */
2525 expand_simple_unop (machine_mode mode, enum rtx_code code, rtx op0,
2526 rtx target, int unsignedp)
2528 optab unop = code_to_optab (code);
2529 gcc_assert (unop);
2531 return expand_unop (mode, unop, op0, target, unsignedp);
2534 /* Try calculating
2535 (clz:narrow x)
2537 (clz:wide (zero_extend:wide x)) - ((width wide) - (width narrow)).
2539 A similar operation can be used for clrsb. UNOPTAB says which operation
2540 we are trying to expand. */
2541 static rtx
2542 widen_leading (machine_mode mode, rtx op0, rtx target, optab unoptab)
2544 enum mode_class mclass = GET_MODE_CLASS (mode);
2545 if (CLASS_HAS_WIDER_MODES_P (mclass))
2547 machine_mode wider_mode;
2548 for (wider_mode = GET_MODE_WIDER_MODE (mode);
2549 wider_mode != VOIDmode;
2550 wider_mode = GET_MODE_WIDER_MODE (wider_mode))
2552 if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing)
2554 rtx xop0, temp;
2555 rtx_insn *last;
2557 last = get_last_insn ();
2559 if (target == 0)
2560 target = gen_reg_rtx (mode);
2561 xop0 = widen_operand (op0, wider_mode, mode,
2562 unoptab != clrsb_optab, false);
2563 temp = expand_unop (wider_mode, unoptab, xop0, NULL_RTX,
2564 unoptab != clrsb_optab);
2565 if (temp != 0)
2566 temp = expand_binop
2567 (wider_mode, sub_optab, temp,
2568 gen_int_mode (GET_MODE_PRECISION (wider_mode)
2569 - GET_MODE_PRECISION (mode),
2570 wider_mode),
2571 target, true, OPTAB_DIRECT);
2572 if (temp == 0)
2573 delete_insns_since (last);
2575 return temp;
2579 return 0;
2582 /* Try calculating clz of a double-word quantity as two clz's of word-sized
2583 quantities, choosing which based on whether the high word is nonzero. */
2584 static rtx
2585 expand_doubleword_clz (machine_mode mode, rtx op0, rtx target)
2587 rtx xop0 = force_reg (mode, op0);
2588 rtx subhi = gen_highpart (word_mode, xop0);
2589 rtx sublo = gen_lowpart (word_mode, xop0);
2590 rtx_code_label *hi0_label = gen_label_rtx ();
2591 rtx_code_label *after_label = gen_label_rtx ();
2592 rtx_insn *seq;
2593 rtx temp, result;
2595 /* If we were not given a target, use a word_mode register, not a
2596 'mode' register. The result will fit, and nobody is expecting
2597 anything bigger (the return type of __builtin_clz* is int). */
2598 if (!target)
2599 target = gen_reg_rtx (word_mode);
2601 /* In any case, write to a word_mode scratch in both branches of the
2602 conditional, so we can ensure there is a single move insn setting
2603 'target' to tag a REG_EQUAL note on. */
2604 result = gen_reg_rtx (word_mode);
2606 start_sequence ();
2608 /* If the high word is not equal to zero,
2609 then clz of the full value is clz of the high word. */
2610 emit_cmp_and_jump_insns (subhi, CONST0_RTX (word_mode), EQ, 0,
2611 word_mode, true, hi0_label);
2613 temp = expand_unop_direct (word_mode, clz_optab, subhi, result, true);
2614 if (!temp)
2615 goto fail;
2617 if (temp != result)
2618 convert_move (result, temp, true);
2620 emit_jump_insn (gen_jump (after_label));
2621 emit_barrier ();
2623 /* Else clz of the full value is clz of the low word plus the number
2624 of bits in the high word. */
2625 emit_label (hi0_label);
2627 temp = expand_unop_direct (word_mode, clz_optab, sublo, 0, true);
2628 if (!temp)
2629 goto fail;
2630 temp = expand_binop (word_mode, add_optab, temp,
2631 gen_int_mode (GET_MODE_BITSIZE (word_mode), word_mode),
2632 result, true, OPTAB_DIRECT);
2633 if (!temp)
2634 goto fail;
2635 if (temp != result)
2636 convert_move (result, temp, true);
2638 emit_label (after_label);
2639 convert_move (target, result, true);
2641 seq = get_insns ();
2642 end_sequence ();
2644 add_equal_note (seq, target, CLZ, xop0, 0);
2645 emit_insn (seq);
2646 return target;
2648 fail:
2649 end_sequence ();
2650 return 0;
2653 /* Try calculating
2654 (bswap:narrow x)
2656 (lshiftrt:wide (bswap:wide x) ((width wide) - (width narrow))). */
2657 static rtx
2658 widen_bswap (machine_mode mode, rtx op0, rtx target)
2660 enum mode_class mclass = GET_MODE_CLASS (mode);
2661 machine_mode wider_mode;
2662 rtx x;
2663 rtx_insn *last;
2665 if (!CLASS_HAS_WIDER_MODES_P (mclass))
2666 return NULL_RTX;
2668 for (wider_mode = GET_MODE_WIDER_MODE (mode);
2669 wider_mode != VOIDmode;
2670 wider_mode = GET_MODE_WIDER_MODE (wider_mode))
2671 if (optab_handler (bswap_optab, wider_mode) != CODE_FOR_nothing)
2672 goto found;
2673 return NULL_RTX;
2675 found:
2676 last = get_last_insn ();
2678 x = widen_operand (op0, wider_mode, mode, true, true);
2679 x = expand_unop (wider_mode, bswap_optab, x, NULL_RTX, true);
2681 gcc_assert (GET_MODE_PRECISION (wider_mode) == GET_MODE_BITSIZE (wider_mode)
2682 && GET_MODE_PRECISION (mode) == GET_MODE_BITSIZE (mode));
2683 if (x != 0)
2684 x = expand_shift (RSHIFT_EXPR, wider_mode, x,
2685 GET_MODE_BITSIZE (wider_mode)
2686 - GET_MODE_BITSIZE (mode),
2687 NULL_RTX, true);
2689 if (x != 0)
2691 if (target == 0)
2692 target = gen_reg_rtx (mode);
2693 emit_move_insn (target, gen_lowpart (mode, x));
2695 else
2696 delete_insns_since (last);
2698 return target;
2701 /* Try calculating bswap as two bswaps of two word-sized operands. */
2703 static rtx
2704 expand_doubleword_bswap (machine_mode mode, rtx op, rtx target)
2706 rtx t0, t1;
2708 t1 = expand_unop (word_mode, bswap_optab,
2709 operand_subword_force (op, 0, mode), NULL_RTX, true);
2710 t0 = expand_unop (word_mode, bswap_optab,
2711 operand_subword_force (op, 1, mode), NULL_RTX, true);
2713 if (target == 0 || !valid_multiword_target_p (target))
2714 target = gen_reg_rtx (mode);
2715 if (REG_P (target))
2716 emit_clobber (target);
2717 emit_move_insn (operand_subword (target, 0, 1, mode), t0);
2718 emit_move_insn (operand_subword (target, 1, 1, mode), t1);
2720 return target;
2723 /* Try calculating (parity x) as (and (popcount x) 1), where
2724 popcount can also be done in a wider mode. */
2725 static rtx
2726 expand_parity (machine_mode mode, rtx op0, rtx target)
2728 enum mode_class mclass = GET_MODE_CLASS (mode);
2729 if (CLASS_HAS_WIDER_MODES_P (mclass))
2731 machine_mode wider_mode;
2732 for (wider_mode = mode; wider_mode != VOIDmode;
2733 wider_mode = GET_MODE_WIDER_MODE (wider_mode))
2735 if (optab_handler (popcount_optab, wider_mode) != CODE_FOR_nothing)
2737 rtx xop0, temp;
2738 rtx_insn *last;
2740 last = get_last_insn ();
2742 if (target == 0)
2743 target = gen_reg_rtx (mode);
2744 xop0 = widen_operand (op0, wider_mode, mode, true, false);
2745 temp = expand_unop (wider_mode, popcount_optab, xop0, NULL_RTX,
2746 true);
2747 if (temp != 0)
2748 temp = expand_binop (wider_mode, and_optab, temp, const1_rtx,
2749 target, true, OPTAB_DIRECT);
2750 if (temp == 0)
2751 delete_insns_since (last);
2753 return temp;
2757 return 0;
2760 /* Try calculating ctz(x) as K - clz(x & -x) ,
2761 where K is GET_MODE_PRECISION(mode) - 1.
2763 Both __builtin_ctz and __builtin_clz are undefined at zero, so we
2764 don't have to worry about what the hardware does in that case. (If
2765 the clz instruction produces the usual value at 0, which is K, the
2766 result of this code sequence will be -1; expand_ffs, below, relies
2767 on this. It might be nice to have it be K instead, for consistency
2768 with the (very few) processors that provide a ctz with a defined
2769 value, but that would take one more instruction, and it would be
2770 less convenient for expand_ffs anyway. */
2772 static rtx
2773 expand_ctz (machine_mode mode, rtx op0, rtx target)
2775 rtx_insn *seq;
2776 rtx temp;
2778 if (optab_handler (clz_optab, mode) == CODE_FOR_nothing)
2779 return 0;
2781 start_sequence ();
2783 temp = expand_unop_direct (mode, neg_optab, op0, NULL_RTX, true);
2784 if (temp)
2785 temp = expand_binop (mode, and_optab, op0, temp, NULL_RTX,
2786 true, OPTAB_DIRECT);
2787 if (temp)
2788 temp = expand_unop_direct (mode, clz_optab, temp, NULL_RTX, true);
2789 if (temp)
2790 temp = expand_binop (mode, sub_optab,
2791 gen_int_mode (GET_MODE_PRECISION (mode) - 1, mode),
2792 temp, target,
2793 true, OPTAB_DIRECT);
2794 if (temp == 0)
2796 end_sequence ();
2797 return 0;
2800 seq = get_insns ();
2801 end_sequence ();
2803 add_equal_note (seq, temp, CTZ, op0, 0);
2804 emit_insn (seq);
2805 return temp;
2809 /* Try calculating ffs(x) using ctz(x) if we have that instruction, or
2810 else with the sequence used by expand_clz.
2812 The ffs builtin promises to return zero for a zero value and ctz/clz
2813 may have an undefined value in that case. If they do not give us a
2814 convenient value, we have to generate a test and branch. */
2815 static rtx
2816 expand_ffs (machine_mode mode, rtx op0, rtx target)
2818 HOST_WIDE_INT val = 0;
2819 bool defined_at_zero = false;
2820 rtx temp;
2821 rtx_insn *seq;
2823 if (optab_handler (ctz_optab, mode) != CODE_FOR_nothing)
2825 start_sequence ();
2827 temp = expand_unop_direct (mode, ctz_optab, op0, 0, true);
2828 if (!temp)
2829 goto fail;
2831 defined_at_zero = (CTZ_DEFINED_VALUE_AT_ZERO (mode, val) == 2);
2833 else if (optab_handler (clz_optab, mode) != CODE_FOR_nothing)
2835 start_sequence ();
2836 temp = expand_ctz (mode, op0, 0);
2837 if (!temp)
2838 goto fail;
2840 if (CLZ_DEFINED_VALUE_AT_ZERO (mode, val) == 2)
2842 defined_at_zero = true;
2843 val = (GET_MODE_PRECISION (mode) - 1) - val;
2846 else
2847 return 0;
2849 if (defined_at_zero && val == -1)
2850 /* No correction needed at zero. */;
2851 else
2853 /* We don't try to do anything clever with the situation found
2854 on some processors (eg Alpha) where ctz(0:mode) ==
2855 bitsize(mode). If someone can think of a way to send N to -1
2856 and leave alone all values in the range 0..N-1 (where N is a
2857 power of two), cheaper than this test-and-branch, please add it.
2859 The test-and-branch is done after the operation itself, in case
2860 the operation sets condition codes that can be recycled for this.
2861 (This is true on i386, for instance.) */
2863 rtx_code_label *nonzero_label = gen_label_rtx ();
2864 emit_cmp_and_jump_insns (op0, CONST0_RTX (mode), NE, 0,
2865 mode, true, nonzero_label);
2867 convert_move (temp, GEN_INT (-1), false);
2868 emit_label (nonzero_label);
2871 /* temp now has a value in the range -1..bitsize-1. ffs is supposed
2872 to produce a value in the range 0..bitsize. */
2873 temp = expand_binop (mode, add_optab, temp, gen_int_mode (1, mode),
2874 target, false, OPTAB_DIRECT);
2875 if (!temp)
2876 goto fail;
2878 seq = get_insns ();
2879 end_sequence ();
2881 add_equal_note (seq, temp, FFS, op0, 0);
2882 emit_insn (seq);
2883 return temp;
2885 fail:
2886 end_sequence ();
2887 return 0;
2890 /* Extract the OMODE lowpart from VAL, which has IMODE. Under certain
2891 conditions, VAL may already be a SUBREG against which we cannot generate
2892 a further SUBREG. In this case, we expect forcing the value into a
2893 register will work around the situation. */
2895 static rtx
2896 lowpart_subreg_maybe_copy (machine_mode omode, rtx val,
2897 machine_mode imode)
2899 rtx ret;
2900 ret = lowpart_subreg (omode, val, imode);
2901 if (ret == NULL)
2903 val = force_reg (imode, val);
2904 ret = lowpart_subreg (omode, val, imode);
2905 gcc_assert (ret != NULL);
2907 return ret;
2910 /* Expand a floating point absolute value or negation operation via a
2911 logical operation on the sign bit. */
2913 static rtx
2914 expand_absneg_bit (enum rtx_code code, machine_mode mode,
2915 rtx op0, rtx target)
2917 const struct real_format *fmt;
2918 int bitpos, word, nwords, i;
2919 machine_mode imode;
2920 rtx temp;
2921 rtx_insn *insns;
2923 /* The format has to have a simple sign bit. */
2924 fmt = REAL_MODE_FORMAT (mode);
2925 if (fmt == NULL)
2926 return NULL_RTX;
2928 bitpos = fmt->signbit_rw;
2929 if (bitpos < 0)
2930 return NULL_RTX;
2932 /* Don't create negative zeros if the format doesn't support them. */
2933 if (code == NEG && !fmt->has_signed_zero)
2934 return NULL_RTX;
2936 if (GET_MODE_SIZE (mode) <= UNITS_PER_WORD)
2938 imode = int_mode_for_mode (mode);
2939 if (imode == BLKmode)
2940 return NULL_RTX;
2941 word = 0;
2942 nwords = 1;
2944 else
2946 imode = word_mode;
2948 if (FLOAT_WORDS_BIG_ENDIAN)
2949 word = (GET_MODE_BITSIZE (mode) - bitpos) / BITS_PER_WORD;
2950 else
2951 word = bitpos / BITS_PER_WORD;
2952 bitpos = bitpos % BITS_PER_WORD;
2953 nwords = (GET_MODE_BITSIZE (mode) + BITS_PER_WORD - 1) / BITS_PER_WORD;
2956 wide_int mask = wi::set_bit_in_zero (bitpos, GET_MODE_PRECISION (imode));
2957 if (code == ABS)
2958 mask = ~mask;
2960 if (target == 0
2961 || target == op0
2962 || (nwords > 1 && !valid_multiword_target_p (target)))
2963 target = gen_reg_rtx (mode);
2965 if (nwords > 1)
2967 start_sequence ();
2969 for (i = 0; i < nwords; ++i)
2971 rtx targ_piece = operand_subword (target, i, 1, mode);
2972 rtx op0_piece = operand_subword_force (op0, i, mode);
2974 if (i == word)
2976 temp = expand_binop (imode, code == ABS ? and_optab : xor_optab,
2977 op0_piece,
2978 immed_wide_int_const (mask, imode),
2979 targ_piece, 1, OPTAB_LIB_WIDEN);
2980 if (temp != targ_piece)
2981 emit_move_insn (targ_piece, temp);
2983 else
2984 emit_move_insn (targ_piece, op0_piece);
2987 insns = get_insns ();
2988 end_sequence ();
2990 emit_insn (insns);
2992 else
2994 temp = expand_binop (imode, code == ABS ? and_optab : xor_optab,
2995 gen_lowpart (imode, op0),
2996 immed_wide_int_const (mask, imode),
2997 gen_lowpart (imode, target), 1, OPTAB_LIB_WIDEN);
2998 target = lowpart_subreg_maybe_copy (mode, temp, imode);
3000 set_dst_reg_note (get_last_insn (), REG_EQUAL,
3001 gen_rtx_fmt_e (code, mode, copy_rtx (op0)),
3002 target);
3005 return target;
3008 /* As expand_unop, but will fail rather than attempt the operation in a
3009 different mode or with a libcall. */
3010 static rtx
3011 expand_unop_direct (machine_mode mode, optab unoptab, rtx op0, rtx target,
3012 int unsignedp)
3014 if (optab_handler (unoptab, mode) != CODE_FOR_nothing)
3016 struct expand_operand ops[2];
3017 enum insn_code icode = optab_handler (unoptab, mode);
3018 rtx_insn *last = get_last_insn ();
3019 rtx pat;
3021 create_output_operand (&ops[0], target, mode);
3022 create_convert_operand_from (&ops[1], op0, mode, unsignedp);
3023 pat = maybe_gen_insn (icode, 2, ops);
3024 if (pat)
3026 if (INSN_P (pat) && NEXT_INSN (as_a <rtx_insn *> (pat)) != NULL_RTX
3027 && ! add_equal_note (as_a <rtx_insn *> (pat), ops[0].value,
3028 optab_to_code (unoptab),
3029 ops[1].value, NULL_RTX))
3031 delete_insns_since (last);
3032 return expand_unop (mode, unoptab, op0, NULL_RTX, unsignedp);
3035 emit_insn (pat);
3037 return ops[0].value;
3040 return 0;
3043 /* Generate code to perform an operation specified by UNOPTAB
3044 on operand OP0, with result having machine-mode MODE.
3046 UNSIGNEDP is for the case where we have to widen the operands
3047 to perform the operation. It says to use zero-extension.
3049 If TARGET is nonzero, the value
3050 is generated there, if it is convenient to do so.
3051 In all cases an rtx is returned for the locus of the value;
3052 this may or may not be TARGET. */
3055 expand_unop (machine_mode mode, optab unoptab, rtx op0, rtx target,
3056 int unsignedp)
3058 enum mode_class mclass = GET_MODE_CLASS (mode);
3059 machine_mode wider_mode;
3060 rtx temp;
3061 rtx libfunc;
3063 temp = expand_unop_direct (mode, unoptab, op0, target, unsignedp);
3064 if (temp)
3065 return temp;
3067 /* It can't be done in this mode. Can we open-code it in a wider mode? */
3069 /* Widening (or narrowing) clz needs special treatment. */
3070 if (unoptab == clz_optab)
3072 temp = widen_leading (mode, op0, target, unoptab);
3073 if (temp)
3074 return temp;
3076 if (GET_MODE_SIZE (mode) == 2 * UNITS_PER_WORD
3077 && optab_handler (unoptab, word_mode) != CODE_FOR_nothing)
3079 temp = expand_doubleword_clz (mode, op0, target);
3080 if (temp)
3081 return temp;
3084 goto try_libcall;
3087 if (unoptab == clrsb_optab)
3089 temp = widen_leading (mode, op0, target, unoptab);
3090 if (temp)
3091 return temp;
3092 goto try_libcall;
3095 /* Widening (or narrowing) bswap needs special treatment. */
3096 if (unoptab == bswap_optab)
3098 /* HImode is special because in this mode BSWAP is equivalent to ROTATE
3099 or ROTATERT. First try these directly; if this fails, then try the
3100 obvious pair of shifts with allowed widening, as this will probably
3101 be always more efficient than the other fallback methods. */
3102 if (mode == HImode)
3104 rtx_insn *last;
3105 rtx temp1, temp2;
3107 if (optab_handler (rotl_optab, mode) != CODE_FOR_nothing)
3109 temp = expand_binop (mode, rotl_optab, op0, GEN_INT (8), target,
3110 unsignedp, OPTAB_DIRECT);
3111 if (temp)
3112 return temp;
3115 if (optab_handler (rotr_optab, mode) != CODE_FOR_nothing)
3117 temp = expand_binop (mode, rotr_optab, op0, GEN_INT (8), target,
3118 unsignedp, OPTAB_DIRECT);
3119 if (temp)
3120 return temp;
3123 last = get_last_insn ();
3125 temp1 = expand_binop (mode, ashl_optab, op0, GEN_INT (8), NULL_RTX,
3126 unsignedp, OPTAB_WIDEN);
3127 temp2 = expand_binop (mode, lshr_optab, op0, GEN_INT (8), NULL_RTX,
3128 unsignedp, OPTAB_WIDEN);
3129 if (temp1 && temp2)
3131 temp = expand_binop (mode, ior_optab, temp1, temp2, target,
3132 unsignedp, OPTAB_WIDEN);
3133 if (temp)
3134 return temp;
3137 delete_insns_since (last);
3140 temp = widen_bswap (mode, op0, target);
3141 if (temp)
3142 return temp;
3144 if (GET_MODE_SIZE (mode) == 2 * UNITS_PER_WORD
3145 && optab_handler (unoptab, word_mode) != CODE_FOR_nothing)
3147 temp = expand_doubleword_bswap (mode, op0, target);
3148 if (temp)
3149 return temp;
3152 goto try_libcall;
3155 if (CLASS_HAS_WIDER_MODES_P (mclass))
3156 for (wider_mode = GET_MODE_WIDER_MODE (mode);
3157 wider_mode != VOIDmode;
3158 wider_mode = GET_MODE_WIDER_MODE (wider_mode))
3160 if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing)
3162 rtx xop0 = op0;
3163 rtx_insn *last = get_last_insn ();
3165 /* For certain operations, we need not actually extend
3166 the narrow operand, as long as we will truncate the
3167 results to the same narrowness. */
3169 xop0 = widen_operand (xop0, wider_mode, mode, unsignedp,
3170 (unoptab == neg_optab
3171 || unoptab == one_cmpl_optab)
3172 && mclass == MODE_INT);
3174 temp = expand_unop (wider_mode, unoptab, xop0, NULL_RTX,
3175 unsignedp);
3177 if (temp)
3179 if (mclass != MODE_INT
3180 || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode))
3182 if (target == 0)
3183 target = gen_reg_rtx (mode);
3184 convert_move (target, temp, 0);
3185 return target;
3187 else
3188 return gen_lowpart (mode, temp);
3190 else
3191 delete_insns_since (last);
3195 /* These can be done a word at a time. */
3196 if (unoptab == one_cmpl_optab
3197 && mclass == MODE_INT
3198 && GET_MODE_SIZE (mode) > UNITS_PER_WORD
3199 && optab_handler (unoptab, word_mode) != CODE_FOR_nothing)
3201 int i;
3202 rtx_insn *insns;
3204 if (target == 0 || target == op0 || !valid_multiword_target_p (target))
3205 target = gen_reg_rtx (mode);
3207 start_sequence ();
3209 /* Do the actual arithmetic. */
3210 for (i = 0; i < GET_MODE_BITSIZE (mode) / BITS_PER_WORD; i++)
3212 rtx target_piece = operand_subword (target, i, 1, mode);
3213 rtx x = expand_unop (word_mode, unoptab,
3214 operand_subword_force (op0, i, mode),
3215 target_piece, unsignedp);
3217 if (target_piece != x)
3218 emit_move_insn (target_piece, x);
3221 insns = get_insns ();
3222 end_sequence ();
3224 emit_insn (insns);
3225 return target;
3228 if (optab_to_code (unoptab) == NEG)
3230 /* Try negating floating point values by flipping the sign bit. */
3231 if (SCALAR_FLOAT_MODE_P (mode))
3233 temp = expand_absneg_bit (NEG, mode, op0, target);
3234 if (temp)
3235 return temp;
3238 /* If there is no negation pattern, and we have no negative zero,
3239 try subtracting from zero. */
3240 if (!HONOR_SIGNED_ZEROS (mode))
3242 temp = expand_binop (mode, (unoptab == negv_optab
3243 ? subv_optab : sub_optab),
3244 CONST0_RTX (mode), op0, target,
3245 unsignedp, OPTAB_DIRECT);
3246 if (temp)
3247 return temp;
3251 /* Try calculating parity (x) as popcount (x) % 2. */
3252 if (unoptab == parity_optab)
3254 temp = expand_parity (mode, op0, target);
3255 if (temp)
3256 return temp;
3259 /* Try implementing ffs (x) in terms of clz (x). */
3260 if (unoptab == ffs_optab)
3262 temp = expand_ffs (mode, op0, target);
3263 if (temp)
3264 return temp;
3267 /* Try implementing ctz (x) in terms of clz (x). */
3268 if (unoptab == ctz_optab)
3270 temp = expand_ctz (mode, op0, target);
3271 if (temp)
3272 return temp;
3275 try_libcall:
3276 /* Now try a library call in this mode. */
3277 libfunc = optab_libfunc (unoptab, mode);
3278 if (libfunc)
3280 rtx_insn *insns;
3281 rtx value;
3282 rtx eq_value;
3283 machine_mode outmode = mode;
3285 /* All of these functions return small values. Thus we choose to
3286 have them return something that isn't a double-word. */
3287 if (unoptab == ffs_optab || unoptab == clz_optab || unoptab == ctz_optab
3288 || unoptab == clrsb_optab || unoptab == popcount_optab
3289 || unoptab == parity_optab)
3290 outmode
3291 = GET_MODE (hard_libcall_value (TYPE_MODE (integer_type_node),
3292 optab_libfunc (unoptab, mode)));
3294 start_sequence ();
3296 /* Pass 1 for NO_QUEUE so we don't lose any increments
3297 if the libcall is cse'd or moved. */
3298 value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST, outmode,
3299 1, op0, mode);
3300 insns = get_insns ();
3301 end_sequence ();
3303 target = gen_reg_rtx (outmode);
3304 eq_value = gen_rtx_fmt_e (optab_to_code (unoptab), mode, op0);
3305 if (GET_MODE_SIZE (outmode) < GET_MODE_SIZE (mode))
3306 eq_value = simplify_gen_unary (TRUNCATE, outmode, eq_value, mode);
3307 else if (GET_MODE_SIZE (outmode) > GET_MODE_SIZE (mode))
3308 eq_value = simplify_gen_unary (ZERO_EXTEND, outmode, eq_value, mode);
3309 emit_libcall_block_1 (insns, target, value, eq_value,
3310 trapv_unoptab_p (unoptab));
3312 return target;
3315 /* It can't be done in this mode. Can we do it in a wider mode? */
3317 if (CLASS_HAS_WIDER_MODES_P (mclass))
3319 for (wider_mode = GET_MODE_WIDER_MODE (mode);
3320 wider_mode != VOIDmode;
3321 wider_mode = GET_MODE_WIDER_MODE (wider_mode))
3323 if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing
3324 || optab_libfunc (unoptab, wider_mode))
3326 rtx xop0 = op0;
3327 rtx_insn *last = get_last_insn ();
3329 /* For certain operations, we need not actually extend
3330 the narrow operand, as long as we will truncate the
3331 results to the same narrowness. */
3332 xop0 = widen_operand (xop0, wider_mode, mode, unsignedp,
3333 (unoptab == neg_optab
3334 || unoptab == one_cmpl_optab
3335 || unoptab == bswap_optab)
3336 && mclass == MODE_INT);
3338 temp = expand_unop (wider_mode, unoptab, xop0, NULL_RTX,
3339 unsignedp);
3341 /* If we are generating clz using wider mode, adjust the
3342 result. Similarly for clrsb. */
3343 if ((unoptab == clz_optab || unoptab == clrsb_optab)
3344 && temp != 0)
3345 temp = expand_binop
3346 (wider_mode, sub_optab, temp,
3347 gen_int_mode (GET_MODE_PRECISION (wider_mode)
3348 - GET_MODE_PRECISION (mode),
3349 wider_mode),
3350 target, true, OPTAB_DIRECT);
3352 /* Likewise for bswap. */
3353 if (unoptab == bswap_optab && temp != 0)
3355 gcc_assert (GET_MODE_PRECISION (wider_mode)
3356 == GET_MODE_BITSIZE (wider_mode)
3357 && GET_MODE_PRECISION (mode)
3358 == GET_MODE_BITSIZE (mode));
3360 temp = expand_shift (RSHIFT_EXPR, wider_mode, temp,
3361 GET_MODE_BITSIZE (wider_mode)
3362 - GET_MODE_BITSIZE (mode),
3363 NULL_RTX, true);
3366 if (temp)
3368 if (mclass != MODE_INT)
3370 if (target == 0)
3371 target = gen_reg_rtx (mode);
3372 convert_move (target, temp, 0);
3373 return target;
3375 else
3376 return gen_lowpart (mode, temp);
3378 else
3379 delete_insns_since (last);
3384 /* One final attempt at implementing negation via subtraction,
3385 this time allowing widening of the operand. */
3386 if (optab_to_code (unoptab) == NEG && !HONOR_SIGNED_ZEROS (mode))
3388 rtx temp;
3389 temp = expand_binop (mode,
3390 unoptab == negv_optab ? subv_optab : sub_optab,
3391 CONST0_RTX (mode), op0,
3392 target, unsignedp, OPTAB_LIB_WIDEN);
3393 if (temp)
3394 return temp;
3397 return 0;
3400 /* Emit code to compute the absolute value of OP0, with result to
3401 TARGET if convenient. (TARGET may be 0.) The return value says
3402 where the result actually is to be found.
3404 MODE is the mode of the operand; the mode of the result is
3405 different but can be deduced from MODE.
3410 expand_abs_nojump (machine_mode mode, rtx op0, rtx target,
3411 int result_unsignedp)
3413 rtx temp;
3415 if (GET_MODE_CLASS (mode) != MODE_INT
3416 || ! flag_trapv)
3417 result_unsignedp = 1;
3419 /* First try to do it with a special abs instruction. */
3420 temp = expand_unop (mode, result_unsignedp ? abs_optab : absv_optab,
3421 op0, target, 0);
3422 if (temp != 0)
3423 return temp;
3425 /* For floating point modes, try clearing the sign bit. */
3426 if (SCALAR_FLOAT_MODE_P (mode))
3428 temp = expand_absneg_bit (ABS, mode, op0, target);
3429 if (temp)
3430 return temp;
3433 /* If we have a MAX insn, we can do this as MAX (x, -x). */
3434 if (optab_handler (smax_optab, mode) != CODE_FOR_nothing
3435 && !HONOR_SIGNED_ZEROS (mode))
3437 rtx_insn *last = get_last_insn ();
3439 temp = expand_unop (mode, result_unsignedp ? neg_optab : negv_optab,
3440 op0, NULL_RTX, 0);
3441 if (temp != 0)
3442 temp = expand_binop (mode, smax_optab, op0, temp, target, 0,
3443 OPTAB_WIDEN);
3445 if (temp != 0)
3446 return temp;
3448 delete_insns_since (last);
3451 /* If this machine has expensive jumps, we can do integer absolute
3452 value of X as (((signed) x >> (W-1)) ^ x) - ((signed) x >> (W-1)),
3453 where W is the width of MODE. */
3455 if (GET_MODE_CLASS (mode) == MODE_INT
3456 && BRANCH_COST (optimize_insn_for_speed_p (),
3457 false) >= 2)
3459 rtx extended = expand_shift (RSHIFT_EXPR, mode, op0,
3460 GET_MODE_PRECISION (mode) - 1,
3461 NULL_RTX, 0);
3463 temp = expand_binop (mode, xor_optab, extended, op0, target, 0,
3464 OPTAB_LIB_WIDEN);
3465 if (temp != 0)
3466 temp = expand_binop (mode, result_unsignedp ? sub_optab : subv_optab,
3467 temp, extended, target, 0, OPTAB_LIB_WIDEN);
3469 if (temp != 0)
3470 return temp;
3473 return NULL_RTX;
3477 expand_abs (machine_mode mode, rtx op0, rtx target,
3478 int result_unsignedp, int safe)
3480 rtx temp;
3481 rtx_code_label *op1;
3483 if (GET_MODE_CLASS (mode) != MODE_INT
3484 || ! flag_trapv)
3485 result_unsignedp = 1;
3487 temp = expand_abs_nojump (mode, op0, target, result_unsignedp);
3488 if (temp != 0)
3489 return temp;
3491 /* If that does not win, use conditional jump and negate. */
3493 /* It is safe to use the target if it is the same
3494 as the source if this is also a pseudo register */
3495 if (op0 == target && REG_P (op0)
3496 && REGNO (op0) >= FIRST_PSEUDO_REGISTER)
3497 safe = 1;
3499 op1 = gen_label_rtx ();
3500 if (target == 0 || ! safe
3501 || GET_MODE (target) != mode
3502 || (MEM_P (target) && MEM_VOLATILE_P (target))
3503 || (REG_P (target)
3504 && REGNO (target) < FIRST_PSEUDO_REGISTER))
3505 target = gen_reg_rtx (mode);
3507 emit_move_insn (target, op0);
3508 NO_DEFER_POP;
3510 do_compare_rtx_and_jump (target, CONST0_RTX (mode), GE, 0, mode,
3511 NULL_RTX, NULL_RTX, op1, -1);
3513 op0 = expand_unop (mode, result_unsignedp ? neg_optab : negv_optab,
3514 target, target, 0);
3515 if (op0 != target)
3516 emit_move_insn (target, op0);
3517 emit_label (op1);
3518 OK_DEFER_POP;
3519 return target;
3522 /* Emit code to compute the one's complement absolute value of OP0
3523 (if (OP0 < 0) OP0 = ~OP0), with result to TARGET if convenient.
3524 (TARGET may be NULL_RTX.) The return value says where the result
3525 actually is to be found.
3527 MODE is the mode of the operand; the mode of the result is
3528 different but can be deduced from MODE. */
3531 expand_one_cmpl_abs_nojump (machine_mode mode, rtx op0, rtx target)
3533 rtx temp;
3535 /* Not applicable for floating point modes. */
3536 if (FLOAT_MODE_P (mode))
3537 return NULL_RTX;
3539 /* If we have a MAX insn, we can do this as MAX (x, ~x). */
3540 if (optab_handler (smax_optab, mode) != CODE_FOR_nothing)
3542 rtx_insn *last = get_last_insn ();
3544 temp = expand_unop (mode, one_cmpl_optab, op0, NULL_RTX, 0);
3545 if (temp != 0)
3546 temp = expand_binop (mode, smax_optab, op0, temp, target, 0,
3547 OPTAB_WIDEN);
3549 if (temp != 0)
3550 return temp;
3552 delete_insns_since (last);
3555 /* If this machine has expensive jumps, we can do one's complement
3556 absolute value of X as (((signed) x >> (W-1)) ^ x). */
3558 if (GET_MODE_CLASS (mode) == MODE_INT
3559 && BRANCH_COST (optimize_insn_for_speed_p (),
3560 false) >= 2)
3562 rtx extended = expand_shift (RSHIFT_EXPR, mode, op0,
3563 GET_MODE_PRECISION (mode) - 1,
3564 NULL_RTX, 0);
3566 temp = expand_binop (mode, xor_optab, extended, op0, target, 0,
3567 OPTAB_LIB_WIDEN);
3569 if (temp != 0)
3570 return temp;
3573 return NULL_RTX;
3576 /* A subroutine of expand_copysign, perform the copysign operation using the
3577 abs and neg primitives advertised to exist on the target. The assumption
3578 is that we have a split register file, and leaving op0 in fp registers,
3579 and not playing with subregs so much, will help the register allocator. */
3581 static rtx
3582 expand_copysign_absneg (machine_mode mode, rtx op0, rtx op1, rtx target,
3583 int bitpos, bool op0_is_abs)
3585 machine_mode imode;
3586 enum insn_code icode;
3587 rtx sign;
3588 rtx_code_label *label;
3590 if (target == op1)
3591 target = NULL_RTX;
3593 /* Check if the back end provides an insn that handles signbit for the
3594 argument's mode. */
3595 icode = optab_handler (signbit_optab, mode);
3596 if (icode != CODE_FOR_nothing)
3598 imode = insn_data[(int) icode].operand[0].mode;
3599 sign = gen_reg_rtx (imode);
3600 emit_unop_insn (icode, sign, op1, UNKNOWN);
3602 else
3604 if (GET_MODE_SIZE (mode) <= UNITS_PER_WORD)
3606 imode = int_mode_for_mode (mode);
3607 if (imode == BLKmode)
3608 return NULL_RTX;
3609 op1 = gen_lowpart (imode, op1);
3611 else
3613 int word;
3615 imode = word_mode;
3616 if (FLOAT_WORDS_BIG_ENDIAN)
3617 word = (GET_MODE_BITSIZE (mode) - bitpos) / BITS_PER_WORD;
3618 else
3619 word = bitpos / BITS_PER_WORD;
3620 bitpos = bitpos % BITS_PER_WORD;
3621 op1 = operand_subword_force (op1, word, mode);
3624 wide_int mask = wi::set_bit_in_zero (bitpos, GET_MODE_PRECISION (imode));
3625 sign = expand_binop (imode, and_optab, op1,
3626 immed_wide_int_const (mask, imode),
3627 NULL_RTX, 1, OPTAB_LIB_WIDEN);
3630 if (!op0_is_abs)
3632 op0 = expand_unop (mode, abs_optab, op0, target, 0);
3633 if (op0 == NULL)
3634 return NULL_RTX;
3635 target = op0;
3637 else
3639 if (target == NULL_RTX)
3640 target = copy_to_reg (op0);
3641 else
3642 emit_move_insn (target, op0);
3645 label = gen_label_rtx ();
3646 emit_cmp_and_jump_insns (sign, const0_rtx, EQ, NULL_RTX, imode, 1, label);
3648 if (CONST_DOUBLE_AS_FLOAT_P (op0))
3649 op0 = simplify_unary_operation (NEG, mode, op0, mode);
3650 else
3651 op0 = expand_unop (mode, neg_optab, op0, target, 0);
3652 if (op0 != target)
3653 emit_move_insn (target, op0);
3655 emit_label (label);
3657 return target;
3661 /* A subroutine of expand_copysign, perform the entire copysign operation
3662 with integer bitmasks. BITPOS is the position of the sign bit; OP0_IS_ABS
3663 is true if op0 is known to have its sign bit clear. */
3665 static rtx
3666 expand_copysign_bit (machine_mode mode, rtx op0, rtx op1, rtx target,
3667 int bitpos, bool op0_is_abs)
3669 machine_mode imode;
3670 int word, nwords, i;
3671 rtx temp;
3672 rtx_insn *insns;
3674 if (GET_MODE_SIZE (mode) <= UNITS_PER_WORD)
3676 imode = int_mode_for_mode (mode);
3677 if (imode == BLKmode)
3678 return NULL_RTX;
3679 word = 0;
3680 nwords = 1;
3682 else
3684 imode = word_mode;
3686 if (FLOAT_WORDS_BIG_ENDIAN)
3687 word = (GET_MODE_BITSIZE (mode) - bitpos) / BITS_PER_WORD;
3688 else
3689 word = bitpos / BITS_PER_WORD;
3690 bitpos = bitpos % BITS_PER_WORD;
3691 nwords = (GET_MODE_BITSIZE (mode) + BITS_PER_WORD - 1) / BITS_PER_WORD;
3694 wide_int mask = wi::set_bit_in_zero (bitpos, GET_MODE_PRECISION (imode));
3696 if (target == 0
3697 || target == op0
3698 || target == op1
3699 || (nwords > 1 && !valid_multiword_target_p (target)))
3700 target = gen_reg_rtx (mode);
3702 if (nwords > 1)
3704 start_sequence ();
3706 for (i = 0; i < nwords; ++i)
3708 rtx targ_piece = operand_subword (target, i, 1, mode);
3709 rtx op0_piece = operand_subword_force (op0, i, mode);
3711 if (i == word)
3713 if (!op0_is_abs)
3714 op0_piece
3715 = expand_binop (imode, and_optab, op0_piece,
3716 immed_wide_int_const (~mask, imode),
3717 NULL_RTX, 1, OPTAB_LIB_WIDEN);
3718 op1 = expand_binop (imode, and_optab,
3719 operand_subword_force (op1, i, mode),
3720 immed_wide_int_const (mask, imode),
3721 NULL_RTX, 1, OPTAB_LIB_WIDEN);
3723 temp = expand_binop (imode, ior_optab, op0_piece, op1,
3724 targ_piece, 1, OPTAB_LIB_WIDEN);
3725 if (temp != targ_piece)
3726 emit_move_insn (targ_piece, temp);
3728 else
3729 emit_move_insn (targ_piece, op0_piece);
3732 insns = get_insns ();
3733 end_sequence ();
3735 emit_insn (insns);
3737 else
3739 op1 = expand_binop (imode, and_optab, gen_lowpart (imode, op1),
3740 immed_wide_int_const (mask, imode),
3741 NULL_RTX, 1, OPTAB_LIB_WIDEN);
3743 op0 = gen_lowpart (imode, op0);
3744 if (!op0_is_abs)
3745 op0 = expand_binop (imode, and_optab, op0,
3746 immed_wide_int_const (~mask, imode),
3747 NULL_RTX, 1, OPTAB_LIB_WIDEN);
3749 temp = expand_binop (imode, ior_optab, op0, op1,
3750 gen_lowpart (imode, target), 1, OPTAB_LIB_WIDEN);
3751 target = lowpart_subreg_maybe_copy (mode, temp, imode);
3754 return target;
3757 /* Expand the C99 copysign operation. OP0 and OP1 must be the same
3758 scalar floating point mode. Return NULL if we do not know how to
3759 expand the operation inline. */
3762 expand_copysign (rtx op0, rtx op1, rtx target)
3764 machine_mode mode = GET_MODE (op0);
3765 const struct real_format *fmt;
3766 bool op0_is_abs;
3767 rtx temp;
3769 gcc_assert (SCALAR_FLOAT_MODE_P (mode));
3770 gcc_assert (GET_MODE (op1) == mode);
3772 /* First try to do it with a special instruction. */
3773 temp = expand_binop (mode, copysign_optab, op0, op1,
3774 target, 0, OPTAB_DIRECT);
3775 if (temp)
3776 return temp;
3778 fmt = REAL_MODE_FORMAT (mode);
3779 if (fmt == NULL || !fmt->has_signed_zero)
3780 return NULL_RTX;
3782 op0_is_abs = false;
3783 if (CONST_DOUBLE_AS_FLOAT_P (op0))
3785 if (real_isneg (CONST_DOUBLE_REAL_VALUE (op0)))
3786 op0 = simplify_unary_operation (ABS, mode, op0, mode);
3787 op0_is_abs = true;
3790 if (fmt->signbit_ro >= 0
3791 && (CONST_DOUBLE_AS_FLOAT_P (op0)
3792 || (optab_handler (neg_optab, mode) != CODE_FOR_nothing
3793 && optab_handler (abs_optab, mode) != CODE_FOR_nothing)))
3795 temp = expand_copysign_absneg (mode, op0, op1, target,
3796 fmt->signbit_ro, op0_is_abs);
3797 if (temp)
3798 return temp;
3801 if (fmt->signbit_rw < 0)
3802 return NULL_RTX;
3803 return expand_copysign_bit (mode, op0, op1, target,
3804 fmt->signbit_rw, op0_is_abs);
3807 /* Generate an instruction whose insn-code is INSN_CODE,
3808 with two operands: an output TARGET and an input OP0.
3809 TARGET *must* be nonzero, and the output is always stored there.
3810 CODE is an rtx code such that (CODE OP0) is an rtx that describes
3811 the value that is stored into TARGET.
3813 Return false if expansion failed. */
3815 bool
3816 maybe_emit_unop_insn (enum insn_code icode, rtx target, rtx op0,
3817 enum rtx_code code)
3819 struct expand_operand ops[2];
3820 rtx pat;
3822 create_output_operand (&ops[0], target, GET_MODE (target));
3823 create_input_operand (&ops[1], op0, GET_MODE (op0));
3824 pat = maybe_gen_insn (icode, 2, ops);
3825 if (!pat)
3826 return false;
3828 if (INSN_P (pat) && NEXT_INSN (as_a <rtx_insn *> (pat)) != NULL_RTX
3829 && code != UNKNOWN)
3830 add_equal_note (as_a <rtx_insn *> (pat), ops[0].value, code, ops[1].value,
3831 NULL_RTX);
3833 emit_insn (pat);
3835 if (ops[0].value != target)
3836 emit_move_insn (target, ops[0].value);
3837 return true;
3839 /* Generate an instruction whose insn-code is INSN_CODE,
3840 with two operands: an output TARGET and an input OP0.
3841 TARGET *must* be nonzero, and the output is always stored there.
3842 CODE is an rtx code such that (CODE OP0) is an rtx that describes
3843 the value that is stored into TARGET. */
3845 void
3846 emit_unop_insn (enum insn_code icode, rtx target, rtx op0, enum rtx_code code)
3848 bool ok = maybe_emit_unop_insn (icode, target, op0, code);
3849 gcc_assert (ok);
3852 struct no_conflict_data
3854 rtx target;
3855 rtx_insn *first, *insn;
3856 bool must_stay;
3859 /* Called via note_stores by emit_libcall_block. Set P->must_stay if
3860 the currently examined clobber / store has to stay in the list of
3861 insns that constitute the actual libcall block. */
3862 static void
3863 no_conflict_move_test (rtx dest, const_rtx set, void *p0)
3865 struct no_conflict_data *p= (struct no_conflict_data *) p0;
3867 /* If this inns directly contributes to setting the target, it must stay. */
3868 if (reg_overlap_mentioned_p (p->target, dest))
3869 p->must_stay = true;
3870 /* If we haven't committed to keeping any other insns in the list yet,
3871 there is nothing more to check. */
3872 else if (p->insn == p->first)
3873 return;
3874 /* If this insn sets / clobbers a register that feeds one of the insns
3875 already in the list, this insn has to stay too. */
3876 else if (reg_overlap_mentioned_p (dest, PATTERN (p->first))
3877 || (CALL_P (p->first) && (find_reg_fusage (p->first, USE, dest)))
3878 || reg_used_between_p (dest, p->first, p->insn)
3879 /* Likewise if this insn depends on a register set by a previous
3880 insn in the list, or if it sets a result (presumably a hard
3881 register) that is set or clobbered by a previous insn.
3882 N.B. the modified_*_p (SET_DEST...) tests applied to a MEM
3883 SET_DEST perform the former check on the address, and the latter
3884 check on the MEM. */
3885 || (GET_CODE (set) == SET
3886 && (modified_in_p (SET_SRC (set), p->first)
3887 || modified_in_p (SET_DEST (set), p->first)
3888 || modified_between_p (SET_SRC (set), p->first, p->insn)
3889 || modified_between_p (SET_DEST (set), p->first, p->insn))))
3890 p->must_stay = true;
3894 /* Emit code to make a call to a constant function or a library call.
3896 INSNS is a list containing all insns emitted in the call.
3897 These insns leave the result in RESULT. Our block is to copy RESULT
3898 to TARGET, which is logically equivalent to EQUIV.
3900 We first emit any insns that set a pseudo on the assumption that these are
3901 loading constants into registers; doing so allows them to be safely cse'ed
3902 between blocks. Then we emit all the other insns in the block, followed by
3903 an insn to move RESULT to TARGET. This last insn will have a REQ_EQUAL
3904 note with an operand of EQUIV. */
3906 static void
3907 emit_libcall_block_1 (rtx_insn *insns, rtx target, rtx result, rtx equiv,
3908 bool equiv_may_trap)
3910 rtx final_dest = target;
3911 rtx_insn *next, *last, *insn;
3913 /* If this is a reg with REG_USERVAR_P set, then it could possibly turn
3914 into a MEM later. Protect the libcall block from this change. */
3915 if (! REG_P (target) || REG_USERVAR_P (target))
3916 target = gen_reg_rtx (GET_MODE (target));
3918 /* If we're using non-call exceptions, a libcall corresponding to an
3919 operation that may trap may also trap. */
3920 /* ??? See the comment in front of make_reg_eh_region_note. */
3921 if (cfun->can_throw_non_call_exceptions
3922 && (equiv_may_trap || may_trap_p (equiv)))
3924 for (insn = insns; insn; insn = NEXT_INSN (insn))
3925 if (CALL_P (insn))
3927 rtx note = find_reg_note (insn, REG_EH_REGION, NULL_RTX);
3928 if (note)
3930 int lp_nr = INTVAL (XEXP (note, 0));
3931 if (lp_nr == 0 || lp_nr == INT_MIN)
3932 remove_note (insn, note);
3936 else
3938 /* Look for any CALL_INSNs in this sequence, and attach a REG_EH_REGION
3939 reg note to indicate that this call cannot throw or execute a nonlocal
3940 goto (unless there is already a REG_EH_REGION note, in which case
3941 we update it). */
3942 for (insn = insns; insn; insn = NEXT_INSN (insn))
3943 if (CALL_P (insn))
3944 make_reg_eh_region_note_nothrow_nononlocal (insn);
3947 /* First emit all insns that set pseudos. Remove them from the list as
3948 we go. Avoid insns that set pseudos which were referenced in previous
3949 insns. These can be generated by move_by_pieces, for example,
3950 to update an address. Similarly, avoid insns that reference things
3951 set in previous insns. */
3953 for (insn = insns; insn; insn = next)
3955 rtx set = single_set (insn);
3957 next = NEXT_INSN (insn);
3959 if (set != 0 && REG_P (SET_DEST (set))
3960 && REGNO (SET_DEST (set)) >= FIRST_PSEUDO_REGISTER)
3962 struct no_conflict_data data;
3964 data.target = const0_rtx;
3965 data.first = insns;
3966 data.insn = insn;
3967 data.must_stay = 0;
3968 note_stores (PATTERN (insn), no_conflict_move_test, &data);
3969 if (! data.must_stay)
3971 if (PREV_INSN (insn))
3972 SET_NEXT_INSN (PREV_INSN (insn)) = next;
3973 else
3974 insns = next;
3976 if (next)
3977 SET_PREV_INSN (next) = PREV_INSN (insn);
3979 add_insn (insn);
3983 /* Some ports use a loop to copy large arguments onto the stack.
3984 Don't move anything outside such a loop. */
3985 if (LABEL_P (insn))
3986 break;
3989 /* Write the remaining insns followed by the final copy. */
3990 for (insn = insns; insn; insn = next)
3992 next = NEXT_INSN (insn);
3994 add_insn (insn);
3997 last = emit_move_insn (target, result);
3998 set_dst_reg_note (last, REG_EQUAL, copy_rtx (equiv), target);
4000 if (final_dest != target)
4001 emit_move_insn (final_dest, target);
4004 void
4005 emit_libcall_block (rtx insns, rtx target, rtx result, rtx equiv)
4007 emit_libcall_block_1 (safe_as_a <rtx_insn *> (insns),
4008 target, result, equiv, false);
4011 /* Nonzero if we can perform a comparison of mode MODE straightforwardly.
4012 PURPOSE describes how this comparison will be used. CODE is the rtx
4013 comparison code we will be using.
4015 ??? Actually, CODE is slightly weaker than that. A target is still
4016 required to implement all of the normal bcc operations, but not
4017 required to implement all (or any) of the unordered bcc operations. */
4020 can_compare_p (enum rtx_code code, machine_mode mode,
4021 enum can_compare_purpose purpose)
4023 rtx test;
4024 test = gen_rtx_fmt_ee (code, mode, const0_rtx, const0_rtx);
4027 enum insn_code icode;
4029 if (purpose == ccp_jump
4030 && (icode = optab_handler (cbranch_optab, mode)) != CODE_FOR_nothing
4031 && insn_operand_matches (icode, 0, test))
4032 return 1;
4033 if (purpose == ccp_store_flag
4034 && (icode = optab_handler (cstore_optab, mode)) != CODE_FOR_nothing
4035 && insn_operand_matches (icode, 1, test))
4036 return 1;
4037 if (purpose == ccp_cmov
4038 && optab_handler (cmov_optab, mode) != CODE_FOR_nothing)
4039 return 1;
4041 mode = GET_MODE_WIDER_MODE (mode);
4042 PUT_MODE (test, mode);
4044 while (mode != VOIDmode);
4046 return 0;
4049 /* This function is called when we are going to emit a compare instruction that
4050 compares the values found in *PX and *PY, using the rtl operator COMPARISON.
4052 *PMODE is the mode of the inputs (in case they are const_int).
4053 *PUNSIGNEDP nonzero says that the operands are unsigned;
4054 this matters if they need to be widened (as given by METHODS).
4056 If they have mode BLKmode, then SIZE specifies the size of both operands.
4058 This function performs all the setup necessary so that the caller only has
4059 to emit a single comparison insn. This setup can involve doing a BLKmode
4060 comparison or emitting a library call to perform the comparison if no insn
4061 is available to handle it.
4062 The values which are passed in through pointers can be modified; the caller
4063 should perform the comparison on the modified values. Constant
4064 comparisons must have already been folded. */
4066 static void
4067 prepare_cmp_insn (rtx x, rtx y, enum rtx_code comparison, rtx size,
4068 int unsignedp, enum optab_methods methods,
4069 rtx *ptest, machine_mode *pmode)
4071 machine_mode mode = *pmode;
4072 rtx libfunc, test;
4073 machine_mode cmp_mode;
4074 enum mode_class mclass;
4076 /* The other methods are not needed. */
4077 gcc_assert (methods == OPTAB_DIRECT || methods == OPTAB_WIDEN
4078 || methods == OPTAB_LIB_WIDEN);
4080 /* If we are optimizing, force expensive constants into a register. */
4081 if (CONSTANT_P (x) && optimize
4082 && (rtx_cost (x, COMPARE, 0, optimize_insn_for_speed_p ())
4083 > COSTS_N_INSNS (1)))
4084 x = force_reg (mode, x);
4086 if (CONSTANT_P (y) && optimize
4087 && (rtx_cost (y, COMPARE, 1, optimize_insn_for_speed_p ())
4088 > COSTS_N_INSNS (1)))
4089 y = force_reg (mode, y);
4091 #ifdef HAVE_cc0
4092 /* Make sure if we have a canonical comparison. The RTL
4093 documentation states that canonical comparisons are required only
4094 for targets which have cc0. */
4095 gcc_assert (!CONSTANT_P (x) || CONSTANT_P (y));
4096 #endif
4098 /* Don't let both operands fail to indicate the mode. */
4099 if (GET_MODE (x) == VOIDmode && GET_MODE (y) == VOIDmode)
4100 x = force_reg (mode, x);
4101 if (mode == VOIDmode)
4102 mode = GET_MODE (x) != VOIDmode ? GET_MODE (x) : GET_MODE (y);
4104 /* Handle all BLKmode compares. */
4106 if (mode == BLKmode)
4108 machine_mode result_mode;
4109 enum insn_code cmp_code;
4110 tree length_type;
4111 rtx libfunc;
4112 rtx result;
4113 rtx opalign
4114 = GEN_INT (MIN (MEM_ALIGN (x), MEM_ALIGN (y)) / BITS_PER_UNIT);
4116 gcc_assert (size);
4118 /* Try to use a memory block compare insn - either cmpstr
4119 or cmpmem will do. */
4120 for (cmp_mode = GET_CLASS_NARROWEST_MODE (MODE_INT);
4121 cmp_mode != VOIDmode;
4122 cmp_mode = GET_MODE_WIDER_MODE (cmp_mode))
4124 cmp_code = direct_optab_handler (cmpmem_optab, cmp_mode);
4125 if (cmp_code == CODE_FOR_nothing)
4126 cmp_code = direct_optab_handler (cmpstr_optab, cmp_mode);
4127 if (cmp_code == CODE_FOR_nothing)
4128 cmp_code = direct_optab_handler (cmpstrn_optab, cmp_mode);
4129 if (cmp_code == CODE_FOR_nothing)
4130 continue;
4132 /* Must make sure the size fits the insn's mode. */
4133 if ((CONST_INT_P (size)
4134 && INTVAL (size) >= (1 << GET_MODE_BITSIZE (cmp_mode)))
4135 || (GET_MODE_BITSIZE (GET_MODE (size))
4136 > GET_MODE_BITSIZE (cmp_mode)))
4137 continue;
4139 result_mode = insn_data[cmp_code].operand[0].mode;
4140 result = gen_reg_rtx (result_mode);
4141 size = convert_to_mode (cmp_mode, size, 1);
4142 emit_insn (GEN_FCN (cmp_code) (result, x, y, size, opalign));
4144 *ptest = gen_rtx_fmt_ee (comparison, VOIDmode, result, const0_rtx);
4145 *pmode = result_mode;
4146 return;
4149 if (methods != OPTAB_LIB && methods != OPTAB_LIB_WIDEN)
4150 goto fail;
4152 /* Otherwise call a library function, memcmp. */
4153 libfunc = memcmp_libfunc;
4154 length_type = sizetype;
4155 result_mode = TYPE_MODE (integer_type_node);
4156 cmp_mode = TYPE_MODE (length_type);
4157 size = convert_to_mode (TYPE_MODE (length_type), size,
4158 TYPE_UNSIGNED (length_type));
4160 result = emit_library_call_value (libfunc, 0, LCT_PURE,
4161 result_mode, 3,
4162 XEXP (x, 0), Pmode,
4163 XEXP (y, 0), Pmode,
4164 size, cmp_mode);
4165 x = result;
4166 y = const0_rtx;
4167 mode = result_mode;
4168 methods = OPTAB_LIB_WIDEN;
4169 unsignedp = false;
4172 /* Don't allow operands to the compare to trap, as that can put the
4173 compare and branch in different basic blocks. */
4174 if (cfun->can_throw_non_call_exceptions)
4176 if (may_trap_p (x))
4177 x = force_reg (mode, x);
4178 if (may_trap_p (y))
4179 y = force_reg (mode, y);
4182 if (GET_MODE_CLASS (mode) == MODE_CC)
4184 enum insn_code icode = optab_handler (cbranch_optab, CCmode);
4185 test = gen_rtx_fmt_ee (comparison, VOIDmode, x, y);
4186 gcc_assert (icode != CODE_FOR_nothing
4187 && insn_operand_matches (icode, 0, test));
4188 *ptest = test;
4189 return;
4192 mclass = GET_MODE_CLASS (mode);
4193 test = gen_rtx_fmt_ee (comparison, VOIDmode, x, y);
4194 cmp_mode = mode;
4197 enum insn_code icode;
4198 icode = optab_handler (cbranch_optab, cmp_mode);
4199 if (icode != CODE_FOR_nothing
4200 && insn_operand_matches (icode, 0, test))
4202 rtx_insn *last = get_last_insn ();
4203 rtx op0 = prepare_operand (icode, x, 1, mode, cmp_mode, unsignedp);
4204 rtx op1 = prepare_operand (icode, y, 2, mode, cmp_mode, unsignedp);
4205 if (op0 && op1
4206 && insn_operand_matches (icode, 1, op0)
4207 && insn_operand_matches (icode, 2, op1))
4209 XEXP (test, 0) = op0;
4210 XEXP (test, 1) = op1;
4211 *ptest = test;
4212 *pmode = cmp_mode;
4213 return;
4215 delete_insns_since (last);
4218 if (methods == OPTAB_DIRECT || !CLASS_HAS_WIDER_MODES_P (mclass))
4219 break;
4220 cmp_mode = GET_MODE_WIDER_MODE (cmp_mode);
4222 while (cmp_mode != VOIDmode);
4224 if (methods != OPTAB_LIB_WIDEN)
4225 goto fail;
4227 if (!SCALAR_FLOAT_MODE_P (mode))
4229 rtx result;
4230 machine_mode ret_mode;
4232 /* Handle a libcall just for the mode we are using. */
4233 libfunc = optab_libfunc (cmp_optab, mode);
4234 gcc_assert (libfunc);
4236 /* If we want unsigned, and this mode has a distinct unsigned
4237 comparison routine, use that. */
4238 if (unsignedp)
4240 rtx ulibfunc = optab_libfunc (ucmp_optab, mode);
4241 if (ulibfunc)
4242 libfunc = ulibfunc;
4245 ret_mode = targetm.libgcc_cmp_return_mode ();
4246 result = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
4247 ret_mode, 2, x, mode, y, mode);
4249 /* There are two kinds of comparison routines. Biased routines
4250 return 0/1/2, and unbiased routines return -1/0/1. Other parts
4251 of gcc expect that the comparison operation is equivalent
4252 to the modified comparison. For signed comparisons compare the
4253 result against 1 in the biased case, and zero in the unbiased
4254 case. For unsigned comparisons always compare against 1 after
4255 biasing the unbiased result by adding 1. This gives us a way to
4256 represent LTU.
4257 The comparisons in the fixed-point helper library are always
4258 biased. */
4259 x = result;
4260 y = const1_rtx;
4262 if (!TARGET_LIB_INT_CMP_BIASED && !ALL_FIXED_POINT_MODE_P (mode))
4264 if (unsignedp)
4265 x = plus_constant (ret_mode, result, 1);
4266 else
4267 y = const0_rtx;
4270 *pmode = ret_mode;
4271 prepare_cmp_insn (x, y, comparison, NULL_RTX, unsignedp, methods,
4272 ptest, pmode);
4274 else
4275 prepare_float_lib_cmp (x, y, comparison, ptest, pmode);
4277 return;
4279 fail:
4280 *ptest = NULL_RTX;
4283 /* Before emitting an insn with code ICODE, make sure that X, which is going
4284 to be used for operand OPNUM of the insn, is converted from mode MODE to
4285 WIDER_MODE (UNSIGNEDP determines whether it is an unsigned conversion), and
4286 that it is accepted by the operand predicate. Return the new value. */
4289 prepare_operand (enum insn_code icode, rtx x, int opnum, machine_mode mode,
4290 machine_mode wider_mode, int unsignedp)
4292 if (mode != wider_mode)
4293 x = convert_modes (wider_mode, mode, x, unsignedp);
4295 if (!insn_operand_matches (icode, opnum, x))
4297 machine_mode op_mode = insn_data[(int) icode].operand[opnum].mode;
4298 if (reload_completed)
4299 return NULL_RTX;
4300 if (GET_MODE (x) != op_mode && GET_MODE (x) != VOIDmode)
4301 return NULL_RTX;
4302 x = copy_to_mode_reg (op_mode, x);
4305 return x;
4308 /* Subroutine of emit_cmp_and_jump_insns; this function is called when we know
4309 we can do the branch. */
4311 static void
4312 emit_cmp_and_jump_insn_1 (rtx test, machine_mode mode, rtx label, int prob)
4314 machine_mode optab_mode;
4315 enum mode_class mclass;
4316 enum insn_code icode;
4317 rtx_insn *insn;
4319 mclass = GET_MODE_CLASS (mode);
4320 optab_mode = (mclass == MODE_CC) ? CCmode : mode;
4321 icode = optab_handler (cbranch_optab, optab_mode);
4323 gcc_assert (icode != CODE_FOR_nothing);
4324 gcc_assert (insn_operand_matches (icode, 0, test));
4325 insn = emit_jump_insn (GEN_FCN (icode) (test, XEXP (test, 0),
4326 XEXP (test, 1), label));
4327 if (prob != -1
4328 && profile_status_for_fn (cfun) != PROFILE_ABSENT
4329 && insn
4330 && JUMP_P (insn)
4331 && any_condjump_p (insn)
4332 && !find_reg_note (insn, REG_BR_PROB, 0))
4333 add_int_reg_note (insn, REG_BR_PROB, prob);
4336 /* Generate code to compare X with Y so that the condition codes are
4337 set and to jump to LABEL if the condition is true. If X is a
4338 constant and Y is not a constant, then the comparison is swapped to
4339 ensure that the comparison RTL has the canonical form.
4341 UNSIGNEDP nonzero says that X and Y are unsigned; this matters if they
4342 need to be widened. UNSIGNEDP is also used to select the proper
4343 branch condition code.
4345 If X and Y have mode BLKmode, then SIZE specifies the size of both X and Y.
4347 MODE is the mode of the inputs (in case they are const_int).
4349 COMPARISON is the rtl operator to compare with (EQ, NE, GT, etc.).
4350 It will be potentially converted into an unsigned variant based on
4351 UNSIGNEDP to select a proper jump instruction.
4353 PROB is the probability of jumping to LABEL. */
4355 void
4356 emit_cmp_and_jump_insns (rtx x, rtx y, enum rtx_code comparison, rtx size,
4357 machine_mode mode, int unsignedp, rtx label,
4358 int prob)
4360 rtx op0 = x, op1 = y;
4361 rtx test;
4363 /* Swap operands and condition to ensure canonical RTL. */
4364 if (swap_commutative_operands_p (x, y)
4365 && can_compare_p (swap_condition (comparison), mode, ccp_jump))
4367 op0 = y, op1 = x;
4368 comparison = swap_condition (comparison);
4371 /* If OP0 is still a constant, then both X and Y must be constants
4372 or the opposite comparison is not supported. Force X into a register
4373 to create canonical RTL. */
4374 if (CONSTANT_P (op0))
4375 op0 = force_reg (mode, op0);
4377 if (unsignedp)
4378 comparison = unsigned_condition (comparison);
4380 prepare_cmp_insn (op0, op1, comparison, size, unsignedp, OPTAB_LIB_WIDEN,
4381 &test, &mode);
4382 emit_cmp_and_jump_insn_1 (test, mode, label, prob);
4386 /* Emit a library call comparison between floating point X and Y.
4387 COMPARISON is the rtl operator to compare with (EQ, NE, GT, etc.). */
4389 static void
4390 prepare_float_lib_cmp (rtx x, rtx y, enum rtx_code comparison,
4391 rtx *ptest, machine_mode *pmode)
4393 enum rtx_code swapped = swap_condition (comparison);
4394 enum rtx_code reversed = reverse_condition_maybe_unordered (comparison);
4395 machine_mode orig_mode = GET_MODE (x);
4396 machine_mode mode, cmp_mode;
4397 rtx true_rtx, false_rtx;
4398 rtx value, target, equiv;
4399 rtx_insn *insns;
4400 rtx libfunc = 0;
4401 bool reversed_p = false;
4402 cmp_mode = targetm.libgcc_cmp_return_mode ();
4404 for (mode = orig_mode;
4405 mode != VOIDmode;
4406 mode = GET_MODE_WIDER_MODE (mode))
4408 if (code_to_optab (comparison)
4409 && (libfunc = optab_libfunc (code_to_optab (comparison), mode)))
4410 break;
4412 if (code_to_optab (swapped)
4413 && (libfunc = optab_libfunc (code_to_optab (swapped), mode)))
4415 rtx tmp;
4416 tmp = x; x = y; y = tmp;
4417 comparison = swapped;
4418 break;
4421 if (code_to_optab (reversed)
4422 && (libfunc = optab_libfunc (code_to_optab (reversed), mode)))
4424 comparison = reversed;
4425 reversed_p = true;
4426 break;
4430 gcc_assert (mode != VOIDmode);
4432 if (mode != orig_mode)
4434 x = convert_to_mode (mode, x, 0);
4435 y = convert_to_mode (mode, y, 0);
4438 /* Attach a REG_EQUAL note describing the semantics of the libcall to
4439 the RTL. The allows the RTL optimizers to delete the libcall if the
4440 condition can be determined at compile-time. */
4441 if (comparison == UNORDERED
4442 || FLOAT_LIB_COMPARE_RETURNS_BOOL (mode, comparison))
4444 true_rtx = const_true_rtx;
4445 false_rtx = const0_rtx;
4447 else
4449 switch (comparison)
4451 case EQ:
4452 true_rtx = const0_rtx;
4453 false_rtx = const_true_rtx;
4454 break;
4456 case NE:
4457 true_rtx = const_true_rtx;
4458 false_rtx = const0_rtx;
4459 break;
4461 case GT:
4462 true_rtx = const1_rtx;
4463 false_rtx = const0_rtx;
4464 break;
4466 case GE:
4467 true_rtx = const0_rtx;
4468 false_rtx = constm1_rtx;
4469 break;
4471 case LT:
4472 true_rtx = constm1_rtx;
4473 false_rtx = const0_rtx;
4474 break;
4476 case LE:
4477 true_rtx = const0_rtx;
4478 false_rtx = const1_rtx;
4479 break;
4481 default:
4482 gcc_unreachable ();
4486 if (comparison == UNORDERED)
4488 rtx temp = simplify_gen_relational (NE, cmp_mode, mode, x, x);
4489 equiv = simplify_gen_relational (NE, cmp_mode, mode, y, y);
4490 equiv = simplify_gen_ternary (IF_THEN_ELSE, cmp_mode, cmp_mode,
4491 temp, const_true_rtx, equiv);
4493 else
4495 equiv = simplify_gen_relational (comparison, cmp_mode, mode, x, y);
4496 if (! FLOAT_LIB_COMPARE_RETURNS_BOOL (mode, comparison))
4497 equiv = simplify_gen_ternary (IF_THEN_ELSE, cmp_mode, cmp_mode,
4498 equiv, true_rtx, false_rtx);
4501 start_sequence ();
4502 value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
4503 cmp_mode, 2, x, mode, y, mode);
4504 insns = get_insns ();
4505 end_sequence ();
4507 target = gen_reg_rtx (cmp_mode);
4508 emit_libcall_block (insns, target, value, equiv);
4510 if (comparison == UNORDERED
4511 || FLOAT_LIB_COMPARE_RETURNS_BOOL (mode, comparison)
4512 || reversed_p)
4513 *ptest = gen_rtx_fmt_ee (reversed_p ? EQ : NE, VOIDmode, target, false_rtx);
4514 else
4515 *ptest = gen_rtx_fmt_ee (comparison, VOIDmode, target, const0_rtx);
4517 *pmode = cmp_mode;
4520 /* Generate code to indirectly jump to a location given in the rtx LOC. */
4522 void
4523 emit_indirect_jump (rtx loc ATTRIBUTE_UNUSED)
4525 #ifndef HAVE_indirect_jump
4526 sorry ("indirect jumps are not available on this target");
4527 #else
4528 struct expand_operand ops[1];
4529 create_address_operand (&ops[0], loc);
4530 expand_jump_insn (CODE_FOR_indirect_jump, 1, ops);
4531 emit_barrier ();
4532 #endif
4535 #ifdef HAVE_conditional_move
4537 /* Emit a conditional move instruction if the machine supports one for that
4538 condition and machine mode.
4540 OP0 and OP1 are the operands that should be compared using CODE. CMODE is
4541 the mode to use should they be constants. If it is VOIDmode, they cannot
4542 both be constants.
4544 OP2 should be stored in TARGET if the comparison is true, otherwise OP3
4545 should be stored there. MODE is the mode to use should they be constants.
4546 If it is VOIDmode, they cannot both be constants.
4548 The result is either TARGET (perhaps modified) or NULL_RTX if the operation
4549 is not supported. */
4552 emit_conditional_move (rtx target, enum rtx_code code, rtx op0, rtx op1,
4553 machine_mode cmode, rtx op2, rtx op3,
4554 machine_mode mode, int unsignedp)
4556 rtx tem, comparison;
4557 rtx_insn *last;
4558 enum insn_code icode;
4559 enum rtx_code reversed;
4561 /* If one operand is constant, make it the second one. Only do this
4562 if the other operand is not constant as well. */
4564 if (swap_commutative_operands_p (op0, op1))
4566 tem = op0;
4567 op0 = op1;
4568 op1 = tem;
4569 code = swap_condition (code);
4572 /* get_condition will prefer to generate LT and GT even if the old
4573 comparison was against zero, so undo that canonicalization here since
4574 comparisons against zero are cheaper. */
4575 if (code == LT && op1 == const1_rtx)
4576 code = LE, op1 = const0_rtx;
4577 else if (code == GT && op1 == constm1_rtx)
4578 code = GE, op1 = const0_rtx;
4580 if (cmode == VOIDmode)
4581 cmode = GET_MODE (op0);
4583 if (swap_commutative_operands_p (op2, op3)
4584 && ((reversed = reversed_comparison_code_parts (code, op0, op1, NULL))
4585 != UNKNOWN))
4587 tem = op2;
4588 op2 = op3;
4589 op3 = tem;
4590 code = reversed;
4593 if (mode == VOIDmode)
4594 mode = GET_MODE (op2);
4596 icode = direct_optab_handler (movcc_optab, mode);
4598 if (icode == CODE_FOR_nothing)
4599 return 0;
4601 if (!target)
4602 target = gen_reg_rtx (mode);
4604 code = unsignedp ? unsigned_condition (code) : code;
4605 comparison = simplify_gen_relational (code, VOIDmode, cmode, op0, op1);
4607 /* We can get const0_rtx or const_true_rtx in some circumstances. Just
4608 return NULL and let the caller figure out how best to deal with this
4609 situation. */
4610 if (!COMPARISON_P (comparison))
4611 return NULL_RTX;
4613 saved_pending_stack_adjust save;
4614 save_pending_stack_adjust (&save);
4615 last = get_last_insn ();
4616 do_pending_stack_adjust ();
4617 prepare_cmp_insn (XEXP (comparison, 0), XEXP (comparison, 1),
4618 GET_CODE (comparison), NULL_RTX, unsignedp, OPTAB_WIDEN,
4619 &comparison, &cmode);
4620 if (comparison)
4622 struct expand_operand ops[4];
4624 create_output_operand (&ops[0], target, mode);
4625 create_fixed_operand (&ops[1], comparison);
4626 create_input_operand (&ops[2], op2, mode);
4627 create_input_operand (&ops[3], op3, mode);
4628 if (maybe_expand_insn (icode, 4, ops))
4630 if (ops[0].value != target)
4631 convert_move (target, ops[0].value, false);
4632 return target;
4635 delete_insns_since (last);
4636 restore_pending_stack_adjust (&save);
4637 return NULL_RTX;
4640 /* Return nonzero if a conditional move of mode MODE is supported.
4642 This function is for combine so it can tell whether an insn that looks
4643 like a conditional move is actually supported by the hardware. If we
4644 guess wrong we lose a bit on optimization, but that's it. */
4645 /* ??? sparc64 supports conditionally moving integers values based on fp
4646 comparisons, and vice versa. How do we handle them? */
4649 can_conditionally_move_p (machine_mode mode)
4651 if (direct_optab_handler (movcc_optab, mode) != CODE_FOR_nothing)
4652 return 1;
4654 return 0;
4657 #endif /* HAVE_conditional_move */
4659 /* Emit a conditional addition instruction if the machine supports one for that
4660 condition and machine mode.
4662 OP0 and OP1 are the operands that should be compared using CODE. CMODE is
4663 the mode to use should they be constants. If it is VOIDmode, they cannot
4664 both be constants.
4666 OP2 should be stored in TARGET if the comparison is false, otherwise OP2+OP3
4667 should be stored there. MODE is the mode to use should they be constants.
4668 If it is VOIDmode, they cannot both be constants.
4670 The result is either TARGET (perhaps modified) or NULL_RTX if the operation
4671 is not supported. */
4674 emit_conditional_add (rtx target, enum rtx_code code, rtx op0, rtx op1,
4675 machine_mode cmode, rtx op2, rtx op3,
4676 machine_mode mode, int unsignedp)
4678 rtx tem, comparison;
4679 rtx_insn *last;
4680 enum insn_code icode;
4682 /* If one operand is constant, make it the second one. Only do this
4683 if the other operand is not constant as well. */
4685 if (swap_commutative_operands_p (op0, op1))
4687 tem = op0;
4688 op0 = op1;
4689 op1 = tem;
4690 code = swap_condition (code);
4693 /* get_condition will prefer to generate LT and GT even if the old
4694 comparison was against zero, so undo that canonicalization here since
4695 comparisons against zero are cheaper. */
4696 if (code == LT && op1 == const1_rtx)
4697 code = LE, op1 = const0_rtx;
4698 else if (code == GT && op1 == constm1_rtx)
4699 code = GE, op1 = const0_rtx;
4701 if (cmode == VOIDmode)
4702 cmode = GET_MODE (op0);
4704 if (mode == VOIDmode)
4705 mode = GET_MODE (op2);
4707 icode = optab_handler (addcc_optab, mode);
4709 if (icode == CODE_FOR_nothing)
4710 return 0;
4712 if (!target)
4713 target = gen_reg_rtx (mode);
4715 code = unsignedp ? unsigned_condition (code) : code;
4716 comparison = simplify_gen_relational (code, VOIDmode, cmode, op0, op1);
4718 /* We can get const0_rtx or const_true_rtx in some circumstances. Just
4719 return NULL and let the caller figure out how best to deal with this
4720 situation. */
4721 if (!COMPARISON_P (comparison))
4722 return NULL_RTX;
4724 do_pending_stack_adjust ();
4725 last = get_last_insn ();
4726 prepare_cmp_insn (XEXP (comparison, 0), XEXP (comparison, 1),
4727 GET_CODE (comparison), NULL_RTX, unsignedp, OPTAB_WIDEN,
4728 &comparison, &cmode);
4729 if (comparison)
4731 struct expand_operand ops[4];
4733 create_output_operand (&ops[0], target, mode);
4734 create_fixed_operand (&ops[1], comparison);
4735 create_input_operand (&ops[2], op2, mode);
4736 create_input_operand (&ops[3], op3, mode);
4737 if (maybe_expand_insn (icode, 4, ops))
4739 if (ops[0].value != target)
4740 convert_move (target, ops[0].value, false);
4741 return target;
4744 delete_insns_since (last);
4745 return NULL_RTX;
4748 /* These functions attempt to generate an insn body, rather than
4749 emitting the insn, but if the gen function already emits them, we
4750 make no attempt to turn them back into naked patterns. */
4752 /* Generate and return an insn body to add Y to X. */
4755 gen_add2_insn (rtx x, rtx y)
4757 enum insn_code icode = optab_handler (add_optab, GET_MODE (x));
4759 gcc_assert (insn_operand_matches (icode, 0, x));
4760 gcc_assert (insn_operand_matches (icode, 1, x));
4761 gcc_assert (insn_operand_matches (icode, 2, y));
4763 return GEN_FCN (icode) (x, x, y);
4766 /* Generate and return an insn body to add r1 and c,
4767 storing the result in r0. */
4770 gen_add3_insn (rtx r0, rtx r1, rtx c)
4772 enum insn_code icode = optab_handler (add_optab, GET_MODE (r0));
4774 if (icode == CODE_FOR_nothing
4775 || !insn_operand_matches (icode, 0, r0)
4776 || !insn_operand_matches (icode, 1, r1)
4777 || !insn_operand_matches (icode, 2, c))
4778 return NULL_RTX;
4780 return GEN_FCN (icode) (r0, r1, c);
4784 have_add2_insn (rtx x, rtx y)
4786 enum insn_code icode;
4788 gcc_assert (GET_MODE (x) != VOIDmode);
4790 icode = optab_handler (add_optab, GET_MODE (x));
4792 if (icode == CODE_FOR_nothing)
4793 return 0;
4795 if (!insn_operand_matches (icode, 0, x)
4796 || !insn_operand_matches (icode, 1, x)
4797 || !insn_operand_matches (icode, 2, y))
4798 return 0;
4800 return 1;
4803 /* Generate and return an insn body to add Y to X. */
4806 gen_addptr3_insn (rtx x, rtx y, rtx z)
4808 enum insn_code icode = optab_handler (addptr3_optab, GET_MODE (x));
4810 gcc_assert (insn_operand_matches (icode, 0, x));
4811 gcc_assert (insn_operand_matches (icode, 1, y));
4812 gcc_assert (insn_operand_matches (icode, 2, z));
4814 return GEN_FCN (icode) (x, y, z);
4817 /* Return true if the target implements an addptr pattern and X, Y,
4818 and Z are valid for the pattern predicates. */
4821 have_addptr3_insn (rtx x, rtx y, rtx z)
4823 enum insn_code icode;
4825 gcc_assert (GET_MODE (x) != VOIDmode);
4827 icode = optab_handler (addptr3_optab, GET_MODE (x));
4829 if (icode == CODE_FOR_nothing)
4830 return 0;
4832 if (!insn_operand_matches (icode, 0, x)
4833 || !insn_operand_matches (icode, 1, y)
4834 || !insn_operand_matches (icode, 2, z))
4835 return 0;
4837 return 1;
4840 /* Generate and return an insn body to subtract Y from X. */
4843 gen_sub2_insn (rtx x, rtx y)
4845 enum insn_code icode = optab_handler (sub_optab, GET_MODE (x));
4847 gcc_assert (insn_operand_matches (icode, 0, x));
4848 gcc_assert (insn_operand_matches (icode, 1, x));
4849 gcc_assert (insn_operand_matches (icode, 2, y));
4851 return GEN_FCN (icode) (x, x, y);
4854 /* Generate and return an insn body to subtract r1 and c,
4855 storing the result in r0. */
4858 gen_sub3_insn (rtx r0, rtx r1, rtx c)
4860 enum insn_code icode = optab_handler (sub_optab, GET_MODE (r0));
4862 if (icode == CODE_FOR_nothing
4863 || !insn_operand_matches (icode, 0, r0)
4864 || !insn_operand_matches (icode, 1, r1)
4865 || !insn_operand_matches (icode, 2, c))
4866 return NULL_RTX;
4868 return GEN_FCN (icode) (r0, r1, c);
4872 have_sub2_insn (rtx x, rtx y)
4874 enum insn_code icode;
4876 gcc_assert (GET_MODE (x) != VOIDmode);
4878 icode = optab_handler (sub_optab, GET_MODE (x));
4880 if (icode == CODE_FOR_nothing)
4881 return 0;
4883 if (!insn_operand_matches (icode, 0, x)
4884 || !insn_operand_matches (icode, 1, x)
4885 || !insn_operand_matches (icode, 2, y))
4886 return 0;
4888 return 1;
4891 /* Return the insn code used to extend FROM_MODE to TO_MODE.
4892 UNSIGNEDP specifies zero-extension instead of sign-extension. If
4893 no such operation exists, CODE_FOR_nothing will be returned. */
4895 enum insn_code
4896 can_extend_p (machine_mode to_mode, machine_mode from_mode,
4897 int unsignedp)
4899 convert_optab tab;
4900 #ifdef HAVE_ptr_extend
4901 if (unsignedp < 0)
4902 return CODE_FOR_ptr_extend;
4903 #endif
4905 tab = unsignedp ? zext_optab : sext_optab;
4906 return convert_optab_handler (tab, to_mode, from_mode);
4909 /* Generate the body of an insn to extend Y (with mode MFROM)
4910 into X (with mode MTO). Do zero-extension if UNSIGNEDP is nonzero. */
4913 gen_extend_insn (rtx x, rtx y, machine_mode mto,
4914 machine_mode mfrom, int unsignedp)
4916 enum insn_code icode = can_extend_p (mto, mfrom, unsignedp);
4917 return GEN_FCN (icode) (x, y);
4920 /* can_fix_p and can_float_p say whether the target machine
4921 can directly convert a given fixed point type to
4922 a given floating point type, or vice versa.
4923 The returned value is the CODE_FOR_... value to use,
4924 or CODE_FOR_nothing if these modes cannot be directly converted.
4926 *TRUNCP_PTR is set to 1 if it is necessary to output
4927 an explicit FTRUNC insn before the fix insn; otherwise 0. */
4929 static enum insn_code
4930 can_fix_p (machine_mode fixmode, machine_mode fltmode,
4931 int unsignedp, int *truncp_ptr)
4933 convert_optab tab;
4934 enum insn_code icode;
4936 tab = unsignedp ? ufixtrunc_optab : sfixtrunc_optab;
4937 icode = convert_optab_handler (tab, fixmode, fltmode);
4938 if (icode != CODE_FOR_nothing)
4940 *truncp_ptr = 0;
4941 return icode;
4944 /* FIXME: This requires a port to define both FIX and FTRUNC pattern
4945 for this to work. We need to rework the fix* and ftrunc* patterns
4946 and documentation. */
4947 tab = unsignedp ? ufix_optab : sfix_optab;
4948 icode = convert_optab_handler (tab, fixmode, fltmode);
4949 if (icode != CODE_FOR_nothing
4950 && optab_handler (ftrunc_optab, fltmode) != CODE_FOR_nothing)
4952 *truncp_ptr = 1;
4953 return icode;
4956 *truncp_ptr = 0;
4957 return CODE_FOR_nothing;
4960 enum insn_code
4961 can_float_p (machine_mode fltmode, machine_mode fixmode,
4962 int unsignedp)
4964 convert_optab tab;
4966 tab = unsignedp ? ufloat_optab : sfloat_optab;
4967 return convert_optab_handler (tab, fltmode, fixmode);
4970 /* Function supportable_convert_operation
4972 Check whether an operation represented by the code CODE is a
4973 convert operation that is supported by the target platform in
4974 vector form (i.e., when operating on arguments of type VECTYPE_IN
4975 producing a result of type VECTYPE_OUT).
4977 Convert operations we currently support directly are FIX_TRUNC and FLOAT.
4978 This function checks if these operations are supported
4979 by the target platform either directly (via vector tree-codes), or via
4980 target builtins.
4982 Output:
4983 - CODE1 is code of vector operation to be used when
4984 vectorizing the operation, if available.
4985 - DECL is decl of target builtin functions to be used
4986 when vectorizing the operation, if available. In this case,
4987 CODE1 is CALL_EXPR. */
4989 bool
4990 supportable_convert_operation (enum tree_code code,
4991 tree vectype_out, tree vectype_in,
4992 tree *decl, enum tree_code *code1)
4994 machine_mode m1,m2;
4995 int truncp;
4997 m1 = TYPE_MODE (vectype_out);
4998 m2 = TYPE_MODE (vectype_in);
5000 /* First check if we can done conversion directly. */
5001 if ((code == FIX_TRUNC_EXPR
5002 && can_fix_p (m1,m2,TYPE_UNSIGNED (vectype_out), &truncp)
5003 != CODE_FOR_nothing)
5004 || (code == FLOAT_EXPR
5005 && can_float_p (m1,m2,TYPE_UNSIGNED (vectype_in))
5006 != CODE_FOR_nothing))
5008 *code1 = code;
5009 return true;
5012 /* Now check for builtin. */
5013 if (targetm.vectorize.builtin_conversion
5014 && targetm.vectorize.builtin_conversion (code, vectype_out, vectype_in))
5016 *code1 = CALL_EXPR;
5017 *decl = targetm.vectorize.builtin_conversion (code, vectype_out, vectype_in);
5018 return true;
5020 return false;
5024 /* Generate code to convert FROM to floating point
5025 and store in TO. FROM must be fixed point and not VOIDmode.
5026 UNSIGNEDP nonzero means regard FROM as unsigned.
5027 Normally this is done by correcting the final value
5028 if it is negative. */
5030 void
5031 expand_float (rtx to, rtx from, int unsignedp)
5033 enum insn_code icode;
5034 rtx target = to;
5035 machine_mode fmode, imode;
5036 bool can_do_signed = false;
5038 /* Crash now, because we won't be able to decide which mode to use. */
5039 gcc_assert (GET_MODE (from) != VOIDmode);
5041 /* Look for an insn to do the conversion. Do it in the specified
5042 modes if possible; otherwise convert either input, output or both to
5043 wider mode. If the integer mode is wider than the mode of FROM,
5044 we can do the conversion signed even if the input is unsigned. */
5046 for (fmode = GET_MODE (to); fmode != VOIDmode;
5047 fmode = GET_MODE_WIDER_MODE (fmode))
5048 for (imode = GET_MODE (from); imode != VOIDmode;
5049 imode = GET_MODE_WIDER_MODE (imode))
5051 int doing_unsigned = unsignedp;
5053 if (fmode != GET_MODE (to)
5054 && significand_size (fmode) < GET_MODE_PRECISION (GET_MODE (from)))
5055 continue;
5057 icode = can_float_p (fmode, imode, unsignedp);
5058 if (icode == CODE_FOR_nothing && unsignedp)
5060 enum insn_code scode = can_float_p (fmode, imode, 0);
5061 if (scode != CODE_FOR_nothing)
5062 can_do_signed = true;
5063 if (imode != GET_MODE (from))
5064 icode = scode, doing_unsigned = 0;
5067 if (icode != CODE_FOR_nothing)
5069 if (imode != GET_MODE (from))
5070 from = convert_to_mode (imode, from, unsignedp);
5072 if (fmode != GET_MODE (to))
5073 target = gen_reg_rtx (fmode);
5075 emit_unop_insn (icode, target, from,
5076 doing_unsigned ? UNSIGNED_FLOAT : FLOAT);
5078 if (target != to)
5079 convert_move (to, target, 0);
5080 return;
5084 /* Unsigned integer, and no way to convert directly. Convert as signed,
5085 then unconditionally adjust the result. */
5086 if (unsignedp && can_do_signed)
5088 rtx_code_label *label = gen_label_rtx ();
5089 rtx temp;
5090 REAL_VALUE_TYPE offset;
5092 /* Look for a usable floating mode FMODE wider than the source and at
5093 least as wide as the target. Using FMODE will avoid rounding woes
5094 with unsigned values greater than the signed maximum value. */
5096 for (fmode = GET_MODE (to); fmode != VOIDmode;
5097 fmode = GET_MODE_WIDER_MODE (fmode))
5098 if (GET_MODE_PRECISION (GET_MODE (from)) < GET_MODE_BITSIZE (fmode)
5099 && can_float_p (fmode, GET_MODE (from), 0) != CODE_FOR_nothing)
5100 break;
5102 if (fmode == VOIDmode)
5104 /* There is no such mode. Pretend the target is wide enough. */
5105 fmode = GET_MODE (to);
5107 /* Avoid double-rounding when TO is narrower than FROM. */
5108 if ((significand_size (fmode) + 1)
5109 < GET_MODE_PRECISION (GET_MODE (from)))
5111 rtx temp1;
5112 rtx_code_label *neglabel = gen_label_rtx ();
5114 /* Don't use TARGET if it isn't a register, is a hard register,
5115 or is the wrong mode. */
5116 if (!REG_P (target)
5117 || REGNO (target) < FIRST_PSEUDO_REGISTER
5118 || GET_MODE (target) != fmode)
5119 target = gen_reg_rtx (fmode);
5121 imode = GET_MODE (from);
5122 do_pending_stack_adjust ();
5124 /* Test whether the sign bit is set. */
5125 emit_cmp_and_jump_insns (from, const0_rtx, LT, NULL_RTX, imode,
5126 0, neglabel);
5128 /* The sign bit is not set. Convert as signed. */
5129 expand_float (target, from, 0);
5130 emit_jump_insn (gen_jump (label));
5131 emit_barrier ();
5133 /* The sign bit is set.
5134 Convert to a usable (positive signed) value by shifting right
5135 one bit, while remembering if a nonzero bit was shifted
5136 out; i.e., compute (from & 1) | (from >> 1). */
5138 emit_label (neglabel);
5139 temp = expand_binop (imode, and_optab, from, const1_rtx,
5140 NULL_RTX, 1, OPTAB_LIB_WIDEN);
5141 temp1 = expand_shift (RSHIFT_EXPR, imode, from, 1, NULL_RTX, 1);
5142 temp = expand_binop (imode, ior_optab, temp, temp1, temp, 1,
5143 OPTAB_LIB_WIDEN);
5144 expand_float (target, temp, 0);
5146 /* Multiply by 2 to undo the shift above. */
5147 temp = expand_binop (fmode, add_optab, target, target,
5148 target, 0, OPTAB_LIB_WIDEN);
5149 if (temp != target)
5150 emit_move_insn (target, temp);
5152 do_pending_stack_adjust ();
5153 emit_label (label);
5154 goto done;
5158 /* If we are about to do some arithmetic to correct for an
5159 unsigned operand, do it in a pseudo-register. */
5161 if (GET_MODE (to) != fmode
5162 || !REG_P (to) || REGNO (to) < FIRST_PSEUDO_REGISTER)
5163 target = gen_reg_rtx (fmode);
5165 /* Convert as signed integer to floating. */
5166 expand_float (target, from, 0);
5168 /* If FROM is negative (and therefore TO is negative),
5169 correct its value by 2**bitwidth. */
5171 do_pending_stack_adjust ();
5172 emit_cmp_and_jump_insns (from, const0_rtx, GE, NULL_RTX, GET_MODE (from),
5173 0, label);
5176 real_2expN (&offset, GET_MODE_PRECISION (GET_MODE (from)), fmode);
5177 temp = expand_binop (fmode, add_optab, target,
5178 CONST_DOUBLE_FROM_REAL_VALUE (offset, fmode),
5179 target, 0, OPTAB_LIB_WIDEN);
5180 if (temp != target)
5181 emit_move_insn (target, temp);
5183 do_pending_stack_adjust ();
5184 emit_label (label);
5185 goto done;
5188 /* No hardware instruction available; call a library routine. */
5190 rtx libfunc;
5191 rtx_insn *insns;
5192 rtx value;
5193 convert_optab tab = unsignedp ? ufloat_optab : sfloat_optab;
5195 if (GET_MODE_PRECISION (GET_MODE (from)) < GET_MODE_PRECISION (SImode))
5196 from = convert_to_mode (SImode, from, unsignedp);
5198 libfunc = convert_optab_libfunc (tab, GET_MODE (to), GET_MODE (from));
5199 gcc_assert (libfunc);
5201 start_sequence ();
5203 value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
5204 GET_MODE (to), 1, from,
5205 GET_MODE (from));
5206 insns = get_insns ();
5207 end_sequence ();
5209 emit_libcall_block (insns, target, value,
5210 gen_rtx_fmt_e (unsignedp ? UNSIGNED_FLOAT : FLOAT,
5211 GET_MODE (to), from));
5214 done:
5216 /* Copy result to requested destination
5217 if we have been computing in a temp location. */
5219 if (target != to)
5221 if (GET_MODE (target) == GET_MODE (to))
5222 emit_move_insn (to, target);
5223 else
5224 convert_move (to, target, 0);
5228 /* Generate code to convert FROM to fixed point and store in TO. FROM
5229 must be floating point. */
5231 void
5232 expand_fix (rtx to, rtx from, int unsignedp)
5234 enum insn_code icode;
5235 rtx target = to;
5236 machine_mode fmode, imode;
5237 int must_trunc = 0;
5239 /* We first try to find a pair of modes, one real and one integer, at
5240 least as wide as FROM and TO, respectively, in which we can open-code
5241 this conversion. If the integer mode is wider than the mode of TO,
5242 we can do the conversion either signed or unsigned. */
5244 for (fmode = GET_MODE (from); fmode != VOIDmode;
5245 fmode = GET_MODE_WIDER_MODE (fmode))
5246 for (imode = GET_MODE (to); imode != VOIDmode;
5247 imode = GET_MODE_WIDER_MODE (imode))
5249 int doing_unsigned = unsignedp;
5251 icode = can_fix_p (imode, fmode, unsignedp, &must_trunc);
5252 if (icode == CODE_FOR_nothing && imode != GET_MODE (to) && unsignedp)
5253 icode = can_fix_p (imode, fmode, 0, &must_trunc), doing_unsigned = 0;
5255 if (icode != CODE_FOR_nothing)
5257 rtx_insn *last = get_last_insn ();
5258 if (fmode != GET_MODE (from))
5259 from = convert_to_mode (fmode, from, 0);
5261 if (must_trunc)
5263 rtx temp = gen_reg_rtx (GET_MODE (from));
5264 from = expand_unop (GET_MODE (from), ftrunc_optab, from,
5265 temp, 0);
5268 if (imode != GET_MODE (to))
5269 target = gen_reg_rtx (imode);
5271 if (maybe_emit_unop_insn (icode, target, from,
5272 doing_unsigned ? UNSIGNED_FIX : FIX))
5274 if (target != to)
5275 convert_move (to, target, unsignedp);
5276 return;
5278 delete_insns_since (last);
5282 /* For an unsigned conversion, there is one more way to do it.
5283 If we have a signed conversion, we generate code that compares
5284 the real value to the largest representable positive number. If if
5285 is smaller, the conversion is done normally. Otherwise, subtract
5286 one plus the highest signed number, convert, and add it back.
5288 We only need to check all real modes, since we know we didn't find
5289 anything with a wider integer mode.
5291 This code used to extend FP value into mode wider than the destination.
5292 This is needed for decimal float modes which cannot accurately
5293 represent one plus the highest signed number of the same size, but
5294 not for binary modes. Consider, for instance conversion from SFmode
5295 into DImode.
5297 The hot path through the code is dealing with inputs smaller than 2^63
5298 and doing just the conversion, so there is no bits to lose.
5300 In the other path we know the value is positive in the range 2^63..2^64-1
5301 inclusive. (as for other input overflow happens and result is undefined)
5302 So we know that the most important bit set in mantissa corresponds to
5303 2^63. The subtraction of 2^63 should not generate any rounding as it
5304 simply clears out that bit. The rest is trivial. */
5306 if (unsignedp && GET_MODE_PRECISION (GET_MODE (to)) <= HOST_BITS_PER_WIDE_INT)
5307 for (fmode = GET_MODE (from); fmode != VOIDmode;
5308 fmode = GET_MODE_WIDER_MODE (fmode))
5309 if (CODE_FOR_nothing != can_fix_p (GET_MODE (to), fmode, 0, &must_trunc)
5310 && (!DECIMAL_FLOAT_MODE_P (fmode)
5311 || GET_MODE_BITSIZE (fmode) > GET_MODE_PRECISION (GET_MODE (to))))
5313 int bitsize;
5314 REAL_VALUE_TYPE offset;
5315 rtx limit;
5316 rtx_code_label *lab1, *lab2;
5317 rtx_insn *insn;
5319 bitsize = GET_MODE_PRECISION (GET_MODE (to));
5320 real_2expN (&offset, bitsize - 1, fmode);
5321 limit = CONST_DOUBLE_FROM_REAL_VALUE (offset, fmode);
5322 lab1 = gen_label_rtx ();
5323 lab2 = gen_label_rtx ();
5325 if (fmode != GET_MODE (from))
5326 from = convert_to_mode (fmode, from, 0);
5328 /* See if we need to do the subtraction. */
5329 do_pending_stack_adjust ();
5330 emit_cmp_and_jump_insns (from, limit, GE, NULL_RTX, GET_MODE (from),
5331 0, lab1);
5333 /* If not, do the signed "fix" and branch around fixup code. */
5334 expand_fix (to, from, 0);
5335 emit_jump_insn (gen_jump (lab2));
5336 emit_barrier ();
5338 /* Otherwise, subtract 2**(N-1), convert to signed number,
5339 then add 2**(N-1). Do the addition using XOR since this
5340 will often generate better code. */
5341 emit_label (lab1);
5342 target = expand_binop (GET_MODE (from), sub_optab, from, limit,
5343 NULL_RTX, 0, OPTAB_LIB_WIDEN);
5344 expand_fix (to, target, 0);
5345 target = expand_binop (GET_MODE (to), xor_optab, to,
5346 gen_int_mode
5347 ((HOST_WIDE_INT) 1 << (bitsize - 1),
5348 GET_MODE (to)),
5349 to, 1, OPTAB_LIB_WIDEN);
5351 if (target != to)
5352 emit_move_insn (to, target);
5354 emit_label (lab2);
5356 if (optab_handler (mov_optab, GET_MODE (to)) != CODE_FOR_nothing)
5358 /* Make a place for a REG_NOTE and add it. */
5359 insn = emit_move_insn (to, to);
5360 set_dst_reg_note (insn, REG_EQUAL,
5361 gen_rtx_fmt_e (UNSIGNED_FIX, GET_MODE (to),
5362 copy_rtx (from)),
5363 to);
5366 return;
5369 /* We can't do it with an insn, so use a library call. But first ensure
5370 that the mode of TO is at least as wide as SImode, since those are the
5371 only library calls we know about. */
5373 if (GET_MODE_PRECISION (GET_MODE (to)) < GET_MODE_PRECISION (SImode))
5375 target = gen_reg_rtx (SImode);
5377 expand_fix (target, from, unsignedp);
5379 else
5381 rtx_insn *insns;
5382 rtx value;
5383 rtx libfunc;
5385 convert_optab tab = unsignedp ? ufix_optab : sfix_optab;
5386 libfunc = convert_optab_libfunc (tab, GET_MODE (to), GET_MODE (from));
5387 gcc_assert (libfunc);
5389 start_sequence ();
5391 value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
5392 GET_MODE (to), 1, from,
5393 GET_MODE (from));
5394 insns = get_insns ();
5395 end_sequence ();
5397 emit_libcall_block (insns, target, value,
5398 gen_rtx_fmt_e (unsignedp ? UNSIGNED_FIX : FIX,
5399 GET_MODE (to), from));
5402 if (target != to)
5404 if (GET_MODE (to) == GET_MODE (target))
5405 emit_move_insn (to, target);
5406 else
5407 convert_move (to, target, 0);
5411 /* Generate code to convert FROM or TO a fixed-point.
5412 If UINTP is true, either TO or FROM is an unsigned integer.
5413 If SATP is true, we need to saturate the result. */
5415 void
5416 expand_fixed_convert (rtx to, rtx from, int uintp, int satp)
5418 machine_mode to_mode = GET_MODE (to);
5419 machine_mode from_mode = GET_MODE (from);
5420 convert_optab tab;
5421 enum rtx_code this_code;
5422 enum insn_code code;
5423 rtx_insn *insns;
5424 rtx value;
5425 rtx libfunc;
5427 if (to_mode == from_mode)
5429 emit_move_insn (to, from);
5430 return;
5433 if (uintp)
5435 tab = satp ? satfractuns_optab : fractuns_optab;
5436 this_code = satp ? UNSIGNED_SAT_FRACT : UNSIGNED_FRACT_CONVERT;
5438 else
5440 tab = satp ? satfract_optab : fract_optab;
5441 this_code = satp ? SAT_FRACT : FRACT_CONVERT;
5443 code = convert_optab_handler (tab, to_mode, from_mode);
5444 if (code != CODE_FOR_nothing)
5446 emit_unop_insn (code, to, from, this_code);
5447 return;
5450 libfunc = convert_optab_libfunc (tab, to_mode, from_mode);
5451 gcc_assert (libfunc);
5453 start_sequence ();
5454 value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST, to_mode,
5455 1, from, from_mode);
5456 insns = get_insns ();
5457 end_sequence ();
5459 emit_libcall_block (insns, to, value,
5460 gen_rtx_fmt_e (optab_to_code (tab), to_mode, from));
5463 /* Generate code to convert FROM to fixed point and store in TO. FROM
5464 must be floating point, TO must be signed. Use the conversion optab
5465 TAB to do the conversion. */
5467 bool
5468 expand_sfix_optab (rtx to, rtx from, convert_optab tab)
5470 enum insn_code icode;
5471 rtx target = to;
5472 machine_mode fmode, imode;
5474 /* We first try to find a pair of modes, one real and one integer, at
5475 least as wide as FROM and TO, respectively, in which we can open-code
5476 this conversion. If the integer mode is wider than the mode of TO,
5477 we can do the conversion either signed or unsigned. */
5479 for (fmode = GET_MODE (from); fmode != VOIDmode;
5480 fmode = GET_MODE_WIDER_MODE (fmode))
5481 for (imode = GET_MODE (to); imode != VOIDmode;
5482 imode = GET_MODE_WIDER_MODE (imode))
5484 icode = convert_optab_handler (tab, imode, fmode);
5485 if (icode != CODE_FOR_nothing)
5487 rtx_insn *last = get_last_insn ();
5488 if (fmode != GET_MODE (from))
5489 from = convert_to_mode (fmode, from, 0);
5491 if (imode != GET_MODE (to))
5492 target = gen_reg_rtx (imode);
5494 if (!maybe_emit_unop_insn (icode, target, from, UNKNOWN))
5496 delete_insns_since (last);
5497 continue;
5499 if (target != to)
5500 convert_move (to, target, 0);
5501 return true;
5505 return false;
5508 /* Report whether we have an instruction to perform the operation
5509 specified by CODE on operands of mode MODE. */
5511 have_insn_for (enum rtx_code code, machine_mode mode)
5513 return (code_to_optab (code)
5514 && (optab_handler (code_to_optab (code), mode)
5515 != CODE_FOR_nothing));
5518 /* Initialize the libfunc fields of an entire group of entries in some
5519 optab. Each entry is set equal to a string consisting of a leading
5520 pair of underscores followed by a generic operation name followed by
5521 a mode name (downshifted to lowercase) followed by a single character
5522 representing the number of operands for the given operation (which is
5523 usually one of the characters '2', '3', or '4').
5525 OPTABLE is the table in which libfunc fields are to be initialized.
5526 OPNAME is the generic (string) name of the operation.
5527 SUFFIX is the character which specifies the number of operands for
5528 the given generic operation.
5529 MODE is the mode to generate for.
5532 static void
5533 gen_libfunc (optab optable, const char *opname, int suffix,
5534 machine_mode mode)
5536 unsigned opname_len = strlen (opname);
5537 const char *mname = GET_MODE_NAME (mode);
5538 unsigned mname_len = strlen (mname);
5539 int prefix_len = targetm.libfunc_gnu_prefix ? 6 : 2;
5540 int len = prefix_len + opname_len + mname_len + 1 + 1;
5541 char *libfunc_name = XALLOCAVEC (char, len);
5542 char *p;
5543 const char *q;
5545 p = libfunc_name;
5546 *p++ = '_';
5547 *p++ = '_';
5548 if (targetm.libfunc_gnu_prefix)
5550 *p++ = 'g';
5551 *p++ = 'n';
5552 *p++ = 'u';
5553 *p++ = '_';
5555 for (q = opname; *q; )
5556 *p++ = *q++;
5557 for (q = mname; *q; q++)
5558 *p++ = TOLOWER (*q);
5559 *p++ = suffix;
5560 *p = '\0';
5562 set_optab_libfunc (optable, mode,
5563 ggc_alloc_string (libfunc_name, p - libfunc_name));
5566 /* Like gen_libfunc, but verify that integer operation is involved. */
5568 void
5569 gen_int_libfunc (optab optable, const char *opname, char suffix,
5570 machine_mode mode)
5572 int maxsize = 2 * BITS_PER_WORD;
5573 int minsize = BITS_PER_WORD;
5575 if (GET_MODE_CLASS (mode) != MODE_INT)
5576 return;
5577 if (maxsize < LONG_LONG_TYPE_SIZE)
5578 maxsize = LONG_LONG_TYPE_SIZE;
5579 if (minsize > INT_TYPE_SIZE
5580 && (trapv_binoptab_p (optable)
5581 || trapv_unoptab_p (optable)))
5582 minsize = INT_TYPE_SIZE;
5583 if (GET_MODE_BITSIZE (mode) < minsize
5584 || GET_MODE_BITSIZE (mode) > maxsize)
5585 return;
5586 gen_libfunc (optable, opname, suffix, mode);
5589 /* Like gen_libfunc, but verify that FP and set decimal prefix if needed. */
5591 void
5592 gen_fp_libfunc (optab optable, const char *opname, char suffix,
5593 machine_mode mode)
5595 char *dec_opname;
5597 if (GET_MODE_CLASS (mode) == MODE_FLOAT)
5598 gen_libfunc (optable, opname, suffix, mode);
5599 if (DECIMAL_FLOAT_MODE_P (mode))
5601 dec_opname = XALLOCAVEC (char, sizeof (DECIMAL_PREFIX) + strlen (opname));
5602 /* For BID support, change the name to have either a bid_ or dpd_ prefix
5603 depending on the low level floating format used. */
5604 memcpy (dec_opname, DECIMAL_PREFIX, sizeof (DECIMAL_PREFIX) - 1);
5605 strcpy (dec_opname + sizeof (DECIMAL_PREFIX) - 1, opname);
5606 gen_libfunc (optable, dec_opname, suffix, mode);
5610 /* Like gen_libfunc, but verify that fixed-point operation is involved. */
5612 void
5613 gen_fixed_libfunc (optab optable, const char *opname, char suffix,
5614 machine_mode mode)
5616 if (!ALL_FIXED_POINT_MODE_P (mode))
5617 return;
5618 gen_libfunc (optable, opname, suffix, mode);
5621 /* Like gen_libfunc, but verify that signed fixed-point operation is
5622 involved. */
5624 void
5625 gen_signed_fixed_libfunc (optab optable, const char *opname, char suffix,
5626 machine_mode mode)
5628 if (!SIGNED_FIXED_POINT_MODE_P (mode))
5629 return;
5630 gen_libfunc (optable, opname, suffix, mode);
5633 /* Like gen_libfunc, but verify that unsigned fixed-point operation is
5634 involved. */
5636 void
5637 gen_unsigned_fixed_libfunc (optab optable, const char *opname, char suffix,
5638 machine_mode mode)
5640 if (!UNSIGNED_FIXED_POINT_MODE_P (mode))
5641 return;
5642 gen_libfunc (optable, opname, suffix, mode);
5645 /* Like gen_libfunc, but verify that FP or INT operation is involved. */
5647 void
5648 gen_int_fp_libfunc (optab optable, const char *name, char suffix,
5649 machine_mode mode)
5651 if (DECIMAL_FLOAT_MODE_P (mode) || GET_MODE_CLASS (mode) == MODE_FLOAT)
5652 gen_fp_libfunc (optable, name, suffix, mode);
5653 if (INTEGRAL_MODE_P (mode))
5654 gen_int_libfunc (optable, name, suffix, mode);
5657 /* Like gen_libfunc, but verify that FP or INT operation is involved
5658 and add 'v' suffix for integer operation. */
5660 void
5661 gen_intv_fp_libfunc (optab optable, const char *name, char suffix,
5662 machine_mode mode)
5664 if (DECIMAL_FLOAT_MODE_P (mode) || GET_MODE_CLASS (mode) == MODE_FLOAT)
5665 gen_fp_libfunc (optable, name, suffix, mode);
5666 if (GET_MODE_CLASS (mode) == MODE_INT)
5668 int len = strlen (name);
5669 char *v_name = XALLOCAVEC (char, len + 2);
5670 strcpy (v_name, name);
5671 v_name[len] = 'v';
5672 v_name[len + 1] = 0;
5673 gen_int_libfunc (optable, v_name, suffix, mode);
5677 /* Like gen_libfunc, but verify that FP or INT or FIXED operation is
5678 involved. */
5680 void
5681 gen_int_fp_fixed_libfunc (optab optable, const char *name, char suffix,
5682 machine_mode mode)
5684 if (DECIMAL_FLOAT_MODE_P (mode) || GET_MODE_CLASS (mode) == MODE_FLOAT)
5685 gen_fp_libfunc (optable, name, suffix, mode);
5686 if (INTEGRAL_MODE_P (mode))
5687 gen_int_libfunc (optable, name, suffix, mode);
5688 if (ALL_FIXED_POINT_MODE_P (mode))
5689 gen_fixed_libfunc (optable, name, suffix, mode);
5692 /* Like gen_libfunc, but verify that FP or INT or signed FIXED operation is
5693 involved. */
5695 void
5696 gen_int_fp_signed_fixed_libfunc (optab optable, const char *name, char suffix,
5697 machine_mode mode)
5699 if (DECIMAL_FLOAT_MODE_P (mode) || GET_MODE_CLASS (mode) == MODE_FLOAT)
5700 gen_fp_libfunc (optable, name, suffix, mode);
5701 if (INTEGRAL_MODE_P (mode))
5702 gen_int_libfunc (optable, name, suffix, mode);
5703 if (SIGNED_FIXED_POINT_MODE_P (mode))
5704 gen_signed_fixed_libfunc (optable, name, suffix, mode);
5707 /* Like gen_libfunc, but verify that INT or FIXED operation is
5708 involved. */
5710 void
5711 gen_int_fixed_libfunc (optab optable, const char *name, char suffix,
5712 machine_mode mode)
5714 if (INTEGRAL_MODE_P (mode))
5715 gen_int_libfunc (optable, name, suffix, mode);
5716 if (ALL_FIXED_POINT_MODE_P (mode))
5717 gen_fixed_libfunc (optable, name, suffix, mode);
5720 /* Like gen_libfunc, but verify that INT or signed FIXED operation is
5721 involved. */
5723 void
5724 gen_int_signed_fixed_libfunc (optab optable, const char *name, char suffix,
5725 machine_mode mode)
5727 if (INTEGRAL_MODE_P (mode))
5728 gen_int_libfunc (optable, name, suffix, mode);
5729 if (SIGNED_FIXED_POINT_MODE_P (mode))
5730 gen_signed_fixed_libfunc (optable, name, suffix, mode);
5733 /* Like gen_libfunc, but verify that INT or unsigned FIXED operation is
5734 involved. */
5736 void
5737 gen_int_unsigned_fixed_libfunc (optab optable, const char *name, char suffix,
5738 machine_mode mode)
5740 if (INTEGRAL_MODE_P (mode))
5741 gen_int_libfunc (optable, name, suffix, mode);
5742 if (UNSIGNED_FIXED_POINT_MODE_P (mode))
5743 gen_unsigned_fixed_libfunc (optable, name, suffix, mode);
5746 /* Initialize the libfunc fields of an entire group of entries of an
5747 inter-mode-class conversion optab. The string formation rules are
5748 similar to the ones for init_libfuncs, above, but instead of having
5749 a mode name and an operand count these functions have two mode names
5750 and no operand count. */
5752 void
5753 gen_interclass_conv_libfunc (convert_optab tab,
5754 const char *opname,
5755 machine_mode tmode,
5756 machine_mode fmode)
5758 size_t opname_len = strlen (opname);
5759 size_t mname_len = 0;
5761 const char *fname, *tname;
5762 const char *q;
5763 int prefix_len = targetm.libfunc_gnu_prefix ? 6 : 2;
5764 char *libfunc_name, *suffix;
5765 char *nondec_name, *dec_name, *nondec_suffix, *dec_suffix;
5766 char *p;
5768 /* If this is a decimal conversion, add the current BID vs. DPD prefix that
5769 depends on which underlying decimal floating point format is used. */
5770 const size_t dec_len = sizeof (DECIMAL_PREFIX) - 1;
5772 mname_len = strlen (GET_MODE_NAME (tmode)) + strlen (GET_MODE_NAME (fmode));
5774 nondec_name = XALLOCAVEC (char, prefix_len + opname_len + mname_len + 1 + 1);
5775 nondec_name[0] = '_';
5776 nondec_name[1] = '_';
5777 if (targetm.libfunc_gnu_prefix)
5779 nondec_name[2] = 'g';
5780 nondec_name[3] = 'n';
5781 nondec_name[4] = 'u';
5782 nondec_name[5] = '_';
5785 memcpy (&nondec_name[prefix_len], opname, opname_len);
5786 nondec_suffix = nondec_name + opname_len + prefix_len;
5788 dec_name = XALLOCAVEC (char, 2 + dec_len + opname_len + mname_len + 1 + 1);
5789 dec_name[0] = '_';
5790 dec_name[1] = '_';
5791 memcpy (&dec_name[2], DECIMAL_PREFIX, dec_len);
5792 memcpy (&dec_name[2+dec_len], opname, opname_len);
5793 dec_suffix = dec_name + dec_len + opname_len + 2;
5795 fname = GET_MODE_NAME (fmode);
5796 tname = GET_MODE_NAME (tmode);
5798 if (DECIMAL_FLOAT_MODE_P (fmode) || DECIMAL_FLOAT_MODE_P (tmode))
5800 libfunc_name = dec_name;
5801 suffix = dec_suffix;
5803 else
5805 libfunc_name = nondec_name;
5806 suffix = nondec_suffix;
5809 p = suffix;
5810 for (q = fname; *q; p++, q++)
5811 *p = TOLOWER (*q);
5812 for (q = tname; *q; p++, q++)
5813 *p = TOLOWER (*q);
5815 *p = '\0';
5817 set_conv_libfunc (tab, tmode, fmode,
5818 ggc_alloc_string (libfunc_name, p - libfunc_name));
5821 /* Same as gen_interclass_conv_libfunc but verify that we are producing
5822 int->fp conversion. */
5824 void
5825 gen_int_to_fp_conv_libfunc (convert_optab tab,
5826 const char *opname,
5827 machine_mode tmode,
5828 machine_mode fmode)
5830 if (GET_MODE_CLASS (fmode) != MODE_INT)
5831 return;
5832 if (GET_MODE_CLASS (tmode) != MODE_FLOAT && !DECIMAL_FLOAT_MODE_P (tmode))
5833 return;
5834 gen_interclass_conv_libfunc (tab, opname, tmode, fmode);
5837 /* ufloat_optab is special by using floatun for FP and floatuns decimal fp
5838 naming scheme. */
5840 void
5841 gen_ufloat_conv_libfunc (convert_optab tab,
5842 const char *opname ATTRIBUTE_UNUSED,
5843 machine_mode tmode,
5844 machine_mode fmode)
5846 if (DECIMAL_FLOAT_MODE_P (tmode))
5847 gen_int_to_fp_conv_libfunc (tab, "floatuns", tmode, fmode);
5848 else
5849 gen_int_to_fp_conv_libfunc (tab, "floatun", tmode, fmode);
5852 /* Same as gen_interclass_conv_libfunc but verify that we are producing
5853 fp->int conversion. */
5855 void
5856 gen_int_to_fp_nondecimal_conv_libfunc (convert_optab tab,
5857 const char *opname,
5858 machine_mode tmode,
5859 machine_mode fmode)
5861 if (GET_MODE_CLASS (fmode) != MODE_INT)
5862 return;
5863 if (GET_MODE_CLASS (tmode) != MODE_FLOAT)
5864 return;
5865 gen_interclass_conv_libfunc (tab, opname, tmode, fmode);
5868 /* Same as gen_interclass_conv_libfunc but verify that we are producing
5869 fp->int conversion with no decimal floating point involved. */
5871 void
5872 gen_fp_to_int_conv_libfunc (convert_optab tab,
5873 const char *opname,
5874 machine_mode tmode,
5875 machine_mode fmode)
5877 if (GET_MODE_CLASS (fmode) != MODE_FLOAT && !DECIMAL_FLOAT_MODE_P (fmode))
5878 return;
5879 if (GET_MODE_CLASS (tmode) != MODE_INT)
5880 return;
5881 gen_interclass_conv_libfunc (tab, opname, tmode, fmode);
5884 /* Initialize the libfunc fields of an of an intra-mode-class conversion optab.
5885 The string formation rules are
5886 similar to the ones for init_libfunc, above. */
5888 void
5889 gen_intraclass_conv_libfunc (convert_optab tab, const char *opname,
5890 machine_mode tmode, machine_mode fmode)
5892 size_t opname_len = strlen (opname);
5893 size_t mname_len = 0;
5895 const char *fname, *tname;
5896 const char *q;
5897 int prefix_len = targetm.libfunc_gnu_prefix ? 6 : 2;
5898 char *nondec_name, *dec_name, *nondec_suffix, *dec_suffix;
5899 char *libfunc_name, *suffix;
5900 char *p;
5902 /* If this is a decimal conversion, add the current BID vs. DPD prefix that
5903 depends on which underlying decimal floating point format is used. */
5904 const size_t dec_len = sizeof (DECIMAL_PREFIX) - 1;
5906 mname_len = strlen (GET_MODE_NAME (tmode)) + strlen (GET_MODE_NAME (fmode));
5908 nondec_name = XALLOCAVEC (char, 2 + opname_len + mname_len + 1 + 1);
5909 nondec_name[0] = '_';
5910 nondec_name[1] = '_';
5911 if (targetm.libfunc_gnu_prefix)
5913 nondec_name[2] = 'g';
5914 nondec_name[3] = 'n';
5915 nondec_name[4] = 'u';
5916 nondec_name[5] = '_';
5918 memcpy (&nondec_name[prefix_len], opname, opname_len);
5919 nondec_suffix = nondec_name + opname_len + prefix_len;
5921 dec_name = XALLOCAVEC (char, 2 + dec_len + opname_len + mname_len + 1 + 1);
5922 dec_name[0] = '_';
5923 dec_name[1] = '_';
5924 memcpy (&dec_name[2], DECIMAL_PREFIX, dec_len);
5925 memcpy (&dec_name[2 + dec_len], opname, opname_len);
5926 dec_suffix = dec_name + dec_len + opname_len + 2;
5928 fname = GET_MODE_NAME (fmode);
5929 tname = GET_MODE_NAME (tmode);
5931 if (DECIMAL_FLOAT_MODE_P (fmode) || DECIMAL_FLOAT_MODE_P (tmode))
5933 libfunc_name = dec_name;
5934 suffix = dec_suffix;
5936 else
5938 libfunc_name = nondec_name;
5939 suffix = nondec_suffix;
5942 p = suffix;
5943 for (q = fname; *q; p++, q++)
5944 *p = TOLOWER (*q);
5945 for (q = tname; *q; p++, q++)
5946 *p = TOLOWER (*q);
5948 *p++ = '2';
5949 *p = '\0';
5951 set_conv_libfunc (tab, tmode, fmode,
5952 ggc_alloc_string (libfunc_name, p - libfunc_name));
5955 /* Pick proper libcall for trunc_optab. We need to chose if we do
5956 truncation or extension and interclass or intraclass. */
5958 void
5959 gen_trunc_conv_libfunc (convert_optab tab,
5960 const char *opname,
5961 machine_mode tmode,
5962 machine_mode fmode)
5964 if (GET_MODE_CLASS (tmode) != MODE_FLOAT && !DECIMAL_FLOAT_MODE_P (tmode))
5965 return;
5966 if (GET_MODE_CLASS (fmode) != MODE_FLOAT && !DECIMAL_FLOAT_MODE_P (fmode))
5967 return;
5968 if (tmode == fmode)
5969 return;
5971 if ((GET_MODE_CLASS (tmode) == MODE_FLOAT && DECIMAL_FLOAT_MODE_P (fmode))
5972 || (GET_MODE_CLASS (fmode) == MODE_FLOAT && DECIMAL_FLOAT_MODE_P (tmode)))
5973 gen_interclass_conv_libfunc (tab, opname, tmode, fmode);
5975 if (GET_MODE_PRECISION (fmode) <= GET_MODE_PRECISION (tmode))
5976 return;
5978 if ((GET_MODE_CLASS (tmode) == MODE_FLOAT
5979 && GET_MODE_CLASS (fmode) == MODE_FLOAT)
5980 || (DECIMAL_FLOAT_MODE_P (fmode) && DECIMAL_FLOAT_MODE_P (tmode)))
5981 gen_intraclass_conv_libfunc (tab, opname, tmode, fmode);
5984 /* Pick proper libcall for extend_optab. We need to chose if we do
5985 truncation or extension and interclass or intraclass. */
5987 void
5988 gen_extend_conv_libfunc (convert_optab tab,
5989 const char *opname ATTRIBUTE_UNUSED,
5990 machine_mode tmode,
5991 machine_mode fmode)
5993 if (GET_MODE_CLASS (tmode) != MODE_FLOAT && !DECIMAL_FLOAT_MODE_P (tmode))
5994 return;
5995 if (GET_MODE_CLASS (fmode) != MODE_FLOAT && !DECIMAL_FLOAT_MODE_P (fmode))
5996 return;
5997 if (tmode == fmode)
5998 return;
6000 if ((GET_MODE_CLASS (tmode) == MODE_FLOAT && DECIMAL_FLOAT_MODE_P (fmode))
6001 || (GET_MODE_CLASS (fmode) == MODE_FLOAT && DECIMAL_FLOAT_MODE_P (tmode)))
6002 gen_interclass_conv_libfunc (tab, opname, tmode, fmode);
6004 if (GET_MODE_PRECISION (fmode) > GET_MODE_PRECISION (tmode))
6005 return;
6007 if ((GET_MODE_CLASS (tmode) == MODE_FLOAT
6008 && GET_MODE_CLASS (fmode) == MODE_FLOAT)
6009 || (DECIMAL_FLOAT_MODE_P (fmode) && DECIMAL_FLOAT_MODE_P (tmode)))
6010 gen_intraclass_conv_libfunc (tab, opname, tmode, fmode);
6013 /* Pick proper libcall for fract_optab. We need to chose if we do
6014 interclass or intraclass. */
6016 void
6017 gen_fract_conv_libfunc (convert_optab tab,
6018 const char *opname,
6019 machine_mode tmode,
6020 machine_mode fmode)
6022 if (tmode == fmode)
6023 return;
6024 if (!(ALL_FIXED_POINT_MODE_P (tmode) || ALL_FIXED_POINT_MODE_P (fmode)))
6025 return;
6027 if (GET_MODE_CLASS (tmode) == GET_MODE_CLASS (fmode))
6028 gen_intraclass_conv_libfunc (tab, opname, tmode, fmode);
6029 else
6030 gen_interclass_conv_libfunc (tab, opname, tmode, fmode);
6033 /* Pick proper libcall for fractuns_optab. */
6035 void
6036 gen_fractuns_conv_libfunc (convert_optab tab,
6037 const char *opname,
6038 machine_mode tmode,
6039 machine_mode fmode)
6041 if (tmode == fmode)
6042 return;
6043 /* One mode must be a fixed-point mode, and the other must be an integer
6044 mode. */
6045 if (!((ALL_FIXED_POINT_MODE_P (tmode) && GET_MODE_CLASS (fmode) == MODE_INT)
6046 || (ALL_FIXED_POINT_MODE_P (fmode)
6047 && GET_MODE_CLASS (tmode) == MODE_INT)))
6048 return;
6050 gen_interclass_conv_libfunc (tab, opname, tmode, fmode);
6053 /* Pick proper libcall for satfract_optab. We need to chose if we do
6054 interclass or intraclass. */
6056 void
6057 gen_satfract_conv_libfunc (convert_optab tab,
6058 const char *opname,
6059 machine_mode tmode,
6060 machine_mode fmode)
6062 if (tmode == fmode)
6063 return;
6064 /* TMODE must be a fixed-point mode. */
6065 if (!ALL_FIXED_POINT_MODE_P (tmode))
6066 return;
6068 if (GET_MODE_CLASS (tmode) == GET_MODE_CLASS (fmode))
6069 gen_intraclass_conv_libfunc (tab, opname, tmode, fmode);
6070 else
6071 gen_interclass_conv_libfunc (tab, opname, tmode, fmode);
6074 /* Pick proper libcall for satfractuns_optab. */
6076 void
6077 gen_satfractuns_conv_libfunc (convert_optab tab,
6078 const char *opname,
6079 machine_mode tmode,
6080 machine_mode fmode)
6082 if (tmode == fmode)
6083 return;
6084 /* TMODE must be a fixed-point mode, and FMODE must be an integer mode. */
6085 if (!(ALL_FIXED_POINT_MODE_P (tmode) && GET_MODE_CLASS (fmode) == MODE_INT))
6086 return;
6088 gen_interclass_conv_libfunc (tab, opname, tmode, fmode);
6091 /* Hashtable callbacks for libfunc_decls. */
6093 struct libfunc_decl_hasher : ggc_hasher<tree>
6095 static hashval_t
6096 hash (tree entry)
6098 return IDENTIFIER_HASH_VALUE (DECL_NAME (entry));
6101 static bool
6102 equal (tree decl, tree name)
6104 return DECL_NAME (decl) == name;
6108 /* A table of previously-created libfuncs, hashed by name. */
6109 static GTY (()) hash_table<libfunc_decl_hasher> *libfunc_decls;
6111 /* Build a decl for a libfunc named NAME. */
6113 tree
6114 build_libfunc_function (const char *name)
6116 tree decl = build_decl (UNKNOWN_LOCATION, FUNCTION_DECL,
6117 get_identifier (name),
6118 build_function_type (integer_type_node, NULL_TREE));
6119 /* ??? We don't have any type information except for this is
6120 a function. Pretend this is "int foo()". */
6121 DECL_ARTIFICIAL (decl) = 1;
6122 DECL_EXTERNAL (decl) = 1;
6123 TREE_PUBLIC (decl) = 1;
6124 gcc_assert (DECL_ASSEMBLER_NAME (decl));
6126 /* Zap the nonsensical SYMBOL_REF_DECL for this. What we're left with
6127 are the flags assigned by targetm.encode_section_info. */
6128 SET_SYMBOL_REF_DECL (XEXP (DECL_RTL (decl), 0), NULL);
6130 return decl;
6134 init_one_libfunc (const char *name)
6136 tree id, decl;
6137 hashval_t hash;
6139 if (libfunc_decls == NULL)
6140 libfunc_decls = hash_table<libfunc_decl_hasher>::create_ggc (37);
6142 /* See if we have already created a libfunc decl for this function. */
6143 id = get_identifier (name);
6144 hash = IDENTIFIER_HASH_VALUE (id);
6145 tree *slot = libfunc_decls->find_slot_with_hash (id, hash, INSERT);
6146 decl = *slot;
6147 if (decl == NULL)
6149 /* Create a new decl, so that it can be passed to
6150 targetm.encode_section_info. */
6151 decl = build_libfunc_function (name);
6152 *slot = decl;
6154 return XEXP (DECL_RTL (decl), 0);
6157 /* Adjust the assembler name of libfunc NAME to ASMSPEC. */
6160 set_user_assembler_libfunc (const char *name, const char *asmspec)
6162 tree id, decl;
6163 hashval_t hash;
6165 id = get_identifier (name);
6166 hash = IDENTIFIER_HASH_VALUE (id);
6167 tree *slot = libfunc_decls->find_slot_with_hash (id, hash, NO_INSERT);
6168 gcc_assert (slot);
6169 decl = (tree) *slot;
6170 set_user_assembler_name (decl, asmspec);
6171 return XEXP (DECL_RTL (decl), 0);
6174 /* Call this to reset the function entry for one optab (OPTABLE) in mode
6175 MODE to NAME, which should be either 0 or a string constant. */
6176 void
6177 set_optab_libfunc (optab op, machine_mode mode, const char *name)
6179 rtx val;
6180 struct libfunc_entry e;
6181 struct libfunc_entry **slot;
6183 e.op = op;
6184 e.mode1 = mode;
6185 e.mode2 = VOIDmode;
6187 if (name)
6188 val = init_one_libfunc (name);
6189 else
6190 val = 0;
6191 slot = libfunc_hash->find_slot (&e, INSERT);
6192 if (*slot == NULL)
6193 *slot = ggc_alloc<libfunc_entry> ();
6194 (*slot)->op = op;
6195 (*slot)->mode1 = mode;
6196 (*slot)->mode2 = VOIDmode;
6197 (*slot)->libfunc = val;
6200 /* Call this to reset the function entry for one conversion optab
6201 (OPTABLE) from mode FMODE to mode TMODE to NAME, which should be
6202 either 0 or a string constant. */
6203 void
6204 set_conv_libfunc (convert_optab optab, machine_mode tmode,
6205 machine_mode fmode, const char *name)
6207 rtx val;
6208 struct libfunc_entry e;
6209 struct libfunc_entry **slot;
6211 e.op = optab;
6212 e.mode1 = tmode;
6213 e.mode2 = fmode;
6215 if (name)
6216 val = init_one_libfunc (name);
6217 else
6218 val = 0;
6219 slot = libfunc_hash->find_slot (&e, INSERT);
6220 if (*slot == NULL)
6221 *slot = ggc_alloc<libfunc_entry> ();
6222 (*slot)->op = optab;
6223 (*slot)->mode1 = tmode;
6224 (*slot)->mode2 = fmode;
6225 (*slot)->libfunc = val;
6228 /* Call this to initialize the contents of the optabs
6229 appropriately for the current target machine. */
6231 void
6232 init_optabs (void)
6234 if (libfunc_hash)
6235 libfunc_hash->empty ();
6236 else
6237 libfunc_hash = hash_table<libfunc_hasher>::create_ggc (10);
6239 /* Fill in the optabs with the insns we support. */
6240 init_all_optabs (this_fn_optabs);
6242 /* The ffs function operates on `int'. Fall back on it if we do not
6243 have a libgcc2 function for that width. */
6244 if (INT_TYPE_SIZE < BITS_PER_WORD)
6245 set_optab_libfunc (ffs_optab, mode_for_size (INT_TYPE_SIZE, MODE_INT, 0),
6246 "ffs");
6248 /* Explicitly initialize the bswap libfuncs since we need them to be
6249 valid for things other than word_mode. */
6250 if (targetm.libfunc_gnu_prefix)
6252 set_optab_libfunc (bswap_optab, SImode, "__gnu_bswapsi2");
6253 set_optab_libfunc (bswap_optab, DImode, "__gnu_bswapdi2");
6255 else
6257 set_optab_libfunc (bswap_optab, SImode, "__bswapsi2");
6258 set_optab_libfunc (bswap_optab, DImode, "__bswapdi2");
6261 /* Use cabs for double complex abs, since systems generally have cabs.
6262 Don't define any libcall for float complex, so that cabs will be used. */
6263 if (complex_double_type_node)
6264 set_optab_libfunc (abs_optab, TYPE_MODE (complex_double_type_node),
6265 "cabs");
6267 abort_libfunc = init_one_libfunc ("abort");
6268 memcpy_libfunc = init_one_libfunc ("memcpy");
6269 memmove_libfunc = init_one_libfunc ("memmove");
6270 memcmp_libfunc = init_one_libfunc ("memcmp");
6271 memset_libfunc = init_one_libfunc ("memset");
6272 setbits_libfunc = init_one_libfunc ("__setbits");
6274 #ifndef DONT_USE_BUILTIN_SETJMP
6275 setjmp_libfunc = init_one_libfunc ("__builtin_setjmp");
6276 longjmp_libfunc = init_one_libfunc ("__builtin_longjmp");
6277 #else
6278 setjmp_libfunc = init_one_libfunc ("setjmp");
6279 longjmp_libfunc = init_one_libfunc ("longjmp");
6280 #endif
6281 unwind_sjlj_register_libfunc = init_one_libfunc ("_Unwind_SjLj_Register");
6282 unwind_sjlj_unregister_libfunc
6283 = init_one_libfunc ("_Unwind_SjLj_Unregister");
6285 /* For function entry/exit instrumentation. */
6286 profile_function_entry_libfunc
6287 = init_one_libfunc ("__cyg_profile_func_enter");
6288 profile_function_exit_libfunc
6289 = init_one_libfunc ("__cyg_profile_func_exit");
6291 gcov_flush_libfunc = init_one_libfunc ("__gcov_flush");
6293 /* Allow the target to add more libcalls or rename some, etc. */
6294 targetm.init_libfuncs ();
6297 /* Use the current target and options to initialize
6298 TREE_OPTIMIZATION_OPTABS (OPTNODE). */
6300 void
6301 init_tree_optimization_optabs (tree optnode)
6303 /* Quick exit if we have already computed optabs for this target. */
6304 if (TREE_OPTIMIZATION_BASE_OPTABS (optnode) == this_target_optabs)
6305 return;
6307 /* Forget any previous information and set up for the current target. */
6308 TREE_OPTIMIZATION_BASE_OPTABS (optnode) = this_target_optabs;
6309 struct target_optabs *tmp_optabs = (struct target_optabs *)
6310 TREE_OPTIMIZATION_OPTABS (optnode);
6311 if (tmp_optabs)
6312 memset (tmp_optabs, 0, sizeof (struct target_optabs));
6313 else
6314 tmp_optabs = ggc_alloc<target_optabs> ();
6316 /* Generate a new set of optabs into tmp_optabs. */
6317 init_all_optabs (tmp_optabs);
6319 /* If the optabs changed, record it. */
6320 if (memcmp (tmp_optabs, this_target_optabs, sizeof (struct target_optabs)))
6321 TREE_OPTIMIZATION_OPTABS (optnode) = tmp_optabs;
6322 else
6324 TREE_OPTIMIZATION_OPTABS (optnode) = NULL;
6325 ggc_free (tmp_optabs);
6329 /* A helper function for init_sync_libfuncs. Using the basename BASE,
6330 install libfuncs into TAB for BASE_N for 1 <= N <= MAX. */
6332 static void
6333 init_sync_libfuncs_1 (optab tab, const char *base, int max)
6335 machine_mode mode;
6336 char buf[64];
6337 size_t len = strlen (base);
6338 int i;
6340 gcc_assert (max <= 8);
6341 gcc_assert (len + 3 < sizeof (buf));
6343 memcpy (buf, base, len);
6344 buf[len] = '_';
6345 buf[len + 1] = '0';
6346 buf[len + 2] = '\0';
6348 mode = QImode;
6349 for (i = 1; i <= max; i *= 2)
6351 buf[len + 1] = '0' + i;
6352 set_optab_libfunc (tab, mode, buf);
6353 mode = GET_MODE_2XWIDER_MODE (mode);
6357 void
6358 init_sync_libfuncs (int max)
6360 if (!flag_sync_libcalls)
6361 return;
6363 init_sync_libfuncs_1 (sync_compare_and_swap_optab,
6364 "__sync_val_compare_and_swap", max);
6365 init_sync_libfuncs_1 (sync_lock_test_and_set_optab,
6366 "__sync_lock_test_and_set", max);
6368 init_sync_libfuncs_1 (sync_old_add_optab, "__sync_fetch_and_add", max);
6369 init_sync_libfuncs_1 (sync_old_sub_optab, "__sync_fetch_and_sub", max);
6370 init_sync_libfuncs_1 (sync_old_ior_optab, "__sync_fetch_and_or", max);
6371 init_sync_libfuncs_1 (sync_old_and_optab, "__sync_fetch_and_and", max);
6372 init_sync_libfuncs_1 (sync_old_xor_optab, "__sync_fetch_and_xor", max);
6373 init_sync_libfuncs_1 (sync_old_nand_optab, "__sync_fetch_and_nand", max);
6375 init_sync_libfuncs_1 (sync_new_add_optab, "__sync_add_and_fetch", max);
6376 init_sync_libfuncs_1 (sync_new_sub_optab, "__sync_sub_and_fetch", max);
6377 init_sync_libfuncs_1 (sync_new_ior_optab, "__sync_or_and_fetch", max);
6378 init_sync_libfuncs_1 (sync_new_and_optab, "__sync_and_and_fetch", max);
6379 init_sync_libfuncs_1 (sync_new_xor_optab, "__sync_xor_and_fetch", max);
6380 init_sync_libfuncs_1 (sync_new_nand_optab, "__sync_nand_and_fetch", max);
6383 /* Print information about the current contents of the optabs on
6384 STDERR. */
6386 DEBUG_FUNCTION void
6387 debug_optab_libfuncs (void)
6389 int i, j, k;
6391 /* Dump the arithmetic optabs. */
6392 for (i = FIRST_NORM_OPTAB; i <= LAST_NORMLIB_OPTAB; ++i)
6393 for (j = 0; j < NUM_MACHINE_MODES; ++j)
6395 rtx l = optab_libfunc ((optab) i, (machine_mode) j);
6396 if (l)
6398 gcc_assert (GET_CODE (l) == SYMBOL_REF);
6399 fprintf (stderr, "%s\t%s:\t%s\n",
6400 GET_RTX_NAME (optab_to_code ((optab) i)),
6401 GET_MODE_NAME (j),
6402 XSTR (l, 0));
6406 /* Dump the conversion optabs. */
6407 for (i = FIRST_CONV_OPTAB; i <= LAST_CONVLIB_OPTAB; ++i)
6408 for (j = 0; j < NUM_MACHINE_MODES; ++j)
6409 for (k = 0; k < NUM_MACHINE_MODES; ++k)
6411 rtx l = convert_optab_libfunc ((optab) i, (machine_mode) j,
6412 (machine_mode) k);
6413 if (l)
6415 gcc_assert (GET_CODE (l) == SYMBOL_REF);
6416 fprintf (stderr, "%s\t%s\t%s:\t%s\n",
6417 GET_RTX_NAME (optab_to_code ((optab) i)),
6418 GET_MODE_NAME (j),
6419 GET_MODE_NAME (k),
6420 XSTR (l, 0));
6426 /* Generate insns to trap with code TCODE if OP1 and OP2 satisfy condition
6427 CODE. Return 0 on failure. */
6430 gen_cond_trap (enum rtx_code code, rtx op1, rtx op2, rtx tcode)
6432 machine_mode mode = GET_MODE (op1);
6433 enum insn_code icode;
6434 rtx insn;
6435 rtx trap_rtx;
6437 if (mode == VOIDmode)
6438 return 0;
6440 icode = optab_handler (ctrap_optab, mode);
6441 if (icode == CODE_FOR_nothing)
6442 return 0;
6444 /* Some targets only accept a zero trap code. */
6445 if (!insn_operand_matches (icode, 3, tcode))
6446 return 0;
6448 do_pending_stack_adjust ();
6449 start_sequence ();
6450 prepare_cmp_insn (op1, op2, code, NULL_RTX, false, OPTAB_DIRECT,
6451 &trap_rtx, &mode);
6452 if (!trap_rtx)
6453 insn = NULL_RTX;
6454 else
6455 insn = GEN_FCN (icode) (trap_rtx, XEXP (trap_rtx, 0), XEXP (trap_rtx, 1),
6456 tcode);
6458 /* If that failed, then give up. */
6459 if (insn == 0)
6461 end_sequence ();
6462 return 0;
6465 emit_insn (insn);
6466 insn = get_insns ();
6467 end_sequence ();
6468 return insn;
6471 /* Return rtx code for TCODE. Use UNSIGNEDP to select signed
6472 or unsigned operation code. */
6474 enum rtx_code
6475 get_rtx_code (enum tree_code tcode, bool unsignedp)
6477 enum rtx_code code;
6478 switch (tcode)
6480 case EQ_EXPR:
6481 code = EQ;
6482 break;
6483 case NE_EXPR:
6484 code = NE;
6485 break;
6486 case LT_EXPR:
6487 code = unsignedp ? LTU : LT;
6488 break;
6489 case LE_EXPR:
6490 code = unsignedp ? LEU : LE;
6491 break;
6492 case GT_EXPR:
6493 code = unsignedp ? GTU : GT;
6494 break;
6495 case GE_EXPR:
6496 code = unsignedp ? GEU : GE;
6497 break;
6499 case UNORDERED_EXPR:
6500 code = UNORDERED;
6501 break;
6502 case ORDERED_EXPR:
6503 code = ORDERED;
6504 break;
6505 case UNLT_EXPR:
6506 code = UNLT;
6507 break;
6508 case UNLE_EXPR:
6509 code = UNLE;
6510 break;
6511 case UNGT_EXPR:
6512 code = UNGT;
6513 break;
6514 case UNGE_EXPR:
6515 code = UNGE;
6516 break;
6517 case UNEQ_EXPR:
6518 code = UNEQ;
6519 break;
6520 case LTGT_EXPR:
6521 code = LTGT;
6522 break;
6524 case BIT_AND_EXPR:
6525 code = AND;
6526 break;
6528 case BIT_IOR_EXPR:
6529 code = IOR;
6530 break;
6532 default:
6533 gcc_unreachable ();
6535 return code;
6538 /* Return comparison rtx for COND. Use UNSIGNEDP to select signed or
6539 unsigned operators. Do not generate compare instruction. */
6541 static rtx
6542 vector_compare_rtx (enum tree_code tcode, tree t_op0, tree t_op1,
6543 bool unsignedp, enum insn_code icode)
6545 struct expand_operand ops[2];
6546 rtx rtx_op0, rtx_op1;
6547 enum rtx_code rcode = get_rtx_code (tcode, unsignedp);
6549 gcc_assert (TREE_CODE_CLASS (tcode) == tcc_comparison);
6551 /* Expand operands. */
6552 rtx_op0 = expand_expr (t_op0, NULL_RTX, TYPE_MODE (TREE_TYPE (t_op0)),
6553 EXPAND_STACK_PARM);
6554 rtx_op1 = expand_expr (t_op1, NULL_RTX, TYPE_MODE (TREE_TYPE (t_op1)),
6555 EXPAND_STACK_PARM);
6557 create_input_operand (&ops[0], rtx_op0, GET_MODE (rtx_op0));
6558 create_input_operand (&ops[1], rtx_op1, GET_MODE (rtx_op1));
6559 if (!maybe_legitimize_operands (icode, 4, 2, ops))
6560 gcc_unreachable ();
6561 return gen_rtx_fmt_ee (rcode, VOIDmode, ops[0].value, ops[1].value);
6564 /* Return true if VEC_PERM_EXPR of arbitrary input vectors can be expanded using
6565 SIMD extensions of the CPU. SEL may be NULL, which stands for an unknown
6566 constant. Note that additional permutations representing whole-vector shifts
6567 may also be handled via the vec_shr optab, but only where the second input
6568 vector is entirely constant zeroes; this case is not dealt with here. */
6570 bool
6571 can_vec_perm_p (machine_mode mode, bool variable,
6572 const unsigned char *sel)
6574 machine_mode qimode;
6576 /* If the target doesn't implement a vector mode for the vector type,
6577 then no operations are supported. */
6578 if (!VECTOR_MODE_P (mode))
6579 return false;
6581 if (!variable)
6583 if (direct_optab_handler (vec_perm_const_optab, mode) != CODE_FOR_nothing
6584 && (sel == NULL
6585 || targetm.vectorize.vec_perm_const_ok == NULL
6586 || targetm.vectorize.vec_perm_const_ok (mode, sel)))
6587 return true;
6590 if (direct_optab_handler (vec_perm_optab, mode) != CODE_FOR_nothing)
6591 return true;
6593 /* We allow fallback to a QI vector mode, and adjust the mask. */
6594 if (GET_MODE_INNER (mode) == QImode)
6595 return false;
6596 qimode = mode_for_vector (QImode, GET_MODE_SIZE (mode));
6597 if (!VECTOR_MODE_P (qimode))
6598 return false;
6600 /* ??? For completeness, we ought to check the QImode version of
6601 vec_perm_const_optab. But all users of this implicit lowering
6602 feature implement the variable vec_perm_optab. */
6603 if (direct_optab_handler (vec_perm_optab, qimode) == CODE_FOR_nothing)
6604 return false;
6606 /* In order to support the lowering of variable permutations,
6607 we need to support shifts and adds. */
6608 if (variable)
6610 if (GET_MODE_UNIT_SIZE (mode) > 2
6611 && optab_handler (ashl_optab, mode) == CODE_FOR_nothing
6612 && optab_handler (vashl_optab, mode) == CODE_FOR_nothing)
6613 return false;
6614 if (optab_handler (add_optab, qimode) == CODE_FOR_nothing)
6615 return false;
6618 return true;
6621 /* Checks if vec_perm mask SEL is a constant equivalent to a shift of the first
6622 vec_perm operand, assuming the second operand is a constant vector of zeroes.
6623 Return the shift distance in bits if so, or NULL_RTX if the vec_perm is not a
6624 shift. */
6625 static rtx
6626 shift_amt_for_vec_perm_mask (rtx sel)
6628 unsigned int i, first, nelt = GET_MODE_NUNITS (GET_MODE (sel));
6629 unsigned int bitsize = GET_MODE_BITSIZE (GET_MODE_INNER (GET_MODE (sel)));
6631 if (GET_CODE (sel) != CONST_VECTOR)
6632 return NULL_RTX;
6634 first = INTVAL (CONST_VECTOR_ELT (sel, 0));
6635 if (first >= 2*nelt)
6636 return NULL_RTX;
6637 for (i = 1; i < nelt; i++)
6639 int idx = INTVAL (CONST_VECTOR_ELT (sel, i));
6640 unsigned int expected = (i + first) & (2 * nelt - 1);
6641 /* Indices into the second vector are all equivalent. */
6642 if (idx < 0 || (MIN (nelt, (unsigned) idx) != MIN (nelt, expected)))
6643 return NULL_RTX;
6646 return GEN_INT (first * bitsize);
6649 /* A subroutine of expand_vec_perm for expanding one vec_perm insn. */
6651 static rtx
6652 expand_vec_perm_1 (enum insn_code icode, rtx target,
6653 rtx v0, rtx v1, rtx sel)
6655 machine_mode tmode = GET_MODE (target);
6656 machine_mode smode = GET_MODE (sel);
6657 struct expand_operand ops[4];
6659 create_output_operand (&ops[0], target, tmode);
6660 create_input_operand (&ops[3], sel, smode);
6662 /* Make an effort to preserve v0 == v1. The target expander is able to
6663 rely on this to determine if we're permuting a single input operand. */
6664 if (rtx_equal_p (v0, v1))
6666 if (!insn_operand_matches (icode, 1, v0))
6667 v0 = force_reg (tmode, v0);
6668 gcc_checking_assert (insn_operand_matches (icode, 1, v0));
6669 gcc_checking_assert (insn_operand_matches (icode, 2, v0));
6671 create_fixed_operand (&ops[1], v0);
6672 create_fixed_operand (&ops[2], v0);
6674 else
6676 create_input_operand (&ops[1], v0, tmode);
6677 /* See if this can be handled with a vec_shr. We only do this if the
6678 second vector is all zeroes. */
6679 enum insn_code shift_code = optab_handler (vec_shr_optab, GET_MODE (v0));
6680 if (v1 == CONST0_RTX (GET_MODE (v1)) && shift_code)
6681 if (rtx shift_amt = shift_amt_for_vec_perm_mask (sel))
6683 create_convert_operand_from_type (&ops[2], shift_amt,
6684 sizetype_tab[(int) stk_sizetype]);
6685 if (maybe_expand_insn (shift_code, 3, ops))
6686 return ops[0].value;
6688 create_input_operand (&ops[2], v1, tmode);
6691 if (maybe_expand_insn (icode, 4, ops))
6692 return ops[0].value;
6693 return NULL_RTX;
6696 /* Generate instructions for vec_perm optab given its mode
6697 and three operands. */
6700 expand_vec_perm (machine_mode mode, rtx v0, rtx v1, rtx sel, rtx target)
6702 enum insn_code icode;
6703 machine_mode qimode;
6704 unsigned int i, w, e, u;
6705 rtx tmp, sel_qi = NULL;
6706 rtvec vec;
6708 if (!target || GET_MODE (target) != mode)
6709 target = gen_reg_rtx (mode);
6711 w = GET_MODE_SIZE (mode);
6712 e = GET_MODE_NUNITS (mode);
6713 u = GET_MODE_UNIT_SIZE (mode);
6715 /* Set QIMODE to a different vector mode with byte elements.
6716 If no such mode, or if MODE already has byte elements, use VOIDmode. */
6717 qimode = VOIDmode;
6718 if (GET_MODE_INNER (mode) != QImode)
6720 qimode = mode_for_vector (QImode, w);
6721 if (!VECTOR_MODE_P (qimode))
6722 qimode = VOIDmode;
6725 /* If the input is a constant, expand it specially. */
6726 gcc_assert (GET_MODE_CLASS (GET_MODE (sel)) == MODE_VECTOR_INT);
6727 if (GET_CODE (sel) == CONST_VECTOR)
6729 icode = direct_optab_handler (vec_perm_const_optab, mode);
6730 if (icode != CODE_FOR_nothing)
6732 tmp = expand_vec_perm_1 (icode, target, v0, v1, sel);
6733 if (tmp)
6734 return tmp;
6737 /* Fall back to a constant byte-based permutation. */
6738 if (qimode != VOIDmode)
6740 vec = rtvec_alloc (w);
6741 for (i = 0; i < e; ++i)
6743 unsigned int j, this_e;
6745 this_e = INTVAL (CONST_VECTOR_ELT (sel, i));
6746 this_e &= 2 * e - 1;
6747 this_e *= u;
6749 for (j = 0; j < u; ++j)
6750 RTVEC_ELT (vec, i * u + j) = GEN_INT (this_e + j);
6752 sel_qi = gen_rtx_CONST_VECTOR (qimode, vec);
6754 icode = direct_optab_handler (vec_perm_const_optab, qimode);
6755 if (icode != CODE_FOR_nothing)
6757 tmp = mode != qimode ? gen_reg_rtx (qimode) : target;
6758 tmp = expand_vec_perm_1 (icode, tmp, gen_lowpart (qimode, v0),
6759 gen_lowpart (qimode, v1), sel_qi);
6760 if (tmp)
6761 return gen_lowpart (mode, tmp);
6766 /* Otherwise expand as a fully variable permuation. */
6767 icode = direct_optab_handler (vec_perm_optab, mode);
6768 if (icode != CODE_FOR_nothing)
6770 tmp = expand_vec_perm_1 (icode, target, v0, v1, sel);
6771 if (tmp)
6772 return tmp;
6775 /* As a special case to aid several targets, lower the element-based
6776 permutation to a byte-based permutation and try again. */
6777 if (qimode == VOIDmode)
6778 return NULL_RTX;
6779 icode = direct_optab_handler (vec_perm_optab, qimode);
6780 if (icode == CODE_FOR_nothing)
6781 return NULL_RTX;
6783 if (sel_qi == NULL)
6785 /* Multiply each element by its byte size. */
6786 machine_mode selmode = GET_MODE (sel);
6787 if (u == 2)
6788 sel = expand_simple_binop (selmode, PLUS, sel, sel,
6789 sel, 0, OPTAB_DIRECT);
6790 else
6791 sel = expand_simple_binop (selmode, ASHIFT, sel,
6792 GEN_INT (exact_log2 (u)),
6793 sel, 0, OPTAB_DIRECT);
6794 gcc_assert (sel != NULL);
6796 /* Broadcast the low byte each element into each of its bytes. */
6797 vec = rtvec_alloc (w);
6798 for (i = 0; i < w; ++i)
6800 int this_e = i / u * u;
6801 if (BYTES_BIG_ENDIAN)
6802 this_e += u - 1;
6803 RTVEC_ELT (vec, i) = GEN_INT (this_e);
6805 tmp = gen_rtx_CONST_VECTOR (qimode, vec);
6806 sel = gen_lowpart (qimode, sel);
6807 sel = expand_vec_perm (qimode, sel, sel, tmp, NULL);
6808 gcc_assert (sel != NULL);
6810 /* Add the byte offset to each byte element. */
6811 /* Note that the definition of the indicies here is memory ordering,
6812 so there should be no difference between big and little endian. */
6813 vec = rtvec_alloc (w);
6814 for (i = 0; i < w; ++i)
6815 RTVEC_ELT (vec, i) = GEN_INT (i % u);
6816 tmp = gen_rtx_CONST_VECTOR (qimode, vec);
6817 sel_qi = expand_simple_binop (qimode, PLUS, sel, tmp,
6818 sel, 0, OPTAB_DIRECT);
6819 gcc_assert (sel_qi != NULL);
6822 tmp = mode != qimode ? gen_reg_rtx (qimode) : target;
6823 tmp = expand_vec_perm_1 (icode, tmp, gen_lowpart (qimode, v0),
6824 gen_lowpart (qimode, v1), sel_qi);
6825 if (tmp)
6826 tmp = gen_lowpart (mode, tmp);
6827 return tmp;
6830 /* Return insn code for a conditional operator with a comparison in
6831 mode CMODE, unsigned if UNS is true, resulting in a value of mode VMODE. */
6833 static inline enum insn_code
6834 get_vcond_icode (machine_mode vmode, machine_mode cmode, bool uns)
6836 enum insn_code icode = CODE_FOR_nothing;
6837 if (uns)
6838 icode = convert_optab_handler (vcondu_optab, vmode, cmode);
6839 else
6840 icode = convert_optab_handler (vcond_optab, vmode, cmode);
6841 return icode;
6844 /* Return TRUE iff, appropriate vector insns are available
6845 for vector cond expr with vector type VALUE_TYPE and a comparison
6846 with operand vector types in CMP_OP_TYPE. */
6848 bool
6849 expand_vec_cond_expr_p (tree value_type, tree cmp_op_type)
6851 machine_mode value_mode = TYPE_MODE (value_type);
6852 machine_mode cmp_op_mode = TYPE_MODE (cmp_op_type);
6853 if (GET_MODE_SIZE (value_mode) != GET_MODE_SIZE (cmp_op_mode)
6854 || GET_MODE_NUNITS (value_mode) != GET_MODE_NUNITS (cmp_op_mode)
6855 || get_vcond_icode (TYPE_MODE (value_type), TYPE_MODE (cmp_op_type),
6856 TYPE_UNSIGNED (cmp_op_type)) == CODE_FOR_nothing)
6857 return false;
6858 return true;
6861 /* Generate insns for a VEC_COND_EXPR, given its TYPE and its
6862 three operands. */
6865 expand_vec_cond_expr (tree vec_cond_type, tree op0, tree op1, tree op2,
6866 rtx target)
6868 struct expand_operand ops[6];
6869 enum insn_code icode;
6870 rtx comparison, rtx_op1, rtx_op2;
6871 machine_mode mode = TYPE_MODE (vec_cond_type);
6872 machine_mode cmp_op_mode;
6873 bool unsignedp;
6874 tree op0a, op0b;
6875 enum tree_code tcode;
6877 if (COMPARISON_CLASS_P (op0))
6879 op0a = TREE_OPERAND (op0, 0);
6880 op0b = TREE_OPERAND (op0, 1);
6881 tcode = TREE_CODE (op0);
6883 else
6885 /* Fake op0 < 0. */
6886 gcc_assert (!TYPE_UNSIGNED (TREE_TYPE (op0)));
6887 op0a = op0;
6888 op0b = build_zero_cst (TREE_TYPE (op0));
6889 tcode = LT_EXPR;
6891 unsignedp = TYPE_UNSIGNED (TREE_TYPE (op0a));
6892 cmp_op_mode = TYPE_MODE (TREE_TYPE (op0a));
6895 gcc_assert (GET_MODE_SIZE (mode) == GET_MODE_SIZE (cmp_op_mode)
6896 && GET_MODE_NUNITS (mode) == GET_MODE_NUNITS (cmp_op_mode));
6898 icode = get_vcond_icode (mode, cmp_op_mode, unsignedp);
6899 if (icode == CODE_FOR_nothing)
6900 return 0;
6902 comparison = vector_compare_rtx (tcode, op0a, op0b, unsignedp, icode);
6903 rtx_op1 = expand_normal (op1);
6904 rtx_op2 = expand_normal (op2);
6906 create_output_operand (&ops[0], target, mode);
6907 create_input_operand (&ops[1], rtx_op1, mode);
6908 create_input_operand (&ops[2], rtx_op2, mode);
6909 create_fixed_operand (&ops[3], comparison);
6910 create_fixed_operand (&ops[4], XEXP (comparison, 0));
6911 create_fixed_operand (&ops[5], XEXP (comparison, 1));
6912 expand_insn (icode, 6, ops);
6913 return ops[0].value;
6916 /* Return non-zero if a highpart multiply is supported of can be synthisized.
6917 For the benefit of expand_mult_highpart, the return value is 1 for direct,
6918 2 for even/odd widening, and 3 for hi/lo widening. */
6921 can_mult_highpart_p (machine_mode mode, bool uns_p)
6923 optab op;
6924 unsigned char *sel;
6925 unsigned i, nunits;
6927 op = uns_p ? umul_highpart_optab : smul_highpart_optab;
6928 if (optab_handler (op, mode) != CODE_FOR_nothing)
6929 return 1;
6931 /* If the mode is an integral vector, synth from widening operations. */
6932 if (GET_MODE_CLASS (mode) != MODE_VECTOR_INT)
6933 return 0;
6935 nunits = GET_MODE_NUNITS (mode);
6936 sel = XALLOCAVEC (unsigned char, nunits);
6938 op = uns_p ? vec_widen_umult_even_optab : vec_widen_smult_even_optab;
6939 if (optab_handler (op, mode) != CODE_FOR_nothing)
6941 op = uns_p ? vec_widen_umult_odd_optab : vec_widen_smult_odd_optab;
6942 if (optab_handler (op, mode) != CODE_FOR_nothing)
6944 for (i = 0; i < nunits; ++i)
6945 sel[i] = !BYTES_BIG_ENDIAN + (i & ~1) + ((i & 1) ? nunits : 0);
6946 if (can_vec_perm_p (mode, false, sel))
6947 return 2;
6951 op = uns_p ? vec_widen_umult_hi_optab : vec_widen_smult_hi_optab;
6952 if (optab_handler (op, mode) != CODE_FOR_nothing)
6954 op = uns_p ? vec_widen_umult_lo_optab : vec_widen_smult_lo_optab;
6955 if (optab_handler (op, mode) != CODE_FOR_nothing)
6957 for (i = 0; i < nunits; ++i)
6958 sel[i] = 2 * i + (BYTES_BIG_ENDIAN ? 0 : 1);
6959 if (can_vec_perm_p (mode, false, sel))
6960 return 3;
6964 return 0;
6967 /* Expand a highpart multiply. */
6970 expand_mult_highpart (machine_mode mode, rtx op0, rtx op1,
6971 rtx target, bool uns_p)
6973 struct expand_operand eops[3];
6974 enum insn_code icode;
6975 int method, i, nunits;
6976 machine_mode wmode;
6977 rtx m1, m2, perm;
6978 optab tab1, tab2;
6979 rtvec v;
6981 method = can_mult_highpart_p (mode, uns_p);
6982 switch (method)
6984 case 0:
6985 return NULL_RTX;
6986 case 1:
6987 tab1 = uns_p ? umul_highpart_optab : smul_highpart_optab;
6988 return expand_binop (mode, tab1, op0, op1, target, uns_p,
6989 OPTAB_LIB_WIDEN);
6990 case 2:
6991 tab1 = uns_p ? vec_widen_umult_even_optab : vec_widen_smult_even_optab;
6992 tab2 = uns_p ? vec_widen_umult_odd_optab : vec_widen_smult_odd_optab;
6993 break;
6994 case 3:
6995 tab1 = uns_p ? vec_widen_umult_lo_optab : vec_widen_smult_lo_optab;
6996 tab2 = uns_p ? vec_widen_umult_hi_optab : vec_widen_smult_hi_optab;
6997 if (BYTES_BIG_ENDIAN)
6999 optab t = tab1;
7000 tab1 = tab2;
7001 tab2 = t;
7003 break;
7004 default:
7005 gcc_unreachable ();
7008 icode = optab_handler (tab1, mode);
7009 nunits = GET_MODE_NUNITS (mode);
7010 wmode = insn_data[icode].operand[0].mode;
7011 gcc_checking_assert (2 * GET_MODE_NUNITS (wmode) == nunits);
7012 gcc_checking_assert (GET_MODE_SIZE (wmode) == GET_MODE_SIZE (mode));
7014 create_output_operand (&eops[0], gen_reg_rtx (wmode), wmode);
7015 create_input_operand (&eops[1], op0, mode);
7016 create_input_operand (&eops[2], op1, mode);
7017 expand_insn (icode, 3, eops);
7018 m1 = gen_lowpart (mode, eops[0].value);
7020 create_output_operand (&eops[0], gen_reg_rtx (wmode), wmode);
7021 create_input_operand (&eops[1], op0, mode);
7022 create_input_operand (&eops[2], op1, mode);
7023 expand_insn (optab_handler (tab2, mode), 3, eops);
7024 m2 = gen_lowpart (mode, eops[0].value);
7026 v = rtvec_alloc (nunits);
7027 if (method == 2)
7029 for (i = 0; i < nunits; ++i)
7030 RTVEC_ELT (v, i) = GEN_INT (!BYTES_BIG_ENDIAN + (i & ~1)
7031 + ((i & 1) ? nunits : 0));
7033 else
7035 for (i = 0; i < nunits; ++i)
7036 RTVEC_ELT (v, i) = GEN_INT (2 * i + (BYTES_BIG_ENDIAN ? 0 : 1));
7038 perm = gen_rtx_CONST_VECTOR (mode, v);
7040 return expand_vec_perm (mode, m1, m2, perm, target);
7043 /* Return true if target supports vector masked load/store for mode. */
7044 bool
7045 can_vec_mask_load_store_p (machine_mode mode, bool is_load)
7047 optab op = is_load ? maskload_optab : maskstore_optab;
7048 machine_mode vmode;
7049 unsigned int vector_sizes;
7051 /* If mode is vector mode, check it directly. */
7052 if (VECTOR_MODE_P (mode))
7053 return optab_handler (op, mode) != CODE_FOR_nothing;
7055 /* Otherwise, return true if there is some vector mode with
7056 the mask load/store supported. */
7058 /* See if there is any chance the mask load or store might be
7059 vectorized. If not, punt. */
7060 vmode = targetm.vectorize.preferred_simd_mode (mode);
7061 if (!VECTOR_MODE_P (vmode))
7062 return false;
7064 if (optab_handler (op, vmode) != CODE_FOR_nothing)
7065 return true;
7067 vector_sizes = targetm.vectorize.autovectorize_vector_sizes ();
7068 while (vector_sizes != 0)
7070 unsigned int cur = 1 << floor_log2 (vector_sizes);
7071 vector_sizes &= ~cur;
7072 if (cur <= GET_MODE_SIZE (mode))
7073 continue;
7074 vmode = mode_for_vector (mode, cur / GET_MODE_SIZE (mode));
7075 if (VECTOR_MODE_P (vmode)
7076 && optab_handler (op, vmode) != CODE_FOR_nothing)
7077 return true;
7079 return false;
7082 /* Return true if there is a compare_and_swap pattern. */
7084 bool
7085 can_compare_and_swap_p (machine_mode mode, bool allow_libcall)
7087 enum insn_code icode;
7089 /* Check for __atomic_compare_and_swap. */
7090 icode = direct_optab_handler (atomic_compare_and_swap_optab, mode);
7091 if (icode != CODE_FOR_nothing)
7092 return true;
7094 /* Check for __sync_compare_and_swap. */
7095 icode = optab_handler (sync_compare_and_swap_optab, mode);
7096 if (icode != CODE_FOR_nothing)
7097 return true;
7098 if (allow_libcall && optab_libfunc (sync_compare_and_swap_optab, mode))
7099 return true;
7101 /* No inline compare and swap. */
7102 return false;
7105 /* Return true if an atomic exchange can be performed. */
7107 bool
7108 can_atomic_exchange_p (machine_mode mode, bool allow_libcall)
7110 enum insn_code icode;
7112 /* Check for __atomic_exchange. */
7113 icode = direct_optab_handler (atomic_exchange_optab, mode);
7114 if (icode != CODE_FOR_nothing)
7115 return true;
7117 /* Don't check __sync_test_and_set, as on some platforms that
7118 has reduced functionality. Targets that really do support
7119 a proper exchange should simply be updated to the __atomics. */
7121 return can_compare_and_swap_p (mode, allow_libcall);
7125 /* Helper function to find the MODE_CC set in a sync_compare_and_swap
7126 pattern. */
7128 static void
7129 find_cc_set (rtx x, const_rtx pat, void *data)
7131 if (REG_P (x) && GET_MODE_CLASS (GET_MODE (x)) == MODE_CC
7132 && GET_CODE (pat) == SET)
7134 rtx *p_cc_reg = (rtx *) data;
7135 gcc_assert (!*p_cc_reg);
7136 *p_cc_reg = x;
7140 /* This is a helper function for the other atomic operations. This function
7141 emits a loop that contains SEQ that iterates until a compare-and-swap
7142 operation at the end succeeds. MEM is the memory to be modified. SEQ is
7143 a set of instructions that takes a value from OLD_REG as an input and
7144 produces a value in NEW_REG as an output. Before SEQ, OLD_REG will be
7145 set to the current contents of MEM. After SEQ, a compare-and-swap will
7146 attempt to update MEM with NEW_REG. The function returns true when the
7147 loop was generated successfully. */
7149 static bool
7150 expand_compare_and_swap_loop (rtx mem, rtx old_reg, rtx new_reg, rtx seq)
7152 machine_mode mode = GET_MODE (mem);
7153 rtx_code_label *label;
7154 rtx cmp_reg, success, oldval;
7156 /* The loop we want to generate looks like
7158 cmp_reg = mem;
7159 label:
7160 old_reg = cmp_reg;
7161 seq;
7162 (success, cmp_reg) = compare-and-swap(mem, old_reg, new_reg)
7163 if (success)
7164 goto label;
7166 Note that we only do the plain load from memory once. Subsequent
7167 iterations use the value loaded by the compare-and-swap pattern. */
7169 label = gen_label_rtx ();
7170 cmp_reg = gen_reg_rtx (mode);
7172 emit_move_insn (cmp_reg, mem);
7173 emit_label (label);
7174 emit_move_insn (old_reg, cmp_reg);
7175 if (seq)
7176 emit_insn (seq);
7178 success = NULL_RTX;
7179 oldval = cmp_reg;
7180 if (!expand_atomic_compare_and_swap (&success, &oldval, mem, old_reg,
7181 new_reg, false, MEMMODEL_SEQ_CST,
7182 MEMMODEL_RELAXED))
7183 return false;
7185 if (oldval != cmp_reg)
7186 emit_move_insn (cmp_reg, oldval);
7188 /* Mark this jump predicted not taken. */
7189 emit_cmp_and_jump_insns (success, const0_rtx, EQ, const0_rtx,
7190 GET_MODE (success), 1, label, 0);
7191 return true;
7195 /* This function tries to emit an atomic_exchange intruction. VAL is written
7196 to *MEM using memory model MODEL. The previous contents of *MEM are returned,
7197 using TARGET if possible. */
7199 static rtx
7200 maybe_emit_atomic_exchange (rtx target, rtx mem, rtx val, enum memmodel model)
7202 machine_mode mode = GET_MODE (mem);
7203 enum insn_code icode;
7205 /* If the target supports the exchange directly, great. */
7206 icode = direct_optab_handler (atomic_exchange_optab, mode);
7207 if (icode != CODE_FOR_nothing)
7209 struct expand_operand ops[4];
7211 create_output_operand (&ops[0], target, mode);
7212 create_fixed_operand (&ops[1], mem);
7213 create_input_operand (&ops[2], val, mode);
7214 create_integer_operand (&ops[3], model);
7215 if (maybe_expand_insn (icode, 4, ops))
7216 return ops[0].value;
7219 return NULL_RTX;
7222 /* This function tries to implement an atomic exchange operation using
7223 __sync_lock_test_and_set. VAL is written to *MEM using memory model MODEL.
7224 The previous contents of *MEM are returned, using TARGET if possible.
7225 Since this instructionn is an acquire barrier only, stronger memory
7226 models may require additional barriers to be emitted. */
7228 static rtx
7229 maybe_emit_sync_lock_test_and_set (rtx target, rtx mem, rtx val,
7230 enum memmodel model)
7232 machine_mode mode = GET_MODE (mem);
7233 enum insn_code icode;
7234 rtx_insn *last_insn = get_last_insn ();
7236 icode = optab_handler (sync_lock_test_and_set_optab, mode);
7238 /* Legacy sync_lock_test_and_set is an acquire barrier. If the pattern
7239 exists, and the memory model is stronger than acquire, add a release
7240 barrier before the instruction. */
7242 if ((model & MEMMODEL_MASK) == MEMMODEL_SEQ_CST
7243 || (model & MEMMODEL_MASK) == MEMMODEL_RELEASE
7244 || (model & MEMMODEL_MASK) == MEMMODEL_ACQ_REL)
7245 expand_mem_thread_fence (model);
7247 if (icode != CODE_FOR_nothing)
7249 struct expand_operand ops[3];
7250 create_output_operand (&ops[0], target, mode);
7251 create_fixed_operand (&ops[1], mem);
7252 create_input_operand (&ops[2], val, mode);
7253 if (maybe_expand_insn (icode, 3, ops))
7254 return ops[0].value;
7257 /* If an external test-and-set libcall is provided, use that instead of
7258 any external compare-and-swap that we might get from the compare-and-
7259 swap-loop expansion later. */
7260 if (!can_compare_and_swap_p (mode, false))
7262 rtx libfunc = optab_libfunc (sync_lock_test_and_set_optab, mode);
7263 if (libfunc != NULL)
7265 rtx addr;
7267 addr = convert_memory_address (ptr_mode, XEXP (mem, 0));
7268 return emit_library_call_value (libfunc, NULL_RTX, LCT_NORMAL,
7269 mode, 2, addr, ptr_mode,
7270 val, mode);
7274 /* If the test_and_set can't be emitted, eliminate any barrier that might
7275 have been emitted. */
7276 delete_insns_since (last_insn);
7277 return NULL_RTX;
7280 /* This function tries to implement an atomic exchange operation using a
7281 compare_and_swap loop. VAL is written to *MEM. The previous contents of
7282 *MEM are returned, using TARGET if possible. No memory model is required
7283 since a compare_and_swap loop is seq-cst. */
7285 static rtx
7286 maybe_emit_compare_and_swap_exchange_loop (rtx target, rtx mem, rtx val)
7288 machine_mode mode = GET_MODE (mem);
7290 if (can_compare_and_swap_p (mode, true))
7292 if (!target || !register_operand (target, mode))
7293 target = gen_reg_rtx (mode);
7294 if (expand_compare_and_swap_loop (mem, target, val, NULL_RTX))
7295 return target;
7298 return NULL_RTX;
7301 /* This function tries to implement an atomic test-and-set operation
7302 using the atomic_test_and_set instruction pattern. A boolean value
7303 is returned from the operation, using TARGET if possible. */
7305 #ifndef HAVE_atomic_test_and_set
7306 #define HAVE_atomic_test_and_set 0
7307 #define CODE_FOR_atomic_test_and_set CODE_FOR_nothing
7308 #endif
7310 static rtx
7311 maybe_emit_atomic_test_and_set (rtx target, rtx mem, enum memmodel model)
7313 machine_mode pat_bool_mode;
7314 struct expand_operand ops[3];
7316 if (!HAVE_atomic_test_and_set)
7317 return NULL_RTX;
7319 /* While we always get QImode from __atomic_test_and_set, we get
7320 other memory modes from __sync_lock_test_and_set. Note that we
7321 use no endian adjustment here. This matches the 4.6 behavior
7322 in the Sparc backend. */
7323 gcc_checking_assert
7324 (insn_data[CODE_FOR_atomic_test_and_set].operand[1].mode == QImode);
7325 if (GET_MODE (mem) != QImode)
7326 mem = adjust_address_nv (mem, QImode, 0);
7328 pat_bool_mode = insn_data[CODE_FOR_atomic_test_and_set].operand[0].mode;
7329 create_output_operand (&ops[0], target, pat_bool_mode);
7330 create_fixed_operand (&ops[1], mem);
7331 create_integer_operand (&ops[2], model);
7333 if (maybe_expand_insn (CODE_FOR_atomic_test_and_set, 3, ops))
7334 return ops[0].value;
7335 return NULL_RTX;
7338 /* This function expands the legacy _sync_lock test_and_set operation which is
7339 generally an atomic exchange. Some limited targets only allow the
7340 constant 1 to be stored. This is an ACQUIRE operation.
7342 TARGET is an optional place to stick the return value.
7343 MEM is where VAL is stored. */
7346 expand_sync_lock_test_and_set (rtx target, rtx mem, rtx val)
7348 rtx ret;
7350 /* Try an atomic_exchange first. */
7351 ret = maybe_emit_atomic_exchange (target, mem, val, MEMMODEL_ACQUIRE);
7352 if (ret)
7353 return ret;
7355 ret = maybe_emit_sync_lock_test_and_set (target, mem, val, MEMMODEL_ACQUIRE);
7356 if (ret)
7357 return ret;
7359 ret = maybe_emit_compare_and_swap_exchange_loop (target, mem, val);
7360 if (ret)
7361 return ret;
7363 /* If there are no other options, try atomic_test_and_set if the value
7364 being stored is 1. */
7365 if (val == const1_rtx)
7366 ret = maybe_emit_atomic_test_and_set (target, mem, MEMMODEL_ACQUIRE);
7368 return ret;
7371 /* This function expands the atomic test_and_set operation:
7372 atomically store a boolean TRUE into MEM and return the previous value.
7374 MEMMODEL is the memory model variant to use.
7375 TARGET is an optional place to stick the return value. */
7378 expand_atomic_test_and_set (rtx target, rtx mem, enum memmodel model)
7380 machine_mode mode = GET_MODE (mem);
7381 rtx ret, trueval, subtarget;
7383 ret = maybe_emit_atomic_test_and_set (target, mem, model);
7384 if (ret)
7385 return ret;
7387 /* Be binary compatible with non-default settings of trueval, and different
7388 cpu revisions. E.g. one revision may have atomic-test-and-set, but
7389 another only has atomic-exchange. */
7390 if (targetm.atomic_test_and_set_trueval == 1)
7392 trueval = const1_rtx;
7393 subtarget = target ? target : gen_reg_rtx (mode);
7395 else
7397 trueval = gen_int_mode (targetm.atomic_test_and_set_trueval, mode);
7398 subtarget = gen_reg_rtx (mode);
7401 /* Try the atomic-exchange optab... */
7402 ret = maybe_emit_atomic_exchange (subtarget, mem, trueval, model);
7404 /* ... then an atomic-compare-and-swap loop ... */
7405 if (!ret)
7406 ret = maybe_emit_compare_and_swap_exchange_loop (subtarget, mem, trueval);
7408 /* ... before trying the vaguely defined legacy lock_test_and_set. */
7409 if (!ret)
7410 ret = maybe_emit_sync_lock_test_and_set (subtarget, mem, trueval, model);
7412 /* Recall that the legacy lock_test_and_set optab was allowed to do magic
7413 things with the value 1. Thus we try again without trueval. */
7414 if (!ret && targetm.atomic_test_and_set_trueval != 1)
7415 ret = maybe_emit_sync_lock_test_and_set (subtarget, mem, const1_rtx, model);
7417 /* Failing all else, assume a single threaded environment and simply
7418 perform the operation. */
7419 if (!ret)
7421 /* If the result is ignored skip the move to target. */
7422 if (subtarget != const0_rtx)
7423 emit_move_insn (subtarget, mem);
7425 emit_move_insn (mem, trueval);
7426 ret = subtarget;
7429 /* Recall that have to return a boolean value; rectify if trueval
7430 is not exactly one. */
7431 if (targetm.atomic_test_and_set_trueval != 1)
7432 ret = emit_store_flag_force (target, NE, ret, const0_rtx, mode, 0, 1);
7434 return ret;
7437 /* This function expands the atomic exchange operation:
7438 atomically store VAL in MEM and return the previous value in MEM.
7440 MEMMODEL is the memory model variant to use.
7441 TARGET is an optional place to stick the return value. */
7444 expand_atomic_exchange (rtx target, rtx mem, rtx val, enum memmodel model)
7446 rtx ret;
7448 ret = maybe_emit_atomic_exchange (target, mem, val, model);
7450 /* Next try a compare-and-swap loop for the exchange. */
7451 if (!ret)
7452 ret = maybe_emit_compare_and_swap_exchange_loop (target, mem, val);
7454 return ret;
7457 /* This function expands the atomic compare exchange operation:
7459 *PTARGET_BOOL is an optional place to store the boolean success/failure.
7460 *PTARGET_OVAL is an optional place to store the old value from memory.
7461 Both target parameters may be NULL to indicate that we do not care about
7462 that return value. Both target parameters are updated on success to
7463 the actual location of the corresponding result.
7465 MEMMODEL is the memory model variant to use.
7467 The return value of the function is true for success. */
7469 bool
7470 expand_atomic_compare_and_swap (rtx *ptarget_bool, rtx *ptarget_oval,
7471 rtx mem, rtx expected, rtx desired,
7472 bool is_weak, enum memmodel succ_model,
7473 enum memmodel fail_model)
7475 machine_mode mode = GET_MODE (mem);
7476 struct expand_operand ops[8];
7477 enum insn_code icode;
7478 rtx target_oval, target_bool = NULL_RTX;
7479 rtx libfunc;
7481 /* Load expected into a register for the compare and swap. */
7482 if (MEM_P (expected))
7483 expected = copy_to_reg (expected);
7485 /* Make sure we always have some place to put the return oldval.
7486 Further, make sure that place is distinct from the input expected,
7487 just in case we need that path down below. */
7488 if (ptarget_oval == NULL
7489 || (target_oval = *ptarget_oval) == NULL
7490 || reg_overlap_mentioned_p (expected, target_oval))
7491 target_oval = gen_reg_rtx (mode);
7493 icode = direct_optab_handler (atomic_compare_and_swap_optab, mode);
7494 if (icode != CODE_FOR_nothing)
7496 machine_mode bool_mode = insn_data[icode].operand[0].mode;
7498 /* Make sure we always have a place for the bool operand. */
7499 if (ptarget_bool == NULL
7500 || (target_bool = *ptarget_bool) == NULL
7501 || GET_MODE (target_bool) != bool_mode)
7502 target_bool = gen_reg_rtx (bool_mode);
7504 /* Emit the compare_and_swap. */
7505 create_output_operand (&ops[0], target_bool, bool_mode);
7506 create_output_operand (&ops[1], target_oval, mode);
7507 create_fixed_operand (&ops[2], mem);
7508 create_input_operand (&ops[3], expected, mode);
7509 create_input_operand (&ops[4], desired, mode);
7510 create_integer_operand (&ops[5], is_weak);
7511 create_integer_operand (&ops[6], succ_model);
7512 create_integer_operand (&ops[7], fail_model);
7513 if (maybe_expand_insn (icode, 8, ops))
7515 /* Return success/failure. */
7516 target_bool = ops[0].value;
7517 target_oval = ops[1].value;
7518 goto success;
7522 /* Otherwise fall back to the original __sync_val_compare_and_swap
7523 which is always seq-cst. */
7524 icode = optab_handler (sync_compare_and_swap_optab, mode);
7525 if (icode != CODE_FOR_nothing)
7527 rtx cc_reg;
7529 create_output_operand (&ops[0], target_oval, mode);
7530 create_fixed_operand (&ops[1], mem);
7531 create_input_operand (&ops[2], expected, mode);
7532 create_input_operand (&ops[3], desired, mode);
7533 if (!maybe_expand_insn (icode, 4, ops))
7534 return false;
7536 target_oval = ops[0].value;
7538 /* If the caller isn't interested in the boolean return value,
7539 skip the computation of it. */
7540 if (ptarget_bool == NULL)
7541 goto success;
7543 /* Otherwise, work out if the compare-and-swap succeeded. */
7544 cc_reg = NULL_RTX;
7545 if (have_insn_for (COMPARE, CCmode))
7546 note_stores (PATTERN (get_last_insn ()), find_cc_set, &cc_reg);
7547 if (cc_reg)
7549 target_bool = emit_store_flag_force (target_bool, EQ, cc_reg,
7550 const0_rtx, VOIDmode, 0, 1);
7551 goto success;
7553 goto success_bool_from_val;
7556 /* Also check for library support for __sync_val_compare_and_swap. */
7557 libfunc = optab_libfunc (sync_compare_and_swap_optab, mode);
7558 if (libfunc != NULL)
7560 rtx addr = convert_memory_address (ptr_mode, XEXP (mem, 0));
7561 target_oval = emit_library_call_value (libfunc, NULL_RTX, LCT_NORMAL,
7562 mode, 3, addr, ptr_mode,
7563 expected, mode, desired, mode);
7565 /* Compute the boolean return value only if requested. */
7566 if (ptarget_bool)
7567 goto success_bool_from_val;
7568 else
7569 goto success;
7572 /* Failure. */
7573 return false;
7575 success_bool_from_val:
7576 target_bool = emit_store_flag_force (target_bool, EQ, target_oval,
7577 expected, VOIDmode, 1, 1);
7578 success:
7579 /* Make sure that the oval output winds up where the caller asked. */
7580 if (ptarget_oval)
7581 *ptarget_oval = target_oval;
7582 if (ptarget_bool)
7583 *ptarget_bool = target_bool;
7584 return true;
7587 /* Generate asm volatile("" : : : "memory") as the memory barrier. */
7589 static void
7590 expand_asm_memory_barrier (void)
7592 rtx asm_op, clob;
7594 asm_op = gen_rtx_ASM_OPERANDS (VOIDmode, empty_string, empty_string, 0,
7595 rtvec_alloc (0), rtvec_alloc (0),
7596 rtvec_alloc (0), UNKNOWN_LOCATION);
7597 MEM_VOLATILE_P (asm_op) = 1;
7599 clob = gen_rtx_SCRATCH (VOIDmode);
7600 clob = gen_rtx_MEM (BLKmode, clob);
7601 clob = gen_rtx_CLOBBER (VOIDmode, clob);
7603 emit_insn (gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, asm_op, clob)));
7606 /* This routine will either emit the mem_thread_fence pattern or issue a
7607 sync_synchronize to generate a fence for memory model MEMMODEL. */
7609 #ifndef HAVE_mem_thread_fence
7610 # define HAVE_mem_thread_fence 0
7611 # define gen_mem_thread_fence(x) (gcc_unreachable (), NULL_RTX)
7612 #endif
7613 #ifndef HAVE_memory_barrier
7614 # define HAVE_memory_barrier 0
7615 # define gen_memory_barrier() (gcc_unreachable (), NULL_RTX)
7616 #endif
7618 void
7619 expand_mem_thread_fence (enum memmodel model)
7621 if (HAVE_mem_thread_fence)
7622 emit_insn (gen_mem_thread_fence (GEN_INT (model)));
7623 else if ((model & MEMMODEL_MASK) != MEMMODEL_RELAXED)
7625 if (HAVE_memory_barrier)
7626 emit_insn (gen_memory_barrier ());
7627 else if (synchronize_libfunc != NULL_RTX)
7628 emit_library_call (synchronize_libfunc, LCT_NORMAL, VOIDmode, 0);
7629 else
7630 expand_asm_memory_barrier ();
7634 /* This routine will either emit the mem_signal_fence pattern or issue a
7635 sync_synchronize to generate a fence for memory model MEMMODEL. */
7637 #ifndef HAVE_mem_signal_fence
7638 # define HAVE_mem_signal_fence 0
7639 # define gen_mem_signal_fence(x) (gcc_unreachable (), NULL_RTX)
7640 #endif
7642 void
7643 expand_mem_signal_fence (enum memmodel model)
7645 if (HAVE_mem_signal_fence)
7646 emit_insn (gen_mem_signal_fence (GEN_INT (model)));
7647 else if ((model & MEMMODEL_MASK) != MEMMODEL_RELAXED)
7649 /* By default targets are coherent between a thread and the signal
7650 handler running on the same thread. Thus this really becomes a
7651 compiler barrier, in that stores must not be sunk past
7652 (or raised above) a given point. */
7653 expand_asm_memory_barrier ();
7657 /* This function expands the atomic load operation:
7658 return the atomically loaded value in MEM.
7660 MEMMODEL is the memory model variant to use.
7661 TARGET is an option place to stick the return value. */
7664 expand_atomic_load (rtx target, rtx mem, enum memmodel model)
7666 machine_mode mode = GET_MODE (mem);
7667 enum insn_code icode;
7669 /* If the target supports the load directly, great. */
7670 icode = direct_optab_handler (atomic_load_optab, mode);
7671 if (icode != CODE_FOR_nothing)
7673 struct expand_operand ops[3];
7675 create_output_operand (&ops[0], target, mode);
7676 create_fixed_operand (&ops[1], mem);
7677 create_integer_operand (&ops[2], model);
7678 if (maybe_expand_insn (icode, 3, ops))
7679 return ops[0].value;
7682 /* If the size of the object is greater than word size on this target,
7683 then we assume that a load will not be atomic. */
7684 if (GET_MODE_PRECISION (mode) > BITS_PER_WORD)
7686 /* Issue val = compare_and_swap (mem, 0, 0).
7687 This may cause the occasional harmless store of 0 when the value is
7688 already 0, but it seems to be OK according to the standards guys. */
7689 if (expand_atomic_compare_and_swap (NULL, &target, mem, const0_rtx,
7690 const0_rtx, false, model, model))
7691 return target;
7692 else
7693 /* Otherwise there is no atomic load, leave the library call. */
7694 return NULL_RTX;
7697 /* Otherwise assume loads are atomic, and emit the proper barriers. */
7698 if (!target || target == const0_rtx)
7699 target = gen_reg_rtx (mode);
7701 /* For SEQ_CST, emit a barrier before the load. */
7702 if ((model & MEMMODEL_MASK) == MEMMODEL_SEQ_CST)
7703 expand_mem_thread_fence (model);
7705 emit_move_insn (target, mem);
7707 /* Emit the appropriate barrier after the load. */
7708 expand_mem_thread_fence (model);
7710 return target;
7713 /* This function expands the atomic store operation:
7714 Atomically store VAL in MEM.
7715 MEMMODEL is the memory model variant to use.
7716 USE_RELEASE is true if __sync_lock_release can be used as a fall back.
7717 function returns const0_rtx if a pattern was emitted. */
7720 expand_atomic_store (rtx mem, rtx val, enum memmodel model, bool use_release)
7722 machine_mode mode = GET_MODE (mem);
7723 enum insn_code icode;
7724 struct expand_operand ops[3];
7726 /* If the target supports the store directly, great. */
7727 icode = direct_optab_handler (atomic_store_optab, mode);
7728 if (icode != CODE_FOR_nothing)
7730 create_fixed_operand (&ops[0], mem);
7731 create_input_operand (&ops[1], val, mode);
7732 create_integer_operand (&ops[2], model);
7733 if (maybe_expand_insn (icode, 3, ops))
7734 return const0_rtx;
7737 /* If using __sync_lock_release is a viable alternative, try it. */
7738 if (use_release)
7740 icode = direct_optab_handler (sync_lock_release_optab, mode);
7741 if (icode != CODE_FOR_nothing)
7743 create_fixed_operand (&ops[0], mem);
7744 create_input_operand (&ops[1], const0_rtx, mode);
7745 if (maybe_expand_insn (icode, 2, ops))
7747 /* lock_release is only a release barrier. */
7748 if ((model & MEMMODEL_MASK) == MEMMODEL_SEQ_CST)
7749 expand_mem_thread_fence (model);
7750 return const0_rtx;
7755 /* If the size of the object is greater than word size on this target,
7756 a default store will not be atomic, Try a mem_exchange and throw away
7757 the result. If that doesn't work, don't do anything. */
7758 if (GET_MODE_PRECISION (mode) > BITS_PER_WORD)
7760 rtx target = maybe_emit_atomic_exchange (NULL_RTX, mem, val, model);
7761 if (!target)
7762 target = maybe_emit_compare_and_swap_exchange_loop (NULL_RTX, mem, val);
7763 if (target)
7764 return const0_rtx;
7765 else
7766 return NULL_RTX;
7769 /* Otherwise assume stores are atomic, and emit the proper barriers. */
7770 expand_mem_thread_fence (model);
7772 emit_move_insn (mem, val);
7774 /* For SEQ_CST, also emit a barrier after the store. */
7775 if ((model & MEMMODEL_MASK) == MEMMODEL_SEQ_CST)
7776 expand_mem_thread_fence (model);
7778 return const0_rtx;
7782 /* Structure containing the pointers and values required to process the
7783 various forms of the atomic_fetch_op and atomic_op_fetch builtins. */
7785 struct atomic_op_functions
7787 direct_optab mem_fetch_before;
7788 direct_optab mem_fetch_after;
7789 direct_optab mem_no_result;
7790 optab fetch_before;
7791 optab fetch_after;
7792 direct_optab no_result;
7793 enum rtx_code reverse_code;
7797 /* Fill in structure pointed to by OP with the various optab entries for an
7798 operation of type CODE. */
7800 static void
7801 get_atomic_op_for_code (struct atomic_op_functions *op, enum rtx_code code)
7803 gcc_assert (op!= NULL);
7805 /* If SWITCHABLE_TARGET is defined, then subtargets can be switched
7806 in the source code during compilation, and the optab entries are not
7807 computable until runtime. Fill in the values at runtime. */
7808 switch (code)
7810 case PLUS:
7811 op->mem_fetch_before = atomic_fetch_add_optab;
7812 op->mem_fetch_after = atomic_add_fetch_optab;
7813 op->mem_no_result = atomic_add_optab;
7814 op->fetch_before = sync_old_add_optab;
7815 op->fetch_after = sync_new_add_optab;
7816 op->no_result = sync_add_optab;
7817 op->reverse_code = MINUS;
7818 break;
7819 case MINUS:
7820 op->mem_fetch_before = atomic_fetch_sub_optab;
7821 op->mem_fetch_after = atomic_sub_fetch_optab;
7822 op->mem_no_result = atomic_sub_optab;
7823 op->fetch_before = sync_old_sub_optab;
7824 op->fetch_after = sync_new_sub_optab;
7825 op->no_result = sync_sub_optab;
7826 op->reverse_code = PLUS;
7827 break;
7828 case XOR:
7829 op->mem_fetch_before = atomic_fetch_xor_optab;
7830 op->mem_fetch_after = atomic_xor_fetch_optab;
7831 op->mem_no_result = atomic_xor_optab;
7832 op->fetch_before = sync_old_xor_optab;
7833 op->fetch_after = sync_new_xor_optab;
7834 op->no_result = sync_xor_optab;
7835 op->reverse_code = XOR;
7836 break;
7837 case AND:
7838 op->mem_fetch_before = atomic_fetch_and_optab;
7839 op->mem_fetch_after = atomic_and_fetch_optab;
7840 op->mem_no_result = atomic_and_optab;
7841 op->fetch_before = sync_old_and_optab;
7842 op->fetch_after = sync_new_and_optab;
7843 op->no_result = sync_and_optab;
7844 op->reverse_code = UNKNOWN;
7845 break;
7846 case IOR:
7847 op->mem_fetch_before = atomic_fetch_or_optab;
7848 op->mem_fetch_after = atomic_or_fetch_optab;
7849 op->mem_no_result = atomic_or_optab;
7850 op->fetch_before = sync_old_ior_optab;
7851 op->fetch_after = sync_new_ior_optab;
7852 op->no_result = sync_ior_optab;
7853 op->reverse_code = UNKNOWN;
7854 break;
7855 case NOT:
7856 op->mem_fetch_before = atomic_fetch_nand_optab;
7857 op->mem_fetch_after = atomic_nand_fetch_optab;
7858 op->mem_no_result = atomic_nand_optab;
7859 op->fetch_before = sync_old_nand_optab;
7860 op->fetch_after = sync_new_nand_optab;
7861 op->no_result = sync_nand_optab;
7862 op->reverse_code = UNKNOWN;
7863 break;
7864 default:
7865 gcc_unreachable ();
7869 /* See if there is a more optimal way to implement the operation "*MEM CODE VAL"
7870 using memory order MODEL. If AFTER is true the operation needs to return
7871 the value of *MEM after the operation, otherwise the previous value.
7872 TARGET is an optional place to place the result. The result is unused if
7873 it is const0_rtx.
7874 Return the result if there is a better sequence, otherwise NULL_RTX. */
7876 static rtx
7877 maybe_optimize_fetch_op (rtx target, rtx mem, rtx val, enum rtx_code code,
7878 enum memmodel model, bool after)
7880 /* If the value is prefetched, or not used, it may be possible to replace
7881 the sequence with a native exchange operation. */
7882 if (!after || target == const0_rtx)
7884 /* fetch_and (&x, 0, m) can be replaced with exchange (&x, 0, m). */
7885 if (code == AND && val == const0_rtx)
7887 if (target == const0_rtx)
7888 target = gen_reg_rtx (GET_MODE (mem));
7889 return maybe_emit_atomic_exchange (target, mem, val, model);
7892 /* fetch_or (&x, -1, m) can be replaced with exchange (&x, -1, m). */
7893 if (code == IOR && val == constm1_rtx)
7895 if (target == const0_rtx)
7896 target = gen_reg_rtx (GET_MODE (mem));
7897 return maybe_emit_atomic_exchange (target, mem, val, model);
7901 return NULL_RTX;
7904 /* Try to emit an instruction for a specific operation varaition.
7905 OPTAB contains the OP functions.
7906 TARGET is an optional place to return the result. const0_rtx means unused.
7907 MEM is the memory location to operate on.
7908 VAL is the value to use in the operation.
7909 USE_MEMMODEL is TRUE if the variation with a memory model should be tried.
7910 MODEL is the memory model, if used.
7911 AFTER is true if the returned result is the value after the operation. */
7913 static rtx
7914 maybe_emit_op (const struct atomic_op_functions *optab, rtx target, rtx mem,
7915 rtx val, bool use_memmodel, enum memmodel model, bool after)
7917 machine_mode mode = GET_MODE (mem);
7918 struct expand_operand ops[4];
7919 enum insn_code icode;
7920 int op_counter = 0;
7921 int num_ops;
7923 /* Check to see if there is a result returned. */
7924 if (target == const0_rtx)
7926 if (use_memmodel)
7928 icode = direct_optab_handler (optab->mem_no_result, mode);
7929 create_integer_operand (&ops[2], model);
7930 num_ops = 3;
7932 else
7934 icode = direct_optab_handler (optab->no_result, mode);
7935 num_ops = 2;
7938 /* Otherwise, we need to generate a result. */
7939 else
7941 if (use_memmodel)
7943 icode = direct_optab_handler (after ? optab->mem_fetch_after
7944 : optab->mem_fetch_before, mode);
7945 create_integer_operand (&ops[3], model);
7946 num_ops = 4;
7948 else
7950 icode = optab_handler (after ? optab->fetch_after
7951 : optab->fetch_before, mode);
7952 num_ops = 3;
7954 create_output_operand (&ops[op_counter++], target, mode);
7956 if (icode == CODE_FOR_nothing)
7957 return NULL_RTX;
7959 create_fixed_operand (&ops[op_counter++], mem);
7960 /* VAL may have been promoted to a wider mode. Shrink it if so. */
7961 create_convert_operand_to (&ops[op_counter++], val, mode, true);
7963 if (maybe_expand_insn (icode, num_ops, ops))
7964 return (target == const0_rtx ? const0_rtx : ops[0].value);
7966 return NULL_RTX;
7970 /* This function expands an atomic fetch_OP or OP_fetch operation:
7971 TARGET is an option place to stick the return value. const0_rtx indicates
7972 the result is unused.
7973 atomically fetch MEM, perform the operation with VAL and return it to MEM.
7974 CODE is the operation being performed (OP)
7975 MEMMODEL is the memory model variant to use.
7976 AFTER is true to return the result of the operation (OP_fetch).
7977 AFTER is false to return the value before the operation (fetch_OP).
7979 This function will *only* generate instructions if there is a direct
7980 optab. No compare and swap loops or libcalls will be generated. */
7982 static rtx
7983 expand_atomic_fetch_op_no_fallback (rtx target, rtx mem, rtx val,
7984 enum rtx_code code, enum memmodel model,
7985 bool after)
7987 machine_mode mode = GET_MODE (mem);
7988 struct atomic_op_functions optab;
7989 rtx result;
7990 bool unused_result = (target == const0_rtx);
7992 get_atomic_op_for_code (&optab, code);
7994 /* Check to see if there are any better instructions. */
7995 result = maybe_optimize_fetch_op (target, mem, val, code, model, after);
7996 if (result)
7997 return result;
7999 /* Check for the case where the result isn't used and try those patterns. */
8000 if (unused_result)
8002 /* Try the memory model variant first. */
8003 result = maybe_emit_op (&optab, target, mem, val, true, model, true);
8004 if (result)
8005 return result;
8007 /* Next try the old style withuot a memory model. */
8008 result = maybe_emit_op (&optab, target, mem, val, false, model, true);
8009 if (result)
8010 return result;
8012 /* There is no no-result pattern, so try patterns with a result. */
8013 target = NULL_RTX;
8016 /* Try the __atomic version. */
8017 result = maybe_emit_op (&optab, target, mem, val, true, model, after);
8018 if (result)
8019 return result;
8021 /* Try the older __sync version. */
8022 result = maybe_emit_op (&optab, target, mem, val, false, model, after);
8023 if (result)
8024 return result;
8026 /* If the fetch value can be calculated from the other variation of fetch,
8027 try that operation. */
8028 if (after || unused_result || optab.reverse_code != UNKNOWN)
8030 /* Try the __atomic version, then the older __sync version. */
8031 result = maybe_emit_op (&optab, target, mem, val, true, model, !after);
8032 if (!result)
8033 result = maybe_emit_op (&optab, target, mem, val, false, model, !after);
8035 if (result)
8037 /* If the result isn't used, no need to do compensation code. */
8038 if (unused_result)
8039 return result;
8041 /* Issue compensation code. Fetch_after == fetch_before OP val.
8042 Fetch_before == after REVERSE_OP val. */
8043 if (!after)
8044 code = optab.reverse_code;
8045 if (code == NOT)
8047 result = expand_simple_binop (mode, AND, result, val, NULL_RTX,
8048 true, OPTAB_LIB_WIDEN);
8049 result = expand_simple_unop (mode, NOT, result, target, true);
8051 else
8052 result = expand_simple_binop (mode, code, result, val, target,
8053 true, OPTAB_LIB_WIDEN);
8054 return result;
8058 /* No direct opcode can be generated. */
8059 return NULL_RTX;
8064 /* This function expands an atomic fetch_OP or OP_fetch operation:
8065 TARGET is an option place to stick the return value. const0_rtx indicates
8066 the result is unused.
8067 atomically fetch MEM, perform the operation with VAL and return it to MEM.
8068 CODE is the operation being performed (OP)
8069 MEMMODEL is the memory model variant to use.
8070 AFTER is true to return the result of the operation (OP_fetch).
8071 AFTER is false to return the value before the operation (fetch_OP). */
8073 expand_atomic_fetch_op (rtx target, rtx mem, rtx val, enum rtx_code code,
8074 enum memmodel model, bool after)
8076 machine_mode mode = GET_MODE (mem);
8077 rtx result;
8078 bool unused_result = (target == const0_rtx);
8080 result = expand_atomic_fetch_op_no_fallback (target, mem, val, code, model,
8081 after);
8083 if (result)
8084 return result;
8086 /* Add/sub can be implemented by doing the reverse operation with -(val). */
8087 if (code == PLUS || code == MINUS)
8089 rtx tmp;
8090 enum rtx_code reverse = (code == PLUS ? MINUS : PLUS);
8092 start_sequence ();
8093 tmp = expand_simple_unop (mode, NEG, val, NULL_RTX, true);
8094 result = expand_atomic_fetch_op_no_fallback (target, mem, tmp, reverse,
8095 model, after);
8096 if (result)
8098 /* PLUS worked so emit the insns and return. */
8099 tmp = get_insns ();
8100 end_sequence ();
8101 emit_insn (tmp);
8102 return result;
8105 /* PLUS did not work, so throw away the negation code and continue. */
8106 end_sequence ();
8109 /* Try the __sync libcalls only if we can't do compare-and-swap inline. */
8110 if (!can_compare_and_swap_p (mode, false))
8112 rtx libfunc;
8113 bool fixup = false;
8114 enum rtx_code orig_code = code;
8115 struct atomic_op_functions optab;
8117 get_atomic_op_for_code (&optab, code);
8118 libfunc = optab_libfunc (after ? optab.fetch_after
8119 : optab.fetch_before, mode);
8120 if (libfunc == NULL
8121 && (after || unused_result || optab.reverse_code != UNKNOWN))
8123 fixup = true;
8124 if (!after)
8125 code = optab.reverse_code;
8126 libfunc = optab_libfunc (after ? optab.fetch_before
8127 : optab.fetch_after, mode);
8129 if (libfunc != NULL)
8131 rtx addr = convert_memory_address (ptr_mode, XEXP (mem, 0));
8132 result = emit_library_call_value (libfunc, NULL, LCT_NORMAL, mode,
8133 2, addr, ptr_mode, val, mode);
8135 if (!unused_result && fixup)
8136 result = expand_simple_binop (mode, code, result, val, target,
8137 true, OPTAB_LIB_WIDEN);
8138 return result;
8141 /* We need the original code for any further attempts. */
8142 code = orig_code;
8145 /* If nothing else has succeeded, default to a compare and swap loop. */
8146 if (can_compare_and_swap_p (mode, true))
8148 rtx_insn *insn;
8149 rtx t0 = gen_reg_rtx (mode), t1;
8151 start_sequence ();
8153 /* If the result is used, get a register for it. */
8154 if (!unused_result)
8156 if (!target || !register_operand (target, mode))
8157 target = gen_reg_rtx (mode);
8158 /* If fetch_before, copy the value now. */
8159 if (!after)
8160 emit_move_insn (target, t0);
8162 else
8163 target = const0_rtx;
8165 t1 = t0;
8166 if (code == NOT)
8168 t1 = expand_simple_binop (mode, AND, t1, val, NULL_RTX,
8169 true, OPTAB_LIB_WIDEN);
8170 t1 = expand_simple_unop (mode, code, t1, NULL_RTX, true);
8172 else
8173 t1 = expand_simple_binop (mode, code, t1, val, NULL_RTX, true,
8174 OPTAB_LIB_WIDEN);
8176 /* For after, copy the value now. */
8177 if (!unused_result && after)
8178 emit_move_insn (target, t1);
8179 insn = get_insns ();
8180 end_sequence ();
8182 if (t1 != NULL && expand_compare_and_swap_loop (mem, t0, t1, insn))
8183 return target;
8186 return NULL_RTX;
8189 /* Return true if OPERAND is suitable for operand number OPNO of
8190 instruction ICODE. */
8192 bool
8193 insn_operand_matches (enum insn_code icode, unsigned int opno, rtx operand)
8195 return (!insn_data[(int) icode].operand[opno].predicate
8196 || (insn_data[(int) icode].operand[opno].predicate
8197 (operand, insn_data[(int) icode].operand[opno].mode)));
8200 /* TARGET is a target of a multiword operation that we are going to
8201 implement as a series of word-mode operations. Return true if
8202 TARGET is suitable for this purpose. */
8204 bool
8205 valid_multiword_target_p (rtx target)
8207 machine_mode mode;
8208 int i;
8210 mode = GET_MODE (target);
8211 for (i = 0; i < GET_MODE_SIZE (mode); i += UNITS_PER_WORD)
8212 if (!validate_subreg (word_mode, mode, target, i))
8213 return false;
8214 return true;
8217 /* Like maybe_legitimize_operand, but do not change the code of the
8218 current rtx value. */
8220 static bool
8221 maybe_legitimize_operand_same_code (enum insn_code icode, unsigned int opno,
8222 struct expand_operand *op)
8224 /* See if the operand matches in its current form. */
8225 if (insn_operand_matches (icode, opno, op->value))
8226 return true;
8228 /* If the operand is a memory whose address has no side effects,
8229 try forcing the address into a non-virtual pseudo register.
8230 The check for side effects is important because copy_to_mode_reg
8231 cannot handle things like auto-modified addresses. */
8232 if (insn_data[(int) icode].operand[opno].allows_mem && MEM_P (op->value))
8234 rtx addr, mem;
8236 mem = op->value;
8237 addr = XEXP (mem, 0);
8238 if (!(REG_P (addr) && REGNO (addr) > LAST_VIRTUAL_REGISTER)
8239 && !side_effects_p (addr))
8241 rtx_insn *last;
8242 machine_mode mode;
8244 last = get_last_insn ();
8245 mode = get_address_mode (mem);
8246 mem = replace_equiv_address (mem, copy_to_mode_reg (mode, addr));
8247 if (insn_operand_matches (icode, opno, mem))
8249 op->value = mem;
8250 return true;
8252 delete_insns_since (last);
8256 return false;
8259 /* Try to make OP match operand OPNO of instruction ICODE. Return true
8260 on success, storing the new operand value back in OP. */
8262 static bool
8263 maybe_legitimize_operand (enum insn_code icode, unsigned int opno,
8264 struct expand_operand *op)
8266 machine_mode mode, imode;
8267 bool old_volatile_ok, result;
8269 mode = op->mode;
8270 switch (op->type)
8272 case EXPAND_FIXED:
8273 old_volatile_ok = volatile_ok;
8274 volatile_ok = true;
8275 result = maybe_legitimize_operand_same_code (icode, opno, op);
8276 volatile_ok = old_volatile_ok;
8277 return result;
8279 case EXPAND_OUTPUT:
8280 gcc_assert (mode != VOIDmode);
8281 if (op->value
8282 && op->value != const0_rtx
8283 && GET_MODE (op->value) == mode
8284 && maybe_legitimize_operand_same_code (icode, opno, op))
8285 return true;
8287 op->value = gen_reg_rtx (mode);
8288 break;
8290 case EXPAND_INPUT:
8291 input:
8292 gcc_assert (mode != VOIDmode);
8293 gcc_assert (GET_MODE (op->value) == VOIDmode
8294 || GET_MODE (op->value) == mode);
8295 if (maybe_legitimize_operand_same_code (icode, opno, op))
8296 return true;
8298 op->value = copy_to_mode_reg (mode, op->value);
8299 break;
8301 case EXPAND_CONVERT_TO:
8302 gcc_assert (mode != VOIDmode);
8303 op->value = convert_to_mode (mode, op->value, op->unsigned_p);
8304 goto input;
8306 case EXPAND_CONVERT_FROM:
8307 if (GET_MODE (op->value) != VOIDmode)
8308 mode = GET_MODE (op->value);
8309 else
8310 /* The caller must tell us what mode this value has. */
8311 gcc_assert (mode != VOIDmode);
8313 imode = insn_data[(int) icode].operand[opno].mode;
8314 if (imode != VOIDmode && imode != mode)
8316 op->value = convert_modes (imode, mode, op->value, op->unsigned_p);
8317 mode = imode;
8319 goto input;
8321 case EXPAND_ADDRESS:
8322 gcc_assert (mode != VOIDmode);
8323 op->value = convert_memory_address (mode, op->value);
8324 goto input;
8326 case EXPAND_INTEGER:
8327 mode = insn_data[(int) icode].operand[opno].mode;
8328 if (mode != VOIDmode && const_int_operand (op->value, mode))
8329 goto input;
8330 break;
8332 return insn_operand_matches (icode, opno, op->value);
8335 /* Make OP describe an input operand that should have the same value
8336 as VALUE, after any mode conversion that the target might request.
8337 TYPE is the type of VALUE. */
8339 void
8340 create_convert_operand_from_type (struct expand_operand *op,
8341 rtx value, tree type)
8343 create_convert_operand_from (op, value, TYPE_MODE (type),
8344 TYPE_UNSIGNED (type));
8347 /* Try to make operands [OPS, OPS + NOPS) match operands [OPNO, OPNO + NOPS)
8348 of instruction ICODE. Return true on success, leaving the new operand
8349 values in the OPS themselves. Emit no code on failure. */
8351 bool
8352 maybe_legitimize_operands (enum insn_code icode, unsigned int opno,
8353 unsigned int nops, struct expand_operand *ops)
8355 rtx_insn *last;
8356 unsigned int i;
8358 last = get_last_insn ();
8359 for (i = 0; i < nops; i++)
8360 if (!maybe_legitimize_operand (icode, opno + i, &ops[i]))
8362 delete_insns_since (last);
8363 return false;
8365 return true;
8368 /* Try to generate instruction ICODE, using operands [OPS, OPS + NOPS)
8369 as its operands. Return the instruction pattern on success,
8370 and emit any necessary set-up code. Return null and emit no
8371 code on failure. */
8374 maybe_gen_insn (enum insn_code icode, unsigned int nops,
8375 struct expand_operand *ops)
8377 gcc_assert (nops == (unsigned int) insn_data[(int) icode].n_generator_args);
8378 if (!maybe_legitimize_operands (icode, 0, nops, ops))
8379 return NULL_RTX;
8381 switch (nops)
8383 case 1:
8384 return GEN_FCN (icode) (ops[0].value);
8385 case 2:
8386 return GEN_FCN (icode) (ops[0].value, ops[1].value);
8387 case 3:
8388 return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value);
8389 case 4:
8390 return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8391 ops[3].value);
8392 case 5:
8393 return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8394 ops[3].value, ops[4].value);
8395 case 6:
8396 return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8397 ops[3].value, ops[4].value, ops[5].value);
8398 case 7:
8399 return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8400 ops[3].value, ops[4].value, ops[5].value,
8401 ops[6].value);
8402 case 8:
8403 return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8404 ops[3].value, ops[4].value, ops[5].value,
8405 ops[6].value, ops[7].value);
8406 case 9:
8407 return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8408 ops[3].value, ops[4].value, ops[5].value,
8409 ops[6].value, ops[7].value, ops[8].value);
8411 gcc_unreachable ();
8414 /* Try to emit instruction ICODE, using operands [OPS, OPS + NOPS)
8415 as its operands. Return true on success and emit no code on failure. */
8417 bool
8418 maybe_expand_insn (enum insn_code icode, unsigned int nops,
8419 struct expand_operand *ops)
8421 rtx pat = maybe_gen_insn (icode, nops, ops);
8422 if (pat)
8424 emit_insn (pat);
8425 return true;
8427 return false;
8430 /* Like maybe_expand_insn, but for jumps. */
8432 bool
8433 maybe_expand_jump_insn (enum insn_code icode, unsigned int nops,
8434 struct expand_operand *ops)
8436 rtx pat = maybe_gen_insn (icode, nops, ops);
8437 if (pat)
8439 emit_jump_insn (pat);
8440 return true;
8442 return false;
8445 /* Emit instruction ICODE, using operands [OPS, OPS + NOPS)
8446 as its operands. */
8448 void
8449 expand_insn (enum insn_code icode, unsigned int nops,
8450 struct expand_operand *ops)
8452 if (!maybe_expand_insn (icode, nops, ops))
8453 gcc_unreachable ();
8456 /* Like expand_insn, but for jumps. */
8458 void
8459 expand_jump_insn (enum insn_code icode, unsigned int nops,
8460 struct expand_operand *ops)
8462 if (!maybe_expand_jump_insn (icode, nops, ops))
8463 gcc_unreachable ();
8466 /* Reduce conditional compilation elsewhere. */
8467 #ifndef HAVE_insv
8468 #define HAVE_insv 0
8469 #define CODE_FOR_insv CODE_FOR_nothing
8470 #endif
8471 #ifndef HAVE_extv
8472 #define HAVE_extv 0
8473 #define CODE_FOR_extv CODE_FOR_nothing
8474 #endif
8475 #ifndef HAVE_extzv
8476 #define HAVE_extzv 0
8477 #define CODE_FOR_extzv CODE_FOR_nothing
8478 #endif
8480 /* Enumerates the possible types of structure operand to an
8481 extraction_insn. */
8482 enum extraction_type { ET_unaligned_mem, ET_reg };
8484 /* Check whether insv, extv or extzv pattern ICODE can be used for an
8485 insertion or extraction of type TYPE on a structure of mode MODE.
8486 Return true if so and fill in *INSN accordingly. STRUCT_OP is the
8487 operand number of the structure (the first sign_extract or zero_extract
8488 operand) and FIELD_OP is the operand number of the field (the other
8489 side of the set from the sign_extract or zero_extract). */
8491 static bool
8492 get_traditional_extraction_insn (extraction_insn *insn,
8493 enum extraction_type type,
8494 machine_mode mode,
8495 enum insn_code icode,
8496 int struct_op, int field_op)
8498 const struct insn_data_d *data = &insn_data[icode];
8500 machine_mode struct_mode = data->operand[struct_op].mode;
8501 if (struct_mode == VOIDmode)
8502 struct_mode = word_mode;
8503 if (mode != struct_mode)
8504 return false;
8506 machine_mode field_mode = data->operand[field_op].mode;
8507 if (field_mode == VOIDmode)
8508 field_mode = word_mode;
8510 machine_mode pos_mode = data->operand[struct_op + 2].mode;
8511 if (pos_mode == VOIDmode)
8512 pos_mode = word_mode;
8514 insn->icode = icode;
8515 insn->field_mode = field_mode;
8516 insn->struct_mode = (type == ET_unaligned_mem ? byte_mode : struct_mode);
8517 insn->pos_mode = pos_mode;
8518 return true;
8521 /* Return true if an optab exists to perform an insertion or extraction
8522 of type TYPE in mode MODE. Describe the instruction in *INSN if so.
8524 REG_OPTAB is the optab to use for register structures and
8525 MISALIGN_OPTAB is the optab to use for misaligned memory structures.
8526 POS_OP is the operand number of the bit position. */
8528 static bool
8529 get_optab_extraction_insn (struct extraction_insn *insn,
8530 enum extraction_type type,
8531 machine_mode mode, direct_optab reg_optab,
8532 direct_optab misalign_optab, int pos_op)
8534 direct_optab optab = (type == ET_unaligned_mem ? misalign_optab : reg_optab);
8535 enum insn_code icode = direct_optab_handler (optab, mode);
8536 if (icode == CODE_FOR_nothing)
8537 return false;
8539 const struct insn_data_d *data = &insn_data[icode];
8541 insn->icode = icode;
8542 insn->field_mode = mode;
8543 insn->struct_mode = (type == ET_unaligned_mem ? BLKmode : mode);
8544 insn->pos_mode = data->operand[pos_op].mode;
8545 if (insn->pos_mode == VOIDmode)
8546 insn->pos_mode = word_mode;
8547 return true;
8550 /* Return true if an instruction exists to perform an insertion or
8551 extraction (PATTERN says which) of type TYPE in mode MODE.
8552 Describe the instruction in *INSN if so. */
8554 static bool
8555 get_extraction_insn (extraction_insn *insn,
8556 enum extraction_pattern pattern,
8557 enum extraction_type type,
8558 machine_mode mode)
8560 switch (pattern)
8562 case EP_insv:
8563 if (HAVE_insv
8564 && get_traditional_extraction_insn (insn, type, mode,
8565 CODE_FOR_insv, 0, 3))
8566 return true;
8567 return get_optab_extraction_insn (insn, type, mode, insv_optab,
8568 insvmisalign_optab, 2);
8570 case EP_extv:
8571 if (HAVE_extv
8572 && get_traditional_extraction_insn (insn, type, mode,
8573 CODE_FOR_extv, 1, 0))
8574 return true;
8575 return get_optab_extraction_insn (insn, type, mode, extv_optab,
8576 extvmisalign_optab, 3);
8578 case EP_extzv:
8579 if (HAVE_extzv
8580 && get_traditional_extraction_insn (insn, type, mode,
8581 CODE_FOR_extzv, 1, 0))
8582 return true;
8583 return get_optab_extraction_insn (insn, type, mode, extzv_optab,
8584 extzvmisalign_optab, 3);
8586 default:
8587 gcc_unreachable ();
8591 /* Return true if an instruction exists to access a field of mode
8592 FIELDMODE in a structure that has STRUCT_BITS significant bits.
8593 Describe the "best" such instruction in *INSN if so. PATTERN and
8594 TYPE describe the type of insertion or extraction we want to perform.
8596 For an insertion, the number of significant structure bits includes
8597 all bits of the target. For an extraction, it need only include the
8598 most significant bit of the field. Larger widths are acceptable
8599 in both cases. */
8601 static bool
8602 get_best_extraction_insn (extraction_insn *insn,
8603 enum extraction_pattern pattern,
8604 enum extraction_type type,
8605 unsigned HOST_WIDE_INT struct_bits,
8606 machine_mode field_mode)
8608 machine_mode mode = smallest_mode_for_size (struct_bits, MODE_INT);
8609 while (mode != VOIDmode)
8611 if (get_extraction_insn (insn, pattern, type, mode))
8613 while (mode != VOIDmode
8614 && GET_MODE_SIZE (mode) <= GET_MODE_SIZE (field_mode)
8615 && !TRULY_NOOP_TRUNCATION_MODES_P (insn->field_mode,
8616 field_mode))
8618 get_extraction_insn (insn, pattern, type, mode);
8619 mode = GET_MODE_WIDER_MODE (mode);
8621 return true;
8623 mode = GET_MODE_WIDER_MODE (mode);
8625 return false;
8628 /* Return true if an instruction exists to access a field of mode
8629 FIELDMODE in a register structure that has STRUCT_BITS significant bits.
8630 Describe the "best" such instruction in *INSN if so. PATTERN describes
8631 the type of insertion or extraction we want to perform.
8633 For an insertion, the number of significant structure bits includes
8634 all bits of the target. For an extraction, it need only include the
8635 most significant bit of the field. Larger widths are acceptable
8636 in both cases. */
8638 bool
8639 get_best_reg_extraction_insn (extraction_insn *insn,
8640 enum extraction_pattern pattern,
8641 unsigned HOST_WIDE_INT struct_bits,
8642 machine_mode field_mode)
8644 return get_best_extraction_insn (insn, pattern, ET_reg, struct_bits,
8645 field_mode);
8648 /* Return true if an instruction exists to access a field of BITSIZE
8649 bits starting BITNUM bits into a memory structure. Describe the
8650 "best" such instruction in *INSN if so. PATTERN describes the type
8651 of insertion or extraction we want to perform and FIELDMODE is the
8652 natural mode of the extracted field.
8654 The instructions considered here only access bytes that overlap
8655 the bitfield; they do not touch any surrounding bytes. */
8657 bool
8658 get_best_mem_extraction_insn (extraction_insn *insn,
8659 enum extraction_pattern pattern,
8660 HOST_WIDE_INT bitsize, HOST_WIDE_INT bitnum,
8661 machine_mode field_mode)
8663 unsigned HOST_WIDE_INT struct_bits = (bitnum % BITS_PER_UNIT
8664 + bitsize
8665 + BITS_PER_UNIT - 1);
8666 struct_bits -= struct_bits % BITS_PER_UNIT;
8667 return get_best_extraction_insn (insn, pattern, ET_unaligned_mem,
8668 struct_bits, field_mode);
8671 /* Determine whether "1 << x" is relatively cheap in word_mode. */
8673 bool
8674 lshift_cheap_p (bool speed_p)
8676 /* FIXME: This should be made target dependent via this "this_target"
8677 mechanism, similar to e.g. can_copy_init_p in gcse.c. */
8678 static bool init[2] = { false, false };
8679 static bool cheap[2] = { true, true };
8681 /* If the targer has no lshift in word_mode, the operation will most
8682 probably not be cheap. ??? Does GCC even work for such targets? */
8683 if (optab_handler (ashl_optab, word_mode) == CODE_FOR_nothing)
8684 return false;
8686 if (!init[speed_p])
8688 rtx reg = gen_raw_REG (word_mode, 10000);
8689 int cost = set_src_cost (gen_rtx_ASHIFT (word_mode, const1_rtx, reg),
8690 speed_p);
8691 cheap[speed_p] = cost < COSTS_N_INSNS (3);
8692 init[speed_p] = true;
8695 return cheap[speed_p];
8698 #include "gt-optabs.h"