/cp
[official-gcc.git] / gcc / optabs.c
blob0719ba2c558ce3cea7da04bccd1babf33e3de4aa
1 /* Expand the basic unary and binary arithmetic operations, for GNU compiler.
2 Copyright (C) 1987-2015 Free Software Foundation, Inc.
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 3, or (at your option) any later
9 version.
11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 for more details.
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
21 #include "config.h"
22 #include "system.h"
23 #include "coretypes.h"
24 #include "backend.h"
25 #include "predict.h"
26 #include "diagnostic-core.h"
28 /* Include insn-config.h before expr.h so that HAVE_conditional_move
29 is properly defined. */
30 #include "insn-config.h"
31 #include "rtl.h"
32 #include "alias.h"
33 #include "tree.h"
34 #include "tree-hasher.h"
35 #include "stor-layout.h"
36 #include "stringpool.h"
37 #include "varasm.h"
38 #include "tm_p.h"
39 #include "flags.h"
40 #include "except.h"
41 #include "expmed.h"
42 #include "dojump.h"
43 #include "explow.h"
44 #include "calls.h"
45 #include "emit-rtl.h"
46 #include "stmt.h"
47 #include "expr.h"
48 #include "insn-codes.h"
49 #include "optabs.h"
50 #include "libfuncs.h"
51 #include "recog.h"
52 #include "reload.h"
53 #include "target.h"
55 struct target_optabs default_target_optabs;
56 struct target_libfuncs default_target_libfuncs;
57 struct target_optabs *this_fn_optabs = &default_target_optabs;
58 #if SWITCHABLE_TARGET
59 struct target_optabs *this_target_optabs = &default_target_optabs;
60 struct target_libfuncs *this_target_libfuncs = &default_target_libfuncs;
61 #endif
63 #define libfunc_hash \
64 (this_target_libfuncs->x_libfunc_hash)
66 static void prepare_float_lib_cmp (rtx, rtx, enum rtx_code, rtx *,
67 machine_mode *);
68 static rtx expand_unop_direct (machine_mode, optab, rtx, rtx, int);
69 static void emit_libcall_block_1 (rtx_insn *, rtx, rtx, rtx, bool);
71 /* Debug facility for use in GDB. */
72 void debug_optab_libfuncs (void);
74 /* Prefixes for the current version of decimal floating point (BID vs. DPD) */
75 #if ENABLE_DECIMAL_BID_FORMAT
76 #define DECIMAL_PREFIX "bid_"
77 #else
78 #define DECIMAL_PREFIX "dpd_"
79 #endif
81 /* Used for libfunc_hash. */
83 hashval_t
84 libfunc_hasher::hash (libfunc_entry *e)
86 return ((e->mode1 + e->mode2 * NUM_MACHINE_MODES) ^ e->op);
89 /* Used for libfunc_hash. */
91 bool
92 libfunc_hasher::equal (libfunc_entry *e1, libfunc_entry *e2)
94 return e1->op == e2->op && e1->mode1 == e2->mode1 && e1->mode2 == e2->mode2;
97 /* Return libfunc corresponding operation defined by OPTAB converting
98 from MODE2 to MODE1. Trigger lazy initialization if needed, return NULL
99 if no libfunc is available. */
101 convert_optab_libfunc (convert_optab optab, machine_mode mode1,
102 machine_mode mode2)
104 struct libfunc_entry e;
105 struct libfunc_entry **slot;
107 /* ??? This ought to be an assert, but not all of the places
108 that we expand optabs know about the optabs that got moved
109 to being direct. */
110 if (!(optab >= FIRST_CONV_OPTAB && optab <= LAST_CONVLIB_OPTAB))
111 return NULL_RTX;
113 e.op = optab;
114 e.mode1 = mode1;
115 e.mode2 = mode2;
116 slot = libfunc_hash->find_slot (&e, NO_INSERT);
117 if (!slot)
119 const struct convert_optab_libcall_d *d
120 = &convlib_def[optab - FIRST_CONV_OPTAB];
122 if (d->libcall_gen == NULL)
123 return NULL;
125 d->libcall_gen (optab, d->libcall_basename, mode1, mode2);
126 slot = libfunc_hash->find_slot (&e, NO_INSERT);
127 if (!slot)
128 return NULL;
130 return (*slot)->libfunc;
133 /* Return libfunc corresponding operation defined by OPTAB in MODE.
134 Trigger lazy initialization if needed, return NULL if no libfunc is
135 available. */
137 optab_libfunc (optab optab, machine_mode mode)
139 struct libfunc_entry e;
140 struct libfunc_entry **slot;
142 /* ??? This ought to be an assert, but not all of the places
143 that we expand optabs know about the optabs that got moved
144 to being direct. */
145 if (!(optab >= FIRST_NORM_OPTAB && optab <= LAST_NORMLIB_OPTAB))
146 return NULL_RTX;
148 e.op = optab;
149 e.mode1 = mode;
150 e.mode2 = VOIDmode;
151 slot = libfunc_hash->find_slot (&e, NO_INSERT);
152 if (!slot)
154 const struct optab_libcall_d *d
155 = &normlib_def[optab - FIRST_NORM_OPTAB];
157 if (d->libcall_gen == NULL)
158 return NULL;
160 d->libcall_gen (optab, d->libcall_basename, d->libcall_suffix, mode);
161 slot = libfunc_hash->find_slot (&e, NO_INSERT);
162 if (!slot)
163 return NULL;
165 return (*slot)->libfunc;
169 /* Add a REG_EQUAL note to the last insn in INSNS. TARGET is being set to
170 the result of operation CODE applied to OP0 (and OP1 if it is a binary
171 operation).
173 If the last insn does not set TARGET, don't do anything, but return 1.
175 If the last insn or a previous insn sets TARGET and TARGET is one of OP0
176 or OP1, don't add the REG_EQUAL note but return 0. Our caller can then
177 try again, ensuring that TARGET is not one of the operands. */
179 static int
180 add_equal_note (rtx_insn *insns, rtx target, enum rtx_code code, rtx op0, rtx op1)
182 rtx_insn *last_insn;
183 rtx set;
184 rtx note;
186 gcc_assert (insns && INSN_P (insns) && NEXT_INSN (insns));
188 if (GET_RTX_CLASS (code) != RTX_COMM_ARITH
189 && GET_RTX_CLASS (code) != RTX_BIN_ARITH
190 && GET_RTX_CLASS (code) != RTX_COMM_COMPARE
191 && GET_RTX_CLASS (code) != RTX_COMPARE
192 && GET_RTX_CLASS (code) != RTX_UNARY)
193 return 1;
195 if (GET_CODE (target) == ZERO_EXTRACT)
196 return 1;
198 for (last_insn = insns;
199 NEXT_INSN (last_insn) != NULL_RTX;
200 last_insn = NEXT_INSN (last_insn))
203 /* If TARGET is in OP0 or OP1, punt. We'd end up with a note referencing
204 a value changing in the insn, so the note would be invalid for CSE. */
205 if (reg_overlap_mentioned_p (target, op0)
206 || (op1 && reg_overlap_mentioned_p (target, op1)))
208 if (MEM_P (target)
209 && (rtx_equal_p (target, op0)
210 || (op1 && rtx_equal_p (target, op1))))
212 /* For MEM target, with MEM = MEM op X, prefer no REG_EQUAL note
213 over expanding it as temp = MEM op X, MEM = temp. If the target
214 supports MEM = MEM op X instructions, it is sometimes too hard
215 to reconstruct that form later, especially if X is also a memory,
216 and due to multiple occurrences of addresses the address might
217 be forced into register unnecessarily.
218 Note that not emitting the REG_EQUIV note might inhibit
219 CSE in some cases. */
220 set = single_set (last_insn);
221 if (set
222 && GET_CODE (SET_SRC (set)) == code
223 && MEM_P (SET_DEST (set))
224 && (rtx_equal_p (SET_DEST (set), XEXP (SET_SRC (set), 0))
225 || (op1 && rtx_equal_p (SET_DEST (set),
226 XEXP (SET_SRC (set), 1)))))
227 return 1;
229 return 0;
232 set = set_for_reg_notes (last_insn);
233 if (set == NULL_RTX)
234 return 1;
236 if (! rtx_equal_p (SET_DEST (set), target)
237 /* For a STRICT_LOW_PART, the REG_NOTE applies to what is inside it. */
238 && (GET_CODE (SET_DEST (set)) != STRICT_LOW_PART
239 || ! rtx_equal_p (XEXP (SET_DEST (set), 0), target)))
240 return 1;
242 if (GET_RTX_CLASS (code) == RTX_UNARY)
243 switch (code)
245 case FFS:
246 case CLZ:
247 case CTZ:
248 case CLRSB:
249 case POPCOUNT:
250 case PARITY:
251 case BSWAP:
252 if (GET_MODE (op0) != VOIDmode && GET_MODE (target) != GET_MODE (op0))
254 note = gen_rtx_fmt_e (code, GET_MODE (op0), copy_rtx (op0));
255 if (GET_MODE_SIZE (GET_MODE (op0))
256 > GET_MODE_SIZE (GET_MODE (target)))
257 note = simplify_gen_unary (TRUNCATE, GET_MODE (target),
258 note, GET_MODE (op0));
259 else
260 note = simplify_gen_unary (ZERO_EXTEND, GET_MODE (target),
261 note, GET_MODE (op0));
262 break;
264 /* FALLTHRU */
265 default:
266 note = gen_rtx_fmt_e (code, GET_MODE (target), copy_rtx (op0));
267 break;
269 else
270 note = gen_rtx_fmt_ee (code, GET_MODE (target), copy_rtx (op0), copy_rtx (op1));
272 set_unique_reg_note (last_insn, REG_EQUAL, note);
274 return 1;
277 /* Given two input operands, OP0 and OP1, determine what the correct from_mode
278 for a widening operation would be. In most cases this would be OP0, but if
279 that's a constant it'll be VOIDmode, which isn't useful. */
281 static machine_mode
282 widened_mode (machine_mode to_mode, rtx op0, rtx op1)
284 machine_mode m0 = GET_MODE (op0);
285 machine_mode m1 = GET_MODE (op1);
286 machine_mode result;
288 if (m0 == VOIDmode && m1 == VOIDmode)
289 return to_mode;
290 else if (m0 == VOIDmode || GET_MODE_SIZE (m0) < GET_MODE_SIZE (m1))
291 result = m1;
292 else
293 result = m0;
295 if (GET_MODE_SIZE (result) > GET_MODE_SIZE (to_mode))
296 return to_mode;
298 return result;
301 /* Like optab_handler, but for widening_operations that have a
302 TO_MODE and a FROM_MODE. */
304 enum insn_code
305 widening_optab_handler (optab op, machine_mode to_mode,
306 machine_mode from_mode)
308 unsigned scode = (op << 16) | to_mode;
309 if (to_mode != from_mode && from_mode != VOIDmode)
311 /* ??? Why does find_widening_optab_handler_and_mode attempt to
312 widen things that can't be widened? E.g. add_optab... */
313 if (op > LAST_CONV_OPTAB)
314 return CODE_FOR_nothing;
315 scode |= from_mode << 8;
317 return raw_optab_handler (scode);
320 /* Find a widening optab even if it doesn't widen as much as we want.
321 E.g. if from_mode is HImode, and to_mode is DImode, and there is no
322 direct HI->SI insn, then return SI->DI, if that exists.
323 If PERMIT_NON_WIDENING is non-zero then this can be used with
324 non-widening optabs also. */
326 enum insn_code
327 find_widening_optab_handler_and_mode (optab op, machine_mode to_mode,
328 machine_mode from_mode,
329 int permit_non_widening,
330 machine_mode *found_mode)
332 for (; (permit_non_widening || from_mode != to_mode)
333 && GET_MODE_SIZE (from_mode) <= GET_MODE_SIZE (to_mode)
334 && from_mode != VOIDmode;
335 from_mode = GET_MODE_WIDER_MODE (from_mode))
337 enum insn_code handler = widening_optab_handler (op, to_mode,
338 from_mode);
340 if (handler != CODE_FOR_nothing)
342 if (found_mode)
343 *found_mode = from_mode;
344 return handler;
348 return CODE_FOR_nothing;
351 /* Widen OP to MODE and return the rtx for the widened operand. UNSIGNEDP
352 says whether OP is signed or unsigned. NO_EXTEND is nonzero if we need
353 not actually do a sign-extend or zero-extend, but can leave the
354 higher-order bits of the result rtx undefined, for example, in the case
355 of logical operations, but not right shifts. */
357 static rtx
358 widen_operand (rtx op, machine_mode mode, machine_mode oldmode,
359 int unsignedp, int no_extend)
361 rtx result;
363 /* If we don't have to extend and this is a constant, return it. */
364 if (no_extend && GET_MODE (op) == VOIDmode)
365 return op;
367 /* If we must extend do so. If OP is a SUBREG for a promoted object, also
368 extend since it will be more efficient to do so unless the signedness of
369 a promoted object differs from our extension. */
370 if (! no_extend
371 || (GET_CODE (op) == SUBREG && SUBREG_PROMOTED_VAR_P (op)
372 && SUBREG_CHECK_PROMOTED_SIGN (op, unsignedp)))
373 return convert_modes (mode, oldmode, op, unsignedp);
375 /* If MODE is no wider than a single word, we return a lowpart or paradoxical
376 SUBREG. */
377 if (GET_MODE_SIZE (mode) <= UNITS_PER_WORD)
378 return gen_lowpart (mode, force_reg (GET_MODE (op), op));
380 /* Otherwise, get an object of MODE, clobber it, and set the low-order
381 part to OP. */
383 result = gen_reg_rtx (mode);
384 emit_clobber (result);
385 emit_move_insn (gen_lowpart (GET_MODE (op), result), op);
386 return result;
389 /* Return the optab used for computing the operation given by the tree code,
390 CODE and the tree EXP. This function is not always usable (for example, it
391 cannot give complete results for multiplication or division) but probably
392 ought to be relied on more widely throughout the expander. */
393 optab
394 optab_for_tree_code (enum tree_code code, const_tree type,
395 enum optab_subtype subtype)
397 bool trapv;
398 switch (code)
400 case BIT_AND_EXPR:
401 return and_optab;
403 case BIT_IOR_EXPR:
404 return ior_optab;
406 case BIT_NOT_EXPR:
407 return one_cmpl_optab;
409 case BIT_XOR_EXPR:
410 return xor_optab;
412 case MULT_HIGHPART_EXPR:
413 return TYPE_UNSIGNED (type) ? umul_highpart_optab : smul_highpart_optab;
415 case TRUNC_MOD_EXPR:
416 case CEIL_MOD_EXPR:
417 case FLOOR_MOD_EXPR:
418 case ROUND_MOD_EXPR:
419 return TYPE_UNSIGNED (type) ? umod_optab : smod_optab;
421 case RDIV_EXPR:
422 case TRUNC_DIV_EXPR:
423 case CEIL_DIV_EXPR:
424 case FLOOR_DIV_EXPR:
425 case ROUND_DIV_EXPR:
426 case EXACT_DIV_EXPR:
427 if (TYPE_SATURATING (type))
428 return TYPE_UNSIGNED (type) ? usdiv_optab : ssdiv_optab;
429 return TYPE_UNSIGNED (type) ? udiv_optab : sdiv_optab;
431 case LSHIFT_EXPR:
432 if (TREE_CODE (type) == VECTOR_TYPE)
434 if (subtype == optab_vector)
435 return TYPE_SATURATING (type) ? unknown_optab : vashl_optab;
437 gcc_assert (subtype == optab_scalar);
439 if (TYPE_SATURATING (type))
440 return TYPE_UNSIGNED (type) ? usashl_optab : ssashl_optab;
441 return ashl_optab;
443 case RSHIFT_EXPR:
444 if (TREE_CODE (type) == VECTOR_TYPE)
446 if (subtype == optab_vector)
447 return TYPE_UNSIGNED (type) ? vlshr_optab : vashr_optab;
449 gcc_assert (subtype == optab_scalar);
451 return TYPE_UNSIGNED (type) ? lshr_optab : ashr_optab;
453 case LROTATE_EXPR:
454 if (TREE_CODE (type) == VECTOR_TYPE)
456 if (subtype == optab_vector)
457 return vrotl_optab;
459 gcc_assert (subtype == optab_scalar);
461 return rotl_optab;
463 case RROTATE_EXPR:
464 if (TREE_CODE (type) == VECTOR_TYPE)
466 if (subtype == optab_vector)
467 return vrotr_optab;
469 gcc_assert (subtype == optab_scalar);
471 return rotr_optab;
473 case MAX_EXPR:
474 return TYPE_UNSIGNED (type) ? umax_optab : smax_optab;
476 case MIN_EXPR:
477 return TYPE_UNSIGNED (type) ? umin_optab : smin_optab;
479 case REALIGN_LOAD_EXPR:
480 return vec_realign_load_optab;
482 case WIDEN_SUM_EXPR:
483 return TYPE_UNSIGNED (type) ? usum_widen_optab : ssum_widen_optab;
485 case DOT_PROD_EXPR:
486 return TYPE_UNSIGNED (type) ? udot_prod_optab : sdot_prod_optab;
488 case SAD_EXPR:
489 return TYPE_UNSIGNED (type) ? usad_optab : ssad_optab;
491 case WIDEN_MULT_PLUS_EXPR:
492 return (TYPE_UNSIGNED (type)
493 ? (TYPE_SATURATING (type)
494 ? usmadd_widen_optab : umadd_widen_optab)
495 : (TYPE_SATURATING (type)
496 ? ssmadd_widen_optab : smadd_widen_optab));
498 case WIDEN_MULT_MINUS_EXPR:
499 return (TYPE_UNSIGNED (type)
500 ? (TYPE_SATURATING (type)
501 ? usmsub_widen_optab : umsub_widen_optab)
502 : (TYPE_SATURATING (type)
503 ? ssmsub_widen_optab : smsub_widen_optab));
505 case FMA_EXPR:
506 return fma_optab;
508 case REDUC_MAX_EXPR:
509 return TYPE_UNSIGNED (type)
510 ? reduc_umax_scal_optab : reduc_smax_scal_optab;
512 case REDUC_MIN_EXPR:
513 return TYPE_UNSIGNED (type)
514 ? reduc_umin_scal_optab : reduc_smin_scal_optab;
516 case REDUC_PLUS_EXPR:
517 return reduc_plus_scal_optab;
519 case VEC_WIDEN_MULT_HI_EXPR:
520 return TYPE_UNSIGNED (type) ?
521 vec_widen_umult_hi_optab : vec_widen_smult_hi_optab;
523 case VEC_WIDEN_MULT_LO_EXPR:
524 return TYPE_UNSIGNED (type) ?
525 vec_widen_umult_lo_optab : vec_widen_smult_lo_optab;
527 case VEC_WIDEN_MULT_EVEN_EXPR:
528 return TYPE_UNSIGNED (type) ?
529 vec_widen_umult_even_optab : vec_widen_smult_even_optab;
531 case VEC_WIDEN_MULT_ODD_EXPR:
532 return TYPE_UNSIGNED (type) ?
533 vec_widen_umult_odd_optab : vec_widen_smult_odd_optab;
535 case VEC_WIDEN_LSHIFT_HI_EXPR:
536 return TYPE_UNSIGNED (type) ?
537 vec_widen_ushiftl_hi_optab : vec_widen_sshiftl_hi_optab;
539 case VEC_WIDEN_LSHIFT_LO_EXPR:
540 return TYPE_UNSIGNED (type) ?
541 vec_widen_ushiftl_lo_optab : vec_widen_sshiftl_lo_optab;
543 case VEC_UNPACK_HI_EXPR:
544 return TYPE_UNSIGNED (type) ?
545 vec_unpacku_hi_optab : vec_unpacks_hi_optab;
547 case VEC_UNPACK_LO_EXPR:
548 return TYPE_UNSIGNED (type) ?
549 vec_unpacku_lo_optab : vec_unpacks_lo_optab;
551 case VEC_UNPACK_FLOAT_HI_EXPR:
552 /* The signedness is determined from input operand. */
553 return TYPE_UNSIGNED (type) ?
554 vec_unpacku_float_hi_optab : vec_unpacks_float_hi_optab;
556 case VEC_UNPACK_FLOAT_LO_EXPR:
557 /* The signedness is determined from input operand. */
558 return TYPE_UNSIGNED (type) ?
559 vec_unpacku_float_lo_optab : vec_unpacks_float_lo_optab;
561 case VEC_PACK_TRUNC_EXPR:
562 return vec_pack_trunc_optab;
564 case VEC_PACK_SAT_EXPR:
565 return TYPE_UNSIGNED (type) ? vec_pack_usat_optab : vec_pack_ssat_optab;
567 case VEC_PACK_FIX_TRUNC_EXPR:
568 /* The signedness is determined from output operand. */
569 return TYPE_UNSIGNED (type) ?
570 vec_pack_ufix_trunc_optab : vec_pack_sfix_trunc_optab;
572 default:
573 break;
576 trapv = INTEGRAL_TYPE_P (type) && TYPE_OVERFLOW_TRAPS (type);
577 switch (code)
579 case POINTER_PLUS_EXPR:
580 case PLUS_EXPR:
581 if (TYPE_SATURATING (type))
582 return TYPE_UNSIGNED (type) ? usadd_optab : ssadd_optab;
583 return trapv ? addv_optab : add_optab;
585 case MINUS_EXPR:
586 if (TYPE_SATURATING (type))
587 return TYPE_UNSIGNED (type) ? ussub_optab : sssub_optab;
588 return trapv ? subv_optab : sub_optab;
590 case MULT_EXPR:
591 if (TYPE_SATURATING (type))
592 return TYPE_UNSIGNED (type) ? usmul_optab : ssmul_optab;
593 return trapv ? smulv_optab : smul_optab;
595 case NEGATE_EXPR:
596 if (TYPE_SATURATING (type))
597 return TYPE_UNSIGNED (type) ? usneg_optab : ssneg_optab;
598 return trapv ? negv_optab : neg_optab;
600 case ABS_EXPR:
601 return trapv ? absv_optab : abs_optab;
603 default:
604 return unknown_optab;
608 /* Given optab UNOPTAB that reduces a vector to a scalar, find instead the old
609 optab that produces a vector with the reduction result in one element,
610 for a tree with type TYPE. */
612 optab
613 scalar_reduc_to_vector (optab unoptab, const_tree type)
615 switch (unoptab)
617 case reduc_plus_scal_optab:
618 return TYPE_UNSIGNED (type) ? reduc_uplus_optab : reduc_splus_optab;
620 case reduc_smin_scal_optab: return reduc_smin_optab;
621 case reduc_umin_scal_optab: return reduc_umin_optab;
622 case reduc_smax_scal_optab: return reduc_smax_optab;
623 case reduc_umax_scal_optab: return reduc_umax_optab;
624 default: return unknown_optab;
628 /* Expand vector widening operations.
630 There are two different classes of operations handled here:
631 1) Operations whose result is wider than all the arguments to the operation.
632 Examples: VEC_UNPACK_HI/LO_EXPR, VEC_WIDEN_MULT_HI/LO_EXPR
633 In this case OP0 and optionally OP1 would be initialized,
634 but WIDE_OP wouldn't (not relevant for this case).
635 2) Operations whose result is of the same size as the last argument to the
636 operation, but wider than all the other arguments to the operation.
637 Examples: WIDEN_SUM_EXPR, VEC_DOT_PROD_EXPR.
638 In the case WIDE_OP, OP0 and optionally OP1 would be initialized.
640 E.g, when called to expand the following operations, this is how
641 the arguments will be initialized:
642 nops OP0 OP1 WIDE_OP
643 widening-sum 2 oprnd0 - oprnd1
644 widening-dot-product 3 oprnd0 oprnd1 oprnd2
645 widening-mult 2 oprnd0 oprnd1 -
646 type-promotion (vec-unpack) 1 oprnd0 - - */
649 expand_widen_pattern_expr (sepops ops, rtx op0, rtx op1, rtx wide_op,
650 rtx target, int unsignedp)
652 struct expand_operand eops[4];
653 tree oprnd0, oprnd1, oprnd2;
654 machine_mode wmode = VOIDmode, tmode0, tmode1 = VOIDmode;
655 optab widen_pattern_optab;
656 enum insn_code icode;
657 int nops = TREE_CODE_LENGTH (ops->code);
658 int op;
660 oprnd0 = ops->op0;
661 tmode0 = TYPE_MODE (TREE_TYPE (oprnd0));
662 widen_pattern_optab =
663 optab_for_tree_code (ops->code, TREE_TYPE (oprnd0), optab_default);
664 if (ops->code == WIDEN_MULT_PLUS_EXPR
665 || ops->code == WIDEN_MULT_MINUS_EXPR)
666 icode = find_widening_optab_handler (widen_pattern_optab,
667 TYPE_MODE (TREE_TYPE (ops->op2)),
668 tmode0, 0);
669 else
670 icode = optab_handler (widen_pattern_optab, tmode0);
671 gcc_assert (icode != CODE_FOR_nothing);
673 if (nops >= 2)
675 oprnd1 = ops->op1;
676 tmode1 = TYPE_MODE (TREE_TYPE (oprnd1));
679 /* The last operand is of a wider mode than the rest of the operands. */
680 if (nops == 2)
681 wmode = tmode1;
682 else if (nops == 3)
684 gcc_assert (tmode1 == tmode0);
685 gcc_assert (op1);
686 oprnd2 = ops->op2;
687 wmode = TYPE_MODE (TREE_TYPE (oprnd2));
690 op = 0;
691 create_output_operand (&eops[op++], target, TYPE_MODE (ops->type));
692 create_convert_operand_from (&eops[op++], op0, tmode0, unsignedp);
693 if (op1)
694 create_convert_operand_from (&eops[op++], op1, tmode1, unsignedp);
695 if (wide_op)
696 create_convert_operand_from (&eops[op++], wide_op, wmode, unsignedp);
697 expand_insn (icode, op, eops);
698 return eops[0].value;
701 /* Generate code to perform an operation specified by TERNARY_OPTAB
702 on operands OP0, OP1 and OP2, with result having machine-mode MODE.
704 UNSIGNEDP is for the case where we have to widen the operands
705 to perform the operation. It says to use zero-extension.
707 If TARGET is nonzero, the value
708 is generated there, if it is convenient to do so.
709 In all cases an rtx is returned for the locus of the value;
710 this may or may not be TARGET. */
713 expand_ternary_op (machine_mode mode, optab ternary_optab, rtx op0,
714 rtx op1, rtx op2, rtx target, int unsignedp)
716 struct expand_operand ops[4];
717 enum insn_code icode = optab_handler (ternary_optab, mode);
719 gcc_assert (optab_handler (ternary_optab, mode) != CODE_FOR_nothing);
721 create_output_operand (&ops[0], target, mode);
722 create_convert_operand_from (&ops[1], op0, mode, unsignedp);
723 create_convert_operand_from (&ops[2], op1, mode, unsignedp);
724 create_convert_operand_from (&ops[3], op2, mode, unsignedp);
725 expand_insn (icode, 4, ops);
726 return ops[0].value;
730 /* Like expand_binop, but return a constant rtx if the result can be
731 calculated at compile time. The arguments and return value are
732 otherwise the same as for expand_binop. */
735 simplify_expand_binop (machine_mode mode, optab binoptab,
736 rtx op0, rtx op1, rtx target, int unsignedp,
737 enum optab_methods methods)
739 if (CONSTANT_P (op0) && CONSTANT_P (op1))
741 rtx x = simplify_binary_operation (optab_to_code (binoptab),
742 mode, op0, op1);
743 if (x)
744 return x;
747 return expand_binop (mode, binoptab, op0, op1, target, unsignedp, methods);
750 /* Like simplify_expand_binop, but always put the result in TARGET.
751 Return true if the expansion succeeded. */
753 bool
754 force_expand_binop (machine_mode mode, optab binoptab,
755 rtx op0, rtx op1, rtx target, int unsignedp,
756 enum optab_methods methods)
758 rtx x = simplify_expand_binop (mode, binoptab, op0, op1,
759 target, unsignedp, methods);
760 if (x == 0)
761 return false;
762 if (x != target)
763 emit_move_insn (target, x);
764 return true;
767 /* Create a new vector value in VMODE with all elements set to OP. The
768 mode of OP must be the element mode of VMODE. If OP is a constant,
769 then the return value will be a constant. */
771 static rtx
772 expand_vector_broadcast (machine_mode vmode, rtx op)
774 enum insn_code icode;
775 rtvec vec;
776 rtx ret;
777 int i, n;
779 gcc_checking_assert (VECTOR_MODE_P (vmode));
781 n = GET_MODE_NUNITS (vmode);
782 vec = rtvec_alloc (n);
783 for (i = 0; i < n; ++i)
784 RTVEC_ELT (vec, i) = op;
786 if (CONSTANT_P (op))
787 return gen_rtx_CONST_VECTOR (vmode, vec);
789 /* ??? If the target doesn't have a vec_init, then we have no easy way
790 of performing this operation. Most of this sort of generic support
791 is hidden away in the vector lowering support in gimple. */
792 icode = optab_handler (vec_init_optab, vmode);
793 if (icode == CODE_FOR_nothing)
794 return NULL;
796 ret = gen_reg_rtx (vmode);
797 emit_insn (GEN_FCN (icode) (ret, gen_rtx_PARALLEL (vmode, vec)));
799 return ret;
802 /* This subroutine of expand_doubleword_shift handles the cases in which
803 the effective shift value is >= BITS_PER_WORD. The arguments and return
804 value are the same as for the parent routine, except that SUPERWORD_OP1
805 is the shift count to use when shifting OUTOF_INPUT into INTO_TARGET.
806 INTO_TARGET may be null if the caller has decided to calculate it. */
808 static bool
809 expand_superword_shift (optab binoptab, rtx outof_input, rtx superword_op1,
810 rtx outof_target, rtx into_target,
811 int unsignedp, enum optab_methods methods)
813 if (into_target != 0)
814 if (!force_expand_binop (word_mode, binoptab, outof_input, superword_op1,
815 into_target, unsignedp, methods))
816 return false;
818 if (outof_target != 0)
820 /* For a signed right shift, we must fill OUTOF_TARGET with copies
821 of the sign bit, otherwise we must fill it with zeros. */
822 if (binoptab != ashr_optab)
823 emit_move_insn (outof_target, CONST0_RTX (word_mode));
824 else
825 if (!force_expand_binop (word_mode, binoptab,
826 outof_input, GEN_INT (BITS_PER_WORD - 1),
827 outof_target, unsignedp, methods))
828 return false;
830 return true;
833 /* This subroutine of expand_doubleword_shift handles the cases in which
834 the effective shift value is < BITS_PER_WORD. The arguments and return
835 value are the same as for the parent routine. */
837 static bool
838 expand_subword_shift (machine_mode op1_mode, optab binoptab,
839 rtx outof_input, rtx into_input, rtx op1,
840 rtx outof_target, rtx into_target,
841 int unsignedp, enum optab_methods methods,
842 unsigned HOST_WIDE_INT shift_mask)
844 optab reverse_unsigned_shift, unsigned_shift;
845 rtx tmp, carries;
847 reverse_unsigned_shift = (binoptab == ashl_optab ? lshr_optab : ashl_optab);
848 unsigned_shift = (binoptab == ashl_optab ? ashl_optab : lshr_optab);
850 /* The low OP1 bits of INTO_TARGET come from the high bits of OUTOF_INPUT.
851 We therefore need to shift OUTOF_INPUT by (BITS_PER_WORD - OP1) bits in
852 the opposite direction to BINOPTAB. */
853 if (CONSTANT_P (op1) || shift_mask >= BITS_PER_WORD)
855 carries = outof_input;
856 tmp = immed_wide_int_const (wi::shwi (BITS_PER_WORD,
857 op1_mode), op1_mode);
858 tmp = simplify_expand_binop (op1_mode, sub_optab, tmp, op1,
859 0, true, methods);
861 else
863 /* We must avoid shifting by BITS_PER_WORD bits since that is either
864 the same as a zero shift (if shift_mask == BITS_PER_WORD - 1) or
865 has unknown behavior. Do a single shift first, then shift by the
866 remainder. It's OK to use ~OP1 as the remainder if shift counts
867 are truncated to the mode size. */
868 carries = expand_binop (word_mode, reverse_unsigned_shift,
869 outof_input, const1_rtx, 0, unsignedp, methods);
870 if (shift_mask == BITS_PER_WORD - 1)
872 tmp = immed_wide_int_const
873 (wi::minus_one (GET_MODE_PRECISION (op1_mode)), op1_mode);
874 tmp = simplify_expand_binop (op1_mode, xor_optab, op1, tmp,
875 0, true, methods);
877 else
879 tmp = immed_wide_int_const (wi::shwi (BITS_PER_WORD - 1,
880 op1_mode), op1_mode);
881 tmp = simplify_expand_binop (op1_mode, sub_optab, tmp, op1,
882 0, true, methods);
885 if (tmp == 0 || carries == 0)
886 return false;
887 carries = expand_binop (word_mode, reverse_unsigned_shift,
888 carries, tmp, 0, unsignedp, methods);
889 if (carries == 0)
890 return false;
892 /* Shift INTO_INPUT logically by OP1. This is the last use of INTO_INPUT
893 so the result can go directly into INTO_TARGET if convenient. */
894 tmp = expand_binop (word_mode, unsigned_shift, into_input, op1,
895 into_target, unsignedp, methods);
896 if (tmp == 0)
897 return false;
899 /* Now OR in the bits carried over from OUTOF_INPUT. */
900 if (!force_expand_binop (word_mode, ior_optab, tmp, carries,
901 into_target, unsignedp, methods))
902 return false;
904 /* Use a standard word_mode shift for the out-of half. */
905 if (outof_target != 0)
906 if (!force_expand_binop (word_mode, binoptab, outof_input, op1,
907 outof_target, unsignedp, methods))
908 return false;
910 return true;
914 /* Try implementing expand_doubleword_shift using conditional moves.
915 The shift is by < BITS_PER_WORD if (CMP_CODE CMP1 CMP2) is true,
916 otherwise it is by >= BITS_PER_WORD. SUBWORD_OP1 and SUPERWORD_OP1
917 are the shift counts to use in the former and latter case. All other
918 arguments are the same as the parent routine. */
920 static bool
921 expand_doubleword_shift_condmove (machine_mode op1_mode, optab binoptab,
922 enum rtx_code cmp_code, rtx cmp1, rtx cmp2,
923 rtx outof_input, rtx into_input,
924 rtx subword_op1, rtx superword_op1,
925 rtx outof_target, rtx into_target,
926 int unsignedp, enum optab_methods methods,
927 unsigned HOST_WIDE_INT shift_mask)
929 rtx outof_superword, into_superword;
931 /* Put the superword version of the output into OUTOF_SUPERWORD and
932 INTO_SUPERWORD. */
933 outof_superword = outof_target != 0 ? gen_reg_rtx (word_mode) : 0;
934 if (outof_target != 0 && subword_op1 == superword_op1)
936 /* The value INTO_TARGET >> SUBWORD_OP1, which we later store in
937 OUTOF_TARGET, is the same as the value of INTO_SUPERWORD. */
938 into_superword = outof_target;
939 if (!expand_superword_shift (binoptab, outof_input, superword_op1,
940 outof_superword, 0, unsignedp, methods))
941 return false;
943 else
945 into_superword = gen_reg_rtx (word_mode);
946 if (!expand_superword_shift (binoptab, outof_input, superword_op1,
947 outof_superword, into_superword,
948 unsignedp, methods))
949 return false;
952 /* Put the subword version directly in OUTOF_TARGET and INTO_TARGET. */
953 if (!expand_subword_shift (op1_mode, binoptab,
954 outof_input, into_input, subword_op1,
955 outof_target, into_target,
956 unsignedp, methods, shift_mask))
957 return false;
959 /* Select between them. Do the INTO half first because INTO_SUPERWORD
960 might be the current value of OUTOF_TARGET. */
961 if (!emit_conditional_move (into_target, cmp_code, cmp1, cmp2, op1_mode,
962 into_target, into_superword, word_mode, false))
963 return false;
965 if (outof_target != 0)
966 if (!emit_conditional_move (outof_target, cmp_code, cmp1, cmp2, op1_mode,
967 outof_target, outof_superword,
968 word_mode, false))
969 return false;
971 return true;
974 /* Expand a doubleword shift (ashl, ashr or lshr) using word-mode shifts.
975 OUTOF_INPUT and INTO_INPUT are the two word-sized halves of the first
976 input operand; the shift moves bits in the direction OUTOF_INPUT->
977 INTO_TARGET. OUTOF_TARGET and INTO_TARGET are the equivalent words
978 of the target. OP1 is the shift count and OP1_MODE is its mode.
979 If OP1 is constant, it will have been truncated as appropriate
980 and is known to be nonzero.
982 If SHIFT_MASK is zero, the result of word shifts is undefined when the
983 shift count is outside the range [0, BITS_PER_WORD). This routine must
984 avoid generating such shifts for OP1s in the range [0, BITS_PER_WORD * 2).
986 If SHIFT_MASK is nonzero, all word-mode shift counts are effectively
987 masked by it and shifts in the range [BITS_PER_WORD, SHIFT_MASK) will
988 fill with zeros or sign bits as appropriate.
990 If SHIFT_MASK is BITS_PER_WORD - 1, this routine will synthesize
991 a doubleword shift whose equivalent mask is BITS_PER_WORD * 2 - 1.
992 Doing this preserves semantics required by SHIFT_COUNT_TRUNCATED.
993 In all other cases, shifts by values outside [0, BITS_PER_UNIT * 2)
994 are undefined.
996 BINOPTAB, UNSIGNEDP and METHODS are as for expand_binop. This function
997 may not use INTO_INPUT after modifying INTO_TARGET, and similarly for
998 OUTOF_INPUT and OUTOF_TARGET. OUTOF_TARGET can be null if the parent
999 function wants to calculate it itself.
1001 Return true if the shift could be successfully synthesized. */
1003 static bool
1004 expand_doubleword_shift (machine_mode op1_mode, optab binoptab,
1005 rtx outof_input, rtx into_input, rtx op1,
1006 rtx outof_target, rtx into_target,
1007 int unsignedp, enum optab_methods methods,
1008 unsigned HOST_WIDE_INT shift_mask)
1010 rtx superword_op1, tmp, cmp1, cmp2;
1011 enum rtx_code cmp_code;
1013 /* See if word-mode shifts by BITS_PER_WORD...BITS_PER_WORD * 2 - 1 will
1014 fill the result with sign or zero bits as appropriate. If so, the value
1015 of OUTOF_TARGET will always be (SHIFT OUTOF_INPUT OP1). Recursively call
1016 this routine to calculate INTO_TARGET (which depends on both OUTOF_INPUT
1017 and INTO_INPUT), then emit code to set up OUTOF_TARGET.
1019 This isn't worthwhile for constant shifts since the optimizers will
1020 cope better with in-range shift counts. */
1021 if (shift_mask >= BITS_PER_WORD
1022 && outof_target != 0
1023 && !CONSTANT_P (op1))
1025 if (!expand_doubleword_shift (op1_mode, binoptab,
1026 outof_input, into_input, op1,
1027 0, into_target,
1028 unsignedp, methods, shift_mask))
1029 return false;
1030 if (!force_expand_binop (word_mode, binoptab, outof_input, op1,
1031 outof_target, unsignedp, methods))
1032 return false;
1033 return true;
1036 /* Set CMP_CODE, CMP1 and CMP2 so that the rtx (CMP_CODE CMP1 CMP2)
1037 is true when the effective shift value is less than BITS_PER_WORD.
1038 Set SUPERWORD_OP1 to the shift count that should be used to shift
1039 OUTOF_INPUT into INTO_TARGET when the condition is false. */
1040 tmp = immed_wide_int_const (wi::shwi (BITS_PER_WORD, op1_mode), op1_mode);
1041 if (!CONSTANT_P (op1) && shift_mask == BITS_PER_WORD - 1)
1043 /* Set CMP1 to OP1 & BITS_PER_WORD. The result is zero iff OP1
1044 is a subword shift count. */
1045 cmp1 = simplify_expand_binop (op1_mode, and_optab, op1, tmp,
1046 0, true, methods);
1047 cmp2 = CONST0_RTX (op1_mode);
1048 cmp_code = EQ;
1049 superword_op1 = op1;
1051 else
1053 /* Set CMP1 to OP1 - BITS_PER_WORD. */
1054 cmp1 = simplify_expand_binop (op1_mode, sub_optab, op1, tmp,
1055 0, true, methods);
1056 cmp2 = CONST0_RTX (op1_mode);
1057 cmp_code = LT;
1058 superword_op1 = cmp1;
1060 if (cmp1 == 0)
1061 return false;
1063 /* If we can compute the condition at compile time, pick the
1064 appropriate subroutine. */
1065 tmp = simplify_relational_operation (cmp_code, SImode, op1_mode, cmp1, cmp2);
1066 if (tmp != 0 && CONST_INT_P (tmp))
1068 if (tmp == const0_rtx)
1069 return expand_superword_shift (binoptab, outof_input, superword_op1,
1070 outof_target, into_target,
1071 unsignedp, methods);
1072 else
1073 return expand_subword_shift (op1_mode, binoptab,
1074 outof_input, into_input, op1,
1075 outof_target, into_target,
1076 unsignedp, methods, shift_mask);
1079 /* Try using conditional moves to generate straight-line code. */
1080 if (HAVE_conditional_move)
1082 rtx_insn *start = get_last_insn ();
1083 if (expand_doubleword_shift_condmove (op1_mode, binoptab,
1084 cmp_code, cmp1, cmp2,
1085 outof_input, into_input,
1086 op1, superword_op1,
1087 outof_target, into_target,
1088 unsignedp, methods, shift_mask))
1089 return true;
1090 delete_insns_since (start);
1093 /* As a last resort, use branches to select the correct alternative. */
1094 rtx_code_label *subword_label = gen_label_rtx ();
1095 rtx_code_label *done_label = gen_label_rtx ();
1097 NO_DEFER_POP;
1098 do_compare_rtx_and_jump (cmp1, cmp2, cmp_code, false, op1_mode,
1099 0, 0, subword_label, -1);
1100 OK_DEFER_POP;
1102 if (!expand_superword_shift (binoptab, outof_input, superword_op1,
1103 outof_target, into_target,
1104 unsignedp, methods))
1105 return false;
1107 emit_jump_insn (targetm.gen_jump (done_label));
1108 emit_barrier ();
1109 emit_label (subword_label);
1111 if (!expand_subword_shift (op1_mode, binoptab,
1112 outof_input, into_input, op1,
1113 outof_target, into_target,
1114 unsignedp, methods, shift_mask))
1115 return false;
1117 emit_label (done_label);
1118 return true;
1121 /* Subroutine of expand_binop. Perform a double word multiplication of
1122 operands OP0 and OP1 both of mode MODE, which is exactly twice as wide
1123 as the target's word_mode. This function return NULL_RTX if anything
1124 goes wrong, in which case it may have already emitted instructions
1125 which need to be deleted.
1127 If we want to multiply two two-word values and have normal and widening
1128 multiplies of single-word values, we can do this with three smaller
1129 multiplications.
1131 The multiplication proceeds as follows:
1132 _______________________
1133 [__op0_high_|__op0_low__]
1134 _______________________
1135 * [__op1_high_|__op1_low__]
1136 _______________________________________________
1137 _______________________
1138 (1) [__op0_low__*__op1_low__]
1139 _______________________
1140 (2a) [__op0_low__*__op1_high_]
1141 _______________________
1142 (2b) [__op0_high_*__op1_low__]
1143 _______________________
1144 (3) [__op0_high_*__op1_high_]
1147 This gives a 4-word result. Since we are only interested in the
1148 lower 2 words, partial result (3) and the upper words of (2a) and
1149 (2b) don't need to be calculated. Hence (2a) and (2b) can be
1150 calculated using non-widening multiplication.
1152 (1), however, needs to be calculated with an unsigned widening
1153 multiplication. If this operation is not directly supported we
1154 try using a signed widening multiplication and adjust the result.
1155 This adjustment works as follows:
1157 If both operands are positive then no adjustment is needed.
1159 If the operands have different signs, for example op0_low < 0 and
1160 op1_low >= 0, the instruction treats the most significant bit of
1161 op0_low as a sign bit instead of a bit with significance
1162 2**(BITS_PER_WORD-1), i.e. the instruction multiplies op1_low
1163 with 2**BITS_PER_WORD - op0_low, and two's complements the
1164 result. Conclusion: We need to add op1_low * 2**BITS_PER_WORD to
1165 the result.
1167 Similarly, if both operands are negative, we need to add
1168 (op0_low + op1_low) * 2**BITS_PER_WORD.
1170 We use a trick to adjust quickly. We logically shift op0_low right
1171 (op1_low) BITS_PER_WORD-1 steps to get 0 or 1, and add this to
1172 op0_high (op1_high) before it is used to calculate 2b (2a). If no
1173 logical shift exists, we do an arithmetic right shift and subtract
1174 the 0 or -1. */
1176 static rtx
1177 expand_doubleword_mult (machine_mode mode, rtx op0, rtx op1, rtx target,
1178 bool umulp, enum optab_methods methods)
1180 int low = (WORDS_BIG_ENDIAN ? 1 : 0);
1181 int high = (WORDS_BIG_ENDIAN ? 0 : 1);
1182 rtx wordm1 = umulp ? NULL_RTX : GEN_INT (BITS_PER_WORD - 1);
1183 rtx product, adjust, product_high, temp;
1185 rtx op0_high = operand_subword_force (op0, high, mode);
1186 rtx op0_low = operand_subword_force (op0, low, mode);
1187 rtx op1_high = operand_subword_force (op1, high, mode);
1188 rtx op1_low = operand_subword_force (op1, low, mode);
1190 /* If we're using an unsigned multiply to directly compute the product
1191 of the low-order words of the operands and perform any required
1192 adjustments of the operands, we begin by trying two more multiplications
1193 and then computing the appropriate sum.
1195 We have checked above that the required addition is provided.
1196 Full-word addition will normally always succeed, especially if
1197 it is provided at all, so we don't worry about its failure. The
1198 multiplication may well fail, however, so we do handle that. */
1200 if (!umulp)
1202 /* ??? This could be done with emit_store_flag where available. */
1203 temp = expand_binop (word_mode, lshr_optab, op0_low, wordm1,
1204 NULL_RTX, 1, methods);
1205 if (temp)
1206 op0_high = expand_binop (word_mode, add_optab, op0_high, temp,
1207 NULL_RTX, 0, OPTAB_DIRECT);
1208 else
1210 temp = expand_binop (word_mode, ashr_optab, op0_low, wordm1,
1211 NULL_RTX, 0, methods);
1212 if (!temp)
1213 return NULL_RTX;
1214 op0_high = expand_binop (word_mode, sub_optab, op0_high, temp,
1215 NULL_RTX, 0, OPTAB_DIRECT);
1218 if (!op0_high)
1219 return NULL_RTX;
1222 adjust = expand_binop (word_mode, smul_optab, op0_high, op1_low,
1223 NULL_RTX, 0, OPTAB_DIRECT);
1224 if (!adjust)
1225 return NULL_RTX;
1227 /* OP0_HIGH should now be dead. */
1229 if (!umulp)
1231 /* ??? This could be done with emit_store_flag where available. */
1232 temp = expand_binop (word_mode, lshr_optab, op1_low, wordm1,
1233 NULL_RTX, 1, methods);
1234 if (temp)
1235 op1_high = expand_binop (word_mode, add_optab, op1_high, temp,
1236 NULL_RTX, 0, OPTAB_DIRECT);
1237 else
1239 temp = expand_binop (word_mode, ashr_optab, op1_low, wordm1,
1240 NULL_RTX, 0, methods);
1241 if (!temp)
1242 return NULL_RTX;
1243 op1_high = expand_binop (word_mode, sub_optab, op1_high, temp,
1244 NULL_RTX, 0, OPTAB_DIRECT);
1247 if (!op1_high)
1248 return NULL_RTX;
1251 temp = expand_binop (word_mode, smul_optab, op1_high, op0_low,
1252 NULL_RTX, 0, OPTAB_DIRECT);
1253 if (!temp)
1254 return NULL_RTX;
1256 /* OP1_HIGH should now be dead. */
1258 adjust = expand_binop (word_mode, add_optab, adjust, temp,
1259 NULL_RTX, 0, OPTAB_DIRECT);
1261 if (target && !REG_P (target))
1262 target = NULL_RTX;
1264 if (umulp)
1265 product = expand_binop (mode, umul_widen_optab, op0_low, op1_low,
1266 target, 1, OPTAB_DIRECT);
1267 else
1268 product = expand_binop (mode, smul_widen_optab, op0_low, op1_low,
1269 target, 1, OPTAB_DIRECT);
1271 if (!product)
1272 return NULL_RTX;
1274 product_high = operand_subword (product, high, 1, mode);
1275 adjust = expand_binop (word_mode, add_optab, product_high, adjust,
1276 NULL_RTX, 0, OPTAB_DIRECT);
1277 emit_move_insn (product_high, adjust);
1278 return product;
1281 /* Wrapper around expand_binop which takes an rtx code to specify
1282 the operation to perform, not an optab pointer. All other
1283 arguments are the same. */
1285 expand_simple_binop (machine_mode mode, enum rtx_code code, rtx op0,
1286 rtx op1, rtx target, int unsignedp,
1287 enum optab_methods methods)
1289 optab binop = code_to_optab (code);
1290 gcc_assert (binop);
1292 return expand_binop (mode, binop, op0, op1, target, unsignedp, methods);
1295 /* Return whether OP0 and OP1 should be swapped when expanding a commutative
1296 binop. Order them according to commutative_operand_precedence and, if
1297 possible, try to put TARGET or a pseudo first. */
1298 static bool
1299 swap_commutative_operands_with_target (rtx target, rtx op0, rtx op1)
1301 int op0_prec = commutative_operand_precedence (op0);
1302 int op1_prec = commutative_operand_precedence (op1);
1304 if (op0_prec < op1_prec)
1305 return true;
1307 if (op0_prec > op1_prec)
1308 return false;
1310 /* With equal precedence, both orders are ok, but it is better if the
1311 first operand is TARGET, or if both TARGET and OP0 are pseudos. */
1312 if (target == 0 || REG_P (target))
1313 return (REG_P (op1) && !REG_P (op0)) || target == op1;
1314 else
1315 return rtx_equal_p (op1, target);
1318 /* Return true if BINOPTAB implements a shift operation. */
1320 static bool
1321 shift_optab_p (optab binoptab)
1323 switch (optab_to_code (binoptab))
1325 case ASHIFT:
1326 case SS_ASHIFT:
1327 case US_ASHIFT:
1328 case ASHIFTRT:
1329 case LSHIFTRT:
1330 case ROTATE:
1331 case ROTATERT:
1332 return true;
1334 default:
1335 return false;
1339 /* Return true if BINOPTAB implements a commutative binary operation. */
1341 static bool
1342 commutative_optab_p (optab binoptab)
1344 return (GET_RTX_CLASS (optab_to_code (binoptab)) == RTX_COMM_ARITH
1345 || binoptab == smul_widen_optab
1346 || binoptab == umul_widen_optab
1347 || binoptab == smul_highpart_optab
1348 || binoptab == umul_highpart_optab);
1351 /* X is to be used in mode MODE as operand OPN to BINOPTAB. If we're
1352 optimizing, and if the operand is a constant that costs more than
1353 1 instruction, force the constant into a register and return that
1354 register. Return X otherwise. UNSIGNEDP says whether X is unsigned. */
1356 static rtx
1357 avoid_expensive_constant (machine_mode mode, optab binoptab,
1358 int opn, rtx x, bool unsignedp)
1360 bool speed = optimize_insn_for_speed_p ();
1362 if (mode != VOIDmode
1363 && optimize
1364 && CONSTANT_P (x)
1365 && (rtx_cost (x, mode, optab_to_code (binoptab), opn, speed)
1366 > set_src_cost (x, mode, speed)))
1368 if (CONST_INT_P (x))
1370 HOST_WIDE_INT intval = trunc_int_for_mode (INTVAL (x), mode);
1371 if (intval != INTVAL (x))
1372 x = GEN_INT (intval);
1374 else
1375 x = convert_modes (mode, VOIDmode, x, unsignedp);
1376 x = force_reg (mode, x);
1378 return x;
1381 /* Helper function for expand_binop: handle the case where there
1382 is an insn that directly implements the indicated operation.
1383 Returns null if this is not possible. */
1384 static rtx
1385 expand_binop_directly (machine_mode mode, optab binoptab,
1386 rtx op0, rtx op1,
1387 rtx target, int unsignedp, enum optab_methods methods,
1388 rtx_insn *last)
1390 machine_mode from_mode = widened_mode (mode, op0, op1);
1391 enum insn_code icode = find_widening_optab_handler (binoptab, mode,
1392 from_mode, 1);
1393 machine_mode xmode0 = insn_data[(int) icode].operand[1].mode;
1394 machine_mode xmode1 = insn_data[(int) icode].operand[2].mode;
1395 machine_mode mode0, mode1, tmp_mode;
1396 struct expand_operand ops[3];
1397 bool commutative_p;
1398 rtx_insn *pat;
1399 rtx xop0 = op0, xop1 = op1;
1401 /* If it is a commutative operator and the modes would match
1402 if we would swap the operands, we can save the conversions. */
1403 commutative_p = commutative_optab_p (binoptab);
1404 if (commutative_p
1405 && GET_MODE (xop0) != xmode0 && GET_MODE (xop1) != xmode1
1406 && GET_MODE (xop0) == xmode1 && GET_MODE (xop1) == xmode1)
1407 std::swap (xop0, xop1);
1409 /* If we are optimizing, force expensive constants into a register. */
1410 xop0 = avoid_expensive_constant (xmode0, binoptab, 0, xop0, unsignedp);
1411 if (!shift_optab_p (binoptab))
1412 xop1 = avoid_expensive_constant (xmode1, binoptab, 1, xop1, unsignedp);
1414 /* In case the insn wants input operands in modes different from
1415 those of the actual operands, convert the operands. It would
1416 seem that we don't need to convert CONST_INTs, but we do, so
1417 that they're properly zero-extended, sign-extended or truncated
1418 for their mode. */
1420 mode0 = GET_MODE (xop0) != VOIDmode ? GET_MODE (xop0) : mode;
1421 if (xmode0 != VOIDmode && xmode0 != mode0)
1423 xop0 = convert_modes (xmode0, mode0, xop0, unsignedp);
1424 mode0 = xmode0;
1427 mode1 = GET_MODE (xop1) != VOIDmode ? GET_MODE (xop1) : mode;
1428 if (xmode1 != VOIDmode && xmode1 != mode1)
1430 xop1 = convert_modes (xmode1, mode1, xop1, unsignedp);
1431 mode1 = xmode1;
1434 /* If operation is commutative,
1435 try to make the first operand a register.
1436 Even better, try to make it the same as the target.
1437 Also try to make the last operand a constant. */
1438 if (commutative_p
1439 && swap_commutative_operands_with_target (target, xop0, xop1))
1440 std::swap (xop0, xop1);
1442 /* Now, if insn's predicates don't allow our operands, put them into
1443 pseudo regs. */
1445 if (binoptab == vec_pack_trunc_optab
1446 || binoptab == vec_pack_usat_optab
1447 || binoptab == vec_pack_ssat_optab
1448 || binoptab == vec_pack_ufix_trunc_optab
1449 || binoptab == vec_pack_sfix_trunc_optab)
1451 /* The mode of the result is different then the mode of the
1452 arguments. */
1453 tmp_mode = insn_data[(int) icode].operand[0].mode;
1454 if (GET_MODE_NUNITS (tmp_mode) != 2 * GET_MODE_NUNITS (mode))
1456 delete_insns_since (last);
1457 return NULL_RTX;
1460 else
1461 tmp_mode = mode;
1463 create_output_operand (&ops[0], target, tmp_mode);
1464 create_input_operand (&ops[1], xop0, mode0);
1465 create_input_operand (&ops[2], xop1, mode1);
1466 pat = maybe_gen_insn (icode, 3, ops);
1467 if (pat)
1469 /* If PAT is composed of more than one insn, try to add an appropriate
1470 REG_EQUAL note to it. If we can't because TEMP conflicts with an
1471 operand, call expand_binop again, this time without a target. */
1472 if (INSN_P (pat) && NEXT_INSN (pat) != NULL_RTX
1473 && ! add_equal_note (pat, ops[0].value,
1474 optab_to_code (binoptab),
1475 ops[1].value, ops[2].value))
1477 delete_insns_since (last);
1478 return expand_binop (mode, binoptab, op0, op1, NULL_RTX,
1479 unsignedp, methods);
1482 emit_insn (pat);
1483 return ops[0].value;
1485 delete_insns_since (last);
1486 return NULL_RTX;
1489 /* Generate code to perform an operation specified by BINOPTAB
1490 on operands OP0 and OP1, with result having machine-mode MODE.
1492 UNSIGNEDP is for the case where we have to widen the operands
1493 to perform the operation. It says to use zero-extension.
1495 If TARGET is nonzero, the value
1496 is generated there, if it is convenient to do so.
1497 In all cases an rtx is returned for the locus of the value;
1498 this may or may not be TARGET. */
1501 expand_binop (machine_mode mode, optab binoptab, rtx op0, rtx op1,
1502 rtx target, int unsignedp, enum optab_methods methods)
1504 enum optab_methods next_methods
1505 = (methods == OPTAB_LIB || methods == OPTAB_LIB_WIDEN
1506 ? OPTAB_WIDEN : methods);
1507 enum mode_class mclass;
1508 machine_mode wider_mode;
1509 rtx libfunc;
1510 rtx temp;
1511 rtx_insn *entry_last = get_last_insn ();
1512 rtx_insn *last;
1514 mclass = GET_MODE_CLASS (mode);
1516 /* If subtracting an integer constant, convert this into an addition of
1517 the negated constant. */
1519 if (binoptab == sub_optab && CONST_INT_P (op1))
1521 op1 = negate_rtx (mode, op1);
1522 binoptab = add_optab;
1525 /* Record where to delete back to if we backtrack. */
1526 last = get_last_insn ();
1528 /* If we can do it with a three-operand insn, do so. */
1530 if (methods != OPTAB_MUST_WIDEN
1531 && find_widening_optab_handler (binoptab, mode,
1532 widened_mode (mode, op0, op1), 1)
1533 != CODE_FOR_nothing)
1535 temp = expand_binop_directly (mode, binoptab, op0, op1, target,
1536 unsignedp, methods, last);
1537 if (temp)
1538 return temp;
1541 /* If we were trying to rotate, and that didn't work, try rotating
1542 the other direction before falling back to shifts and bitwise-or. */
1543 if (((binoptab == rotl_optab
1544 && optab_handler (rotr_optab, mode) != CODE_FOR_nothing)
1545 || (binoptab == rotr_optab
1546 && optab_handler (rotl_optab, mode) != CODE_FOR_nothing))
1547 && mclass == MODE_INT)
1549 optab otheroptab = (binoptab == rotl_optab ? rotr_optab : rotl_optab);
1550 rtx newop1;
1551 unsigned int bits = GET_MODE_PRECISION (mode);
1553 if (CONST_INT_P (op1))
1554 newop1 = GEN_INT (bits - INTVAL (op1));
1555 else if (targetm.shift_truncation_mask (mode) == bits - 1)
1556 newop1 = negate_rtx (GET_MODE (op1), op1);
1557 else
1558 newop1 = expand_binop (GET_MODE (op1), sub_optab,
1559 gen_int_mode (bits, GET_MODE (op1)), op1,
1560 NULL_RTX, unsignedp, OPTAB_DIRECT);
1562 temp = expand_binop_directly (mode, otheroptab, op0, newop1,
1563 target, unsignedp, methods, last);
1564 if (temp)
1565 return temp;
1568 /* If this is a multiply, see if we can do a widening operation that
1569 takes operands of this mode and makes a wider mode. */
1571 if (binoptab == smul_optab
1572 && GET_MODE_2XWIDER_MODE (mode) != VOIDmode
1573 && (widening_optab_handler ((unsignedp ? umul_widen_optab
1574 : smul_widen_optab),
1575 GET_MODE_2XWIDER_MODE (mode), mode)
1576 != CODE_FOR_nothing))
1578 temp = expand_binop (GET_MODE_2XWIDER_MODE (mode),
1579 unsignedp ? umul_widen_optab : smul_widen_optab,
1580 op0, op1, NULL_RTX, unsignedp, OPTAB_DIRECT);
1582 if (temp != 0)
1584 if (GET_MODE_CLASS (mode) == MODE_INT
1585 && TRULY_NOOP_TRUNCATION_MODES_P (mode, GET_MODE (temp)))
1586 return gen_lowpart (mode, temp);
1587 else
1588 return convert_to_mode (mode, temp, unsignedp);
1592 /* If this is a vector shift by a scalar, see if we can do a vector
1593 shift by a vector. If so, broadcast the scalar into a vector. */
1594 if (mclass == MODE_VECTOR_INT)
1596 optab otheroptab = unknown_optab;
1598 if (binoptab == ashl_optab)
1599 otheroptab = vashl_optab;
1600 else if (binoptab == ashr_optab)
1601 otheroptab = vashr_optab;
1602 else if (binoptab == lshr_optab)
1603 otheroptab = vlshr_optab;
1604 else if (binoptab == rotl_optab)
1605 otheroptab = vrotl_optab;
1606 else if (binoptab == rotr_optab)
1607 otheroptab = vrotr_optab;
1609 if (otheroptab && optab_handler (otheroptab, mode) != CODE_FOR_nothing)
1611 rtx vop1 = expand_vector_broadcast (mode, op1);
1612 if (vop1)
1614 temp = expand_binop_directly (mode, otheroptab, op0, vop1,
1615 target, unsignedp, methods, last);
1616 if (temp)
1617 return temp;
1622 /* Look for a wider mode of the same class for which we think we
1623 can open-code the operation. Check for a widening multiply at the
1624 wider mode as well. */
1626 if (CLASS_HAS_WIDER_MODES_P (mclass)
1627 && methods != OPTAB_DIRECT && methods != OPTAB_LIB)
1628 for (wider_mode = GET_MODE_WIDER_MODE (mode);
1629 wider_mode != VOIDmode;
1630 wider_mode = GET_MODE_WIDER_MODE (wider_mode))
1632 if (optab_handler (binoptab, wider_mode) != CODE_FOR_nothing
1633 || (binoptab == smul_optab
1634 && GET_MODE_WIDER_MODE (wider_mode) != VOIDmode
1635 && (find_widening_optab_handler ((unsignedp
1636 ? umul_widen_optab
1637 : smul_widen_optab),
1638 GET_MODE_WIDER_MODE (wider_mode),
1639 mode, 0)
1640 != CODE_FOR_nothing)))
1642 rtx xop0 = op0, xop1 = op1;
1643 int no_extend = 0;
1645 /* For certain integer operations, we need not actually extend
1646 the narrow operands, as long as we will truncate
1647 the results to the same narrowness. */
1649 if ((binoptab == ior_optab || binoptab == and_optab
1650 || binoptab == xor_optab
1651 || binoptab == add_optab || binoptab == sub_optab
1652 || binoptab == smul_optab || binoptab == ashl_optab)
1653 && mclass == MODE_INT)
1655 no_extend = 1;
1656 xop0 = avoid_expensive_constant (mode, binoptab, 0,
1657 xop0, unsignedp);
1658 if (binoptab != ashl_optab)
1659 xop1 = avoid_expensive_constant (mode, binoptab, 1,
1660 xop1, unsignedp);
1663 xop0 = widen_operand (xop0, wider_mode, mode, unsignedp, no_extend);
1665 /* The second operand of a shift must always be extended. */
1666 xop1 = widen_operand (xop1, wider_mode, mode, unsignedp,
1667 no_extend && binoptab != ashl_optab);
1669 temp = expand_binop (wider_mode, binoptab, xop0, xop1, NULL_RTX,
1670 unsignedp, OPTAB_DIRECT);
1671 if (temp)
1673 if (mclass != MODE_INT
1674 || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode))
1676 if (target == 0)
1677 target = gen_reg_rtx (mode);
1678 convert_move (target, temp, 0);
1679 return target;
1681 else
1682 return gen_lowpart (mode, temp);
1684 else
1685 delete_insns_since (last);
1689 /* If operation is commutative,
1690 try to make the first operand a register.
1691 Even better, try to make it the same as the target.
1692 Also try to make the last operand a constant. */
1693 if (commutative_optab_p (binoptab)
1694 && swap_commutative_operands_with_target (target, op0, op1))
1695 std::swap (op0, op1);
1697 /* These can be done a word at a time. */
1698 if ((binoptab == and_optab || binoptab == ior_optab || binoptab == xor_optab)
1699 && mclass == MODE_INT
1700 && GET_MODE_SIZE (mode) > UNITS_PER_WORD
1701 && optab_handler (binoptab, word_mode) != CODE_FOR_nothing)
1703 int i;
1704 rtx_insn *insns;
1706 /* If TARGET is the same as one of the operands, the REG_EQUAL note
1707 won't be accurate, so use a new target. */
1708 if (target == 0
1709 || target == op0
1710 || target == op1
1711 || !valid_multiword_target_p (target))
1712 target = gen_reg_rtx (mode);
1714 start_sequence ();
1716 /* Do the actual arithmetic. */
1717 for (i = 0; i < GET_MODE_BITSIZE (mode) / BITS_PER_WORD; i++)
1719 rtx target_piece = operand_subword (target, i, 1, mode);
1720 rtx x = expand_binop (word_mode, binoptab,
1721 operand_subword_force (op0, i, mode),
1722 operand_subword_force (op1, i, mode),
1723 target_piece, unsignedp, next_methods);
1725 if (x == 0)
1726 break;
1728 if (target_piece != x)
1729 emit_move_insn (target_piece, x);
1732 insns = get_insns ();
1733 end_sequence ();
1735 if (i == GET_MODE_BITSIZE (mode) / BITS_PER_WORD)
1737 emit_insn (insns);
1738 return target;
1742 /* Synthesize double word shifts from single word shifts. */
1743 if ((binoptab == lshr_optab || binoptab == ashl_optab
1744 || binoptab == ashr_optab)
1745 && mclass == MODE_INT
1746 && (CONST_INT_P (op1) || optimize_insn_for_speed_p ())
1747 && GET_MODE_SIZE (mode) == 2 * UNITS_PER_WORD
1748 && GET_MODE_PRECISION (mode) == GET_MODE_BITSIZE (mode)
1749 && optab_handler (binoptab, word_mode) != CODE_FOR_nothing
1750 && optab_handler (ashl_optab, word_mode) != CODE_FOR_nothing
1751 && optab_handler (lshr_optab, word_mode) != CODE_FOR_nothing)
1753 unsigned HOST_WIDE_INT shift_mask, double_shift_mask;
1754 machine_mode op1_mode;
1756 double_shift_mask = targetm.shift_truncation_mask (mode);
1757 shift_mask = targetm.shift_truncation_mask (word_mode);
1758 op1_mode = GET_MODE (op1) != VOIDmode ? GET_MODE (op1) : word_mode;
1760 /* Apply the truncation to constant shifts. */
1761 if (double_shift_mask > 0 && CONST_INT_P (op1))
1762 op1 = GEN_INT (INTVAL (op1) & double_shift_mask);
1764 if (op1 == CONST0_RTX (op1_mode))
1765 return op0;
1767 /* Make sure that this is a combination that expand_doubleword_shift
1768 can handle. See the comments there for details. */
1769 if (double_shift_mask == 0
1770 || (shift_mask == BITS_PER_WORD - 1
1771 && double_shift_mask == BITS_PER_WORD * 2 - 1))
1773 rtx_insn *insns;
1774 rtx into_target, outof_target;
1775 rtx into_input, outof_input;
1776 int left_shift, outof_word;
1778 /* If TARGET is the same as one of the operands, the REG_EQUAL note
1779 won't be accurate, so use a new target. */
1780 if (target == 0
1781 || target == op0
1782 || target == op1
1783 || !valid_multiword_target_p (target))
1784 target = gen_reg_rtx (mode);
1786 start_sequence ();
1788 /* OUTOF_* is the word we are shifting bits away from, and
1789 INTO_* is the word that we are shifting bits towards, thus
1790 they differ depending on the direction of the shift and
1791 WORDS_BIG_ENDIAN. */
1793 left_shift = binoptab == ashl_optab;
1794 outof_word = left_shift ^ ! WORDS_BIG_ENDIAN;
1796 outof_target = operand_subword (target, outof_word, 1, mode);
1797 into_target = operand_subword (target, 1 - outof_word, 1, mode);
1799 outof_input = operand_subword_force (op0, outof_word, mode);
1800 into_input = operand_subword_force (op0, 1 - outof_word, mode);
1802 if (expand_doubleword_shift (op1_mode, binoptab,
1803 outof_input, into_input, op1,
1804 outof_target, into_target,
1805 unsignedp, next_methods, shift_mask))
1807 insns = get_insns ();
1808 end_sequence ();
1810 emit_insn (insns);
1811 return target;
1813 end_sequence ();
1817 /* Synthesize double word rotates from single word shifts. */
1818 if ((binoptab == rotl_optab || binoptab == rotr_optab)
1819 && mclass == MODE_INT
1820 && CONST_INT_P (op1)
1821 && GET_MODE_PRECISION (mode) == 2 * BITS_PER_WORD
1822 && optab_handler (ashl_optab, word_mode) != CODE_FOR_nothing
1823 && optab_handler (lshr_optab, word_mode) != CODE_FOR_nothing)
1825 rtx_insn *insns;
1826 rtx into_target, outof_target;
1827 rtx into_input, outof_input;
1828 rtx inter;
1829 int shift_count, left_shift, outof_word;
1831 /* If TARGET is the same as one of the operands, the REG_EQUAL note
1832 won't be accurate, so use a new target. Do this also if target is not
1833 a REG, first because having a register instead may open optimization
1834 opportunities, and second because if target and op0 happen to be MEMs
1835 designating the same location, we would risk clobbering it too early
1836 in the code sequence we generate below. */
1837 if (target == 0
1838 || target == op0
1839 || target == op1
1840 || !REG_P (target)
1841 || !valid_multiword_target_p (target))
1842 target = gen_reg_rtx (mode);
1844 start_sequence ();
1846 shift_count = INTVAL (op1);
1848 /* OUTOF_* is the word we are shifting bits away from, and
1849 INTO_* is the word that we are shifting bits towards, thus
1850 they differ depending on the direction of the shift and
1851 WORDS_BIG_ENDIAN. */
1853 left_shift = (binoptab == rotl_optab);
1854 outof_word = left_shift ^ ! WORDS_BIG_ENDIAN;
1856 outof_target = operand_subword (target, outof_word, 1, mode);
1857 into_target = operand_subword (target, 1 - outof_word, 1, mode);
1859 outof_input = operand_subword_force (op0, outof_word, mode);
1860 into_input = operand_subword_force (op0, 1 - outof_word, mode);
1862 if (shift_count == BITS_PER_WORD)
1864 /* This is just a word swap. */
1865 emit_move_insn (outof_target, into_input);
1866 emit_move_insn (into_target, outof_input);
1867 inter = const0_rtx;
1869 else
1871 rtx into_temp1, into_temp2, outof_temp1, outof_temp2;
1872 rtx first_shift_count, second_shift_count;
1873 optab reverse_unsigned_shift, unsigned_shift;
1875 reverse_unsigned_shift = (left_shift ^ (shift_count < BITS_PER_WORD)
1876 ? lshr_optab : ashl_optab);
1878 unsigned_shift = (left_shift ^ (shift_count < BITS_PER_WORD)
1879 ? ashl_optab : lshr_optab);
1881 if (shift_count > BITS_PER_WORD)
1883 first_shift_count = GEN_INT (shift_count - BITS_PER_WORD);
1884 second_shift_count = GEN_INT (2 * BITS_PER_WORD - shift_count);
1886 else
1888 first_shift_count = GEN_INT (BITS_PER_WORD - shift_count);
1889 second_shift_count = GEN_INT (shift_count);
1892 into_temp1 = expand_binop (word_mode, unsigned_shift,
1893 outof_input, first_shift_count,
1894 NULL_RTX, unsignedp, next_methods);
1895 into_temp2 = expand_binop (word_mode, reverse_unsigned_shift,
1896 into_input, second_shift_count,
1897 NULL_RTX, unsignedp, next_methods);
1899 if (into_temp1 != 0 && into_temp2 != 0)
1900 inter = expand_binop (word_mode, ior_optab, into_temp1, into_temp2,
1901 into_target, unsignedp, next_methods);
1902 else
1903 inter = 0;
1905 if (inter != 0 && inter != into_target)
1906 emit_move_insn (into_target, inter);
1908 outof_temp1 = expand_binop (word_mode, unsigned_shift,
1909 into_input, first_shift_count,
1910 NULL_RTX, unsignedp, next_methods);
1911 outof_temp2 = expand_binop (word_mode, reverse_unsigned_shift,
1912 outof_input, second_shift_count,
1913 NULL_RTX, unsignedp, next_methods);
1915 if (inter != 0 && outof_temp1 != 0 && outof_temp2 != 0)
1916 inter = expand_binop (word_mode, ior_optab,
1917 outof_temp1, outof_temp2,
1918 outof_target, unsignedp, next_methods);
1920 if (inter != 0 && inter != outof_target)
1921 emit_move_insn (outof_target, inter);
1924 insns = get_insns ();
1925 end_sequence ();
1927 if (inter != 0)
1929 emit_insn (insns);
1930 return target;
1934 /* These can be done a word at a time by propagating carries. */
1935 if ((binoptab == add_optab || binoptab == sub_optab)
1936 && mclass == MODE_INT
1937 && GET_MODE_SIZE (mode) >= 2 * UNITS_PER_WORD
1938 && optab_handler (binoptab, word_mode) != CODE_FOR_nothing)
1940 unsigned int i;
1941 optab otheroptab = binoptab == add_optab ? sub_optab : add_optab;
1942 const unsigned int nwords = GET_MODE_BITSIZE (mode) / BITS_PER_WORD;
1943 rtx carry_in = NULL_RTX, carry_out = NULL_RTX;
1944 rtx xop0, xop1, xtarget;
1946 /* We can handle either a 1 or -1 value for the carry. If STORE_FLAG
1947 value is one of those, use it. Otherwise, use 1 since it is the
1948 one easiest to get. */
1949 #if STORE_FLAG_VALUE == 1 || STORE_FLAG_VALUE == -1
1950 int normalizep = STORE_FLAG_VALUE;
1951 #else
1952 int normalizep = 1;
1953 #endif
1955 /* Prepare the operands. */
1956 xop0 = force_reg (mode, op0);
1957 xop1 = force_reg (mode, op1);
1959 xtarget = gen_reg_rtx (mode);
1961 if (target == 0 || !REG_P (target) || !valid_multiword_target_p (target))
1962 target = xtarget;
1964 /* Indicate for flow that the entire target reg is being set. */
1965 if (REG_P (target))
1966 emit_clobber (xtarget);
1968 /* Do the actual arithmetic. */
1969 for (i = 0; i < nwords; i++)
1971 int index = (WORDS_BIG_ENDIAN ? nwords - i - 1 : i);
1972 rtx target_piece = operand_subword (xtarget, index, 1, mode);
1973 rtx op0_piece = operand_subword_force (xop0, index, mode);
1974 rtx op1_piece = operand_subword_force (xop1, index, mode);
1975 rtx x;
1977 /* Main add/subtract of the input operands. */
1978 x = expand_binop (word_mode, binoptab,
1979 op0_piece, op1_piece,
1980 target_piece, unsignedp, next_methods);
1981 if (x == 0)
1982 break;
1984 if (i + 1 < nwords)
1986 /* Store carry from main add/subtract. */
1987 carry_out = gen_reg_rtx (word_mode);
1988 carry_out = emit_store_flag_force (carry_out,
1989 (binoptab == add_optab
1990 ? LT : GT),
1991 x, op0_piece,
1992 word_mode, 1, normalizep);
1995 if (i > 0)
1997 rtx newx;
1999 /* Add/subtract previous carry to main result. */
2000 newx = expand_binop (word_mode,
2001 normalizep == 1 ? binoptab : otheroptab,
2002 x, carry_in,
2003 NULL_RTX, 1, next_methods);
2005 if (i + 1 < nwords)
2007 /* Get out carry from adding/subtracting carry in. */
2008 rtx carry_tmp = gen_reg_rtx (word_mode);
2009 carry_tmp = emit_store_flag_force (carry_tmp,
2010 (binoptab == add_optab
2011 ? LT : GT),
2012 newx, x,
2013 word_mode, 1, normalizep);
2015 /* Logical-ior the two poss. carry together. */
2016 carry_out = expand_binop (word_mode, ior_optab,
2017 carry_out, carry_tmp,
2018 carry_out, 0, next_methods);
2019 if (carry_out == 0)
2020 break;
2022 emit_move_insn (target_piece, newx);
2024 else
2026 if (x != target_piece)
2027 emit_move_insn (target_piece, x);
2030 carry_in = carry_out;
2033 if (i == GET_MODE_BITSIZE (mode) / (unsigned) BITS_PER_WORD)
2035 if (optab_handler (mov_optab, mode) != CODE_FOR_nothing
2036 || ! rtx_equal_p (target, xtarget))
2038 rtx_insn *temp = emit_move_insn (target, xtarget);
2040 set_dst_reg_note (temp, REG_EQUAL,
2041 gen_rtx_fmt_ee (optab_to_code (binoptab),
2042 mode, copy_rtx (xop0),
2043 copy_rtx (xop1)),
2044 target);
2046 else
2047 target = xtarget;
2049 return target;
2052 else
2053 delete_insns_since (last);
2056 /* Attempt to synthesize double word multiplies using a sequence of word
2057 mode multiplications. We first attempt to generate a sequence using a
2058 more efficient unsigned widening multiply, and if that fails we then
2059 try using a signed widening multiply. */
2061 if (binoptab == smul_optab
2062 && mclass == MODE_INT
2063 && GET_MODE_SIZE (mode) == 2 * UNITS_PER_WORD
2064 && optab_handler (smul_optab, word_mode) != CODE_FOR_nothing
2065 && optab_handler (add_optab, word_mode) != CODE_FOR_nothing)
2067 rtx product = NULL_RTX;
2068 if (widening_optab_handler (umul_widen_optab, mode, word_mode)
2069 != CODE_FOR_nothing)
2071 product = expand_doubleword_mult (mode, op0, op1, target,
2072 true, methods);
2073 if (!product)
2074 delete_insns_since (last);
2077 if (product == NULL_RTX
2078 && widening_optab_handler (smul_widen_optab, mode, word_mode)
2079 != CODE_FOR_nothing)
2081 product = expand_doubleword_mult (mode, op0, op1, target,
2082 false, methods);
2083 if (!product)
2084 delete_insns_since (last);
2087 if (product != NULL_RTX)
2089 if (optab_handler (mov_optab, mode) != CODE_FOR_nothing)
2091 temp = emit_move_insn (target ? target : product, product);
2092 set_dst_reg_note (temp,
2093 REG_EQUAL,
2094 gen_rtx_fmt_ee (MULT, mode,
2095 copy_rtx (op0),
2096 copy_rtx (op1)),
2097 target ? target : product);
2099 return product;
2103 /* It can't be open-coded in this mode.
2104 Use a library call if one is available and caller says that's ok. */
2106 libfunc = optab_libfunc (binoptab, mode);
2107 if (libfunc
2108 && (methods == OPTAB_LIB || methods == OPTAB_LIB_WIDEN))
2110 rtx_insn *insns;
2111 rtx op1x = op1;
2112 machine_mode op1_mode = mode;
2113 rtx value;
2115 start_sequence ();
2117 if (shift_optab_p (binoptab))
2119 op1_mode = targetm.libgcc_shift_count_mode ();
2120 /* Specify unsigned here,
2121 since negative shift counts are meaningless. */
2122 op1x = convert_to_mode (op1_mode, op1, 1);
2125 if (GET_MODE (op0) != VOIDmode
2126 && GET_MODE (op0) != mode)
2127 op0 = convert_to_mode (mode, op0, unsignedp);
2129 /* Pass 1 for NO_QUEUE so we don't lose any increments
2130 if the libcall is cse'd or moved. */
2131 value = emit_library_call_value (libfunc,
2132 NULL_RTX, LCT_CONST, mode, 2,
2133 op0, mode, op1x, op1_mode);
2135 insns = get_insns ();
2136 end_sequence ();
2138 target = gen_reg_rtx (mode);
2139 emit_libcall_block_1 (insns, target, value,
2140 gen_rtx_fmt_ee (optab_to_code (binoptab),
2141 mode, op0, op1),
2142 trapv_binoptab_p (binoptab));
2144 return target;
2147 delete_insns_since (last);
2149 /* It can't be done in this mode. Can we do it in a wider mode? */
2151 if (! (methods == OPTAB_WIDEN || methods == OPTAB_LIB_WIDEN
2152 || methods == OPTAB_MUST_WIDEN))
2154 /* Caller says, don't even try. */
2155 delete_insns_since (entry_last);
2156 return 0;
2159 /* Compute the value of METHODS to pass to recursive calls.
2160 Don't allow widening to be tried recursively. */
2162 methods = (methods == OPTAB_LIB_WIDEN ? OPTAB_LIB : OPTAB_DIRECT);
2164 /* Look for a wider mode of the same class for which it appears we can do
2165 the operation. */
2167 if (CLASS_HAS_WIDER_MODES_P (mclass))
2169 for (wider_mode = GET_MODE_WIDER_MODE (mode);
2170 wider_mode != VOIDmode;
2171 wider_mode = GET_MODE_WIDER_MODE (wider_mode))
2173 if (find_widening_optab_handler (binoptab, wider_mode, mode, 1)
2174 != CODE_FOR_nothing
2175 || (methods == OPTAB_LIB
2176 && optab_libfunc (binoptab, wider_mode)))
2178 rtx xop0 = op0, xop1 = op1;
2179 int no_extend = 0;
2181 /* For certain integer operations, we need not actually extend
2182 the narrow operands, as long as we will truncate
2183 the results to the same narrowness. */
2185 if ((binoptab == ior_optab || binoptab == and_optab
2186 || binoptab == xor_optab
2187 || binoptab == add_optab || binoptab == sub_optab
2188 || binoptab == smul_optab || binoptab == ashl_optab)
2189 && mclass == MODE_INT)
2190 no_extend = 1;
2192 xop0 = widen_operand (xop0, wider_mode, mode,
2193 unsignedp, no_extend);
2195 /* The second operand of a shift must always be extended. */
2196 xop1 = widen_operand (xop1, wider_mode, mode, unsignedp,
2197 no_extend && binoptab != ashl_optab);
2199 temp = expand_binop (wider_mode, binoptab, xop0, xop1, NULL_RTX,
2200 unsignedp, methods);
2201 if (temp)
2203 if (mclass != MODE_INT
2204 || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode))
2206 if (target == 0)
2207 target = gen_reg_rtx (mode);
2208 convert_move (target, temp, 0);
2209 return target;
2211 else
2212 return gen_lowpart (mode, temp);
2214 else
2215 delete_insns_since (last);
2220 delete_insns_since (entry_last);
2221 return 0;
2224 /* Expand a binary operator which has both signed and unsigned forms.
2225 UOPTAB is the optab for unsigned operations, and SOPTAB is for
2226 signed operations.
2228 If we widen unsigned operands, we may use a signed wider operation instead
2229 of an unsigned wider operation, since the result would be the same. */
2232 sign_expand_binop (machine_mode mode, optab uoptab, optab soptab,
2233 rtx op0, rtx op1, rtx target, int unsignedp,
2234 enum optab_methods methods)
2236 rtx temp;
2237 optab direct_optab = unsignedp ? uoptab : soptab;
2238 bool save_enable;
2240 /* Do it without widening, if possible. */
2241 temp = expand_binop (mode, direct_optab, op0, op1, target,
2242 unsignedp, OPTAB_DIRECT);
2243 if (temp || methods == OPTAB_DIRECT)
2244 return temp;
2246 /* Try widening to a signed int. Disable any direct use of any
2247 signed insn in the current mode. */
2248 save_enable = swap_optab_enable (soptab, mode, false);
2250 temp = expand_binop (mode, soptab, op0, op1, target,
2251 unsignedp, OPTAB_WIDEN);
2253 /* For unsigned operands, try widening to an unsigned int. */
2254 if (!temp && unsignedp)
2255 temp = expand_binop (mode, uoptab, op0, op1, target,
2256 unsignedp, OPTAB_WIDEN);
2257 if (temp || methods == OPTAB_WIDEN)
2258 goto egress;
2260 /* Use the right width libcall if that exists. */
2261 temp = expand_binop (mode, direct_optab, op0, op1, target,
2262 unsignedp, OPTAB_LIB);
2263 if (temp || methods == OPTAB_LIB)
2264 goto egress;
2266 /* Must widen and use a libcall, use either signed or unsigned. */
2267 temp = expand_binop (mode, soptab, op0, op1, target,
2268 unsignedp, methods);
2269 if (!temp && unsignedp)
2270 temp = expand_binop (mode, uoptab, op0, op1, target,
2271 unsignedp, methods);
2273 egress:
2274 /* Undo the fiddling above. */
2275 if (save_enable)
2276 swap_optab_enable (soptab, mode, true);
2277 return temp;
2280 /* Generate code to perform an operation specified by UNOPPTAB
2281 on operand OP0, with two results to TARG0 and TARG1.
2282 We assume that the order of the operands for the instruction
2283 is TARG0, TARG1, OP0.
2285 Either TARG0 or TARG1 may be zero, but what that means is that
2286 the result is not actually wanted. We will generate it into
2287 a dummy pseudo-reg and discard it. They may not both be zero.
2289 Returns 1 if this operation can be performed; 0 if not. */
2292 expand_twoval_unop (optab unoptab, rtx op0, rtx targ0, rtx targ1,
2293 int unsignedp)
2295 machine_mode mode = GET_MODE (targ0 ? targ0 : targ1);
2296 enum mode_class mclass;
2297 machine_mode wider_mode;
2298 rtx_insn *entry_last = get_last_insn ();
2299 rtx_insn *last;
2301 mclass = GET_MODE_CLASS (mode);
2303 if (!targ0)
2304 targ0 = gen_reg_rtx (mode);
2305 if (!targ1)
2306 targ1 = gen_reg_rtx (mode);
2308 /* Record where to go back to if we fail. */
2309 last = get_last_insn ();
2311 if (optab_handler (unoptab, mode) != CODE_FOR_nothing)
2313 struct expand_operand ops[3];
2314 enum insn_code icode = optab_handler (unoptab, mode);
2316 create_fixed_operand (&ops[0], targ0);
2317 create_fixed_operand (&ops[1], targ1);
2318 create_convert_operand_from (&ops[2], op0, mode, unsignedp);
2319 if (maybe_expand_insn (icode, 3, ops))
2320 return 1;
2323 /* It can't be done in this mode. Can we do it in a wider mode? */
2325 if (CLASS_HAS_WIDER_MODES_P (mclass))
2327 for (wider_mode = GET_MODE_WIDER_MODE (mode);
2328 wider_mode != VOIDmode;
2329 wider_mode = GET_MODE_WIDER_MODE (wider_mode))
2331 if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing)
2333 rtx t0 = gen_reg_rtx (wider_mode);
2334 rtx t1 = gen_reg_rtx (wider_mode);
2335 rtx cop0 = convert_modes (wider_mode, mode, op0, unsignedp);
2337 if (expand_twoval_unop (unoptab, cop0, t0, t1, unsignedp))
2339 convert_move (targ0, t0, unsignedp);
2340 convert_move (targ1, t1, unsignedp);
2341 return 1;
2343 else
2344 delete_insns_since (last);
2349 delete_insns_since (entry_last);
2350 return 0;
2353 /* Generate code to perform an operation specified by BINOPTAB
2354 on operands OP0 and OP1, with two results to TARG1 and TARG2.
2355 We assume that the order of the operands for the instruction
2356 is TARG0, OP0, OP1, TARG1, which would fit a pattern like
2357 [(set TARG0 (operate OP0 OP1)) (set TARG1 (operate ...))].
2359 Either TARG0 or TARG1 may be zero, but what that means is that
2360 the result is not actually wanted. We will generate it into
2361 a dummy pseudo-reg and discard it. They may not both be zero.
2363 Returns 1 if this operation can be performed; 0 if not. */
2366 expand_twoval_binop (optab binoptab, rtx op0, rtx op1, rtx targ0, rtx targ1,
2367 int unsignedp)
2369 machine_mode mode = GET_MODE (targ0 ? targ0 : targ1);
2370 enum mode_class mclass;
2371 machine_mode wider_mode;
2372 rtx_insn *entry_last = get_last_insn ();
2373 rtx_insn *last;
2375 mclass = GET_MODE_CLASS (mode);
2377 if (!targ0)
2378 targ0 = gen_reg_rtx (mode);
2379 if (!targ1)
2380 targ1 = gen_reg_rtx (mode);
2382 /* Record where to go back to if we fail. */
2383 last = get_last_insn ();
2385 if (optab_handler (binoptab, mode) != CODE_FOR_nothing)
2387 struct expand_operand ops[4];
2388 enum insn_code icode = optab_handler (binoptab, mode);
2389 machine_mode mode0 = insn_data[icode].operand[1].mode;
2390 machine_mode mode1 = insn_data[icode].operand[2].mode;
2391 rtx xop0 = op0, xop1 = op1;
2393 /* If we are optimizing, force expensive constants into a register. */
2394 xop0 = avoid_expensive_constant (mode0, binoptab, 0, xop0, unsignedp);
2395 xop1 = avoid_expensive_constant (mode1, binoptab, 1, xop1, unsignedp);
2397 create_fixed_operand (&ops[0], targ0);
2398 create_convert_operand_from (&ops[1], op0, mode, unsignedp);
2399 create_convert_operand_from (&ops[2], op1, mode, unsignedp);
2400 create_fixed_operand (&ops[3], targ1);
2401 if (maybe_expand_insn (icode, 4, ops))
2402 return 1;
2403 delete_insns_since (last);
2406 /* It can't be done in this mode. Can we do it in a wider mode? */
2408 if (CLASS_HAS_WIDER_MODES_P (mclass))
2410 for (wider_mode = GET_MODE_WIDER_MODE (mode);
2411 wider_mode != VOIDmode;
2412 wider_mode = GET_MODE_WIDER_MODE (wider_mode))
2414 if (optab_handler (binoptab, wider_mode) != CODE_FOR_nothing)
2416 rtx t0 = gen_reg_rtx (wider_mode);
2417 rtx t1 = gen_reg_rtx (wider_mode);
2418 rtx cop0 = convert_modes (wider_mode, mode, op0, unsignedp);
2419 rtx cop1 = convert_modes (wider_mode, mode, op1, unsignedp);
2421 if (expand_twoval_binop (binoptab, cop0, cop1,
2422 t0, t1, unsignedp))
2424 convert_move (targ0, t0, unsignedp);
2425 convert_move (targ1, t1, unsignedp);
2426 return 1;
2428 else
2429 delete_insns_since (last);
2434 delete_insns_since (entry_last);
2435 return 0;
2438 /* Expand the two-valued library call indicated by BINOPTAB, but
2439 preserve only one of the values. If TARG0 is non-NULL, the first
2440 value is placed into TARG0; otherwise the second value is placed
2441 into TARG1. Exactly one of TARG0 and TARG1 must be non-NULL. The
2442 value stored into TARG0 or TARG1 is equivalent to (CODE OP0 OP1).
2443 This routine assumes that the value returned by the library call is
2444 as if the return value was of an integral mode twice as wide as the
2445 mode of OP0. Returns 1 if the call was successful. */
2447 bool
2448 expand_twoval_binop_libfunc (optab binoptab, rtx op0, rtx op1,
2449 rtx targ0, rtx targ1, enum rtx_code code)
2451 machine_mode mode;
2452 machine_mode libval_mode;
2453 rtx libval;
2454 rtx_insn *insns;
2455 rtx libfunc;
2457 /* Exactly one of TARG0 or TARG1 should be non-NULL. */
2458 gcc_assert (!targ0 != !targ1);
2460 mode = GET_MODE (op0);
2461 libfunc = optab_libfunc (binoptab, mode);
2462 if (!libfunc)
2463 return false;
2465 /* The value returned by the library function will have twice as
2466 many bits as the nominal MODE. */
2467 libval_mode = smallest_mode_for_size (2 * GET_MODE_BITSIZE (mode),
2468 MODE_INT);
2469 start_sequence ();
2470 libval = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
2471 libval_mode, 2,
2472 op0, mode,
2473 op1, mode);
2474 /* Get the part of VAL containing the value that we want. */
2475 libval = simplify_gen_subreg (mode, libval, libval_mode,
2476 targ0 ? 0 : GET_MODE_SIZE (mode));
2477 insns = get_insns ();
2478 end_sequence ();
2479 /* Move the into the desired location. */
2480 emit_libcall_block (insns, targ0 ? targ0 : targ1, libval,
2481 gen_rtx_fmt_ee (code, mode, op0, op1));
2483 return true;
2487 /* Wrapper around expand_unop which takes an rtx code to specify
2488 the operation to perform, not an optab pointer. All other
2489 arguments are the same. */
2491 expand_simple_unop (machine_mode mode, enum rtx_code code, rtx op0,
2492 rtx target, int unsignedp)
2494 optab unop = code_to_optab (code);
2495 gcc_assert (unop);
2497 return expand_unop (mode, unop, op0, target, unsignedp);
2500 /* Try calculating
2501 (clz:narrow x)
2503 (clz:wide (zero_extend:wide x)) - ((width wide) - (width narrow)).
2505 A similar operation can be used for clrsb. UNOPTAB says which operation
2506 we are trying to expand. */
2507 static rtx
2508 widen_leading (machine_mode mode, rtx op0, rtx target, optab unoptab)
2510 enum mode_class mclass = GET_MODE_CLASS (mode);
2511 if (CLASS_HAS_WIDER_MODES_P (mclass))
2513 machine_mode wider_mode;
2514 for (wider_mode = GET_MODE_WIDER_MODE (mode);
2515 wider_mode != VOIDmode;
2516 wider_mode = GET_MODE_WIDER_MODE (wider_mode))
2518 if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing)
2520 rtx xop0, temp;
2521 rtx_insn *last;
2523 last = get_last_insn ();
2525 if (target == 0)
2526 target = gen_reg_rtx (mode);
2527 xop0 = widen_operand (op0, wider_mode, mode,
2528 unoptab != clrsb_optab, false);
2529 temp = expand_unop (wider_mode, unoptab, xop0, NULL_RTX,
2530 unoptab != clrsb_optab);
2531 if (temp != 0)
2532 temp = expand_binop
2533 (wider_mode, sub_optab, temp,
2534 gen_int_mode (GET_MODE_PRECISION (wider_mode)
2535 - GET_MODE_PRECISION (mode),
2536 wider_mode),
2537 target, true, OPTAB_DIRECT);
2538 if (temp == 0)
2539 delete_insns_since (last);
2541 return temp;
2545 return 0;
2548 /* Try calculating clz of a double-word quantity as two clz's of word-sized
2549 quantities, choosing which based on whether the high word is nonzero. */
2550 static rtx
2551 expand_doubleword_clz (machine_mode mode, rtx op0, rtx target)
2553 rtx xop0 = force_reg (mode, op0);
2554 rtx subhi = gen_highpart (word_mode, xop0);
2555 rtx sublo = gen_lowpart (word_mode, xop0);
2556 rtx_code_label *hi0_label = gen_label_rtx ();
2557 rtx_code_label *after_label = gen_label_rtx ();
2558 rtx_insn *seq;
2559 rtx temp, result;
2561 /* If we were not given a target, use a word_mode register, not a
2562 'mode' register. The result will fit, and nobody is expecting
2563 anything bigger (the return type of __builtin_clz* is int). */
2564 if (!target)
2565 target = gen_reg_rtx (word_mode);
2567 /* In any case, write to a word_mode scratch in both branches of the
2568 conditional, so we can ensure there is a single move insn setting
2569 'target' to tag a REG_EQUAL note on. */
2570 result = gen_reg_rtx (word_mode);
2572 start_sequence ();
2574 /* If the high word is not equal to zero,
2575 then clz of the full value is clz of the high word. */
2576 emit_cmp_and_jump_insns (subhi, CONST0_RTX (word_mode), EQ, 0,
2577 word_mode, true, hi0_label);
2579 temp = expand_unop_direct (word_mode, clz_optab, subhi, result, true);
2580 if (!temp)
2581 goto fail;
2583 if (temp != result)
2584 convert_move (result, temp, true);
2586 emit_jump_insn (targetm.gen_jump (after_label));
2587 emit_barrier ();
2589 /* Else clz of the full value is clz of the low word plus the number
2590 of bits in the high word. */
2591 emit_label (hi0_label);
2593 temp = expand_unop_direct (word_mode, clz_optab, sublo, 0, true);
2594 if (!temp)
2595 goto fail;
2596 temp = expand_binop (word_mode, add_optab, temp,
2597 gen_int_mode (GET_MODE_BITSIZE (word_mode), word_mode),
2598 result, true, OPTAB_DIRECT);
2599 if (!temp)
2600 goto fail;
2601 if (temp != result)
2602 convert_move (result, temp, true);
2604 emit_label (after_label);
2605 convert_move (target, result, true);
2607 seq = get_insns ();
2608 end_sequence ();
2610 add_equal_note (seq, target, CLZ, xop0, 0);
2611 emit_insn (seq);
2612 return target;
2614 fail:
2615 end_sequence ();
2616 return 0;
2619 /* Try calculating
2620 (bswap:narrow x)
2622 (lshiftrt:wide (bswap:wide x) ((width wide) - (width narrow))). */
2623 static rtx
2624 widen_bswap (machine_mode mode, rtx op0, rtx target)
2626 enum mode_class mclass = GET_MODE_CLASS (mode);
2627 machine_mode wider_mode;
2628 rtx x;
2629 rtx_insn *last;
2631 if (!CLASS_HAS_WIDER_MODES_P (mclass))
2632 return NULL_RTX;
2634 for (wider_mode = GET_MODE_WIDER_MODE (mode);
2635 wider_mode != VOIDmode;
2636 wider_mode = GET_MODE_WIDER_MODE (wider_mode))
2637 if (optab_handler (bswap_optab, wider_mode) != CODE_FOR_nothing)
2638 goto found;
2639 return NULL_RTX;
2641 found:
2642 last = get_last_insn ();
2644 x = widen_operand (op0, wider_mode, mode, true, true);
2645 x = expand_unop (wider_mode, bswap_optab, x, NULL_RTX, true);
2647 gcc_assert (GET_MODE_PRECISION (wider_mode) == GET_MODE_BITSIZE (wider_mode)
2648 && GET_MODE_PRECISION (mode) == GET_MODE_BITSIZE (mode));
2649 if (x != 0)
2650 x = expand_shift (RSHIFT_EXPR, wider_mode, x,
2651 GET_MODE_BITSIZE (wider_mode)
2652 - GET_MODE_BITSIZE (mode),
2653 NULL_RTX, true);
2655 if (x != 0)
2657 if (target == 0)
2658 target = gen_reg_rtx (mode);
2659 emit_move_insn (target, gen_lowpart (mode, x));
2661 else
2662 delete_insns_since (last);
2664 return target;
2667 /* Try calculating bswap as two bswaps of two word-sized operands. */
2669 static rtx
2670 expand_doubleword_bswap (machine_mode mode, rtx op, rtx target)
2672 rtx t0, t1;
2674 t1 = expand_unop (word_mode, bswap_optab,
2675 operand_subword_force (op, 0, mode), NULL_RTX, true);
2676 t0 = expand_unop (word_mode, bswap_optab,
2677 operand_subword_force (op, 1, mode), NULL_RTX, true);
2679 if (target == 0 || !valid_multiword_target_p (target))
2680 target = gen_reg_rtx (mode);
2681 if (REG_P (target))
2682 emit_clobber (target);
2683 emit_move_insn (operand_subword (target, 0, 1, mode), t0);
2684 emit_move_insn (operand_subword (target, 1, 1, mode), t1);
2686 return target;
2689 /* Try calculating (parity x) as (and (popcount x) 1), where
2690 popcount can also be done in a wider mode. */
2691 static rtx
2692 expand_parity (machine_mode mode, rtx op0, rtx target)
2694 enum mode_class mclass = GET_MODE_CLASS (mode);
2695 if (CLASS_HAS_WIDER_MODES_P (mclass))
2697 machine_mode wider_mode;
2698 for (wider_mode = mode; wider_mode != VOIDmode;
2699 wider_mode = GET_MODE_WIDER_MODE (wider_mode))
2701 if (optab_handler (popcount_optab, wider_mode) != CODE_FOR_nothing)
2703 rtx xop0, temp;
2704 rtx_insn *last;
2706 last = get_last_insn ();
2708 if (target == 0)
2709 target = gen_reg_rtx (mode);
2710 xop0 = widen_operand (op0, wider_mode, mode, true, false);
2711 temp = expand_unop (wider_mode, popcount_optab, xop0, NULL_RTX,
2712 true);
2713 if (temp != 0)
2714 temp = expand_binop (wider_mode, and_optab, temp, const1_rtx,
2715 target, true, OPTAB_DIRECT);
2716 if (temp == 0)
2717 delete_insns_since (last);
2719 return temp;
2723 return 0;
2726 /* Try calculating ctz(x) as K - clz(x & -x) ,
2727 where K is GET_MODE_PRECISION(mode) - 1.
2729 Both __builtin_ctz and __builtin_clz are undefined at zero, so we
2730 don't have to worry about what the hardware does in that case. (If
2731 the clz instruction produces the usual value at 0, which is K, the
2732 result of this code sequence will be -1; expand_ffs, below, relies
2733 on this. It might be nice to have it be K instead, for consistency
2734 with the (very few) processors that provide a ctz with a defined
2735 value, but that would take one more instruction, and it would be
2736 less convenient for expand_ffs anyway. */
2738 static rtx
2739 expand_ctz (machine_mode mode, rtx op0, rtx target)
2741 rtx_insn *seq;
2742 rtx temp;
2744 if (optab_handler (clz_optab, mode) == CODE_FOR_nothing)
2745 return 0;
2747 start_sequence ();
2749 temp = expand_unop_direct (mode, neg_optab, op0, NULL_RTX, true);
2750 if (temp)
2751 temp = expand_binop (mode, and_optab, op0, temp, NULL_RTX,
2752 true, OPTAB_DIRECT);
2753 if (temp)
2754 temp = expand_unop_direct (mode, clz_optab, temp, NULL_RTX, true);
2755 if (temp)
2756 temp = expand_binop (mode, sub_optab,
2757 gen_int_mode (GET_MODE_PRECISION (mode) - 1, mode),
2758 temp, target,
2759 true, OPTAB_DIRECT);
2760 if (temp == 0)
2762 end_sequence ();
2763 return 0;
2766 seq = get_insns ();
2767 end_sequence ();
2769 add_equal_note (seq, temp, CTZ, op0, 0);
2770 emit_insn (seq);
2771 return temp;
2775 /* Try calculating ffs(x) using ctz(x) if we have that instruction, or
2776 else with the sequence used by expand_clz.
2778 The ffs builtin promises to return zero for a zero value and ctz/clz
2779 may have an undefined value in that case. If they do not give us a
2780 convenient value, we have to generate a test and branch. */
2781 static rtx
2782 expand_ffs (machine_mode mode, rtx op0, rtx target)
2784 HOST_WIDE_INT val = 0;
2785 bool defined_at_zero = false;
2786 rtx temp;
2787 rtx_insn *seq;
2789 if (optab_handler (ctz_optab, mode) != CODE_FOR_nothing)
2791 start_sequence ();
2793 temp = expand_unop_direct (mode, ctz_optab, op0, 0, true);
2794 if (!temp)
2795 goto fail;
2797 defined_at_zero = (CTZ_DEFINED_VALUE_AT_ZERO (mode, val) == 2);
2799 else if (optab_handler (clz_optab, mode) != CODE_FOR_nothing)
2801 start_sequence ();
2802 temp = expand_ctz (mode, op0, 0);
2803 if (!temp)
2804 goto fail;
2806 if (CLZ_DEFINED_VALUE_AT_ZERO (mode, val) == 2)
2808 defined_at_zero = true;
2809 val = (GET_MODE_PRECISION (mode) - 1) - val;
2812 else
2813 return 0;
2815 if (defined_at_zero && val == -1)
2816 /* No correction needed at zero. */;
2817 else
2819 /* We don't try to do anything clever with the situation found
2820 on some processors (eg Alpha) where ctz(0:mode) ==
2821 bitsize(mode). If someone can think of a way to send N to -1
2822 and leave alone all values in the range 0..N-1 (where N is a
2823 power of two), cheaper than this test-and-branch, please add it.
2825 The test-and-branch is done after the operation itself, in case
2826 the operation sets condition codes that can be recycled for this.
2827 (This is true on i386, for instance.) */
2829 rtx_code_label *nonzero_label = gen_label_rtx ();
2830 emit_cmp_and_jump_insns (op0, CONST0_RTX (mode), NE, 0,
2831 mode, true, nonzero_label);
2833 convert_move (temp, GEN_INT (-1), false);
2834 emit_label (nonzero_label);
2837 /* temp now has a value in the range -1..bitsize-1. ffs is supposed
2838 to produce a value in the range 0..bitsize. */
2839 temp = expand_binop (mode, add_optab, temp, gen_int_mode (1, mode),
2840 target, false, OPTAB_DIRECT);
2841 if (!temp)
2842 goto fail;
2844 seq = get_insns ();
2845 end_sequence ();
2847 add_equal_note (seq, temp, FFS, op0, 0);
2848 emit_insn (seq);
2849 return temp;
2851 fail:
2852 end_sequence ();
2853 return 0;
2856 /* Extract the OMODE lowpart from VAL, which has IMODE. Under certain
2857 conditions, VAL may already be a SUBREG against which we cannot generate
2858 a further SUBREG. In this case, we expect forcing the value into a
2859 register will work around the situation. */
2861 static rtx
2862 lowpart_subreg_maybe_copy (machine_mode omode, rtx val,
2863 machine_mode imode)
2865 rtx ret;
2866 ret = lowpart_subreg (omode, val, imode);
2867 if (ret == NULL)
2869 val = force_reg (imode, val);
2870 ret = lowpart_subreg (omode, val, imode);
2871 gcc_assert (ret != NULL);
2873 return ret;
2876 /* Expand a floating point absolute value or negation operation via a
2877 logical operation on the sign bit. */
2879 static rtx
2880 expand_absneg_bit (enum rtx_code code, machine_mode mode,
2881 rtx op0, rtx target)
2883 const struct real_format *fmt;
2884 int bitpos, word, nwords, i;
2885 machine_mode imode;
2886 rtx temp;
2887 rtx_insn *insns;
2889 /* The format has to have a simple sign bit. */
2890 fmt = REAL_MODE_FORMAT (mode);
2891 if (fmt == NULL)
2892 return NULL_RTX;
2894 bitpos = fmt->signbit_rw;
2895 if (bitpos < 0)
2896 return NULL_RTX;
2898 /* Don't create negative zeros if the format doesn't support them. */
2899 if (code == NEG && !fmt->has_signed_zero)
2900 return NULL_RTX;
2902 if (GET_MODE_SIZE (mode) <= UNITS_PER_WORD)
2904 imode = int_mode_for_mode (mode);
2905 if (imode == BLKmode)
2906 return NULL_RTX;
2907 word = 0;
2908 nwords = 1;
2910 else
2912 imode = word_mode;
2914 if (FLOAT_WORDS_BIG_ENDIAN)
2915 word = (GET_MODE_BITSIZE (mode) - bitpos) / BITS_PER_WORD;
2916 else
2917 word = bitpos / BITS_PER_WORD;
2918 bitpos = bitpos % BITS_PER_WORD;
2919 nwords = (GET_MODE_BITSIZE (mode) + BITS_PER_WORD - 1) / BITS_PER_WORD;
2922 wide_int mask = wi::set_bit_in_zero (bitpos, GET_MODE_PRECISION (imode));
2923 if (code == ABS)
2924 mask = ~mask;
2926 if (target == 0
2927 || target == op0
2928 || (nwords > 1 && !valid_multiword_target_p (target)))
2929 target = gen_reg_rtx (mode);
2931 if (nwords > 1)
2933 start_sequence ();
2935 for (i = 0; i < nwords; ++i)
2937 rtx targ_piece = operand_subword (target, i, 1, mode);
2938 rtx op0_piece = operand_subword_force (op0, i, mode);
2940 if (i == word)
2942 temp = expand_binop (imode, code == ABS ? and_optab : xor_optab,
2943 op0_piece,
2944 immed_wide_int_const (mask, imode),
2945 targ_piece, 1, OPTAB_LIB_WIDEN);
2946 if (temp != targ_piece)
2947 emit_move_insn (targ_piece, temp);
2949 else
2950 emit_move_insn (targ_piece, op0_piece);
2953 insns = get_insns ();
2954 end_sequence ();
2956 emit_insn (insns);
2958 else
2960 temp = expand_binop (imode, code == ABS ? and_optab : xor_optab,
2961 gen_lowpart (imode, op0),
2962 immed_wide_int_const (mask, imode),
2963 gen_lowpart (imode, target), 1, OPTAB_LIB_WIDEN);
2964 target = lowpart_subreg_maybe_copy (mode, temp, imode);
2966 set_dst_reg_note (get_last_insn (), REG_EQUAL,
2967 gen_rtx_fmt_e (code, mode, copy_rtx (op0)),
2968 target);
2971 return target;
2974 /* As expand_unop, but will fail rather than attempt the operation in a
2975 different mode or with a libcall. */
2976 static rtx
2977 expand_unop_direct (machine_mode mode, optab unoptab, rtx op0, rtx target,
2978 int unsignedp)
2980 if (optab_handler (unoptab, mode) != CODE_FOR_nothing)
2982 struct expand_operand ops[2];
2983 enum insn_code icode = optab_handler (unoptab, mode);
2984 rtx_insn *last = get_last_insn ();
2985 rtx_insn *pat;
2987 create_output_operand (&ops[0], target, mode);
2988 create_convert_operand_from (&ops[1], op0, mode, unsignedp);
2989 pat = maybe_gen_insn (icode, 2, ops);
2990 if (pat)
2992 if (INSN_P (pat) && NEXT_INSN (pat) != NULL_RTX
2993 && ! add_equal_note (pat, ops[0].value,
2994 optab_to_code (unoptab),
2995 ops[1].value, NULL_RTX))
2997 delete_insns_since (last);
2998 return expand_unop (mode, unoptab, op0, NULL_RTX, unsignedp);
3001 emit_insn (pat);
3003 return ops[0].value;
3006 return 0;
3009 /* Generate code to perform an operation specified by UNOPTAB
3010 on operand OP0, with result having machine-mode MODE.
3012 UNSIGNEDP is for the case where we have to widen the operands
3013 to perform the operation. It says to use zero-extension.
3015 If TARGET is nonzero, the value
3016 is generated there, if it is convenient to do so.
3017 In all cases an rtx is returned for the locus of the value;
3018 this may or may not be TARGET. */
3021 expand_unop (machine_mode mode, optab unoptab, rtx op0, rtx target,
3022 int unsignedp)
3024 enum mode_class mclass = GET_MODE_CLASS (mode);
3025 machine_mode wider_mode;
3026 rtx temp;
3027 rtx libfunc;
3029 temp = expand_unop_direct (mode, unoptab, op0, target, unsignedp);
3030 if (temp)
3031 return temp;
3033 /* It can't be done in this mode. Can we open-code it in a wider mode? */
3035 /* Widening (or narrowing) clz needs special treatment. */
3036 if (unoptab == clz_optab)
3038 temp = widen_leading (mode, op0, target, unoptab);
3039 if (temp)
3040 return temp;
3042 if (GET_MODE_SIZE (mode) == 2 * UNITS_PER_WORD
3043 && optab_handler (unoptab, word_mode) != CODE_FOR_nothing)
3045 temp = expand_doubleword_clz (mode, op0, target);
3046 if (temp)
3047 return temp;
3050 goto try_libcall;
3053 if (unoptab == clrsb_optab)
3055 temp = widen_leading (mode, op0, target, unoptab);
3056 if (temp)
3057 return temp;
3058 goto try_libcall;
3061 /* Widening (or narrowing) bswap needs special treatment. */
3062 if (unoptab == bswap_optab)
3064 /* HImode is special because in this mode BSWAP is equivalent to ROTATE
3065 or ROTATERT. First try these directly; if this fails, then try the
3066 obvious pair of shifts with allowed widening, as this will probably
3067 be always more efficient than the other fallback methods. */
3068 if (mode == HImode)
3070 rtx_insn *last;
3071 rtx temp1, temp2;
3073 if (optab_handler (rotl_optab, mode) != CODE_FOR_nothing)
3075 temp = expand_binop (mode, rotl_optab, op0, GEN_INT (8), target,
3076 unsignedp, OPTAB_DIRECT);
3077 if (temp)
3078 return temp;
3081 if (optab_handler (rotr_optab, mode) != CODE_FOR_nothing)
3083 temp = expand_binop (mode, rotr_optab, op0, GEN_INT (8), target,
3084 unsignedp, OPTAB_DIRECT);
3085 if (temp)
3086 return temp;
3089 last = get_last_insn ();
3091 temp1 = expand_binop (mode, ashl_optab, op0, GEN_INT (8), NULL_RTX,
3092 unsignedp, OPTAB_WIDEN);
3093 temp2 = expand_binop (mode, lshr_optab, op0, GEN_INT (8), NULL_RTX,
3094 unsignedp, OPTAB_WIDEN);
3095 if (temp1 && temp2)
3097 temp = expand_binop (mode, ior_optab, temp1, temp2, target,
3098 unsignedp, OPTAB_WIDEN);
3099 if (temp)
3100 return temp;
3103 delete_insns_since (last);
3106 temp = widen_bswap (mode, op0, target);
3107 if (temp)
3108 return temp;
3110 if (GET_MODE_SIZE (mode) == 2 * UNITS_PER_WORD
3111 && optab_handler (unoptab, word_mode) != CODE_FOR_nothing)
3113 temp = expand_doubleword_bswap (mode, op0, target);
3114 if (temp)
3115 return temp;
3118 goto try_libcall;
3121 if (CLASS_HAS_WIDER_MODES_P (mclass))
3122 for (wider_mode = GET_MODE_WIDER_MODE (mode);
3123 wider_mode != VOIDmode;
3124 wider_mode = GET_MODE_WIDER_MODE (wider_mode))
3126 if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing)
3128 rtx xop0 = op0;
3129 rtx_insn *last = get_last_insn ();
3131 /* For certain operations, we need not actually extend
3132 the narrow operand, as long as we will truncate the
3133 results to the same narrowness. */
3135 xop0 = widen_operand (xop0, wider_mode, mode, unsignedp,
3136 (unoptab == neg_optab
3137 || unoptab == one_cmpl_optab)
3138 && mclass == MODE_INT);
3140 temp = expand_unop (wider_mode, unoptab, xop0, NULL_RTX,
3141 unsignedp);
3143 if (temp)
3145 if (mclass != MODE_INT
3146 || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode))
3148 if (target == 0)
3149 target = gen_reg_rtx (mode);
3150 convert_move (target, temp, 0);
3151 return target;
3153 else
3154 return gen_lowpart (mode, temp);
3156 else
3157 delete_insns_since (last);
3161 /* These can be done a word at a time. */
3162 if (unoptab == one_cmpl_optab
3163 && mclass == MODE_INT
3164 && GET_MODE_SIZE (mode) > UNITS_PER_WORD
3165 && optab_handler (unoptab, word_mode) != CODE_FOR_nothing)
3167 int i;
3168 rtx_insn *insns;
3170 if (target == 0 || target == op0 || !valid_multiword_target_p (target))
3171 target = gen_reg_rtx (mode);
3173 start_sequence ();
3175 /* Do the actual arithmetic. */
3176 for (i = 0; i < GET_MODE_BITSIZE (mode) / BITS_PER_WORD; i++)
3178 rtx target_piece = operand_subword (target, i, 1, mode);
3179 rtx x = expand_unop (word_mode, unoptab,
3180 operand_subword_force (op0, i, mode),
3181 target_piece, unsignedp);
3183 if (target_piece != x)
3184 emit_move_insn (target_piece, x);
3187 insns = get_insns ();
3188 end_sequence ();
3190 emit_insn (insns);
3191 return target;
3194 if (optab_to_code (unoptab) == NEG)
3196 /* Try negating floating point values by flipping the sign bit. */
3197 if (SCALAR_FLOAT_MODE_P (mode))
3199 temp = expand_absneg_bit (NEG, mode, op0, target);
3200 if (temp)
3201 return temp;
3204 /* If there is no negation pattern, and we have no negative zero,
3205 try subtracting from zero. */
3206 if (!HONOR_SIGNED_ZEROS (mode))
3208 temp = expand_binop (mode, (unoptab == negv_optab
3209 ? subv_optab : sub_optab),
3210 CONST0_RTX (mode), op0, target,
3211 unsignedp, OPTAB_DIRECT);
3212 if (temp)
3213 return temp;
3217 /* Try calculating parity (x) as popcount (x) % 2. */
3218 if (unoptab == parity_optab)
3220 temp = expand_parity (mode, op0, target);
3221 if (temp)
3222 return temp;
3225 /* Try implementing ffs (x) in terms of clz (x). */
3226 if (unoptab == ffs_optab)
3228 temp = expand_ffs (mode, op0, target);
3229 if (temp)
3230 return temp;
3233 /* Try implementing ctz (x) in terms of clz (x). */
3234 if (unoptab == ctz_optab)
3236 temp = expand_ctz (mode, op0, target);
3237 if (temp)
3238 return temp;
3241 try_libcall:
3242 /* Now try a library call in this mode. */
3243 libfunc = optab_libfunc (unoptab, mode);
3244 if (libfunc)
3246 rtx_insn *insns;
3247 rtx value;
3248 rtx eq_value;
3249 machine_mode outmode = mode;
3251 /* All of these functions return small values. Thus we choose to
3252 have them return something that isn't a double-word. */
3253 if (unoptab == ffs_optab || unoptab == clz_optab || unoptab == ctz_optab
3254 || unoptab == clrsb_optab || unoptab == popcount_optab
3255 || unoptab == parity_optab)
3256 outmode
3257 = GET_MODE (hard_libcall_value (TYPE_MODE (integer_type_node),
3258 optab_libfunc (unoptab, mode)));
3260 start_sequence ();
3262 /* Pass 1 for NO_QUEUE so we don't lose any increments
3263 if the libcall is cse'd or moved. */
3264 value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST, outmode,
3265 1, op0, mode);
3266 insns = get_insns ();
3267 end_sequence ();
3269 target = gen_reg_rtx (outmode);
3270 eq_value = gen_rtx_fmt_e (optab_to_code (unoptab), mode, op0);
3271 if (GET_MODE_SIZE (outmode) < GET_MODE_SIZE (mode))
3272 eq_value = simplify_gen_unary (TRUNCATE, outmode, eq_value, mode);
3273 else if (GET_MODE_SIZE (outmode) > GET_MODE_SIZE (mode))
3274 eq_value = simplify_gen_unary (ZERO_EXTEND, outmode, eq_value, mode);
3275 emit_libcall_block_1 (insns, target, value, eq_value,
3276 trapv_unoptab_p (unoptab));
3278 return target;
3281 /* It can't be done in this mode. Can we do it in a wider mode? */
3283 if (CLASS_HAS_WIDER_MODES_P (mclass))
3285 for (wider_mode = GET_MODE_WIDER_MODE (mode);
3286 wider_mode != VOIDmode;
3287 wider_mode = GET_MODE_WIDER_MODE (wider_mode))
3289 if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing
3290 || optab_libfunc (unoptab, wider_mode))
3292 rtx xop0 = op0;
3293 rtx_insn *last = get_last_insn ();
3295 /* For certain operations, we need not actually extend
3296 the narrow operand, as long as we will truncate the
3297 results to the same narrowness. */
3298 xop0 = widen_operand (xop0, wider_mode, mode, unsignedp,
3299 (unoptab == neg_optab
3300 || unoptab == one_cmpl_optab
3301 || unoptab == bswap_optab)
3302 && mclass == MODE_INT);
3304 temp = expand_unop (wider_mode, unoptab, xop0, NULL_RTX,
3305 unsignedp);
3307 /* If we are generating clz using wider mode, adjust the
3308 result. Similarly for clrsb. */
3309 if ((unoptab == clz_optab || unoptab == clrsb_optab)
3310 && temp != 0)
3311 temp = expand_binop
3312 (wider_mode, sub_optab, temp,
3313 gen_int_mode (GET_MODE_PRECISION (wider_mode)
3314 - GET_MODE_PRECISION (mode),
3315 wider_mode),
3316 target, true, OPTAB_DIRECT);
3318 /* Likewise for bswap. */
3319 if (unoptab == bswap_optab && temp != 0)
3321 gcc_assert (GET_MODE_PRECISION (wider_mode)
3322 == GET_MODE_BITSIZE (wider_mode)
3323 && GET_MODE_PRECISION (mode)
3324 == GET_MODE_BITSIZE (mode));
3326 temp = expand_shift (RSHIFT_EXPR, wider_mode, temp,
3327 GET_MODE_BITSIZE (wider_mode)
3328 - GET_MODE_BITSIZE (mode),
3329 NULL_RTX, true);
3332 if (temp)
3334 if (mclass != MODE_INT)
3336 if (target == 0)
3337 target = gen_reg_rtx (mode);
3338 convert_move (target, temp, 0);
3339 return target;
3341 else
3342 return gen_lowpart (mode, temp);
3344 else
3345 delete_insns_since (last);
3350 /* One final attempt at implementing negation via subtraction,
3351 this time allowing widening of the operand. */
3352 if (optab_to_code (unoptab) == NEG && !HONOR_SIGNED_ZEROS (mode))
3354 rtx temp;
3355 temp = expand_binop (mode,
3356 unoptab == negv_optab ? subv_optab : sub_optab,
3357 CONST0_RTX (mode), op0,
3358 target, unsignedp, OPTAB_LIB_WIDEN);
3359 if (temp)
3360 return temp;
3363 return 0;
3366 /* Emit code to compute the absolute value of OP0, with result to
3367 TARGET if convenient. (TARGET may be 0.) The return value says
3368 where the result actually is to be found.
3370 MODE is the mode of the operand; the mode of the result is
3371 different but can be deduced from MODE.
3376 expand_abs_nojump (machine_mode mode, rtx op0, rtx target,
3377 int result_unsignedp)
3379 rtx temp;
3381 if (GET_MODE_CLASS (mode) != MODE_INT
3382 || ! flag_trapv)
3383 result_unsignedp = 1;
3385 /* First try to do it with a special abs instruction. */
3386 temp = expand_unop (mode, result_unsignedp ? abs_optab : absv_optab,
3387 op0, target, 0);
3388 if (temp != 0)
3389 return temp;
3391 /* For floating point modes, try clearing the sign bit. */
3392 if (SCALAR_FLOAT_MODE_P (mode))
3394 temp = expand_absneg_bit (ABS, mode, op0, target);
3395 if (temp)
3396 return temp;
3399 /* If we have a MAX insn, we can do this as MAX (x, -x). */
3400 if (optab_handler (smax_optab, mode) != CODE_FOR_nothing
3401 && !HONOR_SIGNED_ZEROS (mode))
3403 rtx_insn *last = get_last_insn ();
3405 temp = expand_unop (mode, result_unsignedp ? neg_optab : negv_optab,
3406 op0, NULL_RTX, 0);
3407 if (temp != 0)
3408 temp = expand_binop (mode, smax_optab, op0, temp, target, 0,
3409 OPTAB_WIDEN);
3411 if (temp != 0)
3412 return temp;
3414 delete_insns_since (last);
3417 /* If this machine has expensive jumps, we can do integer absolute
3418 value of X as (((signed) x >> (W-1)) ^ x) - ((signed) x >> (W-1)),
3419 where W is the width of MODE. */
3421 if (GET_MODE_CLASS (mode) == MODE_INT
3422 && BRANCH_COST (optimize_insn_for_speed_p (),
3423 false) >= 2)
3425 rtx extended = expand_shift (RSHIFT_EXPR, mode, op0,
3426 GET_MODE_PRECISION (mode) - 1,
3427 NULL_RTX, 0);
3429 temp = expand_binop (mode, xor_optab, extended, op0, target, 0,
3430 OPTAB_LIB_WIDEN);
3431 if (temp != 0)
3432 temp = expand_binop (mode, result_unsignedp ? sub_optab : subv_optab,
3433 temp, extended, target, 0, OPTAB_LIB_WIDEN);
3435 if (temp != 0)
3436 return temp;
3439 return NULL_RTX;
3443 expand_abs (machine_mode mode, rtx op0, rtx target,
3444 int result_unsignedp, int safe)
3446 rtx temp;
3447 rtx_code_label *op1;
3449 if (GET_MODE_CLASS (mode) != MODE_INT
3450 || ! flag_trapv)
3451 result_unsignedp = 1;
3453 temp = expand_abs_nojump (mode, op0, target, result_unsignedp);
3454 if (temp != 0)
3455 return temp;
3457 /* If that does not win, use conditional jump and negate. */
3459 /* It is safe to use the target if it is the same
3460 as the source if this is also a pseudo register */
3461 if (op0 == target && REG_P (op0)
3462 && REGNO (op0) >= FIRST_PSEUDO_REGISTER)
3463 safe = 1;
3465 op1 = gen_label_rtx ();
3466 if (target == 0 || ! safe
3467 || GET_MODE (target) != mode
3468 || (MEM_P (target) && MEM_VOLATILE_P (target))
3469 || (REG_P (target)
3470 && REGNO (target) < FIRST_PSEUDO_REGISTER))
3471 target = gen_reg_rtx (mode);
3473 emit_move_insn (target, op0);
3474 NO_DEFER_POP;
3476 do_compare_rtx_and_jump (target, CONST0_RTX (mode), GE, 0, mode,
3477 NULL_RTX, NULL, op1, -1);
3479 op0 = expand_unop (mode, result_unsignedp ? neg_optab : negv_optab,
3480 target, target, 0);
3481 if (op0 != target)
3482 emit_move_insn (target, op0);
3483 emit_label (op1);
3484 OK_DEFER_POP;
3485 return target;
3488 /* Emit code to compute the one's complement absolute value of OP0
3489 (if (OP0 < 0) OP0 = ~OP0), with result to TARGET if convenient.
3490 (TARGET may be NULL_RTX.) The return value says where the result
3491 actually is to be found.
3493 MODE is the mode of the operand; the mode of the result is
3494 different but can be deduced from MODE. */
3497 expand_one_cmpl_abs_nojump (machine_mode mode, rtx op0, rtx target)
3499 rtx temp;
3501 /* Not applicable for floating point modes. */
3502 if (FLOAT_MODE_P (mode))
3503 return NULL_RTX;
3505 /* If we have a MAX insn, we can do this as MAX (x, ~x). */
3506 if (optab_handler (smax_optab, mode) != CODE_FOR_nothing)
3508 rtx_insn *last = get_last_insn ();
3510 temp = expand_unop (mode, one_cmpl_optab, op0, NULL_RTX, 0);
3511 if (temp != 0)
3512 temp = expand_binop (mode, smax_optab, op0, temp, target, 0,
3513 OPTAB_WIDEN);
3515 if (temp != 0)
3516 return temp;
3518 delete_insns_since (last);
3521 /* If this machine has expensive jumps, we can do one's complement
3522 absolute value of X as (((signed) x >> (W-1)) ^ x). */
3524 if (GET_MODE_CLASS (mode) == MODE_INT
3525 && BRANCH_COST (optimize_insn_for_speed_p (),
3526 false) >= 2)
3528 rtx extended = expand_shift (RSHIFT_EXPR, mode, op0,
3529 GET_MODE_PRECISION (mode) - 1,
3530 NULL_RTX, 0);
3532 temp = expand_binop (mode, xor_optab, extended, op0, target, 0,
3533 OPTAB_LIB_WIDEN);
3535 if (temp != 0)
3536 return temp;
3539 return NULL_RTX;
3542 /* A subroutine of expand_copysign, perform the copysign operation using the
3543 abs and neg primitives advertised to exist on the target. The assumption
3544 is that we have a split register file, and leaving op0 in fp registers,
3545 and not playing with subregs so much, will help the register allocator. */
3547 static rtx
3548 expand_copysign_absneg (machine_mode mode, rtx op0, rtx op1, rtx target,
3549 int bitpos, bool op0_is_abs)
3551 machine_mode imode;
3552 enum insn_code icode;
3553 rtx sign;
3554 rtx_code_label *label;
3556 if (target == op1)
3557 target = NULL_RTX;
3559 /* Check if the back end provides an insn that handles signbit for the
3560 argument's mode. */
3561 icode = optab_handler (signbit_optab, mode);
3562 if (icode != CODE_FOR_nothing)
3564 imode = insn_data[(int) icode].operand[0].mode;
3565 sign = gen_reg_rtx (imode);
3566 emit_unop_insn (icode, sign, op1, UNKNOWN);
3568 else
3570 if (GET_MODE_SIZE (mode) <= UNITS_PER_WORD)
3572 imode = int_mode_for_mode (mode);
3573 if (imode == BLKmode)
3574 return NULL_RTX;
3575 op1 = gen_lowpart (imode, op1);
3577 else
3579 int word;
3581 imode = word_mode;
3582 if (FLOAT_WORDS_BIG_ENDIAN)
3583 word = (GET_MODE_BITSIZE (mode) - bitpos) / BITS_PER_WORD;
3584 else
3585 word = bitpos / BITS_PER_WORD;
3586 bitpos = bitpos % BITS_PER_WORD;
3587 op1 = operand_subword_force (op1, word, mode);
3590 wide_int mask = wi::set_bit_in_zero (bitpos, GET_MODE_PRECISION (imode));
3591 sign = expand_binop (imode, and_optab, op1,
3592 immed_wide_int_const (mask, imode),
3593 NULL_RTX, 1, OPTAB_LIB_WIDEN);
3596 if (!op0_is_abs)
3598 op0 = expand_unop (mode, abs_optab, op0, target, 0);
3599 if (op0 == NULL)
3600 return NULL_RTX;
3601 target = op0;
3603 else
3605 if (target == NULL_RTX)
3606 target = copy_to_reg (op0);
3607 else
3608 emit_move_insn (target, op0);
3611 label = gen_label_rtx ();
3612 emit_cmp_and_jump_insns (sign, const0_rtx, EQ, NULL_RTX, imode, 1, label);
3614 if (CONST_DOUBLE_AS_FLOAT_P (op0))
3615 op0 = simplify_unary_operation (NEG, mode, op0, mode);
3616 else
3617 op0 = expand_unop (mode, neg_optab, op0, target, 0);
3618 if (op0 != target)
3619 emit_move_insn (target, op0);
3621 emit_label (label);
3623 return target;
3627 /* A subroutine of expand_copysign, perform the entire copysign operation
3628 with integer bitmasks. BITPOS is the position of the sign bit; OP0_IS_ABS
3629 is true if op0 is known to have its sign bit clear. */
3631 static rtx
3632 expand_copysign_bit (machine_mode mode, rtx op0, rtx op1, rtx target,
3633 int bitpos, bool op0_is_abs)
3635 machine_mode imode;
3636 int word, nwords, i;
3637 rtx temp;
3638 rtx_insn *insns;
3640 if (GET_MODE_SIZE (mode) <= UNITS_PER_WORD)
3642 imode = int_mode_for_mode (mode);
3643 if (imode == BLKmode)
3644 return NULL_RTX;
3645 word = 0;
3646 nwords = 1;
3648 else
3650 imode = word_mode;
3652 if (FLOAT_WORDS_BIG_ENDIAN)
3653 word = (GET_MODE_BITSIZE (mode) - bitpos) / BITS_PER_WORD;
3654 else
3655 word = bitpos / BITS_PER_WORD;
3656 bitpos = bitpos % BITS_PER_WORD;
3657 nwords = (GET_MODE_BITSIZE (mode) + BITS_PER_WORD - 1) / BITS_PER_WORD;
3660 wide_int mask = wi::set_bit_in_zero (bitpos, GET_MODE_PRECISION (imode));
3662 if (target == 0
3663 || target == op0
3664 || target == op1
3665 || (nwords > 1 && !valid_multiword_target_p (target)))
3666 target = gen_reg_rtx (mode);
3668 if (nwords > 1)
3670 start_sequence ();
3672 for (i = 0; i < nwords; ++i)
3674 rtx targ_piece = operand_subword (target, i, 1, mode);
3675 rtx op0_piece = operand_subword_force (op0, i, mode);
3677 if (i == word)
3679 if (!op0_is_abs)
3680 op0_piece
3681 = expand_binop (imode, and_optab, op0_piece,
3682 immed_wide_int_const (~mask, imode),
3683 NULL_RTX, 1, OPTAB_LIB_WIDEN);
3684 op1 = expand_binop (imode, and_optab,
3685 operand_subword_force (op1, i, mode),
3686 immed_wide_int_const (mask, imode),
3687 NULL_RTX, 1, OPTAB_LIB_WIDEN);
3689 temp = expand_binop (imode, ior_optab, op0_piece, op1,
3690 targ_piece, 1, OPTAB_LIB_WIDEN);
3691 if (temp != targ_piece)
3692 emit_move_insn (targ_piece, temp);
3694 else
3695 emit_move_insn (targ_piece, op0_piece);
3698 insns = get_insns ();
3699 end_sequence ();
3701 emit_insn (insns);
3703 else
3705 op1 = expand_binop (imode, and_optab, gen_lowpart (imode, op1),
3706 immed_wide_int_const (mask, imode),
3707 NULL_RTX, 1, OPTAB_LIB_WIDEN);
3709 op0 = gen_lowpart (imode, op0);
3710 if (!op0_is_abs)
3711 op0 = expand_binop (imode, and_optab, op0,
3712 immed_wide_int_const (~mask, imode),
3713 NULL_RTX, 1, OPTAB_LIB_WIDEN);
3715 temp = expand_binop (imode, ior_optab, op0, op1,
3716 gen_lowpart (imode, target), 1, OPTAB_LIB_WIDEN);
3717 target = lowpart_subreg_maybe_copy (mode, temp, imode);
3720 return target;
3723 /* Expand the C99 copysign operation. OP0 and OP1 must be the same
3724 scalar floating point mode. Return NULL if we do not know how to
3725 expand the operation inline. */
3728 expand_copysign (rtx op0, rtx op1, rtx target)
3730 machine_mode mode = GET_MODE (op0);
3731 const struct real_format *fmt;
3732 bool op0_is_abs;
3733 rtx temp;
3735 gcc_assert (SCALAR_FLOAT_MODE_P (mode));
3736 gcc_assert (GET_MODE (op1) == mode);
3738 /* First try to do it with a special instruction. */
3739 temp = expand_binop (mode, copysign_optab, op0, op1,
3740 target, 0, OPTAB_DIRECT);
3741 if (temp)
3742 return temp;
3744 fmt = REAL_MODE_FORMAT (mode);
3745 if (fmt == NULL || !fmt->has_signed_zero)
3746 return NULL_RTX;
3748 op0_is_abs = false;
3749 if (CONST_DOUBLE_AS_FLOAT_P (op0))
3751 if (real_isneg (CONST_DOUBLE_REAL_VALUE (op0)))
3752 op0 = simplify_unary_operation (ABS, mode, op0, mode);
3753 op0_is_abs = true;
3756 if (fmt->signbit_ro >= 0
3757 && (CONST_DOUBLE_AS_FLOAT_P (op0)
3758 || (optab_handler (neg_optab, mode) != CODE_FOR_nothing
3759 && optab_handler (abs_optab, mode) != CODE_FOR_nothing)))
3761 temp = expand_copysign_absneg (mode, op0, op1, target,
3762 fmt->signbit_ro, op0_is_abs);
3763 if (temp)
3764 return temp;
3767 if (fmt->signbit_rw < 0)
3768 return NULL_RTX;
3769 return expand_copysign_bit (mode, op0, op1, target,
3770 fmt->signbit_rw, op0_is_abs);
3773 /* Generate an instruction whose insn-code is INSN_CODE,
3774 with two operands: an output TARGET and an input OP0.
3775 TARGET *must* be nonzero, and the output is always stored there.
3776 CODE is an rtx code such that (CODE OP0) is an rtx that describes
3777 the value that is stored into TARGET.
3779 Return false if expansion failed. */
3781 bool
3782 maybe_emit_unop_insn (enum insn_code icode, rtx target, rtx op0,
3783 enum rtx_code code)
3785 struct expand_operand ops[2];
3786 rtx_insn *pat;
3788 create_output_operand (&ops[0], target, GET_MODE (target));
3789 create_input_operand (&ops[1], op0, GET_MODE (op0));
3790 pat = maybe_gen_insn (icode, 2, ops);
3791 if (!pat)
3792 return false;
3794 if (INSN_P (pat) && NEXT_INSN (pat) != NULL_RTX
3795 && code != UNKNOWN)
3796 add_equal_note (pat, ops[0].value, code, ops[1].value, NULL_RTX);
3798 emit_insn (pat);
3800 if (ops[0].value != target)
3801 emit_move_insn (target, ops[0].value);
3802 return true;
3804 /* Generate an instruction whose insn-code is INSN_CODE,
3805 with two operands: an output TARGET and an input OP0.
3806 TARGET *must* be nonzero, and the output is always stored there.
3807 CODE is an rtx code such that (CODE OP0) is an rtx that describes
3808 the value that is stored into TARGET. */
3810 void
3811 emit_unop_insn (enum insn_code icode, rtx target, rtx op0, enum rtx_code code)
3813 bool ok = maybe_emit_unop_insn (icode, target, op0, code);
3814 gcc_assert (ok);
3817 struct no_conflict_data
3819 rtx target;
3820 rtx_insn *first, *insn;
3821 bool must_stay;
3824 /* Called via note_stores by emit_libcall_block. Set P->must_stay if
3825 the currently examined clobber / store has to stay in the list of
3826 insns that constitute the actual libcall block. */
3827 static void
3828 no_conflict_move_test (rtx dest, const_rtx set, void *p0)
3830 struct no_conflict_data *p= (struct no_conflict_data *) p0;
3832 /* If this inns directly contributes to setting the target, it must stay. */
3833 if (reg_overlap_mentioned_p (p->target, dest))
3834 p->must_stay = true;
3835 /* If we haven't committed to keeping any other insns in the list yet,
3836 there is nothing more to check. */
3837 else if (p->insn == p->first)
3838 return;
3839 /* If this insn sets / clobbers a register that feeds one of the insns
3840 already in the list, this insn has to stay too. */
3841 else if (reg_overlap_mentioned_p (dest, PATTERN (p->first))
3842 || (CALL_P (p->first) && (find_reg_fusage (p->first, USE, dest)))
3843 || reg_used_between_p (dest, p->first, p->insn)
3844 /* Likewise if this insn depends on a register set by a previous
3845 insn in the list, or if it sets a result (presumably a hard
3846 register) that is set or clobbered by a previous insn.
3847 N.B. the modified_*_p (SET_DEST...) tests applied to a MEM
3848 SET_DEST perform the former check on the address, and the latter
3849 check on the MEM. */
3850 || (GET_CODE (set) == SET
3851 && (modified_in_p (SET_SRC (set), p->first)
3852 || modified_in_p (SET_DEST (set), p->first)
3853 || modified_between_p (SET_SRC (set), p->first, p->insn)
3854 || modified_between_p (SET_DEST (set), p->first, p->insn))))
3855 p->must_stay = true;
3859 /* Emit code to make a call to a constant function or a library call.
3861 INSNS is a list containing all insns emitted in the call.
3862 These insns leave the result in RESULT. Our block is to copy RESULT
3863 to TARGET, which is logically equivalent to EQUIV.
3865 We first emit any insns that set a pseudo on the assumption that these are
3866 loading constants into registers; doing so allows them to be safely cse'ed
3867 between blocks. Then we emit all the other insns in the block, followed by
3868 an insn to move RESULT to TARGET. This last insn will have a REQ_EQUAL
3869 note with an operand of EQUIV. */
3871 static void
3872 emit_libcall_block_1 (rtx_insn *insns, rtx target, rtx result, rtx equiv,
3873 bool equiv_may_trap)
3875 rtx final_dest = target;
3876 rtx_insn *next, *last, *insn;
3878 /* If this is a reg with REG_USERVAR_P set, then it could possibly turn
3879 into a MEM later. Protect the libcall block from this change. */
3880 if (! REG_P (target) || REG_USERVAR_P (target))
3881 target = gen_reg_rtx (GET_MODE (target));
3883 /* If we're using non-call exceptions, a libcall corresponding to an
3884 operation that may trap may also trap. */
3885 /* ??? See the comment in front of make_reg_eh_region_note. */
3886 if (cfun->can_throw_non_call_exceptions
3887 && (equiv_may_trap || may_trap_p (equiv)))
3889 for (insn = insns; insn; insn = NEXT_INSN (insn))
3890 if (CALL_P (insn))
3892 rtx note = find_reg_note (insn, REG_EH_REGION, NULL_RTX);
3893 if (note)
3895 int lp_nr = INTVAL (XEXP (note, 0));
3896 if (lp_nr == 0 || lp_nr == INT_MIN)
3897 remove_note (insn, note);
3901 else
3903 /* Look for any CALL_INSNs in this sequence, and attach a REG_EH_REGION
3904 reg note to indicate that this call cannot throw or execute a nonlocal
3905 goto (unless there is already a REG_EH_REGION note, in which case
3906 we update it). */
3907 for (insn = insns; insn; insn = NEXT_INSN (insn))
3908 if (CALL_P (insn))
3909 make_reg_eh_region_note_nothrow_nononlocal (insn);
3912 /* First emit all insns that set pseudos. Remove them from the list as
3913 we go. Avoid insns that set pseudos which were referenced in previous
3914 insns. These can be generated by move_by_pieces, for example,
3915 to update an address. Similarly, avoid insns that reference things
3916 set in previous insns. */
3918 for (insn = insns; insn; insn = next)
3920 rtx set = single_set (insn);
3922 next = NEXT_INSN (insn);
3924 if (set != 0 && REG_P (SET_DEST (set))
3925 && REGNO (SET_DEST (set)) >= FIRST_PSEUDO_REGISTER)
3927 struct no_conflict_data data;
3929 data.target = const0_rtx;
3930 data.first = insns;
3931 data.insn = insn;
3932 data.must_stay = 0;
3933 note_stores (PATTERN (insn), no_conflict_move_test, &data);
3934 if (! data.must_stay)
3936 if (PREV_INSN (insn))
3937 SET_NEXT_INSN (PREV_INSN (insn)) = next;
3938 else
3939 insns = next;
3941 if (next)
3942 SET_PREV_INSN (next) = PREV_INSN (insn);
3944 add_insn (insn);
3948 /* Some ports use a loop to copy large arguments onto the stack.
3949 Don't move anything outside such a loop. */
3950 if (LABEL_P (insn))
3951 break;
3954 /* Write the remaining insns followed by the final copy. */
3955 for (insn = insns; insn; insn = next)
3957 next = NEXT_INSN (insn);
3959 add_insn (insn);
3962 last = emit_move_insn (target, result);
3963 set_dst_reg_note (last, REG_EQUAL, copy_rtx (equiv), target);
3965 if (final_dest != target)
3966 emit_move_insn (final_dest, target);
3969 void
3970 emit_libcall_block (rtx insns, rtx target, rtx result, rtx equiv)
3972 emit_libcall_block_1 (safe_as_a <rtx_insn *> (insns),
3973 target, result, equiv, false);
3976 /* Nonzero if we can perform a comparison of mode MODE straightforwardly.
3977 PURPOSE describes how this comparison will be used. CODE is the rtx
3978 comparison code we will be using.
3980 ??? Actually, CODE is slightly weaker than that. A target is still
3981 required to implement all of the normal bcc operations, but not
3982 required to implement all (or any) of the unordered bcc operations. */
3985 can_compare_p (enum rtx_code code, machine_mode mode,
3986 enum can_compare_purpose purpose)
3988 rtx test;
3989 test = gen_rtx_fmt_ee (code, mode, const0_rtx, const0_rtx);
3992 enum insn_code icode;
3994 if (purpose == ccp_jump
3995 && (icode = optab_handler (cbranch_optab, mode)) != CODE_FOR_nothing
3996 && insn_operand_matches (icode, 0, test))
3997 return 1;
3998 if (purpose == ccp_store_flag
3999 && (icode = optab_handler (cstore_optab, mode)) != CODE_FOR_nothing
4000 && insn_operand_matches (icode, 1, test))
4001 return 1;
4002 if (purpose == ccp_cmov
4003 && optab_handler (cmov_optab, mode) != CODE_FOR_nothing)
4004 return 1;
4006 mode = GET_MODE_WIDER_MODE (mode);
4007 PUT_MODE (test, mode);
4009 while (mode != VOIDmode);
4011 return 0;
4014 /* This function is called when we are going to emit a compare instruction that
4015 compares the values found in *PX and *PY, using the rtl operator COMPARISON.
4017 *PMODE is the mode of the inputs (in case they are const_int).
4018 *PUNSIGNEDP nonzero says that the operands are unsigned;
4019 this matters if they need to be widened (as given by METHODS).
4021 If they have mode BLKmode, then SIZE specifies the size of both operands.
4023 This function performs all the setup necessary so that the caller only has
4024 to emit a single comparison insn. This setup can involve doing a BLKmode
4025 comparison or emitting a library call to perform the comparison if no insn
4026 is available to handle it.
4027 The values which are passed in through pointers can be modified; the caller
4028 should perform the comparison on the modified values. Constant
4029 comparisons must have already been folded. */
4031 static void
4032 prepare_cmp_insn (rtx x, rtx y, enum rtx_code comparison, rtx size,
4033 int unsignedp, enum optab_methods methods,
4034 rtx *ptest, machine_mode *pmode)
4036 machine_mode mode = *pmode;
4037 rtx libfunc, test;
4038 machine_mode cmp_mode;
4039 enum mode_class mclass;
4041 /* The other methods are not needed. */
4042 gcc_assert (methods == OPTAB_DIRECT || methods == OPTAB_WIDEN
4043 || methods == OPTAB_LIB_WIDEN);
4045 /* If we are optimizing, force expensive constants into a register. */
4046 if (CONSTANT_P (x) && optimize
4047 && (rtx_cost (x, mode, COMPARE, 0, optimize_insn_for_speed_p ())
4048 > COSTS_N_INSNS (1)))
4049 x = force_reg (mode, x);
4051 if (CONSTANT_P (y) && optimize
4052 && (rtx_cost (y, mode, COMPARE, 1, optimize_insn_for_speed_p ())
4053 > COSTS_N_INSNS (1)))
4054 y = force_reg (mode, y);
4056 #if HAVE_cc0
4057 /* Make sure if we have a canonical comparison. The RTL
4058 documentation states that canonical comparisons are required only
4059 for targets which have cc0. */
4060 gcc_assert (!CONSTANT_P (x) || CONSTANT_P (y));
4061 #endif
4063 /* Don't let both operands fail to indicate the mode. */
4064 if (GET_MODE (x) == VOIDmode && GET_MODE (y) == VOIDmode)
4065 x = force_reg (mode, x);
4066 if (mode == VOIDmode)
4067 mode = GET_MODE (x) != VOIDmode ? GET_MODE (x) : GET_MODE (y);
4069 /* Handle all BLKmode compares. */
4071 if (mode == BLKmode)
4073 machine_mode result_mode;
4074 enum insn_code cmp_code;
4075 tree length_type;
4076 rtx libfunc;
4077 rtx result;
4078 rtx opalign
4079 = GEN_INT (MIN (MEM_ALIGN (x), MEM_ALIGN (y)) / BITS_PER_UNIT);
4081 gcc_assert (size);
4083 /* Try to use a memory block compare insn - either cmpstr
4084 or cmpmem will do. */
4085 for (cmp_mode = GET_CLASS_NARROWEST_MODE (MODE_INT);
4086 cmp_mode != VOIDmode;
4087 cmp_mode = GET_MODE_WIDER_MODE (cmp_mode))
4089 cmp_code = direct_optab_handler (cmpmem_optab, cmp_mode);
4090 if (cmp_code == CODE_FOR_nothing)
4091 cmp_code = direct_optab_handler (cmpstr_optab, cmp_mode);
4092 if (cmp_code == CODE_FOR_nothing)
4093 cmp_code = direct_optab_handler (cmpstrn_optab, cmp_mode);
4094 if (cmp_code == CODE_FOR_nothing)
4095 continue;
4097 /* Must make sure the size fits the insn's mode. */
4098 if ((CONST_INT_P (size)
4099 && INTVAL (size) >= (1 << GET_MODE_BITSIZE (cmp_mode)))
4100 || (GET_MODE_BITSIZE (GET_MODE (size))
4101 > GET_MODE_BITSIZE (cmp_mode)))
4102 continue;
4104 result_mode = insn_data[cmp_code].operand[0].mode;
4105 result = gen_reg_rtx (result_mode);
4106 size = convert_to_mode (cmp_mode, size, 1);
4107 emit_insn (GEN_FCN (cmp_code) (result, x, y, size, opalign));
4109 *ptest = gen_rtx_fmt_ee (comparison, VOIDmode, result, const0_rtx);
4110 *pmode = result_mode;
4111 return;
4114 if (methods != OPTAB_LIB && methods != OPTAB_LIB_WIDEN)
4115 goto fail;
4117 /* Otherwise call a library function, memcmp. */
4118 libfunc = memcmp_libfunc;
4119 length_type = sizetype;
4120 result_mode = TYPE_MODE (integer_type_node);
4121 cmp_mode = TYPE_MODE (length_type);
4122 size = convert_to_mode (TYPE_MODE (length_type), size,
4123 TYPE_UNSIGNED (length_type));
4125 result = emit_library_call_value (libfunc, 0, LCT_PURE,
4126 result_mode, 3,
4127 XEXP (x, 0), Pmode,
4128 XEXP (y, 0), Pmode,
4129 size, cmp_mode);
4130 x = result;
4131 y = const0_rtx;
4132 mode = result_mode;
4133 methods = OPTAB_LIB_WIDEN;
4134 unsignedp = false;
4137 /* Don't allow operands to the compare to trap, as that can put the
4138 compare and branch in different basic blocks. */
4139 if (cfun->can_throw_non_call_exceptions)
4141 if (may_trap_p (x))
4142 x = force_reg (mode, x);
4143 if (may_trap_p (y))
4144 y = force_reg (mode, y);
4147 if (GET_MODE_CLASS (mode) == MODE_CC)
4149 enum insn_code icode = optab_handler (cbranch_optab, CCmode);
4150 test = gen_rtx_fmt_ee (comparison, VOIDmode, x, y);
4151 gcc_assert (icode != CODE_FOR_nothing
4152 && insn_operand_matches (icode, 0, test));
4153 *ptest = test;
4154 return;
4157 mclass = GET_MODE_CLASS (mode);
4158 test = gen_rtx_fmt_ee (comparison, VOIDmode, x, y);
4159 cmp_mode = mode;
4162 enum insn_code icode;
4163 icode = optab_handler (cbranch_optab, cmp_mode);
4164 if (icode != CODE_FOR_nothing
4165 && insn_operand_matches (icode, 0, test))
4167 rtx_insn *last = get_last_insn ();
4168 rtx op0 = prepare_operand (icode, x, 1, mode, cmp_mode, unsignedp);
4169 rtx op1 = prepare_operand (icode, y, 2, mode, cmp_mode, unsignedp);
4170 if (op0 && op1
4171 && insn_operand_matches (icode, 1, op0)
4172 && insn_operand_matches (icode, 2, op1))
4174 XEXP (test, 0) = op0;
4175 XEXP (test, 1) = op1;
4176 *ptest = test;
4177 *pmode = cmp_mode;
4178 return;
4180 delete_insns_since (last);
4183 if (methods == OPTAB_DIRECT || !CLASS_HAS_WIDER_MODES_P (mclass))
4184 break;
4185 cmp_mode = GET_MODE_WIDER_MODE (cmp_mode);
4187 while (cmp_mode != VOIDmode);
4189 if (methods != OPTAB_LIB_WIDEN)
4190 goto fail;
4192 if (!SCALAR_FLOAT_MODE_P (mode))
4194 rtx result;
4195 machine_mode ret_mode;
4197 /* Handle a libcall just for the mode we are using. */
4198 libfunc = optab_libfunc (cmp_optab, mode);
4199 gcc_assert (libfunc);
4201 /* If we want unsigned, and this mode has a distinct unsigned
4202 comparison routine, use that. */
4203 if (unsignedp)
4205 rtx ulibfunc = optab_libfunc (ucmp_optab, mode);
4206 if (ulibfunc)
4207 libfunc = ulibfunc;
4210 ret_mode = targetm.libgcc_cmp_return_mode ();
4211 result = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
4212 ret_mode, 2, x, mode, y, mode);
4214 /* There are two kinds of comparison routines. Biased routines
4215 return 0/1/2, and unbiased routines return -1/0/1. Other parts
4216 of gcc expect that the comparison operation is equivalent
4217 to the modified comparison. For signed comparisons compare the
4218 result against 1 in the biased case, and zero in the unbiased
4219 case. For unsigned comparisons always compare against 1 after
4220 biasing the unbiased result by adding 1. This gives us a way to
4221 represent LTU.
4222 The comparisons in the fixed-point helper library are always
4223 biased. */
4224 x = result;
4225 y = const1_rtx;
4227 if (!TARGET_LIB_INT_CMP_BIASED && !ALL_FIXED_POINT_MODE_P (mode))
4229 if (unsignedp)
4230 x = plus_constant (ret_mode, result, 1);
4231 else
4232 y = const0_rtx;
4235 *pmode = ret_mode;
4236 prepare_cmp_insn (x, y, comparison, NULL_RTX, unsignedp, methods,
4237 ptest, pmode);
4239 else
4240 prepare_float_lib_cmp (x, y, comparison, ptest, pmode);
4242 return;
4244 fail:
4245 *ptest = NULL_RTX;
4248 /* Before emitting an insn with code ICODE, make sure that X, which is going
4249 to be used for operand OPNUM of the insn, is converted from mode MODE to
4250 WIDER_MODE (UNSIGNEDP determines whether it is an unsigned conversion), and
4251 that it is accepted by the operand predicate. Return the new value. */
4254 prepare_operand (enum insn_code icode, rtx x, int opnum, machine_mode mode,
4255 machine_mode wider_mode, int unsignedp)
4257 if (mode != wider_mode)
4258 x = convert_modes (wider_mode, mode, x, unsignedp);
4260 if (!insn_operand_matches (icode, opnum, x))
4262 machine_mode op_mode = insn_data[(int) icode].operand[opnum].mode;
4263 if (reload_completed)
4264 return NULL_RTX;
4265 if (GET_MODE (x) != op_mode && GET_MODE (x) != VOIDmode)
4266 return NULL_RTX;
4267 x = copy_to_mode_reg (op_mode, x);
4270 return x;
4273 /* Subroutine of emit_cmp_and_jump_insns; this function is called when we know
4274 we can do the branch. */
4276 static void
4277 emit_cmp_and_jump_insn_1 (rtx test, machine_mode mode, rtx label, int prob)
4279 machine_mode optab_mode;
4280 enum mode_class mclass;
4281 enum insn_code icode;
4282 rtx_insn *insn;
4284 mclass = GET_MODE_CLASS (mode);
4285 optab_mode = (mclass == MODE_CC) ? CCmode : mode;
4286 icode = optab_handler (cbranch_optab, optab_mode);
4288 gcc_assert (icode != CODE_FOR_nothing);
4289 gcc_assert (insn_operand_matches (icode, 0, test));
4290 insn = emit_jump_insn (GEN_FCN (icode) (test, XEXP (test, 0),
4291 XEXP (test, 1), label));
4292 if (prob != -1
4293 && profile_status_for_fn (cfun) != PROFILE_ABSENT
4294 && insn
4295 && JUMP_P (insn)
4296 && any_condjump_p (insn)
4297 && !find_reg_note (insn, REG_BR_PROB, 0))
4298 add_int_reg_note (insn, REG_BR_PROB, prob);
4301 /* Generate code to compare X with Y so that the condition codes are
4302 set and to jump to LABEL if the condition is true. If X is a
4303 constant and Y is not a constant, then the comparison is swapped to
4304 ensure that the comparison RTL has the canonical form.
4306 UNSIGNEDP nonzero says that X and Y are unsigned; this matters if they
4307 need to be widened. UNSIGNEDP is also used to select the proper
4308 branch condition code.
4310 If X and Y have mode BLKmode, then SIZE specifies the size of both X and Y.
4312 MODE is the mode of the inputs (in case they are const_int).
4314 COMPARISON is the rtl operator to compare with (EQ, NE, GT, etc.).
4315 It will be potentially converted into an unsigned variant based on
4316 UNSIGNEDP to select a proper jump instruction.
4318 PROB is the probability of jumping to LABEL. */
4320 void
4321 emit_cmp_and_jump_insns (rtx x, rtx y, enum rtx_code comparison, rtx size,
4322 machine_mode mode, int unsignedp, rtx label,
4323 int prob)
4325 rtx op0 = x, op1 = y;
4326 rtx test;
4328 /* Swap operands and condition to ensure canonical RTL. */
4329 if (swap_commutative_operands_p (x, y)
4330 && can_compare_p (swap_condition (comparison), mode, ccp_jump))
4332 op0 = y, op1 = x;
4333 comparison = swap_condition (comparison);
4336 /* If OP0 is still a constant, then both X and Y must be constants
4337 or the opposite comparison is not supported. Force X into a register
4338 to create canonical RTL. */
4339 if (CONSTANT_P (op0))
4340 op0 = force_reg (mode, op0);
4342 if (unsignedp)
4343 comparison = unsigned_condition (comparison);
4345 prepare_cmp_insn (op0, op1, comparison, size, unsignedp, OPTAB_LIB_WIDEN,
4346 &test, &mode);
4347 emit_cmp_and_jump_insn_1 (test, mode, label, prob);
4351 /* Emit a library call comparison between floating point X and Y.
4352 COMPARISON is the rtl operator to compare with (EQ, NE, GT, etc.). */
4354 static void
4355 prepare_float_lib_cmp (rtx x, rtx y, enum rtx_code comparison,
4356 rtx *ptest, machine_mode *pmode)
4358 enum rtx_code swapped = swap_condition (comparison);
4359 enum rtx_code reversed = reverse_condition_maybe_unordered (comparison);
4360 machine_mode orig_mode = GET_MODE (x);
4361 machine_mode mode, cmp_mode;
4362 rtx true_rtx, false_rtx;
4363 rtx value, target, equiv;
4364 rtx_insn *insns;
4365 rtx libfunc = 0;
4366 bool reversed_p = false;
4367 cmp_mode = targetm.libgcc_cmp_return_mode ();
4369 for (mode = orig_mode;
4370 mode != VOIDmode;
4371 mode = GET_MODE_WIDER_MODE (mode))
4373 if (code_to_optab (comparison)
4374 && (libfunc = optab_libfunc (code_to_optab (comparison), mode)))
4375 break;
4377 if (code_to_optab (swapped)
4378 && (libfunc = optab_libfunc (code_to_optab (swapped), mode)))
4380 std::swap (x, y);
4381 comparison = swapped;
4382 break;
4385 if (code_to_optab (reversed)
4386 && (libfunc = optab_libfunc (code_to_optab (reversed), mode)))
4388 comparison = reversed;
4389 reversed_p = true;
4390 break;
4394 gcc_assert (mode != VOIDmode);
4396 if (mode != orig_mode)
4398 x = convert_to_mode (mode, x, 0);
4399 y = convert_to_mode (mode, y, 0);
4402 /* Attach a REG_EQUAL note describing the semantics of the libcall to
4403 the RTL. The allows the RTL optimizers to delete the libcall if the
4404 condition can be determined at compile-time. */
4405 if (comparison == UNORDERED
4406 || FLOAT_LIB_COMPARE_RETURNS_BOOL (mode, comparison))
4408 true_rtx = const_true_rtx;
4409 false_rtx = const0_rtx;
4411 else
4413 switch (comparison)
4415 case EQ:
4416 true_rtx = const0_rtx;
4417 false_rtx = const_true_rtx;
4418 break;
4420 case NE:
4421 true_rtx = const_true_rtx;
4422 false_rtx = const0_rtx;
4423 break;
4425 case GT:
4426 true_rtx = const1_rtx;
4427 false_rtx = const0_rtx;
4428 break;
4430 case GE:
4431 true_rtx = const0_rtx;
4432 false_rtx = constm1_rtx;
4433 break;
4435 case LT:
4436 true_rtx = constm1_rtx;
4437 false_rtx = const0_rtx;
4438 break;
4440 case LE:
4441 true_rtx = const0_rtx;
4442 false_rtx = const1_rtx;
4443 break;
4445 default:
4446 gcc_unreachable ();
4450 if (comparison == UNORDERED)
4452 rtx temp = simplify_gen_relational (NE, cmp_mode, mode, x, x);
4453 equiv = simplify_gen_relational (NE, cmp_mode, mode, y, y);
4454 equiv = simplify_gen_ternary (IF_THEN_ELSE, cmp_mode, cmp_mode,
4455 temp, const_true_rtx, equiv);
4457 else
4459 equiv = simplify_gen_relational (comparison, cmp_mode, mode, x, y);
4460 if (! FLOAT_LIB_COMPARE_RETURNS_BOOL (mode, comparison))
4461 equiv = simplify_gen_ternary (IF_THEN_ELSE, cmp_mode, cmp_mode,
4462 equiv, true_rtx, false_rtx);
4465 start_sequence ();
4466 value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
4467 cmp_mode, 2, x, mode, y, mode);
4468 insns = get_insns ();
4469 end_sequence ();
4471 target = gen_reg_rtx (cmp_mode);
4472 emit_libcall_block (insns, target, value, equiv);
4474 if (comparison == UNORDERED
4475 || FLOAT_LIB_COMPARE_RETURNS_BOOL (mode, comparison)
4476 || reversed_p)
4477 *ptest = gen_rtx_fmt_ee (reversed_p ? EQ : NE, VOIDmode, target, false_rtx);
4478 else
4479 *ptest = gen_rtx_fmt_ee (comparison, VOIDmode, target, const0_rtx);
4481 *pmode = cmp_mode;
4484 /* Generate code to indirectly jump to a location given in the rtx LOC. */
4486 void
4487 emit_indirect_jump (rtx loc ATTRIBUTE_UNUSED)
4489 #ifndef HAVE_indirect_jump
4490 sorry ("indirect jumps are not available on this target");
4491 #else
4492 struct expand_operand ops[1];
4493 create_address_operand (&ops[0], loc);
4494 expand_jump_insn (CODE_FOR_indirect_jump, 1, ops);
4495 emit_barrier ();
4496 #endif
4500 /* Emit a conditional move instruction if the machine supports one for that
4501 condition and machine mode.
4503 OP0 and OP1 are the operands that should be compared using CODE. CMODE is
4504 the mode to use should they be constants. If it is VOIDmode, they cannot
4505 both be constants.
4507 OP2 should be stored in TARGET if the comparison is true, otherwise OP3
4508 should be stored there. MODE is the mode to use should they be constants.
4509 If it is VOIDmode, they cannot both be constants.
4511 The result is either TARGET (perhaps modified) or NULL_RTX if the operation
4512 is not supported. */
4515 emit_conditional_move (rtx target, enum rtx_code code, rtx op0, rtx op1,
4516 machine_mode cmode, rtx op2, rtx op3,
4517 machine_mode mode, int unsignedp)
4519 rtx comparison;
4520 rtx_insn *last;
4521 enum insn_code icode;
4522 enum rtx_code reversed;
4524 /* If one operand is constant, make it the second one. Only do this
4525 if the other operand is not constant as well. */
4527 if (swap_commutative_operands_p (op0, op1))
4529 std::swap (op0, op1);
4530 code = swap_condition (code);
4533 /* get_condition will prefer to generate LT and GT even if the old
4534 comparison was against zero, so undo that canonicalization here since
4535 comparisons against zero are cheaper. */
4536 if (code == LT && op1 == const1_rtx)
4537 code = LE, op1 = const0_rtx;
4538 else if (code == GT && op1 == constm1_rtx)
4539 code = GE, op1 = const0_rtx;
4541 if (cmode == VOIDmode)
4542 cmode = GET_MODE (op0);
4544 if (swap_commutative_operands_p (op2, op3)
4545 && ((reversed = reversed_comparison_code_parts (code, op0, op1, NULL))
4546 != UNKNOWN))
4548 std::swap (op2, op3);
4549 code = reversed;
4552 if (mode == VOIDmode)
4553 mode = GET_MODE (op2);
4555 icode = direct_optab_handler (movcc_optab, mode);
4557 if (icode == CODE_FOR_nothing)
4558 return 0;
4560 if (!target)
4561 target = gen_reg_rtx (mode);
4563 code = unsignedp ? unsigned_condition (code) : code;
4564 comparison = simplify_gen_relational (code, VOIDmode, cmode, op0, op1);
4566 /* We can get const0_rtx or const_true_rtx in some circumstances. Just
4567 return NULL and let the caller figure out how best to deal with this
4568 situation. */
4569 if (!COMPARISON_P (comparison))
4570 return NULL_RTX;
4572 saved_pending_stack_adjust save;
4573 save_pending_stack_adjust (&save);
4574 last = get_last_insn ();
4575 do_pending_stack_adjust ();
4576 prepare_cmp_insn (XEXP (comparison, 0), XEXP (comparison, 1),
4577 GET_CODE (comparison), NULL_RTX, unsignedp, OPTAB_WIDEN,
4578 &comparison, &cmode);
4579 if (comparison)
4581 struct expand_operand ops[4];
4583 create_output_operand (&ops[0], target, mode);
4584 create_fixed_operand (&ops[1], comparison);
4585 create_input_operand (&ops[2], op2, mode);
4586 create_input_operand (&ops[3], op3, mode);
4587 if (maybe_expand_insn (icode, 4, ops))
4589 if (ops[0].value != target)
4590 convert_move (target, ops[0].value, false);
4591 return target;
4594 delete_insns_since (last);
4595 restore_pending_stack_adjust (&save);
4596 return NULL_RTX;
4599 /* Return nonzero if a conditional move of mode MODE is supported.
4601 This function is for combine so it can tell whether an insn that looks
4602 like a conditional move is actually supported by the hardware. If we
4603 guess wrong we lose a bit on optimization, but that's it. */
4604 /* ??? sparc64 supports conditionally moving integers values based on fp
4605 comparisons, and vice versa. How do we handle them? */
4608 can_conditionally_move_p (machine_mode mode)
4610 if (direct_optab_handler (movcc_optab, mode) != CODE_FOR_nothing)
4611 return 1;
4613 return 0;
4616 /* Emit a conditional addition instruction if the machine supports one for that
4617 condition and machine mode.
4619 OP0 and OP1 are the operands that should be compared using CODE. CMODE is
4620 the mode to use should they be constants. If it is VOIDmode, they cannot
4621 both be constants.
4623 OP2 should be stored in TARGET if the comparison is false, otherwise OP2+OP3
4624 should be stored there. MODE is the mode to use should they be constants.
4625 If it is VOIDmode, they cannot both be constants.
4627 The result is either TARGET (perhaps modified) or NULL_RTX if the operation
4628 is not supported. */
4631 emit_conditional_add (rtx target, enum rtx_code code, rtx op0, rtx op1,
4632 machine_mode cmode, rtx op2, rtx op3,
4633 machine_mode mode, int unsignedp)
4635 rtx comparison;
4636 rtx_insn *last;
4637 enum insn_code icode;
4639 /* If one operand is constant, make it the second one. Only do this
4640 if the other operand is not constant as well. */
4642 if (swap_commutative_operands_p (op0, op1))
4644 std::swap (op0, op1);
4645 code = swap_condition (code);
4648 /* get_condition will prefer to generate LT and GT even if the old
4649 comparison was against zero, so undo that canonicalization here since
4650 comparisons against zero are cheaper. */
4651 if (code == LT && op1 == const1_rtx)
4652 code = LE, op1 = const0_rtx;
4653 else if (code == GT && op1 == constm1_rtx)
4654 code = GE, op1 = const0_rtx;
4656 if (cmode == VOIDmode)
4657 cmode = GET_MODE (op0);
4659 if (mode == VOIDmode)
4660 mode = GET_MODE (op2);
4662 icode = optab_handler (addcc_optab, mode);
4664 if (icode == CODE_FOR_nothing)
4665 return 0;
4667 if (!target)
4668 target = gen_reg_rtx (mode);
4670 code = unsignedp ? unsigned_condition (code) : code;
4671 comparison = simplify_gen_relational (code, VOIDmode, cmode, op0, op1);
4673 /* We can get const0_rtx or const_true_rtx in some circumstances. Just
4674 return NULL and let the caller figure out how best to deal with this
4675 situation. */
4676 if (!COMPARISON_P (comparison))
4677 return NULL_RTX;
4679 do_pending_stack_adjust ();
4680 last = get_last_insn ();
4681 prepare_cmp_insn (XEXP (comparison, 0), XEXP (comparison, 1),
4682 GET_CODE (comparison), NULL_RTX, unsignedp, OPTAB_WIDEN,
4683 &comparison, &cmode);
4684 if (comparison)
4686 struct expand_operand ops[4];
4688 create_output_operand (&ops[0], target, mode);
4689 create_fixed_operand (&ops[1], comparison);
4690 create_input_operand (&ops[2], op2, mode);
4691 create_input_operand (&ops[3], op3, mode);
4692 if (maybe_expand_insn (icode, 4, ops))
4694 if (ops[0].value != target)
4695 convert_move (target, ops[0].value, false);
4696 return target;
4699 delete_insns_since (last);
4700 return NULL_RTX;
4703 /* These functions attempt to generate an insn body, rather than
4704 emitting the insn, but if the gen function already emits them, we
4705 make no attempt to turn them back into naked patterns. */
4707 /* Generate and return an insn body to add Y to X. */
4709 rtx_insn *
4710 gen_add2_insn (rtx x, rtx y)
4712 enum insn_code icode = optab_handler (add_optab, GET_MODE (x));
4714 gcc_assert (insn_operand_matches (icode, 0, x));
4715 gcc_assert (insn_operand_matches (icode, 1, x));
4716 gcc_assert (insn_operand_matches (icode, 2, y));
4718 return GEN_FCN (icode) (x, x, y);
4721 /* Generate and return an insn body to add r1 and c,
4722 storing the result in r0. */
4724 rtx_insn *
4725 gen_add3_insn (rtx r0, rtx r1, rtx c)
4727 enum insn_code icode = optab_handler (add_optab, GET_MODE (r0));
4729 if (icode == CODE_FOR_nothing
4730 || !insn_operand_matches (icode, 0, r0)
4731 || !insn_operand_matches (icode, 1, r1)
4732 || !insn_operand_matches (icode, 2, c))
4733 return NULL;
4735 return GEN_FCN (icode) (r0, r1, c);
4739 have_add2_insn (rtx x, rtx y)
4741 enum insn_code icode;
4743 gcc_assert (GET_MODE (x) != VOIDmode);
4745 icode = optab_handler (add_optab, GET_MODE (x));
4747 if (icode == CODE_FOR_nothing)
4748 return 0;
4750 if (!insn_operand_matches (icode, 0, x)
4751 || !insn_operand_matches (icode, 1, x)
4752 || !insn_operand_matches (icode, 2, y))
4753 return 0;
4755 return 1;
4758 /* Generate and return an insn body to add Y to X. */
4760 rtx_insn *
4761 gen_addptr3_insn (rtx x, rtx y, rtx z)
4763 enum insn_code icode = optab_handler (addptr3_optab, GET_MODE (x));
4765 gcc_assert (insn_operand_matches (icode, 0, x));
4766 gcc_assert (insn_operand_matches (icode, 1, y));
4767 gcc_assert (insn_operand_matches (icode, 2, z));
4769 return GEN_FCN (icode) (x, y, z);
4772 /* Return true if the target implements an addptr pattern and X, Y,
4773 and Z are valid for the pattern predicates. */
4776 have_addptr3_insn (rtx x, rtx y, rtx z)
4778 enum insn_code icode;
4780 gcc_assert (GET_MODE (x) != VOIDmode);
4782 icode = optab_handler (addptr3_optab, GET_MODE (x));
4784 if (icode == CODE_FOR_nothing)
4785 return 0;
4787 if (!insn_operand_matches (icode, 0, x)
4788 || !insn_operand_matches (icode, 1, y)
4789 || !insn_operand_matches (icode, 2, z))
4790 return 0;
4792 return 1;
4795 /* Generate and return an insn body to subtract Y from X. */
4797 rtx_insn *
4798 gen_sub2_insn (rtx x, rtx y)
4800 enum insn_code icode = optab_handler (sub_optab, GET_MODE (x));
4802 gcc_assert (insn_operand_matches (icode, 0, x));
4803 gcc_assert (insn_operand_matches (icode, 1, x));
4804 gcc_assert (insn_operand_matches (icode, 2, y));
4806 return GEN_FCN (icode) (x, x, y);
4809 /* Generate and return an insn body to subtract r1 and c,
4810 storing the result in r0. */
4812 rtx_insn *
4813 gen_sub3_insn (rtx r0, rtx r1, rtx c)
4815 enum insn_code icode = optab_handler (sub_optab, GET_MODE (r0));
4817 if (icode == CODE_FOR_nothing
4818 || !insn_operand_matches (icode, 0, r0)
4819 || !insn_operand_matches (icode, 1, r1)
4820 || !insn_operand_matches (icode, 2, c))
4821 return NULL;
4823 return GEN_FCN (icode) (r0, r1, c);
4827 have_sub2_insn (rtx x, rtx y)
4829 enum insn_code icode;
4831 gcc_assert (GET_MODE (x) != VOIDmode);
4833 icode = optab_handler (sub_optab, GET_MODE (x));
4835 if (icode == CODE_FOR_nothing)
4836 return 0;
4838 if (!insn_operand_matches (icode, 0, x)
4839 || !insn_operand_matches (icode, 1, x)
4840 || !insn_operand_matches (icode, 2, y))
4841 return 0;
4843 return 1;
4846 /* Return the insn code used to extend FROM_MODE to TO_MODE.
4847 UNSIGNEDP specifies zero-extension instead of sign-extension. If
4848 no such operation exists, CODE_FOR_nothing will be returned. */
4850 enum insn_code
4851 can_extend_p (machine_mode to_mode, machine_mode from_mode,
4852 int unsignedp)
4854 convert_optab tab;
4855 #ifdef HAVE_ptr_extend
4856 if (unsignedp < 0)
4857 return CODE_FOR_ptr_extend;
4858 #endif
4860 tab = unsignedp ? zext_optab : sext_optab;
4861 return convert_optab_handler (tab, to_mode, from_mode);
4864 /* Generate the body of an insn to extend Y (with mode MFROM)
4865 into X (with mode MTO). Do zero-extension if UNSIGNEDP is nonzero. */
4867 rtx_insn *
4868 gen_extend_insn (rtx x, rtx y, machine_mode mto,
4869 machine_mode mfrom, int unsignedp)
4871 enum insn_code icode = can_extend_p (mto, mfrom, unsignedp);
4872 return GEN_FCN (icode) (x, y);
4875 /* can_fix_p and can_float_p say whether the target machine
4876 can directly convert a given fixed point type to
4877 a given floating point type, or vice versa.
4878 The returned value is the CODE_FOR_... value to use,
4879 or CODE_FOR_nothing if these modes cannot be directly converted.
4881 *TRUNCP_PTR is set to 1 if it is necessary to output
4882 an explicit FTRUNC insn before the fix insn; otherwise 0. */
4884 static enum insn_code
4885 can_fix_p (machine_mode fixmode, machine_mode fltmode,
4886 int unsignedp, int *truncp_ptr)
4888 convert_optab tab;
4889 enum insn_code icode;
4891 tab = unsignedp ? ufixtrunc_optab : sfixtrunc_optab;
4892 icode = convert_optab_handler (tab, fixmode, fltmode);
4893 if (icode != CODE_FOR_nothing)
4895 *truncp_ptr = 0;
4896 return icode;
4899 /* FIXME: This requires a port to define both FIX and FTRUNC pattern
4900 for this to work. We need to rework the fix* and ftrunc* patterns
4901 and documentation. */
4902 tab = unsignedp ? ufix_optab : sfix_optab;
4903 icode = convert_optab_handler (tab, fixmode, fltmode);
4904 if (icode != CODE_FOR_nothing
4905 && optab_handler (ftrunc_optab, fltmode) != CODE_FOR_nothing)
4907 *truncp_ptr = 1;
4908 return icode;
4911 *truncp_ptr = 0;
4912 return CODE_FOR_nothing;
4915 enum insn_code
4916 can_float_p (machine_mode fltmode, machine_mode fixmode,
4917 int unsignedp)
4919 convert_optab tab;
4921 tab = unsignedp ? ufloat_optab : sfloat_optab;
4922 return convert_optab_handler (tab, fltmode, fixmode);
4925 /* Function supportable_convert_operation
4927 Check whether an operation represented by the code CODE is a
4928 convert operation that is supported by the target platform in
4929 vector form (i.e., when operating on arguments of type VECTYPE_IN
4930 producing a result of type VECTYPE_OUT).
4932 Convert operations we currently support directly are FIX_TRUNC and FLOAT.
4933 This function checks if these operations are supported
4934 by the target platform either directly (via vector tree-codes), or via
4935 target builtins.
4937 Output:
4938 - CODE1 is code of vector operation to be used when
4939 vectorizing the operation, if available.
4940 - DECL is decl of target builtin functions to be used
4941 when vectorizing the operation, if available. In this case,
4942 CODE1 is CALL_EXPR. */
4944 bool
4945 supportable_convert_operation (enum tree_code code,
4946 tree vectype_out, tree vectype_in,
4947 tree *decl, enum tree_code *code1)
4949 machine_mode m1,m2;
4950 int truncp;
4952 m1 = TYPE_MODE (vectype_out);
4953 m2 = TYPE_MODE (vectype_in);
4955 /* First check if we can done conversion directly. */
4956 if ((code == FIX_TRUNC_EXPR
4957 && can_fix_p (m1,m2,TYPE_UNSIGNED (vectype_out), &truncp)
4958 != CODE_FOR_nothing)
4959 || (code == FLOAT_EXPR
4960 && can_float_p (m1,m2,TYPE_UNSIGNED (vectype_in))
4961 != CODE_FOR_nothing))
4963 *code1 = code;
4964 return true;
4967 /* Now check for builtin. */
4968 if (targetm.vectorize.builtin_conversion
4969 && targetm.vectorize.builtin_conversion (code, vectype_out, vectype_in))
4971 *code1 = CALL_EXPR;
4972 *decl = targetm.vectorize.builtin_conversion (code, vectype_out, vectype_in);
4973 return true;
4975 return false;
4979 /* Generate code to convert FROM to floating point
4980 and store in TO. FROM must be fixed point and not VOIDmode.
4981 UNSIGNEDP nonzero means regard FROM as unsigned.
4982 Normally this is done by correcting the final value
4983 if it is negative. */
4985 void
4986 expand_float (rtx to, rtx from, int unsignedp)
4988 enum insn_code icode;
4989 rtx target = to;
4990 machine_mode fmode, imode;
4991 bool can_do_signed = false;
4993 /* Crash now, because we won't be able to decide which mode to use. */
4994 gcc_assert (GET_MODE (from) != VOIDmode);
4996 /* Look for an insn to do the conversion. Do it in the specified
4997 modes if possible; otherwise convert either input, output or both to
4998 wider mode. If the integer mode is wider than the mode of FROM,
4999 we can do the conversion signed even if the input is unsigned. */
5001 for (fmode = GET_MODE (to); fmode != VOIDmode;
5002 fmode = GET_MODE_WIDER_MODE (fmode))
5003 for (imode = GET_MODE (from); imode != VOIDmode;
5004 imode = GET_MODE_WIDER_MODE (imode))
5006 int doing_unsigned = unsignedp;
5008 if (fmode != GET_MODE (to)
5009 && significand_size (fmode) < GET_MODE_PRECISION (GET_MODE (from)))
5010 continue;
5012 icode = can_float_p (fmode, imode, unsignedp);
5013 if (icode == CODE_FOR_nothing && unsignedp)
5015 enum insn_code scode = can_float_p (fmode, imode, 0);
5016 if (scode != CODE_FOR_nothing)
5017 can_do_signed = true;
5018 if (imode != GET_MODE (from))
5019 icode = scode, doing_unsigned = 0;
5022 if (icode != CODE_FOR_nothing)
5024 if (imode != GET_MODE (from))
5025 from = convert_to_mode (imode, from, unsignedp);
5027 if (fmode != GET_MODE (to))
5028 target = gen_reg_rtx (fmode);
5030 emit_unop_insn (icode, target, from,
5031 doing_unsigned ? UNSIGNED_FLOAT : FLOAT);
5033 if (target != to)
5034 convert_move (to, target, 0);
5035 return;
5039 /* Unsigned integer, and no way to convert directly. Convert as signed,
5040 then unconditionally adjust the result. */
5041 if (unsignedp && can_do_signed)
5043 rtx_code_label *label = gen_label_rtx ();
5044 rtx temp;
5045 REAL_VALUE_TYPE offset;
5047 /* Look for a usable floating mode FMODE wider than the source and at
5048 least as wide as the target. Using FMODE will avoid rounding woes
5049 with unsigned values greater than the signed maximum value. */
5051 for (fmode = GET_MODE (to); fmode != VOIDmode;
5052 fmode = GET_MODE_WIDER_MODE (fmode))
5053 if (GET_MODE_PRECISION (GET_MODE (from)) < GET_MODE_BITSIZE (fmode)
5054 && can_float_p (fmode, GET_MODE (from), 0) != CODE_FOR_nothing)
5055 break;
5057 if (fmode == VOIDmode)
5059 /* There is no such mode. Pretend the target is wide enough. */
5060 fmode = GET_MODE (to);
5062 /* Avoid double-rounding when TO is narrower than FROM. */
5063 if ((significand_size (fmode) + 1)
5064 < GET_MODE_PRECISION (GET_MODE (from)))
5066 rtx temp1;
5067 rtx_code_label *neglabel = gen_label_rtx ();
5069 /* Don't use TARGET if it isn't a register, is a hard register,
5070 or is the wrong mode. */
5071 if (!REG_P (target)
5072 || REGNO (target) < FIRST_PSEUDO_REGISTER
5073 || GET_MODE (target) != fmode)
5074 target = gen_reg_rtx (fmode);
5076 imode = GET_MODE (from);
5077 do_pending_stack_adjust ();
5079 /* Test whether the sign bit is set. */
5080 emit_cmp_and_jump_insns (from, const0_rtx, LT, NULL_RTX, imode,
5081 0, neglabel);
5083 /* The sign bit is not set. Convert as signed. */
5084 expand_float (target, from, 0);
5085 emit_jump_insn (targetm.gen_jump (label));
5086 emit_barrier ();
5088 /* The sign bit is set.
5089 Convert to a usable (positive signed) value by shifting right
5090 one bit, while remembering if a nonzero bit was shifted
5091 out; i.e., compute (from & 1) | (from >> 1). */
5093 emit_label (neglabel);
5094 temp = expand_binop (imode, and_optab, from, const1_rtx,
5095 NULL_RTX, 1, OPTAB_LIB_WIDEN);
5096 temp1 = expand_shift (RSHIFT_EXPR, imode, from, 1, NULL_RTX, 1);
5097 temp = expand_binop (imode, ior_optab, temp, temp1, temp, 1,
5098 OPTAB_LIB_WIDEN);
5099 expand_float (target, temp, 0);
5101 /* Multiply by 2 to undo the shift above. */
5102 temp = expand_binop (fmode, add_optab, target, target,
5103 target, 0, OPTAB_LIB_WIDEN);
5104 if (temp != target)
5105 emit_move_insn (target, temp);
5107 do_pending_stack_adjust ();
5108 emit_label (label);
5109 goto done;
5113 /* If we are about to do some arithmetic to correct for an
5114 unsigned operand, do it in a pseudo-register. */
5116 if (GET_MODE (to) != fmode
5117 || !REG_P (to) || REGNO (to) < FIRST_PSEUDO_REGISTER)
5118 target = gen_reg_rtx (fmode);
5120 /* Convert as signed integer to floating. */
5121 expand_float (target, from, 0);
5123 /* If FROM is negative (and therefore TO is negative),
5124 correct its value by 2**bitwidth. */
5126 do_pending_stack_adjust ();
5127 emit_cmp_and_jump_insns (from, const0_rtx, GE, NULL_RTX, GET_MODE (from),
5128 0, label);
5131 real_2expN (&offset, GET_MODE_PRECISION (GET_MODE (from)), fmode);
5132 temp = expand_binop (fmode, add_optab, target,
5133 CONST_DOUBLE_FROM_REAL_VALUE (offset, fmode),
5134 target, 0, OPTAB_LIB_WIDEN);
5135 if (temp != target)
5136 emit_move_insn (target, temp);
5138 do_pending_stack_adjust ();
5139 emit_label (label);
5140 goto done;
5143 /* No hardware instruction available; call a library routine. */
5145 rtx libfunc;
5146 rtx_insn *insns;
5147 rtx value;
5148 convert_optab tab = unsignedp ? ufloat_optab : sfloat_optab;
5150 if (GET_MODE_PRECISION (GET_MODE (from)) < GET_MODE_PRECISION (SImode))
5151 from = convert_to_mode (SImode, from, unsignedp);
5153 libfunc = convert_optab_libfunc (tab, GET_MODE (to), GET_MODE (from));
5154 gcc_assert (libfunc);
5156 start_sequence ();
5158 value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
5159 GET_MODE (to), 1, from,
5160 GET_MODE (from));
5161 insns = get_insns ();
5162 end_sequence ();
5164 emit_libcall_block (insns, target, value,
5165 gen_rtx_fmt_e (unsignedp ? UNSIGNED_FLOAT : FLOAT,
5166 GET_MODE (to), from));
5169 done:
5171 /* Copy result to requested destination
5172 if we have been computing in a temp location. */
5174 if (target != to)
5176 if (GET_MODE (target) == GET_MODE (to))
5177 emit_move_insn (to, target);
5178 else
5179 convert_move (to, target, 0);
5183 /* Generate code to convert FROM to fixed point and store in TO. FROM
5184 must be floating point. */
5186 void
5187 expand_fix (rtx to, rtx from, int unsignedp)
5189 enum insn_code icode;
5190 rtx target = to;
5191 machine_mode fmode, imode;
5192 int must_trunc = 0;
5194 /* We first try to find a pair of modes, one real and one integer, at
5195 least as wide as FROM and TO, respectively, in which we can open-code
5196 this conversion. If the integer mode is wider than the mode of TO,
5197 we can do the conversion either signed or unsigned. */
5199 for (fmode = GET_MODE (from); fmode != VOIDmode;
5200 fmode = GET_MODE_WIDER_MODE (fmode))
5201 for (imode = GET_MODE (to); imode != VOIDmode;
5202 imode = GET_MODE_WIDER_MODE (imode))
5204 int doing_unsigned = unsignedp;
5206 icode = can_fix_p (imode, fmode, unsignedp, &must_trunc);
5207 if (icode == CODE_FOR_nothing && imode != GET_MODE (to) && unsignedp)
5208 icode = can_fix_p (imode, fmode, 0, &must_trunc), doing_unsigned = 0;
5210 if (icode != CODE_FOR_nothing)
5212 rtx_insn *last = get_last_insn ();
5213 if (fmode != GET_MODE (from))
5214 from = convert_to_mode (fmode, from, 0);
5216 if (must_trunc)
5218 rtx temp = gen_reg_rtx (GET_MODE (from));
5219 from = expand_unop (GET_MODE (from), ftrunc_optab, from,
5220 temp, 0);
5223 if (imode != GET_MODE (to))
5224 target = gen_reg_rtx (imode);
5226 if (maybe_emit_unop_insn (icode, target, from,
5227 doing_unsigned ? UNSIGNED_FIX : FIX))
5229 if (target != to)
5230 convert_move (to, target, unsignedp);
5231 return;
5233 delete_insns_since (last);
5237 /* For an unsigned conversion, there is one more way to do it.
5238 If we have a signed conversion, we generate code that compares
5239 the real value to the largest representable positive number. If if
5240 is smaller, the conversion is done normally. Otherwise, subtract
5241 one plus the highest signed number, convert, and add it back.
5243 We only need to check all real modes, since we know we didn't find
5244 anything with a wider integer mode.
5246 This code used to extend FP value into mode wider than the destination.
5247 This is needed for decimal float modes which cannot accurately
5248 represent one plus the highest signed number of the same size, but
5249 not for binary modes. Consider, for instance conversion from SFmode
5250 into DImode.
5252 The hot path through the code is dealing with inputs smaller than 2^63
5253 and doing just the conversion, so there is no bits to lose.
5255 In the other path we know the value is positive in the range 2^63..2^64-1
5256 inclusive. (as for other input overflow happens and result is undefined)
5257 So we know that the most important bit set in mantissa corresponds to
5258 2^63. The subtraction of 2^63 should not generate any rounding as it
5259 simply clears out that bit. The rest is trivial. */
5261 if (unsignedp && GET_MODE_PRECISION (GET_MODE (to)) <= HOST_BITS_PER_WIDE_INT)
5262 for (fmode = GET_MODE (from); fmode != VOIDmode;
5263 fmode = GET_MODE_WIDER_MODE (fmode))
5264 if (CODE_FOR_nothing != can_fix_p (GET_MODE (to), fmode, 0, &must_trunc)
5265 && (!DECIMAL_FLOAT_MODE_P (fmode)
5266 || GET_MODE_BITSIZE (fmode) > GET_MODE_PRECISION (GET_MODE (to))))
5268 int bitsize;
5269 REAL_VALUE_TYPE offset;
5270 rtx limit;
5271 rtx_code_label *lab1, *lab2;
5272 rtx_insn *insn;
5274 bitsize = GET_MODE_PRECISION (GET_MODE (to));
5275 real_2expN (&offset, bitsize - 1, fmode);
5276 limit = CONST_DOUBLE_FROM_REAL_VALUE (offset, fmode);
5277 lab1 = gen_label_rtx ();
5278 lab2 = gen_label_rtx ();
5280 if (fmode != GET_MODE (from))
5281 from = convert_to_mode (fmode, from, 0);
5283 /* See if we need to do the subtraction. */
5284 do_pending_stack_adjust ();
5285 emit_cmp_and_jump_insns (from, limit, GE, NULL_RTX, GET_MODE (from),
5286 0, lab1);
5288 /* If not, do the signed "fix" and branch around fixup code. */
5289 expand_fix (to, from, 0);
5290 emit_jump_insn (targetm.gen_jump (lab2));
5291 emit_barrier ();
5293 /* Otherwise, subtract 2**(N-1), convert to signed number,
5294 then add 2**(N-1). Do the addition using XOR since this
5295 will often generate better code. */
5296 emit_label (lab1);
5297 target = expand_binop (GET_MODE (from), sub_optab, from, limit,
5298 NULL_RTX, 0, OPTAB_LIB_WIDEN);
5299 expand_fix (to, target, 0);
5300 target = expand_binop (GET_MODE (to), xor_optab, to,
5301 gen_int_mode
5302 ((HOST_WIDE_INT) 1 << (bitsize - 1),
5303 GET_MODE (to)),
5304 to, 1, OPTAB_LIB_WIDEN);
5306 if (target != to)
5307 emit_move_insn (to, target);
5309 emit_label (lab2);
5311 if (optab_handler (mov_optab, GET_MODE (to)) != CODE_FOR_nothing)
5313 /* Make a place for a REG_NOTE and add it. */
5314 insn = emit_move_insn (to, to);
5315 set_dst_reg_note (insn, REG_EQUAL,
5316 gen_rtx_fmt_e (UNSIGNED_FIX, GET_MODE (to),
5317 copy_rtx (from)),
5318 to);
5321 return;
5324 /* We can't do it with an insn, so use a library call. But first ensure
5325 that the mode of TO is at least as wide as SImode, since those are the
5326 only library calls we know about. */
5328 if (GET_MODE_PRECISION (GET_MODE (to)) < GET_MODE_PRECISION (SImode))
5330 target = gen_reg_rtx (SImode);
5332 expand_fix (target, from, unsignedp);
5334 else
5336 rtx_insn *insns;
5337 rtx value;
5338 rtx libfunc;
5340 convert_optab tab = unsignedp ? ufix_optab : sfix_optab;
5341 libfunc = convert_optab_libfunc (tab, GET_MODE (to), GET_MODE (from));
5342 gcc_assert (libfunc);
5344 start_sequence ();
5346 value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
5347 GET_MODE (to), 1, from,
5348 GET_MODE (from));
5349 insns = get_insns ();
5350 end_sequence ();
5352 emit_libcall_block (insns, target, value,
5353 gen_rtx_fmt_e (unsignedp ? UNSIGNED_FIX : FIX,
5354 GET_MODE (to), from));
5357 if (target != to)
5359 if (GET_MODE (to) == GET_MODE (target))
5360 emit_move_insn (to, target);
5361 else
5362 convert_move (to, target, 0);
5366 /* Generate code to convert FROM or TO a fixed-point.
5367 If UINTP is true, either TO or FROM is an unsigned integer.
5368 If SATP is true, we need to saturate the result. */
5370 void
5371 expand_fixed_convert (rtx to, rtx from, int uintp, int satp)
5373 machine_mode to_mode = GET_MODE (to);
5374 machine_mode from_mode = GET_MODE (from);
5375 convert_optab tab;
5376 enum rtx_code this_code;
5377 enum insn_code code;
5378 rtx_insn *insns;
5379 rtx value;
5380 rtx libfunc;
5382 if (to_mode == from_mode)
5384 emit_move_insn (to, from);
5385 return;
5388 if (uintp)
5390 tab = satp ? satfractuns_optab : fractuns_optab;
5391 this_code = satp ? UNSIGNED_SAT_FRACT : UNSIGNED_FRACT_CONVERT;
5393 else
5395 tab = satp ? satfract_optab : fract_optab;
5396 this_code = satp ? SAT_FRACT : FRACT_CONVERT;
5398 code = convert_optab_handler (tab, to_mode, from_mode);
5399 if (code != CODE_FOR_nothing)
5401 emit_unop_insn (code, to, from, this_code);
5402 return;
5405 libfunc = convert_optab_libfunc (tab, to_mode, from_mode);
5406 gcc_assert (libfunc);
5408 start_sequence ();
5409 value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST, to_mode,
5410 1, from, from_mode);
5411 insns = get_insns ();
5412 end_sequence ();
5414 emit_libcall_block (insns, to, value,
5415 gen_rtx_fmt_e (optab_to_code (tab), to_mode, from));
5418 /* Generate code to convert FROM to fixed point and store in TO. FROM
5419 must be floating point, TO must be signed. Use the conversion optab
5420 TAB to do the conversion. */
5422 bool
5423 expand_sfix_optab (rtx to, rtx from, convert_optab tab)
5425 enum insn_code icode;
5426 rtx target = to;
5427 machine_mode fmode, imode;
5429 /* We first try to find a pair of modes, one real and one integer, at
5430 least as wide as FROM and TO, respectively, in which we can open-code
5431 this conversion. If the integer mode is wider than the mode of TO,
5432 we can do the conversion either signed or unsigned. */
5434 for (fmode = GET_MODE (from); fmode != VOIDmode;
5435 fmode = GET_MODE_WIDER_MODE (fmode))
5436 for (imode = GET_MODE (to); imode != VOIDmode;
5437 imode = GET_MODE_WIDER_MODE (imode))
5439 icode = convert_optab_handler (tab, imode, fmode);
5440 if (icode != CODE_FOR_nothing)
5442 rtx_insn *last = get_last_insn ();
5443 if (fmode != GET_MODE (from))
5444 from = convert_to_mode (fmode, from, 0);
5446 if (imode != GET_MODE (to))
5447 target = gen_reg_rtx (imode);
5449 if (!maybe_emit_unop_insn (icode, target, from, UNKNOWN))
5451 delete_insns_since (last);
5452 continue;
5454 if (target != to)
5455 convert_move (to, target, 0);
5456 return true;
5460 return false;
5463 /* Report whether we have an instruction to perform the operation
5464 specified by CODE on operands of mode MODE. */
5466 have_insn_for (enum rtx_code code, machine_mode mode)
5468 return (code_to_optab (code)
5469 && (optab_handler (code_to_optab (code), mode)
5470 != CODE_FOR_nothing));
5473 /* Initialize the libfunc fields of an entire group of entries in some
5474 optab. Each entry is set equal to a string consisting of a leading
5475 pair of underscores followed by a generic operation name followed by
5476 a mode name (downshifted to lowercase) followed by a single character
5477 representing the number of operands for the given operation (which is
5478 usually one of the characters '2', '3', or '4').
5480 OPTABLE is the table in which libfunc fields are to be initialized.
5481 OPNAME is the generic (string) name of the operation.
5482 SUFFIX is the character which specifies the number of operands for
5483 the given generic operation.
5484 MODE is the mode to generate for.
5487 static void
5488 gen_libfunc (optab optable, const char *opname, int suffix,
5489 machine_mode mode)
5491 unsigned opname_len = strlen (opname);
5492 const char *mname = GET_MODE_NAME (mode);
5493 unsigned mname_len = strlen (mname);
5494 int prefix_len = targetm.libfunc_gnu_prefix ? 6 : 2;
5495 int len = prefix_len + opname_len + mname_len + 1 + 1;
5496 char *libfunc_name = XALLOCAVEC (char, len);
5497 char *p;
5498 const char *q;
5500 p = libfunc_name;
5501 *p++ = '_';
5502 *p++ = '_';
5503 if (targetm.libfunc_gnu_prefix)
5505 *p++ = 'g';
5506 *p++ = 'n';
5507 *p++ = 'u';
5508 *p++ = '_';
5510 for (q = opname; *q; )
5511 *p++ = *q++;
5512 for (q = mname; *q; q++)
5513 *p++ = TOLOWER (*q);
5514 *p++ = suffix;
5515 *p = '\0';
5517 set_optab_libfunc (optable, mode,
5518 ggc_alloc_string (libfunc_name, p - libfunc_name));
5521 /* Like gen_libfunc, but verify that integer operation is involved. */
5523 void
5524 gen_int_libfunc (optab optable, const char *opname, char suffix,
5525 machine_mode mode)
5527 int maxsize = 2 * BITS_PER_WORD;
5528 int minsize = BITS_PER_WORD;
5530 if (GET_MODE_CLASS (mode) != MODE_INT)
5531 return;
5532 if (maxsize < LONG_LONG_TYPE_SIZE)
5533 maxsize = LONG_LONG_TYPE_SIZE;
5534 if (minsize > INT_TYPE_SIZE
5535 && (trapv_binoptab_p (optable)
5536 || trapv_unoptab_p (optable)))
5537 minsize = INT_TYPE_SIZE;
5538 if (GET_MODE_BITSIZE (mode) < minsize
5539 || GET_MODE_BITSIZE (mode) > maxsize)
5540 return;
5541 gen_libfunc (optable, opname, suffix, mode);
5544 /* Like gen_libfunc, but verify that FP and set decimal prefix if needed. */
5546 void
5547 gen_fp_libfunc (optab optable, const char *opname, char suffix,
5548 machine_mode mode)
5550 char *dec_opname;
5552 if (GET_MODE_CLASS (mode) == MODE_FLOAT)
5553 gen_libfunc (optable, opname, suffix, mode);
5554 if (DECIMAL_FLOAT_MODE_P (mode))
5556 dec_opname = XALLOCAVEC (char, sizeof (DECIMAL_PREFIX) + strlen (opname));
5557 /* For BID support, change the name to have either a bid_ or dpd_ prefix
5558 depending on the low level floating format used. */
5559 memcpy (dec_opname, DECIMAL_PREFIX, sizeof (DECIMAL_PREFIX) - 1);
5560 strcpy (dec_opname + sizeof (DECIMAL_PREFIX) - 1, opname);
5561 gen_libfunc (optable, dec_opname, suffix, mode);
5565 /* Like gen_libfunc, but verify that fixed-point operation is involved. */
5567 void
5568 gen_fixed_libfunc (optab optable, const char *opname, char suffix,
5569 machine_mode mode)
5571 if (!ALL_FIXED_POINT_MODE_P (mode))
5572 return;
5573 gen_libfunc (optable, opname, suffix, mode);
5576 /* Like gen_libfunc, but verify that signed fixed-point operation is
5577 involved. */
5579 void
5580 gen_signed_fixed_libfunc (optab optable, const char *opname, char suffix,
5581 machine_mode mode)
5583 if (!SIGNED_FIXED_POINT_MODE_P (mode))
5584 return;
5585 gen_libfunc (optable, opname, suffix, mode);
5588 /* Like gen_libfunc, but verify that unsigned fixed-point operation is
5589 involved. */
5591 void
5592 gen_unsigned_fixed_libfunc (optab optable, const char *opname, char suffix,
5593 machine_mode mode)
5595 if (!UNSIGNED_FIXED_POINT_MODE_P (mode))
5596 return;
5597 gen_libfunc (optable, opname, suffix, mode);
5600 /* Like gen_libfunc, but verify that FP or INT operation is involved. */
5602 void
5603 gen_int_fp_libfunc (optab optable, const char *name, char suffix,
5604 machine_mode mode)
5606 if (DECIMAL_FLOAT_MODE_P (mode) || GET_MODE_CLASS (mode) == MODE_FLOAT)
5607 gen_fp_libfunc (optable, name, suffix, mode);
5608 if (INTEGRAL_MODE_P (mode))
5609 gen_int_libfunc (optable, name, suffix, mode);
5612 /* Like gen_libfunc, but verify that FP or INT operation is involved
5613 and add 'v' suffix for integer operation. */
5615 void
5616 gen_intv_fp_libfunc (optab optable, const char *name, char suffix,
5617 machine_mode mode)
5619 if (DECIMAL_FLOAT_MODE_P (mode) || GET_MODE_CLASS (mode) == MODE_FLOAT)
5620 gen_fp_libfunc (optable, name, suffix, mode);
5621 if (GET_MODE_CLASS (mode) == MODE_INT)
5623 int len = strlen (name);
5624 char *v_name = XALLOCAVEC (char, len + 2);
5625 strcpy (v_name, name);
5626 v_name[len] = 'v';
5627 v_name[len + 1] = 0;
5628 gen_int_libfunc (optable, v_name, suffix, mode);
5632 /* Like gen_libfunc, but verify that FP or INT or FIXED operation is
5633 involved. */
5635 void
5636 gen_int_fp_fixed_libfunc (optab optable, const char *name, char suffix,
5637 machine_mode mode)
5639 if (DECIMAL_FLOAT_MODE_P (mode) || GET_MODE_CLASS (mode) == MODE_FLOAT)
5640 gen_fp_libfunc (optable, name, suffix, mode);
5641 if (INTEGRAL_MODE_P (mode))
5642 gen_int_libfunc (optable, name, suffix, mode);
5643 if (ALL_FIXED_POINT_MODE_P (mode))
5644 gen_fixed_libfunc (optable, name, suffix, mode);
5647 /* Like gen_libfunc, but verify that FP or INT or signed FIXED operation is
5648 involved. */
5650 void
5651 gen_int_fp_signed_fixed_libfunc (optab optable, const char *name, char suffix,
5652 machine_mode mode)
5654 if (DECIMAL_FLOAT_MODE_P (mode) || GET_MODE_CLASS (mode) == MODE_FLOAT)
5655 gen_fp_libfunc (optable, name, suffix, mode);
5656 if (INTEGRAL_MODE_P (mode))
5657 gen_int_libfunc (optable, name, suffix, mode);
5658 if (SIGNED_FIXED_POINT_MODE_P (mode))
5659 gen_signed_fixed_libfunc (optable, name, suffix, mode);
5662 /* Like gen_libfunc, but verify that INT or FIXED operation is
5663 involved. */
5665 void
5666 gen_int_fixed_libfunc (optab optable, const char *name, char suffix,
5667 machine_mode mode)
5669 if (INTEGRAL_MODE_P (mode))
5670 gen_int_libfunc (optable, name, suffix, mode);
5671 if (ALL_FIXED_POINT_MODE_P (mode))
5672 gen_fixed_libfunc (optable, name, suffix, mode);
5675 /* Like gen_libfunc, but verify that INT or signed FIXED operation is
5676 involved. */
5678 void
5679 gen_int_signed_fixed_libfunc (optab optable, const char *name, char suffix,
5680 machine_mode mode)
5682 if (INTEGRAL_MODE_P (mode))
5683 gen_int_libfunc (optable, name, suffix, mode);
5684 if (SIGNED_FIXED_POINT_MODE_P (mode))
5685 gen_signed_fixed_libfunc (optable, name, suffix, mode);
5688 /* Like gen_libfunc, but verify that INT or unsigned FIXED operation is
5689 involved. */
5691 void
5692 gen_int_unsigned_fixed_libfunc (optab optable, const char *name, char suffix,
5693 machine_mode mode)
5695 if (INTEGRAL_MODE_P (mode))
5696 gen_int_libfunc (optable, name, suffix, mode);
5697 if (UNSIGNED_FIXED_POINT_MODE_P (mode))
5698 gen_unsigned_fixed_libfunc (optable, name, suffix, mode);
5701 /* Initialize the libfunc fields of an entire group of entries of an
5702 inter-mode-class conversion optab. The string formation rules are
5703 similar to the ones for init_libfuncs, above, but instead of having
5704 a mode name and an operand count these functions have two mode names
5705 and no operand count. */
5707 void
5708 gen_interclass_conv_libfunc (convert_optab tab,
5709 const char *opname,
5710 machine_mode tmode,
5711 machine_mode fmode)
5713 size_t opname_len = strlen (opname);
5714 size_t mname_len = 0;
5716 const char *fname, *tname;
5717 const char *q;
5718 int prefix_len = targetm.libfunc_gnu_prefix ? 6 : 2;
5719 char *libfunc_name, *suffix;
5720 char *nondec_name, *dec_name, *nondec_suffix, *dec_suffix;
5721 char *p;
5723 /* If this is a decimal conversion, add the current BID vs. DPD prefix that
5724 depends on which underlying decimal floating point format is used. */
5725 const size_t dec_len = sizeof (DECIMAL_PREFIX) - 1;
5727 mname_len = strlen (GET_MODE_NAME (tmode)) + strlen (GET_MODE_NAME (fmode));
5729 nondec_name = XALLOCAVEC (char, prefix_len + opname_len + mname_len + 1 + 1);
5730 nondec_name[0] = '_';
5731 nondec_name[1] = '_';
5732 if (targetm.libfunc_gnu_prefix)
5734 nondec_name[2] = 'g';
5735 nondec_name[3] = 'n';
5736 nondec_name[4] = 'u';
5737 nondec_name[5] = '_';
5740 memcpy (&nondec_name[prefix_len], opname, opname_len);
5741 nondec_suffix = nondec_name + opname_len + prefix_len;
5743 dec_name = XALLOCAVEC (char, 2 + dec_len + opname_len + mname_len + 1 + 1);
5744 dec_name[0] = '_';
5745 dec_name[1] = '_';
5746 memcpy (&dec_name[2], DECIMAL_PREFIX, dec_len);
5747 memcpy (&dec_name[2+dec_len], opname, opname_len);
5748 dec_suffix = dec_name + dec_len + opname_len + 2;
5750 fname = GET_MODE_NAME (fmode);
5751 tname = GET_MODE_NAME (tmode);
5753 if (DECIMAL_FLOAT_MODE_P (fmode) || DECIMAL_FLOAT_MODE_P (tmode))
5755 libfunc_name = dec_name;
5756 suffix = dec_suffix;
5758 else
5760 libfunc_name = nondec_name;
5761 suffix = nondec_suffix;
5764 p = suffix;
5765 for (q = fname; *q; p++, q++)
5766 *p = TOLOWER (*q);
5767 for (q = tname; *q; p++, q++)
5768 *p = TOLOWER (*q);
5770 *p = '\0';
5772 set_conv_libfunc (tab, tmode, fmode,
5773 ggc_alloc_string (libfunc_name, p - libfunc_name));
5776 /* Same as gen_interclass_conv_libfunc but verify that we are producing
5777 int->fp conversion. */
5779 void
5780 gen_int_to_fp_conv_libfunc (convert_optab tab,
5781 const char *opname,
5782 machine_mode tmode,
5783 machine_mode fmode)
5785 if (GET_MODE_CLASS (fmode) != MODE_INT)
5786 return;
5787 if (GET_MODE_CLASS (tmode) != MODE_FLOAT && !DECIMAL_FLOAT_MODE_P (tmode))
5788 return;
5789 gen_interclass_conv_libfunc (tab, opname, tmode, fmode);
5792 /* ufloat_optab is special by using floatun for FP and floatuns decimal fp
5793 naming scheme. */
5795 void
5796 gen_ufloat_conv_libfunc (convert_optab tab,
5797 const char *opname ATTRIBUTE_UNUSED,
5798 machine_mode tmode,
5799 machine_mode fmode)
5801 if (DECIMAL_FLOAT_MODE_P (tmode))
5802 gen_int_to_fp_conv_libfunc (tab, "floatuns", tmode, fmode);
5803 else
5804 gen_int_to_fp_conv_libfunc (tab, "floatun", tmode, fmode);
5807 /* Same as gen_interclass_conv_libfunc but verify that we are producing
5808 fp->int conversion. */
5810 void
5811 gen_int_to_fp_nondecimal_conv_libfunc (convert_optab tab,
5812 const char *opname,
5813 machine_mode tmode,
5814 machine_mode fmode)
5816 if (GET_MODE_CLASS (fmode) != MODE_INT)
5817 return;
5818 if (GET_MODE_CLASS (tmode) != MODE_FLOAT)
5819 return;
5820 gen_interclass_conv_libfunc (tab, opname, tmode, fmode);
5823 /* Same as gen_interclass_conv_libfunc but verify that we are producing
5824 fp->int conversion with no decimal floating point involved. */
5826 void
5827 gen_fp_to_int_conv_libfunc (convert_optab tab,
5828 const char *opname,
5829 machine_mode tmode,
5830 machine_mode fmode)
5832 if (GET_MODE_CLASS (fmode) != MODE_FLOAT && !DECIMAL_FLOAT_MODE_P (fmode))
5833 return;
5834 if (GET_MODE_CLASS (tmode) != MODE_INT)
5835 return;
5836 gen_interclass_conv_libfunc (tab, opname, tmode, fmode);
5839 /* Initialize the libfunc fields of an of an intra-mode-class conversion optab.
5840 The string formation rules are
5841 similar to the ones for init_libfunc, above. */
5843 void
5844 gen_intraclass_conv_libfunc (convert_optab tab, const char *opname,
5845 machine_mode tmode, machine_mode fmode)
5847 size_t opname_len = strlen (opname);
5848 size_t mname_len = 0;
5850 const char *fname, *tname;
5851 const char *q;
5852 int prefix_len = targetm.libfunc_gnu_prefix ? 6 : 2;
5853 char *nondec_name, *dec_name, *nondec_suffix, *dec_suffix;
5854 char *libfunc_name, *suffix;
5855 char *p;
5857 /* If this is a decimal conversion, add the current BID vs. DPD prefix that
5858 depends on which underlying decimal floating point format is used. */
5859 const size_t dec_len = sizeof (DECIMAL_PREFIX) - 1;
5861 mname_len = strlen (GET_MODE_NAME (tmode)) + strlen (GET_MODE_NAME (fmode));
5863 nondec_name = XALLOCAVEC (char, 2 + opname_len + mname_len + 1 + 1);
5864 nondec_name[0] = '_';
5865 nondec_name[1] = '_';
5866 if (targetm.libfunc_gnu_prefix)
5868 nondec_name[2] = 'g';
5869 nondec_name[3] = 'n';
5870 nondec_name[4] = 'u';
5871 nondec_name[5] = '_';
5873 memcpy (&nondec_name[prefix_len], opname, opname_len);
5874 nondec_suffix = nondec_name + opname_len + prefix_len;
5876 dec_name = XALLOCAVEC (char, 2 + dec_len + opname_len + mname_len + 1 + 1);
5877 dec_name[0] = '_';
5878 dec_name[1] = '_';
5879 memcpy (&dec_name[2], DECIMAL_PREFIX, dec_len);
5880 memcpy (&dec_name[2 + dec_len], opname, opname_len);
5881 dec_suffix = dec_name + dec_len + opname_len + 2;
5883 fname = GET_MODE_NAME (fmode);
5884 tname = GET_MODE_NAME (tmode);
5886 if (DECIMAL_FLOAT_MODE_P (fmode) || DECIMAL_FLOAT_MODE_P (tmode))
5888 libfunc_name = dec_name;
5889 suffix = dec_suffix;
5891 else
5893 libfunc_name = nondec_name;
5894 suffix = nondec_suffix;
5897 p = suffix;
5898 for (q = fname; *q; p++, q++)
5899 *p = TOLOWER (*q);
5900 for (q = tname; *q; p++, q++)
5901 *p = TOLOWER (*q);
5903 *p++ = '2';
5904 *p = '\0';
5906 set_conv_libfunc (tab, tmode, fmode,
5907 ggc_alloc_string (libfunc_name, p - libfunc_name));
5910 /* Pick proper libcall for trunc_optab. We need to chose if we do
5911 truncation or extension and interclass or intraclass. */
5913 void
5914 gen_trunc_conv_libfunc (convert_optab tab,
5915 const char *opname,
5916 machine_mode tmode,
5917 machine_mode fmode)
5919 if (GET_MODE_CLASS (tmode) != MODE_FLOAT && !DECIMAL_FLOAT_MODE_P (tmode))
5920 return;
5921 if (GET_MODE_CLASS (fmode) != MODE_FLOAT && !DECIMAL_FLOAT_MODE_P (fmode))
5922 return;
5923 if (tmode == fmode)
5924 return;
5926 if ((GET_MODE_CLASS (tmode) == MODE_FLOAT && DECIMAL_FLOAT_MODE_P (fmode))
5927 || (GET_MODE_CLASS (fmode) == MODE_FLOAT && DECIMAL_FLOAT_MODE_P (tmode)))
5928 gen_interclass_conv_libfunc (tab, opname, tmode, fmode);
5930 if (GET_MODE_PRECISION (fmode) <= GET_MODE_PRECISION (tmode))
5931 return;
5933 if ((GET_MODE_CLASS (tmode) == MODE_FLOAT
5934 && GET_MODE_CLASS (fmode) == MODE_FLOAT)
5935 || (DECIMAL_FLOAT_MODE_P (fmode) && DECIMAL_FLOAT_MODE_P (tmode)))
5936 gen_intraclass_conv_libfunc (tab, opname, tmode, fmode);
5939 /* Pick proper libcall for extend_optab. We need to chose if we do
5940 truncation or extension and interclass or intraclass. */
5942 void
5943 gen_extend_conv_libfunc (convert_optab tab,
5944 const char *opname ATTRIBUTE_UNUSED,
5945 machine_mode tmode,
5946 machine_mode fmode)
5948 if (GET_MODE_CLASS (tmode) != MODE_FLOAT && !DECIMAL_FLOAT_MODE_P (tmode))
5949 return;
5950 if (GET_MODE_CLASS (fmode) != MODE_FLOAT && !DECIMAL_FLOAT_MODE_P (fmode))
5951 return;
5952 if (tmode == fmode)
5953 return;
5955 if ((GET_MODE_CLASS (tmode) == MODE_FLOAT && DECIMAL_FLOAT_MODE_P (fmode))
5956 || (GET_MODE_CLASS (fmode) == MODE_FLOAT && DECIMAL_FLOAT_MODE_P (tmode)))
5957 gen_interclass_conv_libfunc (tab, opname, tmode, fmode);
5959 if (GET_MODE_PRECISION (fmode) > GET_MODE_PRECISION (tmode))
5960 return;
5962 if ((GET_MODE_CLASS (tmode) == MODE_FLOAT
5963 && GET_MODE_CLASS (fmode) == MODE_FLOAT)
5964 || (DECIMAL_FLOAT_MODE_P (fmode) && DECIMAL_FLOAT_MODE_P (tmode)))
5965 gen_intraclass_conv_libfunc (tab, opname, tmode, fmode);
5968 /* Pick proper libcall for fract_optab. We need to chose if we do
5969 interclass or intraclass. */
5971 void
5972 gen_fract_conv_libfunc (convert_optab tab,
5973 const char *opname,
5974 machine_mode tmode,
5975 machine_mode fmode)
5977 if (tmode == fmode)
5978 return;
5979 if (!(ALL_FIXED_POINT_MODE_P (tmode) || ALL_FIXED_POINT_MODE_P (fmode)))
5980 return;
5982 if (GET_MODE_CLASS (tmode) == GET_MODE_CLASS (fmode))
5983 gen_intraclass_conv_libfunc (tab, opname, tmode, fmode);
5984 else
5985 gen_interclass_conv_libfunc (tab, opname, tmode, fmode);
5988 /* Pick proper libcall for fractuns_optab. */
5990 void
5991 gen_fractuns_conv_libfunc (convert_optab tab,
5992 const char *opname,
5993 machine_mode tmode,
5994 machine_mode fmode)
5996 if (tmode == fmode)
5997 return;
5998 /* One mode must be a fixed-point mode, and the other must be an integer
5999 mode. */
6000 if (!((ALL_FIXED_POINT_MODE_P (tmode) && GET_MODE_CLASS (fmode) == MODE_INT)
6001 || (ALL_FIXED_POINT_MODE_P (fmode)
6002 && GET_MODE_CLASS (tmode) == MODE_INT)))
6003 return;
6005 gen_interclass_conv_libfunc (tab, opname, tmode, fmode);
6008 /* Pick proper libcall for satfract_optab. We need to chose if we do
6009 interclass or intraclass. */
6011 void
6012 gen_satfract_conv_libfunc (convert_optab tab,
6013 const char *opname,
6014 machine_mode tmode,
6015 machine_mode fmode)
6017 if (tmode == fmode)
6018 return;
6019 /* TMODE must be a fixed-point mode. */
6020 if (!ALL_FIXED_POINT_MODE_P (tmode))
6021 return;
6023 if (GET_MODE_CLASS (tmode) == GET_MODE_CLASS (fmode))
6024 gen_intraclass_conv_libfunc (tab, opname, tmode, fmode);
6025 else
6026 gen_interclass_conv_libfunc (tab, opname, tmode, fmode);
6029 /* Pick proper libcall for satfractuns_optab. */
6031 void
6032 gen_satfractuns_conv_libfunc (convert_optab tab,
6033 const char *opname,
6034 machine_mode tmode,
6035 machine_mode fmode)
6037 if (tmode == fmode)
6038 return;
6039 /* TMODE must be a fixed-point mode, and FMODE must be an integer mode. */
6040 if (!(ALL_FIXED_POINT_MODE_P (tmode) && GET_MODE_CLASS (fmode) == MODE_INT))
6041 return;
6043 gen_interclass_conv_libfunc (tab, opname, tmode, fmode);
6046 /* Hashtable callbacks for libfunc_decls. */
6048 struct libfunc_decl_hasher : ggc_ptr_hash<tree_node>
6050 static hashval_t
6051 hash (tree entry)
6053 return IDENTIFIER_HASH_VALUE (DECL_NAME (entry));
6056 static bool
6057 equal (tree decl, tree name)
6059 return DECL_NAME (decl) == name;
6063 /* A table of previously-created libfuncs, hashed by name. */
6064 static GTY (()) hash_table<libfunc_decl_hasher> *libfunc_decls;
6066 /* Build a decl for a libfunc named NAME. */
6068 tree
6069 build_libfunc_function (const char *name)
6071 tree decl = build_decl (UNKNOWN_LOCATION, FUNCTION_DECL,
6072 get_identifier (name),
6073 build_function_type (integer_type_node, NULL_TREE));
6074 /* ??? We don't have any type information except for this is
6075 a function. Pretend this is "int foo()". */
6076 DECL_ARTIFICIAL (decl) = 1;
6077 DECL_EXTERNAL (decl) = 1;
6078 TREE_PUBLIC (decl) = 1;
6079 gcc_assert (DECL_ASSEMBLER_NAME (decl));
6081 /* Zap the nonsensical SYMBOL_REF_DECL for this. What we're left with
6082 are the flags assigned by targetm.encode_section_info. */
6083 SET_SYMBOL_REF_DECL (XEXP (DECL_RTL (decl), 0), NULL);
6085 return decl;
6089 init_one_libfunc (const char *name)
6091 tree id, decl;
6092 hashval_t hash;
6094 if (libfunc_decls == NULL)
6095 libfunc_decls = hash_table<libfunc_decl_hasher>::create_ggc (37);
6097 /* See if we have already created a libfunc decl for this function. */
6098 id = get_identifier (name);
6099 hash = IDENTIFIER_HASH_VALUE (id);
6100 tree *slot = libfunc_decls->find_slot_with_hash (id, hash, INSERT);
6101 decl = *slot;
6102 if (decl == NULL)
6104 /* Create a new decl, so that it can be passed to
6105 targetm.encode_section_info. */
6106 decl = build_libfunc_function (name);
6107 *slot = decl;
6109 return XEXP (DECL_RTL (decl), 0);
6112 /* Adjust the assembler name of libfunc NAME to ASMSPEC. */
6115 set_user_assembler_libfunc (const char *name, const char *asmspec)
6117 tree id, decl;
6118 hashval_t hash;
6120 id = get_identifier (name);
6121 hash = IDENTIFIER_HASH_VALUE (id);
6122 tree *slot = libfunc_decls->find_slot_with_hash (id, hash, NO_INSERT);
6123 gcc_assert (slot);
6124 decl = (tree) *slot;
6125 set_user_assembler_name (decl, asmspec);
6126 return XEXP (DECL_RTL (decl), 0);
6129 /* Call this to reset the function entry for one optab (OPTABLE) in mode
6130 MODE to NAME, which should be either 0 or a string constant. */
6131 void
6132 set_optab_libfunc (optab op, machine_mode mode, const char *name)
6134 rtx val;
6135 struct libfunc_entry e;
6136 struct libfunc_entry **slot;
6138 e.op = op;
6139 e.mode1 = mode;
6140 e.mode2 = VOIDmode;
6142 if (name)
6143 val = init_one_libfunc (name);
6144 else
6145 val = 0;
6146 slot = libfunc_hash->find_slot (&e, INSERT);
6147 if (*slot == NULL)
6148 *slot = ggc_alloc<libfunc_entry> ();
6149 (*slot)->op = op;
6150 (*slot)->mode1 = mode;
6151 (*slot)->mode2 = VOIDmode;
6152 (*slot)->libfunc = val;
6155 /* Call this to reset the function entry for one conversion optab
6156 (OPTABLE) from mode FMODE to mode TMODE to NAME, which should be
6157 either 0 or a string constant. */
6158 void
6159 set_conv_libfunc (convert_optab optab, machine_mode tmode,
6160 machine_mode fmode, const char *name)
6162 rtx val;
6163 struct libfunc_entry e;
6164 struct libfunc_entry **slot;
6166 e.op = optab;
6167 e.mode1 = tmode;
6168 e.mode2 = fmode;
6170 if (name)
6171 val = init_one_libfunc (name);
6172 else
6173 val = 0;
6174 slot = libfunc_hash->find_slot (&e, INSERT);
6175 if (*slot == NULL)
6176 *slot = ggc_alloc<libfunc_entry> ();
6177 (*slot)->op = optab;
6178 (*slot)->mode1 = tmode;
6179 (*slot)->mode2 = fmode;
6180 (*slot)->libfunc = val;
6183 /* Call this to initialize the contents of the optabs
6184 appropriately for the current target machine. */
6186 void
6187 init_optabs (void)
6189 if (libfunc_hash)
6190 libfunc_hash->empty ();
6191 else
6192 libfunc_hash = hash_table<libfunc_hasher>::create_ggc (10);
6194 /* Fill in the optabs with the insns we support. */
6195 init_all_optabs (this_fn_optabs);
6197 /* The ffs function operates on `int'. Fall back on it if we do not
6198 have a libgcc2 function for that width. */
6199 if (INT_TYPE_SIZE < BITS_PER_WORD)
6200 set_optab_libfunc (ffs_optab, mode_for_size (INT_TYPE_SIZE, MODE_INT, 0),
6201 "ffs");
6203 /* Explicitly initialize the bswap libfuncs since we need them to be
6204 valid for things other than word_mode. */
6205 if (targetm.libfunc_gnu_prefix)
6207 set_optab_libfunc (bswap_optab, SImode, "__gnu_bswapsi2");
6208 set_optab_libfunc (bswap_optab, DImode, "__gnu_bswapdi2");
6210 else
6212 set_optab_libfunc (bswap_optab, SImode, "__bswapsi2");
6213 set_optab_libfunc (bswap_optab, DImode, "__bswapdi2");
6216 /* Use cabs for double complex abs, since systems generally have cabs.
6217 Don't define any libcall for float complex, so that cabs will be used. */
6218 if (complex_double_type_node)
6219 set_optab_libfunc (abs_optab, TYPE_MODE (complex_double_type_node),
6220 "cabs");
6222 abort_libfunc = init_one_libfunc ("abort");
6223 memcpy_libfunc = init_one_libfunc ("memcpy");
6224 memmove_libfunc = init_one_libfunc ("memmove");
6225 memcmp_libfunc = init_one_libfunc ("memcmp");
6226 memset_libfunc = init_one_libfunc ("memset");
6227 setbits_libfunc = init_one_libfunc ("__setbits");
6229 #ifndef DONT_USE_BUILTIN_SETJMP
6230 setjmp_libfunc = init_one_libfunc ("__builtin_setjmp");
6231 longjmp_libfunc = init_one_libfunc ("__builtin_longjmp");
6232 #else
6233 setjmp_libfunc = init_one_libfunc ("setjmp");
6234 longjmp_libfunc = init_one_libfunc ("longjmp");
6235 #endif
6236 unwind_sjlj_register_libfunc = init_one_libfunc ("_Unwind_SjLj_Register");
6237 unwind_sjlj_unregister_libfunc
6238 = init_one_libfunc ("_Unwind_SjLj_Unregister");
6240 /* For function entry/exit instrumentation. */
6241 profile_function_entry_libfunc
6242 = init_one_libfunc ("__cyg_profile_func_enter");
6243 profile_function_exit_libfunc
6244 = init_one_libfunc ("__cyg_profile_func_exit");
6246 gcov_flush_libfunc = init_one_libfunc ("__gcov_flush");
6248 /* Allow the target to add more libcalls or rename some, etc. */
6249 targetm.init_libfuncs ();
6252 /* Use the current target and options to initialize
6253 TREE_OPTIMIZATION_OPTABS (OPTNODE). */
6255 void
6256 init_tree_optimization_optabs (tree optnode)
6258 /* Quick exit if we have already computed optabs for this target. */
6259 if (TREE_OPTIMIZATION_BASE_OPTABS (optnode) == this_target_optabs)
6260 return;
6262 /* Forget any previous information and set up for the current target. */
6263 TREE_OPTIMIZATION_BASE_OPTABS (optnode) = this_target_optabs;
6264 struct target_optabs *tmp_optabs = (struct target_optabs *)
6265 TREE_OPTIMIZATION_OPTABS (optnode);
6266 if (tmp_optabs)
6267 memset (tmp_optabs, 0, sizeof (struct target_optabs));
6268 else
6269 tmp_optabs = ggc_alloc<target_optabs> ();
6271 /* Generate a new set of optabs into tmp_optabs. */
6272 init_all_optabs (tmp_optabs);
6274 /* If the optabs changed, record it. */
6275 if (memcmp (tmp_optabs, this_target_optabs, sizeof (struct target_optabs)))
6276 TREE_OPTIMIZATION_OPTABS (optnode) = tmp_optabs;
6277 else
6279 TREE_OPTIMIZATION_OPTABS (optnode) = NULL;
6280 ggc_free (tmp_optabs);
6284 /* A helper function for init_sync_libfuncs. Using the basename BASE,
6285 install libfuncs into TAB for BASE_N for 1 <= N <= MAX. */
6287 static void
6288 init_sync_libfuncs_1 (optab tab, const char *base, int max)
6290 machine_mode mode;
6291 char buf[64];
6292 size_t len = strlen (base);
6293 int i;
6295 gcc_assert (max <= 8);
6296 gcc_assert (len + 3 < sizeof (buf));
6298 memcpy (buf, base, len);
6299 buf[len] = '_';
6300 buf[len + 1] = '0';
6301 buf[len + 2] = '\0';
6303 mode = QImode;
6304 for (i = 1; i <= max; i *= 2)
6306 buf[len + 1] = '0' + i;
6307 set_optab_libfunc (tab, mode, buf);
6308 mode = GET_MODE_2XWIDER_MODE (mode);
6312 void
6313 init_sync_libfuncs (int max)
6315 if (!flag_sync_libcalls)
6316 return;
6318 init_sync_libfuncs_1 (sync_compare_and_swap_optab,
6319 "__sync_val_compare_and_swap", max);
6320 init_sync_libfuncs_1 (sync_lock_test_and_set_optab,
6321 "__sync_lock_test_and_set", max);
6323 init_sync_libfuncs_1 (sync_old_add_optab, "__sync_fetch_and_add", max);
6324 init_sync_libfuncs_1 (sync_old_sub_optab, "__sync_fetch_and_sub", max);
6325 init_sync_libfuncs_1 (sync_old_ior_optab, "__sync_fetch_and_or", max);
6326 init_sync_libfuncs_1 (sync_old_and_optab, "__sync_fetch_and_and", max);
6327 init_sync_libfuncs_1 (sync_old_xor_optab, "__sync_fetch_and_xor", max);
6328 init_sync_libfuncs_1 (sync_old_nand_optab, "__sync_fetch_and_nand", max);
6330 init_sync_libfuncs_1 (sync_new_add_optab, "__sync_add_and_fetch", max);
6331 init_sync_libfuncs_1 (sync_new_sub_optab, "__sync_sub_and_fetch", max);
6332 init_sync_libfuncs_1 (sync_new_ior_optab, "__sync_or_and_fetch", max);
6333 init_sync_libfuncs_1 (sync_new_and_optab, "__sync_and_and_fetch", max);
6334 init_sync_libfuncs_1 (sync_new_xor_optab, "__sync_xor_and_fetch", max);
6335 init_sync_libfuncs_1 (sync_new_nand_optab, "__sync_nand_and_fetch", max);
6338 /* Print information about the current contents of the optabs on
6339 STDERR. */
6341 DEBUG_FUNCTION void
6342 debug_optab_libfuncs (void)
6344 int i, j, k;
6346 /* Dump the arithmetic optabs. */
6347 for (i = FIRST_NORM_OPTAB; i <= LAST_NORMLIB_OPTAB; ++i)
6348 for (j = 0; j < NUM_MACHINE_MODES; ++j)
6350 rtx l = optab_libfunc ((optab) i, (machine_mode) j);
6351 if (l)
6353 gcc_assert (GET_CODE (l) == SYMBOL_REF);
6354 fprintf (stderr, "%s\t%s:\t%s\n",
6355 GET_RTX_NAME (optab_to_code ((optab) i)),
6356 GET_MODE_NAME (j),
6357 XSTR (l, 0));
6361 /* Dump the conversion optabs. */
6362 for (i = FIRST_CONV_OPTAB; i <= LAST_CONVLIB_OPTAB; ++i)
6363 for (j = 0; j < NUM_MACHINE_MODES; ++j)
6364 for (k = 0; k < NUM_MACHINE_MODES; ++k)
6366 rtx l = convert_optab_libfunc ((optab) i, (machine_mode) j,
6367 (machine_mode) k);
6368 if (l)
6370 gcc_assert (GET_CODE (l) == SYMBOL_REF);
6371 fprintf (stderr, "%s\t%s\t%s:\t%s\n",
6372 GET_RTX_NAME (optab_to_code ((optab) i)),
6373 GET_MODE_NAME (j),
6374 GET_MODE_NAME (k),
6375 XSTR (l, 0));
6381 /* Generate insns to trap with code TCODE if OP1 and OP2 satisfy condition
6382 CODE. Return 0 on failure. */
6384 rtx_insn *
6385 gen_cond_trap (enum rtx_code code, rtx op1, rtx op2, rtx tcode)
6387 machine_mode mode = GET_MODE (op1);
6388 enum insn_code icode;
6389 rtx_insn *insn;
6390 rtx trap_rtx;
6392 if (mode == VOIDmode)
6393 return 0;
6395 icode = optab_handler (ctrap_optab, mode);
6396 if (icode == CODE_FOR_nothing)
6397 return 0;
6399 /* Some targets only accept a zero trap code. */
6400 if (!insn_operand_matches (icode, 3, tcode))
6401 return 0;
6403 do_pending_stack_adjust ();
6404 start_sequence ();
6405 prepare_cmp_insn (op1, op2, code, NULL_RTX, false, OPTAB_DIRECT,
6406 &trap_rtx, &mode);
6407 if (!trap_rtx)
6408 insn = NULL;
6409 else
6410 insn = GEN_FCN (icode) (trap_rtx, XEXP (trap_rtx, 0), XEXP (trap_rtx, 1),
6411 tcode);
6413 /* If that failed, then give up. */
6414 if (insn == 0)
6416 end_sequence ();
6417 return 0;
6420 emit_insn (insn);
6421 insn = get_insns ();
6422 end_sequence ();
6423 return insn;
6426 /* Return rtx code for TCODE. Use UNSIGNEDP to select signed
6427 or unsigned operation code. */
6429 enum rtx_code
6430 get_rtx_code (enum tree_code tcode, bool unsignedp)
6432 enum rtx_code code;
6433 switch (tcode)
6435 case EQ_EXPR:
6436 code = EQ;
6437 break;
6438 case NE_EXPR:
6439 code = NE;
6440 break;
6441 case LT_EXPR:
6442 code = unsignedp ? LTU : LT;
6443 break;
6444 case LE_EXPR:
6445 code = unsignedp ? LEU : LE;
6446 break;
6447 case GT_EXPR:
6448 code = unsignedp ? GTU : GT;
6449 break;
6450 case GE_EXPR:
6451 code = unsignedp ? GEU : GE;
6452 break;
6454 case UNORDERED_EXPR:
6455 code = UNORDERED;
6456 break;
6457 case ORDERED_EXPR:
6458 code = ORDERED;
6459 break;
6460 case UNLT_EXPR:
6461 code = UNLT;
6462 break;
6463 case UNLE_EXPR:
6464 code = UNLE;
6465 break;
6466 case UNGT_EXPR:
6467 code = UNGT;
6468 break;
6469 case UNGE_EXPR:
6470 code = UNGE;
6471 break;
6472 case UNEQ_EXPR:
6473 code = UNEQ;
6474 break;
6475 case LTGT_EXPR:
6476 code = LTGT;
6477 break;
6479 case BIT_AND_EXPR:
6480 code = AND;
6481 break;
6483 case BIT_IOR_EXPR:
6484 code = IOR;
6485 break;
6487 default:
6488 gcc_unreachable ();
6490 return code;
6493 /* Return comparison rtx for COND. Use UNSIGNEDP to select signed or
6494 unsigned operators. Do not generate compare instruction. */
6496 static rtx
6497 vector_compare_rtx (enum tree_code tcode, tree t_op0, tree t_op1,
6498 bool unsignedp, enum insn_code icode)
6500 struct expand_operand ops[2];
6501 rtx rtx_op0, rtx_op1;
6502 machine_mode m0, m1;
6503 enum rtx_code rcode = get_rtx_code (tcode, unsignedp);
6505 gcc_assert (TREE_CODE_CLASS (tcode) == tcc_comparison);
6507 /* Expand operands. For vector types with scalar modes, e.g. where int64x1_t
6508 has mode DImode, this can produce a constant RTX of mode VOIDmode; in such
6509 cases, use the original mode. */
6510 rtx_op0 = expand_expr (t_op0, NULL_RTX, TYPE_MODE (TREE_TYPE (t_op0)),
6511 EXPAND_STACK_PARM);
6512 m0 = GET_MODE (rtx_op0);
6513 if (m0 == VOIDmode)
6514 m0 = TYPE_MODE (TREE_TYPE (t_op0));
6516 rtx_op1 = expand_expr (t_op1, NULL_RTX, TYPE_MODE (TREE_TYPE (t_op1)),
6517 EXPAND_STACK_PARM);
6518 m1 = GET_MODE (rtx_op1);
6519 if (m1 == VOIDmode)
6520 m1 = TYPE_MODE (TREE_TYPE (t_op1));
6522 create_input_operand (&ops[0], rtx_op0, m0);
6523 create_input_operand (&ops[1], rtx_op1, m1);
6524 if (!maybe_legitimize_operands (icode, 4, 2, ops))
6525 gcc_unreachable ();
6526 return gen_rtx_fmt_ee (rcode, VOIDmode, ops[0].value, ops[1].value);
6529 /* Return true if VEC_PERM_EXPR of arbitrary input vectors can be expanded using
6530 SIMD extensions of the CPU. SEL may be NULL, which stands for an unknown
6531 constant. Note that additional permutations representing whole-vector shifts
6532 may also be handled via the vec_shr optab, but only where the second input
6533 vector is entirely constant zeroes; this case is not dealt with here. */
6535 bool
6536 can_vec_perm_p (machine_mode mode, bool variable,
6537 const unsigned char *sel)
6539 machine_mode qimode;
6541 /* If the target doesn't implement a vector mode for the vector type,
6542 then no operations are supported. */
6543 if (!VECTOR_MODE_P (mode))
6544 return false;
6546 if (!variable)
6548 if (direct_optab_handler (vec_perm_const_optab, mode) != CODE_FOR_nothing
6549 && (sel == NULL
6550 || targetm.vectorize.vec_perm_const_ok == NULL
6551 || targetm.vectorize.vec_perm_const_ok (mode, sel)))
6552 return true;
6555 if (direct_optab_handler (vec_perm_optab, mode) != CODE_FOR_nothing)
6556 return true;
6558 /* We allow fallback to a QI vector mode, and adjust the mask. */
6559 if (GET_MODE_INNER (mode) == QImode)
6560 return false;
6561 qimode = mode_for_vector (QImode, GET_MODE_SIZE (mode));
6562 if (!VECTOR_MODE_P (qimode))
6563 return false;
6565 /* ??? For completeness, we ought to check the QImode version of
6566 vec_perm_const_optab. But all users of this implicit lowering
6567 feature implement the variable vec_perm_optab. */
6568 if (direct_optab_handler (vec_perm_optab, qimode) == CODE_FOR_nothing)
6569 return false;
6571 /* In order to support the lowering of variable permutations,
6572 we need to support shifts and adds. */
6573 if (variable)
6575 if (GET_MODE_UNIT_SIZE (mode) > 2
6576 && optab_handler (ashl_optab, mode) == CODE_FOR_nothing
6577 && optab_handler (vashl_optab, mode) == CODE_FOR_nothing)
6578 return false;
6579 if (optab_handler (add_optab, qimode) == CODE_FOR_nothing)
6580 return false;
6583 return true;
6586 /* Checks if vec_perm mask SEL is a constant equivalent to a shift of the first
6587 vec_perm operand, assuming the second operand is a constant vector of zeroes.
6588 Return the shift distance in bits if so, or NULL_RTX if the vec_perm is not a
6589 shift. */
6590 static rtx
6591 shift_amt_for_vec_perm_mask (rtx sel)
6593 unsigned int i, first, nelt = GET_MODE_NUNITS (GET_MODE (sel));
6594 unsigned int bitsize = GET_MODE_BITSIZE (GET_MODE_INNER (GET_MODE (sel)));
6596 if (GET_CODE (sel) != CONST_VECTOR)
6597 return NULL_RTX;
6599 first = INTVAL (CONST_VECTOR_ELT (sel, 0));
6600 if (first >= 2*nelt)
6601 return NULL_RTX;
6602 for (i = 1; i < nelt; i++)
6604 int idx = INTVAL (CONST_VECTOR_ELT (sel, i));
6605 unsigned int expected = (i + first) & (2 * nelt - 1);
6606 /* Indices into the second vector are all equivalent. */
6607 if (idx < 0 || (MIN (nelt, (unsigned) idx) != MIN (nelt, expected)))
6608 return NULL_RTX;
6611 return GEN_INT (first * bitsize);
6614 /* A subroutine of expand_vec_perm for expanding one vec_perm insn. */
6616 static rtx
6617 expand_vec_perm_1 (enum insn_code icode, rtx target,
6618 rtx v0, rtx v1, rtx sel)
6620 machine_mode tmode = GET_MODE (target);
6621 machine_mode smode = GET_MODE (sel);
6622 struct expand_operand ops[4];
6624 create_output_operand (&ops[0], target, tmode);
6625 create_input_operand (&ops[3], sel, smode);
6627 /* Make an effort to preserve v0 == v1. The target expander is able to
6628 rely on this to determine if we're permuting a single input operand. */
6629 if (rtx_equal_p (v0, v1))
6631 if (!insn_operand_matches (icode, 1, v0))
6632 v0 = force_reg (tmode, v0);
6633 gcc_checking_assert (insn_operand_matches (icode, 1, v0));
6634 gcc_checking_assert (insn_operand_matches (icode, 2, v0));
6636 create_fixed_operand (&ops[1], v0);
6637 create_fixed_operand (&ops[2], v0);
6639 else
6641 create_input_operand (&ops[1], v0, tmode);
6642 /* See if this can be handled with a vec_shr. We only do this if the
6643 second vector is all zeroes. */
6644 enum insn_code shift_code = optab_handler (vec_shr_optab, GET_MODE (v0));
6645 if (v1 == CONST0_RTX (GET_MODE (v1)) && shift_code)
6646 if (rtx shift_amt = shift_amt_for_vec_perm_mask (sel))
6648 create_convert_operand_from_type (&ops[2], shift_amt,
6649 sizetype_tab[(int) stk_sizetype]);
6650 if (maybe_expand_insn (shift_code, 3, ops))
6651 return ops[0].value;
6653 create_input_operand (&ops[2], v1, tmode);
6656 if (maybe_expand_insn (icode, 4, ops))
6657 return ops[0].value;
6658 return NULL_RTX;
6661 /* Generate instructions for vec_perm optab given its mode
6662 and three operands. */
6665 expand_vec_perm (machine_mode mode, rtx v0, rtx v1, rtx sel, rtx target)
6667 enum insn_code icode;
6668 machine_mode qimode;
6669 unsigned int i, w, e, u;
6670 rtx tmp, sel_qi = NULL;
6671 rtvec vec;
6673 if (!target || GET_MODE (target) != mode)
6674 target = gen_reg_rtx (mode);
6676 w = GET_MODE_SIZE (mode);
6677 e = GET_MODE_NUNITS (mode);
6678 u = GET_MODE_UNIT_SIZE (mode);
6680 /* Set QIMODE to a different vector mode with byte elements.
6681 If no such mode, or if MODE already has byte elements, use VOIDmode. */
6682 qimode = VOIDmode;
6683 if (GET_MODE_INNER (mode) != QImode)
6685 qimode = mode_for_vector (QImode, w);
6686 if (!VECTOR_MODE_P (qimode))
6687 qimode = VOIDmode;
6690 /* If the input is a constant, expand it specially. */
6691 gcc_assert (GET_MODE_CLASS (GET_MODE (sel)) == MODE_VECTOR_INT);
6692 if (GET_CODE (sel) == CONST_VECTOR)
6694 icode = direct_optab_handler (vec_perm_const_optab, mode);
6695 if (icode != CODE_FOR_nothing)
6697 tmp = expand_vec_perm_1 (icode, target, v0, v1, sel);
6698 if (tmp)
6699 return tmp;
6702 /* Fall back to a constant byte-based permutation. */
6703 if (qimode != VOIDmode)
6705 vec = rtvec_alloc (w);
6706 for (i = 0; i < e; ++i)
6708 unsigned int j, this_e;
6710 this_e = INTVAL (CONST_VECTOR_ELT (sel, i));
6711 this_e &= 2 * e - 1;
6712 this_e *= u;
6714 for (j = 0; j < u; ++j)
6715 RTVEC_ELT (vec, i * u + j) = GEN_INT (this_e + j);
6717 sel_qi = gen_rtx_CONST_VECTOR (qimode, vec);
6719 icode = direct_optab_handler (vec_perm_const_optab, qimode);
6720 if (icode != CODE_FOR_nothing)
6722 tmp = mode != qimode ? gen_reg_rtx (qimode) : target;
6723 tmp = expand_vec_perm_1 (icode, tmp, gen_lowpart (qimode, v0),
6724 gen_lowpart (qimode, v1), sel_qi);
6725 if (tmp)
6726 return gen_lowpart (mode, tmp);
6731 /* Otherwise expand as a fully variable permuation. */
6732 icode = direct_optab_handler (vec_perm_optab, mode);
6733 if (icode != CODE_FOR_nothing)
6735 tmp = expand_vec_perm_1 (icode, target, v0, v1, sel);
6736 if (tmp)
6737 return tmp;
6740 /* As a special case to aid several targets, lower the element-based
6741 permutation to a byte-based permutation and try again. */
6742 if (qimode == VOIDmode)
6743 return NULL_RTX;
6744 icode = direct_optab_handler (vec_perm_optab, qimode);
6745 if (icode == CODE_FOR_nothing)
6746 return NULL_RTX;
6748 if (sel_qi == NULL)
6750 /* Multiply each element by its byte size. */
6751 machine_mode selmode = GET_MODE (sel);
6752 if (u == 2)
6753 sel = expand_simple_binop (selmode, PLUS, sel, sel,
6754 NULL, 0, OPTAB_DIRECT);
6755 else
6756 sel = expand_simple_binop (selmode, ASHIFT, sel,
6757 GEN_INT (exact_log2 (u)),
6758 NULL, 0, OPTAB_DIRECT);
6759 gcc_assert (sel != NULL);
6761 /* Broadcast the low byte each element into each of its bytes. */
6762 vec = rtvec_alloc (w);
6763 for (i = 0; i < w; ++i)
6765 int this_e = i / u * u;
6766 if (BYTES_BIG_ENDIAN)
6767 this_e += u - 1;
6768 RTVEC_ELT (vec, i) = GEN_INT (this_e);
6770 tmp = gen_rtx_CONST_VECTOR (qimode, vec);
6771 sel = gen_lowpart (qimode, sel);
6772 sel = expand_vec_perm (qimode, sel, sel, tmp, NULL);
6773 gcc_assert (sel != NULL);
6775 /* Add the byte offset to each byte element. */
6776 /* Note that the definition of the indicies here is memory ordering,
6777 so there should be no difference between big and little endian. */
6778 vec = rtvec_alloc (w);
6779 for (i = 0; i < w; ++i)
6780 RTVEC_ELT (vec, i) = GEN_INT (i % u);
6781 tmp = gen_rtx_CONST_VECTOR (qimode, vec);
6782 sel_qi = expand_simple_binop (qimode, PLUS, sel, tmp,
6783 sel, 0, OPTAB_DIRECT);
6784 gcc_assert (sel_qi != NULL);
6787 tmp = mode != qimode ? gen_reg_rtx (qimode) : target;
6788 tmp = expand_vec_perm_1 (icode, tmp, gen_lowpart (qimode, v0),
6789 gen_lowpart (qimode, v1), sel_qi);
6790 if (tmp)
6791 tmp = gen_lowpart (mode, tmp);
6792 return tmp;
6795 /* Return insn code for a conditional operator with a comparison in
6796 mode CMODE, unsigned if UNS is true, resulting in a value of mode VMODE. */
6798 static inline enum insn_code
6799 get_vcond_icode (machine_mode vmode, machine_mode cmode, bool uns)
6801 enum insn_code icode = CODE_FOR_nothing;
6802 if (uns)
6803 icode = convert_optab_handler (vcondu_optab, vmode, cmode);
6804 else
6805 icode = convert_optab_handler (vcond_optab, vmode, cmode);
6806 return icode;
6809 /* Return TRUE iff, appropriate vector insns are available
6810 for vector cond expr with vector type VALUE_TYPE and a comparison
6811 with operand vector types in CMP_OP_TYPE. */
6813 bool
6814 expand_vec_cond_expr_p (tree value_type, tree cmp_op_type)
6816 machine_mode value_mode = TYPE_MODE (value_type);
6817 machine_mode cmp_op_mode = TYPE_MODE (cmp_op_type);
6818 if (GET_MODE_SIZE (value_mode) != GET_MODE_SIZE (cmp_op_mode)
6819 || GET_MODE_NUNITS (value_mode) != GET_MODE_NUNITS (cmp_op_mode)
6820 || get_vcond_icode (TYPE_MODE (value_type), TYPE_MODE (cmp_op_type),
6821 TYPE_UNSIGNED (cmp_op_type)) == CODE_FOR_nothing)
6822 return false;
6823 return true;
6826 /* Generate insns for a VEC_COND_EXPR, given its TYPE and its
6827 three operands. */
6830 expand_vec_cond_expr (tree vec_cond_type, tree op0, tree op1, tree op2,
6831 rtx target)
6833 struct expand_operand ops[6];
6834 enum insn_code icode;
6835 rtx comparison, rtx_op1, rtx_op2;
6836 machine_mode mode = TYPE_MODE (vec_cond_type);
6837 machine_mode cmp_op_mode;
6838 bool unsignedp;
6839 tree op0a, op0b;
6840 enum tree_code tcode;
6842 if (COMPARISON_CLASS_P (op0))
6844 op0a = TREE_OPERAND (op0, 0);
6845 op0b = TREE_OPERAND (op0, 1);
6846 tcode = TREE_CODE (op0);
6848 else
6850 /* Fake op0 < 0. */
6851 gcc_assert (!TYPE_UNSIGNED (TREE_TYPE (op0)));
6852 op0a = op0;
6853 op0b = build_zero_cst (TREE_TYPE (op0));
6854 tcode = LT_EXPR;
6856 unsignedp = TYPE_UNSIGNED (TREE_TYPE (op0a));
6857 cmp_op_mode = TYPE_MODE (TREE_TYPE (op0a));
6860 gcc_assert (GET_MODE_SIZE (mode) == GET_MODE_SIZE (cmp_op_mode)
6861 && GET_MODE_NUNITS (mode) == GET_MODE_NUNITS (cmp_op_mode));
6863 icode = get_vcond_icode (mode, cmp_op_mode, unsignedp);
6864 if (icode == CODE_FOR_nothing)
6865 return 0;
6867 comparison = vector_compare_rtx (tcode, op0a, op0b, unsignedp, icode);
6868 rtx_op1 = expand_normal (op1);
6869 rtx_op2 = expand_normal (op2);
6871 create_output_operand (&ops[0], target, mode);
6872 create_input_operand (&ops[1], rtx_op1, mode);
6873 create_input_operand (&ops[2], rtx_op2, mode);
6874 create_fixed_operand (&ops[3], comparison);
6875 create_fixed_operand (&ops[4], XEXP (comparison, 0));
6876 create_fixed_operand (&ops[5], XEXP (comparison, 1));
6877 expand_insn (icode, 6, ops);
6878 return ops[0].value;
6881 /* Return non-zero if a highpart multiply is supported of can be synthisized.
6882 For the benefit of expand_mult_highpart, the return value is 1 for direct,
6883 2 for even/odd widening, and 3 for hi/lo widening. */
6886 can_mult_highpart_p (machine_mode mode, bool uns_p)
6888 optab op;
6889 unsigned char *sel;
6890 unsigned i, nunits;
6892 op = uns_p ? umul_highpart_optab : smul_highpart_optab;
6893 if (optab_handler (op, mode) != CODE_FOR_nothing)
6894 return 1;
6896 /* If the mode is an integral vector, synth from widening operations. */
6897 if (GET_MODE_CLASS (mode) != MODE_VECTOR_INT)
6898 return 0;
6900 nunits = GET_MODE_NUNITS (mode);
6901 sel = XALLOCAVEC (unsigned char, nunits);
6903 op = uns_p ? vec_widen_umult_even_optab : vec_widen_smult_even_optab;
6904 if (optab_handler (op, mode) != CODE_FOR_nothing)
6906 op = uns_p ? vec_widen_umult_odd_optab : vec_widen_smult_odd_optab;
6907 if (optab_handler (op, mode) != CODE_FOR_nothing)
6909 for (i = 0; i < nunits; ++i)
6910 sel[i] = !BYTES_BIG_ENDIAN + (i & ~1) + ((i & 1) ? nunits : 0);
6911 if (can_vec_perm_p (mode, false, sel))
6912 return 2;
6916 op = uns_p ? vec_widen_umult_hi_optab : vec_widen_smult_hi_optab;
6917 if (optab_handler (op, mode) != CODE_FOR_nothing)
6919 op = uns_p ? vec_widen_umult_lo_optab : vec_widen_smult_lo_optab;
6920 if (optab_handler (op, mode) != CODE_FOR_nothing)
6922 for (i = 0; i < nunits; ++i)
6923 sel[i] = 2 * i + (BYTES_BIG_ENDIAN ? 0 : 1);
6924 if (can_vec_perm_p (mode, false, sel))
6925 return 3;
6929 return 0;
6932 /* Expand a highpart multiply. */
6935 expand_mult_highpart (machine_mode mode, rtx op0, rtx op1,
6936 rtx target, bool uns_p)
6938 struct expand_operand eops[3];
6939 enum insn_code icode;
6940 int method, i, nunits;
6941 machine_mode wmode;
6942 rtx m1, m2, perm;
6943 optab tab1, tab2;
6944 rtvec v;
6946 method = can_mult_highpart_p (mode, uns_p);
6947 switch (method)
6949 case 0:
6950 return NULL_RTX;
6951 case 1:
6952 tab1 = uns_p ? umul_highpart_optab : smul_highpart_optab;
6953 return expand_binop (mode, tab1, op0, op1, target, uns_p,
6954 OPTAB_LIB_WIDEN);
6955 case 2:
6956 tab1 = uns_p ? vec_widen_umult_even_optab : vec_widen_smult_even_optab;
6957 tab2 = uns_p ? vec_widen_umult_odd_optab : vec_widen_smult_odd_optab;
6958 break;
6959 case 3:
6960 tab1 = uns_p ? vec_widen_umult_lo_optab : vec_widen_smult_lo_optab;
6961 tab2 = uns_p ? vec_widen_umult_hi_optab : vec_widen_smult_hi_optab;
6962 if (BYTES_BIG_ENDIAN)
6963 std::swap (tab1, tab2);
6964 break;
6965 default:
6966 gcc_unreachable ();
6969 icode = optab_handler (tab1, mode);
6970 nunits = GET_MODE_NUNITS (mode);
6971 wmode = insn_data[icode].operand[0].mode;
6972 gcc_checking_assert (2 * GET_MODE_NUNITS (wmode) == nunits);
6973 gcc_checking_assert (GET_MODE_SIZE (wmode) == GET_MODE_SIZE (mode));
6975 create_output_operand (&eops[0], gen_reg_rtx (wmode), wmode);
6976 create_input_operand (&eops[1], op0, mode);
6977 create_input_operand (&eops[2], op1, mode);
6978 expand_insn (icode, 3, eops);
6979 m1 = gen_lowpart (mode, eops[0].value);
6981 create_output_operand (&eops[0], gen_reg_rtx (wmode), wmode);
6982 create_input_operand (&eops[1], op0, mode);
6983 create_input_operand (&eops[2], op1, mode);
6984 expand_insn (optab_handler (tab2, mode), 3, eops);
6985 m2 = gen_lowpart (mode, eops[0].value);
6987 v = rtvec_alloc (nunits);
6988 if (method == 2)
6990 for (i = 0; i < nunits; ++i)
6991 RTVEC_ELT (v, i) = GEN_INT (!BYTES_BIG_ENDIAN + (i & ~1)
6992 + ((i & 1) ? nunits : 0));
6994 else
6996 for (i = 0; i < nunits; ++i)
6997 RTVEC_ELT (v, i) = GEN_INT (2 * i + (BYTES_BIG_ENDIAN ? 0 : 1));
6999 perm = gen_rtx_CONST_VECTOR (mode, v);
7001 return expand_vec_perm (mode, m1, m2, perm, target);
7004 /* Return true if target supports vector masked load/store for mode. */
7005 bool
7006 can_vec_mask_load_store_p (machine_mode mode, bool is_load)
7008 optab op = is_load ? maskload_optab : maskstore_optab;
7009 machine_mode vmode;
7010 unsigned int vector_sizes;
7012 /* If mode is vector mode, check it directly. */
7013 if (VECTOR_MODE_P (mode))
7014 return optab_handler (op, mode) != CODE_FOR_nothing;
7016 /* Otherwise, return true if there is some vector mode with
7017 the mask load/store supported. */
7019 /* See if there is any chance the mask load or store might be
7020 vectorized. If not, punt. */
7021 vmode = targetm.vectorize.preferred_simd_mode (mode);
7022 if (!VECTOR_MODE_P (vmode))
7023 return false;
7025 if (optab_handler (op, vmode) != CODE_FOR_nothing)
7026 return true;
7028 vector_sizes = targetm.vectorize.autovectorize_vector_sizes ();
7029 while (vector_sizes != 0)
7031 unsigned int cur = 1 << floor_log2 (vector_sizes);
7032 vector_sizes &= ~cur;
7033 if (cur <= GET_MODE_SIZE (mode))
7034 continue;
7035 vmode = mode_for_vector (mode, cur / GET_MODE_SIZE (mode));
7036 if (VECTOR_MODE_P (vmode)
7037 && optab_handler (op, vmode) != CODE_FOR_nothing)
7038 return true;
7040 return false;
7043 /* Return true if there is a compare_and_swap pattern. */
7045 bool
7046 can_compare_and_swap_p (machine_mode mode, bool allow_libcall)
7048 enum insn_code icode;
7050 /* Check for __atomic_compare_and_swap. */
7051 icode = direct_optab_handler (atomic_compare_and_swap_optab, mode);
7052 if (icode != CODE_FOR_nothing)
7053 return true;
7055 /* Check for __sync_compare_and_swap. */
7056 icode = optab_handler (sync_compare_and_swap_optab, mode);
7057 if (icode != CODE_FOR_nothing)
7058 return true;
7059 if (allow_libcall && optab_libfunc (sync_compare_and_swap_optab, mode))
7060 return true;
7062 /* No inline compare and swap. */
7063 return false;
7066 /* Return true if an atomic exchange can be performed. */
7068 bool
7069 can_atomic_exchange_p (machine_mode mode, bool allow_libcall)
7071 enum insn_code icode;
7073 /* Check for __atomic_exchange. */
7074 icode = direct_optab_handler (atomic_exchange_optab, mode);
7075 if (icode != CODE_FOR_nothing)
7076 return true;
7078 /* Don't check __sync_test_and_set, as on some platforms that
7079 has reduced functionality. Targets that really do support
7080 a proper exchange should simply be updated to the __atomics. */
7082 return can_compare_and_swap_p (mode, allow_libcall);
7086 /* Helper function to find the MODE_CC set in a sync_compare_and_swap
7087 pattern. */
7089 static void
7090 find_cc_set (rtx x, const_rtx pat, void *data)
7092 if (REG_P (x) && GET_MODE_CLASS (GET_MODE (x)) == MODE_CC
7093 && GET_CODE (pat) == SET)
7095 rtx *p_cc_reg = (rtx *) data;
7096 gcc_assert (!*p_cc_reg);
7097 *p_cc_reg = x;
7101 /* This is a helper function for the other atomic operations. This function
7102 emits a loop that contains SEQ that iterates until a compare-and-swap
7103 operation at the end succeeds. MEM is the memory to be modified. SEQ is
7104 a set of instructions that takes a value from OLD_REG as an input and
7105 produces a value in NEW_REG as an output. Before SEQ, OLD_REG will be
7106 set to the current contents of MEM. After SEQ, a compare-and-swap will
7107 attempt to update MEM with NEW_REG. The function returns true when the
7108 loop was generated successfully. */
7110 static bool
7111 expand_compare_and_swap_loop (rtx mem, rtx old_reg, rtx new_reg, rtx seq)
7113 machine_mode mode = GET_MODE (mem);
7114 rtx_code_label *label;
7115 rtx cmp_reg, success, oldval;
7117 /* The loop we want to generate looks like
7119 cmp_reg = mem;
7120 label:
7121 old_reg = cmp_reg;
7122 seq;
7123 (success, cmp_reg) = compare-and-swap(mem, old_reg, new_reg)
7124 if (success)
7125 goto label;
7127 Note that we only do the plain load from memory once. Subsequent
7128 iterations use the value loaded by the compare-and-swap pattern. */
7130 label = gen_label_rtx ();
7131 cmp_reg = gen_reg_rtx (mode);
7133 emit_move_insn (cmp_reg, mem);
7134 emit_label (label);
7135 emit_move_insn (old_reg, cmp_reg);
7136 if (seq)
7137 emit_insn (seq);
7139 success = NULL_RTX;
7140 oldval = cmp_reg;
7141 if (!expand_atomic_compare_and_swap (&success, &oldval, mem, old_reg,
7142 new_reg, false, MEMMODEL_SYNC_SEQ_CST,
7143 MEMMODEL_RELAXED))
7144 return false;
7146 if (oldval != cmp_reg)
7147 emit_move_insn (cmp_reg, oldval);
7149 /* Mark this jump predicted not taken. */
7150 emit_cmp_and_jump_insns (success, const0_rtx, EQ, const0_rtx,
7151 GET_MODE (success), 1, label, 0);
7152 return true;
7156 /* This function tries to emit an atomic_exchange intruction. VAL is written
7157 to *MEM using memory model MODEL. The previous contents of *MEM are returned,
7158 using TARGET if possible. */
7160 static rtx
7161 maybe_emit_atomic_exchange (rtx target, rtx mem, rtx val, enum memmodel model)
7163 machine_mode mode = GET_MODE (mem);
7164 enum insn_code icode;
7166 /* If the target supports the exchange directly, great. */
7167 icode = direct_optab_handler (atomic_exchange_optab, mode);
7168 if (icode != CODE_FOR_nothing)
7170 struct expand_operand ops[4];
7172 create_output_operand (&ops[0], target, mode);
7173 create_fixed_operand (&ops[1], mem);
7174 create_input_operand (&ops[2], val, mode);
7175 create_integer_operand (&ops[3], model);
7176 if (maybe_expand_insn (icode, 4, ops))
7177 return ops[0].value;
7180 return NULL_RTX;
7183 /* This function tries to implement an atomic exchange operation using
7184 __sync_lock_test_and_set. VAL is written to *MEM using memory model MODEL.
7185 The previous contents of *MEM are returned, using TARGET if possible.
7186 Since this instructionn is an acquire barrier only, stronger memory
7187 models may require additional barriers to be emitted. */
7189 static rtx
7190 maybe_emit_sync_lock_test_and_set (rtx target, rtx mem, rtx val,
7191 enum memmodel model)
7193 machine_mode mode = GET_MODE (mem);
7194 enum insn_code icode;
7195 rtx_insn *last_insn = get_last_insn ();
7197 icode = optab_handler (sync_lock_test_and_set_optab, mode);
7199 /* Legacy sync_lock_test_and_set is an acquire barrier. If the pattern
7200 exists, and the memory model is stronger than acquire, add a release
7201 barrier before the instruction. */
7203 if (is_mm_seq_cst (model) || is_mm_release (model) || is_mm_acq_rel (model))
7204 expand_mem_thread_fence (model);
7206 if (icode != CODE_FOR_nothing)
7208 struct expand_operand ops[3];
7209 create_output_operand (&ops[0], target, mode);
7210 create_fixed_operand (&ops[1], mem);
7211 create_input_operand (&ops[2], val, mode);
7212 if (maybe_expand_insn (icode, 3, ops))
7213 return ops[0].value;
7216 /* If an external test-and-set libcall is provided, use that instead of
7217 any external compare-and-swap that we might get from the compare-and-
7218 swap-loop expansion later. */
7219 if (!can_compare_and_swap_p (mode, false))
7221 rtx libfunc = optab_libfunc (sync_lock_test_and_set_optab, mode);
7222 if (libfunc != NULL)
7224 rtx addr;
7226 addr = convert_memory_address (ptr_mode, XEXP (mem, 0));
7227 return emit_library_call_value (libfunc, NULL_RTX, LCT_NORMAL,
7228 mode, 2, addr, ptr_mode,
7229 val, mode);
7233 /* If the test_and_set can't be emitted, eliminate any barrier that might
7234 have been emitted. */
7235 delete_insns_since (last_insn);
7236 return NULL_RTX;
7239 /* This function tries to implement an atomic exchange operation using a
7240 compare_and_swap loop. VAL is written to *MEM. The previous contents of
7241 *MEM are returned, using TARGET if possible. No memory model is required
7242 since a compare_and_swap loop is seq-cst. */
7244 static rtx
7245 maybe_emit_compare_and_swap_exchange_loop (rtx target, rtx mem, rtx val)
7247 machine_mode mode = GET_MODE (mem);
7249 if (can_compare_and_swap_p (mode, true))
7251 if (!target || !register_operand (target, mode))
7252 target = gen_reg_rtx (mode);
7253 if (expand_compare_and_swap_loop (mem, target, val, NULL_RTX))
7254 return target;
7257 return NULL_RTX;
7260 /* This function tries to implement an atomic test-and-set operation
7261 using the atomic_test_and_set instruction pattern. A boolean value
7262 is returned from the operation, using TARGET if possible. */
7264 #ifndef HAVE_atomic_test_and_set
7265 #define HAVE_atomic_test_and_set 0
7266 #define CODE_FOR_atomic_test_and_set CODE_FOR_nothing
7267 #endif
7269 static rtx
7270 maybe_emit_atomic_test_and_set (rtx target, rtx mem, enum memmodel model)
7272 machine_mode pat_bool_mode;
7273 struct expand_operand ops[3];
7275 if (!HAVE_atomic_test_and_set)
7276 return NULL_RTX;
7278 /* While we always get QImode from __atomic_test_and_set, we get
7279 other memory modes from __sync_lock_test_and_set. Note that we
7280 use no endian adjustment here. This matches the 4.6 behavior
7281 in the Sparc backend. */
7282 gcc_checking_assert
7283 (insn_data[CODE_FOR_atomic_test_and_set].operand[1].mode == QImode);
7284 if (GET_MODE (mem) != QImode)
7285 mem = adjust_address_nv (mem, QImode, 0);
7287 pat_bool_mode = insn_data[CODE_FOR_atomic_test_and_set].operand[0].mode;
7288 create_output_operand (&ops[0], target, pat_bool_mode);
7289 create_fixed_operand (&ops[1], mem);
7290 create_integer_operand (&ops[2], model);
7292 if (maybe_expand_insn (CODE_FOR_atomic_test_and_set, 3, ops))
7293 return ops[0].value;
7294 return NULL_RTX;
7297 /* This function expands the legacy _sync_lock test_and_set operation which is
7298 generally an atomic exchange. Some limited targets only allow the
7299 constant 1 to be stored. This is an ACQUIRE operation.
7301 TARGET is an optional place to stick the return value.
7302 MEM is where VAL is stored. */
7305 expand_sync_lock_test_and_set (rtx target, rtx mem, rtx val)
7307 rtx ret;
7309 /* Try an atomic_exchange first. */
7310 ret = maybe_emit_atomic_exchange (target, mem, val, MEMMODEL_SYNC_ACQUIRE);
7311 if (ret)
7312 return ret;
7314 ret = maybe_emit_sync_lock_test_and_set (target, mem, val,
7315 MEMMODEL_SYNC_ACQUIRE);
7316 if (ret)
7317 return ret;
7319 ret = maybe_emit_compare_and_swap_exchange_loop (target, mem, val);
7320 if (ret)
7321 return ret;
7323 /* If there are no other options, try atomic_test_and_set if the value
7324 being stored is 1. */
7325 if (val == const1_rtx)
7326 ret = maybe_emit_atomic_test_and_set (target, mem, MEMMODEL_SYNC_ACQUIRE);
7328 return ret;
7331 /* This function expands the atomic test_and_set operation:
7332 atomically store a boolean TRUE into MEM and return the previous value.
7334 MEMMODEL is the memory model variant to use.
7335 TARGET is an optional place to stick the return value. */
7338 expand_atomic_test_and_set (rtx target, rtx mem, enum memmodel model)
7340 machine_mode mode = GET_MODE (mem);
7341 rtx ret, trueval, subtarget;
7343 ret = maybe_emit_atomic_test_and_set (target, mem, model);
7344 if (ret)
7345 return ret;
7347 /* Be binary compatible with non-default settings of trueval, and different
7348 cpu revisions. E.g. one revision may have atomic-test-and-set, but
7349 another only has atomic-exchange. */
7350 if (targetm.atomic_test_and_set_trueval == 1)
7352 trueval = const1_rtx;
7353 subtarget = target ? target : gen_reg_rtx (mode);
7355 else
7357 trueval = gen_int_mode (targetm.atomic_test_and_set_trueval, mode);
7358 subtarget = gen_reg_rtx (mode);
7361 /* Try the atomic-exchange optab... */
7362 ret = maybe_emit_atomic_exchange (subtarget, mem, trueval, model);
7364 /* ... then an atomic-compare-and-swap loop ... */
7365 if (!ret)
7366 ret = maybe_emit_compare_and_swap_exchange_loop (subtarget, mem, trueval);
7368 /* ... before trying the vaguely defined legacy lock_test_and_set. */
7369 if (!ret)
7370 ret = maybe_emit_sync_lock_test_and_set (subtarget, mem, trueval, model);
7372 /* Recall that the legacy lock_test_and_set optab was allowed to do magic
7373 things with the value 1. Thus we try again without trueval. */
7374 if (!ret && targetm.atomic_test_and_set_trueval != 1)
7375 ret = maybe_emit_sync_lock_test_and_set (subtarget, mem, const1_rtx, model);
7377 /* Failing all else, assume a single threaded environment and simply
7378 perform the operation. */
7379 if (!ret)
7381 /* If the result is ignored skip the move to target. */
7382 if (subtarget != const0_rtx)
7383 emit_move_insn (subtarget, mem);
7385 emit_move_insn (mem, trueval);
7386 ret = subtarget;
7389 /* Recall that have to return a boolean value; rectify if trueval
7390 is not exactly one. */
7391 if (targetm.atomic_test_and_set_trueval != 1)
7392 ret = emit_store_flag_force (target, NE, ret, const0_rtx, mode, 0, 1);
7394 return ret;
7397 /* This function expands the atomic exchange operation:
7398 atomically store VAL in MEM and return the previous value in MEM.
7400 MEMMODEL is the memory model variant to use.
7401 TARGET is an optional place to stick the return value. */
7404 expand_atomic_exchange (rtx target, rtx mem, rtx val, enum memmodel model)
7406 rtx ret;
7408 ret = maybe_emit_atomic_exchange (target, mem, val, model);
7410 /* Next try a compare-and-swap loop for the exchange. */
7411 if (!ret)
7412 ret = maybe_emit_compare_and_swap_exchange_loop (target, mem, val);
7414 return ret;
7417 /* This function expands the atomic compare exchange operation:
7419 *PTARGET_BOOL is an optional place to store the boolean success/failure.
7420 *PTARGET_OVAL is an optional place to store the old value from memory.
7421 Both target parameters may be NULL to indicate that we do not care about
7422 that return value. Both target parameters are updated on success to
7423 the actual location of the corresponding result.
7425 MEMMODEL is the memory model variant to use.
7427 The return value of the function is true for success. */
7429 bool
7430 expand_atomic_compare_and_swap (rtx *ptarget_bool, rtx *ptarget_oval,
7431 rtx mem, rtx expected, rtx desired,
7432 bool is_weak, enum memmodel succ_model,
7433 enum memmodel fail_model)
7435 machine_mode mode = GET_MODE (mem);
7436 struct expand_operand ops[8];
7437 enum insn_code icode;
7438 rtx target_oval, target_bool = NULL_RTX;
7439 rtx libfunc;
7441 /* Load expected into a register for the compare and swap. */
7442 if (MEM_P (expected))
7443 expected = copy_to_reg (expected);
7445 /* Make sure we always have some place to put the return oldval.
7446 Further, make sure that place is distinct from the input expected,
7447 just in case we need that path down below. */
7448 if (ptarget_oval == NULL
7449 || (target_oval = *ptarget_oval) == NULL
7450 || reg_overlap_mentioned_p (expected, target_oval))
7451 target_oval = gen_reg_rtx (mode);
7453 icode = direct_optab_handler (atomic_compare_and_swap_optab, mode);
7454 if (icode != CODE_FOR_nothing)
7456 machine_mode bool_mode = insn_data[icode].operand[0].mode;
7458 /* Make sure we always have a place for the bool operand. */
7459 if (ptarget_bool == NULL
7460 || (target_bool = *ptarget_bool) == NULL
7461 || GET_MODE (target_bool) != bool_mode)
7462 target_bool = gen_reg_rtx (bool_mode);
7464 /* Emit the compare_and_swap. */
7465 create_output_operand (&ops[0], target_bool, bool_mode);
7466 create_output_operand (&ops[1], target_oval, mode);
7467 create_fixed_operand (&ops[2], mem);
7468 create_input_operand (&ops[3], expected, mode);
7469 create_input_operand (&ops[4], desired, mode);
7470 create_integer_operand (&ops[5], is_weak);
7471 create_integer_operand (&ops[6], succ_model);
7472 create_integer_operand (&ops[7], fail_model);
7473 if (maybe_expand_insn (icode, 8, ops))
7475 /* Return success/failure. */
7476 target_bool = ops[0].value;
7477 target_oval = ops[1].value;
7478 goto success;
7482 /* Otherwise fall back to the original __sync_val_compare_and_swap
7483 which is always seq-cst. */
7484 icode = optab_handler (sync_compare_and_swap_optab, mode);
7485 if (icode != CODE_FOR_nothing)
7487 rtx cc_reg;
7489 create_output_operand (&ops[0], target_oval, mode);
7490 create_fixed_operand (&ops[1], mem);
7491 create_input_operand (&ops[2], expected, mode);
7492 create_input_operand (&ops[3], desired, mode);
7493 if (!maybe_expand_insn (icode, 4, ops))
7494 return false;
7496 target_oval = ops[0].value;
7498 /* If the caller isn't interested in the boolean return value,
7499 skip the computation of it. */
7500 if (ptarget_bool == NULL)
7501 goto success;
7503 /* Otherwise, work out if the compare-and-swap succeeded. */
7504 cc_reg = NULL_RTX;
7505 if (have_insn_for (COMPARE, CCmode))
7506 note_stores (PATTERN (get_last_insn ()), find_cc_set, &cc_reg);
7507 if (cc_reg)
7509 target_bool = emit_store_flag_force (target_bool, EQ, cc_reg,
7510 const0_rtx, VOIDmode, 0, 1);
7511 goto success;
7513 goto success_bool_from_val;
7516 /* Also check for library support for __sync_val_compare_and_swap. */
7517 libfunc = optab_libfunc (sync_compare_and_swap_optab, mode);
7518 if (libfunc != NULL)
7520 rtx addr = convert_memory_address (ptr_mode, XEXP (mem, 0));
7521 target_oval = emit_library_call_value (libfunc, NULL_RTX, LCT_NORMAL,
7522 mode, 3, addr, ptr_mode,
7523 expected, mode, desired, mode);
7525 /* Compute the boolean return value only if requested. */
7526 if (ptarget_bool)
7527 goto success_bool_from_val;
7528 else
7529 goto success;
7532 /* Failure. */
7533 return false;
7535 success_bool_from_val:
7536 target_bool = emit_store_flag_force (target_bool, EQ, target_oval,
7537 expected, VOIDmode, 1, 1);
7538 success:
7539 /* Make sure that the oval output winds up where the caller asked. */
7540 if (ptarget_oval)
7541 *ptarget_oval = target_oval;
7542 if (ptarget_bool)
7543 *ptarget_bool = target_bool;
7544 return true;
7547 /* Generate asm volatile("" : : : "memory") as the memory barrier. */
7549 static void
7550 expand_asm_memory_barrier (void)
7552 rtx asm_op, clob;
7554 asm_op = gen_rtx_ASM_OPERANDS (VOIDmode, empty_string, empty_string, 0,
7555 rtvec_alloc (0), rtvec_alloc (0),
7556 rtvec_alloc (0), UNKNOWN_LOCATION);
7557 MEM_VOLATILE_P (asm_op) = 1;
7559 clob = gen_rtx_SCRATCH (VOIDmode);
7560 clob = gen_rtx_MEM (BLKmode, clob);
7561 clob = gen_rtx_CLOBBER (VOIDmode, clob);
7563 emit_insn (gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, asm_op, clob)));
7566 /* This routine will either emit the mem_thread_fence pattern or issue a
7567 sync_synchronize to generate a fence for memory model MEMMODEL. */
7569 void
7570 expand_mem_thread_fence (enum memmodel model)
7572 if (targetm.have_mem_thread_fence ())
7573 emit_insn (targetm.gen_mem_thread_fence (GEN_INT (model)));
7574 else if (!is_mm_relaxed (model))
7576 if (targetm.have_memory_barrier ())
7577 emit_insn (targetm.gen_memory_barrier ());
7578 else if (synchronize_libfunc != NULL_RTX)
7579 emit_library_call (synchronize_libfunc, LCT_NORMAL, VOIDmode, 0);
7580 else
7581 expand_asm_memory_barrier ();
7585 /* This routine will either emit the mem_signal_fence pattern or issue a
7586 sync_synchronize to generate a fence for memory model MEMMODEL. */
7588 void
7589 expand_mem_signal_fence (enum memmodel model)
7591 if (targetm.have_mem_signal_fence ())
7592 emit_insn (targetm.gen_mem_signal_fence (GEN_INT (model)));
7593 else if (!is_mm_relaxed (model))
7595 /* By default targets are coherent between a thread and the signal
7596 handler running on the same thread. Thus this really becomes a
7597 compiler barrier, in that stores must not be sunk past
7598 (or raised above) a given point. */
7599 expand_asm_memory_barrier ();
7603 /* This function expands the atomic load operation:
7604 return the atomically loaded value in MEM.
7606 MEMMODEL is the memory model variant to use.
7607 TARGET is an option place to stick the return value. */
7610 expand_atomic_load (rtx target, rtx mem, enum memmodel model)
7612 machine_mode mode = GET_MODE (mem);
7613 enum insn_code icode;
7615 /* If the target supports the load directly, great. */
7616 icode = direct_optab_handler (atomic_load_optab, mode);
7617 if (icode != CODE_FOR_nothing)
7619 struct expand_operand ops[3];
7621 create_output_operand (&ops[0], target, mode);
7622 create_fixed_operand (&ops[1], mem);
7623 create_integer_operand (&ops[2], model);
7624 if (maybe_expand_insn (icode, 3, ops))
7625 return ops[0].value;
7628 /* If the size of the object is greater than word size on this target,
7629 then we assume that a load will not be atomic. */
7630 if (GET_MODE_PRECISION (mode) > BITS_PER_WORD)
7632 /* Issue val = compare_and_swap (mem, 0, 0).
7633 This may cause the occasional harmless store of 0 when the value is
7634 already 0, but it seems to be OK according to the standards guys. */
7635 if (expand_atomic_compare_and_swap (NULL, &target, mem, const0_rtx,
7636 const0_rtx, false, model, model))
7637 return target;
7638 else
7639 /* Otherwise there is no atomic load, leave the library call. */
7640 return NULL_RTX;
7643 /* Otherwise assume loads are atomic, and emit the proper barriers. */
7644 if (!target || target == const0_rtx)
7645 target = gen_reg_rtx (mode);
7647 /* For SEQ_CST, emit a barrier before the load. */
7648 if (is_mm_seq_cst (model))
7649 expand_mem_thread_fence (model);
7651 emit_move_insn (target, mem);
7653 /* Emit the appropriate barrier after the load. */
7654 expand_mem_thread_fence (model);
7656 return target;
7659 /* This function expands the atomic store operation:
7660 Atomically store VAL in MEM.
7661 MEMMODEL is the memory model variant to use.
7662 USE_RELEASE is true if __sync_lock_release can be used as a fall back.
7663 function returns const0_rtx if a pattern was emitted. */
7666 expand_atomic_store (rtx mem, rtx val, enum memmodel model, bool use_release)
7668 machine_mode mode = GET_MODE (mem);
7669 enum insn_code icode;
7670 struct expand_operand ops[3];
7672 /* If the target supports the store directly, great. */
7673 icode = direct_optab_handler (atomic_store_optab, mode);
7674 if (icode != CODE_FOR_nothing)
7676 create_fixed_operand (&ops[0], mem);
7677 create_input_operand (&ops[1], val, mode);
7678 create_integer_operand (&ops[2], model);
7679 if (maybe_expand_insn (icode, 3, ops))
7680 return const0_rtx;
7683 /* If using __sync_lock_release is a viable alternative, try it. */
7684 if (use_release)
7686 icode = direct_optab_handler (sync_lock_release_optab, mode);
7687 if (icode != CODE_FOR_nothing)
7689 create_fixed_operand (&ops[0], mem);
7690 create_input_operand (&ops[1], const0_rtx, mode);
7691 if (maybe_expand_insn (icode, 2, ops))
7693 /* lock_release is only a release barrier. */
7694 if (is_mm_seq_cst (model))
7695 expand_mem_thread_fence (model);
7696 return const0_rtx;
7701 /* If the size of the object is greater than word size on this target,
7702 a default store will not be atomic, Try a mem_exchange and throw away
7703 the result. If that doesn't work, don't do anything. */
7704 if (GET_MODE_PRECISION (mode) > BITS_PER_WORD)
7706 rtx target = maybe_emit_atomic_exchange (NULL_RTX, mem, val, model);
7707 if (!target)
7708 target = maybe_emit_compare_and_swap_exchange_loop (NULL_RTX, mem, val);
7709 if (target)
7710 return const0_rtx;
7711 else
7712 return NULL_RTX;
7715 /* Otherwise assume stores are atomic, and emit the proper barriers. */
7716 expand_mem_thread_fence (model);
7718 emit_move_insn (mem, val);
7720 /* For SEQ_CST, also emit a barrier after the store. */
7721 if (is_mm_seq_cst (model))
7722 expand_mem_thread_fence (model);
7724 return const0_rtx;
7728 /* Structure containing the pointers and values required to process the
7729 various forms of the atomic_fetch_op and atomic_op_fetch builtins. */
7731 struct atomic_op_functions
7733 direct_optab mem_fetch_before;
7734 direct_optab mem_fetch_after;
7735 direct_optab mem_no_result;
7736 optab fetch_before;
7737 optab fetch_after;
7738 direct_optab no_result;
7739 enum rtx_code reverse_code;
7743 /* Fill in structure pointed to by OP with the various optab entries for an
7744 operation of type CODE. */
7746 static void
7747 get_atomic_op_for_code (struct atomic_op_functions *op, enum rtx_code code)
7749 gcc_assert (op!= NULL);
7751 /* If SWITCHABLE_TARGET is defined, then subtargets can be switched
7752 in the source code during compilation, and the optab entries are not
7753 computable until runtime. Fill in the values at runtime. */
7754 switch (code)
7756 case PLUS:
7757 op->mem_fetch_before = atomic_fetch_add_optab;
7758 op->mem_fetch_after = atomic_add_fetch_optab;
7759 op->mem_no_result = atomic_add_optab;
7760 op->fetch_before = sync_old_add_optab;
7761 op->fetch_after = sync_new_add_optab;
7762 op->no_result = sync_add_optab;
7763 op->reverse_code = MINUS;
7764 break;
7765 case MINUS:
7766 op->mem_fetch_before = atomic_fetch_sub_optab;
7767 op->mem_fetch_after = atomic_sub_fetch_optab;
7768 op->mem_no_result = atomic_sub_optab;
7769 op->fetch_before = sync_old_sub_optab;
7770 op->fetch_after = sync_new_sub_optab;
7771 op->no_result = sync_sub_optab;
7772 op->reverse_code = PLUS;
7773 break;
7774 case XOR:
7775 op->mem_fetch_before = atomic_fetch_xor_optab;
7776 op->mem_fetch_after = atomic_xor_fetch_optab;
7777 op->mem_no_result = atomic_xor_optab;
7778 op->fetch_before = sync_old_xor_optab;
7779 op->fetch_after = sync_new_xor_optab;
7780 op->no_result = sync_xor_optab;
7781 op->reverse_code = XOR;
7782 break;
7783 case AND:
7784 op->mem_fetch_before = atomic_fetch_and_optab;
7785 op->mem_fetch_after = atomic_and_fetch_optab;
7786 op->mem_no_result = atomic_and_optab;
7787 op->fetch_before = sync_old_and_optab;
7788 op->fetch_after = sync_new_and_optab;
7789 op->no_result = sync_and_optab;
7790 op->reverse_code = UNKNOWN;
7791 break;
7792 case IOR:
7793 op->mem_fetch_before = atomic_fetch_or_optab;
7794 op->mem_fetch_after = atomic_or_fetch_optab;
7795 op->mem_no_result = atomic_or_optab;
7796 op->fetch_before = sync_old_ior_optab;
7797 op->fetch_after = sync_new_ior_optab;
7798 op->no_result = sync_ior_optab;
7799 op->reverse_code = UNKNOWN;
7800 break;
7801 case NOT:
7802 op->mem_fetch_before = atomic_fetch_nand_optab;
7803 op->mem_fetch_after = atomic_nand_fetch_optab;
7804 op->mem_no_result = atomic_nand_optab;
7805 op->fetch_before = sync_old_nand_optab;
7806 op->fetch_after = sync_new_nand_optab;
7807 op->no_result = sync_nand_optab;
7808 op->reverse_code = UNKNOWN;
7809 break;
7810 default:
7811 gcc_unreachable ();
7815 /* See if there is a more optimal way to implement the operation "*MEM CODE VAL"
7816 using memory order MODEL. If AFTER is true the operation needs to return
7817 the value of *MEM after the operation, otherwise the previous value.
7818 TARGET is an optional place to place the result. The result is unused if
7819 it is const0_rtx.
7820 Return the result if there is a better sequence, otherwise NULL_RTX. */
7822 static rtx
7823 maybe_optimize_fetch_op (rtx target, rtx mem, rtx val, enum rtx_code code,
7824 enum memmodel model, bool after)
7826 /* If the value is prefetched, or not used, it may be possible to replace
7827 the sequence with a native exchange operation. */
7828 if (!after || target == const0_rtx)
7830 /* fetch_and (&x, 0, m) can be replaced with exchange (&x, 0, m). */
7831 if (code == AND && val == const0_rtx)
7833 if (target == const0_rtx)
7834 target = gen_reg_rtx (GET_MODE (mem));
7835 return maybe_emit_atomic_exchange (target, mem, val, model);
7838 /* fetch_or (&x, -1, m) can be replaced with exchange (&x, -1, m). */
7839 if (code == IOR && val == constm1_rtx)
7841 if (target == const0_rtx)
7842 target = gen_reg_rtx (GET_MODE (mem));
7843 return maybe_emit_atomic_exchange (target, mem, val, model);
7847 return NULL_RTX;
7850 /* Try to emit an instruction for a specific operation varaition.
7851 OPTAB contains the OP functions.
7852 TARGET is an optional place to return the result. const0_rtx means unused.
7853 MEM is the memory location to operate on.
7854 VAL is the value to use in the operation.
7855 USE_MEMMODEL is TRUE if the variation with a memory model should be tried.
7856 MODEL is the memory model, if used.
7857 AFTER is true if the returned result is the value after the operation. */
7859 static rtx
7860 maybe_emit_op (const struct atomic_op_functions *optab, rtx target, rtx mem,
7861 rtx val, bool use_memmodel, enum memmodel model, bool after)
7863 machine_mode mode = GET_MODE (mem);
7864 struct expand_operand ops[4];
7865 enum insn_code icode;
7866 int op_counter = 0;
7867 int num_ops;
7869 /* Check to see if there is a result returned. */
7870 if (target == const0_rtx)
7872 if (use_memmodel)
7874 icode = direct_optab_handler (optab->mem_no_result, mode);
7875 create_integer_operand (&ops[2], model);
7876 num_ops = 3;
7878 else
7880 icode = direct_optab_handler (optab->no_result, mode);
7881 num_ops = 2;
7884 /* Otherwise, we need to generate a result. */
7885 else
7887 if (use_memmodel)
7889 icode = direct_optab_handler (after ? optab->mem_fetch_after
7890 : optab->mem_fetch_before, mode);
7891 create_integer_operand (&ops[3], model);
7892 num_ops = 4;
7894 else
7896 icode = optab_handler (after ? optab->fetch_after
7897 : optab->fetch_before, mode);
7898 num_ops = 3;
7900 create_output_operand (&ops[op_counter++], target, mode);
7902 if (icode == CODE_FOR_nothing)
7903 return NULL_RTX;
7905 create_fixed_operand (&ops[op_counter++], mem);
7906 /* VAL may have been promoted to a wider mode. Shrink it if so. */
7907 create_convert_operand_to (&ops[op_counter++], val, mode, true);
7909 if (maybe_expand_insn (icode, num_ops, ops))
7910 return (target == const0_rtx ? const0_rtx : ops[0].value);
7912 return NULL_RTX;
7916 /* This function expands an atomic fetch_OP or OP_fetch operation:
7917 TARGET is an option place to stick the return value. const0_rtx indicates
7918 the result is unused.
7919 atomically fetch MEM, perform the operation with VAL and return it to MEM.
7920 CODE is the operation being performed (OP)
7921 MEMMODEL is the memory model variant to use.
7922 AFTER is true to return the result of the operation (OP_fetch).
7923 AFTER is false to return the value before the operation (fetch_OP).
7925 This function will *only* generate instructions if there is a direct
7926 optab. No compare and swap loops or libcalls will be generated. */
7928 static rtx
7929 expand_atomic_fetch_op_no_fallback (rtx target, rtx mem, rtx val,
7930 enum rtx_code code, enum memmodel model,
7931 bool after)
7933 machine_mode mode = GET_MODE (mem);
7934 struct atomic_op_functions optab;
7935 rtx result;
7936 bool unused_result = (target == const0_rtx);
7938 get_atomic_op_for_code (&optab, code);
7940 /* Check to see if there are any better instructions. */
7941 result = maybe_optimize_fetch_op (target, mem, val, code, model, after);
7942 if (result)
7943 return result;
7945 /* Check for the case where the result isn't used and try those patterns. */
7946 if (unused_result)
7948 /* Try the memory model variant first. */
7949 result = maybe_emit_op (&optab, target, mem, val, true, model, true);
7950 if (result)
7951 return result;
7953 /* Next try the old style withuot a memory model. */
7954 result = maybe_emit_op (&optab, target, mem, val, false, model, true);
7955 if (result)
7956 return result;
7958 /* There is no no-result pattern, so try patterns with a result. */
7959 target = NULL_RTX;
7962 /* Try the __atomic version. */
7963 result = maybe_emit_op (&optab, target, mem, val, true, model, after);
7964 if (result)
7965 return result;
7967 /* Try the older __sync version. */
7968 result = maybe_emit_op (&optab, target, mem, val, false, model, after);
7969 if (result)
7970 return result;
7972 /* If the fetch value can be calculated from the other variation of fetch,
7973 try that operation. */
7974 if (after || unused_result || optab.reverse_code != UNKNOWN)
7976 /* Try the __atomic version, then the older __sync version. */
7977 result = maybe_emit_op (&optab, target, mem, val, true, model, !after);
7978 if (!result)
7979 result = maybe_emit_op (&optab, target, mem, val, false, model, !after);
7981 if (result)
7983 /* If the result isn't used, no need to do compensation code. */
7984 if (unused_result)
7985 return result;
7987 /* Issue compensation code. Fetch_after == fetch_before OP val.
7988 Fetch_before == after REVERSE_OP val. */
7989 if (!after)
7990 code = optab.reverse_code;
7991 if (code == NOT)
7993 result = expand_simple_binop (mode, AND, result, val, NULL_RTX,
7994 true, OPTAB_LIB_WIDEN);
7995 result = expand_simple_unop (mode, NOT, result, target, true);
7997 else
7998 result = expand_simple_binop (mode, code, result, val, target,
7999 true, OPTAB_LIB_WIDEN);
8000 return result;
8004 /* No direct opcode can be generated. */
8005 return NULL_RTX;
8010 /* This function expands an atomic fetch_OP or OP_fetch operation:
8011 TARGET is an option place to stick the return value. const0_rtx indicates
8012 the result is unused.
8013 atomically fetch MEM, perform the operation with VAL and return it to MEM.
8014 CODE is the operation being performed (OP)
8015 MEMMODEL is the memory model variant to use.
8016 AFTER is true to return the result of the operation (OP_fetch).
8017 AFTER is false to return the value before the operation (fetch_OP). */
8019 expand_atomic_fetch_op (rtx target, rtx mem, rtx val, enum rtx_code code,
8020 enum memmodel model, bool after)
8022 machine_mode mode = GET_MODE (mem);
8023 rtx result;
8024 bool unused_result = (target == const0_rtx);
8026 result = expand_atomic_fetch_op_no_fallback (target, mem, val, code, model,
8027 after);
8029 if (result)
8030 return result;
8032 /* Add/sub can be implemented by doing the reverse operation with -(val). */
8033 if (code == PLUS || code == MINUS)
8035 rtx tmp;
8036 enum rtx_code reverse = (code == PLUS ? MINUS : PLUS);
8038 start_sequence ();
8039 tmp = expand_simple_unop (mode, NEG, val, NULL_RTX, true);
8040 result = expand_atomic_fetch_op_no_fallback (target, mem, tmp, reverse,
8041 model, after);
8042 if (result)
8044 /* PLUS worked so emit the insns and return. */
8045 tmp = get_insns ();
8046 end_sequence ();
8047 emit_insn (tmp);
8048 return result;
8051 /* PLUS did not work, so throw away the negation code and continue. */
8052 end_sequence ();
8055 /* Try the __sync libcalls only if we can't do compare-and-swap inline. */
8056 if (!can_compare_and_swap_p (mode, false))
8058 rtx libfunc;
8059 bool fixup = false;
8060 enum rtx_code orig_code = code;
8061 struct atomic_op_functions optab;
8063 get_atomic_op_for_code (&optab, code);
8064 libfunc = optab_libfunc (after ? optab.fetch_after
8065 : optab.fetch_before, mode);
8066 if (libfunc == NULL
8067 && (after || unused_result || optab.reverse_code != UNKNOWN))
8069 fixup = true;
8070 if (!after)
8071 code = optab.reverse_code;
8072 libfunc = optab_libfunc (after ? optab.fetch_before
8073 : optab.fetch_after, mode);
8075 if (libfunc != NULL)
8077 rtx addr = convert_memory_address (ptr_mode, XEXP (mem, 0));
8078 result = emit_library_call_value (libfunc, NULL, LCT_NORMAL, mode,
8079 2, addr, ptr_mode, val, mode);
8081 if (!unused_result && fixup)
8082 result = expand_simple_binop (mode, code, result, val, target,
8083 true, OPTAB_LIB_WIDEN);
8084 return result;
8087 /* We need the original code for any further attempts. */
8088 code = orig_code;
8091 /* If nothing else has succeeded, default to a compare and swap loop. */
8092 if (can_compare_and_swap_p (mode, true))
8094 rtx_insn *insn;
8095 rtx t0 = gen_reg_rtx (mode), t1;
8097 start_sequence ();
8099 /* If the result is used, get a register for it. */
8100 if (!unused_result)
8102 if (!target || !register_operand (target, mode))
8103 target = gen_reg_rtx (mode);
8104 /* If fetch_before, copy the value now. */
8105 if (!after)
8106 emit_move_insn (target, t0);
8108 else
8109 target = const0_rtx;
8111 t1 = t0;
8112 if (code == NOT)
8114 t1 = expand_simple_binop (mode, AND, t1, val, NULL_RTX,
8115 true, OPTAB_LIB_WIDEN);
8116 t1 = expand_simple_unop (mode, code, t1, NULL_RTX, true);
8118 else
8119 t1 = expand_simple_binop (mode, code, t1, val, NULL_RTX, true,
8120 OPTAB_LIB_WIDEN);
8122 /* For after, copy the value now. */
8123 if (!unused_result && after)
8124 emit_move_insn (target, t1);
8125 insn = get_insns ();
8126 end_sequence ();
8128 if (t1 != NULL && expand_compare_and_swap_loop (mem, t0, t1, insn))
8129 return target;
8132 return NULL_RTX;
8135 /* Return true if OPERAND is suitable for operand number OPNO of
8136 instruction ICODE. */
8138 bool
8139 insn_operand_matches (enum insn_code icode, unsigned int opno, rtx operand)
8141 return (!insn_data[(int) icode].operand[opno].predicate
8142 || (insn_data[(int) icode].operand[opno].predicate
8143 (operand, insn_data[(int) icode].operand[opno].mode)));
8146 /* TARGET is a target of a multiword operation that we are going to
8147 implement as a series of word-mode operations. Return true if
8148 TARGET is suitable for this purpose. */
8150 bool
8151 valid_multiword_target_p (rtx target)
8153 machine_mode mode;
8154 int i;
8156 mode = GET_MODE (target);
8157 for (i = 0; i < GET_MODE_SIZE (mode); i += UNITS_PER_WORD)
8158 if (!validate_subreg (word_mode, mode, target, i))
8159 return false;
8160 return true;
8163 /* Like maybe_legitimize_operand, but do not change the code of the
8164 current rtx value. */
8166 static bool
8167 maybe_legitimize_operand_same_code (enum insn_code icode, unsigned int opno,
8168 struct expand_operand *op)
8170 /* See if the operand matches in its current form. */
8171 if (insn_operand_matches (icode, opno, op->value))
8172 return true;
8174 /* If the operand is a memory whose address has no side effects,
8175 try forcing the address into a non-virtual pseudo register.
8176 The check for side effects is important because copy_to_mode_reg
8177 cannot handle things like auto-modified addresses. */
8178 if (insn_data[(int) icode].operand[opno].allows_mem && MEM_P (op->value))
8180 rtx addr, mem;
8182 mem = op->value;
8183 addr = XEXP (mem, 0);
8184 if (!(REG_P (addr) && REGNO (addr) > LAST_VIRTUAL_REGISTER)
8185 && !side_effects_p (addr))
8187 rtx_insn *last;
8188 machine_mode mode;
8190 last = get_last_insn ();
8191 mode = get_address_mode (mem);
8192 mem = replace_equiv_address (mem, copy_to_mode_reg (mode, addr));
8193 if (insn_operand_matches (icode, opno, mem))
8195 op->value = mem;
8196 return true;
8198 delete_insns_since (last);
8202 return false;
8205 /* Try to make OP match operand OPNO of instruction ICODE. Return true
8206 on success, storing the new operand value back in OP. */
8208 static bool
8209 maybe_legitimize_operand (enum insn_code icode, unsigned int opno,
8210 struct expand_operand *op)
8212 machine_mode mode, imode;
8213 bool old_volatile_ok, result;
8215 mode = op->mode;
8216 switch (op->type)
8218 case EXPAND_FIXED:
8219 old_volatile_ok = volatile_ok;
8220 volatile_ok = true;
8221 result = maybe_legitimize_operand_same_code (icode, opno, op);
8222 volatile_ok = old_volatile_ok;
8223 return result;
8225 case EXPAND_OUTPUT:
8226 gcc_assert (mode != VOIDmode);
8227 if (op->value
8228 && op->value != const0_rtx
8229 && GET_MODE (op->value) == mode
8230 && maybe_legitimize_operand_same_code (icode, opno, op))
8231 return true;
8233 op->value = gen_reg_rtx (mode);
8234 break;
8236 case EXPAND_INPUT:
8237 input:
8238 gcc_assert (mode != VOIDmode);
8239 gcc_assert (GET_MODE (op->value) == VOIDmode
8240 || GET_MODE (op->value) == mode);
8241 if (maybe_legitimize_operand_same_code (icode, opno, op))
8242 return true;
8244 op->value = copy_to_mode_reg (mode, op->value);
8245 break;
8247 case EXPAND_CONVERT_TO:
8248 gcc_assert (mode != VOIDmode);
8249 op->value = convert_to_mode (mode, op->value, op->unsigned_p);
8250 goto input;
8252 case EXPAND_CONVERT_FROM:
8253 if (GET_MODE (op->value) != VOIDmode)
8254 mode = GET_MODE (op->value);
8255 else
8256 /* The caller must tell us what mode this value has. */
8257 gcc_assert (mode != VOIDmode);
8259 imode = insn_data[(int) icode].operand[opno].mode;
8260 if (imode != VOIDmode && imode != mode)
8262 op->value = convert_modes (imode, mode, op->value, op->unsigned_p);
8263 mode = imode;
8265 goto input;
8267 case EXPAND_ADDRESS:
8268 gcc_assert (mode != VOIDmode);
8269 op->value = convert_memory_address (mode, op->value);
8270 goto input;
8272 case EXPAND_INTEGER:
8273 mode = insn_data[(int) icode].operand[opno].mode;
8274 if (mode != VOIDmode && const_int_operand (op->value, mode))
8275 goto input;
8276 break;
8278 return insn_operand_matches (icode, opno, op->value);
8281 /* Make OP describe an input operand that should have the same value
8282 as VALUE, after any mode conversion that the target might request.
8283 TYPE is the type of VALUE. */
8285 void
8286 create_convert_operand_from_type (struct expand_operand *op,
8287 rtx value, tree type)
8289 create_convert_operand_from (op, value, TYPE_MODE (type),
8290 TYPE_UNSIGNED (type));
8293 /* Try to make operands [OPS, OPS + NOPS) match operands [OPNO, OPNO + NOPS)
8294 of instruction ICODE. Return true on success, leaving the new operand
8295 values in the OPS themselves. Emit no code on failure. */
8297 bool
8298 maybe_legitimize_operands (enum insn_code icode, unsigned int opno,
8299 unsigned int nops, struct expand_operand *ops)
8301 rtx_insn *last;
8302 unsigned int i;
8304 last = get_last_insn ();
8305 for (i = 0; i < nops; i++)
8306 if (!maybe_legitimize_operand (icode, opno + i, &ops[i]))
8308 delete_insns_since (last);
8309 return false;
8311 return true;
8314 /* Try to generate instruction ICODE, using operands [OPS, OPS + NOPS)
8315 as its operands. Return the instruction pattern on success,
8316 and emit any necessary set-up code. Return null and emit no
8317 code on failure. */
8319 rtx_insn *
8320 maybe_gen_insn (enum insn_code icode, unsigned int nops,
8321 struct expand_operand *ops)
8323 gcc_assert (nops == (unsigned int) insn_data[(int) icode].n_generator_args);
8324 if (!maybe_legitimize_operands (icode, 0, nops, ops))
8325 return NULL;
8327 switch (nops)
8329 case 1:
8330 return GEN_FCN (icode) (ops[0].value);
8331 case 2:
8332 return GEN_FCN (icode) (ops[0].value, ops[1].value);
8333 case 3:
8334 return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value);
8335 case 4:
8336 return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8337 ops[3].value);
8338 case 5:
8339 return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8340 ops[3].value, ops[4].value);
8341 case 6:
8342 return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8343 ops[3].value, ops[4].value, ops[5].value);
8344 case 7:
8345 return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8346 ops[3].value, ops[4].value, ops[5].value,
8347 ops[6].value);
8348 case 8:
8349 return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8350 ops[3].value, ops[4].value, ops[5].value,
8351 ops[6].value, ops[7].value);
8352 case 9:
8353 return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8354 ops[3].value, ops[4].value, ops[5].value,
8355 ops[6].value, ops[7].value, ops[8].value);
8357 gcc_unreachable ();
8360 /* Try to emit instruction ICODE, using operands [OPS, OPS + NOPS)
8361 as its operands. Return true on success and emit no code on failure. */
8363 bool
8364 maybe_expand_insn (enum insn_code icode, unsigned int nops,
8365 struct expand_operand *ops)
8367 rtx_insn *pat = maybe_gen_insn (icode, nops, ops);
8368 if (pat)
8370 emit_insn (pat);
8371 return true;
8373 return false;
8376 /* Like maybe_expand_insn, but for jumps. */
8378 bool
8379 maybe_expand_jump_insn (enum insn_code icode, unsigned int nops,
8380 struct expand_operand *ops)
8382 rtx_insn *pat = maybe_gen_insn (icode, nops, ops);
8383 if (pat)
8385 emit_jump_insn (pat);
8386 return true;
8388 return false;
8391 /* Emit instruction ICODE, using operands [OPS, OPS + NOPS)
8392 as its operands. */
8394 void
8395 expand_insn (enum insn_code icode, unsigned int nops,
8396 struct expand_operand *ops)
8398 if (!maybe_expand_insn (icode, nops, ops))
8399 gcc_unreachable ();
8402 /* Like expand_insn, but for jumps. */
8404 void
8405 expand_jump_insn (enum insn_code icode, unsigned int nops,
8406 struct expand_operand *ops)
8408 if (!maybe_expand_jump_insn (icode, nops, ops))
8409 gcc_unreachable ();
8412 /* Reduce conditional compilation elsewhere. */
8414 /* Enumerates the possible types of structure operand to an
8415 extraction_insn. */
8416 enum extraction_type { ET_unaligned_mem, ET_reg };
8418 /* Check whether insv, extv or extzv pattern ICODE can be used for an
8419 insertion or extraction of type TYPE on a structure of mode MODE.
8420 Return true if so and fill in *INSN accordingly. STRUCT_OP is the
8421 operand number of the structure (the first sign_extract or zero_extract
8422 operand) and FIELD_OP is the operand number of the field (the other
8423 side of the set from the sign_extract or zero_extract). */
8425 static bool
8426 get_traditional_extraction_insn (extraction_insn *insn,
8427 enum extraction_type type,
8428 machine_mode mode,
8429 enum insn_code icode,
8430 int struct_op, int field_op)
8432 const struct insn_data_d *data = &insn_data[icode];
8434 machine_mode struct_mode = data->operand[struct_op].mode;
8435 if (struct_mode == VOIDmode)
8436 struct_mode = word_mode;
8437 if (mode != struct_mode)
8438 return false;
8440 machine_mode field_mode = data->operand[field_op].mode;
8441 if (field_mode == VOIDmode)
8442 field_mode = word_mode;
8444 machine_mode pos_mode = data->operand[struct_op + 2].mode;
8445 if (pos_mode == VOIDmode)
8446 pos_mode = word_mode;
8448 insn->icode = icode;
8449 insn->field_mode = field_mode;
8450 insn->struct_mode = (type == ET_unaligned_mem ? byte_mode : struct_mode);
8451 insn->pos_mode = pos_mode;
8452 return true;
8455 /* Return true if an optab exists to perform an insertion or extraction
8456 of type TYPE in mode MODE. Describe the instruction in *INSN if so.
8458 REG_OPTAB is the optab to use for register structures and
8459 MISALIGN_OPTAB is the optab to use for misaligned memory structures.
8460 POS_OP is the operand number of the bit position. */
8462 static bool
8463 get_optab_extraction_insn (struct extraction_insn *insn,
8464 enum extraction_type type,
8465 machine_mode mode, direct_optab reg_optab,
8466 direct_optab misalign_optab, int pos_op)
8468 direct_optab optab = (type == ET_unaligned_mem ? misalign_optab : reg_optab);
8469 enum insn_code icode = direct_optab_handler (optab, mode);
8470 if (icode == CODE_FOR_nothing)
8471 return false;
8473 const struct insn_data_d *data = &insn_data[icode];
8475 insn->icode = icode;
8476 insn->field_mode = mode;
8477 insn->struct_mode = (type == ET_unaligned_mem ? BLKmode : mode);
8478 insn->pos_mode = data->operand[pos_op].mode;
8479 if (insn->pos_mode == VOIDmode)
8480 insn->pos_mode = word_mode;
8481 return true;
8484 /* Return true if an instruction exists to perform an insertion or
8485 extraction (PATTERN says which) of type TYPE in mode MODE.
8486 Describe the instruction in *INSN if so. */
8488 static bool
8489 get_extraction_insn (extraction_insn *insn,
8490 enum extraction_pattern pattern,
8491 enum extraction_type type,
8492 machine_mode mode)
8494 switch (pattern)
8496 case EP_insv:
8497 if (targetm.have_insv ()
8498 && get_traditional_extraction_insn (insn, type, mode,
8499 targetm.code_for_insv, 0, 3))
8500 return true;
8501 return get_optab_extraction_insn (insn, type, mode, insv_optab,
8502 insvmisalign_optab, 2);
8504 case EP_extv:
8505 if (targetm.have_extv ()
8506 && get_traditional_extraction_insn (insn, type, mode,
8507 targetm.code_for_extv, 1, 0))
8508 return true;
8509 return get_optab_extraction_insn (insn, type, mode, extv_optab,
8510 extvmisalign_optab, 3);
8512 case EP_extzv:
8513 if (targetm.have_extzv ()
8514 && get_traditional_extraction_insn (insn, type, mode,
8515 targetm.code_for_extzv, 1, 0))
8516 return true;
8517 return get_optab_extraction_insn (insn, type, mode, extzv_optab,
8518 extzvmisalign_optab, 3);
8520 default:
8521 gcc_unreachable ();
8525 /* Return true if an instruction exists to access a field of mode
8526 FIELDMODE in a structure that has STRUCT_BITS significant bits.
8527 Describe the "best" such instruction in *INSN if so. PATTERN and
8528 TYPE describe the type of insertion or extraction we want to perform.
8530 For an insertion, the number of significant structure bits includes
8531 all bits of the target. For an extraction, it need only include the
8532 most significant bit of the field. Larger widths are acceptable
8533 in both cases. */
8535 static bool
8536 get_best_extraction_insn (extraction_insn *insn,
8537 enum extraction_pattern pattern,
8538 enum extraction_type type,
8539 unsigned HOST_WIDE_INT struct_bits,
8540 machine_mode field_mode)
8542 machine_mode mode = smallest_mode_for_size (struct_bits, MODE_INT);
8543 while (mode != VOIDmode)
8545 if (get_extraction_insn (insn, pattern, type, mode))
8547 while (mode != VOIDmode
8548 && GET_MODE_SIZE (mode) <= GET_MODE_SIZE (field_mode)
8549 && !TRULY_NOOP_TRUNCATION_MODES_P (insn->field_mode,
8550 field_mode))
8552 get_extraction_insn (insn, pattern, type, mode);
8553 mode = GET_MODE_WIDER_MODE (mode);
8555 return true;
8557 mode = GET_MODE_WIDER_MODE (mode);
8559 return false;
8562 /* Return true if an instruction exists to access a field of mode
8563 FIELDMODE in a register structure that has STRUCT_BITS significant bits.
8564 Describe the "best" such instruction in *INSN if so. PATTERN describes
8565 the type of insertion or extraction we want to perform.
8567 For an insertion, the number of significant structure bits includes
8568 all bits of the target. For an extraction, it need only include the
8569 most significant bit of the field. Larger widths are acceptable
8570 in both cases. */
8572 bool
8573 get_best_reg_extraction_insn (extraction_insn *insn,
8574 enum extraction_pattern pattern,
8575 unsigned HOST_WIDE_INT struct_bits,
8576 machine_mode field_mode)
8578 return get_best_extraction_insn (insn, pattern, ET_reg, struct_bits,
8579 field_mode);
8582 /* Return true if an instruction exists to access a field of BITSIZE
8583 bits starting BITNUM bits into a memory structure. Describe the
8584 "best" such instruction in *INSN if so. PATTERN describes the type
8585 of insertion or extraction we want to perform and FIELDMODE is the
8586 natural mode of the extracted field.
8588 The instructions considered here only access bytes that overlap
8589 the bitfield; they do not touch any surrounding bytes. */
8591 bool
8592 get_best_mem_extraction_insn (extraction_insn *insn,
8593 enum extraction_pattern pattern,
8594 HOST_WIDE_INT bitsize, HOST_WIDE_INT bitnum,
8595 machine_mode field_mode)
8597 unsigned HOST_WIDE_INT struct_bits = (bitnum % BITS_PER_UNIT
8598 + bitsize
8599 + BITS_PER_UNIT - 1);
8600 struct_bits -= struct_bits % BITS_PER_UNIT;
8601 return get_best_extraction_insn (insn, pattern, ET_unaligned_mem,
8602 struct_bits, field_mode);
8605 /* Determine whether "1 << x" is relatively cheap in word_mode. */
8607 bool
8608 lshift_cheap_p (bool speed_p)
8610 /* FIXME: This should be made target dependent via this "this_target"
8611 mechanism, similar to e.g. can_copy_init_p in gcse.c. */
8612 static bool init[2] = { false, false };
8613 static bool cheap[2] = { true, true };
8615 /* If the targer has no lshift in word_mode, the operation will most
8616 probably not be cheap. ??? Does GCC even work for such targets? */
8617 if (optab_handler (ashl_optab, word_mode) == CODE_FOR_nothing)
8618 return false;
8620 if (!init[speed_p])
8622 rtx reg = gen_raw_REG (word_mode, 10000);
8623 int cost = set_src_cost (gen_rtx_ASHIFT (word_mode, const1_rtx, reg),
8624 word_mode, speed_p);
8625 cheap[speed_p] = cost < COSTS_N_INSNS (3);
8626 init[speed_p] = true;
8629 return cheap[speed_p];
8632 #include "gt-optabs.h"