1 ;; Machine description for AArch64 architecture.
2 ;; Copyright (C) 2009-2018 Free Software Foundation, Inc.
3 ;; Contributed by ARM Ltd.
5 ;; This file is part of GCC.
7 ;; GCC is free software; you can redistribute it and/or modify it
8 ;; under the terms of the GNU General Public License as published by
9 ;; the Free Software Foundation; either version 3, or (at your option)
12 ;; GCC is distributed in the hope that it will be useful, but
13 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
14 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 ;; General Public License for more details.
17 ;; You should have received a copy of the GNU General Public License
18 ;; along with GCC; see the file COPYING3. If not see
19 ;; <http://www.gnu.org/licenses/>.
91 (LAST_SAVED_REGNUM 63)
95 ;; Defined only to make the DWARF description simpler.
113 ;; A couple of call-clobbered registers that we need to reserve when
114 ;; tracking speculation this is not ABI, so is subject to change.
115 (SPECULATION_TRACKER_REGNUM 15)
116 (SPECULATION_SCRATCH_REGNUM 14)
120 (define_c_enum "unspec" [
146 UNSPEC_GOTSMALLPIC28K
224 UNSPEC_SPECULATION_TRACKER
227 (define_c_enum "unspecv" [
228 UNSPECV_EH_RETURN ; Represent EH_RETURN
229 UNSPECV_GET_FPCR ; Represent fetch of FPCR content.
230 UNSPECV_SET_FPCR ; Represent assign of FPCR content.
231 UNSPECV_GET_FPSR ; Represent fetch of FPSR content.
232 UNSPECV_SET_FPSR ; Represent assign of FPSR content.
233 UNSPECV_BLOCKAGE ; Represent a blockage
234 UNSPECV_PROBE_STACK_RANGE ; Represent stack range probing.
235 UNSPECV_SPECULATION_BARRIER ; Represent speculation barrier.
239 ;; If further include files are added the defintion of MD_INCLUDES
242 (include "constraints.md")
243 (include "predicates.md")
244 (include "iterators.md")
246 ;; -------------------------------------------------------------------
247 ;; Instruction types and attributes
248 ;; -------------------------------------------------------------------
250 ; The "type" attribute is included here from AArch32 backend to be able
251 ; to share pipeline descriptions.
252 (include "../arm/types.md")
254 ;; It is important to set the fp or simd attributes to yes when a pattern
255 ;; alternative uses the FP or SIMD register files, usually signified by use of
256 ;; the 'w' constraint. This will ensure that the alternative will be
257 ;; disabled when compiling with -mgeneral-regs-only or with the +nofp/+nosimd
258 ;; architecture extensions. If all the alternatives in a pattern use the
259 ;; FP or SIMD registers then the pattern predicate should include TARGET_FLOAT
262 ;; Attributes of the architecture required to support the instruction (or
263 ;; alternative). This attribute is used to compute attribute "enabled", use type
264 ;; "any" to enable an alternative in all cases.
266 (define_enum "arches" [ any fp simd sve fp16])
268 (define_enum_attr "arch" "arches" (const_string "any"))
270 ;; [For compatibility with Arm in pipeline models]
271 ;; Attribute that specifies whether or not the instruction touches fp
273 ;; Note that this attribute is not used anywhere in either the arm or aarch64
274 ;; backends except in the scheduling description for xgene1. In that
275 ;; scheduling description this attribute is used to subclass the load_4 and
277 (define_attr "fp" "no,yes"
279 (eq_attr "arch" "fp")
281 (const_string "no")))
283 (define_attr "arch_enabled" "no,yes"
286 (eq_attr "arch" "any")
288 (and (eq_attr "arch" "fp")
289 (match_test "TARGET_FLOAT"))
291 (and (eq_attr "arch" "simd")
292 (match_test "TARGET_SIMD"))
294 (and (eq_attr "arch" "fp16")
295 (match_test "TARGET_FP_F16INST"))
297 (and (eq_attr "arch" "sve")
298 (match_test "TARGET_SVE")))
300 (const_string "no")))
302 ;; Attribute that controls whether an alternative is enabled or not.
303 ;; Currently it is only used to disable alternatives which touch fp or simd
304 ;; registers when -mgeneral-regs-only is specified or to require a special
305 ;; architecture support.
306 (define_attr "enabled" "no,yes" (attr "arch_enabled"))
308 ;; Attribute that specifies whether we are dealing with a branch to a
309 ;; label that is far away, i.e. further away than the maximum/minimum
310 ;; representable in a signed 21-bits number.
313 (define_attr "far_branch" "" (const_int 0))
315 ;; Attribute that specifies whether the alternative uses MOVPRFX.
316 (define_attr "movprfx" "no,yes" (const_string "no"))
318 (define_attr "length" ""
319 (cond [(eq_attr "movprfx" "yes")
323 ;; Strictly for compatibility with AArch32 in pipeline models, since AArch64 has
324 ;; no predicated insns.
325 (define_attr "predicated" "yes,no" (const_string "no"))
327 ;; Set to true on an insn that requires the speculation tracking state to be
328 ;; in the tracking register before the insn issues. Otherwise the compiler
329 ;; may chose to hold the tracking state encoded in SP.
330 (define_attr "speculation_barrier" "true,false" (const_string "false"))
332 ;; -------------------------------------------------------------------
333 ;; Pipeline descriptions and scheduling
334 ;; -------------------------------------------------------------------
337 (include "aarch64-tune.md")
340 (include "../arm/cortex-a53.md")
341 (include "../arm/cortex-a57.md")
342 (include "../arm/exynos-m1.md")
343 (include "falkor.md")
344 (include "thunderx.md")
345 (include "../arm/xgene1.md")
346 (include "thunderx2t99.md")
348 ;; -------------------------------------------------------------------
349 ;; Jumps and other miscellaneous insns
350 ;; -------------------------------------------------------------------
352 (define_insn "indirect_jump"
353 [(set (pc) (match_operand:DI 0 "register_operand" "r"))]
356 [(set_attr "type" "branch")]
360 [(set (pc) (label_ref (match_operand 0 "" "")))]
363 [(set_attr "type" "branch")]
366 (define_expand "cbranch<mode>4"
367 [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
368 [(match_operand:GPI 1 "register_operand" "")
369 (match_operand:GPI 2 "aarch64_plus_operand" "")])
370 (label_ref (match_operand 3 "" ""))
374 operands[1] = aarch64_gen_compare_reg (GET_CODE (operands[0]), operands[1],
376 operands[2] = const0_rtx;
380 (define_expand "cbranch<mode>4"
381 [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
382 [(match_operand:GPF 1 "register_operand" "")
383 (match_operand:GPF 2 "aarch64_fp_compare_operand" "")])
384 (label_ref (match_operand 3 "" ""))
388 operands[1] = aarch64_gen_compare_reg (GET_CODE (operands[0]), operands[1],
390 operands[2] = const0_rtx;
394 (define_expand "cbranchcc4"
395 [(set (pc) (if_then_else
396 (match_operator 0 "aarch64_comparison_operator"
397 [(match_operand 1 "cc_register" "")
398 (match_operand 2 "const0_operand")])
399 (label_ref (match_operand 3 "" ""))
404 (define_insn "ccmp<mode>"
405 [(set (match_operand:CC 1 "cc_register" "")
407 (match_operator 4 "aarch64_comparison_operator"
408 [(match_operand 0 "cc_register" "")
411 (match_operand:GPI 2 "register_operand" "r,r,r")
412 (match_operand:GPI 3 "aarch64_ccmp_operand" "r,Uss,Usn"))
413 (unspec:CC [(match_operand 5 "immediate_operand")] UNSPEC_NZCV)))]
416 ccmp\\t%<w>2, %<w>3, %k5, %m4
417 ccmp\\t%<w>2, %3, %k5, %m4
418 ccmn\\t%<w>2, #%n3, %k5, %m4"
419 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
422 (define_insn "fccmp<mode>"
423 [(set (match_operand:CCFP 1 "cc_register" "")
425 (match_operator 4 "aarch64_comparison_operator"
426 [(match_operand 0 "cc_register" "")
429 (match_operand:GPF 2 "register_operand" "w")
430 (match_operand:GPF 3 "register_operand" "w"))
431 (unspec:CCFP [(match_operand 5 "immediate_operand")] UNSPEC_NZCV)))]
433 "fccmp\\t%<s>2, %<s>3, %k5, %m4"
434 [(set_attr "type" "fccmp<s>")]
437 (define_insn "fccmpe<mode>"
438 [(set (match_operand:CCFPE 1 "cc_register" "")
440 (match_operator 4 "aarch64_comparison_operator"
441 [(match_operand 0 "cc_register" "")
444 (match_operand:GPF 2 "register_operand" "w")
445 (match_operand:GPF 3 "register_operand" "w"))
446 (unspec:CCFPE [(match_operand 5 "immediate_operand")] UNSPEC_NZCV)))]
448 "fccmpe\\t%<s>2, %<s>3, %k5, %m4"
449 [(set_attr "type" "fccmp<s>")]
452 ;; Expansion of signed mod by a power of 2 using CSNEG.
453 ;; For x0 % n where n is a power of 2 produce:
455 ;; and x0, x0, #(n - 1)
456 ;; and x1, x1, #(n - 1)
457 ;; csneg x0, x0, x1, mi
459 (define_expand "mod<mode>3"
460 [(match_operand:GPI 0 "register_operand" "")
461 (match_operand:GPI 1 "register_operand" "")
462 (match_operand:GPI 2 "const_int_operand" "")]
465 HOST_WIDE_INT val = INTVAL (operands[2]);
468 || exact_log2 (val) <= 0
469 || !aarch64_bitmask_imm (val - 1, <MODE>mode))
472 rtx mask = GEN_INT (val - 1);
474 /* In the special case of x0 % 2 we can do the even shorter:
480 rtx masked = gen_reg_rtx (<MODE>mode);
481 rtx ccreg = aarch64_gen_compare_reg (LT, operands[1], const0_rtx);
482 emit_insn (gen_and<mode>3 (masked, operands[1], mask));
483 rtx x = gen_rtx_LT (VOIDmode, ccreg, const0_rtx);
484 emit_insn (gen_csneg3<mode>_insn (operands[0], x, masked, masked));
488 rtx neg_op = gen_reg_rtx (<MODE>mode);
489 rtx_insn *insn = emit_insn (gen_neg<mode>2_compare0 (neg_op, operands[1]));
491 /* Extract the condition register and mode. */
492 rtx cmp = XVECEXP (PATTERN (insn), 0, 0);
493 rtx cc_reg = SET_DEST (cmp);
494 rtx cond = gen_rtx_GE (VOIDmode, cc_reg, const0_rtx);
496 rtx masked_pos = gen_reg_rtx (<MODE>mode);
497 emit_insn (gen_and<mode>3 (masked_pos, operands[1], mask));
499 rtx masked_neg = gen_reg_rtx (<MODE>mode);
500 emit_insn (gen_and<mode>3 (masked_neg, neg_op, mask));
502 emit_insn (gen_csneg3<mode>_insn (operands[0], cond,
503 masked_neg, masked_pos));
508 (define_insn "condjump"
509 [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
510 [(match_operand 1 "cc_register" "") (const_int 0)])
511 (label_ref (match_operand 2 "" ""))
515 if (get_attr_length (insn) == 8)
516 return aarch64_gen_far_branch (operands, 2, "Lbcond", "b%M0\\t");
520 [(set_attr "type" "branch")
522 (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
523 (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
526 (set (attr "far_branch")
527 (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
528 (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
533 ;; For a 24-bit immediate CST we can optimize the compare for equality
534 ;; and branch sequence from:
536 ;; movk x0, #imm2, lsl 16 /* x0 contains CST. */
540 ;; sub x0, x1, #(CST & 0xfff000)
541 ;; subs x0, x0, #(CST & 0x000fff)
543 (define_insn_and_split "*compare_condjump<mode>"
544 [(set (pc) (if_then_else (EQL
545 (match_operand:GPI 0 "register_operand" "r")
546 (match_operand:GPI 1 "aarch64_imm24" "n"))
547 (label_ref:P (match_operand 2 "" ""))
549 "!aarch64_move_imm (INTVAL (operands[1]), <MODE>mode)
550 && !aarch64_plus_operand (operands[1], <MODE>mode)
551 && !reload_completed"
556 HOST_WIDE_INT lo_imm = UINTVAL (operands[1]) & 0xfff;
557 HOST_WIDE_INT hi_imm = UINTVAL (operands[1]) & 0xfff000;
558 rtx tmp = gen_reg_rtx (<MODE>mode);
559 emit_insn (gen_add<mode>3 (tmp, operands[0], GEN_INT (-hi_imm)));
560 emit_insn (gen_add<mode>3_compare0 (tmp, tmp, GEN_INT (-lo_imm)));
561 rtx cc_reg = gen_rtx_REG (CC_NZmode, CC_REGNUM);
562 rtx cmp_rtx = gen_rtx_fmt_ee (<EQL:CMP>, <MODE>mode, cc_reg, const0_rtx);
563 emit_jump_insn (gen_condjump (cmp_rtx, cc_reg, operands[2]));
568 (define_expand "casesi"
569 [(match_operand:SI 0 "register_operand" "") ; Index
570 (match_operand:SI 1 "const_int_operand" "") ; Lower bound
571 (match_operand:SI 2 "const_int_operand" "") ; Total range
572 (match_operand:DI 3 "" "") ; Table label
573 (match_operand:DI 4 "" "")] ; Out of range label
576 if (operands[1] != const0_rtx)
578 rtx reg = gen_reg_rtx (SImode);
580 /* Canonical RTL says that if you have:
584 then this should be emitted as:
588 The use of trunc_int_for_mode ensures that the resulting
589 constant can be represented in SImode, this is important
590 for the corner case where operand[1] is INT_MIN. */
592 operands[1] = GEN_INT (trunc_int_for_mode (-INTVAL (operands[1]), SImode));
594 if (!(*insn_data[CODE_FOR_addsi3].operand[2].predicate)
595 (operands[1], SImode))
596 operands[1] = force_reg (SImode, operands[1]);
597 emit_insn (gen_addsi3 (reg, operands[0], operands[1]));
601 if (!aarch64_plus_operand (operands[2], SImode))
602 operands[2] = force_reg (SImode, operands[2]);
603 emit_jump_insn (gen_cbranchsi4 (gen_rtx_GTU (SImode, const0_rtx,
605 operands[0], operands[2], operands[4]));
607 operands[2] = force_reg (DImode, gen_rtx_LABEL_REF (DImode, operands[3]));
608 emit_jump_insn (gen_casesi_dispatch (operands[2], operands[0],
614 (define_insn "casesi_dispatch"
617 (mem:DI (unspec [(match_operand:DI 0 "register_operand" "r")
618 (match_operand:SI 1 "register_operand" "r")]
620 (clobber (reg:CC CC_REGNUM))
621 (clobber (match_scratch:DI 3 "=r"))
622 (clobber (match_scratch:DI 4 "=r"))
623 (use (label_ref (match_operand 2 "" "")))])]
626 return aarch64_output_casesi (operands);
628 [(set_attr "length" "16")
629 (set_attr "type" "branch")]
633 [(unspec[(const_int 0)] UNSPEC_NOP)]
636 [(set_attr "type" "no_insn")]
639 (define_insn "prefetch"
640 [(prefetch (match_operand:DI 0 "aarch64_prefetch_operand" "Dp")
641 (match_operand:QI 1 "const_int_operand" "")
642 (match_operand:QI 2 "const_int_operand" ""))]
645 const char * pftype[2][4] =
647 {"prfm\\tPLDL1STRM, %0",
648 "prfm\\tPLDL3KEEP, %0",
649 "prfm\\tPLDL2KEEP, %0",
650 "prfm\\tPLDL1KEEP, %0"},
651 {"prfm\\tPSTL1STRM, %0",
652 "prfm\\tPSTL3KEEP, %0",
653 "prfm\\tPSTL2KEEP, %0",
654 "prfm\\tPSTL1KEEP, %0"},
657 int locality = INTVAL (operands[2]);
659 gcc_assert (IN_RANGE (locality, 0, 3));
661 /* PRFM accepts the same addresses as a 64-bit LDR so wrap
662 the address into a DImode MEM so that aarch64_print_operand knows
664 operands[0] = gen_rtx_MEM (DImode, operands[0]);
665 return pftype[INTVAL(operands[1])][locality];
667 [(set_attr "type" "load_4")]
671 [(trap_if (const_int 1) (const_int 8))]
674 [(set_attr "type" "trap")])
676 (define_expand "prologue"
677 [(clobber (const_int 0))]
680 aarch64_expand_prologue ();
685 (define_expand "epilogue"
686 [(clobber (const_int 0))]
689 aarch64_expand_epilogue (false);
694 (define_expand "sibcall_epilogue"
695 [(clobber (const_int 0))]
698 aarch64_expand_epilogue (true);
703 (define_insn "*do_return"
707 if (aarch64_return_address_signing_enabled ()
709 && !crtl->calls_eh_return)
714 [(set_attr "type" "branch")]
717 (define_expand "return"
719 "aarch64_use_return_insn_p ()"
723 (define_insn "simple_return"
727 [(set_attr "type" "branch")]
730 (define_insn "*cb<optab><mode>1"
731 [(set (pc) (if_then_else (EQL (match_operand:GPI 0 "register_operand" "r")
733 (label_ref (match_operand 1 "" ""))
735 "!aarch64_track_speculation"
737 if (get_attr_length (insn) == 8)
738 return aarch64_gen_far_branch (operands, 1, "Lcb", "<inv_cb>\\t%<w>0, ");
740 return "<cbz>\\t%<w>0, %l1";
742 [(set_attr "type" "branch")
744 (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -1048576))
745 (lt (minus (match_dup 1) (pc)) (const_int 1048572)))
748 (set (attr "far_branch")
749 (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
750 (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
755 (define_insn "*tb<optab><mode>1"
756 [(set (pc) (if_then_else
757 (EQL (zero_extract:DI (match_operand:GPI 0 "register_operand" "r")
760 "aarch64_simd_shift_imm_<mode>" "n"))
762 (label_ref (match_operand 2 "" ""))
764 (clobber (reg:CC CC_REGNUM))]
765 "!aarch64_track_speculation"
767 if (get_attr_length (insn) == 8)
769 if (get_attr_far_branch (insn) == 1)
770 return aarch64_gen_far_branch (operands, 2, "Ltb",
771 "<inv_tb>\\t%<w>0, %1, ");
774 operands[1] = GEN_INT (HOST_WIDE_INT_1U << UINTVAL (operands[1]));
775 return "tst\t%<w>0, %1\;<bcond>\t%l2";
779 return "<tbz>\t%<w>0, %1, %l2";
781 [(set_attr "type" "branch")
783 (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -32768))
784 (lt (minus (match_dup 2) (pc)) (const_int 32764)))
787 (set (attr "far_branch")
788 (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
789 (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
795 (define_insn "*cb<optab><mode>1"
796 [(set (pc) (if_then_else (LTGE (match_operand:ALLI 0 "register_operand" "r")
798 (label_ref (match_operand 1 "" ""))
800 (clobber (reg:CC CC_REGNUM))]
801 "!aarch64_track_speculation"
803 if (get_attr_length (insn) == 8)
805 if (get_attr_far_branch (insn) == 1)
806 return aarch64_gen_far_branch (operands, 1, "Ltb",
807 "<inv_tb>\\t%<w>0, <sizem1>, ");
811 uint64_t val = ((uint64_t) 1)
812 << (GET_MODE_SIZE (<MODE>mode) * BITS_PER_UNIT - 1);
813 sprintf (buf, "tst\t%%<w>0, %" PRId64, val);
814 output_asm_insn (buf, operands);
815 return "<bcond>\t%l1";
819 return "<tbz>\t%<w>0, <sizem1>, %l1";
821 [(set_attr "type" "branch")
823 (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -32768))
824 (lt (minus (match_dup 1) (pc)) (const_int 32764)))
827 (set (attr "far_branch")
828 (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -1048576))
829 (lt (minus (match_dup 1) (pc)) (const_int 1048572)))
834 ;; -------------------------------------------------------------------
835 ;; Subroutine calls and sibcalls
836 ;; -------------------------------------------------------------------
838 (define_expand "call"
839 [(parallel [(call (match_operand 0 "memory_operand" "")
840 (match_operand 1 "general_operand" ""))
841 (use (match_operand 2 "" ""))
842 (clobber (reg:DI LR_REGNUM))])]
846 aarch64_expand_call (NULL_RTX, operands[0], false);
851 (define_insn "*call_insn"
852 [(call (mem:DI (match_operand:DI 0 "aarch64_call_insn_operand" "r, Usf"))
853 (match_operand 1 "" ""))
854 (clobber (reg:DI LR_REGNUM))]
859 [(set_attr "type" "call, call")]
862 (define_expand "call_value"
863 [(parallel [(set (match_operand 0 "" "")
864 (call (match_operand 1 "memory_operand" "")
865 (match_operand 2 "general_operand" "")))
866 (use (match_operand 3 "" ""))
867 (clobber (reg:DI LR_REGNUM))])]
871 aarch64_expand_call (operands[0], operands[1], false);
876 (define_insn "*call_value_insn"
877 [(set (match_operand 0 "" "")
878 (call (mem:DI (match_operand:DI 1 "aarch64_call_insn_operand" "r, Usf"))
879 (match_operand 2 "" "")))
880 (clobber (reg:DI LR_REGNUM))]
885 [(set_attr "type" "call, call")]
888 (define_expand "sibcall"
889 [(parallel [(call (match_operand 0 "memory_operand" "")
890 (match_operand 1 "general_operand" ""))
892 (use (match_operand 2 "" ""))])]
895 aarch64_expand_call (NULL_RTX, operands[0], true);
900 (define_expand "sibcall_value"
901 [(parallel [(set (match_operand 0 "" "")
902 (call (match_operand 1 "memory_operand" "")
903 (match_operand 2 "general_operand" "")))
905 (use (match_operand 3 "" ""))])]
908 aarch64_expand_call (operands[0], operands[1], true);
913 (define_insn "*sibcall_insn"
914 [(call (mem:DI (match_operand:DI 0 "aarch64_call_insn_operand" "Ucs, Usf"))
915 (match_operand 1 "" ""))
917 "SIBLING_CALL_P (insn)"
921 [(set_attr "type" "branch, branch")]
924 (define_insn "*sibcall_value_insn"
925 [(set (match_operand 0 "" "")
927 (match_operand:DI 1 "aarch64_call_insn_operand" "Ucs, Usf"))
928 (match_operand 2 "" "")))
930 "SIBLING_CALL_P (insn)"
934 [(set_attr "type" "branch, branch")]
937 ;; Call subroutine returning any type.
939 (define_expand "untyped_call"
940 [(parallel [(call (match_operand 0 "")
943 (match_operand 2 "")])]
948 emit_call_insn (gen_call (operands[0], const0_rtx, NULL));
950 for (i = 0; i < XVECLEN (operands[2], 0); i++)
952 rtx set = XVECEXP (operands[2], 0, i);
953 emit_move_insn (SET_DEST (set), SET_SRC (set));
956 /* The optimizer does not know that the call sets the function value
957 registers we stored in the result block. We avoid problems by
958 claiming that all hard registers are used and clobbered at this
960 emit_insn (gen_blockage ());
964 ;; -------------------------------------------------------------------
966 ;; -------------------------------------------------------------------
968 (define_expand "mov<mode>"
969 [(set (match_operand:SHORT 0 "nonimmediate_operand" "")
970 (match_operand:SHORT 1 "general_operand" ""))]
973 if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
974 operands[1] = force_reg (<MODE>mode, operands[1]);
976 if (GET_CODE (operands[1]) == CONST_POLY_INT)
978 aarch64_expand_mov_immediate (operands[0], operands[1]);
984 (define_insn "*mov<mode>_aarch64"
985 [(set (match_operand:SHORT 0 "nonimmediate_operand" "=r,r, w,r ,r,w, m,m,r,w,w")
986 (match_operand:SHORT 1 "aarch64_mov_operand" " r,M,D<hq>,Usv,m,m,rZ,w,w,r,w"))]
987 "(register_operand (operands[0], <MODE>mode)
988 || aarch64_reg_or_zero (operands[1], <MODE>mode))"
990 switch (which_alternative)
993 return "mov\t%w0, %w1";
995 return "mov\t%w0, %1";
997 return aarch64_output_scalar_simd_mov_immediate (operands[1],
1000 return aarch64_output_sve_cnt_immediate (\"cnt\", \"%x0\", operands[1]);
1002 return "ldr<size>\t%w0, %1";
1004 return "ldr\t%<size>0, %1";
1006 return "str<size>\t%w1, %0";
1008 return "str\t%<size>1, %0";
1010 return "umov\t%w0, %1.<v>[0]";
1012 return "dup\t%0.<Vallxd>, %w1";
1014 return "dup\t%<Vetype>0, %1.<v>[0]";
1019 ;; The "mov_imm" type for CNT is just a placeholder.
1020 [(set_attr "type" "mov_reg,mov_imm,neon_move,mov_imm,load_4,load_4,store_4,
1021 store_4,neon_to_gp<q>,neon_from_gp<q>,neon_dup")
1022 (set_attr "arch" "*,*,simd,sve,*,*,*,*,simd,simd,simd")]
1025 (define_expand "mov<mode>"
1026 [(set (match_operand:GPI 0 "nonimmediate_operand" "")
1027 (match_operand:GPI 1 "general_operand" ""))]
1030 if (MEM_P (operands[0]) && CONST_INT_P (operands[1])
1031 && <MODE>mode == DImode
1032 && aarch64_split_dimode_const_store (operands[0], operands[1]))
1035 if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
1036 operands[1] = force_reg (<MODE>mode, operands[1]);
1038 /* FIXME: RR we still need to fix up what we are doing with
1039 symbol_refs and other types of constants. */
1040 if (CONSTANT_P (operands[1])
1041 && !CONST_INT_P (operands[1]))
1043 aarch64_expand_mov_immediate (operands[0], operands[1]);
1049 (define_insn_and_split "*movsi_aarch64"
1050 [(set (match_operand:SI 0 "nonimmediate_operand" "=r,k,r,r,r,r, r,w, m, m, r, r, w,r,w, w")
1051 (match_operand:SI 1 "aarch64_mov_operand" " r,r,k,M,n,Usv,m,m,rZ,w,Usa,Ush,rZ,w,w,Ds"))]
1052 "(register_operand (operands[0], SImode)
1053 || aarch64_reg_or_zero (operands[1], SImode))"
1060 * return aarch64_output_sve_cnt_immediate (\"cnt\", \"%x0\", operands[1]);
1070 * return aarch64_output_scalar_simd_mov_immediate (operands[1], SImode);"
1071 "CONST_INT_P (operands[1]) && !aarch64_move_imm (INTVAL (operands[1]), SImode)
1072 && REG_P (operands[0]) && GP_REGNUM_P (REGNO (operands[0]))"
1075 aarch64_expand_mov_immediate (operands[0], operands[1]);
1078 ;; The "mov_imm" type for CNT is just a placeholder.
1079 [(set_attr "type" "mov_reg,mov_reg,mov_reg,mov_imm,mov_imm,mov_imm,load_4,
1080 load_4,store_4,store_4,adr,adr,f_mcr,f_mrc,fmov,neon_move")
1081 (set_attr "arch" "*,*,*,*,*,sve,*,fp,*,fp,*,*,fp,fp,fp,simd")]
1084 (define_insn_and_split "*movdi_aarch64"
1085 [(set (match_operand:DI 0 "nonimmediate_operand" "=r,k,r,r,r,r,r, r,w, m,m, r, r, w,r,w, w")
1086 (match_operand:DI 1 "aarch64_mov_operand" " r,r,k,N,M,n,Usv,m,m,rZ,w,Usa,Ush,rZ,w,w,Dd"))]
1087 "(register_operand (operands[0], DImode)
1088 || aarch64_reg_or_zero (operands[1], DImode))"
1096 * return aarch64_output_sve_cnt_immediate (\"cnt\", \"%x0\", operands[1]);
1106 * return aarch64_output_scalar_simd_mov_immediate (operands[1], DImode);"
1107 "(CONST_INT_P (operands[1]) && !aarch64_move_imm (INTVAL (operands[1]), DImode))
1108 && REG_P (operands[0]) && GP_REGNUM_P (REGNO (operands[0]))"
1111 aarch64_expand_mov_immediate (operands[0], operands[1]);
1114 ;; The "mov_imm" type for CNTD is just a placeholder.
1115 [(set_attr "type" "mov_reg,mov_reg,mov_reg,mov_imm,mov_imm,mov_imm,mov_imm,
1116 load_8,load_8,store_8,store_8,adr,adr,f_mcr,f_mrc,fmov,
1118 (set_attr "arch" "*,*,*,*,*,*,sve,*,fp,*,fp,*,*,fp,fp,fp,simd")]
1121 (define_insn "insv_imm<mode>"
1122 [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
1124 (match_operand:GPI 1 "const_int_operand" "n"))
1125 (match_operand:GPI 2 "const_int_operand" "n"))]
1126 "UINTVAL (operands[1]) < GET_MODE_BITSIZE (<MODE>mode)
1127 && UINTVAL (operands[1]) % 16 == 0"
1128 "movk\\t%<w>0, %X2, lsl %1"
1129 [(set_attr "type" "mov_imm")]
1132 (define_expand "movti"
1133 [(set (match_operand:TI 0 "nonimmediate_operand" "")
1134 (match_operand:TI 1 "general_operand" ""))]
1137 if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
1138 operands[1] = force_reg (TImode, operands[1]);
1140 if (GET_CODE (operands[1]) == CONST_POLY_INT)
1142 emit_move_insn (gen_lowpart (DImode, operands[0]),
1143 gen_lowpart (DImode, operands[1]));
1144 emit_move_insn (gen_highpart (DImode, operands[0]), const0_rtx);
1150 (define_insn "*movti_aarch64"
1151 [(set (match_operand:TI 0
1152 "nonimmediate_operand" "= r,w, r,w,r,m,m,w,m")
1154 "aarch64_movti_operand" " rUti,r, w,w,m,r,Z,m,w"))]
1155 "(register_operand (operands[0], TImode)
1156 || aarch64_reg_or_zero (operands[1], TImode))"
1161 mov\\t%0.16b, %1.16b
1167 [(set_attr "type" "multiple,f_mcr,f_mrc,neon_logic_q, \
1168 load_16,store_16,store_16,\
1170 (set_attr "length" "8,8,8,4,4,4,4,4,4")
1171 (set_attr "arch" "*,*,*,simd,*,*,*,fp,fp")]
1174 ;; Split a TImode register-register or register-immediate move into
1175 ;; its component DImode pieces, taking care to handle overlapping
1176 ;; source and dest registers.
1178 [(set (match_operand:TI 0 "register_operand" "")
1179 (match_operand:TI 1 "aarch64_reg_or_imm" ""))]
1180 "reload_completed && aarch64_split_128bit_move_p (operands[0], operands[1])"
1183 aarch64_split_128bit_move (operands[0], operands[1]);
1187 (define_expand "mov<mode>"
1188 [(set (match_operand:GPF_TF_F16 0 "nonimmediate_operand" "")
1189 (match_operand:GPF_TF_F16 1 "general_operand" ""))]
1194 aarch64_err_no_fpadvsimd (<MODE>mode);
1198 if (GET_CODE (operands[0]) == MEM
1199 && ! (GET_CODE (operands[1]) == CONST_DOUBLE
1200 && aarch64_float_const_zero_rtx_p (operands[1])))
1201 operands[1] = force_reg (<MODE>mode, operands[1]);
1205 (define_insn "*movhf_aarch64"
1206 [(set (match_operand:HF 0 "nonimmediate_operand" "=w,w , w,?r,w,w ,w ,w,m,r,m ,r")
1207 (match_operand:HF 1 "general_operand" "Y ,?rY,?r, w,w,Ufc,Uvi,m,w,m,rY,r"))]
1208 "TARGET_FLOAT && (register_operand (operands[0], HFmode)
1209 || aarch64_reg_or_fp_zero (operands[1], HFmode))"
1215 mov\\t%0.h[0], %1.h[0]
1217 * return aarch64_output_scalar_simd_mov_immediate (operands[1], HImode);
1223 [(set_attr "type" "neon_move,f_mcr,neon_move,neon_to_gp, neon_move,fconsts, \
1224 neon_move,f_loads,f_stores,load_4,store_4,mov_reg")
1225 (set_attr "arch" "simd,fp16,simd,simd,simd,fp16,simd,*,*,*,*,*")]
1228 (define_insn "*movsf_aarch64"
1229 [(set (match_operand:SF 0 "nonimmediate_operand" "=w,w ,?r,w,w ,w ,w,m,r,m ,r,r")
1230 (match_operand:SF 1 "general_operand" "Y ,?rY, w,w,Ufc,Uvi,m,w,m,rY,r,M"))]
1231 "TARGET_FLOAT && (register_operand (operands[0], SFmode)
1232 || aarch64_reg_or_fp_zero (operands[1], SFmode))"
1239 * return aarch64_output_scalar_simd_mov_immediate (operands[1], SImode);
1246 [(set_attr "type" "neon_move,f_mcr,f_mrc,fmov,fconsts,neon_move,\
1247 f_loads,f_stores,load_4,store_4,mov_reg,\
1249 (set_attr "arch" "simd,*,*,*,*,simd,*,*,*,*,*,*")]
1252 (define_insn "*movdf_aarch64"
1253 [(set (match_operand:DF 0 "nonimmediate_operand" "=w, w ,?r,w,w ,w ,w,m,r,m ,r,r")
1254 (match_operand:DF 1 "general_operand" "Y , ?rY, w,w,Ufc,Uvi,m,w,m,rY,r,N"))]
1255 "TARGET_FLOAT && (register_operand (operands[0], DFmode)
1256 || aarch64_reg_or_fp_zero (operands[1], DFmode))"
1263 * return aarch64_output_scalar_simd_mov_immediate (operands[1], DImode);
1270 [(set_attr "type" "neon_move,f_mcr,f_mrc,fmov,fconstd,neon_move,\
1271 f_loadd,f_stored,load_8,store_8,mov_reg,\
1273 (set_attr "arch" "simd,*,*,*,*,simd,*,*,*,*,*,*")]
1277 [(set (match_operand:GPF_HF 0 "nonimmediate_operand")
1278 (match_operand:GPF_HF 1 "general_operand"))]
1279 "can_create_pseudo_p ()
1280 && !aarch64_can_const_movi_rtx_p (operands[1], <MODE>mode)
1281 && !aarch64_float_const_representable_p (operands[1])
1282 && aarch64_float_const_rtx_p (operands[1])"
1285 unsigned HOST_WIDE_INT ival;
1286 if (!aarch64_reinterpret_float_as_int (operands[1], &ival))
1289 rtx tmp = gen_reg_rtx (<FCVT_TARGET>mode);
1290 emit_move_insn (tmp, gen_int_mode (ival, <FCVT_TARGET>mode));
1291 emit_move_insn (operands[0], gen_lowpart (<MODE>mode, tmp));
1296 (define_insn "*movtf_aarch64"
1297 [(set (match_operand:TF 0
1298 "nonimmediate_operand" "=w,?&r,w ,?r,w,?w,w,m,?r,m ,m")
1300 "general_operand" " w,?r, ?r,w ,Y,Y ,m,w,m ,?r,Y"))]
1301 "TARGET_FLOAT && (register_operand (operands[0], TFmode)
1302 || aarch64_reg_or_fp_zero (operands[1], TFmode))"
1304 mov\\t%0.16b, %1.16b
1315 [(set_attr "type" "logic_reg,multiple,f_mcr,f_mrc,neon_move_q,f_mcr,\
1316 f_loadd,f_stored,load_16,store_16,store_16")
1317 (set_attr "length" "4,8,8,8,4,4,4,4,4,4,4")
1318 (set_attr "arch" "simd,*,*,*,simd,*,*,*,*,*,*")]
1322 [(set (match_operand:TF 0 "register_operand" "")
1323 (match_operand:TF 1 "aarch64_reg_or_imm" ""))]
1324 "reload_completed && aarch64_split_128bit_move_p (operands[0], operands[1])"
1327 aarch64_split_128bit_move (operands[0], operands[1]);
1334 ;; 2 is size of move in bytes
1337 (define_expand "movmemdi"
1338 [(match_operand:BLK 0 "memory_operand")
1339 (match_operand:BLK 1 "memory_operand")
1340 (match_operand:DI 2 "immediate_operand")
1341 (match_operand:DI 3 "immediate_operand")]
1344 if (aarch64_expand_movmem (operands))
1350 ;; Operands 1 and 3 are tied together by the final condition; so we allow
1351 ;; fairly lax checking on the second memory operation.
1352 (define_insn "load_pair_sw_<SX:mode><SX2:mode>"
1353 [(set (match_operand:SX 0 "register_operand" "=r,w")
1354 (match_operand:SX 1 "aarch64_mem_pair_operand" "Ump,Ump"))
1355 (set (match_operand:SX2 2 "register_operand" "=r,w")
1356 (match_operand:SX2 3 "memory_operand" "m,m"))]
1357 "rtx_equal_p (XEXP (operands[3], 0),
1358 plus_constant (Pmode,
1359 XEXP (operands[1], 0),
1360 GET_MODE_SIZE (<SX:MODE>mode)))"
1364 [(set_attr "type" "load_8,neon_load1_2reg")
1365 (set_attr "arch" "*,fp")]
1368 ;; Storing different modes that can still be merged
1369 (define_insn "load_pair_dw_<DX:mode><DX2:mode>"
1370 [(set (match_operand:DX 0 "register_operand" "=r,w")
1371 (match_operand:DX 1 "aarch64_mem_pair_operand" "Ump,Ump"))
1372 (set (match_operand:DX2 2 "register_operand" "=r,w")
1373 (match_operand:DX2 3 "memory_operand" "m,m"))]
1374 "rtx_equal_p (XEXP (operands[3], 0),
1375 plus_constant (Pmode,
1376 XEXP (operands[1], 0),
1377 GET_MODE_SIZE (<DX:MODE>mode)))"
1381 [(set_attr "type" "load_16,neon_load1_2reg")
1382 (set_attr "arch" "*,fp")]
1385 ;; Operands 0 and 2 are tied together by the final condition; so we allow
1386 ;; fairly lax checking on the second memory operation.
1387 (define_insn "store_pair_sw_<SX:mode><SX2:mode>"
1388 [(set (match_operand:SX 0 "aarch64_mem_pair_operand" "=Ump,Ump")
1389 (match_operand:SX 1 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))
1390 (set (match_operand:SX2 2 "memory_operand" "=m,m")
1391 (match_operand:SX2 3 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))]
1392 "rtx_equal_p (XEXP (operands[2], 0),
1393 plus_constant (Pmode,
1394 XEXP (operands[0], 0),
1395 GET_MODE_SIZE (<SX:MODE>mode)))"
1399 [(set_attr "type" "store_8,neon_store1_2reg")
1400 (set_attr "arch" "*,fp")]
1403 ;; Storing different modes that can still be merged
1404 (define_insn "store_pair_dw_<DX:mode><DX2:mode>"
1405 [(set (match_operand:DX 0 "aarch64_mem_pair_operand" "=Ump,Ump")
1406 (match_operand:DX 1 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))
1407 (set (match_operand:DX2 2 "memory_operand" "=m,m")
1408 (match_operand:DX2 3 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))]
1409 "rtx_equal_p (XEXP (operands[2], 0),
1410 plus_constant (Pmode,
1411 XEXP (operands[0], 0),
1412 GET_MODE_SIZE (<DX:MODE>mode)))"
1416 [(set_attr "type" "store_16,neon_store1_2reg")
1417 (set_attr "arch" "*,fp")]
1420 ;; Load pair with post-index writeback. This is primarily used in function
1422 (define_insn "loadwb_pair<GPI:mode>_<P:mode>"
1424 [(set (match_operand:P 0 "register_operand" "=k")
1425 (plus:P (match_operand:P 1 "register_operand" "0")
1426 (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1427 (set (match_operand:GPI 2 "register_operand" "=r")
1428 (mem:GPI (match_dup 1)))
1429 (set (match_operand:GPI 3 "register_operand" "=r")
1430 (mem:GPI (plus:P (match_dup 1)
1431 (match_operand:P 5 "const_int_operand" "n"))))])]
1432 "INTVAL (operands[5]) == GET_MODE_SIZE (<GPI:MODE>mode)"
1433 "ldp\\t%<w>2, %<w>3, [%1], %4"
1434 [(set_attr "type" "load_<ldpstp_sz>")]
1437 (define_insn "loadwb_pair<GPF:mode>_<P:mode>"
1439 [(set (match_operand:P 0 "register_operand" "=k")
1440 (plus:P (match_operand:P 1 "register_operand" "0")
1441 (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1442 (set (match_operand:GPF 2 "register_operand" "=w")
1443 (mem:GPF (match_dup 1)))
1444 (set (match_operand:GPF 3 "register_operand" "=w")
1445 (mem:GPF (plus:P (match_dup 1)
1446 (match_operand:P 5 "const_int_operand" "n"))))])]
1447 "INTVAL (operands[5]) == GET_MODE_SIZE (<GPF:MODE>mode)"
1448 "ldp\\t%<w>2, %<w>3, [%1], %4"
1449 [(set_attr "type" "neon_load1_2reg")]
1452 ;; Store pair with pre-index writeback. This is primarily used in function
1454 (define_insn "storewb_pair<GPI:mode>_<P:mode>"
1456 [(set (match_operand:P 0 "register_operand" "=&k")
1457 (plus:P (match_operand:P 1 "register_operand" "0")
1458 (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1459 (set (mem:GPI (plus:P (match_dup 0)
1461 (match_operand:GPI 2 "register_operand" "r"))
1462 (set (mem:GPI (plus:P (match_dup 0)
1463 (match_operand:P 5 "const_int_operand" "n")))
1464 (match_operand:GPI 3 "register_operand" "r"))])]
1465 "INTVAL (operands[5]) == INTVAL (operands[4]) + GET_MODE_SIZE (<GPI:MODE>mode)"
1466 "stp\\t%<w>2, %<w>3, [%0, %4]!"
1467 [(set_attr "type" "store_<ldpstp_sz>")]
1470 (define_insn "storewb_pair<GPF:mode>_<P:mode>"
1472 [(set (match_operand:P 0 "register_operand" "=&k")
1473 (plus:P (match_operand:P 1 "register_operand" "0")
1474 (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1475 (set (mem:GPF (plus:P (match_dup 0)
1477 (match_operand:GPF 2 "register_operand" "w"))
1478 (set (mem:GPF (plus:P (match_dup 0)
1479 (match_operand:P 5 "const_int_operand" "n")))
1480 (match_operand:GPF 3 "register_operand" "w"))])]
1481 "INTVAL (operands[5]) == INTVAL (operands[4]) + GET_MODE_SIZE (<GPF:MODE>mode)"
1482 "stp\\t%<w>2, %<w>3, [%0, %4]!"
1483 [(set_attr "type" "neon_store1_2reg<q>")]
1486 ;; -------------------------------------------------------------------
1487 ;; Sign/Zero extension
1488 ;; -------------------------------------------------------------------
1490 (define_expand "<optab>sidi2"
1491 [(set (match_operand:DI 0 "register_operand")
1492 (ANY_EXTEND:DI (match_operand:SI 1 "nonimmediate_operand")))]
1496 (define_insn "*extendsidi2_aarch64"
1497 [(set (match_operand:DI 0 "register_operand" "=r,r")
1498 (sign_extend:DI (match_operand:SI 1 "nonimmediate_operand" "r,m")))]
1503 [(set_attr "type" "extend,load_4")]
1506 (define_insn "*load_pair_extendsidi2_aarch64"
1507 [(set (match_operand:DI 0 "register_operand" "=r")
1508 (sign_extend:DI (match_operand:SI 1 "aarch64_mem_pair_operand" "Ump")))
1509 (set (match_operand:DI 2 "register_operand" "=r")
1510 (sign_extend:DI (match_operand:SI 3 "memory_operand" "m")))]
1511 "rtx_equal_p (XEXP (operands[3], 0),
1512 plus_constant (Pmode,
1513 XEXP (operands[1], 0),
1514 GET_MODE_SIZE (SImode)))"
1515 "ldpsw\\t%0, %2, %1"
1516 [(set_attr "type" "load_8")]
1519 (define_insn "*zero_extendsidi2_aarch64"
1520 [(set (match_operand:DI 0 "register_operand" "=r,r")
1521 (zero_extend:DI (match_operand:SI 1 "nonimmediate_operand" "r,m")))]
1526 [(set_attr "type" "extend,load_4")]
1529 (define_insn "*load_pair_zero_extendsidi2_aarch64"
1530 [(set (match_operand:DI 0 "register_operand" "=r")
1531 (zero_extend:DI (match_operand:SI 1 "aarch64_mem_pair_operand" "Ump")))
1532 (set (match_operand:DI 2 "register_operand" "=r")
1533 (zero_extend:DI (match_operand:SI 3 "memory_operand" "m")))]
1534 "rtx_equal_p (XEXP (operands[3], 0),
1535 plus_constant (Pmode,
1536 XEXP (operands[1], 0),
1537 GET_MODE_SIZE (SImode)))"
1538 "ldp\\t%w0, %w2, %1"
1539 [(set_attr "type" "load_8")]
1542 (define_expand "<ANY_EXTEND:optab><SHORT:mode><GPI:mode>2"
1543 [(set (match_operand:GPI 0 "register_operand")
1544 (ANY_EXTEND:GPI (match_operand:SHORT 1 "nonimmediate_operand")))]
1548 (define_insn "*extend<SHORT:mode><GPI:mode>2_aarch64"
1549 [(set (match_operand:GPI 0 "register_operand" "=r,r")
1550 (sign_extend:GPI (match_operand:SHORT 1 "nonimmediate_operand" "r,m")))]
1553 sxt<SHORT:size>\t%<GPI:w>0, %w1
1554 ldrs<SHORT:size>\t%<GPI:w>0, %1"
1555 [(set_attr "type" "extend,load_4")]
1558 (define_insn "*zero_extend<SHORT:mode><GPI:mode>2_aarch64"
1559 [(set (match_operand:GPI 0 "register_operand" "=r,r,w")
1560 (zero_extend:GPI (match_operand:SHORT 1 "nonimmediate_operand" "r,m,m")))]
1563 and\t%<GPI:w>0, %<GPI:w>1, <SHORT:short_mask>
1564 ldr<SHORT:size>\t%w0, %1
1565 ldr\t%<SHORT:size>0, %1"
1566 [(set_attr "type" "logic_imm,load_4,load_4")]
1569 (define_expand "<optab>qihi2"
1570 [(set (match_operand:HI 0 "register_operand")
1571 (ANY_EXTEND:HI (match_operand:QI 1 "nonimmediate_operand")))]
1575 (define_insn "*extendqihi2_aarch64"
1576 [(set (match_operand:HI 0 "register_operand" "=r,r")
1577 (sign_extend:HI (match_operand:QI 1 "nonimmediate_operand" "r,m")))]
1582 [(set_attr "type" "extend,load_4")]
1585 (define_insn "*zero_extendqihi2_aarch64"
1586 [(set (match_operand:HI 0 "register_operand" "=r,r")
1587 (zero_extend:HI (match_operand:QI 1 "nonimmediate_operand" "r,m")))]
1592 [(set_attr "type" "logic_imm,load_4")]
1595 ;; -------------------------------------------------------------------
1596 ;; Simple arithmetic
1597 ;; -------------------------------------------------------------------
1599 (define_expand "add<mode>3"
1601 (match_operand:GPI 0 "register_operand" "")
1602 (plus:GPI (match_operand:GPI 1 "register_operand" "")
1603 (match_operand:GPI 2 "aarch64_pluslong_or_poly_operand" "")))]
1606 /* If operands[1] is a subreg extract the inner RTX. */
1607 rtx op1 = REG_P (operands[1]) ? operands[1] : SUBREG_REG (operands[1]);
1609 /* If the constant is too large for a single instruction and isn't frame
1610 based, split off the immediate so it is available for CSE. */
1611 if (!aarch64_plus_immediate (operands[2], <MODE>mode)
1612 && can_create_pseudo_p ()
1614 || !REGNO_PTR_FRAME_P (REGNO (op1))))
1615 operands[2] = force_reg (<MODE>mode, operands[2]);
1616 /* Expand polynomial additions now if the destination is the stack
1617 pointer, since we don't want to use that as a temporary. */
1618 else if (operands[0] == stack_pointer_rtx
1619 && aarch64_split_add_offset_immediate (operands[2], <MODE>mode))
1621 aarch64_split_add_offset (<MODE>mode, operands[0], operands[1],
1622 operands[2], NULL_RTX, NULL_RTX);
1627 (define_insn "*add<mode>3_aarch64"
1629 (match_operand:GPI 0 "register_operand" "=rk,rk,w,rk,r,rk")
1631 (match_operand:GPI 1 "register_operand" "%rk,rk,w,rk,rk,rk")
1632 (match_operand:GPI 2 "aarch64_pluslong_operand" "I,r,w,J,Uaa,Uav")))]
1635 add\\t%<w>0, %<w>1, %2
1636 add\\t%<w>0, %<w>1, %<w>2
1637 add\\t%<rtn>0<vas>, %<rtn>1<vas>, %<rtn>2<vas>
1638 sub\\t%<w>0, %<w>1, #%n2
1640 * return aarch64_output_sve_addvl_addpl (operands[0], operands[1], operands[2]);"
1641 ;; The "alu_imm" type for ADDVL/ADDPL is just a placeholder.
1642 [(set_attr "type" "alu_imm,alu_sreg,neon_add,alu_imm,multiple,alu_imm")
1643 (set_attr "arch" "*,*,simd,*,*,*")]
1646 ;; zero_extend version of above
1647 (define_insn "*addsi3_aarch64_uxtw"
1649 (match_operand:DI 0 "register_operand" "=rk,rk,rk,r")
1651 (plus:SI (match_operand:SI 1 "register_operand" "%rk,rk,rk,rk")
1652 (match_operand:SI 2 "aarch64_pluslong_operand" "I,r,J,Uaa"))))]
1657 sub\\t%w0, %w1, #%n2
1659 [(set_attr "type" "alu_imm,alu_sreg,alu_imm,multiple")]
1662 ;; If there's a free register, and we can load the constant with a
1663 ;; single instruction, do so. This has a chance to improve scheduling.
1665 [(match_scratch:GPI 3 "r")
1666 (set (match_operand:GPI 0 "register_operand")
1668 (match_operand:GPI 1 "register_operand")
1669 (match_operand:GPI 2 "aarch64_pluslong_strict_immedate")))]
1670 "aarch64_move_imm (INTVAL (operands[2]), <MODE>mode)"
1671 [(set (match_dup 3) (match_dup 2))
1672 (set (match_dup 0) (plus:GPI (match_dup 1) (match_dup 3)))]
1676 [(match_scratch:SI 3 "r")
1677 (set (match_operand:DI 0 "register_operand")
1680 (match_operand:SI 1 "register_operand")
1681 (match_operand:SI 2 "aarch64_pluslong_strict_immedate"))))]
1682 "aarch64_move_imm (INTVAL (operands[2]), SImode)"
1683 [(set (match_dup 3) (match_dup 2))
1684 (set (match_dup 0) (zero_extend:DI (plus:SI (match_dup 1) (match_dup 3))))]
1687 ;; After peephole2 has had a chance to run, split any remaining long
1688 ;; additions into two add immediates.
1690 [(set (match_operand:GPI 0 "register_operand")
1692 (match_operand:GPI 1 "register_operand")
1693 (match_operand:GPI 2 "aarch64_pluslong_strict_immedate")))]
1694 "epilogue_completed"
1695 [(set (match_dup 0) (plus:GPI (match_dup 1) (match_dup 3)))
1696 (set (match_dup 0) (plus:GPI (match_dup 0) (match_dup 4)))]
1698 HOST_WIDE_INT i = INTVAL (operands[2]);
1699 HOST_WIDE_INT s = (i >= 0 ? i & 0xfff : -(-i & 0xfff));
1700 operands[3] = GEN_INT (i - s);
1701 operands[4] = GEN_INT (s);
1705 ;; Match addition of polynomial offsets that require one temporary, for which
1706 ;; we can use the early-clobbered destination register. This is a separate
1707 ;; pattern so that the early clobber doesn't affect register allocation
1708 ;; for other forms of addition. However, we still need to provide an
1709 ;; all-register alternative, in case the offset goes out of range after
1710 ;; elimination. For completeness we might as well provide all GPR-based
1711 ;; alternatives from the main pattern.
1713 ;; We don't have a pattern for additions requiring two temporaries since at
1714 ;; present LRA doesn't allow new scratches to be added during elimination.
1715 ;; Such offsets should be rare anyway.
1717 ;; ??? But if we added LRA support for new scratches, much of the ugliness
1718 ;; here would go away. We could just handle all polynomial constants in
1720 (define_insn_and_split "*add<mode>3_poly_1"
1722 (match_operand:GPI 0 "register_operand" "=r,r,r,r,r,&r")
1724 (match_operand:GPI 1 "register_operand" "%rk,rk,rk,rk,rk,rk")
1725 (match_operand:GPI 2 "aarch64_pluslong_or_poly_operand" "I,r,J,Uaa,Uav,Uat")))]
1726 "TARGET_SVE && operands[0] != stack_pointer_rtx"
1728 add\\t%<w>0, %<w>1, %2
1729 add\\t%<w>0, %<w>1, %<w>2
1730 sub\\t%<w>0, %<w>1, #%n2
1732 * return aarch64_output_sve_addvl_addpl (operands[0], operands[1], operands[2]);
1734 "&& epilogue_completed
1735 && !reg_overlap_mentioned_p (operands[0], operands[1])
1736 && aarch64_split_add_offset_immediate (operands[2], <MODE>mode)"
1739 aarch64_split_add_offset (<MODE>mode, operands[0], operands[1],
1740 operands[2], operands[0], NULL_RTX);
1743 ;; The "alu_imm" type for ADDVL/ADDPL is just a placeholder.
1744 [(set_attr "type" "alu_imm,alu_sreg,alu_imm,multiple,alu_imm,multiple")]
1748 [(set (match_operand:DI 0 "register_operand")
1751 (match_operand:SI 1 "register_operand")
1752 (match_operand:SI 2 "aarch64_pluslong_strict_immedate"))))]
1753 "epilogue_completed"
1754 [(set (match_dup 5) (plus:SI (match_dup 1) (match_dup 3)))
1755 (set (match_dup 0) (zero_extend:DI (plus:SI (match_dup 5) (match_dup 4))))]
1757 HOST_WIDE_INT i = INTVAL (operands[2]);
1758 HOST_WIDE_INT s = (i >= 0 ? i & 0xfff : -(-i & 0xfff));
1759 operands[3] = GEN_INT (i - s);
1760 operands[4] = GEN_INT (s);
1761 operands[5] = gen_lowpart (SImode, operands[0]);
1765 (define_expand "addv<mode>4"
1766 [(match_operand:GPI 0 "register_operand")
1767 (match_operand:GPI 1 "register_operand")
1768 (match_operand:GPI 2 "register_operand")
1769 (label_ref (match_operand 3 "" ""))]
1772 emit_insn (gen_add<mode>3_compareV (operands[0], operands[1], operands[2]));
1773 aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]);
1778 (define_expand "uaddv<mode>4"
1779 [(match_operand:GPI 0 "register_operand")
1780 (match_operand:GPI 1 "register_operand")
1781 (match_operand:GPI 2 "register_operand")
1782 (label_ref (match_operand 3 "" ""))]
1785 emit_insn (gen_add<mode>3_compareC (operands[0], operands[1], operands[2]));
1786 aarch64_gen_unlikely_cbranch (NE, CC_Cmode, operands[3]);
1791 (define_expand "addti3"
1792 [(set (match_operand:TI 0 "register_operand" "")
1793 (plus:TI (match_operand:TI 1 "register_operand" "")
1794 (match_operand:TI 2 "aarch64_reg_or_imm" "")))]
1797 rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
1799 aarch64_addti_scratch_regs (operands[1], operands[2],
1800 &low_dest, &op1_low, &op2_low,
1801 &high_dest, &op1_high, &op2_high);
1803 if (op2_low == const0_rtx)
1806 if (!aarch64_pluslong_operand (op2_high, DImode))
1807 op2_high = force_reg (DImode, op2_high);
1808 emit_insn (gen_adddi3 (high_dest, op1_high, op2_high));
1812 emit_insn (gen_adddi3_compareC (low_dest, op1_low,
1813 force_reg (DImode, op2_low)));
1814 emit_insn (gen_adddi3_carryin (high_dest, op1_high,
1815 force_reg (DImode, op2_high)));
1818 emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest);
1819 emit_move_insn (gen_highpart (DImode, operands[0]), high_dest);
1824 (define_expand "addvti4"
1825 [(match_operand:TI 0 "register_operand" "")
1826 (match_operand:TI 1 "register_operand" "")
1827 (match_operand:TI 2 "aarch64_reg_or_imm" "")
1828 (label_ref (match_operand 3 "" ""))]
1831 rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
1833 aarch64_addti_scratch_regs (operands[1], operands[2],
1834 &low_dest, &op1_low, &op2_low,
1835 &high_dest, &op1_high, &op2_high);
1837 if (op2_low == const0_rtx)
1840 emit_insn (gen_adddi3_compareV (high_dest, op1_high,
1841 force_reg (DImode, op2_high)));
1845 emit_insn (gen_adddi3_compareC (low_dest, op1_low,
1846 force_reg (DImode, op2_low)));
1847 emit_insn (gen_adddi3_carryinV (high_dest, op1_high,
1848 force_reg (DImode, op2_high)));
1851 emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest);
1852 emit_move_insn (gen_highpart (DImode, operands[0]), high_dest);
1854 aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]);
1858 (define_expand "uaddvti4"
1859 [(match_operand:TI 0 "register_operand" "")
1860 (match_operand:TI 1 "register_operand" "")
1861 (match_operand:TI 2 "aarch64_reg_or_imm" "")
1862 (label_ref (match_operand 3 "" ""))]
1865 rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
1867 aarch64_addti_scratch_regs (operands[1], operands[2],
1868 &low_dest, &op1_low, &op2_low,
1869 &high_dest, &op1_high, &op2_high);
1871 if (op2_low == const0_rtx)
1874 emit_insn (gen_adddi3_compareC (high_dest, op1_high,
1875 force_reg (DImode, op2_high)));
1879 emit_insn (gen_adddi3_compareC (low_dest, op1_low,
1880 force_reg (DImode, op2_low)));
1881 emit_insn (gen_adddi3_carryinC (high_dest, op1_high,
1882 force_reg (DImode, op2_high)));
1885 emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest);
1886 emit_move_insn (gen_highpart (DImode, operands[0]), high_dest);
1888 aarch64_gen_unlikely_cbranch (NE, CC_Cmode, operands[3]);
1892 (define_insn "add<mode>3_compare0"
1893 [(set (reg:CC_NZ CC_REGNUM)
1895 (plus:GPI (match_operand:GPI 1 "register_operand" "%r,r,r")
1896 (match_operand:GPI 2 "aarch64_plus_operand" "r,I,J"))
1898 (set (match_operand:GPI 0 "register_operand" "=r,r,r")
1899 (plus:GPI (match_dup 1) (match_dup 2)))]
1902 adds\\t%<w>0, %<w>1, %<w>2
1903 adds\\t%<w>0, %<w>1, %2
1904 subs\\t%<w>0, %<w>1, #%n2"
1905 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
1908 ;; zero_extend version of above
1909 (define_insn "*addsi3_compare0_uxtw"
1910 [(set (reg:CC_NZ CC_REGNUM)
1912 (plus:SI (match_operand:SI 1 "register_operand" "%r,r,r")
1913 (match_operand:SI 2 "aarch64_plus_operand" "r,I,J"))
1915 (set (match_operand:DI 0 "register_operand" "=r,r,r")
1916 (zero_extend:DI (plus:SI (match_dup 1) (match_dup 2))))]
1919 adds\\t%w0, %w1, %w2
1921 subs\\t%w0, %w1, #%n2"
1922 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
1925 (define_insn "*add<mode>3_compareC_cconly_imm"
1926 [(set (reg:CC_C CC_REGNUM)
1929 (zero_extend:<DWI> (match_operand:GPI 0 "register_operand" "r,r"))
1930 (match_operand:<DWI> 2 "const_scalar_int_operand" ""))
1934 (match_operand:GPI 1 "aarch64_plus_immediate" "I,J")))))]
1935 "aarch64_zero_extend_const_eq (<DWI>mode, operands[2],
1936 <MODE>mode, operands[1])"
1940 [(set_attr "type" "alus_imm")]
1943 (define_insn "*add<mode>3_compareC_cconly"
1944 [(set (reg:CC_C CC_REGNUM)
1947 (zero_extend:<DWI> (match_operand:GPI 0 "register_operand" "r"))
1948 (zero_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
1949 (zero_extend:<DWI> (plus:GPI (match_dup 0) (match_dup 1)))))]
1951 "cmn\\t%<w>0, %<w>1"
1952 [(set_attr "type" "alus_sreg")]
1955 (define_insn "*add<mode>3_compareC_imm"
1956 [(set (reg:CC_C CC_REGNUM)
1959 (zero_extend:<DWI> (match_operand:GPI 1 "register_operand" "r,r"))
1960 (match_operand:<DWI> 3 "const_scalar_int_operand" ""))
1964 (match_operand:GPI 2 "aarch64_plus_immediate" "I,J")))))
1965 (set (match_operand:GPI 0 "register_operand" "=r,r")
1966 (plus:GPI (match_dup 1) (match_dup 2)))]
1967 "aarch64_zero_extend_const_eq (<DWI>mode, operands[3],
1968 <MODE>mode, operands[2])"
1970 adds\\t%<w>0, %<w>1, %2
1971 subs\\t%<w>0, %<w>1, #%n2"
1972 [(set_attr "type" "alus_imm")]
1975 (define_insn "add<mode>3_compareC"
1976 [(set (reg:CC_C CC_REGNUM)
1979 (zero_extend:<DWI> (match_operand:GPI 1 "register_operand" "r"))
1980 (zero_extend:<DWI> (match_operand:GPI 2 "register_operand" "r")))
1982 (plus:GPI (match_dup 1) (match_dup 2)))))
1983 (set (match_operand:GPI 0 "register_operand" "=r")
1984 (plus:GPI (match_dup 1) (match_dup 2)))]
1986 "adds\\t%<w>0, %<w>1, %<w>2"
1987 [(set_attr "type" "alus_sreg")]
1990 (define_insn "*add<mode>3_compareV_cconly_imm"
1991 [(set (reg:CC_V CC_REGNUM)
1994 (sign_extend:<DWI> (match_operand:GPI 0 "register_operand" "r,r"))
1995 (match_operand:<DWI> 1 "const_scalar_int_operand" ""))
1999 (match_operand:GPI 2 "aarch64_plus_immediate" "I,J")))))]
2000 "INTVAL (operands[1]) == INTVAL (operands[2])"
2004 [(set_attr "type" "alus_imm")]
2007 (define_insn "*add<mode>3_compareV_cconly"
2008 [(set (reg:CC_V CC_REGNUM)
2011 (sign_extend:<DWI> (match_operand:GPI 0 "register_operand" "r"))
2012 (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2013 (sign_extend:<DWI> (plus:GPI (match_dup 0) (match_dup 1)))))]
2015 "cmn\\t%<w>0, %<w>1"
2016 [(set_attr "type" "alus_sreg")]
2019 (define_insn "*add<mode>3_compareV_imm"
2020 [(set (reg:CC_V CC_REGNUM)
2024 (match_operand:GPI 1 "register_operand" "r,r"))
2025 (match_operand:GPI 2 "aarch64_plus_immediate" "I,J"))
2027 (plus:GPI (match_dup 1) (match_dup 2)))))
2028 (set (match_operand:GPI 0 "register_operand" "=r,r")
2029 (plus:GPI (match_dup 1) (match_dup 2)))]
2032 adds\\t%<w>0, %<w>1, %<w>2
2033 subs\\t%<w>0, %<w>1, #%n2"
2034 [(set_attr "type" "alus_imm,alus_imm")]
2037 (define_insn "add<mode>3_compareV"
2038 [(set (reg:CC_V CC_REGNUM)
2041 (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r"))
2042 (sign_extend:<DWI> (match_operand:GPI 2 "register_operand" "r")))
2043 (sign_extend:<DWI> (plus:GPI (match_dup 1) (match_dup 2)))))
2044 (set (match_operand:GPI 0 "register_operand" "=r")
2045 (plus:GPI (match_dup 1) (match_dup 2)))]
2047 "adds\\t%<w>0, %<w>1, %<w>2"
2048 [(set_attr "type" "alus_sreg")]
2051 (define_insn "*adds_shift_imm_<mode>"
2052 [(set (reg:CC_NZ CC_REGNUM)
2054 (plus:GPI (ASHIFT:GPI
2055 (match_operand:GPI 1 "register_operand" "r")
2056 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
2057 (match_operand:GPI 3 "register_operand" "r"))
2059 (set (match_operand:GPI 0 "register_operand" "=r")
2060 (plus:GPI (ASHIFT:GPI (match_dup 1) (match_dup 2))
2063 "adds\\t%<w>0, %<w>3, %<w>1, <shift> %2"
2064 [(set_attr "type" "alus_shift_imm")]
2067 (define_insn "*subs_shift_imm_<mode>"
2068 [(set (reg:CC_NZ CC_REGNUM)
2070 (minus:GPI (match_operand:GPI 1 "register_operand" "r")
2072 (match_operand:GPI 2 "register_operand" "r")
2073 (match_operand:QI 3 "aarch64_shift_imm_<mode>" "n")))
2075 (set (match_operand:GPI 0 "register_operand" "=r")
2076 (minus:GPI (match_dup 1)
2077 (ASHIFT:GPI (match_dup 2) (match_dup 3))))]
2079 "subs\\t%<w>0, %<w>1, %<w>2, <shift> %3"
2080 [(set_attr "type" "alus_shift_imm")]
2083 (define_insn "*adds_mul_imm_<mode>"
2084 [(set (reg:CC_NZ CC_REGNUM)
2087 (match_operand:GPI 1 "register_operand" "r")
2088 (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))
2089 (match_operand:GPI 3 "register_operand" "r"))
2091 (set (match_operand:GPI 0 "register_operand" "=r")
2092 (plus:GPI (mult:GPI (match_dup 1) (match_dup 2))
2095 "adds\\t%<w>0, %<w>3, %<w>1, lsl %p2"
2096 [(set_attr "type" "alus_shift_imm")]
2099 (define_insn "*subs_mul_imm_<mode>"
2100 [(set (reg:CC_NZ CC_REGNUM)
2102 (minus:GPI (match_operand:GPI 1 "register_operand" "r")
2104 (match_operand:GPI 2 "register_operand" "r")
2105 (match_operand:QI 3 "aarch64_pwr_2_<mode>" "n")))
2107 (set (match_operand:GPI 0 "register_operand" "=r")
2108 (minus:GPI (match_dup 1)
2109 (mult:GPI (match_dup 2) (match_dup 3))))]
2111 "subs\\t%<w>0, %<w>1, %<w>2, lsl %p3"
2112 [(set_attr "type" "alus_shift_imm")]
2115 (define_insn "*adds_<optab><ALLX:mode>_<GPI:mode>"
2116 [(set (reg:CC_NZ CC_REGNUM)
2119 (ANY_EXTEND:GPI (match_operand:ALLX 1 "register_operand" "r"))
2120 (match_operand:GPI 2 "register_operand" "r"))
2122 (set (match_operand:GPI 0 "register_operand" "=r")
2123 (plus:GPI (ANY_EXTEND:GPI (match_dup 1)) (match_dup 2)))]
2125 "adds\\t%<GPI:w>0, %<GPI:w>2, %<GPI:w>1, <su>xt<ALLX:size>"
2126 [(set_attr "type" "alus_ext")]
2129 (define_insn "*subs_<optab><ALLX:mode>_<GPI:mode>"
2130 [(set (reg:CC_NZ CC_REGNUM)
2132 (minus:GPI (match_operand:GPI 1 "register_operand" "r")
2134 (match_operand:ALLX 2 "register_operand" "r")))
2136 (set (match_operand:GPI 0 "register_operand" "=r")
2137 (minus:GPI (match_dup 1) (ANY_EXTEND:GPI (match_dup 2))))]
2139 "subs\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size>"
2140 [(set_attr "type" "alus_ext")]
2143 (define_insn "*adds_<optab><ALLX:mode>_shift_<GPI:mode>"
2144 [(set (reg:CC_NZ CC_REGNUM)
2146 (plus:GPI (ashift:GPI
2148 (match_operand:ALLX 1 "register_operand" "r"))
2149 (match_operand 2 "aarch64_imm3" "Ui3"))
2150 (match_operand:GPI 3 "register_operand" "r"))
2152 (set (match_operand:GPI 0 "register_operand" "=rk")
2153 (plus:GPI (ashift:GPI (ANY_EXTEND:GPI (match_dup 1))
2157 "adds\\t%<GPI:w>0, %<GPI:w>3, %<GPI:w>1, <su>xt<ALLX:size> %2"
2158 [(set_attr "type" "alus_ext")]
2161 (define_insn "*subs_<optab><ALLX:mode>_shift_<GPI:mode>"
2162 [(set (reg:CC_NZ CC_REGNUM)
2164 (minus:GPI (match_operand:GPI 1 "register_operand" "r")
2167 (match_operand:ALLX 2 "register_operand" "r"))
2168 (match_operand 3 "aarch64_imm3" "Ui3")))
2170 (set (match_operand:GPI 0 "register_operand" "=rk")
2171 (minus:GPI (match_dup 1)
2172 (ashift:GPI (ANY_EXTEND:GPI (match_dup 2))
2175 "subs\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size> %3"
2176 [(set_attr "type" "alus_ext")]
2179 (define_insn "*adds_<optab><mode>_multp2"
2180 [(set (reg:CC_NZ CC_REGNUM)
2182 (plus:GPI (ANY_EXTRACT:GPI
2183 (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2184 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2185 (match_operand 3 "const_int_operand" "n")
2187 (match_operand:GPI 4 "register_operand" "r"))
2189 (set (match_operand:GPI 0 "register_operand" "=r")
2190 (plus:GPI (ANY_EXTRACT:GPI (mult:GPI (match_dup 1) (match_dup 2))
2194 "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
2195 "adds\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
2196 [(set_attr "type" "alus_ext")]
2199 (define_insn "*subs_<optab><mode>_multp2"
2200 [(set (reg:CC_NZ CC_REGNUM)
2202 (minus:GPI (match_operand:GPI 4 "register_operand" "r")
2204 (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2205 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2206 (match_operand 3 "const_int_operand" "n")
2209 (set (match_operand:GPI 0 "register_operand" "=r")
2210 (minus:GPI (match_dup 4) (ANY_EXTRACT:GPI
2211 (mult:GPI (match_dup 1) (match_dup 2))
2214 "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
2215 "subs\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
2216 [(set_attr "type" "alus_ext")]
2219 (define_insn "*add<mode>3nr_compare0"
2220 [(set (reg:CC_NZ CC_REGNUM)
2222 (plus:GPI (match_operand:GPI 0 "register_operand" "%r,r,r")
2223 (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J"))
2230 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
2233 (define_insn "aarch64_sub<mode>_compare0"
2234 [(set (reg:CC_NZ CC_REGNUM)
2236 (minus:GPI (match_operand:GPI 0 "register_operand" "r")
2237 (match_operand:GPI 1 "aarch64_plus_operand" "r"))
2240 "cmp\\t%<w>0, %<w>1"
2241 [(set_attr "type" "alus_sreg")]
2244 (define_insn "*compare_neg<mode>"
2245 [(set (reg:CC_Z CC_REGNUM)
2247 (neg:GPI (match_operand:GPI 0 "register_operand" "r"))
2248 (match_operand:GPI 1 "register_operand" "r")))]
2250 "cmn\\t%<w>1, %<w>0"
2251 [(set_attr "type" "alus_sreg")]
2254 (define_insn "*add_<shift>_<mode>"
2255 [(set (match_operand:GPI 0 "register_operand" "=r")
2256 (plus:GPI (ASHIFT:GPI (match_operand:GPI 1 "register_operand" "r")
2257 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
2258 (match_operand:GPI 3 "register_operand" "r")))]
2260 "add\\t%<w>0, %<w>3, %<w>1, <shift> %2"
2261 [(set_attr "type" "alu_shift_imm")]
2264 ;; zero_extend version of above
2265 (define_insn "*add_<shift>_si_uxtw"
2266 [(set (match_operand:DI 0 "register_operand" "=r")
2268 (plus:SI (ASHIFT:SI (match_operand:SI 1 "register_operand" "r")
2269 (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
2270 (match_operand:SI 3 "register_operand" "r"))))]
2272 "add\\t%w0, %w3, %w1, <shift> %2"
2273 [(set_attr "type" "alu_shift_imm")]
2276 (define_insn "*add_mul_imm_<mode>"
2277 [(set (match_operand:GPI 0 "register_operand" "=r")
2278 (plus:GPI (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2279 (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))
2280 (match_operand:GPI 3 "register_operand" "r")))]
2282 "add\\t%<w>0, %<w>3, %<w>1, lsl %p2"
2283 [(set_attr "type" "alu_shift_imm")]
2286 (define_insn "*add_<optab><ALLX:mode>_<GPI:mode>"
2287 [(set (match_operand:GPI 0 "register_operand" "=rk")
2288 (plus:GPI (ANY_EXTEND:GPI (match_operand:ALLX 1 "register_operand" "r"))
2289 (match_operand:GPI 2 "register_operand" "r")))]
2291 "add\\t%<GPI:w>0, %<GPI:w>2, %<GPI:w>1, <su>xt<ALLX:size>"
2292 [(set_attr "type" "alu_ext")]
2295 ;; zero_extend version of above
2296 (define_insn "*add_<optab><SHORT:mode>_si_uxtw"
2297 [(set (match_operand:DI 0 "register_operand" "=rk")
2299 (plus:SI (ANY_EXTEND:SI (match_operand:SHORT 1 "register_operand" "r"))
2300 (match_operand:GPI 2 "register_operand" "r"))))]
2302 "add\\t%w0, %w2, %w1, <su>xt<SHORT:size>"
2303 [(set_attr "type" "alu_ext")]
2306 (define_insn "*add_<optab><ALLX:mode>_shft_<GPI:mode>"
2307 [(set (match_operand:GPI 0 "register_operand" "=rk")
2308 (plus:GPI (ashift:GPI (ANY_EXTEND:GPI
2309 (match_operand:ALLX 1 "register_operand" "r"))
2310 (match_operand 2 "aarch64_imm3" "Ui3"))
2311 (match_operand:GPI 3 "register_operand" "r")))]
2313 "add\\t%<GPI:w>0, %<GPI:w>3, %<GPI:w>1, <su>xt<ALLX:size> %2"
2314 [(set_attr "type" "alu_ext")]
2317 ;; zero_extend version of above
2318 (define_insn "*add_<optab><SHORT:mode>_shft_si_uxtw"
2319 [(set (match_operand:DI 0 "register_operand" "=rk")
2321 (plus:SI (ashift:SI (ANY_EXTEND:SI
2322 (match_operand:SHORT 1 "register_operand" "r"))
2323 (match_operand 2 "aarch64_imm3" "Ui3"))
2324 (match_operand:SI 3 "register_operand" "r"))))]
2326 "add\\t%w0, %w3, %w1, <su>xt<SHORT:size> %2"
2327 [(set_attr "type" "alu_ext")]
2330 (define_insn "*add_<optab><ALLX:mode>_mult_<GPI:mode>"
2331 [(set (match_operand:GPI 0 "register_operand" "=rk")
2332 (plus:GPI (mult:GPI (ANY_EXTEND:GPI
2333 (match_operand:ALLX 1 "register_operand" "r"))
2334 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2335 (match_operand:GPI 3 "register_operand" "r")))]
2337 "add\\t%<GPI:w>0, %<GPI:w>3, %<GPI:w>1, <su>xt<ALLX:size> %p2"
2338 [(set_attr "type" "alu_ext")]
2341 ;; zero_extend version of above
2342 (define_insn "*add_<optab><SHORT:mode>_mult_si_uxtw"
2343 [(set (match_operand:DI 0 "register_operand" "=rk")
2344 (zero_extend:DI (plus:SI (mult:SI (ANY_EXTEND:SI
2345 (match_operand:SHORT 1 "register_operand" "r"))
2346 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2347 (match_operand:SI 3 "register_operand" "r"))))]
2349 "add\\t%w0, %w3, %w1, <su>xt<SHORT:size> %p2"
2350 [(set_attr "type" "alu_ext")]
2353 (define_insn "*add_<optab><mode>_multp2"
2354 [(set (match_operand:GPI 0 "register_operand" "=rk")
2355 (plus:GPI (ANY_EXTRACT:GPI
2356 (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2357 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2358 (match_operand 3 "const_int_operand" "n")
2360 (match_operand:GPI 4 "register_operand" "r")))]
2361 "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
2362 "add\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
2363 [(set_attr "type" "alu_ext")]
2366 ;; zero_extend version of above
2367 (define_insn "*add_<optab>si_multp2_uxtw"
2368 [(set (match_operand:DI 0 "register_operand" "=rk")
2370 (plus:SI (ANY_EXTRACT:SI
2371 (mult:SI (match_operand:SI 1 "register_operand" "r")
2372 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2373 (match_operand 3 "const_int_operand" "n")
2375 (match_operand:SI 4 "register_operand" "r"))))]
2376 "aarch64_is_extend_from_extract (SImode, operands[2], operands[3])"
2377 "add\\t%w0, %w4, %w1, <su>xt%e3 %p2"
2378 [(set_attr "type" "alu_ext")]
2381 (define_expand "add<mode>3_carryin"
2382 [(set (match_operand:GPI 0 "register_operand")
2385 (ne:GPI (reg:CC_C CC_REGNUM) (const_int 0))
2386 (match_operand:GPI 1 "aarch64_reg_or_zero"))
2387 (match_operand:GPI 2 "aarch64_reg_or_zero")))]
2392 ;; Note that add with carry with two zero inputs is matched by cset,
2393 ;; and that add with carry with one zero input is matched by cinc.
2395 (define_insn "*add<mode>3_carryin"
2396 [(set (match_operand:GPI 0 "register_operand" "=r")
2399 (match_operand:GPI 3 "aarch64_carry_operation" "")
2400 (match_operand:GPI 1 "register_operand" "r"))
2401 (match_operand:GPI 2 "register_operand" "r")))]
2403 "adc\\t%<w>0, %<w>1, %<w>2"
2404 [(set_attr "type" "adc_reg")]
2407 ;; zero_extend version of above
2408 (define_insn "*addsi3_carryin_uxtw"
2409 [(set (match_operand:DI 0 "register_operand" "=r")
2413 (match_operand:SI 3 "aarch64_carry_operation" "")
2414 (match_operand:SI 1 "register_operand" "r"))
2415 (match_operand:SI 2 "register_operand" "r"))))]
2417 "adc\\t%w0, %w1, %w2"
2418 [(set_attr "type" "adc_reg")]
2421 (define_expand "add<mode>3_carryinC"
2429 (match_operand:GPI 1 "register_operand" "")))
2431 (match_operand:GPI 2 "register_operand" "")))
2434 (plus:GPI (match_dup 5) (match_dup 1))
2436 (set (match_operand:GPI 0 "register_operand")
2438 (plus:GPI (match_dup 5) (match_dup 1))
2442 operands[3] = gen_rtx_REG (CC_Cmode, CC_REGNUM);
2443 operands[4] = gen_rtx_NE (<DWI>mode, operands[3], const0_rtx);
2444 operands[5] = gen_rtx_NE (<MODE>mode, operands[3], const0_rtx);
2447 (define_insn "*add<mode>3_carryinC_zero"
2448 [(set (reg:CC_C CC_REGNUM)
2451 (match_operand:<DWI> 2 "aarch64_carry_operation" "")
2452 (zero_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2455 (match_operand:GPI 3 "aarch64_carry_operation" "")
2457 (set (match_operand:GPI 0 "register_operand" "=r")
2458 (plus:GPI (match_dup 3) (match_dup 1)))]
2460 "adcs\\t%<w>0, %<w>1, <w>zr"
2461 [(set_attr "type" "adc_reg")]
2464 (define_insn "*add<mode>3_carryinC"
2465 [(set (reg:CC_C CC_REGNUM)
2469 (match_operand:<DWI> 3 "aarch64_carry_operation" "")
2470 (zero_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2471 (zero_extend:<DWI> (match_operand:GPI 2 "register_operand" "r")))
2475 (match_operand:GPI 4 "aarch64_carry_operation" "")
2478 (set (match_operand:GPI 0 "register_operand" "=r")
2480 (plus:GPI (match_dup 4) (match_dup 1))
2483 "adcs\\t%<w>0, %<w>1, %<w>2"
2484 [(set_attr "type" "adc_reg")]
2487 (define_expand "add<mode>3_carryinV"
2489 [(set (reg:CC_V CC_REGNUM)
2495 (match_operand:GPI 1 "register_operand" "")))
2497 (match_operand:GPI 2 "register_operand" "")))
2500 (plus:GPI (match_dup 4) (match_dup 1))
2502 (set (match_operand:GPI 0 "register_operand")
2504 (plus:GPI (match_dup 4) (match_dup 1))
2508 rtx cc = gen_rtx_REG (CC_Cmode, CC_REGNUM);
2509 operands[3] = gen_rtx_NE (<DWI>mode, cc, const0_rtx);
2510 operands[4] = gen_rtx_NE (<MODE>mode, cc, const0_rtx);
2513 (define_insn "*add<mode>3_carryinV_zero"
2514 [(set (reg:CC_V CC_REGNUM)
2517 (match_operand:<DWI> 2 "aarch64_carry_operation" "")
2518 (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2521 (match_operand:GPI 3 "aarch64_carry_operation" "")
2523 (set (match_operand:GPI 0 "register_operand" "=r")
2524 (plus:GPI (match_dup 3) (match_dup 1)))]
2526 "adcs\\t%<w>0, %<w>1, <w>zr"
2527 [(set_attr "type" "adc_reg")]
2530 (define_insn "*add<mode>3_carryinV"
2531 [(set (reg:CC_V CC_REGNUM)
2535 (match_operand:<DWI> 3 "aarch64_carry_operation" "")
2536 (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
2537 (sign_extend:<DWI> (match_operand:GPI 2 "register_operand" "r")))
2541 (match_operand:GPI 4 "aarch64_carry_operation" "")
2544 (set (match_operand:GPI 0 "register_operand" "=r")
2546 (plus:GPI (match_dup 4) (match_dup 1))
2549 "adcs\\t%<w>0, %<w>1, %<w>2"
2550 [(set_attr "type" "adc_reg")]
2553 (define_insn "*add_uxt<mode>_shift2"
2554 [(set (match_operand:GPI 0 "register_operand" "=rk")
2556 (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
2557 (match_operand 2 "aarch64_imm3" "Ui3"))
2558 (match_operand 3 "const_int_operand" "n"))
2559 (match_operand:GPI 4 "register_operand" "r")))]
2560 "aarch64_uxt_size (INTVAL (operands[2]), INTVAL (operands[3])) != 0"
2562 operands[3] = GEN_INT (aarch64_uxt_size (INTVAL(operands[2]),
2563 INTVAL (operands[3])));
2564 return \"add\t%<w>0, %<w>4, %<w>1, uxt%e3 %2\";"
2565 [(set_attr "type" "alu_ext")]
2568 ;; zero_extend version of above
2569 (define_insn "*add_uxtsi_shift2_uxtw"
2570 [(set (match_operand:DI 0 "register_operand" "=rk")
2573 (ashift:SI (match_operand:SI 1 "register_operand" "r")
2574 (match_operand 2 "aarch64_imm3" "Ui3"))
2575 (match_operand 3 "const_int_operand" "n"))
2576 (match_operand:SI 4 "register_operand" "r"))))]
2577 "aarch64_uxt_size (INTVAL (operands[2]), INTVAL (operands[3])) != 0"
2579 operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
2580 INTVAL (operands[3])));
2581 return \"add\t%w0, %w4, %w1, uxt%e3 %2\";"
2582 [(set_attr "type" "alu_ext")]
2585 (define_insn "*add_uxt<mode>_multp2"
2586 [(set (match_operand:GPI 0 "register_operand" "=rk")
2588 (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2589 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2590 (match_operand 3 "const_int_operand" "n"))
2591 (match_operand:GPI 4 "register_operand" "r")))]
2592 "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])), INTVAL (operands[3])) != 0"
2594 operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
2595 INTVAL (operands[3])));
2596 return \"add\t%<w>0, %<w>4, %<w>1, uxt%e3 %p2\";"
2597 [(set_attr "type" "alu_ext")]
2600 ;; zero_extend version of above
2601 (define_insn "*add_uxtsi_multp2_uxtw"
2602 [(set (match_operand:DI 0 "register_operand" "=rk")
2605 (mult:SI (match_operand:SI 1 "register_operand" "r")
2606 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2607 (match_operand 3 "const_int_operand" "n"))
2608 (match_operand:SI 4 "register_operand" "r"))))]
2609 "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])), INTVAL (operands[3])) != 0"
2611 operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
2612 INTVAL (operands[3])));
2613 return \"add\t%w0, %w4, %w1, uxt%e3 %p2\";"
2614 [(set_attr "type" "alu_ext")]
2617 (define_insn "subsi3"
2618 [(set (match_operand:SI 0 "register_operand" "=rk")
2619 (minus:SI (match_operand:SI 1 "register_operand" "rk")
2620 (match_operand:SI 2 "register_operand" "r")))]
2622 "sub\\t%w0, %w1, %w2"
2623 [(set_attr "type" "alu_sreg")]
2626 ;; zero_extend version of above
2627 (define_insn "*subsi3_uxtw"
2628 [(set (match_operand:DI 0 "register_operand" "=rk")
2630 (minus:SI (match_operand:SI 1 "register_operand" "rk")
2631 (match_operand:SI 2 "register_operand" "r"))))]
2633 "sub\\t%w0, %w1, %w2"
2634 [(set_attr "type" "alu_sreg")]
2637 (define_insn "subdi3"
2638 [(set (match_operand:DI 0 "register_operand" "=rk,w")
2639 (minus:DI (match_operand:DI 1 "register_operand" "rk,w")
2640 (match_operand:DI 2 "register_operand" "r,w")))]
2644 sub\\t%d0, %d1, %d2"
2645 [(set_attr "type" "alu_sreg, neon_sub")
2646 (set_attr "arch" "*,simd")]
2649 (define_expand "subv<mode>4"
2650 [(match_operand:GPI 0 "register_operand")
2651 (match_operand:GPI 1 "aarch64_reg_or_zero")
2652 (match_operand:GPI 2 "aarch64_reg_or_zero")
2653 (label_ref (match_operand 3 "" ""))]
2656 emit_insn (gen_sub<mode>3_compare1 (operands[0], operands[1], operands[2]));
2657 aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]);
2662 (define_expand "usubv<mode>4"
2663 [(match_operand:GPI 0 "register_operand")
2664 (match_operand:GPI 1 "aarch64_reg_or_zero")
2665 (match_operand:GPI 2 "aarch64_reg_or_zero")
2666 (label_ref (match_operand 3 "" ""))]
2669 emit_insn (gen_sub<mode>3_compare1 (operands[0], operands[1], operands[2]));
2670 aarch64_gen_unlikely_cbranch (LTU, CCmode, operands[3]);
2675 (define_expand "subti3"
2676 [(set (match_operand:TI 0 "register_operand" "")
2677 (minus:TI (match_operand:TI 1 "aarch64_reg_or_zero" "")
2678 (match_operand:TI 2 "register_operand" "")))]
2681 rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
2683 aarch64_subvti_scratch_regs (operands[1], operands[2],
2684 &low_dest, &op1_low, &op2_low,
2685 &high_dest, &op1_high, &op2_high);
2687 emit_insn (gen_subdi3_compare1 (low_dest, op1_low, op2_low));
2688 emit_insn (gen_subdi3_carryin (high_dest, op1_high, op2_high));
2690 emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest);
2691 emit_move_insn (gen_highpart (DImode, operands[0]), high_dest);
2695 (define_expand "subvti4"
2696 [(match_operand:TI 0 "register_operand")
2697 (match_operand:TI 1 "aarch64_reg_or_zero")
2698 (match_operand:TI 2 "aarch64_reg_or_imm")
2699 (label_ref (match_operand 3 "" ""))]
2702 rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
2704 aarch64_subvti_scratch_regs (operands[1], operands[2],
2705 &low_dest, &op1_low, &op2_low,
2706 &high_dest, &op1_high, &op2_high);
2707 aarch64_expand_subvti (operands[0], low_dest, op1_low, op2_low,
2708 high_dest, op1_high, op2_high);
2710 aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]);
2714 (define_expand "usubvti4"
2715 [(match_operand:TI 0 "register_operand")
2716 (match_operand:TI 1 "aarch64_reg_or_zero")
2717 (match_operand:TI 2 "aarch64_reg_or_imm")
2718 (label_ref (match_operand 3 "" ""))]
2721 rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high;
2723 aarch64_subvti_scratch_regs (operands[1], operands[2],
2724 &low_dest, &op1_low, &op2_low,
2725 &high_dest, &op1_high, &op2_high);
2726 aarch64_expand_subvti (operands[0], low_dest, op1_low, op2_low,
2727 high_dest, op1_high, op2_high);
2729 aarch64_gen_unlikely_cbranch (LTU, CCmode, operands[3]);
2733 (define_insn "*sub<mode>3_compare0"
2734 [(set (reg:CC_NZ CC_REGNUM)
2735 (compare:CC_NZ (minus:GPI (match_operand:GPI 1 "register_operand" "r")
2736 (match_operand:GPI 2 "register_operand" "r"))
2738 (set (match_operand:GPI 0 "register_operand" "=r")
2739 (minus:GPI (match_dup 1) (match_dup 2)))]
2741 "subs\\t%<w>0, %<w>1, %<w>2"
2742 [(set_attr "type" "alus_sreg")]
2745 ;; zero_extend version of above
2746 (define_insn "*subsi3_compare0_uxtw"
2747 [(set (reg:CC_NZ CC_REGNUM)
2748 (compare:CC_NZ (minus:SI (match_operand:SI 1 "register_operand" "r")
2749 (match_operand:SI 2 "register_operand" "r"))
2751 (set (match_operand:DI 0 "register_operand" "=r")
2752 (zero_extend:DI (minus:SI (match_dup 1) (match_dup 2))))]
2754 "subs\\t%w0, %w1, %w2"
2755 [(set_attr "type" "alus_sreg")]
2758 (define_insn "*sub<mode>3_compare1_imm"
2759 [(set (reg:CC CC_REGNUM)
2761 (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ,rZ")
2762 (match_operand:GPI 2 "aarch64_plus_immediate" "I,J")))
2763 (set (match_operand:GPI 0 "register_operand" "=r,r")
2766 (match_operand:GPI 3 "aarch64_plus_immediate" "J,I")))]
2767 "UINTVAL (operands[2]) == -UINTVAL (operands[3])"
2769 subs\\t%<w>0, %<w>1, #%n3
2770 adds\\t%<w>0, %<w>1, %3"
2771 [(set_attr "type" "alus_imm")]
2774 (define_insn "sub<mode>3_compare1"
2775 [(set (reg:CC CC_REGNUM)
2777 (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
2778 (match_operand:GPI 2 "aarch64_reg_or_zero" "rZ")))
2779 (set (match_operand:GPI 0 "register_operand" "=r")
2780 (minus:GPI (match_dup 1) (match_dup 2)))]
2782 "subs\\t%<w>0, %<w>1, %<w>2"
2783 [(set_attr "type" "alus_sreg")]
2786 (define_insn "sub<mode>3_compare1_imm"
2787 [(set (reg:CC CC_REGNUM)
2789 (match_operand:GPI 1 "register_operand" "r")
2790 (match_operand:GPI 3 "const_int_operand" "n")))
2791 (set (match_operand:GPI 0 "register_operand" "=r")
2792 (plus:GPI (match_dup 1)
2793 (match_operand:GPI 2 "aarch64_sub_immediate" "J")))]
2794 "INTVAL (operands[3]) == -INTVAL (operands[2])"
2795 "subs\\t%<w>0, %<w>1, #%n2"
2796 [(set_attr "type" "alus_sreg")]
2800 [(set (match_operand:GPI 0 "register_operand")
2801 (minus:GPI (match_operand:GPI 1 "aarch64_reg_or_zero")
2802 (match_operand:GPI 2 "aarch64_reg_or_zero")))
2803 (set (reg:CC CC_REGNUM)
2807 "!reg_overlap_mentioned_p (operands[0], operands[1])
2808 && !reg_overlap_mentioned_p (operands[0], operands[2])"
2811 emit_insn (gen_sub<mode>3_compare1 (operands[0], operands[1],
2817 ;; Same as the above peephole but with the compare and minus in
2818 ;; swapped order. The restriction on overlap between operand 0
2819 ;; and operands 1 and 2 doesn't apply here.
2821 [(set (reg:CC CC_REGNUM)
2823 (match_operand:GPI 1 "aarch64_reg_or_zero")
2824 (match_operand:GPI 2 "aarch64_reg_or_zero")))
2825 (set (match_operand:GPI 0 "register_operand")
2826 (minus:GPI (match_dup 1)
2831 emit_insn (gen_sub<mode>3_compare1 (operands[0], operands[1],
2838 [(set (match_operand:GPI 0 "register_operand")
2839 (plus:GPI (match_operand:GPI 1 "register_operand")
2840 (match_operand:GPI 2 "aarch64_sub_immediate")))
2841 (set (reg:CC CC_REGNUM)
2844 (match_operand:GPI 3 "const_int_operand")))]
2845 "!reg_overlap_mentioned_p (operands[0], operands[1])
2846 && INTVAL (operands[3]) == -INTVAL (operands[2])"
2849 emit_insn (gen_sub<mode>3_compare1_imm (operands[0], operands[1],
2850 operands[2], operands[3]));
2855 ;; Same as the above peephole but with the compare and minus in
2856 ;; swapped order. The restriction on overlap between operand 0
2857 ;; and operands 1 doesn't apply here.
2859 [(set (reg:CC CC_REGNUM)
2861 (match_operand:GPI 1 "register_operand")
2862 (match_operand:GPI 3 "const_int_operand")))
2863 (set (match_operand:GPI 0 "register_operand")
2864 (plus:GPI (match_dup 1)
2865 (match_operand:GPI 2 "aarch64_sub_immediate")))]
2866 "INTVAL (operands[3]) == -INTVAL (operands[2])"
2869 emit_insn (gen_sub<mode>3_compare1_imm (operands[0], operands[1],
2870 operands[2], operands[3]));
2875 (define_insn "*sub_<shift>_<mode>"
2876 [(set (match_operand:GPI 0 "register_operand" "=r")
2877 (minus:GPI (match_operand:GPI 3 "register_operand" "r")
2879 (match_operand:GPI 1 "register_operand" "r")
2880 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
2882 "sub\\t%<w>0, %<w>3, %<w>1, <shift> %2"
2883 [(set_attr "type" "alu_shift_imm")]
2886 ;; zero_extend version of above
2887 (define_insn "*sub_<shift>_si_uxtw"
2888 [(set (match_operand:DI 0 "register_operand" "=r")
2890 (minus:SI (match_operand:SI 3 "register_operand" "r")
2892 (match_operand:SI 1 "register_operand" "r")
2893 (match_operand:QI 2 "aarch64_shift_imm_si" "n")))))]
2895 "sub\\t%w0, %w3, %w1, <shift> %2"
2896 [(set_attr "type" "alu_shift_imm")]
2899 (define_insn "*sub_mul_imm_<mode>"
2900 [(set (match_operand:GPI 0 "register_operand" "=r")
2901 (minus:GPI (match_operand:GPI 3 "register_operand" "r")
2903 (match_operand:GPI 1 "register_operand" "r")
2904 (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))))]
2906 "sub\\t%<w>0, %<w>3, %<w>1, lsl %p2"
2907 [(set_attr "type" "alu_shift_imm")]
2910 ;; zero_extend version of above
2911 (define_insn "*sub_mul_imm_si_uxtw"
2912 [(set (match_operand:DI 0 "register_operand" "=r")
2914 (minus:SI (match_operand:SI 3 "register_operand" "r")
2916 (match_operand:SI 1 "register_operand" "r")
2917 (match_operand:QI 2 "aarch64_pwr_2_si" "n")))))]
2919 "sub\\t%w0, %w3, %w1, lsl %p2"
2920 [(set_attr "type" "alu_shift_imm")]
2923 (define_insn "*sub_<optab><ALLX:mode>_<GPI:mode>"
2924 [(set (match_operand:GPI 0 "register_operand" "=rk")
2925 (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
2927 (match_operand:ALLX 2 "register_operand" "r"))))]
2929 "sub\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size>"
2930 [(set_attr "type" "alu_ext")]
2933 ;; zero_extend version of above
2934 (define_insn "*sub_<optab><SHORT:mode>_si_uxtw"
2935 [(set (match_operand:DI 0 "register_operand" "=rk")
2937 (minus:SI (match_operand:SI 1 "register_operand" "rk")
2939 (match_operand:SHORT 2 "register_operand" "r")))))]
2941 "sub\\t%w0, %w1, %w2, <su>xt<SHORT:size>"
2942 [(set_attr "type" "alu_ext")]
2945 (define_insn "*sub_<optab><ALLX:mode>_shft_<GPI:mode>"
2946 [(set (match_operand:GPI 0 "register_operand" "=rk")
2947 (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
2948 (ashift:GPI (ANY_EXTEND:GPI
2949 (match_operand:ALLX 2 "register_operand" "r"))
2950 (match_operand 3 "aarch64_imm3" "Ui3"))))]
2952 "sub\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size> %3"
2953 [(set_attr "type" "alu_ext")]
2956 ;; zero_extend version of above
2957 (define_insn "*sub_<optab><SHORT:mode>_shft_si_uxtw"
2958 [(set (match_operand:DI 0 "register_operand" "=rk")
2960 (minus:SI (match_operand:SI 1 "register_operand" "rk")
2961 (ashift:SI (ANY_EXTEND:SI
2962 (match_operand:SHORT 2 "register_operand" "r"))
2963 (match_operand 3 "aarch64_imm3" "Ui3")))))]
2965 "sub\\t%w0, %w1, %w2, <su>xt<SHORT:size> %3"
2966 [(set_attr "type" "alu_ext")]
2969 (define_insn "*sub_<optab><mode>_multp2"
2970 [(set (match_operand:GPI 0 "register_operand" "=rk")
2971 (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
2973 (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2974 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2975 (match_operand 3 "const_int_operand" "n")
2977 "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
2978 "sub\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
2979 [(set_attr "type" "alu_ext")]
2982 ;; zero_extend version of above
2983 (define_insn "*sub_<optab>si_multp2_uxtw"
2984 [(set (match_operand:DI 0 "register_operand" "=rk")
2986 (minus:SI (match_operand:SI 4 "register_operand" "rk")
2988 (mult:SI (match_operand:SI 1 "register_operand" "r")
2989 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2990 (match_operand 3 "const_int_operand" "n")
2992 "aarch64_is_extend_from_extract (SImode, operands[2], operands[3])"
2993 "sub\\t%w0, %w4, %w1, <su>xt%e3 %p2"
2994 [(set_attr "type" "alu_ext")]
2997 ;; The hardware description is op1 + ~op2 + C.
2998 ;; = op1 + (-op2 + 1) + (1 - !C)
2999 ;; = op1 - op2 - 1 + 1 - !C
3000 ;; = op1 - op2 - !C.
3001 ;; We describe the latter.
3003 (define_insn "*sub<mode>3_carryin0"
3004 [(set (match_operand:GPI 0 "register_operand" "=r")
3006 (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
3007 (match_operand:GPI 2 "aarch64_borrow_operation" "")))]
3009 "sbc\\t%<w>0, %<w>1, <w>zr"
3010 [(set_attr "type" "adc_reg")]
3013 ;; zero_extend version of the above
3014 (define_insn "*subsi3_carryin_uxtw"
3015 [(set (match_operand:DI 0 "register_operand" "=r")
3018 (match_operand:SI 1 "aarch64_reg_or_zero" "rZ")
3019 (match_operand:SI 2 "aarch64_borrow_operation" ""))))]
3021 "sbc\\t%w0, %w1, wzr"
3022 [(set_attr "type" "adc_reg")]
3025 (define_expand "sub<mode>3_carryin"
3026 [(set (match_operand:GPI 0 "register_operand")
3029 (match_operand:GPI 1 "aarch64_reg_or_zero")
3030 (match_operand:GPI 2 "register_operand"))
3031 (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))))]
3036 (define_insn "*sub<mode>3_carryin"
3037 [(set (match_operand:GPI 0 "register_operand" "=r")
3040 (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
3041 (match_operand:GPI 2 "register_operand" "r"))
3042 (match_operand:GPI 3 "aarch64_borrow_operation" "")))]
3045 "sbc\\t%<w>0, %<w>1, %<w>2"
3046 [(set_attr "type" "adc_reg")]
3049 ;; zero_extend version of the above
3050 (define_insn "*subsi3_carryin_uxtw"
3051 [(set (match_operand:DI 0 "register_operand" "=r")
3055 (match_operand:SI 1 "aarch64_reg_or_zero" "rZ")
3056 (match_operand:SI 2 "register_operand" "r"))
3057 (match_operand:SI 3 "aarch64_borrow_operation" ""))))]
3060 "sbc\\t%w0, %w1, %w2"
3061 [(set_attr "type" "adc_reg")]
3064 (define_insn "*sub<mode>3_carryin_alt"
3065 [(set (match_operand:GPI 0 "register_operand" "=r")
3068 (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
3069 (match_operand:GPI 3 "aarch64_borrow_operation" ""))
3070 (match_operand:GPI 2 "register_operand" "r")))]
3072 "sbc\\t%<w>0, %<w>1, %<w>2"
3073 [(set_attr "type" "adc_reg")]
3076 ;; zero_extend version of the above
3077 (define_insn "*subsi3_carryin_alt_uxtw"
3078 [(set (match_operand:DI 0 "register_operand" "=r")
3082 (match_operand:SI 1 "aarch64_reg_or_zero" "rZ")
3083 (match_operand:SI 3 "aarch64_borrow_operation" ""))
3084 (match_operand:SI 2 "register_operand" "r"))))]
3086 "sbc\\t%w0, %w1, %w2"
3087 [(set_attr "type" "adc_reg")]
3090 (define_expand "sub<mode>3_carryinCV"
3092 [(set (reg:CC CC_REGNUM)
3095 (match_operand:GPI 1 "aarch64_reg_or_zero" ""))
3098 (match_operand:GPI 2 "register_operand" ""))
3099 (ltu:<DWI> (reg:CC CC_REGNUM) (const_int 0)))))
3100 (set (match_operand:GPI 0 "register_operand" "")
3102 (minus:GPI (match_dup 1) (match_dup 2))
3103 (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))))])]
3107 (define_insn "*sub<mode>3_carryinCV_z1_z2"
3108 [(set (reg:CC CC_REGNUM)
3111 (match_operand:<DWI> 2 "aarch64_borrow_operation" "")))
3112 (set (match_operand:GPI 0 "register_operand" "=r")
3113 (neg:GPI (match_operand:GPI 1 "aarch64_borrow_operation" "")))]
3115 "sbcs\\t%<w>0, <w>zr, <w>zr"
3116 [(set_attr "type" "adc_reg")]
3119 (define_insn "*sub<mode>3_carryinCV_z1"
3120 [(set (reg:CC CC_REGNUM)
3125 (match_operand:GPI 1 "register_operand" "r"))
3126 (match_operand:<DWI> 2 "aarch64_borrow_operation" ""))))
3127 (set (match_operand:GPI 0 "register_operand" "=r")
3129 (neg:GPI (match_dup 1))
3130 (match_operand:GPI 3 "aarch64_borrow_operation" "")))]
3132 "sbcs\\t%<w>0, <w>zr, %<w>1"
3133 [(set_attr "type" "adc_reg")]
3136 (define_insn "*sub<mode>3_carryinCV_z2"
3137 [(set (reg:CC CC_REGNUM)
3140 (match_operand:GPI 1 "register_operand" "r"))
3141 (match_operand:<DWI> 2 "aarch64_borrow_operation" "")))
3142 (set (match_operand:GPI 0 "register_operand" "=r")
3145 (match_operand:GPI 3 "aarch64_borrow_operation" "")))]
3147 "sbcs\\t%<w>0, %<w>1, <w>zr"
3148 [(set_attr "type" "adc_reg")]
3151 (define_insn "*sub<mode>3_carryinCV"
3152 [(set (reg:CC CC_REGNUM)
3155 (match_operand:GPI 1 "register_operand" "r"))
3158 (match_operand:GPI 2 "register_operand" "r"))
3159 (match_operand:<DWI> 3 "aarch64_borrow_operation" ""))))
3160 (set (match_operand:GPI 0 "register_operand" "=r")
3162 (minus:GPI (match_dup 1) (match_dup 2))
3163 (match_operand:GPI 4 "aarch64_borrow_operation" "")))]
3165 "sbcs\\t%<w>0, %<w>1, %<w>2"
3166 [(set_attr "type" "adc_reg")]
3169 (define_insn "*sub_uxt<mode>_shift2"
3170 [(set (match_operand:GPI 0 "register_operand" "=rk")
3171 (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
3173 (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
3174 (match_operand 2 "aarch64_imm3" "Ui3"))
3175 (match_operand 3 "const_int_operand" "n"))))]
3176 "aarch64_uxt_size (INTVAL (operands[2]),INTVAL (operands[3])) != 0"
3178 operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
3179 INTVAL (operands[3])));
3180 return \"sub\t%<w>0, %<w>4, %<w>1, uxt%e3 %2\";"
3181 [(set_attr "type" "alu_ext")]
3184 ;; zero_extend version of above
3185 (define_insn "*sub_uxtsi_shift2_uxtw"
3186 [(set (match_operand:DI 0 "register_operand" "=rk")
3188 (minus:SI (match_operand:SI 4 "register_operand" "rk")
3190 (ashift:SI (match_operand:SI 1 "register_operand" "r")
3191 (match_operand 2 "aarch64_imm3" "Ui3"))
3192 (match_operand 3 "const_int_operand" "n")))))]
3193 "aarch64_uxt_size (INTVAL (operands[2]),INTVAL (operands[3])) != 0"
3195 operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
3196 INTVAL (operands[3])));
3197 return \"sub\t%w0, %w4, %w1, uxt%e3 %2\";"
3198 [(set_attr "type" "alu_ext")]
3201 (define_insn "*sub_uxt<mode>_multp2"
3202 [(set (match_operand:GPI 0 "register_operand" "=rk")
3203 (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
3205 (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3206 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
3207 (match_operand 3 "const_int_operand" "n"))))]
3208 "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),INTVAL (operands[3])) != 0"
3210 operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
3211 INTVAL (operands[3])));
3212 return \"sub\t%<w>0, %<w>4, %<w>1, uxt%e3 %p2\";"
3213 [(set_attr "type" "alu_ext")]
3216 ;; zero_extend version of above
3217 (define_insn "*sub_uxtsi_multp2_uxtw"
3218 [(set (match_operand:DI 0 "register_operand" "=rk")
3220 (minus:SI (match_operand:SI 4 "register_operand" "rk")
3222 (mult:SI (match_operand:SI 1 "register_operand" "r")
3223 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
3224 (match_operand 3 "const_int_operand" "n")))))]
3225 "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),INTVAL (operands[3])) != 0"
3227 operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
3228 INTVAL (operands[3])));
3229 return \"sub\t%w0, %w4, %w1, uxt%e3 %p2\";"
3230 [(set_attr "type" "alu_ext")]
3233 (define_expand "abs<mode>2"
3234 [(match_operand:GPI 0 "register_operand" "")
3235 (match_operand:GPI 1 "register_operand" "")]
3238 rtx ccreg = aarch64_gen_compare_reg (LT, operands[1], const0_rtx);
3239 rtx x = gen_rtx_LT (VOIDmode, ccreg, const0_rtx);
3240 emit_insn (gen_csneg3<mode>_insn (operands[0], x, operands[1], operands[1]));
3245 (define_insn "neg<mode>2"
3246 [(set (match_operand:GPI 0 "register_operand" "=r,w")
3247 (neg:GPI (match_operand:GPI 1 "register_operand" "r,w")))]
3251 neg\\t%<rtn>0<vas>, %<rtn>1<vas>"
3252 [(set_attr "type" "alu_sreg, neon_neg<q>")
3253 (set_attr "arch" "*,simd")]
3256 ;; zero_extend version of above
3257 (define_insn "*negsi2_uxtw"
3258 [(set (match_operand:DI 0 "register_operand" "=r")
3259 (zero_extend:DI (neg:SI (match_operand:SI 1 "register_operand" "r"))))]
3262 [(set_attr "type" "alu_sreg")]
3265 (define_insn "*ngc<mode>"
3266 [(set (match_operand:GPI 0 "register_operand" "=r")
3268 (neg:GPI (match_operand:GPI 2 "aarch64_borrow_operation" ""))
3269 (match_operand:GPI 1 "register_operand" "r")))]
3271 "ngc\\t%<w>0, %<w>1"
3272 [(set_attr "type" "adc_reg")]
3275 (define_insn "*ngcsi_uxtw"
3276 [(set (match_operand:DI 0 "register_operand" "=r")
3279 (neg:SI (match_operand:SI 2 "aarch64_borrow_operation" ""))
3280 (match_operand:SI 1 "register_operand" "r"))))]
3283 [(set_attr "type" "adc_reg")]
3286 (define_insn "neg<mode>2_compare0"
3287 [(set (reg:CC_NZ CC_REGNUM)
3288 (compare:CC_NZ (neg:GPI (match_operand:GPI 1 "register_operand" "r"))
3290 (set (match_operand:GPI 0 "register_operand" "=r")
3291 (neg:GPI (match_dup 1)))]
3293 "negs\\t%<w>0, %<w>1"
3294 [(set_attr "type" "alus_sreg")]
3297 ;; zero_extend version of above
3298 (define_insn "*negsi2_compare0_uxtw"
3299 [(set (reg:CC_NZ CC_REGNUM)
3300 (compare:CC_NZ (neg:SI (match_operand:SI 1 "register_operand" "r"))
3302 (set (match_operand:DI 0 "register_operand" "=r")
3303 (zero_extend:DI (neg:SI (match_dup 1))))]
3306 [(set_attr "type" "alus_sreg")]
3309 (define_insn "*neg_<shift><mode>3_compare0"
3310 [(set (reg:CC_NZ CC_REGNUM)
3312 (neg:GPI (ASHIFT:GPI
3313 (match_operand:GPI 1 "register_operand" "r")
3314 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
3316 (set (match_operand:GPI 0 "register_operand" "=r")
3317 (neg:GPI (ASHIFT:GPI (match_dup 1) (match_dup 2))))]
3319 "negs\\t%<w>0, %<w>1, <shift> %2"
3320 [(set_attr "type" "alus_shift_imm")]
3323 (define_insn "*neg_<shift>_<mode>2"
3324 [(set (match_operand:GPI 0 "register_operand" "=r")
3325 (neg:GPI (ASHIFT:GPI
3326 (match_operand:GPI 1 "register_operand" "r")
3327 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
3329 "neg\\t%<w>0, %<w>1, <shift> %2"
3330 [(set_attr "type" "alu_shift_imm")]
3333 ;; zero_extend version of above
3334 (define_insn "*neg_<shift>_si2_uxtw"
3335 [(set (match_operand:DI 0 "register_operand" "=r")
3338 (match_operand:SI 1 "register_operand" "r")
3339 (match_operand:QI 2 "aarch64_shift_imm_si" "n")))))]
3341 "neg\\t%w0, %w1, <shift> %2"
3342 [(set_attr "type" "alu_shift_imm")]
3345 (define_insn "*neg_mul_imm_<mode>2"
3346 [(set (match_operand:GPI 0 "register_operand" "=r")
3348 (match_operand:GPI 1 "register_operand" "r")
3349 (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))))]
3351 "neg\\t%<w>0, %<w>1, lsl %p2"
3352 [(set_attr "type" "alu_shift_imm")]
3355 ;; zero_extend version of above
3356 (define_insn "*neg_mul_imm_si2_uxtw"
3357 [(set (match_operand:DI 0 "register_operand" "=r")
3360 (match_operand:SI 1 "register_operand" "r")
3361 (match_operand:QI 2 "aarch64_pwr_2_si" "n")))))]
3363 "neg\\t%w0, %w1, lsl %p2"
3364 [(set_attr "type" "alu_shift_imm")]
3367 (define_insn "mul<mode>3"
3368 [(set (match_operand:GPI 0 "register_operand" "=r")
3369 (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3370 (match_operand:GPI 2 "register_operand" "r")))]
3372 "mul\\t%<w>0, %<w>1, %<w>2"
3373 [(set_attr "type" "mul")]
3376 ;; zero_extend version of above
3377 (define_insn "*mulsi3_uxtw"
3378 [(set (match_operand:DI 0 "register_operand" "=r")
3380 (mult:SI (match_operand:SI 1 "register_operand" "r")
3381 (match_operand:SI 2 "register_operand" "r"))))]
3383 "mul\\t%w0, %w1, %w2"
3384 [(set_attr "type" "mul")]
3387 (define_insn "madd<mode>"
3388 [(set (match_operand:GPI 0 "register_operand" "=r")
3389 (plus:GPI (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3390 (match_operand:GPI 2 "register_operand" "r"))
3391 (match_operand:GPI 3 "register_operand" "r")))]
3393 "madd\\t%<w>0, %<w>1, %<w>2, %<w>3"
3394 [(set_attr "type" "mla")]
3397 ;; zero_extend version of above
3398 (define_insn "*maddsi_uxtw"
3399 [(set (match_operand:DI 0 "register_operand" "=r")
3401 (plus:SI (mult:SI (match_operand:SI 1 "register_operand" "r")
3402 (match_operand:SI 2 "register_operand" "r"))
3403 (match_operand:SI 3 "register_operand" "r"))))]
3405 "madd\\t%w0, %w1, %w2, %w3"
3406 [(set_attr "type" "mla")]
3409 (define_insn "*msub<mode>"
3410 [(set (match_operand:GPI 0 "register_operand" "=r")
3411 (minus:GPI (match_operand:GPI 3 "register_operand" "r")
3412 (mult:GPI (match_operand:GPI 1 "register_operand" "r")
3413 (match_operand:GPI 2 "register_operand" "r"))))]
3416 "msub\\t%<w>0, %<w>1, %<w>2, %<w>3"
3417 [(set_attr "type" "mla")]
3420 ;; zero_extend version of above
3421 (define_insn "*msubsi_uxtw"
3422 [(set (match_operand:DI 0 "register_operand" "=r")
3424 (minus:SI (match_operand:SI 3 "register_operand" "r")
3425 (mult:SI (match_operand:SI 1 "register_operand" "r")
3426 (match_operand:SI 2 "register_operand" "r")))))]
3429 "msub\\t%w0, %w1, %w2, %w3"
3430 [(set_attr "type" "mla")]
3433 (define_insn "*mul<mode>_neg"
3434 [(set (match_operand:GPI 0 "register_operand" "=r")
3435 (mult:GPI (neg:GPI (match_operand:GPI 1 "register_operand" "r"))
3436 (match_operand:GPI 2 "register_operand" "r")))]
3439 "mneg\\t%<w>0, %<w>1, %<w>2"
3440 [(set_attr "type" "mul")]
3443 ;; zero_extend version of above
3444 (define_insn "*mulsi_neg_uxtw"
3445 [(set (match_operand:DI 0 "register_operand" "=r")
3447 (mult:SI (neg:SI (match_operand:SI 1 "register_operand" "r"))
3448 (match_operand:SI 2 "register_operand" "r"))))]
3451 "mneg\\t%w0, %w1, %w2"
3452 [(set_attr "type" "mul")]
3455 (define_insn "<su_optab>mulsidi3"
3456 [(set (match_operand:DI 0 "register_operand" "=r")
3457 (mult:DI (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
3458 (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r"))))]
3460 "<su>mull\\t%0, %w1, %w2"
3461 [(set_attr "type" "<su>mull")]
3464 (define_insn "<su_optab>maddsidi4"
3465 [(set (match_operand:DI 0 "register_operand" "=r")
3467 (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
3468 (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r")))
3469 (match_operand:DI 3 "register_operand" "r")))]
3471 "<su>maddl\\t%0, %w1, %w2, %3"
3472 [(set_attr "type" "<su>mlal")]
3475 (define_insn "<su_optab>msubsidi4"
3476 [(set (match_operand:DI 0 "register_operand" "=r")
3478 (match_operand:DI 3 "register_operand" "r")
3479 (mult:DI (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
3481 (match_operand:SI 2 "register_operand" "r")))))]
3483 "<su>msubl\\t%0, %w1, %w2, %3"
3484 [(set_attr "type" "<su>mlal")]
3487 (define_insn "*<su_optab>mulsidi_neg"
3488 [(set (match_operand:DI 0 "register_operand" "=r")
3490 (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r")))
3491 (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r"))))]
3493 "<su>mnegl\\t%0, %w1, %w2"
3494 [(set_attr "type" "<su>mull")]
3497 (define_expand "<su_optab>mulditi3"
3498 [(set (match_operand:TI 0 "register_operand")
3499 (mult:TI (ANY_EXTEND:TI (match_operand:DI 1 "register_operand"))
3500 (ANY_EXTEND:TI (match_operand:DI 2 "register_operand"))))]
3503 rtx low = gen_reg_rtx (DImode);
3504 emit_insn (gen_muldi3 (low, operands[1], operands[2]));
3506 rtx high = gen_reg_rtx (DImode);
3507 emit_insn (gen_<su>muldi3_highpart (high, operands[1], operands[2]));
3509 emit_move_insn (gen_lowpart (DImode, operands[0]), low);
3510 emit_move_insn (gen_highpart (DImode, operands[0]), high);
3514 ;; The default expansion of multi3 using umuldi3_highpart will perform
3515 ;; the additions in an order that fails to combine into two madd insns.
3516 (define_expand "multi3"
3517 [(set (match_operand:TI 0 "register_operand")
3518 (mult:TI (match_operand:TI 1 "register_operand")
3519 (match_operand:TI 2 "register_operand")))]
3522 rtx l0 = gen_reg_rtx (DImode);
3523 rtx l1 = gen_lowpart (DImode, operands[1]);
3524 rtx l2 = gen_lowpart (DImode, operands[2]);
3525 rtx h0 = gen_reg_rtx (DImode);
3526 rtx h1 = gen_highpart (DImode, operands[1]);
3527 rtx h2 = gen_highpart (DImode, operands[2]);
3529 emit_insn (gen_muldi3 (l0, l1, l2));
3530 emit_insn (gen_umuldi3_highpart (h0, l1, l2));
3531 emit_insn (gen_madddi (h0, h1, l2, h0));
3532 emit_insn (gen_madddi (h0, l1, h2, h0));
3534 emit_move_insn (gen_lowpart (DImode, operands[0]), l0);
3535 emit_move_insn (gen_highpart (DImode, operands[0]), h0);
3539 (define_insn "<su>muldi3_highpart"
3540 [(set (match_operand:DI 0 "register_operand" "=r")
3544 (ANY_EXTEND:TI (match_operand:DI 1 "register_operand" "r"))
3545 (ANY_EXTEND:TI (match_operand:DI 2 "register_operand" "r")))
3548 "<su>mulh\\t%0, %1, %2"
3549 [(set_attr "type" "<su>mull")]
3552 (define_insn "<su_optab>div<mode>3"
3553 [(set (match_operand:GPI 0 "register_operand" "=r")
3554 (ANY_DIV:GPI (match_operand:GPI 1 "register_operand" "r")
3555 (match_operand:GPI 2 "register_operand" "r")))]
3557 "<su>div\\t%<w>0, %<w>1, %<w>2"
3558 [(set_attr "type" "<su>div")]
3561 ;; zero_extend version of above
3562 (define_insn "*<su_optab>divsi3_uxtw"
3563 [(set (match_operand:DI 0 "register_operand" "=r")
3565 (ANY_DIV:SI (match_operand:SI 1 "register_operand" "r")
3566 (match_operand:SI 2 "register_operand" "r"))))]
3568 "<su>div\\t%w0, %w1, %w2"
3569 [(set_attr "type" "<su>div")]
3572 ;; -------------------------------------------------------------------
3574 ;; -------------------------------------------------------------------
3576 (define_insn "cmp<mode>"
3577 [(set (reg:CC CC_REGNUM)
3578 (compare:CC (match_operand:GPI 0 "register_operand" "rk,rk,rk")
3579 (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J")))]
3585 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
3588 (define_insn "fcmp<mode>"
3589 [(set (reg:CCFP CC_REGNUM)
3590 (compare:CCFP (match_operand:GPF 0 "register_operand" "w,w")
3591 (match_operand:GPF 1 "aarch64_fp_compare_operand" "Y,w")))]
3595 fcmp\\t%<s>0, %<s>1"
3596 [(set_attr "type" "fcmp<s>")]
3599 (define_insn "fcmpe<mode>"
3600 [(set (reg:CCFPE CC_REGNUM)
3601 (compare:CCFPE (match_operand:GPF 0 "register_operand" "w,w")
3602 (match_operand:GPF 1 "aarch64_fp_compare_operand" "Y,w")))]
3606 fcmpe\\t%<s>0, %<s>1"
3607 [(set_attr "type" "fcmp<s>")]
3610 (define_insn "*cmp_swp_<shift>_reg<mode>"
3611 [(set (reg:CC_SWP CC_REGNUM)
3612 (compare:CC_SWP (ASHIFT:GPI
3613 (match_operand:GPI 0 "register_operand" "r")
3614 (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n"))
3615 (match_operand:GPI 2 "aarch64_reg_or_zero" "rZ")))]
3617 "cmp\\t%<w>2, %<w>0, <shift> %1"
3618 [(set_attr "type" "alus_shift_imm")]
3621 (define_insn "*cmp_swp_<optab><ALLX:mode>_reg<GPI:mode>"
3622 [(set (reg:CC_SWP CC_REGNUM)
3623 (compare:CC_SWP (ANY_EXTEND:GPI
3624 (match_operand:ALLX 0 "register_operand" "r"))
3625 (match_operand:GPI 1 "register_operand" "r")))]
3627 "cmp\\t%<GPI:w>1, %<GPI:w>0, <su>xt<ALLX:size>"
3628 [(set_attr "type" "alus_ext")]
3631 (define_insn "*cmp_swp_<optab><ALLX:mode>_shft_<GPI:mode>"
3632 [(set (reg:CC_SWP CC_REGNUM)
3633 (compare:CC_SWP (ashift:GPI
3635 (match_operand:ALLX 0 "register_operand" "r"))
3636 (match_operand 1 "aarch64_imm3" "Ui3"))
3637 (match_operand:GPI 2 "register_operand" "r")))]
3639 "cmp\\t%<GPI:w>2, %<GPI:w>0, <su>xt<ALLX:size> %1"
3640 [(set_attr "type" "alus_ext")]
3643 ;; -------------------------------------------------------------------
3644 ;; Store-flag and conditional select insns
3645 ;; -------------------------------------------------------------------
3647 (define_expand "cstore<mode>4"
3648 [(set (match_operand:SI 0 "register_operand" "")
3649 (match_operator:SI 1 "aarch64_comparison_operator"
3650 [(match_operand:GPI 2 "register_operand" "")
3651 (match_operand:GPI 3 "aarch64_plus_operand" "")]))]
3654 operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
3656 operands[3] = const0_rtx;
3660 (define_expand "cstorecc4"
3661 [(set (match_operand:SI 0 "register_operand")
3662 (match_operator 1 "aarch64_comparison_operator_mode"
3663 [(match_operand 2 "cc_register")
3664 (match_operand 3 "const0_operand")]))]
3667 emit_insn (gen_rtx_SET (operands[0], operands[1]));
3672 (define_expand "cstore<mode>4"
3673 [(set (match_operand:SI 0 "register_operand" "")
3674 (match_operator:SI 1 "aarch64_comparison_operator_mode"
3675 [(match_operand:GPF 2 "register_operand" "")
3676 (match_operand:GPF 3 "aarch64_fp_compare_operand" "")]))]
3679 operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
3681 operands[3] = const0_rtx;
3685 (define_insn "aarch64_cstore<mode>"
3686 [(set (match_operand:ALLI 0 "register_operand" "=r")
3687 (match_operator:ALLI 1 "aarch64_comparison_operator_mode"
3688 [(match_operand 2 "cc_register" "") (const_int 0)]))]
3691 [(set_attr "type" "csel")]
3694 ;; For a 24-bit immediate CST we can optimize the compare for equality
3695 ;; and branch sequence from:
3697 ;; movk x0, #imm2, lsl 16 /* x0 contains CST. */
3700 ;; into the shorter:
3701 ;; sub x0, x1, #(CST & 0xfff000)
3702 ;; subs x0, x0, #(CST & 0x000fff)
3703 ;; cset x2, <ne, eq>.
3704 (define_insn_and_split "*compare_cstore<mode>_insn"
3705 [(set (match_operand:GPI 0 "register_operand" "=r")
3706 (EQL:GPI (match_operand:GPI 1 "register_operand" "r")
3707 (match_operand:GPI 2 "aarch64_imm24" "n")))
3708 (clobber (reg:CC CC_REGNUM))]
3709 "!aarch64_move_imm (INTVAL (operands[2]), <MODE>mode)
3710 && !aarch64_plus_operand (operands[2], <MODE>mode)
3711 && !reload_completed"
3716 HOST_WIDE_INT lo_imm = UINTVAL (operands[2]) & 0xfff;
3717 HOST_WIDE_INT hi_imm = UINTVAL (operands[2]) & 0xfff000;
3718 rtx tmp = gen_reg_rtx (<MODE>mode);
3719 emit_insn (gen_add<mode>3 (tmp, operands[1], GEN_INT (-hi_imm)));
3720 emit_insn (gen_add<mode>3_compare0 (tmp, tmp, GEN_INT (-lo_imm)));
3721 rtx cc_reg = gen_rtx_REG (CC_NZmode, CC_REGNUM);
3722 rtx cmp_rtx = gen_rtx_fmt_ee (<EQL:CMP>, <MODE>mode, cc_reg, const0_rtx);
3723 emit_insn (gen_aarch64_cstore<mode> (operands[0], cmp_rtx, cc_reg));
3726 [(set_attr "type" "csel")]
3729 ;; zero_extend version of the above
3730 (define_insn "*cstoresi_insn_uxtw"
3731 [(set (match_operand:DI 0 "register_operand" "=r")
3733 (match_operator:SI 1 "aarch64_comparison_operator_mode"
3734 [(match_operand 2 "cc_register" "") (const_int 0)])))]
3737 [(set_attr "type" "csel")]
3740 (define_insn "cstore<mode>_neg"
3741 [(set (match_operand:ALLI 0 "register_operand" "=r")
3742 (neg:ALLI (match_operator:ALLI 1 "aarch64_comparison_operator_mode"
3743 [(match_operand 2 "cc_register" "") (const_int 0)])))]
3745 "csetm\\t%<w>0, %m1"
3746 [(set_attr "type" "csel")]
3749 ;; zero_extend version of the above
3750 (define_insn "*cstoresi_neg_uxtw"
3751 [(set (match_operand:DI 0 "register_operand" "=r")
3753 (neg:SI (match_operator:SI 1 "aarch64_comparison_operator_mode"
3754 [(match_operand 2 "cc_register" "") (const_int 0)]))))]
3757 [(set_attr "type" "csel")]
3760 (define_expand "cmov<mode>6"
3761 [(set (match_operand:GPI 0 "register_operand" "")
3763 (match_operator 1 "aarch64_comparison_operator"
3764 [(match_operand:GPI 2 "register_operand" "")
3765 (match_operand:GPI 3 "aarch64_plus_operand" "")])
3766 (match_operand:GPI 4 "register_operand" "")
3767 (match_operand:GPI 5 "register_operand" "")))]
3770 operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
3772 operands[3] = const0_rtx;
3776 (define_expand "cmov<mode>6"
3777 [(set (match_operand:GPF 0 "register_operand" "")
3779 (match_operator 1 "aarch64_comparison_operator"
3780 [(match_operand:GPF 2 "register_operand" "")
3781 (match_operand:GPF 3 "aarch64_fp_compare_operand" "")])
3782 (match_operand:GPF 4 "register_operand" "")
3783 (match_operand:GPF 5 "register_operand" "")))]
3786 operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
3788 operands[3] = const0_rtx;
3792 (define_insn "*cmov<mode>_insn"
3793 [(set (match_operand:ALLI 0 "register_operand" "=r,r,r,r,r,r,r")
3795 (match_operator 1 "aarch64_comparison_operator"
3796 [(match_operand 2 "cc_register" "") (const_int 0)])
3797 (match_operand:ALLI 3 "aarch64_reg_zero_or_m1_or_1" "rZ,rZ,UsM,rZ,Ui1,UsM,Ui1")
3798 (match_operand:ALLI 4 "aarch64_reg_zero_or_m1_or_1" "rZ,UsM,rZ,Ui1,rZ,UsM,Ui1")))]
3799 "!((operands[3] == const1_rtx && operands[4] == constm1_rtx)
3800 || (operands[3] == constm1_rtx && operands[4] == const1_rtx))"
3801 ;; Final two alternatives should be unreachable, but included for completeness
3803 csel\\t%<w>0, %<w>3, %<w>4, %m1
3804 csinv\\t%<w>0, %<w>3, <w>zr, %m1
3805 csinv\\t%<w>0, %<w>4, <w>zr, %M1
3806 csinc\\t%<w>0, %<w>3, <w>zr, %m1
3807 csinc\\t%<w>0, %<w>4, <w>zr, %M1
3810 [(set_attr "type" "csel, csel, csel, csel, csel, mov_imm, mov_imm")]
3813 ;; zero_extend version of above
3814 (define_insn "*cmovsi_insn_uxtw"
3815 [(set (match_operand:DI 0 "register_operand" "=r,r,r,r,r,r,r")
3818 (match_operator 1 "aarch64_comparison_operator"
3819 [(match_operand 2 "cc_register" "") (const_int 0)])
3820 (match_operand:SI 3 "aarch64_reg_zero_or_m1_or_1" "rZ,rZ,UsM,rZ,Ui1,UsM,Ui1")
3821 (match_operand:SI 4 "aarch64_reg_zero_or_m1_or_1" "rZ,UsM,rZ,Ui1,rZ,UsM,Ui1"))))]
3822 "!((operands[3] == const1_rtx && operands[4] == constm1_rtx)
3823 || (operands[3] == constm1_rtx && operands[4] == const1_rtx))"
3824 ;; Final two alternatives should be unreachable, but included for completeness
3826 csel\\t%w0, %w3, %w4, %m1
3827 csinv\\t%w0, %w3, wzr, %m1
3828 csinv\\t%w0, %w4, wzr, %M1
3829 csinc\\t%w0, %w3, wzr, %m1
3830 csinc\\t%w0, %w4, wzr, %M1
3833 [(set_attr "type" "csel, csel, csel, csel, csel, mov_imm, mov_imm")]
3836 (define_insn "*cmovdi_insn_uxtw"
3837 [(set (match_operand:DI 0 "register_operand" "=r")
3839 (match_operator 1 "aarch64_comparison_operator"
3840 [(match_operand 2 "cc_register" "") (const_int 0)])
3841 (zero_extend:DI (match_operand:SI 3 "register_operand" "r"))
3842 (zero_extend:DI (match_operand:SI 4 "register_operand" "r"))))]
3844 "csel\\t%w0, %w3, %w4, %m1"
3845 [(set_attr "type" "csel")]
3848 (define_insn "*cmov<mode>_insn"
3849 [(set (match_operand:GPF 0 "register_operand" "=w")
3851 (match_operator 1 "aarch64_comparison_operator"
3852 [(match_operand 2 "cc_register" "") (const_int 0)])
3853 (match_operand:GPF 3 "register_operand" "w")
3854 (match_operand:GPF 4 "register_operand" "w")))]
3856 "fcsel\\t%<s>0, %<s>3, %<s>4, %m1"
3857 [(set_attr "type" "fcsel")]
3860 (define_expand "mov<mode>cc"
3861 [(set (match_operand:ALLI 0 "register_operand" "")
3862 (if_then_else:ALLI (match_operand 1 "aarch64_comparison_operator" "")
3863 (match_operand:ALLI 2 "register_operand" "")
3864 (match_operand:ALLI 3 "register_operand" "")))]
3868 enum rtx_code code = GET_CODE (operands[1]);
3870 if (code == UNEQ || code == LTGT)
3873 ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
3874 XEXP (operands[1], 1));
3875 operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
3879 (define_expand "mov<GPF:mode><GPI:mode>cc"
3880 [(set (match_operand:GPI 0 "register_operand" "")
3881 (if_then_else:GPI (match_operand 1 "aarch64_comparison_operator" "")
3882 (match_operand:GPF 2 "register_operand" "")
3883 (match_operand:GPF 3 "register_operand" "")))]
3887 enum rtx_code code = GET_CODE (operands[1]);
3889 if (code == UNEQ || code == LTGT)
3892 ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
3893 XEXP (operands[1], 1));
3894 operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
3898 (define_expand "mov<mode>cc"
3899 [(set (match_operand:GPF 0 "register_operand" "")
3900 (if_then_else:GPF (match_operand 1 "aarch64_comparison_operator" "")
3901 (match_operand:GPF 2 "register_operand" "")
3902 (match_operand:GPF 3 "register_operand" "")))]
3906 enum rtx_code code = GET_CODE (operands[1]);
3908 if (code == UNEQ || code == LTGT)
3911 ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
3912 XEXP (operands[1], 1));
3913 operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
3917 (define_expand "<neg_not_op><mode>cc"
3918 [(set (match_operand:GPI 0 "register_operand" "")
3919 (if_then_else:GPI (match_operand 1 "aarch64_comparison_operator" "")
3920 (NEG_NOT:GPI (match_operand:GPI 2 "register_operand" ""))
3921 (match_operand:GPI 3 "register_operand" "")))]
3925 enum rtx_code code = GET_CODE (operands[1]);
3927 if (code == UNEQ || code == LTGT)
3930 ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
3931 XEXP (operands[1], 1));
3932 operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
3936 ;; CRC32 instructions.
3937 (define_insn "aarch64_<crc_variant>"
3938 [(set (match_operand:SI 0 "register_operand" "=r")
3939 (unspec:SI [(match_operand:SI 1 "register_operand" "r")
3940 (match_operand:<crc_mode> 2 "register_operand" "r")]
3944 if (GET_MODE_BITSIZE (<crc_mode>mode) >= 64)
3945 return "<crc_variant>\\t%w0, %w1, %x2";
3947 return "<crc_variant>\\t%w0, %w1, %w2";
3949 [(set_attr "type" "crc")]
3952 (define_insn "*csinc2<mode>_insn"
3953 [(set (match_operand:GPI 0 "register_operand" "=r")
3954 (plus:GPI (match_operand 2 "aarch64_comparison_operation" "")
3955 (match_operand:GPI 1 "register_operand" "r")))]
3957 "cinc\\t%<w>0, %<w>1, %m2"
3958 [(set_attr "type" "csel")]
3961 (define_insn "csinc3<mode>_insn"
3962 [(set (match_operand:GPI 0 "register_operand" "=r")
3964 (match_operand 1 "aarch64_comparison_operation" "")
3965 (plus:GPI (match_operand:GPI 2 "register_operand" "r")
3967 (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
3969 "csinc\\t%<w>0, %<w>3, %<w>2, %M1"
3970 [(set_attr "type" "csel")]
3973 (define_insn "*csinv3<mode>_insn"
3974 [(set (match_operand:GPI 0 "register_operand" "=r")
3976 (match_operand 1 "aarch64_comparison_operation" "")
3977 (not:GPI (match_operand:GPI 2 "register_operand" "r"))
3978 (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
3980 "csinv\\t%<w>0, %<w>3, %<w>2, %M1"
3981 [(set_attr "type" "csel")]
3984 (define_insn "csneg3_uxtw_insn"
3985 [(set (match_operand:DI 0 "register_operand" "=r")
3988 (match_operand 1 "aarch64_comparison_operation" "")
3989 (neg:SI (match_operand:SI 2 "register_operand" "r"))
3990 (match_operand:SI 3 "aarch64_reg_or_zero" "rZ"))))]
3992 "csneg\\t%w0, %w3, %w2, %M1"
3993 [(set_attr "type" "csel")]
3996 (define_insn "csneg3<mode>_insn"
3997 [(set (match_operand:GPI 0 "register_operand" "=r")
3999 (match_operand 1 "aarch64_comparison_operation" "")
4000 (neg:GPI (match_operand:GPI 2 "register_operand" "r"))
4001 (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
4003 "csneg\\t%<w>0, %<w>3, %<w>2, %M1"
4004 [(set_attr "type" "csel")]
4007 ;; If X can be loaded by a single CNT[BHWD] instruction,
4011 ;; is equivalent to:
4013 ;; TMP = UQDEC[BHWD] (B, X)
4016 ;; Defining the pattern this way means that:
4018 ;; A = UMAX (B, X) - X
4022 ;; TMP1 = UQDEC[BHWD] (B, X)
4026 ;; which combine can optimize to:
4028 ;; A = UQDEC[BHWD] (B, X)
4030 ;; We don't use match_operand predicates because the order of the operands
4031 ;; can vary: the CNT[BHWD] constant will come first if the other operand is
4032 ;; a simpler constant (such as a CONST_INT), otherwise it will come second.
4033 (define_expand "umax<mode>3"
4034 [(set (match_operand:GPI 0 "register_operand")
4035 (umax:GPI (match_operand:GPI 1 "")
4036 (match_operand:GPI 2 "")))]
4039 if (aarch64_sve_cnt_immediate (operands[1], <MODE>mode))
4040 std::swap (operands[1], operands[2]);
4041 else if (!aarch64_sve_cnt_immediate (operands[2], <MODE>mode))
4043 rtx temp = gen_reg_rtx (<MODE>mode);
4044 operands[1] = force_reg (<MODE>mode, operands[1]);
4045 emit_insn (gen_aarch64_uqdec<mode> (temp, operands[1], operands[2]));
4046 emit_insn (gen_add<mode>3 (operands[0], temp, operands[2]));
4051 ;; Saturating unsigned subtraction of a CNT[BHWD] immediate.
4052 (define_insn "aarch64_uqdec<mode>"
4053 [(set (match_operand:GPI 0 "register_operand" "=r")
4055 (umax:GPI (match_operand:GPI 1 "register_operand" "0")
4056 (match_operand:GPI 2 "aarch64_sve_cnt_immediate" "Usv"))
4060 return aarch64_output_sve_cnt_immediate ("uqdec", "%<w>0", operands[2]);
4064 ;; -------------------------------------------------------------------
4065 ;; Logical operations
4066 ;; -------------------------------------------------------------------
4069 (define_insn_and_split "*aarch64_and<mode>_imm2"
4070 [(set (match_operand:GPI 0 "register_operand" "=rk")
4071 (and:GPI (match_operand:GPI 1 "register_operand" "%r")
4072 (match_operand:GPI 2 "aarch64_logical_and_immediate" "<lconst2>")))]
4078 HOST_WIDE_INT val = INTVAL (operands[2]);
4079 rtx imm1 = GEN_INT (aarch64_and_split_imm1 (val));
4080 rtx imm2 = GEN_INT (aarch64_and_split_imm2 (val));
4082 emit_insn (gen_and<mode>3 (operands[0], operands[1], imm1));
4083 emit_insn (gen_and<mode>3 (operands[0], operands[0], imm2));
4088 (define_insn "<optab><mode>3"
4089 [(set (match_operand:GPI 0 "register_operand" "=r,rk,w")
4090 (LOGICAL:GPI (match_operand:GPI 1 "register_operand" "%r,r,w")
4091 (match_operand:GPI 2 "aarch64_logical_operand" "r,<lconst>,w")))]
4094 <logical>\\t%<w>0, %<w>1, %<w>2
4095 <logical>\\t%<w>0, %<w>1, %2
4096 <logical>\\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>"
4097 [(set_attr "type" "logic_reg,logic_imm,neon_logic")
4098 (set_attr "arch" "*,*,simd")]
4101 ;; zero_extend version of above
4102 (define_insn "*<optab>si3_uxtw"
4103 [(set (match_operand:DI 0 "register_operand" "=r,rk")
4105 (LOGICAL:SI (match_operand:SI 1 "register_operand" "%r,r")
4106 (match_operand:SI 2 "aarch64_logical_operand" "r,K"))))]
4109 <logical>\\t%w0, %w1, %w2
4110 <logical>\\t%w0, %w1, %2"
4111 [(set_attr "type" "logic_reg,logic_imm")]
4114 (define_insn "*and<mode>3_compare0"
4115 [(set (reg:CC_NZ CC_REGNUM)
4117 (and:GPI (match_operand:GPI 1 "register_operand" "%r,r")
4118 (match_operand:GPI 2 "aarch64_logical_operand" "r,<lconst>"))
4120 (set (match_operand:GPI 0 "register_operand" "=r,r")
4121 (and:GPI (match_dup 1) (match_dup 2)))]
4124 ands\\t%<w>0, %<w>1, %<w>2
4125 ands\\t%<w>0, %<w>1, %2"
4126 [(set_attr "type" "logics_reg,logics_imm")]
4129 ;; zero_extend version of above
4130 (define_insn "*andsi3_compare0_uxtw"
4131 [(set (reg:CC_NZ CC_REGNUM)
4133 (and:SI (match_operand:SI 1 "register_operand" "%r,r")
4134 (match_operand:SI 2 "aarch64_logical_operand" "r,K"))
4136 (set (match_operand:DI 0 "register_operand" "=r,r")
4137 (zero_extend:DI (and:SI (match_dup 1) (match_dup 2))))]
4140 ands\\t%w0, %w1, %w2
4141 ands\\t%w0, %w1, %2"
4142 [(set_attr "type" "logics_reg,logics_imm")]
4145 (define_insn "*and_<SHIFT:optab><mode>3_compare0"
4146 [(set (reg:CC_NZ CC_REGNUM)
4149 (match_operand:GPI 1 "register_operand" "r")
4150 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
4151 (match_operand:GPI 3 "register_operand" "r"))
4153 (set (match_operand:GPI 0 "register_operand" "=r")
4154 (and:GPI (SHIFT:GPI (match_dup 1) (match_dup 2)) (match_dup 3)))]
4156 "ands\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
4157 [(set_attr "type" "logics_shift_imm")]
4160 ;; zero_extend version of above
4161 (define_insn "*and_<SHIFT:optab>si3_compare0_uxtw"
4162 [(set (reg:CC_NZ CC_REGNUM)
4165 (match_operand:SI 1 "register_operand" "r")
4166 (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
4167 (match_operand:SI 3 "register_operand" "r"))
4169 (set (match_operand:DI 0 "register_operand" "=r")
4170 (zero_extend:DI (and:SI (SHIFT:SI (match_dup 1) (match_dup 2))
4173 "ands\\t%w0, %w3, %w1, <SHIFT:shift> %2"
4174 [(set_attr "type" "logics_shift_imm")]
4177 (define_insn "*<LOGICAL:optab>_<SHIFT:optab><mode>3"
4178 [(set (match_operand:GPI 0 "register_operand" "=r")
4179 (LOGICAL:GPI (SHIFT:GPI
4180 (match_operand:GPI 1 "register_operand" "r")
4181 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
4182 (match_operand:GPI 3 "register_operand" "r")))]
4184 "<LOGICAL:logical>\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
4185 [(set_attr "type" "logic_shift_imm")]
4188 (define_insn "*<optab>_rol<mode>3"
4189 [(set (match_operand:GPI 0 "register_operand" "=r")
4190 (LOGICAL:GPI (rotate:GPI
4191 (match_operand:GPI 1 "register_operand" "r")
4192 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
4193 (match_operand:GPI 3 "register_operand" "r")))]
4195 "<logical>\\t%<w>0, %<w>3, %<w>1, ror (<sizen> - %2)"
4196 [(set_attr "type" "logic_shift_imm")]
4199 ;; zero_extend versions of above
4200 (define_insn "*<LOGICAL:optab>_<SHIFT:optab>si3_uxtw"
4201 [(set (match_operand:DI 0 "register_operand" "=r")
4203 (LOGICAL:SI (SHIFT:SI
4204 (match_operand:SI 1 "register_operand" "r")
4205 (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
4206 (match_operand:SI 3 "register_operand" "r"))))]
4208 "<LOGICAL:logical>\\t%w0, %w3, %w1, <SHIFT:shift> %2"
4209 [(set_attr "type" "logic_shift_imm")]
4212 (define_insn "*<optab>_rolsi3_uxtw"
4213 [(set (match_operand:DI 0 "register_operand" "=r")
4215 (LOGICAL:SI (rotate:SI
4216 (match_operand:SI 1 "register_operand" "r")
4217 (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
4218 (match_operand:SI 3 "register_operand" "r"))))]
4220 "<logical>\\t%w0, %w3, %w1, ror (32 - %2)"
4221 [(set_attr "type" "logic_shift_imm")]
4224 (define_insn "one_cmpl<mode>2"
4225 [(set (match_operand:GPI 0 "register_operand" "=r,w")
4226 (not:GPI (match_operand:GPI 1 "register_operand" "r,w")))]
4231 [(set_attr "type" "logic_reg,neon_logic")
4232 (set_attr "arch" "*,simd")]
4235 (define_insn "*one_cmpl_<optab><mode>2"
4236 [(set (match_operand:GPI 0 "register_operand" "=r")
4237 (not:GPI (SHIFT:GPI (match_operand:GPI 1 "register_operand" "r")
4238 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
4240 "mvn\\t%<w>0, %<w>1, <shift> %2"
4241 [(set_attr "type" "logic_shift_imm")]
4244 ;; Binary logical operators negating one operand, i.e. (a & !b), (a | !b).
4246 (define_insn "*<NLOGICAL:optab>_one_cmpl<mode>3"
4247 [(set (match_operand:GPI 0 "register_operand" "=r,w")
4248 (NLOGICAL:GPI (not:GPI (match_operand:GPI 1 "register_operand" "r,w"))
4249 (match_operand:GPI 2 "register_operand" "r,w")))]
4252 <NLOGICAL:nlogical>\\t%<w>0, %<w>2, %<w>1
4253 <NLOGICAL:nlogical>\\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>"
4254 [(set_attr "type" "logic_reg,neon_logic")
4255 (set_attr "arch" "*,simd")]
4258 (define_insn "*<NLOGICAL:optab>_one_cmplsidi3_ze"
4259 [(set (match_operand:DI 0 "register_operand" "=r")
4261 (NLOGICAL:SI (not:SI (match_operand:SI 1 "register_operand" "r"))
4262 (match_operand:SI 2 "register_operand" "r"))))]
4264 "<NLOGICAL:nlogical>\\t%w0, %w2, %w1"
4265 [(set_attr "type" "logic_reg")]
4268 (define_insn "*xor_one_cmplsidi3_ze"
4269 [(set (match_operand:DI 0 "register_operand" "=r")
4271 (not:SI (xor:SI (match_operand:SI 1 "register_operand" "r")
4272 (match_operand:SI 2 "register_operand" "r")))))]
4274 "eon\\t%w0, %w1, %w2"
4275 [(set_attr "type" "logic_reg")]
4278 ;; (xor (not a) b) is simplify_rtx-ed down to (not (xor a b)).
4279 ;; eon does not operate on SIMD registers so the vector variant must be split.
4280 (define_insn_and_split "*xor_one_cmpl<mode>3"
4281 [(set (match_operand:GPI 0 "register_operand" "=r,w")
4282 (not:GPI (xor:GPI (match_operand:GPI 1 "register_operand" "r,?w")
4283 (match_operand:GPI 2 "register_operand" "r,w"))))]
4286 eon\\t%<w>0, %<w>1, %<w>2
4288 "reload_completed && FP_REGNUM_P (REGNO (operands[0]))" ;; For SIMD registers.
4289 [(set (match_operand:GPI 0 "register_operand" "=w")
4290 (xor:GPI (match_operand:GPI 1 "register_operand" "w")
4291 (match_operand:GPI 2 "register_operand" "w")))
4292 (set (match_dup 0) (not:GPI (match_dup 0)))]
4294 [(set_attr "type" "logic_reg,multiple")
4295 (set_attr "arch" "*,simd")]
4298 (define_insn "*and_one_cmpl<mode>3_compare0"
4299 [(set (reg:CC_NZ CC_REGNUM)
4302 (match_operand:GPI 1 "register_operand" "r"))
4303 (match_operand:GPI 2 "register_operand" "r"))
4305 (set (match_operand:GPI 0 "register_operand" "=r")
4306 (and:GPI (not:GPI (match_dup 1)) (match_dup 2)))]
4308 "bics\\t%<w>0, %<w>2, %<w>1"
4309 [(set_attr "type" "logics_reg")]
4312 ;; zero_extend version of above
4313 (define_insn "*and_one_cmplsi3_compare0_uxtw"
4314 [(set (reg:CC_NZ CC_REGNUM)
4317 (match_operand:SI 1 "register_operand" "r"))
4318 (match_operand:SI 2 "register_operand" "r"))
4320 (set (match_operand:DI 0 "register_operand" "=r")
4321 (zero_extend:DI (and:SI (not:SI (match_dup 1)) (match_dup 2))))]
4323 "bics\\t%w0, %w2, %w1"
4324 [(set_attr "type" "logics_reg")]
4327 (define_insn "*and_one_cmpl<mode>3_compare0_no_reuse"
4328 [(set (reg:CC_NZ CC_REGNUM)
4331 (match_operand:GPI 0 "register_operand" "r"))
4332 (match_operand:GPI 1 "register_operand" "r"))
4335 "bics\\t<w>zr, %<w>1, %<w>0"
4336 [(set_attr "type" "logics_reg")]
4339 (define_insn "<LOGICAL:optab>_one_cmpl_<SHIFT:optab><mode>3"
4340 [(set (match_operand:GPI 0 "register_operand" "=r")
4341 (LOGICAL:GPI (not:GPI
4343 (match_operand:GPI 1 "register_operand" "r")
4344 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
4345 (match_operand:GPI 3 "register_operand" "r")))]
4347 "<LOGICAL:nlogical>\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
4348 [(set_attr "type" "logic_shift_imm")]
4351 (define_insn "*eor_one_cmpl_<SHIFT:optab><mode>3_alt"
4352 [(set (match_operand:GPI 0 "register_operand" "=r")
4355 (match_operand:GPI 1 "register_operand" "r")
4356 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
4357 (match_operand:GPI 3 "register_operand" "r"))))]
4359 "eon\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
4360 [(set_attr "type" "logic_shift_imm")]
4363 ;; Zero-extend version of the above.
4364 (define_insn "*eor_one_cmpl_<SHIFT:optab>sidi3_alt_ze"
4365 [(set (match_operand:DI 0 "register_operand" "=r")
4369 (match_operand:SI 1 "register_operand" "r")
4370 (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
4371 (match_operand:SI 3 "register_operand" "r")))))]
4373 "eon\\t%w0, %w3, %w1, <SHIFT:shift> %2"
4374 [(set_attr "type" "logic_shift_imm")]
4377 (define_insn "*and_one_cmpl_<SHIFT:optab><mode>3_compare0"
4378 [(set (reg:CC_NZ CC_REGNUM)
4382 (match_operand:GPI 1 "register_operand" "r")
4383 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
4384 (match_operand:GPI 3 "register_operand" "r"))
4386 (set (match_operand:GPI 0 "register_operand" "=r")
4389 (match_dup 1) (match_dup 2))) (match_dup 3)))]
4391 "bics\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
4392 [(set_attr "type" "logics_shift_imm")]
4395 ;; zero_extend version of above
4396 (define_insn "*and_one_cmpl_<SHIFT:optab>si3_compare0_uxtw"
4397 [(set (reg:CC_NZ CC_REGNUM)
4401 (match_operand:SI 1 "register_operand" "r")
4402 (match_operand:QI 2 "aarch64_shift_imm_si" "n")))
4403 (match_operand:SI 3 "register_operand" "r"))
4405 (set (match_operand:DI 0 "register_operand" "=r")
4406 (zero_extend:DI (and:SI
4408 (SHIFT:SI (match_dup 1) (match_dup 2))) (match_dup 3))))]
4410 "bics\\t%w0, %w3, %w1, <SHIFT:shift> %2"
4411 [(set_attr "type" "logics_shift_imm")]
4414 (define_insn "*and_one_cmpl_<SHIFT:optab><mode>3_compare0_no_reuse"
4415 [(set (reg:CC_NZ CC_REGNUM)
4419 (match_operand:GPI 0 "register_operand" "r")
4420 (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n")))
4421 (match_operand:GPI 2 "register_operand" "r"))
4424 "bics\\t<w>zr, %<w>2, %<w>0, <SHIFT:shift> %1"
4425 [(set_attr "type" "logics_shift_imm")]
4428 (define_insn "clz<mode>2"
4429 [(set (match_operand:GPI 0 "register_operand" "=r")
4430 (clz:GPI (match_operand:GPI 1 "register_operand" "r")))]
4432 "clz\\t%<w>0, %<w>1"
4433 [(set_attr "type" "clz")]
4436 (define_expand "ffs<mode>2"
4437 [(match_operand:GPI 0 "register_operand")
4438 (match_operand:GPI 1 "register_operand")]
4441 rtx ccreg = aarch64_gen_compare_reg (EQ, operands[1], const0_rtx);
4442 rtx x = gen_rtx_NE (VOIDmode, ccreg, const0_rtx);
4444 emit_insn (gen_rbit<mode>2 (operands[0], operands[1]));
4445 emit_insn (gen_clz<mode>2 (operands[0], operands[0]));
4446 emit_insn (gen_csinc3<mode>_insn (operands[0], x, operands[0], const0_rtx));
4451 ;; Pop count be done via the "CNT" instruction in AdvSIMD.
4458 (define_expand "popcount<mode>2"
4459 [(match_operand:GPI 0 "register_operand")
4460 (match_operand:GPI 1 "register_operand")]
4463 rtx v = gen_reg_rtx (V8QImode);
4464 rtx v1 = gen_reg_rtx (V8QImode);
4465 rtx r = gen_reg_rtx (QImode);
4466 rtx in = operands[1];
4467 rtx out = operands[0];
4468 if(<MODE>mode == SImode)
4471 tmp = gen_reg_rtx (DImode);
4472 /* If we have SImode, zero extend to DImode, pop count does
4473 not change if we have extra zeros. */
4474 emit_insn (gen_zero_extendsidi2 (tmp, in));
4477 emit_move_insn (v, gen_lowpart (V8QImode, in));
4478 emit_insn (gen_popcountv8qi2 (v1, v));
4479 emit_insn (gen_reduc_plus_scal_v8qi (r, v1));
4480 emit_insn (gen_zero_extendqi<mode>2 (out, r));
4484 (define_insn "clrsb<mode>2"
4485 [(set (match_operand:GPI 0 "register_operand" "=r")
4486 (clrsb:GPI (match_operand:GPI 1 "register_operand" "r")))]
4488 "cls\\t%<w>0, %<w>1"
4489 [(set_attr "type" "clz")]
4492 (define_insn "rbit<mode>2"
4493 [(set (match_operand:GPI 0 "register_operand" "=r")
4494 (unspec:GPI [(match_operand:GPI 1 "register_operand" "r")] UNSPEC_RBIT))]
4496 "rbit\\t%<w>0, %<w>1"
4497 [(set_attr "type" "rbit")]
4500 ;; Split after reload into RBIT + CLZ. Since RBIT is represented as an UNSPEC
4501 ;; it is unlikely to fold with any other operation, so keep this as a CTZ
4502 ;; expression and split after reload to enable scheduling them apart if
4505 (define_insn_and_split "ctz<mode>2"
4506 [(set (match_operand:GPI 0 "register_operand" "=r")
4507 (ctz:GPI (match_operand:GPI 1 "register_operand" "r")))]
4513 emit_insn (gen_rbit<mode>2 (operands[0], operands[1]));
4514 emit_insn (gen_clz<mode>2 (operands[0], operands[0]));
4518 (define_insn "*and<mode>_compare0"
4519 [(set (reg:CC_NZ CC_REGNUM)
4521 (match_operand:SHORT 0 "register_operand" "r")
4524 "tst\\t%<w>0, <short_mask>"
4525 [(set_attr "type" "alus_imm")]
4528 (define_insn "*ands<mode>_compare0"
4529 [(set (reg:CC_NZ CC_REGNUM)
4531 (zero_extend:GPI (match_operand:SHORT 1 "register_operand" "r"))
4533 (set (match_operand:GPI 0 "register_operand" "=r")
4534 (zero_extend:GPI (match_dup 1)))]
4536 "ands\\t%<GPI:w>0, %<GPI:w>1, <short_mask>"
4537 [(set_attr "type" "alus_imm")]
4540 (define_insn "*and<mode>3nr_compare0"
4541 [(set (reg:CC_NZ CC_REGNUM)
4543 (and:GPI (match_operand:GPI 0 "register_operand" "%r,r")
4544 (match_operand:GPI 1 "aarch64_logical_operand" "r,<lconst>"))
4550 [(set_attr "type" "logics_reg,logics_imm")]
4554 [(set (reg:CC_NZ CC_REGNUM)
4556 (and:GPI (match_operand:GPI 0 "register_operand")
4557 (match_operand:GPI 1 "aarch64_mov_imm_operand"))
4559 (clobber (match_operand:SI 2 "register_operand"))]
4561 [(set (match_dup 2) (match_dup 1))
4562 (set (reg:CC_NZ CC_REGNUM)
4564 (and:GPI (match_dup 0)
4569 (define_insn "*and<mode>3nr_compare0_zextract"
4570 [(set (reg:CC_NZ CC_REGNUM)
4572 (zero_extract:GPI (match_operand:GPI 0 "register_operand" "r")
4573 (match_operand:GPI 1 "const_int_operand" "n")
4574 (match_operand:GPI 2 "const_int_operand" "n"))
4576 "INTVAL (operands[1]) > 0
4577 && ((INTVAL (operands[1]) + INTVAL (operands[2]))
4578 <= GET_MODE_BITSIZE (<MODE>mode))
4579 && aarch64_bitmask_imm (
4580 UINTVAL (aarch64_mask_from_zextract_ops (operands[1],
4585 = aarch64_mask_from_zextract_ops (operands[1], operands[2]);
4586 return "tst\\t%<w>0, %1";
4588 [(set_attr "type" "logics_shift_imm")]
4591 (define_insn "*and_<SHIFT:optab><mode>3nr_compare0"
4592 [(set (reg:CC_NZ CC_REGNUM)
4595 (match_operand:GPI 0 "register_operand" "r")
4596 (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n"))
4597 (match_operand:GPI 2 "register_operand" "r"))
4600 "tst\\t%<w>2, %<w>0, <SHIFT:shift> %1"
4601 [(set_attr "type" "logics_shift_imm")]
4605 [(set (reg:CC_NZ CC_REGNUM)
4608 (match_operand:GPI 0 "register_operand")
4609 (match_operand:QI 1 "aarch64_shift_imm_<mode>"))
4610 (match_operand:GPI 2 "aarch64_mov_imm_operand"))
4612 (clobber (match_operand:SI 3 "register_operand"))]
4614 [(set (match_dup 3) (match_dup 2))
4615 (set (reg:CC_NZ CC_REGNUM)
4624 ;; -------------------------------------------------------------------
4626 ;; -------------------------------------------------------------------
4628 (define_expand "<optab><mode>3"
4629 [(set (match_operand:GPI 0 "register_operand")
4630 (ASHIFT:GPI (match_operand:GPI 1 "register_operand")
4631 (match_operand:QI 2 "aarch64_reg_or_imm")))]
4634 if (CONST_INT_P (operands[2]))
4636 operands[2] = GEN_INT (INTVAL (operands[2])
4637 & (GET_MODE_BITSIZE (<MODE>mode) - 1));
4639 if (operands[2] == const0_rtx)
4641 emit_insn (gen_mov<mode> (operands[0], operands[1]));
4648 (define_expand "ashl<mode>3"
4649 [(set (match_operand:SHORT 0 "register_operand")
4650 (ashift:SHORT (match_operand:SHORT 1 "register_operand")
4651 (match_operand:QI 2 "const_int_operand")))]
4654 operands[2] = GEN_INT (INTVAL (operands[2]) & GET_MODE_MASK (<MODE>mode));
4656 if (operands[2] == const0_rtx)
4658 emit_insn (gen_mov<mode> (operands[0], operands[1]));
4664 (define_expand "rotr<mode>3"
4665 [(set (match_operand:GPI 0 "register_operand")
4666 (rotatert:GPI (match_operand:GPI 1 "register_operand")
4667 (match_operand:QI 2 "aarch64_reg_or_imm")))]
4670 if (CONST_INT_P (operands[2]))
4672 operands[2] = GEN_INT (INTVAL (operands[2])
4673 & (GET_MODE_BITSIZE (<MODE>mode) - 1));
4675 if (operands[2] == const0_rtx)
4677 emit_insn (gen_mov<mode> (operands[0], operands[1]));
4684 (define_expand "rotl<mode>3"
4685 [(set (match_operand:GPI 0 "register_operand")
4686 (rotatert:GPI (match_operand:GPI 1 "register_operand")
4687 (match_operand:QI 2 "aarch64_reg_or_imm")))]
4690 /* (SZ - cnt) % SZ == -cnt % SZ */
4691 if (CONST_INT_P (operands[2]))
4693 operands[2] = GEN_INT ((-INTVAL (operands[2]))
4694 & (GET_MODE_BITSIZE (<MODE>mode) - 1));
4695 if (operands[2] == const0_rtx)
4697 emit_insn (gen_mov<mode> (operands[0], operands[1]));
4702 operands[2] = expand_simple_unop (QImode, NEG, operands[2],
4707 ;; When the LSL, LSR, ASR, ROR instructions operate on all register arguments
4708 ;; they truncate the shift/rotate amount by the size of the registers they
4709 ;; operate on: 32 for W-regs, 64 for X-regs. This allows us to optimise away
4710 ;; such redundant masking instructions. GCC can do that automatically when
4711 ;; SHIFT_COUNT_TRUNCATED is true, but we can't enable it for TARGET_SIMD
4712 ;; because some of the SISD shift alternatives don't perform this truncations.
4713 ;; So this pattern exists to catch such cases.
4715 (define_insn "*aarch64_<optab>_reg_<mode>3_mask1"
4716 [(set (match_operand:GPI 0 "register_operand" "=r")
4718 (match_operand:GPI 1 "register_operand" "r")
4719 (match_operator 4 "subreg_lowpart_operator"
4720 [(and:GPI (match_operand:GPI 2 "register_operand" "r")
4721 (match_operand 3 "const_int_operand" "n"))])))]
4722 "(~INTVAL (operands[3]) & (GET_MODE_BITSIZE (<MODE>mode) - 1)) == 0"
4723 "<shift>\t%<w>0, %<w>1, %<w>2"
4724 [(set_attr "type" "shift_reg")]
4727 (define_insn_and_split "*aarch64_<optab>_reg_<mode>3_neg_mask2"
4728 [(set (match_operand:GPI 0 "register_operand" "=&r")
4730 (match_operand:GPI 1 "register_operand" "r")
4731 (match_operator 4 "subreg_lowpart_operator"
4732 [(neg:SI (and:SI (match_operand:SI 2 "register_operand" "r")
4733 (match_operand 3 "const_int_operand" "n")))])))]
4734 "((~INTVAL (operands[3]) & (GET_MODE_BITSIZE (<MODE>mode) - 1)) == 0)"
4739 rtx tmp = (can_create_pseudo_p () ? gen_reg_rtx (SImode)
4740 : lowpart_subreg (SImode, operands[0], <MODE>mode));
4741 emit_insn (gen_negsi2 (tmp, operands[2]));
4743 rtx and_op = gen_rtx_AND (SImode, tmp, operands[3]);
4744 rtx subreg_tmp = gen_rtx_SUBREG (GET_MODE (operands[4]), and_op,
4745 SUBREG_BYTE (operands[4]));
4746 emit_insn (gen_<optab><mode>3 (operands[0], operands[1], subreg_tmp));
4751 (define_insn_and_split "*aarch64_ashl_reg_<mode>3_minus_mask"
4752 [(set (match_operand:GPI 0 "register_operand" "=&r")
4754 (match_operand:GPI 1 "register_operand" "r")
4755 (minus:QI (match_operand 2 "const_int_operand" "n")
4756 (match_operator 5 "subreg_lowpart_operator"
4757 [(and:SI (match_operand:SI 3 "register_operand" "r")
4758 (match_operand 4 "const_int_operand" "n"))]))))]
4759 "((~INTVAL (operands[4]) & (GET_MODE_BITSIZE (<MODE>mode) - 1)) == 0)
4760 && INTVAL (operands[2]) == GET_MODE_BITSIZE (<MODE>mode)"
4765 rtx tmp = (can_create_pseudo_p () ? gen_reg_rtx (SImode)
4768 emit_insn (gen_negsi2 (tmp, operands[3]));
4770 rtx and_op = gen_rtx_AND (SImode, tmp, operands[4]);
4771 rtx subreg_tmp = gen_rtx_SUBREG (GET_MODE (operands[5]), and_op,
4772 SUBREG_BYTE (operands[5]));
4774 emit_insn (gen_ashl<mode>3 (operands[0], operands[1], subreg_tmp));
4779 (define_insn "*aarch64_<optab>_reg_di3_mask2"
4780 [(set (match_operand:DI 0 "register_operand" "=r")
4782 (match_operand:DI 1 "register_operand" "r")
4783 (match_operator 4 "subreg_lowpart_operator"
4784 [(and:SI (match_operand:SI 2 "register_operand" "r")
4785 (match_operand 3 "const_int_operand" "n"))])))]
4786 "((~INTVAL (operands[3]) & (GET_MODE_BITSIZE (DImode) - 1)) == 0)"
4789 xop[0] = operands[0];
4790 xop[1] = operands[1];
4791 xop[2] = gen_lowpart (GET_MODE (operands[4]), operands[2]);
4792 output_asm_insn ("<shift>\t%x0, %x1, %x2", xop);
4795 [(set_attr "type" "shift_reg")]
4798 (define_insn_and_split "*aarch64_<optab>_reg_minus<mode>3"
4799 [(set (match_operand:GPI 0 "register_operand" "=&r")
4801 (match_operand:GPI 1 "register_operand" "r")
4802 (minus:QI (match_operand 2 "const_int_operand" "n")
4803 (match_operand:QI 3 "register_operand" "r"))))]
4804 "INTVAL (operands[2]) == GET_MODE_BITSIZE (<MODE>mode)"
4809 rtx subreg_tmp = gen_lowpart (SImode, operands[3]);
4811 rtx tmp = (can_create_pseudo_p () ? gen_reg_rtx (SImode)
4812 : gen_lowpart (SImode, operands[0]));
4814 emit_insn (gen_negsi2 (tmp, subreg_tmp));
4816 rtx and_op = gen_rtx_AND (SImode, tmp,
4817 GEN_INT (GET_MODE_BITSIZE (<MODE>mode) - 1));
4819 rtx subreg_tmp2 = gen_lowpart_SUBREG (QImode, and_op);
4821 emit_insn (gen_<optab><mode>3 (operands[0], operands[1], subreg_tmp2));
4824 [(set_attr "length" "8")]
4827 ;; Logical left shift using SISD or Integer instruction
4828 (define_insn "*aarch64_ashl_sisd_or_int_<mode>3"
4829 [(set (match_operand:GPI 0 "register_operand" "=r,r,w,w")
4831 (match_operand:GPI 1 "register_operand" "r,r,w,w")
4832 (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>" "Us<cmode>,r,Us<cmode>,w")))]
4835 lsl\t%<w>0, %<w>1, %2
4836 lsl\t%<w>0, %<w>1, %<w>2
4837 shl\t%<rtn>0<vas>, %<rtn>1<vas>, %2
4838 ushl\t%<rtn>0<vas>, %<rtn>1<vas>, %<rtn>2<vas>"
4839 [(set_attr "type" "bfx,shift_reg,neon_shift_imm<q>, neon_shift_reg<q>")
4840 (set_attr "arch" "*,*,simd,simd")]
4843 ;; Logical right shift using SISD or Integer instruction
4844 (define_insn "*aarch64_lshr_sisd_or_int_<mode>3"
4845 [(set (match_operand:GPI 0 "register_operand" "=r,r,w,&w,&w")
4847 (match_operand:GPI 1 "register_operand" "r,r,w,w,w")
4848 (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>"
4849 "Us<cmode>,r,Us<cmode_simd>,w,0")))]
4852 lsr\t%<w>0, %<w>1, %2
4853 lsr\t%<w>0, %<w>1, %<w>2
4854 ushr\t%<rtn>0<vas>, %<rtn>1<vas>, %2
4857 [(set_attr "type" "bfx,shift_reg,neon_shift_imm<q>,neon_shift_reg<q>,neon_shift_reg<q>")
4858 (set_attr "arch" "*,*,simd,simd,simd")]
4862 [(set (match_operand:DI 0 "aarch64_simd_register")
4864 (match_operand:DI 1 "aarch64_simd_register")
4865 (match_operand:QI 2 "aarch64_simd_register")))]
4866 "TARGET_SIMD && reload_completed"
4868 (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
4870 (unspec:DI [(match_dup 1) (match_dup 3)] UNSPEC_SISD_USHL))]
4872 operands[3] = gen_lowpart (QImode, operands[0]);
4877 [(set (match_operand:SI 0 "aarch64_simd_register")
4879 (match_operand:SI 1 "aarch64_simd_register")
4880 (match_operand:QI 2 "aarch64_simd_register")))]
4881 "TARGET_SIMD && reload_completed"
4883 (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
4885 (unspec:SI [(match_dup 1) (match_dup 3)] UNSPEC_USHL_2S))]
4887 operands[3] = gen_lowpart (QImode, operands[0]);
4891 ;; Arithmetic right shift using SISD or Integer instruction
4892 (define_insn "*aarch64_ashr_sisd_or_int_<mode>3"
4893 [(set (match_operand:GPI 0 "register_operand" "=r,r,w,&w,&w")
4895 (match_operand:GPI 1 "register_operand" "r,r,w,w,w")
4896 (match_operand:QI 2 "aarch64_reg_or_shift_imm_di"
4897 "Us<cmode>,r,Us<cmode_simd>,w,0")))]
4900 asr\t%<w>0, %<w>1, %2
4901 asr\t%<w>0, %<w>1, %<w>2
4902 sshr\t%<rtn>0<vas>, %<rtn>1<vas>, %2
4905 [(set_attr "type" "bfx,shift_reg,neon_shift_imm<q>,neon_shift_reg<q>,neon_shift_reg<q>")
4906 (set_attr "arch" "*,*,simd,simd,simd")]
4910 [(set (match_operand:DI 0 "aarch64_simd_register")
4912 (match_operand:DI 1 "aarch64_simd_register")
4913 (match_operand:QI 2 "aarch64_simd_register")))]
4914 "TARGET_SIMD && reload_completed"
4916 (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
4918 (unspec:DI [(match_dup 1) (match_dup 3)] UNSPEC_SISD_SSHL))]
4920 operands[3] = gen_lowpart (QImode, operands[0]);
4925 [(set (match_operand:SI 0 "aarch64_simd_register")
4927 (match_operand:SI 1 "aarch64_simd_register")
4928 (match_operand:QI 2 "aarch64_simd_register")))]
4929 "TARGET_SIMD && reload_completed"
4931 (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
4933 (unspec:SI [(match_dup 1) (match_dup 3)] UNSPEC_SSHL_2S))]
4935 operands[3] = gen_lowpart (QImode, operands[0]);
4939 (define_insn "*aarch64_sisd_ushl"
4940 [(set (match_operand:DI 0 "register_operand" "=w")
4941 (unspec:DI [(match_operand:DI 1 "register_operand" "w")
4942 (match_operand:QI 2 "register_operand" "w")]
4945 "ushl\t%d0, %d1, %d2"
4946 [(set_attr "type" "neon_shift_reg")]
4949 (define_insn "*aarch64_ushl_2s"
4950 [(set (match_operand:SI 0 "register_operand" "=w")
4951 (unspec:SI [(match_operand:SI 1 "register_operand" "w")
4952 (match_operand:QI 2 "register_operand" "w")]
4955 "ushl\t%0.2s, %1.2s, %2.2s"
4956 [(set_attr "type" "neon_shift_reg")]
4959 (define_insn "*aarch64_sisd_sshl"
4960 [(set (match_operand:DI 0 "register_operand" "=w")
4961 (unspec:DI [(match_operand:DI 1 "register_operand" "w")
4962 (match_operand:QI 2 "register_operand" "w")]
4965 "sshl\t%d0, %d1, %d2"
4966 [(set_attr "type" "neon_shift_reg")]
4969 (define_insn "*aarch64_sshl_2s"
4970 [(set (match_operand:SI 0 "register_operand" "=w")
4971 (unspec:SI [(match_operand:SI 1 "register_operand" "w")
4972 (match_operand:QI 2 "register_operand" "w")]
4975 "sshl\t%0.2s, %1.2s, %2.2s"
4976 [(set_attr "type" "neon_shift_reg")]
4979 (define_insn "*aarch64_sisd_neg_qi"
4980 [(set (match_operand:QI 0 "register_operand" "=w")
4981 (unspec:QI [(match_operand:QI 1 "register_operand" "w")]
4985 [(set_attr "type" "neon_neg")]
4989 (define_insn "*ror<mode>3_insn"
4990 [(set (match_operand:GPI 0 "register_operand" "=r,r")
4992 (match_operand:GPI 1 "register_operand" "r,r")
4993 (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>" "Us<cmode>,r")))]
4996 ror\\t%<w>0, %<w>1, %2
4997 ror\\t%<w>0, %<w>1, %<w>2"
4998 [(set_attr "type" "rotate_imm,shift_reg")]
5001 ;; zero_extend version of above
5002 (define_insn "*<optab>si3_insn_uxtw"
5003 [(set (match_operand:DI 0 "register_operand" "=r,r")
5004 (zero_extend:DI (SHIFT:SI
5005 (match_operand:SI 1 "register_operand" "r,r")
5006 (match_operand:QI 2 "aarch64_reg_or_shift_imm_si" "Uss,r"))))]
5009 <shift>\\t%w0, %w1, %2
5010 <shift>\\t%w0, %w1, %w2"
5011 [(set_attr "type" "bfx,shift_reg")]
5014 (define_insn "*<optab><mode>3_insn"
5015 [(set (match_operand:SHORT 0 "register_operand" "=r")
5016 (ASHIFT:SHORT (match_operand:SHORT 1 "register_operand" "r")
5017 (match_operand 2 "const_int_operand" "n")))]
5018 "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<MODE>mode)"
5020 operands[3] = GEN_INT (<sizen> - UINTVAL (operands[2]));
5021 return "<bfshift>\t%w0, %w1, %2, %3";
5023 [(set_attr "type" "bfx")]
5026 (define_insn "*extr<mode>5_insn"
5027 [(set (match_operand:GPI 0 "register_operand" "=r")
5028 (ior:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
5029 (match_operand 3 "const_int_operand" "n"))
5030 (lshiftrt:GPI (match_operand:GPI 2 "register_operand" "r")
5031 (match_operand 4 "const_int_operand" "n"))))]
5032 "UINTVAL (operands[3]) < GET_MODE_BITSIZE (<MODE>mode) &&
5033 (UINTVAL (operands[3]) + UINTVAL (operands[4]) == GET_MODE_BITSIZE (<MODE>mode))"
5034 "extr\\t%<w>0, %<w>1, %<w>2, %4"
5035 [(set_attr "type" "rotate_imm")]
5038 ;; There are no canonicalisation rules for ashift and lshiftrt inside an ior
5039 ;; so we have to match both orderings.
5040 (define_insn "*extr<mode>5_insn_alt"
5041 [(set (match_operand:GPI 0 "register_operand" "=r")
5042 (ior:GPI (lshiftrt:GPI (match_operand:GPI 2 "register_operand" "r")
5043 (match_operand 4 "const_int_operand" "n"))
5044 (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
5045 (match_operand 3 "const_int_operand" "n"))))]
5046 "UINTVAL (operands[3]) < GET_MODE_BITSIZE (<MODE>mode)
5047 && (UINTVAL (operands[3]) + UINTVAL (operands[4])
5048 == GET_MODE_BITSIZE (<MODE>mode))"
5049 "extr\\t%<w>0, %<w>1, %<w>2, %4"
5050 [(set_attr "type" "rotate_imm")]
5053 ;; zero_extend version of the above
5054 (define_insn "*extrsi5_insn_uxtw"
5055 [(set (match_operand:DI 0 "register_operand" "=r")
5057 (ior:SI (ashift:SI (match_operand:SI 1 "register_operand" "r")
5058 (match_operand 3 "const_int_operand" "n"))
5059 (lshiftrt:SI (match_operand:SI 2 "register_operand" "r")
5060 (match_operand 4 "const_int_operand" "n")))))]
5061 "UINTVAL (operands[3]) < 32 &&
5062 (UINTVAL (operands[3]) + UINTVAL (operands[4]) == 32)"
5063 "extr\\t%w0, %w1, %w2, %4"
5064 [(set_attr "type" "rotate_imm")]
5067 (define_insn "*extrsi5_insn_uxtw_alt"
5068 [(set (match_operand:DI 0 "register_operand" "=r")
5070 (ior:SI (lshiftrt:SI (match_operand:SI 2 "register_operand" "r")
5071 (match_operand 4 "const_int_operand" "n"))
5072 (ashift:SI (match_operand:SI 1 "register_operand" "r")
5073 (match_operand 3 "const_int_operand" "n")))))]
5074 "UINTVAL (operands[3]) < 32 &&
5075 (UINTVAL (operands[3]) + UINTVAL (operands[4]) == 32)"
5076 "extr\\t%w0, %w1, %w2, %4"
5077 [(set_attr "type" "rotate_imm")]
5080 (define_insn "*ror<mode>3_insn"
5081 [(set (match_operand:GPI 0 "register_operand" "=r")
5082 (rotate:GPI (match_operand:GPI 1 "register_operand" "r")
5083 (match_operand 2 "const_int_operand" "n")))]
5084 "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<MODE>mode)"
5086 operands[3] = GEN_INT (<sizen> - UINTVAL (operands[2]));
5087 return "ror\\t%<w>0, %<w>1, %3";
5089 [(set_attr "type" "rotate_imm")]
5092 ;; zero_extend version of the above
5093 (define_insn "*rorsi3_insn_uxtw"
5094 [(set (match_operand:DI 0 "register_operand" "=r")
5096 (rotate:SI (match_operand:SI 1 "register_operand" "r")
5097 (match_operand 2 "const_int_operand" "n"))))]
5098 "UINTVAL (operands[2]) < 32"
5100 operands[3] = GEN_INT (32 - UINTVAL (operands[2]));
5101 return "ror\\t%w0, %w1, %3";
5103 [(set_attr "type" "rotate_imm")]
5106 (define_insn "*<ANY_EXTEND:optab><GPI:mode>_ashl<SHORT:mode>"
5107 [(set (match_operand:GPI 0 "register_operand" "=r")
5109 (ashift:SHORT (match_operand:SHORT 1 "register_operand" "r")
5110 (match_operand 2 "const_int_operand" "n"))))]
5111 "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
5113 operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
5114 return "<su>bfiz\t%<GPI:w>0, %<GPI:w>1, %2, %3";
5116 [(set_attr "type" "bfx")]
5119 (define_insn "*zero_extend<GPI:mode>_lshr<SHORT:mode>"
5120 [(set (match_operand:GPI 0 "register_operand" "=r")
5122 (lshiftrt:SHORT (match_operand:SHORT 1 "register_operand" "r")
5123 (match_operand 2 "const_int_operand" "n"))))]
5124 "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
5126 operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
5127 return "ubfx\t%<GPI:w>0, %<GPI:w>1, %2, %3";
5129 [(set_attr "type" "bfx")]
5132 (define_insn "*extend<GPI:mode>_ashr<SHORT:mode>"
5133 [(set (match_operand:GPI 0 "register_operand" "=r")
5135 (ashiftrt:SHORT (match_operand:SHORT 1 "register_operand" "r")
5136 (match_operand 2 "const_int_operand" "n"))))]
5137 "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
5139 operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
5140 return "sbfx\\t%<GPI:w>0, %<GPI:w>1, %2, %3";
5142 [(set_attr "type" "bfx")]
5145 ;; -------------------------------------------------------------------
5147 ;; -------------------------------------------------------------------
5149 (define_expand "<optab>"
5150 [(set (match_operand:DI 0 "register_operand" "=r")
5151 (ANY_EXTRACT:DI (match_operand:DI 1 "register_operand")
5153 "aarch64_simd_shift_imm_offset_di")
5154 (match_operand 3 "aarch64_simd_shift_imm_di")))]
5157 if (!IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]),
5158 1, GET_MODE_BITSIZE (DImode) - 1))
5164 (define_insn "*<optab><mode>"
5165 [(set (match_operand:GPI 0 "register_operand" "=r")
5166 (ANY_EXTRACT:GPI (match_operand:GPI 1 "register_operand" "r")
5168 "aarch64_simd_shift_imm_offset_<mode>" "n")
5170 "aarch64_simd_shift_imm_<mode>" "n")))]
5171 "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]),
5172 1, GET_MODE_BITSIZE (<MODE>mode) - 1)"
5173 "<su>bfx\\t%<w>0, %<w>1, %3, %2"
5174 [(set_attr "type" "bfx")]
5177 ;; When the bit position and width add up to 32 we can use a W-reg LSR
5178 ;; instruction taking advantage of the implicit zero-extension of the X-reg.
5180 [(set (match_operand:DI 0 "register_operand")
5181 (zero_extract:DI (match_operand:DI 1 "register_operand")
5183 "aarch64_simd_shift_imm_offset_di")
5185 "aarch64_simd_shift_imm_di")))]
5186 "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]), 1,
5187 GET_MODE_BITSIZE (DImode) - 1)
5188 && (INTVAL (operands[2]) + INTVAL (operands[3]))
5189 == GET_MODE_BITSIZE (SImode)"
5191 (zero_extend:DI (lshiftrt:SI (match_dup 4) (match_dup 3))))]
5193 operands[4] = gen_lowpart (SImode, operands[1]);
5197 ;; Bitfield Insert (insv)
5198 (define_expand "insv<mode>"
5199 [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand")
5200 (match_operand 1 "const_int_operand")
5201 (match_operand 2 "const_int_operand"))
5202 (match_operand:GPI 3 "general_operand"))]
5205 unsigned HOST_WIDE_INT width = UINTVAL (operands[1]);
5206 unsigned HOST_WIDE_INT pos = UINTVAL (operands[2]);
5207 rtx value = operands[3];
5209 if (width == 0 || (pos + width) > GET_MODE_BITSIZE (<MODE>mode))
5212 if (CONST_INT_P (value))
5214 unsigned HOST_WIDE_INT mask = ((unsigned HOST_WIDE_INT)1 << width) - 1;
5216 /* Prefer AND/OR for inserting all zeros or all ones. */
5217 if ((UINTVAL (value) & mask) == 0
5218 || (UINTVAL (value) & mask) == mask)
5221 /* 16-bit aligned 16-bit wide insert is handled by insv_imm. */
5222 if (width == 16 && (pos % 16) == 0)
5225 operands[3] = force_reg (<MODE>mode, value);
5228 (define_insn "*insv_reg<mode>"
5229 [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
5230 (match_operand 1 "const_int_operand" "n")
5231 (match_operand 2 "const_int_operand" "n"))
5232 (match_operand:GPI 3 "register_operand" "r"))]
5233 "!(UINTVAL (operands[1]) == 0
5234 || (UINTVAL (operands[2]) + UINTVAL (operands[1])
5235 > GET_MODE_BITSIZE (<MODE>mode)))"
5236 "bfi\\t%<w>0, %<w>3, %2, %1"
5237 [(set_attr "type" "bfm")]
5240 (define_insn "*aarch64_bfi<GPI:mode><ALLX:mode>4"
5241 [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
5242 (match_operand 1 "const_int_operand" "n")
5243 (match_operand 2 "const_int_operand" "n"))
5244 (zero_extend:GPI (match_operand:ALLX 3 "register_operand" "r")))]
5245 "UINTVAL (operands[1]) <= <ALLX:sizen>"
5246 "bfi\\t%<GPI:w>0, %<GPI:w>3, %2, %1"
5247 [(set_attr "type" "bfm")]
5250 (define_insn "*extr_insv_lower_reg<mode>"
5251 [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
5252 (match_operand 1 "const_int_operand" "n")
5254 (zero_extract:GPI (match_operand:GPI 2 "register_operand" "r")
5256 (match_operand 3 "const_int_operand" "n")))]
5257 "!(UINTVAL (operands[1]) == 0
5258 || (UINTVAL (operands[3]) + UINTVAL (operands[1])
5259 > GET_MODE_BITSIZE (<MODE>mode)))"
5260 "bfxil\\t%<w>0, %<w>2, %3, %1"
5261 [(set_attr "type" "bfm")]
5264 (define_insn "*<optab><ALLX:mode>_shft_<GPI:mode>"
5265 [(set (match_operand:GPI 0 "register_operand" "=r")
5266 (ashift:GPI (ANY_EXTEND:GPI
5267 (match_operand:ALLX 1 "register_operand" "r"))
5268 (match_operand 2 "const_int_operand" "n")))]
5269 "UINTVAL (operands[2]) < <GPI:sizen>"
5271 operands[3] = (<ALLX:sizen> <= (<GPI:sizen> - UINTVAL (operands[2])))
5272 ? GEN_INT (<ALLX:sizen>)
5273 : GEN_INT (<GPI:sizen> - UINTVAL (operands[2]));
5274 return "<su>bfiz\t%<GPI:w>0, %<GPI:w>1, %2, %3";
5276 [(set_attr "type" "bfx")]
5279 ;; XXX We should match (any_extend (ashift)) here, like (and (ashift)) below
5281 (define_insn "*andim_ashift<mode>_bfiz"
5282 [(set (match_operand:GPI 0 "register_operand" "=r")
5283 (and:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
5284 (match_operand 2 "const_int_operand" "n"))
5285 (match_operand 3 "const_int_operand" "n")))]
5286 "aarch64_mask_and_shift_for_ubfiz_p (<MODE>mode, operands[3], operands[2])"
5287 "ubfiz\\t%<w>0, %<w>1, %2, %P3"
5288 [(set_attr "type" "bfx")]
5291 ;; Match sbfiz pattern in a shift left + shift right operation.
5293 (define_insn "*ashift<mode>_extv_bfiz"
5294 [(set (match_operand:GPI 0 "register_operand" "=r")
5295 (ashift:GPI (sign_extract:GPI (match_operand:GPI 1 "register_operand" "r")
5296 (match_operand 2 "aarch64_simd_shift_imm_offset_<mode>" "n")
5298 (match_operand 3 "aarch64_simd_shift_imm_<mode>" "n")))]
5299 "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]),
5300 1, GET_MODE_BITSIZE (<MODE>mode) - 1)"
5301 "sbfiz\\t%<w>0, %<w>1, %3, %2"
5302 [(set_attr "type" "bfx")]
5305 ;; When the bit position and width of the equivalent extraction add up to 32
5306 ;; we can use a W-reg LSL instruction taking advantage of the implicit
5307 ;; zero-extension of the X-reg.
5309 [(set (match_operand:DI 0 "register_operand")
5310 (and:DI (ashift:DI (match_operand:DI 1 "register_operand")
5311 (match_operand 2 "const_int_operand"))
5312 (match_operand 3 "const_int_operand")))]
5313 "aarch64_mask_and_shift_for_ubfiz_p (DImode, operands[3], operands[2])
5314 && (INTVAL (operands[2]) + popcount_hwi (INTVAL (operands[3])))
5315 == GET_MODE_BITSIZE (SImode)"
5317 (zero_extend:DI (ashift:SI (match_dup 4) (match_dup 2))))]
5319 operands[4] = gen_lowpart (SImode, operands[1]);
5323 (define_insn "bswap<mode>2"
5324 [(set (match_operand:GPI 0 "register_operand" "=r")
5325 (bswap:GPI (match_operand:GPI 1 "register_operand" "r")))]
5327 "rev\\t%<w>0, %<w>1"
5328 [(set_attr "type" "rev")]
5331 (define_insn "bswaphi2"
5332 [(set (match_operand:HI 0 "register_operand" "=r")
5333 (bswap:HI (match_operand:HI 1 "register_operand" "r")))]
5336 [(set_attr "type" "rev")]
5339 ;; There are no canonicalisation rules for the position of the lshiftrt, ashift
5340 ;; operations within an IOR/AND RTX, therefore we have two patterns matching
5341 ;; each valid permutation.
5343 (define_insn "rev16<mode>2"
5344 [(set (match_operand:GPI 0 "register_operand" "=r")
5345 (ior:GPI (and:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
5347 (match_operand:GPI 3 "const_int_operand" "n"))
5348 (and:GPI (lshiftrt:GPI (match_dup 1)
5350 (match_operand:GPI 2 "const_int_operand" "n"))))]
5351 "aarch_rev16_shleft_mask_imm_p (operands[3], <MODE>mode)
5352 && aarch_rev16_shright_mask_imm_p (operands[2], <MODE>mode)"
5353 "rev16\\t%<w>0, %<w>1"
5354 [(set_attr "type" "rev")]
5357 (define_insn "rev16<mode>2_alt"
5358 [(set (match_operand:GPI 0 "register_operand" "=r")
5359 (ior:GPI (and:GPI (lshiftrt:GPI (match_operand:GPI 1 "register_operand" "r")
5361 (match_operand:GPI 2 "const_int_operand" "n"))
5362 (and:GPI (ashift:GPI (match_dup 1)
5364 (match_operand:GPI 3 "const_int_operand" "n"))))]
5365 "aarch_rev16_shleft_mask_imm_p (operands[3], <MODE>mode)
5366 && aarch_rev16_shright_mask_imm_p (operands[2], <MODE>mode)"
5367 "rev16\\t%<w>0, %<w>1"
5368 [(set_attr "type" "rev")]
5371 ;; zero_extend version of above
5372 (define_insn "*bswapsi2_uxtw"
5373 [(set (match_operand:DI 0 "register_operand" "=r")
5374 (zero_extend:DI (bswap:SI (match_operand:SI 1 "register_operand" "r"))))]
5377 [(set_attr "type" "rev")]
5380 ;; -------------------------------------------------------------------
5381 ;; Floating-point intrinsics
5382 ;; -------------------------------------------------------------------
5384 ;; frint floating-point round to integral standard patterns.
5385 ;; Expands to btrunc, ceil, floor, nearbyint, rint, round, frintn.
5387 (define_insn "<frint_pattern><mode>2"
5388 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5389 (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")]
5392 "frint<frint_suffix>\\t%<s>0, %<s>1"
5393 [(set_attr "type" "f_rint<stype>")]
5396 ;; frcvt floating-point round to integer and convert standard patterns.
5397 ;; Expands to lbtrunc, lceil, lfloor, lround.
5398 (define_insn "l<fcvt_pattern><su_optab><GPF_F16:mode><GPI:mode>2"
5399 [(set (match_operand:GPI 0 "register_operand" "=r")
5401 (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")]
5404 "fcvt<frint_suffix><su>\\t%<GPI:w>0, %<GPF_F16:s>1"
5405 [(set_attr "type" "f_cvtf2i")]
5408 (define_insn "*aarch64_fcvt<su_optab><GPF:mode><GPI:mode>2_mult"
5409 [(set (match_operand:GPI 0 "register_operand" "=r")
5412 (match_operand:GPF 1 "register_operand" "w")
5413 (match_operand:GPF 2 "aarch64_fp_pow2" "F"))))]
5415 && IN_RANGE (aarch64_fpconst_pow_of_2 (operands[2]), 1,
5416 GET_MODE_BITSIZE (<GPI:MODE>mode))"
5418 int fbits = aarch64_fpconst_pow_of_2 (operands[2]);
5420 snprintf (buf, 64, "fcvtz<su>\\t%%<GPI:w>0, %%<GPF:s>1, #%d", fbits);
5421 output_asm_insn (buf, operands);
5424 [(set_attr "type" "f_cvtf2i")]
5427 ;; fma - expand fma into patterns with the accumulator operand first since
5428 ;; reusing the accumulator results in better register allocation.
5429 ;; The register allocator considers copy preferences in operand order,
5430 ;; so this prefers fmadd s0, s1, s2, s0 over fmadd s1, s1, s2, s0.
5432 (define_expand "fma<mode>4"
5433 [(set (match_operand:GPF_F16 0 "register_operand")
5434 (fma:GPF_F16 (match_operand:GPF_F16 1 "register_operand")
5435 (match_operand:GPF_F16 2 "register_operand")
5436 (match_operand:GPF_F16 3 "register_operand")))]
5440 (define_insn "*aarch64_fma<mode>4"
5441 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5442 (fma:GPF_F16 (match_operand:GPF_F16 2 "register_operand" "w")
5443 (match_operand:GPF_F16 3 "register_operand" "w")
5444 (match_operand:GPF_F16 1 "register_operand" "w")))]
5446 "fmadd\\t%<s>0, %<s>2, %<s>3, %<s>1"
5447 [(set_attr "type" "fmac<stype>")]
5450 (define_expand "fnma<mode>4"
5451 [(set (match_operand:GPF_F16 0 "register_operand")
5453 (neg:GPF_F16 (match_operand:GPF_F16 1 "register_operand"))
5454 (match_operand:GPF_F16 2 "register_operand")
5455 (match_operand:GPF_F16 3 "register_operand")))]
5459 (define_insn "*aarch64_fnma<mode>4"
5460 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5462 (neg:GPF_F16 (match_operand:GPF_F16 2 "register_operand" "w"))
5463 (match_operand:GPF_F16 3 "register_operand" "w")
5464 (match_operand:GPF_F16 1 "register_operand" "w")))]
5466 "fmsub\\t%<s>0, %<s>2, %<s>3, %<s>1"
5467 [(set_attr "type" "fmac<stype>")]
5471 (define_expand "fms<mode>4"
5472 [(set (match_operand:GPF 0 "register_operand")
5473 (fma:GPF (match_operand:GPF 1 "register_operand")
5474 (match_operand:GPF 2 "register_operand")
5475 (neg:GPF (match_operand:GPF 3 "register_operand"))))]
5479 (define_insn "*aarch64_fms<mode>4"
5480 [(set (match_operand:GPF 0 "register_operand" "=w")
5481 (fma:GPF (match_operand:GPF 2 "register_operand" "w")
5482 (match_operand:GPF 3 "register_operand" "w")
5483 (neg:GPF (match_operand:GPF 1 "register_operand" "w"))))]
5485 "fnmsub\\t%<s>0, %<s>2, %<s>3, %<s>1"
5486 [(set_attr "type" "fmac<s>")]
5489 (define_expand "fnms<mode>4"
5490 [(set (match_operand:GPF 0 "register_operand")
5491 (fma:GPF (neg:GPF (match_operand:GPF 1 "register_operand"))
5492 (match_operand:GPF 2 "register_operand")
5493 (neg:GPF (match_operand:GPF 3 "register_operand"))))]
5497 (define_insn "*aarch64_fnms<mode>4"
5498 [(set (match_operand:GPF 0 "register_operand" "=w")
5499 (fma:GPF (neg:GPF (match_operand:GPF 2 "register_operand" "w"))
5500 (match_operand:GPF 3 "register_operand" "w")
5501 (neg:GPF (match_operand:GPF 1 "register_operand" "w"))))]
5503 "fnmadd\\t%<s>0, %<s>2, %<s>3, %<s>1"
5504 [(set_attr "type" "fmac<s>")]
5507 ;; If signed zeros are ignored, -(a * b + c) = -a * b - c.
5508 (define_insn "*aarch64_fnmadd<mode>4"
5509 [(set (match_operand:GPF 0 "register_operand" "=w")
5510 (neg:GPF (fma:GPF (match_operand:GPF 2 "register_operand" "w")
5511 (match_operand:GPF 3 "register_operand" "w")
5512 (match_operand:GPF 1 "register_operand" "w"))))]
5513 "!HONOR_SIGNED_ZEROS (<MODE>mode) && TARGET_FLOAT"
5514 "fnmadd\\t%<s>0, %<s>2, %<s>3, %<s>1"
5515 [(set_attr "type" "fmac<s>")]
5518 ;; -------------------------------------------------------------------
5519 ;; Floating-point conversions
5520 ;; -------------------------------------------------------------------
5522 (define_insn "extendsfdf2"
5523 [(set (match_operand:DF 0 "register_operand" "=w")
5524 (float_extend:DF (match_operand:SF 1 "register_operand" "w")))]
5527 [(set_attr "type" "f_cvt")]
5530 (define_insn "extendhfsf2"
5531 [(set (match_operand:SF 0 "register_operand" "=w")
5532 (float_extend:SF (match_operand:HF 1 "register_operand" "w")))]
5535 [(set_attr "type" "f_cvt")]
5538 (define_insn "extendhfdf2"
5539 [(set (match_operand:DF 0 "register_operand" "=w")
5540 (float_extend:DF (match_operand:HF 1 "register_operand" "w")))]
5543 [(set_attr "type" "f_cvt")]
5546 (define_insn "truncdfsf2"
5547 [(set (match_operand:SF 0 "register_operand" "=w")
5548 (float_truncate:SF (match_operand:DF 1 "register_operand" "w")))]
5551 [(set_attr "type" "f_cvt")]
5554 (define_insn "truncsfhf2"
5555 [(set (match_operand:HF 0 "register_operand" "=w")
5556 (float_truncate:HF (match_operand:SF 1 "register_operand" "w")))]
5559 [(set_attr "type" "f_cvt")]
5562 (define_insn "truncdfhf2"
5563 [(set (match_operand:HF 0 "register_operand" "=w")
5564 (float_truncate:HF (match_operand:DF 1 "register_operand" "w")))]
5567 [(set_attr "type" "f_cvt")]
5570 ;; Convert SF -> SI or DF -> DI while preferring w = w register constraints
5571 ;; and making r = w more expensive
5573 (define_insn "<optab>_trunc<fcvt_target><GPI:mode>2"
5574 [(set (match_operand:GPI 0 "register_operand" "=w,?r")
5575 (FIXUORS:GPI (match_operand:<FCVT_TARGET> 1 "register_operand" "w,w")))]
5578 fcvtz<su>\t%<s>0, %<s>1
5579 fcvtz<su>\t%<w>0, %<s>1"
5580 [(set_attr "type" "neon_fp_to_int_s,f_cvtf2i")]
5583 ;; Convert HF -> SI or DI
5585 (define_insn "<optab>_trunchf<GPI:mode>2"
5586 [(set (match_operand:GPI 0 "register_operand" "=r")
5587 (FIXUORS:GPI (match_operand:HF 1 "register_operand" "w")))]
5589 "fcvtz<su>\t%<w>0, %h1"
5590 [(set_attr "type" "f_cvtf2i")]
5593 ;; Convert DF -> SI or SF -> DI which can only be accomplished with
5594 ;; input in a fp register and output in a integer register
5596 (define_insn "<optab>_trunc<fcvt_change_mode><GPI:mode>2"
5597 [(set (match_operand:GPI 0 "register_operand" "=r")
5598 (FIXUORS:GPI (match_operand:<FCVT_CHANGE_MODE> 1 "register_operand" "w")))]
5600 "fcvtz<su>\t%<w>0, %<fpw>1"
5601 [(set_attr "type" "f_cvtf2i")]
5604 (define_insn "*fix_to_zero_extend<mode>di2"
5605 [(set (match_operand:DI 0 "register_operand" "=r")
5608 (match_operand:GPF 1 "register_operand" "w"))))]
5610 "fcvtzu\t%w0, %<s>1"
5611 [(set_attr "type" "f_cvtf2i")]
5614 (define_insn "<optab><fcvt_target><GPF:mode>2"
5615 [(set (match_operand:GPF 0 "register_operand" "=w,w")
5616 (FLOATUORS:GPF (match_operand:<FCVT_TARGET> 1 "register_operand" "w,?r")))]
5619 <su_optab>cvtf\t%<GPF:s>0, %<s>1
5620 <su_optab>cvtf\t%<GPF:s>0, %<w1>1"
5621 [(set_attr "type" "neon_int_to_fp_<Vetype>,f_cvti2f")
5622 (set_attr "arch" "simd,fp")]
5625 (define_insn "<optab><fcvt_iesize><GPF:mode>2"
5626 [(set (match_operand:GPF 0 "register_operand" "=w")
5627 (FLOATUORS:GPF (match_operand:<FCVT_IESIZE> 1 "register_operand" "r")))]
5629 "<su_optab>cvtf\t%<GPF:s>0, %<w2>1"
5630 [(set_attr "type" "f_cvti2f")]
5633 ;; If we do not have ARMv8.2-A 16-bit floating point extensions, the
5634 ;; midend will arrange for an SImode conversion to HFmode to first go
5635 ;; through DFmode, then to HFmode. But first it will try converting
5636 ;; to DImode then down, which would match our DImode pattern below and
5637 ;; give very poor code-generation. So, we must provide our own emulation
5638 ;; of the mid-end logic.
5640 (define_insn "aarch64_fp16_<optab><mode>hf2"
5641 [(set (match_operand:HF 0 "register_operand" "=w")
5642 (FLOATUORS:HF (match_operand:GPI 1 "register_operand" "r")))]
5644 "<su_optab>cvtf\t%h0, %<w>1"
5645 [(set_attr "type" "f_cvti2f")]
5648 (define_expand "<optab>sihf2"
5649 [(set (match_operand:HF 0 "register_operand")
5650 (FLOATUORS:HF (match_operand:SI 1 "register_operand")))]
5653 if (TARGET_FP_F16INST)
5654 emit_insn (gen_aarch64_fp16_<optab>sihf2 (operands[0], operands[1]));
5657 rtx convert_target = gen_reg_rtx (DFmode);
5658 emit_insn (gen_<optab>sidf2 (convert_target, operands[1]));
5659 emit_insn (gen_truncdfhf2 (operands[0], convert_target));
5665 ;; For DImode there is no wide enough floating-point mode that we
5666 ;; can convert through natively (TFmode would work, but requires a library
5667 ;; call). However, we know that any value >= 65504 will be rounded
5668 ;; to infinity on conversion. This is well within the range of SImode, so
5670 ;; Saturate to SImode.
5671 ;; Convert from that to DFmode
5672 ;; Convert from that to HFmode (phew!).
5673 ;; Note that the saturation to SImode requires the SIMD extensions. If
5674 ;; we ever need to provide this pattern where the SIMD extensions are not
5675 ;; available, we would need a different approach.
5677 (define_expand "<optab>dihf2"
5678 [(set (match_operand:HF 0 "register_operand")
5679 (FLOATUORS:HF (match_operand:DI 1 "register_operand")))]
5680 "TARGET_FLOAT && (TARGET_FP_F16INST || TARGET_SIMD)"
5682 if (TARGET_FP_F16INST)
5683 emit_insn (gen_aarch64_fp16_<optab>dihf2 (operands[0], operands[1]));
5686 rtx sat_target = gen_reg_rtx (SImode);
5687 emit_insn (gen_aarch64_<su_optab>qmovndi (sat_target, operands[1]));
5688 emit_insn (gen_<optab>sihf2 (operands[0], sat_target));
5695 ;; Convert between fixed-point and floating-point (scalar modes)
5697 (define_insn "<FCVT_F2FIXED:fcvt_fixed_insn><GPF:mode>3"
5698 [(set (match_operand:<GPF:FCVT_TARGET> 0 "register_operand" "=r, w")
5699 (unspec:<GPF:FCVT_TARGET> [(match_operand:GPF 1 "register_operand" "w, w")
5700 (match_operand:SI 2 "immediate_operand" "i, i")]
5704 <FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPF:w1>0, %<GPF:s>1, #%2
5705 <FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPF:s>0, %<GPF:s>1, #%2"
5706 [(set_attr "type" "f_cvtf2i, neon_fp_to_int_<GPF:Vetype>")
5707 (set_attr "arch" "fp,simd")]
5710 (define_insn "<FCVT_FIXED2F:fcvt_fixed_insn><GPI:mode>3"
5711 [(set (match_operand:<GPI:FCVT_TARGET> 0 "register_operand" "=w, w")
5712 (unspec:<GPI:FCVT_TARGET> [(match_operand:GPI 1 "register_operand" "r, w")
5713 (match_operand:SI 2 "immediate_operand" "i, i")]
5717 <FCVT_FIXED2F:fcvt_fixed_insn>\t%<GPI:v>0, %<GPI:w>1, #%2
5718 <FCVT_FIXED2F:fcvt_fixed_insn>\t%<GPI:v>0, %<GPI:v>1, #%2"
5719 [(set_attr "type" "f_cvti2f, neon_int_to_fp_<GPI:Vetype>")
5720 (set_attr "arch" "fp,simd")]
5723 (define_insn "<FCVT_F2FIXED:fcvt_fixed_insn>hf<mode>3"
5724 [(set (match_operand:GPI 0 "register_operand" "=r")
5725 (unspec:GPI [(match_operand:HF 1 "register_operand" "w")
5726 (match_operand:SI 2 "immediate_operand" "i")]
5729 "<FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPI:w>0, %h1, #%2"
5730 [(set_attr "type" "f_cvtf2i")]
5733 (define_insn "<FCVT_FIXED2F:fcvt_fixed_insn><mode>hf3"
5734 [(set (match_operand:HF 0 "register_operand" "=w")
5735 (unspec:HF [(match_operand:GPI 1 "register_operand" "r")
5736 (match_operand:SI 2 "immediate_operand" "i")]
5739 "<FCVT_FIXED2F:fcvt_fixed_insn>\t%h0, %<GPI:w>1, #%2"
5740 [(set_attr "type" "f_cvti2f")]
5743 (define_insn "<FCVT_F2FIXED:fcvt_fixed_insn>hf3"
5744 [(set (match_operand:HI 0 "register_operand" "=w")
5745 (unspec:HI [(match_operand:HF 1 "register_operand" "w")
5746 (match_operand:SI 2 "immediate_operand" "i")]
5749 "<FCVT_F2FIXED:fcvt_fixed_insn>\t%h0, %h1, #%2"
5750 [(set_attr "type" "neon_fp_to_int_s")]
5753 (define_insn "<FCVT_FIXED2F:fcvt_fixed_insn>hi3"
5754 [(set (match_operand:HF 0 "register_operand" "=w")
5755 (unspec:HF [(match_operand:HI 1 "register_operand" "w")
5756 (match_operand:SI 2 "immediate_operand" "i")]
5759 "<FCVT_FIXED2F:fcvt_fixed_insn>\t%h0, %h1, #%2"
5760 [(set_attr "type" "neon_int_to_fp_s")]
5763 ;; -------------------------------------------------------------------
5764 ;; Floating-point arithmetic
5765 ;; -------------------------------------------------------------------
5767 (define_insn "add<mode>3"
5768 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5770 (match_operand:GPF_F16 1 "register_operand" "w")
5771 (match_operand:GPF_F16 2 "register_operand" "w")))]
5773 "fadd\\t%<s>0, %<s>1, %<s>2"
5774 [(set_attr "type" "fadd<stype>")]
5777 (define_insn "sub<mode>3"
5778 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5780 (match_operand:GPF_F16 1 "register_operand" "w")
5781 (match_operand:GPF_F16 2 "register_operand" "w")))]
5783 "fsub\\t%<s>0, %<s>1, %<s>2"
5784 [(set_attr "type" "fadd<stype>")]
5787 (define_insn "mul<mode>3"
5788 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5790 (match_operand:GPF_F16 1 "register_operand" "w")
5791 (match_operand:GPF_F16 2 "register_operand" "w")))]
5793 "fmul\\t%<s>0, %<s>1, %<s>2"
5794 [(set_attr "type" "fmul<stype>")]
5797 (define_insn "*fnmul<mode>3"
5798 [(set (match_operand:GPF 0 "register_operand" "=w")
5800 (neg:GPF (match_operand:GPF 1 "register_operand" "w"))
5801 (match_operand:GPF 2 "register_operand" "w")))]
5802 "TARGET_FLOAT && !flag_rounding_math"
5803 "fnmul\\t%<s>0, %<s>1, %<s>2"
5804 [(set_attr "type" "fmul<s>")]
5807 (define_insn "*fnmul<mode>3"
5808 [(set (match_operand:GPF 0 "register_operand" "=w")
5810 (match_operand:GPF 1 "register_operand" "w")
5811 (match_operand:GPF 2 "register_operand" "w"))))]
5813 "fnmul\\t%<s>0, %<s>1, %<s>2"
5814 [(set_attr "type" "fmul<s>")]
5817 (define_expand "div<mode>3"
5818 [(set (match_operand:GPF_F16 0 "register_operand")
5819 (div:GPF_F16 (match_operand:GPF_F16 1 "general_operand")
5820 (match_operand:GPF_F16 2 "register_operand")))]
5823 if (aarch64_emit_approx_div (operands[0], operands[1], operands[2]))
5826 operands[1] = force_reg (<MODE>mode, operands[1]);
5829 (define_insn "*div<mode>3"
5830 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5831 (div:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")
5832 (match_operand:GPF_F16 2 "register_operand" "w")))]
5834 "fdiv\\t%<s>0, %<s>1, %<s>2"
5835 [(set_attr "type" "fdiv<stype>")]
5838 (define_insn "neg<mode>2"
5839 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5840 (neg:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
5842 "fneg\\t%<s>0, %<s>1"
5843 [(set_attr "type" "ffarith<stype>")]
5846 (define_expand "sqrt<mode>2"
5847 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5848 (sqrt:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
5851 if (aarch64_emit_approx_sqrt (operands[0], operands[1], false))
5855 (define_insn "*sqrt<mode>2"
5856 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5857 (sqrt:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
5859 "fsqrt\\t%<s>0, %<s>1"
5860 [(set_attr "type" "fsqrt<stype>")]
5863 (define_insn "abs<mode>2"
5864 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5865 (abs:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
5867 "fabs\\t%<s>0, %<s>1"
5868 [(set_attr "type" "ffarith<stype>")]
5871 ;; Given that smax/smin do not specify the result when either input is NaN,
5872 ;; we could use either FMAXNM or FMAX for smax, and either FMINNM or FMIN
5875 (define_insn "smax<mode>3"
5876 [(set (match_operand:GPF 0 "register_operand" "=w")
5877 (smax:GPF (match_operand:GPF 1 "register_operand" "w")
5878 (match_operand:GPF 2 "register_operand" "w")))]
5880 "fmaxnm\\t%<s>0, %<s>1, %<s>2"
5881 [(set_attr "type" "f_minmax<s>")]
5884 (define_insn "smin<mode>3"
5885 [(set (match_operand:GPF 0 "register_operand" "=w")
5886 (smin:GPF (match_operand:GPF 1 "register_operand" "w")
5887 (match_operand:GPF 2 "register_operand" "w")))]
5889 "fminnm\\t%<s>0, %<s>1, %<s>2"
5890 [(set_attr "type" "f_minmax<s>")]
5893 ;; Scalar forms for fmax, fmin, fmaxnm, fminnm.
5894 ;; fmaxnm and fminnm are used for the fmax<mode>3 standard pattern names,
5895 ;; which implement the IEEE fmax ()/fmin () functions.
5896 (define_insn "<maxmin_uns><mode>3"
5897 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5898 (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")
5899 (match_operand:GPF_F16 2 "register_operand" "w")]
5902 "<maxmin_uns_op>\\t%<s>0, %<s>1, %<s>2"
5903 [(set_attr "type" "f_minmax<stype>")]
5906 (define_expand "lrint<GPF:mode><GPI:mode>2"
5907 [(match_operand:GPI 0 "register_operand")
5908 (match_operand:GPF 1 "register_operand")]
5910 && ((GET_MODE_SIZE (<GPF:MODE>mode) <= GET_MODE_SIZE (<GPI:MODE>mode))
5911 || !flag_trapping_math || flag_fp_int_builtin_inexact)"
5913 rtx cvt = gen_reg_rtx (<GPF:MODE>mode);
5914 emit_insn (gen_rint<GPF:mode>2 (cvt, operands[1]));
5915 emit_insn (gen_lbtrunc<GPF:mode><GPI:mode>2 (operands[0], cvt));
5920 ;; For copysign (x, y), we want to generate:
5922 ;; LDR d2, #(1 << 63)
5923 ;; BSL v2.8b, [y], [x]
5925 ;; or another, equivalent, sequence using one of BSL/BIT/BIF.
5926 ;; aarch64_simd_bsldf will select the best suited of these instructions
5927 ;; to generate based on register allocation, and knows how to partially
5928 ;; constant fold based on the values of X and Y, so expand through that.
5930 (define_expand "copysigndf3"
5931 [(match_operand:DF 0 "register_operand")
5932 (match_operand:DF 1 "register_operand")
5933 (match_operand:DF 2 "register_operand")]
5934 "TARGET_FLOAT && TARGET_SIMD"
5936 rtx mask = gen_reg_rtx (DImode);
5937 emit_move_insn (mask, GEN_INT (HOST_WIDE_INT_1U << 63));
5938 emit_insn (gen_aarch64_simd_bsldf (operands[0], mask,
5939 operands[2], operands[1]));
5944 ;; As above, but we must first get to a 64-bit value if we wish to use
5945 ;; aarch64_simd_bslv2sf.
5947 (define_expand "copysignsf3"
5948 [(match_operand:SF 0 "register_operand")
5949 (match_operand:SF 1 "register_operand")
5950 (match_operand:SF 2 "register_operand")]
5951 "TARGET_FLOAT && TARGET_SIMD"
5953 rtx v_bitmask = gen_reg_rtx (V2SImode);
5955 /* Juggle modes to get us in to a vector mode for BSL. */
5956 rtx op1 = lowpart_subreg (DImode, operands[1], SFmode);
5957 rtx op2 = lowpart_subreg (V2SFmode, operands[2], SFmode);
5958 rtx tmp = gen_reg_rtx (V2SFmode);
5959 emit_move_insn (v_bitmask,
5960 aarch64_simd_gen_const_vector_dup (V2SImode,
5961 HOST_WIDE_INT_M1U << 31));
5962 emit_insn (gen_aarch64_simd_bslv2sf (tmp, v_bitmask, op2, op1));
5963 emit_move_insn (operands[0], lowpart_subreg (SFmode, tmp, V2SFmode));
5968 ;; For xorsign (x, y), we want to generate:
5971 ;; AND v3.8B, v1.8B, v2.8B
5972 ;; EOR v0.8B, v0.8B, v3.8B
5975 (define_expand "xorsign<mode>3"
5976 [(match_operand:GPF 0 "register_operand")
5977 (match_operand:GPF 1 "register_operand")
5978 (match_operand:GPF 2 "register_operand")]
5979 "TARGET_FLOAT && TARGET_SIMD"
5982 machine_mode imode = <V_INT_EQUIV>mode;
5983 rtx mask = gen_reg_rtx (imode);
5984 rtx op1x = gen_reg_rtx (imode);
5985 rtx op2x = gen_reg_rtx (imode);
5987 int bits = GET_MODE_BITSIZE (<MODE>mode) - 1;
5988 emit_move_insn (mask, GEN_INT (trunc_int_for_mode (HOST_WIDE_INT_M1U << bits,
5991 emit_insn (gen_and<v_int_equiv>3 (op2x, mask,
5992 lowpart_subreg (imode, operands[2],
5994 emit_insn (gen_xor<v_int_equiv>3 (op1x,
5995 lowpart_subreg (imode, operands[1],
5998 emit_move_insn (operands[0],
5999 lowpart_subreg (<MODE>mode, op1x, imode));
6004 ;; -------------------------------------------------------------------
6006 ;; -------------------------------------------------------------------
6007 ;; Reload Scalar Floating point modes from constant pool.
6008 ;; The AArch64 port doesn't have __int128 constant move support.
6009 (define_expand "@aarch64_reload_movcp<GPF_TF:mode><P:mode>"
6010 [(set (match_operand:GPF_TF 0 "register_operand" "=w")
6011 (mem:GPF_TF (match_operand 1 "aarch64_constant_pool_symref" "S")))
6012 (clobber (match_operand:P 2 "register_operand" "=&r"))]
6015 aarch64_expand_mov_immediate (operands[2], XEXP (operands[1], 0));
6016 emit_move_insn (operands[0], gen_rtx_MEM (<GPF_TF:MODE>mode, operands[2]));
6021 ;; Reload Vector modes from constant pool.
6022 (define_expand "@aarch64_reload_movcp<VALL:mode><P:mode>"
6023 [(set (match_operand:VALL 0 "register_operand" "=w")
6024 (mem:VALL (match_operand 1 "aarch64_constant_pool_symref" "S")))
6025 (clobber (match_operand:P 2 "register_operand" "=&r"))]
6028 aarch64_expand_mov_immediate (operands[2], XEXP (operands[1], 0));
6029 emit_move_insn (operands[0], gen_rtx_MEM (<VALL:MODE>mode, operands[2]));
6034 (define_expand "@aarch64_reload_mov<mode>"
6035 [(set (match_operand:TX 0 "register_operand" "=w")
6036 (match_operand:TX 1 "register_operand" "w"))
6037 (clobber (match_operand:DI 2 "register_operand" "=&r"))
6041 rtx op0 = simplify_gen_subreg (TImode, operands[0], <MODE>mode, 0);
6042 rtx op1 = simplify_gen_subreg (TImode, operands[1], <MODE>mode, 0);
6043 gen_aarch64_movtilow_tilow (op0, op1);
6044 gen_aarch64_movdi_tihigh (operands[2], op1);
6045 gen_aarch64_movtihigh_di (op0, operands[2]);
6050 ;; The following secondary reload helpers patterns are invoked
6051 ;; after or during reload as we don't want these patterns to start
6052 ;; kicking in during the combiner.
6054 (define_insn "@aarch64_movdi_<mode>low"
6055 [(set (match_operand:DI 0 "register_operand" "=r")
6056 (zero_extract:DI (match_operand:TX 1 "register_operand" "w")
6057 (const_int 64) (const_int 0)))]
6058 "TARGET_FLOAT && (reload_completed || reload_in_progress)"
6060 [(set_attr "type" "f_mrc")
6061 (set_attr "length" "4")
6064 (define_insn "@aarch64_movdi_<mode>high"
6065 [(set (match_operand:DI 0 "register_operand" "=r")
6066 (zero_extract:DI (match_operand:TX 1 "register_operand" "w")
6067 (const_int 64) (const_int 64)))]
6068 "TARGET_FLOAT && (reload_completed || reload_in_progress)"
6069 "fmov\\t%x0, %1.d[1]"
6070 [(set_attr "type" "f_mrc")
6071 (set_attr "length" "4")
6074 (define_insn "@aarch64_mov<mode>high_di"
6075 [(set (zero_extract:TX (match_operand:TX 0 "register_operand" "+w")
6076 (const_int 64) (const_int 64))
6077 (zero_extend:TX (match_operand:DI 1 "register_operand" "r")))]
6078 "TARGET_FLOAT && (reload_completed || reload_in_progress)"
6079 "fmov\\t%0.d[1], %x1"
6080 [(set_attr "type" "f_mcr")
6081 (set_attr "length" "4")
6084 (define_insn "@aarch64_mov<mode>low_di"
6085 [(set (match_operand:TX 0 "register_operand" "=w")
6086 (zero_extend:TX (match_operand:DI 1 "register_operand" "r")))]
6087 "TARGET_FLOAT && (reload_completed || reload_in_progress)"
6089 [(set_attr "type" "f_mcr")
6090 (set_attr "length" "4")
6093 (define_insn "aarch64_movtilow_tilow"
6094 [(set (match_operand:TI 0 "register_operand" "=w")
6096 (truncate:DI (match_operand:TI 1 "register_operand" "w"))))]
6097 "TARGET_FLOAT && (reload_completed || reload_in_progress)"
6099 [(set_attr "type" "fmov")
6100 (set_attr "length" "4")
6103 ;; There is a deliberate reason why the parameters of high and lo_sum's
6104 ;; don't have modes for ADRP and ADD instructions. This is to allow high
6105 ;; and lo_sum's to be used with the labels defining the jump tables in
6108 (define_expand "add_losym"
6109 [(set (match_operand 0 "register_operand" "=r")
6110 (lo_sum (match_operand 1 "register_operand" "r")
6111 (match_operand 2 "aarch64_valid_symref" "S")))]
6114 machine_mode mode = GET_MODE (operands[0]);
6116 emit_insn ((mode == DImode
6118 : gen_add_losym_si) (operands[0],
6124 (define_insn "add_losym_<mode>"
6125 [(set (match_operand:P 0 "register_operand" "=r")
6126 (lo_sum:P (match_operand:P 1 "register_operand" "r")
6127 (match_operand 2 "aarch64_valid_symref" "S")))]
6129 "add\\t%<w>0, %<w>1, :lo12:%c2"
6130 [(set_attr "type" "alu_imm")]
6133 (define_insn "ldr_got_small_<mode>"
6134 [(set (match_operand:PTR 0 "register_operand" "=r")
6135 (unspec:PTR [(mem:PTR (lo_sum:PTR
6136 (match_operand:PTR 1 "register_operand" "r")
6137 (match_operand:PTR 2 "aarch64_valid_symref" "S")))]
6138 UNSPEC_GOTSMALLPIC))]
6140 "ldr\\t%<w>0, [%1, #:got_lo12:%c2]"
6141 [(set_attr "type" "load_<ldst_sz>")]
6144 (define_insn "ldr_got_small_sidi"
6145 [(set (match_operand:DI 0 "register_operand" "=r")
6147 (unspec:SI [(mem:SI (lo_sum:DI
6148 (match_operand:DI 1 "register_operand" "r")
6149 (match_operand:DI 2 "aarch64_valid_symref" "S")))]
6150 UNSPEC_GOTSMALLPIC)))]
6152 "ldr\\t%w0, [%1, #:got_lo12:%c2]"
6153 [(set_attr "type" "load_4")]
6156 (define_insn "ldr_got_small_28k_<mode>"
6157 [(set (match_operand:PTR 0 "register_operand" "=r")
6158 (unspec:PTR [(mem:PTR (lo_sum:PTR
6159 (match_operand:PTR 1 "register_operand" "r")
6160 (match_operand:PTR 2 "aarch64_valid_symref" "S")))]
6161 UNSPEC_GOTSMALLPIC28K))]
6163 "ldr\\t%<w>0, [%1, #:<got_modifier>:%c2]"
6164 [(set_attr "type" "load_<ldst_sz>")]
6167 (define_insn "ldr_got_small_28k_sidi"
6168 [(set (match_operand:DI 0 "register_operand" "=r")
6170 (unspec:SI [(mem:SI (lo_sum:DI
6171 (match_operand:DI 1 "register_operand" "r")
6172 (match_operand:DI 2 "aarch64_valid_symref" "S")))]
6173 UNSPEC_GOTSMALLPIC28K)))]
6175 "ldr\\t%w0, [%1, #:gotpage_lo14:%c2]"
6176 [(set_attr "type" "load_4")]
6179 (define_insn "ldr_got_tiny"
6180 [(set (match_operand:DI 0 "register_operand" "=r")
6181 (unspec:DI [(match_operand:DI 1 "aarch64_valid_symref" "S")]
6182 UNSPEC_GOTTINYPIC))]
6185 [(set_attr "type" "load_8")]
6188 (define_insn "aarch64_load_tp_hard"
6189 [(set (match_operand:DI 0 "register_operand" "=r")
6190 (unspec:DI [(const_int 0)] UNSPEC_TLS))]
6192 "mrs\\t%0, tpidr_el0"
6193 [(set_attr "type" "mrs")]
6196 ;; The TLS ABI specifically requires that the compiler does not schedule
6197 ;; instructions in the TLS stubs, in order to enable linker relaxation.
6198 ;; Therefore we treat the stubs as an atomic sequence.
6199 (define_expand "tlsgd_small_<mode>"
6200 [(parallel [(set (match_operand 0 "register_operand" "")
6201 (call (mem:DI (match_dup 2)) (const_int 1)))
6202 (unspec:DI [(match_operand:PTR 1 "aarch64_valid_symref" "")] UNSPEC_GOTSMALLTLS)
6203 (clobber (reg:DI LR_REGNUM))])]
6206 operands[2] = aarch64_tls_get_addr ();
6209 (define_insn "*tlsgd_small_<mode>"
6210 [(set (match_operand 0 "register_operand" "")
6211 (call (mem:DI (match_operand:DI 2 "" "")) (const_int 1)))
6212 (unspec:DI [(match_operand:PTR 1 "aarch64_valid_symref" "S")] UNSPEC_GOTSMALLTLS)
6213 (clobber (reg:DI LR_REGNUM))
6216 "adrp\\tx0, %A1\;add\\tx0, x0, %L1\;bl\\t%2\;nop"
6217 [(set_attr "type" "call")
6218 (set_attr "length" "16")])
6220 (define_insn "tlsie_small_<mode>"
6221 [(set (match_operand:PTR 0 "register_operand" "=r")
6222 (unspec:PTR [(match_operand 1 "aarch64_tls_ie_symref" "S")]
6223 UNSPEC_GOTSMALLTLS))]
6225 "adrp\\t%0, %A1\;ldr\\t%<w>0, [%0, #%L1]"
6226 [(set_attr "type" "load_4")
6227 (set_attr "length" "8")]
6230 (define_insn "tlsie_small_sidi"
6231 [(set (match_operand:DI 0 "register_operand" "=r")
6233 (unspec:SI [(match_operand 1 "aarch64_tls_ie_symref" "S")]
6234 UNSPEC_GOTSMALLTLS)))]
6236 "adrp\\t%0, %A1\;ldr\\t%w0, [%0, #%L1]"
6237 [(set_attr "type" "load_4")
6238 (set_attr "length" "8")]
6241 (define_insn "tlsie_tiny_<mode>"
6242 [(set (match_operand:PTR 0 "register_operand" "=&r")
6243 (unspec:PTR [(match_operand 1 "aarch64_tls_ie_symref" "S")
6244 (match_operand:PTR 2 "register_operand" "r")]
6245 UNSPEC_GOTTINYTLS))]
6247 "ldr\\t%<w>0, %L1\;add\\t%<w>0, %<w>0, %<w>2"
6248 [(set_attr "type" "multiple")
6249 (set_attr "length" "8")]
6252 (define_insn "tlsie_tiny_sidi"
6253 [(set (match_operand:DI 0 "register_operand" "=&r")
6255 (unspec:SI [(match_operand 1 "aarch64_tls_ie_symref" "S")
6256 (match_operand:DI 2 "register_operand" "r")
6258 UNSPEC_GOTTINYTLS)))]
6260 "ldr\\t%w0, %L1\;add\\t%w0, %w0, %w2"
6261 [(set_attr "type" "multiple")
6262 (set_attr "length" "8")]
6265 (define_insn "tlsle12_<mode>"
6266 [(set (match_operand:P 0 "register_operand" "=r")
6267 (unspec:P [(match_operand:P 1 "register_operand" "r")
6268 (match_operand 2 "aarch64_tls_le_symref" "S")]
6271 "add\\t%<w>0, %<w>1, #%L2";
6272 [(set_attr "type" "alu_sreg")
6273 (set_attr "length" "4")]
6276 (define_insn "tlsle24_<mode>"
6277 [(set (match_operand:P 0 "register_operand" "=r")
6278 (unspec:P [(match_operand:P 1 "register_operand" "r")
6279 (match_operand 2 "aarch64_tls_le_symref" "S")]
6282 "add\\t%<w>0, %<w>1, #%G2, lsl #12\;add\\t%<w>0, %<w>0, #%L2"
6283 [(set_attr "type" "multiple")
6284 (set_attr "length" "8")]
6287 (define_insn "tlsle32_<mode>"
6288 [(set (match_operand:P 0 "register_operand" "=r")
6289 (unspec:P [(match_operand 1 "aarch64_tls_le_symref" "S")]
6292 "movz\\t%<w>0, #:tprel_g1:%1\;movk\\t%<w>0, #:tprel_g0_nc:%1"
6293 [(set_attr "type" "multiple")
6294 (set_attr "length" "8")]
6297 (define_insn "tlsle48_<mode>"
6298 [(set (match_operand:P 0 "register_operand" "=r")
6299 (unspec:P [(match_operand 1 "aarch64_tls_le_symref" "S")]
6302 "movz\\t%<w>0, #:tprel_g2:%1\;movk\\t%<w>0, #:tprel_g1_nc:%1\;movk\\t%<w>0, #:tprel_g0_nc:%1"
6303 [(set_attr "type" "multiple")
6304 (set_attr "length" "12")]
6307 (define_expand "tlsdesc_small_<mode>"
6308 [(unspec:PTR [(match_operand 0 "aarch64_valid_symref")] UNSPEC_TLSDESC)]
6312 emit_insn (gen_tlsdesc_small_sve_<mode> (operands[0]));
6314 emit_insn (gen_tlsdesc_small_advsimd_<mode> (operands[0]));
6319 ;; tlsdesc calls preserve all core and Advanced SIMD registers except
6321 (define_insn "tlsdesc_small_advsimd_<mode>"
6322 [(set (reg:PTR R0_REGNUM)
6323 (unspec:PTR [(match_operand 0 "aarch64_valid_symref" "S")]
6325 (clobber (reg:DI LR_REGNUM))
6326 (clobber (reg:CC CC_REGNUM))
6327 (clobber (match_scratch:DI 1 "=r"))]
6328 "TARGET_TLS_DESC && !TARGET_SVE"
6329 "adrp\\tx0, %A0\;ldr\\t%<w>1, [x0, #%L0]\;add\\t<w>0, <w>0, %L0\;.tlsdesccall\\t%0\;blr\\t%1"
6330 [(set_attr "type" "call")
6331 (set_attr "length" "16")])
6333 ;; For SVE, model tlsdesc calls as clobbering the lower 128 bits of
6334 ;; all vector registers, and clobber all predicate registers, on
6335 ;; top of the usual R0 and LR.
6336 (define_insn "tlsdesc_small_sve_<mode>"
6337 [(set (reg:PTR R0_REGNUM)
6338 (unspec:PTR [(match_operand 0 "aarch64_valid_symref" "S")]
6340 (clobber (reg:DI LR_REGNUM))
6341 (clobber (reg:CC CC_REGNUM))
6342 (clobber_high (reg:TI V0_REGNUM))
6343 (clobber_high (reg:TI V1_REGNUM))
6344 (clobber_high (reg:TI V2_REGNUM))
6345 (clobber_high (reg:TI V3_REGNUM))
6346 (clobber_high (reg:TI V4_REGNUM))
6347 (clobber_high (reg:TI V5_REGNUM))
6348 (clobber_high (reg:TI V6_REGNUM))
6349 (clobber_high (reg:TI V7_REGNUM))
6350 (clobber_high (reg:TI V8_REGNUM))
6351 (clobber_high (reg:TI V9_REGNUM))
6352 (clobber_high (reg:TI V10_REGNUM))
6353 (clobber_high (reg:TI V11_REGNUM))
6354 (clobber_high (reg:TI V12_REGNUM))
6355 (clobber_high (reg:TI V13_REGNUM))
6356 (clobber_high (reg:TI V14_REGNUM))
6357 (clobber_high (reg:TI V15_REGNUM))
6358 (clobber_high (reg:TI V16_REGNUM))
6359 (clobber_high (reg:TI V17_REGNUM))
6360 (clobber_high (reg:TI V18_REGNUM))
6361 (clobber_high (reg:TI V19_REGNUM))
6362 (clobber_high (reg:TI V20_REGNUM))
6363 (clobber_high (reg:TI V21_REGNUM))
6364 (clobber_high (reg:TI V22_REGNUM))
6365 (clobber_high (reg:TI V23_REGNUM))
6366 (clobber_high (reg:TI V24_REGNUM))
6367 (clobber_high (reg:TI V25_REGNUM))
6368 (clobber_high (reg:TI V26_REGNUM))
6369 (clobber_high (reg:TI V27_REGNUM))
6370 (clobber_high (reg:TI V28_REGNUM))
6371 (clobber_high (reg:TI V29_REGNUM))
6372 (clobber_high (reg:TI V30_REGNUM))
6373 (clobber_high (reg:TI V31_REGNUM))
6374 (clobber (reg:VNx2BI P0_REGNUM))
6375 (clobber (reg:VNx2BI P1_REGNUM))
6376 (clobber (reg:VNx2BI P2_REGNUM))
6377 (clobber (reg:VNx2BI P3_REGNUM))
6378 (clobber (reg:VNx2BI P4_REGNUM))
6379 (clobber (reg:VNx2BI P5_REGNUM))
6380 (clobber (reg:VNx2BI P6_REGNUM))
6381 (clobber (reg:VNx2BI P7_REGNUM))
6382 (clobber (reg:VNx2BI P8_REGNUM))
6383 (clobber (reg:VNx2BI P9_REGNUM))
6384 (clobber (reg:VNx2BI P10_REGNUM))
6385 (clobber (reg:VNx2BI P11_REGNUM))
6386 (clobber (reg:VNx2BI P12_REGNUM))
6387 (clobber (reg:VNx2BI P13_REGNUM))
6388 (clobber (reg:VNx2BI P14_REGNUM))
6389 (clobber (reg:VNx2BI P15_REGNUM))
6390 (clobber (match_scratch:DI 1 "=r"))]
6391 "TARGET_TLS_DESC && TARGET_SVE"
6392 "adrp\\tx0, %A0\;ldr\\t%<w>1, [x0, #%L0]\;add\\t<w>0, <w>0, %L0\;.tlsdesccall\\t%0\;blr\\t%1"
6393 [(set_attr "type" "call")
6394 (set_attr "length" "16")])
6396 (define_insn "stack_tie"
6397 [(set (mem:BLK (scratch))
6398 (unspec:BLK [(match_operand:DI 0 "register_operand" "rk")
6399 (match_operand:DI 1 "register_operand" "rk")]
6403 [(set_attr "length" "0")]
6406 ;; Pointer authentication patterns are always provided. In architecture
6407 ;; revisions prior to ARMv8.3-A these HINT instructions operate as NOPs.
6408 ;; This lets the user write portable software which authenticates pointers
6409 ;; when run on something which implements ARMv8.3-A, and which runs
6410 ;; correctly, but does not authenticate pointers, where ARMv8.3-A is not
6413 ;; Signing/Authenticating R30 using SP as the salt.
6415 (define_insn "<pauth_mnem_prefix>sp"
6416 [(set (reg:DI R30_REGNUM)
6417 (unspec:DI [(reg:DI R30_REGNUM) (reg:DI SP_REGNUM)] PAUTH_LR_SP))]
6419 "hint\t<pauth_hint_num_a> // <pauth_mnem_prefix>asp";
6422 ;; Signing/Authenticating X17 using X16 as the salt.
6424 (define_insn "<pauth_mnem_prefix>1716"
6425 [(set (reg:DI R17_REGNUM)
6426 (unspec:DI [(reg:DI R17_REGNUM) (reg:DI R16_REGNUM)] PAUTH_17_16))]
6428 "hint\t<pauth_hint_num_a> // <pauth_mnem_prefix>a1716";
6431 ;; Stripping the signature in R30.
6433 (define_insn "xpaclri"
6434 [(set (reg:DI R30_REGNUM) (unspec:DI [(reg:DI R30_REGNUM)] UNSPEC_XPACLRI))]
6436 "hint\t7 // xpaclri"
6439 ;; UNSPEC_VOLATILE is considered to use and clobber all hard registers and
6440 ;; all of memory. This blocks insns from being moved across this point.
6442 (define_insn "blockage"
6443 [(unspec_volatile [(const_int 0)] UNSPECV_BLOCKAGE)]
6446 [(set_attr "length" "0")
6447 (set_attr "type" "block")]
6450 (define_insn "probe_stack_range"
6451 [(set (match_operand:DI 0 "register_operand" "=r")
6452 (unspec_volatile:DI [(match_operand:DI 1 "register_operand" "0")
6453 (match_operand:DI 2 "register_operand" "r")]
6454 UNSPECV_PROBE_STACK_RANGE))]
6457 return aarch64_output_probe_stack_range (operands[0], operands[2]);
6459 [(set_attr "length" "32")]
6462 ;; Named pattern for expanding thread pointer reference.
6463 (define_expand "get_thread_pointerdi"
6464 [(match_operand:DI 0 "register_operand" "=r")]
6467 rtx tmp = aarch64_load_tp (operands[0]);
6468 if (tmp != operands[0])
6469 emit_move_insn (operands[0], tmp);
6473 ;; Named patterns for stack smashing protection.
6474 (define_expand "stack_protect_set"
6475 [(match_operand 0 "memory_operand")
6476 (match_operand 1 "memory_operand")]
6479 machine_mode mode = GET_MODE (operands[0]);
6481 emit_insn ((mode == DImode
6482 ? gen_stack_protect_set_di
6483 : gen_stack_protect_set_si) (operands[0], operands[1]));
6487 (define_insn "stack_protect_set_<mode>"
6488 [(set (match_operand:PTR 0 "memory_operand" "=m")
6489 (unspec:PTR [(match_operand:PTR 1 "memory_operand" "m")]
6491 (set (match_scratch:PTR 2 "=&r") (const_int 0))]
6493 "ldr\\t%<w>2, %1\;str\\t%<w>2, %0\;mov\t%<w>2,0"
6494 [(set_attr "length" "12")
6495 (set_attr "type" "multiple")])
6497 (define_expand "stack_protect_test"
6498 [(match_operand 0 "memory_operand")
6499 (match_operand 1 "memory_operand")
6504 machine_mode mode = GET_MODE (operands[0]);
6506 result = gen_reg_rtx(mode);
6508 emit_insn ((mode == DImode
6509 ? gen_stack_protect_test_di
6510 : gen_stack_protect_test_si) (result,
6515 emit_jump_insn (gen_cbranchdi4 (gen_rtx_EQ (VOIDmode, result, const0_rtx),
6516 result, const0_rtx, operands[2]));
6518 emit_jump_insn (gen_cbranchsi4 (gen_rtx_EQ (VOIDmode, result, const0_rtx),
6519 result, const0_rtx, operands[2]));
6523 (define_insn "stack_protect_test_<mode>"
6524 [(set (match_operand:PTR 0 "register_operand" "=r")
6525 (unspec:PTR [(match_operand:PTR 1 "memory_operand" "m")
6526 (match_operand:PTR 2 "memory_operand" "m")]
6528 (clobber (match_scratch:PTR 3 "=&r"))]
6530 "ldr\t%<w>3, %1\;ldr\t%<w>0, %2\;eor\t%<w>0, %<w>3, %<w>0"
6531 [(set_attr "length" "12")
6532 (set_attr "type" "multiple")])
6534 ;; Write Floating-point Control Register.
6535 (define_insn "set_fpcr"
6536 [(unspec_volatile [(match_operand:SI 0 "register_operand" "r")] UNSPECV_SET_FPCR)]
6539 [(set_attr "type" "mrs")])
6541 ;; Read Floating-point Control Register.
6542 (define_insn "get_fpcr"
6543 [(set (match_operand:SI 0 "register_operand" "=r")
6544 (unspec_volatile:SI [(const_int 0)] UNSPECV_GET_FPCR))]
6547 [(set_attr "type" "mrs")])
6549 ;; Write Floating-point Status Register.
6550 (define_insn "set_fpsr"
6551 [(unspec_volatile [(match_operand:SI 0 "register_operand" "r")] UNSPECV_SET_FPSR)]
6554 [(set_attr "type" "mrs")])
6556 ;; Read Floating-point Status Register.
6557 (define_insn "get_fpsr"
6558 [(set (match_operand:SI 0 "register_operand" "=r")
6559 (unspec_volatile:SI [(const_int 0)] UNSPECV_GET_FPSR))]
6562 [(set_attr "type" "mrs")])
6565 ;; Define the subtract-one-and-jump insns so loop.c
6566 ;; knows what to generate.
6567 (define_expand "doloop_end"
6568 [(use (match_operand 0 "" "")) ; loop pseudo
6569 (use (match_operand 1 "" ""))] ; label
6570 "optimize > 0 && flag_modulo_sched"
6579 /* Currently SMS relies on the do-loop pattern to recognize loops
6580 where (1) the control part consists of all insns defining and/or
6581 using a certain 'count' register and (2) the loop count can be
6582 adjusted by modifying this register prior to the loop.
6583 ??? The possible introduction of a new block to initialize the
6584 new IV can potentially affect branch optimizations. */
6586 if (GET_MODE (operands[0]) != DImode)
6590 insn = emit_insn (gen_adddi3_compare0 (s0, s0, GEN_INT (-1)));
6592 cmp = XVECEXP (PATTERN (insn), 0, 0);
6593 cc_reg = SET_DEST (cmp);
6594 bcomp = gen_rtx_NE (VOIDmode, cc_reg, const0_rtx);
6595 loc_ref = gen_rtx_LABEL_REF (VOIDmode, operands [1]);
6596 emit_jump_insn (gen_rtx_SET (pc_rtx,
6597 gen_rtx_IF_THEN_ELSE (VOIDmode, bcomp,
6602 ;; Track speculation through conditional branches. We assume that
6603 ;; SPECULATION_TRACKER_REGNUM is reserved for this purpose when necessary.
6604 (define_insn "speculation_tracker"
6605 [(set (reg:DI SPECULATION_TRACKER_REGNUM)
6606 (unspec [(reg:DI SPECULATION_TRACKER_REGNUM) (match_operand 0)]
6607 UNSPEC_SPECULATION_TRACKER))]
6610 operands[1] = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
6611 output_asm_insn ("csel\\t%1, %1, xzr, %m0", operands);
6614 [(set_attr "type" "csel")]
6617 ;; Helper for aarch64.c code.
6618 (define_expand "set_clobber_cc"
6619 [(parallel [(set (match_operand 0)
6621 (clobber (reg:CC CC_REGNUM))])])
6623 ;; Hard speculation barrier.
6624 (define_insn "speculation_barrier"
6625 [(unspec_volatile [(const_int 0)] UNSPECV_SPECULATION_BARRIER)]
6628 [(set_attr "length" "8")
6629 (set_attr "type" "block")
6630 (set_attr "speculation_barrier" "true")]
6633 ;; Support for __builtin_speculation_safe_value when we have speculation
6634 ;; tracking enabled. Use the speculation tracker to decide whether to
6635 ;; copy operand 1 to the target, or to copy the fail value (operand 2).
6636 (define_expand "despeculate_copy<ALLI_TI:mode>"
6637 [(set (match_operand:ALLI_TI 0 "register_operand" "=r")
6638 (unspec_volatile:ALLI_TI
6639 [(match_operand:ALLI_TI 1 "register_operand" "r")
6640 (match_operand:ALLI_TI 2 "aarch64_reg_or_zero" "rZ")
6641 (use (reg:DI SPECULATION_TRACKER_REGNUM))
6642 (clobber (reg:CC CC_REGNUM))] UNSPECV_SPECULATION_BARRIER))]
6646 if (operands[2] == const0_rtx)
6649 if (<MODE>mode == TImode)
6650 tracker = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
6652 tracker = gen_rtx_REG (<MODE>mode, SPECULATION_TRACKER_REGNUM);
6654 emit_insn (gen_despeculate_simple<mode> (operands[0], operands[1],
6662 ;; Patterns to match despeculate_copy<mode>. Note that "hint 0x14" is the
6663 ;; encoding for CSDB, but will work in older versions of the assembler.
6664 (define_insn "*despeculate_copy<ALLI:mode>_insn"
6665 [(set (match_operand:ALLI 0 "register_operand" "=r")
6666 (unspec_volatile:ALLI
6667 [(match_operand:ALLI 1 "register_operand" "r")
6668 (match_operand:ALLI 2 "aarch64_reg_or_zero" "rZ")
6669 (use (reg:DI SPECULATION_TRACKER_REGNUM))
6670 (clobber (reg:CC CC_REGNUM))] UNSPECV_SPECULATION_BARRIER))]
6673 operands[3] = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
6674 output_asm_insn ("cmp\\t%3, #0\;csel\\t%<w>0, %<w>1, %<w>2, ne\;hint\t0x14 // csdb",
6678 [(set_attr "length" "12")
6679 (set_attr "type" "block")
6680 (set_attr "speculation_barrier" "true")]
6683 ;; Pattern to match despeculate_copyti
6684 (define_insn "*despeculate_copyti_insn"
6685 [(set (match_operand:TI 0 "register_operand" "=r")
6687 [(match_operand:TI 1 "register_operand" "r")
6688 (match_operand:TI 2 "aarch64_reg_or_zero" "rZ")
6689 (use (reg:DI SPECULATION_TRACKER_REGNUM))
6690 (clobber (reg:CC CC_REGNUM))] UNSPECV_SPECULATION_BARRIER))]
6693 operands[3] = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM);
6695 ("cmp\\t%3, #0\;csel\\t%0, %1, %2, ne\;csel\\t%H0, %H1, %H2, ne\;hint\t0x14 // csdb",
6699 [(set_attr "length" "16")
6700 (set_attr "type" "block")
6701 (set_attr "speculation_barrier" "true")]
6704 (define_insn "despeculate_simple<ALLI:mode>"
6705 [(set (match_operand:ALLI 0 "register_operand" "=r")
6706 (unspec_volatile:ALLI
6707 [(match_operand:ALLI 1 "register_operand" "r")
6708 (use (match_operand:ALLI 2 "register_operand" ""))]
6709 UNSPECV_SPECULATION_BARRIER))]
6711 "and\\t%<w>0, %<w>1, %<w>2\;hint\t0x14 // csdb"
6712 [(set_attr "type" "block")
6713 (set_attr "length" "8")
6714 (set_attr "speculation_barrier" "true")]
6717 (define_insn "despeculate_simpleti"
6718 [(set (match_operand:TI 0 "register_operand" "=r")
6720 [(match_operand:TI 1 "register_operand" "r")
6721 (use (match_operand:DI 2 "register_operand" ""))]
6722 UNSPECV_SPECULATION_BARRIER))]
6724 "and\\t%0, %1, %2\;and\\t%H0, %H1, %2\;hint\t0x14 // csdb"
6725 [(set_attr "type" "block")
6726 (set_attr "length" "12")
6727 (set_attr "speculation_barrier" "true")]
6731 (include "aarch64-simd.md")
6733 ;; Atomic Operations
6734 (include "atomics.md")
6736 ;; ldp/stp peephole patterns
6737 (include "aarch64-ldpstp.md")
6740 (include "aarch64-sve.md")