1 ;; Machine description for AArch64 architecture.
2 ;; Copyright (C) 2009-2018 Free Software Foundation, Inc.
3 ;; Contributed by ARM Ltd.
5 ;; This file is part of GCC.
7 ;; GCC is free software; you can redistribute it and/or modify it
8 ;; under the terms of the GNU General Public License as published by
9 ;; the Free Software Foundation; either version 3, or (at your option)
12 ;; GCC is distributed in the hope that it will be useful, but
13 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
14 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 ;; General Public License for more details.
17 ;; You should have received a copy of the GNU General Public License
18 ;; along with GCC; see the file COPYING3. If not see
19 ;; <http://www.gnu.org/licenses/>.
62 (LAST_SAVED_REGNUM 63)
66 ;; Defined only to make the DWARF description simpler.
74 (define_c_enum "unspec" [
100 UNSPEC_GOTSMALLPIC28K
174 (define_c_enum "unspecv" [
175 UNSPECV_EH_RETURN ; Represent EH_RETURN
176 UNSPECV_GET_FPCR ; Represent fetch of FPCR content.
177 UNSPECV_SET_FPCR ; Represent assign of FPCR content.
178 UNSPECV_GET_FPSR ; Represent fetch of FPSR content.
179 UNSPECV_SET_FPSR ; Represent assign of FPSR content.
180 UNSPECV_BLOCKAGE ; Represent a blockage
181 UNSPECV_PROBE_STACK_RANGE ; Represent stack range probing.
185 ;; If further include files are added the defintion of MD_INCLUDES
188 (include "constraints.md")
189 (include "predicates.md")
190 (include "iterators.md")
192 ;; -------------------------------------------------------------------
193 ;; Instruction types and attributes
194 ;; -------------------------------------------------------------------
196 ; The "type" attribute is included here from AArch32 backend to be able
197 ; to share pipeline descriptions.
198 (include "../arm/types.md")
200 ;; It is important to set the fp or simd attributes to yes when a pattern
201 ;; alternative uses the FP or SIMD register files, usually signified by use of
202 ;; the 'w' constraint. This will ensure that the alternative will be
203 ;; disabled when compiling with -mgeneral-regs-only or with the +nofp/+nosimd
204 ;; architecture extensions. If all the alternatives in a pattern use the
205 ;; FP or SIMD registers then the pattern predicate should include TARGET_FLOAT
208 ;; Attribute that specifies whether or not the instruction touches fp
209 ;; registers. When this is set to yes for an alternative, that alternative
210 ;; will be disabled when !TARGET_FLOAT.
211 (define_attr "fp" "no,yes" (const_string "no"))
213 ;; Attribute that specifies whether or not the instruction touches half
214 ;; precision fp registers. When this is set to yes for an alternative,
215 ;; that alternative will be disabled when !TARGET_FP_F16INST.
216 (define_attr "fp16" "no,yes" (const_string "no"))
218 ;; Attribute that specifies whether or not the instruction touches simd
219 ;; registers. When this is set to yes for an alternative, that alternative
220 ;; will be disabled when !TARGET_SIMD.
221 (define_attr "simd" "no,yes" (const_string "no"))
223 ;; Attribute that specifies whether or not the instruction uses SVE.
224 ;; When this is set to yes for an alternative, that alternative
225 ;; will be disabled when !TARGET_SVE.
226 (define_attr "sve" "no,yes" (const_string "no"))
228 (define_attr "length" ""
231 ;; Attribute that controls whether an alternative is enabled or not.
232 ;; Currently it is only used to disable alternatives which touch fp or simd
233 ;; registers when -mgeneral-regs-only is specified.
234 (define_attr "enabled" "no,yes"
236 (and (eq_attr "fp" "yes")
237 (eq (symbol_ref "TARGET_FLOAT") (const_int 0)))
238 (and (eq_attr "simd" "yes")
239 (eq (symbol_ref "TARGET_SIMD") (const_int 0)))
240 (and (eq_attr "fp16" "yes")
241 (eq (symbol_ref "TARGET_FP_F16INST") (const_int 0)))
242 (and (eq_attr "sve" "yes")
243 (eq (symbol_ref "TARGET_SVE") (const_int 0))))
245 ] (const_string "yes")))
247 ;; Attribute that specifies whether we are dealing with a branch to a
248 ;; label that is far away, i.e. further away than the maximum/minimum
249 ;; representable in a signed 21-bits number.
252 (define_attr "far_branch" "" (const_int 0))
254 ;; Strictly for compatibility with AArch32 in pipeline models, since AArch64 has
255 ;; no predicated insns.
256 (define_attr "predicated" "yes,no" (const_string "no"))
258 ;; -------------------------------------------------------------------
259 ;; Pipeline descriptions and scheduling
260 ;; -------------------------------------------------------------------
263 (include "aarch64-tune.md")
266 (include "../arm/cortex-a53.md")
267 (include "../arm/cortex-a57.md")
268 (include "../arm/exynos-m1.md")
269 (include "falkor.md")
270 (include "thunderx.md")
271 (include "../arm/xgene1.md")
272 (include "thunderx2t99.md")
274 ;; -------------------------------------------------------------------
275 ;; Jumps and other miscellaneous insns
276 ;; -------------------------------------------------------------------
278 (define_insn "indirect_jump"
279 [(set (pc) (match_operand:DI 0 "register_operand" "r"))]
282 [(set_attr "type" "branch")]
286 [(set (pc) (label_ref (match_operand 0 "" "")))]
289 [(set_attr "type" "branch")]
292 (define_expand "cbranch<mode>4"
293 [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
294 [(match_operand:GPI 1 "register_operand" "")
295 (match_operand:GPI 2 "aarch64_plus_operand" "")])
296 (label_ref (match_operand 3 "" ""))
300 operands[1] = aarch64_gen_compare_reg (GET_CODE (operands[0]), operands[1],
302 operands[2] = const0_rtx;
306 (define_expand "cbranch<mode>4"
307 [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
308 [(match_operand:GPF 1 "register_operand" "")
309 (match_operand:GPF 2 "aarch64_fp_compare_operand" "")])
310 (label_ref (match_operand 3 "" ""))
314 operands[1] = aarch64_gen_compare_reg (GET_CODE (operands[0]), operands[1],
316 operands[2] = const0_rtx;
320 (define_expand "cbranchcc4"
321 [(set (pc) (if_then_else
322 (match_operator 0 "aarch64_comparison_operator"
323 [(match_operand 1 "cc_register" "")
324 (match_operand 2 "const0_operand")])
325 (label_ref (match_operand 3 "" ""))
330 (define_insn "ccmp<mode>"
331 [(set (match_operand:CC 1 "cc_register" "")
333 (match_operator 4 "aarch64_comparison_operator"
334 [(match_operand 0 "cc_register" "")
337 (match_operand:GPI 2 "register_operand" "r,r,r")
338 (match_operand:GPI 3 "aarch64_ccmp_operand" "r,Uss,Usn"))
339 (unspec:CC [(match_operand 5 "immediate_operand")] UNSPEC_NZCV)))]
342 ccmp\\t%<w>2, %<w>3, %k5, %m4
343 ccmp\\t%<w>2, %3, %k5, %m4
344 ccmn\\t%<w>2, #%n3, %k5, %m4"
345 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
348 (define_insn "fccmp<mode>"
349 [(set (match_operand:CCFP 1 "cc_register" "")
351 (match_operator 4 "aarch64_comparison_operator"
352 [(match_operand 0 "cc_register" "")
355 (match_operand:GPF 2 "register_operand" "w")
356 (match_operand:GPF 3 "register_operand" "w"))
357 (unspec:CCFP [(match_operand 5 "immediate_operand")] UNSPEC_NZCV)))]
359 "fccmp\\t%<s>2, %<s>3, %k5, %m4"
360 [(set_attr "type" "fccmp<s>")]
363 (define_insn "fccmpe<mode>"
364 [(set (match_operand:CCFPE 1 "cc_register" "")
366 (match_operator 4 "aarch64_comparison_operator"
367 [(match_operand 0 "cc_register" "")
370 (match_operand:GPF 2 "register_operand" "w")
371 (match_operand:GPF 3 "register_operand" "w"))
372 (unspec:CCFPE [(match_operand 5 "immediate_operand")] UNSPEC_NZCV)))]
374 "fccmpe\\t%<s>2, %<s>3, %k5, %m4"
375 [(set_attr "type" "fccmp<s>")]
378 ;; Expansion of signed mod by a power of 2 using CSNEG.
379 ;; For x0 % n where n is a power of 2 produce:
381 ;; and x0, x0, #(n - 1)
382 ;; and x1, x1, #(n - 1)
383 ;; csneg x0, x0, x1, mi
385 (define_expand "mod<mode>3"
386 [(match_operand:GPI 0 "register_operand" "")
387 (match_operand:GPI 1 "register_operand" "")
388 (match_operand:GPI 2 "const_int_operand" "")]
391 HOST_WIDE_INT val = INTVAL (operands[2]);
394 || exact_log2 (val) <= 0
395 || !aarch64_bitmask_imm (val - 1, <MODE>mode))
398 rtx mask = GEN_INT (val - 1);
400 /* In the special case of x0 % 2 we can do the even shorter:
406 rtx masked = gen_reg_rtx (<MODE>mode);
407 rtx ccreg = aarch64_gen_compare_reg (LT, operands[1], const0_rtx);
408 emit_insn (gen_and<mode>3 (masked, operands[1], mask));
409 rtx x = gen_rtx_LT (VOIDmode, ccreg, const0_rtx);
410 emit_insn (gen_csneg3<mode>_insn (operands[0], x, masked, masked));
414 rtx neg_op = gen_reg_rtx (<MODE>mode);
415 rtx_insn *insn = emit_insn (gen_neg<mode>2_compare0 (neg_op, operands[1]));
417 /* Extract the condition register and mode. */
418 rtx cmp = XVECEXP (PATTERN (insn), 0, 0);
419 rtx cc_reg = SET_DEST (cmp);
420 rtx cond = gen_rtx_GE (VOIDmode, cc_reg, const0_rtx);
422 rtx masked_pos = gen_reg_rtx (<MODE>mode);
423 emit_insn (gen_and<mode>3 (masked_pos, operands[1], mask));
425 rtx masked_neg = gen_reg_rtx (<MODE>mode);
426 emit_insn (gen_and<mode>3 (masked_neg, neg_op, mask));
428 emit_insn (gen_csneg3<mode>_insn (operands[0], cond,
429 masked_neg, masked_pos));
434 (define_insn "condjump"
435 [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
436 [(match_operand 1 "cc_register" "") (const_int 0)])
437 (label_ref (match_operand 2 "" ""))
441 if (get_attr_length (insn) == 8)
442 return aarch64_gen_far_branch (operands, 2, "Lbcond", "b%M0\\t");
446 [(set_attr "type" "branch")
448 (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
449 (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
452 (set (attr "far_branch")
453 (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
454 (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
459 ;; For a 24-bit immediate CST we can optimize the compare for equality
460 ;; and branch sequence from:
462 ;; movk x0, #imm2, lsl 16 /* x0 contains CST. */
466 ;; sub x0, x1, #(CST & 0xfff000)
467 ;; subs x0, x0, #(CST & 0x000fff)
469 (define_insn_and_split "*compare_condjump<mode>"
470 [(set (pc) (if_then_else (EQL
471 (match_operand:GPI 0 "register_operand" "r")
472 (match_operand:GPI 1 "aarch64_imm24" "n"))
473 (label_ref:P (match_operand 2 "" ""))
475 "!aarch64_move_imm (INTVAL (operands[1]), <MODE>mode)
476 && !aarch64_plus_operand (operands[1], <MODE>mode)
477 && !reload_completed"
482 HOST_WIDE_INT lo_imm = UINTVAL (operands[1]) & 0xfff;
483 HOST_WIDE_INT hi_imm = UINTVAL (operands[1]) & 0xfff000;
484 rtx tmp = gen_reg_rtx (<MODE>mode);
485 emit_insn (gen_add<mode>3 (tmp, operands[0], GEN_INT (-hi_imm)));
486 emit_insn (gen_add<mode>3_compare0 (tmp, tmp, GEN_INT (-lo_imm)));
487 rtx cc_reg = gen_rtx_REG (CC_NZmode, CC_REGNUM);
488 rtx cmp_rtx = gen_rtx_fmt_ee (<EQL:CMP>, <MODE>mode, cc_reg, const0_rtx);
489 emit_jump_insn (gen_condjump (cmp_rtx, cc_reg, operands[2]));
494 (define_expand "casesi"
495 [(match_operand:SI 0 "register_operand" "") ; Index
496 (match_operand:SI 1 "const_int_operand" "") ; Lower bound
497 (match_operand:SI 2 "const_int_operand" "") ; Total range
498 (match_operand:DI 3 "" "") ; Table label
499 (match_operand:DI 4 "" "")] ; Out of range label
502 if (operands[1] != const0_rtx)
504 rtx reg = gen_reg_rtx (SImode);
506 /* Canonical RTL says that if you have:
510 then this should be emitted as:
514 The use of trunc_int_for_mode ensures that the resulting
515 constant can be represented in SImode, this is important
516 for the corner case where operand[1] is INT_MIN. */
518 operands[1] = GEN_INT (trunc_int_for_mode (-INTVAL (operands[1]), SImode));
520 if (!(*insn_data[CODE_FOR_addsi3].operand[2].predicate)
521 (operands[1], SImode))
522 operands[1] = force_reg (SImode, operands[1]);
523 emit_insn (gen_addsi3 (reg, operands[0], operands[1]));
527 if (!aarch64_plus_operand (operands[2], SImode))
528 operands[2] = force_reg (SImode, operands[2]);
529 emit_jump_insn (gen_cbranchsi4 (gen_rtx_GTU (SImode, const0_rtx,
531 operands[0], operands[2], operands[4]));
533 operands[2] = force_reg (DImode, gen_rtx_LABEL_REF (DImode, operands[3]));
534 emit_jump_insn (gen_casesi_dispatch (operands[2], operands[0],
540 (define_insn "casesi_dispatch"
543 (mem:DI (unspec [(match_operand:DI 0 "register_operand" "r")
544 (match_operand:SI 1 "register_operand" "r")]
546 (clobber (reg:CC CC_REGNUM))
547 (clobber (match_scratch:DI 3 "=r"))
548 (clobber (match_scratch:DI 4 "=r"))
549 (use (label_ref (match_operand 2 "" "")))])]
552 return aarch64_output_casesi (operands);
554 [(set_attr "length" "16")
555 (set_attr "type" "branch")]
559 [(unspec[(const_int 0)] UNSPEC_NOP)]
562 [(set_attr "type" "no_insn")]
565 (define_insn "prefetch"
566 [(prefetch (match_operand:DI 0 "aarch64_prefetch_operand" "Dp")
567 (match_operand:QI 1 "const_int_operand" "")
568 (match_operand:QI 2 "const_int_operand" ""))]
571 const char * pftype[2][4] =
573 {"prfm\\tPLDL1STRM, %0",
574 "prfm\\tPLDL3KEEP, %0",
575 "prfm\\tPLDL2KEEP, %0",
576 "prfm\\tPLDL1KEEP, %0"},
577 {"prfm\\tPSTL1STRM, %0",
578 "prfm\\tPSTL3KEEP, %0",
579 "prfm\\tPSTL2KEEP, %0",
580 "prfm\\tPSTL1KEEP, %0"},
583 int locality = INTVAL (operands[2]);
585 gcc_assert (IN_RANGE (locality, 0, 3));
587 /* PRFM accepts the same addresses as a 64-bit LDR so wrap
588 the address into a DImode MEM so that aarch64_print_operand knows
590 operands[0] = gen_rtx_MEM (DImode, operands[0]);
591 return pftype[INTVAL(operands[1])][locality];
593 [(set_attr "type" "load_4")]
597 [(trap_if (const_int 1) (const_int 8))]
600 [(set_attr "type" "trap")])
602 (define_expand "prologue"
603 [(clobber (const_int 0))]
606 aarch64_expand_prologue ();
611 (define_expand "epilogue"
612 [(clobber (const_int 0))]
615 aarch64_expand_epilogue (false);
620 (define_expand "sibcall_epilogue"
621 [(clobber (const_int 0))]
624 aarch64_expand_epilogue (true);
629 (define_insn "*do_return"
633 if (aarch64_return_address_signing_enabled ()
635 && !crtl->calls_eh_return)
640 [(set_attr "type" "branch")]
643 (define_expand "return"
645 "aarch64_use_return_insn_p ()"
649 (define_insn "simple_return"
653 [(set_attr "type" "branch")]
656 (define_insn "*cb<optab><mode>1"
657 [(set (pc) (if_then_else (EQL (match_operand:GPI 0 "register_operand" "r")
659 (label_ref (match_operand 1 "" ""))
663 if (get_attr_length (insn) == 8)
664 return aarch64_gen_far_branch (operands, 1, "Lcb", "<inv_cb>\\t%<w>0, ");
666 return "<cbz>\\t%<w>0, %l1";
668 [(set_attr "type" "branch")
670 (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -1048576))
671 (lt (minus (match_dup 1) (pc)) (const_int 1048572)))
674 (set (attr "far_branch")
675 (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
676 (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
681 (define_insn "*tb<optab><mode>1"
682 [(set (pc) (if_then_else
683 (EQL (zero_extract:DI (match_operand:GPI 0 "register_operand" "r")
686 "aarch64_simd_shift_imm_<mode>" "n"))
688 (label_ref (match_operand 2 "" ""))
690 (clobber (reg:CC CC_REGNUM))]
693 if (get_attr_length (insn) == 8)
695 if (get_attr_far_branch (insn) == 1)
696 return aarch64_gen_far_branch (operands, 2, "Ltb",
697 "<inv_tb>\\t%<w>0, %1, ");
700 operands[1] = GEN_INT (HOST_WIDE_INT_1U << UINTVAL (operands[1]));
701 return "tst\t%<w>0, %1\;<bcond>\t%l2";
705 return "<tbz>\t%<w>0, %1, %l2";
707 [(set_attr "type" "branch")
709 (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -32768))
710 (lt (minus (match_dup 2) (pc)) (const_int 32764)))
713 (set (attr "far_branch")
714 (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
715 (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
721 (define_insn "*cb<optab><mode>1"
722 [(set (pc) (if_then_else (LTGE (match_operand:ALLI 0 "register_operand" "r")
724 (label_ref (match_operand 1 "" ""))
726 (clobber (reg:CC CC_REGNUM))]
729 if (get_attr_length (insn) == 8)
731 if (get_attr_far_branch (insn) == 1)
732 return aarch64_gen_far_branch (operands, 1, "Ltb",
733 "<inv_tb>\\t%<w>0, <sizem1>, ");
737 uint64_t val = ((uint64_t) 1)
738 << (GET_MODE_SIZE (<MODE>mode) * BITS_PER_UNIT - 1);
739 sprintf (buf, "tst\t%%<w>0, %" PRId64, val);
740 output_asm_insn (buf, operands);
741 return "<bcond>\t%l1";
745 return "<tbz>\t%<w>0, <sizem1>, %l1";
747 [(set_attr "type" "branch")
749 (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -32768))
750 (lt (minus (match_dup 1) (pc)) (const_int 32764)))
753 (set (attr "far_branch")
754 (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -1048576))
755 (lt (minus (match_dup 1) (pc)) (const_int 1048572)))
760 ;; -------------------------------------------------------------------
761 ;; Subroutine calls and sibcalls
762 ;; -------------------------------------------------------------------
764 (define_expand "call"
765 [(parallel [(call (match_operand 0 "memory_operand" "")
766 (match_operand 1 "general_operand" ""))
767 (use (match_operand 2 "" ""))
768 (clobber (reg:DI LR_REGNUM))])]
772 aarch64_expand_call (NULL_RTX, operands[0], false);
777 (define_insn "*call_insn"
778 [(call (mem:DI (match_operand:DI 0 "aarch64_call_insn_operand" "r, Usf"))
779 (match_operand 1 "" ""))
780 (clobber (reg:DI LR_REGNUM))]
785 [(set_attr "type" "call, call")]
788 (define_expand "call_value"
789 [(parallel [(set (match_operand 0 "" "")
790 (call (match_operand 1 "memory_operand" "")
791 (match_operand 2 "general_operand" "")))
792 (use (match_operand 3 "" ""))
793 (clobber (reg:DI LR_REGNUM))])]
797 aarch64_expand_call (operands[0], operands[1], false);
802 (define_insn "*call_value_insn"
803 [(set (match_operand 0 "" "")
804 (call (mem:DI (match_operand:DI 1 "aarch64_call_insn_operand" "r, Usf"))
805 (match_operand 2 "" "")))
806 (clobber (reg:DI LR_REGNUM))]
811 [(set_attr "type" "call, call")]
814 (define_expand "sibcall"
815 [(parallel [(call (match_operand 0 "memory_operand" "")
816 (match_operand 1 "general_operand" ""))
818 (use (match_operand 2 "" ""))])]
821 aarch64_expand_call (NULL_RTX, operands[0], true);
826 (define_expand "sibcall_value"
827 [(parallel [(set (match_operand 0 "" "")
828 (call (match_operand 1 "memory_operand" "")
829 (match_operand 2 "general_operand" "")))
831 (use (match_operand 3 "" ""))])]
834 aarch64_expand_call (operands[0], operands[1], true);
839 (define_insn "*sibcall_insn"
840 [(call (mem:DI (match_operand:DI 0 "aarch64_call_insn_operand" "Ucs, Usf"))
841 (match_operand 1 "" ""))
843 "SIBLING_CALL_P (insn)"
847 [(set_attr "type" "branch, branch")]
850 (define_insn "*sibcall_value_insn"
851 [(set (match_operand 0 "" "")
853 (match_operand:DI 1 "aarch64_call_insn_operand" "Ucs, Usf"))
854 (match_operand 2 "" "")))
856 "SIBLING_CALL_P (insn)"
860 [(set_attr "type" "branch, branch")]
863 ;; Call subroutine returning any type.
865 (define_expand "untyped_call"
866 [(parallel [(call (match_operand 0 "")
869 (match_operand 2 "")])]
874 emit_call_insn (gen_call (operands[0], const0_rtx, NULL));
876 for (i = 0; i < XVECLEN (operands[2], 0); i++)
878 rtx set = XVECEXP (operands[2], 0, i);
879 emit_move_insn (SET_DEST (set), SET_SRC (set));
882 /* The optimizer does not know that the call sets the function value
883 registers we stored in the result block. We avoid problems by
884 claiming that all hard registers are used and clobbered at this
886 emit_insn (gen_blockage ());
890 ;; -------------------------------------------------------------------
892 ;; -------------------------------------------------------------------
894 (define_expand "mov<mode>"
895 [(set (match_operand:SHORT 0 "nonimmediate_operand" "")
896 (match_operand:SHORT 1 "general_operand" ""))]
899 if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
900 operands[1] = force_reg (<MODE>mode, operands[1]);
902 if (GET_CODE (operands[1]) == CONST_POLY_INT)
904 aarch64_expand_mov_immediate (operands[0], operands[1]);
910 (define_insn "*mov<mode>_aarch64"
911 [(set (match_operand:SHORT 0 "nonimmediate_operand" "=r,r, *w,r ,r,*w, m, m, r,*w,*w")
912 (match_operand:SHORT 1 "aarch64_mov_operand" " r,M,D<hq>,Usv,m, m,rZ,*w,*w, r,*w"))]
913 "(register_operand (operands[0], <MODE>mode)
914 || aarch64_reg_or_zero (operands[1], <MODE>mode))"
916 switch (which_alternative)
919 return "mov\t%w0, %w1";
921 return "mov\t%w0, %1";
923 return aarch64_output_scalar_simd_mov_immediate (operands[1],
926 return aarch64_output_sve_cnt_immediate (\"cnt\", \"%x0\", operands[1]);
928 return "ldr<size>\t%w0, %1";
930 return "ldr\t%<size>0, %1";
932 return "str<size>\t%w1, %0";
934 return "str\t%<size>1, %0";
936 return "umov\t%w0, %1.<v>[0]";
938 return "dup\t%0.<Vallxd>, %w1";
940 return "dup\t%<Vetype>0, %1.<v>[0]";
945 ;; The "mov_imm" type for CNT is just a placeholder.
946 [(set_attr "type" "mov_reg,mov_imm,neon_move,mov_imm,load_4,load_4,store_4,
947 store_4,neon_to_gp<q>,neon_from_gp<q>,neon_dup")
948 (set_attr "simd" "*,*,yes,*,*,*,*,*,yes,yes,yes")
949 (set_attr "sve" "*,*,*,yes,*,*,*,*,*,*,*")]
952 (define_expand "mov<mode>"
953 [(set (match_operand:GPI 0 "nonimmediate_operand" "")
954 (match_operand:GPI 1 "general_operand" ""))]
957 if (MEM_P (operands[0]) && CONST_INT_P (operands[1])
958 && <MODE>mode == DImode
959 && aarch64_split_dimode_const_store (operands[0], operands[1]))
962 if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
963 operands[1] = force_reg (<MODE>mode, operands[1]);
965 /* FIXME: RR we still need to fix up what we are doing with
966 symbol_refs and other types of constants. */
967 if (CONSTANT_P (operands[1])
968 && !CONST_INT_P (operands[1]))
970 aarch64_expand_mov_immediate (operands[0], operands[1]);
976 (define_insn_and_split "*movsi_aarch64"
977 [(set (match_operand:SI 0 "nonimmediate_operand" "=r,k,r,r,r,r, r,w, m, m, r, r, w,r,w, w")
978 (match_operand:SI 1 "aarch64_mov_operand" " r,r,k,M,n,Usv,m,m,rZ,*w,Usa,Ush,rZ,w,w,Ds"))]
979 "(register_operand (operands[0], SImode)
980 || aarch64_reg_or_zero (operands[1], SImode))"
987 * return aarch64_output_sve_cnt_immediate (\"cnt\", \"%x0\", operands[1]);
997 * return aarch64_output_scalar_simd_mov_immediate (operands[1], SImode);"
998 "CONST_INT_P (operands[1]) && !aarch64_move_imm (INTVAL (operands[1]), SImode)
999 && REG_P (operands[0]) && GP_REGNUM_P (REGNO (operands[0]))"
1002 aarch64_expand_mov_immediate (operands[0], operands[1]);
1005 ;; The "mov_imm" type for CNT is just a placeholder.
1006 [(set_attr "type" "mov_reg,mov_reg,mov_reg,mov_imm,mov_imm,mov_imm,load_4,
1007 load_4,store_4,store_4,adr,adr,f_mcr,f_mrc,fmov,neon_move")
1008 (set_attr "fp" "*,*,*,*,*,*,*,yes,*,yes,*,*,yes,yes,yes,*")
1009 (set_attr "simd" "*,*,*,*,*,*,*,*,*,*,*,*,*,*,*,yes")
1010 (set_attr "sve" "*,*,*,*,*,yes,*,*,*,*,*,*,*,*,*,*")]
1013 (define_insn_and_split "*movdi_aarch64"
1014 [(set (match_operand:DI 0 "nonimmediate_operand" "=r,k,r,r,r,r,r, r,w, m,m, r, r, w,r,w, w")
1015 (match_operand:DI 1 "aarch64_mov_operand" " r,r,k,N,M,n,Usv,m,m,rZ,w,Usa,Ush,rZ,w,w,Dd"))]
1016 "(register_operand (operands[0], DImode)
1017 || aarch64_reg_or_zero (operands[1], DImode))"
1025 * return aarch64_output_sve_cnt_immediate (\"cnt\", \"%x0\", operands[1]);
1035 * return aarch64_output_scalar_simd_mov_immediate (operands[1], DImode);"
1036 "(CONST_INT_P (operands[1]) && !aarch64_move_imm (INTVAL (operands[1]), DImode))
1037 && REG_P (operands[0]) && GP_REGNUM_P (REGNO (operands[0]))"
1040 aarch64_expand_mov_immediate (operands[0], operands[1]);
1043 ;; The "mov_imm" type for CNTD is just a placeholder.
1044 [(set_attr "type" "mov_reg,mov_reg,mov_reg,mov_imm,mov_imm,mov_imm,mov_imm,
1045 load_8,load_8,store_8,store_8,adr,adr,f_mcr,f_mrc,fmov,
1047 (set_attr "fp" "*,*,*,*,*,*,*,*,yes,*,yes,*,*,yes,yes,yes,*")
1048 (set_attr "simd" "*,*,*,*,*,*,*,*,*,*,*,*,*,*,*,*,yes")
1049 (set_attr "sve" "*,*,*,*,*,*,yes,*,*,*,*,*,*,*,*,*,*")]
1052 (define_insn "insv_imm<mode>"
1053 [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
1055 (match_operand:GPI 1 "const_int_operand" "n"))
1056 (match_operand:GPI 2 "const_int_operand" "n"))]
1057 "UINTVAL (operands[1]) < GET_MODE_BITSIZE (<MODE>mode)
1058 && UINTVAL (operands[1]) % 16 == 0"
1059 "movk\\t%<w>0, %X2, lsl %1"
1060 [(set_attr "type" "mov_imm")]
1063 (define_expand "movti"
1064 [(set (match_operand:TI 0 "nonimmediate_operand" "")
1065 (match_operand:TI 1 "general_operand" ""))]
1068 if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
1069 operands[1] = force_reg (TImode, operands[1]);
1071 if (GET_CODE (operands[1]) == CONST_POLY_INT)
1073 emit_move_insn (gen_lowpart (DImode, operands[0]),
1074 gen_lowpart (DImode, operands[1]));
1075 emit_move_insn (gen_highpart (DImode, operands[0]), const0_rtx);
1081 (define_insn "*movti_aarch64"
1082 [(set (match_operand:TI 0
1083 "nonimmediate_operand" "= r,w, r,w,r,m,m,w,m")
1085 "aarch64_movti_operand" " rUti,r, w,w,m,r,Z,m,w"))]
1086 "(register_operand (operands[0], TImode)
1087 || aarch64_reg_or_zero (operands[1], TImode))"
1092 mov\\t%0.16b, %1.16b
1098 [(set_attr "type" "multiple,f_mcr,f_mrc,neon_logic_q, \
1099 load_16,store_16,store_16,\
1101 (set_attr "length" "8,8,8,4,4,4,4,4,4")
1102 (set_attr "simd" "*,*,*,yes,*,*,*,*,*")
1103 (set_attr "fp" "*,*,*,*,*,*,*,yes,yes")]
1106 ;; Split a TImode register-register or register-immediate move into
1107 ;; its component DImode pieces, taking care to handle overlapping
1108 ;; source and dest registers.
1110 [(set (match_operand:TI 0 "register_operand" "")
1111 (match_operand:TI 1 "aarch64_reg_or_imm" ""))]
1112 "reload_completed && aarch64_split_128bit_move_p (operands[0], operands[1])"
1115 aarch64_split_128bit_move (operands[0], operands[1]);
1119 (define_expand "mov<mode>"
1120 [(set (match_operand:GPF_TF_F16 0 "nonimmediate_operand" "")
1121 (match_operand:GPF_TF_F16 1 "general_operand" ""))]
1126 aarch64_err_no_fpadvsimd (<MODE>mode, "code");
1130 if (GET_CODE (operands[0]) == MEM
1131 && ! (GET_CODE (operands[1]) == CONST_DOUBLE
1132 && aarch64_float_const_zero_rtx_p (operands[1])))
1133 operands[1] = force_reg (<MODE>mode, operands[1]);
1137 (define_insn "*movhf_aarch64"
1138 [(set (match_operand:HF 0 "nonimmediate_operand" "=w,w ,?r,w,w ,w ,w,m,r,m ,r")
1139 (match_operand:HF 1 "general_operand" "Y ,?rY, w,w,Ufc,Uvi,m,w,m,rY,r"))]
1140 "TARGET_FLOAT && (register_operand (operands[0], HFmode)
1141 || aarch64_reg_or_fp_zero (operands[1], HFmode))"
1146 mov\\t%0.h[0], %1.h[0]
1148 * return aarch64_output_scalar_simd_mov_immediate (operands[1], SImode);
1154 [(set_attr "type" "neon_move,f_mcr,neon_to_gp,neon_move,fconsts, \
1155 neon_move,f_loads,f_stores,load_4,store_4,mov_reg")
1156 (set_attr "simd" "yes,*,yes,yes,*,yes,*,*,*,*,*")
1157 (set_attr "fp16" "*,yes,*,*,yes,*,*,*,*,*,*")]
1160 (define_insn "*movsf_aarch64"
1161 [(set (match_operand:SF 0 "nonimmediate_operand" "=w,w ,?r,w,w ,w ,w,m,r,m ,r,r")
1162 (match_operand:SF 1 "general_operand" "Y ,?rY, w,w,Ufc,Uvi,m,w,m,rY,r,M"))]
1163 "TARGET_FLOAT && (register_operand (operands[0], SFmode)
1164 || aarch64_reg_or_fp_zero (operands[1], SFmode))"
1171 * return aarch64_output_scalar_simd_mov_immediate (operands[1], SImode);
1178 [(set_attr "type" "neon_move,f_mcr,f_mrc,fmov,fconsts,neon_move,\
1179 f_loads,f_stores,load_4,store_4,mov_reg,\
1181 (set_attr "simd" "yes,*,*,*,*,yes,*,*,*,*,*,*")]
1184 (define_insn "*movdf_aarch64"
1185 [(set (match_operand:DF 0 "nonimmediate_operand" "=w, w ,?r,w,w ,w ,w,m,r,m ,r,r")
1186 (match_operand:DF 1 "general_operand" "Y , ?rY, w,w,Ufc,Uvi,m,w,m,rY,r,N"))]
1187 "TARGET_FLOAT && (register_operand (operands[0], DFmode)
1188 || aarch64_reg_or_fp_zero (operands[1], DFmode))"
1195 * return aarch64_output_scalar_simd_mov_immediate (operands[1], DImode);
1202 [(set_attr "type" "neon_move,f_mcr,f_mrc,fmov,fconstd,neon_move,\
1203 f_loadd,f_stored,load_8,store_8,mov_reg,\
1205 (set_attr "simd" "yes,*,*,*,*,yes,*,*,*,*,*,*")]
1209 [(set (match_operand:GPF_HF 0 "nonimmediate_operand")
1210 (match_operand:GPF_HF 1 "general_operand"))]
1211 "can_create_pseudo_p ()
1212 && !aarch64_can_const_movi_rtx_p (operands[1], <MODE>mode)
1213 && !aarch64_float_const_representable_p (operands[1])
1214 && aarch64_float_const_rtx_p (operands[1])"
1217 unsigned HOST_WIDE_INT ival;
1218 if (!aarch64_reinterpret_float_as_int (operands[1], &ival))
1221 rtx tmp = gen_reg_rtx (<FCVT_TARGET>mode);
1222 emit_move_insn (tmp, gen_int_mode (ival, <FCVT_TARGET>mode));
1223 emit_move_insn (operands[0], gen_lowpart (<MODE>mode, tmp));
1228 (define_insn "*movtf_aarch64"
1229 [(set (match_operand:TF 0
1230 "nonimmediate_operand" "=w,?&r,w ,?r,w,?w,w,m,?r,m ,m")
1232 "general_operand" " w,?r, ?r,w ,Y,Y ,m,w,m ,?r,Y"))]
1233 "TARGET_FLOAT && (register_operand (operands[0], TFmode)
1234 || aarch64_reg_or_fp_zero (operands[1], TFmode))"
1236 mov\\t%0.16b, %1.16b
1247 [(set_attr "type" "logic_reg,multiple,f_mcr,f_mrc,neon_move_q,f_mcr,\
1248 f_loadd,f_stored,load_16,store_16,store_16")
1249 (set_attr "length" "4,8,8,8,4,4,4,4,4,4,4")
1250 (set_attr "simd" "yes,*,*,*,yes,*,*,*,*,*,*")]
1254 [(set (match_operand:TF 0 "register_operand" "")
1255 (match_operand:TF 1 "aarch64_reg_or_imm" ""))]
1256 "reload_completed && aarch64_split_128bit_move_p (operands[0], operands[1])"
1259 aarch64_split_128bit_move (operands[0], operands[1]);
1266 ;; 2 is size of move in bytes
1269 (define_expand "movmemdi"
1270 [(match_operand:BLK 0 "memory_operand")
1271 (match_operand:BLK 1 "memory_operand")
1272 (match_operand:DI 2 "immediate_operand")
1273 (match_operand:DI 3 "immediate_operand")]
1276 if (aarch64_expand_movmem (operands))
1282 ;; Operands 1 and 3 are tied together by the final condition; so we allow
1283 ;; fairly lax checking on the second memory operation.
1284 (define_insn "load_pairsi"
1285 [(set (match_operand:SI 0 "register_operand" "=r,*w")
1286 (match_operand:SI 1 "aarch64_mem_pair_operand" "Ump,Ump"))
1287 (set (match_operand:SI 2 "register_operand" "=r,*w")
1288 (match_operand:SI 3 "memory_operand" "m,m"))]
1289 "rtx_equal_p (XEXP (operands[3], 0),
1290 plus_constant (Pmode,
1291 XEXP (operands[1], 0),
1292 GET_MODE_SIZE (SImode)))"
1296 [(set_attr "type" "load_8,neon_load1_2reg")
1297 (set_attr "fp" "*,yes")]
1300 (define_insn "load_pairdi"
1301 [(set (match_operand:DI 0 "register_operand" "=r,*w")
1302 (match_operand:DI 1 "aarch64_mem_pair_operand" "Ump,Ump"))
1303 (set (match_operand:DI 2 "register_operand" "=r,*w")
1304 (match_operand:DI 3 "memory_operand" "m,m"))]
1305 "rtx_equal_p (XEXP (operands[3], 0),
1306 plus_constant (Pmode,
1307 XEXP (operands[1], 0),
1308 GET_MODE_SIZE (DImode)))"
1312 [(set_attr "type" "load_16,neon_load1_2reg")
1313 (set_attr "fp" "*,yes")]
1317 ;; Operands 0 and 2 are tied together by the final condition; so we allow
1318 ;; fairly lax checking on the second memory operation.
1319 (define_insn "store_pairsi"
1320 [(set (match_operand:SI 0 "aarch64_mem_pair_operand" "=Ump,Ump")
1321 (match_operand:SI 1 "aarch64_reg_or_zero" "rZ,*w"))
1322 (set (match_operand:SI 2 "memory_operand" "=m,m")
1323 (match_operand:SI 3 "aarch64_reg_or_zero" "rZ,*w"))]
1324 "rtx_equal_p (XEXP (operands[2], 0),
1325 plus_constant (Pmode,
1326 XEXP (operands[0], 0),
1327 GET_MODE_SIZE (SImode)))"
1331 [(set_attr "type" "store_8,neon_store1_2reg")
1332 (set_attr "fp" "*,yes")]
1335 (define_insn "store_pairdi"
1336 [(set (match_operand:DI 0 "aarch64_mem_pair_operand" "=Ump,Ump")
1337 (match_operand:DI 1 "aarch64_reg_or_zero" "rZ,*w"))
1338 (set (match_operand:DI 2 "memory_operand" "=m,m")
1339 (match_operand:DI 3 "aarch64_reg_or_zero" "rZ,*w"))]
1340 "rtx_equal_p (XEXP (operands[2], 0),
1341 plus_constant (Pmode,
1342 XEXP (operands[0], 0),
1343 GET_MODE_SIZE (DImode)))"
1347 [(set_attr "type" "store_16,neon_store1_2reg")
1348 (set_attr "fp" "*,yes")]
1351 ;; Operands 1 and 3 are tied together by the final condition; so we allow
1352 ;; fairly lax checking on the second memory operation.
1353 (define_insn "load_pairsf"
1354 [(set (match_operand:SF 0 "register_operand" "=w,*r")
1355 (match_operand:SF 1 "aarch64_mem_pair_operand" "Ump,Ump"))
1356 (set (match_operand:SF 2 "register_operand" "=w,*r")
1357 (match_operand:SF 3 "memory_operand" "m,m"))]
1358 "rtx_equal_p (XEXP (operands[3], 0),
1359 plus_constant (Pmode,
1360 XEXP (operands[1], 0),
1361 GET_MODE_SIZE (SFmode)))"
1365 [(set_attr "type" "neon_load1_2reg,load_8")
1366 (set_attr "fp" "yes,*")]
1369 (define_insn "load_pairdf"
1370 [(set (match_operand:DF 0 "register_operand" "=w,*r")
1371 (match_operand:DF 1 "aarch64_mem_pair_operand" "Ump,Ump"))
1372 (set (match_operand:DF 2 "register_operand" "=w,*r")
1373 (match_operand:DF 3 "memory_operand" "m,m"))]
1374 "rtx_equal_p (XEXP (operands[3], 0),
1375 plus_constant (Pmode,
1376 XEXP (operands[1], 0),
1377 GET_MODE_SIZE (DFmode)))"
1381 [(set_attr "type" "neon_load1_2reg,load_16")
1382 (set_attr "fp" "yes,*")]
1385 ;; Operands 0 and 2 are tied together by the final condition; so we allow
1386 ;; fairly lax checking on the second memory operation.
1387 (define_insn "store_pairsf"
1388 [(set (match_operand:SF 0 "aarch64_mem_pair_operand" "=Ump,Ump")
1389 (match_operand:SF 1 "aarch64_reg_or_fp_zero" "w,*rY"))
1390 (set (match_operand:SF 2 "memory_operand" "=m,m")
1391 (match_operand:SF 3 "aarch64_reg_or_fp_zero" "w,*rY"))]
1392 "rtx_equal_p (XEXP (operands[2], 0),
1393 plus_constant (Pmode,
1394 XEXP (operands[0], 0),
1395 GET_MODE_SIZE (SFmode)))"
1399 [(set_attr "type" "neon_store1_2reg,store_8")
1400 (set_attr "fp" "yes,*")]
1403 (define_insn "store_pairdf"
1404 [(set (match_operand:DF 0 "aarch64_mem_pair_operand" "=Ump,Ump")
1405 (match_operand:DF 1 "aarch64_reg_or_fp_zero" "w,*rY"))
1406 (set (match_operand:DF 2 "memory_operand" "=m,m")
1407 (match_operand:DF 3 "aarch64_reg_or_fp_zero" "w,*rY"))]
1408 "rtx_equal_p (XEXP (operands[2], 0),
1409 plus_constant (Pmode,
1410 XEXP (operands[0], 0),
1411 GET_MODE_SIZE (DFmode)))"
1415 [(set_attr "type" "neon_store1_2reg,store_16")
1416 (set_attr "fp" "yes,*")]
1419 ;; Load pair with post-index writeback. This is primarily used in function
1421 (define_insn "loadwb_pair<GPI:mode>_<P:mode>"
1423 [(set (match_operand:P 0 "register_operand" "=k")
1424 (plus:P (match_operand:P 1 "register_operand" "0")
1425 (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1426 (set (match_operand:GPI 2 "register_operand" "=r")
1427 (mem:GPI (match_dup 1)))
1428 (set (match_operand:GPI 3 "register_operand" "=r")
1429 (mem:GPI (plus:P (match_dup 1)
1430 (match_operand:P 5 "const_int_operand" "n"))))])]
1431 "INTVAL (operands[5]) == GET_MODE_SIZE (<GPI:MODE>mode)"
1432 "ldp\\t%<w>2, %<w>3, [%1], %4"
1433 [(set_attr "type" "load_<ldpstp_sz>")]
1436 (define_insn "loadwb_pair<GPF:mode>_<P:mode>"
1438 [(set (match_operand:P 0 "register_operand" "=k")
1439 (plus:P (match_operand:P 1 "register_operand" "0")
1440 (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1441 (set (match_operand:GPF 2 "register_operand" "=w")
1442 (mem:GPF (match_dup 1)))
1443 (set (match_operand:GPF 3 "register_operand" "=w")
1444 (mem:GPF (plus:P (match_dup 1)
1445 (match_operand:P 5 "const_int_operand" "n"))))])]
1446 "INTVAL (operands[5]) == GET_MODE_SIZE (<GPF:MODE>mode)"
1447 "ldp\\t%<w>2, %<w>3, [%1], %4"
1448 [(set_attr "type" "neon_load1_2reg")]
1451 ;; Store pair with pre-index writeback. This is primarily used in function
1453 (define_insn "storewb_pair<GPI:mode>_<P:mode>"
1455 [(set (match_operand:P 0 "register_operand" "=&k")
1456 (plus:P (match_operand:P 1 "register_operand" "0")
1457 (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1458 (set (mem:GPI (plus:P (match_dup 0)
1460 (match_operand:GPI 2 "register_operand" "r"))
1461 (set (mem:GPI (plus:P (match_dup 0)
1462 (match_operand:P 5 "const_int_operand" "n")))
1463 (match_operand:GPI 3 "register_operand" "r"))])]
1464 "INTVAL (operands[5]) == INTVAL (operands[4]) + GET_MODE_SIZE (<GPI:MODE>mode)"
1465 "stp\\t%<w>2, %<w>3, [%0, %4]!"
1466 [(set_attr "type" "store_<ldpstp_sz>")]
1469 (define_insn "storewb_pair<GPF:mode>_<P:mode>"
1471 [(set (match_operand:P 0 "register_operand" "=&k")
1472 (plus:P (match_operand:P 1 "register_operand" "0")
1473 (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1474 (set (mem:GPF (plus:P (match_dup 0)
1476 (match_operand:GPF 2 "register_operand" "w"))
1477 (set (mem:GPF (plus:P (match_dup 0)
1478 (match_operand:P 5 "const_int_operand" "n")))
1479 (match_operand:GPF 3 "register_operand" "w"))])]
1480 "INTVAL (operands[5]) == INTVAL (operands[4]) + GET_MODE_SIZE (<GPF:MODE>mode)"
1481 "stp\\t%<w>2, %<w>3, [%0, %4]!"
1482 [(set_attr "type" "neon_store1_2reg<q>")]
1485 ;; -------------------------------------------------------------------
1486 ;; Sign/Zero extension
1487 ;; -------------------------------------------------------------------
1489 (define_expand "<optab>sidi2"
1490 [(set (match_operand:DI 0 "register_operand")
1491 (ANY_EXTEND:DI (match_operand:SI 1 "nonimmediate_operand")))]
1495 (define_insn "*extendsidi2_aarch64"
1496 [(set (match_operand:DI 0 "register_operand" "=r,r")
1497 (sign_extend:DI (match_operand:SI 1 "nonimmediate_operand" "r,m")))]
1502 [(set_attr "type" "extend,load_4")]
1505 (define_insn "*load_pair_extendsidi2_aarch64"
1506 [(set (match_operand:DI 0 "register_operand" "=r")
1507 (sign_extend:DI (match_operand:SI 1 "aarch64_mem_pair_operand" "Ump")))
1508 (set (match_operand:DI 2 "register_operand" "=r")
1509 (sign_extend:DI (match_operand:SI 3 "memory_operand" "m")))]
1510 "rtx_equal_p (XEXP (operands[3], 0),
1511 plus_constant (Pmode,
1512 XEXP (operands[1], 0),
1513 GET_MODE_SIZE (SImode)))"
1514 "ldpsw\\t%0, %2, %1"
1515 [(set_attr "type" "load_8")]
1518 (define_insn "*zero_extendsidi2_aarch64"
1519 [(set (match_operand:DI 0 "register_operand" "=r,r")
1520 (zero_extend:DI (match_operand:SI 1 "nonimmediate_operand" "r,m")))]
1525 [(set_attr "type" "extend,load_4")]
1528 (define_insn "*load_pair_zero_extendsidi2_aarch64"
1529 [(set (match_operand:DI 0 "register_operand" "=r")
1530 (zero_extend:DI (match_operand:SI 1 "aarch64_mem_pair_operand" "Ump")))
1531 (set (match_operand:DI 2 "register_operand" "=r")
1532 (zero_extend:DI (match_operand:SI 3 "memory_operand" "m")))]
1533 "rtx_equal_p (XEXP (operands[3], 0),
1534 plus_constant (Pmode,
1535 XEXP (operands[1], 0),
1536 GET_MODE_SIZE (SImode)))"
1537 "ldp\\t%w0, %w2, %1"
1538 [(set_attr "type" "load_8")]
1541 (define_expand "<ANY_EXTEND:optab><SHORT:mode><GPI:mode>2"
1542 [(set (match_operand:GPI 0 "register_operand")
1543 (ANY_EXTEND:GPI (match_operand:SHORT 1 "nonimmediate_operand")))]
1547 (define_insn "*extend<SHORT:mode><GPI:mode>2_aarch64"
1548 [(set (match_operand:GPI 0 "register_operand" "=r,r")
1549 (sign_extend:GPI (match_operand:SHORT 1 "nonimmediate_operand" "r,m")))]
1552 sxt<SHORT:size>\t%<GPI:w>0, %w1
1553 ldrs<SHORT:size>\t%<GPI:w>0, %1"
1554 [(set_attr "type" "extend,load_4")]
1557 (define_insn "*zero_extend<SHORT:mode><GPI:mode>2_aarch64"
1558 [(set (match_operand:GPI 0 "register_operand" "=r,r,*w")
1559 (zero_extend:GPI (match_operand:SHORT 1 "nonimmediate_operand" "r,m,m")))]
1562 and\t%<GPI:w>0, %<GPI:w>1, <SHORT:short_mask>
1563 ldr<SHORT:size>\t%w0, %1
1564 ldr\t%<SHORT:size>0, %1"
1565 [(set_attr "type" "logic_imm,load_4,load_4")]
1568 (define_expand "<optab>qihi2"
1569 [(set (match_operand:HI 0 "register_operand")
1570 (ANY_EXTEND:HI (match_operand:QI 1 "nonimmediate_operand")))]
1574 (define_insn "*extendqihi2_aarch64"
1575 [(set (match_operand:HI 0 "register_operand" "=r,r")
1576 (sign_extend:HI (match_operand:QI 1 "nonimmediate_operand" "r,m")))]
1581 [(set_attr "type" "extend,load_4")]
1584 (define_insn "*zero_extendqihi2_aarch64"
1585 [(set (match_operand:HI 0 "register_operand" "=r,r")
1586 (zero_extend:HI (match_operand:QI 1 "nonimmediate_operand" "r,m")))]
1591 [(set_attr "type" "logic_imm,load_4")]
1594 ;; -------------------------------------------------------------------
1595 ;; Simple arithmetic
1596 ;; -------------------------------------------------------------------
1598 (define_expand "add<mode>3"
1600 (match_operand:GPI 0 "register_operand" "")
1601 (plus:GPI (match_operand:GPI 1 "register_operand" "")
1602 (match_operand:GPI 2 "aarch64_pluslong_or_poly_operand" "")))]
1605 /* If operands[1] is a subreg extract the inner RTX. */
1606 rtx op1 = REG_P (operands[1]) ? operands[1] : SUBREG_REG (operands[1]);
1608 /* If the constant is too large for a single instruction and isn't frame
1609 based, split off the immediate so it is available for CSE. */
1610 if (!aarch64_plus_immediate (operands[2], <MODE>mode)
1611 && can_create_pseudo_p ()
1613 || !REGNO_PTR_FRAME_P (REGNO (op1))))
1614 operands[2] = force_reg (<MODE>mode, operands[2]);
1615 /* Expand polynomial additions now if the destination is the stack
1616 pointer, since we don't want to use that as a temporary. */
1617 else if (operands[0] == stack_pointer_rtx
1618 && aarch64_split_add_offset_immediate (operands[2], <MODE>mode))
1620 aarch64_split_add_offset (<MODE>mode, operands[0], operands[1],
1621 operands[2], NULL_RTX, NULL_RTX);
1626 (define_insn "*add<mode>3_aarch64"
1628 (match_operand:GPI 0 "register_operand" "=rk,rk,w,rk,r,rk")
1630 (match_operand:GPI 1 "register_operand" "%rk,rk,w,rk,rk,rk")
1631 (match_operand:GPI 2 "aarch64_pluslong_operand" "I,r,w,J,Uaa,Uav")))]
1634 add\\t%<w>0, %<w>1, %2
1635 add\\t%<w>0, %<w>1, %<w>2
1636 add\\t%<rtn>0<vas>, %<rtn>1<vas>, %<rtn>2<vas>
1637 sub\\t%<w>0, %<w>1, #%n2
1639 * return aarch64_output_sve_addvl_addpl (operands[0], operands[1], operands[2]);"
1640 ;; The "alu_imm" type for ADDVL/ADDPL is just a placeholder.
1641 [(set_attr "type" "alu_imm,alu_sreg,neon_add,alu_imm,multiple,alu_imm")
1642 (set_attr "simd" "*,*,yes,*,*,*")]
1645 ;; zero_extend version of above
1646 (define_insn "*addsi3_aarch64_uxtw"
1648 (match_operand:DI 0 "register_operand" "=rk,rk,rk,r")
1650 (plus:SI (match_operand:SI 1 "register_operand" "%rk,rk,rk,rk")
1651 (match_operand:SI 2 "aarch64_pluslong_operand" "I,r,J,Uaa"))))]
1656 sub\\t%w0, %w1, #%n2
1658 [(set_attr "type" "alu_imm,alu_sreg,alu_imm,multiple")]
1661 ;; If there's a free register, and we can load the constant with a
1662 ;; single instruction, do so. This has a chance to improve scheduling.
1664 [(match_scratch:GPI 3 "r")
1665 (set (match_operand:GPI 0 "register_operand")
1667 (match_operand:GPI 1 "register_operand")
1668 (match_operand:GPI 2 "aarch64_pluslong_strict_immedate")))]
1669 "aarch64_move_imm (INTVAL (operands[2]), <MODE>mode)"
1670 [(set (match_dup 3) (match_dup 2))
1671 (set (match_dup 0) (plus:GPI (match_dup 1) (match_dup 3)))]
1675 [(match_scratch:SI 3 "r")
1676 (set (match_operand:DI 0 "register_operand")
1679 (match_operand:SI 1 "register_operand")
1680 (match_operand:SI 2 "aarch64_pluslong_strict_immedate"))))]
1681 "aarch64_move_imm (INTVAL (operands[2]), SImode)"
1682 [(set (match_dup 3) (match_dup 2))
1683 (set (match_dup 0) (zero_extend:DI (plus:SI (match_dup 1) (match_dup 3))))]
1686 ;; After peephole2 has had a chance to run, split any remaining long
1687 ;; additions into two add immediates.
1689 [(set (match_operand:GPI 0 "register_operand")
1691 (match_operand:GPI 1 "register_operand")
1692 (match_operand:GPI 2 "aarch64_pluslong_strict_immedate")))]
1693 "epilogue_completed"
1694 [(set (match_dup 0) (plus:GPI (match_dup 1) (match_dup 3)))
1695 (set (match_dup 0) (plus:GPI (match_dup 0) (match_dup 4)))]
1697 HOST_WIDE_INT i = INTVAL (operands[2]);
1698 HOST_WIDE_INT s = (i >= 0 ? i & 0xfff : -(-i & 0xfff));
1699 operands[3] = GEN_INT (i - s);
1700 operands[4] = GEN_INT (s);
1704 ;; Match addition of polynomial offsets that require one temporary, for which
1705 ;; we can use the early-clobbered destination register. This is a separate
1706 ;; pattern so that the early clobber doesn't affect register allocation
1707 ;; for other forms of addition. However, we still need to provide an
1708 ;; all-register alternative, in case the offset goes out of range after
1709 ;; elimination. For completeness we might as well provide all GPR-based
1710 ;; alternatives from the main pattern.
1712 ;; We don't have a pattern for additions requiring two temporaries since at
1713 ;; present LRA doesn't allow new scratches to be added during elimination.
1714 ;; Such offsets should be rare anyway.
1716 ;; ??? But if we added LRA support for new scratches, much of the ugliness
1717 ;; here would go away. We could just handle all polynomial constants in
1719 (define_insn_and_split "*add<mode>3_poly_1"
1721 (match_operand:GPI 0 "register_operand" "=r,r,r,r,r,&r")
1723 (match_operand:GPI 1 "register_operand" "%rk,rk,rk,rk,rk,rk")
1724 (match_operand:GPI 2 "aarch64_pluslong_or_poly_operand" "I,r,J,Uaa,Uav,Uat")))]
1725 "TARGET_SVE && operands[0] != stack_pointer_rtx"
1727 add\\t%<w>0, %<w>1, %2
1728 add\\t%<w>0, %<w>1, %<w>2
1729 sub\\t%<w>0, %<w>1, #%n2
1731 * return aarch64_output_sve_addvl_addpl (operands[0], operands[1], operands[2]);
1733 "&& epilogue_completed
1734 && !reg_overlap_mentioned_p (operands[0], operands[1])
1735 && aarch64_split_add_offset_immediate (operands[2], <MODE>mode)"
1738 aarch64_split_add_offset (<MODE>mode, operands[0], operands[1],
1739 operands[2], operands[0], NULL_RTX);
1742 ;; The "alu_imm" type for ADDVL/ADDPL is just a placeholder.
1743 [(set_attr "type" "alu_imm,alu_sreg,alu_imm,multiple,alu_imm,multiple")]
1747 [(set (match_operand:DI 0 "register_operand")
1750 (match_operand:SI 1 "register_operand")
1751 (match_operand:SI 2 "aarch64_pluslong_strict_immedate"))))]
1752 "epilogue_completed"
1753 [(set (match_dup 5) (plus:SI (match_dup 1) (match_dup 3)))
1754 (set (match_dup 0) (zero_extend:DI (plus:SI (match_dup 5) (match_dup 4))))]
1756 HOST_WIDE_INT i = INTVAL (operands[2]);
1757 HOST_WIDE_INT s = (i >= 0 ? i & 0xfff : -(-i & 0xfff));
1758 operands[3] = GEN_INT (i - s);
1759 operands[4] = GEN_INT (s);
1760 operands[5] = gen_lowpart (SImode, operands[0]);
1764 (define_expand "addti3"
1765 [(set (match_operand:TI 0 "register_operand" "")
1766 (plus:TI (match_operand:TI 1 "register_operand" "")
1767 (match_operand:TI 2 "register_operand" "")))]
1770 rtx low = gen_reg_rtx (DImode);
1771 emit_insn (gen_adddi3_compareC (low, gen_lowpart (DImode, operands[1]),
1772 gen_lowpart (DImode, operands[2])));
1774 rtx high = gen_reg_rtx (DImode);
1775 emit_insn (gen_adddi3_carryin (high, gen_highpart (DImode, operands[1]),
1776 gen_highpart (DImode, operands[2])));
1778 emit_move_insn (gen_lowpart (DImode, operands[0]), low);
1779 emit_move_insn (gen_highpart (DImode, operands[0]), high);
1783 (define_insn "add<mode>3_compare0"
1784 [(set (reg:CC_NZ CC_REGNUM)
1786 (plus:GPI (match_operand:GPI 1 "register_operand" "%r,r,r")
1787 (match_operand:GPI 2 "aarch64_plus_operand" "r,I,J"))
1789 (set (match_operand:GPI 0 "register_operand" "=r,r,r")
1790 (plus:GPI (match_dup 1) (match_dup 2)))]
1793 adds\\t%<w>0, %<w>1, %<w>2
1794 adds\\t%<w>0, %<w>1, %2
1795 subs\\t%<w>0, %<w>1, #%n2"
1796 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
1799 ;; zero_extend version of above
1800 (define_insn "*addsi3_compare0_uxtw"
1801 [(set (reg:CC_NZ CC_REGNUM)
1803 (plus:SI (match_operand:SI 1 "register_operand" "%r,r,r")
1804 (match_operand:SI 2 "aarch64_plus_operand" "r,I,J"))
1806 (set (match_operand:DI 0 "register_operand" "=r,r,r")
1807 (zero_extend:DI (plus:SI (match_dup 1) (match_dup 2))))]
1810 adds\\t%w0, %w1, %w2
1812 subs\\t%w0, %w1, #%n2"
1813 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
1816 (define_insn "*add<mode>3_compareC_cconly_imm"
1817 [(set (reg:CC_C CC_REGNUM)
1820 (zero_extend:<DWI> (match_operand:GPI 0 "register_operand" "r,r"))
1821 (match_operand:<DWI> 2 "const_scalar_int_operand" ""))
1825 (match_operand:GPI 1 "aarch64_plus_immediate" "I,J")))))]
1826 "aarch64_zero_extend_const_eq (<DWI>mode, operands[2],
1827 <MODE>mode, operands[1])"
1831 [(set_attr "type" "alus_imm")]
1834 (define_insn "*add<mode>3_compareC_cconly"
1835 [(set (reg:CC_C CC_REGNUM)
1838 (zero_extend:<DWI> (match_operand:GPI 0 "register_operand" "r"))
1839 (zero_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
1840 (zero_extend:<DWI> (plus:GPI (match_dup 0) (match_dup 1)))))]
1842 "cmn\\t%<w>0, %<w>1"
1843 [(set_attr "type" "alus_sreg")]
1846 (define_insn "*add<mode>3_compareC_imm"
1847 [(set (reg:CC_C CC_REGNUM)
1850 (zero_extend:<DWI> (match_operand:GPI 1 "register_operand" "r,r"))
1851 (match_operand:<DWI> 3 "const_scalar_int_operand" ""))
1855 (match_operand:GPI 2 "aarch64_plus_immediate" "I,J")))))
1856 (set (match_operand:GPI 0 "register_operand" "=r,r")
1857 (plus:GPI (match_dup 1) (match_dup 2)))]
1858 "aarch64_zero_extend_const_eq (<DWI>mode, operands[3],
1859 <MODE>mode, operands[2])"
1861 adds\\t%<w>0, %<w>1, %2
1862 subs\\t%<w>0, %<w>1, #%n2"
1863 [(set_attr "type" "alus_imm")]
1866 (define_insn "add<mode>3_compareC"
1867 [(set (reg:CC_C CC_REGNUM)
1870 (zero_extend:<DWI> (match_operand:GPI 1 "register_operand" "r"))
1871 (zero_extend:<DWI> (match_operand:GPI 2 "register_operand" "r")))
1873 (plus:GPI (match_dup 1) (match_dup 2)))))
1874 (set (match_operand:GPI 0 "register_operand" "=r")
1875 (plus:GPI (match_dup 1) (match_dup 2)))]
1877 "adds\\t%<w>0, %<w>1, %<w>2"
1878 [(set_attr "type" "alus_sreg")]
1881 (define_insn "*adds_shift_imm_<mode>"
1882 [(set (reg:CC_NZ CC_REGNUM)
1884 (plus:GPI (ASHIFT:GPI
1885 (match_operand:GPI 1 "register_operand" "r")
1886 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
1887 (match_operand:GPI 3 "register_operand" "r"))
1889 (set (match_operand:GPI 0 "register_operand" "=r")
1890 (plus:GPI (ASHIFT:GPI (match_dup 1) (match_dup 2))
1893 "adds\\t%<w>0, %<w>3, %<w>1, <shift> %2"
1894 [(set_attr "type" "alus_shift_imm")]
1897 (define_insn "*subs_shift_imm_<mode>"
1898 [(set (reg:CC_NZ CC_REGNUM)
1900 (minus:GPI (match_operand:GPI 1 "register_operand" "r")
1902 (match_operand:GPI 2 "register_operand" "r")
1903 (match_operand:QI 3 "aarch64_shift_imm_<mode>" "n")))
1905 (set (match_operand:GPI 0 "register_operand" "=r")
1906 (minus:GPI (match_dup 1)
1907 (ASHIFT:GPI (match_dup 2) (match_dup 3))))]
1909 "subs\\t%<w>0, %<w>1, %<w>2, <shift> %3"
1910 [(set_attr "type" "alus_shift_imm")]
1913 (define_insn "*adds_mul_imm_<mode>"
1914 [(set (reg:CC_NZ CC_REGNUM)
1917 (match_operand:GPI 1 "register_operand" "r")
1918 (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))
1919 (match_operand:GPI 3 "register_operand" "r"))
1921 (set (match_operand:GPI 0 "register_operand" "=r")
1922 (plus:GPI (mult:GPI (match_dup 1) (match_dup 2))
1925 "adds\\t%<w>0, %<w>3, %<w>1, lsl %p2"
1926 [(set_attr "type" "alus_shift_imm")]
1929 (define_insn "*subs_mul_imm_<mode>"
1930 [(set (reg:CC_NZ CC_REGNUM)
1932 (minus:GPI (match_operand:GPI 1 "register_operand" "r")
1934 (match_operand:GPI 2 "register_operand" "r")
1935 (match_operand:QI 3 "aarch64_pwr_2_<mode>" "n")))
1937 (set (match_operand:GPI 0 "register_operand" "=r")
1938 (minus:GPI (match_dup 1)
1939 (mult:GPI (match_dup 2) (match_dup 3))))]
1941 "subs\\t%<w>0, %<w>1, %<w>2, lsl %p3"
1942 [(set_attr "type" "alus_shift_imm")]
1945 (define_insn "*adds_<optab><ALLX:mode>_<GPI:mode>"
1946 [(set (reg:CC_NZ CC_REGNUM)
1949 (ANY_EXTEND:GPI (match_operand:ALLX 1 "register_operand" "r"))
1950 (match_operand:GPI 2 "register_operand" "r"))
1952 (set (match_operand:GPI 0 "register_operand" "=r")
1953 (plus:GPI (ANY_EXTEND:GPI (match_dup 1)) (match_dup 2)))]
1955 "adds\\t%<GPI:w>0, %<GPI:w>2, %<GPI:w>1, <su>xt<ALLX:size>"
1956 [(set_attr "type" "alus_ext")]
1959 (define_insn "*subs_<optab><ALLX:mode>_<GPI:mode>"
1960 [(set (reg:CC_NZ CC_REGNUM)
1962 (minus:GPI (match_operand:GPI 1 "register_operand" "r")
1964 (match_operand:ALLX 2 "register_operand" "r")))
1966 (set (match_operand:GPI 0 "register_operand" "=r")
1967 (minus:GPI (match_dup 1) (ANY_EXTEND:GPI (match_dup 2))))]
1969 "subs\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size>"
1970 [(set_attr "type" "alus_ext")]
1973 (define_insn "*adds_<optab><ALLX:mode>_shift_<GPI:mode>"
1974 [(set (reg:CC_NZ CC_REGNUM)
1976 (plus:GPI (ashift:GPI
1978 (match_operand:ALLX 1 "register_operand" "r"))
1979 (match_operand 2 "aarch64_imm3" "Ui3"))
1980 (match_operand:GPI 3 "register_operand" "r"))
1982 (set (match_operand:GPI 0 "register_operand" "=rk")
1983 (plus:GPI (ashift:GPI (ANY_EXTEND:GPI (match_dup 1))
1987 "adds\\t%<GPI:w>0, %<GPI:w>3, %<GPI:w>1, <su>xt<ALLX:size> %2"
1988 [(set_attr "type" "alus_ext")]
1991 (define_insn "*subs_<optab><ALLX:mode>_shift_<GPI:mode>"
1992 [(set (reg:CC_NZ CC_REGNUM)
1994 (minus:GPI (match_operand:GPI 1 "register_operand" "r")
1997 (match_operand:ALLX 2 "register_operand" "r"))
1998 (match_operand 3 "aarch64_imm3" "Ui3")))
2000 (set (match_operand:GPI 0 "register_operand" "=rk")
2001 (minus:GPI (match_dup 1)
2002 (ashift:GPI (ANY_EXTEND:GPI (match_dup 2))
2005 "subs\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size> %3"
2006 [(set_attr "type" "alus_ext")]
2009 (define_insn "*adds_<optab><mode>_multp2"
2010 [(set (reg:CC_NZ CC_REGNUM)
2012 (plus:GPI (ANY_EXTRACT:GPI
2013 (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2014 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2015 (match_operand 3 "const_int_operand" "n")
2017 (match_operand:GPI 4 "register_operand" "r"))
2019 (set (match_operand:GPI 0 "register_operand" "=r")
2020 (plus:GPI (ANY_EXTRACT:GPI (mult:GPI (match_dup 1) (match_dup 2))
2024 "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
2025 "adds\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
2026 [(set_attr "type" "alus_ext")]
2029 (define_insn "*subs_<optab><mode>_multp2"
2030 [(set (reg:CC_NZ CC_REGNUM)
2032 (minus:GPI (match_operand:GPI 4 "register_operand" "r")
2034 (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2035 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2036 (match_operand 3 "const_int_operand" "n")
2039 (set (match_operand:GPI 0 "register_operand" "=r")
2040 (minus:GPI (match_dup 4) (ANY_EXTRACT:GPI
2041 (mult:GPI (match_dup 1) (match_dup 2))
2044 "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
2045 "subs\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
2046 [(set_attr "type" "alus_ext")]
2049 (define_insn "*add<mode>3nr_compare0"
2050 [(set (reg:CC_NZ CC_REGNUM)
2052 (plus:GPI (match_operand:GPI 0 "register_operand" "%r,r,r")
2053 (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J"))
2060 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
2063 (define_insn "aarch64_sub<mode>_compare0"
2064 [(set (reg:CC_NZ CC_REGNUM)
2066 (minus:GPI (match_operand:GPI 0 "register_operand" "r")
2067 (match_operand:GPI 1 "aarch64_plus_operand" "r"))
2070 "cmp\\t%<w>0, %<w>1"
2071 [(set_attr "type" "alus_sreg")]
2074 (define_insn "*compare_neg<mode>"
2075 [(set (reg:CC_Z CC_REGNUM)
2077 (neg:GPI (match_operand:GPI 0 "register_operand" "r"))
2078 (match_operand:GPI 1 "register_operand" "r")))]
2080 "cmn\\t%<w>1, %<w>0"
2081 [(set_attr "type" "alus_sreg")]
2084 (define_insn "*add_<shift>_<mode>"
2085 [(set (match_operand:GPI 0 "register_operand" "=r")
2086 (plus:GPI (ASHIFT:GPI (match_operand:GPI 1 "register_operand" "r")
2087 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
2088 (match_operand:GPI 3 "register_operand" "r")))]
2090 "add\\t%<w>0, %<w>3, %<w>1, <shift> %2"
2091 [(set_attr "type" "alu_shift_imm")]
2094 ;; zero_extend version of above
2095 (define_insn "*add_<shift>_si_uxtw"
2096 [(set (match_operand:DI 0 "register_operand" "=r")
2098 (plus:SI (ASHIFT:SI (match_operand:SI 1 "register_operand" "r")
2099 (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
2100 (match_operand:SI 3 "register_operand" "r"))))]
2102 "add\\t%w0, %w3, %w1, <shift> %2"
2103 [(set_attr "type" "alu_shift_imm")]
2106 (define_insn "*add_mul_imm_<mode>"
2107 [(set (match_operand:GPI 0 "register_operand" "=r")
2108 (plus:GPI (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2109 (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))
2110 (match_operand:GPI 3 "register_operand" "r")))]
2112 "add\\t%<w>0, %<w>3, %<w>1, lsl %p2"
2113 [(set_attr "type" "alu_shift_imm")]
2116 (define_insn "*add_<optab><ALLX:mode>_<GPI:mode>"
2117 [(set (match_operand:GPI 0 "register_operand" "=rk")
2118 (plus:GPI (ANY_EXTEND:GPI (match_operand:ALLX 1 "register_operand" "r"))
2119 (match_operand:GPI 2 "register_operand" "r")))]
2121 "add\\t%<GPI:w>0, %<GPI:w>2, %<GPI:w>1, <su>xt<ALLX:size>"
2122 [(set_attr "type" "alu_ext")]
2125 ;; zero_extend version of above
2126 (define_insn "*add_<optab><SHORT:mode>_si_uxtw"
2127 [(set (match_operand:DI 0 "register_operand" "=rk")
2129 (plus:SI (ANY_EXTEND:SI (match_operand:SHORT 1 "register_operand" "r"))
2130 (match_operand:GPI 2 "register_operand" "r"))))]
2132 "add\\t%w0, %w2, %w1, <su>xt<SHORT:size>"
2133 [(set_attr "type" "alu_ext")]
2136 (define_insn "*add_<optab><ALLX:mode>_shft_<GPI:mode>"
2137 [(set (match_operand:GPI 0 "register_operand" "=rk")
2138 (plus:GPI (ashift:GPI (ANY_EXTEND:GPI
2139 (match_operand:ALLX 1 "register_operand" "r"))
2140 (match_operand 2 "aarch64_imm3" "Ui3"))
2141 (match_operand:GPI 3 "register_operand" "r")))]
2143 "add\\t%<GPI:w>0, %<GPI:w>3, %<GPI:w>1, <su>xt<ALLX:size> %2"
2144 [(set_attr "type" "alu_ext")]
2147 ;; zero_extend version of above
2148 (define_insn "*add_<optab><SHORT:mode>_shft_si_uxtw"
2149 [(set (match_operand:DI 0 "register_operand" "=rk")
2151 (plus:SI (ashift:SI (ANY_EXTEND:SI
2152 (match_operand:SHORT 1 "register_operand" "r"))
2153 (match_operand 2 "aarch64_imm3" "Ui3"))
2154 (match_operand:SI 3 "register_operand" "r"))))]
2156 "add\\t%w0, %w3, %w1, <su>xt<SHORT:size> %2"
2157 [(set_attr "type" "alu_ext")]
2160 (define_insn "*add_<optab><ALLX:mode>_mult_<GPI:mode>"
2161 [(set (match_operand:GPI 0 "register_operand" "=rk")
2162 (plus:GPI (mult:GPI (ANY_EXTEND:GPI
2163 (match_operand:ALLX 1 "register_operand" "r"))
2164 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2165 (match_operand:GPI 3 "register_operand" "r")))]
2167 "add\\t%<GPI:w>0, %<GPI:w>3, %<GPI:w>1, <su>xt<ALLX:size> %p2"
2168 [(set_attr "type" "alu_ext")]
2171 ;; zero_extend version of above
2172 (define_insn "*add_<optab><SHORT:mode>_mult_si_uxtw"
2173 [(set (match_operand:DI 0 "register_operand" "=rk")
2174 (zero_extend:DI (plus:SI (mult:SI (ANY_EXTEND:SI
2175 (match_operand:SHORT 1 "register_operand" "r"))
2176 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2177 (match_operand:SI 3 "register_operand" "r"))))]
2179 "add\\t%w0, %w3, %w1, <su>xt<SHORT:size> %p2"
2180 [(set_attr "type" "alu_ext")]
2183 (define_insn "*add_<optab><mode>_multp2"
2184 [(set (match_operand:GPI 0 "register_operand" "=rk")
2185 (plus:GPI (ANY_EXTRACT:GPI
2186 (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2187 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2188 (match_operand 3 "const_int_operand" "n")
2190 (match_operand:GPI 4 "register_operand" "r")))]
2191 "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
2192 "add\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
2193 [(set_attr "type" "alu_ext")]
2196 ;; zero_extend version of above
2197 (define_insn "*add_<optab>si_multp2_uxtw"
2198 [(set (match_operand:DI 0 "register_operand" "=rk")
2200 (plus:SI (ANY_EXTRACT:SI
2201 (mult:SI (match_operand:SI 1 "register_operand" "r")
2202 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2203 (match_operand 3 "const_int_operand" "n")
2205 (match_operand:SI 4 "register_operand" "r"))))]
2206 "aarch64_is_extend_from_extract (SImode, operands[2], operands[3])"
2207 "add\\t%w0, %w4, %w1, <su>xt%e3 %p2"
2208 [(set_attr "type" "alu_ext")]
2211 (define_expand "add<mode>3_carryin"
2212 [(set (match_operand:GPI 0 "register_operand")
2215 (ne:GPI (reg:CC_C CC_REGNUM) (const_int 0))
2216 (match_operand:GPI 1 "aarch64_reg_or_zero"))
2217 (match_operand:GPI 2 "aarch64_reg_or_zero")))]
2222 ;; Note that add with carry with two zero inputs is matched by cset,
2223 ;; and that add with carry with one zero input is matched by cinc.
2225 (define_insn "*add<mode>3_carryin"
2226 [(set (match_operand:GPI 0 "register_operand" "=r")
2229 (match_operand:GPI 3 "aarch64_carry_operation" "")
2230 (match_operand:GPI 1 "register_operand" "r"))
2231 (match_operand:GPI 2 "register_operand" "r")))]
2233 "adc\\t%<w>0, %<w>1, %<w>2"
2234 [(set_attr "type" "adc_reg")]
2237 ;; zero_extend version of above
2238 (define_insn "*addsi3_carryin_uxtw"
2239 [(set (match_operand:DI 0 "register_operand" "=r")
2243 (match_operand:SI 3 "aarch64_carry_operation" "")
2244 (match_operand:SI 1 "register_operand" "r"))
2245 (match_operand:SI 2 "register_operand" "r"))))]
2247 "adc\\t%w0, %w1, %w2"
2248 [(set_attr "type" "adc_reg")]
2251 (define_insn "*add_uxt<mode>_shift2"
2252 [(set (match_operand:GPI 0 "register_operand" "=rk")
2254 (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
2255 (match_operand 2 "aarch64_imm3" "Ui3"))
2256 (match_operand 3 "const_int_operand" "n"))
2257 (match_operand:GPI 4 "register_operand" "r")))]
2258 "aarch64_uxt_size (INTVAL (operands[2]), INTVAL (operands[3])) != 0"
2260 operands[3] = GEN_INT (aarch64_uxt_size (INTVAL(operands[2]),
2261 INTVAL (operands[3])));
2262 return \"add\t%<w>0, %<w>4, %<w>1, uxt%e3 %2\";"
2263 [(set_attr "type" "alu_ext")]
2266 ;; zero_extend version of above
2267 (define_insn "*add_uxtsi_shift2_uxtw"
2268 [(set (match_operand:DI 0 "register_operand" "=rk")
2271 (ashift:SI (match_operand:SI 1 "register_operand" "r")
2272 (match_operand 2 "aarch64_imm3" "Ui3"))
2273 (match_operand 3 "const_int_operand" "n"))
2274 (match_operand:SI 4 "register_operand" "r"))))]
2275 "aarch64_uxt_size (INTVAL (operands[2]), INTVAL (operands[3])) != 0"
2277 operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
2278 INTVAL (operands[3])));
2279 return \"add\t%w0, %w4, %w1, uxt%e3 %2\";"
2280 [(set_attr "type" "alu_ext")]
2283 (define_insn "*add_uxt<mode>_multp2"
2284 [(set (match_operand:GPI 0 "register_operand" "=rk")
2286 (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2287 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2288 (match_operand 3 "const_int_operand" "n"))
2289 (match_operand:GPI 4 "register_operand" "r")))]
2290 "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])), INTVAL (operands[3])) != 0"
2292 operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
2293 INTVAL (operands[3])));
2294 return \"add\t%<w>0, %<w>4, %<w>1, uxt%e3 %p2\";"
2295 [(set_attr "type" "alu_ext")]
2298 ;; zero_extend version of above
2299 (define_insn "*add_uxtsi_multp2_uxtw"
2300 [(set (match_operand:DI 0 "register_operand" "=rk")
2303 (mult:SI (match_operand:SI 1 "register_operand" "r")
2304 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2305 (match_operand 3 "const_int_operand" "n"))
2306 (match_operand:SI 4 "register_operand" "r"))))]
2307 "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])), INTVAL (operands[3])) != 0"
2309 operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
2310 INTVAL (operands[3])));
2311 return \"add\t%w0, %w4, %w1, uxt%e3 %p2\";"
2312 [(set_attr "type" "alu_ext")]
2315 (define_insn "subsi3"
2316 [(set (match_operand:SI 0 "register_operand" "=rk")
2317 (minus:SI (match_operand:SI 1 "register_operand" "rk")
2318 (match_operand:SI 2 "register_operand" "r")))]
2320 "sub\\t%w0, %w1, %w2"
2321 [(set_attr "type" "alu_sreg")]
2324 ;; zero_extend version of above
2325 (define_insn "*subsi3_uxtw"
2326 [(set (match_operand:DI 0 "register_operand" "=rk")
2328 (minus:SI (match_operand:SI 1 "register_operand" "rk")
2329 (match_operand:SI 2 "register_operand" "r"))))]
2331 "sub\\t%w0, %w1, %w2"
2332 [(set_attr "type" "alu_sreg")]
2335 (define_insn "subdi3"
2336 [(set (match_operand:DI 0 "register_operand" "=rk,w")
2337 (minus:DI (match_operand:DI 1 "register_operand" "rk,w")
2338 (match_operand:DI 2 "register_operand" "r,w")))]
2342 sub\\t%d0, %d1, %d2"
2343 [(set_attr "type" "alu_sreg, neon_sub")
2344 (set_attr "simd" "*,yes")]
2347 (define_expand "subti3"
2348 [(set (match_operand:TI 0 "register_operand" "")
2349 (minus:TI (match_operand:TI 1 "register_operand" "")
2350 (match_operand:TI 2 "register_operand" "")))]
2353 rtx low = gen_reg_rtx (DImode);
2354 emit_insn (gen_subdi3_compare1 (low, gen_lowpart (DImode, operands[1]),
2355 gen_lowpart (DImode, operands[2])));
2357 rtx high = gen_reg_rtx (DImode);
2358 emit_insn (gen_subdi3_carryin (high, gen_highpart (DImode, operands[1]),
2359 gen_highpart (DImode, operands[2])));
2361 emit_move_insn (gen_lowpart (DImode, operands[0]), low);
2362 emit_move_insn (gen_highpart (DImode, operands[0]), high);
2366 (define_insn "*sub<mode>3_compare0"
2367 [(set (reg:CC_NZ CC_REGNUM)
2368 (compare:CC_NZ (minus:GPI (match_operand:GPI 1 "register_operand" "r")
2369 (match_operand:GPI 2 "register_operand" "r"))
2371 (set (match_operand:GPI 0 "register_operand" "=r")
2372 (minus:GPI (match_dup 1) (match_dup 2)))]
2374 "subs\\t%<w>0, %<w>1, %<w>2"
2375 [(set_attr "type" "alus_sreg")]
2378 ;; zero_extend version of above
2379 (define_insn "*subsi3_compare0_uxtw"
2380 [(set (reg:CC_NZ CC_REGNUM)
2381 (compare:CC_NZ (minus:SI (match_operand:SI 1 "register_operand" "r")
2382 (match_operand:SI 2 "register_operand" "r"))
2384 (set (match_operand:DI 0 "register_operand" "=r")
2385 (zero_extend:DI (minus:SI (match_dup 1) (match_dup 2))))]
2387 "subs\\t%w0, %w1, %w2"
2388 [(set_attr "type" "alus_sreg")]
2391 (define_insn "sub<mode>3_compare1"
2392 [(set (reg:CC CC_REGNUM)
2394 (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
2395 (match_operand:GPI 2 "aarch64_reg_or_zero" "rZ")))
2396 (set (match_operand:GPI 0 "register_operand" "=r")
2397 (minus:GPI (match_dup 1) (match_dup 2)))]
2399 "subs\\t%<w>0, %<w>1, %<w>2"
2400 [(set_attr "type" "alus_sreg")]
2403 (define_insn "sub<mode>3_compare1_imm"
2404 [(set (reg:CC CC_REGNUM)
2406 (match_operand:GPI 1 "register_operand" "r")
2407 (match_operand:GPI 3 "const_int_operand" "n")))
2408 (set (match_operand:GPI 0 "register_operand" "=r")
2409 (plus:GPI (match_dup 1)
2410 (match_operand:GPI 2 "aarch64_sub_immediate" "J")))]
2411 "INTVAL (operands[3]) == -INTVAL (operands[2])"
2412 "subs\\t%<w>0, %<w>1, #%n2"
2413 [(set_attr "type" "alus_sreg")]
2417 [(set (match_operand:GPI 0 "register_operand")
2418 (minus:GPI (match_operand:GPI 1 "aarch64_reg_or_zero")
2419 (match_operand:GPI 2 "aarch64_reg_or_zero")))
2420 (set (reg:CC CC_REGNUM)
2424 "!reg_overlap_mentioned_p (operands[0], operands[1])
2425 && !reg_overlap_mentioned_p (operands[0], operands[2])"
2428 emit_insn (gen_sub<mode>3_compare1 (operands[0], operands[1],
2434 ;; Same as the above peephole but with the compare and minus in
2435 ;; swapped order. The restriction on overlap between operand 0
2436 ;; and operands 1 and 2 doesn't apply here.
2438 [(set (reg:CC CC_REGNUM)
2440 (match_operand:GPI 1 "aarch64_reg_or_zero")
2441 (match_operand:GPI 2 "aarch64_reg_or_zero")))
2442 (set (match_operand:GPI 0 "register_operand")
2443 (minus:GPI (match_dup 1)
2448 emit_insn (gen_sub<mode>3_compare1 (operands[0], operands[1],
2455 [(set (match_operand:GPI 0 "register_operand")
2456 (plus:GPI (match_operand:GPI 1 "register_operand")
2457 (match_operand:GPI 2 "aarch64_sub_immediate")))
2458 (set (reg:CC CC_REGNUM)
2461 (match_operand:GPI 3 "const_int_operand")))]
2462 "!reg_overlap_mentioned_p (operands[0], operands[1])
2463 && INTVAL (operands[3]) == -INTVAL (operands[2])"
2466 emit_insn (gen_sub<mode>3_compare1_imm (operands[0], operands[1],
2467 operands[2], operands[3]));
2472 ;; Same as the above peephole but with the compare and minus in
2473 ;; swapped order. The restriction on overlap between operand 0
2474 ;; and operands 1 doesn't apply here.
2476 [(set (reg:CC CC_REGNUM)
2478 (match_operand:GPI 1 "register_operand")
2479 (match_operand:GPI 3 "const_int_operand")))
2480 (set (match_operand:GPI 0 "register_operand")
2481 (plus:GPI (match_dup 1)
2482 (match_operand:GPI 2 "aarch64_sub_immediate")))]
2483 "INTVAL (operands[3]) == -INTVAL (operands[2])"
2486 emit_insn (gen_sub<mode>3_compare1_imm (operands[0], operands[1],
2487 operands[2], operands[3]));
2492 (define_insn "*sub_<shift>_<mode>"
2493 [(set (match_operand:GPI 0 "register_operand" "=r")
2494 (minus:GPI (match_operand:GPI 3 "register_operand" "r")
2496 (match_operand:GPI 1 "register_operand" "r")
2497 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
2499 "sub\\t%<w>0, %<w>3, %<w>1, <shift> %2"
2500 [(set_attr "type" "alu_shift_imm")]
2503 ;; zero_extend version of above
2504 (define_insn "*sub_<shift>_si_uxtw"
2505 [(set (match_operand:DI 0 "register_operand" "=r")
2507 (minus:SI (match_operand:SI 3 "register_operand" "r")
2509 (match_operand:SI 1 "register_operand" "r")
2510 (match_operand:QI 2 "aarch64_shift_imm_si" "n")))))]
2512 "sub\\t%w0, %w3, %w1, <shift> %2"
2513 [(set_attr "type" "alu_shift_imm")]
2516 (define_insn "*sub_mul_imm_<mode>"
2517 [(set (match_operand:GPI 0 "register_operand" "=r")
2518 (minus:GPI (match_operand:GPI 3 "register_operand" "r")
2520 (match_operand:GPI 1 "register_operand" "r")
2521 (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))))]
2523 "sub\\t%<w>0, %<w>3, %<w>1, lsl %p2"
2524 [(set_attr "type" "alu_shift_imm")]
2527 ;; zero_extend version of above
2528 (define_insn "*sub_mul_imm_si_uxtw"
2529 [(set (match_operand:DI 0 "register_operand" "=r")
2531 (minus:SI (match_operand:SI 3 "register_operand" "r")
2533 (match_operand:SI 1 "register_operand" "r")
2534 (match_operand:QI 2 "aarch64_pwr_2_si" "n")))))]
2536 "sub\\t%w0, %w3, %w1, lsl %p2"
2537 [(set_attr "type" "alu_shift_imm")]
2540 (define_insn "*sub_<optab><ALLX:mode>_<GPI:mode>"
2541 [(set (match_operand:GPI 0 "register_operand" "=rk")
2542 (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
2544 (match_operand:ALLX 2 "register_operand" "r"))))]
2546 "sub\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size>"
2547 [(set_attr "type" "alu_ext")]
2550 ;; zero_extend version of above
2551 (define_insn "*sub_<optab><SHORT:mode>_si_uxtw"
2552 [(set (match_operand:DI 0 "register_operand" "=rk")
2554 (minus:SI (match_operand:SI 1 "register_operand" "rk")
2556 (match_operand:SHORT 2 "register_operand" "r")))))]
2558 "sub\\t%w0, %w1, %w2, <su>xt<SHORT:size>"
2559 [(set_attr "type" "alu_ext")]
2562 (define_insn "*sub_<optab><ALLX:mode>_shft_<GPI:mode>"
2563 [(set (match_operand:GPI 0 "register_operand" "=rk")
2564 (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
2565 (ashift:GPI (ANY_EXTEND:GPI
2566 (match_operand:ALLX 2 "register_operand" "r"))
2567 (match_operand 3 "aarch64_imm3" "Ui3"))))]
2569 "sub\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size> %3"
2570 [(set_attr "type" "alu_ext")]
2573 ;; zero_extend version of above
2574 (define_insn "*sub_<optab><SHORT:mode>_shft_si_uxtw"
2575 [(set (match_operand:DI 0 "register_operand" "=rk")
2577 (minus:SI (match_operand:SI 1 "register_operand" "rk")
2578 (ashift:SI (ANY_EXTEND:SI
2579 (match_operand:SHORT 2 "register_operand" "r"))
2580 (match_operand 3 "aarch64_imm3" "Ui3")))))]
2582 "sub\\t%w0, %w1, %w2, <su>xt<SHORT:size> %3"
2583 [(set_attr "type" "alu_ext")]
2586 (define_insn "*sub_<optab><mode>_multp2"
2587 [(set (match_operand:GPI 0 "register_operand" "=rk")
2588 (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
2590 (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2591 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2592 (match_operand 3 "const_int_operand" "n")
2594 "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
2595 "sub\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
2596 [(set_attr "type" "alu_ext")]
2599 ;; zero_extend version of above
2600 (define_insn "*sub_<optab>si_multp2_uxtw"
2601 [(set (match_operand:DI 0 "register_operand" "=rk")
2603 (minus:SI (match_operand:SI 4 "register_operand" "rk")
2605 (mult:SI (match_operand:SI 1 "register_operand" "r")
2606 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2607 (match_operand 3 "const_int_operand" "n")
2609 "aarch64_is_extend_from_extract (SImode, operands[2], operands[3])"
2610 "sub\\t%w0, %w4, %w1, <su>xt%e3 %p2"
2611 [(set_attr "type" "alu_ext")]
2614 ;; The hardware description is op1 + ~op2 + C.
2615 ;; = op1 + (-op2 + 1) + (1 - !C)
2616 ;; = op1 - op2 - 1 + 1 - !C
2617 ;; = op1 - op2 - !C.
2618 ;; We describe the latter.
2620 (define_insn "*sub<mode>3_carryin0"
2621 [(set (match_operand:GPI 0 "register_operand" "=r")
2623 (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
2624 (match_operand:GPI 2 "aarch64_borrow_operation" "")))]
2626 "sbc\\t%<w>0, %<w>1, <w>zr"
2627 [(set_attr "type" "adc_reg")]
2630 ;; zero_extend version of the above
2631 (define_insn "*subsi3_carryin_uxtw"
2632 [(set (match_operand:DI 0 "register_operand" "=r")
2635 (match_operand:SI 1 "aarch64_reg_or_zero" "rZ")
2636 (match_operand:SI 2 "aarch64_borrow_operation" ""))))]
2638 "sbc\\t%w0, %w1, wzr"
2639 [(set_attr "type" "adc_reg")]
2642 (define_expand "sub<mode>3_carryin"
2643 [(set (match_operand:GPI 0 "register_operand")
2646 (match_operand:GPI 1 "aarch64_reg_or_zero")
2647 (match_operand:GPI 2 "register_operand"))
2648 (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))))]
2653 (define_insn "*sub<mode>3_carryin"
2654 [(set (match_operand:GPI 0 "register_operand" "=r")
2657 (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
2658 (match_operand:GPI 2 "register_operand" "r"))
2659 (match_operand:GPI 3 "aarch64_borrow_operation" "")))]
2662 "sbc\\t%<w>0, %<w>1, %<w>2"
2663 [(set_attr "type" "adc_reg")]
2666 ;; zero_extend version of the above
2667 (define_insn "*subsi3_carryin_uxtw"
2668 [(set (match_operand:DI 0 "register_operand" "=r")
2672 (match_operand:SI 1 "aarch64_reg_or_zero" "rZ")
2673 (match_operand:SI 2 "register_operand" "r"))
2674 (match_operand:SI 3 "aarch64_borrow_operation" ""))))]
2677 "sbc\\t%w0, %w1, %w2"
2678 [(set_attr "type" "adc_reg")]
2681 (define_insn "*sub<mode>3_carryin_alt"
2682 [(set (match_operand:GPI 0 "register_operand" "=r")
2685 (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
2686 (match_operand:GPI 3 "aarch64_borrow_operation" ""))
2687 (match_operand:GPI 2 "register_operand" "r")))]
2689 "sbc\\t%<w>0, %<w>1, %<w>2"
2690 [(set_attr "type" "adc_reg")]
2693 ;; zero_extend version of the above
2694 (define_insn "*subsi3_carryin_alt_uxtw"
2695 [(set (match_operand:DI 0 "register_operand" "=r")
2699 (match_operand:SI 1 "aarch64_reg_or_zero" "rZ")
2700 (match_operand:SI 3 "aarch64_borrow_operation" ""))
2701 (match_operand:SI 2 "register_operand" "r"))))]
2703 "sbc\\t%w0, %w1, %w2"
2704 [(set_attr "type" "adc_reg")]
2707 (define_insn "*sub_uxt<mode>_shift2"
2708 [(set (match_operand:GPI 0 "register_operand" "=rk")
2709 (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
2711 (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
2712 (match_operand 2 "aarch64_imm3" "Ui3"))
2713 (match_operand 3 "const_int_operand" "n"))))]
2714 "aarch64_uxt_size (INTVAL (operands[2]),INTVAL (operands[3])) != 0"
2716 operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
2717 INTVAL (operands[3])));
2718 return \"sub\t%<w>0, %<w>4, %<w>1, uxt%e3 %2\";"
2719 [(set_attr "type" "alu_ext")]
2722 ;; zero_extend version of above
2723 (define_insn "*sub_uxtsi_shift2_uxtw"
2724 [(set (match_operand:DI 0 "register_operand" "=rk")
2726 (minus:SI (match_operand:SI 4 "register_operand" "rk")
2728 (ashift:SI (match_operand:SI 1 "register_operand" "r")
2729 (match_operand 2 "aarch64_imm3" "Ui3"))
2730 (match_operand 3 "const_int_operand" "n")))))]
2731 "aarch64_uxt_size (INTVAL (operands[2]),INTVAL (operands[3])) != 0"
2733 operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
2734 INTVAL (operands[3])));
2735 return \"sub\t%w0, %w4, %w1, uxt%e3 %2\";"
2736 [(set_attr "type" "alu_ext")]
2739 (define_insn "*sub_uxt<mode>_multp2"
2740 [(set (match_operand:GPI 0 "register_operand" "=rk")
2741 (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
2743 (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2744 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2745 (match_operand 3 "const_int_operand" "n"))))]
2746 "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),INTVAL (operands[3])) != 0"
2748 operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
2749 INTVAL (operands[3])));
2750 return \"sub\t%<w>0, %<w>4, %<w>1, uxt%e3 %p2\";"
2751 [(set_attr "type" "alu_ext")]
2754 ;; zero_extend version of above
2755 (define_insn "*sub_uxtsi_multp2_uxtw"
2756 [(set (match_operand:DI 0 "register_operand" "=rk")
2758 (minus:SI (match_operand:SI 4 "register_operand" "rk")
2760 (mult:SI (match_operand:SI 1 "register_operand" "r")
2761 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2762 (match_operand 3 "const_int_operand" "n")))))]
2763 "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),INTVAL (operands[3])) != 0"
2765 operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
2766 INTVAL (operands[3])));
2767 return \"sub\t%w0, %w4, %w1, uxt%e3 %p2\";"
2768 [(set_attr "type" "alu_ext")]
2771 (define_expand "abs<mode>2"
2772 [(match_operand:GPI 0 "register_operand" "")
2773 (match_operand:GPI 1 "register_operand" "")]
2776 rtx ccreg = aarch64_gen_compare_reg (LT, operands[1], const0_rtx);
2777 rtx x = gen_rtx_LT (VOIDmode, ccreg, const0_rtx);
2778 emit_insn (gen_csneg3<mode>_insn (operands[0], x, operands[1], operands[1]));
2783 (define_insn "neg<mode>2"
2784 [(set (match_operand:GPI 0 "register_operand" "=r,w")
2785 (neg:GPI (match_operand:GPI 1 "register_operand" "r,w")))]
2789 neg\\t%<rtn>0<vas>, %<rtn>1<vas>"
2790 [(set_attr "type" "alu_sreg, neon_neg<q>")
2791 (set_attr "simd" "*,yes")]
2794 ;; zero_extend version of above
2795 (define_insn "*negsi2_uxtw"
2796 [(set (match_operand:DI 0 "register_operand" "=r")
2797 (zero_extend:DI (neg:SI (match_operand:SI 1 "register_operand" "r"))))]
2800 [(set_attr "type" "alu_sreg")]
2803 (define_insn "*ngc<mode>"
2804 [(set (match_operand:GPI 0 "register_operand" "=r")
2806 (neg:GPI (match_operand:GPI 2 "aarch64_borrow_operation" ""))
2807 (match_operand:GPI 1 "register_operand" "r")))]
2809 "ngc\\t%<w>0, %<w>1"
2810 [(set_attr "type" "adc_reg")]
2813 (define_insn "*ngcsi_uxtw"
2814 [(set (match_operand:DI 0 "register_operand" "=r")
2817 (neg:SI (match_operand:SI 2 "aarch64_borrow_operation" ""))
2818 (match_operand:SI 1 "register_operand" "r"))))]
2821 [(set_attr "type" "adc_reg")]
2824 (define_insn "neg<mode>2_compare0"
2825 [(set (reg:CC_NZ CC_REGNUM)
2826 (compare:CC_NZ (neg:GPI (match_operand:GPI 1 "register_operand" "r"))
2828 (set (match_operand:GPI 0 "register_operand" "=r")
2829 (neg:GPI (match_dup 1)))]
2831 "negs\\t%<w>0, %<w>1"
2832 [(set_attr "type" "alus_sreg")]
2835 ;; zero_extend version of above
2836 (define_insn "*negsi2_compare0_uxtw"
2837 [(set (reg:CC_NZ CC_REGNUM)
2838 (compare:CC_NZ (neg:SI (match_operand:SI 1 "register_operand" "r"))
2840 (set (match_operand:DI 0 "register_operand" "=r")
2841 (zero_extend:DI (neg:SI (match_dup 1))))]
2844 [(set_attr "type" "alus_sreg")]
2847 (define_insn "*neg_<shift><mode>3_compare0"
2848 [(set (reg:CC_NZ CC_REGNUM)
2850 (neg:GPI (ASHIFT:GPI
2851 (match_operand:GPI 1 "register_operand" "r")
2852 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
2854 (set (match_operand:GPI 0 "register_operand" "=r")
2855 (neg:GPI (ASHIFT:GPI (match_dup 1) (match_dup 2))))]
2857 "negs\\t%<w>0, %<w>1, <shift> %2"
2858 [(set_attr "type" "alus_shift_imm")]
2861 (define_insn "*neg_<shift>_<mode>2"
2862 [(set (match_operand:GPI 0 "register_operand" "=r")
2863 (neg:GPI (ASHIFT:GPI
2864 (match_operand:GPI 1 "register_operand" "r")
2865 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
2867 "neg\\t%<w>0, %<w>1, <shift> %2"
2868 [(set_attr "type" "alu_shift_imm")]
2871 ;; zero_extend version of above
2872 (define_insn "*neg_<shift>_si2_uxtw"
2873 [(set (match_operand:DI 0 "register_operand" "=r")
2876 (match_operand:SI 1 "register_operand" "r")
2877 (match_operand:QI 2 "aarch64_shift_imm_si" "n")))))]
2879 "neg\\t%w0, %w1, <shift> %2"
2880 [(set_attr "type" "alu_shift_imm")]
2883 (define_insn "*neg_mul_imm_<mode>2"
2884 [(set (match_operand:GPI 0 "register_operand" "=r")
2886 (match_operand:GPI 1 "register_operand" "r")
2887 (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))))]
2889 "neg\\t%<w>0, %<w>1, lsl %p2"
2890 [(set_attr "type" "alu_shift_imm")]
2893 ;; zero_extend version of above
2894 (define_insn "*neg_mul_imm_si2_uxtw"
2895 [(set (match_operand:DI 0 "register_operand" "=r")
2898 (match_operand:SI 1 "register_operand" "r")
2899 (match_operand:QI 2 "aarch64_pwr_2_si" "n")))))]
2901 "neg\\t%w0, %w1, lsl %p2"
2902 [(set_attr "type" "alu_shift_imm")]
2905 (define_insn "mul<mode>3"
2906 [(set (match_operand:GPI 0 "register_operand" "=r")
2907 (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2908 (match_operand:GPI 2 "register_operand" "r")))]
2910 "mul\\t%<w>0, %<w>1, %<w>2"
2911 [(set_attr "type" "mul")]
2914 ;; zero_extend version of above
2915 (define_insn "*mulsi3_uxtw"
2916 [(set (match_operand:DI 0 "register_operand" "=r")
2918 (mult:SI (match_operand:SI 1 "register_operand" "r")
2919 (match_operand:SI 2 "register_operand" "r"))))]
2921 "mul\\t%w0, %w1, %w2"
2922 [(set_attr "type" "mul")]
2925 (define_insn "madd<mode>"
2926 [(set (match_operand:GPI 0 "register_operand" "=r")
2927 (plus:GPI (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2928 (match_operand:GPI 2 "register_operand" "r"))
2929 (match_operand:GPI 3 "register_operand" "r")))]
2931 "madd\\t%<w>0, %<w>1, %<w>2, %<w>3"
2932 [(set_attr "type" "mla")]
2935 ;; zero_extend version of above
2936 (define_insn "*maddsi_uxtw"
2937 [(set (match_operand:DI 0 "register_operand" "=r")
2939 (plus:SI (mult:SI (match_operand:SI 1 "register_operand" "r")
2940 (match_operand:SI 2 "register_operand" "r"))
2941 (match_operand:SI 3 "register_operand" "r"))))]
2943 "madd\\t%w0, %w1, %w2, %w3"
2944 [(set_attr "type" "mla")]
2947 (define_insn "*msub<mode>"
2948 [(set (match_operand:GPI 0 "register_operand" "=r")
2949 (minus:GPI (match_operand:GPI 3 "register_operand" "r")
2950 (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2951 (match_operand:GPI 2 "register_operand" "r"))))]
2954 "msub\\t%<w>0, %<w>1, %<w>2, %<w>3"
2955 [(set_attr "type" "mla")]
2958 ;; zero_extend version of above
2959 (define_insn "*msubsi_uxtw"
2960 [(set (match_operand:DI 0 "register_operand" "=r")
2962 (minus:SI (match_operand:SI 3 "register_operand" "r")
2963 (mult:SI (match_operand:SI 1 "register_operand" "r")
2964 (match_operand:SI 2 "register_operand" "r")))))]
2967 "msub\\t%w0, %w1, %w2, %w3"
2968 [(set_attr "type" "mla")]
2971 (define_insn "*mul<mode>_neg"
2972 [(set (match_operand:GPI 0 "register_operand" "=r")
2973 (mult:GPI (neg:GPI (match_operand:GPI 1 "register_operand" "r"))
2974 (match_operand:GPI 2 "register_operand" "r")))]
2977 "mneg\\t%<w>0, %<w>1, %<w>2"
2978 [(set_attr "type" "mul")]
2981 ;; zero_extend version of above
2982 (define_insn "*mulsi_neg_uxtw"
2983 [(set (match_operand:DI 0 "register_operand" "=r")
2985 (mult:SI (neg:SI (match_operand:SI 1 "register_operand" "r"))
2986 (match_operand:SI 2 "register_operand" "r"))))]
2989 "mneg\\t%w0, %w1, %w2"
2990 [(set_attr "type" "mul")]
2993 (define_insn "<su_optab>mulsidi3"
2994 [(set (match_operand:DI 0 "register_operand" "=r")
2995 (mult:DI (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
2996 (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r"))))]
2998 "<su>mull\\t%0, %w1, %w2"
2999 [(set_attr "type" "<su>mull")]
3002 (define_insn "<su_optab>maddsidi4"
3003 [(set (match_operand:DI 0 "register_operand" "=r")
3005 (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
3006 (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r")))
3007 (match_operand:DI 3 "register_operand" "r")))]
3009 "<su>maddl\\t%0, %w1, %w2, %3"
3010 [(set_attr "type" "<su>mlal")]
3013 (define_insn "<su_optab>msubsidi4"
3014 [(set (match_operand:DI 0 "register_operand" "=r")
3016 (match_operand:DI 3 "register_operand" "r")
3017 (mult:DI (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
3019 (match_operand:SI 2 "register_operand" "r")))))]
3021 "<su>msubl\\t%0, %w1, %w2, %3"
3022 [(set_attr "type" "<su>mlal")]
3025 (define_insn "*<su_optab>mulsidi_neg"
3026 [(set (match_operand:DI 0 "register_operand" "=r")
3028 (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r")))
3029 (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r"))))]
3031 "<su>mnegl\\t%0, %w1, %w2"
3032 [(set_attr "type" "<su>mull")]
3035 (define_expand "<su_optab>mulditi3"
3036 [(set (match_operand:TI 0 "register_operand")
3037 (mult:TI (ANY_EXTEND:TI (match_operand:DI 1 "register_operand"))
3038 (ANY_EXTEND:TI (match_operand:DI 2 "register_operand"))))]
3041 rtx low = gen_reg_rtx (DImode);
3042 emit_insn (gen_muldi3 (low, operands[1], operands[2]));
3044 rtx high = gen_reg_rtx (DImode);
3045 emit_insn (gen_<su>muldi3_highpart (high, operands[1], operands[2]));
3047 emit_move_insn (gen_lowpart (DImode, operands[0]), low);
3048 emit_move_insn (gen_highpart (DImode, operands[0]), high);
3052 ;; The default expansion of multi3 using umuldi3_highpart will perform
3053 ;; the additions in an order that fails to combine into two madd insns.
3054 (define_expand "multi3"
3055 [(set (match_operand:TI 0 "register_operand")
3056 (mult:TI (match_operand:TI 1 "register_operand")
3057 (match_operand:TI 2 "register_operand")))]
3060 rtx l0 = gen_reg_rtx (DImode);
3061 rtx l1 = gen_lowpart (DImode, operands[1]);
3062 rtx l2 = gen_lowpart (DImode, operands[2]);
3063 rtx h0 = gen_reg_rtx (DImode);
3064 rtx h1 = gen_highpart (DImode, operands[1]);
3065 rtx h2 = gen_highpart (DImode, operands[2]);
3067 emit_insn (gen_muldi3 (l0, l1, l2));
3068 emit_insn (gen_umuldi3_highpart (h0, l1, l2));
3069 emit_insn (gen_madddi (h0, h1, l2, h0));
3070 emit_insn (gen_madddi (h0, l1, h2, h0));
3072 emit_move_insn (gen_lowpart (DImode, operands[0]), l0);
3073 emit_move_insn (gen_highpart (DImode, operands[0]), h0);
3077 (define_insn "<su>muldi3_highpart"
3078 [(set (match_operand:DI 0 "register_operand" "=r")
3082 (ANY_EXTEND:TI (match_operand:DI 1 "register_operand" "r"))
3083 (ANY_EXTEND:TI (match_operand:DI 2 "register_operand" "r")))
3086 "<su>mulh\\t%0, %1, %2"
3087 [(set_attr "type" "<su>mull")]
3090 (define_insn "<su_optab>div<mode>3"
3091 [(set (match_operand:GPI 0 "register_operand" "=r")
3092 (ANY_DIV:GPI (match_operand:GPI 1 "register_operand" "r")
3093 (match_operand:GPI 2 "register_operand" "r")))]
3095 "<su>div\\t%<w>0, %<w>1, %<w>2"
3096 [(set_attr "type" "<su>div")]
3099 ;; zero_extend version of above
3100 (define_insn "*<su_optab>divsi3_uxtw"
3101 [(set (match_operand:DI 0 "register_operand" "=r")
3103 (ANY_DIV:SI (match_operand:SI 1 "register_operand" "r")
3104 (match_operand:SI 2 "register_operand" "r"))))]
3106 "<su>div\\t%w0, %w1, %w2"
3107 [(set_attr "type" "<su>div")]
3110 ;; -------------------------------------------------------------------
3112 ;; -------------------------------------------------------------------
3114 (define_insn "cmp<mode>"
3115 [(set (reg:CC CC_REGNUM)
3116 (compare:CC (match_operand:GPI 0 "register_operand" "r,r,r")
3117 (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J")))]
3123 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
3126 (define_insn "fcmp<mode>"
3127 [(set (reg:CCFP CC_REGNUM)
3128 (compare:CCFP (match_operand:GPF 0 "register_operand" "w,w")
3129 (match_operand:GPF 1 "aarch64_fp_compare_operand" "Y,w")))]
3133 fcmp\\t%<s>0, %<s>1"
3134 [(set_attr "type" "fcmp<s>")]
3137 (define_insn "fcmpe<mode>"
3138 [(set (reg:CCFPE CC_REGNUM)
3139 (compare:CCFPE (match_operand:GPF 0 "register_operand" "w,w")
3140 (match_operand:GPF 1 "aarch64_fp_compare_operand" "Y,w")))]
3144 fcmpe\\t%<s>0, %<s>1"
3145 [(set_attr "type" "fcmp<s>")]
3148 (define_insn "*cmp_swp_<shift>_reg<mode>"
3149 [(set (reg:CC_SWP CC_REGNUM)
3150 (compare:CC_SWP (ASHIFT:GPI
3151 (match_operand:GPI 0 "register_operand" "r")
3152 (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n"))
3153 (match_operand:GPI 2 "aarch64_reg_or_zero" "rZ")))]
3155 "cmp\\t%<w>2, %<w>0, <shift> %1"
3156 [(set_attr "type" "alus_shift_imm")]
3159 (define_insn "*cmp_swp_<optab><ALLX:mode>_reg<GPI:mode>"
3160 [(set (reg:CC_SWP CC_REGNUM)
3161 (compare:CC_SWP (ANY_EXTEND:GPI
3162 (match_operand:ALLX 0 "register_operand" "r"))
3163 (match_operand:GPI 1 "register_operand" "r")))]
3165 "cmp\\t%<GPI:w>1, %<GPI:w>0, <su>xt<ALLX:size>"
3166 [(set_attr "type" "alus_ext")]
3169 (define_insn "*cmp_swp_<optab><ALLX:mode>_shft_<GPI:mode>"
3170 [(set (reg:CC_SWP CC_REGNUM)
3171 (compare:CC_SWP (ashift:GPI
3173 (match_operand:ALLX 0 "register_operand" "r"))
3174 (match_operand 1 "aarch64_imm3" "Ui3"))
3175 (match_operand:GPI 2 "register_operand" "r")))]
3177 "cmp\\t%<GPI:w>2, %<GPI:w>0, <su>xt<ALLX:size> %1"
3178 [(set_attr "type" "alus_ext")]
3181 ;; -------------------------------------------------------------------
3182 ;; Store-flag and conditional select insns
3183 ;; -------------------------------------------------------------------
3185 (define_expand "cstore<mode>4"
3186 [(set (match_operand:SI 0 "register_operand" "")
3187 (match_operator:SI 1 "aarch64_comparison_operator"
3188 [(match_operand:GPI 2 "register_operand" "")
3189 (match_operand:GPI 3 "aarch64_plus_operand" "")]))]
3192 operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
3194 operands[3] = const0_rtx;
3198 (define_expand "cstorecc4"
3199 [(set (match_operand:SI 0 "register_operand")
3200 (match_operator 1 "aarch64_comparison_operator_mode"
3201 [(match_operand 2 "cc_register")
3202 (match_operand 3 "const0_operand")]))]
3205 emit_insn (gen_rtx_SET (operands[0], operands[1]));
3210 (define_expand "cstore<mode>4"
3211 [(set (match_operand:SI 0 "register_operand" "")
3212 (match_operator:SI 1 "aarch64_comparison_operator_mode"
3213 [(match_operand:GPF 2 "register_operand" "")
3214 (match_operand:GPF 3 "aarch64_fp_compare_operand" "")]))]
3217 operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
3219 operands[3] = const0_rtx;
3223 (define_insn "aarch64_cstore<mode>"
3224 [(set (match_operand:ALLI 0 "register_operand" "=r")
3225 (match_operator:ALLI 1 "aarch64_comparison_operator_mode"
3226 [(match_operand 2 "cc_register" "") (const_int 0)]))]
3229 [(set_attr "type" "csel")]
3232 ;; For a 24-bit immediate CST we can optimize the compare for equality
3233 ;; and branch sequence from:
3235 ;; movk x0, #imm2, lsl 16 /* x0 contains CST. */
3238 ;; into the shorter:
3239 ;; sub x0, x1, #(CST & 0xfff000)
3240 ;; subs x0, x0, #(CST & 0x000fff)
3241 ;; cset x2, <ne, eq>.
3242 (define_insn_and_split "*compare_cstore<mode>_insn"
3243 [(set (match_operand:GPI 0 "register_operand" "=r")
3244 (EQL:GPI (match_operand:GPI 1 "register_operand" "r")
3245 (match_operand:GPI 2 "aarch64_imm24" "n")))
3246 (clobber (reg:CC CC_REGNUM))]
3247 "!aarch64_move_imm (INTVAL (operands[2]), <MODE>mode)
3248 && !aarch64_plus_operand (operands[2], <MODE>mode)
3249 && !reload_completed"
3254 HOST_WIDE_INT lo_imm = UINTVAL (operands[2]) & 0xfff;
3255 HOST_WIDE_INT hi_imm = UINTVAL (operands[2]) & 0xfff000;
3256 rtx tmp = gen_reg_rtx (<MODE>mode);
3257 emit_insn (gen_add<mode>3 (tmp, operands[1], GEN_INT (-hi_imm)));
3258 emit_insn (gen_add<mode>3_compare0 (tmp, tmp, GEN_INT (-lo_imm)));
3259 rtx cc_reg = gen_rtx_REG (CC_NZmode, CC_REGNUM);
3260 rtx cmp_rtx = gen_rtx_fmt_ee (<EQL:CMP>, <MODE>mode, cc_reg, const0_rtx);
3261 emit_insn (gen_aarch64_cstore<mode> (operands[0], cmp_rtx, cc_reg));
3264 [(set_attr "type" "csel")]
3267 ;; zero_extend version of the above
3268 (define_insn "*cstoresi_insn_uxtw"
3269 [(set (match_operand:DI 0 "register_operand" "=r")
3271 (match_operator:SI 1 "aarch64_comparison_operator_mode"
3272 [(match_operand 2 "cc_register" "") (const_int 0)])))]
3275 [(set_attr "type" "csel")]
3278 (define_insn "cstore<mode>_neg"
3279 [(set (match_operand:ALLI 0 "register_operand" "=r")
3280 (neg:ALLI (match_operator:ALLI 1 "aarch64_comparison_operator_mode"
3281 [(match_operand 2 "cc_register" "") (const_int 0)])))]
3283 "csetm\\t%<w>0, %m1"
3284 [(set_attr "type" "csel")]
3287 ;; zero_extend version of the above
3288 (define_insn "*cstoresi_neg_uxtw"
3289 [(set (match_operand:DI 0 "register_operand" "=r")
3291 (neg:SI (match_operator:SI 1 "aarch64_comparison_operator_mode"
3292 [(match_operand 2 "cc_register" "") (const_int 0)]))))]
3295 [(set_attr "type" "csel")]
3298 (define_expand "cmov<mode>6"
3299 [(set (match_operand:GPI 0 "register_operand" "")
3301 (match_operator 1 "aarch64_comparison_operator"
3302 [(match_operand:GPI 2 "register_operand" "")
3303 (match_operand:GPI 3 "aarch64_plus_operand" "")])
3304 (match_operand:GPI 4 "register_operand" "")
3305 (match_operand:GPI 5 "register_operand" "")))]
3308 operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
3310 operands[3] = const0_rtx;
3314 (define_expand "cmov<mode>6"
3315 [(set (match_operand:GPF 0 "register_operand" "")
3317 (match_operator 1 "aarch64_comparison_operator"
3318 [(match_operand:GPF 2 "register_operand" "")
3319 (match_operand:GPF 3 "aarch64_fp_compare_operand" "")])
3320 (match_operand:GPF 4 "register_operand" "")
3321 (match_operand:GPF 5 "register_operand" "")))]
3324 operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
3326 operands[3] = const0_rtx;
3330 (define_insn "*cmov<mode>_insn"
3331 [(set (match_operand:ALLI 0 "register_operand" "=r,r,r,r,r,r,r")
3333 (match_operator 1 "aarch64_comparison_operator"
3334 [(match_operand 2 "cc_register" "") (const_int 0)])
3335 (match_operand:ALLI 3 "aarch64_reg_zero_or_m1_or_1" "rZ,rZ,UsM,rZ,Ui1,UsM,Ui1")
3336 (match_operand:ALLI 4 "aarch64_reg_zero_or_m1_or_1" "rZ,UsM,rZ,Ui1,rZ,UsM,Ui1")))]
3337 "!((operands[3] == const1_rtx && operands[4] == constm1_rtx)
3338 || (operands[3] == constm1_rtx && operands[4] == const1_rtx))"
3339 ;; Final two alternatives should be unreachable, but included for completeness
3341 csel\\t%<w>0, %<w>3, %<w>4, %m1
3342 csinv\\t%<w>0, %<w>3, <w>zr, %m1
3343 csinv\\t%<w>0, %<w>4, <w>zr, %M1
3344 csinc\\t%<w>0, %<w>3, <w>zr, %m1
3345 csinc\\t%<w>0, %<w>4, <w>zr, %M1
3348 [(set_attr "type" "csel, csel, csel, csel, csel, mov_imm, mov_imm")]
3351 ;; zero_extend version of above
3352 (define_insn "*cmovsi_insn_uxtw"
3353 [(set (match_operand:DI 0 "register_operand" "=r,r,r,r,r,r,r")
3356 (match_operator 1 "aarch64_comparison_operator"
3357 [(match_operand 2 "cc_register" "") (const_int 0)])
3358 (match_operand:SI 3 "aarch64_reg_zero_or_m1_or_1" "rZ,rZ,UsM,rZ,Ui1,UsM,Ui1")
3359 (match_operand:SI 4 "aarch64_reg_zero_or_m1_or_1" "rZ,UsM,rZ,Ui1,rZ,UsM,Ui1"))))]
3360 "!((operands[3] == const1_rtx && operands[4] == constm1_rtx)
3361 || (operands[3] == constm1_rtx && operands[4] == const1_rtx))"
3362 ;; Final two alternatives should be unreachable, but included for completeness
3364 csel\\t%w0, %w3, %w4, %m1
3365 csinv\\t%w0, %w3, wzr, %m1
3366 csinv\\t%w0, %w4, wzr, %M1
3367 csinc\\t%w0, %w3, wzr, %m1
3368 csinc\\t%w0, %w4, wzr, %M1
3371 [(set_attr "type" "csel, csel, csel, csel, csel, mov_imm, mov_imm")]
3374 (define_insn "*cmovdi_insn_uxtw"
3375 [(set (match_operand:DI 0 "register_operand" "=r")
3377 (match_operator 1 "aarch64_comparison_operator"
3378 [(match_operand 2 "cc_register" "") (const_int 0)])
3379 (zero_extend:DI (match_operand:SI 3 "register_operand" "r"))
3380 (zero_extend:DI (match_operand:SI 4 "register_operand" "r"))))]
3382 "csel\\t%w0, %w3, %w4, %m1"
3383 [(set_attr "type" "csel")]
3386 (define_insn "*cmov<mode>_insn"
3387 [(set (match_operand:GPF 0 "register_operand" "=w")
3389 (match_operator 1 "aarch64_comparison_operator"
3390 [(match_operand 2 "cc_register" "") (const_int 0)])
3391 (match_operand:GPF 3 "register_operand" "w")
3392 (match_operand:GPF 4 "register_operand" "w")))]
3394 "fcsel\\t%<s>0, %<s>3, %<s>4, %m1"
3395 [(set_attr "type" "fcsel")]
3398 (define_expand "mov<mode>cc"
3399 [(set (match_operand:ALLI 0 "register_operand" "")
3400 (if_then_else:ALLI (match_operand 1 "aarch64_comparison_operator" "")
3401 (match_operand:ALLI 2 "register_operand" "")
3402 (match_operand:ALLI 3 "register_operand" "")))]
3406 enum rtx_code code = GET_CODE (operands[1]);
3408 if (code == UNEQ || code == LTGT)
3411 ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
3412 XEXP (operands[1], 1));
3413 operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
3417 (define_expand "mov<GPF:mode><GPI:mode>cc"
3418 [(set (match_operand:GPI 0 "register_operand" "")
3419 (if_then_else:GPI (match_operand 1 "aarch64_comparison_operator" "")
3420 (match_operand:GPF 2 "register_operand" "")
3421 (match_operand:GPF 3 "register_operand" "")))]
3425 enum rtx_code code = GET_CODE (operands[1]);
3427 if (code == UNEQ || code == LTGT)
3430 ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
3431 XEXP (operands[1], 1));
3432 operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
3436 (define_expand "mov<mode>cc"
3437 [(set (match_operand:GPF 0 "register_operand" "")
3438 (if_then_else:GPF (match_operand 1 "aarch64_comparison_operator" "")
3439 (match_operand:GPF 2 "register_operand" "")
3440 (match_operand:GPF 3 "register_operand" "")))]
3444 enum rtx_code code = GET_CODE (operands[1]);
3446 if (code == UNEQ || code == LTGT)
3449 ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
3450 XEXP (operands[1], 1));
3451 operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
3455 (define_expand "<neg_not_op><mode>cc"
3456 [(set (match_operand:GPI 0 "register_operand" "")
3457 (if_then_else:GPI (match_operand 1 "aarch64_comparison_operator" "")
3458 (NEG_NOT:GPI (match_operand:GPI 2 "register_operand" ""))
3459 (match_operand:GPI 3 "register_operand" "")))]
3463 enum rtx_code code = GET_CODE (operands[1]);
3465 if (code == UNEQ || code == LTGT)
3468 ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
3469 XEXP (operands[1], 1));
3470 operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
3474 ;; CRC32 instructions.
3475 (define_insn "aarch64_<crc_variant>"
3476 [(set (match_operand:SI 0 "register_operand" "=r")
3477 (unspec:SI [(match_operand:SI 1 "register_operand" "r")
3478 (match_operand:<crc_mode> 2 "register_operand" "r")]
3482 if (GET_MODE_BITSIZE (<crc_mode>mode) >= 64)
3483 return "<crc_variant>\\t%w0, %w1, %x2";
3485 return "<crc_variant>\\t%w0, %w1, %w2";
3487 [(set_attr "type" "crc")]
3490 (define_insn "*csinc2<mode>_insn"
3491 [(set (match_operand:GPI 0 "register_operand" "=r")
3492 (plus:GPI (match_operand 2 "aarch64_comparison_operation" "")
3493 (match_operand:GPI 1 "register_operand" "r")))]
3495 "cinc\\t%<w>0, %<w>1, %m2"
3496 [(set_attr "type" "csel")]
3499 (define_insn "csinc3<mode>_insn"
3500 [(set (match_operand:GPI 0 "register_operand" "=r")
3502 (match_operand 1 "aarch64_comparison_operation" "")
3503 (plus:GPI (match_operand:GPI 2 "register_operand" "r")
3505 (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
3507 "csinc\\t%<w>0, %<w>3, %<w>2, %M1"
3508 [(set_attr "type" "csel")]
3511 (define_insn "*csinv3<mode>_insn"
3512 [(set (match_operand:GPI 0 "register_operand" "=r")
3514 (match_operand 1 "aarch64_comparison_operation" "")
3515 (not:GPI (match_operand:GPI 2 "register_operand" "r"))
3516 (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
3518 "csinv\\t%<w>0, %<w>3, %<w>2, %M1"
3519 [(set_attr "type" "csel")]
3522 (define_insn "csneg3_uxtw_insn"
3523 [(set (match_operand:DI 0 "register_operand" "=r")
3526 (match_operand 1 "aarch64_comparison_operation" "")
3527 (neg:SI (match_operand:SI 2 "register_operand" "r"))
3528 (match_operand:SI 3 "aarch64_reg_or_zero" "rZ"))))]
3530 "csneg\\t%w0, %w3, %w2, %M1"
3531 [(set_attr "type" "csel")]
3534 (define_insn "csneg3<mode>_insn"
3535 [(set (match_operand:GPI 0 "register_operand" "=r")
3537 (match_operand 1 "aarch64_comparison_operation" "")
3538 (neg:GPI (match_operand:GPI 2 "register_operand" "r"))
3539 (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
3541 "csneg\\t%<w>0, %<w>3, %<w>2, %M1"
3542 [(set_attr "type" "csel")]
3545 ;; If X can be loaded by a single CNT[BHWD] instruction,
3549 ;; is equivalent to:
3551 ;; TMP = UQDEC[BHWD] (B, X)
3554 ;; Defining the pattern this way means that:
3556 ;; A = UMAX (B, X) - X
3560 ;; TMP1 = UQDEC[BHWD] (B, X)
3564 ;; which combine can optimize to:
3566 ;; A = UQDEC[BHWD] (B, X)
3568 ;; We don't use match_operand predicates because the order of the operands
3569 ;; can vary: the CNT[BHWD] constant will come first if the other operand is
3570 ;; a simpler constant (such as a CONST_INT), otherwise it will come second.
3571 (define_expand "umax<mode>3"
3572 [(set (match_operand:GPI 0 "register_operand")
3573 (umax:GPI (match_operand:GPI 1 "")
3574 (match_operand:GPI 2 "")))]
3577 if (aarch64_sve_cnt_immediate (operands[1], <MODE>mode))
3578 std::swap (operands[1], operands[2]);
3579 else if (!aarch64_sve_cnt_immediate (operands[2], <MODE>mode))
3581 rtx temp = gen_reg_rtx (<MODE>mode);
3582 operands[1] = force_reg (<MODE>mode, operands[1]);
3583 emit_insn (gen_aarch64_uqdec<mode> (temp, operands[1], operands[2]));
3584 emit_insn (gen_add<mode>3 (operands[0], temp, operands[2]));
3589 ;; Saturating unsigned subtraction of a CNT[BHWD] immediate.
3590 (define_insn "aarch64_uqdec<mode>"
3591 [(set (match_operand:GPI 0 "register_operand" "=r")
3593 (umax:GPI (match_operand:GPI 1 "register_operand" "0")
3594 (match_operand:GPI 2 "aarch64_sve_cnt_immediate" "Usv"))
3598 return aarch64_output_sve_cnt_immediate ("uqdec", "%<w>0", operands[2]);
3602 ;; -------------------------------------------------------------------
3603 ;; Logical operations
3604 ;; -------------------------------------------------------------------
3607 (define_insn_and_split "*aarch64_and<mode>_imm2"
3608 [(set (match_operand:GPI 0 "register_operand" "=rk")
3609 (and:GPI (match_operand:GPI 1 "register_operand" "%r")
3610 (match_operand:GPI 2 "aarch64_logical_and_immediate" "<lconst2>")))]
3616 HOST_WIDE_INT val = INTVAL (operands[2]);
3617 rtx imm1 = GEN_INT (aarch64_and_split_imm1 (val));
3618 rtx imm2 = GEN_INT (aarch64_and_split_imm2 (val));
3620 emit_insn (gen_and<mode>3 (operands[0], operands[1], imm1));
3621 emit_insn (gen_and<mode>3 (operands[0], operands[0], imm2));
3626 (define_insn "<optab><mode>3"
3627 [(set (match_operand:GPI 0 "register_operand" "=r,rk,w")
3628 (LOGICAL:GPI (match_operand:GPI 1 "register_operand" "%r,r,w")
3629 (match_operand:GPI 2 "aarch64_logical_operand" "r,<lconst>,w")))]
3632 <logical>\\t%<w>0, %<w>1, %<w>2
3633 <logical>\\t%<w>0, %<w>1, %2
3634 <logical>\\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>"
3635 [(set_attr "type" "logic_reg,logic_imm,neon_logic")
3636 (set_attr "simd" "*,*,yes")]
3639 ;; zero_extend version of above
3640 (define_insn "*<optab>si3_uxtw"
3641 [(set (match_operand:DI 0 "register_operand" "=r,rk")
3643 (LOGICAL:SI (match_operand:SI 1 "register_operand" "%r,r")
3644 (match_operand:SI 2 "aarch64_logical_operand" "r,K"))))]
3647 <logical>\\t%w0, %w1, %w2
3648 <logical>\\t%w0, %w1, %2"
3649 [(set_attr "type" "logic_reg,logic_imm")]
3652 (define_insn "*and<mode>3_compare0"
3653 [(set (reg:CC_NZ CC_REGNUM)
3655 (and:GPI (match_operand:GPI 1 "register_operand" "%r,r")
3656 (match_operand:GPI 2 "aarch64_logical_operand" "r,<lconst>"))
3658 (set (match_operand:GPI 0 "register_operand" "=r,r")
3659 (and:GPI (match_dup 1) (match_dup 2)))]
3662 ands\\t%<w>0, %<w>1, %<w>2
3663 ands\\t%<w>0, %<w>1, %2"
3664 [(set_attr "type" "logics_reg,logics_imm")]
3667 ;; zero_extend version of above
3668 (define_insn "*andsi3_compare0_uxtw"
3669 [(set (reg:CC_NZ CC_REGNUM)
3671 (and:SI (match_operand:SI 1 "register_operand" "%r,r")
3672 (match_operand:SI 2 "aarch64_logical_operand" "r,K"))
3674 (set (match_operand:DI 0 "register_operand" "=r,r")
3675 (zero_extend:DI (and:SI (match_dup 1) (match_dup 2))))]
3678 ands\\t%w0, %w1, %w2
3679 ands\\t%w0, %w1, %2"
3680 [(set_attr "type" "logics_reg,logics_imm")]
3683 (define_insn "*and_<SHIFT:optab><mode>3_compare0"
3684 [(set (reg:CC_NZ CC_REGNUM)
3687 (match_operand:GPI 1 "register_operand" "r")
3688 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
3689 (match_operand:GPI 3 "register_operand" "r"))
3691 (set (match_operand:GPI 0 "register_operand" "=r")
3692 (and:GPI (SHIFT:GPI (match_dup 1) (match_dup 2)) (match_dup 3)))]
3694 "ands\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
3695 [(set_attr "type" "logics_shift_imm")]
3698 ;; zero_extend version of above
3699 (define_insn "*and_<SHIFT:optab>si3_compare0_uxtw"
3700 [(set (reg:CC_NZ CC_REGNUM)
3703 (match_operand:SI 1 "register_operand" "r")
3704 (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
3705 (match_operand:SI 3 "register_operand" "r"))
3707 (set (match_operand:DI 0 "register_operand" "=r")
3708 (zero_extend:DI (and:SI (SHIFT:SI (match_dup 1) (match_dup 2))
3711 "ands\\t%w0, %w3, %w1, <SHIFT:shift> %2"
3712 [(set_attr "type" "logics_shift_imm")]
3715 (define_insn "*<LOGICAL:optab>_<SHIFT:optab><mode>3"
3716 [(set (match_operand:GPI 0 "register_operand" "=r")
3717 (LOGICAL:GPI (SHIFT:GPI
3718 (match_operand:GPI 1 "register_operand" "r")
3719 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
3720 (match_operand:GPI 3 "register_operand" "r")))]
3722 "<LOGICAL:logical>\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
3723 [(set_attr "type" "logic_shift_imm")]
3726 (define_insn "*<optab>_rol<mode>3"
3727 [(set (match_operand:GPI 0 "register_operand" "=r")
3728 (LOGICAL:GPI (rotate:GPI
3729 (match_operand:GPI 1 "register_operand" "r")
3730 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
3731 (match_operand:GPI 3 "register_operand" "r")))]
3733 "<logical>\\t%<w>0, %<w>3, %<w>1, ror (<sizen> - %2)"
3734 [(set_attr "type" "logic_shift_imm")]
3737 ;; zero_extend versions of above
3738 (define_insn "*<LOGICAL:optab>_<SHIFT:optab>si3_uxtw"
3739 [(set (match_operand:DI 0 "register_operand" "=r")
3741 (LOGICAL:SI (SHIFT:SI
3742 (match_operand:SI 1 "register_operand" "r")
3743 (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
3744 (match_operand:SI 3 "register_operand" "r"))))]
3746 "<LOGICAL:logical>\\t%w0, %w3, %w1, <SHIFT:shift> %2"
3747 [(set_attr "type" "logic_shift_imm")]
3750 (define_insn "*<optab>_rolsi3_uxtw"
3751 [(set (match_operand:DI 0 "register_operand" "=r")
3753 (LOGICAL:SI (rotate:SI
3754 (match_operand:SI 1 "register_operand" "r")
3755 (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
3756 (match_operand:SI 3 "register_operand" "r"))))]
3758 "<logical>\\t%w0, %w3, %w1, ror (32 - %2)"
3759 [(set_attr "type" "logic_shift_imm")]
3762 (define_insn "one_cmpl<mode>2"
3763 [(set (match_operand:GPI 0 "register_operand" "=r,w")
3764 (not:GPI (match_operand:GPI 1 "register_operand" "r,w")))]
3769 [(set_attr "type" "logic_reg,neon_logic")
3770 (set_attr "simd" "*,yes")]
3773 (define_insn "*one_cmpl_<optab><mode>2"
3774 [(set (match_operand:GPI 0 "register_operand" "=r")
3775 (not:GPI (SHIFT:GPI (match_operand:GPI 1 "register_operand" "r")
3776 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
3778 "mvn\\t%<w>0, %<w>1, <shift> %2"
3779 [(set_attr "type" "logic_shift_imm")]
3782 ;; Binary logical operators negating one operand, i.e. (a & !b), (a | !b).
3784 (define_insn "*<NLOGICAL:optab>_one_cmpl<mode>3"
3785 [(set (match_operand:GPI 0 "register_operand" "=r,w")
3786 (NLOGICAL:GPI (not:GPI (match_operand:GPI 1 "register_operand" "r,w"))
3787 (match_operand:GPI 2 "register_operand" "r,w")))]
3790 <NLOGICAL:nlogical>\\t%<w>0, %<w>2, %<w>1
3791 <NLOGICAL:nlogical>\\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>"
3792 [(set_attr "type" "logic_reg,neon_logic")
3793 (set_attr "simd" "*,yes")]
3796 (define_insn "*<NLOGICAL:optab>_one_cmplsidi3_ze"
3797 [(set (match_operand:DI 0 "register_operand" "=r")
3799 (NLOGICAL:SI (not:SI (match_operand:SI 1 "register_operand" "r"))
3800 (match_operand:SI 2 "register_operand" "r"))))]
3802 "<NLOGICAL:nlogical>\\t%w0, %w2, %w1"
3803 [(set_attr "type" "logic_reg")]
3806 (define_insn "*xor_one_cmplsidi3_ze"
3807 [(set (match_operand:DI 0 "register_operand" "=r")
3809 (not:SI (xor:SI (match_operand:SI 1 "register_operand" "r")
3810 (match_operand:SI 2 "register_operand" "r")))))]
3812 "eon\\t%w0, %w1, %w2"
3813 [(set_attr "type" "logic_reg")]
3816 ;; (xor (not a) b) is simplify_rtx-ed down to (not (xor a b)).
3817 ;; eon does not operate on SIMD registers so the vector variant must be split.
3818 (define_insn_and_split "*xor_one_cmpl<mode>3"
3819 [(set (match_operand:GPI 0 "register_operand" "=r,w")
3820 (not:GPI (xor:GPI (match_operand:GPI 1 "register_operand" "r,?w")
3821 (match_operand:GPI 2 "register_operand" "r,w"))))]
3824 eon\\t%<w>0, %<w>1, %<w>2
3826 "reload_completed && FP_REGNUM_P (REGNO (operands[0]))" ;; For SIMD registers.
3827 [(set (match_operand:GPI 0 "register_operand" "=w")
3828 (xor:GPI (match_operand:GPI 1 "register_operand" "w")
3829 (match_operand:GPI 2 "register_operand" "w")))
3830 (set (match_dup 0) (not:GPI (match_dup 0)))]
3832 [(set_attr "type" "logic_reg,multiple")
3833 (set_attr "simd" "*,yes")]
3836 (define_insn "*and_one_cmpl<mode>3_compare0"
3837 [(set (reg:CC_NZ CC_REGNUM)
3840 (match_operand:GPI 1 "register_operand" "r"))
3841 (match_operand:GPI 2 "register_operand" "r"))
3843 (set (match_operand:GPI 0 "register_operand" "=r")
3844 (and:GPI (not:GPI (match_dup 1)) (match_dup 2)))]
3846 "bics\\t%<w>0, %<w>2, %<w>1"
3847 [(set_attr "type" "logics_reg")]
3850 ;; zero_extend version of above
3851 (define_insn "*and_one_cmplsi3_compare0_uxtw"
3852 [(set (reg:CC_NZ CC_REGNUM)
3855 (match_operand:SI 1 "register_operand" "r"))
3856 (match_operand:SI 2 "register_operand" "r"))
3858 (set (match_operand:DI 0 "register_operand" "=r")
3859 (zero_extend:DI (and:SI (not:SI (match_dup 1)) (match_dup 2))))]
3861 "bics\\t%w0, %w2, %w1"
3862 [(set_attr "type" "logics_reg")]
3865 (define_insn "*and_one_cmpl<mode>3_compare0_no_reuse"
3866 [(set (reg:CC_NZ CC_REGNUM)
3869 (match_operand:GPI 0 "register_operand" "r"))
3870 (match_operand:GPI 1 "register_operand" "r"))
3873 "bics\\t<w>zr, %<w>1, %<w>0"
3874 [(set_attr "type" "logics_reg")]
3877 (define_insn "<LOGICAL:optab>_one_cmpl_<SHIFT:optab><mode>3"
3878 [(set (match_operand:GPI 0 "register_operand" "=r")
3879 (LOGICAL:GPI (not:GPI
3881 (match_operand:GPI 1 "register_operand" "r")
3882 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
3883 (match_operand:GPI 3 "register_operand" "r")))]
3885 "<LOGICAL:nlogical>\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
3886 [(set_attr "type" "logic_shift_imm")]
3889 (define_insn "*eor_one_cmpl_<SHIFT:optab><mode>3_alt"
3890 [(set (match_operand:GPI 0 "register_operand" "=r")
3893 (match_operand:GPI 1 "register_operand" "r")
3894 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
3895 (match_operand:GPI 3 "register_operand" "r"))))]
3897 "eon\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
3898 [(set_attr "type" "logic_shift_imm")]
3901 ;; Zero-extend version of the above.
3902 (define_insn "*eor_one_cmpl_<SHIFT:optab>sidi3_alt_ze"
3903 [(set (match_operand:DI 0 "register_operand" "=r")
3907 (match_operand:SI 1 "register_operand" "r")
3908 (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
3909 (match_operand:SI 3 "register_operand" "r")))))]
3911 "eon\\t%w0, %w3, %w1, <SHIFT:shift> %2"
3912 [(set_attr "type" "logic_shift_imm")]
3915 (define_insn "*and_one_cmpl_<SHIFT:optab><mode>3_compare0"
3916 [(set (reg:CC_NZ CC_REGNUM)
3920 (match_operand:GPI 1 "register_operand" "r")
3921 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
3922 (match_operand:GPI 3 "register_operand" "r"))
3924 (set (match_operand:GPI 0 "register_operand" "=r")
3927 (match_dup 1) (match_dup 2))) (match_dup 3)))]
3929 "bics\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
3930 [(set_attr "type" "logics_shift_imm")]
3933 ;; zero_extend version of above
3934 (define_insn "*and_one_cmpl_<SHIFT:optab>si3_compare0_uxtw"
3935 [(set (reg:CC_NZ CC_REGNUM)
3939 (match_operand:SI 1 "register_operand" "r")
3940 (match_operand:QI 2 "aarch64_shift_imm_si" "n")))
3941 (match_operand:SI 3 "register_operand" "r"))
3943 (set (match_operand:DI 0 "register_operand" "=r")
3944 (zero_extend:DI (and:SI
3946 (SHIFT:SI (match_dup 1) (match_dup 2))) (match_dup 3))))]
3948 "bics\\t%w0, %w3, %w1, <SHIFT:shift> %2"
3949 [(set_attr "type" "logics_shift_imm")]
3952 (define_insn "*and_one_cmpl_<SHIFT:optab><mode>3_compare0_no_reuse"
3953 [(set (reg:CC_NZ CC_REGNUM)
3957 (match_operand:GPI 0 "register_operand" "r")
3958 (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n")))
3959 (match_operand:GPI 2 "register_operand" "r"))
3962 "bics\\t<w>zr, %<w>2, %<w>0, <SHIFT:shift> %1"
3963 [(set_attr "type" "logics_shift_imm")]
3966 (define_insn "clz<mode>2"
3967 [(set (match_operand:GPI 0 "register_operand" "=r")
3968 (clz:GPI (match_operand:GPI 1 "register_operand" "r")))]
3970 "clz\\t%<w>0, %<w>1"
3971 [(set_attr "type" "clz")]
3974 (define_expand "ffs<mode>2"
3975 [(match_operand:GPI 0 "register_operand")
3976 (match_operand:GPI 1 "register_operand")]
3979 rtx ccreg = aarch64_gen_compare_reg (EQ, operands[1], const0_rtx);
3980 rtx x = gen_rtx_NE (VOIDmode, ccreg, const0_rtx);
3982 emit_insn (gen_rbit<mode>2 (operands[0], operands[1]));
3983 emit_insn (gen_clz<mode>2 (operands[0], operands[0]));
3984 emit_insn (gen_csinc3<mode>_insn (operands[0], x, operands[0], const0_rtx));
3989 ;; Pop count be done via the "CNT" instruction in AdvSIMD.
3996 (define_expand "popcount<mode>2"
3997 [(match_operand:GPI 0 "register_operand")
3998 (match_operand:GPI 1 "register_operand")]
4001 rtx v = gen_reg_rtx (V8QImode);
4002 rtx v1 = gen_reg_rtx (V8QImode);
4003 rtx r = gen_reg_rtx (QImode);
4004 rtx in = operands[1];
4005 rtx out = operands[0];
4006 if(<MODE>mode == SImode)
4009 tmp = gen_reg_rtx (DImode);
4010 /* If we have SImode, zero extend to DImode, pop count does
4011 not change if we have extra zeros. */
4012 emit_insn (gen_zero_extendsidi2 (tmp, in));
4015 emit_move_insn (v, gen_lowpart (V8QImode, in));
4016 emit_insn (gen_popcountv8qi2 (v1, v));
4017 emit_insn (gen_reduc_plus_scal_v8qi (r, v1));
4018 emit_insn (gen_zero_extendqi<mode>2 (out, r));
4022 (define_insn "clrsb<mode>2"
4023 [(set (match_operand:GPI 0 "register_operand" "=r")
4024 (clrsb:GPI (match_operand:GPI 1 "register_operand" "r")))]
4026 "cls\\t%<w>0, %<w>1"
4027 [(set_attr "type" "clz")]
4030 (define_insn "rbit<mode>2"
4031 [(set (match_operand:GPI 0 "register_operand" "=r")
4032 (unspec:GPI [(match_operand:GPI 1 "register_operand" "r")] UNSPEC_RBIT))]
4034 "rbit\\t%<w>0, %<w>1"
4035 [(set_attr "type" "rbit")]
4038 ;; Split after reload into RBIT + CLZ. Since RBIT is represented as an UNSPEC
4039 ;; it is unlikely to fold with any other operation, so keep this as a CTZ
4040 ;; expression and split after reload to enable scheduling them apart if
4043 (define_insn_and_split "ctz<mode>2"
4044 [(set (match_operand:GPI 0 "register_operand" "=r")
4045 (ctz:GPI (match_operand:GPI 1 "register_operand" "r")))]
4051 emit_insn (gen_rbit<mode>2 (operands[0], operands[1]));
4052 emit_insn (gen_clz<mode>2 (operands[0], operands[0]));
4056 (define_insn "*and<mode>_compare0"
4057 [(set (reg:CC_NZ CC_REGNUM)
4059 (match_operand:SHORT 0 "register_operand" "r")
4062 "tst\\t%<w>0, <short_mask>"
4063 [(set_attr "type" "alus_imm")]
4066 (define_insn "*ands<mode>_compare0"
4067 [(set (reg:CC_NZ CC_REGNUM)
4069 (zero_extend:GPI (match_operand:SHORT 1 "register_operand" "r"))
4071 (set (match_operand:GPI 0 "register_operand" "=r")
4072 (zero_extend:GPI (match_dup 1)))]
4074 "ands\\t%<GPI:w>0, %<GPI:w>1, <short_mask>"
4075 [(set_attr "type" "alus_imm")]
4078 (define_insn "*and<mode>3nr_compare0"
4079 [(set (reg:CC_NZ CC_REGNUM)
4081 (and:GPI (match_operand:GPI 0 "register_operand" "%r,r")
4082 (match_operand:GPI 1 "aarch64_logical_operand" "r,<lconst>"))
4088 [(set_attr "type" "logics_reg,logics_imm")]
4092 [(set (reg:CC_NZ CC_REGNUM)
4094 (and:GPI (match_operand:GPI 0 "register_operand")
4095 (match_operand:GPI 1 "aarch64_mov_imm_operand"))
4097 (clobber (match_operand:SI 2 "register_operand"))]
4099 [(set (match_dup 2) (match_dup 1))
4100 (set (reg:CC_NZ CC_REGNUM)
4102 (and:GPI (match_dup 0)
4107 (define_insn "*and<mode>3nr_compare0_zextract"
4108 [(set (reg:CC_NZ CC_REGNUM)
4110 (zero_extract:GPI (match_operand:GPI 0 "register_operand" "r")
4111 (match_operand:GPI 1 "const_int_operand" "n")
4112 (match_operand:GPI 2 "const_int_operand" "n"))
4114 "INTVAL (operands[1]) > 0
4115 && ((INTVAL (operands[1]) + INTVAL (operands[2]))
4116 <= GET_MODE_BITSIZE (<MODE>mode))
4117 && aarch64_bitmask_imm (
4118 UINTVAL (aarch64_mask_from_zextract_ops (operands[1],
4123 = aarch64_mask_from_zextract_ops (operands[1], operands[2]);
4124 return "tst\\t%<w>0, %1";
4126 [(set_attr "type" "logics_shift_imm")]
4129 (define_insn "*and_<SHIFT:optab><mode>3nr_compare0"
4130 [(set (reg:CC_NZ CC_REGNUM)
4133 (match_operand:GPI 0 "register_operand" "r")
4134 (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n"))
4135 (match_operand:GPI 2 "register_operand" "r"))
4138 "tst\\t%<w>2, %<w>0, <SHIFT:shift> %1"
4139 [(set_attr "type" "logics_shift_imm")]
4143 [(set (reg:CC_NZ CC_REGNUM)
4146 (match_operand:GPI 0 "register_operand")
4147 (match_operand:QI 1 "aarch64_shift_imm_<mode>"))
4148 (match_operand:GPI 2 "aarch64_mov_imm_operand"))
4150 (clobber (match_operand:SI 3 "register_operand"))]
4152 [(set (match_dup 3) (match_dup 2))
4153 (set (reg:CC_NZ CC_REGNUM)
4162 ;; -------------------------------------------------------------------
4164 ;; -------------------------------------------------------------------
4166 (define_expand "<optab><mode>3"
4167 [(set (match_operand:GPI 0 "register_operand")
4168 (ASHIFT:GPI (match_operand:GPI 1 "register_operand")
4169 (match_operand:QI 2 "aarch64_reg_or_imm")))]
4172 if (CONST_INT_P (operands[2]))
4174 operands[2] = GEN_INT (INTVAL (operands[2])
4175 & (GET_MODE_BITSIZE (<MODE>mode) - 1));
4177 if (operands[2] == const0_rtx)
4179 emit_insn (gen_mov<mode> (operands[0], operands[1]));
4186 (define_expand "ashl<mode>3"
4187 [(set (match_operand:SHORT 0 "register_operand")
4188 (ashift:SHORT (match_operand:SHORT 1 "register_operand")
4189 (match_operand:QI 2 "const_int_operand")))]
4192 operands[2] = GEN_INT (INTVAL (operands[2]) & GET_MODE_MASK (<MODE>mode));
4194 if (operands[2] == const0_rtx)
4196 emit_insn (gen_mov<mode> (operands[0], operands[1]));
4202 (define_expand "rotr<mode>3"
4203 [(set (match_operand:GPI 0 "register_operand")
4204 (rotatert:GPI (match_operand:GPI 1 "register_operand")
4205 (match_operand:QI 2 "aarch64_reg_or_imm")))]
4208 if (CONST_INT_P (operands[2]))
4210 operands[2] = GEN_INT (INTVAL (operands[2])
4211 & (GET_MODE_BITSIZE (<MODE>mode) - 1));
4213 if (operands[2] == const0_rtx)
4215 emit_insn (gen_mov<mode> (operands[0], operands[1]));
4222 (define_expand "rotl<mode>3"
4223 [(set (match_operand:GPI 0 "register_operand")
4224 (rotatert:GPI (match_operand:GPI 1 "register_operand")
4225 (match_operand:QI 2 "aarch64_reg_or_imm")))]
4228 /* (SZ - cnt) % SZ == -cnt % SZ */
4229 if (CONST_INT_P (operands[2]))
4231 operands[2] = GEN_INT ((-INTVAL (operands[2]))
4232 & (GET_MODE_BITSIZE (<MODE>mode) - 1));
4233 if (operands[2] == const0_rtx)
4235 emit_insn (gen_mov<mode> (operands[0], operands[1]));
4240 operands[2] = expand_simple_unop (QImode, NEG, operands[2],
4245 ;; When the LSL, LSR, ASR, ROR instructions operate on all register arguments
4246 ;; they truncate the shift/rotate amount by the size of the registers they
4247 ;; operate on: 32 for W-regs, 64 for X-regs. This allows us to optimise away
4248 ;; such redundant masking instructions. GCC can do that automatically when
4249 ;; SHIFT_COUNT_TRUNCATED is true, but we can't enable it for TARGET_SIMD
4250 ;; because some of the SISD shift alternatives don't perform this truncations.
4251 ;; So this pattern exists to catch such cases.
4253 (define_insn "*aarch64_<optab>_reg_<mode>3_mask1"
4254 [(set (match_operand:GPI 0 "register_operand" "=r")
4256 (match_operand:GPI 1 "register_operand" "r")
4257 (match_operator 4 "subreg_lowpart_operator"
4258 [(and:GPI (match_operand:GPI 2 "register_operand" "r")
4259 (match_operand 3 "const_int_operand" "n"))])))]
4260 "(~INTVAL (operands[3]) & (GET_MODE_BITSIZE (<MODE>mode) - 1)) == 0"
4261 "<shift>\t%<w>0, %<w>1, %<w>2"
4262 [(set_attr "type" "shift_reg")]
4265 (define_insn_and_split "*aarch64_reg_<mode>3_neg_mask2"
4266 [(set (match_operand:GPI 0 "register_operand" "=&r")
4268 (match_operand:GPI 1 "register_operand" "r")
4269 (match_operator 4 "subreg_lowpart_operator"
4270 [(neg:SI (and:SI (match_operand:SI 2 "register_operand" "r")
4271 (match_operand 3 "const_int_operand" "n")))])))]
4272 "((~INTVAL (operands[3]) & (GET_MODE_BITSIZE (<MODE>mode) - 1)) == 0)"
4277 rtx tmp = (can_create_pseudo_p () ? gen_reg_rtx (SImode)
4279 emit_insn (gen_negsi2 (tmp, operands[2]));
4281 rtx and_op = gen_rtx_AND (SImode, tmp, operands[3]);
4282 rtx subreg_tmp = gen_rtx_SUBREG (GET_MODE (operands[4]), and_op,
4283 SUBREG_BYTE (operands[4]));
4284 emit_insn (gen_<optab><mode>3 (operands[0], operands[1], subreg_tmp));
4289 (define_insn_and_split "*aarch64_reg_<mode>3_minus_mask"
4290 [(set (match_operand:GPI 0 "register_operand" "=&r")
4292 (match_operand:GPI 1 "register_operand" "r")
4293 (minus:QI (match_operand 2 "const_int_operand" "n")
4294 (match_operator 5 "subreg_lowpart_operator"
4295 [(and:SI (match_operand:SI 3 "register_operand" "r")
4296 (match_operand 4 "const_int_operand" "n"))]))))]
4297 "((~INTVAL (operands[4]) & (GET_MODE_BITSIZE (<MODE>mode) - 1)) == 0)
4298 && INTVAL (operands[2]) == GET_MODE_BITSIZE (<MODE>mode)"
4303 rtx tmp = (can_create_pseudo_p () ? gen_reg_rtx (SImode)
4306 emit_insn (gen_negsi2 (tmp, operands[3]));
4308 rtx and_op = gen_rtx_AND (SImode, tmp, operands[4]);
4309 rtx subreg_tmp = gen_rtx_SUBREG (GET_MODE (operands[5]), and_op,
4310 SUBREG_BYTE (operands[5]));
4312 emit_insn (gen_ashl<mode>3 (operands[0], operands[1], subreg_tmp));
4317 (define_insn "*aarch64_<optab>_reg_di3_mask2"
4318 [(set (match_operand:DI 0 "register_operand" "=r")
4320 (match_operand:DI 1 "register_operand" "r")
4321 (match_operator 4 "subreg_lowpart_operator"
4322 [(and:SI (match_operand:SI 2 "register_operand" "r")
4323 (match_operand 3 "aarch64_shift_imm_di" "Usd"))])))]
4324 "((~INTVAL (operands[3]) & (GET_MODE_BITSIZE (DImode)-1)) == 0)"
4327 xop[0] = operands[0];
4328 xop[1] = operands[1];
4329 xop[2] = gen_lowpart (GET_MODE (operands[4]), operands[2]);
4330 output_asm_insn ("<shift>\t%x0, %x1, %x2", xop);
4333 [(set_attr "type" "shift_reg")]
4336 (define_insn_and_split "*aarch64_reg_<optab>_minus<mode>3"
4337 [(set (match_operand:GPI 0 "register_operand" "=&r")
4339 (match_operand:GPI 1 "register_operand" "r")
4340 (minus:QI (match_operand 2 "const_int_operand" "n")
4341 (match_operand:QI 3 "register_operand" "r"))))]
4342 "INTVAL (operands[2]) == GET_MODE_BITSIZE (<MODE>mode)"
4347 rtx subreg_tmp = gen_lowpart (SImode, operands[3]);
4349 rtx tmp = (can_create_pseudo_p () ? gen_reg_rtx (SImode)
4350 : gen_lowpart (SImode, operands[0]));
4352 emit_insn (gen_negsi2 (tmp, subreg_tmp));
4354 rtx and_op = gen_rtx_AND (SImode, tmp,
4355 GEN_INT (GET_MODE_BITSIZE (<MODE>mode) - 1));
4357 rtx subreg_tmp2 = gen_lowpart_SUBREG (QImode, and_op);
4359 emit_insn (gen_<optab><mode>3 (operands[0], operands[1], subreg_tmp2));
4362 [(set_attr "length" "8")]
4365 ;; Logical left shift using SISD or Integer instruction
4366 (define_insn "*aarch64_ashl_sisd_or_int_<mode>3"
4367 [(set (match_operand:GPI 0 "register_operand" "=r,r,w,w")
4369 (match_operand:GPI 1 "register_operand" "r,r,w,w")
4370 (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>" "Us<cmode>,r,Us<cmode>,w")))]
4373 lsl\t%<w>0, %<w>1, %2
4374 lsl\t%<w>0, %<w>1, %<w>2
4375 shl\t%<rtn>0<vas>, %<rtn>1<vas>, %2
4376 ushl\t%<rtn>0<vas>, %<rtn>1<vas>, %<rtn>2<vas>"
4377 [(set_attr "simd" "no,no,yes,yes")
4378 (set_attr "type" "bfx,shift_reg,neon_shift_imm<q>, neon_shift_reg<q>")]
4381 ;; Logical right shift using SISD or Integer instruction
4382 (define_insn "*aarch64_lshr_sisd_or_int_<mode>3"
4383 [(set (match_operand:GPI 0 "register_operand" "=r,r,w,&w,&w")
4385 (match_operand:GPI 1 "register_operand" "r,r,w,w,w")
4386 (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>" "Us<cmode>,r,Us<cmode>,w,0")))]
4389 lsr\t%<w>0, %<w>1, %2
4390 lsr\t%<w>0, %<w>1, %<w>2
4391 ushr\t%<rtn>0<vas>, %<rtn>1<vas>, %2
4394 [(set_attr "simd" "no,no,yes,yes,yes")
4395 (set_attr "type" "bfx,shift_reg,neon_shift_imm<q>,neon_shift_reg<q>,neon_shift_reg<q>")]
4399 [(set (match_operand:DI 0 "aarch64_simd_register")
4401 (match_operand:DI 1 "aarch64_simd_register")
4402 (match_operand:QI 2 "aarch64_simd_register")))]
4403 "TARGET_SIMD && reload_completed"
4405 (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
4407 (unspec:DI [(match_dup 1) (match_dup 3)] UNSPEC_SISD_USHL))]
4409 operands[3] = gen_lowpart (QImode, operands[0]);
4414 [(set (match_operand:SI 0 "aarch64_simd_register")
4416 (match_operand:SI 1 "aarch64_simd_register")
4417 (match_operand:QI 2 "aarch64_simd_register")))]
4418 "TARGET_SIMD && reload_completed"
4420 (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
4422 (unspec:SI [(match_dup 1) (match_dup 3)] UNSPEC_USHL_2S))]
4424 operands[3] = gen_lowpart (QImode, operands[0]);
4428 ;; Arithmetic right shift using SISD or Integer instruction
4429 (define_insn "*aarch64_ashr_sisd_or_int_<mode>3"
4430 [(set (match_operand:GPI 0 "register_operand" "=r,r,w,&w,&w")
4432 (match_operand:GPI 1 "register_operand" "r,r,w,w,w")
4433 (match_operand:QI 2 "aarch64_reg_or_shift_imm_di" "Us<cmode>,r,Us<cmode>,w,0")))]
4436 asr\t%<w>0, %<w>1, %2
4437 asr\t%<w>0, %<w>1, %<w>2
4438 sshr\t%<rtn>0<vas>, %<rtn>1<vas>, %2
4441 [(set_attr "simd" "no,no,yes,yes,yes")
4442 (set_attr "type" "bfx,shift_reg,neon_shift_imm<q>,neon_shift_reg<q>,neon_shift_reg<q>")]
4446 [(set (match_operand:DI 0 "aarch64_simd_register")
4448 (match_operand:DI 1 "aarch64_simd_register")
4449 (match_operand:QI 2 "aarch64_simd_register")))]
4450 "TARGET_SIMD && reload_completed"
4452 (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
4454 (unspec:DI [(match_dup 1) (match_dup 3)] UNSPEC_SISD_SSHL))]
4456 operands[3] = gen_lowpart (QImode, operands[0]);
4461 [(set (match_operand:SI 0 "aarch64_simd_register")
4463 (match_operand:SI 1 "aarch64_simd_register")
4464 (match_operand:QI 2 "aarch64_simd_register")))]
4465 "TARGET_SIMD && reload_completed"
4467 (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
4469 (unspec:SI [(match_dup 1) (match_dup 3)] UNSPEC_SSHL_2S))]
4471 operands[3] = gen_lowpart (QImode, operands[0]);
4475 (define_insn "*aarch64_sisd_ushl"
4476 [(set (match_operand:DI 0 "register_operand" "=w")
4477 (unspec:DI [(match_operand:DI 1 "register_operand" "w")
4478 (match_operand:QI 2 "register_operand" "w")]
4481 "ushl\t%d0, %d1, %d2"
4482 [(set_attr "simd" "yes")
4483 (set_attr "type" "neon_shift_reg")]
4486 (define_insn "*aarch64_ushl_2s"
4487 [(set (match_operand:SI 0 "register_operand" "=w")
4488 (unspec:SI [(match_operand:SI 1 "register_operand" "w")
4489 (match_operand:QI 2 "register_operand" "w")]
4492 "ushl\t%0.2s, %1.2s, %2.2s"
4493 [(set_attr "simd" "yes")
4494 (set_attr "type" "neon_shift_reg")]
4497 (define_insn "*aarch64_sisd_sshl"
4498 [(set (match_operand:DI 0 "register_operand" "=w")
4499 (unspec:DI [(match_operand:DI 1 "register_operand" "w")
4500 (match_operand:QI 2 "register_operand" "w")]
4503 "sshl\t%d0, %d1, %d2"
4504 [(set_attr "simd" "yes")
4505 (set_attr "type" "neon_shift_reg")]
4508 (define_insn "*aarch64_sshl_2s"
4509 [(set (match_operand:SI 0 "register_operand" "=w")
4510 (unspec:SI [(match_operand:SI 1 "register_operand" "w")
4511 (match_operand:QI 2 "register_operand" "w")]
4514 "sshl\t%0.2s, %1.2s, %2.2s"
4515 [(set_attr "simd" "yes")
4516 (set_attr "type" "neon_shift_reg")]
4519 (define_insn "*aarch64_sisd_neg_qi"
4520 [(set (match_operand:QI 0 "register_operand" "=w")
4521 (unspec:QI [(match_operand:QI 1 "register_operand" "w")]
4525 [(set_attr "simd" "yes")
4526 (set_attr "type" "neon_neg")]
4530 (define_insn "*ror<mode>3_insn"
4531 [(set (match_operand:GPI 0 "register_operand" "=r,r")
4533 (match_operand:GPI 1 "register_operand" "r,r")
4534 (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>" "Us<cmode>,r")))]
4537 ror\\t%<w>0, %<w>1, %2
4538 ror\\t%<w>0, %<w>1, %<w>2"
4539 [(set_attr "type" "rotate_imm,shift_reg")]
4542 ;; zero_extend version of above
4543 (define_insn "*<optab>si3_insn_uxtw"
4544 [(set (match_operand:DI 0 "register_operand" "=r,r")
4545 (zero_extend:DI (SHIFT:SI
4546 (match_operand:SI 1 "register_operand" "r,r")
4547 (match_operand:QI 2 "aarch64_reg_or_shift_imm_si" "Uss,r"))))]
4550 <shift>\\t%w0, %w1, %2
4551 <shift>\\t%w0, %w1, %w2"
4552 [(set_attr "type" "bfx,shift_reg")]
4555 (define_insn "*<optab><mode>3_insn"
4556 [(set (match_operand:SHORT 0 "register_operand" "=r")
4557 (ASHIFT:SHORT (match_operand:SHORT 1 "register_operand" "r")
4558 (match_operand 2 "const_int_operand" "n")))]
4559 "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<MODE>mode)"
4561 operands[3] = GEN_INT (<sizen> - UINTVAL (operands[2]));
4562 return "<bfshift>\t%w0, %w1, %2, %3";
4564 [(set_attr "type" "bfx")]
4567 (define_insn "*extr<mode>5_insn"
4568 [(set (match_operand:GPI 0 "register_operand" "=r")
4569 (ior:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
4570 (match_operand 3 "const_int_operand" "n"))
4571 (lshiftrt:GPI (match_operand:GPI 2 "register_operand" "r")
4572 (match_operand 4 "const_int_operand" "n"))))]
4573 "UINTVAL (operands[3]) < GET_MODE_BITSIZE (<MODE>mode) &&
4574 (UINTVAL (operands[3]) + UINTVAL (operands[4]) == GET_MODE_BITSIZE (<MODE>mode))"
4575 "extr\\t%<w>0, %<w>1, %<w>2, %4"
4576 [(set_attr "type" "rotate_imm")]
4579 ;; There are no canonicalisation rules for ashift and lshiftrt inside an ior
4580 ;; so we have to match both orderings.
4581 (define_insn "*extr<mode>5_insn_alt"
4582 [(set (match_operand:GPI 0 "register_operand" "=r")
4583 (ior:GPI (lshiftrt:GPI (match_operand:GPI 2 "register_operand" "r")
4584 (match_operand 4 "const_int_operand" "n"))
4585 (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
4586 (match_operand 3 "const_int_operand" "n"))))]
4587 "UINTVAL (operands[3]) < GET_MODE_BITSIZE (<MODE>mode)
4588 && (UINTVAL (operands[3]) + UINTVAL (operands[4])
4589 == GET_MODE_BITSIZE (<MODE>mode))"
4590 "extr\\t%<w>0, %<w>1, %<w>2, %4"
4591 [(set_attr "type" "rotate_imm")]
4594 ;; zero_extend version of the above
4595 (define_insn "*extrsi5_insn_uxtw"
4596 [(set (match_operand:DI 0 "register_operand" "=r")
4598 (ior:SI (ashift:SI (match_operand:SI 1 "register_operand" "r")
4599 (match_operand 3 "const_int_operand" "n"))
4600 (lshiftrt:SI (match_operand:SI 2 "register_operand" "r")
4601 (match_operand 4 "const_int_operand" "n")))))]
4602 "UINTVAL (operands[3]) < 32 &&
4603 (UINTVAL (operands[3]) + UINTVAL (operands[4]) == 32)"
4604 "extr\\t%w0, %w1, %w2, %4"
4605 [(set_attr "type" "rotate_imm")]
4608 (define_insn "*extrsi5_insn_uxtw_alt"
4609 [(set (match_operand:DI 0 "register_operand" "=r")
4611 (ior:SI (lshiftrt:SI (match_operand:SI 2 "register_operand" "r")
4612 (match_operand 4 "const_int_operand" "n"))
4613 (ashift:SI (match_operand:SI 1 "register_operand" "r")
4614 (match_operand 3 "const_int_operand" "n")))))]
4615 "UINTVAL (operands[3]) < 32 &&
4616 (UINTVAL (operands[3]) + UINTVAL (operands[4]) == 32)"
4617 "extr\\t%w0, %w1, %w2, %4"
4618 [(set_attr "type" "rotate_imm")]
4621 (define_insn "*ror<mode>3_insn"
4622 [(set (match_operand:GPI 0 "register_operand" "=r")
4623 (rotate:GPI (match_operand:GPI 1 "register_operand" "r")
4624 (match_operand 2 "const_int_operand" "n")))]
4625 "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<MODE>mode)"
4627 operands[3] = GEN_INT (<sizen> - UINTVAL (operands[2]));
4628 return "ror\\t%<w>0, %<w>1, %3";
4630 [(set_attr "type" "rotate_imm")]
4633 ;; zero_extend version of the above
4634 (define_insn "*rorsi3_insn_uxtw"
4635 [(set (match_operand:DI 0 "register_operand" "=r")
4637 (rotate:SI (match_operand:SI 1 "register_operand" "r")
4638 (match_operand 2 "const_int_operand" "n"))))]
4639 "UINTVAL (operands[2]) < 32"
4641 operands[3] = GEN_INT (32 - UINTVAL (operands[2]));
4642 return "ror\\t%w0, %w1, %3";
4644 [(set_attr "type" "rotate_imm")]
4647 (define_insn "*<ANY_EXTEND:optab><GPI:mode>_ashl<SHORT:mode>"
4648 [(set (match_operand:GPI 0 "register_operand" "=r")
4650 (ashift:SHORT (match_operand:SHORT 1 "register_operand" "r")
4651 (match_operand 2 "const_int_operand" "n"))))]
4652 "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
4654 operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
4655 return "<su>bfiz\t%<GPI:w>0, %<GPI:w>1, %2, %3";
4657 [(set_attr "type" "bfx")]
4660 (define_insn "*zero_extend<GPI:mode>_lshr<SHORT:mode>"
4661 [(set (match_operand:GPI 0 "register_operand" "=r")
4663 (lshiftrt:SHORT (match_operand:SHORT 1 "register_operand" "r")
4664 (match_operand 2 "const_int_operand" "n"))))]
4665 "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
4667 operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
4668 return "ubfx\t%<GPI:w>0, %<GPI:w>1, %2, %3";
4670 [(set_attr "type" "bfx")]
4673 (define_insn "*extend<GPI:mode>_ashr<SHORT:mode>"
4674 [(set (match_operand:GPI 0 "register_operand" "=r")
4676 (ashiftrt:SHORT (match_operand:SHORT 1 "register_operand" "r")
4677 (match_operand 2 "const_int_operand" "n"))))]
4678 "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
4680 operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
4681 return "sbfx\\t%<GPI:w>0, %<GPI:w>1, %2, %3";
4683 [(set_attr "type" "bfx")]
4686 ;; -------------------------------------------------------------------
4688 ;; -------------------------------------------------------------------
4690 (define_expand "<optab>"
4691 [(set (match_operand:DI 0 "register_operand" "=r")
4692 (ANY_EXTRACT:DI (match_operand:DI 1 "register_operand")
4694 "aarch64_simd_shift_imm_offset_di")
4695 (match_operand 3 "aarch64_simd_shift_imm_di")))]
4698 if (!IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]),
4699 1, GET_MODE_BITSIZE (DImode) - 1))
4705 (define_insn "*<optab><mode>"
4706 [(set (match_operand:GPI 0 "register_operand" "=r")
4707 (ANY_EXTRACT:GPI (match_operand:GPI 1 "register_operand" "r")
4709 "aarch64_simd_shift_imm_offset_<mode>" "n")
4711 "aarch64_simd_shift_imm_<mode>" "n")))]
4712 "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]),
4713 1, GET_MODE_BITSIZE (<MODE>mode) - 1)"
4714 "<su>bfx\\t%<w>0, %<w>1, %3, %2"
4715 [(set_attr "type" "bfx")]
4718 ;; When the bit position and width add up to 32 we can use a W-reg LSR
4719 ;; instruction taking advantage of the implicit zero-extension of the X-reg.
4721 [(set (match_operand:DI 0 "register_operand")
4722 (zero_extract:DI (match_operand:DI 1 "register_operand")
4724 "aarch64_simd_shift_imm_offset_di")
4726 "aarch64_simd_shift_imm_di")))]
4727 "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]), 1,
4728 GET_MODE_BITSIZE (DImode) - 1)
4729 && (INTVAL (operands[2]) + INTVAL (operands[3]))
4730 == GET_MODE_BITSIZE (SImode)"
4732 (zero_extend:DI (lshiftrt:SI (match_dup 4) (match_dup 3))))]
4734 operands[4] = gen_lowpart (SImode, operands[1]);
4738 ;; Bitfield Insert (insv)
4739 (define_expand "insv<mode>"
4740 [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand")
4741 (match_operand 1 "const_int_operand")
4742 (match_operand 2 "const_int_operand"))
4743 (match_operand:GPI 3 "general_operand"))]
4746 unsigned HOST_WIDE_INT width = UINTVAL (operands[1]);
4747 unsigned HOST_WIDE_INT pos = UINTVAL (operands[2]);
4748 rtx value = operands[3];
4750 if (width == 0 || (pos + width) > GET_MODE_BITSIZE (<MODE>mode))
4753 if (CONST_INT_P (value))
4755 unsigned HOST_WIDE_INT mask = ((unsigned HOST_WIDE_INT)1 << width) - 1;
4757 /* Prefer AND/OR for inserting all zeros or all ones. */
4758 if ((UINTVAL (value) & mask) == 0
4759 || (UINTVAL (value) & mask) == mask)
4762 /* 16-bit aligned 16-bit wide insert is handled by insv_imm. */
4763 if (width == 16 && (pos % 16) == 0)
4766 operands[3] = force_reg (<MODE>mode, value);
4769 (define_insn "*insv_reg<mode>"
4770 [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
4771 (match_operand 1 "const_int_operand" "n")
4772 (match_operand 2 "const_int_operand" "n"))
4773 (match_operand:GPI 3 "register_operand" "r"))]
4774 "!(UINTVAL (operands[1]) == 0
4775 || (UINTVAL (operands[2]) + UINTVAL (operands[1])
4776 > GET_MODE_BITSIZE (<MODE>mode)))"
4777 "bfi\\t%<w>0, %<w>3, %2, %1"
4778 [(set_attr "type" "bfm")]
4781 (define_insn "*aarch64_bfi<GPI:mode><ALLX:mode>4"
4782 [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
4783 (match_operand 1 "const_int_operand" "n")
4784 (match_operand 2 "const_int_operand" "n"))
4785 (zero_extend:GPI (match_operand:ALLX 3 "register_operand" "r")))]
4786 "UINTVAL (operands[1]) <= <ALLX:sizen>"
4787 "bfi\\t%<GPI:w>0, %<GPI:w>3, %2, %1"
4788 [(set_attr "type" "bfm")]
4791 (define_insn "*extr_insv_lower_reg<mode>"
4792 [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
4793 (match_operand 1 "const_int_operand" "n")
4795 (zero_extract:GPI (match_operand:GPI 2 "register_operand" "r")
4797 (match_operand 3 "const_int_operand" "n")))]
4798 "!(UINTVAL (operands[1]) == 0
4799 || (UINTVAL (operands[3]) + UINTVAL (operands[1])
4800 > GET_MODE_BITSIZE (<MODE>mode)))"
4801 "bfxil\\t%<w>0, %<w>2, %3, %1"
4802 [(set_attr "type" "bfm")]
4805 (define_insn "*<optab><ALLX:mode>_shft_<GPI:mode>"
4806 [(set (match_operand:GPI 0 "register_operand" "=r")
4807 (ashift:GPI (ANY_EXTEND:GPI
4808 (match_operand:ALLX 1 "register_operand" "r"))
4809 (match_operand 2 "const_int_operand" "n")))]
4810 "UINTVAL (operands[2]) < <GPI:sizen>"
4812 operands[3] = (<ALLX:sizen> <= (<GPI:sizen> - UINTVAL (operands[2])))
4813 ? GEN_INT (<ALLX:sizen>)
4814 : GEN_INT (<GPI:sizen> - UINTVAL (operands[2]));
4815 return "<su>bfiz\t%<GPI:w>0, %<GPI:w>1, %2, %3";
4817 [(set_attr "type" "bfx")]
4820 ;; XXX We should match (any_extend (ashift)) here, like (and (ashift)) below
4822 (define_insn "*andim_ashift<mode>_bfiz"
4823 [(set (match_operand:GPI 0 "register_operand" "=r")
4824 (and:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
4825 (match_operand 2 "const_int_operand" "n"))
4826 (match_operand 3 "const_int_operand" "n")))]
4827 "aarch64_mask_and_shift_for_ubfiz_p (<MODE>mode, operands[3], operands[2])"
4828 "ubfiz\\t%<w>0, %<w>1, %2, %P3"
4829 [(set_attr "type" "bfx")]
4832 ;; When the bit position and width of the equivalent extraction add up to 32
4833 ;; we can use a W-reg LSL instruction taking advantage of the implicit
4834 ;; zero-extension of the X-reg.
4836 [(set (match_operand:DI 0 "register_operand")
4837 (and:DI (ashift:DI (match_operand:DI 1 "register_operand")
4838 (match_operand 2 "const_int_operand"))
4839 (match_operand 3 "const_int_operand")))]
4840 "aarch64_mask_and_shift_for_ubfiz_p (DImode, operands[3], operands[2])
4841 && (INTVAL (operands[2]) + popcount_hwi (INTVAL (operands[3])))
4842 == GET_MODE_BITSIZE (SImode)"
4844 (zero_extend:DI (ashift:SI (match_dup 4) (match_dup 2))))]
4846 operands[4] = gen_lowpart (SImode, operands[1]);
4850 (define_insn "bswap<mode>2"
4851 [(set (match_operand:GPI 0 "register_operand" "=r")
4852 (bswap:GPI (match_operand:GPI 1 "register_operand" "r")))]
4854 "rev\\t%<w>0, %<w>1"
4855 [(set_attr "type" "rev")]
4858 (define_insn "bswaphi2"
4859 [(set (match_operand:HI 0 "register_operand" "=r")
4860 (bswap:HI (match_operand:HI 1 "register_operand" "r")))]
4863 [(set_attr "type" "rev")]
4866 ;; There are no canonicalisation rules for the position of the lshiftrt, ashift
4867 ;; operations within an IOR/AND RTX, therefore we have two patterns matching
4868 ;; each valid permutation.
4870 (define_insn "rev16<mode>2"
4871 [(set (match_operand:GPI 0 "register_operand" "=r")
4872 (ior:GPI (and:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
4874 (match_operand:GPI 3 "const_int_operand" "n"))
4875 (and:GPI (lshiftrt:GPI (match_dup 1)
4877 (match_operand:GPI 2 "const_int_operand" "n"))))]
4878 "aarch_rev16_shleft_mask_imm_p (operands[3], <MODE>mode)
4879 && aarch_rev16_shright_mask_imm_p (operands[2], <MODE>mode)"
4880 "rev16\\t%<w>0, %<w>1"
4881 [(set_attr "type" "rev")]
4884 (define_insn "rev16<mode>2_alt"
4885 [(set (match_operand:GPI 0 "register_operand" "=r")
4886 (ior:GPI (and:GPI (lshiftrt:GPI (match_operand:GPI 1 "register_operand" "r")
4888 (match_operand:GPI 2 "const_int_operand" "n"))
4889 (and:GPI (ashift:GPI (match_dup 1)
4891 (match_operand:GPI 3 "const_int_operand" "n"))))]
4892 "aarch_rev16_shleft_mask_imm_p (operands[3], <MODE>mode)
4893 && aarch_rev16_shright_mask_imm_p (operands[2], <MODE>mode)"
4894 "rev16\\t%<w>0, %<w>1"
4895 [(set_attr "type" "rev")]
4898 ;; zero_extend version of above
4899 (define_insn "*bswapsi2_uxtw"
4900 [(set (match_operand:DI 0 "register_operand" "=r")
4901 (zero_extend:DI (bswap:SI (match_operand:SI 1 "register_operand" "r"))))]
4904 [(set_attr "type" "rev")]
4907 ;; -------------------------------------------------------------------
4908 ;; Floating-point intrinsics
4909 ;; -------------------------------------------------------------------
4911 ;; frint floating-point round to integral standard patterns.
4912 ;; Expands to btrunc, ceil, floor, nearbyint, rint, round, frintn.
4914 (define_insn "<frint_pattern><mode>2"
4915 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
4916 (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")]
4919 "frint<frint_suffix>\\t%<s>0, %<s>1"
4920 [(set_attr "type" "f_rint<stype>")]
4923 ;; frcvt floating-point round to integer and convert standard patterns.
4924 ;; Expands to lbtrunc, lceil, lfloor, lround.
4925 (define_insn "l<fcvt_pattern><su_optab><GPF_F16:mode><GPI:mode>2"
4926 [(set (match_operand:GPI 0 "register_operand" "=r")
4928 (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")]
4931 "fcvt<frint_suffix><su>\\t%<GPI:w>0, %<GPF_F16:s>1"
4932 [(set_attr "type" "f_cvtf2i")]
4935 (define_insn "*aarch64_fcvt<su_optab><GPF:mode><GPI:mode>2_mult"
4936 [(set (match_operand:GPI 0 "register_operand" "=r")
4939 (match_operand:GPF 1 "register_operand" "w")
4940 (match_operand:GPF 2 "aarch64_fp_pow2" "F"))))]
4942 && IN_RANGE (aarch64_fpconst_pow_of_2 (operands[2]), 1,
4943 GET_MODE_BITSIZE (<GPI:MODE>mode))"
4945 int fbits = aarch64_fpconst_pow_of_2 (operands[2]);
4947 snprintf (buf, 64, "fcvtz<su>\\t%%<GPI:w>0, %%<GPF:s>1, #%d", fbits);
4948 output_asm_insn (buf, operands);
4951 [(set_attr "type" "f_cvtf2i")]
4956 (define_insn "fma<mode>4"
4957 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
4958 (fma:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")
4959 (match_operand:GPF_F16 2 "register_operand" "w")
4960 (match_operand:GPF_F16 3 "register_operand" "w")))]
4962 "fmadd\\t%<s>0, %<s>1, %<s>2, %<s>3"
4963 [(set_attr "type" "fmac<stype>")]
4966 (define_insn "fnma<mode>4"
4967 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
4969 (neg:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w"))
4970 (match_operand:GPF_F16 2 "register_operand" "w")
4971 (match_operand:GPF_F16 3 "register_operand" "w")))]
4973 "fmsub\\t%<s>0, %<s>1, %<s>2, %<s>3"
4974 [(set_attr "type" "fmac<stype>")]
4977 (define_insn "fms<mode>4"
4978 [(set (match_operand:GPF 0 "register_operand" "=w")
4979 (fma:GPF (match_operand:GPF 1 "register_operand" "w")
4980 (match_operand:GPF 2 "register_operand" "w")
4981 (neg:GPF (match_operand:GPF 3 "register_operand" "w"))))]
4983 "fnmsub\\t%<s>0, %<s>1, %<s>2, %<s>3"
4984 [(set_attr "type" "fmac<s>")]
4987 (define_insn "fnms<mode>4"
4988 [(set (match_operand:GPF 0 "register_operand" "=w")
4989 (fma:GPF (neg:GPF (match_operand:GPF 1 "register_operand" "w"))
4990 (match_operand:GPF 2 "register_operand" "w")
4991 (neg:GPF (match_operand:GPF 3 "register_operand" "w"))))]
4993 "fnmadd\\t%<s>0, %<s>1, %<s>2, %<s>3"
4994 [(set_attr "type" "fmac<s>")]
4997 ;; If signed zeros are ignored, -(a * b + c) = -a * b - c.
4998 (define_insn "*fnmadd<mode>4"
4999 [(set (match_operand:GPF 0 "register_operand" "=w")
5000 (neg:GPF (fma:GPF (match_operand:GPF 1 "register_operand" "w")
5001 (match_operand:GPF 2 "register_operand" "w")
5002 (match_operand:GPF 3 "register_operand" "w"))))]
5003 "!HONOR_SIGNED_ZEROS (<MODE>mode) && TARGET_FLOAT"
5004 "fnmadd\\t%<s>0, %<s>1, %<s>2, %<s>3"
5005 [(set_attr "type" "fmac<s>")]
5008 ;; -------------------------------------------------------------------
5009 ;; Floating-point conversions
5010 ;; -------------------------------------------------------------------
5012 (define_insn "extendsfdf2"
5013 [(set (match_operand:DF 0 "register_operand" "=w")
5014 (float_extend:DF (match_operand:SF 1 "register_operand" "w")))]
5017 [(set_attr "type" "f_cvt")]
5020 (define_insn "extendhfsf2"
5021 [(set (match_operand:SF 0 "register_operand" "=w")
5022 (float_extend:SF (match_operand:HF 1 "register_operand" "w")))]
5025 [(set_attr "type" "f_cvt")]
5028 (define_insn "extendhfdf2"
5029 [(set (match_operand:DF 0 "register_operand" "=w")
5030 (float_extend:DF (match_operand:HF 1 "register_operand" "w")))]
5033 [(set_attr "type" "f_cvt")]
5036 (define_insn "truncdfsf2"
5037 [(set (match_operand:SF 0 "register_operand" "=w")
5038 (float_truncate:SF (match_operand:DF 1 "register_operand" "w")))]
5041 [(set_attr "type" "f_cvt")]
5044 (define_insn "truncsfhf2"
5045 [(set (match_operand:HF 0 "register_operand" "=w")
5046 (float_truncate:HF (match_operand:SF 1 "register_operand" "w")))]
5049 [(set_attr "type" "f_cvt")]
5052 (define_insn "truncdfhf2"
5053 [(set (match_operand:HF 0 "register_operand" "=w")
5054 (float_truncate:HF (match_operand:DF 1 "register_operand" "w")))]
5057 [(set_attr "type" "f_cvt")]
5060 ;; Convert SF -> SI or DF -> DI while preferring w = w register constraints
5061 ;; and making r = w more expensive
5063 (define_insn "<optab>_trunc<fcvt_target><GPI:mode>2"
5064 [(set (match_operand:GPI 0 "register_operand" "=?r,w")
5065 (FIXUORS:GPI (match_operand:<FCVT_TARGET> 1 "register_operand" "w,w")))]
5068 fcvtz<su>\t%<w>0, %<s>1
5069 fcvtz<su>\t%<s>0, %<s>1"
5070 [(set_attr "type" "f_cvtf2i,neon_fp_to_int_s")]
5073 ;; Convert HF -> SI or DI
5075 (define_insn "<optab>_trunchf<GPI:mode>2"
5076 [(set (match_operand:GPI 0 "register_operand" "=r")
5077 (FIXUORS:GPI (match_operand:HF 1 "register_operand" "w")))]
5079 "fcvtz<su>\t%<w>0, %h1"
5080 [(set_attr "type" "f_cvtf2i")]
5083 ;; Convert DF -> SI or SF -> DI which can only be accomplished with
5084 ;; input in a fp register and output in a integer register
5086 (define_insn "<optab>_trunc<fcvt_change_mode><GPI:mode>2"
5087 [(set (match_operand:GPI 0 "register_operand" "=r")
5088 (FIXUORS:GPI (match_operand:<FCVT_CHANGE_MODE> 1 "register_operand" "w")))]
5090 "fcvtz<su>\t%<w>0, %<fpw>1"
5091 [(set_attr "type" "f_cvtf2i")]
5094 (define_insn "<optab><fcvt_target><GPF:mode>2"
5095 [(set (match_operand:GPF 0 "register_operand" "=w,w")
5096 (FLOATUORS:GPF (match_operand:<FCVT_TARGET> 1 "register_operand" "w,r")))]
5099 <su_optab>cvtf\t%<GPF:s>0, %<s>1
5100 <su_optab>cvtf\t%<GPF:s>0, %<w1>1"
5101 [(set_attr "simd" "yes,no")
5102 (set_attr "fp" "no,yes")
5103 (set_attr "type" "neon_int_to_fp_<Vetype>,f_cvti2f")]
5106 (define_insn "<optab><fcvt_iesize><GPF:mode>2"
5107 [(set (match_operand:GPF 0 "register_operand" "=w")
5108 (FLOATUORS:GPF (match_operand:<FCVT_IESIZE> 1 "register_operand" "r")))]
5110 "<su_optab>cvtf\t%<GPF:s>0, %<w2>1"
5111 [(set_attr "type" "f_cvti2f")]
5114 ;; If we do not have ARMv8.2-A 16-bit floating point extensions, the
5115 ;; midend will arrange for an SImode conversion to HFmode to first go
5116 ;; through DFmode, then to HFmode. But first it will try converting
5117 ;; to DImode then down, which would match our DImode pattern below and
5118 ;; give very poor code-generation. So, we must provide our own emulation
5119 ;; of the mid-end logic.
5121 (define_insn "aarch64_fp16_<optab><mode>hf2"
5122 [(set (match_operand:HF 0 "register_operand" "=w")
5123 (FLOATUORS:HF (match_operand:GPI 1 "register_operand" "r")))]
5125 "<su_optab>cvtf\t%h0, %<w>1"
5126 [(set_attr "type" "f_cvti2f")]
5129 (define_expand "<optab>sihf2"
5130 [(set (match_operand:HF 0 "register_operand")
5131 (FLOATUORS:HF (match_operand:SI 1 "register_operand")))]
5134 if (TARGET_FP_F16INST)
5135 emit_insn (gen_aarch64_fp16_<optab>sihf2 (operands[0], operands[1]));
5138 rtx convert_target = gen_reg_rtx (DFmode);
5139 emit_insn (gen_<optab>sidf2 (convert_target, operands[1]));
5140 emit_insn (gen_truncdfhf2 (operands[0], convert_target));
5146 ;; For DImode there is no wide enough floating-point mode that we
5147 ;; can convert through natively (TFmode would work, but requires a library
5148 ;; call). However, we know that any value >= 65504 will be rounded
5149 ;; to infinity on conversion. This is well within the range of SImode, so
5151 ;; Saturate to SImode.
5152 ;; Convert from that to DFmode
5153 ;; Convert from that to HFmode (phew!).
5154 ;; Note that the saturation to SImode requires the SIMD extensions. If
5155 ;; we ever need to provide this pattern where the SIMD extensions are not
5156 ;; available, we would need a different approach.
5158 (define_expand "<optab>dihf2"
5159 [(set (match_operand:HF 0 "register_operand")
5160 (FLOATUORS:HF (match_operand:DI 1 "register_operand")))]
5161 "TARGET_FLOAT && (TARGET_FP_F16INST || TARGET_SIMD)"
5163 if (TARGET_FP_F16INST)
5164 emit_insn (gen_aarch64_fp16_<optab>dihf2 (operands[0], operands[1]));
5167 rtx sat_target = gen_reg_rtx (SImode);
5168 emit_insn (gen_aarch64_<su_optab>qmovndi (sat_target, operands[1]));
5169 emit_insn (gen_<optab>sihf2 (operands[0], sat_target));
5176 ;; Convert between fixed-point and floating-point (scalar modes)
5178 (define_insn "<FCVT_F2FIXED:fcvt_fixed_insn><GPF:mode>3"
5179 [(set (match_operand:<GPF:FCVT_TARGET> 0 "register_operand" "=r, w")
5180 (unspec:<GPF:FCVT_TARGET> [(match_operand:GPF 1 "register_operand" "w, w")
5181 (match_operand:SI 2 "immediate_operand" "i, i")]
5185 <FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPF:w1>0, %<GPF:s>1, #%2
5186 <FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPF:s>0, %<GPF:s>1, #%2"
5187 [(set_attr "type" "f_cvtf2i, neon_fp_to_int_<GPF:Vetype>")
5188 (set_attr "fp" "yes, *")
5189 (set_attr "simd" "*, yes")]
5192 (define_insn "<FCVT_FIXED2F:fcvt_fixed_insn><GPI:mode>3"
5193 [(set (match_operand:<GPI:FCVT_TARGET> 0 "register_operand" "=w, w")
5194 (unspec:<GPI:FCVT_TARGET> [(match_operand:GPI 1 "register_operand" "r, w")
5195 (match_operand:SI 2 "immediate_operand" "i, i")]
5199 <FCVT_FIXED2F:fcvt_fixed_insn>\t%<GPI:v>0, %<GPI:w>1, #%2
5200 <FCVT_FIXED2F:fcvt_fixed_insn>\t%<GPI:v>0, %<GPI:v>1, #%2"
5201 [(set_attr "type" "f_cvti2f, neon_int_to_fp_<GPI:Vetype>")
5202 (set_attr "fp" "yes, *")
5203 (set_attr "simd" "*, yes")]
5206 (define_insn "<FCVT_F2FIXED:fcvt_fixed_insn>hf<mode>3"
5207 [(set (match_operand:GPI 0 "register_operand" "=r")
5208 (unspec:GPI [(match_operand:HF 1 "register_operand" "w")
5209 (match_operand:SI 2 "immediate_operand" "i")]
5212 "<FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPI:w>0, %h1, #%2"
5213 [(set_attr "type" "f_cvtf2i")]
5216 (define_insn "<FCVT_FIXED2F:fcvt_fixed_insn><mode>hf3"
5217 [(set (match_operand:HF 0 "register_operand" "=w")
5218 (unspec:HF [(match_operand:GPI 1 "register_operand" "r")
5219 (match_operand:SI 2 "immediate_operand" "i")]
5222 "<FCVT_FIXED2F:fcvt_fixed_insn>\t%h0, %<GPI:w>1, #%2"
5223 [(set_attr "type" "f_cvti2f")]
5226 (define_insn "<FCVT_F2FIXED:fcvt_fixed_insn>hf3"
5227 [(set (match_operand:HI 0 "register_operand" "=w")
5228 (unspec:HI [(match_operand:HF 1 "register_operand" "w")
5229 (match_operand:SI 2 "immediate_operand" "i")]
5232 "<FCVT_F2FIXED:fcvt_fixed_insn>\t%h0, %h1, #%2"
5233 [(set_attr "type" "neon_fp_to_int_s")]
5236 (define_insn "<FCVT_FIXED2F:fcvt_fixed_insn>hi3"
5237 [(set (match_operand:HF 0 "register_operand" "=w")
5238 (unspec:HF [(match_operand:HI 1 "register_operand" "w")
5239 (match_operand:SI 2 "immediate_operand" "i")]
5242 "<FCVT_FIXED2F:fcvt_fixed_insn>\t%h0, %h1, #%2"
5243 [(set_attr "type" "neon_int_to_fp_s")]
5246 ;; -------------------------------------------------------------------
5247 ;; Floating-point arithmetic
5248 ;; -------------------------------------------------------------------
5250 (define_insn "add<mode>3"
5251 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5253 (match_operand:GPF_F16 1 "register_operand" "w")
5254 (match_operand:GPF_F16 2 "register_operand" "w")))]
5256 "fadd\\t%<s>0, %<s>1, %<s>2"
5257 [(set_attr "type" "fadd<stype>")]
5260 (define_insn "sub<mode>3"
5261 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5263 (match_operand:GPF_F16 1 "register_operand" "w")
5264 (match_operand:GPF_F16 2 "register_operand" "w")))]
5266 "fsub\\t%<s>0, %<s>1, %<s>2"
5267 [(set_attr "type" "fadd<stype>")]
5270 (define_insn "mul<mode>3"
5271 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5273 (match_operand:GPF_F16 1 "register_operand" "w")
5274 (match_operand:GPF_F16 2 "register_operand" "w")))]
5276 "fmul\\t%<s>0, %<s>1, %<s>2"
5277 [(set_attr "type" "fmul<stype>")]
5280 (define_insn "*fnmul<mode>3"
5281 [(set (match_operand:GPF 0 "register_operand" "=w")
5283 (neg:GPF (match_operand:GPF 1 "register_operand" "w"))
5284 (match_operand:GPF 2 "register_operand" "w")))]
5285 "TARGET_FLOAT && !flag_rounding_math"
5286 "fnmul\\t%<s>0, %<s>1, %<s>2"
5287 [(set_attr "type" "fmul<s>")]
5290 (define_insn "*fnmul<mode>3"
5291 [(set (match_operand:GPF 0 "register_operand" "=w")
5293 (match_operand:GPF 1 "register_operand" "w")
5294 (match_operand:GPF 2 "register_operand" "w"))))]
5296 "fnmul\\t%<s>0, %<s>1, %<s>2"
5297 [(set_attr "type" "fmul<s>")]
5300 (define_expand "div<mode>3"
5301 [(set (match_operand:GPF_F16 0 "register_operand")
5302 (div:GPF_F16 (match_operand:GPF_F16 1 "general_operand")
5303 (match_operand:GPF_F16 2 "register_operand")))]
5306 if (aarch64_emit_approx_div (operands[0], operands[1], operands[2]))
5309 operands[1] = force_reg (<MODE>mode, operands[1]);
5312 (define_insn "*div<mode>3"
5313 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5314 (div:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")
5315 (match_operand:GPF_F16 2 "register_operand" "w")))]
5317 "fdiv\\t%<s>0, %<s>1, %<s>2"
5318 [(set_attr "type" "fdiv<stype>")]
5321 (define_insn "neg<mode>2"
5322 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5323 (neg:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
5325 "fneg\\t%<s>0, %<s>1"
5326 [(set_attr "type" "ffarith<stype>")]
5329 (define_expand "sqrt<mode>2"
5330 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5331 (sqrt:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
5334 if (aarch64_emit_approx_sqrt (operands[0], operands[1], false))
5338 (define_insn "*sqrt<mode>2"
5339 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5340 (sqrt:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
5342 "fsqrt\\t%<s>0, %<s>1"
5343 [(set_attr "type" "fsqrt<stype>")]
5346 (define_insn "abs<mode>2"
5347 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5348 (abs:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
5350 "fabs\\t%<s>0, %<s>1"
5351 [(set_attr "type" "ffarith<stype>")]
5354 ;; Given that smax/smin do not specify the result when either input is NaN,
5355 ;; we could use either FMAXNM or FMAX for smax, and either FMINNM or FMIN
5358 (define_insn "smax<mode>3"
5359 [(set (match_operand:GPF 0 "register_operand" "=w")
5360 (smax:GPF (match_operand:GPF 1 "register_operand" "w")
5361 (match_operand:GPF 2 "register_operand" "w")))]
5363 "fmaxnm\\t%<s>0, %<s>1, %<s>2"
5364 [(set_attr "type" "f_minmax<s>")]
5367 (define_insn "smin<mode>3"
5368 [(set (match_operand:GPF 0 "register_operand" "=w")
5369 (smin:GPF (match_operand:GPF 1 "register_operand" "w")
5370 (match_operand:GPF 2 "register_operand" "w")))]
5372 "fminnm\\t%<s>0, %<s>1, %<s>2"
5373 [(set_attr "type" "f_minmax<s>")]
5376 ;; Scalar forms for fmax, fmin, fmaxnm, fminnm.
5377 ;; fmaxnm and fminnm are used for the fmax<mode>3 standard pattern names,
5378 ;; which implement the IEEE fmax ()/fmin () functions.
5379 (define_insn "<maxmin_uns><mode>3"
5380 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5381 (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")
5382 (match_operand:GPF_F16 2 "register_operand" "w")]
5385 "<maxmin_uns_op>\\t%<s>0, %<s>1, %<s>2"
5386 [(set_attr "type" "f_minmax<stype>")]
5389 (define_expand "lrint<GPF:mode><GPI:mode>2"
5390 [(match_operand:GPI 0 "register_operand")
5391 (match_operand:GPF 1 "register_operand")]
5393 && ((GET_MODE_SIZE (<GPF:MODE>mode) <= GET_MODE_SIZE (<GPI:MODE>mode))
5394 || !flag_trapping_math || flag_fp_int_builtin_inexact)"
5396 rtx cvt = gen_reg_rtx (<GPF:MODE>mode);
5397 emit_insn (gen_rint<GPF:mode>2 (cvt, operands[1]));
5398 emit_insn (gen_lbtrunc<GPF:mode><GPI:mode>2 (operands[0], cvt));
5403 ;; For copysign (x, y), we want to generate:
5405 ;; LDR d2, #(1 << 63)
5406 ;; BSL v2.8b, [y], [x]
5408 ;; or another, equivalent, sequence using one of BSL/BIT/BIF.
5409 ;; aarch64_simd_bsldf will select the best suited of these instructions
5410 ;; to generate based on register allocation, and knows how to partially
5411 ;; constant fold based on the values of X and Y, so expand through that.
5413 (define_expand "copysigndf3"
5414 [(match_operand:DF 0 "register_operand")
5415 (match_operand:DF 1 "register_operand")
5416 (match_operand:DF 2 "register_operand")]
5417 "TARGET_FLOAT && TARGET_SIMD"
5419 rtx mask = gen_reg_rtx (DImode);
5420 emit_move_insn (mask, GEN_INT (HOST_WIDE_INT_1U << 63));
5421 emit_insn (gen_aarch64_simd_bsldf (operands[0], mask,
5422 operands[2], operands[1]));
5427 ;; As above, but we must first get to a 64-bit value if we wish to use
5428 ;; aarch64_simd_bslv2sf.
5430 (define_expand "copysignsf3"
5431 [(match_operand:SF 0 "register_operand")
5432 (match_operand:SF 1 "register_operand")
5433 (match_operand:SF 2 "register_operand")]
5434 "TARGET_FLOAT && TARGET_SIMD"
5436 rtx v_bitmask = gen_reg_rtx (V2SImode);
5438 /* Juggle modes to get us in to a vector mode for BSL. */
5439 rtx op1 = lowpart_subreg (DImode, operands[1], SFmode);
5440 rtx op2 = lowpart_subreg (V2SFmode, operands[2], SFmode);
5441 rtx tmp = gen_reg_rtx (V2SFmode);
5442 emit_move_insn (v_bitmask,
5443 aarch64_simd_gen_const_vector_dup (V2SImode,
5444 HOST_WIDE_INT_M1U << 31));
5445 emit_insn (gen_aarch64_simd_bslv2sf (tmp, v_bitmask, op2, op1));
5446 emit_move_insn (operands[0], lowpart_subreg (SFmode, tmp, V2SFmode));
5451 ;; For xorsign (x, y), we want to generate:
5454 ;; AND v3.8B, v1.8B, v2.8B
5455 ;; EOR v0.8B, v0.8B, v3.8B
5458 (define_expand "xorsign<mode>3"
5459 [(match_operand:GPF 0 "register_operand")
5460 (match_operand:GPF 1 "register_operand")
5461 (match_operand:GPF 2 "register_operand")]
5462 "TARGET_FLOAT && TARGET_SIMD"
5465 machine_mode imode = <V_INT_EQUIV>mode;
5466 rtx mask = gen_reg_rtx (imode);
5467 rtx op1x = gen_reg_rtx (imode);
5468 rtx op2x = gen_reg_rtx (imode);
5470 int bits = GET_MODE_BITSIZE (<MODE>mode) - 1;
5471 emit_move_insn (mask, GEN_INT (trunc_int_for_mode (HOST_WIDE_INT_M1U << bits,
5474 emit_insn (gen_and<v_int_equiv>3 (op2x, mask,
5475 lowpart_subreg (imode, operands[2],
5477 emit_insn (gen_xor<v_int_equiv>3 (op1x,
5478 lowpart_subreg (imode, operands[1],
5481 emit_move_insn (operands[0],
5482 lowpart_subreg (<MODE>mode, op1x, imode));
5487 ;; -------------------------------------------------------------------
5489 ;; -------------------------------------------------------------------
5490 ;; Reload Scalar Floating point modes from constant pool.
5491 ;; The AArch64 port doesn't have __int128 constant move support.
5492 (define_expand "aarch64_reload_movcp<GPF_TF:mode><P:mode>"
5493 [(set (match_operand:GPF_TF 0 "register_operand" "=w")
5494 (mem:GPF_TF (match_operand 1 "aarch64_constant_pool_symref" "S")))
5495 (clobber (match_operand:P 2 "register_operand" "=&r"))]
5498 aarch64_expand_mov_immediate (operands[2], XEXP (operands[1], 0));
5499 emit_move_insn (operands[0], gen_rtx_MEM (<GPF_TF:MODE>mode, operands[2]));
5504 ;; Reload Vector modes from constant pool.
5505 (define_expand "aarch64_reload_movcp<VALL:mode><P:mode>"
5506 [(set (match_operand:VALL 0 "register_operand" "=w")
5507 (mem:VALL (match_operand 1 "aarch64_constant_pool_symref" "S")))
5508 (clobber (match_operand:P 2 "register_operand" "=&r"))]
5511 aarch64_expand_mov_immediate (operands[2], XEXP (operands[1], 0));
5512 emit_move_insn (operands[0], gen_rtx_MEM (<VALL:MODE>mode, operands[2]));
5517 (define_expand "aarch64_reload_mov<mode>"
5518 [(set (match_operand:TX 0 "register_operand" "=w")
5519 (match_operand:TX 1 "register_operand" "w"))
5520 (clobber (match_operand:DI 2 "register_operand" "=&r"))
5524 rtx op0 = simplify_gen_subreg (TImode, operands[0], <MODE>mode, 0);
5525 rtx op1 = simplify_gen_subreg (TImode, operands[1], <MODE>mode, 0);
5526 gen_aarch64_movtilow_tilow (op0, op1);
5527 gen_aarch64_movdi_tihigh (operands[2], op1);
5528 gen_aarch64_movtihigh_di (op0, operands[2]);
5533 ;; The following secondary reload helpers patterns are invoked
5534 ;; after or during reload as we don't want these patterns to start
5535 ;; kicking in during the combiner.
5537 (define_insn "aarch64_movdi_<mode>low"
5538 [(set (match_operand:DI 0 "register_operand" "=r")
5539 (zero_extract:DI (match_operand:TX 1 "register_operand" "w")
5540 (const_int 64) (const_int 0)))]
5541 "TARGET_FLOAT && (reload_completed || reload_in_progress)"
5543 [(set_attr "type" "f_mrc")
5544 (set_attr "length" "4")
5547 (define_insn "aarch64_movdi_<mode>high"
5548 [(set (match_operand:DI 0 "register_operand" "=r")
5549 (zero_extract:DI (match_operand:TX 1 "register_operand" "w")
5550 (const_int 64) (const_int 64)))]
5551 "TARGET_FLOAT && (reload_completed || reload_in_progress)"
5552 "fmov\\t%x0, %1.d[1]"
5553 [(set_attr "type" "f_mrc")
5554 (set_attr "length" "4")
5557 (define_insn "aarch64_mov<mode>high_di"
5558 [(set (zero_extract:TX (match_operand:TX 0 "register_operand" "+w")
5559 (const_int 64) (const_int 64))
5560 (zero_extend:TX (match_operand:DI 1 "register_operand" "r")))]
5561 "TARGET_FLOAT && (reload_completed || reload_in_progress)"
5562 "fmov\\t%0.d[1], %x1"
5563 [(set_attr "type" "f_mcr")
5564 (set_attr "length" "4")
5567 (define_insn "aarch64_mov<mode>low_di"
5568 [(set (match_operand:TX 0 "register_operand" "=w")
5569 (zero_extend:TX (match_operand:DI 1 "register_operand" "r")))]
5570 "TARGET_FLOAT && (reload_completed || reload_in_progress)"
5572 [(set_attr "type" "f_mcr")
5573 (set_attr "length" "4")
5576 (define_insn "aarch64_movtilow_tilow"
5577 [(set (match_operand:TI 0 "register_operand" "=w")
5579 (truncate:DI (match_operand:TI 1 "register_operand" "w"))))]
5580 "TARGET_FLOAT && (reload_completed || reload_in_progress)"
5582 [(set_attr "type" "fmov")
5583 (set_attr "length" "4")
5586 ;; There is a deliberate reason why the parameters of high and lo_sum's
5587 ;; don't have modes for ADRP and ADD instructions. This is to allow high
5588 ;; and lo_sum's to be used with the labels defining the jump tables in
5591 (define_expand "add_losym"
5592 [(set (match_operand 0 "register_operand" "=r")
5593 (lo_sum (match_operand 1 "register_operand" "r")
5594 (match_operand 2 "aarch64_valid_symref" "S")))]
5597 machine_mode mode = GET_MODE (operands[0]);
5599 emit_insn ((mode == DImode
5601 : gen_add_losym_si) (operands[0],
5607 (define_insn "add_losym_<mode>"
5608 [(set (match_operand:P 0 "register_operand" "=r")
5609 (lo_sum:P (match_operand:P 1 "register_operand" "r")
5610 (match_operand 2 "aarch64_valid_symref" "S")))]
5612 "add\\t%<w>0, %<w>1, :lo12:%c2"
5613 [(set_attr "type" "alu_imm")]
5616 (define_insn "ldr_got_small_<mode>"
5617 [(set (match_operand:PTR 0 "register_operand" "=r")
5618 (unspec:PTR [(mem:PTR (lo_sum:PTR
5619 (match_operand:PTR 1 "register_operand" "r")
5620 (match_operand:PTR 2 "aarch64_valid_symref" "S")))]
5621 UNSPEC_GOTSMALLPIC))]
5623 "ldr\\t%<w>0, [%1, #:got_lo12:%c2]"
5624 [(set_attr "type" "load_<ldst_sz>")]
5627 (define_insn "ldr_got_small_sidi"
5628 [(set (match_operand:DI 0 "register_operand" "=r")
5630 (unspec:SI [(mem:SI (lo_sum:DI
5631 (match_operand:DI 1 "register_operand" "r")
5632 (match_operand:DI 2 "aarch64_valid_symref" "S")))]
5633 UNSPEC_GOTSMALLPIC)))]
5635 "ldr\\t%w0, [%1, #:got_lo12:%c2]"
5636 [(set_attr "type" "load_4")]
5639 (define_insn "ldr_got_small_28k_<mode>"
5640 [(set (match_operand:PTR 0 "register_operand" "=r")
5641 (unspec:PTR [(mem:PTR (lo_sum:PTR
5642 (match_operand:PTR 1 "register_operand" "r")
5643 (match_operand:PTR 2 "aarch64_valid_symref" "S")))]
5644 UNSPEC_GOTSMALLPIC28K))]
5646 "ldr\\t%<w>0, [%1, #:<got_modifier>:%c2]"
5647 [(set_attr "type" "load_<ldst_sz>")]
5650 (define_insn "ldr_got_small_28k_sidi"
5651 [(set (match_operand:DI 0 "register_operand" "=r")
5653 (unspec:SI [(mem:SI (lo_sum:DI
5654 (match_operand:DI 1 "register_operand" "r")
5655 (match_operand:DI 2 "aarch64_valid_symref" "S")))]
5656 UNSPEC_GOTSMALLPIC28K)))]
5658 "ldr\\t%w0, [%1, #:gotpage_lo14:%c2]"
5659 [(set_attr "type" "load_4")]
5662 (define_insn "ldr_got_tiny"
5663 [(set (match_operand:DI 0 "register_operand" "=r")
5664 (unspec:DI [(match_operand:DI 1 "aarch64_valid_symref" "S")]
5665 UNSPEC_GOTTINYPIC))]
5668 [(set_attr "type" "load_8")]
5671 (define_insn "aarch64_load_tp_hard"
5672 [(set (match_operand:DI 0 "register_operand" "=r")
5673 (unspec:DI [(const_int 0)] UNSPEC_TLS))]
5675 "mrs\\t%0, tpidr_el0"
5676 [(set_attr "type" "mrs")]
5679 ;; The TLS ABI specifically requires that the compiler does not schedule
5680 ;; instructions in the TLS stubs, in order to enable linker relaxation.
5681 ;; Therefore we treat the stubs as an atomic sequence.
5682 (define_expand "tlsgd_small_<mode>"
5683 [(parallel [(set (match_operand 0 "register_operand" "")
5684 (call (mem:DI (match_dup 2)) (const_int 1)))
5685 (unspec:DI [(match_operand:PTR 1 "aarch64_valid_symref" "")] UNSPEC_GOTSMALLTLS)
5686 (clobber (reg:DI LR_REGNUM))])]
5689 operands[2] = aarch64_tls_get_addr ();
5692 (define_insn "*tlsgd_small_<mode>"
5693 [(set (match_operand 0 "register_operand" "")
5694 (call (mem:DI (match_operand:DI 2 "" "")) (const_int 1)))
5695 (unspec:DI [(match_operand:PTR 1 "aarch64_valid_symref" "S")] UNSPEC_GOTSMALLTLS)
5696 (clobber (reg:DI LR_REGNUM))
5699 "adrp\\tx0, %A1\;add\\tx0, x0, %L1\;bl\\t%2\;nop"
5700 [(set_attr "type" "call")
5701 (set_attr "length" "16")])
5703 (define_insn "tlsie_small_<mode>"
5704 [(set (match_operand:PTR 0 "register_operand" "=r")
5705 (unspec:PTR [(match_operand 1 "aarch64_tls_ie_symref" "S")]
5706 UNSPEC_GOTSMALLTLS))]
5708 "adrp\\t%0, %A1\;ldr\\t%<w>0, [%0, #%L1]"
5709 [(set_attr "type" "load_4")
5710 (set_attr "length" "8")]
5713 (define_insn "tlsie_small_sidi"
5714 [(set (match_operand:DI 0 "register_operand" "=r")
5716 (unspec:SI [(match_operand 1 "aarch64_tls_ie_symref" "S")]
5717 UNSPEC_GOTSMALLTLS)))]
5719 "adrp\\t%0, %A1\;ldr\\t%w0, [%0, #%L1]"
5720 [(set_attr "type" "load_4")
5721 (set_attr "length" "8")]
5724 (define_insn "tlsie_tiny_<mode>"
5725 [(set (match_operand:PTR 0 "register_operand" "=&r")
5726 (unspec:PTR [(match_operand 1 "aarch64_tls_ie_symref" "S")
5727 (match_operand:PTR 2 "register_operand" "r")]
5728 UNSPEC_GOTTINYTLS))]
5730 "ldr\\t%<w>0, %L1\;add\\t%<w>0, %<w>0, %<w>2"
5731 [(set_attr "type" "multiple")
5732 (set_attr "length" "8")]
5735 (define_insn "tlsie_tiny_sidi"
5736 [(set (match_operand:DI 0 "register_operand" "=&r")
5738 (unspec:SI [(match_operand 1 "aarch64_tls_ie_symref" "S")
5739 (match_operand:DI 2 "register_operand" "r")
5741 UNSPEC_GOTTINYTLS)))]
5743 "ldr\\t%w0, %L1\;add\\t%w0, %w0, %w2"
5744 [(set_attr "type" "multiple")
5745 (set_attr "length" "8")]
5748 (define_insn "tlsle12_<mode>"
5749 [(set (match_operand:P 0 "register_operand" "=r")
5750 (unspec:P [(match_operand:P 1 "register_operand" "r")
5751 (match_operand 2 "aarch64_tls_le_symref" "S")]
5754 "add\\t%<w>0, %<w>1, #%L2";
5755 [(set_attr "type" "alu_sreg")
5756 (set_attr "length" "4")]
5759 (define_insn "tlsle24_<mode>"
5760 [(set (match_operand:P 0 "register_operand" "=r")
5761 (unspec:P [(match_operand:P 1 "register_operand" "r")
5762 (match_operand 2 "aarch64_tls_le_symref" "S")]
5765 "add\\t%<w>0, %<w>1, #%G2, lsl #12\;add\\t%<w>0, %<w>0, #%L2"
5766 [(set_attr "type" "multiple")
5767 (set_attr "length" "8")]
5770 (define_insn "tlsle32_<mode>"
5771 [(set (match_operand:P 0 "register_operand" "=r")
5772 (unspec:P [(match_operand 1 "aarch64_tls_le_symref" "S")]
5775 "movz\\t%<w>0, #:tprel_g1:%1\;movk\\t%<w>0, #:tprel_g0_nc:%1"
5776 [(set_attr "type" "multiple")
5777 (set_attr "length" "8")]
5780 (define_insn "tlsle48_<mode>"
5781 [(set (match_operand:P 0 "register_operand" "=r")
5782 (unspec:P [(match_operand 1 "aarch64_tls_le_symref" "S")]
5785 "movz\\t%<w>0, #:tprel_g2:%1\;movk\\t%<w>0, #:tprel_g1_nc:%1\;movk\\t%<w>0, #:tprel_g0_nc:%1"
5786 [(set_attr "type" "multiple")
5787 (set_attr "length" "12")]
5790 (define_insn "tlsdesc_small_<mode>"
5791 [(set (reg:PTR R0_REGNUM)
5792 (unspec:PTR [(match_operand 0 "aarch64_valid_symref" "S")]
5794 (clobber (reg:DI LR_REGNUM))
5795 (clobber (reg:CC CC_REGNUM))
5796 (clobber (match_scratch:DI 1 "=r"))]
5798 "adrp\\tx0, %A0\;ldr\\t%<w>1, [x0, #%L0]\;add\\t<w>0, <w>0, %L0\;.tlsdesccall\\t%0\;blr\\t%1"
5799 [(set_attr "type" "call")
5800 (set_attr "length" "16")])
5802 (define_insn "stack_tie"
5803 [(set (mem:BLK (scratch))
5804 (unspec:BLK [(match_operand:DI 0 "register_operand" "rk")
5805 (match_operand:DI 1 "register_operand" "rk")]
5809 [(set_attr "length" "0")]
5812 ;; Pointer authentication patterns are always provided. In architecture
5813 ;; revisions prior to ARMv8.3-A these HINT instructions operate as NOPs.
5814 ;; This lets the user write portable software which authenticates pointers
5815 ;; when run on something which implements ARMv8.3-A, and which runs
5816 ;; correctly, but does not authenticate pointers, where ARMv8.3-A is not
5819 ;; Signing/Authenticating R30 using SP as the salt.
5821 (define_insn "<pauth_mnem_prefix>sp"
5822 [(set (reg:DI R30_REGNUM)
5823 (unspec:DI [(reg:DI R30_REGNUM) (reg:DI SP_REGNUM)] PAUTH_LR_SP))]
5825 "hint\t<pauth_hint_num_a> // <pauth_mnem_prefix>asp";
5828 ;; Signing/Authenticating X17 using X16 as the salt.
5830 (define_insn "<pauth_mnem_prefix>1716"
5831 [(set (reg:DI R17_REGNUM)
5832 (unspec:DI [(reg:DI R17_REGNUM) (reg:DI R16_REGNUM)] PAUTH_17_16))]
5834 "hint\t<pauth_hint_num_a> // <pauth_mnem_prefix>a1716";
5837 ;; Stripping the signature in R30.
5839 (define_insn "xpaclri"
5840 [(set (reg:DI R30_REGNUM) (unspec:DI [(reg:DI R30_REGNUM)] UNSPEC_XPACLRI))]
5842 "hint\t7 // xpaclri"
5845 ;; UNSPEC_VOLATILE is considered to use and clobber all hard registers and
5846 ;; all of memory. This blocks insns from being moved across this point.
5848 (define_insn "blockage"
5849 [(unspec_volatile [(const_int 0)] UNSPECV_BLOCKAGE)]
5852 [(set_attr "length" "0")
5853 (set_attr "type" "block")]
5856 (define_insn "probe_stack_range"
5857 [(set (match_operand:DI 0 "register_operand" "=r")
5858 (unspec_volatile:DI [(match_operand:DI 1 "register_operand" "0")
5859 (match_operand:DI 2 "register_operand" "r")]
5860 UNSPECV_PROBE_STACK_RANGE))]
5863 return aarch64_output_probe_stack_range (operands[0], operands[2]);
5865 [(set_attr "length" "32")]
5868 ;; Named pattern for expanding thread pointer reference.
5869 (define_expand "get_thread_pointerdi"
5870 [(match_operand:DI 0 "register_operand" "=r")]
5873 rtx tmp = aarch64_load_tp (operands[0]);
5874 if (tmp != operands[0])
5875 emit_move_insn (operands[0], tmp);
5879 ;; Named patterns for stack smashing protection.
5880 (define_expand "stack_protect_set"
5881 [(match_operand 0 "memory_operand")
5882 (match_operand 1 "memory_operand")]
5885 machine_mode mode = GET_MODE (operands[0]);
5887 emit_insn ((mode == DImode
5888 ? gen_stack_protect_set_di
5889 : gen_stack_protect_set_si) (operands[0], operands[1]));
5893 (define_insn "stack_protect_set_<mode>"
5894 [(set (match_operand:PTR 0 "memory_operand" "=m")
5895 (unspec:PTR [(match_operand:PTR 1 "memory_operand" "m")]
5897 (set (match_scratch:PTR 2 "=&r") (const_int 0))]
5899 "ldr\\t%<w>2, %1\;str\\t%<w>2, %0\;mov\t%<w>2,0"
5900 [(set_attr "length" "12")
5901 (set_attr "type" "multiple")])
5903 (define_expand "stack_protect_test"
5904 [(match_operand 0 "memory_operand")
5905 (match_operand 1 "memory_operand")
5910 machine_mode mode = GET_MODE (operands[0]);
5912 result = gen_reg_rtx(mode);
5914 emit_insn ((mode == DImode
5915 ? gen_stack_protect_test_di
5916 : gen_stack_protect_test_si) (result,
5921 emit_jump_insn (gen_cbranchdi4 (gen_rtx_EQ (VOIDmode, result, const0_rtx),
5922 result, const0_rtx, operands[2]));
5924 emit_jump_insn (gen_cbranchsi4 (gen_rtx_EQ (VOIDmode, result, const0_rtx),
5925 result, const0_rtx, operands[2]));
5929 (define_insn "stack_protect_test_<mode>"
5930 [(set (match_operand:PTR 0 "register_operand" "=r")
5931 (unspec:PTR [(match_operand:PTR 1 "memory_operand" "m")
5932 (match_operand:PTR 2 "memory_operand" "m")]
5934 (clobber (match_scratch:PTR 3 "=&r"))]
5936 "ldr\t%<w>3, %1\;ldr\t%<w>0, %2\;eor\t%<w>0, %<w>3, %<w>0"
5937 [(set_attr "length" "12")
5938 (set_attr "type" "multiple")])
5940 ;; Write Floating-point Control Register.
5941 (define_insn "set_fpcr"
5942 [(unspec_volatile [(match_operand:SI 0 "register_operand" "r")] UNSPECV_SET_FPCR)]
5945 [(set_attr "type" "mrs")])
5947 ;; Read Floating-point Control Register.
5948 (define_insn "get_fpcr"
5949 [(set (match_operand:SI 0 "register_operand" "=r")
5950 (unspec_volatile:SI [(const_int 0)] UNSPECV_GET_FPCR))]
5953 [(set_attr "type" "mrs")])
5955 ;; Write Floating-point Status Register.
5956 (define_insn "set_fpsr"
5957 [(unspec_volatile [(match_operand:SI 0 "register_operand" "r")] UNSPECV_SET_FPSR)]
5960 [(set_attr "type" "mrs")])
5962 ;; Read Floating-point Status Register.
5963 (define_insn "get_fpsr"
5964 [(set (match_operand:SI 0 "register_operand" "=r")
5965 (unspec_volatile:SI [(const_int 0)] UNSPECV_GET_FPSR))]
5968 [(set_attr "type" "mrs")])
5971 ;; Define the subtract-one-and-jump insns so loop.c
5972 ;; knows what to generate.
5973 (define_expand "doloop_end"
5974 [(use (match_operand 0 "" "")) ; loop pseudo
5975 (use (match_operand 1 "" ""))] ; label
5976 "optimize > 0 && flag_modulo_sched"
5985 /* Currently SMS relies on the do-loop pattern to recognize loops
5986 where (1) the control part consists of all insns defining and/or
5987 using a certain 'count' register and (2) the loop count can be
5988 adjusted by modifying this register prior to the loop.
5989 ??? The possible introduction of a new block to initialize the
5990 new IV can potentially affect branch optimizations. */
5992 if (GET_MODE (operands[0]) != DImode)
5996 insn = emit_insn (gen_adddi3_compare0 (s0, s0, GEN_INT (-1)));
5998 cmp = XVECEXP (PATTERN (insn), 0, 0);
5999 cc_reg = SET_DEST (cmp);
6000 bcomp = gen_rtx_NE (VOIDmode, cc_reg, const0_rtx);
6001 loc_ref = gen_rtx_LABEL_REF (VOIDmode, operands [1]);
6002 emit_jump_insn (gen_rtx_SET (pc_rtx,
6003 gen_rtx_IF_THEN_ELSE (VOIDmode, bcomp,
6008 ;; Helper for aarch64.c code.
6009 (define_expand "set_clobber_cc"
6010 [(parallel [(set (match_operand 0)
6012 (clobber (reg:CC CC_REGNUM))])])
6015 (include "aarch64-simd.md")
6017 ;; Atomic Operations
6018 (include "atomics.md")
6020 ;; ldp/stp peephole patterns
6021 (include "aarch64-ldpstp.md")
6024 (include "aarch64-sve.md")