1 ;; Machine description for AArch64 architecture.
2 ;; Copyright (C) 2009-2018 Free Software Foundation, Inc.
3 ;; Contributed by ARM Ltd.
5 ;; This file is part of GCC.
7 ;; GCC is free software; you can redistribute it and/or modify it
8 ;; under the terms of the GNU General Public License as published by
9 ;; the Free Software Foundation; either version 3, or (at your option)
12 ;; GCC is distributed in the hope that it will be useful, but
13 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
14 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 ;; General Public License for more details.
17 ;; You should have received a copy of the GNU General Public License
18 ;; along with GCC; see the file COPYING3. If not see
19 ;; <http://www.gnu.org/licenses/>.
62 (LAST_SAVED_REGNUM 63)
66 ;; Defined only to make the DWARF description simpler.
74 (define_c_enum "unspec" [
100 UNSPEC_GOTSMALLPIC28K
174 (define_c_enum "unspecv" [
175 UNSPECV_EH_RETURN ; Represent EH_RETURN
176 UNSPECV_GET_FPCR ; Represent fetch of FPCR content.
177 UNSPECV_SET_FPCR ; Represent assign of FPCR content.
178 UNSPECV_GET_FPSR ; Represent fetch of FPSR content.
179 UNSPECV_SET_FPSR ; Represent assign of FPSR content.
180 UNSPECV_BLOCKAGE ; Represent a blockage
181 UNSPECV_PROBE_STACK_RANGE ; Represent stack range probing.
185 ;; If further include files are added the defintion of MD_INCLUDES
188 (include "constraints.md")
189 (include "predicates.md")
190 (include "iterators.md")
192 ;; -------------------------------------------------------------------
193 ;; Instruction types and attributes
194 ;; -------------------------------------------------------------------
196 ; The "type" attribute is included here from AArch32 backend to be able
197 ; to share pipeline descriptions.
198 (include "../arm/types.md")
200 ;; It is important to set the fp or simd attributes to yes when a pattern
201 ;; alternative uses the FP or SIMD register files, usually signified by use of
202 ;; the 'w' constraint. This will ensure that the alternative will be
203 ;; disabled when compiling with -mgeneral-regs-only or with the +nofp/+nosimd
204 ;; architecture extensions. If all the alternatives in a pattern use the
205 ;; FP or SIMD registers then the pattern predicate should include TARGET_FLOAT
208 ;; Attribute that specifies whether or not the instruction touches fp
209 ;; registers. When this is set to yes for an alternative, that alternative
210 ;; will be disabled when !TARGET_FLOAT.
211 (define_attr "fp" "no,yes" (const_string "no"))
213 ;; Attribute that specifies whether or not the instruction touches half
214 ;; precision fp registers. When this is set to yes for an alternative,
215 ;; that alternative will be disabled when !TARGET_FP_F16INST.
216 (define_attr "fp16" "no,yes" (const_string "no"))
218 ;; Attribute that specifies whether or not the instruction touches simd
219 ;; registers. When this is set to yes for an alternative, that alternative
220 ;; will be disabled when !TARGET_SIMD.
221 (define_attr "simd" "no,yes" (const_string "no"))
223 ;; Attribute that specifies whether or not the instruction uses SVE.
224 ;; When this is set to yes for an alternative, that alternative
225 ;; will be disabled when !TARGET_SVE.
226 (define_attr "sve" "no,yes" (const_string "no"))
228 (define_attr "length" ""
231 ;; Attribute that controls whether an alternative is enabled or not.
232 ;; Currently it is only used to disable alternatives which touch fp or simd
233 ;; registers when -mgeneral-regs-only is specified.
234 (define_attr "enabled" "no,yes"
236 (and (eq_attr "fp" "yes")
237 (eq (symbol_ref "TARGET_FLOAT") (const_int 0)))
238 (and (eq_attr "simd" "yes")
239 (eq (symbol_ref "TARGET_SIMD") (const_int 0)))
240 (and (eq_attr "fp16" "yes")
241 (eq (symbol_ref "TARGET_FP_F16INST") (const_int 0)))
242 (and (eq_attr "sve" "yes")
243 (eq (symbol_ref "TARGET_SVE") (const_int 0))))
245 ] (const_string "yes")))
247 ;; Attribute that specifies whether we are dealing with a branch to a
248 ;; label that is far away, i.e. further away than the maximum/minimum
249 ;; representable in a signed 21-bits number.
252 (define_attr "far_branch" "" (const_int 0))
254 ;; Strictly for compatibility with AArch32 in pipeline models, since AArch64 has
255 ;; no predicated insns.
256 (define_attr "predicated" "yes,no" (const_string "no"))
258 ;; -------------------------------------------------------------------
259 ;; Pipeline descriptions and scheduling
260 ;; -------------------------------------------------------------------
263 (include "aarch64-tune.md")
266 (include "../arm/cortex-a53.md")
267 (include "../arm/cortex-a57.md")
268 (include "../arm/exynos-m1.md")
269 (include "falkor.md")
270 (include "thunderx.md")
271 (include "../arm/xgene1.md")
272 (include "thunderx2t99.md")
274 ;; -------------------------------------------------------------------
275 ;; Jumps and other miscellaneous insns
276 ;; -------------------------------------------------------------------
278 (define_insn "indirect_jump"
279 [(set (pc) (match_operand:DI 0 "register_operand" "r"))]
282 [(set_attr "type" "branch")]
286 [(set (pc) (label_ref (match_operand 0 "" "")))]
289 [(set_attr "type" "branch")]
292 (define_expand "cbranch<mode>4"
293 [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
294 [(match_operand:GPI 1 "register_operand" "")
295 (match_operand:GPI 2 "aarch64_plus_operand" "")])
296 (label_ref (match_operand 3 "" ""))
300 operands[1] = aarch64_gen_compare_reg (GET_CODE (operands[0]), operands[1],
302 operands[2] = const0_rtx;
306 (define_expand "cbranch<mode>4"
307 [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
308 [(match_operand:GPF 1 "register_operand" "")
309 (match_operand:GPF 2 "aarch64_fp_compare_operand" "")])
310 (label_ref (match_operand 3 "" ""))
314 operands[1] = aarch64_gen_compare_reg (GET_CODE (operands[0]), operands[1],
316 operands[2] = const0_rtx;
320 (define_expand "cbranchcc4"
321 [(set (pc) (if_then_else
322 (match_operator 0 "aarch64_comparison_operator"
323 [(match_operand 1 "cc_register" "")
324 (match_operand 2 "const0_operand")])
325 (label_ref (match_operand 3 "" ""))
330 (define_insn "ccmp<mode>"
331 [(set (match_operand:CC 1 "cc_register" "")
333 (match_operator 4 "aarch64_comparison_operator"
334 [(match_operand 0 "cc_register" "")
337 (match_operand:GPI 2 "register_operand" "r,r,r")
338 (match_operand:GPI 3 "aarch64_ccmp_operand" "r,Uss,Usn"))
339 (unspec:CC [(match_operand 5 "immediate_operand")] UNSPEC_NZCV)))]
342 ccmp\\t%<w>2, %<w>3, %k5, %m4
343 ccmp\\t%<w>2, %3, %k5, %m4
344 ccmn\\t%<w>2, #%n3, %k5, %m4"
345 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
348 (define_insn "fccmp<mode>"
349 [(set (match_operand:CCFP 1 "cc_register" "")
351 (match_operator 4 "aarch64_comparison_operator"
352 [(match_operand 0 "cc_register" "")
355 (match_operand:GPF 2 "register_operand" "w")
356 (match_operand:GPF 3 "register_operand" "w"))
357 (unspec:CCFP [(match_operand 5 "immediate_operand")] UNSPEC_NZCV)))]
359 "fccmp\\t%<s>2, %<s>3, %k5, %m4"
360 [(set_attr "type" "fccmp<s>")]
363 (define_insn "fccmpe<mode>"
364 [(set (match_operand:CCFPE 1 "cc_register" "")
366 (match_operator 4 "aarch64_comparison_operator"
367 [(match_operand 0 "cc_register" "")
370 (match_operand:GPF 2 "register_operand" "w")
371 (match_operand:GPF 3 "register_operand" "w"))
372 (unspec:CCFPE [(match_operand 5 "immediate_operand")] UNSPEC_NZCV)))]
374 "fccmpe\\t%<s>2, %<s>3, %k5, %m4"
375 [(set_attr "type" "fccmp<s>")]
378 ;; Expansion of signed mod by a power of 2 using CSNEG.
379 ;; For x0 % n where n is a power of 2 produce:
381 ;; and x0, x0, #(n - 1)
382 ;; and x1, x1, #(n - 1)
383 ;; csneg x0, x0, x1, mi
385 (define_expand "mod<mode>3"
386 [(match_operand:GPI 0 "register_operand" "")
387 (match_operand:GPI 1 "register_operand" "")
388 (match_operand:GPI 2 "const_int_operand" "")]
391 HOST_WIDE_INT val = INTVAL (operands[2]);
394 || exact_log2 (val) <= 0
395 || !aarch64_bitmask_imm (val - 1, <MODE>mode))
398 rtx mask = GEN_INT (val - 1);
400 /* In the special case of x0 % 2 we can do the even shorter:
406 rtx masked = gen_reg_rtx (<MODE>mode);
407 rtx ccreg = aarch64_gen_compare_reg (LT, operands[1], const0_rtx);
408 emit_insn (gen_and<mode>3 (masked, operands[1], mask));
409 rtx x = gen_rtx_LT (VOIDmode, ccreg, const0_rtx);
410 emit_insn (gen_csneg3<mode>_insn (operands[0], x, masked, masked));
414 rtx neg_op = gen_reg_rtx (<MODE>mode);
415 rtx_insn *insn = emit_insn (gen_neg<mode>2_compare0 (neg_op, operands[1]));
417 /* Extract the condition register and mode. */
418 rtx cmp = XVECEXP (PATTERN (insn), 0, 0);
419 rtx cc_reg = SET_DEST (cmp);
420 rtx cond = gen_rtx_GE (VOIDmode, cc_reg, const0_rtx);
422 rtx masked_pos = gen_reg_rtx (<MODE>mode);
423 emit_insn (gen_and<mode>3 (masked_pos, operands[1], mask));
425 rtx masked_neg = gen_reg_rtx (<MODE>mode);
426 emit_insn (gen_and<mode>3 (masked_neg, neg_op, mask));
428 emit_insn (gen_csneg3<mode>_insn (operands[0], cond,
429 masked_neg, masked_pos));
434 (define_insn "condjump"
435 [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator"
436 [(match_operand 1 "cc_register" "") (const_int 0)])
437 (label_ref (match_operand 2 "" ""))
441 if (get_attr_length (insn) == 8)
442 return aarch64_gen_far_branch (operands, 2, "Lbcond", "b%M0\\t");
446 [(set_attr "type" "branch")
448 (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
449 (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
452 (set (attr "far_branch")
453 (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
454 (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
459 ;; For a 24-bit immediate CST we can optimize the compare for equality
460 ;; and branch sequence from:
462 ;; movk x0, #imm2, lsl 16 /* x0 contains CST. */
466 ;; sub x0, x1, #(CST & 0xfff000)
467 ;; subs x0, x0, #(CST & 0x000fff)
469 (define_insn_and_split "*compare_condjump<mode>"
470 [(set (pc) (if_then_else (EQL
471 (match_operand:GPI 0 "register_operand" "r")
472 (match_operand:GPI 1 "aarch64_imm24" "n"))
473 (label_ref:P (match_operand 2 "" ""))
475 "!aarch64_move_imm (INTVAL (operands[1]), <MODE>mode)
476 && !aarch64_plus_operand (operands[1], <MODE>mode)
477 && !reload_completed"
482 HOST_WIDE_INT lo_imm = UINTVAL (operands[1]) & 0xfff;
483 HOST_WIDE_INT hi_imm = UINTVAL (operands[1]) & 0xfff000;
484 rtx tmp = gen_reg_rtx (<MODE>mode);
485 emit_insn (gen_add<mode>3 (tmp, operands[0], GEN_INT (-hi_imm)));
486 emit_insn (gen_add<mode>3_compare0 (tmp, tmp, GEN_INT (-lo_imm)));
487 rtx cc_reg = gen_rtx_REG (CC_NZmode, CC_REGNUM);
488 rtx cmp_rtx = gen_rtx_fmt_ee (<EQL:CMP>, <MODE>mode, cc_reg, const0_rtx);
489 emit_jump_insn (gen_condjump (cmp_rtx, cc_reg, operands[2]));
494 (define_expand "casesi"
495 [(match_operand:SI 0 "register_operand" "") ; Index
496 (match_operand:SI 1 "const_int_operand" "") ; Lower bound
497 (match_operand:SI 2 "const_int_operand" "") ; Total range
498 (match_operand:DI 3 "" "") ; Table label
499 (match_operand:DI 4 "" "")] ; Out of range label
502 if (operands[1] != const0_rtx)
504 rtx reg = gen_reg_rtx (SImode);
506 /* Canonical RTL says that if you have:
510 then this should be emitted as:
514 The use of trunc_int_for_mode ensures that the resulting
515 constant can be represented in SImode, this is important
516 for the corner case where operand[1] is INT_MIN. */
518 operands[1] = GEN_INT (trunc_int_for_mode (-INTVAL (operands[1]), SImode));
520 if (!(*insn_data[CODE_FOR_addsi3].operand[2].predicate)
521 (operands[1], SImode))
522 operands[1] = force_reg (SImode, operands[1]);
523 emit_insn (gen_addsi3 (reg, operands[0], operands[1]));
527 if (!aarch64_plus_operand (operands[2], SImode))
528 operands[2] = force_reg (SImode, operands[2]);
529 emit_jump_insn (gen_cbranchsi4 (gen_rtx_GTU (SImode, const0_rtx,
531 operands[0], operands[2], operands[4]));
533 operands[2] = force_reg (DImode, gen_rtx_LABEL_REF (DImode, operands[3]));
534 emit_jump_insn (gen_casesi_dispatch (operands[2], operands[0],
540 (define_insn "casesi_dispatch"
543 (mem:DI (unspec [(match_operand:DI 0 "register_operand" "r")
544 (match_operand:SI 1 "register_operand" "r")]
546 (clobber (reg:CC CC_REGNUM))
547 (clobber (match_scratch:DI 3 "=r"))
548 (clobber (match_scratch:DI 4 "=r"))
549 (use (label_ref (match_operand 2 "" "")))])]
552 return aarch64_output_casesi (operands);
554 [(set_attr "length" "16")
555 (set_attr "type" "branch")]
559 [(unspec[(const_int 0)] UNSPEC_NOP)]
562 [(set_attr "type" "no_insn")]
565 (define_insn "prefetch"
566 [(prefetch (match_operand:DI 0 "aarch64_prefetch_operand" "Dp")
567 (match_operand:QI 1 "const_int_operand" "")
568 (match_operand:QI 2 "const_int_operand" ""))]
571 const char * pftype[2][4] =
573 {"prfm\\tPLDL1STRM, %0",
574 "prfm\\tPLDL3KEEP, %0",
575 "prfm\\tPLDL2KEEP, %0",
576 "prfm\\tPLDL1KEEP, %0"},
577 {"prfm\\tPSTL1STRM, %0",
578 "prfm\\tPSTL3KEEP, %0",
579 "prfm\\tPSTL2KEEP, %0",
580 "prfm\\tPSTL1KEEP, %0"},
583 int locality = INTVAL (operands[2]);
585 gcc_assert (IN_RANGE (locality, 0, 3));
587 /* PRFM accepts the same addresses as a 64-bit LDR so wrap
588 the address into a DImode MEM so that aarch64_print_operand knows
590 operands[0] = gen_rtx_MEM (DImode, operands[0]);
591 return pftype[INTVAL(operands[1])][locality];
593 [(set_attr "type" "load_4")]
597 [(trap_if (const_int 1) (const_int 8))]
600 [(set_attr "type" "trap")])
602 (define_expand "prologue"
603 [(clobber (const_int 0))]
606 aarch64_expand_prologue ();
611 (define_expand "epilogue"
612 [(clobber (const_int 0))]
615 aarch64_expand_epilogue (false);
620 (define_expand "sibcall_epilogue"
621 [(clobber (const_int 0))]
624 aarch64_expand_epilogue (true);
629 (define_insn "*do_return"
633 if (aarch64_return_address_signing_enabled ()
635 && !crtl->calls_eh_return)
640 [(set_attr "type" "branch")]
643 (define_expand "return"
645 "aarch64_use_return_insn_p ()"
649 (define_insn "simple_return"
653 [(set_attr "type" "branch")]
656 (define_insn "*cb<optab><mode>1"
657 [(set (pc) (if_then_else (EQL (match_operand:GPI 0 "register_operand" "r")
659 (label_ref (match_operand 1 "" ""))
663 if (get_attr_length (insn) == 8)
664 return aarch64_gen_far_branch (operands, 1, "Lcb", "<inv_cb>\\t%<w>0, ");
666 return "<cbz>\\t%<w>0, %l1";
668 [(set_attr "type" "branch")
670 (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -1048576))
671 (lt (minus (match_dup 1) (pc)) (const_int 1048572)))
674 (set (attr "far_branch")
675 (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
676 (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
681 (define_insn "*tb<optab><mode>1"
682 [(set (pc) (if_then_else
683 (EQL (zero_extract:DI (match_operand:GPI 0 "register_operand" "r")
686 "aarch64_simd_shift_imm_<mode>" "n"))
688 (label_ref (match_operand 2 "" ""))
690 (clobber (reg:CC CC_REGNUM))]
693 if (get_attr_length (insn) == 8)
695 if (get_attr_far_branch (insn) == 1)
696 return aarch64_gen_far_branch (operands, 2, "Ltb",
697 "<inv_tb>\\t%<w>0, %1, ");
700 operands[1] = GEN_INT (HOST_WIDE_INT_1U << UINTVAL (operands[1]));
701 return "tst\t%<w>0, %1\;<bcond>\t%l2";
705 return "<tbz>\t%<w>0, %1, %l2";
707 [(set_attr "type" "branch")
709 (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -32768))
710 (lt (minus (match_dup 2) (pc)) (const_int 32764)))
713 (set (attr "far_branch")
714 (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576))
715 (lt (minus (match_dup 2) (pc)) (const_int 1048572)))
721 (define_insn "*cb<optab><mode>1"
722 [(set (pc) (if_then_else (LTGE (match_operand:ALLI 0 "register_operand" "r")
724 (label_ref (match_operand 1 "" ""))
726 (clobber (reg:CC CC_REGNUM))]
729 if (get_attr_length (insn) == 8)
731 if (get_attr_far_branch (insn) == 1)
732 return aarch64_gen_far_branch (operands, 1, "Ltb",
733 "<inv_tb>\\t%<w>0, <sizem1>, ");
737 uint64_t val = ((uint64_t) 1)
738 << (GET_MODE_SIZE (<MODE>mode) * BITS_PER_UNIT - 1);
739 sprintf (buf, "tst\t%%<w>0, %" PRId64, val);
740 output_asm_insn (buf, operands);
741 return "<bcond>\t%l1";
745 return "<tbz>\t%<w>0, <sizem1>, %l1";
747 [(set_attr "type" "branch")
749 (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -32768))
750 (lt (minus (match_dup 1) (pc)) (const_int 32764)))
753 (set (attr "far_branch")
754 (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -1048576))
755 (lt (minus (match_dup 1) (pc)) (const_int 1048572)))
760 ;; -------------------------------------------------------------------
761 ;; Subroutine calls and sibcalls
762 ;; -------------------------------------------------------------------
764 (define_expand "call"
765 [(parallel [(call (match_operand 0 "memory_operand" "")
766 (match_operand 1 "general_operand" ""))
767 (use (match_operand 2 "" ""))
768 (clobber (reg:DI LR_REGNUM))])]
772 aarch64_expand_call (NULL_RTX, operands[0], false);
777 (define_insn "*call_insn"
778 [(call (mem:DI (match_operand:DI 0 "aarch64_call_insn_operand" "r, Usf"))
779 (match_operand 1 "" ""))
780 (clobber (reg:DI LR_REGNUM))]
785 [(set_attr "type" "call, call")]
788 (define_expand "call_value"
789 [(parallel [(set (match_operand 0 "" "")
790 (call (match_operand 1 "memory_operand" "")
791 (match_operand 2 "general_operand" "")))
792 (use (match_operand 3 "" ""))
793 (clobber (reg:DI LR_REGNUM))])]
797 aarch64_expand_call (operands[0], operands[1], false);
802 (define_insn "*call_value_insn"
803 [(set (match_operand 0 "" "")
804 (call (mem:DI (match_operand:DI 1 "aarch64_call_insn_operand" "r, Usf"))
805 (match_operand 2 "" "")))
806 (clobber (reg:DI LR_REGNUM))]
811 [(set_attr "type" "call, call")]
814 (define_expand "sibcall"
815 [(parallel [(call (match_operand 0 "memory_operand" "")
816 (match_operand 1 "general_operand" ""))
818 (use (match_operand 2 "" ""))])]
821 aarch64_expand_call (NULL_RTX, operands[0], true);
826 (define_expand "sibcall_value"
827 [(parallel [(set (match_operand 0 "" "")
828 (call (match_operand 1 "memory_operand" "")
829 (match_operand 2 "general_operand" "")))
831 (use (match_operand 3 "" ""))])]
834 aarch64_expand_call (operands[0], operands[1], true);
839 (define_insn "*sibcall_insn"
840 [(call (mem:DI (match_operand:DI 0 "aarch64_call_insn_operand" "Ucs, Usf"))
841 (match_operand 1 "" ""))
843 "SIBLING_CALL_P (insn)"
847 [(set_attr "type" "branch, branch")]
850 (define_insn "*sibcall_value_insn"
851 [(set (match_operand 0 "" "")
853 (match_operand:DI 1 "aarch64_call_insn_operand" "Ucs, Usf"))
854 (match_operand 2 "" "")))
856 "SIBLING_CALL_P (insn)"
860 [(set_attr "type" "branch, branch")]
863 ;; Call subroutine returning any type.
865 (define_expand "untyped_call"
866 [(parallel [(call (match_operand 0 "")
869 (match_operand 2 "")])]
874 emit_call_insn (gen_call (operands[0], const0_rtx, NULL));
876 for (i = 0; i < XVECLEN (operands[2], 0); i++)
878 rtx set = XVECEXP (operands[2], 0, i);
879 emit_move_insn (SET_DEST (set), SET_SRC (set));
882 /* The optimizer does not know that the call sets the function value
883 registers we stored in the result block. We avoid problems by
884 claiming that all hard registers are used and clobbered at this
886 emit_insn (gen_blockage ());
890 ;; -------------------------------------------------------------------
892 ;; -------------------------------------------------------------------
894 (define_expand "mov<mode>"
895 [(set (match_operand:SHORT 0 "nonimmediate_operand" "")
896 (match_operand:SHORT 1 "general_operand" ""))]
899 if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
900 operands[1] = force_reg (<MODE>mode, operands[1]);
902 if (GET_CODE (operands[1]) == CONST_POLY_INT)
904 aarch64_expand_mov_immediate (operands[0], operands[1]);
910 (define_insn "*mov<mode>_aarch64"
911 [(set (match_operand:SHORT 0 "nonimmediate_operand" "=r,r, *w,r ,r,*w, m, m, r,*w,*w")
912 (match_operand:SHORT 1 "aarch64_mov_operand" " r,M,D<hq>,Usv,m, m,rZ,*w,*w, r,*w"))]
913 "(register_operand (operands[0], <MODE>mode)
914 || aarch64_reg_or_zero (operands[1], <MODE>mode))"
916 switch (which_alternative)
919 return "mov\t%w0, %w1";
921 return "mov\t%w0, %1";
923 return aarch64_output_scalar_simd_mov_immediate (operands[1],
926 return aarch64_output_sve_cnt_immediate (\"cnt\", \"%x0\", operands[1]);
928 return "ldr<size>\t%w0, %1";
930 return "ldr\t%<size>0, %1";
932 return "str<size>\t%w1, %0";
934 return "str\t%<size>1, %0";
936 return "umov\t%w0, %1.<v>[0]";
938 return "dup\t%0.<Vallxd>, %w1";
940 return "dup\t%<Vetype>0, %1.<v>[0]";
945 ;; The "mov_imm" type for CNT is just a placeholder.
946 [(set_attr "type" "mov_reg,mov_imm,neon_move,mov_imm,load_4,load_4,store_4,
947 store_4,neon_to_gp<q>,neon_from_gp<q>,neon_dup")
948 (set_attr "simd" "*,*,yes,*,*,*,*,*,yes,yes,yes")
949 (set_attr "sve" "*,*,*,yes,*,*,*,*,*,*,*")]
952 (define_expand "mov<mode>"
953 [(set (match_operand:GPI 0 "nonimmediate_operand" "")
954 (match_operand:GPI 1 "general_operand" ""))]
957 if (MEM_P (operands[0]) && CONST_INT_P (operands[1])
958 && <MODE>mode == DImode
959 && aarch64_split_dimode_const_store (operands[0], operands[1]))
962 if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
963 operands[1] = force_reg (<MODE>mode, operands[1]);
965 /* FIXME: RR we still need to fix up what we are doing with
966 symbol_refs and other types of constants. */
967 if (CONSTANT_P (operands[1])
968 && !CONST_INT_P (operands[1]))
970 aarch64_expand_mov_immediate (operands[0], operands[1]);
976 (define_insn_and_split "*movsi_aarch64"
977 [(set (match_operand:SI 0 "nonimmediate_operand" "=r,k,r,r,r,r, r,w, m, m, r, r, w,r,w, w")
978 (match_operand:SI 1 "aarch64_mov_operand" " r,r,k,M,n,Usv,m,m,rZ,*w,Usa,Ush,rZ,w,w,Ds"))]
979 "(register_operand (operands[0], SImode)
980 || aarch64_reg_or_zero (operands[1], SImode))"
987 * return aarch64_output_sve_cnt_immediate (\"cnt\", \"%x0\", operands[1]);
997 * return aarch64_output_scalar_simd_mov_immediate (operands[1], SImode);"
998 "CONST_INT_P (operands[1]) && !aarch64_move_imm (INTVAL (operands[1]), SImode)
999 && REG_P (operands[0]) && GP_REGNUM_P (REGNO (operands[0]))"
1002 aarch64_expand_mov_immediate (operands[0], operands[1]);
1005 ;; The "mov_imm" type for CNT is just a placeholder.
1006 [(set_attr "type" "mov_reg,mov_reg,mov_reg,mov_imm,mov_imm,mov_imm,load_4,
1007 load_4,store_4,store_4,adr,adr,f_mcr,f_mrc,fmov,neon_move")
1008 (set_attr "fp" "*,*,*,*,*,*,*,yes,*,yes,*,*,yes,yes,yes,*")
1009 (set_attr "simd" "*,*,*,*,*,*,*,*,*,*,*,*,*,*,*,yes")
1010 (set_attr "sve" "*,*,*,*,*,yes,*,*,*,*,*,*,*,*,*,*")]
1013 (define_insn_and_split "*movdi_aarch64"
1014 [(set (match_operand:DI 0 "nonimmediate_operand" "=r,k,r,r,r,r,r, r,w, m,m, r, r, w,r,w, w")
1015 (match_operand:DI 1 "aarch64_mov_operand" " r,r,k,N,M,n,Usv,m,m,rZ,w,Usa,Ush,rZ,w,w,Dd"))]
1016 "(register_operand (operands[0], DImode)
1017 || aarch64_reg_or_zero (operands[1], DImode))"
1025 * return aarch64_output_sve_cnt_immediate (\"cnt\", \"%x0\", operands[1]);
1035 * return aarch64_output_scalar_simd_mov_immediate (operands[1], DImode);"
1036 "(CONST_INT_P (operands[1]) && !aarch64_move_imm (INTVAL (operands[1]), DImode))
1037 && REG_P (operands[0]) && GP_REGNUM_P (REGNO (operands[0]))"
1040 aarch64_expand_mov_immediate (operands[0], operands[1]);
1043 ;; The "mov_imm" type for CNTD is just a placeholder.
1044 [(set_attr "type" "mov_reg,mov_reg,mov_reg,mov_imm,mov_imm,mov_imm,mov_imm,
1045 load_8,load_8,store_8,store_8,adr,adr,f_mcr,f_mrc,fmov,
1047 (set_attr "fp" "*,*,*,*,*,*,*,*,yes,*,yes,*,*,yes,yes,yes,*")
1048 (set_attr "simd" "*,*,*,*,*,*,*,*,*,*,*,*,*,*,*,*,yes")
1049 (set_attr "sve" "*,*,*,*,*,*,yes,*,*,*,*,*,*,*,*,*,*")]
1052 (define_insn "insv_imm<mode>"
1053 [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
1055 (match_operand:GPI 1 "const_int_operand" "n"))
1056 (match_operand:GPI 2 "const_int_operand" "n"))]
1057 "UINTVAL (operands[1]) < GET_MODE_BITSIZE (<MODE>mode)
1058 && UINTVAL (operands[1]) % 16 == 0"
1059 "movk\\t%<w>0, %X2, lsl %1"
1060 [(set_attr "type" "mov_imm")]
1063 (define_expand "movti"
1064 [(set (match_operand:TI 0 "nonimmediate_operand" "")
1065 (match_operand:TI 1 "general_operand" ""))]
1068 if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx)
1069 operands[1] = force_reg (TImode, operands[1]);
1071 if (GET_CODE (operands[1]) == CONST_POLY_INT)
1073 emit_move_insn (gen_lowpart (DImode, operands[0]),
1074 gen_lowpart (DImode, operands[1]));
1075 emit_move_insn (gen_highpart (DImode, operands[0]), const0_rtx);
1081 (define_insn "*movti_aarch64"
1082 [(set (match_operand:TI 0
1083 "nonimmediate_operand" "= r,w, r,w,r,m,m,w,m")
1085 "aarch64_movti_operand" " rUti,r, w,w,m,r,Z,m,w"))]
1086 "(register_operand (operands[0], TImode)
1087 || aarch64_reg_or_zero (operands[1], TImode))"
1092 mov\\t%0.16b, %1.16b
1098 [(set_attr "type" "multiple,f_mcr,f_mrc,neon_logic_q, \
1099 load_16,store_16,store_16,\
1101 (set_attr "length" "8,8,8,4,4,4,4,4,4")
1102 (set_attr "simd" "*,*,*,yes,*,*,*,*,*")
1103 (set_attr "fp" "*,*,*,*,*,*,*,yes,yes")]
1106 ;; Split a TImode register-register or register-immediate move into
1107 ;; its component DImode pieces, taking care to handle overlapping
1108 ;; source and dest registers.
1110 [(set (match_operand:TI 0 "register_operand" "")
1111 (match_operand:TI 1 "aarch64_reg_or_imm" ""))]
1112 "reload_completed && aarch64_split_128bit_move_p (operands[0], operands[1])"
1115 aarch64_split_128bit_move (operands[0], operands[1]);
1119 (define_expand "mov<mode>"
1120 [(set (match_operand:GPF_TF_F16 0 "nonimmediate_operand" "")
1121 (match_operand:GPF_TF_F16 1 "general_operand" ""))]
1126 aarch64_err_no_fpadvsimd (<MODE>mode, "code");
1130 if (GET_CODE (operands[0]) == MEM
1131 && ! (GET_CODE (operands[1]) == CONST_DOUBLE
1132 && aarch64_float_const_zero_rtx_p (operands[1])))
1133 operands[1] = force_reg (<MODE>mode, operands[1]);
1137 (define_insn "*movhf_aarch64"
1138 [(set (match_operand:HF 0 "nonimmediate_operand" "=w,w ,?r,w,w ,w ,w,m,r,m ,r")
1139 (match_operand:HF 1 "general_operand" "Y ,?rY, w,w,Ufc,Uvi,m,w,m,rY,r"))]
1140 "TARGET_FLOAT && (register_operand (operands[0], HFmode)
1141 || aarch64_reg_or_fp_zero (operands[1], HFmode))"
1146 mov\\t%0.h[0], %1.h[0]
1148 * return aarch64_output_scalar_simd_mov_immediate (operands[1], SImode);
1154 [(set_attr "type" "neon_move,f_mcr,neon_to_gp,neon_move,fconsts, \
1155 neon_move,f_loads,f_stores,load_4,store_4,mov_reg")
1156 (set_attr "simd" "yes,*,yes,yes,*,yes,*,*,*,*,*")
1157 (set_attr "fp16" "*,yes,*,*,yes,*,*,*,*,*,*")]
1160 (define_insn "*movsf_aarch64"
1161 [(set (match_operand:SF 0 "nonimmediate_operand" "=w,w ,?r,w,w ,w ,w,m,r,m ,r,r")
1162 (match_operand:SF 1 "general_operand" "Y ,?rY, w,w,Ufc,Uvi,m,w,m,rY,r,M"))]
1163 "TARGET_FLOAT && (register_operand (operands[0], SFmode)
1164 || aarch64_reg_or_fp_zero (operands[1], SFmode))"
1171 * return aarch64_output_scalar_simd_mov_immediate (operands[1], SImode);
1178 [(set_attr "type" "neon_move,f_mcr,f_mrc,fmov,fconsts,neon_move,\
1179 f_loads,f_stores,load_4,store_4,mov_reg,\
1181 (set_attr "simd" "yes,*,*,*,*,yes,*,*,*,*,*,*")]
1184 (define_insn "*movdf_aarch64"
1185 [(set (match_operand:DF 0 "nonimmediate_operand" "=w, w ,?r,w,w ,w ,w,m,r,m ,r,r")
1186 (match_operand:DF 1 "general_operand" "Y , ?rY, w,w,Ufc,Uvi,m,w,m,rY,r,N"))]
1187 "TARGET_FLOAT && (register_operand (operands[0], DFmode)
1188 || aarch64_reg_or_fp_zero (operands[1], DFmode))"
1195 * return aarch64_output_scalar_simd_mov_immediate (operands[1], DImode);
1202 [(set_attr "type" "neon_move,f_mcr,f_mrc,fmov,fconstd,neon_move,\
1203 f_loadd,f_stored,load_8,store_8,mov_reg,\
1205 (set_attr "simd" "yes,*,*,*,*,yes,*,*,*,*,*,*")]
1209 [(set (match_operand:GPF_HF 0 "nonimmediate_operand")
1210 (match_operand:GPF_HF 1 "general_operand"))]
1211 "can_create_pseudo_p ()
1212 && !aarch64_can_const_movi_rtx_p (operands[1], <MODE>mode)
1213 && !aarch64_float_const_representable_p (operands[1])
1214 && aarch64_float_const_rtx_p (operands[1])"
1217 unsigned HOST_WIDE_INT ival;
1218 if (!aarch64_reinterpret_float_as_int (operands[1], &ival))
1221 rtx tmp = gen_reg_rtx (<FCVT_TARGET>mode);
1222 emit_move_insn (tmp, gen_int_mode (ival, <FCVT_TARGET>mode));
1223 emit_move_insn (operands[0], gen_lowpart (<MODE>mode, tmp));
1228 (define_insn "*movtf_aarch64"
1229 [(set (match_operand:TF 0
1230 "nonimmediate_operand" "=w,?&r,w ,?r,w,?w,w,m,?r,m ,m")
1232 "general_operand" " w,?r, ?r,w ,Y,Y ,m,w,m ,?r,Y"))]
1233 "TARGET_FLOAT && (register_operand (operands[0], TFmode)
1234 || aarch64_reg_or_fp_zero (operands[1], TFmode))"
1236 mov\\t%0.16b, %1.16b
1247 [(set_attr "type" "logic_reg,multiple,f_mcr,f_mrc,neon_move_q,f_mcr,\
1248 f_loadd,f_stored,load_16,store_16,store_16")
1249 (set_attr "length" "4,8,8,8,4,4,4,4,4,4,4")
1250 (set_attr "simd" "yes,*,*,*,yes,*,*,*,*,*,*")]
1254 [(set (match_operand:TF 0 "register_operand" "")
1255 (match_operand:TF 1 "aarch64_reg_or_imm" ""))]
1256 "reload_completed && aarch64_split_128bit_move_p (operands[0], operands[1])"
1259 aarch64_split_128bit_move (operands[0], operands[1]);
1266 ;; 2 is size of move in bytes
1269 (define_expand "movmemdi"
1270 [(match_operand:BLK 0 "memory_operand")
1271 (match_operand:BLK 1 "memory_operand")
1272 (match_operand:DI 2 "immediate_operand")
1273 (match_operand:DI 3 "immediate_operand")]
1276 if (aarch64_expand_movmem (operands))
1282 ;; Operands 1 and 3 are tied together by the final condition; so we allow
1283 ;; fairly lax checking on the second memory operation.
1284 (define_insn "load_pairsi"
1285 [(set (match_operand:SI 0 "register_operand" "=r,*w")
1286 (match_operand:SI 1 "aarch64_mem_pair_operand" "Ump,Ump"))
1287 (set (match_operand:SI 2 "register_operand" "=r,*w")
1288 (match_operand:SI 3 "memory_operand" "m,m"))]
1289 "rtx_equal_p (XEXP (operands[3], 0),
1290 plus_constant (Pmode,
1291 XEXP (operands[1], 0),
1292 GET_MODE_SIZE (SImode)))"
1296 [(set_attr "type" "load_8,neon_load1_2reg")
1297 (set_attr "fp" "*,yes")]
1300 (define_insn "load_pairdi"
1301 [(set (match_operand:DI 0 "register_operand" "=r,*w")
1302 (match_operand:DI 1 "aarch64_mem_pair_operand" "Ump,Ump"))
1303 (set (match_operand:DI 2 "register_operand" "=r,*w")
1304 (match_operand:DI 3 "memory_operand" "m,m"))]
1305 "rtx_equal_p (XEXP (operands[3], 0),
1306 plus_constant (Pmode,
1307 XEXP (operands[1], 0),
1308 GET_MODE_SIZE (DImode)))"
1312 [(set_attr "type" "load_16,neon_load1_2reg")
1313 (set_attr "fp" "*,yes")]
1317 ;; Operands 0 and 2 are tied together by the final condition; so we allow
1318 ;; fairly lax checking on the second memory operation.
1319 (define_insn "store_pairsi"
1320 [(set (match_operand:SI 0 "aarch64_mem_pair_operand" "=Ump,Ump")
1321 (match_operand:SI 1 "aarch64_reg_or_zero" "rZ,*w"))
1322 (set (match_operand:SI 2 "memory_operand" "=m,m")
1323 (match_operand:SI 3 "aarch64_reg_or_zero" "rZ,*w"))]
1324 "rtx_equal_p (XEXP (operands[2], 0),
1325 plus_constant (Pmode,
1326 XEXP (operands[0], 0),
1327 GET_MODE_SIZE (SImode)))"
1331 [(set_attr "type" "store_8,neon_store1_2reg")
1332 (set_attr "fp" "*,yes")]
1335 (define_insn "store_pairdi"
1336 [(set (match_operand:DI 0 "aarch64_mem_pair_operand" "=Ump,Ump")
1337 (match_operand:DI 1 "aarch64_reg_or_zero" "rZ,*w"))
1338 (set (match_operand:DI 2 "memory_operand" "=m,m")
1339 (match_operand:DI 3 "aarch64_reg_or_zero" "rZ,*w"))]
1340 "rtx_equal_p (XEXP (operands[2], 0),
1341 plus_constant (Pmode,
1342 XEXP (operands[0], 0),
1343 GET_MODE_SIZE (DImode)))"
1347 [(set_attr "type" "store_16,neon_store1_2reg")
1348 (set_attr "fp" "*,yes")]
1351 ;; Operands 1 and 3 are tied together by the final condition; so we allow
1352 ;; fairly lax checking on the second memory operation.
1353 (define_insn "load_pairsf"
1354 [(set (match_operand:SF 0 "register_operand" "=w,*r")
1355 (match_operand:SF 1 "aarch64_mem_pair_operand" "Ump,Ump"))
1356 (set (match_operand:SF 2 "register_operand" "=w,*r")
1357 (match_operand:SF 3 "memory_operand" "m,m"))]
1358 "rtx_equal_p (XEXP (operands[3], 0),
1359 plus_constant (Pmode,
1360 XEXP (operands[1], 0),
1361 GET_MODE_SIZE (SFmode)))"
1365 [(set_attr "type" "neon_load1_2reg,load_8")
1366 (set_attr "fp" "yes,*")]
1369 (define_insn "load_pairdf"
1370 [(set (match_operand:DF 0 "register_operand" "=w,*r")
1371 (match_operand:DF 1 "aarch64_mem_pair_operand" "Ump,Ump"))
1372 (set (match_operand:DF 2 "register_operand" "=w,*r")
1373 (match_operand:DF 3 "memory_operand" "m,m"))]
1374 "rtx_equal_p (XEXP (operands[3], 0),
1375 plus_constant (Pmode,
1376 XEXP (operands[1], 0),
1377 GET_MODE_SIZE (DFmode)))"
1381 [(set_attr "type" "neon_load1_2reg,load_16")
1382 (set_attr "fp" "yes,*")]
1385 ;; Operands 0 and 2 are tied together by the final condition; so we allow
1386 ;; fairly lax checking on the second memory operation.
1387 (define_insn "store_pairsf"
1388 [(set (match_operand:SF 0 "aarch64_mem_pair_operand" "=Ump,Ump")
1389 (match_operand:SF 1 "aarch64_reg_or_fp_zero" "w,*rY"))
1390 (set (match_operand:SF 2 "memory_operand" "=m,m")
1391 (match_operand:SF 3 "aarch64_reg_or_fp_zero" "w,*rY"))]
1392 "rtx_equal_p (XEXP (operands[2], 0),
1393 plus_constant (Pmode,
1394 XEXP (operands[0], 0),
1395 GET_MODE_SIZE (SFmode)))"
1399 [(set_attr "type" "neon_store1_2reg,store_8")
1400 (set_attr "fp" "yes,*")]
1403 (define_insn "store_pairdf"
1404 [(set (match_operand:DF 0 "aarch64_mem_pair_operand" "=Ump,Ump")
1405 (match_operand:DF 1 "aarch64_reg_or_fp_zero" "w,*rY"))
1406 (set (match_operand:DF 2 "memory_operand" "=m,m")
1407 (match_operand:DF 3 "aarch64_reg_or_fp_zero" "w,*rY"))]
1408 "rtx_equal_p (XEXP (operands[2], 0),
1409 plus_constant (Pmode,
1410 XEXP (operands[0], 0),
1411 GET_MODE_SIZE (DFmode)))"
1415 [(set_attr "type" "neon_store1_2reg,store_16")
1416 (set_attr "fp" "yes,*")]
1419 ;; Load pair with post-index writeback. This is primarily used in function
1421 (define_insn "loadwb_pair<GPI:mode>_<P:mode>"
1423 [(set (match_operand:P 0 "register_operand" "=k")
1424 (plus:P (match_operand:P 1 "register_operand" "0")
1425 (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1426 (set (match_operand:GPI 2 "register_operand" "=r")
1427 (mem:GPI (match_dup 1)))
1428 (set (match_operand:GPI 3 "register_operand" "=r")
1429 (mem:GPI (plus:P (match_dup 1)
1430 (match_operand:P 5 "const_int_operand" "n"))))])]
1431 "INTVAL (operands[5]) == GET_MODE_SIZE (<GPI:MODE>mode)"
1432 "ldp\\t%<w>2, %<w>3, [%1], %4"
1433 [(set_attr "type" "load_<ldpstp_sz>")]
1436 (define_insn "loadwb_pair<GPF:mode>_<P:mode>"
1438 [(set (match_operand:P 0 "register_operand" "=k")
1439 (plus:P (match_operand:P 1 "register_operand" "0")
1440 (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1441 (set (match_operand:GPF 2 "register_operand" "=w")
1442 (mem:GPF (match_dup 1)))
1443 (set (match_operand:GPF 3 "register_operand" "=w")
1444 (mem:GPF (plus:P (match_dup 1)
1445 (match_operand:P 5 "const_int_operand" "n"))))])]
1446 "INTVAL (operands[5]) == GET_MODE_SIZE (<GPF:MODE>mode)"
1447 "ldp\\t%<w>2, %<w>3, [%1], %4"
1448 [(set_attr "type" "neon_load1_2reg")]
1451 ;; Store pair with pre-index writeback. This is primarily used in function
1453 (define_insn "storewb_pair<GPI:mode>_<P:mode>"
1455 [(set (match_operand:P 0 "register_operand" "=&k")
1456 (plus:P (match_operand:P 1 "register_operand" "0")
1457 (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1458 (set (mem:GPI (plus:P (match_dup 0)
1460 (match_operand:GPI 2 "register_operand" "r"))
1461 (set (mem:GPI (plus:P (match_dup 0)
1462 (match_operand:P 5 "const_int_operand" "n")))
1463 (match_operand:GPI 3 "register_operand" "r"))])]
1464 "INTVAL (operands[5]) == INTVAL (operands[4]) + GET_MODE_SIZE (<GPI:MODE>mode)"
1465 "stp\\t%<w>2, %<w>3, [%0, %4]!"
1466 [(set_attr "type" "store_<ldpstp_sz>")]
1469 (define_insn "storewb_pair<GPF:mode>_<P:mode>"
1471 [(set (match_operand:P 0 "register_operand" "=&k")
1472 (plus:P (match_operand:P 1 "register_operand" "0")
1473 (match_operand:P 4 "aarch64_mem_pair_offset" "n")))
1474 (set (mem:GPF (plus:P (match_dup 0)
1476 (match_operand:GPF 2 "register_operand" "w"))
1477 (set (mem:GPF (plus:P (match_dup 0)
1478 (match_operand:P 5 "const_int_operand" "n")))
1479 (match_operand:GPF 3 "register_operand" "w"))])]
1480 "INTVAL (operands[5]) == INTVAL (operands[4]) + GET_MODE_SIZE (<GPF:MODE>mode)"
1481 "stp\\t%<w>2, %<w>3, [%0, %4]!"
1482 [(set_attr "type" "neon_store1_2reg<q>")]
1485 ;; -------------------------------------------------------------------
1486 ;; Sign/Zero extension
1487 ;; -------------------------------------------------------------------
1489 (define_expand "<optab>sidi2"
1490 [(set (match_operand:DI 0 "register_operand")
1491 (ANY_EXTEND:DI (match_operand:SI 1 "nonimmediate_operand")))]
1495 (define_insn "*extendsidi2_aarch64"
1496 [(set (match_operand:DI 0 "register_operand" "=r,r")
1497 (sign_extend:DI (match_operand:SI 1 "nonimmediate_operand" "r,m")))]
1502 [(set_attr "type" "extend,load_4")]
1505 (define_insn "*load_pair_extendsidi2_aarch64"
1506 [(set (match_operand:DI 0 "register_operand" "=r")
1507 (sign_extend:DI (match_operand:SI 1 "aarch64_mem_pair_operand" "Ump")))
1508 (set (match_operand:DI 2 "register_operand" "=r")
1509 (sign_extend:DI (match_operand:SI 3 "memory_operand" "m")))]
1510 "rtx_equal_p (XEXP (operands[3], 0),
1511 plus_constant (Pmode,
1512 XEXP (operands[1], 0),
1513 GET_MODE_SIZE (SImode)))"
1514 "ldpsw\\t%0, %2, %1"
1515 [(set_attr "type" "load_8")]
1518 (define_insn "*zero_extendsidi2_aarch64"
1519 [(set (match_operand:DI 0 "register_operand" "=r,r")
1520 (zero_extend:DI (match_operand:SI 1 "nonimmediate_operand" "r,m")))]
1525 [(set_attr "type" "extend,load_4")]
1528 (define_insn "*load_pair_zero_extendsidi2_aarch64"
1529 [(set (match_operand:DI 0 "register_operand" "=r")
1530 (zero_extend:DI (match_operand:SI 1 "aarch64_mem_pair_operand" "Ump")))
1531 (set (match_operand:DI 2 "register_operand" "=r")
1532 (zero_extend:DI (match_operand:SI 3 "memory_operand" "m")))]
1533 "rtx_equal_p (XEXP (operands[3], 0),
1534 plus_constant (Pmode,
1535 XEXP (operands[1], 0),
1536 GET_MODE_SIZE (SImode)))"
1537 "ldp\\t%w0, %w2, %1"
1538 [(set_attr "type" "load_8")]
1541 (define_expand "<ANY_EXTEND:optab><SHORT:mode><GPI:mode>2"
1542 [(set (match_operand:GPI 0 "register_operand")
1543 (ANY_EXTEND:GPI (match_operand:SHORT 1 "nonimmediate_operand")))]
1547 (define_insn "*extend<SHORT:mode><GPI:mode>2_aarch64"
1548 [(set (match_operand:GPI 0 "register_operand" "=r,r")
1549 (sign_extend:GPI (match_operand:SHORT 1 "nonimmediate_operand" "r,m")))]
1552 sxt<SHORT:size>\t%<GPI:w>0, %w1
1553 ldrs<SHORT:size>\t%<GPI:w>0, %1"
1554 [(set_attr "type" "extend,load_4")]
1557 (define_insn "*zero_extend<SHORT:mode><GPI:mode>2_aarch64"
1558 [(set (match_operand:GPI 0 "register_operand" "=r,r,*w")
1559 (zero_extend:GPI (match_operand:SHORT 1 "nonimmediate_operand" "r,m,m")))]
1562 and\t%<GPI:w>0, %<GPI:w>1, <SHORT:short_mask>
1563 ldr<SHORT:size>\t%w0, %1
1564 ldr\t%<SHORT:size>0, %1"
1565 [(set_attr "type" "logic_imm,load_4,load_4")]
1568 (define_expand "<optab>qihi2"
1569 [(set (match_operand:HI 0 "register_operand")
1570 (ANY_EXTEND:HI (match_operand:QI 1 "nonimmediate_operand")))]
1574 (define_insn "*extendqihi2_aarch64"
1575 [(set (match_operand:HI 0 "register_operand" "=r,r")
1576 (sign_extend:HI (match_operand:QI 1 "nonimmediate_operand" "r,m")))]
1581 [(set_attr "type" "extend,load_4")]
1584 (define_insn "*zero_extendqihi2_aarch64"
1585 [(set (match_operand:HI 0 "register_operand" "=r,r")
1586 (zero_extend:HI (match_operand:QI 1 "nonimmediate_operand" "r,m")))]
1591 [(set_attr "type" "logic_imm,load_4")]
1594 ;; -------------------------------------------------------------------
1595 ;; Simple arithmetic
1596 ;; -------------------------------------------------------------------
1598 (define_expand "add<mode>3"
1600 (match_operand:GPI 0 "register_operand" "")
1601 (plus:GPI (match_operand:GPI 1 "register_operand" "")
1602 (match_operand:GPI 2 "aarch64_pluslong_or_poly_operand" "")))]
1605 /* If operands[1] is a subreg extract the inner RTX. */
1606 rtx op1 = REG_P (operands[1]) ? operands[1] : SUBREG_REG (operands[1]);
1608 /* If the constant is too large for a single instruction and isn't frame
1609 based, split off the immediate so it is available for CSE. */
1610 if (!aarch64_plus_immediate (operands[2], <MODE>mode)
1611 && can_create_pseudo_p ()
1613 || !REGNO_PTR_FRAME_P (REGNO (op1))))
1614 operands[2] = force_reg (<MODE>mode, operands[2]);
1615 /* Expand polynomial additions now if the destination is the stack
1616 pointer, since we don't want to use that as a temporary. */
1617 else if (operands[0] == stack_pointer_rtx
1618 && aarch64_split_add_offset_immediate (operands[2], <MODE>mode))
1620 aarch64_split_add_offset (<MODE>mode, operands[0], operands[1],
1621 operands[2], NULL_RTX, NULL_RTX);
1626 (define_insn "*add<mode>3_aarch64"
1628 (match_operand:GPI 0 "register_operand" "=rk,rk,w,rk,r,rk")
1630 (match_operand:GPI 1 "register_operand" "%rk,rk,w,rk,rk,rk")
1631 (match_operand:GPI 2 "aarch64_pluslong_operand" "I,r,w,J,Uaa,Uav")))]
1634 add\\t%<w>0, %<w>1, %2
1635 add\\t%<w>0, %<w>1, %<w>2
1636 add\\t%<rtn>0<vas>, %<rtn>1<vas>, %<rtn>2<vas>
1637 sub\\t%<w>0, %<w>1, #%n2
1639 * return aarch64_output_sve_addvl_addpl (operands[0], operands[1], operands[2]);"
1640 ;; The "alu_imm" type for ADDVL/ADDPL is just a placeholder.
1641 [(set_attr "type" "alu_imm,alu_sreg,neon_add,alu_imm,multiple,alu_imm")
1642 (set_attr "simd" "*,*,yes,*,*,*")]
1645 ;; zero_extend version of above
1646 (define_insn "*addsi3_aarch64_uxtw"
1648 (match_operand:DI 0 "register_operand" "=rk,rk,rk,r")
1650 (plus:SI (match_operand:SI 1 "register_operand" "%rk,rk,rk,rk")
1651 (match_operand:SI 2 "aarch64_pluslong_operand" "I,r,J,Uaa"))))]
1656 sub\\t%w0, %w1, #%n2
1658 [(set_attr "type" "alu_imm,alu_sreg,alu_imm,multiple")]
1661 ;; If there's a free register, and we can load the constant with a
1662 ;; single instruction, do so. This has a chance to improve scheduling.
1664 [(match_scratch:GPI 3 "r")
1665 (set (match_operand:GPI 0 "register_operand")
1667 (match_operand:GPI 1 "register_operand")
1668 (match_operand:GPI 2 "aarch64_pluslong_strict_immedate")))]
1669 "aarch64_move_imm (INTVAL (operands[2]), <MODE>mode)"
1670 [(set (match_dup 3) (match_dup 2))
1671 (set (match_dup 0) (plus:GPI (match_dup 1) (match_dup 3)))]
1675 [(match_scratch:SI 3 "r")
1676 (set (match_operand:DI 0 "register_operand")
1679 (match_operand:SI 1 "register_operand")
1680 (match_operand:SI 2 "aarch64_pluslong_strict_immedate"))))]
1681 "aarch64_move_imm (INTVAL (operands[2]), SImode)"
1682 [(set (match_dup 3) (match_dup 2))
1683 (set (match_dup 0) (zero_extend:DI (plus:SI (match_dup 1) (match_dup 3))))]
1686 ;; After peephole2 has had a chance to run, split any remaining long
1687 ;; additions into two add immediates.
1689 [(set (match_operand:GPI 0 "register_operand")
1691 (match_operand:GPI 1 "register_operand")
1692 (match_operand:GPI 2 "aarch64_pluslong_strict_immedate")))]
1693 "epilogue_completed"
1694 [(set (match_dup 0) (plus:GPI (match_dup 1) (match_dup 3)))
1695 (set (match_dup 0) (plus:GPI (match_dup 0) (match_dup 4)))]
1697 HOST_WIDE_INT i = INTVAL (operands[2]);
1698 HOST_WIDE_INT s = (i >= 0 ? i & 0xfff : -(-i & 0xfff));
1699 operands[3] = GEN_INT (i - s);
1700 operands[4] = GEN_INT (s);
1704 ;; Match addition of polynomial offsets that require one temporary, for which
1705 ;; we can use the early-clobbered destination register. This is a separate
1706 ;; pattern so that the early clobber doesn't affect register allocation
1707 ;; for other forms of addition. However, we still need to provide an
1708 ;; all-register alternative, in case the offset goes out of range after
1709 ;; elimination. For completeness we might as well provide all GPR-based
1710 ;; alternatives from the main pattern.
1712 ;; We don't have a pattern for additions requiring two temporaries since at
1713 ;; present LRA doesn't allow new scratches to be added during elimination.
1714 ;; Such offsets should be rare anyway.
1716 ;; ??? But if we added LRA support for new scratches, much of the ugliness
1717 ;; here would go away. We could just handle all polynomial constants in
1719 (define_insn_and_split "*add<mode>3_poly_1"
1721 (match_operand:GPI 0 "register_operand" "=r,r,r,r,r,&r")
1723 (match_operand:GPI 1 "register_operand" "%rk,rk,rk,rk,rk,rk")
1724 (match_operand:GPI 2 "aarch64_pluslong_or_poly_operand" "I,r,J,Uaa,Uav,Uat")))]
1725 "TARGET_SVE && operands[0] != stack_pointer_rtx"
1727 add\\t%<w>0, %<w>1, %2
1728 add\\t%<w>0, %<w>1, %<w>2
1729 sub\\t%<w>0, %<w>1, #%n2
1731 * return aarch64_output_sve_addvl_addpl (operands[0], operands[1], operands[2]);
1733 "&& epilogue_completed
1734 && !reg_overlap_mentioned_p (operands[0], operands[1])
1735 && aarch64_split_add_offset_immediate (operands[2], <MODE>mode)"
1738 aarch64_split_add_offset (<MODE>mode, operands[0], operands[1],
1739 operands[2], operands[0], NULL_RTX);
1742 ;; The "alu_imm" type for ADDVL/ADDPL is just a placeholder.
1743 [(set_attr "type" "alu_imm,alu_sreg,alu_imm,multiple,alu_imm,multiple")]
1747 [(set (match_operand:DI 0 "register_operand")
1750 (match_operand:SI 1 "register_operand")
1751 (match_operand:SI 2 "aarch64_pluslong_strict_immedate"))))]
1752 "epilogue_completed"
1753 [(set (match_dup 5) (plus:SI (match_dup 1) (match_dup 3)))
1754 (set (match_dup 0) (zero_extend:DI (plus:SI (match_dup 5) (match_dup 4))))]
1756 HOST_WIDE_INT i = INTVAL (operands[2]);
1757 HOST_WIDE_INT s = (i >= 0 ? i & 0xfff : -(-i & 0xfff));
1758 operands[3] = GEN_INT (i - s);
1759 operands[4] = GEN_INT (s);
1760 operands[5] = gen_lowpart (SImode, operands[0]);
1764 (define_expand "addti3"
1765 [(set (match_operand:TI 0 "register_operand" "")
1766 (plus:TI (match_operand:TI 1 "register_operand" "")
1767 (match_operand:TI 2 "register_operand" "")))]
1770 rtx low = gen_reg_rtx (DImode);
1771 emit_insn (gen_adddi3_compareC (low, gen_lowpart (DImode, operands[1]),
1772 gen_lowpart (DImode, operands[2])));
1774 rtx high = gen_reg_rtx (DImode);
1775 emit_insn (gen_adddi3_carryin (high, gen_highpart (DImode, operands[1]),
1776 gen_highpart (DImode, operands[2])));
1778 emit_move_insn (gen_lowpart (DImode, operands[0]), low);
1779 emit_move_insn (gen_highpart (DImode, operands[0]), high);
1783 (define_insn "add<mode>3_compare0"
1784 [(set (reg:CC_NZ CC_REGNUM)
1786 (plus:GPI (match_operand:GPI 1 "register_operand" "%r,r,r")
1787 (match_operand:GPI 2 "aarch64_plus_operand" "r,I,J"))
1789 (set (match_operand:GPI 0 "register_operand" "=r,r,r")
1790 (plus:GPI (match_dup 1) (match_dup 2)))]
1793 adds\\t%<w>0, %<w>1, %<w>2
1794 adds\\t%<w>0, %<w>1, %2
1795 subs\\t%<w>0, %<w>1, #%n2"
1796 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
1799 ;; zero_extend version of above
1800 (define_insn "*addsi3_compare0_uxtw"
1801 [(set (reg:CC_NZ CC_REGNUM)
1803 (plus:SI (match_operand:SI 1 "register_operand" "%r,r,r")
1804 (match_operand:SI 2 "aarch64_plus_operand" "r,I,J"))
1806 (set (match_operand:DI 0 "register_operand" "=r,r,r")
1807 (zero_extend:DI (plus:SI (match_dup 1) (match_dup 2))))]
1810 adds\\t%w0, %w1, %w2
1812 subs\\t%w0, %w1, #%n2"
1813 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
1816 (define_insn "*add<mode>3_compareC_cconly_imm"
1817 [(set (reg:CC_C CC_REGNUM)
1820 (zero_extend:<DWI> (match_operand:GPI 0 "register_operand" "r,r"))
1821 (match_operand:<DWI> 2 "const_scalar_int_operand" ""))
1825 (match_operand:GPI 1 "aarch64_plus_immediate" "I,J")))))]
1826 "aarch64_zero_extend_const_eq (<DWI>mode, operands[2],
1827 <MODE>mode, operands[1])"
1831 [(set_attr "type" "alus_imm")]
1834 (define_insn "*add<mode>3_compareC_cconly"
1835 [(set (reg:CC_C CC_REGNUM)
1838 (zero_extend:<DWI> (match_operand:GPI 0 "register_operand" "r"))
1839 (zero_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")))
1840 (zero_extend:<DWI> (plus:GPI (match_dup 0) (match_dup 1)))))]
1842 "cmn\\t%<w>0, %<w>1"
1843 [(set_attr "type" "alus_sreg")]
1846 (define_insn "*add<mode>3_compareC_imm"
1847 [(set (reg:CC_C CC_REGNUM)
1850 (zero_extend:<DWI> (match_operand:GPI 1 "register_operand" "r,r"))
1851 (match_operand:<DWI> 3 "const_scalar_int_operand" ""))
1855 (match_operand:GPI 2 "aarch64_plus_immediate" "I,J")))))
1856 (set (match_operand:GPI 0 "register_operand" "=r,r")
1857 (plus:GPI (match_dup 1) (match_dup 2)))]
1858 "aarch64_zero_extend_const_eq (<DWI>mode, operands[3],
1859 <MODE>mode, operands[2])"
1861 adds\\t%<w>0, %<w>1, %2
1862 subs\\t%<w>0, %<w>1, #%n2"
1863 [(set_attr "type" "alus_imm")]
1866 (define_insn "add<mode>3_compareC"
1867 [(set (reg:CC_C CC_REGNUM)
1870 (zero_extend:<DWI> (match_operand:GPI 1 "register_operand" "r"))
1871 (zero_extend:<DWI> (match_operand:GPI 2 "register_operand" "r")))
1873 (plus:GPI (match_dup 1) (match_dup 2)))))
1874 (set (match_operand:GPI 0 "register_operand" "=r")
1875 (plus:GPI (match_dup 1) (match_dup 2)))]
1877 "adds\\t%<w>0, %<w>1, %<w>2"
1878 [(set_attr "type" "alus_sreg")]
1881 (define_insn "*adds_shift_imm_<mode>"
1882 [(set (reg:CC_NZ CC_REGNUM)
1884 (plus:GPI (ASHIFT:GPI
1885 (match_operand:GPI 1 "register_operand" "r")
1886 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
1887 (match_operand:GPI 3 "register_operand" "r"))
1889 (set (match_operand:GPI 0 "register_operand" "=r")
1890 (plus:GPI (ASHIFT:GPI (match_dup 1) (match_dup 2))
1893 "adds\\t%<w>0, %<w>3, %<w>1, <shift> %2"
1894 [(set_attr "type" "alus_shift_imm")]
1897 (define_insn "*subs_shift_imm_<mode>"
1898 [(set (reg:CC_NZ CC_REGNUM)
1900 (minus:GPI (match_operand:GPI 1 "register_operand" "r")
1902 (match_operand:GPI 2 "register_operand" "r")
1903 (match_operand:QI 3 "aarch64_shift_imm_<mode>" "n")))
1905 (set (match_operand:GPI 0 "register_operand" "=r")
1906 (minus:GPI (match_dup 1)
1907 (ASHIFT:GPI (match_dup 2) (match_dup 3))))]
1909 "subs\\t%<w>0, %<w>1, %<w>2, <shift> %3"
1910 [(set_attr "type" "alus_shift_imm")]
1913 (define_insn "*adds_mul_imm_<mode>"
1914 [(set (reg:CC_NZ CC_REGNUM)
1917 (match_operand:GPI 1 "register_operand" "r")
1918 (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))
1919 (match_operand:GPI 3 "register_operand" "r"))
1921 (set (match_operand:GPI 0 "register_operand" "=r")
1922 (plus:GPI (mult:GPI (match_dup 1) (match_dup 2))
1925 "adds\\t%<w>0, %<w>3, %<w>1, lsl %p2"
1926 [(set_attr "type" "alus_shift_imm")]
1929 (define_insn "*subs_mul_imm_<mode>"
1930 [(set (reg:CC_NZ CC_REGNUM)
1932 (minus:GPI (match_operand:GPI 1 "register_operand" "r")
1934 (match_operand:GPI 2 "register_operand" "r")
1935 (match_operand:QI 3 "aarch64_pwr_2_<mode>" "n")))
1937 (set (match_operand:GPI 0 "register_operand" "=r")
1938 (minus:GPI (match_dup 1)
1939 (mult:GPI (match_dup 2) (match_dup 3))))]
1941 "subs\\t%<w>0, %<w>1, %<w>2, lsl %p3"
1942 [(set_attr "type" "alus_shift_imm")]
1945 (define_insn "*adds_<optab><ALLX:mode>_<GPI:mode>"
1946 [(set (reg:CC_NZ CC_REGNUM)
1949 (ANY_EXTEND:GPI (match_operand:ALLX 1 "register_operand" "r"))
1950 (match_operand:GPI 2 "register_operand" "r"))
1952 (set (match_operand:GPI 0 "register_operand" "=r")
1953 (plus:GPI (ANY_EXTEND:GPI (match_dup 1)) (match_dup 2)))]
1955 "adds\\t%<GPI:w>0, %<GPI:w>2, %<GPI:w>1, <su>xt<ALLX:size>"
1956 [(set_attr "type" "alus_ext")]
1959 (define_insn "*subs_<optab><ALLX:mode>_<GPI:mode>"
1960 [(set (reg:CC_NZ CC_REGNUM)
1962 (minus:GPI (match_operand:GPI 1 "register_operand" "r")
1964 (match_operand:ALLX 2 "register_operand" "r")))
1966 (set (match_operand:GPI 0 "register_operand" "=r")
1967 (minus:GPI (match_dup 1) (ANY_EXTEND:GPI (match_dup 2))))]
1969 "subs\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size>"
1970 [(set_attr "type" "alus_ext")]
1973 (define_insn "*adds_<optab><ALLX:mode>_shift_<GPI:mode>"
1974 [(set (reg:CC_NZ CC_REGNUM)
1976 (plus:GPI (ashift:GPI
1978 (match_operand:ALLX 1 "register_operand" "r"))
1979 (match_operand 2 "aarch64_imm3" "Ui3"))
1980 (match_operand:GPI 3 "register_operand" "r"))
1982 (set (match_operand:GPI 0 "register_operand" "=rk")
1983 (plus:GPI (ashift:GPI (ANY_EXTEND:GPI (match_dup 1))
1987 "adds\\t%<GPI:w>0, %<GPI:w>3, %<GPI:w>1, <su>xt<ALLX:size> %2"
1988 [(set_attr "type" "alus_ext")]
1991 (define_insn "*subs_<optab><ALLX:mode>_shift_<GPI:mode>"
1992 [(set (reg:CC_NZ CC_REGNUM)
1994 (minus:GPI (match_operand:GPI 1 "register_operand" "r")
1997 (match_operand:ALLX 2 "register_operand" "r"))
1998 (match_operand 3 "aarch64_imm3" "Ui3")))
2000 (set (match_operand:GPI 0 "register_operand" "=rk")
2001 (minus:GPI (match_dup 1)
2002 (ashift:GPI (ANY_EXTEND:GPI (match_dup 2))
2005 "subs\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size> %3"
2006 [(set_attr "type" "alus_ext")]
2009 (define_insn "*adds_<optab><mode>_multp2"
2010 [(set (reg:CC_NZ CC_REGNUM)
2012 (plus:GPI (ANY_EXTRACT:GPI
2013 (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2014 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2015 (match_operand 3 "const_int_operand" "n")
2017 (match_operand:GPI 4 "register_operand" "r"))
2019 (set (match_operand:GPI 0 "register_operand" "=r")
2020 (plus:GPI (ANY_EXTRACT:GPI (mult:GPI (match_dup 1) (match_dup 2))
2024 "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
2025 "adds\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
2026 [(set_attr "type" "alus_ext")]
2029 (define_insn "*subs_<optab><mode>_multp2"
2030 [(set (reg:CC_NZ CC_REGNUM)
2032 (minus:GPI (match_operand:GPI 4 "register_operand" "r")
2034 (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2035 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2036 (match_operand 3 "const_int_operand" "n")
2039 (set (match_operand:GPI 0 "register_operand" "=r")
2040 (minus:GPI (match_dup 4) (ANY_EXTRACT:GPI
2041 (mult:GPI (match_dup 1) (match_dup 2))
2044 "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
2045 "subs\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
2046 [(set_attr "type" "alus_ext")]
2049 (define_insn "*add<mode>3nr_compare0"
2050 [(set (reg:CC_NZ CC_REGNUM)
2052 (plus:GPI (match_operand:GPI 0 "register_operand" "%r,r,r")
2053 (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J"))
2060 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
2063 (define_insn "aarch64_sub<mode>_compare0"
2064 [(set (reg:CC_NZ CC_REGNUM)
2066 (minus:GPI (match_operand:GPI 0 "register_operand" "r")
2067 (match_operand:GPI 1 "aarch64_plus_operand" "r"))
2070 "cmp\\t%<w>0, %<w>1"
2071 [(set_attr "type" "alus_sreg")]
2074 (define_insn "*compare_neg<mode>"
2075 [(set (reg:CC_Z CC_REGNUM)
2077 (neg:GPI (match_operand:GPI 0 "register_operand" "r"))
2078 (match_operand:GPI 1 "register_operand" "r")))]
2080 "cmn\\t%<w>1, %<w>0"
2081 [(set_attr "type" "alus_sreg")]
2084 (define_insn "*add_<shift>_<mode>"
2085 [(set (match_operand:GPI 0 "register_operand" "=r")
2086 (plus:GPI (ASHIFT:GPI (match_operand:GPI 1 "register_operand" "r")
2087 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
2088 (match_operand:GPI 3 "register_operand" "r")))]
2090 "add\\t%<w>0, %<w>3, %<w>1, <shift> %2"
2091 [(set_attr "type" "alu_shift_imm")]
2094 ;; zero_extend version of above
2095 (define_insn "*add_<shift>_si_uxtw"
2096 [(set (match_operand:DI 0 "register_operand" "=r")
2098 (plus:SI (ASHIFT:SI (match_operand:SI 1 "register_operand" "r")
2099 (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
2100 (match_operand:SI 3 "register_operand" "r"))))]
2102 "add\\t%w0, %w3, %w1, <shift> %2"
2103 [(set_attr "type" "alu_shift_imm")]
2106 (define_insn "*add_mul_imm_<mode>"
2107 [(set (match_operand:GPI 0 "register_operand" "=r")
2108 (plus:GPI (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2109 (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))
2110 (match_operand:GPI 3 "register_operand" "r")))]
2112 "add\\t%<w>0, %<w>3, %<w>1, lsl %p2"
2113 [(set_attr "type" "alu_shift_imm")]
2116 (define_insn "*add_<optab><ALLX:mode>_<GPI:mode>"
2117 [(set (match_operand:GPI 0 "register_operand" "=rk")
2118 (plus:GPI (ANY_EXTEND:GPI (match_operand:ALLX 1 "register_operand" "r"))
2119 (match_operand:GPI 2 "register_operand" "r")))]
2121 "add\\t%<GPI:w>0, %<GPI:w>2, %<GPI:w>1, <su>xt<ALLX:size>"
2122 [(set_attr "type" "alu_ext")]
2125 ;; zero_extend version of above
2126 (define_insn "*add_<optab><SHORT:mode>_si_uxtw"
2127 [(set (match_operand:DI 0 "register_operand" "=rk")
2129 (plus:SI (ANY_EXTEND:SI (match_operand:SHORT 1 "register_operand" "r"))
2130 (match_operand:GPI 2 "register_operand" "r"))))]
2132 "add\\t%w0, %w2, %w1, <su>xt<SHORT:size>"
2133 [(set_attr "type" "alu_ext")]
2136 (define_insn "*add_<optab><ALLX:mode>_shft_<GPI:mode>"
2137 [(set (match_operand:GPI 0 "register_operand" "=rk")
2138 (plus:GPI (ashift:GPI (ANY_EXTEND:GPI
2139 (match_operand:ALLX 1 "register_operand" "r"))
2140 (match_operand 2 "aarch64_imm3" "Ui3"))
2141 (match_operand:GPI 3 "register_operand" "r")))]
2143 "add\\t%<GPI:w>0, %<GPI:w>3, %<GPI:w>1, <su>xt<ALLX:size> %2"
2144 [(set_attr "type" "alu_ext")]
2147 ;; zero_extend version of above
2148 (define_insn "*add_<optab><SHORT:mode>_shft_si_uxtw"
2149 [(set (match_operand:DI 0 "register_operand" "=rk")
2151 (plus:SI (ashift:SI (ANY_EXTEND:SI
2152 (match_operand:SHORT 1 "register_operand" "r"))
2153 (match_operand 2 "aarch64_imm3" "Ui3"))
2154 (match_operand:SI 3 "register_operand" "r"))))]
2156 "add\\t%w0, %w3, %w1, <su>xt<SHORT:size> %2"
2157 [(set_attr "type" "alu_ext")]
2160 (define_insn "*add_<optab><ALLX:mode>_mult_<GPI:mode>"
2161 [(set (match_operand:GPI 0 "register_operand" "=rk")
2162 (plus:GPI (mult:GPI (ANY_EXTEND:GPI
2163 (match_operand:ALLX 1 "register_operand" "r"))
2164 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2165 (match_operand:GPI 3 "register_operand" "r")))]
2167 "add\\t%<GPI:w>0, %<GPI:w>3, %<GPI:w>1, <su>xt<ALLX:size> %p2"
2168 [(set_attr "type" "alu_ext")]
2171 ;; zero_extend version of above
2172 (define_insn "*add_<optab><SHORT:mode>_mult_si_uxtw"
2173 [(set (match_operand:DI 0 "register_operand" "=rk")
2174 (zero_extend:DI (plus:SI (mult:SI (ANY_EXTEND:SI
2175 (match_operand:SHORT 1 "register_operand" "r"))
2176 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2177 (match_operand:SI 3 "register_operand" "r"))))]
2179 "add\\t%w0, %w3, %w1, <su>xt<SHORT:size> %p2"
2180 [(set_attr "type" "alu_ext")]
2183 (define_insn "*add_<optab><mode>_multp2"
2184 [(set (match_operand:GPI 0 "register_operand" "=rk")
2185 (plus:GPI (ANY_EXTRACT:GPI
2186 (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2187 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2188 (match_operand 3 "const_int_operand" "n")
2190 (match_operand:GPI 4 "register_operand" "r")))]
2191 "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
2192 "add\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
2193 [(set_attr "type" "alu_ext")]
2196 ;; zero_extend version of above
2197 (define_insn "*add_<optab>si_multp2_uxtw"
2198 [(set (match_operand:DI 0 "register_operand" "=rk")
2200 (plus:SI (ANY_EXTRACT:SI
2201 (mult:SI (match_operand:SI 1 "register_operand" "r")
2202 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2203 (match_operand 3 "const_int_operand" "n")
2205 (match_operand:SI 4 "register_operand" "r"))))]
2206 "aarch64_is_extend_from_extract (SImode, operands[2], operands[3])"
2207 "add\\t%w0, %w4, %w1, <su>xt%e3 %p2"
2208 [(set_attr "type" "alu_ext")]
2211 (define_expand "add<mode>3_carryin"
2212 [(set (match_operand:GPI 0 "register_operand")
2215 (ne:GPI (reg:CC_C CC_REGNUM) (const_int 0))
2216 (match_operand:GPI 1 "aarch64_reg_or_zero"))
2217 (match_operand:GPI 2 "aarch64_reg_or_zero")))]
2222 ;; Note that add with carry with two zero inputs is matched by cset,
2223 ;; and that add with carry with one zero input is matched by cinc.
2225 (define_insn "*add<mode>3_carryin"
2226 [(set (match_operand:GPI 0 "register_operand" "=r")
2229 (match_operand:GPI 3 "aarch64_carry_operation" "")
2230 (match_operand:GPI 1 "register_operand" "r"))
2231 (match_operand:GPI 2 "register_operand" "r")))]
2233 "adc\\t%<w>0, %<w>1, %<w>2"
2234 [(set_attr "type" "adc_reg")]
2237 ;; zero_extend version of above
2238 (define_insn "*addsi3_carryin_uxtw"
2239 [(set (match_operand:DI 0 "register_operand" "=r")
2243 (match_operand:SI 3 "aarch64_carry_operation" "")
2244 (match_operand:SI 1 "register_operand" "r"))
2245 (match_operand:SI 2 "register_operand" "r"))))]
2247 "adc\\t%w0, %w1, %w2"
2248 [(set_attr "type" "adc_reg")]
2251 (define_insn "*add_uxt<mode>_shift2"
2252 [(set (match_operand:GPI 0 "register_operand" "=rk")
2254 (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
2255 (match_operand 2 "aarch64_imm3" "Ui3"))
2256 (match_operand 3 "const_int_operand" "n"))
2257 (match_operand:GPI 4 "register_operand" "r")))]
2258 "aarch64_uxt_size (INTVAL (operands[2]), INTVAL (operands[3])) != 0"
2260 operands[3] = GEN_INT (aarch64_uxt_size (INTVAL(operands[2]),
2261 INTVAL (operands[3])));
2262 return \"add\t%<w>0, %<w>4, %<w>1, uxt%e3 %2\";"
2263 [(set_attr "type" "alu_ext")]
2266 ;; zero_extend version of above
2267 (define_insn "*add_uxtsi_shift2_uxtw"
2268 [(set (match_operand:DI 0 "register_operand" "=rk")
2271 (ashift:SI (match_operand:SI 1 "register_operand" "r")
2272 (match_operand 2 "aarch64_imm3" "Ui3"))
2273 (match_operand 3 "const_int_operand" "n"))
2274 (match_operand:SI 4 "register_operand" "r"))))]
2275 "aarch64_uxt_size (INTVAL (operands[2]), INTVAL (operands[3])) != 0"
2277 operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
2278 INTVAL (operands[3])));
2279 return \"add\t%w0, %w4, %w1, uxt%e3 %2\";"
2280 [(set_attr "type" "alu_ext")]
2283 (define_insn "*add_uxt<mode>_multp2"
2284 [(set (match_operand:GPI 0 "register_operand" "=rk")
2286 (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2287 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2288 (match_operand 3 "const_int_operand" "n"))
2289 (match_operand:GPI 4 "register_operand" "r")))]
2290 "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])), INTVAL (operands[3])) != 0"
2292 operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
2293 INTVAL (operands[3])));
2294 return \"add\t%<w>0, %<w>4, %<w>1, uxt%e3 %p2\";"
2295 [(set_attr "type" "alu_ext")]
2298 ;; zero_extend version of above
2299 (define_insn "*add_uxtsi_multp2_uxtw"
2300 [(set (match_operand:DI 0 "register_operand" "=rk")
2303 (mult:SI (match_operand:SI 1 "register_operand" "r")
2304 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2305 (match_operand 3 "const_int_operand" "n"))
2306 (match_operand:SI 4 "register_operand" "r"))))]
2307 "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])), INTVAL (operands[3])) != 0"
2309 operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
2310 INTVAL (operands[3])));
2311 return \"add\t%w0, %w4, %w1, uxt%e3 %p2\";"
2312 [(set_attr "type" "alu_ext")]
2315 (define_insn "subsi3"
2316 [(set (match_operand:SI 0 "register_operand" "=rk")
2317 (minus:SI (match_operand:SI 1 "register_operand" "rk")
2318 (match_operand:SI 2 "register_operand" "r")))]
2320 "sub\\t%w0, %w1, %w2"
2321 [(set_attr "type" "alu_sreg")]
2324 ;; zero_extend version of above
2325 (define_insn "*subsi3_uxtw"
2326 [(set (match_operand:DI 0 "register_operand" "=rk")
2328 (minus:SI (match_operand:SI 1 "register_operand" "rk")
2329 (match_operand:SI 2 "register_operand" "r"))))]
2331 "sub\\t%w0, %w1, %w2"
2332 [(set_attr "type" "alu_sreg")]
2335 (define_insn "subdi3"
2336 [(set (match_operand:DI 0 "register_operand" "=rk,w")
2337 (minus:DI (match_operand:DI 1 "register_operand" "rk,w")
2338 (match_operand:DI 2 "register_operand" "r,w")))]
2342 sub\\t%d0, %d1, %d2"
2343 [(set_attr "type" "alu_sreg, neon_sub")
2344 (set_attr "simd" "*,yes")]
2347 (define_expand "subti3"
2348 [(set (match_operand:TI 0 "register_operand" "")
2349 (minus:TI (match_operand:TI 1 "register_operand" "")
2350 (match_operand:TI 2 "register_operand" "")))]
2353 rtx low = gen_reg_rtx (DImode);
2354 emit_insn (gen_subdi3_compare1 (low, gen_lowpart (DImode, operands[1]),
2355 gen_lowpart (DImode, operands[2])));
2357 rtx high = gen_reg_rtx (DImode);
2358 emit_insn (gen_subdi3_carryin (high, gen_highpart (DImode, operands[1]),
2359 gen_highpart (DImode, operands[2])));
2361 emit_move_insn (gen_lowpart (DImode, operands[0]), low);
2362 emit_move_insn (gen_highpart (DImode, operands[0]), high);
2366 (define_insn "*sub<mode>3_compare0"
2367 [(set (reg:CC_NZ CC_REGNUM)
2368 (compare:CC_NZ (minus:GPI (match_operand:GPI 1 "register_operand" "r")
2369 (match_operand:GPI 2 "register_operand" "r"))
2371 (set (match_operand:GPI 0 "register_operand" "=r")
2372 (minus:GPI (match_dup 1) (match_dup 2)))]
2374 "subs\\t%<w>0, %<w>1, %<w>2"
2375 [(set_attr "type" "alus_sreg")]
2378 ;; zero_extend version of above
2379 (define_insn "*subsi3_compare0_uxtw"
2380 [(set (reg:CC_NZ CC_REGNUM)
2381 (compare:CC_NZ (minus:SI (match_operand:SI 1 "register_operand" "r")
2382 (match_operand:SI 2 "register_operand" "r"))
2384 (set (match_operand:DI 0 "register_operand" "=r")
2385 (zero_extend:DI (minus:SI (match_dup 1) (match_dup 2))))]
2387 "subs\\t%w0, %w1, %w2"
2388 [(set_attr "type" "alus_sreg")]
2391 (define_insn "sub<mode>3_compare1"
2392 [(set (reg:CC CC_REGNUM)
2394 (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
2395 (match_operand:GPI 2 "aarch64_reg_or_zero" "rZ")))
2396 (set (match_operand:GPI 0 "register_operand" "=r")
2397 (minus:GPI (match_dup 1) (match_dup 2)))]
2399 "subs\\t%<w>0, %<w>1, %<w>2"
2400 [(set_attr "type" "alus_sreg")]
2403 (define_insn "sub<mode>3_compare1_imm"
2404 [(set (reg:CC CC_REGNUM)
2406 (match_operand:GPI 1 "register_operand" "r")
2407 (match_operand:GPI 3 "const_int_operand" "n")))
2408 (set (match_operand:GPI 0 "register_operand" "=r")
2409 (plus:GPI (match_dup 1)
2410 (match_operand:GPI 2 "aarch64_sub_immediate" "J")))]
2411 "INTVAL (operands[3]) == -INTVAL (operands[2])"
2412 "subs\\t%<w>0, %<w>1, #%n2"
2413 [(set_attr "type" "alus_sreg")]
2417 [(set (match_operand:GPI 0 "register_operand")
2418 (minus:GPI (match_operand:GPI 1 "aarch64_reg_or_zero")
2419 (match_operand:GPI 2 "aarch64_reg_or_zero")))
2420 (set (reg:CC CC_REGNUM)
2424 "!reg_overlap_mentioned_p (operands[0], operands[1])
2425 && !reg_overlap_mentioned_p (operands[0], operands[2])"
2428 emit_insn (gen_sub<mode>3_compare1 (operands[0], operands[1],
2434 ;; Same as the above peephole but with the compare and minus in
2435 ;; swapped order. The restriction on overlap between operand 0
2436 ;; and operands 1 and 2 doesn't apply here.
2438 [(set (reg:CC CC_REGNUM)
2440 (match_operand:GPI 1 "aarch64_reg_or_zero")
2441 (match_operand:GPI 2 "aarch64_reg_or_zero")))
2442 (set (match_operand:GPI 0 "register_operand")
2443 (minus:GPI (match_dup 1)
2448 emit_insn (gen_sub<mode>3_compare1 (operands[0], operands[1],
2455 [(set (match_operand:GPI 0 "register_operand")
2456 (plus:GPI (match_operand:GPI 1 "register_operand")
2457 (match_operand:GPI 2 "aarch64_sub_immediate")))
2458 (set (reg:CC CC_REGNUM)
2461 (match_operand:GPI 3 "const_int_operand")))]
2462 "!reg_overlap_mentioned_p (operands[0], operands[1])
2463 && INTVAL (operands[3]) == -INTVAL (operands[2])"
2466 emit_insn (gen_sub<mode>3_compare1_imm (operands[0], operands[1],
2467 operands[2], operands[3]));
2472 ;; Same as the above peephole but with the compare and minus in
2473 ;; swapped order. The restriction on overlap between operand 0
2474 ;; and operands 1 doesn't apply here.
2476 [(set (reg:CC CC_REGNUM)
2478 (match_operand:GPI 1 "register_operand")
2479 (match_operand:GPI 3 "const_int_operand")))
2480 (set (match_operand:GPI 0 "register_operand")
2481 (plus:GPI (match_dup 1)
2482 (match_operand:GPI 2 "aarch64_sub_immediate")))]
2483 "INTVAL (operands[3]) == -INTVAL (operands[2])"
2486 emit_insn (gen_sub<mode>3_compare1_imm (operands[0], operands[1],
2487 operands[2], operands[3]));
2492 (define_insn "*sub_<shift>_<mode>"
2493 [(set (match_operand:GPI 0 "register_operand" "=r")
2494 (minus:GPI (match_operand:GPI 3 "register_operand" "r")
2496 (match_operand:GPI 1 "register_operand" "r")
2497 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
2499 "sub\\t%<w>0, %<w>3, %<w>1, <shift> %2"
2500 [(set_attr "type" "alu_shift_imm")]
2503 ;; zero_extend version of above
2504 (define_insn "*sub_<shift>_si_uxtw"
2505 [(set (match_operand:DI 0 "register_operand" "=r")
2507 (minus:SI (match_operand:SI 3 "register_operand" "r")
2509 (match_operand:SI 1 "register_operand" "r")
2510 (match_operand:QI 2 "aarch64_shift_imm_si" "n")))))]
2512 "sub\\t%w0, %w3, %w1, <shift> %2"
2513 [(set_attr "type" "alu_shift_imm")]
2516 (define_insn "*sub_mul_imm_<mode>"
2517 [(set (match_operand:GPI 0 "register_operand" "=r")
2518 (minus:GPI (match_operand:GPI 3 "register_operand" "r")
2520 (match_operand:GPI 1 "register_operand" "r")
2521 (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))))]
2523 "sub\\t%<w>0, %<w>3, %<w>1, lsl %p2"
2524 [(set_attr "type" "alu_shift_imm")]
2527 ;; zero_extend version of above
2528 (define_insn "*sub_mul_imm_si_uxtw"
2529 [(set (match_operand:DI 0 "register_operand" "=r")
2531 (minus:SI (match_operand:SI 3 "register_operand" "r")
2533 (match_operand:SI 1 "register_operand" "r")
2534 (match_operand:QI 2 "aarch64_pwr_2_si" "n")))))]
2536 "sub\\t%w0, %w3, %w1, lsl %p2"
2537 [(set_attr "type" "alu_shift_imm")]
2540 (define_insn "*sub_<optab><ALLX:mode>_<GPI:mode>"
2541 [(set (match_operand:GPI 0 "register_operand" "=rk")
2542 (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
2544 (match_operand:ALLX 2 "register_operand" "r"))))]
2546 "sub\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size>"
2547 [(set_attr "type" "alu_ext")]
2550 ;; zero_extend version of above
2551 (define_insn "*sub_<optab><SHORT:mode>_si_uxtw"
2552 [(set (match_operand:DI 0 "register_operand" "=rk")
2554 (minus:SI (match_operand:SI 1 "register_operand" "rk")
2556 (match_operand:SHORT 2 "register_operand" "r")))))]
2558 "sub\\t%w0, %w1, %w2, <su>xt<SHORT:size>"
2559 [(set_attr "type" "alu_ext")]
2562 (define_insn "*sub_<optab><ALLX:mode>_shft_<GPI:mode>"
2563 [(set (match_operand:GPI 0 "register_operand" "=rk")
2564 (minus:GPI (match_operand:GPI 1 "register_operand" "rk")
2565 (ashift:GPI (ANY_EXTEND:GPI
2566 (match_operand:ALLX 2 "register_operand" "r"))
2567 (match_operand 3 "aarch64_imm3" "Ui3"))))]
2569 "sub\\t%<GPI:w>0, %<GPI:w>1, %<GPI:w>2, <su>xt<ALLX:size> %3"
2570 [(set_attr "type" "alu_ext")]
2573 ;; zero_extend version of above
2574 (define_insn "*sub_<optab><SHORT:mode>_shft_si_uxtw"
2575 [(set (match_operand:DI 0 "register_operand" "=rk")
2577 (minus:SI (match_operand:SI 1 "register_operand" "rk")
2578 (ashift:SI (ANY_EXTEND:SI
2579 (match_operand:SHORT 2 "register_operand" "r"))
2580 (match_operand 3 "aarch64_imm3" "Ui3")))))]
2582 "sub\\t%w0, %w1, %w2, <su>xt<SHORT:size> %3"
2583 [(set_attr "type" "alu_ext")]
2586 (define_insn "*sub_<optab><mode>_multp2"
2587 [(set (match_operand:GPI 0 "register_operand" "=rk")
2588 (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
2590 (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2591 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2592 (match_operand 3 "const_int_operand" "n")
2594 "aarch64_is_extend_from_extract (<MODE>mode, operands[2], operands[3])"
2595 "sub\\t%<w>0, %<w>4, %<w>1, <su>xt%e3 %p2"
2596 [(set_attr "type" "alu_ext")]
2599 ;; zero_extend version of above
2600 (define_insn "*sub_<optab>si_multp2_uxtw"
2601 [(set (match_operand:DI 0 "register_operand" "=rk")
2603 (minus:SI (match_operand:SI 4 "register_operand" "rk")
2605 (mult:SI (match_operand:SI 1 "register_operand" "r")
2606 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2607 (match_operand 3 "const_int_operand" "n")
2609 "aarch64_is_extend_from_extract (SImode, operands[2], operands[3])"
2610 "sub\\t%w0, %w4, %w1, <su>xt%e3 %p2"
2611 [(set_attr "type" "alu_ext")]
2614 ;; The hardware description is op1 + ~op2 + C.
2615 ;; = op1 + (-op2 + 1) + (1 - !C)
2616 ;; = op1 - op2 - 1 + 1 - !C
2617 ;; = op1 - op2 - !C.
2618 ;; We describe the latter.
2620 (define_insn "*sub<mode>3_carryin0"
2621 [(set (match_operand:GPI 0 "register_operand" "=r")
2623 (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
2624 (match_operand:GPI 2 "aarch64_borrow_operation" "")))]
2626 "sbc\\t%<w>0, %<w>1, <w>zr"
2627 [(set_attr "type" "adc_reg")]
2630 ;; zero_extend version of the above
2631 (define_insn "*subsi3_carryin_uxtw"
2632 [(set (match_operand:DI 0 "register_operand" "=r")
2635 (match_operand:SI 1 "aarch64_reg_or_zero" "rZ")
2636 (match_operand:SI 2 "aarch64_borrow_operation" ""))))]
2638 "sbc\\t%w0, %w1, wzr"
2639 [(set_attr "type" "adc_reg")]
2642 (define_expand "sub<mode>3_carryin"
2643 [(set (match_operand:GPI 0 "register_operand")
2646 (match_operand:GPI 1 "aarch64_reg_or_zero")
2647 (match_operand:GPI 2 "register_operand"))
2648 (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))))]
2653 (define_insn "*sub<mode>3_carryin"
2654 [(set (match_operand:GPI 0 "register_operand" "=r")
2657 (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
2658 (match_operand:GPI 2 "register_operand" "r"))
2659 (match_operand:GPI 3 "aarch64_borrow_operation" "")))]
2662 "sbc\\t%<w>0, %<w>1, %<w>2"
2663 [(set_attr "type" "adc_reg")]
2666 ;; zero_extend version of the above
2667 (define_insn "*subsi3_carryin_uxtw"
2668 [(set (match_operand:DI 0 "register_operand" "=r")
2672 (match_operand:SI 1 "aarch64_reg_or_zero" "rZ")
2673 (match_operand:SI 2 "register_operand" "r"))
2674 (match_operand:SI 3 "aarch64_borrow_operation" ""))))]
2677 "sbc\\t%w0, %w1, %w2"
2678 [(set_attr "type" "adc_reg")]
2681 (define_insn "*sub<mode>3_carryin_alt"
2682 [(set (match_operand:GPI 0 "register_operand" "=r")
2685 (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ")
2686 (match_operand:GPI 3 "aarch64_borrow_operation" ""))
2687 (match_operand:GPI 2 "register_operand" "r")))]
2689 "sbc\\t%<w>0, %<w>1, %<w>2"
2690 [(set_attr "type" "adc_reg")]
2693 ;; zero_extend version of the above
2694 (define_insn "*subsi3_carryin_alt_uxtw"
2695 [(set (match_operand:DI 0 "register_operand" "=r")
2699 (match_operand:SI 1 "aarch64_reg_or_zero" "rZ")
2700 (match_operand:SI 3 "aarch64_borrow_operation" ""))
2701 (match_operand:SI 2 "register_operand" "r"))))]
2703 "sbc\\t%w0, %w1, %w2"
2704 [(set_attr "type" "adc_reg")]
2707 (define_insn "*sub_uxt<mode>_shift2"
2708 [(set (match_operand:GPI 0 "register_operand" "=rk")
2709 (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
2711 (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
2712 (match_operand 2 "aarch64_imm3" "Ui3"))
2713 (match_operand 3 "const_int_operand" "n"))))]
2714 "aarch64_uxt_size (INTVAL (operands[2]),INTVAL (operands[3])) != 0"
2716 operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
2717 INTVAL (operands[3])));
2718 return \"sub\t%<w>0, %<w>4, %<w>1, uxt%e3 %2\";"
2719 [(set_attr "type" "alu_ext")]
2722 ;; zero_extend version of above
2723 (define_insn "*sub_uxtsi_shift2_uxtw"
2724 [(set (match_operand:DI 0 "register_operand" "=rk")
2726 (minus:SI (match_operand:SI 4 "register_operand" "rk")
2728 (ashift:SI (match_operand:SI 1 "register_operand" "r")
2729 (match_operand 2 "aarch64_imm3" "Ui3"))
2730 (match_operand 3 "const_int_operand" "n")))))]
2731 "aarch64_uxt_size (INTVAL (operands[2]),INTVAL (operands[3])) != 0"
2733 operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]),
2734 INTVAL (operands[3])));
2735 return \"sub\t%w0, %w4, %w1, uxt%e3 %2\";"
2736 [(set_attr "type" "alu_ext")]
2739 (define_insn "*sub_uxt<mode>_multp2"
2740 [(set (match_operand:GPI 0 "register_operand" "=rk")
2741 (minus:GPI (match_operand:GPI 4 "register_operand" "rk")
2743 (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2744 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2745 (match_operand 3 "const_int_operand" "n"))))]
2746 "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),INTVAL (operands[3])) != 0"
2748 operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
2749 INTVAL (operands[3])));
2750 return \"sub\t%<w>0, %<w>4, %<w>1, uxt%e3 %p2\";"
2751 [(set_attr "type" "alu_ext")]
2754 ;; zero_extend version of above
2755 (define_insn "*sub_uxtsi_multp2_uxtw"
2756 [(set (match_operand:DI 0 "register_operand" "=rk")
2758 (minus:SI (match_operand:SI 4 "register_operand" "rk")
2760 (mult:SI (match_operand:SI 1 "register_operand" "r")
2761 (match_operand 2 "aarch64_pwr_imm3" "Up3"))
2762 (match_operand 3 "const_int_operand" "n")))))]
2763 "aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),INTVAL (operands[3])) != 0"
2765 operands[3] = GEN_INT (aarch64_uxt_size (exact_log2 (INTVAL (operands[2])),
2766 INTVAL (operands[3])));
2767 return \"sub\t%w0, %w4, %w1, uxt%e3 %p2\";"
2768 [(set_attr "type" "alu_ext")]
2771 (define_expand "abs<mode>2"
2772 [(match_operand:GPI 0 "register_operand" "")
2773 (match_operand:GPI 1 "register_operand" "")]
2776 rtx ccreg = aarch64_gen_compare_reg (LT, operands[1], const0_rtx);
2777 rtx x = gen_rtx_LT (VOIDmode, ccreg, const0_rtx);
2778 emit_insn (gen_csneg3<mode>_insn (operands[0], x, operands[1], operands[1]));
2783 (define_insn "neg<mode>2"
2784 [(set (match_operand:GPI 0 "register_operand" "=r,w")
2785 (neg:GPI (match_operand:GPI 1 "register_operand" "r,w")))]
2789 neg\\t%<rtn>0<vas>, %<rtn>1<vas>"
2790 [(set_attr "type" "alu_sreg, neon_neg<q>")
2791 (set_attr "simd" "*,yes")]
2794 ;; zero_extend version of above
2795 (define_insn "*negsi2_uxtw"
2796 [(set (match_operand:DI 0 "register_operand" "=r")
2797 (zero_extend:DI (neg:SI (match_operand:SI 1 "register_operand" "r"))))]
2800 [(set_attr "type" "alu_sreg")]
2803 (define_insn "*ngc<mode>"
2804 [(set (match_operand:GPI 0 "register_operand" "=r")
2806 (neg:GPI (match_operand:GPI 2 "aarch64_borrow_operation" ""))
2807 (match_operand:GPI 1 "register_operand" "r")))]
2809 "ngc\\t%<w>0, %<w>1"
2810 [(set_attr "type" "adc_reg")]
2813 (define_insn "*ngcsi_uxtw"
2814 [(set (match_operand:DI 0 "register_operand" "=r")
2817 (neg:SI (match_operand:SI 2 "aarch64_borrow_operation" ""))
2818 (match_operand:SI 1 "register_operand" "r"))))]
2821 [(set_attr "type" "adc_reg")]
2824 (define_insn "neg<mode>2_compare0"
2825 [(set (reg:CC_NZ CC_REGNUM)
2826 (compare:CC_NZ (neg:GPI (match_operand:GPI 1 "register_operand" "r"))
2828 (set (match_operand:GPI 0 "register_operand" "=r")
2829 (neg:GPI (match_dup 1)))]
2831 "negs\\t%<w>0, %<w>1"
2832 [(set_attr "type" "alus_sreg")]
2835 ;; zero_extend version of above
2836 (define_insn "*negsi2_compare0_uxtw"
2837 [(set (reg:CC_NZ CC_REGNUM)
2838 (compare:CC_NZ (neg:SI (match_operand:SI 1 "register_operand" "r"))
2840 (set (match_operand:DI 0 "register_operand" "=r")
2841 (zero_extend:DI (neg:SI (match_dup 1))))]
2844 [(set_attr "type" "alus_sreg")]
2847 (define_insn "*neg_<shift><mode>3_compare0"
2848 [(set (reg:CC_NZ CC_REGNUM)
2850 (neg:GPI (ASHIFT:GPI
2851 (match_operand:GPI 1 "register_operand" "r")
2852 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
2854 (set (match_operand:GPI 0 "register_operand" "=r")
2855 (neg:GPI (ASHIFT:GPI (match_dup 1) (match_dup 2))))]
2857 "negs\\t%<w>0, %<w>1, <shift> %2"
2858 [(set_attr "type" "alus_shift_imm")]
2861 (define_insn "*neg_<shift>_<mode>2"
2862 [(set (match_operand:GPI 0 "register_operand" "=r")
2863 (neg:GPI (ASHIFT:GPI
2864 (match_operand:GPI 1 "register_operand" "r")
2865 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
2867 "neg\\t%<w>0, %<w>1, <shift> %2"
2868 [(set_attr "type" "alu_shift_imm")]
2871 ;; zero_extend version of above
2872 (define_insn "*neg_<shift>_si2_uxtw"
2873 [(set (match_operand:DI 0 "register_operand" "=r")
2876 (match_operand:SI 1 "register_operand" "r")
2877 (match_operand:QI 2 "aarch64_shift_imm_si" "n")))))]
2879 "neg\\t%w0, %w1, <shift> %2"
2880 [(set_attr "type" "alu_shift_imm")]
2883 (define_insn "*neg_mul_imm_<mode>2"
2884 [(set (match_operand:GPI 0 "register_operand" "=r")
2886 (match_operand:GPI 1 "register_operand" "r")
2887 (match_operand:QI 2 "aarch64_pwr_2_<mode>" "n"))))]
2889 "neg\\t%<w>0, %<w>1, lsl %p2"
2890 [(set_attr "type" "alu_shift_imm")]
2893 ;; zero_extend version of above
2894 (define_insn "*neg_mul_imm_si2_uxtw"
2895 [(set (match_operand:DI 0 "register_operand" "=r")
2898 (match_operand:SI 1 "register_operand" "r")
2899 (match_operand:QI 2 "aarch64_pwr_2_si" "n")))))]
2901 "neg\\t%w0, %w1, lsl %p2"
2902 [(set_attr "type" "alu_shift_imm")]
2905 (define_insn "mul<mode>3"
2906 [(set (match_operand:GPI 0 "register_operand" "=r")
2907 (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2908 (match_operand:GPI 2 "register_operand" "r")))]
2910 "mul\\t%<w>0, %<w>1, %<w>2"
2911 [(set_attr "type" "mul")]
2914 ;; zero_extend version of above
2915 (define_insn "*mulsi3_uxtw"
2916 [(set (match_operand:DI 0 "register_operand" "=r")
2918 (mult:SI (match_operand:SI 1 "register_operand" "r")
2919 (match_operand:SI 2 "register_operand" "r"))))]
2921 "mul\\t%w0, %w1, %w2"
2922 [(set_attr "type" "mul")]
2925 (define_insn "madd<mode>"
2926 [(set (match_operand:GPI 0 "register_operand" "=r")
2927 (plus:GPI (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2928 (match_operand:GPI 2 "register_operand" "r"))
2929 (match_operand:GPI 3 "register_operand" "r")))]
2931 "madd\\t%<w>0, %<w>1, %<w>2, %<w>3"
2932 [(set_attr "type" "mla")]
2935 ;; zero_extend version of above
2936 (define_insn "*maddsi_uxtw"
2937 [(set (match_operand:DI 0 "register_operand" "=r")
2939 (plus:SI (mult:SI (match_operand:SI 1 "register_operand" "r")
2940 (match_operand:SI 2 "register_operand" "r"))
2941 (match_operand:SI 3 "register_operand" "r"))))]
2943 "madd\\t%w0, %w1, %w2, %w3"
2944 [(set_attr "type" "mla")]
2947 (define_insn "*msub<mode>"
2948 [(set (match_operand:GPI 0 "register_operand" "=r")
2949 (minus:GPI (match_operand:GPI 3 "register_operand" "r")
2950 (mult:GPI (match_operand:GPI 1 "register_operand" "r")
2951 (match_operand:GPI 2 "register_operand" "r"))))]
2954 "msub\\t%<w>0, %<w>1, %<w>2, %<w>3"
2955 [(set_attr "type" "mla")]
2958 ;; zero_extend version of above
2959 (define_insn "*msubsi_uxtw"
2960 [(set (match_operand:DI 0 "register_operand" "=r")
2962 (minus:SI (match_operand:SI 3 "register_operand" "r")
2963 (mult:SI (match_operand:SI 1 "register_operand" "r")
2964 (match_operand:SI 2 "register_operand" "r")))))]
2967 "msub\\t%w0, %w1, %w2, %w3"
2968 [(set_attr "type" "mla")]
2971 (define_insn "*mul<mode>_neg"
2972 [(set (match_operand:GPI 0 "register_operand" "=r")
2973 (mult:GPI (neg:GPI (match_operand:GPI 1 "register_operand" "r"))
2974 (match_operand:GPI 2 "register_operand" "r")))]
2977 "mneg\\t%<w>0, %<w>1, %<w>2"
2978 [(set_attr "type" "mul")]
2981 ;; zero_extend version of above
2982 (define_insn "*mulsi_neg_uxtw"
2983 [(set (match_operand:DI 0 "register_operand" "=r")
2985 (mult:SI (neg:SI (match_operand:SI 1 "register_operand" "r"))
2986 (match_operand:SI 2 "register_operand" "r"))))]
2989 "mneg\\t%w0, %w1, %w2"
2990 [(set_attr "type" "mul")]
2993 (define_insn "<su_optab>mulsidi3"
2994 [(set (match_operand:DI 0 "register_operand" "=r")
2995 (mult:DI (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
2996 (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r"))))]
2998 "<su>mull\\t%0, %w1, %w2"
2999 [(set_attr "type" "<su>mull")]
3002 (define_insn "<su_optab>maddsidi4"
3003 [(set (match_operand:DI 0 "register_operand" "=r")
3005 (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
3006 (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r")))
3007 (match_operand:DI 3 "register_operand" "r")))]
3009 "<su>maddl\\t%0, %w1, %w2, %3"
3010 [(set_attr "type" "<su>mlal")]
3013 (define_insn "<su_optab>msubsidi4"
3014 [(set (match_operand:DI 0 "register_operand" "=r")
3016 (match_operand:DI 3 "register_operand" "r")
3017 (mult:DI (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))
3019 (match_operand:SI 2 "register_operand" "r")))))]
3021 "<su>msubl\\t%0, %w1, %w2, %3"
3022 [(set_attr "type" "<su>mlal")]
3025 (define_insn "*<su_optab>mulsidi_neg"
3026 [(set (match_operand:DI 0 "register_operand" "=r")
3028 (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r")))
3029 (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r"))))]
3031 "<su>mnegl\\t%0, %w1, %w2"
3032 [(set_attr "type" "<su>mull")]
3035 (define_expand "<su_optab>mulditi3"
3036 [(set (match_operand:TI 0 "register_operand")
3037 (mult:TI (ANY_EXTEND:TI (match_operand:DI 1 "register_operand"))
3038 (ANY_EXTEND:TI (match_operand:DI 2 "register_operand"))))]
3041 rtx low = gen_reg_rtx (DImode);
3042 emit_insn (gen_muldi3 (low, operands[1], operands[2]));
3044 rtx high = gen_reg_rtx (DImode);
3045 emit_insn (gen_<su>muldi3_highpart (high, operands[1], operands[2]));
3047 emit_move_insn (gen_lowpart (DImode, operands[0]), low);
3048 emit_move_insn (gen_highpart (DImode, operands[0]), high);
3052 ;; The default expansion of multi3 using umuldi3_highpart will perform
3053 ;; the additions in an order that fails to combine into two madd insns.
3054 (define_expand "multi3"
3055 [(set (match_operand:TI 0 "register_operand")
3056 (mult:TI (match_operand:TI 1 "register_operand")
3057 (match_operand:TI 2 "register_operand")))]
3060 rtx l0 = gen_reg_rtx (DImode);
3061 rtx l1 = gen_lowpart (DImode, operands[1]);
3062 rtx l2 = gen_lowpart (DImode, operands[2]);
3063 rtx h0 = gen_reg_rtx (DImode);
3064 rtx h1 = gen_highpart (DImode, operands[1]);
3065 rtx h2 = gen_highpart (DImode, operands[2]);
3067 emit_insn (gen_muldi3 (l0, l1, l2));
3068 emit_insn (gen_umuldi3_highpart (h0, l1, l2));
3069 emit_insn (gen_madddi (h0, h1, l2, h0));
3070 emit_insn (gen_madddi (h0, l1, h2, h0));
3072 emit_move_insn (gen_lowpart (DImode, operands[0]), l0);
3073 emit_move_insn (gen_highpart (DImode, operands[0]), h0);
3077 (define_insn "<su>muldi3_highpart"
3078 [(set (match_operand:DI 0 "register_operand" "=r")
3082 (ANY_EXTEND:TI (match_operand:DI 1 "register_operand" "r"))
3083 (ANY_EXTEND:TI (match_operand:DI 2 "register_operand" "r")))
3086 "<su>mulh\\t%0, %1, %2"
3087 [(set_attr "type" "<su>mull")]
3090 (define_insn "<su_optab>div<mode>3"
3091 [(set (match_operand:GPI 0 "register_operand" "=r")
3092 (ANY_DIV:GPI (match_operand:GPI 1 "register_operand" "r")
3093 (match_operand:GPI 2 "register_operand" "r")))]
3095 "<su>div\\t%<w>0, %<w>1, %<w>2"
3096 [(set_attr "type" "<su>div")]
3099 ;; zero_extend version of above
3100 (define_insn "*<su_optab>divsi3_uxtw"
3101 [(set (match_operand:DI 0 "register_operand" "=r")
3103 (ANY_DIV:SI (match_operand:SI 1 "register_operand" "r")
3104 (match_operand:SI 2 "register_operand" "r"))))]
3106 "<su>div\\t%w0, %w1, %w2"
3107 [(set_attr "type" "<su>div")]
3110 ;; -------------------------------------------------------------------
3112 ;; -------------------------------------------------------------------
3114 (define_insn "cmp<mode>"
3115 [(set (reg:CC CC_REGNUM)
3116 (compare:CC (match_operand:GPI 0 "register_operand" "r,r,r")
3117 (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J")))]
3123 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")]
3126 (define_insn "fcmp<mode>"
3127 [(set (reg:CCFP CC_REGNUM)
3128 (compare:CCFP (match_operand:GPF 0 "register_operand" "w,w")
3129 (match_operand:GPF 1 "aarch64_fp_compare_operand" "Y,w")))]
3133 fcmp\\t%<s>0, %<s>1"
3134 [(set_attr "type" "fcmp<s>")]
3137 (define_insn "fcmpe<mode>"
3138 [(set (reg:CCFPE CC_REGNUM)
3139 (compare:CCFPE (match_operand:GPF 0 "register_operand" "w,w")
3140 (match_operand:GPF 1 "aarch64_fp_compare_operand" "Y,w")))]
3144 fcmpe\\t%<s>0, %<s>1"
3145 [(set_attr "type" "fcmp<s>")]
3148 (define_insn "*cmp_swp_<shift>_reg<mode>"
3149 [(set (reg:CC_SWP CC_REGNUM)
3150 (compare:CC_SWP (ASHIFT:GPI
3151 (match_operand:GPI 0 "register_operand" "r")
3152 (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n"))
3153 (match_operand:GPI 2 "aarch64_reg_or_zero" "rZ")))]
3155 "cmp\\t%<w>2, %<w>0, <shift> %1"
3156 [(set_attr "type" "alus_shift_imm")]
3159 (define_insn "*cmp_swp_<optab><ALLX:mode>_reg<GPI:mode>"
3160 [(set (reg:CC_SWP CC_REGNUM)
3161 (compare:CC_SWP (ANY_EXTEND:GPI
3162 (match_operand:ALLX 0 "register_operand" "r"))
3163 (match_operand:GPI 1 "register_operand" "r")))]
3165 "cmp\\t%<GPI:w>1, %<GPI:w>0, <su>xt<ALLX:size>"
3166 [(set_attr "type" "alus_ext")]
3169 (define_insn "*cmp_swp_<optab><ALLX:mode>_shft_<GPI:mode>"
3170 [(set (reg:CC_SWP CC_REGNUM)
3171 (compare:CC_SWP (ashift:GPI
3173 (match_operand:ALLX 0 "register_operand" "r"))
3174 (match_operand 1 "aarch64_imm3" "Ui3"))
3175 (match_operand:GPI 2 "register_operand" "r")))]
3177 "cmp\\t%<GPI:w>2, %<GPI:w>0, <su>xt<ALLX:size> %1"
3178 [(set_attr "type" "alus_ext")]
3181 ;; -------------------------------------------------------------------
3182 ;; Store-flag and conditional select insns
3183 ;; -------------------------------------------------------------------
3185 (define_expand "cstore<mode>4"
3186 [(set (match_operand:SI 0 "register_operand" "")
3187 (match_operator:SI 1 "aarch64_comparison_operator"
3188 [(match_operand:GPI 2 "register_operand" "")
3189 (match_operand:GPI 3 "aarch64_plus_operand" "")]))]
3192 operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
3194 operands[3] = const0_rtx;
3198 (define_expand "cstorecc4"
3199 [(set (match_operand:SI 0 "register_operand")
3200 (match_operator 1 "aarch64_comparison_operator_mode"
3201 [(match_operand 2 "cc_register")
3202 (match_operand 3 "const0_operand")]))]
3205 emit_insn (gen_rtx_SET (operands[0], operands[1]));
3210 (define_expand "cstore<mode>4"
3211 [(set (match_operand:SI 0 "register_operand" "")
3212 (match_operator:SI 1 "aarch64_comparison_operator_mode"
3213 [(match_operand:GPF 2 "register_operand" "")
3214 (match_operand:GPF 3 "aarch64_fp_compare_operand" "")]))]
3217 operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
3219 operands[3] = const0_rtx;
3223 (define_insn "aarch64_cstore<mode>"
3224 [(set (match_operand:ALLI 0 "register_operand" "=r")
3225 (match_operator:ALLI 1 "aarch64_comparison_operator_mode"
3226 [(match_operand 2 "cc_register" "") (const_int 0)]))]
3229 [(set_attr "type" "csel")]
3232 ;; For a 24-bit immediate CST we can optimize the compare for equality
3233 ;; and branch sequence from:
3235 ;; movk x0, #imm2, lsl 16 /* x0 contains CST. */
3238 ;; into the shorter:
3239 ;; sub x0, x1, #(CST & 0xfff000)
3240 ;; subs x0, x0, #(CST & 0x000fff)
3241 ;; cset x2, <ne, eq>.
3242 (define_insn_and_split "*compare_cstore<mode>_insn"
3243 [(set (match_operand:GPI 0 "register_operand" "=r")
3244 (EQL:GPI (match_operand:GPI 1 "register_operand" "r")
3245 (match_operand:GPI 2 "aarch64_imm24" "n")))]
3246 "!aarch64_move_imm (INTVAL (operands[2]), <MODE>mode)
3247 && !aarch64_plus_operand (operands[2], <MODE>mode)
3248 && !reload_completed"
3253 HOST_WIDE_INT lo_imm = UINTVAL (operands[2]) & 0xfff;
3254 HOST_WIDE_INT hi_imm = UINTVAL (operands[2]) & 0xfff000;
3255 rtx tmp = gen_reg_rtx (<MODE>mode);
3256 emit_insn (gen_add<mode>3 (tmp, operands[1], GEN_INT (-hi_imm)));
3257 emit_insn (gen_add<mode>3_compare0 (tmp, tmp, GEN_INT (-lo_imm)));
3258 rtx cc_reg = gen_rtx_REG (CC_NZmode, CC_REGNUM);
3259 rtx cmp_rtx = gen_rtx_fmt_ee (<EQL:CMP>, <MODE>mode, cc_reg, const0_rtx);
3260 emit_insn (gen_aarch64_cstore<mode> (operands[0], cmp_rtx, cc_reg));
3263 [(set_attr "type" "csel")]
3266 ;; zero_extend version of the above
3267 (define_insn "*cstoresi_insn_uxtw"
3268 [(set (match_operand:DI 0 "register_operand" "=r")
3270 (match_operator:SI 1 "aarch64_comparison_operator_mode"
3271 [(match_operand 2 "cc_register" "") (const_int 0)])))]
3274 [(set_attr "type" "csel")]
3277 (define_insn "cstore<mode>_neg"
3278 [(set (match_operand:ALLI 0 "register_operand" "=r")
3279 (neg:ALLI (match_operator:ALLI 1 "aarch64_comparison_operator_mode"
3280 [(match_operand 2 "cc_register" "") (const_int 0)])))]
3282 "csetm\\t%<w>0, %m1"
3283 [(set_attr "type" "csel")]
3286 ;; zero_extend version of the above
3287 (define_insn "*cstoresi_neg_uxtw"
3288 [(set (match_operand:DI 0 "register_operand" "=r")
3290 (neg:SI (match_operator:SI 1 "aarch64_comparison_operator_mode"
3291 [(match_operand 2 "cc_register" "") (const_int 0)]))))]
3294 [(set_attr "type" "csel")]
3297 (define_expand "cmov<mode>6"
3298 [(set (match_operand:GPI 0 "register_operand" "")
3300 (match_operator 1 "aarch64_comparison_operator"
3301 [(match_operand:GPI 2 "register_operand" "")
3302 (match_operand:GPI 3 "aarch64_plus_operand" "")])
3303 (match_operand:GPI 4 "register_operand" "")
3304 (match_operand:GPI 5 "register_operand" "")))]
3307 operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
3309 operands[3] = const0_rtx;
3313 (define_expand "cmov<mode>6"
3314 [(set (match_operand:GPF 0 "register_operand" "")
3316 (match_operator 1 "aarch64_comparison_operator"
3317 [(match_operand:GPF 2 "register_operand" "")
3318 (match_operand:GPF 3 "aarch64_fp_compare_operand" "")])
3319 (match_operand:GPF 4 "register_operand" "")
3320 (match_operand:GPF 5 "register_operand" "")))]
3323 operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2],
3325 operands[3] = const0_rtx;
3329 (define_insn "*cmov<mode>_insn"
3330 [(set (match_operand:ALLI 0 "register_operand" "=r,r,r,r,r,r,r")
3332 (match_operator 1 "aarch64_comparison_operator"
3333 [(match_operand 2 "cc_register" "") (const_int 0)])
3334 (match_operand:ALLI 3 "aarch64_reg_zero_or_m1_or_1" "rZ,rZ,UsM,rZ,Ui1,UsM,Ui1")
3335 (match_operand:ALLI 4 "aarch64_reg_zero_or_m1_or_1" "rZ,UsM,rZ,Ui1,rZ,UsM,Ui1")))]
3336 "!((operands[3] == const1_rtx && operands[4] == constm1_rtx)
3337 || (operands[3] == constm1_rtx && operands[4] == const1_rtx))"
3338 ;; Final two alternatives should be unreachable, but included for completeness
3340 csel\\t%<w>0, %<w>3, %<w>4, %m1
3341 csinv\\t%<w>0, %<w>3, <w>zr, %m1
3342 csinv\\t%<w>0, %<w>4, <w>zr, %M1
3343 csinc\\t%<w>0, %<w>3, <w>zr, %m1
3344 csinc\\t%<w>0, %<w>4, <w>zr, %M1
3347 [(set_attr "type" "csel, csel, csel, csel, csel, mov_imm, mov_imm")]
3350 ;; zero_extend version of above
3351 (define_insn "*cmovsi_insn_uxtw"
3352 [(set (match_operand:DI 0 "register_operand" "=r,r,r,r,r,r,r")
3355 (match_operator 1 "aarch64_comparison_operator"
3356 [(match_operand 2 "cc_register" "") (const_int 0)])
3357 (match_operand:SI 3 "aarch64_reg_zero_or_m1_or_1" "rZ,rZ,UsM,rZ,Ui1,UsM,Ui1")
3358 (match_operand:SI 4 "aarch64_reg_zero_or_m1_or_1" "rZ,UsM,rZ,Ui1,rZ,UsM,Ui1"))))]
3359 "!((operands[3] == const1_rtx && operands[4] == constm1_rtx)
3360 || (operands[3] == constm1_rtx && operands[4] == const1_rtx))"
3361 ;; Final two alternatives should be unreachable, but included for completeness
3363 csel\\t%w0, %w3, %w4, %m1
3364 csinv\\t%w0, %w3, wzr, %m1
3365 csinv\\t%w0, %w4, wzr, %M1
3366 csinc\\t%w0, %w3, wzr, %m1
3367 csinc\\t%w0, %w4, wzr, %M1
3370 [(set_attr "type" "csel, csel, csel, csel, csel, mov_imm, mov_imm")]
3373 (define_insn "*cmovdi_insn_uxtw"
3374 [(set (match_operand:DI 0 "register_operand" "=r")
3376 (match_operator 1 "aarch64_comparison_operator"
3377 [(match_operand 2 "cc_register" "") (const_int 0)])
3378 (zero_extend:DI (match_operand:SI 3 "register_operand" "r"))
3379 (zero_extend:DI (match_operand:SI 4 "register_operand" "r"))))]
3381 "csel\\t%w0, %w3, %w4, %m1"
3382 [(set_attr "type" "csel")]
3385 (define_insn "*cmov<mode>_insn"
3386 [(set (match_operand:GPF 0 "register_operand" "=w")
3388 (match_operator 1 "aarch64_comparison_operator"
3389 [(match_operand 2 "cc_register" "") (const_int 0)])
3390 (match_operand:GPF 3 "register_operand" "w")
3391 (match_operand:GPF 4 "register_operand" "w")))]
3393 "fcsel\\t%<s>0, %<s>3, %<s>4, %m1"
3394 [(set_attr "type" "fcsel")]
3397 (define_expand "mov<mode>cc"
3398 [(set (match_operand:ALLI 0 "register_operand" "")
3399 (if_then_else:ALLI (match_operand 1 "aarch64_comparison_operator" "")
3400 (match_operand:ALLI 2 "register_operand" "")
3401 (match_operand:ALLI 3 "register_operand" "")))]
3405 enum rtx_code code = GET_CODE (operands[1]);
3407 if (code == UNEQ || code == LTGT)
3410 ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
3411 XEXP (operands[1], 1));
3412 operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
3416 (define_expand "mov<GPF:mode><GPI:mode>cc"
3417 [(set (match_operand:GPI 0 "register_operand" "")
3418 (if_then_else:GPI (match_operand 1 "aarch64_comparison_operator" "")
3419 (match_operand:GPF 2 "register_operand" "")
3420 (match_operand:GPF 3 "register_operand" "")))]
3424 enum rtx_code code = GET_CODE (operands[1]);
3426 if (code == UNEQ || code == LTGT)
3429 ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
3430 XEXP (operands[1], 1));
3431 operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
3435 (define_expand "mov<mode>cc"
3436 [(set (match_operand:GPF 0 "register_operand" "")
3437 (if_then_else:GPF (match_operand 1 "aarch64_comparison_operator" "")
3438 (match_operand:GPF 2 "register_operand" "")
3439 (match_operand:GPF 3 "register_operand" "")))]
3443 enum rtx_code code = GET_CODE (operands[1]);
3445 if (code == UNEQ || code == LTGT)
3448 ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
3449 XEXP (operands[1], 1));
3450 operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
3454 (define_expand "<neg_not_op><mode>cc"
3455 [(set (match_operand:GPI 0 "register_operand" "")
3456 (if_then_else:GPI (match_operand 1 "aarch64_comparison_operator" "")
3457 (NEG_NOT:GPI (match_operand:GPI 2 "register_operand" ""))
3458 (match_operand:GPI 3 "register_operand" "")))]
3462 enum rtx_code code = GET_CODE (operands[1]);
3464 if (code == UNEQ || code == LTGT)
3467 ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0),
3468 XEXP (operands[1], 1));
3469 operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx);
3473 ;; CRC32 instructions.
3474 (define_insn "aarch64_<crc_variant>"
3475 [(set (match_operand:SI 0 "register_operand" "=r")
3476 (unspec:SI [(match_operand:SI 1 "register_operand" "r")
3477 (match_operand:<crc_mode> 2 "register_operand" "r")]
3481 if (GET_MODE_BITSIZE (<crc_mode>mode) >= 64)
3482 return "<crc_variant>\\t%w0, %w1, %x2";
3484 return "<crc_variant>\\t%w0, %w1, %w2";
3486 [(set_attr "type" "crc")]
3489 (define_insn "*csinc2<mode>_insn"
3490 [(set (match_operand:GPI 0 "register_operand" "=r")
3491 (plus:GPI (match_operand 2 "aarch64_comparison_operation" "")
3492 (match_operand:GPI 1 "register_operand" "r")))]
3494 "cinc\\t%<w>0, %<w>1, %m2"
3495 [(set_attr "type" "csel")]
3498 (define_insn "csinc3<mode>_insn"
3499 [(set (match_operand:GPI 0 "register_operand" "=r")
3501 (match_operand 1 "aarch64_comparison_operation" "")
3502 (plus:GPI (match_operand:GPI 2 "register_operand" "r")
3504 (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
3506 "csinc\\t%<w>0, %<w>3, %<w>2, %M1"
3507 [(set_attr "type" "csel")]
3510 (define_insn "*csinv3<mode>_insn"
3511 [(set (match_operand:GPI 0 "register_operand" "=r")
3513 (match_operand 1 "aarch64_comparison_operation" "")
3514 (not:GPI (match_operand:GPI 2 "register_operand" "r"))
3515 (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
3517 "csinv\\t%<w>0, %<w>3, %<w>2, %M1"
3518 [(set_attr "type" "csel")]
3521 (define_insn "csneg3_uxtw_insn"
3522 [(set (match_operand:DI 0 "register_operand" "=r")
3525 (match_operand 1 "aarch64_comparison_operation" "")
3526 (neg:SI (match_operand:SI 2 "register_operand" "r"))
3527 (match_operand:SI 3 "aarch64_reg_or_zero" "rZ"))))]
3529 "csneg\\t%w0, %w3, %w2, %M1"
3530 [(set_attr "type" "csel")]
3533 (define_insn "csneg3<mode>_insn"
3534 [(set (match_operand:GPI 0 "register_operand" "=r")
3536 (match_operand 1 "aarch64_comparison_operation" "")
3537 (neg:GPI (match_operand:GPI 2 "register_operand" "r"))
3538 (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))]
3540 "csneg\\t%<w>0, %<w>3, %<w>2, %M1"
3541 [(set_attr "type" "csel")]
3544 ;; If X can be loaded by a single CNT[BHWD] instruction,
3548 ;; is equivalent to:
3550 ;; TMP = UQDEC[BHWD] (B, X)
3553 ;; Defining the pattern this way means that:
3555 ;; A = UMAX (B, X) - X
3559 ;; TMP1 = UQDEC[BHWD] (B, X)
3563 ;; which combine can optimize to:
3565 ;; A = UQDEC[BHWD] (B, X)
3567 ;; We don't use match_operand predicates because the order of the operands
3568 ;; can vary: the CNT[BHWD] constant will come first if the other operand is
3569 ;; a simpler constant (such as a CONST_INT), otherwise it will come second.
3570 (define_expand "umax<mode>3"
3571 [(set (match_operand:GPI 0 "register_operand")
3572 (umax:GPI (match_operand:GPI 1 "")
3573 (match_operand:GPI 2 "")))]
3576 if (aarch64_sve_cnt_immediate (operands[1], <MODE>mode))
3577 std::swap (operands[1], operands[2]);
3578 else if (!aarch64_sve_cnt_immediate (operands[2], <MODE>mode))
3580 rtx temp = gen_reg_rtx (<MODE>mode);
3581 operands[1] = force_reg (<MODE>mode, operands[1]);
3582 emit_insn (gen_aarch64_uqdec<mode> (temp, operands[1], operands[2]));
3583 emit_insn (gen_add<mode>3 (operands[0], temp, operands[2]));
3588 ;; Saturating unsigned subtraction of a CNT[BHWD] immediate.
3589 (define_insn "aarch64_uqdec<mode>"
3590 [(set (match_operand:GPI 0 "register_operand" "=r")
3592 (umax:GPI (match_operand:GPI 1 "register_operand" "0")
3593 (match_operand:GPI 2 "aarch64_sve_cnt_immediate" "Usv"))
3597 return aarch64_output_sve_cnt_immediate ("uqdec", "%<w>0", operands[2]);
3601 ;; -------------------------------------------------------------------
3602 ;; Logical operations
3603 ;; -------------------------------------------------------------------
3606 (define_insn_and_split "*aarch64_and<mode>_imm2"
3607 [(set (match_operand:GPI 0 "register_operand" "=rk")
3608 (and:GPI (match_operand:GPI 1 "register_operand" "%r")
3609 (match_operand:GPI 2 "aarch64_logical_and_immediate" "<lconst2>")))]
3615 HOST_WIDE_INT val = INTVAL (operands[2]);
3616 rtx imm1 = GEN_INT (aarch64_and_split_imm1 (val));
3617 rtx imm2 = GEN_INT (aarch64_and_split_imm2 (val));
3619 emit_insn (gen_and<mode>3 (operands[0], operands[1], imm1));
3620 emit_insn (gen_and<mode>3 (operands[0], operands[0], imm2));
3625 (define_insn "<optab><mode>3"
3626 [(set (match_operand:GPI 0 "register_operand" "=r,rk,w")
3627 (LOGICAL:GPI (match_operand:GPI 1 "register_operand" "%r,r,w")
3628 (match_operand:GPI 2 "aarch64_logical_operand" "r,<lconst>,w")))]
3631 <logical>\\t%<w>0, %<w>1, %<w>2
3632 <logical>\\t%<w>0, %<w>1, %2
3633 <logical>\\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>"
3634 [(set_attr "type" "logic_reg,logic_imm,neon_logic")
3635 (set_attr "simd" "*,*,yes")]
3638 ;; zero_extend version of above
3639 (define_insn "*<optab>si3_uxtw"
3640 [(set (match_operand:DI 0 "register_operand" "=r,rk")
3642 (LOGICAL:SI (match_operand:SI 1 "register_operand" "%r,r")
3643 (match_operand:SI 2 "aarch64_logical_operand" "r,K"))))]
3646 <logical>\\t%w0, %w1, %w2
3647 <logical>\\t%w0, %w1, %2"
3648 [(set_attr "type" "logic_reg,logic_imm")]
3651 (define_insn "*and<mode>3_compare0"
3652 [(set (reg:CC_NZ CC_REGNUM)
3654 (and:GPI (match_operand:GPI 1 "register_operand" "%r,r")
3655 (match_operand:GPI 2 "aarch64_logical_operand" "r,<lconst>"))
3657 (set (match_operand:GPI 0 "register_operand" "=r,r")
3658 (and:GPI (match_dup 1) (match_dup 2)))]
3661 ands\\t%<w>0, %<w>1, %<w>2
3662 ands\\t%<w>0, %<w>1, %2"
3663 [(set_attr "type" "logics_reg,logics_imm")]
3666 ;; zero_extend version of above
3667 (define_insn "*andsi3_compare0_uxtw"
3668 [(set (reg:CC_NZ CC_REGNUM)
3670 (and:SI (match_operand:SI 1 "register_operand" "%r,r")
3671 (match_operand:SI 2 "aarch64_logical_operand" "r,K"))
3673 (set (match_operand:DI 0 "register_operand" "=r,r")
3674 (zero_extend:DI (and:SI (match_dup 1) (match_dup 2))))]
3677 ands\\t%w0, %w1, %w2
3678 ands\\t%w0, %w1, %2"
3679 [(set_attr "type" "logics_reg,logics_imm")]
3682 (define_insn "*and_<SHIFT:optab><mode>3_compare0"
3683 [(set (reg:CC_NZ CC_REGNUM)
3686 (match_operand:GPI 1 "register_operand" "r")
3687 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
3688 (match_operand:GPI 3 "register_operand" "r"))
3690 (set (match_operand:GPI 0 "register_operand" "=r")
3691 (and:GPI (SHIFT:GPI (match_dup 1) (match_dup 2)) (match_dup 3)))]
3693 "ands\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
3694 [(set_attr "type" "logics_shift_imm")]
3697 ;; zero_extend version of above
3698 (define_insn "*and_<SHIFT:optab>si3_compare0_uxtw"
3699 [(set (reg:CC_NZ CC_REGNUM)
3702 (match_operand:SI 1 "register_operand" "r")
3703 (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
3704 (match_operand:SI 3 "register_operand" "r"))
3706 (set (match_operand:DI 0 "register_operand" "=r")
3707 (zero_extend:DI (and:SI (SHIFT:SI (match_dup 1) (match_dup 2))
3710 "ands\\t%w0, %w3, %w1, <SHIFT:shift> %2"
3711 [(set_attr "type" "logics_shift_imm")]
3714 (define_insn "*<LOGICAL:optab>_<SHIFT:optab><mode>3"
3715 [(set (match_operand:GPI 0 "register_operand" "=r")
3716 (LOGICAL:GPI (SHIFT:GPI
3717 (match_operand:GPI 1 "register_operand" "r")
3718 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
3719 (match_operand:GPI 3 "register_operand" "r")))]
3721 "<LOGICAL:logical>\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
3722 [(set_attr "type" "logic_shift_imm")]
3725 (define_insn "*<optab>_rol<mode>3"
3726 [(set (match_operand:GPI 0 "register_operand" "=r")
3727 (LOGICAL:GPI (rotate:GPI
3728 (match_operand:GPI 1 "register_operand" "r")
3729 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
3730 (match_operand:GPI 3 "register_operand" "r")))]
3732 "<logical>\\t%<w>0, %<w>3, %<w>1, ror (<sizen> - %2)"
3733 [(set_attr "type" "logic_shift_imm")]
3736 ;; zero_extend versions of above
3737 (define_insn "*<LOGICAL:optab>_<SHIFT:optab>si3_uxtw"
3738 [(set (match_operand:DI 0 "register_operand" "=r")
3740 (LOGICAL:SI (SHIFT:SI
3741 (match_operand:SI 1 "register_operand" "r")
3742 (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
3743 (match_operand:SI 3 "register_operand" "r"))))]
3745 "<LOGICAL:logical>\\t%w0, %w3, %w1, <SHIFT:shift> %2"
3746 [(set_attr "type" "logic_shift_imm")]
3749 (define_insn "*<optab>_rolsi3_uxtw"
3750 [(set (match_operand:DI 0 "register_operand" "=r")
3752 (LOGICAL:SI (rotate:SI
3753 (match_operand:SI 1 "register_operand" "r")
3754 (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
3755 (match_operand:SI 3 "register_operand" "r"))))]
3757 "<logical>\\t%w0, %w3, %w1, ror (32 - %2)"
3758 [(set_attr "type" "logic_shift_imm")]
3761 (define_insn "one_cmpl<mode>2"
3762 [(set (match_operand:GPI 0 "register_operand" "=r,w")
3763 (not:GPI (match_operand:GPI 1 "register_operand" "r,w")))]
3768 [(set_attr "type" "logic_reg,neon_logic")
3769 (set_attr "simd" "*,yes")]
3772 (define_insn "*one_cmpl_<optab><mode>2"
3773 [(set (match_operand:GPI 0 "register_operand" "=r")
3774 (not:GPI (SHIFT:GPI (match_operand:GPI 1 "register_operand" "r")
3775 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))]
3777 "mvn\\t%<w>0, %<w>1, <shift> %2"
3778 [(set_attr "type" "logic_shift_imm")]
3781 ;; Binary logical operators negating one operand, i.e. (a & !b), (a | !b).
3783 (define_insn "*<NLOGICAL:optab>_one_cmpl<mode>3"
3784 [(set (match_operand:GPI 0 "register_operand" "=r,w")
3785 (NLOGICAL:GPI (not:GPI (match_operand:GPI 1 "register_operand" "r,w"))
3786 (match_operand:GPI 2 "register_operand" "r,w")))]
3789 <NLOGICAL:nlogical>\\t%<w>0, %<w>2, %<w>1
3790 <NLOGICAL:nlogical>\\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>"
3791 [(set_attr "type" "logic_reg,neon_logic")
3792 (set_attr "simd" "*,yes")]
3795 (define_insn "*<NLOGICAL:optab>_one_cmplsidi3_ze"
3796 [(set (match_operand:DI 0 "register_operand" "=r")
3798 (NLOGICAL:SI (not:SI (match_operand:SI 1 "register_operand" "r"))
3799 (match_operand:SI 2 "register_operand" "r"))))]
3801 "<NLOGICAL:nlogical>\\t%w0, %w2, %w1"
3802 [(set_attr "type" "logic_reg")]
3805 (define_insn "*xor_one_cmplsidi3_ze"
3806 [(set (match_operand:DI 0 "register_operand" "=r")
3808 (not:SI (xor:SI (match_operand:SI 1 "register_operand" "r")
3809 (match_operand:SI 2 "register_operand" "r")))))]
3811 "eon\\t%w0, %w1, %w2"
3812 [(set_attr "type" "logic_reg")]
3815 ;; (xor (not a) b) is simplify_rtx-ed down to (not (xor a b)).
3816 ;; eon does not operate on SIMD registers so the vector variant must be split.
3817 (define_insn_and_split "*xor_one_cmpl<mode>3"
3818 [(set (match_operand:GPI 0 "register_operand" "=r,w")
3819 (not:GPI (xor:GPI (match_operand:GPI 1 "register_operand" "r,?w")
3820 (match_operand:GPI 2 "register_operand" "r,w"))))]
3823 eon\\t%<w>0, %<w>1, %<w>2
3825 "reload_completed && FP_REGNUM_P (REGNO (operands[0]))" ;; For SIMD registers.
3826 [(set (match_operand:GPI 0 "register_operand" "=w")
3827 (xor:GPI (match_operand:GPI 1 "register_operand" "w")
3828 (match_operand:GPI 2 "register_operand" "w")))
3829 (set (match_dup 0) (not:GPI (match_dup 0)))]
3831 [(set_attr "type" "logic_reg,multiple")
3832 (set_attr "simd" "*,yes")]
3835 (define_insn "*and_one_cmpl<mode>3_compare0"
3836 [(set (reg:CC_NZ CC_REGNUM)
3839 (match_operand:GPI 1 "register_operand" "r"))
3840 (match_operand:GPI 2 "register_operand" "r"))
3842 (set (match_operand:GPI 0 "register_operand" "=r")
3843 (and:GPI (not:GPI (match_dup 1)) (match_dup 2)))]
3845 "bics\\t%<w>0, %<w>2, %<w>1"
3846 [(set_attr "type" "logics_reg")]
3849 ;; zero_extend version of above
3850 (define_insn "*and_one_cmplsi3_compare0_uxtw"
3851 [(set (reg:CC_NZ CC_REGNUM)
3854 (match_operand:SI 1 "register_operand" "r"))
3855 (match_operand:SI 2 "register_operand" "r"))
3857 (set (match_operand:DI 0 "register_operand" "=r")
3858 (zero_extend:DI (and:SI (not:SI (match_dup 1)) (match_dup 2))))]
3860 "bics\\t%w0, %w2, %w1"
3861 [(set_attr "type" "logics_reg")]
3864 (define_insn "*and_one_cmpl<mode>3_compare0_no_reuse"
3865 [(set (reg:CC_NZ CC_REGNUM)
3868 (match_operand:GPI 0 "register_operand" "r"))
3869 (match_operand:GPI 1 "register_operand" "r"))
3872 "bics\\t<w>zr, %<w>1, %<w>0"
3873 [(set_attr "type" "logics_reg")]
3876 (define_insn "<LOGICAL:optab>_one_cmpl_<SHIFT:optab><mode>3"
3877 [(set (match_operand:GPI 0 "register_operand" "=r")
3878 (LOGICAL:GPI (not:GPI
3880 (match_operand:GPI 1 "register_operand" "r")
3881 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
3882 (match_operand:GPI 3 "register_operand" "r")))]
3884 "<LOGICAL:nlogical>\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
3885 [(set_attr "type" "logic_shift_imm")]
3888 (define_insn "*eor_one_cmpl_<SHIFT:optab><mode>3_alt"
3889 [(set (match_operand:GPI 0 "register_operand" "=r")
3892 (match_operand:GPI 1 "register_operand" "r")
3893 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))
3894 (match_operand:GPI 3 "register_operand" "r"))))]
3896 "eon\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
3897 [(set_attr "type" "logic_shift_imm")]
3900 ;; Zero-extend version of the above.
3901 (define_insn "*eor_one_cmpl_<SHIFT:optab>sidi3_alt_ze"
3902 [(set (match_operand:DI 0 "register_operand" "=r")
3906 (match_operand:SI 1 "register_operand" "r")
3907 (match_operand:QI 2 "aarch64_shift_imm_si" "n"))
3908 (match_operand:SI 3 "register_operand" "r")))))]
3910 "eon\\t%w0, %w3, %w1, <SHIFT:shift> %2"
3911 [(set_attr "type" "logic_shift_imm")]
3914 (define_insn "*and_one_cmpl_<SHIFT:optab><mode>3_compare0"
3915 [(set (reg:CC_NZ CC_REGNUM)
3919 (match_operand:GPI 1 "register_operand" "r")
3920 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")))
3921 (match_operand:GPI 3 "register_operand" "r"))
3923 (set (match_operand:GPI 0 "register_operand" "=r")
3926 (match_dup 1) (match_dup 2))) (match_dup 3)))]
3928 "bics\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2"
3929 [(set_attr "type" "logics_shift_imm")]
3932 ;; zero_extend version of above
3933 (define_insn "*and_one_cmpl_<SHIFT:optab>si3_compare0_uxtw"
3934 [(set (reg:CC_NZ CC_REGNUM)
3938 (match_operand:SI 1 "register_operand" "r")
3939 (match_operand:QI 2 "aarch64_shift_imm_si" "n")))
3940 (match_operand:SI 3 "register_operand" "r"))
3942 (set (match_operand:DI 0 "register_operand" "=r")
3943 (zero_extend:DI (and:SI
3945 (SHIFT:SI (match_dup 1) (match_dup 2))) (match_dup 3))))]
3947 "bics\\t%w0, %w3, %w1, <SHIFT:shift> %2"
3948 [(set_attr "type" "logics_shift_imm")]
3951 (define_insn "*and_one_cmpl_<SHIFT:optab><mode>3_compare0_no_reuse"
3952 [(set (reg:CC_NZ CC_REGNUM)
3956 (match_operand:GPI 0 "register_operand" "r")
3957 (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n")))
3958 (match_operand:GPI 2 "register_operand" "r"))
3961 "bics\\t<w>zr, %<w>2, %<w>0, <SHIFT:shift> %1"
3962 [(set_attr "type" "logics_shift_imm")]
3965 (define_insn "clz<mode>2"
3966 [(set (match_operand:GPI 0 "register_operand" "=r")
3967 (clz:GPI (match_operand:GPI 1 "register_operand" "r")))]
3969 "clz\\t%<w>0, %<w>1"
3970 [(set_attr "type" "clz")]
3973 (define_expand "ffs<mode>2"
3974 [(match_operand:GPI 0 "register_operand")
3975 (match_operand:GPI 1 "register_operand")]
3978 rtx ccreg = aarch64_gen_compare_reg (EQ, operands[1], const0_rtx);
3979 rtx x = gen_rtx_NE (VOIDmode, ccreg, const0_rtx);
3981 emit_insn (gen_rbit<mode>2 (operands[0], operands[1]));
3982 emit_insn (gen_clz<mode>2 (operands[0], operands[0]));
3983 emit_insn (gen_csinc3<mode>_insn (operands[0], x, operands[0], const0_rtx));
3988 ;; Pop count be done via the "CNT" instruction in AdvSIMD.
3995 (define_expand "popcount<mode>2"
3996 [(match_operand:GPI 0 "register_operand")
3997 (match_operand:GPI 1 "register_operand")]
4000 rtx v = gen_reg_rtx (V8QImode);
4001 rtx v1 = gen_reg_rtx (V8QImode);
4002 rtx r = gen_reg_rtx (QImode);
4003 rtx in = operands[1];
4004 rtx out = operands[0];
4005 if(<MODE>mode == SImode)
4008 tmp = gen_reg_rtx (DImode);
4009 /* If we have SImode, zero extend to DImode, pop count does
4010 not change if we have extra zeros. */
4011 emit_insn (gen_zero_extendsidi2 (tmp, in));
4014 emit_move_insn (v, gen_lowpart (V8QImode, in));
4015 emit_insn (gen_popcountv8qi2 (v1, v));
4016 emit_insn (gen_reduc_plus_scal_v8qi (r, v1));
4017 emit_insn (gen_zero_extendqi<mode>2 (out, r));
4021 (define_insn "clrsb<mode>2"
4022 [(set (match_operand:GPI 0 "register_operand" "=r")
4023 (clrsb:GPI (match_operand:GPI 1 "register_operand" "r")))]
4025 "cls\\t%<w>0, %<w>1"
4026 [(set_attr "type" "clz")]
4029 (define_insn "rbit<mode>2"
4030 [(set (match_operand:GPI 0 "register_operand" "=r")
4031 (unspec:GPI [(match_operand:GPI 1 "register_operand" "r")] UNSPEC_RBIT))]
4033 "rbit\\t%<w>0, %<w>1"
4034 [(set_attr "type" "rbit")]
4037 ;; Split after reload into RBIT + CLZ. Since RBIT is represented as an UNSPEC
4038 ;; it is unlikely to fold with any other operation, so keep this as a CTZ
4039 ;; expression and split after reload to enable scheduling them apart if
4042 (define_insn_and_split "ctz<mode>2"
4043 [(set (match_operand:GPI 0 "register_operand" "=r")
4044 (ctz:GPI (match_operand:GPI 1 "register_operand" "r")))]
4050 emit_insn (gen_rbit<mode>2 (operands[0], operands[1]));
4051 emit_insn (gen_clz<mode>2 (operands[0], operands[0]));
4055 (define_insn "*and<mode>_compare0"
4056 [(set (reg:CC_NZ CC_REGNUM)
4058 (match_operand:SHORT 0 "register_operand" "r")
4061 "tst\\t%<w>0, <short_mask>"
4062 [(set_attr "type" "alus_imm")]
4065 (define_insn "*ands<mode>_compare0"
4066 [(set (reg:CC_NZ CC_REGNUM)
4068 (zero_extend:GPI (match_operand:SHORT 1 "register_operand" "r"))
4070 (set (match_operand:GPI 0 "register_operand" "=r")
4071 (zero_extend:GPI (match_dup 1)))]
4073 "ands\\t%<GPI:w>0, %<GPI:w>1, <short_mask>"
4074 [(set_attr "type" "alus_imm")]
4077 (define_insn "*and<mode>3nr_compare0"
4078 [(set (reg:CC_NZ CC_REGNUM)
4080 (and:GPI (match_operand:GPI 0 "register_operand" "%r,r")
4081 (match_operand:GPI 1 "aarch64_logical_operand" "r,<lconst>"))
4087 [(set_attr "type" "logics_reg,logics_imm")]
4091 [(set (reg:CC_NZ CC_REGNUM)
4093 (and:GPI (match_operand:GPI 0 "register_operand")
4094 (match_operand:GPI 1 "aarch64_mov_imm_operand"))
4096 (clobber (match_operand:SI 2 "register_operand"))]
4098 [(set (match_dup 2) (match_dup 1))
4099 (set (reg:CC_NZ CC_REGNUM)
4101 (and:GPI (match_dup 0)
4106 (define_insn "*and<mode>3nr_compare0_zextract"
4107 [(set (reg:CC_NZ CC_REGNUM)
4109 (zero_extract:GPI (match_operand:GPI 0 "register_operand" "r")
4110 (match_operand:GPI 1 "const_int_operand" "n")
4111 (match_operand:GPI 2 "const_int_operand" "n"))
4113 "INTVAL (operands[1]) > 0
4114 && ((INTVAL (operands[1]) + INTVAL (operands[2]))
4115 <= GET_MODE_BITSIZE (<MODE>mode))
4116 && aarch64_bitmask_imm (
4117 UINTVAL (aarch64_mask_from_zextract_ops (operands[1],
4122 = aarch64_mask_from_zextract_ops (operands[1], operands[2]);
4123 return "tst\\t%<w>0, %1";
4125 [(set_attr "type" "logics_shift_imm")]
4128 (define_insn "*and_<SHIFT:optab><mode>3nr_compare0"
4129 [(set (reg:CC_NZ CC_REGNUM)
4132 (match_operand:GPI 0 "register_operand" "r")
4133 (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n"))
4134 (match_operand:GPI 2 "register_operand" "r"))
4137 "tst\\t%<w>2, %<w>0, <SHIFT:shift> %1"
4138 [(set_attr "type" "logics_shift_imm")]
4142 [(set (reg:CC_NZ CC_REGNUM)
4145 (match_operand:GPI 0 "register_operand")
4146 (match_operand:QI 1 "aarch64_shift_imm_<mode>"))
4147 (match_operand:GPI 2 "aarch64_mov_imm_operand"))
4149 (clobber (match_operand:SI 3 "register_operand"))]
4151 [(set (match_dup 3) (match_dup 2))
4152 (set (reg:CC_NZ CC_REGNUM)
4161 ;; -------------------------------------------------------------------
4163 ;; -------------------------------------------------------------------
4165 (define_expand "<optab><mode>3"
4166 [(set (match_operand:GPI 0 "register_operand")
4167 (ASHIFT:GPI (match_operand:GPI 1 "register_operand")
4168 (match_operand:QI 2 "aarch64_reg_or_imm")))]
4171 if (CONST_INT_P (operands[2]))
4173 operands[2] = GEN_INT (INTVAL (operands[2])
4174 & (GET_MODE_BITSIZE (<MODE>mode) - 1));
4176 if (operands[2] == const0_rtx)
4178 emit_insn (gen_mov<mode> (operands[0], operands[1]));
4185 (define_expand "ashl<mode>3"
4186 [(set (match_operand:SHORT 0 "register_operand")
4187 (ashift:SHORT (match_operand:SHORT 1 "register_operand")
4188 (match_operand:QI 2 "const_int_operand")))]
4191 operands[2] = GEN_INT (INTVAL (operands[2]) & GET_MODE_MASK (<MODE>mode));
4193 if (operands[2] == const0_rtx)
4195 emit_insn (gen_mov<mode> (operands[0], operands[1]));
4201 (define_expand "rotr<mode>3"
4202 [(set (match_operand:GPI 0 "register_operand")
4203 (rotatert:GPI (match_operand:GPI 1 "register_operand")
4204 (match_operand:QI 2 "aarch64_reg_or_imm")))]
4207 if (CONST_INT_P (operands[2]))
4209 operands[2] = GEN_INT (INTVAL (operands[2])
4210 & (GET_MODE_BITSIZE (<MODE>mode) - 1));
4212 if (operands[2] == const0_rtx)
4214 emit_insn (gen_mov<mode> (operands[0], operands[1]));
4221 (define_expand "rotl<mode>3"
4222 [(set (match_operand:GPI 0 "register_operand")
4223 (rotatert:GPI (match_operand:GPI 1 "register_operand")
4224 (match_operand:QI 2 "aarch64_reg_or_imm")))]
4227 /* (SZ - cnt) % SZ == -cnt % SZ */
4228 if (CONST_INT_P (operands[2]))
4230 operands[2] = GEN_INT ((-INTVAL (operands[2]))
4231 & (GET_MODE_BITSIZE (<MODE>mode) - 1));
4232 if (operands[2] == const0_rtx)
4234 emit_insn (gen_mov<mode> (operands[0], operands[1]));
4239 operands[2] = expand_simple_unop (QImode, NEG, operands[2],
4244 ;; When the LSL, LSR, ASR, ROR instructions operate on all register arguments
4245 ;; they truncate the shift/rotate amount by the size of the registers they
4246 ;; operate on: 32 for W-regs, 64 for X-regs. This allows us to optimise away
4247 ;; such redundant masking instructions. GCC can do that automatically when
4248 ;; SHIFT_COUNT_TRUNCATED is true, but we can't enable it for TARGET_SIMD
4249 ;; because some of the SISD shift alternatives don't perform this truncations.
4250 ;; So this pattern exists to catch such cases.
4252 (define_insn "*aarch64_<optab>_reg_<mode>3_mask1"
4253 [(set (match_operand:GPI 0 "register_operand" "=r")
4255 (match_operand:GPI 1 "register_operand" "r")
4256 (match_operator 4 "subreg_lowpart_operator"
4257 [(and:GPI (match_operand:GPI 2 "register_operand" "r")
4258 (match_operand 3 "const_int_operand" "n"))])))]
4259 "(~INTVAL (operands[3]) & (GET_MODE_BITSIZE (<MODE>mode) - 1)) == 0"
4260 "<shift>\t%<w>0, %<w>1, %<w>2"
4261 [(set_attr "type" "shift_reg")]
4264 (define_insn_and_split "*aarch64_reg_<mode>3_neg_mask2"
4265 [(set (match_operand:GPI 0 "register_operand" "=&r")
4267 (match_operand:GPI 1 "register_operand" "r")
4268 (match_operator 4 "subreg_lowpart_operator"
4269 [(neg:SI (and:SI (match_operand:SI 2 "register_operand" "r")
4270 (match_operand 3 "const_int_operand" "n")))])))]
4271 "((~INTVAL (operands[3]) & (GET_MODE_BITSIZE (<MODE>mode) - 1)) == 0)"
4276 rtx tmp = (can_create_pseudo_p () ? gen_reg_rtx (SImode)
4278 emit_insn (gen_negsi2 (tmp, operands[2]));
4280 rtx and_op = gen_rtx_AND (SImode, tmp, operands[3]);
4281 rtx subreg_tmp = gen_rtx_SUBREG (GET_MODE (operands[4]), and_op,
4282 SUBREG_BYTE (operands[4]));
4283 emit_insn (gen_<optab><mode>3 (operands[0], operands[1], subreg_tmp));
4288 (define_insn_and_split "*aarch64_reg_<mode>3_minus_mask"
4289 [(set (match_operand:GPI 0 "register_operand" "=&r")
4291 (match_operand:GPI 1 "register_operand" "r")
4292 (minus:QI (match_operand 2 "const_int_operand" "n")
4293 (match_operator 5 "subreg_lowpart_operator"
4294 [(and:SI (match_operand:SI 3 "register_operand" "r")
4295 (match_operand 4 "const_int_operand" "n"))]))))]
4296 "((~INTVAL (operands[4]) & (GET_MODE_BITSIZE (<MODE>mode) - 1)) == 0)
4297 && INTVAL (operands[2]) == GET_MODE_BITSIZE (<MODE>mode)"
4302 rtx tmp = (can_create_pseudo_p () ? gen_reg_rtx (SImode)
4305 emit_insn (gen_negsi2 (tmp, operands[3]));
4307 rtx and_op = gen_rtx_AND (SImode, tmp, operands[4]);
4308 rtx subreg_tmp = gen_rtx_SUBREG (GET_MODE (operands[5]), and_op,
4309 SUBREG_BYTE (operands[5]));
4311 emit_insn (gen_ashl<mode>3 (operands[0], operands[1], subreg_tmp));
4316 (define_insn "*aarch64_<optab>_reg_di3_mask2"
4317 [(set (match_operand:DI 0 "register_operand" "=r")
4319 (match_operand:DI 1 "register_operand" "r")
4320 (match_operator 4 "subreg_lowpart_operator"
4321 [(and:SI (match_operand:SI 2 "register_operand" "r")
4322 (match_operand 3 "aarch64_shift_imm_di" "Usd"))])))]
4323 "((~INTVAL (operands[3]) & (GET_MODE_BITSIZE (DImode)-1)) == 0)"
4326 xop[0] = operands[0];
4327 xop[1] = operands[1];
4328 xop[2] = gen_lowpart (GET_MODE (operands[4]), operands[2]);
4329 output_asm_insn ("<shift>\t%x0, %x1, %x2", xop);
4332 [(set_attr "type" "shift_reg")]
4335 (define_insn_and_split "*aarch64_reg_<optab>_minus<mode>3"
4336 [(set (match_operand:GPI 0 "register_operand" "=&r")
4338 (match_operand:GPI 1 "register_operand" "r")
4339 (minus:QI (match_operand 2 "const_int_operand" "n")
4340 (match_operand:QI 3 "register_operand" "r"))))]
4341 "INTVAL (operands[2]) == GET_MODE_BITSIZE (<MODE>mode)"
4346 rtx subreg_tmp = gen_lowpart (SImode, operands[3]);
4348 rtx tmp = (can_create_pseudo_p () ? gen_reg_rtx (SImode)
4349 : gen_lowpart (SImode, operands[0]));
4351 emit_insn (gen_negsi2 (tmp, subreg_tmp));
4353 rtx and_op = gen_rtx_AND (SImode, tmp,
4354 GEN_INT (GET_MODE_BITSIZE (<MODE>mode) - 1));
4356 rtx subreg_tmp2 = gen_lowpart_SUBREG (QImode, and_op);
4358 emit_insn (gen_<optab><mode>3 (operands[0], operands[1], subreg_tmp2));
4361 [(set_attr "length" "8")]
4364 ;; Logical left shift using SISD or Integer instruction
4365 (define_insn "*aarch64_ashl_sisd_or_int_<mode>3"
4366 [(set (match_operand:GPI 0 "register_operand" "=r,r,w,w")
4368 (match_operand:GPI 1 "register_operand" "r,r,w,w")
4369 (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>" "Us<cmode>,r,Us<cmode>,w")))]
4372 lsl\t%<w>0, %<w>1, %2
4373 lsl\t%<w>0, %<w>1, %<w>2
4374 shl\t%<rtn>0<vas>, %<rtn>1<vas>, %2
4375 ushl\t%<rtn>0<vas>, %<rtn>1<vas>, %<rtn>2<vas>"
4376 [(set_attr "simd" "no,no,yes,yes")
4377 (set_attr "type" "bfx,shift_reg,neon_shift_imm<q>, neon_shift_reg<q>")]
4380 ;; Logical right shift using SISD or Integer instruction
4381 (define_insn "*aarch64_lshr_sisd_or_int_<mode>3"
4382 [(set (match_operand:GPI 0 "register_operand" "=r,r,w,&w,&w")
4384 (match_operand:GPI 1 "register_operand" "r,r,w,w,w")
4385 (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>" "Us<cmode>,r,Us<cmode>,w,0")))]
4388 lsr\t%<w>0, %<w>1, %2
4389 lsr\t%<w>0, %<w>1, %<w>2
4390 ushr\t%<rtn>0<vas>, %<rtn>1<vas>, %2
4393 [(set_attr "simd" "no,no,yes,yes,yes")
4394 (set_attr "type" "bfx,shift_reg,neon_shift_imm<q>,neon_shift_reg<q>,neon_shift_reg<q>")]
4398 [(set (match_operand:DI 0 "aarch64_simd_register")
4400 (match_operand:DI 1 "aarch64_simd_register")
4401 (match_operand:QI 2 "aarch64_simd_register")))]
4402 "TARGET_SIMD && reload_completed"
4404 (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
4406 (unspec:DI [(match_dup 1) (match_dup 3)] UNSPEC_SISD_USHL))]
4408 operands[3] = gen_lowpart (QImode, operands[0]);
4413 [(set (match_operand:SI 0 "aarch64_simd_register")
4415 (match_operand:SI 1 "aarch64_simd_register")
4416 (match_operand:QI 2 "aarch64_simd_register")))]
4417 "TARGET_SIMD && reload_completed"
4419 (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
4421 (unspec:SI [(match_dup 1) (match_dup 3)] UNSPEC_USHL_2S))]
4423 operands[3] = gen_lowpart (QImode, operands[0]);
4427 ;; Arithmetic right shift using SISD or Integer instruction
4428 (define_insn "*aarch64_ashr_sisd_or_int_<mode>3"
4429 [(set (match_operand:GPI 0 "register_operand" "=r,r,w,&w,&w")
4431 (match_operand:GPI 1 "register_operand" "r,r,w,w,w")
4432 (match_operand:QI 2 "aarch64_reg_or_shift_imm_di" "Us<cmode>,r,Us<cmode>,w,0")))]
4435 asr\t%<w>0, %<w>1, %2
4436 asr\t%<w>0, %<w>1, %<w>2
4437 sshr\t%<rtn>0<vas>, %<rtn>1<vas>, %2
4440 [(set_attr "simd" "no,no,yes,yes,yes")
4441 (set_attr "type" "bfx,shift_reg,neon_shift_imm<q>,neon_shift_reg<q>,neon_shift_reg<q>")]
4445 [(set (match_operand:DI 0 "aarch64_simd_register")
4447 (match_operand:DI 1 "aarch64_simd_register")
4448 (match_operand:QI 2 "aarch64_simd_register")))]
4449 "TARGET_SIMD && reload_completed"
4451 (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
4453 (unspec:DI [(match_dup 1) (match_dup 3)] UNSPEC_SISD_SSHL))]
4455 operands[3] = gen_lowpart (QImode, operands[0]);
4460 [(set (match_operand:SI 0 "aarch64_simd_register")
4462 (match_operand:SI 1 "aarch64_simd_register")
4463 (match_operand:QI 2 "aarch64_simd_register")))]
4464 "TARGET_SIMD && reload_completed"
4466 (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG))
4468 (unspec:SI [(match_dup 1) (match_dup 3)] UNSPEC_SSHL_2S))]
4470 operands[3] = gen_lowpart (QImode, operands[0]);
4474 (define_insn "*aarch64_sisd_ushl"
4475 [(set (match_operand:DI 0 "register_operand" "=w")
4476 (unspec:DI [(match_operand:DI 1 "register_operand" "w")
4477 (match_operand:QI 2 "register_operand" "w")]
4480 "ushl\t%d0, %d1, %d2"
4481 [(set_attr "simd" "yes")
4482 (set_attr "type" "neon_shift_reg")]
4485 (define_insn "*aarch64_ushl_2s"
4486 [(set (match_operand:SI 0 "register_operand" "=w")
4487 (unspec:SI [(match_operand:SI 1 "register_operand" "w")
4488 (match_operand:QI 2 "register_operand" "w")]
4491 "ushl\t%0.2s, %1.2s, %2.2s"
4492 [(set_attr "simd" "yes")
4493 (set_attr "type" "neon_shift_reg")]
4496 (define_insn "*aarch64_sisd_sshl"
4497 [(set (match_operand:DI 0 "register_operand" "=w")
4498 (unspec:DI [(match_operand:DI 1 "register_operand" "w")
4499 (match_operand:QI 2 "register_operand" "w")]
4502 "sshl\t%d0, %d1, %d2"
4503 [(set_attr "simd" "yes")
4504 (set_attr "type" "neon_shift_reg")]
4507 (define_insn "*aarch64_sshl_2s"
4508 [(set (match_operand:SI 0 "register_operand" "=w")
4509 (unspec:SI [(match_operand:SI 1 "register_operand" "w")
4510 (match_operand:QI 2 "register_operand" "w")]
4513 "sshl\t%0.2s, %1.2s, %2.2s"
4514 [(set_attr "simd" "yes")
4515 (set_attr "type" "neon_shift_reg")]
4518 (define_insn "*aarch64_sisd_neg_qi"
4519 [(set (match_operand:QI 0 "register_operand" "=w")
4520 (unspec:QI [(match_operand:QI 1 "register_operand" "w")]
4524 [(set_attr "simd" "yes")
4525 (set_attr "type" "neon_neg")]
4529 (define_insn "*ror<mode>3_insn"
4530 [(set (match_operand:GPI 0 "register_operand" "=r,r")
4532 (match_operand:GPI 1 "register_operand" "r,r")
4533 (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>" "Us<cmode>,r")))]
4536 ror\\t%<w>0, %<w>1, %2
4537 ror\\t%<w>0, %<w>1, %<w>2"
4538 [(set_attr "type" "rotate_imm,shift_reg")]
4541 ;; zero_extend version of above
4542 (define_insn "*<optab>si3_insn_uxtw"
4543 [(set (match_operand:DI 0 "register_operand" "=r,r")
4544 (zero_extend:DI (SHIFT:SI
4545 (match_operand:SI 1 "register_operand" "r,r")
4546 (match_operand:QI 2 "aarch64_reg_or_shift_imm_si" "Uss,r"))))]
4549 <shift>\\t%w0, %w1, %2
4550 <shift>\\t%w0, %w1, %w2"
4551 [(set_attr "type" "bfx,shift_reg")]
4554 (define_insn "*<optab><mode>3_insn"
4555 [(set (match_operand:SHORT 0 "register_operand" "=r")
4556 (ASHIFT:SHORT (match_operand:SHORT 1 "register_operand" "r")
4557 (match_operand 2 "const_int_operand" "n")))]
4558 "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<MODE>mode)"
4560 operands[3] = GEN_INT (<sizen> - UINTVAL (operands[2]));
4561 return "<bfshift>\t%w0, %w1, %2, %3";
4563 [(set_attr "type" "bfx")]
4566 (define_insn "*extr<mode>5_insn"
4567 [(set (match_operand:GPI 0 "register_operand" "=r")
4568 (ior:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
4569 (match_operand 3 "const_int_operand" "n"))
4570 (lshiftrt:GPI (match_operand:GPI 2 "register_operand" "r")
4571 (match_operand 4 "const_int_operand" "n"))))]
4572 "UINTVAL (operands[3]) < GET_MODE_BITSIZE (<MODE>mode) &&
4573 (UINTVAL (operands[3]) + UINTVAL (operands[4]) == GET_MODE_BITSIZE (<MODE>mode))"
4574 "extr\\t%<w>0, %<w>1, %<w>2, %4"
4575 [(set_attr "type" "rotate_imm")]
4578 ;; There are no canonicalisation rules for ashift and lshiftrt inside an ior
4579 ;; so we have to match both orderings.
4580 (define_insn "*extr<mode>5_insn_alt"
4581 [(set (match_operand:GPI 0 "register_operand" "=r")
4582 (ior:GPI (lshiftrt:GPI (match_operand:GPI 2 "register_operand" "r")
4583 (match_operand 4 "const_int_operand" "n"))
4584 (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
4585 (match_operand 3 "const_int_operand" "n"))))]
4586 "UINTVAL (operands[3]) < GET_MODE_BITSIZE (<MODE>mode)
4587 && (UINTVAL (operands[3]) + UINTVAL (operands[4])
4588 == GET_MODE_BITSIZE (<MODE>mode))"
4589 "extr\\t%<w>0, %<w>1, %<w>2, %4"
4590 [(set_attr "type" "rotate_imm")]
4593 ;; zero_extend version of the above
4594 (define_insn "*extrsi5_insn_uxtw"
4595 [(set (match_operand:DI 0 "register_operand" "=r")
4597 (ior:SI (ashift:SI (match_operand:SI 1 "register_operand" "r")
4598 (match_operand 3 "const_int_operand" "n"))
4599 (lshiftrt:SI (match_operand:SI 2 "register_operand" "r")
4600 (match_operand 4 "const_int_operand" "n")))))]
4601 "UINTVAL (operands[3]) < 32 &&
4602 (UINTVAL (operands[3]) + UINTVAL (operands[4]) == 32)"
4603 "extr\\t%w0, %w1, %w2, %4"
4604 [(set_attr "type" "rotate_imm")]
4607 (define_insn "*extrsi5_insn_uxtw_alt"
4608 [(set (match_operand:DI 0 "register_operand" "=r")
4610 (ior:SI (lshiftrt:SI (match_operand:SI 2 "register_operand" "r")
4611 (match_operand 4 "const_int_operand" "n"))
4612 (ashift:SI (match_operand:SI 1 "register_operand" "r")
4613 (match_operand 3 "const_int_operand" "n")))))]
4614 "UINTVAL (operands[3]) < 32 &&
4615 (UINTVAL (operands[3]) + UINTVAL (operands[4]) == 32)"
4616 "extr\\t%w0, %w1, %w2, %4"
4617 [(set_attr "type" "rotate_imm")]
4620 (define_insn "*ror<mode>3_insn"
4621 [(set (match_operand:GPI 0 "register_operand" "=r")
4622 (rotate:GPI (match_operand:GPI 1 "register_operand" "r")
4623 (match_operand 2 "const_int_operand" "n")))]
4624 "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<MODE>mode)"
4626 operands[3] = GEN_INT (<sizen> - UINTVAL (operands[2]));
4627 return "ror\\t%<w>0, %<w>1, %3";
4629 [(set_attr "type" "rotate_imm")]
4632 ;; zero_extend version of the above
4633 (define_insn "*rorsi3_insn_uxtw"
4634 [(set (match_operand:DI 0 "register_operand" "=r")
4636 (rotate:SI (match_operand:SI 1 "register_operand" "r")
4637 (match_operand 2 "const_int_operand" "n"))))]
4638 "UINTVAL (operands[2]) < 32"
4640 operands[3] = GEN_INT (32 - UINTVAL (operands[2]));
4641 return "ror\\t%w0, %w1, %3";
4643 [(set_attr "type" "rotate_imm")]
4646 (define_insn "*<ANY_EXTEND:optab><GPI:mode>_ashl<SHORT:mode>"
4647 [(set (match_operand:GPI 0 "register_operand" "=r")
4649 (ashift:SHORT (match_operand:SHORT 1 "register_operand" "r")
4650 (match_operand 2 "const_int_operand" "n"))))]
4651 "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
4653 operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
4654 return "<su>bfiz\t%<GPI:w>0, %<GPI:w>1, %2, %3";
4656 [(set_attr "type" "bfx")]
4659 (define_insn "*zero_extend<GPI:mode>_lshr<SHORT:mode>"
4660 [(set (match_operand:GPI 0 "register_operand" "=r")
4662 (lshiftrt:SHORT (match_operand:SHORT 1 "register_operand" "r")
4663 (match_operand 2 "const_int_operand" "n"))))]
4664 "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
4666 operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
4667 return "ubfx\t%<GPI:w>0, %<GPI:w>1, %2, %3";
4669 [(set_attr "type" "bfx")]
4672 (define_insn "*extend<GPI:mode>_ashr<SHORT:mode>"
4673 [(set (match_operand:GPI 0 "register_operand" "=r")
4675 (ashiftrt:SHORT (match_operand:SHORT 1 "register_operand" "r")
4676 (match_operand 2 "const_int_operand" "n"))))]
4677 "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)"
4679 operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2]));
4680 return "sbfx\\t%<GPI:w>0, %<GPI:w>1, %2, %3";
4682 [(set_attr "type" "bfx")]
4685 ;; -------------------------------------------------------------------
4687 ;; -------------------------------------------------------------------
4689 (define_expand "<optab>"
4690 [(set (match_operand:DI 0 "register_operand" "=r")
4691 (ANY_EXTRACT:DI (match_operand:DI 1 "register_operand")
4693 "aarch64_simd_shift_imm_offset_di")
4694 (match_operand 3 "aarch64_simd_shift_imm_di")))]
4697 if (!IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]),
4698 1, GET_MODE_BITSIZE (DImode) - 1))
4704 (define_insn "*<optab><mode>"
4705 [(set (match_operand:GPI 0 "register_operand" "=r")
4706 (ANY_EXTRACT:GPI (match_operand:GPI 1 "register_operand" "r")
4708 "aarch64_simd_shift_imm_offset_<mode>" "n")
4710 "aarch64_simd_shift_imm_<mode>" "n")))]
4711 "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]),
4712 1, GET_MODE_BITSIZE (<MODE>mode) - 1)"
4713 "<su>bfx\\t%<w>0, %<w>1, %3, %2"
4714 [(set_attr "type" "bfx")]
4717 ;; When the bit position and width add up to 32 we can use a W-reg LSR
4718 ;; instruction taking advantage of the implicit zero-extension of the X-reg.
4720 [(set (match_operand:DI 0 "register_operand")
4721 (zero_extract:DI (match_operand:DI 1 "register_operand")
4723 "aarch64_simd_shift_imm_offset_di")
4725 "aarch64_simd_shift_imm_di")))]
4726 "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]), 1,
4727 GET_MODE_BITSIZE (DImode) - 1)
4728 && (INTVAL (operands[2]) + INTVAL (operands[3]))
4729 == GET_MODE_BITSIZE (SImode)"
4731 (zero_extend:DI (lshiftrt:SI (match_dup 4) (match_dup 3))))]
4733 operands[4] = gen_lowpart (SImode, operands[1]);
4737 ;; Bitfield Insert (insv)
4738 (define_expand "insv<mode>"
4739 [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand")
4740 (match_operand 1 "const_int_operand")
4741 (match_operand 2 "const_int_operand"))
4742 (match_operand:GPI 3 "general_operand"))]
4745 unsigned HOST_WIDE_INT width = UINTVAL (operands[1]);
4746 unsigned HOST_WIDE_INT pos = UINTVAL (operands[2]);
4747 rtx value = operands[3];
4749 if (width == 0 || (pos + width) > GET_MODE_BITSIZE (<MODE>mode))
4752 if (CONST_INT_P (value))
4754 unsigned HOST_WIDE_INT mask = ((unsigned HOST_WIDE_INT)1 << width) - 1;
4756 /* Prefer AND/OR for inserting all zeros or all ones. */
4757 if ((UINTVAL (value) & mask) == 0
4758 || (UINTVAL (value) & mask) == mask)
4761 /* 16-bit aligned 16-bit wide insert is handled by insv_imm. */
4762 if (width == 16 && (pos % 16) == 0)
4765 operands[3] = force_reg (<MODE>mode, value);
4768 (define_insn "*insv_reg<mode>"
4769 [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
4770 (match_operand 1 "const_int_operand" "n")
4771 (match_operand 2 "const_int_operand" "n"))
4772 (match_operand:GPI 3 "register_operand" "r"))]
4773 "!(UINTVAL (operands[1]) == 0
4774 || (UINTVAL (operands[2]) + UINTVAL (operands[1])
4775 > GET_MODE_BITSIZE (<MODE>mode)))"
4776 "bfi\\t%<w>0, %<w>3, %2, %1"
4777 [(set_attr "type" "bfm")]
4780 (define_insn "*aarch64_bfi<GPI:mode><ALLX:mode>4"
4781 [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
4782 (match_operand 1 "const_int_operand" "n")
4783 (match_operand 2 "const_int_operand" "n"))
4784 (zero_extend:GPI (match_operand:ALLX 3 "register_operand" "r")))]
4785 "UINTVAL (operands[1]) <= <ALLX:sizen>"
4786 "bfi\\t%<GPI:w>0, %<GPI:w>3, %2, %1"
4787 [(set_attr "type" "bfm")]
4790 (define_insn "*extr_insv_lower_reg<mode>"
4791 [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r")
4792 (match_operand 1 "const_int_operand" "n")
4794 (zero_extract:GPI (match_operand:GPI 2 "register_operand" "r")
4796 (match_operand 3 "const_int_operand" "n")))]
4797 "!(UINTVAL (operands[1]) == 0
4798 || (UINTVAL (operands[3]) + UINTVAL (operands[1])
4799 > GET_MODE_BITSIZE (<MODE>mode)))"
4800 "bfxil\\t%<w>0, %<w>2, %3, %1"
4801 [(set_attr "type" "bfm")]
4804 (define_insn "*<optab><ALLX:mode>_shft_<GPI:mode>"
4805 [(set (match_operand:GPI 0 "register_operand" "=r")
4806 (ashift:GPI (ANY_EXTEND:GPI
4807 (match_operand:ALLX 1 "register_operand" "r"))
4808 (match_operand 2 "const_int_operand" "n")))]
4809 "UINTVAL (operands[2]) < <GPI:sizen>"
4811 operands[3] = (<ALLX:sizen> <= (<GPI:sizen> - UINTVAL (operands[2])))
4812 ? GEN_INT (<ALLX:sizen>)
4813 : GEN_INT (<GPI:sizen> - UINTVAL (operands[2]));
4814 return "<su>bfiz\t%<GPI:w>0, %<GPI:w>1, %2, %3";
4816 [(set_attr "type" "bfx")]
4819 ;; XXX We should match (any_extend (ashift)) here, like (and (ashift)) below
4821 (define_insn "*andim_ashift<mode>_bfiz"
4822 [(set (match_operand:GPI 0 "register_operand" "=r")
4823 (and:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
4824 (match_operand 2 "const_int_operand" "n"))
4825 (match_operand 3 "const_int_operand" "n")))]
4826 "aarch64_mask_and_shift_for_ubfiz_p (<MODE>mode, operands[3], operands[2])"
4827 "ubfiz\\t%<w>0, %<w>1, %2, %P3"
4828 [(set_attr "type" "bfx")]
4831 ;; When the bit position and width of the equivalent extraction add up to 32
4832 ;; we can use a W-reg LSL instruction taking advantage of the implicit
4833 ;; zero-extension of the X-reg.
4835 [(set (match_operand:DI 0 "register_operand")
4836 (and:DI (ashift:DI (match_operand:DI 1 "register_operand")
4837 (match_operand 2 "const_int_operand"))
4838 (match_operand 3 "const_int_operand")))]
4839 "aarch64_mask_and_shift_for_ubfiz_p (DImode, operands[3], operands[2])
4840 && (INTVAL (operands[2]) + popcount_hwi (INTVAL (operands[3])))
4841 == GET_MODE_BITSIZE (SImode)"
4843 (zero_extend:DI (ashift:SI (match_dup 4) (match_dup 2))))]
4845 operands[4] = gen_lowpart (SImode, operands[1]);
4849 (define_insn "bswap<mode>2"
4850 [(set (match_operand:GPI 0 "register_operand" "=r")
4851 (bswap:GPI (match_operand:GPI 1 "register_operand" "r")))]
4853 "rev\\t%<w>0, %<w>1"
4854 [(set_attr "type" "rev")]
4857 (define_insn "bswaphi2"
4858 [(set (match_operand:HI 0 "register_operand" "=r")
4859 (bswap:HI (match_operand:HI 1 "register_operand" "r")))]
4862 [(set_attr "type" "rev")]
4865 ;; There are no canonicalisation rules for the position of the lshiftrt, ashift
4866 ;; operations within an IOR/AND RTX, therefore we have two patterns matching
4867 ;; each valid permutation.
4869 (define_insn "rev16<mode>2"
4870 [(set (match_operand:GPI 0 "register_operand" "=r")
4871 (ior:GPI (and:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r")
4873 (match_operand:GPI 3 "const_int_operand" "n"))
4874 (and:GPI (lshiftrt:GPI (match_dup 1)
4876 (match_operand:GPI 2 "const_int_operand" "n"))))]
4877 "aarch_rev16_shleft_mask_imm_p (operands[3], <MODE>mode)
4878 && aarch_rev16_shright_mask_imm_p (operands[2], <MODE>mode)"
4879 "rev16\\t%<w>0, %<w>1"
4880 [(set_attr "type" "rev")]
4883 (define_insn "rev16<mode>2_alt"
4884 [(set (match_operand:GPI 0 "register_operand" "=r")
4885 (ior:GPI (and:GPI (lshiftrt:GPI (match_operand:GPI 1 "register_operand" "r")
4887 (match_operand:GPI 2 "const_int_operand" "n"))
4888 (and:GPI (ashift:GPI (match_dup 1)
4890 (match_operand:GPI 3 "const_int_operand" "n"))))]
4891 "aarch_rev16_shleft_mask_imm_p (operands[3], <MODE>mode)
4892 && aarch_rev16_shright_mask_imm_p (operands[2], <MODE>mode)"
4893 "rev16\\t%<w>0, %<w>1"
4894 [(set_attr "type" "rev")]
4897 ;; zero_extend version of above
4898 (define_insn "*bswapsi2_uxtw"
4899 [(set (match_operand:DI 0 "register_operand" "=r")
4900 (zero_extend:DI (bswap:SI (match_operand:SI 1 "register_operand" "r"))))]
4903 [(set_attr "type" "rev")]
4906 ;; -------------------------------------------------------------------
4907 ;; Floating-point intrinsics
4908 ;; -------------------------------------------------------------------
4910 ;; frint floating-point round to integral standard patterns.
4911 ;; Expands to btrunc, ceil, floor, nearbyint, rint, round, frintn.
4913 (define_insn "<frint_pattern><mode>2"
4914 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
4915 (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")]
4918 "frint<frint_suffix>\\t%<s>0, %<s>1"
4919 [(set_attr "type" "f_rint<stype>")]
4922 ;; frcvt floating-point round to integer and convert standard patterns.
4923 ;; Expands to lbtrunc, lceil, lfloor, lround.
4924 (define_insn "l<fcvt_pattern><su_optab><GPF_F16:mode><GPI:mode>2"
4925 [(set (match_operand:GPI 0 "register_operand" "=r")
4927 (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")]
4930 "fcvt<frint_suffix><su>\\t%<GPI:w>0, %<GPF_F16:s>1"
4931 [(set_attr "type" "f_cvtf2i")]
4934 (define_insn "*aarch64_fcvt<su_optab><GPF:mode><GPI:mode>2_mult"
4935 [(set (match_operand:GPI 0 "register_operand" "=r")
4938 (match_operand:GPF 1 "register_operand" "w")
4939 (match_operand:GPF 2 "aarch64_fp_pow2" "F"))))]
4941 && IN_RANGE (aarch64_fpconst_pow_of_2 (operands[2]), 1,
4942 GET_MODE_BITSIZE (<GPI:MODE>mode))"
4944 int fbits = aarch64_fpconst_pow_of_2 (operands[2]);
4946 snprintf (buf, 64, "fcvtz<su>\\t%%<GPI:w>0, %%<GPF:s>1, #%d", fbits);
4947 output_asm_insn (buf, operands);
4950 [(set_attr "type" "f_cvtf2i")]
4955 (define_insn "fma<mode>4"
4956 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
4957 (fma:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")
4958 (match_operand:GPF_F16 2 "register_operand" "w")
4959 (match_operand:GPF_F16 3 "register_operand" "w")))]
4961 "fmadd\\t%<s>0, %<s>1, %<s>2, %<s>3"
4962 [(set_attr "type" "fmac<stype>")]
4965 (define_insn "fnma<mode>4"
4966 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
4968 (neg:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w"))
4969 (match_operand:GPF_F16 2 "register_operand" "w")
4970 (match_operand:GPF_F16 3 "register_operand" "w")))]
4972 "fmsub\\t%<s>0, %<s>1, %<s>2, %<s>3"
4973 [(set_attr "type" "fmac<stype>")]
4976 (define_insn "fms<mode>4"
4977 [(set (match_operand:GPF 0 "register_operand" "=w")
4978 (fma:GPF (match_operand:GPF 1 "register_operand" "w")
4979 (match_operand:GPF 2 "register_operand" "w")
4980 (neg:GPF (match_operand:GPF 3 "register_operand" "w"))))]
4982 "fnmsub\\t%<s>0, %<s>1, %<s>2, %<s>3"
4983 [(set_attr "type" "fmac<s>")]
4986 (define_insn "fnms<mode>4"
4987 [(set (match_operand:GPF 0 "register_operand" "=w")
4988 (fma:GPF (neg:GPF (match_operand:GPF 1 "register_operand" "w"))
4989 (match_operand:GPF 2 "register_operand" "w")
4990 (neg:GPF (match_operand:GPF 3 "register_operand" "w"))))]
4992 "fnmadd\\t%<s>0, %<s>1, %<s>2, %<s>3"
4993 [(set_attr "type" "fmac<s>")]
4996 ;; If signed zeros are ignored, -(a * b + c) = -a * b - c.
4997 (define_insn "*fnmadd<mode>4"
4998 [(set (match_operand:GPF 0 "register_operand" "=w")
4999 (neg:GPF (fma:GPF (match_operand:GPF 1 "register_operand" "w")
5000 (match_operand:GPF 2 "register_operand" "w")
5001 (match_operand:GPF 3 "register_operand" "w"))))]
5002 "!HONOR_SIGNED_ZEROS (<MODE>mode) && TARGET_FLOAT"
5003 "fnmadd\\t%<s>0, %<s>1, %<s>2, %<s>3"
5004 [(set_attr "type" "fmac<s>")]
5007 ;; -------------------------------------------------------------------
5008 ;; Floating-point conversions
5009 ;; -------------------------------------------------------------------
5011 (define_insn "extendsfdf2"
5012 [(set (match_operand:DF 0 "register_operand" "=w")
5013 (float_extend:DF (match_operand:SF 1 "register_operand" "w")))]
5016 [(set_attr "type" "f_cvt")]
5019 (define_insn "extendhfsf2"
5020 [(set (match_operand:SF 0 "register_operand" "=w")
5021 (float_extend:SF (match_operand:HF 1 "register_operand" "w")))]
5024 [(set_attr "type" "f_cvt")]
5027 (define_insn "extendhfdf2"
5028 [(set (match_operand:DF 0 "register_operand" "=w")
5029 (float_extend:DF (match_operand:HF 1 "register_operand" "w")))]
5032 [(set_attr "type" "f_cvt")]
5035 (define_insn "truncdfsf2"
5036 [(set (match_operand:SF 0 "register_operand" "=w")
5037 (float_truncate:SF (match_operand:DF 1 "register_operand" "w")))]
5040 [(set_attr "type" "f_cvt")]
5043 (define_insn "truncsfhf2"
5044 [(set (match_operand:HF 0 "register_operand" "=w")
5045 (float_truncate:HF (match_operand:SF 1 "register_operand" "w")))]
5048 [(set_attr "type" "f_cvt")]
5051 (define_insn "truncdfhf2"
5052 [(set (match_operand:HF 0 "register_operand" "=w")
5053 (float_truncate:HF (match_operand:DF 1 "register_operand" "w")))]
5056 [(set_attr "type" "f_cvt")]
5059 ;; Convert SF -> SI or DF -> DI while preferring w = w register constraints
5060 ;; and making r = w more expensive
5062 (define_insn "<optab>_trunc<fcvt_target><GPI:mode>2"
5063 [(set (match_operand:GPI 0 "register_operand" "=?r,w")
5064 (FIXUORS:GPI (match_operand:<FCVT_TARGET> 1 "register_operand" "w,w")))]
5067 fcvtz<su>\t%<w>0, %<s>1
5068 fcvtz<su>\t%<s>0, %<s>1"
5069 [(set_attr "type" "f_cvtf2i,neon_fp_to_int_s")]
5072 ;; Convert HF -> SI or DI
5074 (define_insn "<optab>_trunchf<GPI:mode>2"
5075 [(set (match_operand:GPI 0 "register_operand" "=r")
5076 (FIXUORS:GPI (match_operand:HF 1 "register_operand" "w")))]
5078 "fcvtz<su>\t%<w>0, %h1"
5079 [(set_attr "type" "f_cvtf2i")]
5082 ;; Convert DF -> SI or SF -> DI which can only be accomplished with
5083 ;; input in a fp register and output in a integer register
5085 (define_insn "<optab>_trunc<fcvt_change_mode><GPI:mode>2"
5086 [(set (match_operand:GPI 0 "register_operand" "=r")
5087 (FIXUORS:GPI (match_operand:<FCVT_CHANGE_MODE> 1 "register_operand" "w")))]
5089 "fcvtz<su>\t%<w>0, %<fpw>1"
5090 [(set_attr "type" "f_cvtf2i")]
5093 (define_insn "<optab><fcvt_target><GPF:mode>2"
5094 [(set (match_operand:GPF 0 "register_operand" "=w,w")
5095 (FLOATUORS:GPF (match_operand:<FCVT_TARGET> 1 "register_operand" "w,r")))]
5098 <su_optab>cvtf\t%<GPF:s>0, %<s>1
5099 <su_optab>cvtf\t%<GPF:s>0, %<w1>1"
5100 [(set_attr "simd" "yes,no")
5101 (set_attr "fp" "no,yes")
5102 (set_attr "type" "neon_int_to_fp_<Vetype>,f_cvti2f")]
5105 (define_insn "<optab><fcvt_iesize><GPF:mode>2"
5106 [(set (match_operand:GPF 0 "register_operand" "=w")
5107 (FLOATUORS:GPF (match_operand:<FCVT_IESIZE> 1 "register_operand" "r")))]
5109 "<su_optab>cvtf\t%<GPF:s>0, %<w2>1"
5110 [(set_attr "type" "f_cvti2f")]
5113 ;; If we do not have ARMv8.2-A 16-bit floating point extensions, the
5114 ;; midend will arrange for an SImode conversion to HFmode to first go
5115 ;; through DFmode, then to HFmode. But first it will try converting
5116 ;; to DImode then down, which would match our DImode pattern below and
5117 ;; give very poor code-generation. So, we must provide our own emulation
5118 ;; of the mid-end logic.
5120 (define_insn "aarch64_fp16_<optab><mode>hf2"
5121 [(set (match_operand:HF 0 "register_operand" "=w")
5122 (FLOATUORS:HF (match_operand:GPI 1 "register_operand" "r")))]
5124 "<su_optab>cvtf\t%h0, %<w>1"
5125 [(set_attr "type" "f_cvti2f")]
5128 (define_expand "<optab>sihf2"
5129 [(set (match_operand:HF 0 "register_operand")
5130 (FLOATUORS:HF (match_operand:SI 1 "register_operand")))]
5133 if (TARGET_FP_F16INST)
5134 emit_insn (gen_aarch64_fp16_<optab>sihf2 (operands[0], operands[1]));
5137 rtx convert_target = gen_reg_rtx (DFmode);
5138 emit_insn (gen_<optab>sidf2 (convert_target, operands[1]));
5139 emit_insn (gen_truncdfhf2 (operands[0], convert_target));
5145 ;; For DImode there is no wide enough floating-point mode that we
5146 ;; can convert through natively (TFmode would work, but requires a library
5147 ;; call). However, we know that any value >= 65504 will be rounded
5148 ;; to infinity on conversion. This is well within the range of SImode, so
5150 ;; Saturate to SImode.
5151 ;; Convert from that to DFmode
5152 ;; Convert from that to HFmode (phew!).
5153 ;; Note that the saturation to SImode requires the SIMD extensions. If
5154 ;; we ever need to provide this pattern where the SIMD extensions are not
5155 ;; available, we would need a different approach.
5157 (define_expand "<optab>dihf2"
5158 [(set (match_operand:HF 0 "register_operand")
5159 (FLOATUORS:HF (match_operand:DI 1 "register_operand")))]
5160 "TARGET_FLOAT && (TARGET_FP_F16INST || TARGET_SIMD)"
5162 if (TARGET_FP_F16INST)
5163 emit_insn (gen_aarch64_fp16_<optab>dihf2 (operands[0], operands[1]));
5166 rtx sat_target = gen_reg_rtx (SImode);
5167 emit_insn (gen_aarch64_<su_optab>qmovndi (sat_target, operands[1]));
5168 emit_insn (gen_<optab>sihf2 (operands[0], sat_target));
5175 ;; Convert between fixed-point and floating-point (scalar modes)
5177 (define_insn "<FCVT_F2FIXED:fcvt_fixed_insn><GPF:mode>3"
5178 [(set (match_operand:<GPF:FCVT_TARGET> 0 "register_operand" "=r, w")
5179 (unspec:<GPF:FCVT_TARGET> [(match_operand:GPF 1 "register_operand" "w, w")
5180 (match_operand:SI 2 "immediate_operand" "i, i")]
5184 <FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPF:w1>0, %<GPF:s>1, #%2
5185 <FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPF:s>0, %<GPF:s>1, #%2"
5186 [(set_attr "type" "f_cvtf2i, neon_fp_to_int_<GPF:Vetype>")
5187 (set_attr "fp" "yes, *")
5188 (set_attr "simd" "*, yes")]
5191 (define_insn "<FCVT_FIXED2F:fcvt_fixed_insn><GPI:mode>3"
5192 [(set (match_operand:<GPI:FCVT_TARGET> 0 "register_operand" "=w, w")
5193 (unspec:<GPI:FCVT_TARGET> [(match_operand:GPI 1 "register_operand" "r, w")
5194 (match_operand:SI 2 "immediate_operand" "i, i")]
5198 <FCVT_FIXED2F:fcvt_fixed_insn>\t%<GPI:v>0, %<GPI:w>1, #%2
5199 <FCVT_FIXED2F:fcvt_fixed_insn>\t%<GPI:v>0, %<GPI:v>1, #%2"
5200 [(set_attr "type" "f_cvti2f, neon_int_to_fp_<GPI:Vetype>")
5201 (set_attr "fp" "yes, *")
5202 (set_attr "simd" "*, yes")]
5205 (define_insn "<FCVT_F2FIXED:fcvt_fixed_insn>hf<mode>3"
5206 [(set (match_operand:GPI 0 "register_operand" "=r")
5207 (unspec:GPI [(match_operand:HF 1 "register_operand" "w")
5208 (match_operand:SI 2 "immediate_operand" "i")]
5211 "<FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPI:w>0, %h1, #%2"
5212 [(set_attr "type" "f_cvtf2i")]
5215 (define_insn "<FCVT_FIXED2F:fcvt_fixed_insn><mode>hf3"
5216 [(set (match_operand:HF 0 "register_operand" "=w")
5217 (unspec:HF [(match_operand:GPI 1 "register_operand" "r")
5218 (match_operand:SI 2 "immediate_operand" "i")]
5221 "<FCVT_FIXED2F:fcvt_fixed_insn>\t%h0, %<GPI:w>1, #%2"
5222 [(set_attr "type" "f_cvti2f")]
5225 (define_insn "<FCVT_F2FIXED:fcvt_fixed_insn>hf3"
5226 [(set (match_operand:HI 0 "register_operand" "=w")
5227 (unspec:HI [(match_operand:HF 1 "register_operand" "w")
5228 (match_operand:SI 2 "immediate_operand" "i")]
5231 "<FCVT_F2FIXED:fcvt_fixed_insn>\t%h0, %h1, #%2"
5232 [(set_attr "type" "neon_fp_to_int_s")]
5235 (define_insn "<FCVT_FIXED2F:fcvt_fixed_insn>hi3"
5236 [(set (match_operand:HF 0 "register_operand" "=w")
5237 (unspec:HF [(match_operand:HI 1 "register_operand" "w")
5238 (match_operand:SI 2 "immediate_operand" "i")]
5241 "<FCVT_FIXED2F:fcvt_fixed_insn>\t%h0, %h1, #%2"
5242 [(set_attr "type" "neon_int_to_fp_s")]
5245 ;; -------------------------------------------------------------------
5246 ;; Floating-point arithmetic
5247 ;; -------------------------------------------------------------------
5249 (define_insn "add<mode>3"
5250 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5252 (match_operand:GPF_F16 1 "register_operand" "w")
5253 (match_operand:GPF_F16 2 "register_operand" "w")))]
5255 "fadd\\t%<s>0, %<s>1, %<s>2"
5256 [(set_attr "type" "fadd<stype>")]
5259 (define_insn "sub<mode>3"
5260 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5262 (match_operand:GPF_F16 1 "register_operand" "w")
5263 (match_operand:GPF_F16 2 "register_operand" "w")))]
5265 "fsub\\t%<s>0, %<s>1, %<s>2"
5266 [(set_attr "type" "fadd<stype>")]
5269 (define_insn "mul<mode>3"
5270 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5272 (match_operand:GPF_F16 1 "register_operand" "w")
5273 (match_operand:GPF_F16 2 "register_operand" "w")))]
5275 "fmul\\t%<s>0, %<s>1, %<s>2"
5276 [(set_attr "type" "fmul<stype>")]
5279 (define_insn "*fnmul<mode>3"
5280 [(set (match_operand:GPF 0 "register_operand" "=w")
5282 (neg:GPF (match_operand:GPF 1 "register_operand" "w"))
5283 (match_operand:GPF 2 "register_operand" "w")))]
5284 "TARGET_FLOAT && !flag_rounding_math"
5285 "fnmul\\t%<s>0, %<s>1, %<s>2"
5286 [(set_attr "type" "fmul<s>")]
5289 (define_insn "*fnmul<mode>3"
5290 [(set (match_operand:GPF 0 "register_operand" "=w")
5292 (match_operand:GPF 1 "register_operand" "w")
5293 (match_operand:GPF 2 "register_operand" "w"))))]
5295 "fnmul\\t%<s>0, %<s>1, %<s>2"
5296 [(set_attr "type" "fmul<s>")]
5299 (define_expand "div<mode>3"
5300 [(set (match_operand:GPF_F16 0 "register_operand")
5301 (div:GPF_F16 (match_operand:GPF_F16 1 "general_operand")
5302 (match_operand:GPF_F16 2 "register_operand")))]
5305 if (aarch64_emit_approx_div (operands[0], operands[1], operands[2]))
5308 operands[1] = force_reg (<MODE>mode, operands[1]);
5311 (define_insn "*div<mode>3"
5312 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5313 (div:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")
5314 (match_operand:GPF_F16 2 "register_operand" "w")))]
5316 "fdiv\\t%<s>0, %<s>1, %<s>2"
5317 [(set_attr "type" "fdiv<stype>")]
5320 (define_insn "neg<mode>2"
5321 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5322 (neg:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
5324 "fneg\\t%<s>0, %<s>1"
5325 [(set_attr "type" "ffarith<stype>")]
5328 (define_expand "sqrt<mode>2"
5329 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5330 (sqrt:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
5333 if (aarch64_emit_approx_sqrt (operands[0], operands[1], false))
5337 (define_insn "*sqrt<mode>2"
5338 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5339 (sqrt:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
5341 "fsqrt\\t%<s>0, %<s>1"
5342 [(set_attr "type" "fsqrt<stype>")]
5345 (define_insn "abs<mode>2"
5346 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5347 (abs:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))]
5349 "fabs\\t%<s>0, %<s>1"
5350 [(set_attr "type" "ffarith<stype>")]
5353 ;; Given that smax/smin do not specify the result when either input is NaN,
5354 ;; we could use either FMAXNM or FMAX for smax, and either FMINNM or FMIN
5357 (define_insn "smax<mode>3"
5358 [(set (match_operand:GPF 0 "register_operand" "=w")
5359 (smax:GPF (match_operand:GPF 1 "register_operand" "w")
5360 (match_operand:GPF 2 "register_operand" "w")))]
5362 "fmaxnm\\t%<s>0, %<s>1, %<s>2"
5363 [(set_attr "type" "f_minmax<s>")]
5366 (define_insn "smin<mode>3"
5367 [(set (match_operand:GPF 0 "register_operand" "=w")
5368 (smin:GPF (match_operand:GPF 1 "register_operand" "w")
5369 (match_operand:GPF 2 "register_operand" "w")))]
5371 "fminnm\\t%<s>0, %<s>1, %<s>2"
5372 [(set_attr "type" "f_minmax<s>")]
5375 ;; Scalar forms for fmax, fmin, fmaxnm, fminnm.
5376 ;; fmaxnm and fminnm are used for the fmax<mode>3 standard pattern names,
5377 ;; which implement the IEEE fmax ()/fmin () functions.
5378 (define_insn "<maxmin_uns><mode>3"
5379 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5380 (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")
5381 (match_operand:GPF_F16 2 "register_operand" "w")]
5384 "<maxmin_uns_op>\\t%<s>0, %<s>1, %<s>2"
5385 [(set_attr "type" "f_minmax<stype>")]
5388 (define_expand "lrint<GPF:mode><GPI:mode>2"
5389 [(match_operand:GPI 0 "register_operand")
5390 (match_operand:GPF 1 "register_operand")]
5392 && ((GET_MODE_SIZE (<GPF:MODE>mode) <= GET_MODE_SIZE (<GPI:MODE>mode))
5393 || !flag_trapping_math || flag_fp_int_builtin_inexact)"
5395 rtx cvt = gen_reg_rtx (<GPF:MODE>mode);
5396 emit_insn (gen_rint<GPF:mode>2 (cvt, operands[1]));
5397 emit_insn (gen_lbtrunc<GPF:mode><GPI:mode>2 (operands[0], cvt));
5402 ;; For copysign (x, y), we want to generate:
5404 ;; LDR d2, #(1 << 63)
5405 ;; BSL v2.8b, [y], [x]
5407 ;; or another, equivalent, sequence using one of BSL/BIT/BIF.
5408 ;; aarch64_simd_bsldf will select the best suited of these instructions
5409 ;; to generate based on register allocation, and knows how to partially
5410 ;; constant fold based on the values of X and Y, so expand through that.
5412 (define_expand "copysigndf3"
5413 [(match_operand:DF 0 "register_operand")
5414 (match_operand:DF 1 "register_operand")
5415 (match_operand:DF 2 "register_operand")]
5416 "TARGET_FLOAT && TARGET_SIMD"
5418 rtx mask = gen_reg_rtx (DImode);
5419 emit_move_insn (mask, GEN_INT (HOST_WIDE_INT_1U << 63));
5420 emit_insn (gen_aarch64_simd_bsldf (operands[0], mask,
5421 operands[2], operands[1]));
5426 ;; As above, but we must first get to a 64-bit value if we wish to use
5427 ;; aarch64_simd_bslv2sf.
5429 (define_expand "copysignsf3"
5430 [(match_operand:SF 0 "register_operand")
5431 (match_operand:SF 1 "register_operand")
5432 (match_operand:SF 2 "register_operand")]
5433 "TARGET_FLOAT && TARGET_SIMD"
5435 rtx v_bitmask = gen_reg_rtx (V2SImode);
5437 /* Juggle modes to get us in to a vector mode for BSL. */
5438 rtx op1 = lowpart_subreg (DImode, operands[1], SFmode);
5439 rtx op2 = lowpart_subreg (V2SFmode, operands[2], SFmode);
5440 rtx tmp = gen_reg_rtx (V2SFmode);
5441 emit_move_insn (v_bitmask,
5442 aarch64_simd_gen_const_vector_dup (V2SImode,
5443 HOST_WIDE_INT_M1U << 31));
5444 emit_insn (gen_aarch64_simd_bslv2sf (tmp, v_bitmask, op2, op1));
5445 emit_move_insn (operands[0], lowpart_subreg (SFmode, tmp, V2SFmode));
5450 ;; For xorsign (x, y), we want to generate:
5453 ;; AND v3.8B, v1.8B, v2.8B
5454 ;; EOR v0.8B, v0.8B, v3.8B
5457 (define_expand "xorsign<mode>3"
5458 [(match_operand:GPF 0 "register_operand")
5459 (match_operand:GPF 1 "register_operand")
5460 (match_operand:GPF 2 "register_operand")]
5461 "TARGET_FLOAT && TARGET_SIMD"
5464 machine_mode imode = <V_INT_EQUIV>mode;
5465 rtx mask = gen_reg_rtx (imode);
5466 rtx op1x = gen_reg_rtx (imode);
5467 rtx op2x = gen_reg_rtx (imode);
5469 int bits = GET_MODE_BITSIZE (<MODE>mode) - 1;
5470 emit_move_insn (mask, GEN_INT (trunc_int_for_mode (HOST_WIDE_INT_M1U << bits,
5473 emit_insn (gen_and<v_int_equiv>3 (op2x, mask,
5474 lowpart_subreg (imode, operands[2],
5476 emit_insn (gen_xor<v_int_equiv>3 (op1x,
5477 lowpart_subreg (imode, operands[1],
5480 emit_move_insn (operands[0],
5481 lowpart_subreg (<MODE>mode, op1x, imode));
5486 ;; -------------------------------------------------------------------
5488 ;; -------------------------------------------------------------------
5489 ;; Reload Scalar Floating point modes from constant pool.
5490 ;; The AArch64 port doesn't have __int128 constant move support.
5491 (define_expand "aarch64_reload_movcp<GPF_TF:mode><P:mode>"
5492 [(set (match_operand:GPF_TF 0 "register_operand" "=w")
5493 (mem:GPF_TF (match_operand 1 "aarch64_constant_pool_symref" "S")))
5494 (clobber (match_operand:P 2 "register_operand" "=&r"))]
5497 aarch64_expand_mov_immediate (operands[2], XEXP (operands[1], 0));
5498 emit_move_insn (operands[0], gen_rtx_MEM (<GPF_TF:MODE>mode, operands[2]));
5503 ;; Reload Vector modes from constant pool.
5504 (define_expand "aarch64_reload_movcp<VALL:mode><P:mode>"
5505 [(set (match_operand:VALL 0 "register_operand" "=w")
5506 (mem:VALL (match_operand 1 "aarch64_constant_pool_symref" "S")))
5507 (clobber (match_operand:P 2 "register_operand" "=&r"))]
5510 aarch64_expand_mov_immediate (operands[2], XEXP (operands[1], 0));
5511 emit_move_insn (operands[0], gen_rtx_MEM (<VALL:MODE>mode, operands[2]));
5516 (define_expand "aarch64_reload_mov<mode>"
5517 [(set (match_operand:TX 0 "register_operand" "=w")
5518 (match_operand:TX 1 "register_operand" "w"))
5519 (clobber (match_operand:DI 2 "register_operand" "=&r"))
5523 rtx op0 = simplify_gen_subreg (TImode, operands[0], <MODE>mode, 0);
5524 rtx op1 = simplify_gen_subreg (TImode, operands[1], <MODE>mode, 0);
5525 gen_aarch64_movtilow_tilow (op0, op1);
5526 gen_aarch64_movdi_tihigh (operands[2], op1);
5527 gen_aarch64_movtihigh_di (op0, operands[2]);
5532 ;; The following secondary reload helpers patterns are invoked
5533 ;; after or during reload as we don't want these patterns to start
5534 ;; kicking in during the combiner.
5536 (define_insn "aarch64_movdi_<mode>low"
5537 [(set (match_operand:DI 0 "register_operand" "=r")
5538 (zero_extract:DI (match_operand:TX 1 "register_operand" "w")
5539 (const_int 64) (const_int 0)))]
5540 "TARGET_FLOAT && (reload_completed || reload_in_progress)"
5542 [(set_attr "type" "f_mrc")
5543 (set_attr "length" "4")
5546 (define_insn "aarch64_movdi_<mode>high"
5547 [(set (match_operand:DI 0 "register_operand" "=r")
5548 (zero_extract:DI (match_operand:TX 1 "register_operand" "w")
5549 (const_int 64) (const_int 64)))]
5550 "TARGET_FLOAT && (reload_completed || reload_in_progress)"
5551 "fmov\\t%x0, %1.d[1]"
5552 [(set_attr "type" "f_mrc")
5553 (set_attr "length" "4")
5556 (define_insn "aarch64_mov<mode>high_di"
5557 [(set (zero_extract:TX (match_operand:TX 0 "register_operand" "+w")
5558 (const_int 64) (const_int 64))
5559 (zero_extend:TX (match_operand:DI 1 "register_operand" "r")))]
5560 "TARGET_FLOAT && (reload_completed || reload_in_progress)"
5561 "fmov\\t%0.d[1], %x1"
5562 [(set_attr "type" "f_mcr")
5563 (set_attr "length" "4")
5566 (define_insn "aarch64_mov<mode>low_di"
5567 [(set (match_operand:TX 0 "register_operand" "=w")
5568 (zero_extend:TX (match_operand:DI 1 "register_operand" "r")))]
5569 "TARGET_FLOAT && (reload_completed || reload_in_progress)"
5571 [(set_attr "type" "f_mcr")
5572 (set_attr "length" "4")
5575 (define_insn "aarch64_movtilow_tilow"
5576 [(set (match_operand:TI 0 "register_operand" "=w")
5578 (truncate:DI (match_operand:TI 1 "register_operand" "w"))))]
5579 "TARGET_FLOAT && (reload_completed || reload_in_progress)"
5581 [(set_attr "type" "fmov")
5582 (set_attr "length" "4")
5585 ;; There is a deliberate reason why the parameters of high and lo_sum's
5586 ;; don't have modes for ADRP and ADD instructions. This is to allow high
5587 ;; and lo_sum's to be used with the labels defining the jump tables in
5590 (define_expand "add_losym"
5591 [(set (match_operand 0 "register_operand" "=r")
5592 (lo_sum (match_operand 1 "register_operand" "r")
5593 (match_operand 2 "aarch64_valid_symref" "S")))]
5596 machine_mode mode = GET_MODE (operands[0]);
5598 emit_insn ((mode == DImode
5600 : gen_add_losym_si) (operands[0],
5606 (define_insn "add_losym_<mode>"
5607 [(set (match_operand:P 0 "register_operand" "=r")
5608 (lo_sum:P (match_operand:P 1 "register_operand" "r")
5609 (match_operand 2 "aarch64_valid_symref" "S")))]
5611 "add\\t%<w>0, %<w>1, :lo12:%c2"
5612 [(set_attr "type" "alu_imm")]
5615 (define_insn "ldr_got_small_<mode>"
5616 [(set (match_operand:PTR 0 "register_operand" "=r")
5617 (unspec:PTR [(mem:PTR (lo_sum:PTR
5618 (match_operand:PTR 1 "register_operand" "r")
5619 (match_operand:PTR 2 "aarch64_valid_symref" "S")))]
5620 UNSPEC_GOTSMALLPIC))]
5622 "ldr\\t%<w>0, [%1, #:got_lo12:%c2]"
5623 [(set_attr "type" "load_<ldst_sz>")]
5626 (define_insn "ldr_got_small_sidi"
5627 [(set (match_operand:DI 0 "register_operand" "=r")
5629 (unspec:SI [(mem:SI (lo_sum:DI
5630 (match_operand:DI 1 "register_operand" "r")
5631 (match_operand:DI 2 "aarch64_valid_symref" "S")))]
5632 UNSPEC_GOTSMALLPIC)))]
5634 "ldr\\t%w0, [%1, #:got_lo12:%c2]"
5635 [(set_attr "type" "load_4")]
5638 (define_insn "ldr_got_small_28k_<mode>"
5639 [(set (match_operand:PTR 0 "register_operand" "=r")
5640 (unspec:PTR [(mem:PTR (lo_sum:PTR
5641 (match_operand:PTR 1 "register_operand" "r")
5642 (match_operand:PTR 2 "aarch64_valid_symref" "S")))]
5643 UNSPEC_GOTSMALLPIC28K))]
5645 "ldr\\t%<w>0, [%1, #:<got_modifier>:%c2]"
5646 [(set_attr "type" "load_<ldst_sz>")]
5649 (define_insn "ldr_got_small_28k_sidi"
5650 [(set (match_operand:DI 0 "register_operand" "=r")
5652 (unspec:SI [(mem:SI (lo_sum:DI
5653 (match_operand:DI 1 "register_operand" "r")
5654 (match_operand:DI 2 "aarch64_valid_symref" "S")))]
5655 UNSPEC_GOTSMALLPIC28K)))]
5657 "ldr\\t%w0, [%1, #:gotpage_lo14:%c2]"
5658 [(set_attr "type" "load_4")]
5661 (define_insn "ldr_got_tiny"
5662 [(set (match_operand:DI 0 "register_operand" "=r")
5663 (unspec:DI [(match_operand:DI 1 "aarch64_valid_symref" "S")]
5664 UNSPEC_GOTTINYPIC))]
5667 [(set_attr "type" "load_8")]
5670 (define_insn "aarch64_load_tp_hard"
5671 [(set (match_operand:DI 0 "register_operand" "=r")
5672 (unspec:DI [(const_int 0)] UNSPEC_TLS))]
5674 "mrs\\t%0, tpidr_el0"
5675 [(set_attr "type" "mrs")]
5678 ;; The TLS ABI specifically requires that the compiler does not schedule
5679 ;; instructions in the TLS stubs, in order to enable linker relaxation.
5680 ;; Therefore we treat the stubs as an atomic sequence.
5681 (define_expand "tlsgd_small_<mode>"
5682 [(parallel [(set (match_operand 0 "register_operand" "")
5683 (call (mem:DI (match_dup 2)) (const_int 1)))
5684 (unspec:DI [(match_operand:PTR 1 "aarch64_valid_symref" "")] UNSPEC_GOTSMALLTLS)
5685 (clobber (reg:DI LR_REGNUM))])]
5688 operands[2] = aarch64_tls_get_addr ();
5691 (define_insn "*tlsgd_small_<mode>"
5692 [(set (match_operand 0 "register_operand" "")
5693 (call (mem:DI (match_operand:DI 2 "" "")) (const_int 1)))
5694 (unspec:DI [(match_operand:PTR 1 "aarch64_valid_symref" "S")] UNSPEC_GOTSMALLTLS)
5695 (clobber (reg:DI LR_REGNUM))
5698 "adrp\\tx0, %A1\;add\\tx0, x0, %L1\;bl\\t%2\;nop"
5699 [(set_attr "type" "call")
5700 (set_attr "length" "16")])
5702 (define_insn "tlsie_small_<mode>"
5703 [(set (match_operand:PTR 0 "register_operand" "=r")
5704 (unspec:PTR [(match_operand 1 "aarch64_tls_ie_symref" "S")]
5705 UNSPEC_GOTSMALLTLS))]
5707 "adrp\\t%0, %A1\;ldr\\t%<w>0, [%0, #%L1]"
5708 [(set_attr "type" "load_4")
5709 (set_attr "length" "8")]
5712 (define_insn "tlsie_small_sidi"
5713 [(set (match_operand:DI 0 "register_operand" "=r")
5715 (unspec:SI [(match_operand 1 "aarch64_tls_ie_symref" "S")]
5716 UNSPEC_GOTSMALLTLS)))]
5718 "adrp\\t%0, %A1\;ldr\\t%w0, [%0, #%L1]"
5719 [(set_attr "type" "load_4")
5720 (set_attr "length" "8")]
5723 (define_insn "tlsie_tiny_<mode>"
5724 [(set (match_operand:PTR 0 "register_operand" "=&r")
5725 (unspec:PTR [(match_operand 1 "aarch64_tls_ie_symref" "S")
5726 (match_operand:PTR 2 "register_operand" "r")]
5727 UNSPEC_GOTTINYTLS))]
5729 "ldr\\t%<w>0, %L1\;add\\t%<w>0, %<w>0, %<w>2"
5730 [(set_attr "type" "multiple")
5731 (set_attr "length" "8")]
5734 (define_insn "tlsie_tiny_sidi"
5735 [(set (match_operand:DI 0 "register_operand" "=&r")
5737 (unspec:SI [(match_operand 1 "aarch64_tls_ie_symref" "S")
5738 (match_operand:DI 2 "register_operand" "r")
5740 UNSPEC_GOTTINYTLS)))]
5742 "ldr\\t%w0, %L1\;add\\t%w0, %w0, %w2"
5743 [(set_attr "type" "multiple")
5744 (set_attr "length" "8")]
5747 (define_insn "tlsle12_<mode>"
5748 [(set (match_operand:P 0 "register_operand" "=r")
5749 (unspec:P [(match_operand:P 1 "register_operand" "r")
5750 (match_operand 2 "aarch64_tls_le_symref" "S")]
5753 "add\\t%<w>0, %<w>1, #%L2";
5754 [(set_attr "type" "alu_sreg")
5755 (set_attr "length" "4")]
5758 (define_insn "tlsle24_<mode>"
5759 [(set (match_operand:P 0 "register_operand" "=r")
5760 (unspec:P [(match_operand:P 1 "register_operand" "r")
5761 (match_operand 2 "aarch64_tls_le_symref" "S")]
5764 "add\\t%<w>0, %<w>1, #%G2, lsl #12\;add\\t%<w>0, %<w>0, #%L2"
5765 [(set_attr "type" "multiple")
5766 (set_attr "length" "8")]
5769 (define_insn "tlsle32_<mode>"
5770 [(set (match_operand:P 0 "register_operand" "=r")
5771 (unspec:P [(match_operand 1 "aarch64_tls_le_symref" "S")]
5774 "movz\\t%<w>0, #:tprel_g1:%1\;movk\\t%<w>0, #:tprel_g0_nc:%1"
5775 [(set_attr "type" "multiple")
5776 (set_attr "length" "8")]
5779 (define_insn "tlsle48_<mode>"
5780 [(set (match_operand:P 0 "register_operand" "=r")
5781 (unspec:P [(match_operand 1 "aarch64_tls_le_symref" "S")]
5784 "movz\\t%<w>0, #:tprel_g2:%1\;movk\\t%<w>0, #:tprel_g1_nc:%1\;movk\\t%<w>0, #:tprel_g0_nc:%1"
5785 [(set_attr "type" "multiple")
5786 (set_attr "length" "12")]
5789 (define_insn "tlsdesc_small_<mode>"
5790 [(set (reg:PTR R0_REGNUM)
5791 (unspec:PTR [(match_operand 0 "aarch64_valid_symref" "S")]
5793 (clobber (reg:DI LR_REGNUM))
5794 (clobber (reg:CC CC_REGNUM))
5795 (clobber (match_scratch:DI 1 "=r"))]
5797 "adrp\\tx0, %A0\;ldr\\t%<w>1, [x0, #%L0]\;add\\t<w>0, <w>0, %L0\;.tlsdesccall\\t%0\;blr\\t%1"
5798 [(set_attr "type" "call")
5799 (set_attr "length" "16")])
5801 (define_insn "stack_tie"
5802 [(set (mem:BLK (scratch))
5803 (unspec:BLK [(match_operand:DI 0 "register_operand" "rk")
5804 (match_operand:DI 1 "register_operand" "rk")]
5808 [(set_attr "length" "0")]
5811 ;; Pointer authentication patterns are always provided. In architecture
5812 ;; revisions prior to ARMv8.3-A these HINT instructions operate as NOPs.
5813 ;; This lets the user write portable software which authenticates pointers
5814 ;; when run on something which implements ARMv8.3-A, and which runs
5815 ;; correctly, but does not authenticate pointers, where ARMv8.3-A is not
5818 ;; Signing/Authenticating R30 using SP as the salt.
5820 (define_insn "<pauth_mnem_prefix>sp"
5821 [(set (reg:DI R30_REGNUM)
5822 (unspec:DI [(reg:DI R30_REGNUM) (reg:DI SP_REGNUM)] PAUTH_LR_SP))]
5824 "hint\t<pauth_hint_num_a> // <pauth_mnem_prefix>asp";
5827 ;; Signing/Authenticating X17 using X16 as the salt.
5829 (define_insn "<pauth_mnem_prefix>1716"
5830 [(set (reg:DI R17_REGNUM)
5831 (unspec:DI [(reg:DI R17_REGNUM) (reg:DI R16_REGNUM)] PAUTH_17_16))]
5833 "hint\t<pauth_hint_num_a> // <pauth_mnem_prefix>a1716";
5836 ;; Stripping the signature in R30.
5838 (define_insn "xpaclri"
5839 [(set (reg:DI R30_REGNUM) (unspec:DI [(reg:DI R30_REGNUM)] UNSPEC_XPACLRI))]
5841 "hint\t7 // xpaclri"
5844 ;; UNSPEC_VOLATILE is considered to use and clobber all hard registers and
5845 ;; all of memory. This blocks insns from being moved across this point.
5847 (define_insn "blockage"
5848 [(unspec_volatile [(const_int 0)] UNSPECV_BLOCKAGE)]
5851 [(set_attr "length" "0")
5852 (set_attr "type" "block")]
5855 (define_insn "probe_stack_range"
5856 [(set (match_operand:DI 0 "register_operand" "=r")
5857 (unspec_volatile:DI [(match_operand:DI 1 "register_operand" "0")
5858 (match_operand:DI 2 "register_operand" "r")]
5859 UNSPECV_PROBE_STACK_RANGE))]
5862 return aarch64_output_probe_stack_range (operands[0], operands[2]);
5864 [(set_attr "length" "32")]
5867 ;; Named pattern for expanding thread pointer reference.
5868 (define_expand "get_thread_pointerdi"
5869 [(match_operand:DI 0 "register_operand" "=r")]
5872 rtx tmp = aarch64_load_tp (operands[0]);
5873 if (tmp != operands[0])
5874 emit_move_insn (operands[0], tmp);
5878 ;; Named patterns for stack smashing protection.
5879 (define_expand "stack_protect_set"
5880 [(match_operand 0 "memory_operand")
5881 (match_operand 1 "memory_operand")]
5884 machine_mode mode = GET_MODE (operands[0]);
5886 emit_insn ((mode == DImode
5887 ? gen_stack_protect_set_di
5888 : gen_stack_protect_set_si) (operands[0], operands[1]));
5892 (define_insn "stack_protect_set_<mode>"
5893 [(set (match_operand:PTR 0 "memory_operand" "=m")
5894 (unspec:PTR [(match_operand:PTR 1 "memory_operand" "m")]
5896 (set (match_scratch:PTR 2 "=&r") (const_int 0))]
5898 "ldr\\t%<w>2, %1\;str\\t%<w>2, %0\;mov\t%<w>2,0"
5899 [(set_attr "length" "12")
5900 (set_attr "type" "multiple")])
5902 (define_expand "stack_protect_test"
5903 [(match_operand 0 "memory_operand")
5904 (match_operand 1 "memory_operand")
5909 machine_mode mode = GET_MODE (operands[0]);
5911 result = gen_reg_rtx(mode);
5913 emit_insn ((mode == DImode
5914 ? gen_stack_protect_test_di
5915 : gen_stack_protect_test_si) (result,
5920 emit_jump_insn (gen_cbranchdi4 (gen_rtx_EQ (VOIDmode, result, const0_rtx),
5921 result, const0_rtx, operands[2]));
5923 emit_jump_insn (gen_cbranchsi4 (gen_rtx_EQ (VOIDmode, result, const0_rtx),
5924 result, const0_rtx, operands[2]));
5928 (define_insn "stack_protect_test_<mode>"
5929 [(set (match_operand:PTR 0 "register_operand" "=r")
5930 (unspec:PTR [(match_operand:PTR 1 "memory_operand" "m")
5931 (match_operand:PTR 2 "memory_operand" "m")]
5933 (clobber (match_scratch:PTR 3 "=&r"))]
5935 "ldr\t%<w>3, %1\;ldr\t%<w>0, %2\;eor\t%<w>0, %<w>3, %<w>0"
5936 [(set_attr "length" "12")
5937 (set_attr "type" "multiple")])
5939 ;; Write Floating-point Control Register.
5940 (define_insn "set_fpcr"
5941 [(unspec_volatile [(match_operand:SI 0 "register_operand" "r")] UNSPECV_SET_FPCR)]
5944 [(set_attr "type" "mrs")])
5946 ;; Read Floating-point Control Register.
5947 (define_insn "get_fpcr"
5948 [(set (match_operand:SI 0 "register_operand" "=r")
5949 (unspec_volatile:SI [(const_int 0)] UNSPECV_GET_FPCR))]
5952 [(set_attr "type" "mrs")])
5954 ;; Write Floating-point Status Register.
5955 (define_insn "set_fpsr"
5956 [(unspec_volatile [(match_operand:SI 0 "register_operand" "r")] UNSPECV_SET_FPSR)]
5959 [(set_attr "type" "mrs")])
5961 ;; Read Floating-point Status Register.
5962 (define_insn "get_fpsr"
5963 [(set (match_operand:SI 0 "register_operand" "=r")
5964 (unspec_volatile:SI [(const_int 0)] UNSPECV_GET_FPSR))]
5967 [(set_attr "type" "mrs")])
5970 ;; Define the subtract-one-and-jump insns so loop.c
5971 ;; knows what to generate.
5972 (define_expand "doloop_end"
5973 [(use (match_operand 0 "" "")) ; loop pseudo
5974 (use (match_operand 1 "" ""))] ; label
5975 "optimize > 0 && flag_modulo_sched"
5984 /* Currently SMS relies on the do-loop pattern to recognize loops
5985 where (1) the control part consists of all insns defining and/or
5986 using a certain 'count' register and (2) the loop count can be
5987 adjusted by modifying this register prior to the loop.
5988 ??? The possible introduction of a new block to initialize the
5989 new IV can potentially affect branch optimizations. */
5991 if (GET_MODE (operands[0]) != DImode)
5995 insn = emit_insn (gen_adddi3_compare0 (s0, s0, GEN_INT (-1)));
5997 cmp = XVECEXP (PATTERN (insn), 0, 0);
5998 cc_reg = SET_DEST (cmp);
5999 bcomp = gen_rtx_NE (VOIDmode, cc_reg, const0_rtx);
6000 loc_ref = gen_rtx_LABEL_REF (VOIDmode, operands [1]);
6001 emit_jump_insn (gen_rtx_SET (pc_rtx,
6002 gen_rtx_IF_THEN_ELSE (VOIDmode, bcomp,
6007 ;; Helper for aarch64.c code.
6008 (define_expand "set_clobber_cc"
6009 [(parallel [(set (match_operand 0)
6011 (clobber (reg:CC CC_REGNUM))])])
6014 (include "aarch64-simd.md")
6016 ;; Atomic Operations
6017 (include "atomics.md")
6019 ;; ldp/stp peephole patterns
6020 (include "aarch64-ldpstp.md")
6023 (include "aarch64-sve.md")