gcc/config/aarch64/aarch64-simd.md

   1 ;; Machine description for AArch64 AdvSIMD architecture.
   2 ;; Copyright (C) 2011-2013 Free Software Foundation, Inc.
   3 ;; Contributed by ARM Ltd.
   4 ;;
   5 ;; This file is part of GCC.
   6 ;;
   7 ;; GCC is free software; you can redistribute it and/or modify it
   8 ;; under the terms of the GNU General Public License as published by
   9 ;; the Free Software Foundation; either version 3, or (at your option)
  10 ;; any later version.
  11 ;;
  12 ;; GCC is distributed in the hope that it will be useful, but
  13 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
  14 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15 ;; General Public License for more details.
  16 ;;
  17 ;; You should have received a copy of the GNU General Public License
  18 ;; along with GCC; see the file COPYING3.  If not see
  19 ;; <http://www.gnu.org/licenses/>.
  20
  21
  22 ; Main data types used by the insntructions
  23
  24 (define_attr "simd_mode" "unknown,none,V8QI,V16QI,V4HI,V8HI,V2SI,V4SI,V2DI,V2SF,V4SF,V2DF,OI,CI,XI,DI,DF,SI,HI,QI"
  25   (const_string "unknown"))
  26
  27
  28 ; Classification of AdvSIMD instructions for scheduling purposes.
  29 ; Do not set this attribute and the "v8type" attribute together in
  30 ; any instruction pattern.
  31
  32 ; simd_abd              integer absolute difference and accumulate.
  33 ; simd_abdl             integer absolute difference and accumulate (long).
  34 ; simd_adal             integer add and accumulate (long).
  35 ; simd_add              integer addition/subtraction.
  36 ; simd_addl             integer addition/subtraction (long).
  37 ; simd_addlv            across lanes integer sum (long).
  38 ; simd_addn             integer addition/subtraction (narrow).
  39 ; simd_addn2            integer addition/subtraction (narrow, high).
  40 ; simd_addv             across lanes integer sum.
  41 ; simd_cls              count leading sign/zero bits.
  42 ; simd_cmp              compare / create mask.
  43 ; simd_cnt              population count.
  44 ; simd_dup              duplicate element.
  45 ; simd_dupgp            duplicate general purpose register.
  46 ; simd_ext              bitwise extract from pair.
  47 ; simd_fabd             floating absolute difference and accumulate.
  48 ; simd_fadd             floating point add/sub.
  49 ; simd_fcmp             floating point compare.
  50 ; simd_fcvti            floating point convert to integer.
  51 ; simd_fcvtl            floating-point convert upsize.
  52 ; simd_fcvtn            floating-point convert downsize (narrow).
  53 ; simd_fcvtn2           floating-point convert downsize (narrow, high).
  54 ; simd_fdiv             floating point division.
  55 ; simd_fminmax          floating point min/max.
  56 ; simd_fminmaxv         across lanes floating point min/max.
  57 ; simd_fmla             floating point multiply-add.
  58 ; simd_fmla_elt         floating point multiply-add (by element).
  59 ; simd_fmul             floating point multiply.
  60 ; simd_fmul_elt         floating point multiply (by element).
  61 ; simd_fnegabs          floating point neg/abs.
  62 ; simd_frecpe            floating point reciprocal estimate.
  63 ; simd_frecps            floating point reciprocal step.
  64 ; simd_frecpx            floating point reciprocal exponent.
  65 ; simd_frint            floating point round to integer.
  66 ; simd_fsqrt            floating point square root.
  67 ; simd_icvtf            integer convert to floating point.
  68 ; simd_ins              insert element.
  69 ; simd_insgp            insert general purpose register.
  70 ; simd_load1            load multiple structures to one register (LD1).
  71 ; simd_load1r           load single structure to all lanes of one register (LD1R).
  72 ; simd_load1s           load single structure to one lane of one register (LD1 [index]).
  73 ; simd_load2            load multiple structures to two registers (LD1, LD2).
  74 ; simd_load2r           load single structure to all lanes of two registers (LD1R, LD2R).
  75 ; simd_load2s           load single structure to one lane of two registers (LD2 [index]).
  76 ; simd_load3            load multiple structures to three registers (LD1, LD3).
  77 ; simd_load3r           load single structure to all lanes of three registers (LD3R).
  78 ; simd_load3s           load single structure to one lane of three registers (LD3 [index]).
  79 ; simd_load4            load multiple structures to four registers (LD1, LD2, LD4).
  80 ; simd_load4r           load single structure to all lanes of four registers (LD4R).
  81 ; simd_load4s           load single structure to one lane of four registers (LD4 [index]).
  82 ; simd_logic            logical operation.
  83 ; simd_logic_imm        logcial operation (immediate).
  84 ; simd_minmax           integer min/max.
  85 ; simd_minmaxv          across lanes integer min/max,
  86 ; simd_mla              integer multiply-accumulate.
  87 ; simd_mla_elt          integer multiply-accumulate (by element).
  88 ; simd_mlal             integer multiply-accumulate (long).
  89 ; simd_mlal_elt         integer multiply-accumulate (by element, long).
  90 ; simd_move             move register.
  91 ; simd_move_imm         move immediate.
  92 ; simd_movgp            move element to general purpose register.
  93 ; simd_mul              integer multiply.
  94 ; simd_mul_elt          integer multiply (by element).
  95 ; simd_mull             integer multiply (long).
  96 ; simd_mull_elt         integer multiply (by element, long).
  97 ; simd_negabs           integer negate/absolute.
  98 ; simd_rbit             bitwise reverse.
  99 ; simd_rcpe             integer reciprocal estimate.
 100 ; simd_rcps             integer reciprocal square root.
 101 ; simd_rev              element reverse.
 102 ; simd_sat_add          integer saturating addition/subtraction.
 103 ; simd_sat_mlal         integer saturating multiply-accumulate (long).
 104 ; simd_sat_mlal_elt     integer saturating multiply-accumulate (by element, long).
 105 ; simd_sat_mul          integer saturating multiply.
 106 ; simd_sat_mul_elt      integer saturating multiply (by element).
 107 ; simd_sat_mull         integer saturating multiply (long).
 108 ; simd_sat_mull_elt     integer saturating multiply (by element, long).
 109 ; simd_sat_negabs       integer saturating negate/absolute.
 110 ; simd_sat_shift        integer saturating shift.
 111 ; simd_sat_shift_imm    integer saturating shift (immediate).
 112 ; simd_sat_shiftn_imm   integer saturating shift (narrow, immediate).
 113 ; simd_sat_shiftn2_imm  integer saturating shift (narrow, high, immediate).
 114 ; simd_shift            shift register/vector.
 115 ; simd_shift_acc        shift accumulate.
 116 ; simd_shift_imm        shift immediate.
 117 ; simd_shift_imm_acc    shift immediate and accumualte.
 118 ; simd_shiftl           shift register/vector (long).
 119 ; simd_shiftl_imm       shift register/vector (long, immediate).
 120 ; simd_shiftn_imm       shift register/vector (narrow, immediate).
 121 ; simd_shiftn2_imm      shift register/vector (narrow, high, immediate).
 122 ; simd_store1           store multiple structures from one register (ST1).
 123 ; simd_store1s          store single structure from one lane of one register (ST1 [index]).
 124 ; simd_store2           store multiple structures from two registers (ST1, ST2).
 125 ; simd_store2s          store single structure from one lane of two registers (ST2 [index]).
 126 ; simd_store3           store multiple structures from three registers (ST1, ST3).
 127 ; simd_store3s          store single structure from one lane of three register (ST3 [index]).
 128 ; simd_store4           store multiple structures from four registers (ST1, ST2, ST4).
 129 ; simd_store4s          store single structure from one lane for four registers (ST4 [index]).
 130 ; simd_tbl              table lookup.
 131 ; simd_trn              transpose.
 132 ; simd_uzp              unzip.
 133 ; simd_zip              zip.
 134
 135 (define_attr "simd_type"
 136    "simd_abd,\
 137    simd_abdl,\
 138    simd_adal,\
 139    simd_add,\
 140    simd_addl,\
 141    simd_addlv,\
 142    simd_addn,\
 143    simd_addn2,\
 144    simd_addv,\
 145    simd_cls,\
 146    simd_cmp,\
 147    simd_cnt,\
 148    simd_dup,\
 149    simd_dupgp,\
 150    simd_ext,\
 151    simd_fabd,\
 152    simd_fadd,\
 153    simd_fcmp,\
 154    simd_fcvti,\
 155    simd_fcvtl,\
 156    simd_fcvtn,\
 157    simd_fcvtn2,\
 158    simd_fdiv,\
 159    simd_fminmax,\
 160    simd_fminmaxv,\
 161    simd_fmla,\
 162    simd_fmla_elt,\
 163    simd_fmul,\
 164    simd_fmul_elt,\
 165    simd_fnegabs,\
 166    simd_frecpe,\
 167    simd_frecps,\
 168    simd_frecpx,\
 169    simd_frint,\
 170    simd_fsqrt,\
 171    simd_icvtf,\
 172    simd_ins,\
 173    simd_insgp,\
 174    simd_load1,\
 175    simd_load1r,\
 176    simd_load1s,\
 177    simd_load2,\
 178    simd_load2r,\
 179    simd_load2s,\
 180    simd_load3,\
 181    simd_load3r,\
 182    simd_load3s,\
 183    simd_load4,\
 184    simd_load4r,\
 185    simd_load4s,\
 186    simd_logic,\
 187    simd_logic_imm,\
 188    simd_minmax,\
 189    simd_minmaxv,\
 190    simd_mla,\
 191    simd_mla_elt,\
 192    simd_mlal,\
 193    simd_mlal_elt,\
 194    simd_movgp,\
 195    simd_move,\
 196    simd_move_imm,\
 197    simd_mul,\
 198    simd_mul_elt,\
 199    simd_mull,\
 200    simd_mull_elt,\
 201    simd_negabs,\
 202    simd_rbit,\
 203    simd_rcpe,\
 204    simd_rcps,\
 205    simd_rev,\
 206    simd_sat_add,\
 207    simd_sat_mlal,\
 208    simd_sat_mlal_elt,\
 209    simd_sat_mul,\
 210    simd_sat_mul_elt,\
 211    simd_sat_mull,\
 212    simd_sat_mull_elt,\
 213    simd_sat_negabs,\
 214    simd_sat_shift,\
 215    simd_sat_shift_imm,\
 216    simd_sat_shiftn_imm,\
 217    simd_sat_shiftn2_imm,\
 218    simd_shift,\
 219    simd_shift_acc,\
 220    simd_shift_imm,\
 221    simd_shift_imm_acc,\
 222    simd_shiftl,\
 223    simd_shiftl_imm,\
 224    simd_shiftn_imm,\
 225    simd_shiftn2_imm,\
 226    simd_store1,\
 227    simd_store1s,\
 228    simd_store2,\
 229    simd_store2s,\
 230    simd_store3,\
 231    simd_store3s,\
 232    simd_store4,\
 233    simd_store4s,\
 234    simd_tbl,\
 235    simd_trn,\
 236    simd_uzp,\
 237    simd_zip,\
 238    none"
 239   (const_string "none"))
 240
 241
 242 ; The "neon_type" attribute is used by the AArch32 backend.  Below is a mapping
 243 ; from "simd_type" to "neon_type".
 244
 245 (define_attr "neon_type"
 246    "neon_int_1,neon_int_2,neon_int_3,neon_int_4,neon_int_5,neon_vqneg_vqabs,
 247    neon_vmov,neon_vaba,neon_vsma,neon_vaba_qqq,
 248    neon_mul_ddd_8_16_qdd_16_8_long_32_16_long,neon_mul_qqq_8_16_32_ddd_32,
 249    neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar,
 250    neon_mla_ddd_8_16_qdd_16_8_long_32_16_long,neon_mla_qqq_8_16,
 251    neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long,
 252    neon_mla_qqq_32_qqd_32_scalar,neon_mul_ddd_16_scalar_32_16_long_scalar,
 253    neon_mul_qqd_32_scalar,neon_mla_ddd_16_scalar_qdd_32_16_long_scalar,
 254    neon_shift_1,neon_shift_2,neon_shift_3,neon_vshl_ddd,
 255    neon_vqshl_vrshl_vqrshl_qqq,neon_vsra_vrsra,neon_fp_vadd_ddd_vabs_dd,
 256    neon_fp_vadd_qqq_vabs_qq,neon_fp_vsum,neon_fp_vmul_ddd,neon_fp_vmul_qqd,
 257    neon_fp_vmla_ddd,neon_fp_vmla_qqq,neon_fp_vmla_ddd_scalar,
 258    neon_fp_vmla_qqq_scalar,neon_fp_vrecps_vrsqrts_ddd,
 259    neon_fp_vrecps_vrsqrts_qqq,neon_bp_simple,neon_bp_2cycle,neon_bp_3cycle,
 260    neon_ldr,neon_str,neon_vld1_1_2_regs,neon_vld1_3_4_regs,
 261    neon_vld2_2_regs_vld1_vld2_all_lanes,neon_vld2_4_regs,neon_vld3_vld4,
 262    neon_vst1_1_2_regs_vst2_2_regs,neon_vst1_3_4_regs,
 263    neon_vst2_4_regs_vst3_vst4,neon_vst3_vst4,neon_vld1_vld2_lane,
 264    neon_vld3_vld4_lane,neon_vst1_vst2_lane,neon_vst3_vst4_lane,
 265    neon_vld3_vld4_all_lanes,neon_mcr,neon_mcr_2_mcrr,neon_mrc,neon_mrrc,
 266    neon_ldm_2,neon_stm_2,none,unknown"
 267   (cond [
 268           (eq_attr "simd_type" "simd_dup") (const_string "neon_bp_simple")
 269           (eq_attr "simd_type" "simd_movgp") (const_string "neon_bp_simple")
 270           (eq_attr "simd_type" "simd_add,simd_logic,simd_logic_imm") (const_string "neon_int_1")
 271           (eq_attr "simd_type" "simd_negabs,simd_addlv") (const_string "neon_int_3")
 272           (eq_attr "simd_type" "simd_addn,simd_addn2,simd_addl,simd_sat_add,simd_sat_negabs") (const_string "neon_int_4")
 273           (eq_attr "simd_type" "simd_move") (const_string "neon_vmov")
 274           (eq_attr "simd_type" "simd_ins") (const_string "neon_mcr")
 275           (and (eq_attr "simd_type" "simd_mul,simd_sat_mul") (eq_attr "simd_mode" "V8QI,V4HI")) (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long")
 276           (and (eq_attr "simd_type" "simd_mul,simd_sat_mul") (eq_attr "simd_mode" "V2SI,V8QI,V16QI,V2SI")) (const_string "neon_mul_qqq_8_16_32_ddd_32")
 277           (and (eq_attr "simd_type" "simd_mull,simd_sat_mull") (eq_attr "simd_mode" "V8QI,V16QI,V4HI,V8HI")) (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long")
 278           (and (eq_attr "simd_type" "simd_mull,simd_sat_mull") (eq_attr "simd_mode" "V2SI,V4SI,V2DI")) (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")
 279           (and (eq_attr "simd_type" "simd_mla,simd_sat_mlal") (eq_attr "simd_mode" "V8QI,V4HI")) (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
 280           (and (eq_attr "simd_type" "simd_mla,simd_sat_mlal") (eq_attr "simd_mode" "V2SI")) (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")
 281           (and (eq_attr "simd_type" "simd_mla,simd_sat_mlal") (eq_attr "simd_mode" "V16QI,V8HI")) (const_string "neon_mla_qqq_8_16")
 282           (and (eq_attr "simd_type" "simd_mla,simd_sat_mlal") (eq_attr "simd_mode" "V4SI")) (const_string "neon_mla_qqq_32_qqd_32_scalar")
 283           (and (eq_attr "simd_type" "simd_mlal") (eq_attr "simd_mode" "V8QI,V16QI,V4HI,V8HI")) (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
 284           (and (eq_attr "simd_type" "simd_mlal") (eq_attr "simd_mode" "V2SI,V4SI,V2DI")) (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")
 285           (and (eq_attr "simd_type" "simd_fmla") (eq_attr "simd_mode" "V2SF")) (const_string "neon_fp_vmla_ddd")
 286           (and (eq_attr "simd_type" "simd_fmla") (eq_attr "simd_mode" "V4SF,V2DF")) (const_string "neon_fp_vmla_qqq")
 287           (and (eq_attr "simd_type" "simd_fmla_elt") (eq_attr "simd_mode" "V2SF")) (const_string "neon_fp_vmla_ddd_scalar")
 288           (and (eq_attr "simd_type" "simd_fmla_elt") (eq_attr "simd_mode" "V4SF,V2DF")) (const_string "neon_fp_vmla_qqq_scalar")
 289           (and (eq_attr "simd_type" "simd_fmul,simd_fmul_elt,simd_fdiv,simd_fsqrt") (eq_attr "simd_mode" "V2SF")) (const_string "neon_fp_vmul_ddd")
 290           (and (eq_attr "simd_type" "simd_fmul,simd_fmul_elt,simd_fdiv,simd_fsqrt") (eq_attr "simd_mode" "V4SF,V2DF")) (const_string "neon_fp_vmul_qqd")
 291           (and (eq_attr "simd_type" "simd_fadd") (eq_attr "simd_mode" "V2SF")) (const_string "neon_fp_vadd_ddd_vabs_dd")
 292           (and (eq_attr "simd_type" "simd_fadd") (eq_attr "simd_mode" "V4SF,V2DF")) (const_string "neon_fp_vadd_qqq_vabs_qq")
 293           (and (eq_attr "simd_type" "simd_fnegabs,simd_fminmax,simd_fminmaxv") (eq_attr "simd_mode" "V2SF")) (const_string "neon_fp_vadd_ddd_vabs_dd")
 294           (and (eq_attr "simd_type" "simd_fnegabs,simd_fminmax,simd_fminmaxv") (eq_attr "simd_mode" "V4SF,V2DF")) (const_string "neon_fp_vadd_qqq_vabs_qq")
 295           (and (eq_attr "simd_type" "simd_shift,simd_shift_acc") (eq_attr "simd_mode" "V8QI,V4HI,V2SI")) (const_string "neon_vshl_ddd")
 296           (and (eq_attr "simd_type" "simd_shift,simd_shift_acc") (eq_attr "simd_mode" "V16QI,V8HI,V4SI,V2DI")) (const_string "neon_shift_3")
 297           (eq_attr "simd_type" "simd_minmax,simd_minmaxv") (const_string "neon_int_5")
 298           (eq_attr "simd_type" "simd_shiftn_imm,simd_shiftn2_imm,simd_shiftl_imm,") (const_string "neon_shift_1")
 299           (eq_attr "simd_type" "simd_load1,simd_load2") (const_string "neon_vld1_1_2_regs")
 300           (eq_attr "simd_type" "simd_load3,simd_load3") (const_string "neon_vld1_3_4_regs")
 301           (eq_attr "simd_type" "simd_load1r,simd_load2r,simd_load3r,simd_load4r") (const_string "neon_vld2_2_regs_vld1_vld2_all_lanes")
 302           (eq_attr "simd_type" "simd_load1s,simd_load2s") (const_string "neon_vld1_vld2_lane")
 303           (eq_attr "simd_type" "simd_load3s,simd_load4s") (const_string "neon_vld3_vld4_lane")
 304           (eq_attr "simd_type" "simd_store1,simd_store2") (const_string "neon_vst1_1_2_regs_vst2_2_regs")
 305           (eq_attr "simd_type" "simd_store3,simd_store4") (const_string "neon_vst1_3_4_regs")
 306           (eq_attr "simd_type" "simd_store1s,simd_store2s") (const_string "neon_vst1_vst2_lane")
 307           (eq_attr "simd_type" "simd_store3s,simd_store4s") (const_string "neon_vst3_vst4_lane")
 308           (and (eq_attr "simd_type" "simd_frecpe,simd_frecps") (eq_attr "simd_mode" "V2SF")) (const_string "neon_fp_vrecps_vrsqrts_ddd")
 309           (and (eq_attr "simd_type" "simd_frecpe,simd_frecps") (eq_attr "simd_mode" "V4SF,V2DF")) (const_string "neon_fp_vrecps_vrsqrts_qqq")
 310           (eq_attr "simd_type" "none") (const_string "none")
 311   ]
 312   (const_string "unknown")))
 313
 314
 315 (define_expand "mov<mode>"
 316   [(set (match_operand:VALL 0 "aarch64_simd_nonimmediate_operand" "")
 317         (match_operand:VALL 1 "aarch64_simd_general_operand" ""))]
 318   "TARGET_SIMD"
 319   "
 320     if (GET_CODE (operands[0]) == MEM)
 321       operands[1] = force_reg (<MODE>mode, operands[1]);
 322   "
 323 )
 324
 325 (define_expand "movmisalign<mode>"
 326   [(set (match_operand:VALL 0 "aarch64_simd_nonimmediate_operand" "")
 327         (match_operand:VALL 1 "aarch64_simd_general_operand" ""))]
 328   "TARGET_SIMD"
 329 {
 330   /* This pattern is not permitted to fail during expansion: if both arguments
 331      are non-registers (e.g. memory := constant, which can be created by the
 332      auto-vectorizer), force operand 1 into a register.  */
 333   if (!register_operand (operands[0], <MODE>mode)
 334       && !register_operand (operands[1], <MODE>mode))
 335     operands[1] = force_reg (<MODE>mode, operands[1]);
 336 })
 337
 338 (define_insn "aarch64_simd_dup<mode>"
 339   [(set (match_operand:VDQ 0 "register_operand" "=w")
 340         (vec_duplicate:VDQ (match_operand:<VEL> 1 "register_operand" "r")))]
 341   "TARGET_SIMD"
 342   "dup\\t%0.<Vtype>, %<vw>1"
 343   [(set_attr "simd_type" "simd_dupgp")
 344    (set_attr "simd_mode" "<MODE>")]
 345 )
 346
 347 (define_insn "aarch64_dup_lane<mode>"
 348   [(set (match_operand:VDQ_I 0 "register_operand" "=w")
 349         (vec_duplicate:VDQ_I
 350           (vec_select:<VEL>
 351             (match_operand:<VCON> 1 "register_operand" "w")
 352             (parallel [(match_operand:SI 2 "immediate_operand" "i")])
 353           )))]
 354   "TARGET_SIMD"
 355   "dup\\t%<v>0<Vmtype>, %1.<Vetype>[%2]"
 356   [(set_attr "simd_type" "simd_dup")
 357    (set_attr "simd_mode" "<MODE>")]
 358 )
 359
 360 (define_insn "aarch64_dup_lane<mode>"
 361   [(set (match_operand:SDQ_I 0 "register_operand" "=w")
 362         (vec_select:<VEL>
 363           (match_operand:<VCON> 1 "register_operand" "w")
 364           (parallel [(match_operand:SI 2 "immediate_operand" "i")])
 365         ))]
 366   "TARGET_SIMD"
 367   "dup\\t%<v>0<Vmtype>, %1.<Vetype>[%2]"
 368   [(set_attr "simd_type" "simd_dup")
 369    (set_attr "simd_mode" "<MODE>")]
 370 )
 371
 372 (define_insn "aarch64_simd_dup<mode>"
 373   [(set (match_operand:VDQF 0 "register_operand" "=w")
 374         (vec_duplicate:VDQF (match_operand:<VEL> 1 "register_operand" "w")))]
 375   "TARGET_SIMD"
 376   "dup\\t%0.<Vtype>, %1.<Vetype>[0]"
 377   [(set_attr "simd_type" "simd_dup")
 378    (set_attr "simd_mode" "<MODE>")]
 379 )
 380
 381 (define_insn "*aarch64_simd_mov<mode>"
 382   [(set (match_operand:VD 0 "aarch64_simd_nonimmediate_operand"
 383                 "=w, Utv,  w, ?r, ?w, ?r, w")
 384         (match_operand:VD 1 "aarch64_simd_general_operand"
 385                 "Utv,  w,  w,  w,  r,  r, Dn"))]
 386   "TARGET_SIMD
 387    && (register_operand (operands[0], <MODE>mode)
 388        || register_operand (operands[1], <MODE>mode))"
 389 {
 390    switch (which_alternative)
 391      {
 392      case 0: return "ld1\t{%0.<Vtype>}, %1";
 393      case 1: return "st1\t{%1.<Vtype>}, %0";
 394      case 2: return "orr\t%0.<Vbtype>, %1.<Vbtype>, %1.<Vbtype>";
 395      case 3: return "umov\t%0, %1.d[0]";
 396      case 4: return "ins\t%0.d[0], %1";
 397      case 5: return "mov\t%0, %1";
 398      case 6:
 399         return aarch64_output_simd_mov_immediate (&operands[1],
 400                                                   <MODE>mode, 64);
 401      default: gcc_unreachable ();
 402      }
 403 }
 404   [(set_attr "simd_type" "simd_load1,simd_store1,simd_move,simd_movgp,simd_insgp,simd_move,simd_move_imm")
 405    (set_attr "simd_mode" "<MODE>")]
 406 )
 407
 408 (define_insn "*aarch64_simd_mov<mode>"
 409   [(set (match_operand:VQ 0 "aarch64_simd_nonimmediate_operand"
 410                 "=w, Utv,  w, ?r, ?w, ?r, w")
 411         (match_operand:VQ 1 "aarch64_simd_general_operand"
 412                 "Utv,  w,  w,  w,  r,  r, Dn"))]
 413   "TARGET_SIMD
 414    && (register_operand (operands[0], <MODE>mode)
 415        || register_operand (operands[1], <MODE>mode))"
 416 {
 417   switch (which_alternative)
 418     {
 419     case 0: return "ld1\t{%0.<Vtype>}, %1";
 420     case 1: return "st1\t{%1.<Vtype>}, %0";
 421     case 2: return "orr\t%0.<Vbtype>, %1.<Vbtype>, %1.<Vbtype>";
 422     case 3: return "umov\t%0, %1.d[0]\;umov\t%H0, %1.d[1]";
 423     case 4: return "ins\t%0.d[0], %1\;ins\t%0.d[1], %H1";
 424     case 5: return "#";
 425     case 6:
 426         return aarch64_output_simd_mov_immediate (&operands[1],
 427                                                   <MODE>mode, 128);
 428     default: gcc_unreachable ();
 429     }
 430 }
 431   [(set_attr "simd_type" "simd_load1,simd_store1,simd_move,simd_movgp,simd_insgp,simd_move,simd_move_imm")
 432    (set_attr "simd_mode" "<MODE>")
 433    (set_attr "length" "4,4,4,8,8,8,4")]
 434 )
 435
 436 (define_split
 437   [(set (match_operand:VQ 0 "register_operand" "")
 438       (match_operand:VQ 1 "register_operand" ""))]
 439   "TARGET_SIMD && reload_completed
 440    && GP_REGNUM_P (REGNO (operands[0]))
 441    && GP_REGNUM_P (REGNO (operands[1]))"
 442   [(set (match_dup 0) (match_dup 1))
 443    (set (match_dup 2) (match_dup 3))]
 444 {
 445   int rdest = REGNO (operands[0]);
 446   int rsrc = REGNO (operands[1]);
 447   rtx dest[2], src[2];
 448
 449   dest[0] = gen_rtx_REG (DImode, rdest);
 450   src[0] = gen_rtx_REG (DImode, rsrc);
 451   dest[1] = gen_rtx_REG (DImode, rdest + 1);
 452   src[1] = gen_rtx_REG (DImode, rsrc + 1);
 453
 454   aarch64_simd_disambiguate_copy (operands, dest, src, 2);
 455 })
 456
 457 (define_insn "orn<mode>3"
 458  [(set (match_operand:VDQ 0 "register_operand" "=w")
 459        (ior:VDQ (not:VDQ (match_operand:VDQ 1 "register_operand" "w"))
 460                 (match_operand:VDQ 2 "register_operand" "w")))]
 461  "TARGET_SIMD"
 462  "orn\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>"
 463   [(set_attr "simd_type" "simd_logic")
 464    (set_attr "simd_mode" "<MODE>")]
 465 )
 466
 467 (define_insn "bic<mode>3"
 468  [(set (match_operand:VDQ 0 "register_operand" "=w")
 469        (and:VDQ (not:VDQ (match_operand:VDQ 1 "register_operand" "w"))
 470                 (match_operand:VDQ 2 "register_operand" "w")))]
 471  "TARGET_SIMD"
 472  "bic\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>"
 473   [(set_attr "simd_type" "simd_logic")
 474    (set_attr "simd_mode" "<MODE>")]
 475 )
 476
 477 (define_insn "add<mode>3"
 478   [(set (match_operand:VDQ 0 "register_operand" "=w")
 479         (plus:VDQ (match_operand:VDQ 1 "register_operand" "w")
 480                   (match_operand:VDQ 2 "register_operand" "w")))]
 481   "TARGET_SIMD"
 482   "add\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
 483   [(set_attr "simd_type" "simd_add")
 484    (set_attr "simd_mode" "<MODE>")]
 485 )
 486
 487 (define_insn "sub<mode>3"
 488   [(set (match_operand:VDQ 0 "register_operand" "=w")
 489         (minus:VDQ (match_operand:VDQ 1 "register_operand" "w")
 490                    (match_operand:VDQ 2 "register_operand" "w")))]
 491   "TARGET_SIMD"
 492   "sub\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
 493   [(set_attr "simd_type" "simd_add")
 494    (set_attr "simd_mode" "<MODE>")]
 495 )
 496
 497 (define_insn "mul<mode>3"
 498   [(set (match_operand:VDQM 0 "register_operand" "=w")
 499         (mult:VDQM (match_operand:VDQM 1 "register_operand" "w")
 500                    (match_operand:VDQM 2 "register_operand" "w")))]
 501   "TARGET_SIMD"
 502   "mul\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
 503   [(set_attr "simd_type" "simd_mul")
 504    (set_attr "simd_mode" "<MODE>")]
 505 )
 506
 507 (define_insn "neg<mode>2"
 508   [(set (match_operand:VDQ 0 "register_operand" "=w")
 509         (neg:VDQ (match_operand:VDQ 1 "register_operand" "w")))]
 510   "TARGET_SIMD"
 511   "neg\t%0.<Vtype>, %1.<Vtype>"
 512   [(set_attr "simd_type" "simd_negabs")
 513    (set_attr "simd_mode" "<MODE>")]
 514 )
 515
 516 (define_insn "abs<mode>2"
 517   [(set (match_operand:VDQ 0 "register_operand" "=w")
 518         (abs:VDQ (match_operand:VDQ 1 "register_operand" "w")))]
 519   "TARGET_SIMD"
 520   "abs\t%0.<Vtype>, %1.<Vtype>"
 521   [(set_attr "simd_type" "simd_negabs")
 522    (set_attr "simd_mode" "<MODE>")]
 523 )
 524
 525 (define_insn "abd<mode>_3"
 526   [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
 527         (abs:VDQ_BHSI (minus:VDQ_BHSI
 528                        (match_operand:VDQ_BHSI 1 "register_operand" "w")
 529                        (match_operand:VDQ_BHSI 2 "register_operand" "w"))))]
 530   "TARGET_SIMD"
 531   "sabd\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
 532   [(set_attr "simd_type" "simd_abd")
 533    (set_attr "simd_mode" "<MODE>")]
 534 )
 535
 536 (define_insn "aba<mode>_3"
 537   [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
 538         (plus:VDQ_BHSI (abs:VDQ_BHSI (minus:VDQ_BHSI
 539                          (match_operand:VDQ_BHSI 1 "register_operand" "w")
 540                          (match_operand:VDQ_BHSI 2 "register_operand" "w")))
 541                        (match_operand:VDQ_BHSI 3 "register_operand" "0")))]
 542   "TARGET_SIMD"
 543   "saba\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
 544   [(set_attr "simd_type" "simd_abd")
 545    (set_attr "simd_mode" "<MODE>")]
 546 )
 547
 548 (define_insn "fabd<mode>_3"
 549   [(set (match_operand:VDQF 0 "register_operand" "=w")
 550         (abs:VDQF (minus:VDQF
 551                    (match_operand:VDQF 1 "register_operand" "w")
 552                    (match_operand:VDQF 2 "register_operand" "w"))))]
 553   "TARGET_SIMD"
 554   "fabd\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
 555   [(set_attr "simd_type" "simd_fabd")
 556    (set_attr "simd_mode" "<MODE>")]
 557 )
 558
 559 (define_insn "and<mode>3"
 560   [(set (match_operand:VDQ 0 "register_operand" "=w")
 561         (and:VDQ (match_operand:VDQ 1 "register_operand" "w")
 562                  (match_operand:VDQ 2 "register_operand" "w")))]
 563   "TARGET_SIMD"
 564   "and\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>"
 565   [(set_attr "simd_type" "simd_logic")
 566    (set_attr "simd_mode" "<MODE>")]
 567 )
 568
 569 (define_insn "ior<mode>3"
 570   [(set (match_operand:VDQ 0 "register_operand" "=w")
 571         (ior:VDQ (match_operand:VDQ 1 "register_operand" "w")
 572                  (match_operand:VDQ 2 "register_operand" "w")))]
 573   "TARGET_SIMD"
 574   "orr\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>"
 575   [(set_attr "simd_type" "simd_logic")
 576    (set_attr "simd_mode" "<MODE>")]
 577 )
 578
 579 (define_insn "xor<mode>3"
 580   [(set (match_operand:VDQ 0 "register_operand" "=w")
 581         (xor:VDQ (match_operand:VDQ 1 "register_operand" "w")
 582                  (match_operand:VDQ 2 "register_operand" "w")))]
 583   "TARGET_SIMD"
 584   "eor\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>"
 585   [(set_attr "simd_type" "simd_logic")
 586    (set_attr "simd_mode" "<MODE>")]
 587 )
 588
 589 (define_insn "one_cmpl<mode>2"
 590   [(set (match_operand:VDQ 0 "register_operand" "=w")
 591         (not:VDQ (match_operand:VDQ 1 "register_operand" "w")))]
 592   "TARGET_SIMD"
 593   "not\t%0.<Vbtype>, %1.<Vbtype>"
 594   [(set_attr "simd_type" "simd_logic")
 595    (set_attr "simd_mode" "<MODE>")]
 596 )
 597
 598 (define_insn "aarch64_simd_vec_set<mode>"
 599   [(set (match_operand:VQ_S 0 "register_operand" "=w")
 600         (vec_merge:VQ_S
 601             (vec_duplicate:VQ_S
 602                 (match_operand:<VEL> 1 "register_operand" "r"))
 603             (match_operand:VQ_S 3 "register_operand" "0")
 604             (match_operand:SI 2 "immediate_operand" "i")))]
 605   "TARGET_SIMD"
 606   "ins\t%0.<Vetype>[%p2], %w1";
 607   [(set_attr "simd_type" "simd_insgp")
 608    (set_attr "simd_mode" "<MODE>")]
 609 )
 610
 611 (define_insn "aarch64_simd_lshr<mode>"
 612  [(set (match_operand:VDQ 0 "register_operand" "=w")
 613        (lshiftrt:VDQ (match_operand:VDQ 1 "register_operand" "w")
 614                      (match_operand:VDQ  2 "aarch64_simd_rshift_imm" "Dr")))]
 615  "TARGET_SIMD"
 616  "ushr\t%0.<Vtype>, %1.<Vtype>, %2"
 617   [(set_attr "simd_type" "simd_shift_imm")
 618    (set_attr "simd_mode" "<MODE>")]
 619 )
 620
 621 (define_insn "aarch64_simd_ashr<mode>"
 622  [(set (match_operand:VDQ 0 "register_operand" "=w")
 623        (ashiftrt:VDQ (match_operand:VDQ 1 "register_operand" "w")
 624                      (match_operand:VDQ  2 "aarch64_simd_rshift_imm" "Dr")))]
 625  "TARGET_SIMD"
 626  "sshr\t%0.<Vtype>, %1.<Vtype>, %2"
 627   [(set_attr "simd_type" "simd_shift_imm")
 628    (set_attr "simd_mode" "<MODE>")]
 629 )
 630
 631 (define_insn "aarch64_simd_imm_shl<mode>"
 632  [(set (match_operand:VDQ 0 "register_operand" "=w")
 633        (ashift:VDQ (match_operand:VDQ 1 "register_operand" "w")
 634                    (match_operand:VDQ  2 "aarch64_simd_lshift_imm" "Dl")))]
 635  "TARGET_SIMD"
 636   "shl\t%0.<Vtype>, %1.<Vtype>, %2"
 637   [(set_attr "simd_type" "simd_shift_imm")
 638    (set_attr "simd_mode" "<MODE>")]
 639 )
 640
 641 (define_insn "aarch64_simd_reg_sshl<mode>"
 642  [(set (match_operand:VDQ 0 "register_operand" "=w")
 643        (ashift:VDQ (match_operand:VDQ 1 "register_operand" "w")
 644                    (match_operand:VDQ 2 "register_operand" "w")))]
 645  "TARGET_SIMD"
 646  "sshl\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
 647   [(set_attr "simd_type" "simd_shift")
 648    (set_attr "simd_mode" "<MODE>")]
 649 )
 650
 651 (define_insn "aarch64_simd_reg_shl<mode>_unsigned"
 652  [(set (match_operand:VDQ 0 "register_operand" "=w")
 653        (unspec:VDQ [(match_operand:VDQ 1 "register_operand" "w")
 654                     (match_operand:VDQ 2 "register_operand" "w")]
 655                    UNSPEC_ASHIFT_UNSIGNED))]
 656  "TARGET_SIMD"
 657  "ushl\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
 658   [(set_attr "simd_type" "simd_shift")
 659    (set_attr "simd_mode" "<MODE>")]
 660 )
 661
 662 (define_insn "aarch64_simd_reg_shl<mode>_signed"
 663  [(set (match_operand:VDQ 0 "register_operand" "=w")
 664        (unspec:VDQ [(match_operand:VDQ 1 "register_operand" "w")
 665                     (match_operand:VDQ 2 "register_operand" "w")]
 666                    UNSPEC_ASHIFT_SIGNED))]
 667  "TARGET_SIMD"
 668  "sshl\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
 669   [(set_attr "simd_type" "simd_shift")
 670    (set_attr "simd_mode" "<MODE>")]
 671 )
 672
 673 (define_expand "ashl<mode>3"
 674   [(match_operand:VDQ 0 "register_operand" "")
 675    (match_operand:VDQ 1 "register_operand" "")
 676    (match_operand:SI  2 "general_operand" "")]
 677  "TARGET_SIMD"
 678 {
 679   int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
 680   int shift_amount;
 681
 682   if (CONST_INT_P (operands[2]))
 683     {
 684       shift_amount = INTVAL (operands[2]);
 685       if (shift_amount >= 0 && shift_amount < bit_width)
 686         {
 687           rtx tmp = aarch64_simd_gen_const_vector_dup (<MODE>mode,
 688                                                        shift_amount);
 689           emit_insn (gen_aarch64_simd_imm_shl<mode> (operands[0],
 690                                                      operands[1],
 691                                                      tmp));
 692           DONE;
 693         }
 694       else
 695         {
 696           operands[2] = force_reg (SImode, operands[2]);
 697         }
 698     }
 699   else if (MEM_P (operands[2]))
 700     {
 701       operands[2] = force_reg (SImode, operands[2]);
 702     }
 703
 704   if (REG_P (operands[2]))
 705     {
 706       rtx tmp = gen_reg_rtx (<MODE>mode);
 707       emit_insn (gen_aarch64_simd_dup<mode> (tmp,
 708                                              convert_to_mode (<VEL>mode,
 709                                                               operands[2],
 710                                                               0)));
 711       emit_insn (gen_aarch64_simd_reg_sshl<mode> (operands[0], operands[1],
 712                                                   tmp));
 713       DONE;
 714     }
 715   else
 716     FAIL;
 717 }
 718 )
 719
 720 (define_expand "lshr<mode>3"
 721   [(match_operand:VDQ 0 "register_operand" "")
 722    (match_operand:VDQ 1 "register_operand" "")
 723    (match_operand:SI  2 "general_operand" "")]
 724  "TARGET_SIMD"
 725 {
 726   int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
 727   int shift_amount;
 728
 729   if (CONST_INT_P (operands[2]))
 730     {
 731       shift_amount = INTVAL (operands[2]);
 732       if (shift_amount > 0 && shift_amount <= bit_width)
 733         {
 734           rtx tmp = aarch64_simd_gen_const_vector_dup (<MODE>mode,
 735                                                        shift_amount);
 736           emit_insn (gen_aarch64_simd_lshr<mode> (operands[0],
 737                                                   operands[1],
 738                                                   tmp));
 739           DONE;
 740         }
 741       else
 742         operands[2] = force_reg (SImode, operands[2]);
 743     }
 744   else if (MEM_P (operands[2]))
 745     {
 746       operands[2] = force_reg (SImode, operands[2]);
 747     }
 748
 749   if (REG_P (operands[2]))
 750     {
 751       rtx tmp = gen_reg_rtx (SImode);
 752       rtx tmp1 = gen_reg_rtx (<MODE>mode);
 753       emit_insn (gen_negsi2 (tmp, operands[2]));
 754       emit_insn (gen_aarch64_simd_dup<mode> (tmp1,
 755                                              convert_to_mode (<VEL>mode,
 756                                                               tmp, 0)));
 757       emit_insn (gen_aarch64_simd_reg_shl<mode>_unsigned (operands[0],
 758                                                           operands[1],
 759                                                           tmp1));
 760       DONE;
 761     }
 762   else
 763     FAIL;
 764 }
 765 )
 766
 767 (define_expand "ashr<mode>3"
 768   [(match_operand:VDQ 0 "register_operand" "")
 769    (match_operand:VDQ 1 "register_operand" "")
 770    (match_operand:SI  2 "general_operand" "")]
 771  "TARGET_SIMD"
 772 {
 773   int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
 774   int shift_amount;
 775
 776   if (CONST_INT_P (operands[2]))
 777     {
 778       shift_amount = INTVAL (operands[2]);
 779       if (shift_amount > 0 && shift_amount <= bit_width)
 780         {
 781           rtx tmp = aarch64_simd_gen_const_vector_dup (<MODE>mode,
 782                                                        shift_amount);
 783           emit_insn (gen_aarch64_simd_ashr<mode> (operands[0],
 784                                                   operands[1],
 785                                                   tmp));
 786           DONE;
 787         }
 788       else
 789         operands[2] = force_reg (SImode, operands[2]);
 790     }
 791   else if (MEM_P (operands[2]))
 792     {
 793       operands[2] = force_reg (SImode, operands[2]);
 794     }
 795
 796   if (REG_P (operands[2]))
 797     {
 798       rtx tmp = gen_reg_rtx (SImode);
 799       rtx tmp1 = gen_reg_rtx (<MODE>mode);
 800       emit_insn (gen_negsi2 (tmp, operands[2]));
 801       emit_insn (gen_aarch64_simd_dup<mode> (tmp1,
 802                                              convert_to_mode (<VEL>mode,
 803                                                               tmp, 0)));
 804       emit_insn (gen_aarch64_simd_reg_shl<mode>_signed (operands[0],
 805                                                         operands[1],
 806                                                         tmp1));
 807       DONE;
 808     }
 809   else
 810     FAIL;
 811 }
 812 )
 813
 814 (define_expand "vashl<mode>3"
 815  [(match_operand:VDQ 0 "register_operand" "")
 816   (match_operand:VDQ 1 "register_operand" "")
 817   (match_operand:VDQ 2 "register_operand" "")]
 818  "TARGET_SIMD"
 819 {
 820   emit_insn (gen_aarch64_simd_reg_sshl<mode> (operands[0], operands[1],
 821                                               operands[2]));
 822   DONE;
 823 })
 824
 825 ;; Using mode VQ_S as there is no V2DImode neg!
 826 ;; Negating individual lanes most certainly offsets the
 827 ;; gain from vectorization.
 828 (define_expand "vashr<mode>3"
 829  [(match_operand:VQ_S 0 "register_operand" "")
 830   (match_operand:VQ_S 1 "register_operand" "")
 831   (match_operand:VQ_S 2 "register_operand" "")]
 832  "TARGET_SIMD"
 833 {
 834   rtx neg = gen_reg_rtx (<MODE>mode);
 835   emit (gen_neg<mode>2 (neg, operands[2]));
 836   emit_insn (gen_aarch64_simd_reg_shl<mode>_signed (operands[0], operands[1],
 837                                                     neg));
 838   DONE;
 839 })
 840
 841 (define_expand "vlshr<mode>3"
 842  [(match_operand:VQ_S 0 "register_operand" "")
 843   (match_operand:VQ_S 1 "register_operand" "")
 844   (match_operand:VQ_S 2 "register_operand" "")]
 845  "TARGET_SIMD"
 846 {
 847   rtx neg = gen_reg_rtx (<MODE>mode);
 848   emit (gen_neg<mode>2 (neg, operands[2]));
 849   emit_insn (gen_aarch64_simd_reg_shl<mode>_unsigned (operands[0], operands[1],
 850                                                       neg));
 851   DONE;
 852 })
 853
 854 (define_expand "vec_set<mode>"
 855   [(match_operand:VQ_S 0 "register_operand" "+w")
 856    (match_operand:<VEL> 1 "register_operand" "r")
 857    (match_operand:SI 2 "immediate_operand" "")]
 858   "TARGET_SIMD"
 859   {
 860     HOST_WIDE_INT elem = (HOST_WIDE_INT) 1 << INTVAL (operands[2]);
 861     emit_insn (gen_aarch64_simd_vec_set<mode> (operands[0], operands[1],
 862                                             GEN_INT (elem), operands[0]));
 863     DONE;
 864   }
 865 )
 866
 867 (define_insn "aarch64_simd_vec_setv2di"
 868   [(set (match_operand:V2DI 0 "register_operand" "=w")
 869         (vec_merge:V2DI
 870             (vec_duplicate:V2DI
 871                 (match_operand:DI 1 "register_operand" "r"))
 872             (match_operand:V2DI 3 "register_operand" "0")
 873             (match_operand:SI 2 "immediate_operand" "i")))]
 874   "TARGET_SIMD"
 875   "ins\t%0.d[%p2], %1";
 876   [(set_attr "simd_type" "simd_insgp")
 877    (set_attr "simd_mode" "V2DI")]
 878 )
 879
 880 (define_expand "vec_setv2di"
 881   [(match_operand:V2DI 0 "register_operand" "+w")
 882    (match_operand:DI 1 "register_operand" "r")
 883    (match_operand:SI 2 "immediate_operand" "")]
 884   "TARGET_SIMD"
 885   {
 886     HOST_WIDE_INT elem = (HOST_WIDE_INT) 1 << INTVAL (operands[2]);
 887     emit_insn (gen_aarch64_simd_vec_setv2di (operands[0], operands[1],
 888                                           GEN_INT (elem), operands[0]));
 889     DONE;
 890   }
 891 )
 892
 893 (define_insn "aarch64_simd_vec_set<mode>"
 894   [(set (match_operand:VDQF 0 "register_operand" "=w")
 895         (vec_merge:VDQF
 896             (vec_duplicate:VDQF
 897                 (match_operand:<VEL> 1 "register_operand" "w"))
 898             (match_operand:VDQF 3 "register_operand" "0")
 899             (match_operand:SI 2 "immediate_operand" "i")))]
 900   "TARGET_SIMD"
 901   "ins\t%0.<Vetype>[%p2], %1.<Vetype>[0]";
 902   [(set_attr "simd_type" "simd_ins")
 903    (set_attr "simd_mode" "<MODE>")]
 904 )
 905
 906 (define_expand "vec_set<mode>"
 907   [(match_operand:VDQF 0 "register_operand" "+w")
 908    (match_operand:<VEL> 1 "register_operand" "w")
 909    (match_operand:SI 2 "immediate_operand" "")]
 910   "TARGET_SIMD"
 911   {
 912     HOST_WIDE_INT elem = (HOST_WIDE_INT) 1 << INTVAL (operands[2]);
 913     emit_insn (gen_aarch64_simd_vec_set<mode> (operands[0], operands[1],
 914                                           GEN_INT (elem), operands[0]));
 915     DONE;
 916   }
 917 )
 918
 919
 920 (define_insn "aarch64_mla<mode>"
 921  [(set (match_operand:VQ_S 0 "register_operand" "=w")
 922        (plus:VQ_S (mult:VQ_S (match_operand:VQ_S 2 "register_operand" "w")
 923                              (match_operand:VQ_S 3 "register_operand" "w"))
 924                   (match_operand:VQ_S 1 "register_operand" "0")))]
 925  "TARGET_SIMD"
 926  "mla\t%0.<Vtype>, %2.<Vtype>, %3.<Vtype>"
 927   [(set_attr "simd_type" "simd_mla")
 928    (set_attr "simd_mode" "<MODE>")]
 929 )
 930
 931 (define_insn "aarch64_mls<mode>"
 932  [(set (match_operand:VQ_S 0 "register_operand" "=w")
 933        (minus:VQ_S (match_operand:VQ_S 1 "register_operand" "0")
 934                    (mult:VQ_S (match_operand:VQ_S 2 "register_operand" "w")
 935                               (match_operand:VQ_S 3 "register_operand" "w"))))]
 936  "TARGET_SIMD"
 937  "mls\t%0.<Vtype>, %2.<Vtype>, %3.<Vtype>"
 938   [(set_attr "simd_type" "simd_mla")
 939    (set_attr "simd_mode" "<MODE>")]
 940 )
 941
 942 ;; Max/Min operations.
 943 (define_insn "<maxmin><mode>3"
 944  [(set (match_operand:VQ_S 0 "register_operand" "=w")
 945        (MAXMIN:VQ_S (match_operand:VQ_S 1 "register_operand" "w")
 946                     (match_operand:VQ_S 2 "register_operand" "w")))]
 947  "TARGET_SIMD"
 948  "<maxmin>\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
 949   [(set_attr "simd_type" "simd_minmax")
 950    (set_attr "simd_mode" "<MODE>")]
 951 )
 952
 953 ;; Move into low-half clearing high half to 0.
 954
 955 (define_insn "move_lo_quad_<mode>"
 956   [(set (match_operand:VQ 0 "register_operand" "=w")
 957         (vec_concat:VQ
 958           (match_operand:<VHALF> 1 "register_operand" "w")
 959           (vec_duplicate:<VHALF> (const_int 0))))]
 960   "TARGET_SIMD"
 961   "mov\\t%d0, %d1";
 962   [(set_attr "simd_type" "simd_dup")
 963    (set_attr "simd_mode" "<MODE>")]
 964 )
 965
 966 ;; Move into high-half.
 967
 968 (define_insn "aarch64_simd_move_hi_quad_<mode>"
 969   [(set (match_operand:VQ 0 "register_operand" "+w")
 970         (vec_concat:VQ
 971           (vec_select:<VHALF>
 972                 (match_dup 0)
 973                 (match_operand:VQ 2 "vect_par_cnst_lo_half" ""))
 974           (match_operand:<VHALF> 1 "register_operand" "w")))]
 975   "TARGET_SIMD"
 976   "ins\\t%0.d[1], %1.d[0]";
 977   [(set_attr "simd_type" "simd_ins")
 978    (set_attr "simd_mode" "<MODE>")]
 979 )
 980
 981 (define_expand "move_hi_quad_<mode>"
 982  [(match_operand:VQ 0 "register_operand" "")
 983   (match_operand:<VHALF> 1 "register_operand" "")]
 984  "TARGET_SIMD"
 985 {
 986   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
 987   emit_insn (gen_aarch64_simd_move_hi_quad_<mode> (operands[0],
 988                                                    operands[1], p));
 989   DONE;
 990 })
 991
 992 ;; Narrowing operations.
 993
 994 ;; For doubles.
 995 (define_insn "aarch64_simd_vec_pack_trunc_<mode>"
 996  [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
 997        (truncate:<VNARROWQ> (match_operand:VQN 1 "register_operand" "w")))]
 998  "TARGET_SIMD"
 999  "xtn\\t%0.<Vntype>, %1.<Vtype>"
1000   [(set_attr "simd_type" "simd_shiftn_imm")
1001    (set_attr "simd_mode" "<MODE>")]
1002 )
1003
1004 (define_expand "vec_pack_trunc_<mode>"
1005  [(match_operand:<VNARROWD> 0 "register_operand" "")
1006   (match_operand:VDN 1 "register_operand" "")
1007   (match_operand:VDN 2 "register_operand" "")]
1008  "TARGET_SIMD"
1009 {
1010   rtx tempreg = gen_reg_rtx (<VDBL>mode);
1011
1012   emit_insn (gen_move_lo_quad_<Vdbl> (tempreg, operands[1]));
1013   emit_insn (gen_move_hi_quad_<Vdbl> (tempreg, operands[2]));
1014   emit_insn (gen_aarch64_simd_vec_pack_trunc_<Vdbl> (operands[0], tempreg));
1015   DONE;
1016 })
1017
1018 ;; For quads.
1019
1020 (define_insn "vec_pack_trunc_<mode>"
1021  [(set (match_operand:<VNARROWQ2> 0 "register_operand" "+&w")
1022        (vec_concat:<VNARROWQ2>
1023          (truncate:<VNARROWQ> (match_operand:VQN 1 "register_operand" "w"))
1024          (truncate:<VNARROWQ> (match_operand:VQN 2 "register_operand" "w"))))]
1025  "TARGET_SIMD"
1026  "xtn\\t%0.<Vntype>, %1.<Vtype>\;xtn2\\t%0.<V2ntype>, %2.<Vtype>"
1027   [(set_attr "simd_type" "simd_shiftn2_imm")
1028    (set_attr "simd_mode" "<MODE>")
1029    (set_attr "length" "8")]
1030 )
1031
1032 ;; Widening operations.
1033
1034 (define_insn "aarch64_simd_vec_unpack<su>_lo_<mode>"
1035   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1036         (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1037                                (match_operand:VQW 1 "register_operand" "w")
1038                                (match_operand:VQW 2 "vect_par_cnst_lo_half" "")
1039                             )))]
1040   "TARGET_SIMD"
1041   "<su>shll %0.<Vwtype>, %1.<Vhalftype>, 0"
1042   [(set_attr "simd_type" "simd_shiftl_imm")
1043    (set_attr "simd_mode" "<MODE>")]
1044 )
1045
1046 (define_insn "aarch64_simd_vec_unpack<su>_hi_<mode>"
1047   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1048         (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1049                                (match_operand:VQW 1 "register_operand" "w")
1050                                (match_operand:VQW 2 "vect_par_cnst_hi_half" "")
1051                             )))]
1052   "TARGET_SIMD"
1053   "<su>shll2 %0.<Vwtype>, %1.<Vtype>, 0"
1054   [(set_attr "simd_type" "simd_shiftl_imm")
1055    (set_attr "simd_mode" "<MODE>")]
1056 )
1057
1058 (define_expand "vec_unpack<su>_hi_<mode>"
1059   [(match_operand:<VWIDE> 0 "register_operand" "")
1060    (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand"))]
1061   "TARGET_SIMD"
1062   {
1063     rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
1064     emit_insn (gen_aarch64_simd_vec_unpack<su>_hi_<mode> (operands[0],
1065                                                           operands[1], p));
1066     DONE;
1067   }
1068 )
1069
1070 (define_expand "vec_unpack<su>_lo_<mode>"
1071   [(match_operand:<VWIDE> 0 "register_operand" "")
1072    (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand" ""))]
1073   "TARGET_SIMD"
1074   {
1075     rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
1076     emit_insn (gen_aarch64_simd_vec_unpack<su>_lo_<mode> (operands[0],
1077                                                           operands[1], p));
1078     DONE;
1079   }
1080 )
1081
1082 ;; Widening arithmetic.
1083
1084 (define_insn "aarch64_simd_vec_<su>mult_lo_<mode>"
1085  [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1086        (mult:<VWIDE> (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1087                            (match_operand:VQW 1 "register_operand" "w")
1088                            (match_operand:VQW 3 "vect_par_cnst_lo_half" "")))
1089                      (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1090                            (match_operand:VQW 2 "register_operand" "w")
1091                            (match_dup 3)))))]
1092   "TARGET_SIMD"
1093   "<su>mull\\t%0.<Vwtype>, %1.<Vhalftype>, %2.<Vhalftype>"
1094   [(set_attr "simd_type" "simd_mull")
1095    (set_attr "simd_mode" "<MODE>")]
1096 )
1097
1098 (define_expand "vec_widen_<su>mult_lo_<mode>"
1099   [(match_operand:<VWIDE> 0 "register_operand" "")
1100    (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand" ""))
1101    (ANY_EXTEND:<VWIDE> (match_operand:VQW 2 "register_operand" ""))]
1102  "TARGET_SIMD"
1103  {
1104    rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
1105    emit_insn (gen_aarch64_simd_vec_<su>mult_lo_<mode> (operands[0],
1106                                                        operands[1],
1107                                                        operands[2], p));
1108    DONE;
1109  }
1110 )
1111
1112 (define_insn "aarch64_simd_vec_<su>mult_hi_<mode>"
1113  [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1114       (mult:<VWIDE> (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1115                             (match_operand:VQW 1 "register_operand" "w")
1116                             (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))
1117                     (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1118                             (match_operand:VQW 2 "register_operand" "w")
1119                             (match_dup 3)))))]
1120   "TARGET_SIMD"
1121   "<su>mull2\\t%0.<Vwtype>, %1.<Vtype>, %2.<Vtype>"
1122   [(set_attr "simd_type" "simd_mull")
1123    (set_attr "simd_mode" "<MODE>")]
1124 )
1125
1126 (define_expand "vec_widen_<su>mult_hi_<mode>"
1127   [(match_operand:<VWIDE> 0 "register_operand" "")
1128    (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand" ""))
1129    (ANY_EXTEND:<VWIDE> (match_operand:VQW 2 "register_operand" ""))]
1130  "TARGET_SIMD"
1131  {
1132    rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
1133    emit_insn (gen_aarch64_simd_vec_<su>mult_hi_<mode> (operands[0],
1134                                                        operands[1],
1135                                                        operands[2], p));
1136    DONE;
1137
1138  }
1139 )
1140
1141 ;; FP vector operations.
1142 ;; AArch64 AdvSIMD supports single-precision (32-bit) and
1143 ;; double-precision (64-bit) floating-point data types and arithmetic as
1144 ;; defined by the IEEE 754-2008 standard.  This makes them vectorizable
1145 ;; without the need for -ffast-math or -funsafe-math-optimizations.
1146 ;;
1147 ;; Floating-point operations can raise an exception.  Vectorizing such
1148 ;; operations are safe because of reasons explained below.
1149 ;;
1150 ;; ARMv8 permits an extension to enable trapped floating-point
1151 ;; exception handling, however this is an optional feature.  In the
1152 ;; event of a floating-point exception being raised by vectorised
1153 ;; code then:
1154 ;; 1.  If trapped floating-point exceptions are available, then a trap
1155 ;;     will be taken when any lane raises an enabled exception.  A trap
1156 ;;     handler may determine which lane raised the exception.
1157 ;; 2.  Alternatively a sticky exception flag is set in the
1158 ;;     floating-point status register (FPSR).  Software may explicitly
1159 ;;     test the exception flags, in which case the tests will either
1160 ;;     prevent vectorisation, allowing precise identification of the
1161 ;;     failing operation, or if tested outside of vectorisable regions
1162 ;;     then the specific operation and lane are not of interest.
1163
1164 ;; FP arithmetic operations.
1165
1166 (define_insn "add<mode>3"
1167  [(set (match_operand:VDQF 0 "register_operand" "=w")
1168        (plus:VDQF (match_operand:VDQF 1 "register_operand" "w")
1169                   (match_operand:VDQF 2 "register_operand" "w")))]
1170  "TARGET_SIMD"
1171  "fadd\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1172   [(set_attr "simd_type" "simd_fadd")
1173    (set_attr "simd_mode" "<MODE>")]
1174 )
1175
1176 (define_insn "sub<mode>3"
1177  [(set (match_operand:VDQF 0 "register_operand" "=w")
1178        (minus:VDQF (match_operand:VDQF 1 "register_operand" "w")
1179                    (match_operand:VDQF 2 "register_operand" "w")))]
1180  "TARGET_SIMD"
1181  "fsub\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1182   [(set_attr "simd_type" "simd_fadd")
1183    (set_attr "simd_mode" "<MODE>")]
1184 )
1185
1186 (define_insn "mul<mode>3"
1187  [(set (match_operand:VDQF 0 "register_operand" "=w")
1188        (mult:VDQF (match_operand:VDQF 1 "register_operand" "w")
1189                   (match_operand:VDQF 2 "register_operand" "w")))]
1190  "TARGET_SIMD"
1191  "fmul\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1192   [(set_attr "simd_type" "simd_fmul")
1193    (set_attr "simd_mode" "<MODE>")]
1194 )
1195
1196 (define_insn "div<mode>3"
1197  [(set (match_operand:VDQF 0 "register_operand" "=w")
1198        (div:VDQF (match_operand:VDQF 1 "register_operand" "w")
1199                  (match_operand:VDQF 2 "register_operand" "w")))]
1200  "TARGET_SIMD"
1201  "fdiv\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1202   [(set_attr "simd_type" "simd_fdiv")
1203    (set_attr "simd_mode" "<MODE>")]
1204 )
1205
1206 (define_insn "neg<mode>2"
1207  [(set (match_operand:VDQF 0 "register_operand" "=w")
1208        (neg:VDQF (match_operand:VDQF 1 "register_operand" "w")))]
1209  "TARGET_SIMD"
1210  "fneg\\t%0.<Vtype>, %1.<Vtype>"
1211   [(set_attr "simd_type" "simd_fnegabs")
1212    (set_attr "simd_mode" "<MODE>")]
1213 )
1214
1215 (define_insn "abs<mode>2"
1216  [(set (match_operand:VDQF 0 "register_operand" "=w")
1217        (abs:VDQF (match_operand:VDQF 1 "register_operand" "w")))]
1218  "TARGET_SIMD"
1219  "fabs\\t%0.<Vtype>, %1.<Vtype>"
1220   [(set_attr "simd_type" "simd_fnegabs")
1221    (set_attr "simd_mode" "<MODE>")]
1222 )
1223
1224 (define_insn "fma<mode>4"
1225   [(set (match_operand:VDQF 0 "register_operand" "=w")
1226        (fma:VDQF (match_operand:VDQF 1 "register_operand" "w")
1227                 (match_operand:VDQF 2 "register_operand" "w")
1228                 (match_operand:VDQF 3 "register_operand" "0")))]
1229   "TARGET_SIMD"
1230  "fmla\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1231   [(set_attr "simd_type" "simd_fmla")
1232    (set_attr "simd_mode" "<MODE>")]
1233 )
1234
1235 ;; Vector versions of the floating-point frint patterns.
1236 ;; Expands to btrunc, ceil, floor, nearbyint, rint, round.
1237 (define_insn "<frint_pattern><mode>2"
1238   [(set (match_operand:VDQF 0 "register_operand" "=w")
1239         (unspec:VDQF [(match_operand:VDQF 1 "register_operand" "w")]
1240                       FRINT))]
1241   "TARGET_SIMD"
1242   "frint<frint_suffix>\\t%0.<Vtype>, %1.<Vtype>"
1243   [(set_attr "simd_type" "simd_frint")
1244    (set_attr "simd_mode" "<MODE>")]
1245 )
1246
1247 ;; Vector versions of the fcvt standard patterns.
1248 ;; Expands to lbtrunc, lround, lceil, lfloor
1249 (define_insn "l<fcvt_pattern><su_optab><VDQF:mode><fcvt_target>2"
1250   [(set (match_operand:<FCVT_TARGET> 0 "register_operand" "=w")
1251         (FIXUORS:<FCVT_TARGET> (unspec:<FCVT_TARGET>
1252                                [(match_operand:VDQF 1 "register_operand" "w")]
1253                                FCVT)))]
1254   "TARGET_SIMD"
1255   "fcvt<frint_suffix><su>\\t%0.<Vtype>, %1.<Vtype>"
1256   [(set_attr "simd_type" "simd_fcvti")
1257    (set_attr "simd_mode" "<MODE>")]
1258 )
1259
1260 (define_insn "<optab><fcvt_target><VDQF:mode>2"
1261   [(set (match_operand:VDQF 0 "register_operand" "=w")
1262         (FLOATUORS:VDQF
1263           (match_operand:<FCVT_TARGET> 1 "register_operand" "w")))]
1264   "TARGET_SIMD"
1265   "<su_optab>cvtf\\t%0.<Vtype>, %1.<Vtype>"
1266   [(set_attr "simd_type" "simd_icvtf")
1267    (set_attr "simd_mode" "<MODE>")]
1268 )
1269
1270 ;; Conversions between vectors of floats and doubles.
1271 ;; Contains a mix of patterns to match standard pattern names
1272 ;; and those for intrinsics.
1273
1274 ;; Float widening operations.
1275
1276 (define_insn "vec_unpacks_lo_v4sf"
1277   [(set (match_operand:V2DF 0 "register_operand" "=w")
1278         (float_extend:V2DF
1279           (vec_select:V2SF
1280             (match_operand:V4SF 1 "register_operand" "w")
1281             (parallel [(const_int 0) (const_int 1)])
1282           )))]
1283   "TARGET_SIMD"
1284   "fcvtl\\t%0.2d, %1.2s"
1285   [(set_attr "simd_type" "simd_fcvtl")
1286    (set_attr "simd_mode" "V2DF")]
1287 )
1288
1289 (define_insn "aarch64_float_extend_lo_v2df"
1290   [(set (match_operand:V2DF 0 "register_operand" "=w")
1291         (float_extend:V2DF
1292           (match_operand:V2SF 1 "register_operand" "w")))]
1293   "TARGET_SIMD"
1294   "fcvtl\\t%0.2d, %1.2s"
1295   [(set_attr "simd_type" "simd_fcvtl")
1296    (set_attr "simd_mode" "V2DF")]
1297 )
1298
1299 (define_insn "vec_unpacks_hi_v4sf"
1300   [(set (match_operand:V2DF 0 "register_operand" "=w")
1301         (float_extend:V2DF
1302           (vec_select:V2SF
1303             (match_operand:V4SF 1 "register_operand" "w")
1304             (parallel [(const_int 2) (const_int 3)])
1305           )))]
1306   "TARGET_SIMD"
1307   "fcvtl2\\t%0.2d, %1.4s"
1308   [(set_attr "simd_type" "simd_fcvtl")
1309    (set_attr "simd_mode" "V2DF")]
1310 )
1311
1312 ;; Float narrowing operations.
1313
1314 (define_insn "aarch64_float_truncate_lo_v2sf"
1315   [(set (match_operand:V2SF 0 "register_operand" "=w")
1316       (float_truncate:V2SF
1317         (match_operand:V2DF 1 "register_operand" "w")))]
1318   "TARGET_SIMD"
1319   "fcvtn\\t%0.2s, %1.2d"
1320   [(set_attr "simd_type" "simd_fcvtl")
1321    (set_attr "simd_mode" "V2SF")]
1322 )
1323
1324 (define_insn "aarch64_float_truncate_hi_v4sf"
1325   [(set (match_operand:V4SF 0 "register_operand" "=w")
1326     (vec_concat:V4SF
1327       (match_operand:V2SF 1 "register_operand" "0")
1328       (float_truncate:V2SF
1329         (match_operand:V2DF 2 "register_operand" "w"))))]
1330   "TARGET_SIMD"
1331   "fcvtn2\\t%0.4s, %2.2d"
1332   [(set_attr "simd_type" "simd_fcvtl")
1333    (set_attr "simd_mode" "V4SF")]
1334 )
1335
1336 (define_expand "vec_pack_trunc_v2df"
1337   [(set (match_operand:V4SF 0 "register_operand")
1338       (vec_concat:V4SF
1339         (float_truncate:V2SF
1340             (match_operand:V2DF 1 "register_operand"))
1341         (float_truncate:V2SF
1342             (match_operand:V2DF 2 "register_operand"))
1343           ))]
1344   "TARGET_SIMD"
1345   {
1346     rtx tmp = gen_reg_rtx (V2SFmode);
1347     emit_insn (gen_aarch64_float_truncate_lo_v2sf (tmp, operands[1]));
1348     emit_insn (gen_aarch64_float_truncate_hi_v4sf (operands[0],
1349                                                    tmp, operands[2]));
1350     DONE;
1351   }
1352 )
1353
1354 (define_expand "vec_pack_trunc_df"
1355   [(set (match_operand:V2SF 0 "register_operand")
1356       (vec_concat:V2SF
1357         (float_truncate:SF
1358             (match_operand:DF 1 "register_operand"))
1359         (float_truncate:SF
1360             (match_operand:DF 2 "register_operand"))
1361           ))]
1362   "TARGET_SIMD"
1363   {
1364     rtx tmp = gen_reg_rtx (V2SFmode);
1365     emit_insn (gen_move_lo_quad_v2df (tmp, operands[1]));
1366     emit_insn (gen_move_hi_quad_v2df (tmp, operands[2]));
1367     emit_insn (gen_aarch64_float_truncate_lo_v2sf (operands[0], tmp));
1368     DONE;
1369   }
1370 )
1371
1372 (define_insn "aarch64_vmls<mode>"
1373   [(set (match_operand:VDQF 0 "register_operand" "=w")
1374        (minus:VDQF (match_operand:VDQF 1 "register_operand" "0")
1375                    (mult:VDQF (match_operand:VDQF 2 "register_operand" "w")
1376                               (match_operand:VDQF 3 "register_operand" "w"))))]
1377   "TARGET_SIMD"
1378  "fmls\\t%0.<Vtype>, %2.<Vtype>, %3.<Vtype>"
1379   [(set_attr "simd_type" "simd_fmla")
1380    (set_attr "simd_mode" "<MODE>")]
1381 )
1382
1383 ;; FP Max/Min
1384 ;; Max/Min are introduced by idiom recognition by GCC's mid-end.  An
1385 ;; expression like:
1386 ;;      a = (b < c) ? b : c;
1387 ;; is idiom-matched as MIN_EXPR<b,c> only if -ffinite-math-only is enabled
1388 ;; either explicitly or indirectly via -ffast-math.
1389 ;;
1390 ;; MIN_EXPR and MAX_EXPR eventually map to 'smin' and 'smax' in RTL.
1391 ;; The 'smax' and 'smin' RTL standard pattern names do not specify which
1392 ;; operand will be returned when both operands are zero (i.e. they may not
1393 ;; honour signed zeroes), or when either operand is NaN.  Therefore GCC
1394 ;; only introduces MIN_EXPR/MAX_EXPR in fast math mode or when not honouring
1395 ;; NaNs.
1396
1397 (define_insn "smax<mode>3"
1398   [(set (match_operand:VDQF 0 "register_operand" "=w")
1399         (smax:VDQF (match_operand:VDQF 1 "register_operand" "w")
1400                    (match_operand:VDQF 2 "register_operand" "w")))]
1401   "TARGET_SIMD"
1402   "fmaxnm\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1403   [(set_attr "simd_type" "simd_fminmax")
1404    (set_attr "simd_mode" "<MODE>")]
1405 )
1406
1407 (define_insn "smin<mode>3"
1408   [(set (match_operand:VDQF 0 "register_operand" "=w")
1409         (smin:VDQF (match_operand:VDQF 1 "register_operand" "w")
1410                    (match_operand:VDQF 2 "register_operand" "w")))]
1411   "TARGET_SIMD"
1412   "fminnm\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1413   [(set_attr "simd_type" "simd_fminmax")
1414    (set_attr "simd_mode" "<MODE>")]
1415 )
1416
1417 ;; FP 'across lanes' max and min ops.
1418
1419 (define_insn "reduc_s<fmaxminv>_v4sf"
1420  [(set (match_operand:V4SF 0 "register_operand" "=w")
1421        (unspec:V4SF [(match_operand:V4SF 1 "register_operand" "w")]
1422                     FMAXMINV))]
1423  "TARGET_SIMD"
1424  "f<fmaxminv>nmv\\t%s0, %1.4s";
1425   [(set_attr "simd_type" "simd_fminmaxv")
1426    (set_attr "simd_mode" "V4SF")]
1427 )
1428
1429 (define_insn "reduc_s<fmaxminv>_<mode>"
1430  [(set (match_operand:V2F 0 "register_operand" "=w")
1431        (unspec:V2F [(match_operand:V2F 1 "register_operand" "w")]
1432                     FMAXMINV))]
1433  "TARGET_SIMD"
1434  "f<fmaxminv>nmp\\t%0.<Vtype>, %1.<Vtype>, %1.<Vtype>";
1435   [(set_attr "simd_type" "simd_fminmax")
1436    (set_attr "simd_mode" "<MODE>")]
1437 )
1438
1439 ;; FP 'across lanes' add.
1440
1441 (define_insn "aarch64_addpv4sf"
1442  [(set (match_operand:V4SF 0 "register_operand" "=w")
1443        (unspec:V4SF [(match_operand:V4SF 1 "register_operand" "w")]
1444                     UNSPEC_FADDV))]
1445  "TARGET_SIMD"
1446  "faddp\\t%0.4s, %1.4s, %1.4s"
1447   [(set_attr "simd_type" "simd_fadd")
1448    (set_attr "simd_mode" "V4SF")]
1449 )
1450
1451 (define_expand "reduc_uplus_v4sf"
1452  [(set (match_operand:V4SF 0 "register_operand" "=w")
1453        (match_operand:V4SF 1 "register_operand" "w"))]
1454  "TARGET_SIMD"
1455 {
1456   rtx tmp = gen_reg_rtx (V4SFmode);
1457   emit_insn (gen_aarch64_addpv4sf (tmp, operands[1]));
1458   emit_insn (gen_aarch64_addpv4sf (operands[0], tmp));
1459   DONE;
1460 })
1461
1462 (define_expand "reduc_splus_v4sf"
1463  [(set (match_operand:V4SF 0 "register_operand" "=w")
1464        (match_operand:V4SF 1 "register_operand" "w"))]
1465  "TARGET_SIMD"
1466 {
1467   rtx tmp = gen_reg_rtx (V4SFmode);
1468   emit_insn (gen_aarch64_addpv4sf (tmp, operands[1]));
1469   emit_insn (gen_aarch64_addpv4sf (operands[0], tmp));
1470   DONE;
1471 })
1472
1473 (define_expand "aarch64_addvv4sf"
1474  [(set (match_operand:V4SF 0 "register_operand" "=w")
1475         (unspec:V4SF [(match_operand:V4SF 1 "register_operand" "w")]
1476                      UNSPEC_FADDV))]
1477  "TARGET_SIMD"
1478 {
1479   emit_insn (gen_reduc_splus_v4sf (operands[0], operands[1]));
1480   DONE;
1481 })
1482
1483 (define_insn "aarch64_addv<mode>"
1484  [(set (match_operand:V2F 0 "register_operand" "=w")
1485        (unspec:V2F [(match_operand:V2F 1 "register_operand" "w")]
1486                     UNSPEC_FADDV))]
1487  "TARGET_SIMD"
1488  "faddp\\t%<Vetype>0, %1.<Vtype>"
1489   [(set_attr "simd_type" "simd_fadd")
1490    (set_attr "simd_mode" "<MODE>")]
1491 )
1492
1493 (define_expand "reduc_uplus_<mode>"
1494  [(set (match_operand:V2F 0 "register_operand" "=w")
1495        (unspec:V2F [(match_operand:V2F 1 "register_operand" "w")]
1496                     UNSPEC_FADDV))]
1497  "TARGET_SIMD"
1498  ""
1499 )
1500
1501 (define_expand "reduc_splus_<mode>"
1502  [(set (match_operand:V2F 0 "register_operand" "=w")
1503        (unspec:V2F [(match_operand:V2F 1 "register_operand" "w")]
1504                     UNSPEC_FADDV))]
1505  "TARGET_SIMD"
1506  ""
1507 )
1508
1509 ;; Reduction across lanes.
1510
1511 (define_insn "aarch64_addv<mode>"
1512  [(set (match_operand:VDQV 0 "register_operand" "=w")
1513        (unspec:VDQV [(match_operand:VDQV 1 "register_operand" "w")]
1514                     UNSPEC_ADDV))]
1515  "TARGET_SIMD"
1516  "addv\\t%<Vetype>0, %1.<Vtype>"
1517   [(set_attr "simd_type" "simd_addv")
1518    (set_attr "simd_mode" "<MODE>")]
1519 )
1520
1521 (define_expand "reduc_splus_<mode>"
1522  [(set (match_operand:VDQV 0 "register_operand" "=w")
1523        (unspec:VDQV [(match_operand:VDQV 1 "register_operand" "w")]
1524                     UNSPEC_ADDV))]
1525  "TARGET_SIMD"
1526  ""
1527 )
1528
1529 (define_expand "reduc_uplus_<mode>"
1530  [(set (match_operand:VDQV 0 "register_operand" "=w")
1531        (unspec:VDQV [(match_operand:VDQV 1 "register_operand" "w")]
1532                     UNSPEC_ADDV))]
1533  "TARGET_SIMD"
1534  ""
1535 )
1536
1537 (define_insn "aarch64_addvv2di"
1538  [(set (match_operand:V2DI 0 "register_operand" "=w")
1539        (unspec:V2DI [(match_operand:V2DI 1 "register_operand" "w")]
1540                     UNSPEC_ADDV))]
1541  "TARGET_SIMD"
1542  "addp\\t%d0, %1.2d"
1543   [(set_attr "simd_type" "simd_add")
1544    (set_attr "simd_mode" "V2DI")]
1545 )
1546
1547 (define_expand "reduc_uplus_v2di"
1548  [(set (match_operand:V2DI 0 "register_operand" "=w")
1549        (unspec:V2DI [(match_operand:V2DI 1 "register_operand" "w")]
1550                     UNSPEC_ADDV))]
1551  "TARGET_SIMD"
1552  ""
1553 )
1554
1555 (define_expand "reduc_splus_v2di"
1556  [(set (match_operand:V2DI 0 "register_operand" "=w")
1557        (unspec:V2DI [(match_operand:V2DI 1 "register_operand" "w")]
1558                     UNSPEC_ADDV))]
1559  "TARGET_SIMD"
1560  ""
1561 )
1562
1563 (define_insn "aarch64_addvv2si"
1564  [(set (match_operand:V2SI 0 "register_operand" "=w")
1565        (unspec:V2SI [(match_operand:V2SI 1 "register_operand" "w")]
1566                     UNSPEC_ADDV))]
1567  "TARGET_SIMD"
1568  "addp\\t%0.2s, %1.2s, %1.2s"
1569   [(set_attr "simd_type" "simd_add")
1570    (set_attr "simd_mode" "V2SI")]
1571 )
1572
1573 (define_expand "reduc_uplus_v2si"
1574  [(set (match_operand:V2SI 0 "register_operand" "=w")
1575        (unspec:V2SI [(match_operand:V2SI 1 "register_operand" "w")]
1576                     UNSPEC_ADDV))]
1577  "TARGET_SIMD"
1578  ""
1579 )
1580
1581 (define_expand "reduc_splus_v2si"
1582  [(set (match_operand:V2SI 0 "register_operand" "=w")
1583        (unspec:V2SI [(match_operand:V2SI 1 "register_operand" "w")]
1584                     UNSPEC_ADDV))]
1585  "TARGET_SIMD"
1586  ""
1587 )
1588
1589 (define_insn "reduc_<maxminv>_<mode>"
1590  [(set (match_operand:VDQV 0 "register_operand" "=w")
1591        (unspec:VDQV [(match_operand:VDQV 1 "register_operand" "w")]
1592                     MAXMINV))]
1593  "TARGET_SIMD"
1594  "<maxminv>v\\t%<Vetype>0, %1.<Vtype>"
1595   [(set_attr "simd_type" "simd_minmaxv")
1596    (set_attr "simd_mode" "<MODE>")]
1597 )
1598
1599 (define_insn "reduc_<maxminv>_v2si"
1600  [(set (match_operand:V2SI 0 "register_operand" "=w")
1601        (unspec:V2SI [(match_operand:V2SI 1 "register_operand" "w")]
1602                     MAXMINV))]
1603  "TARGET_SIMD"
1604  "<maxminv>p\\t%0.2s, %1.2s, %1.2s"
1605   [(set_attr "simd_type" "simd_minmax")
1606    (set_attr "simd_mode" "V2SI")]
1607 )
1608
1609 ;; aarch64_simd_bsl may compile to any of bsl/bif/bit depending on register
1610 ;; allocation.
1611 ;; Operand 1 is the mask, operands 2 and 3 are the bitfields from which
1612 ;; to select.
1613 ;;
1614 ;; Thus our BSL is of the form:
1615 ;;   op0 = bsl (mask, op2, op3)
1616 ;; We can use any of:
1617 ;;
1618 ;;   if (op0 = mask)
1619 ;;     bsl mask, op1, op2
1620 ;;   if (op0 = op1) (so 1-bits in mask choose bits from op2, else op0)
1621 ;;     bit op0, op2, mask
1622 ;;   if (op0 = op2) (so 0-bits in mask choose bits from op1, else op0)
1623 ;;     bif op0, op1, mask
1624
1625 (define_insn "aarch64_simd_bsl<mode>_internal"
1626   [(set (match_operand:VALL 0 "register_operand"                "=w,w,w")
1627         (ior:VALL
1628            (and:VALL
1629              (match_operand:<V_cmp_result> 1 "register_operand" " 0,w,w")
1630              (match_operand:VALL 2 "register_operand"           " w,w,0"))
1631            (and:VALL
1632              (not:<V_cmp_result>
1633                 (match_dup:<V_cmp_result> 1))
1634              (match_operand:VALL 3 "register_operand"           " w,0,w"))
1635         ))]
1636   "TARGET_SIMD"
1637   "@
1638   bsl\\t%0.<Vbtype>, %2.<Vbtype>, %3.<Vbtype>
1639   bit\\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>
1640   bif\\t%0.<Vbtype>, %3.<Vbtype>, %1.<Vbtype>"
1641 )
1642
1643 (define_expand "aarch64_simd_bsl<mode>"
1644   [(match_operand:VALL 0 "register_operand")
1645    (match_operand:<V_cmp_result> 1 "register_operand")
1646    (match_operand:VALL 2 "register_operand")
1647    (match_operand:VALL 3 "register_operand")]
1648  "TARGET_SIMD"
1649 {
1650   /* We can't alias operands together if they have different modes.  */
1651   operands[1] = gen_lowpart (<V_cmp_result>mode, operands[1]);
1652   emit_insn (gen_aarch64_simd_bsl<mode>_internal (operands[0], operands[1],
1653                                                   operands[2], operands[3]));
1654   DONE;
1655 })
1656
1657 (define_expand "aarch64_vcond_internal<mode>"
1658   [(set (match_operand:VDQ 0 "register_operand")
1659         (if_then_else:VDQ
1660           (match_operator 3 "comparison_operator"
1661             [(match_operand:VDQ 4 "register_operand")
1662              (match_operand:VDQ 5 "nonmemory_operand")])
1663           (match_operand:VDQ 1 "register_operand")
1664           (match_operand:VDQ 2 "register_operand")))]
1665   "TARGET_SIMD"
1666 {
1667   int inverse = 0, has_zero_imm_form = 0;
1668   rtx mask = gen_reg_rtx (<MODE>mode);
1669
1670   switch (GET_CODE (operands[3]))
1671     {
1672     case LE:
1673     case LT:
1674     case NE:
1675       inverse = 1;
1676       /* Fall through.  */
1677     case GE:
1678     case GT:
1679     case EQ:
1680       has_zero_imm_form = 1;
1681       break;
1682     case LEU:
1683     case LTU:
1684       inverse = 1;
1685       break;
1686     default:
1687       break;
1688     }
1689
1690   if (!REG_P (operands[5])
1691       && (operands[5] != CONST0_RTX (<MODE>mode) || !has_zero_imm_form))
1692     operands[5] = force_reg (<MODE>mode, operands[5]);
1693
1694   switch (GET_CODE (operands[3]))
1695     {
1696     case LT:
1697     case GE:
1698       emit_insn (gen_aarch64_cmge<mode> (mask, operands[4], operands[5]));
1699       break;
1700
1701     case LE:
1702     case GT:
1703       emit_insn (gen_aarch64_cmgt<mode> (mask, operands[4], operands[5]));
1704       break;
1705
1706     case LTU:
1707     case GEU:
1708       emit_insn (gen_aarch64_cmhs<mode> (mask, operands[4], operands[5]));
1709       break;
1710
1711     case LEU:
1712     case GTU:
1713       emit_insn (gen_aarch64_cmhi<mode> (mask, operands[4], operands[5]));
1714       break;
1715
1716     case NE:
1717     case EQ:
1718       emit_insn (gen_aarch64_cmeq<mode> (mask, operands[4], operands[5]));
1719       break;
1720
1721     default:
1722       gcc_unreachable ();
1723     }
1724
1725   if (inverse)
1726     emit_insn (gen_aarch64_simd_bsl<mode> (operands[0], mask, operands[2],
1727                                     operands[1]));
1728   else
1729     emit_insn (gen_aarch64_simd_bsl<mode> (operands[0], mask, operands[1],
1730                                     operands[2]));
1731
1732   DONE;
1733 })
1734
1735 (define_expand "aarch64_vcond_internal<mode>"
1736   [(set (match_operand:VDQF 0 "register_operand")
1737         (if_then_else:VDQF
1738           (match_operator 3 "comparison_operator"
1739             [(match_operand:VDQF 4 "register_operand")
1740              (match_operand:VDQF 5 "nonmemory_operand")])
1741           (match_operand:VDQF 1 "register_operand")
1742           (match_operand:VDQF 2 "register_operand")))]
1743   "TARGET_SIMD"
1744 {
1745   int inverse = 0;
1746   int use_zero_form = 0;
1747   int swap_bsl_operands = 0;
1748   rtx mask = gen_reg_rtx (<V_cmp_result>mode);
1749   rtx tmp = gen_reg_rtx (<V_cmp_result>mode);
1750
1751   rtx (*base_comparison) (rtx, rtx, rtx);
1752   rtx (*complimentary_comparison) (rtx, rtx, rtx);
1753
1754   switch (GET_CODE (operands[3]))
1755     {
1756     case GE:
1757     case GT:
1758     case LE:
1759     case LT:
1760     case EQ:
1761       if (operands[5] == CONST0_RTX (<MODE>mode))
1762         {
1763           use_zero_form = 1;
1764           break;
1765         }
1766       /* Fall through.  */
1767     default:
1768       if (!REG_P (operands[5]))
1769         operands[5] = force_reg (<MODE>mode, operands[5]);
1770     }
1771
1772   switch (GET_CODE (operands[3]))
1773     {
1774     case LT:
1775     case UNLT:
1776       inverse = 1;
1777       /* Fall through.  */
1778     case GE:
1779     case UNGE:
1780     case ORDERED:
1781     case UNORDERED:
1782       base_comparison = gen_aarch64_cmge<mode>;
1783       complimentary_comparison = gen_aarch64_cmgt<mode>;
1784       break;
1785     case LE:
1786     case UNLE:
1787       inverse = 1;
1788       /* Fall through.  */
1789     case GT:
1790     case UNGT:
1791       base_comparison = gen_aarch64_cmgt<mode>;
1792       complimentary_comparison = gen_aarch64_cmge<mode>;
1793       break;
1794     case EQ:
1795     case NE:
1796     case UNEQ:
1797       base_comparison = gen_aarch64_cmeq<mode>;
1798       complimentary_comparison = gen_aarch64_cmeq<mode>;
1799       break;
1800     default:
1801       gcc_unreachable ();
1802     }
1803
1804   switch (GET_CODE (operands[3]))
1805     {
1806     case LT:
1807     case LE:
1808     case GT:
1809     case GE:
1810     case EQ:
1811       /* The easy case.  Here we emit one of FCMGE, FCMGT or FCMEQ.
1812          As a LT b <=> b GE a && a LE b <=> b GT a.  Our transformations are:
1813          a GE b -> a GE b
1814          a GT b -> a GT b
1815          a LE b -> b GE a
1816          a LT b -> b GT a
1817          a EQ b -> a EQ b
1818          Note that there also exist direct comparison against 0 forms,
1819          so catch those as a special case.  */
1820       if (use_zero_form)
1821         {
1822           inverse = 0;
1823           switch (GET_CODE (operands[3]))
1824             {
1825             case LT:
1826               base_comparison = gen_aarch64_cmlt<mode>;
1827               break;
1828             case LE:
1829               base_comparison = gen_aarch64_cmle<mode>;
1830               break;
1831             default:
1832               /* Do nothing, other zero form cases already have the correct
1833                  base_comparison.  */
1834               break;
1835             }
1836         }
1837
1838       if (!inverse)
1839         emit_insn (base_comparison (mask, operands[4], operands[5]));
1840       else
1841         emit_insn (complimentary_comparison (mask, operands[5], operands[4]));
1842       break;
1843     case UNLT:
1844     case UNLE:
1845     case UNGT:
1846     case UNGE:
1847     case NE:
1848       /* FCM returns false for lanes which are unordered, so if we use
1849          the inverse of the comparison we actually want to emit, then
1850          swap the operands to BSL, we will end up with the correct result.
1851          Note that a NE NaN and NaN NE b are true for all a, b.
1852
1853          Our transformations are:
1854          a GE b -> !(b GT a)
1855          a GT b -> !(b GE a)
1856          a LE b -> !(a GT b)
1857          a LT b -> !(a GE b)
1858          a NE b -> !(a EQ b)  */
1859
1860       if (inverse)
1861         emit_insn (base_comparison (mask, operands[4], operands[5]));
1862       else
1863         emit_insn (complimentary_comparison (mask, operands[5], operands[4]));
1864
1865       swap_bsl_operands = 1;
1866       break;
1867     case UNEQ:
1868       /* We check (a > b ||  b > a).  combining these comparisons give us
1869          true iff !(a != b && a ORDERED b), swapping the operands to BSL
1870          will then give us (a == b ||  a UNORDERED b) as intended.  */
1871
1872       emit_insn (gen_aarch64_cmgt<mode> (mask, operands[4], operands[5]));
1873       emit_insn (gen_aarch64_cmgt<mode> (tmp, operands[5], operands[4]));
1874       emit_insn (gen_ior<v_cmp_result>3 (mask, mask, tmp));
1875       swap_bsl_operands = 1;
1876       break;
1877     case UNORDERED:
1878        /* Operands are ORDERED iff (a > b || b >= a).
1879          Swapping the operands to BSL will give the UNORDERED case.  */
1880      swap_bsl_operands = 1;
1881      /* Fall through.  */
1882     case ORDERED:
1883       emit_insn (gen_aarch64_cmgt<mode> (tmp, operands[4], operands[5]));
1884       emit_insn (gen_aarch64_cmge<mode> (mask, operands[5], operands[4]));
1885       emit_insn (gen_ior<v_cmp_result>3 (mask, mask, tmp));
1886       break;
1887     default:
1888       gcc_unreachable ();
1889     }
1890
1891   if (swap_bsl_operands)
1892     emit_insn (gen_aarch64_simd_bsl<mode> (operands[0], mask, operands[2],
1893                                     operands[1]));
1894   else
1895     emit_insn (gen_aarch64_simd_bsl<mode> (operands[0], mask, operands[1],
1896                                     operands[2]));
1897   DONE;
1898 })
1899
1900 (define_expand "vcond<mode><mode>"
1901   [(set (match_operand:VALL 0 "register_operand")
1902         (if_then_else:VALL
1903           (match_operator 3 "comparison_operator"
1904             [(match_operand:VALL 4 "register_operand")
1905              (match_operand:VALL 5 "nonmemory_operand")])
1906           (match_operand:VALL 1 "register_operand")
1907           (match_operand:VALL 2 "register_operand")))]
1908   "TARGET_SIMD"
1909 {
1910   emit_insn (gen_aarch64_vcond_internal<mode> (operands[0], operands[1],
1911                                                operands[2], operands[3],
1912                                                operands[4], operands[5]));
1913   DONE;
1914 })
1915
1916
1917 (define_expand "vcondu<mode><mode>"
1918   [(set (match_operand:VDQ 0 "register_operand")
1919         (if_then_else:VDQ
1920           (match_operator 3 "comparison_operator"
1921             [(match_operand:VDQ 4 "register_operand")
1922              (match_operand:VDQ 5 "nonmemory_operand")])
1923           (match_operand:VDQ 1 "register_operand")
1924           (match_operand:VDQ 2 "register_operand")))]
1925   "TARGET_SIMD"
1926 {
1927   emit_insn (gen_aarch64_vcond_internal<mode> (operands[0], operands[1],
1928                                                operands[2], operands[3],
1929                                                operands[4], operands[5]));
1930   DONE;
1931 })
1932
1933 ;; Patterns for AArch64 SIMD Intrinsics.
1934
1935 (define_expand "aarch64_create<mode>"
1936   [(match_operand:VD_RE 0 "register_operand" "")
1937    (match_operand:DI 1 "general_operand" "")]
1938   "TARGET_SIMD"
1939 {
1940   rtx src = gen_lowpart (<MODE>mode, operands[1]);
1941   emit_move_insn (operands[0], src);
1942   DONE;
1943 })
1944
1945 (define_insn "aarch64_get_lane_signed<mode>"
1946   [(set (match_operand:<VEL> 0 "register_operand" "=r")
1947         (sign_extend:<VEL>
1948           (vec_select:<VEL>
1949             (match_operand:VQ_S 1 "register_operand" "w")
1950             (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
1951   "TARGET_SIMD"
1952   "smov\\t%0, %1.<Vetype>[%2]"
1953   [(set_attr "simd_type" "simd_movgp")
1954    (set_attr "simd_mode" "<MODE>")]
1955 )
1956
1957 (define_insn "aarch64_get_lane_unsigned<mode>"
1958   [(set (match_operand:<VEL> 0 "register_operand" "=r")
1959         (zero_extend:<VEL>
1960           (vec_select:<VEL>
1961             (match_operand:VDQ 1 "register_operand" "w")
1962             (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
1963   "TARGET_SIMD"
1964   "umov\\t%<vw>0, %1.<Vetype>[%2]"
1965   [(set_attr "simd_type" "simd_movgp")
1966    (set_attr "simd_mode" "<MODE>")]
1967 )
1968
1969 (define_insn "aarch64_get_lane<mode>"
1970   [(set (match_operand:<VEL> 0 "register_operand" "=w")
1971         (vec_select:<VEL>
1972             (match_operand:VDQF 1 "register_operand" "w")
1973             (parallel [(match_operand:SI 2 "immediate_operand" "i")])))]
1974   "TARGET_SIMD"
1975   "mov\\t%0.<Vetype>[0], %1.<Vetype>[%2]"
1976   [(set_attr "simd_type" "simd_ins")
1977    (set_attr "simd_mode" "<MODE>")]
1978 )
1979
1980 (define_expand "aarch64_get_lanedi"
1981   [(match_operand:DI 0 "register_operand" "=r")
1982    (match_operand:DI 1 "register_operand" "w")
1983    (match_operand:SI 2 "immediate_operand" "i")]
1984   "TARGET_SIMD"
1985 {
1986   aarch64_simd_lane_bounds (operands[2], 0, 1);
1987   emit_move_insn (operands[0], operands[1]);
1988   DONE;
1989 })
1990
1991 (define_expand "aarch64_reinterpretv8qi<mode>"
1992   [(match_operand:V8QI 0 "register_operand" "")
1993    (match_operand:VDC 1 "register_operand" "")]
1994   "TARGET_SIMD"
1995 {
1996   aarch64_simd_reinterpret (operands[0], operands[1]);
1997   DONE;
1998 })
1999
2000 (define_expand "aarch64_reinterpretv4hi<mode>"
2001   [(match_operand:V4HI 0 "register_operand" "")
2002    (match_operand:VDC 1 "register_operand" "")]
2003   "TARGET_SIMD"
2004 {
2005   aarch64_simd_reinterpret (operands[0], operands[1]);
2006   DONE;
2007 })
2008
2009 (define_expand "aarch64_reinterpretv2si<mode>"
2010   [(match_operand:V2SI 0 "register_operand" "")
2011    (match_operand:VDC 1 "register_operand" "")]
2012   "TARGET_SIMD"
2013 {
2014   aarch64_simd_reinterpret (operands[0], operands[1]);
2015   DONE;
2016 })
2017
2018 (define_expand "aarch64_reinterpretv2sf<mode>"
2019   [(match_operand:V2SF 0 "register_operand" "")
2020    (match_operand:VDC 1 "register_operand" "")]
2021   "TARGET_SIMD"
2022 {
2023   aarch64_simd_reinterpret (operands[0], operands[1]);
2024   DONE;
2025 })
2026
2027 (define_expand "aarch64_reinterpretdi<mode>"
2028   [(match_operand:DI 0 "register_operand" "")
2029    (match_operand:VD_RE 1 "register_operand" "")]
2030   "TARGET_SIMD"
2031 {
2032   aarch64_simd_reinterpret (operands[0], operands[1]);
2033   DONE;
2034 })
2035
2036 (define_expand "aarch64_reinterpretv16qi<mode>"
2037   [(match_operand:V16QI 0 "register_operand" "")
2038    (match_operand:VQ 1 "register_operand" "")]
2039   "TARGET_SIMD"
2040 {
2041   aarch64_simd_reinterpret (operands[0], operands[1]);
2042   DONE;
2043 })
2044
2045 (define_expand "aarch64_reinterpretv8hi<mode>"
2046   [(match_operand:V8HI 0 "register_operand" "")
2047    (match_operand:VQ 1 "register_operand" "")]
2048   "TARGET_SIMD"
2049 {
2050   aarch64_simd_reinterpret (operands[0], operands[1]);
2051   DONE;
2052 })
2053
2054 (define_expand "aarch64_reinterpretv4si<mode>"
2055   [(match_operand:V4SI 0 "register_operand" "")
2056    (match_operand:VQ 1 "register_operand" "")]
2057   "TARGET_SIMD"
2058 {
2059   aarch64_simd_reinterpret (operands[0], operands[1]);
2060   DONE;
2061 })
2062
2063 (define_expand "aarch64_reinterpretv4sf<mode>"
2064   [(match_operand:V4SF 0 "register_operand" "")
2065    (match_operand:VQ 1 "register_operand" "")]
2066   "TARGET_SIMD"
2067 {
2068   aarch64_simd_reinterpret (operands[0], operands[1]);
2069   DONE;
2070 })
2071
2072 (define_expand "aarch64_reinterpretv2di<mode>"
2073   [(match_operand:V2DI 0 "register_operand" "")
2074    (match_operand:VQ 1 "register_operand" "")]
2075   "TARGET_SIMD"
2076 {
2077   aarch64_simd_reinterpret (operands[0], operands[1]);
2078   DONE;
2079 })
2080
2081 (define_expand "aarch64_reinterpretv2df<mode>"
2082   [(match_operand:V2DF 0 "register_operand" "")
2083    (match_operand:VQ 1 "register_operand" "")]
2084   "TARGET_SIMD"
2085 {
2086   aarch64_simd_reinterpret (operands[0], operands[1]);
2087   DONE;
2088 })
2089
2090 ;; In this insn, operand 1 should be low, and operand 2 the high part of the
2091 ;; dest vector.
2092
2093 (define_insn "*aarch64_combinez<mode>"
2094   [(set (match_operand:<VDBL> 0 "register_operand" "=&w")
2095         (vec_concat:<VDBL>
2096            (match_operand:VDIC 1 "register_operand" "w")
2097            (match_operand:VDIC 2 "aarch64_simd_imm_zero" "Dz")))]
2098   "TARGET_SIMD"
2099   "mov\\t%0.8b, %1.8b"
2100   [(set_attr "simd_type" "simd_move")
2101    (set_attr "simd_mode" "<MODE>")]
2102 )
2103
2104 (define_insn "aarch64_combine<mode>"
2105   [(set (match_operand:<VDBL> 0 "register_operand" "=&w")
2106         (vec_concat:<VDBL> (match_operand:VDC 1 "register_operand" "w")
2107                            (match_operand:VDC 2 "register_operand" "w")))]
2108   "TARGET_SIMD"
2109   "mov\\t%0.d[0], %1.d[0]\;ins\\t%0.d[1], %2.d[0]"
2110   [(set_attr "simd_type" "simd_ins")
2111    (set_attr "simd_mode" "<MODE>")]
2112 )
2113
2114 ;; <su><addsub>l<q>.
2115
2116 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>l2<mode>_internal"
2117  [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2118        (ADDSUB:<VWIDE> (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
2119                            (match_operand:VQW 1 "register_operand" "w")
2120                            (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))
2121                        (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
2122                            (match_operand:VQW 2 "register_operand" "w")
2123                            (match_dup 3)))))]
2124   "TARGET_SIMD"
2125   "<ANY_EXTEND:su><ADDSUB:optab>l2 %0.<Vwtype>, %1.<Vtype>, %2.<Vtype>"
2126   [(set_attr "simd_type" "simd_addl")
2127    (set_attr "simd_mode" "<MODE>")]
2128 )
2129
2130 (define_expand "aarch64_saddl2<mode>"
2131   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2132    (match_operand:VQW 1 "register_operand" "w")
2133    (match_operand:VQW 2 "register_operand" "w")]
2134   "TARGET_SIMD"
2135 {
2136   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2137   emit_insn (gen_aarch64_saddl2<mode>_internal (operands[0], operands[1],
2138                                                 operands[2], p));
2139   DONE;
2140 })
2141
2142 (define_expand "aarch64_uaddl2<mode>"
2143   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2144    (match_operand:VQW 1 "register_operand" "w")
2145    (match_operand:VQW 2 "register_operand" "w")]
2146   "TARGET_SIMD"
2147 {
2148   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2149   emit_insn (gen_aarch64_uaddl2<mode>_internal (operands[0], operands[1],
2150                                                 operands[2], p));
2151   DONE;
2152 })
2153
2154 (define_expand "aarch64_ssubl2<mode>"
2155   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2156    (match_operand:VQW 1 "register_operand" "w")
2157    (match_operand:VQW 2 "register_operand" "w")]
2158   "TARGET_SIMD"
2159 {
2160   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2161   emit_insn (gen_aarch64_ssubl2<mode>_internal (operands[0], operands[1],
2162                                                 operands[2], p));
2163   DONE;
2164 })
2165
2166 (define_expand "aarch64_usubl2<mode>"
2167   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2168    (match_operand:VQW 1 "register_operand" "w")
2169    (match_operand:VQW 2 "register_operand" "w")]
2170   "TARGET_SIMD"
2171 {
2172   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2173   emit_insn (gen_aarch64_usubl2<mode>_internal (operands[0], operands[1],
2174                                                 operands[2], p));
2175   DONE;
2176 })
2177
2178 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>l<mode>"
2179  [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2180        (ADDSUB:<VWIDE> (ANY_EXTEND:<VWIDE>
2181                            (match_operand:VDW 1 "register_operand" "w"))
2182                        (ANY_EXTEND:<VWIDE>
2183                            (match_operand:VDW 2 "register_operand" "w"))))]
2184   "TARGET_SIMD"
2185   "<ANY_EXTEND:su><ADDSUB:optab>l %0.<Vwtype>, %1.<Vtype>, %2.<Vtype>"
2186   [(set_attr "simd_type" "simd_addl")
2187    (set_attr "simd_mode" "<MODE>")]
2188 )
2189
2190 ;; <su><addsub>w<q>.
2191
2192 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>w<mode>"
2193   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2194         (ADDSUB:<VWIDE> (match_operand:<VWIDE> 1 "register_operand" "w")
2195                         (ANY_EXTEND:<VWIDE>
2196                           (match_operand:VDW 2 "register_operand" "w"))))]
2197   "TARGET_SIMD"
2198   "<ANY_EXTEND:su><ADDSUB:optab>w\\t%0.<Vwtype>, %1.<Vwtype>, %2.<Vtype>"
2199   [(set_attr "simd_type" "simd_addl")
2200    (set_attr "simd_mode" "<MODE>")]
2201 )
2202
2203 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>w2<mode>_internal"
2204   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2205         (ADDSUB:<VWIDE> (match_operand:<VWIDE> 1 "register_operand" "w")
2206                         (ANY_EXTEND:<VWIDE>
2207                           (vec_select:<VHALF>
2208                            (match_operand:VQW 2 "register_operand" "w")
2209                            (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))))]
2210   "TARGET_SIMD"
2211   "<ANY_EXTEND:su><ADDSUB:optab>w2\\t%0.<Vwtype>, %1.<Vwtype>, %2.<Vtype>"
2212   [(set_attr "simd_type" "simd_addl")
2213    (set_attr "simd_mode" "<MODE>")]
2214 )
2215
2216 (define_expand "aarch64_saddw2<mode>"
2217   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2218    (match_operand:<VWIDE> 1 "register_operand" "w")
2219    (match_operand:VQW 2 "register_operand" "w")]
2220   "TARGET_SIMD"
2221 {
2222   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2223   emit_insn (gen_aarch64_saddw2<mode>_internal (operands[0], operands[1],
2224                                                 operands[2], p));
2225   DONE;
2226 })
2227
2228 (define_expand "aarch64_uaddw2<mode>"
2229   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2230    (match_operand:<VWIDE> 1 "register_operand" "w")
2231    (match_operand:VQW 2 "register_operand" "w")]
2232   "TARGET_SIMD"
2233 {
2234   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2235   emit_insn (gen_aarch64_uaddw2<mode>_internal (operands[0], operands[1],
2236                                                 operands[2], p));
2237   DONE;
2238 })
2239
2240
2241 (define_expand "aarch64_ssubw2<mode>"
2242   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2243    (match_operand:<VWIDE> 1 "register_operand" "w")
2244    (match_operand:VQW 2 "register_operand" "w")]
2245   "TARGET_SIMD"
2246 {
2247   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2248   emit_insn (gen_aarch64_ssubw2<mode>_internal (operands[0], operands[1],
2249                                                 operands[2], p));
2250   DONE;
2251 })
2252
2253 (define_expand "aarch64_usubw2<mode>"
2254   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2255    (match_operand:<VWIDE> 1 "register_operand" "w")
2256    (match_operand:VQW 2 "register_operand" "w")]
2257   "TARGET_SIMD"
2258 {
2259   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2260   emit_insn (gen_aarch64_usubw2<mode>_internal (operands[0], operands[1],
2261                                                 operands[2], p));
2262   DONE;
2263 })
2264
2265 ;; <su><r>h<addsub>.
2266
2267 (define_insn "aarch64_<sur>h<addsub><mode>"
2268   [(set (match_operand:VQ_S 0 "register_operand" "=w")
2269         (unspec:VQ_S [(match_operand:VQ_S 1 "register_operand" "w")
2270                       (match_operand:VQ_S 2 "register_operand" "w")]
2271                      HADDSUB))]
2272   "TARGET_SIMD"
2273   "<sur>h<addsub>\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
2274   [(set_attr "simd_type" "simd_add")
2275    (set_attr "simd_mode" "<MODE>")]
2276 )
2277
2278 ;; <r><addsub>hn<q>.
2279
2280 (define_insn "aarch64_<sur><addsub>hn<mode>"
2281   [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
2282         (unspec:<VNARROWQ> [(match_operand:VQN 1 "register_operand" "w")
2283                             (match_operand:VQN 2 "register_operand" "w")]
2284                            ADDSUBHN))]
2285   "TARGET_SIMD"
2286   "<sur><addsub>hn\\t%0.<Vntype>, %1.<Vtype>, %2.<Vtype>"
2287   [(set_attr "simd_type" "simd_addn")
2288    (set_attr "simd_mode" "<MODE>")]
2289 )
2290
2291 (define_insn "aarch64_<sur><addsub>hn2<mode>"
2292   [(set (match_operand:<VNARROWQ2> 0 "register_operand" "=w")
2293         (unspec:<VNARROWQ2> [(match_operand:<VNARROWQ> 1 "register_operand" "0")
2294                              (match_operand:VQN 2 "register_operand" "w")
2295                              (match_operand:VQN 3 "register_operand" "w")]
2296                             ADDSUBHN2))]
2297   "TARGET_SIMD"
2298   "<sur><addsub>hn2\\t%0.<V2ntype>, %2.<Vtype>, %3.<Vtype>"
2299   [(set_attr "simd_type" "simd_addn2")
2300    (set_attr "simd_mode" "<MODE>")]
2301 )
2302
2303 ;; pmul.
2304
2305 (define_insn "aarch64_pmul<mode>"
2306   [(set (match_operand:VB 0 "register_operand" "=w")
2307         (unspec:VB [(match_operand:VB 1 "register_operand" "w")
2308                     (match_operand:VB 2 "register_operand" "w")]
2309                    UNSPEC_PMUL))]
2310  "TARGET_SIMD"
2311  "pmul\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
2312   [(set_attr "simd_type" "simd_mul")
2313    (set_attr "simd_mode" "<MODE>")]
2314 )
2315
2316 ;; <su>q<addsub>
2317
2318 (define_insn "aarch64_<su_optab><optab><mode>"
2319   [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
2320         (BINQOPS:VSDQ_I (match_operand:VSDQ_I 1 "register_operand" "w")
2321                           (match_operand:VSDQ_I 2 "register_operand" "w")))]
2322   "TARGET_SIMD"
2323   "<su_optab><optab>\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
2324   [(set_attr "simd_type" "simd_add")
2325    (set_attr "simd_mode" "<MODE>")]
2326 )
2327
2328 ;; suqadd and usqadd
2329
2330 (define_insn "aarch64_<sur>qadd<mode>"
2331   [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
2332         (unspec:VSDQ_I [(match_operand:VSDQ_I 1 "register_operand" "0")
2333                         (match_operand:VSDQ_I 2 "register_operand" "w")]
2334                        USSUQADD))]
2335   "TARGET_SIMD"
2336   "<sur>qadd\\t%<v>0<Vmtype>, %<v>2<Vmtype>"
2337   [(set_attr "simd_type" "simd_sat_add")
2338    (set_attr "simd_mode" "<MODE>")]
2339 )
2340
2341 ;; sqmovun
2342
2343 (define_insn "aarch64_sqmovun<mode>"
2344   [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
2345         (unspec:<VNARROWQ> [(match_operand:VSQN_HSDI 1 "register_operand" "w")]
2346                             UNSPEC_SQXTUN))]
2347    "TARGET_SIMD"
2348    "sqxtun\\t%<vn2>0<Vmntype>, %<v>1<Vmtype>"
2349    [(set_attr "simd_type" "simd_sat_shiftn_imm")
2350     (set_attr "simd_mode" "<MODE>")]
2351  )
2352
2353 ;; sqmovn and uqmovn
2354
2355 (define_insn "aarch64_<sur>qmovn<mode>"
2356   [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
2357         (unspec:<VNARROWQ> [(match_operand:VSQN_HSDI 1 "register_operand" "w")]
2358                             SUQMOVN))]
2359   "TARGET_SIMD"
2360   "<sur>qxtn\\t%<vn2>0<Vmntype>, %<v>1<Vmtype>"
2361    [(set_attr "simd_type" "simd_sat_shiftn_imm")
2362     (set_attr "simd_mode" "<MODE>")]
2363  )
2364
2365 ;; <su>q<absneg>
2366
2367 (define_insn "aarch64_s<optab><mode>"
2368   [(set (match_operand:VSDQ_I_BHSI 0 "register_operand" "=w")
2369         (UNQOPS:VSDQ_I_BHSI
2370           (match_operand:VSDQ_I_BHSI 1 "register_operand" "w")))]
2371   "TARGET_SIMD"
2372   "s<optab>\\t%<v>0<Vmtype>, %<v>1<Vmtype>"
2373   [(set_attr "simd_type" "simd_sat_negabs")
2374    (set_attr "simd_mode" "<MODE>")]
2375 )
2376
2377 ;; sq<r>dmulh.
2378
2379 (define_insn "aarch64_sq<r>dmulh<mode>"
2380   [(set (match_operand:VSDQ_HSI 0 "register_operand" "=w")
2381         (unspec:VSDQ_HSI
2382           [(match_operand:VSDQ_HSI 1 "register_operand" "w")
2383            (match_operand:VSDQ_HSI 2 "register_operand" "w")]
2384          VQDMULH))]
2385   "TARGET_SIMD"
2386   "sq<r>dmulh\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
2387   [(set_attr "simd_type" "simd_sat_mul")
2388    (set_attr "simd_mode" "<MODE>")]
2389 )
2390
2391 ;; sq<r>dmulh_lane
2392
2393 (define_insn "aarch64_sq<r>dmulh_lane<mode>"
2394   [(set (match_operand:VDQHS 0 "register_operand" "=w")
2395         (unspec:VDQHS
2396           [(match_operand:VDQHS 1 "register_operand" "w")
2397            (vec_select:<VEL>
2398              (match_operand:<VCOND> 2 "register_operand" "<vwx>")
2399              (parallel [(match_operand:SI 3 "immediate_operand" "i")]))]
2400          VQDMULH))]
2401   "TARGET_SIMD"
2402   "*
2403    aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCOND>mode));
2404    return \"sq<r>dmulh\\t%0.<Vtype>, %1.<Vtype>, %2.<Vetype>[%3]\";"
2405   [(set_attr "simd_type" "simd_sat_mul")
2406    (set_attr "simd_mode" "<MODE>")]
2407 )
2408
2409 (define_insn "aarch64_sq<r>dmulh_laneq<mode>"
2410   [(set (match_operand:VDQHS 0 "register_operand" "=w")
2411         (unspec:VDQHS
2412           [(match_operand:VDQHS 1 "register_operand" "w")
2413            (vec_select:<VEL>
2414              (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
2415              (parallel [(match_operand:SI 3 "immediate_operand" "i")]))]
2416          VQDMULH))]
2417   "TARGET_SIMD"
2418   "*
2419    aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCONQ>mode));
2420    return \"sq<r>dmulh\\t%0.<Vtype>, %1.<Vtype>, %2.<Vetype>[%3]\";"
2421   [(set_attr "simd_type" "simd_sat_mul")
2422    (set_attr "simd_mode" "<MODE>")]
2423 )
2424
2425 (define_insn "aarch64_sq<r>dmulh_lane<mode>"
2426   [(set (match_operand:SD_HSI 0 "register_operand" "=w")
2427         (unspec:SD_HSI
2428           [(match_operand:SD_HSI 1 "register_operand" "w")
2429            (vec_select:<VEL>
2430              (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
2431              (parallel [(match_operand:SI 3 "immediate_operand" "i")]))]
2432          VQDMULH))]
2433   "TARGET_SIMD"
2434   "*
2435    aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCONQ>mode));
2436    return \"sq<r>dmulh\\t%<v>0, %<v>1, %2.<v>[%3]\";"
2437   [(set_attr "simd_type" "simd_sat_mul")
2438    (set_attr "simd_mode" "<MODE>")]
2439 )
2440
2441 ;; vqdml[sa]l
2442
2443 (define_insn "aarch64_sqdml<SBINQOPS:as>l<mode>"
2444   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2445         (SBINQOPS:<VWIDE>
2446           (match_operand:<VWIDE> 1 "register_operand" "0")
2447           (ss_ashift:<VWIDE>
2448               (mult:<VWIDE>
2449                 (sign_extend:<VWIDE>
2450                       (match_operand:VSD_HSI 2 "register_operand" "w"))
2451                 (sign_extend:<VWIDE>
2452                       (match_operand:VSD_HSI 3 "register_operand" "w")))
2453               (const_int 1))))]
2454   "TARGET_SIMD"
2455   "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %<v>3<Vmtype>"
2456   [(set_attr "simd_type" "simd_sat_mlal")
2457    (set_attr "simd_mode" "<MODE>")]
2458 )
2459
2460 ;; vqdml[sa]l_lane
2461
2462 (define_insn "aarch64_sqdml<SBINQOPS:as>l_lane<mode>_internal"
2463   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2464         (SBINQOPS:<VWIDE>
2465           (match_operand:<VWIDE> 1 "register_operand" "0")
2466           (ss_ashift:<VWIDE>
2467             (mult:<VWIDE>
2468               (sign_extend:<VWIDE>
2469                 (match_operand:VD_HSI 2 "register_operand" "w"))
2470               (sign_extend:<VWIDE>
2471                 (vec_duplicate:VD_HSI
2472                   (vec_select:<VEL>
2473                     (match_operand:<VCON> 3 "register_operand" "<vwx>")
2474                     (parallel [(match_operand:SI 4 "immediate_operand" "i")])))
2475               ))
2476             (const_int 1))))]
2477   "TARGET_SIMD"
2478   "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]"
2479   [(set_attr "simd_type" "simd_sat_mlal")
2480    (set_attr "simd_mode" "<MODE>")]
2481 )
2482
2483 (define_insn "aarch64_sqdml<SBINQOPS:as>l_lane<mode>_internal"
2484   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2485         (SBINQOPS:<VWIDE>
2486           (match_operand:<VWIDE> 1 "register_operand" "0")
2487           (ss_ashift:<VWIDE>
2488             (mult:<VWIDE>
2489               (sign_extend:<VWIDE>
2490                 (match_operand:SD_HSI 2 "register_operand" "w"))
2491               (sign_extend:<VWIDE>
2492                 (vec_select:<VEL>
2493                   (match_operand:<VCON> 3 "register_operand" "<vwx>")
2494                   (parallel [(match_operand:SI 4 "immediate_operand" "i")])))
2495               )
2496             (const_int 1))))]
2497   "TARGET_SIMD"
2498   "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]"
2499   [(set_attr "simd_type" "simd_sat_mlal")
2500    (set_attr "simd_mode" "<MODE>")]
2501 )
2502
2503 (define_expand "aarch64_sqdmlal_lane<mode>"
2504   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2505    (match_operand:<VWIDE> 1 "register_operand" "0")
2506    (match_operand:VSD_HSI 2 "register_operand" "w")
2507    (match_operand:<VCON> 3 "register_operand" "<vwx>")
2508    (match_operand:SI 4 "immediate_operand" "i")]
2509   "TARGET_SIMD"
2510 {
2511   aarch64_simd_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<VCON>mode) / 2);
2512   emit_insn (gen_aarch64_sqdmlal_lane<mode>_internal (operands[0], operands[1],
2513                                                       operands[2], operands[3],
2514                                                       operands[4]));
2515   DONE;
2516 })
2517
2518 (define_expand "aarch64_sqdmlal_laneq<mode>"
2519   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2520    (match_operand:<VWIDE> 1 "register_operand" "0")
2521    (match_operand:VSD_HSI 2 "register_operand" "w")
2522    (match_operand:<VCON> 3 "register_operand" "<vwx>")
2523    (match_operand:SI 4 "immediate_operand" "i")]
2524   "TARGET_SIMD"
2525 {
2526   aarch64_simd_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<VCON>mode));
2527   emit_insn (gen_aarch64_sqdmlal_lane<mode>_internal (operands[0], operands[1],
2528                                                       operands[2], operands[3],
2529                                                       operands[4]));
2530   DONE;
2531 })
2532
2533 (define_expand "aarch64_sqdmlsl_lane<mode>"
2534   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2535    (match_operand:<VWIDE> 1 "register_operand" "0")
2536    (match_operand:VSD_HSI 2 "register_operand" "w")
2537    (match_operand:<VCON> 3 "register_operand" "<vwx>")
2538    (match_operand:SI 4 "immediate_operand" "i")]
2539   "TARGET_SIMD"
2540 {
2541   aarch64_simd_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<VCON>mode) / 2);
2542   emit_insn (gen_aarch64_sqdmlsl_lane<mode>_internal (operands[0], operands[1],
2543                                                       operands[2], operands[3],
2544                                                       operands[4]));
2545   DONE;
2546 })
2547
2548 (define_expand "aarch64_sqdmlsl_laneq<mode>"
2549   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2550    (match_operand:<VWIDE> 1 "register_operand" "0")
2551    (match_operand:VSD_HSI 2 "register_operand" "w")
2552    (match_operand:<VCON> 3 "register_operand" "<vwx>")
2553    (match_operand:SI 4 "immediate_operand" "i")]
2554   "TARGET_SIMD"
2555 {
2556   aarch64_simd_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<VCON>mode));
2557   emit_insn (gen_aarch64_sqdmlsl_lane<mode>_internal (operands[0], operands[1],
2558                                                       operands[2], operands[3],
2559                                                       operands[4]));
2560   DONE;
2561 })
2562
2563 ;; vqdml[sa]l_n
2564
2565 (define_insn "aarch64_sqdml<SBINQOPS:as>l_n<mode>"
2566   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2567         (SBINQOPS:<VWIDE>
2568           (match_operand:<VWIDE> 1 "register_operand" "0")
2569           (ss_ashift:<VWIDE>
2570               (mult:<VWIDE>
2571                 (sign_extend:<VWIDE>
2572                       (match_operand:VD_HSI 2 "register_operand" "w"))
2573                 (sign_extend:<VWIDE>
2574                   (vec_duplicate:VD_HSI
2575                     (match_operand:<VEL> 3 "register_operand" "w"))))
2576               (const_int 1))))]
2577   "TARGET_SIMD"
2578   "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[0]"
2579   [(set_attr "simd_type" "simd_sat_mlal")
2580    (set_attr "simd_mode" "<MODE>")]
2581 )
2582
2583 ;; sqdml[as]l2
2584
2585 (define_insn "aarch64_sqdml<SBINQOPS:as>l2<mode>_internal"
2586   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2587         (SBINQOPS:<VWIDE>
2588          (match_operand:<VWIDE> 1 "register_operand" "0")
2589          (ss_ashift:<VWIDE>
2590              (mult:<VWIDE>
2591                (sign_extend:<VWIDE>
2592                  (vec_select:<VHALF>
2593                      (match_operand:VQ_HSI 2 "register_operand" "w")
2594                      (match_operand:VQ_HSI 4 "vect_par_cnst_hi_half" "")))
2595                (sign_extend:<VWIDE>
2596                  (vec_select:<VHALF>
2597                      (match_operand:VQ_HSI 3 "register_operand" "w")
2598                      (match_dup 4))))
2599              (const_int 1))))]
2600   "TARGET_SIMD"
2601   "sqdml<SBINQOPS:as>l2\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %<v>3<Vmtype>"
2602   [(set_attr "simd_type" "simd_sat_mlal")
2603    (set_attr "simd_mode" "<MODE>")]
2604 )
2605
2606 (define_expand "aarch64_sqdmlal2<mode>"
2607   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2608    (match_operand:<VWIDE> 1 "register_operand" "w")
2609    (match_operand:VQ_HSI 2 "register_operand" "w")
2610    (match_operand:VQ_HSI 3 "register_operand" "w")]
2611   "TARGET_SIMD"
2612 {
2613   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2614   emit_insn (gen_aarch64_sqdmlal2<mode>_internal (operands[0], operands[1],
2615                                                   operands[2], operands[3], p));
2616   DONE;
2617 })
2618
2619 (define_expand "aarch64_sqdmlsl2<mode>"
2620   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2621    (match_operand:<VWIDE> 1 "register_operand" "w")
2622    (match_operand:VQ_HSI 2 "register_operand" "w")
2623    (match_operand:VQ_HSI 3 "register_operand" "w")]
2624   "TARGET_SIMD"
2625 {
2626   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2627   emit_insn (gen_aarch64_sqdmlsl2<mode>_internal (operands[0], operands[1],
2628                                                   operands[2], operands[3], p));
2629   DONE;
2630 })
2631
2632 ;; vqdml[sa]l2_lane
2633
2634 (define_insn "aarch64_sqdml<SBINQOPS:as>l2_lane<mode>_internal"
2635   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2636         (SBINQOPS:<VWIDE>
2637           (match_operand:<VWIDE> 1 "register_operand" "0")
2638           (ss_ashift:<VWIDE>
2639               (mult:<VWIDE>
2640                 (sign_extend:<VWIDE>
2641                   (vec_select:<VHALF>
2642                     (match_operand:VQ_HSI 2 "register_operand" "w")
2643                     (match_operand:VQ_HSI 5 "vect_par_cnst_hi_half" "")))
2644                 (sign_extend:<VWIDE>
2645                   (vec_duplicate:<VHALF>
2646                     (vec_select:<VEL>
2647                       (match_operand:<VCON> 3 "register_operand" "<vwx>")
2648                       (parallel [(match_operand:SI 4 "immediate_operand" "i")])
2649                     ))))
2650               (const_int 1))))]
2651   "TARGET_SIMD"
2652   "sqdml<SBINQOPS:as>l2\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]"
2653   [(set_attr "simd_type" "simd_sat_mlal")
2654    (set_attr "simd_mode" "<MODE>")]
2655 )
2656
2657 (define_expand "aarch64_sqdmlal2_lane<mode>"
2658   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2659    (match_operand:<VWIDE> 1 "register_operand" "w")
2660    (match_operand:VQ_HSI 2 "register_operand" "w")
2661    (match_operand:<VCON> 3 "register_operand" "<vwx>")
2662    (match_operand:SI 4 "immediate_operand" "i")]
2663   "TARGET_SIMD"
2664 {
2665   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2666   aarch64_simd_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode) / 2);
2667   emit_insn (gen_aarch64_sqdmlal2_lane<mode>_internal (operands[0], operands[1],
2668                                                        operands[2], operands[3],
2669                                                        operands[4], p));
2670   DONE;
2671 })
2672
2673 (define_expand "aarch64_sqdmlal2_laneq<mode>"
2674   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2675    (match_operand:<VWIDE> 1 "register_operand" "w")
2676    (match_operand:VQ_HSI 2 "register_operand" "w")
2677    (match_operand:<VCON> 3 "register_operand" "<vwx>")
2678    (match_operand:SI 4 "immediate_operand" "i")]
2679   "TARGET_SIMD"
2680 {
2681   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2682   aarch64_simd_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
2683   emit_insn (gen_aarch64_sqdmlal2_lane<mode>_internal (operands[0], operands[1],
2684                                                        operands[2], operands[3],
2685                                                        operands[4], p));
2686   DONE;
2687 })
2688
2689 (define_expand "aarch64_sqdmlsl2_lane<mode>"
2690   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2691    (match_operand:<VWIDE> 1 "register_operand" "w")
2692    (match_operand:VQ_HSI 2 "register_operand" "w")
2693    (match_operand:<VCON> 3 "register_operand" "<vwx>")
2694    (match_operand:SI 4 "immediate_operand" "i")]
2695   "TARGET_SIMD"
2696 {
2697   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2698   aarch64_simd_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode) / 2);
2699   emit_insn (gen_aarch64_sqdmlsl2_lane<mode>_internal (operands[0], operands[1],
2700                                                        operands[2], operands[3],
2701                                                        operands[4], p));
2702   DONE;
2703 })
2704
2705 (define_expand "aarch64_sqdmlsl2_laneq<mode>"
2706   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2707    (match_operand:<VWIDE> 1 "register_operand" "w")
2708    (match_operand:VQ_HSI 2 "register_operand" "w")
2709    (match_operand:<VCON> 3 "register_operand" "<vwx>")
2710    (match_operand:SI 4 "immediate_operand" "i")]
2711   "TARGET_SIMD"
2712 {
2713   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2714   aarch64_simd_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
2715   emit_insn (gen_aarch64_sqdmlsl2_lane<mode>_internal (operands[0], operands[1],
2716                                                        operands[2], operands[3],
2717                                                        operands[4], p));
2718   DONE;
2719 })
2720
2721 (define_insn "aarch64_sqdml<SBINQOPS:as>l2_n<mode>_internal"
2722   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2723         (SBINQOPS:<VWIDE>
2724           (match_operand:<VWIDE> 1 "register_operand" "0")
2725           (ss_ashift:<VWIDE>
2726             (mult:<VWIDE>
2727               (sign_extend:<VWIDE>
2728                 (vec_select:<VHALF>
2729                   (match_operand:VQ_HSI 2 "register_operand" "w")
2730                   (match_operand:VQ_HSI 4 "vect_par_cnst_hi_half" "")))
2731               (sign_extend:<VWIDE>
2732                 (vec_duplicate:<VHALF>
2733                   (match_operand:<VEL> 3 "register_operand" "w"))))
2734             (const_int 1))))]
2735   "TARGET_SIMD"
2736   "sqdml<SBINQOPS:as>l2\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[0]"
2737   [(set_attr "simd_type" "simd_sat_mlal")
2738    (set_attr "simd_mode" "<MODE>")]
2739 )
2740
2741 (define_expand "aarch64_sqdmlal2_n<mode>"
2742   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2743    (match_operand:<VWIDE> 1 "register_operand" "w")
2744    (match_operand:VQ_HSI 2 "register_operand" "w")
2745    (match_operand:<VEL> 3 "register_operand" "w")]
2746   "TARGET_SIMD"
2747 {
2748   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2749   emit_insn (gen_aarch64_sqdmlal2_n<mode>_internal (operands[0], operands[1],
2750                                                     operands[2], operands[3],
2751                                                     p));
2752   DONE;
2753 })
2754
2755 (define_expand "aarch64_sqdmlsl2_n<mode>"
2756   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2757    (match_operand:<VWIDE> 1 "register_operand" "w")
2758    (match_operand:VQ_HSI 2 "register_operand" "w")
2759    (match_operand:<VEL> 3 "register_operand" "w")]
2760   "TARGET_SIMD"
2761 {
2762   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2763   emit_insn (gen_aarch64_sqdmlsl2_n<mode>_internal (operands[0], operands[1],
2764                                                     operands[2], operands[3],
2765                                                     p));
2766   DONE;
2767 })
2768
2769 ;; vqdmull
2770
2771 (define_insn "aarch64_sqdmull<mode>"
2772   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2773         (ss_ashift:<VWIDE>
2774              (mult:<VWIDE>
2775                (sign_extend:<VWIDE>
2776                      (match_operand:VSD_HSI 1 "register_operand" "w"))
2777                (sign_extend:<VWIDE>
2778                      (match_operand:VSD_HSI 2 "register_operand" "w")))
2779              (const_int 1)))]
2780   "TARGET_SIMD"
2781   "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
2782   [(set_attr "simd_type" "simd_sat_mul")
2783    (set_attr "simd_mode" "<MODE>")]
2784 )
2785
2786 ;; vqdmull_lane
2787
2788 (define_insn "aarch64_sqdmull_lane<mode>_internal"
2789   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2790         (ss_ashift:<VWIDE>
2791              (mult:<VWIDE>
2792                (sign_extend:<VWIDE>
2793                  (match_operand:VD_HSI 1 "register_operand" "w"))
2794                (sign_extend:<VWIDE>
2795                  (vec_duplicate:VD_HSI
2796                    (vec_select:<VEL>
2797                      (match_operand:<VCON> 2 "register_operand" "<vwx>")
2798                      (parallel [(match_operand:SI 3 "immediate_operand" "i")])))
2799                ))
2800              (const_int 1)))]
2801   "TARGET_SIMD"
2802   "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]"
2803   [(set_attr "simd_type" "simd_sat_mul")
2804    (set_attr "simd_mode" "<MODE>")]
2805 )
2806
2807 (define_insn "aarch64_sqdmull_lane<mode>_internal"
2808   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2809         (ss_ashift:<VWIDE>
2810              (mult:<VWIDE>
2811                (sign_extend:<VWIDE>
2812                  (match_operand:SD_HSI 1 "register_operand" "w"))
2813                (sign_extend:<VWIDE>
2814                  (vec_select:<VEL>
2815                    (match_operand:<VCON> 2 "register_operand" "<vwx>")
2816                    (parallel [(match_operand:SI 3 "immediate_operand" "i")]))
2817                ))
2818              (const_int 1)))]
2819   "TARGET_SIMD"
2820   "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]"
2821   [(set_attr "simd_type" "simd_sat_mul")
2822    (set_attr "simd_mode" "<MODE>")]
2823 )
2824
2825 (define_expand "aarch64_sqdmull_lane<mode>"
2826   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2827    (match_operand:VSD_HSI 1 "register_operand" "w")
2828    (match_operand:<VCON> 2 "register_operand" "<vwx>")
2829    (match_operand:SI 3 "immediate_operand" "i")]
2830   "TARGET_SIMD"
2831 {
2832   aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCON>mode) / 2);
2833   emit_insn (gen_aarch64_sqdmull_lane<mode>_internal (operands[0], operands[1],
2834                                                       operands[2], operands[3]));
2835   DONE;
2836 })
2837
2838 (define_expand "aarch64_sqdmull_laneq<mode>"
2839   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2840    (match_operand:VD_HSI 1 "register_operand" "w")
2841    (match_operand:<VCON> 2 "register_operand" "<vwx>")
2842    (match_operand:SI 3 "immediate_operand" "i")]
2843   "TARGET_SIMD"
2844 {
2845   aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCON>mode));
2846   emit_insn (gen_aarch64_sqdmull_lane<mode>_internal
2847                (operands[0], operands[1], operands[2], operands[3]));
2848   DONE;
2849 })
2850
2851 ;; vqdmull_n
2852
2853 (define_insn "aarch64_sqdmull_n<mode>"
2854   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2855         (ss_ashift:<VWIDE>
2856              (mult:<VWIDE>
2857                (sign_extend:<VWIDE>
2858                  (match_operand:VD_HSI 1 "register_operand" "w"))
2859                (sign_extend:<VWIDE>
2860                  (vec_duplicate:VD_HSI
2861                    (match_operand:<VEL> 2 "register_operand" "w")))
2862                )
2863              (const_int 1)))]
2864   "TARGET_SIMD"
2865   "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[0]"
2866   [(set_attr "simd_type" "simd_sat_mul")
2867    (set_attr "simd_mode" "<MODE>")]
2868 )
2869
2870 ;; vqdmull2
2871
2872
2873
2874 (define_insn "aarch64_sqdmull2<mode>_internal"
2875   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2876         (ss_ashift:<VWIDE>
2877              (mult:<VWIDE>
2878                (sign_extend:<VWIDE>
2879                  (vec_select:<VHALF>
2880                    (match_operand:VQ_HSI 1 "register_operand" "w")
2881                    (match_operand:VQ_HSI 3 "vect_par_cnst_hi_half" "")))
2882                (sign_extend:<VWIDE>
2883                  (vec_select:<VHALF>
2884                    (match_operand:VQ_HSI 2 "register_operand" "w")
2885                    (match_dup 3)))
2886                )
2887              (const_int 1)))]
2888   "TARGET_SIMD"
2889   "sqdmull2\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
2890   [(set_attr "simd_type" "simd_sat_mul")
2891    (set_attr "simd_mode" "<MODE>")]
2892 )
2893
2894 (define_expand "aarch64_sqdmull2<mode>"
2895   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2896    (match_operand:VQ_HSI 1 "register_operand" "w")
2897    (match_operand:<VCON> 2 "register_operand" "w")]
2898   "TARGET_SIMD"
2899 {
2900   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2901   emit_insn (gen_aarch64_sqdmull2<mode>_internal (operands[0], operands[1],
2902                                                   operands[2], p));
2903   DONE;
2904 })
2905
2906 ;; vqdmull2_lane
2907
2908 (define_insn "aarch64_sqdmull2_lane<mode>_internal"
2909   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2910         (ss_ashift:<VWIDE>
2911              (mult:<VWIDE>
2912                (sign_extend:<VWIDE>
2913                  (vec_select:<VHALF>
2914                    (match_operand:VQ_HSI 1 "register_operand" "w")
2915                    (match_operand:VQ_HSI 4 "vect_par_cnst_hi_half" "")))
2916                (sign_extend:<VWIDE>
2917                  (vec_duplicate:<VHALF>
2918                    (vec_select:<VEL>
2919                      (match_operand:<VCON> 2 "register_operand" "<vwx>")
2920                      (parallel [(match_operand:SI 3 "immediate_operand" "i")])))
2921                ))
2922              (const_int 1)))]
2923   "TARGET_SIMD"
2924   "sqdmull2\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]"
2925   [(set_attr "simd_type" "simd_sat_mul")
2926    (set_attr "simd_mode" "<MODE>")]
2927 )
2928
2929 (define_expand "aarch64_sqdmull2_lane<mode>"
2930   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2931    (match_operand:VQ_HSI 1 "register_operand" "w")
2932    (match_operand:<VCON> 2 "register_operand" "<vwx>")
2933    (match_operand:SI 3 "immediate_operand" "i")]
2934   "TARGET_SIMD"
2935 {
2936   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2937   aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode) / 2);
2938   emit_insn (gen_aarch64_sqdmull2_lane<mode>_internal (operands[0], operands[1],
2939                                                        operands[2], operands[3],
2940                                                        p));
2941   DONE;
2942 })
2943
2944 (define_expand "aarch64_sqdmull2_laneq<mode>"
2945   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2946    (match_operand:VQ_HSI 1 "register_operand" "w")
2947    (match_operand:<VCON> 2 "register_operand" "<vwx>")
2948    (match_operand:SI 3 "immediate_operand" "i")]
2949   "TARGET_SIMD"
2950 {
2951   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2952   aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
2953   emit_insn (gen_aarch64_sqdmull2_lane<mode>_internal (operands[0], operands[1],
2954                                                        operands[2], operands[3],
2955                                                        p));
2956   DONE;
2957 })
2958
2959 ;; vqdmull2_n
2960
2961 (define_insn "aarch64_sqdmull2_n<mode>_internal"
2962   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2963         (ss_ashift:<VWIDE>
2964              (mult:<VWIDE>
2965                (sign_extend:<VWIDE>
2966                  (vec_select:<VHALF>
2967                    (match_operand:VQ_HSI 1 "register_operand" "w")
2968                    (match_operand:VQ_HSI 3 "vect_par_cnst_hi_half" "")))
2969                (sign_extend:<VWIDE>
2970                  (vec_duplicate:<VHALF>
2971                    (match_operand:<VEL> 2 "register_operand" "w")))
2972                )
2973              (const_int 1)))]
2974   "TARGET_SIMD"
2975   "sqdmull2\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[0]"
2976   [(set_attr "simd_type" "simd_sat_mul")
2977    (set_attr "simd_mode" "<MODE>")]
2978 )
2979
2980 (define_expand "aarch64_sqdmull2_n<mode>"
2981   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2982    (match_operand:VQ_HSI 1 "register_operand" "w")
2983    (match_operand:<VEL> 2 "register_operand" "w")]
2984   "TARGET_SIMD"
2985 {
2986   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2987   emit_insn (gen_aarch64_sqdmull2_n<mode>_internal (operands[0], operands[1],
2988                                                     operands[2], p));
2989   DONE;
2990 })
2991
2992 ;; vshl
2993
2994 (define_insn "aarch64_<sur>shl<mode>"
2995   [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
2996         (unspec:VSDQ_I_DI
2997           [(match_operand:VSDQ_I_DI 1 "register_operand" "w")
2998            (match_operand:VSDQ_I_DI 2 "register_operand" "w")]
2999          VSHL))]
3000   "TARGET_SIMD"
3001   "<sur>shl\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>";
3002   [(set_attr "simd_type" "simd_shift")
3003    (set_attr "simd_mode" "<MODE>")]
3004 )
3005
3006
3007 ;; vqshl
3008
3009 (define_insn "aarch64_<sur>q<r>shl<mode>"
3010   [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
3011         (unspec:VSDQ_I
3012           [(match_operand:VSDQ_I 1 "register_operand" "w")
3013            (match_operand:VSDQ_I 2 "register_operand" "w")]
3014          VQSHL))]
3015   "TARGET_SIMD"
3016   "<sur>q<r>shl\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>";
3017   [(set_attr "simd_type" "simd_sat_shift")
3018    (set_attr "simd_mode" "<MODE>")]
3019 )
3020
3021 ;; vshll_n
3022
3023 (define_insn "aarch64_<sur>shll_n<mode>"
3024   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3025         (unspec:<VWIDE> [(match_operand:VDW 1 "register_operand" "w")
3026                          (match_operand:SI 2 "immediate_operand" "i")]
3027                          VSHLL))]
3028   "TARGET_SIMD"
3029   "*
3030   int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
3031   aarch64_simd_const_bounds (operands[2], 0, bit_width + 1);
3032   if (INTVAL (operands[2]) == bit_width)
3033   {
3034     return \"shll\\t%0.<Vwtype>, %1.<Vtype>, %2\";
3035   }
3036   else {
3037     return \"<sur>shll\\t%0.<Vwtype>, %1.<Vtype>, %2\";
3038   }"
3039   [(set_attr "simd_type" "simd_shift_imm")
3040    (set_attr "simd_mode" "<MODE>")]
3041 )
3042
3043 ;; vshll_high_n
3044
3045 (define_insn "aarch64_<sur>shll2_n<mode>"
3046   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3047         (unspec:<VWIDE> [(match_operand:VQW 1 "register_operand" "w")
3048                          (match_operand:SI 2 "immediate_operand" "i")]
3049                          VSHLL))]
3050   "TARGET_SIMD"
3051   "*
3052   int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
3053   aarch64_simd_const_bounds (operands[2], 0, bit_width + 1);
3054   if (INTVAL (operands[2]) == bit_width)
3055   {
3056     return \"shll2\\t%0.<Vwtype>, %1.<Vtype>, %2\";
3057   }
3058   else {
3059     return \"<sur>shll2\\t%0.<Vwtype>, %1.<Vtype>, %2\";
3060   }"
3061   [(set_attr "simd_type" "simd_shift_imm")
3062    (set_attr "simd_mode" "<MODE>")]
3063 )
3064
3065 ;; vrshr_n
3066
3067 (define_insn "aarch64_<sur>shr_n<mode>"
3068   [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
3069         (unspec:VSDQ_I_DI [(match_operand:VSDQ_I_DI 1 "register_operand" "w")
3070                            (match_operand:SI 2 "immediate_operand" "i")]
3071                           VRSHR_N))]
3072   "TARGET_SIMD"
3073   "*
3074   int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
3075   aarch64_simd_const_bounds (operands[2], 1, bit_width + 1);
3076   return \"<sur>shr\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %2\";"
3077   [(set_attr "simd_type" "simd_shift_imm")
3078    (set_attr "simd_mode" "<MODE>")]
3079 )
3080
3081 ;; v(r)sra_n
3082
3083 (define_insn "aarch64_<sur>sra_n<mode>"
3084   [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
3085         (unspec:VSDQ_I_DI [(match_operand:VSDQ_I_DI 1 "register_operand" "0")
3086                        (match_operand:VSDQ_I_DI 2 "register_operand" "w")
3087                        (match_operand:SI 3 "immediate_operand" "i")]
3088                       VSRA))]
3089   "TARGET_SIMD"
3090   "*
3091   int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
3092   aarch64_simd_const_bounds (operands[3], 1, bit_width + 1);
3093   return \"<sur>sra\\t%<v>0<Vmtype>, %<v>2<Vmtype>, %3\";"
3094   [(set_attr "simd_type" "simd_shift_imm_acc")
3095    (set_attr "simd_mode" "<MODE>")]
3096 )
3097
3098 ;; vs<lr>i_n
3099
3100 (define_insn "aarch64_<sur>s<lr>i_n<mode>"
3101   [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
3102         (unspec:VSDQ_I_DI [(match_operand:VSDQ_I_DI 1 "register_operand" "0")
3103                        (match_operand:VSDQ_I_DI 2 "register_operand" "w")
3104                        (match_operand:SI 3 "immediate_operand" "i")]
3105                       VSLRI))]
3106   "TARGET_SIMD"
3107   "*
3108   int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
3109   aarch64_simd_const_bounds (operands[3], 1 - <VSLRI:offsetlr>,
3110                              bit_width - <VSLRI:offsetlr> + 1);
3111   return \"s<lr>i\\t%<v>0<Vmtype>, %<v>2<Vmtype>, %3\";"
3112   [(set_attr "simd_type" "simd_shift_imm")
3113    (set_attr "simd_mode" "<MODE>")]
3114 )
3115
3116 ;; vqshl(u)
3117
3118 (define_insn "aarch64_<sur>qshl<u>_n<mode>"
3119   [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
3120         (unspec:VSDQ_I [(match_operand:VSDQ_I 1 "register_operand" "w")
3121                        (match_operand:SI 2 "immediate_operand" "i")]
3122                       VQSHL_N))]
3123   "TARGET_SIMD"
3124   "*
3125   int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
3126   aarch64_simd_const_bounds (operands[2], 0, bit_width);
3127   return \"<sur>qshl<u>\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %2\";"
3128   [(set_attr "simd_type" "simd_sat_shift_imm")
3129    (set_attr "simd_mode" "<MODE>")]
3130 )
3131
3132
3133 ;; vq(r)shr(u)n_n
3134
3135 (define_insn "aarch64_<sur>q<r>shr<u>n_n<mode>"
3136   [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
3137         (unspec:<VNARROWQ> [(match_operand:VSQN_HSDI 1 "register_operand" "w")
3138                             (match_operand:SI 2 "immediate_operand" "i")]
3139                            VQSHRN_N))]
3140   "TARGET_SIMD"
3141   "*
3142   int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
3143   aarch64_simd_const_bounds (operands[2], 1, bit_width + 1);
3144   return \"<sur>q<r>shr<u>n\\t%<vn2>0<Vmntype>, %<v>1<Vmtype>, %2\";"
3145   [(set_attr "simd_type" "simd_sat_shiftn_imm")
3146    (set_attr "simd_mode" "<MODE>")]
3147 )
3148
3149
3150 ;; cm(eq|ge|le|lt|gt)
3151
3152 (define_insn "aarch64_cm<cmp><mode>"
3153   [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w,w")
3154         (unspec:<V_cmp_result>
3155           [(match_operand:VSDQ_I_DI 1 "register_operand" "w,w")
3156            (match_operand:VSDQ_I_DI 2 "aarch64_simd_reg_or_zero" "w,Z")]
3157           VCMP_S))]
3158   "TARGET_SIMD"
3159   "@
3160   cm<cmp>\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>
3161   cm<cmp>\t%<v>0<Vmtype>, %<v>1<Vmtype>, #0"
3162   [(set_attr "simd_type" "simd_cmp")
3163    (set_attr "simd_mode" "<MODE>")]
3164 )
3165
3166 ;; cm(hs|hi|tst)
3167
3168 (define_insn "aarch64_cm<cmp><mode>"
3169   [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w")
3170         (unspec:<V_cmp_result>
3171           [(match_operand:VSDQ_I_DI 1 "register_operand" "w")
3172            (match_operand:VSDQ_I_DI 2 "register_operand" "w")]
3173           VCMP_U))]
3174   "TARGET_SIMD"
3175   "cm<cmp>\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
3176   [(set_attr "simd_type" "simd_cmp")
3177    (set_attr "simd_mode" "<MODE>")]
3178 )
3179
3180 ;; fcm(eq|ge|le|lt|gt)
3181
3182 (define_insn "aarch64_cm<cmp><mode>"
3183   [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w,w")
3184         (unspec:<V_cmp_result>
3185           [(match_operand:VDQF 1 "register_operand" "w,w")
3186            (match_operand:VDQF 2 "aarch64_simd_reg_or_zero" "w,Dz")]
3187            VCMP_S))]
3188   "TARGET_SIMD"
3189   "@
3190   fcm<cmp>\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>
3191   fcm<cmp>\t%<v>0<Vmtype>, %<v>1<Vmtype>, 0"
3192   [(set_attr "simd_type" "simd_fcmp")
3193    (set_attr "simd_mode" "<MODE>")]
3194 )
3195
3196 ;; addp
3197
3198 (define_insn "aarch64_addp<mode>"
3199   [(set (match_operand:VD_BHSI 0 "register_operand" "=w")
3200         (unspec:VD_BHSI
3201           [(match_operand:VD_BHSI 1 "register_operand" "w")
3202            (match_operand:VD_BHSI 2 "register_operand" "w")]
3203           UNSPEC_ADDP))]
3204   "TARGET_SIMD"
3205   "addp\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
3206   [(set_attr "simd_type" "simd_add")
3207    (set_attr "simd_mode" "<MODE>")]
3208 )
3209
3210 (define_insn "aarch64_addpdi"
3211   [(set (match_operand:DI 0 "register_operand" "=w")
3212         (unspec:DI
3213           [(match_operand:V2DI 1 "register_operand" "w")]
3214           UNSPEC_ADDP))]
3215   "TARGET_SIMD"
3216   "addp\t%d0, %1.2d"
3217   [(set_attr "simd_type" "simd_add")
3218    (set_attr "simd_mode" "DI")]
3219 )
3220
3221 (define_insn "aarch64_<fmaxmin><mode>"
3222   [(set (match_operand:VDQF 0 "register_operand" "=w")
3223         (unspec:VDQF [(match_operand:VDQF 1 "register_operand" "w")
3224                       (match_operand:VDQF 2 "register_operand" "w")]
3225                       FMAXMIN))]
3226   "TARGET_SIMD"
3227   "<fmaxmin>\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
3228   [(set_attr "simd_type" "simd_fminmax")
3229    (set_attr "simd_mode" "<MODE>")]
3230 )
3231
3232 ;; sqrt
3233
3234 (define_insn "sqrt<mode>2"
3235   [(set (match_operand:VDQF 0 "register_operand" "=w")
3236         (sqrt:VDQF (match_operand:VDQF 1 "register_operand" "w")))]
3237   "TARGET_SIMD"
3238   "fsqrt\\t%0.<Vtype>, %1.<Vtype>"
3239   [(set_attr "simd_type" "simd_fsqrt")
3240    (set_attr "simd_mode" "<MODE>")]
3241 )
3242
3243 ;; Patterns for vector struct loads and stores.
3244
3245 (define_insn "vec_load_lanesoi<mode>"
3246   [(set (match_operand:OI 0 "register_operand" "=w")
3247         (unspec:OI [(match_operand:OI 1 "aarch64_simd_struct_operand" "Utv")
3248                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3249                    UNSPEC_LD2))]
3250   "TARGET_SIMD"
3251   "ld2\\t{%S0.<Vtype> - %T0.<Vtype>}, %1"
3252   [(set_attr "simd_type" "simd_load2")
3253    (set_attr "simd_mode" "<MODE>")])
3254
3255 (define_insn "vec_store_lanesoi<mode>"
3256   [(set (match_operand:OI 0 "aarch64_simd_struct_operand" "=Utv")
3257         (unspec:OI [(match_operand:OI 1 "register_operand" "w")
3258                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3259                    UNSPEC_ST2))]
3260   "TARGET_SIMD"
3261   "st2\\t{%S1.<Vtype> - %T1.<Vtype>}, %0"
3262   [(set_attr "simd_type" "simd_store2")
3263    (set_attr "simd_mode" "<MODE>")])
3264
3265 (define_insn "vec_load_lanesci<mode>"
3266   [(set (match_operand:CI 0 "register_operand" "=w")
3267         (unspec:CI [(match_operand:CI 1 "aarch64_simd_struct_operand" "Utv")
3268                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3269                    UNSPEC_LD3))]
3270   "TARGET_SIMD"
3271   "ld3\\t{%S0.<Vtype> - %U0.<Vtype>}, %1"
3272   [(set_attr "simd_type" "simd_load3")
3273    (set_attr "simd_mode" "<MODE>")])
3274
3275 (define_insn "vec_store_lanesci<mode>"
3276   [(set (match_operand:CI 0 "aarch64_simd_struct_operand" "=Utv")
3277         (unspec:CI [(match_operand:CI 1 "register_operand" "w")
3278                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3279                    UNSPEC_ST3))]
3280   "TARGET_SIMD"
3281   "st3\\t{%S1.<Vtype> - %U1.<Vtype>}, %0"
3282   [(set_attr "simd_type" "simd_store3")
3283    (set_attr "simd_mode" "<MODE>")])
3284
3285 (define_insn "vec_load_lanesxi<mode>"
3286   [(set (match_operand:XI 0 "register_operand" "=w")
3287         (unspec:XI [(match_operand:XI 1 "aarch64_simd_struct_operand" "Utv")
3288                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3289                    UNSPEC_LD4))]
3290   "TARGET_SIMD"
3291   "ld4\\t{%S0.<Vtype> - %V0.<Vtype>}, %1"
3292   [(set_attr "simd_type" "simd_load4")
3293    (set_attr "simd_mode" "<MODE>")])
3294
3295 (define_insn "vec_store_lanesxi<mode>"
3296   [(set (match_operand:XI 0 "aarch64_simd_struct_operand" "=Utv")
3297         (unspec:XI [(match_operand:XI 1 "register_operand" "w")
3298                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3299                    UNSPEC_ST4))]
3300   "TARGET_SIMD"
3301   "st4\\t{%S1.<Vtype> - %V1.<Vtype>}, %0"
3302   [(set_attr "simd_type" "simd_store4")
3303    (set_attr "simd_mode" "<MODE>")])
3304
3305 ;; Reload patterns for AdvSIMD register list operands.
3306
3307 (define_expand "mov<mode>"
3308   [(set (match_operand:VSTRUCT 0 "aarch64_simd_nonimmediate_operand" "")
3309         (match_operand:VSTRUCT 1 "aarch64_simd_general_operand" ""))]
3310   "TARGET_SIMD"
3311 {
3312   if (can_create_pseudo_p ())
3313     {
3314       if (GET_CODE (operands[0]) != REG)
3315         operands[1] = force_reg (<MODE>mode, operands[1]);
3316     }
3317 })
3318
3319 (define_insn "*aarch64_mov<mode>"
3320   [(set (match_operand:VSTRUCT 0 "aarch64_simd_nonimmediate_operand" "=w,Utv,w")
3321         (match_operand:VSTRUCT 1 "aarch64_simd_general_operand" " w,w,Utv"))]
3322   "TARGET_SIMD
3323    && (register_operand (operands[0], <MODE>mode)
3324        || register_operand (operands[1], <MODE>mode))"
3325
3326 {
3327   switch (which_alternative)
3328     {
3329     case 0: return "#";
3330     case 1: return "st1\\t{%S1.16b - %<Vendreg>1.16b}, %0";
3331     case 2: return "ld1\\t{%S0.16b - %<Vendreg>0.16b}, %1";
3332     default: gcc_unreachable ();
3333     }
3334 }
3335   [(set_attr "simd_type" "simd_move,simd_store<nregs>,simd_load<nregs>")
3336    (set (attr "length") (symbol_ref "aarch64_simd_attr_length_move (insn)"))
3337    (set_attr "simd_mode" "<MODE>")])
3338
3339 (define_split
3340   [(set (match_operand:OI 0 "register_operand" "")
3341         (match_operand:OI 1 "register_operand" ""))]
3342   "TARGET_SIMD && reload_completed"
3343   [(set (match_dup 0) (match_dup 1))
3344    (set (match_dup 2) (match_dup 3))]
3345 {
3346   int rdest = REGNO (operands[0]);
3347   int rsrc = REGNO (operands[1]);
3348   rtx dest[2], src[2];
3349
3350   dest[0] = gen_rtx_REG (TFmode, rdest);
3351   src[0] = gen_rtx_REG (TFmode, rsrc);
3352   dest[1] = gen_rtx_REG (TFmode, rdest + 1);
3353   src[1] = gen_rtx_REG (TFmode, rsrc + 1);
3354
3355   aarch64_simd_disambiguate_copy (operands, dest, src, 2);
3356 })
3357
3358 (define_split
3359   [(set (match_operand:CI 0 "register_operand" "")
3360         (match_operand:CI 1 "register_operand" ""))]
3361   "TARGET_SIMD && reload_completed"
3362   [(set (match_dup 0) (match_dup 1))
3363    (set (match_dup 2) (match_dup 3))
3364    (set (match_dup 4) (match_dup 5))]
3365 {
3366   int rdest = REGNO (operands[0]);
3367   int rsrc = REGNO (operands[1]);
3368   rtx dest[3], src[3];
3369
3370   dest[0] = gen_rtx_REG (TFmode, rdest);
3371   src[0] = gen_rtx_REG (TFmode, rsrc);
3372   dest[1] = gen_rtx_REG (TFmode, rdest + 1);
3373   src[1] = gen_rtx_REG (TFmode, rsrc + 1);
3374   dest[2] = gen_rtx_REG (TFmode, rdest + 2);
3375   src[2] = gen_rtx_REG (TFmode, rsrc + 2);
3376
3377   aarch64_simd_disambiguate_copy (operands, dest, src, 3);
3378 })
3379
3380 (define_split
3381   [(set (match_operand:XI 0 "register_operand" "")
3382         (match_operand:XI 1 "register_operand" ""))]
3383   "TARGET_SIMD && reload_completed"
3384   [(set (match_dup 0) (match_dup 1))
3385    (set (match_dup 2) (match_dup 3))
3386    (set (match_dup 4) (match_dup 5))
3387    (set (match_dup 6) (match_dup 7))]
3388 {
3389   int rdest = REGNO (operands[0]);
3390   int rsrc = REGNO (operands[1]);
3391   rtx dest[4], src[4];
3392
3393   dest[0] = gen_rtx_REG (TFmode, rdest);
3394   src[0] = gen_rtx_REG (TFmode, rsrc);
3395   dest[1] = gen_rtx_REG (TFmode, rdest + 1);
3396   src[1] = gen_rtx_REG (TFmode, rsrc + 1);
3397   dest[2] = gen_rtx_REG (TFmode, rdest + 2);
3398   src[2] = gen_rtx_REG (TFmode, rsrc + 2);
3399   dest[3] = gen_rtx_REG (TFmode, rdest + 3);
3400   src[3] = gen_rtx_REG (TFmode, rsrc + 3);
3401
3402   aarch64_simd_disambiguate_copy (operands, dest, src, 4);
3403 })
3404
3405 (define_insn "aarch64_ld2<mode>_dreg"
3406   [(set (match_operand:OI 0 "register_operand" "=w")
3407         (subreg:OI
3408           (vec_concat:<VRL2>
3409             (vec_concat:<VDBL>
3410              (unspec:VD [(match_operand:TI 1 "aarch64_simd_struct_operand" "Utv")]
3411                         UNSPEC_LD2)
3412              (vec_duplicate:VD (const_int 0)))
3413             (vec_concat:<VDBL>
3414              (unspec:VD [(match_dup 1)]
3415                         UNSPEC_LD2)
3416              (vec_duplicate:VD (const_int 0)))) 0))]
3417   "TARGET_SIMD"
3418   "ld2\\t{%S0.<Vtype> - %T0.<Vtype>}, %1"
3419   [(set_attr "simd_type" "simd_load2")
3420    (set_attr "simd_mode" "<MODE>")])
3421
3422 (define_insn "aarch64_ld2<mode>_dreg"
3423   [(set (match_operand:OI 0 "register_operand" "=w")
3424         (subreg:OI
3425           (vec_concat:<VRL2>
3426             (vec_concat:<VDBL>
3427              (unspec:DX [(match_operand:TI 1 "aarch64_simd_struct_operand" "Utv")]
3428                         UNSPEC_LD2)
3429              (const_int 0))
3430             (vec_concat:<VDBL>
3431              (unspec:DX [(match_dup 1)]
3432                         UNSPEC_LD2)
3433              (const_int 0))) 0))]
3434   "TARGET_SIMD"
3435   "ld1\\t{%S0.1d - %T0.1d}, %1"
3436   [(set_attr "simd_type" "simd_load2")
3437    (set_attr "simd_mode" "<MODE>")])
3438
3439 (define_insn "aarch64_ld3<mode>_dreg"
3440   [(set (match_operand:CI 0 "register_operand" "=w")
3441         (subreg:CI
3442          (vec_concat:<VRL3>
3443           (vec_concat:<VRL2>
3444             (vec_concat:<VDBL>
3445              (unspec:VD [(match_operand:EI 1 "aarch64_simd_struct_operand" "Utv")]
3446                         UNSPEC_LD3)
3447              (vec_duplicate:VD (const_int 0)))
3448             (vec_concat:<VDBL>
3449              (unspec:VD [(match_dup 1)]
3450                         UNSPEC_LD3)
3451              (vec_duplicate:VD (const_int 0))))
3452           (vec_concat:<VDBL>
3453              (unspec:VD [(match_dup 1)]
3454                         UNSPEC_LD3)
3455              (vec_duplicate:VD (const_int 0)))) 0))]
3456   "TARGET_SIMD"
3457   "ld3\\t{%S0.<Vtype> - %U0.<Vtype>}, %1"
3458   [(set_attr "simd_type" "simd_load3")
3459    (set_attr "simd_mode" "<MODE>")])
3460
3461 (define_insn "aarch64_ld3<mode>_dreg"
3462   [(set (match_operand:CI 0 "register_operand" "=w")
3463         (subreg:CI
3464          (vec_concat:<VRL3>
3465           (vec_concat:<VRL2>
3466             (vec_concat:<VDBL>
3467              (unspec:DX [(match_operand:EI 1 "aarch64_simd_struct_operand" "Utv")]
3468                         UNSPEC_LD3)
3469              (const_int 0))
3470             (vec_concat:<VDBL>
3471              (unspec:DX [(match_dup 1)]
3472                         UNSPEC_LD3)
3473              (const_int 0)))
3474           (vec_concat:<VDBL>
3475              (unspec:DX [(match_dup 1)]
3476                         UNSPEC_LD3)
3477              (const_int 0))) 0))]
3478   "TARGET_SIMD"
3479   "ld1\\t{%S0.1d - %U0.1d}, %1"
3480   [(set_attr "simd_type" "simd_load3")
3481    (set_attr "simd_mode" "<MODE>")])
3482
3483 (define_insn "aarch64_ld4<mode>_dreg"
3484   [(set (match_operand:XI 0 "register_operand" "=w")
3485         (subreg:XI
3486          (vec_concat:<VRL4>
3487            (vec_concat:<VRL2>
3488              (vec_concat:<VDBL>
3489                (unspec:VD [(match_operand:OI 1 "aarch64_simd_struct_operand" "Utv")]
3490                           UNSPEC_LD4)
3491                (vec_duplicate:VD (const_int 0)))
3492               (vec_concat:<VDBL>
3493                 (unspec:VD [(match_dup 1)]
3494                         UNSPEC_LD4)
3495                 (vec_duplicate:VD (const_int 0))))
3496            (vec_concat:<VRL2>
3497              (vec_concat:<VDBL>
3498                (unspec:VD [(match_dup 1)]
3499                         UNSPEC_LD4)
3500                (vec_duplicate:VD (const_int 0)))
3501              (vec_concat:<VDBL>
3502                (unspec:VD [(match_dup 1)]
3503                         UNSPEC_LD4)
3504                (vec_duplicate:VD (const_int 0))))) 0))]
3505   "TARGET_SIMD"
3506   "ld4\\t{%S0.<Vtype> - %V0.<Vtype>}, %1"
3507   [(set_attr "simd_type" "simd_load4")
3508    (set_attr "simd_mode" "<MODE>")])
3509
3510 (define_insn "aarch64_ld4<mode>_dreg"
3511   [(set (match_operand:XI 0 "register_operand" "=w")
3512         (subreg:XI
3513          (vec_concat:<VRL4>
3514            (vec_concat:<VRL2>
3515              (vec_concat:<VDBL>
3516                (unspec:DX [(match_operand:OI 1 "aarch64_simd_struct_operand" "Utv")]
3517                           UNSPEC_LD4)
3518                (const_int 0))
3519               (vec_concat:<VDBL>
3520                 (unspec:DX [(match_dup 1)]
3521                         UNSPEC_LD4)
3522                 (const_int 0)))
3523            (vec_concat:<VRL2>
3524              (vec_concat:<VDBL>
3525                (unspec:DX [(match_dup 1)]
3526                         UNSPEC_LD4)
3527                (const_int 0))
3528              (vec_concat:<VDBL>
3529                (unspec:DX [(match_dup 1)]
3530                         UNSPEC_LD4)
3531                (const_int 0)))) 0))]
3532   "TARGET_SIMD"
3533   "ld1\\t{%S0.1d - %V0.1d}, %1"
3534   [(set_attr "simd_type" "simd_load4")
3535    (set_attr "simd_mode" "<MODE>")])
3536
3537 (define_expand "aarch64_ld<VSTRUCT:nregs><VDC:mode>"
3538  [(match_operand:VSTRUCT 0 "register_operand" "=w")
3539   (match_operand:DI 1 "register_operand" "r")
3540   (unspec:VDC [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3541   "TARGET_SIMD"
3542 {
3543   enum machine_mode mode = <VSTRUCT:VSTRUCT_DREG>mode;
3544   rtx mem = gen_rtx_MEM (mode, operands[1]);
3545
3546   emit_insn (gen_aarch64_ld<VSTRUCT:nregs><VDC:mode>_dreg (operands[0], mem));
3547   DONE;
3548 })
3549
3550 (define_expand "aarch64_ld<VSTRUCT:nregs><VQ:mode>"
3551  [(match_operand:VSTRUCT 0 "register_operand" "=w")
3552   (match_operand:DI 1 "register_operand" "r")
3553   (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3554   "TARGET_SIMD"
3555 {
3556   enum machine_mode mode = <VSTRUCT:MODE>mode;
3557   rtx mem = gen_rtx_MEM (mode, operands[1]);
3558
3559   emit_insn (gen_vec_load_lanes<VSTRUCT:mode><VQ:mode> (operands[0], mem));
3560   DONE;
3561 })
3562
3563 ;; Expanders for builtins to extract vector registers from large
3564 ;; opaque integer modes.
3565
3566 ;; D-register list.
3567
3568 (define_expand "aarch64_get_dreg<VSTRUCT:mode><VDC:mode>"
3569  [(match_operand:VDC 0 "register_operand" "=w")
3570   (match_operand:VSTRUCT 1 "register_operand" "w")
3571   (match_operand:SI 2 "immediate_operand" "i")]
3572   "TARGET_SIMD"
3573 {
3574   int part = INTVAL (operands[2]);
3575   rtx temp = gen_reg_rtx (<VDC:VDBL>mode);
3576   int offset = part * 16;
3577
3578   emit_move_insn (temp, gen_rtx_SUBREG (<VDC:VDBL>mode, operands[1], offset));
3579   emit_move_insn (operands[0], gen_lowpart (<VDC:MODE>mode, temp));
3580   DONE;
3581 })
3582
3583 ;; Q-register list.
3584
3585 (define_expand "aarch64_get_qreg<VSTRUCT:mode><VQ:mode>"
3586  [(match_operand:VQ 0 "register_operand" "=w")
3587   (match_operand:VSTRUCT 1 "register_operand" "w")
3588   (match_operand:SI 2 "immediate_operand" "i")]
3589   "TARGET_SIMD"
3590 {
3591   int part = INTVAL (operands[2]);
3592   int offset = part * 16;
3593
3594   emit_move_insn (operands[0],
3595                   gen_rtx_SUBREG (<VQ:MODE>mode, operands[1], offset));
3596   DONE;
3597 })
3598
3599 ;; Permuted-store expanders for neon intrinsics.
3600
3601 ;; Permute instructions
3602
3603 ;; vec_perm support
3604
3605 (define_expand "vec_perm_const<mode>"
3606   [(match_operand:VALL 0 "register_operand")
3607    (match_operand:VALL 1 "register_operand")
3608    (match_operand:VALL 2 "register_operand")
3609    (match_operand:<V_cmp_result> 3)]
3610   "TARGET_SIMD"
3611 {
3612   if (aarch64_expand_vec_perm_const (operands[0], operands[1],
3613                                      operands[2], operands[3]))
3614     DONE;
3615   else
3616     FAIL;
3617 })
3618
3619 (define_expand "vec_perm<mode>"
3620   [(match_operand:VB 0 "register_operand")
3621    (match_operand:VB 1 "register_operand")
3622    (match_operand:VB 2 "register_operand")
3623    (match_operand:VB 3 "register_operand")]
3624   "TARGET_SIMD"
3625 {
3626   aarch64_expand_vec_perm (operands[0], operands[1],
3627                            operands[2], operands[3]);
3628   DONE;
3629 })
3630
3631 (define_insn "aarch64_tbl1<mode>"
3632   [(set (match_operand:VB 0 "register_operand" "=w")
3633         (unspec:VB [(match_operand:V16QI 1 "register_operand" "w")
3634                     (match_operand:VB 2 "register_operand" "w")]
3635                    UNSPEC_TBL))]
3636   "TARGET_SIMD"
3637   "tbl\\t%0.<Vtype>, {%1.16b}, %2.<Vtype>"
3638   [(set_attr "simd_type" "simd_tbl")
3639    (set_attr "simd_mode" "<MODE>")]
3640 )
3641
3642 ;; Two source registers.
3643
3644 (define_insn "aarch64_tbl2v16qi"
3645   [(set (match_operand:V16QI 0 "register_operand" "=w")
3646         (unspec:V16QI [(match_operand:OI 1 "register_operand" "w")
3647                        (match_operand:V16QI 2 "register_operand" "w")]
3648                       UNSPEC_TBL))]
3649   "TARGET_SIMD"
3650   "tbl\\t%0.16b, {%S1.16b - %T1.16b}, %2.16b"
3651   [(set_attr "simd_type" "simd_tbl")
3652    (set_attr "simd_mode" "V16QI")]
3653 )
3654
3655 (define_insn_and_split "aarch64_combinev16qi"
3656   [(set (match_operand:OI 0 "register_operand" "=w")
3657         (unspec:OI [(match_operand:V16QI 1 "register_operand" "w")
3658                     (match_operand:V16QI 2 "register_operand" "w")]
3659                    UNSPEC_CONCAT))]
3660   "TARGET_SIMD"
3661   "#"
3662   "&& reload_completed"
3663   [(const_int 0)]
3664 {
3665   aarch64_split_combinev16qi (operands);
3666   DONE;
3667 })
3668
3669 (define_insn "aarch64_<PERMUTE:perm_insn><PERMUTE:perm_hilo><mode>"
3670   [(set (match_operand:VALL 0 "register_operand" "=w")
3671         (unspec:VALL [(match_operand:VALL 1 "register_operand" "w")
3672                       (match_operand:VALL 2 "register_operand" "w")]
3673                        PERMUTE))]
3674   "TARGET_SIMD"
3675   "<PERMUTE:perm_insn><PERMUTE:perm_hilo>\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
3676   [(set_attr "simd_type" "simd_<PERMUTE:perm_insn>")
3677    (set_attr "simd_mode" "<MODE>")]
3678 )
3679
3680 (define_insn "aarch64_st2<mode>_dreg"
3681   [(set (match_operand:TI 0 "aarch64_simd_struct_operand" "=Utv")
3682         (unspec:TI [(match_operand:OI 1 "register_operand" "w")
3683                     (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3684                    UNSPEC_ST2))]
3685   "TARGET_SIMD"
3686   "st2\\t{%S1.<Vtype> - %T1.<Vtype>}, %0"
3687   [(set_attr "simd_type" "simd_store2")
3688    (set_attr "simd_mode" "<MODE>")])
3689
3690 (define_insn "aarch64_st2<mode>_dreg"
3691   [(set (match_operand:TI 0 "aarch64_simd_struct_operand" "=Utv")
3692         (unspec:TI [(match_operand:OI 1 "register_operand" "w")
3693                     (unspec:DX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3694                    UNSPEC_ST2))]
3695   "TARGET_SIMD"
3696   "st1\\t{%S1.1d - %T1.1d}, %0"
3697   [(set_attr "simd_type" "simd_store2")
3698    (set_attr "simd_mode" "<MODE>")])
3699
3700 (define_insn "aarch64_st3<mode>_dreg"
3701   [(set (match_operand:EI 0 "aarch64_simd_struct_operand" "=Utv")
3702         (unspec:EI [(match_operand:CI 1 "register_operand" "w")
3703                     (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3704                    UNSPEC_ST3))]
3705   "TARGET_SIMD"
3706   "st3\\t{%S1.<Vtype> - %U1.<Vtype>}, %0"
3707   [(set_attr "simd_type" "simd_store3")
3708    (set_attr "simd_mode" "<MODE>")])
3709
3710 (define_insn "aarch64_st3<mode>_dreg"
3711   [(set (match_operand:EI 0 "aarch64_simd_struct_operand" "=Utv")
3712         (unspec:EI [(match_operand:CI 1 "register_operand" "w")
3713                     (unspec:DX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3714                    UNSPEC_ST3))]
3715   "TARGET_SIMD"
3716   "st1\\t{%S1.1d - %U1.1d}, %0"
3717   [(set_attr "simd_type" "simd_store3")
3718    (set_attr "simd_mode" "<MODE>")])
3719
3720 (define_insn "aarch64_st4<mode>_dreg"
3721   [(set (match_operand:OI 0 "aarch64_simd_struct_operand" "=Utv")
3722         (unspec:OI [(match_operand:XI 1 "register_operand" "w")
3723                     (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3724                    UNSPEC_ST4))]
3725   "TARGET_SIMD"
3726   "st4\\t{%S1.<Vtype> - %V1.<Vtype>}, %0"
3727   [(set_attr "simd_type" "simd_store4")
3728    (set_attr "simd_mode" "<MODE>")])
3729
3730 (define_insn "aarch64_st4<mode>_dreg"
3731   [(set (match_operand:OI 0 "aarch64_simd_struct_operand" "=Utv")
3732         (unspec:OI [(match_operand:XI 1 "register_operand" "w")
3733                     (unspec:DX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3734                    UNSPEC_ST4))]
3735   "TARGET_SIMD"
3736   "st1\\t{%S1.1d - %V1.1d}, %0"
3737   [(set_attr "simd_type" "simd_store4")
3738    (set_attr "simd_mode" "<MODE>")])
3739
3740 (define_expand "aarch64_st<VSTRUCT:nregs><VDC:mode>"
3741  [(match_operand:DI 0 "register_operand" "r")
3742   (match_operand:VSTRUCT 1 "register_operand" "w")
3743   (unspec:VDC [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3744   "TARGET_SIMD"
3745 {
3746   enum machine_mode mode = <VSTRUCT:VSTRUCT_DREG>mode;
3747   rtx mem = gen_rtx_MEM (mode, operands[0]);
3748
3749   emit_insn (gen_aarch64_st<VSTRUCT:nregs><VDC:mode>_dreg (mem, operands[1]));
3750   DONE;
3751 })
3752
3753 (define_expand "aarch64_st<VSTRUCT:nregs><VQ:mode>"
3754  [(match_operand:DI 0 "register_operand" "r")
3755   (match_operand:VSTRUCT 1 "register_operand" "w")
3756   (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3757   "TARGET_SIMD"
3758 {
3759   enum machine_mode mode = <VSTRUCT:MODE>mode;
3760   rtx mem = gen_rtx_MEM (mode, operands[0]);
3761
3762   emit_insn (gen_vec_store_lanes<VSTRUCT:mode><VQ:mode> (mem, operands[1]));
3763   DONE;
3764 })
3765
3766 ;; Expander for builtins to insert vector registers into large
3767 ;; opaque integer modes.
3768
3769 ;; Q-register list.  We don't need a D-reg inserter as we zero
3770 ;; extend them in arm_neon.h and insert the resulting Q-regs.
3771
3772 (define_expand "aarch64_set_qreg<VSTRUCT:mode><VQ:mode>"
3773  [(match_operand:VSTRUCT 0 "register_operand" "+w")
3774   (match_operand:VSTRUCT 1 "register_operand" "0")
3775   (match_operand:VQ 2 "register_operand" "w")
3776   (match_operand:SI 3 "immediate_operand" "i")]
3777   "TARGET_SIMD"
3778 {
3779   int part = INTVAL (operands[3]);
3780   int offset = part * 16;
3781
3782   emit_move_insn (operands[0], operands[1]);
3783   emit_move_insn (gen_rtx_SUBREG (<VQ:MODE>mode, operands[0], offset),
3784                   operands[2]);
3785   DONE;
3786 })
3787
3788 ;; Standard pattern name vec_init<mode>.
3789
3790 (define_expand "vec_init<mode>"
3791   [(match_operand:VALL 0 "register_operand" "")
3792    (match_operand 1 "" "")]
3793   "TARGET_SIMD"
3794 {
3795   aarch64_expand_vector_init (operands[0], operands[1]);
3796   DONE;
3797 })
3798
3799 (define_insn "*aarch64_simd_ld1r<mode>"
3800   [(set (match_operand:VALLDI 0 "register_operand" "=w")
3801         (vec_duplicate:VALLDI
3802           (match_operand:<VEL> 1 "aarch64_simd_struct_operand" "Utv")))]
3803   "TARGET_SIMD"
3804   "ld1r\\t{%0.<Vtype>}, %1"
3805   [(set_attr "simd_type" "simd_load1r")
3806    (set_attr "simd_mode" "<MODE>")])
3807
3808 (define_insn "aarch64_frecpe<mode>"
3809   [(set (match_operand:VDQF 0 "register_operand" "=w")
3810         (unspec:VDQF [(match_operand:VDQF 1 "register_operand" "w")]
3811                     UNSPEC_FRECPE))]
3812   "TARGET_SIMD"
3813   "frecpe\\t%0.<Vtype>, %1.<Vtype>"
3814   [(set_attr "simd_type" "simd_frecpe")
3815    (set_attr "simd_mode" "<MODE>")]
3816 )
3817
3818 (define_insn "aarch64_frecps<mode>"
3819   [(set (match_operand:VDQF 0 "register_operand" "=w")
3820         (unspec:VDQF [(match_operand:VDQF 1 "register_operand" "w")
3821                      (match_operand:VDQF 2 "register_operand" "w")]
3822                     UNSPEC_FRECPS))]
3823   "TARGET_SIMD"
3824   "frecps\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
3825   [(set_attr "simd_type" "simd_frecps")
3826    (set_attr "simd_mode" "<MODE>")]
3827 )
3828