gcc/config/aarch64/aarch64-simd.md

   1 ;; Machine description for AArch64 AdvSIMD architecture.
   2 ;; Copyright (C) 2011-2013 Free Software Foundation, Inc.
   3 ;; Contributed by ARM Ltd.
   4 ;;
   5 ;; This file is part of GCC.
   6 ;;
   7 ;; GCC is free software; you can redistribute it and/or modify it
   8 ;; under the terms of the GNU General Public License as published by
   9 ;; the Free Software Foundation; either version 3, or (at your option)
  10 ;; any later version.
  11 ;;
  12 ;; GCC is distributed in the hope that it will be useful, but
  13 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
  14 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15 ;; General Public License for more details.
  16 ;;
  17 ;; You should have received a copy of the GNU General Public License
  18 ;; along with GCC; see the file COPYING3.  If not see
  19 ;; <http://www.gnu.org/licenses/>.
  20
  21
  22 ; Main data types used by the insntructions
  23
  24 (define_attr "simd_mode" "unknown,none,V8QI,V16QI,V4HI,V8HI,V2SI,V4SI,V2DI,V2SF,V4SF,V2DF,OI,CI,XI,DI,DF,SI,SF,HI,QI"
  25   (const_string "unknown"))
  26
  27
  28 ; Classification of AdvSIMD instructions for scheduling purposes.
  29 ; Do not set this attribute and the "v8type" attribute together in
  30 ; any instruction pattern.
  31
  32 ; simd_abd              integer absolute difference and accumulate.
  33 ; simd_abdl             integer absolute difference and accumulate (long).
  34 ; simd_adal             integer add and accumulate (long).
  35 ; simd_add              integer addition/subtraction.
  36 ; simd_addl             integer addition/subtraction (long).
  37 ; simd_addlv            across lanes integer sum (long).
  38 ; simd_addn             integer addition/subtraction (narrow).
  39 ; simd_addn2            integer addition/subtraction (narrow, high).
  40 ; simd_addv             across lanes integer sum.
  41 ; simd_cls              count leading sign/zero bits.
  42 ; simd_cmp              compare / create mask.
  43 ; simd_cnt              population count.
  44 ; simd_dup              duplicate element.
  45 ; simd_dupgp            duplicate general purpose register.
  46 ; simd_ext              bitwise extract from pair.
  47 ; simd_fabd             floating absolute difference and accumulate.
  48 ; simd_fadd             floating point add/sub.
  49 ; simd_fcmp             floating point compare.
  50 ; simd_fcvti            floating point convert to integer.
  51 ; simd_fcvtl            floating-point convert upsize.
  52 ; simd_fcvtn            floating-point convert downsize (narrow).
  53 ; simd_fcvtn2           floating-point convert downsize (narrow, high).
  54 ; simd_fdiv             floating point division.
  55 ; simd_fminmax          floating point min/max.
  56 ; simd_fminmaxv         across lanes floating point min/max.
  57 ; simd_fmla             floating point multiply-add.
  58 ; simd_fmla_elt         floating point multiply-add (by element).
  59 ; simd_fmul             floating point multiply.
  60 ; simd_fmul_elt         floating point multiply (by element).
  61 ; simd_fnegabs          floating point neg/abs.
  62 ; simd_frecpe            floating point reciprocal estimate.
  63 ; simd_frecps            floating point reciprocal step.
  64 ; simd_frecpx            floating point reciprocal exponent.
  65 ; simd_frint            floating point round to integer.
  66 ; simd_fsqrt            floating point square root.
  67 ; simd_icvtf            integer convert to floating point.
  68 ; simd_ins              insert element.
  69 ; simd_insgp            insert general purpose register.
  70 ; simd_load1            load multiple structures to one register (LD1).
  71 ; simd_load1r           load single structure to all lanes of one register (LD1R).
  72 ; simd_load1s           load single structure to one lane of one register (LD1 [index]).
  73 ; simd_load2            load multiple structures to two registers (LD1, LD2).
  74 ; simd_load2r           load single structure to all lanes of two registers (LD1R, LD2R).
  75 ; simd_load2s           load single structure to one lane of two registers (LD2 [index]).
  76 ; simd_load3            load multiple structures to three registers (LD1, LD3).
  77 ; simd_load3r           load single structure to all lanes of three registers (LD3R).
  78 ; simd_load3s           load single structure to one lane of three registers (LD3 [index]).
  79 ; simd_load4            load multiple structures to four registers (LD1, LD2, LD4).
  80 ; simd_load4r           load single structure to all lanes of four registers (LD4R).
  81 ; simd_load4s           load single structure to one lane of four registers (LD4 [index]).
  82 ; simd_logic            logical operation.
  83 ; simd_logic_imm        logcial operation (immediate).
  84 ; simd_minmax           integer min/max.
  85 ; simd_minmaxv          across lanes integer min/max,
  86 ; simd_mla              integer multiply-accumulate.
  87 ; simd_mla_elt          integer multiply-accumulate (by element).
  88 ; simd_mlal             integer multiply-accumulate (long).
  89 ; simd_mlal_elt         integer multiply-accumulate (by element, long).
  90 ; simd_move             move register.
  91 ; simd_move_imm         move immediate.
  92 ; simd_movgp            move element to general purpose register.
  93 ; simd_mul              integer multiply.
  94 ; simd_mul_elt          integer multiply (by element).
  95 ; simd_mull             integer multiply (long).
  96 ; simd_mull_elt         integer multiply (by element, long).
  97 ; simd_negabs           integer negate/absolute.
  98 ; simd_rbit             bitwise reverse.
  99 ; simd_rcpe             integer reciprocal estimate.
 100 ; simd_rcps             integer reciprocal square root.
 101 ; simd_rev              element reverse.
 102 ; simd_sat_add          integer saturating addition/subtraction.
 103 ; simd_sat_mlal         integer saturating multiply-accumulate (long).
 104 ; simd_sat_mlal_elt     integer saturating multiply-accumulate (by element, long).
 105 ; simd_sat_mul          integer saturating multiply.
 106 ; simd_sat_mul_elt      integer saturating multiply (by element).
 107 ; simd_sat_mull         integer saturating multiply (long).
 108 ; simd_sat_mull_elt     integer saturating multiply (by element, long).
 109 ; simd_sat_negabs       integer saturating negate/absolute.
 110 ; simd_sat_shift        integer saturating shift.
 111 ; simd_sat_shift_imm    integer saturating shift (immediate).
 112 ; simd_sat_shiftn_imm   integer saturating shift (narrow, immediate).
 113 ; simd_sat_shiftn2_imm  integer saturating shift (narrow, high, immediate).
 114 ; simd_shift            shift register/vector.
 115 ; simd_shift_acc        shift accumulate.
 116 ; simd_shift_imm        shift immediate.
 117 ; simd_shift_imm_acc    shift immediate and accumualte.
 118 ; simd_shiftl           shift register/vector (long).
 119 ; simd_shiftl_imm       shift register/vector (long, immediate).
 120 ; simd_shiftn_imm       shift register/vector (narrow, immediate).
 121 ; simd_shiftn2_imm      shift register/vector (narrow, high, immediate).
 122 ; simd_store1           store multiple structures from one register (ST1).
 123 ; simd_store1s          store single structure from one lane of one register (ST1 [index]).
 124 ; simd_store2           store multiple structures from two registers (ST1, ST2).
 125 ; simd_store2s          store single structure from one lane of two registers (ST2 [index]).
 126 ; simd_store3           store multiple structures from three registers (ST1, ST3).
 127 ; simd_store3s          store single structure from one lane of three register (ST3 [index]).
 128 ; simd_store4           store multiple structures from four registers (ST1, ST2, ST4).
 129 ; simd_store4s          store single structure from one lane for four registers (ST4 [index]).
 130 ; simd_tbl              table lookup.
 131 ; simd_trn              transpose.
 132 ; simd_uzp              unzip.
 133 ; simd_zip              zip.
 134
 135 (define_attr "simd_type"
 136    "simd_abd,\
 137    simd_abdl,\
 138    simd_adal,\
 139    simd_add,\
 140    simd_addl,\
 141    simd_addlv,\
 142    simd_addn,\
 143    simd_addn2,\
 144    simd_addv,\
 145    simd_cls,\
 146    simd_cmp,\
 147    simd_cnt,\
 148    simd_dup,\
 149    simd_dupgp,\
 150    simd_ext,\
 151    simd_fabd,\
 152    simd_fadd,\
 153    simd_fcmp,\
 154    simd_fcvti,\
 155    simd_fcvtl,\
 156    simd_fcvtn,\
 157    simd_fcvtn2,\
 158    simd_fdiv,\
 159    simd_fminmax,\
 160    simd_fminmaxv,\
 161    simd_fmla,\
 162    simd_fmla_elt,\
 163    simd_fmul,\
 164    simd_fmul_elt,\
 165    simd_fnegabs,\
 166    simd_frecpe,\
 167    simd_frecps,\
 168    simd_frecpx,\
 169    simd_frint,\
 170    simd_fsqrt,\
 171    simd_icvtf,\
 172    simd_ins,\
 173    simd_insgp,\
 174    simd_load1,\
 175    simd_load1r,\
 176    simd_load1s,\
 177    simd_load2,\
 178    simd_load2r,\
 179    simd_load2s,\
 180    simd_load3,\
 181    simd_load3r,\
 182    simd_load3s,\
 183    simd_load4,\
 184    simd_load4r,\
 185    simd_load4s,\
 186    simd_logic,\
 187    simd_logic_imm,\
 188    simd_minmax,\
 189    simd_minmaxv,\
 190    simd_mla,\
 191    simd_mla_elt,\
 192    simd_mlal,\
 193    simd_mlal_elt,\
 194    simd_movgp,\
 195    simd_move,\
 196    simd_move_imm,\
 197    simd_mul,\
 198    simd_mul_elt,\
 199    simd_mull,\
 200    simd_mull_elt,\
 201    simd_negabs,\
 202    simd_rbit,\
 203    simd_rcpe,\
 204    simd_rcps,\
 205    simd_rev,\
 206    simd_sat_add,\
 207    simd_sat_mlal,\
 208    simd_sat_mlal_elt,\
 209    simd_sat_mul,\
 210    simd_sat_mul_elt,\
 211    simd_sat_mull,\
 212    simd_sat_mull_elt,\
 213    simd_sat_negabs,\
 214    simd_sat_shift,\
 215    simd_sat_shift_imm,\
 216    simd_sat_shiftn_imm,\
 217    simd_sat_shiftn2_imm,\
 218    simd_shift,\
 219    simd_shift_acc,\
 220    simd_shift_imm,\
 221    simd_shift_imm_acc,\
 222    simd_shiftl,\
 223    simd_shiftl_imm,\
 224    simd_shiftn_imm,\
 225    simd_shiftn2_imm,\
 226    simd_store1,\
 227    simd_store1s,\
 228    simd_store2,\
 229    simd_store2s,\
 230    simd_store3,\
 231    simd_store3s,\
 232    simd_store4,\
 233    simd_store4s,\
 234    simd_tbl,\
 235    simd_trn,\
 236    simd_uzp,\
 237    simd_zip,\
 238    none"
 239   (const_string "none"))
 240
 241
 242 ; The "neon_type" attribute is used by the AArch32 backend.  Below is a mapping
 243 ; from "simd_type" to "neon_type".
 244
 245 (define_attr "neon_type"
 246    "neon_int_1,neon_int_2,neon_int_3,neon_int_4,neon_int_5,neon_vqneg_vqabs,
 247    neon_vmov,neon_vaba,neon_vsma,neon_vaba_qqq,
 248    neon_mul_ddd_8_16_qdd_16_8_long_32_16_long,neon_mul_qqq_8_16_32_ddd_32,
 249    neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar,
 250    neon_mla_ddd_8_16_qdd_16_8_long_32_16_long,neon_mla_qqq_8_16,
 251    neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long,
 252    neon_mla_qqq_32_qqd_32_scalar,neon_mul_ddd_16_scalar_32_16_long_scalar,
 253    neon_mul_qqd_32_scalar,neon_mla_ddd_16_scalar_qdd_32_16_long_scalar,
 254    neon_shift_1,neon_shift_2,neon_shift_3,neon_vshl_ddd,
 255    neon_vqshl_vrshl_vqrshl_qqq,neon_vsra_vrsra,neon_fp_vadd_ddd_vabs_dd,
 256    neon_fp_vadd_qqq_vabs_qq,neon_fp_vsum,neon_fp_vmul_ddd,neon_fp_vmul_qqd,
 257    neon_fp_vmla_ddd,neon_fp_vmla_qqq,neon_fp_vmla_ddd_scalar,
 258    neon_fp_vmla_qqq_scalar,neon_fp_vrecps_vrsqrts_ddd,
 259    neon_fp_vrecps_vrsqrts_qqq,neon_bp_simple,neon_bp_2cycle,neon_bp_3cycle,
 260    neon_ldr,neon_str,neon_vld1_1_2_regs,neon_vld1_3_4_regs,
 261    neon_vld2_2_regs_vld1_vld2_all_lanes,neon_vld2_4_regs,neon_vld3_vld4,
 262    neon_vst1_1_2_regs_vst2_2_regs,neon_vst1_3_4_regs,
 263    neon_vst2_4_regs_vst3_vst4,neon_vst3_vst4,neon_vld1_vld2_lane,
 264    neon_vld3_vld4_lane,neon_vst1_vst2_lane,neon_vst3_vst4_lane,
 265    neon_vld3_vld4_all_lanes,neon_mcr,neon_mcr_2_mcrr,neon_mrc,neon_mrrc,
 266    neon_ldm_2,neon_stm_2,none,unknown"
 267   (cond [
 268           (eq_attr "simd_type" "simd_dup") (const_string "neon_bp_simple")
 269           (eq_attr "simd_type" "simd_movgp") (const_string "neon_bp_simple")
 270           (eq_attr "simd_type" "simd_add,simd_logic,simd_logic_imm") (const_string "neon_int_1")
 271           (eq_attr "simd_type" "simd_negabs,simd_addlv") (const_string "neon_int_3")
 272           (eq_attr "simd_type" "simd_addn,simd_addn2,simd_addl,simd_sat_add,simd_sat_negabs") (const_string "neon_int_4")
 273           (eq_attr "simd_type" "simd_move") (const_string "neon_vmov")
 274           (eq_attr "simd_type" "simd_ins") (const_string "neon_mcr")
 275           (and (eq_attr "simd_type" "simd_mul,simd_sat_mul") (eq_attr "simd_mode" "V8QI,V4HI")) (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long")
 276           (and (eq_attr "simd_type" "simd_mul,simd_sat_mul") (eq_attr "simd_mode" "V2SI,V8QI,V16QI,V2SI")) (const_string "neon_mul_qqq_8_16_32_ddd_32")
 277           (and (eq_attr "simd_type" "simd_mull,simd_sat_mull") (eq_attr "simd_mode" "V8QI,V16QI,V4HI,V8HI")) (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long")
 278           (and (eq_attr "simd_type" "simd_mull,simd_sat_mull") (eq_attr "simd_mode" "V2SI,V4SI,V2DI")) (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")
 279           (and (eq_attr "simd_type" "simd_mla,simd_sat_mlal") (eq_attr "simd_mode" "V8QI,V4HI")) (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
 280           (and (eq_attr "simd_type" "simd_mla,simd_sat_mlal") (eq_attr "simd_mode" "V2SI")) (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")
 281           (and (eq_attr "simd_type" "simd_mla,simd_sat_mlal") (eq_attr "simd_mode" "V16QI,V8HI")) (const_string "neon_mla_qqq_8_16")
 282           (and (eq_attr "simd_type" "simd_mla,simd_sat_mlal") (eq_attr "simd_mode" "V4SI")) (const_string "neon_mla_qqq_32_qqd_32_scalar")
 283           (and (eq_attr "simd_type" "simd_mlal") (eq_attr "simd_mode" "V8QI,V16QI,V4HI,V8HI")) (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
 284           (and (eq_attr "simd_type" "simd_mlal") (eq_attr "simd_mode" "V2SI,V4SI,V2DI")) (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")
 285           (and (eq_attr "simd_type" "simd_fmla") (eq_attr "simd_mode" "V2SF")) (const_string "neon_fp_vmla_ddd")
 286           (and (eq_attr "simd_type" "simd_fmla") (eq_attr "simd_mode" "V4SF,V2DF")) (const_string "neon_fp_vmla_qqq")
 287           (and (eq_attr "simd_type" "simd_fmla_elt") (eq_attr "simd_mode" "V2SF")) (const_string "neon_fp_vmla_ddd_scalar")
 288           (and (eq_attr "simd_type" "simd_fmla_elt") (eq_attr "simd_mode" "V4SF,V2DF")) (const_string "neon_fp_vmla_qqq_scalar")
 289           (and (eq_attr "simd_type" "simd_fmul,simd_fmul_elt,simd_fdiv,simd_fsqrt") (eq_attr "simd_mode" "V2SF")) (const_string "neon_fp_vmul_ddd")
 290           (and (eq_attr "simd_type" "simd_fmul,simd_fmul_elt,simd_fdiv,simd_fsqrt") (eq_attr "simd_mode" "V4SF,V2DF")) (const_string "neon_fp_vmul_qqd")
 291           (and (eq_attr "simd_type" "simd_fadd") (eq_attr "simd_mode" "V2SF")) (const_string "neon_fp_vadd_ddd_vabs_dd")
 292           (and (eq_attr "simd_type" "simd_fadd") (eq_attr "simd_mode" "V4SF,V2DF")) (const_string "neon_fp_vadd_qqq_vabs_qq")
 293           (and (eq_attr "simd_type" "simd_fnegabs,simd_fminmax,simd_fminmaxv") (eq_attr "simd_mode" "V2SF")) (const_string "neon_fp_vadd_ddd_vabs_dd")
 294           (and (eq_attr "simd_type" "simd_fnegabs,simd_fminmax,simd_fminmaxv") (eq_attr "simd_mode" "V4SF,V2DF")) (const_string "neon_fp_vadd_qqq_vabs_qq")
 295           (and (eq_attr "simd_type" "simd_shift,simd_shift_acc") (eq_attr "simd_mode" "V8QI,V4HI,V2SI")) (const_string "neon_vshl_ddd")
 296           (and (eq_attr "simd_type" "simd_shift,simd_shift_acc") (eq_attr "simd_mode" "V16QI,V8HI,V4SI,V2DI")) (const_string "neon_shift_3")
 297           (eq_attr "simd_type" "simd_minmax,simd_minmaxv") (const_string "neon_int_5")
 298           (eq_attr "simd_type" "simd_shiftn_imm,simd_shiftn2_imm,simd_shiftl_imm,") (const_string "neon_shift_1")
 299           (eq_attr "simd_type" "simd_load1,simd_load2") (const_string "neon_vld1_1_2_regs")
 300           (eq_attr "simd_type" "simd_load3,simd_load3") (const_string "neon_vld1_3_4_regs")
 301           (eq_attr "simd_type" "simd_load1r,simd_load2r,simd_load3r,simd_load4r") (const_string "neon_vld2_2_regs_vld1_vld2_all_lanes")
 302           (eq_attr "simd_type" "simd_load1s,simd_load2s") (const_string "neon_vld1_vld2_lane")
 303           (eq_attr "simd_type" "simd_load3s,simd_load4s") (const_string "neon_vld3_vld4_lane")
 304           (eq_attr "simd_type" "simd_store1,simd_store2") (const_string "neon_vst1_1_2_regs_vst2_2_regs")
 305           (eq_attr "simd_type" "simd_store3,simd_store4") (const_string "neon_vst1_3_4_regs")
 306           (eq_attr "simd_type" "simd_store1s,simd_store2s") (const_string "neon_vst1_vst2_lane")
 307           (eq_attr "simd_type" "simd_store3s,simd_store4s") (const_string "neon_vst3_vst4_lane")
 308           (and (eq_attr "simd_type" "simd_frecpe,simd_frecps") (eq_attr "simd_mode" "V2SF")) (const_string "neon_fp_vrecps_vrsqrts_ddd")
 309           (and (eq_attr "simd_type" "simd_frecpe,simd_frecps") (eq_attr "simd_mode" "V4SF,V2DF")) (const_string "neon_fp_vrecps_vrsqrts_qqq")
 310           (eq_attr "simd_type" "none") (const_string "none")
 311   ]
 312   (const_string "unknown")))
 313
 314
 315 (define_expand "mov<mode>"
 316   [(set (match_operand:VALL 0 "aarch64_simd_nonimmediate_operand" "")
 317         (match_operand:VALL 1 "aarch64_simd_general_operand" ""))]
 318   "TARGET_SIMD"
 319   "
 320     if (GET_CODE (operands[0]) == MEM)
 321       operands[1] = force_reg (<MODE>mode, operands[1]);
 322   "
 323 )
 324
 325 (define_expand "movmisalign<mode>"
 326   [(set (match_operand:VALL 0 "aarch64_simd_nonimmediate_operand" "")
 327         (match_operand:VALL 1 "aarch64_simd_general_operand" ""))]
 328   "TARGET_SIMD"
 329 {
 330   /* This pattern is not permitted to fail during expansion: if both arguments
 331      are non-registers (e.g. memory := constant, which can be created by the
 332      auto-vectorizer), force operand 1 into a register.  */
 333   if (!register_operand (operands[0], <MODE>mode)
 334       && !register_operand (operands[1], <MODE>mode))
 335     operands[1] = force_reg (<MODE>mode, operands[1]);
 336 })
 337
 338 (define_insn "aarch64_simd_dup<mode>"
 339   [(set (match_operand:VDQ 0 "register_operand" "=w")
 340         (vec_duplicate:VDQ (match_operand:<VEL> 1 "register_operand" "r")))]
 341   "TARGET_SIMD"
 342   "dup\\t%0.<Vtype>, %<vw>1"
 343   [(set_attr "simd_type" "simd_dupgp")
 344    (set_attr "simd_mode" "<MODE>")]
 345 )
 346
 347 (define_insn "aarch64_dup_lane<mode>"
 348   [(set (match_operand:VDQ_I 0 "register_operand" "=w")
 349         (vec_duplicate:VDQ_I
 350           (vec_select:<VEL>
 351             (match_operand:<VCON> 1 "register_operand" "w")
 352             (parallel [(match_operand:SI 2 "immediate_operand" "i")])
 353           )))]
 354   "TARGET_SIMD"
 355   "dup\\t%<v>0<Vmtype>, %1.<Vetype>[%2]"
 356   [(set_attr "simd_type" "simd_dup")
 357    (set_attr "simd_mode" "<MODE>")]
 358 )
 359
 360 (define_insn "aarch64_dup_lane<mode>"
 361   [(set (match_operand:SDQ_I 0 "register_operand" "=w")
 362         (vec_select:<VEL>
 363           (match_operand:<VCON> 1 "register_operand" "w")
 364           (parallel [(match_operand:SI 2 "immediate_operand" "i")])
 365         ))]
 366   "TARGET_SIMD"
 367   "dup\\t%<v>0<Vmtype>, %1.<Vetype>[%2]"
 368   [(set_attr "simd_type" "simd_dup")
 369    (set_attr "simd_mode" "<MODE>")]
 370 )
 371
 372 (define_insn "aarch64_simd_dup<mode>"
 373   [(set (match_operand:VDQF 0 "register_operand" "=w")
 374         (vec_duplicate:VDQF (match_operand:<VEL> 1 "register_operand" "w")))]
 375   "TARGET_SIMD"
 376   "dup\\t%0.<Vtype>, %1.<Vetype>[0]"
 377   [(set_attr "simd_type" "simd_dup")
 378    (set_attr "simd_mode" "<MODE>")]
 379 )
 380
 381 (define_insn "*aarch64_simd_mov<mode>"
 382   [(set (match_operand:VD 0 "aarch64_simd_nonimmediate_operand"
 383                 "=w, Utv,  w, ?r, ?w, ?r, w")
 384         (match_operand:VD 1 "aarch64_simd_general_operand"
 385                 "Utv,  w,  w,  w,  r,  r, Dn"))]
 386   "TARGET_SIMD
 387    && (register_operand (operands[0], <MODE>mode)
 388        || register_operand (operands[1], <MODE>mode))"
 389 {
 390    switch (which_alternative)
 391      {
 392      case 0: return "ld1\t{%0.<Vtype>}, %1";
 393      case 1: return "st1\t{%1.<Vtype>}, %0";
 394      case 2: return "orr\t%0.<Vbtype>, %1.<Vbtype>, %1.<Vbtype>";
 395      case 3: return "umov\t%0, %1.d[0]";
 396      case 4: return "ins\t%0.d[0], %1";
 397      case 5: return "mov\t%0, %1";
 398      case 6:
 399         return aarch64_output_simd_mov_immediate (&operands[1],
 400                                                   <MODE>mode, 64);
 401      default: gcc_unreachable ();
 402      }
 403 }
 404   [(set_attr "simd_type" "simd_load1,simd_store1,simd_move,simd_movgp,simd_insgp,simd_move,simd_move_imm")
 405    (set_attr "simd_mode" "<MODE>")]
 406 )
 407
 408 (define_insn "*aarch64_simd_mov<mode>"
 409   [(set (match_operand:VQ 0 "aarch64_simd_nonimmediate_operand"
 410                 "=w, Utv,  w, ?r, ?w, ?r, w")
 411         (match_operand:VQ 1 "aarch64_simd_general_operand"
 412                 "Utv,  w,  w,  w,  r,  r, Dn"))]
 413   "TARGET_SIMD
 414    && (register_operand (operands[0], <MODE>mode)
 415        || register_operand (operands[1], <MODE>mode))"
 416 {
 417   switch (which_alternative)
 418     {
 419     case 0: return "ld1\t{%0.<Vtype>}, %1";
 420     case 1: return "st1\t{%1.<Vtype>}, %0";
 421     case 2: return "orr\t%0.<Vbtype>, %1.<Vbtype>, %1.<Vbtype>";
 422     case 3: return "umov\t%0, %1.d[0]\;umov\t%H0, %1.d[1]";
 423     case 4: return "ins\t%0.d[0], %1\;ins\t%0.d[1], %H1";
 424     case 5: return "#";
 425     case 6:
 426         return aarch64_output_simd_mov_immediate (&operands[1],
 427                                                   <MODE>mode, 128);
 428     default: gcc_unreachable ();
 429     }
 430 }
 431   [(set_attr "simd_type" "simd_load1,simd_store1,simd_move,simd_movgp,simd_insgp,simd_move,simd_move_imm")
 432    (set_attr "simd_mode" "<MODE>")
 433    (set_attr "length" "4,4,4,8,8,8,4")]
 434 )
 435
 436 (define_split
 437   [(set (match_operand:VQ 0 "register_operand" "")
 438       (match_operand:VQ 1 "register_operand" ""))]
 439   "TARGET_SIMD && reload_completed
 440    && GP_REGNUM_P (REGNO (operands[0]))
 441    && GP_REGNUM_P (REGNO (operands[1]))"
 442   [(set (match_dup 0) (match_dup 1))
 443    (set (match_dup 2) (match_dup 3))]
 444 {
 445   int rdest = REGNO (operands[0]);
 446   int rsrc = REGNO (operands[1]);
 447   rtx dest[2], src[2];
 448
 449   dest[0] = gen_rtx_REG (DImode, rdest);
 450   src[0] = gen_rtx_REG (DImode, rsrc);
 451   dest[1] = gen_rtx_REG (DImode, rdest + 1);
 452   src[1] = gen_rtx_REG (DImode, rsrc + 1);
 453
 454   aarch64_simd_disambiguate_copy (operands, dest, src, 2);
 455 })
 456
 457 (define_insn "orn<mode>3"
 458  [(set (match_operand:VDQ 0 "register_operand" "=w")
 459        (ior:VDQ (not:VDQ (match_operand:VDQ 1 "register_operand" "w"))
 460                 (match_operand:VDQ 2 "register_operand" "w")))]
 461  "TARGET_SIMD"
 462  "orn\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>"
 463   [(set_attr "simd_type" "simd_logic")
 464    (set_attr "simd_mode" "<MODE>")]
 465 )
 466
 467 (define_insn "bic<mode>3"
 468  [(set (match_operand:VDQ 0 "register_operand" "=w")
 469        (and:VDQ (not:VDQ (match_operand:VDQ 1 "register_operand" "w"))
 470                 (match_operand:VDQ 2 "register_operand" "w")))]
 471  "TARGET_SIMD"
 472  "bic\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>"
 473   [(set_attr "simd_type" "simd_logic")
 474    (set_attr "simd_mode" "<MODE>")]
 475 )
 476
 477 (define_insn "add<mode>3"
 478   [(set (match_operand:VDQ 0 "register_operand" "=w")
 479         (plus:VDQ (match_operand:VDQ 1 "register_operand" "w")
 480                   (match_operand:VDQ 2 "register_operand" "w")))]
 481   "TARGET_SIMD"
 482   "add\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
 483   [(set_attr "simd_type" "simd_add")
 484    (set_attr "simd_mode" "<MODE>")]
 485 )
 486
 487 (define_insn "sub<mode>3"
 488   [(set (match_operand:VDQ 0 "register_operand" "=w")
 489         (minus:VDQ (match_operand:VDQ 1 "register_operand" "w")
 490                    (match_operand:VDQ 2 "register_operand" "w")))]
 491   "TARGET_SIMD"
 492   "sub\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
 493   [(set_attr "simd_type" "simd_add")
 494    (set_attr "simd_mode" "<MODE>")]
 495 )
 496
 497 (define_insn "mul<mode>3"
 498   [(set (match_operand:VDQM 0 "register_operand" "=w")
 499         (mult:VDQM (match_operand:VDQM 1 "register_operand" "w")
 500                    (match_operand:VDQM 2 "register_operand" "w")))]
 501   "TARGET_SIMD"
 502   "mul\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
 503   [(set_attr "simd_type" "simd_mul")
 504    (set_attr "simd_mode" "<MODE>")]
 505 )
 506
 507 (define_insn "neg<mode>2"
 508   [(set (match_operand:VDQ 0 "register_operand" "=w")
 509         (neg:VDQ (match_operand:VDQ 1 "register_operand" "w")))]
 510   "TARGET_SIMD"
 511   "neg\t%0.<Vtype>, %1.<Vtype>"
 512   [(set_attr "simd_type" "simd_negabs")
 513    (set_attr "simd_mode" "<MODE>")]
 514 )
 515
 516 (define_insn "abs<mode>2"
 517   [(set (match_operand:VDQ 0 "register_operand" "=w")
 518         (abs:VDQ (match_operand:VDQ 1 "register_operand" "w")))]
 519   "TARGET_SIMD"
 520   "abs\t%0.<Vtype>, %1.<Vtype>"
 521   [(set_attr "simd_type" "simd_negabs")
 522    (set_attr "simd_mode" "<MODE>")]
 523 )
 524
 525 (define_insn "abd<mode>_3"
 526   [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
 527         (abs:VDQ_BHSI (minus:VDQ_BHSI
 528                        (match_operand:VDQ_BHSI 1 "register_operand" "w")
 529                        (match_operand:VDQ_BHSI 2 "register_operand" "w"))))]
 530   "TARGET_SIMD"
 531   "sabd\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
 532   [(set_attr "simd_type" "simd_abd")
 533    (set_attr "simd_mode" "<MODE>")]
 534 )
 535
 536 (define_insn "aba<mode>_3"
 537   [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
 538         (plus:VDQ_BHSI (abs:VDQ_BHSI (minus:VDQ_BHSI
 539                          (match_operand:VDQ_BHSI 1 "register_operand" "w")
 540                          (match_operand:VDQ_BHSI 2 "register_operand" "w")))
 541                        (match_operand:VDQ_BHSI 3 "register_operand" "0")))]
 542   "TARGET_SIMD"
 543   "saba\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
 544   [(set_attr "simd_type" "simd_abd")
 545    (set_attr "simd_mode" "<MODE>")]
 546 )
 547
 548 (define_insn "fabd<mode>_3"
 549   [(set (match_operand:VDQF 0 "register_operand" "=w")
 550         (abs:VDQF (minus:VDQF
 551                    (match_operand:VDQF 1 "register_operand" "w")
 552                    (match_operand:VDQF 2 "register_operand" "w"))))]
 553   "TARGET_SIMD"
 554   "fabd\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
 555   [(set_attr "simd_type" "simd_fabd")
 556    (set_attr "simd_mode" "<MODE>")]
 557 )
 558
 559 (define_insn "and<mode>3"
 560   [(set (match_operand:VDQ 0 "register_operand" "=w")
 561         (and:VDQ (match_operand:VDQ 1 "register_operand" "w")
 562                  (match_operand:VDQ 2 "register_operand" "w")))]
 563   "TARGET_SIMD"
 564   "and\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>"
 565   [(set_attr "simd_type" "simd_logic")
 566    (set_attr "simd_mode" "<MODE>")]
 567 )
 568
 569 (define_insn "ior<mode>3"
 570   [(set (match_operand:VDQ 0 "register_operand" "=w")
 571         (ior:VDQ (match_operand:VDQ 1 "register_operand" "w")
 572                  (match_operand:VDQ 2 "register_operand" "w")))]
 573   "TARGET_SIMD"
 574   "orr\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>"
 575   [(set_attr "simd_type" "simd_logic")
 576    (set_attr "simd_mode" "<MODE>")]
 577 )
 578
 579 (define_insn "xor<mode>3"
 580   [(set (match_operand:VDQ 0 "register_operand" "=w")
 581         (xor:VDQ (match_operand:VDQ 1 "register_operand" "w")
 582                  (match_operand:VDQ 2 "register_operand" "w")))]
 583   "TARGET_SIMD"
 584   "eor\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>"
 585   [(set_attr "simd_type" "simd_logic")
 586    (set_attr "simd_mode" "<MODE>")]
 587 )
 588
 589 (define_insn "one_cmpl<mode>2"
 590   [(set (match_operand:VDQ 0 "register_operand" "=w")
 591         (not:VDQ (match_operand:VDQ 1 "register_operand" "w")))]
 592   "TARGET_SIMD"
 593   "not\t%0.<Vbtype>, %1.<Vbtype>"
 594   [(set_attr "simd_type" "simd_logic")
 595    (set_attr "simd_mode" "<MODE>")]
 596 )
 597
 598 (define_insn "aarch64_simd_vec_set<mode>"
 599   [(set (match_operand:VQ_S 0 "register_operand" "=w")
 600         (vec_merge:VQ_S
 601             (vec_duplicate:VQ_S
 602                 (match_operand:<VEL> 1 "register_operand" "r"))
 603             (match_operand:VQ_S 3 "register_operand" "0")
 604             (match_operand:SI 2 "immediate_operand" "i")))]
 605   "TARGET_SIMD"
 606   "ins\t%0.<Vetype>[%p2], %w1";
 607   [(set_attr "simd_type" "simd_insgp")
 608    (set_attr "simd_mode" "<MODE>")]
 609 )
 610
 611 (define_insn "aarch64_simd_lshr<mode>"
 612  [(set (match_operand:VDQ 0 "register_operand" "=w")
 613        (lshiftrt:VDQ (match_operand:VDQ 1 "register_operand" "w")
 614                      (match_operand:VDQ  2 "aarch64_simd_rshift_imm" "Dr")))]
 615  "TARGET_SIMD"
 616  "ushr\t%0.<Vtype>, %1.<Vtype>, %2"
 617   [(set_attr "simd_type" "simd_shift_imm")
 618    (set_attr "simd_mode" "<MODE>")]
 619 )
 620
 621 (define_insn "aarch64_simd_ashr<mode>"
 622  [(set (match_operand:VDQ 0 "register_operand" "=w")
 623        (ashiftrt:VDQ (match_operand:VDQ 1 "register_operand" "w")
 624                      (match_operand:VDQ  2 "aarch64_simd_rshift_imm" "Dr")))]
 625  "TARGET_SIMD"
 626  "sshr\t%0.<Vtype>, %1.<Vtype>, %2"
 627   [(set_attr "simd_type" "simd_shift_imm")
 628    (set_attr "simd_mode" "<MODE>")]
 629 )
 630
 631 (define_insn "aarch64_simd_imm_shl<mode>"
 632  [(set (match_operand:VDQ 0 "register_operand" "=w")
 633        (ashift:VDQ (match_operand:VDQ 1 "register_operand" "w")
 634                    (match_operand:VDQ  2 "aarch64_simd_lshift_imm" "Dl")))]
 635  "TARGET_SIMD"
 636   "shl\t%0.<Vtype>, %1.<Vtype>, %2"
 637   [(set_attr "simd_type" "simd_shift_imm")
 638    (set_attr "simd_mode" "<MODE>")]
 639 )
 640
 641 (define_insn "aarch64_simd_reg_sshl<mode>"
 642  [(set (match_operand:VDQ 0 "register_operand" "=w")
 643        (ashift:VDQ (match_operand:VDQ 1 "register_operand" "w")
 644                    (match_operand:VDQ 2 "register_operand" "w")))]
 645  "TARGET_SIMD"
 646  "sshl\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
 647   [(set_attr "simd_type" "simd_shift")
 648    (set_attr "simd_mode" "<MODE>")]
 649 )
 650
 651 (define_insn "aarch64_simd_reg_shl<mode>_unsigned"
 652  [(set (match_operand:VDQ 0 "register_operand" "=w")
 653        (unspec:VDQ [(match_operand:VDQ 1 "register_operand" "w")
 654                     (match_operand:VDQ 2 "register_operand" "w")]
 655                    UNSPEC_ASHIFT_UNSIGNED))]
 656  "TARGET_SIMD"
 657  "ushl\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
 658   [(set_attr "simd_type" "simd_shift")
 659    (set_attr "simd_mode" "<MODE>")]
 660 )
 661
 662 (define_insn "aarch64_simd_reg_shl<mode>_signed"
 663  [(set (match_operand:VDQ 0 "register_operand" "=w")
 664        (unspec:VDQ [(match_operand:VDQ 1 "register_operand" "w")
 665                     (match_operand:VDQ 2 "register_operand" "w")]
 666                    UNSPEC_ASHIFT_SIGNED))]
 667  "TARGET_SIMD"
 668  "sshl\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
 669   [(set_attr "simd_type" "simd_shift")
 670    (set_attr "simd_mode" "<MODE>")]
 671 )
 672
 673 (define_expand "ashl<mode>3"
 674   [(match_operand:VDQ 0 "register_operand" "")
 675    (match_operand:VDQ 1 "register_operand" "")
 676    (match_operand:SI  2 "general_operand" "")]
 677  "TARGET_SIMD"
 678 {
 679   int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
 680   int shift_amount;
 681
 682   if (CONST_INT_P (operands[2]))
 683     {
 684       shift_amount = INTVAL (operands[2]);
 685       if (shift_amount >= 0 && shift_amount < bit_width)
 686         {
 687           rtx tmp = aarch64_simd_gen_const_vector_dup (<MODE>mode,
 688                                                        shift_amount);
 689           emit_insn (gen_aarch64_simd_imm_shl<mode> (operands[0],
 690                                                      operands[1],
 691                                                      tmp));
 692           DONE;
 693         }
 694       else
 695         {
 696           operands[2] = force_reg (SImode, operands[2]);
 697         }
 698     }
 699   else if (MEM_P (operands[2]))
 700     {
 701       operands[2] = force_reg (SImode, operands[2]);
 702     }
 703
 704   if (REG_P (operands[2]))
 705     {
 706       rtx tmp = gen_reg_rtx (<MODE>mode);
 707       emit_insn (gen_aarch64_simd_dup<mode> (tmp,
 708                                              convert_to_mode (<VEL>mode,
 709                                                               operands[2],
 710                                                               0)));
 711       emit_insn (gen_aarch64_simd_reg_sshl<mode> (operands[0], operands[1],
 712                                                   tmp));
 713       DONE;
 714     }
 715   else
 716     FAIL;
 717 }
 718 )
 719
 720 (define_expand "lshr<mode>3"
 721   [(match_operand:VDQ 0 "register_operand" "")
 722    (match_operand:VDQ 1 "register_operand" "")
 723    (match_operand:SI  2 "general_operand" "")]
 724  "TARGET_SIMD"
 725 {
 726   int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
 727   int shift_amount;
 728
 729   if (CONST_INT_P (operands[2]))
 730     {
 731       shift_amount = INTVAL (operands[2]);
 732       if (shift_amount > 0 && shift_amount <= bit_width)
 733         {
 734           rtx tmp = aarch64_simd_gen_const_vector_dup (<MODE>mode,
 735                                                        shift_amount);
 736           emit_insn (gen_aarch64_simd_lshr<mode> (operands[0],
 737                                                   operands[1],
 738                                                   tmp));
 739           DONE;
 740         }
 741       else
 742         operands[2] = force_reg (SImode, operands[2]);
 743     }
 744   else if (MEM_P (operands[2]))
 745     {
 746       operands[2] = force_reg (SImode, operands[2]);
 747     }
 748
 749   if (REG_P (operands[2]))
 750     {
 751       rtx tmp = gen_reg_rtx (SImode);
 752       rtx tmp1 = gen_reg_rtx (<MODE>mode);
 753       emit_insn (gen_negsi2 (tmp, operands[2]));
 754       emit_insn (gen_aarch64_simd_dup<mode> (tmp1,
 755                                              convert_to_mode (<VEL>mode,
 756                                                               tmp, 0)));
 757       emit_insn (gen_aarch64_simd_reg_shl<mode>_unsigned (operands[0],
 758                                                           operands[1],
 759                                                           tmp1));
 760       DONE;
 761     }
 762   else
 763     FAIL;
 764 }
 765 )
 766
 767 (define_expand "ashr<mode>3"
 768   [(match_operand:VDQ 0 "register_operand" "")
 769    (match_operand:VDQ 1 "register_operand" "")
 770    (match_operand:SI  2 "general_operand" "")]
 771  "TARGET_SIMD"
 772 {
 773   int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
 774   int shift_amount;
 775
 776   if (CONST_INT_P (operands[2]))
 777     {
 778       shift_amount = INTVAL (operands[2]);
 779       if (shift_amount > 0 && shift_amount <= bit_width)
 780         {
 781           rtx tmp = aarch64_simd_gen_const_vector_dup (<MODE>mode,
 782                                                        shift_amount);
 783           emit_insn (gen_aarch64_simd_ashr<mode> (operands[0],
 784                                                   operands[1],
 785                                                   tmp));
 786           DONE;
 787         }
 788       else
 789         operands[2] = force_reg (SImode, operands[2]);
 790     }
 791   else if (MEM_P (operands[2]))
 792     {
 793       operands[2] = force_reg (SImode, operands[2]);
 794     }
 795
 796   if (REG_P (operands[2]))
 797     {
 798       rtx tmp = gen_reg_rtx (SImode);
 799       rtx tmp1 = gen_reg_rtx (<MODE>mode);
 800       emit_insn (gen_negsi2 (tmp, operands[2]));
 801       emit_insn (gen_aarch64_simd_dup<mode> (tmp1,
 802                                              convert_to_mode (<VEL>mode,
 803                                                               tmp, 0)));
 804       emit_insn (gen_aarch64_simd_reg_shl<mode>_signed (operands[0],
 805                                                         operands[1],
 806                                                         tmp1));
 807       DONE;
 808     }
 809   else
 810     FAIL;
 811 }
 812 )
 813
 814 (define_expand "vashl<mode>3"
 815  [(match_operand:VDQ 0 "register_operand" "")
 816   (match_operand:VDQ 1 "register_operand" "")
 817   (match_operand:VDQ 2 "register_operand" "")]
 818  "TARGET_SIMD"
 819 {
 820   emit_insn (gen_aarch64_simd_reg_sshl<mode> (operands[0], operands[1],
 821                                               operands[2]));
 822   DONE;
 823 })
 824
 825 ;; Using mode VQ_S as there is no V2DImode neg!
 826 ;; Negating individual lanes most certainly offsets the
 827 ;; gain from vectorization.
 828 (define_expand "vashr<mode>3"
 829  [(match_operand:VQ_S 0 "register_operand" "")
 830   (match_operand:VQ_S 1 "register_operand" "")
 831   (match_operand:VQ_S 2 "register_operand" "")]
 832  "TARGET_SIMD"
 833 {
 834   rtx neg = gen_reg_rtx (<MODE>mode);
 835   emit (gen_neg<mode>2 (neg, operands[2]));
 836   emit_insn (gen_aarch64_simd_reg_shl<mode>_signed (operands[0], operands[1],
 837                                                     neg));
 838   DONE;
 839 })
 840
 841 (define_expand "vlshr<mode>3"
 842  [(match_operand:VQ_S 0 "register_operand" "")
 843   (match_operand:VQ_S 1 "register_operand" "")
 844   (match_operand:VQ_S 2 "register_operand" "")]
 845  "TARGET_SIMD"
 846 {
 847   rtx neg = gen_reg_rtx (<MODE>mode);
 848   emit (gen_neg<mode>2 (neg, operands[2]));
 849   emit_insn (gen_aarch64_simd_reg_shl<mode>_unsigned (operands[0], operands[1],
 850                                                       neg));
 851   DONE;
 852 })
 853
 854 (define_expand "vec_set<mode>"
 855   [(match_operand:VQ_S 0 "register_operand" "+w")
 856    (match_operand:<VEL> 1 "register_operand" "r")
 857    (match_operand:SI 2 "immediate_operand" "")]
 858   "TARGET_SIMD"
 859   {
 860     HOST_WIDE_INT elem = (HOST_WIDE_INT) 1 << INTVAL (operands[2]);
 861     emit_insn (gen_aarch64_simd_vec_set<mode> (operands[0], operands[1],
 862                                             GEN_INT (elem), operands[0]));
 863     DONE;
 864   }
 865 )
 866
 867 (define_insn "aarch64_simd_vec_setv2di"
 868   [(set (match_operand:V2DI 0 "register_operand" "=w")
 869         (vec_merge:V2DI
 870             (vec_duplicate:V2DI
 871                 (match_operand:DI 1 "register_operand" "r"))
 872             (match_operand:V2DI 3 "register_operand" "0")
 873             (match_operand:SI 2 "immediate_operand" "i")))]
 874   "TARGET_SIMD"
 875   "ins\t%0.d[%p2], %1";
 876   [(set_attr "simd_type" "simd_insgp")
 877    (set_attr "simd_mode" "V2DI")]
 878 )
 879
 880 (define_expand "vec_setv2di"
 881   [(match_operand:V2DI 0 "register_operand" "+w")
 882    (match_operand:DI 1 "register_operand" "r")
 883    (match_operand:SI 2 "immediate_operand" "")]
 884   "TARGET_SIMD"
 885   {
 886     HOST_WIDE_INT elem = (HOST_WIDE_INT) 1 << INTVAL (operands[2]);
 887     emit_insn (gen_aarch64_simd_vec_setv2di (operands[0], operands[1],
 888                                           GEN_INT (elem), operands[0]));
 889     DONE;
 890   }
 891 )
 892
 893 (define_insn "aarch64_simd_vec_set<mode>"
 894   [(set (match_operand:VDQF 0 "register_operand" "=w")
 895         (vec_merge:VDQF
 896             (vec_duplicate:VDQF
 897                 (match_operand:<VEL> 1 "register_operand" "w"))
 898             (match_operand:VDQF 3 "register_operand" "0")
 899             (match_operand:SI 2 "immediate_operand" "i")))]
 900   "TARGET_SIMD"
 901   "ins\t%0.<Vetype>[%p2], %1.<Vetype>[0]";
 902   [(set_attr "simd_type" "simd_ins")
 903    (set_attr "simd_mode" "<MODE>")]
 904 )
 905
 906 (define_expand "vec_set<mode>"
 907   [(match_operand:VDQF 0 "register_operand" "+w")
 908    (match_operand:<VEL> 1 "register_operand" "w")
 909    (match_operand:SI 2 "immediate_operand" "")]
 910   "TARGET_SIMD"
 911   {
 912     HOST_WIDE_INT elem = (HOST_WIDE_INT) 1 << INTVAL (operands[2]);
 913     emit_insn (gen_aarch64_simd_vec_set<mode> (operands[0], operands[1],
 914                                           GEN_INT (elem), operands[0]));
 915     DONE;
 916   }
 917 )
 918
 919
 920 (define_insn "aarch64_mla<mode>"
 921  [(set (match_operand:VQ_S 0 "register_operand" "=w")
 922        (plus:VQ_S (mult:VQ_S (match_operand:VQ_S 2 "register_operand" "w")
 923                              (match_operand:VQ_S 3 "register_operand" "w"))
 924                   (match_operand:VQ_S 1 "register_operand" "0")))]
 925  "TARGET_SIMD"
 926  "mla\t%0.<Vtype>, %2.<Vtype>, %3.<Vtype>"
 927   [(set_attr "simd_type" "simd_mla")
 928    (set_attr "simd_mode" "<MODE>")]
 929 )
 930
 931 (define_insn "aarch64_mls<mode>"
 932  [(set (match_operand:VQ_S 0 "register_operand" "=w")
 933        (minus:VQ_S (match_operand:VQ_S 1 "register_operand" "0")
 934                    (mult:VQ_S (match_operand:VQ_S 2 "register_operand" "w")
 935                               (match_operand:VQ_S 3 "register_operand" "w"))))]
 936  "TARGET_SIMD"
 937  "mls\t%0.<Vtype>, %2.<Vtype>, %3.<Vtype>"
 938   [(set_attr "simd_type" "simd_mla")
 939    (set_attr "simd_mode" "<MODE>")]
 940 )
 941
 942 ;; Max/Min operations.
 943 (define_insn "<su><maxmin><mode>3"
 944  [(set (match_operand:VQ_S 0 "register_operand" "=w")
 945        (MAXMIN:VQ_S (match_operand:VQ_S 1 "register_operand" "w")
 946                     (match_operand:VQ_S 2 "register_operand" "w")))]
 947  "TARGET_SIMD"
 948  "<su><maxmin>\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
 949   [(set_attr "simd_type" "simd_minmax")
 950    (set_attr "simd_mode" "<MODE>")]
 951 )
 952
 953 ;; Move into low-half clearing high half to 0.
 954
 955 (define_insn "move_lo_quad_<mode>"
 956   [(set (match_operand:VQ 0 "register_operand" "=w")
 957         (vec_concat:VQ
 958           (match_operand:<VHALF> 1 "register_operand" "w")
 959           (vec_duplicate:<VHALF> (const_int 0))))]
 960   "TARGET_SIMD"
 961   "mov\\t%d0, %d1";
 962   [(set_attr "simd_type" "simd_dup")
 963    (set_attr "simd_mode" "<MODE>")]
 964 )
 965
 966 ;; Move into high-half.
 967
 968 (define_insn "aarch64_simd_move_hi_quad_<mode>"
 969   [(set (match_operand:VQ 0 "register_operand" "+w")
 970         (vec_concat:VQ
 971           (vec_select:<VHALF>
 972                 (match_dup 0)
 973                 (match_operand:VQ 2 "vect_par_cnst_lo_half" ""))
 974           (match_operand:<VHALF> 1 "register_operand" "w")))]
 975   "TARGET_SIMD"
 976   "ins\\t%0.d[1], %1.d[0]";
 977   [(set_attr "simd_type" "simd_ins")
 978    (set_attr "simd_mode" "<MODE>")]
 979 )
 980
 981 (define_expand "move_hi_quad_<mode>"
 982  [(match_operand:VQ 0 "register_operand" "")
 983   (match_operand:<VHALF> 1 "register_operand" "")]
 984  "TARGET_SIMD"
 985 {
 986   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
 987   emit_insn (gen_aarch64_simd_move_hi_quad_<mode> (operands[0],
 988                                                    operands[1], p));
 989   DONE;
 990 })
 991
 992 ;; Narrowing operations.
 993
 994 ;; For doubles.
 995 (define_insn "aarch64_simd_vec_pack_trunc_<mode>"
 996  [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
 997        (truncate:<VNARROWQ> (match_operand:VQN 1 "register_operand" "w")))]
 998  "TARGET_SIMD"
 999  "xtn\\t%0.<Vntype>, %1.<Vtype>"
1000   [(set_attr "simd_type" "simd_shiftn_imm")
1001    (set_attr "simd_mode" "<MODE>")]
1002 )
1003
1004 (define_expand "vec_pack_trunc_<mode>"
1005  [(match_operand:<VNARROWD> 0 "register_operand" "")
1006   (match_operand:VDN 1 "register_operand" "")
1007   (match_operand:VDN 2 "register_operand" "")]
1008  "TARGET_SIMD"
1009 {
1010   rtx tempreg = gen_reg_rtx (<VDBL>mode);
1011
1012   emit_insn (gen_move_lo_quad_<Vdbl> (tempreg, operands[1]));
1013   emit_insn (gen_move_hi_quad_<Vdbl> (tempreg, operands[2]));
1014   emit_insn (gen_aarch64_simd_vec_pack_trunc_<Vdbl> (operands[0], tempreg));
1015   DONE;
1016 })
1017
1018 ;; For quads.
1019
1020 (define_insn "vec_pack_trunc_<mode>"
1021  [(set (match_operand:<VNARROWQ2> 0 "register_operand" "+&w")
1022        (vec_concat:<VNARROWQ2>
1023          (truncate:<VNARROWQ> (match_operand:VQN 1 "register_operand" "w"))
1024          (truncate:<VNARROWQ> (match_operand:VQN 2 "register_operand" "w"))))]
1025  "TARGET_SIMD"
1026  "xtn\\t%0.<Vntype>, %1.<Vtype>\;xtn2\\t%0.<V2ntype>, %2.<Vtype>"
1027   [(set_attr "simd_type" "simd_shiftn2_imm")
1028    (set_attr "simd_mode" "<MODE>")
1029    (set_attr "length" "8")]
1030 )
1031
1032 ;; Widening operations.
1033
1034 (define_insn "aarch64_simd_vec_unpack<su>_lo_<mode>"
1035   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1036         (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1037                                (match_operand:VQW 1 "register_operand" "w")
1038                                (match_operand:VQW 2 "vect_par_cnst_lo_half" "")
1039                             )))]
1040   "TARGET_SIMD"
1041   "<su>shll %0.<Vwtype>, %1.<Vhalftype>, 0"
1042   [(set_attr "simd_type" "simd_shiftl_imm")
1043    (set_attr "simd_mode" "<MODE>")]
1044 )
1045
1046 (define_insn "aarch64_simd_vec_unpack<su>_hi_<mode>"
1047   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1048         (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1049                                (match_operand:VQW 1 "register_operand" "w")
1050                                (match_operand:VQW 2 "vect_par_cnst_hi_half" "")
1051                             )))]
1052   "TARGET_SIMD"
1053   "<su>shll2 %0.<Vwtype>, %1.<Vtype>, 0"
1054   [(set_attr "simd_type" "simd_shiftl_imm")
1055    (set_attr "simd_mode" "<MODE>")]
1056 )
1057
1058 (define_expand "vec_unpack<su>_hi_<mode>"
1059   [(match_operand:<VWIDE> 0 "register_operand" "")
1060    (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand"))]
1061   "TARGET_SIMD"
1062   {
1063     rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
1064     emit_insn (gen_aarch64_simd_vec_unpack<su>_hi_<mode> (operands[0],
1065                                                           operands[1], p));
1066     DONE;
1067   }
1068 )
1069
1070 (define_expand "vec_unpack<su>_lo_<mode>"
1071   [(match_operand:<VWIDE> 0 "register_operand" "")
1072    (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand" ""))]
1073   "TARGET_SIMD"
1074   {
1075     rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
1076     emit_insn (gen_aarch64_simd_vec_unpack<su>_lo_<mode> (operands[0],
1077                                                           operands[1], p));
1078     DONE;
1079   }
1080 )
1081
1082 ;; Widening arithmetic.
1083
1084 (define_insn "aarch64_simd_vec_<su>mult_lo_<mode>"
1085  [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1086        (mult:<VWIDE> (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1087                            (match_operand:VQW 1 "register_operand" "w")
1088                            (match_operand:VQW 3 "vect_par_cnst_lo_half" "")))
1089                      (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1090                            (match_operand:VQW 2 "register_operand" "w")
1091                            (match_dup 3)))))]
1092   "TARGET_SIMD"
1093   "<su>mull\\t%0.<Vwtype>, %1.<Vhalftype>, %2.<Vhalftype>"
1094   [(set_attr "simd_type" "simd_mull")
1095    (set_attr "simd_mode" "<MODE>")]
1096 )
1097
1098 (define_expand "vec_widen_<su>mult_lo_<mode>"
1099   [(match_operand:<VWIDE> 0 "register_operand" "")
1100    (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand" ""))
1101    (ANY_EXTEND:<VWIDE> (match_operand:VQW 2 "register_operand" ""))]
1102  "TARGET_SIMD"
1103  {
1104    rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
1105    emit_insn (gen_aarch64_simd_vec_<su>mult_lo_<mode> (operands[0],
1106                                                        operands[1],
1107                                                        operands[2], p));
1108    DONE;
1109  }
1110 )
1111
1112 (define_insn "aarch64_simd_vec_<su>mult_hi_<mode>"
1113  [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1114       (mult:<VWIDE> (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1115                             (match_operand:VQW 1 "register_operand" "w")
1116                             (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))
1117                     (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1118                             (match_operand:VQW 2 "register_operand" "w")
1119                             (match_dup 3)))))]
1120   "TARGET_SIMD"
1121   "<su>mull2\\t%0.<Vwtype>, %1.<Vtype>, %2.<Vtype>"
1122   [(set_attr "simd_type" "simd_mull")
1123    (set_attr "simd_mode" "<MODE>")]
1124 )
1125
1126 (define_expand "vec_widen_<su>mult_hi_<mode>"
1127   [(match_operand:<VWIDE> 0 "register_operand" "")
1128    (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand" ""))
1129    (ANY_EXTEND:<VWIDE> (match_operand:VQW 2 "register_operand" ""))]
1130  "TARGET_SIMD"
1131  {
1132    rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
1133    emit_insn (gen_aarch64_simd_vec_<su>mult_hi_<mode> (operands[0],
1134                                                        operands[1],
1135                                                        operands[2], p));
1136    DONE;
1137
1138  }
1139 )
1140
1141 ;; FP vector operations.
1142 ;; AArch64 AdvSIMD supports single-precision (32-bit) and
1143 ;; double-precision (64-bit) floating-point data types and arithmetic as
1144 ;; defined by the IEEE 754-2008 standard.  This makes them vectorizable
1145 ;; without the need for -ffast-math or -funsafe-math-optimizations.
1146 ;;
1147 ;; Floating-point operations can raise an exception.  Vectorizing such
1148 ;; operations are safe because of reasons explained below.
1149 ;;
1150 ;; ARMv8 permits an extension to enable trapped floating-point
1151 ;; exception handling, however this is an optional feature.  In the
1152 ;; event of a floating-point exception being raised by vectorised
1153 ;; code then:
1154 ;; 1.  If trapped floating-point exceptions are available, then a trap
1155 ;;     will be taken when any lane raises an enabled exception.  A trap
1156 ;;     handler may determine which lane raised the exception.
1157 ;; 2.  Alternatively a sticky exception flag is set in the
1158 ;;     floating-point status register (FPSR).  Software may explicitly
1159 ;;     test the exception flags, in which case the tests will either
1160 ;;     prevent vectorisation, allowing precise identification of the
1161 ;;     failing operation, or if tested outside of vectorisable regions
1162 ;;     then the specific operation and lane are not of interest.
1163
1164 ;; FP arithmetic operations.
1165
1166 (define_insn "add<mode>3"
1167  [(set (match_operand:VDQF 0 "register_operand" "=w")
1168        (plus:VDQF (match_operand:VDQF 1 "register_operand" "w")
1169                   (match_operand:VDQF 2 "register_operand" "w")))]
1170  "TARGET_SIMD"
1171  "fadd\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1172   [(set_attr "simd_type" "simd_fadd")
1173    (set_attr "simd_mode" "<MODE>")]
1174 )
1175
1176 (define_insn "sub<mode>3"
1177  [(set (match_operand:VDQF 0 "register_operand" "=w")
1178        (minus:VDQF (match_operand:VDQF 1 "register_operand" "w")
1179                    (match_operand:VDQF 2 "register_operand" "w")))]
1180  "TARGET_SIMD"
1181  "fsub\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1182   [(set_attr "simd_type" "simd_fadd")
1183    (set_attr "simd_mode" "<MODE>")]
1184 )
1185
1186 (define_insn "mul<mode>3"
1187  [(set (match_operand:VDQF 0 "register_operand" "=w")
1188        (mult:VDQF (match_operand:VDQF 1 "register_operand" "w")
1189                   (match_operand:VDQF 2 "register_operand" "w")))]
1190  "TARGET_SIMD"
1191  "fmul\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1192   [(set_attr "simd_type" "simd_fmul")
1193    (set_attr "simd_mode" "<MODE>")]
1194 )
1195
1196 (define_insn "div<mode>3"
1197  [(set (match_operand:VDQF 0 "register_operand" "=w")
1198        (div:VDQF (match_operand:VDQF 1 "register_operand" "w")
1199                  (match_operand:VDQF 2 "register_operand" "w")))]
1200  "TARGET_SIMD"
1201  "fdiv\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1202   [(set_attr "simd_type" "simd_fdiv")
1203    (set_attr "simd_mode" "<MODE>")]
1204 )
1205
1206 (define_insn "neg<mode>2"
1207  [(set (match_operand:VDQF 0 "register_operand" "=w")
1208        (neg:VDQF (match_operand:VDQF 1 "register_operand" "w")))]
1209  "TARGET_SIMD"
1210  "fneg\\t%0.<Vtype>, %1.<Vtype>"
1211   [(set_attr "simd_type" "simd_fnegabs")
1212    (set_attr "simd_mode" "<MODE>")]
1213 )
1214
1215 (define_insn "abs<mode>2"
1216  [(set (match_operand:VDQF 0 "register_operand" "=w")
1217        (abs:VDQF (match_operand:VDQF 1 "register_operand" "w")))]
1218  "TARGET_SIMD"
1219  "fabs\\t%0.<Vtype>, %1.<Vtype>"
1220   [(set_attr "simd_type" "simd_fnegabs")
1221    (set_attr "simd_mode" "<MODE>")]
1222 )
1223
1224 (define_insn "fma<mode>4"
1225   [(set (match_operand:VDQF 0 "register_operand" "=w")
1226        (fma:VDQF (match_operand:VDQF 1 "register_operand" "w")
1227                 (match_operand:VDQF 2 "register_operand" "w")
1228                 (match_operand:VDQF 3 "register_operand" "0")))]
1229   "TARGET_SIMD"
1230  "fmla\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1231   [(set_attr "simd_type" "simd_fmla")
1232    (set_attr "simd_mode" "<MODE>")]
1233 )
1234
1235 ;; Vector versions of the floating-point frint patterns.
1236 ;; Expands to btrunc, ceil, floor, nearbyint, rint, round.
1237 (define_insn "<frint_pattern><mode>2"
1238   [(set (match_operand:VDQF 0 "register_operand" "=w")
1239         (unspec:VDQF [(match_operand:VDQF 1 "register_operand" "w")]
1240                       FRINT))]
1241   "TARGET_SIMD"
1242   "frint<frint_suffix>\\t%0.<Vtype>, %1.<Vtype>"
1243   [(set_attr "simd_type" "simd_frint")
1244    (set_attr "simd_mode" "<MODE>")]
1245 )
1246
1247 ;; Vector versions of the fcvt standard patterns.
1248 ;; Expands to lbtrunc, lround, lceil, lfloor
1249 (define_insn "l<fcvt_pattern><su_optab><VDQF:mode><fcvt_target>2"
1250   [(set (match_operand:<FCVT_TARGET> 0 "register_operand" "=w")
1251         (FIXUORS:<FCVT_TARGET> (unspec:<FCVT_TARGET>
1252                                [(match_operand:VDQF 1 "register_operand" "w")]
1253                                FCVT)))]
1254   "TARGET_SIMD"
1255   "fcvt<frint_suffix><su>\\t%0.<Vtype>, %1.<Vtype>"
1256   [(set_attr "simd_type" "simd_fcvti")
1257    (set_attr "simd_mode" "<MODE>")]
1258 )
1259
1260 (define_expand "<optab><VDQF:mode><fcvt_target>2"
1261   [(set (match_operand:<FCVT_TARGET> 0 "register_operand")
1262         (FIXUORS:<FCVT_TARGET> (unspec:<FCVT_TARGET>
1263                                [(match_operand:VDQF 1 "register_operand")]
1264                                UNSPEC_FRINTZ)))]
1265   "TARGET_SIMD"
1266   {})
1267
1268 (define_expand "<fix_trunc_optab><VDQF:mode><fcvt_target>2"
1269   [(set (match_operand:<FCVT_TARGET> 0 "register_operand")
1270         (FIXUORS:<FCVT_TARGET> (unspec:<FCVT_TARGET>
1271                                [(match_operand:VDQF 1 "register_operand")]
1272                                UNSPEC_FRINTZ)))]
1273   "TARGET_SIMD"
1274   {})
1275
1276 (define_expand "ftrunc<VDQF:mode>2"
1277   [(set (match_operand:VDQF 0 "register_operand")
1278         (unspec:VDQF [(match_operand:VDQF 1 "register_operand")]
1279                       UNSPEC_FRINTZ))]
1280   "TARGET_SIMD"
1281   {})
1282
1283 (define_insn "<optab><fcvt_target><VDQF:mode>2"
1284   [(set (match_operand:VDQF 0 "register_operand" "=w")
1285         (FLOATUORS:VDQF
1286           (match_operand:<FCVT_TARGET> 1 "register_operand" "w")))]
1287   "TARGET_SIMD"
1288   "<su_optab>cvtf\\t%0.<Vtype>, %1.<Vtype>"
1289   [(set_attr "simd_type" "simd_icvtf")
1290    (set_attr "simd_mode" "<MODE>")]
1291 )
1292
1293 ;; Conversions between vectors of floats and doubles.
1294 ;; Contains a mix of patterns to match standard pattern names
1295 ;; and those for intrinsics.
1296
1297 ;; Float widening operations.
1298
1299 (define_insn "vec_unpacks_lo_v4sf"
1300   [(set (match_operand:V2DF 0 "register_operand" "=w")
1301         (float_extend:V2DF
1302           (vec_select:V2SF
1303             (match_operand:V4SF 1 "register_operand" "w")
1304             (parallel [(const_int 0) (const_int 1)])
1305           )))]
1306   "TARGET_SIMD"
1307   "fcvtl\\t%0.2d, %1.2s"
1308   [(set_attr "simd_type" "simd_fcvtl")
1309    (set_attr "simd_mode" "V2DF")]
1310 )
1311
1312 (define_insn "aarch64_float_extend_lo_v2df"
1313   [(set (match_operand:V2DF 0 "register_operand" "=w")
1314         (float_extend:V2DF
1315           (match_operand:V2SF 1 "register_operand" "w")))]
1316   "TARGET_SIMD"
1317   "fcvtl\\t%0.2d, %1.2s"
1318   [(set_attr "simd_type" "simd_fcvtl")
1319    (set_attr "simd_mode" "V2DF")]
1320 )
1321
1322 (define_insn "vec_unpacks_hi_v4sf"
1323   [(set (match_operand:V2DF 0 "register_operand" "=w")
1324         (float_extend:V2DF
1325           (vec_select:V2SF
1326             (match_operand:V4SF 1 "register_operand" "w")
1327             (parallel [(const_int 2) (const_int 3)])
1328           )))]
1329   "TARGET_SIMD"
1330   "fcvtl2\\t%0.2d, %1.4s"
1331   [(set_attr "simd_type" "simd_fcvtl")
1332    (set_attr "simd_mode" "V2DF")]
1333 )
1334
1335 ;; Float narrowing operations.
1336
1337 (define_insn "aarch64_float_truncate_lo_v2sf"
1338   [(set (match_operand:V2SF 0 "register_operand" "=w")
1339       (float_truncate:V2SF
1340         (match_operand:V2DF 1 "register_operand" "w")))]
1341   "TARGET_SIMD"
1342   "fcvtn\\t%0.2s, %1.2d"
1343   [(set_attr "simd_type" "simd_fcvtl")
1344    (set_attr "simd_mode" "V2SF")]
1345 )
1346
1347 (define_insn "aarch64_float_truncate_hi_v4sf"
1348   [(set (match_operand:V4SF 0 "register_operand" "=w")
1349     (vec_concat:V4SF
1350       (match_operand:V2SF 1 "register_operand" "0")
1351       (float_truncate:V2SF
1352         (match_operand:V2DF 2 "register_operand" "w"))))]
1353   "TARGET_SIMD"
1354   "fcvtn2\\t%0.4s, %2.2d"
1355   [(set_attr "simd_type" "simd_fcvtl")
1356    (set_attr "simd_mode" "V4SF")]
1357 )
1358
1359 (define_expand "vec_pack_trunc_v2df"
1360   [(set (match_operand:V4SF 0 "register_operand")
1361       (vec_concat:V4SF
1362         (float_truncate:V2SF
1363             (match_operand:V2DF 1 "register_operand"))
1364         (float_truncate:V2SF
1365             (match_operand:V2DF 2 "register_operand"))
1366           ))]
1367   "TARGET_SIMD"
1368   {
1369     rtx tmp = gen_reg_rtx (V2SFmode);
1370     emit_insn (gen_aarch64_float_truncate_lo_v2sf (tmp, operands[1]));
1371     emit_insn (gen_aarch64_float_truncate_hi_v4sf (operands[0],
1372                                                    tmp, operands[2]));
1373     DONE;
1374   }
1375 )
1376
1377 (define_expand "vec_pack_trunc_df"
1378   [(set (match_operand:V2SF 0 "register_operand")
1379       (vec_concat:V2SF
1380         (float_truncate:SF
1381             (match_operand:DF 1 "register_operand"))
1382         (float_truncate:SF
1383             (match_operand:DF 2 "register_operand"))
1384           ))]
1385   "TARGET_SIMD"
1386   {
1387     rtx tmp = gen_reg_rtx (V2SFmode);
1388     emit_insn (gen_move_lo_quad_v2df (tmp, operands[1]));
1389     emit_insn (gen_move_hi_quad_v2df (tmp, operands[2]));
1390     emit_insn (gen_aarch64_float_truncate_lo_v2sf (operands[0], tmp));
1391     DONE;
1392   }
1393 )
1394
1395 (define_insn "aarch64_vmls<mode>"
1396   [(set (match_operand:VDQF 0 "register_operand" "=w")
1397        (minus:VDQF (match_operand:VDQF 1 "register_operand" "0")
1398                    (mult:VDQF (match_operand:VDQF 2 "register_operand" "w")
1399                               (match_operand:VDQF 3 "register_operand" "w"))))]
1400   "TARGET_SIMD"
1401  "fmls\\t%0.<Vtype>, %2.<Vtype>, %3.<Vtype>"
1402   [(set_attr "simd_type" "simd_fmla")
1403    (set_attr "simd_mode" "<MODE>")]
1404 )
1405
1406 ;; FP Max/Min
1407 ;; Max/Min are introduced by idiom recognition by GCC's mid-end.  An
1408 ;; expression like:
1409 ;;      a = (b < c) ? b : c;
1410 ;; is idiom-matched as MIN_EXPR<b,c> only if -ffinite-math-only is enabled
1411 ;; either explicitly or indirectly via -ffast-math.
1412 ;;
1413 ;; MIN_EXPR and MAX_EXPR eventually map to 'smin' and 'smax' in RTL.
1414 ;; The 'smax' and 'smin' RTL standard pattern names do not specify which
1415 ;; operand will be returned when both operands are zero (i.e. they may not
1416 ;; honour signed zeroes), or when either operand is NaN.  Therefore GCC
1417 ;; only introduces MIN_EXPR/MAX_EXPR in fast math mode or when not honouring
1418 ;; NaNs.
1419
1420 (define_insn "<su><maxmin><mode>3"
1421   [(set (match_operand:VDQF 0 "register_operand" "=w")
1422         (FMAXMIN:VDQF (match_operand:VDQF 1 "register_operand" "w")
1423                    (match_operand:VDQF 2 "register_operand" "w")))]
1424   "TARGET_SIMD"
1425   "f<maxmin>nm\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1426   [(set_attr "simd_type" "simd_fminmax")
1427    (set_attr "simd_mode" "<MODE>")]
1428 )
1429
1430 (define_insn "<maxmin_uns><mode>3"
1431   [(set (match_operand:VDQF 0 "register_operand" "=w")
1432        (unspec:VDQF [(match_operand:VDQF 1 "register_operand" "w")
1433                      (match_operand:VDQF 2 "register_operand" "w")]
1434                     FMAXMIN_UNS))]
1435   "TARGET_SIMD"
1436   "<maxmin_uns_op>\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1437   [(set_attr "simd_type" "simd_fminmax")
1438    (set_attr "simd_mode" "<MODE>")]
1439 )
1440
1441 ;; 'across lanes' add.
1442
1443 (define_insn "reduc_<sur>plus_<mode>"
1444  [(set (match_operand:VDQV 0 "register_operand" "=w")
1445        (unspec:VDQV [(match_operand:VDQV 1 "register_operand" "w")]
1446                     SUADDV))]
1447  "TARGET_SIMD"
1448  "addv\\t%<Vetype>0, %1.<Vtype>"
1449   [(set_attr "simd_type" "simd_addv")
1450    (set_attr "simd_mode" "<MODE>")]
1451 )
1452
1453 (define_insn "reduc_<sur>plus_v2di"
1454  [(set (match_operand:V2DI 0 "register_operand" "=w")
1455        (unspec:V2DI [(match_operand:V2DI 1 "register_operand" "w")]
1456                     SUADDV))]
1457  "TARGET_SIMD"
1458  "addp\\t%d0, %1.2d"
1459   [(set_attr "simd_type" "simd_addv")
1460    (set_attr "simd_mode" "V2DI")]
1461 )
1462
1463 (define_insn "reduc_<sur>plus_v2si"
1464  [(set (match_operand:V2SI 0 "register_operand" "=w")
1465        (unspec:V2SI [(match_operand:V2SI 1 "register_operand" "w")]
1466                     SUADDV))]
1467  "TARGET_SIMD"
1468  "addp\\t%0.2s, %1.2s, %1.2s"
1469   [(set_attr "simd_type" "simd_addv")
1470    (set_attr "simd_mode" "V2SI")]
1471 )
1472
1473 (define_insn "reduc_<sur>plus_<mode>"
1474  [(set (match_operand:V2F 0 "register_operand" "=w")
1475        (unspec:V2F [(match_operand:V2F 1 "register_operand" "w")]
1476                     SUADDV))]
1477  "TARGET_SIMD"
1478  "faddp\\t%<Vetype>0, %1.<Vtype>"
1479   [(set_attr "simd_type" "simd_fadd")
1480    (set_attr "simd_mode" "<MODE>")]
1481 )
1482
1483 (define_insn "aarch64_addpv4sf"
1484  [(set (match_operand:V4SF 0 "register_operand" "=w")
1485        (unspec:V4SF [(match_operand:V4SF 1 "register_operand" "w")]
1486                     UNSPEC_FADDV))]
1487  "TARGET_SIMD"
1488  "faddp\\t%0.4s, %1.4s, %1.4s"
1489   [(set_attr "simd_type" "simd_fadd")
1490    (set_attr "simd_mode" "V4SF")]
1491 )
1492
1493 (define_expand "reduc_<sur>plus_v4sf"
1494  [(set (match_operand:V4SF 0 "register_operand")
1495        (unspec:V4SF [(match_operand:V4SF 1 "register_operand")]
1496                     SUADDV))]
1497  "TARGET_SIMD"
1498 {
1499   rtx tmp = gen_reg_rtx (V4SFmode);
1500   emit_insn (gen_aarch64_addpv4sf (tmp, operands[1]));
1501   emit_insn (gen_aarch64_addpv4sf (operands[0], tmp));
1502   DONE;
1503 })
1504
1505 ;; 'across lanes' max and min ops.
1506
1507 (define_insn "reduc_<maxmin_uns>_<mode>"
1508  [(set (match_operand:VDQV 0 "register_operand" "=w")
1509        (unspec:VDQV [(match_operand:VDQV 1 "register_operand" "w")]
1510                     MAXMINV))]
1511  "TARGET_SIMD"
1512  "<maxmin_uns_op>v\\t%<Vetype>0, %1.<Vtype>"
1513   [(set_attr "simd_type" "simd_minmaxv")
1514    (set_attr "simd_mode" "<MODE>")]
1515 )
1516
1517 (define_insn "reduc_<maxmin_uns>_v2di"
1518  [(set (match_operand:V2DI 0 "register_operand" "=w")
1519        (unspec:V2DI [(match_operand:V2DI 1 "register_operand" "w")]
1520                     MAXMINV))]
1521  "TARGET_SIMD"
1522  "<maxmin_uns_op>p\\t%d0, %1.2d"
1523   [(set_attr "simd_type" "simd_minmaxv")
1524    (set_attr "simd_mode" "V2DI")]
1525 )
1526
1527 (define_insn "reduc_<maxmin_uns>_v2si"
1528  [(set (match_operand:V2SI 0 "register_operand" "=w")
1529        (unspec:V2SI [(match_operand:V2SI 1 "register_operand" "w")]
1530                     MAXMINV))]
1531  "TARGET_SIMD"
1532  "<maxmin_uns_op>p\\t%0.2s, %1.2s, %1.2s"
1533   [(set_attr "simd_type" "simd_minmaxv")
1534    (set_attr "simd_mode" "V2SI")]
1535 )
1536
1537 (define_insn "reduc_<maxmin_uns>_<mode>"
1538  [(set (match_operand:V2F 0 "register_operand" "=w")
1539        (unspec:V2F [(match_operand:V2F 1 "register_operand" "w")]
1540                     FMAXMINV))]
1541  "TARGET_SIMD"
1542  "<maxmin_uns_op>p\\t%<Vetype>0, %1.<Vtype>"
1543   [(set_attr "simd_type" "simd_fminmaxv")
1544    (set_attr "simd_mode" "<MODE>")]
1545 )
1546
1547 (define_insn "reduc_<maxmin_uns>_v4sf"
1548  [(set (match_operand:V4SF 0 "register_operand" "=w")
1549        (unspec:V4SF [(match_operand:V4SF 1 "register_operand" "w")]
1550                     FMAXMINV))]
1551  "TARGET_SIMD"
1552  "<maxmin_uns_op>v\\t%s0, %1.4s"
1553   [(set_attr "simd_type" "simd_fminmaxv")
1554    (set_attr "simd_mode" "V4SF")]
1555 )
1556
1557 ;; aarch64_simd_bsl may compile to any of bsl/bif/bit depending on register
1558 ;; allocation.
1559 ;; Operand 1 is the mask, operands 2 and 3 are the bitfields from which
1560 ;; to select.
1561 ;;
1562 ;; Thus our BSL is of the form:
1563 ;;   op0 = bsl (mask, op2, op3)
1564 ;; We can use any of:
1565 ;;
1566 ;;   if (op0 = mask)
1567 ;;     bsl mask, op1, op2
1568 ;;   if (op0 = op1) (so 1-bits in mask choose bits from op2, else op0)
1569 ;;     bit op0, op2, mask
1570 ;;   if (op0 = op2) (so 0-bits in mask choose bits from op1, else op0)
1571 ;;     bif op0, op1, mask
1572
1573 (define_insn "aarch64_simd_bsl<mode>_internal"
1574   [(set (match_operand:VALL 0 "register_operand"                "=w,w,w")
1575         (ior:VALL
1576            (and:VALL
1577              (match_operand:<V_cmp_result> 1 "register_operand" " 0,w,w")
1578              (match_operand:VALL 2 "register_operand"           " w,w,0"))
1579            (and:VALL
1580              (not:<V_cmp_result>
1581                 (match_dup:<V_cmp_result> 1))
1582              (match_operand:VALL 3 "register_operand"           " w,0,w"))
1583         ))]
1584   "TARGET_SIMD"
1585   "@
1586   bsl\\t%0.<Vbtype>, %2.<Vbtype>, %3.<Vbtype>
1587   bit\\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>
1588   bif\\t%0.<Vbtype>, %3.<Vbtype>, %1.<Vbtype>"
1589 )
1590
1591 (define_expand "aarch64_simd_bsl<mode>"
1592   [(match_operand:VALL 0 "register_operand")
1593    (match_operand:<V_cmp_result> 1 "register_operand")
1594    (match_operand:VALL 2 "register_operand")
1595    (match_operand:VALL 3 "register_operand")]
1596  "TARGET_SIMD"
1597 {
1598   /* We can't alias operands together if they have different modes.  */
1599   operands[1] = gen_lowpart (<V_cmp_result>mode, operands[1]);
1600   emit_insn (gen_aarch64_simd_bsl<mode>_internal (operands[0], operands[1],
1601                                                   operands[2], operands[3]));
1602   DONE;
1603 })
1604
1605 (define_expand "aarch64_vcond_internal<mode>"
1606   [(set (match_operand:VDQ 0 "register_operand")
1607         (if_then_else:VDQ
1608           (match_operator 3 "comparison_operator"
1609             [(match_operand:VDQ 4 "register_operand")
1610              (match_operand:VDQ 5 "nonmemory_operand")])
1611           (match_operand:VDQ 1 "nonmemory_operand")
1612           (match_operand:VDQ 2 "nonmemory_operand")))]
1613   "TARGET_SIMD"
1614 {
1615   int inverse = 0, has_zero_imm_form = 0;
1616   rtx op1 = operands[1];
1617   rtx op2 = operands[2];
1618   rtx mask = gen_reg_rtx (<MODE>mode);
1619
1620   switch (GET_CODE (operands[3]))
1621     {
1622     case LE:
1623     case LT:
1624     case NE:
1625       inverse = 1;
1626       /* Fall through.  */
1627     case GE:
1628     case GT:
1629     case EQ:
1630       has_zero_imm_form = 1;
1631       break;
1632     case LEU:
1633     case LTU:
1634       inverse = 1;
1635       break;
1636     default:
1637       break;
1638     }
1639
1640   if (!REG_P (operands[5])
1641       && (operands[5] != CONST0_RTX (<MODE>mode) || !has_zero_imm_form))
1642     operands[5] = force_reg (<MODE>mode, operands[5]);
1643
1644   switch (GET_CODE (operands[3]))
1645     {
1646     case LT:
1647     case GE:
1648       emit_insn (gen_aarch64_cmge<mode> (mask, operands[4], operands[5]));
1649       break;
1650
1651     case LE:
1652     case GT:
1653       emit_insn (gen_aarch64_cmgt<mode> (mask, operands[4], operands[5]));
1654       break;
1655
1656     case LTU:
1657     case GEU:
1658       emit_insn (gen_aarch64_cmgeu<mode> (mask, operands[4], operands[5]));
1659       break;
1660
1661     case LEU:
1662     case GTU:
1663       emit_insn (gen_aarch64_cmgtu<mode> (mask, operands[4], operands[5]));
1664       break;
1665
1666     case NE:
1667     case EQ:
1668       emit_insn (gen_aarch64_cmeq<mode> (mask, operands[4], operands[5]));
1669       break;
1670
1671     default:
1672       gcc_unreachable ();
1673     }
1674
1675   if (inverse)
1676     {
1677       op1 = operands[2];
1678       op2 = operands[1];
1679     }
1680
1681     /* If we have (a = (b CMP c) ? -1 : 0);
1682        Then we can simply move the generated mask.  */
1683
1684     if (op1 == CONSTM1_RTX (<V_cmp_result>mode)
1685         && op2 == CONST0_RTX (<V_cmp_result>mode))
1686       emit_move_insn (operands[0], mask);
1687     else
1688       {
1689         if (!REG_P (op1))
1690           op1 = force_reg (<MODE>mode, op1);
1691         if (!REG_P (op2))
1692           op2 = force_reg (<MODE>mode, op2);
1693         emit_insn (gen_aarch64_simd_bsl<mode> (operands[0], mask,
1694                                                op1, op2));
1695       }
1696
1697   DONE;
1698 })
1699
1700 (define_expand "aarch64_vcond_internal<mode>"
1701   [(set (match_operand:VDQF 0 "register_operand")
1702         (if_then_else:VDQF
1703           (match_operator 3 "comparison_operator"
1704             [(match_operand:VDQF 4 "register_operand")
1705              (match_operand:VDQF 5 "nonmemory_operand")])
1706           (match_operand:VDQF 1 "nonmemory_operand")
1707           (match_operand:VDQF 2 "nonmemory_operand")))]
1708   "TARGET_SIMD"
1709 {
1710   int inverse = 0;
1711   int use_zero_form = 0;
1712   int swap_bsl_operands = 0;
1713   rtx op1 = operands[1];
1714   rtx op2 = operands[2];
1715   rtx mask = gen_reg_rtx (<V_cmp_result>mode);
1716   rtx tmp = gen_reg_rtx (<V_cmp_result>mode);
1717
1718   rtx (*base_comparison) (rtx, rtx, rtx);
1719   rtx (*complimentary_comparison) (rtx, rtx, rtx);
1720
1721   switch (GET_CODE (operands[3]))
1722     {
1723     case GE:
1724     case GT:
1725     case LE:
1726     case LT:
1727     case EQ:
1728       if (operands[5] == CONST0_RTX (<MODE>mode))
1729         {
1730           use_zero_form = 1;
1731           break;
1732         }
1733       /* Fall through.  */
1734     default:
1735       if (!REG_P (operands[5]))
1736         operands[5] = force_reg (<MODE>mode, operands[5]);
1737     }
1738
1739   switch (GET_CODE (operands[3]))
1740     {
1741     case LT:
1742     case UNLT:
1743       inverse = 1;
1744       /* Fall through.  */
1745     case GE:
1746     case UNGE:
1747     case ORDERED:
1748     case UNORDERED:
1749       base_comparison = gen_aarch64_cmge<mode>;
1750       complimentary_comparison = gen_aarch64_cmgt<mode>;
1751       break;
1752     case LE:
1753     case UNLE:
1754       inverse = 1;
1755       /* Fall through.  */
1756     case GT:
1757     case UNGT:
1758       base_comparison = gen_aarch64_cmgt<mode>;
1759       complimentary_comparison = gen_aarch64_cmge<mode>;
1760       break;
1761     case EQ:
1762     case NE:
1763     case UNEQ:
1764       base_comparison = gen_aarch64_cmeq<mode>;
1765       complimentary_comparison = gen_aarch64_cmeq<mode>;
1766       break;
1767     default:
1768       gcc_unreachable ();
1769     }
1770
1771   switch (GET_CODE (operands[3]))
1772     {
1773     case LT:
1774     case LE:
1775     case GT:
1776     case GE:
1777     case EQ:
1778       /* The easy case.  Here we emit one of FCMGE, FCMGT or FCMEQ.
1779          As a LT b <=> b GE a && a LE b <=> b GT a.  Our transformations are:
1780          a GE b -> a GE b
1781          a GT b -> a GT b
1782          a LE b -> b GE a
1783          a LT b -> b GT a
1784          a EQ b -> a EQ b
1785          Note that there also exist direct comparison against 0 forms,
1786          so catch those as a special case.  */
1787       if (use_zero_form)
1788         {
1789           inverse = 0;
1790           switch (GET_CODE (operands[3]))
1791             {
1792             case LT:
1793               base_comparison = gen_aarch64_cmlt<mode>;
1794               break;
1795             case LE:
1796               base_comparison = gen_aarch64_cmle<mode>;
1797               break;
1798             default:
1799               /* Do nothing, other zero form cases already have the correct
1800                  base_comparison.  */
1801               break;
1802             }
1803         }
1804
1805       if (!inverse)
1806         emit_insn (base_comparison (mask, operands[4], operands[5]));
1807       else
1808         emit_insn (complimentary_comparison (mask, operands[5], operands[4]));
1809       break;
1810     case UNLT:
1811     case UNLE:
1812     case UNGT:
1813     case UNGE:
1814     case NE:
1815       /* FCM returns false for lanes which are unordered, so if we use
1816          the inverse of the comparison we actually want to emit, then
1817          swap the operands to BSL, we will end up with the correct result.
1818          Note that a NE NaN and NaN NE b are true for all a, b.
1819
1820          Our transformations are:
1821          a GE b -> !(b GT a)
1822          a GT b -> !(b GE a)
1823          a LE b -> !(a GT b)
1824          a LT b -> !(a GE b)
1825          a NE b -> !(a EQ b)  */
1826
1827       if (inverse)
1828         emit_insn (base_comparison (mask, operands[4], operands[5]));
1829       else
1830         emit_insn (complimentary_comparison (mask, operands[5], operands[4]));
1831
1832       swap_bsl_operands = 1;
1833       break;
1834     case UNEQ:
1835       /* We check (a > b ||  b > a).  combining these comparisons give us
1836          true iff !(a != b && a ORDERED b), swapping the operands to BSL
1837          will then give us (a == b ||  a UNORDERED b) as intended.  */
1838
1839       emit_insn (gen_aarch64_cmgt<mode> (mask, operands[4], operands[5]));
1840       emit_insn (gen_aarch64_cmgt<mode> (tmp, operands[5], operands[4]));
1841       emit_insn (gen_ior<v_cmp_result>3 (mask, mask, tmp));
1842       swap_bsl_operands = 1;
1843       break;
1844     case UNORDERED:
1845        /* Operands are ORDERED iff (a > b || b >= a).
1846          Swapping the operands to BSL will give the UNORDERED case.  */
1847      swap_bsl_operands = 1;
1848      /* Fall through.  */
1849     case ORDERED:
1850       emit_insn (gen_aarch64_cmgt<mode> (tmp, operands[4], operands[5]));
1851       emit_insn (gen_aarch64_cmge<mode> (mask, operands[5], operands[4]));
1852       emit_insn (gen_ior<v_cmp_result>3 (mask, mask, tmp));
1853       break;
1854     default:
1855       gcc_unreachable ();
1856     }
1857
1858   if (swap_bsl_operands)
1859     {
1860       op1 = operands[2];
1861       op2 = operands[1];
1862     }
1863
1864     /* If we have (a = (b CMP c) ? -1 : 0);
1865        Then we can simply move the generated mask.  */
1866
1867     if (op1 == CONSTM1_RTX (<V_cmp_result>mode)
1868         && op2 == CONST0_RTX (<V_cmp_result>mode))
1869       emit_move_insn (operands[0], mask);
1870     else
1871       {
1872         if (!REG_P (op1))
1873           op1 = force_reg (<MODE>mode, op1);
1874         if (!REG_P (op2))
1875           op2 = force_reg (<MODE>mode, op2);
1876         emit_insn (gen_aarch64_simd_bsl<mode> (operands[0], mask,
1877                                                op1, op2));
1878       }
1879
1880   DONE;
1881 })
1882
1883 (define_expand "vcond<mode><mode>"
1884   [(set (match_operand:VALL 0 "register_operand")
1885         (if_then_else:VALL
1886           (match_operator 3 "comparison_operator"
1887             [(match_operand:VALL 4 "register_operand")
1888              (match_operand:VALL 5 "nonmemory_operand")])
1889           (match_operand:VALL 1 "nonmemory_operand")
1890           (match_operand:VALL 2 "nonmemory_operand")))]
1891   "TARGET_SIMD"
1892 {
1893   emit_insn (gen_aarch64_vcond_internal<mode> (operands[0], operands[1],
1894                                                operands[2], operands[3],
1895                                                operands[4], operands[5]));
1896   DONE;
1897 })
1898
1899 (define_expand "vcond<v_cmp_result><mode>"
1900   [(set (match_operand:<V_cmp_result> 0 "register_operand")
1901         (if_then_else:<V_cmp_result>
1902           (match_operator 3 "comparison_operator"
1903             [(match_operand:VDQF 4 "register_operand")
1904              (match_operand:VDQF 5 "nonmemory_operand")])
1905           (match_operand:<V_cmp_result> 1 "nonmemory_operand")
1906           (match_operand:<V_cmp_result> 2 "nonmemory_operand")))]
1907   "TARGET_SIMD"
1908 {
1909   emit_insn (gen_aarch64_vcond_internal<v_cmp_result> (
1910                                                 operands[0], operands[1],
1911                                                 operands[2], operands[3],
1912                                                 operands[4], operands[5]));
1913   DONE;
1914 })
1915
1916 (define_expand "vcondu<mode><mode>"
1917   [(set (match_operand:VDQ 0 "register_operand")
1918         (if_then_else:VDQ
1919           (match_operator 3 "comparison_operator"
1920             [(match_operand:VDQ 4 "register_operand")
1921              (match_operand:VDQ 5 "nonmemory_operand")])
1922           (match_operand:VDQ 1 "nonmemory_operand")
1923           (match_operand:VDQ 2 "nonmemory_operand")))]
1924   "TARGET_SIMD"
1925 {
1926   emit_insn (gen_aarch64_vcond_internal<mode> (operands[0], operands[1],
1927                                                operands[2], operands[3],
1928                                                operands[4], operands[5]));
1929   DONE;
1930 })
1931
1932 ;; Patterns for AArch64 SIMD Intrinsics.
1933
1934 (define_expand "aarch64_create<mode>"
1935   [(match_operand:VD_RE 0 "register_operand" "")
1936    (match_operand:DI 1 "general_operand" "")]
1937   "TARGET_SIMD"
1938 {
1939   rtx src = gen_lowpart (<MODE>mode, operands[1]);
1940   emit_move_insn (operands[0], src);
1941   DONE;
1942 })
1943
1944 (define_insn "aarch64_get_lane_signed<mode>"
1945   [(set (match_operand:<VEL> 0 "register_operand" "=r")
1946         (sign_extend:<VEL>
1947           (vec_select:<VEL>
1948             (match_operand:VQ_S 1 "register_operand" "w")
1949             (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
1950   "TARGET_SIMD"
1951   "smov\\t%0, %1.<Vetype>[%2]"
1952   [(set_attr "simd_type" "simd_movgp")
1953    (set_attr "simd_mode" "<MODE>")]
1954 )
1955
1956 (define_insn "aarch64_get_lane_unsigned<mode>"
1957   [(set (match_operand:<VEL> 0 "register_operand" "=r")
1958         (zero_extend:<VEL>
1959           (vec_select:<VEL>
1960             (match_operand:VDQ 1 "register_operand" "w")
1961             (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
1962   "TARGET_SIMD"
1963   "umov\\t%<vw>0, %1.<Vetype>[%2]"
1964   [(set_attr "simd_type" "simd_movgp")
1965    (set_attr "simd_mode" "<MODE>")]
1966 )
1967
1968 (define_insn "aarch64_get_lane<mode>"
1969   [(set (match_operand:<VEL> 0 "register_operand" "=w")
1970         (vec_select:<VEL>
1971             (match_operand:VDQF 1 "register_operand" "w")
1972             (parallel [(match_operand:SI 2 "immediate_operand" "i")])))]
1973   "TARGET_SIMD"
1974   "mov\\t%0.<Vetype>[0], %1.<Vetype>[%2]"
1975   [(set_attr "simd_type" "simd_ins")
1976    (set_attr "simd_mode" "<MODE>")]
1977 )
1978
1979 (define_expand "aarch64_get_lanedi"
1980   [(match_operand:DI 0 "register_operand" "=r")
1981    (match_operand:DI 1 "register_operand" "w")
1982    (match_operand:SI 2 "immediate_operand" "i")]
1983   "TARGET_SIMD"
1984 {
1985   aarch64_simd_lane_bounds (operands[2], 0, 1);
1986   emit_move_insn (operands[0], operands[1]);
1987   DONE;
1988 })
1989
1990 (define_expand "aarch64_reinterpretv8qi<mode>"
1991   [(match_operand:V8QI 0 "register_operand" "")
1992    (match_operand:VDC 1 "register_operand" "")]
1993   "TARGET_SIMD"
1994 {
1995   aarch64_simd_reinterpret (operands[0], operands[1]);
1996   DONE;
1997 })
1998
1999 (define_expand "aarch64_reinterpretv4hi<mode>"
2000   [(match_operand:V4HI 0 "register_operand" "")
2001    (match_operand:VDC 1 "register_operand" "")]
2002   "TARGET_SIMD"
2003 {
2004   aarch64_simd_reinterpret (operands[0], operands[1]);
2005   DONE;
2006 })
2007
2008 (define_expand "aarch64_reinterpretv2si<mode>"
2009   [(match_operand:V2SI 0 "register_operand" "")
2010    (match_operand:VDC 1 "register_operand" "")]
2011   "TARGET_SIMD"
2012 {
2013   aarch64_simd_reinterpret (operands[0], operands[1]);
2014   DONE;
2015 })
2016
2017 (define_expand "aarch64_reinterpretv2sf<mode>"
2018   [(match_operand:V2SF 0 "register_operand" "")
2019    (match_operand:VDC 1 "register_operand" "")]
2020   "TARGET_SIMD"
2021 {
2022   aarch64_simd_reinterpret (operands[0], operands[1]);
2023   DONE;
2024 })
2025
2026 (define_expand "aarch64_reinterpretdi<mode>"
2027   [(match_operand:DI 0 "register_operand" "")
2028    (match_operand:VD_RE 1 "register_operand" "")]
2029   "TARGET_SIMD"
2030 {
2031   aarch64_simd_reinterpret (operands[0], operands[1]);
2032   DONE;
2033 })
2034
2035 (define_expand "aarch64_reinterpretv16qi<mode>"
2036   [(match_operand:V16QI 0 "register_operand" "")
2037    (match_operand:VQ 1 "register_operand" "")]
2038   "TARGET_SIMD"
2039 {
2040   aarch64_simd_reinterpret (operands[0], operands[1]);
2041   DONE;
2042 })
2043
2044 (define_expand "aarch64_reinterpretv8hi<mode>"
2045   [(match_operand:V8HI 0 "register_operand" "")
2046    (match_operand:VQ 1 "register_operand" "")]
2047   "TARGET_SIMD"
2048 {
2049   aarch64_simd_reinterpret (operands[0], operands[1]);
2050   DONE;
2051 })
2052
2053 (define_expand "aarch64_reinterpretv4si<mode>"
2054   [(match_operand:V4SI 0 "register_operand" "")
2055    (match_operand:VQ 1 "register_operand" "")]
2056   "TARGET_SIMD"
2057 {
2058   aarch64_simd_reinterpret (operands[0], operands[1]);
2059   DONE;
2060 })
2061
2062 (define_expand "aarch64_reinterpretv4sf<mode>"
2063   [(match_operand:V4SF 0 "register_operand" "")
2064    (match_operand:VQ 1 "register_operand" "")]
2065   "TARGET_SIMD"
2066 {
2067   aarch64_simd_reinterpret (operands[0], operands[1]);
2068   DONE;
2069 })
2070
2071 (define_expand "aarch64_reinterpretv2di<mode>"
2072   [(match_operand:V2DI 0 "register_operand" "")
2073    (match_operand:VQ 1 "register_operand" "")]
2074   "TARGET_SIMD"
2075 {
2076   aarch64_simd_reinterpret (operands[0], operands[1]);
2077   DONE;
2078 })
2079
2080 (define_expand "aarch64_reinterpretv2df<mode>"
2081   [(match_operand:V2DF 0 "register_operand" "")
2082    (match_operand:VQ 1 "register_operand" "")]
2083   "TARGET_SIMD"
2084 {
2085   aarch64_simd_reinterpret (operands[0], operands[1]);
2086   DONE;
2087 })
2088
2089 ;; In this insn, operand 1 should be low, and operand 2 the high part of the
2090 ;; dest vector.
2091
2092 (define_insn "*aarch64_combinez<mode>"
2093   [(set (match_operand:<VDBL> 0 "register_operand" "=&w")
2094         (vec_concat:<VDBL>
2095            (match_operand:VDIC 1 "register_operand" "w")
2096            (match_operand:VDIC 2 "aarch64_simd_imm_zero" "Dz")))]
2097   "TARGET_SIMD"
2098   "mov\\t%0.8b, %1.8b"
2099   [(set_attr "simd_type" "simd_move")
2100    (set_attr "simd_mode" "<MODE>")]
2101 )
2102
2103 (define_insn "aarch64_combine<mode>"
2104   [(set (match_operand:<VDBL> 0 "register_operand" "=&w")
2105         (vec_concat:<VDBL> (match_operand:VDC 1 "register_operand" "w")
2106                            (match_operand:VDC 2 "register_operand" "w")))]
2107   "TARGET_SIMD"
2108   "mov\\t%0.d[0], %1.d[0]\;ins\\t%0.d[1], %2.d[0]"
2109   [(set_attr "simd_type" "simd_ins")
2110    (set_attr "simd_mode" "<MODE>")]
2111 )
2112
2113 ;; <su><addsub>l<q>.
2114
2115 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>l2<mode>_internal"
2116  [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2117        (ADDSUB:<VWIDE> (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
2118                            (match_operand:VQW 1 "register_operand" "w")
2119                            (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))
2120                        (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
2121                            (match_operand:VQW 2 "register_operand" "w")
2122                            (match_dup 3)))))]
2123   "TARGET_SIMD"
2124   "<ANY_EXTEND:su><ADDSUB:optab>l2 %0.<Vwtype>, %1.<Vtype>, %2.<Vtype>"
2125   [(set_attr "simd_type" "simd_addl")
2126    (set_attr "simd_mode" "<MODE>")]
2127 )
2128
2129 (define_expand "aarch64_saddl2<mode>"
2130   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2131    (match_operand:VQW 1 "register_operand" "w")
2132    (match_operand:VQW 2 "register_operand" "w")]
2133   "TARGET_SIMD"
2134 {
2135   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2136   emit_insn (gen_aarch64_saddl2<mode>_internal (operands[0], operands[1],
2137                                                 operands[2], p));
2138   DONE;
2139 })
2140
2141 (define_expand "aarch64_uaddl2<mode>"
2142   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2143    (match_operand:VQW 1 "register_operand" "w")
2144    (match_operand:VQW 2 "register_operand" "w")]
2145   "TARGET_SIMD"
2146 {
2147   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2148   emit_insn (gen_aarch64_uaddl2<mode>_internal (operands[0], operands[1],
2149                                                 operands[2], p));
2150   DONE;
2151 })
2152
2153 (define_expand "aarch64_ssubl2<mode>"
2154   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2155    (match_operand:VQW 1 "register_operand" "w")
2156    (match_operand:VQW 2 "register_operand" "w")]
2157   "TARGET_SIMD"
2158 {
2159   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2160   emit_insn (gen_aarch64_ssubl2<mode>_internal (operands[0], operands[1],
2161                                                 operands[2], p));
2162   DONE;
2163 })
2164
2165 (define_expand "aarch64_usubl2<mode>"
2166   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2167    (match_operand:VQW 1 "register_operand" "w")
2168    (match_operand:VQW 2 "register_operand" "w")]
2169   "TARGET_SIMD"
2170 {
2171   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2172   emit_insn (gen_aarch64_usubl2<mode>_internal (operands[0], operands[1],
2173                                                 operands[2], p));
2174   DONE;
2175 })
2176
2177 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>l<mode>"
2178  [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2179        (ADDSUB:<VWIDE> (ANY_EXTEND:<VWIDE>
2180                            (match_operand:VDW 1 "register_operand" "w"))
2181                        (ANY_EXTEND:<VWIDE>
2182                            (match_operand:VDW 2 "register_operand" "w"))))]
2183   "TARGET_SIMD"
2184   "<ANY_EXTEND:su><ADDSUB:optab>l %0.<Vwtype>, %1.<Vtype>, %2.<Vtype>"
2185   [(set_attr "simd_type" "simd_addl")
2186    (set_attr "simd_mode" "<MODE>")]
2187 )
2188
2189 ;; <su><addsub>w<q>.
2190
2191 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>w<mode>"
2192   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2193         (ADDSUB:<VWIDE> (match_operand:<VWIDE> 1 "register_operand" "w")
2194                         (ANY_EXTEND:<VWIDE>
2195                           (match_operand:VDW 2 "register_operand" "w"))))]
2196   "TARGET_SIMD"
2197   "<ANY_EXTEND:su><ADDSUB:optab>w\\t%0.<Vwtype>, %1.<Vwtype>, %2.<Vtype>"
2198   [(set_attr "simd_type" "simd_addl")
2199    (set_attr "simd_mode" "<MODE>")]
2200 )
2201
2202 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>w2<mode>_internal"
2203   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2204         (ADDSUB:<VWIDE> (match_operand:<VWIDE> 1 "register_operand" "w")
2205                         (ANY_EXTEND:<VWIDE>
2206                           (vec_select:<VHALF>
2207                            (match_operand:VQW 2 "register_operand" "w")
2208                            (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))))]
2209   "TARGET_SIMD"
2210   "<ANY_EXTEND:su><ADDSUB:optab>w2\\t%0.<Vwtype>, %1.<Vwtype>, %2.<Vtype>"
2211   [(set_attr "simd_type" "simd_addl")
2212    (set_attr "simd_mode" "<MODE>")]
2213 )
2214
2215 (define_expand "aarch64_saddw2<mode>"
2216   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2217    (match_operand:<VWIDE> 1 "register_operand" "w")
2218    (match_operand:VQW 2 "register_operand" "w")]
2219   "TARGET_SIMD"
2220 {
2221   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2222   emit_insn (gen_aarch64_saddw2<mode>_internal (operands[0], operands[1],
2223                                                 operands[2], p));
2224   DONE;
2225 })
2226
2227 (define_expand "aarch64_uaddw2<mode>"
2228   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2229    (match_operand:<VWIDE> 1 "register_operand" "w")
2230    (match_operand:VQW 2 "register_operand" "w")]
2231   "TARGET_SIMD"
2232 {
2233   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2234   emit_insn (gen_aarch64_uaddw2<mode>_internal (operands[0], operands[1],
2235                                                 operands[2], p));
2236   DONE;
2237 })
2238
2239
2240 (define_expand "aarch64_ssubw2<mode>"
2241   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2242    (match_operand:<VWIDE> 1 "register_operand" "w")
2243    (match_operand:VQW 2 "register_operand" "w")]
2244   "TARGET_SIMD"
2245 {
2246   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2247   emit_insn (gen_aarch64_ssubw2<mode>_internal (operands[0], operands[1],
2248                                                 operands[2], p));
2249   DONE;
2250 })
2251
2252 (define_expand "aarch64_usubw2<mode>"
2253   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2254    (match_operand:<VWIDE> 1 "register_operand" "w")
2255    (match_operand:VQW 2 "register_operand" "w")]
2256   "TARGET_SIMD"
2257 {
2258   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2259   emit_insn (gen_aarch64_usubw2<mode>_internal (operands[0], operands[1],
2260                                                 operands[2], p));
2261   DONE;
2262 })
2263
2264 ;; <su><r>h<addsub>.
2265
2266 (define_insn "aarch64_<sur>h<addsub><mode>"
2267   [(set (match_operand:VQ_S 0 "register_operand" "=w")
2268         (unspec:VQ_S [(match_operand:VQ_S 1 "register_operand" "w")
2269                       (match_operand:VQ_S 2 "register_operand" "w")]
2270                      HADDSUB))]
2271   "TARGET_SIMD"
2272   "<sur>h<addsub>\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
2273   [(set_attr "simd_type" "simd_add")
2274    (set_attr "simd_mode" "<MODE>")]
2275 )
2276
2277 ;; <r><addsub>hn<q>.
2278
2279 (define_insn "aarch64_<sur><addsub>hn<mode>"
2280   [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
2281         (unspec:<VNARROWQ> [(match_operand:VQN 1 "register_operand" "w")
2282                             (match_operand:VQN 2 "register_operand" "w")]
2283                            ADDSUBHN))]
2284   "TARGET_SIMD"
2285   "<sur><addsub>hn\\t%0.<Vntype>, %1.<Vtype>, %2.<Vtype>"
2286   [(set_attr "simd_type" "simd_addn")
2287    (set_attr "simd_mode" "<MODE>")]
2288 )
2289
2290 (define_insn "aarch64_<sur><addsub>hn2<mode>"
2291   [(set (match_operand:<VNARROWQ2> 0 "register_operand" "=w")
2292         (unspec:<VNARROWQ2> [(match_operand:<VNARROWQ> 1 "register_operand" "0")
2293                              (match_operand:VQN 2 "register_operand" "w")
2294                              (match_operand:VQN 3 "register_operand" "w")]
2295                             ADDSUBHN2))]
2296   "TARGET_SIMD"
2297   "<sur><addsub>hn2\\t%0.<V2ntype>, %2.<Vtype>, %3.<Vtype>"
2298   [(set_attr "simd_type" "simd_addn2")
2299    (set_attr "simd_mode" "<MODE>")]
2300 )
2301
2302 ;; pmul.
2303
2304 (define_insn "aarch64_pmul<mode>"
2305   [(set (match_operand:VB 0 "register_operand" "=w")
2306         (unspec:VB [(match_operand:VB 1 "register_operand" "w")
2307                     (match_operand:VB 2 "register_operand" "w")]
2308                    UNSPEC_PMUL))]
2309  "TARGET_SIMD"
2310  "pmul\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
2311   [(set_attr "simd_type" "simd_mul")
2312    (set_attr "simd_mode" "<MODE>")]
2313 )
2314
2315 ;; <su>q<addsub>
2316
2317 (define_insn "aarch64_<su_optab><optab><mode>"
2318   [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
2319         (BINQOPS:VSDQ_I (match_operand:VSDQ_I 1 "register_operand" "w")
2320                           (match_operand:VSDQ_I 2 "register_operand" "w")))]
2321   "TARGET_SIMD"
2322   "<su_optab><optab>\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
2323   [(set_attr "simd_type" "simd_add")
2324    (set_attr "simd_mode" "<MODE>")]
2325 )
2326
2327 ;; suqadd and usqadd
2328
2329 (define_insn "aarch64_<sur>qadd<mode>"
2330   [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
2331         (unspec:VSDQ_I [(match_operand:VSDQ_I 1 "register_operand" "0")
2332                         (match_operand:VSDQ_I 2 "register_operand" "w")]
2333                        USSUQADD))]
2334   "TARGET_SIMD"
2335   "<sur>qadd\\t%<v>0<Vmtype>, %<v>2<Vmtype>"
2336   [(set_attr "simd_type" "simd_sat_add")
2337    (set_attr "simd_mode" "<MODE>")]
2338 )
2339
2340 ;; sqmovun
2341
2342 (define_insn "aarch64_sqmovun<mode>"
2343   [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
2344         (unspec:<VNARROWQ> [(match_operand:VSQN_HSDI 1 "register_operand" "w")]
2345                             UNSPEC_SQXTUN))]
2346    "TARGET_SIMD"
2347    "sqxtun\\t%<vn2>0<Vmntype>, %<v>1<Vmtype>"
2348    [(set_attr "simd_type" "simd_sat_shiftn_imm")
2349     (set_attr "simd_mode" "<MODE>")]
2350  )
2351
2352 ;; sqmovn and uqmovn
2353
2354 (define_insn "aarch64_<sur>qmovn<mode>"
2355   [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
2356         (unspec:<VNARROWQ> [(match_operand:VSQN_HSDI 1 "register_operand" "w")]
2357                             SUQMOVN))]
2358   "TARGET_SIMD"
2359   "<sur>qxtn\\t%<vn2>0<Vmntype>, %<v>1<Vmtype>"
2360    [(set_attr "simd_type" "simd_sat_shiftn_imm")
2361     (set_attr "simd_mode" "<MODE>")]
2362  )
2363
2364 ;; <su>q<absneg>
2365
2366 (define_insn "aarch64_s<optab><mode>"
2367   [(set (match_operand:VSDQ_I_BHSI 0 "register_operand" "=w")
2368         (UNQOPS:VSDQ_I_BHSI
2369           (match_operand:VSDQ_I_BHSI 1 "register_operand" "w")))]
2370   "TARGET_SIMD"
2371   "s<optab>\\t%<v>0<Vmtype>, %<v>1<Vmtype>"
2372   [(set_attr "simd_type" "simd_sat_negabs")
2373    (set_attr "simd_mode" "<MODE>")]
2374 )
2375
2376 ;; sq<r>dmulh.
2377
2378 (define_insn "aarch64_sq<r>dmulh<mode>"
2379   [(set (match_operand:VSDQ_HSI 0 "register_operand" "=w")
2380         (unspec:VSDQ_HSI
2381           [(match_operand:VSDQ_HSI 1 "register_operand" "w")
2382            (match_operand:VSDQ_HSI 2 "register_operand" "w")]
2383          VQDMULH))]
2384   "TARGET_SIMD"
2385   "sq<r>dmulh\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
2386   [(set_attr "simd_type" "simd_sat_mul")
2387    (set_attr "simd_mode" "<MODE>")]
2388 )
2389
2390 ;; sq<r>dmulh_lane
2391
2392 (define_insn "aarch64_sq<r>dmulh_lane<mode>"
2393   [(set (match_operand:VDQHS 0 "register_operand" "=w")
2394         (unspec:VDQHS
2395           [(match_operand:VDQHS 1 "register_operand" "w")
2396            (vec_select:<VEL>
2397              (match_operand:<VCOND> 2 "register_operand" "<vwx>")
2398              (parallel [(match_operand:SI 3 "immediate_operand" "i")]))]
2399          VQDMULH))]
2400   "TARGET_SIMD"
2401   "*
2402    aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCOND>mode));
2403    return \"sq<r>dmulh\\t%0.<Vtype>, %1.<Vtype>, %2.<Vetype>[%3]\";"
2404   [(set_attr "simd_type" "simd_sat_mul")
2405    (set_attr "simd_mode" "<MODE>")]
2406 )
2407
2408 (define_insn "aarch64_sq<r>dmulh_laneq<mode>"
2409   [(set (match_operand:VDQHS 0 "register_operand" "=w")
2410         (unspec:VDQHS
2411           [(match_operand:VDQHS 1 "register_operand" "w")
2412            (vec_select:<VEL>
2413              (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
2414              (parallel [(match_operand:SI 3 "immediate_operand" "i")]))]
2415          VQDMULH))]
2416   "TARGET_SIMD"
2417   "*
2418    aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCONQ>mode));
2419    return \"sq<r>dmulh\\t%0.<Vtype>, %1.<Vtype>, %2.<Vetype>[%3]\";"
2420   [(set_attr "simd_type" "simd_sat_mul")
2421    (set_attr "simd_mode" "<MODE>")]
2422 )
2423
2424 (define_insn "aarch64_sq<r>dmulh_lane<mode>"
2425   [(set (match_operand:SD_HSI 0 "register_operand" "=w")
2426         (unspec:SD_HSI
2427           [(match_operand:SD_HSI 1 "register_operand" "w")
2428            (vec_select:<VEL>
2429              (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
2430              (parallel [(match_operand:SI 3 "immediate_operand" "i")]))]
2431          VQDMULH))]
2432   "TARGET_SIMD"
2433   "*
2434    aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCONQ>mode));
2435    return \"sq<r>dmulh\\t%<v>0, %<v>1, %2.<v>[%3]\";"
2436   [(set_attr "simd_type" "simd_sat_mul")
2437    (set_attr "simd_mode" "<MODE>")]
2438 )
2439
2440 ;; vqdml[sa]l
2441
2442 (define_insn "aarch64_sqdml<SBINQOPS:as>l<mode>"
2443   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2444         (SBINQOPS:<VWIDE>
2445           (match_operand:<VWIDE> 1 "register_operand" "0")
2446           (ss_ashift:<VWIDE>
2447               (mult:<VWIDE>
2448                 (sign_extend:<VWIDE>
2449                       (match_operand:VSD_HSI 2 "register_operand" "w"))
2450                 (sign_extend:<VWIDE>
2451                       (match_operand:VSD_HSI 3 "register_operand" "w")))
2452               (const_int 1))))]
2453   "TARGET_SIMD"
2454   "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %<v>3<Vmtype>"
2455   [(set_attr "simd_type" "simd_sat_mlal")
2456    (set_attr "simd_mode" "<MODE>")]
2457 )
2458
2459 ;; vqdml[sa]l_lane
2460
2461 (define_insn "aarch64_sqdml<SBINQOPS:as>l_lane<mode>_internal"
2462   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2463         (SBINQOPS:<VWIDE>
2464           (match_operand:<VWIDE> 1 "register_operand" "0")
2465           (ss_ashift:<VWIDE>
2466             (mult:<VWIDE>
2467               (sign_extend:<VWIDE>
2468                 (match_operand:VD_HSI 2 "register_operand" "w"))
2469               (sign_extend:<VWIDE>
2470                 (vec_duplicate:VD_HSI
2471                   (vec_select:<VEL>
2472                     (match_operand:<VCON> 3 "register_operand" "<vwx>")
2473                     (parallel [(match_operand:SI 4 "immediate_operand" "i")])))
2474               ))
2475             (const_int 1))))]
2476   "TARGET_SIMD"
2477   "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]"
2478   [(set_attr "simd_type" "simd_sat_mlal")
2479    (set_attr "simd_mode" "<MODE>")]
2480 )
2481
2482 (define_insn "aarch64_sqdml<SBINQOPS:as>l_lane<mode>_internal"
2483   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2484         (SBINQOPS:<VWIDE>
2485           (match_operand:<VWIDE> 1 "register_operand" "0")
2486           (ss_ashift:<VWIDE>
2487             (mult:<VWIDE>
2488               (sign_extend:<VWIDE>
2489                 (match_operand:SD_HSI 2 "register_operand" "w"))
2490               (sign_extend:<VWIDE>
2491                 (vec_select:<VEL>
2492                   (match_operand:<VCON> 3 "register_operand" "<vwx>")
2493                   (parallel [(match_operand:SI 4 "immediate_operand" "i")])))
2494               )
2495             (const_int 1))))]
2496   "TARGET_SIMD"
2497   "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]"
2498   [(set_attr "simd_type" "simd_sat_mlal")
2499    (set_attr "simd_mode" "<MODE>")]
2500 )
2501
2502 (define_expand "aarch64_sqdmlal_lane<mode>"
2503   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2504    (match_operand:<VWIDE> 1 "register_operand" "0")
2505    (match_operand:VSD_HSI 2 "register_operand" "w")
2506    (match_operand:<VCON> 3 "register_operand" "<vwx>")
2507    (match_operand:SI 4 "immediate_operand" "i")]
2508   "TARGET_SIMD"
2509 {
2510   aarch64_simd_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<VCON>mode) / 2);
2511   emit_insn (gen_aarch64_sqdmlal_lane<mode>_internal (operands[0], operands[1],
2512                                                       operands[2], operands[3],
2513                                                       operands[4]));
2514   DONE;
2515 })
2516
2517 (define_expand "aarch64_sqdmlal_laneq<mode>"
2518   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2519    (match_operand:<VWIDE> 1 "register_operand" "0")
2520    (match_operand:VSD_HSI 2 "register_operand" "w")
2521    (match_operand:<VCON> 3 "register_operand" "<vwx>")
2522    (match_operand:SI 4 "immediate_operand" "i")]
2523   "TARGET_SIMD"
2524 {
2525   aarch64_simd_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<VCON>mode));
2526   emit_insn (gen_aarch64_sqdmlal_lane<mode>_internal (operands[0], operands[1],
2527                                                       operands[2], operands[3],
2528                                                       operands[4]));
2529   DONE;
2530 })
2531
2532 (define_expand "aarch64_sqdmlsl_lane<mode>"
2533   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2534    (match_operand:<VWIDE> 1 "register_operand" "0")
2535    (match_operand:VSD_HSI 2 "register_operand" "w")
2536    (match_operand:<VCON> 3 "register_operand" "<vwx>")
2537    (match_operand:SI 4 "immediate_operand" "i")]
2538   "TARGET_SIMD"
2539 {
2540   aarch64_simd_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<VCON>mode) / 2);
2541   emit_insn (gen_aarch64_sqdmlsl_lane<mode>_internal (operands[0], operands[1],
2542                                                       operands[2], operands[3],
2543                                                       operands[4]));
2544   DONE;
2545 })
2546
2547 (define_expand "aarch64_sqdmlsl_laneq<mode>"
2548   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2549    (match_operand:<VWIDE> 1 "register_operand" "0")
2550    (match_operand:VSD_HSI 2 "register_operand" "w")
2551    (match_operand:<VCON> 3 "register_operand" "<vwx>")
2552    (match_operand:SI 4 "immediate_operand" "i")]
2553   "TARGET_SIMD"
2554 {
2555   aarch64_simd_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<VCON>mode));
2556   emit_insn (gen_aarch64_sqdmlsl_lane<mode>_internal (operands[0], operands[1],
2557                                                       operands[2], operands[3],
2558                                                       operands[4]));
2559   DONE;
2560 })
2561
2562 ;; vqdml[sa]l_n
2563
2564 (define_insn "aarch64_sqdml<SBINQOPS:as>l_n<mode>"
2565   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2566         (SBINQOPS:<VWIDE>
2567           (match_operand:<VWIDE> 1 "register_operand" "0")
2568           (ss_ashift:<VWIDE>
2569               (mult:<VWIDE>
2570                 (sign_extend:<VWIDE>
2571                       (match_operand:VD_HSI 2 "register_operand" "w"))
2572                 (sign_extend:<VWIDE>
2573                   (vec_duplicate:VD_HSI
2574                     (match_operand:<VEL> 3 "register_operand" "w"))))
2575               (const_int 1))))]
2576   "TARGET_SIMD"
2577   "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[0]"
2578   [(set_attr "simd_type" "simd_sat_mlal")
2579    (set_attr "simd_mode" "<MODE>")]
2580 )
2581
2582 ;; sqdml[as]l2
2583
2584 (define_insn "aarch64_sqdml<SBINQOPS:as>l2<mode>_internal"
2585   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2586         (SBINQOPS:<VWIDE>
2587          (match_operand:<VWIDE> 1 "register_operand" "0")
2588          (ss_ashift:<VWIDE>
2589              (mult:<VWIDE>
2590                (sign_extend:<VWIDE>
2591                  (vec_select:<VHALF>
2592                      (match_operand:VQ_HSI 2 "register_operand" "w")
2593                      (match_operand:VQ_HSI 4 "vect_par_cnst_hi_half" "")))
2594                (sign_extend:<VWIDE>
2595                  (vec_select:<VHALF>
2596                      (match_operand:VQ_HSI 3 "register_operand" "w")
2597                      (match_dup 4))))
2598              (const_int 1))))]
2599   "TARGET_SIMD"
2600   "sqdml<SBINQOPS:as>l2\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %<v>3<Vmtype>"
2601   [(set_attr "simd_type" "simd_sat_mlal")
2602    (set_attr "simd_mode" "<MODE>")]
2603 )
2604
2605 (define_expand "aarch64_sqdmlal2<mode>"
2606   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2607    (match_operand:<VWIDE> 1 "register_operand" "w")
2608    (match_operand:VQ_HSI 2 "register_operand" "w")
2609    (match_operand:VQ_HSI 3 "register_operand" "w")]
2610   "TARGET_SIMD"
2611 {
2612   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2613   emit_insn (gen_aarch64_sqdmlal2<mode>_internal (operands[0], operands[1],
2614                                                   operands[2], operands[3], p));
2615   DONE;
2616 })
2617
2618 (define_expand "aarch64_sqdmlsl2<mode>"
2619   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2620    (match_operand:<VWIDE> 1 "register_operand" "w")
2621    (match_operand:VQ_HSI 2 "register_operand" "w")
2622    (match_operand:VQ_HSI 3 "register_operand" "w")]
2623   "TARGET_SIMD"
2624 {
2625   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2626   emit_insn (gen_aarch64_sqdmlsl2<mode>_internal (operands[0], operands[1],
2627                                                   operands[2], operands[3], p));
2628   DONE;
2629 })
2630
2631 ;; vqdml[sa]l2_lane
2632
2633 (define_insn "aarch64_sqdml<SBINQOPS:as>l2_lane<mode>_internal"
2634   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2635         (SBINQOPS:<VWIDE>
2636           (match_operand:<VWIDE> 1 "register_operand" "0")
2637           (ss_ashift:<VWIDE>
2638               (mult:<VWIDE>
2639                 (sign_extend:<VWIDE>
2640                   (vec_select:<VHALF>
2641                     (match_operand:VQ_HSI 2 "register_operand" "w")
2642                     (match_operand:VQ_HSI 5 "vect_par_cnst_hi_half" "")))
2643                 (sign_extend:<VWIDE>
2644                   (vec_duplicate:<VHALF>
2645                     (vec_select:<VEL>
2646                       (match_operand:<VCON> 3 "register_operand" "<vwx>")
2647                       (parallel [(match_operand:SI 4 "immediate_operand" "i")])
2648                     ))))
2649               (const_int 1))))]
2650   "TARGET_SIMD"
2651   "sqdml<SBINQOPS:as>l2\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]"
2652   [(set_attr "simd_type" "simd_sat_mlal")
2653    (set_attr "simd_mode" "<MODE>")]
2654 )
2655
2656 (define_expand "aarch64_sqdmlal2_lane<mode>"
2657   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2658    (match_operand:<VWIDE> 1 "register_operand" "w")
2659    (match_operand:VQ_HSI 2 "register_operand" "w")
2660    (match_operand:<VCON> 3 "register_operand" "<vwx>")
2661    (match_operand:SI 4 "immediate_operand" "i")]
2662   "TARGET_SIMD"
2663 {
2664   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2665   aarch64_simd_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode) / 2);
2666   emit_insn (gen_aarch64_sqdmlal2_lane<mode>_internal (operands[0], operands[1],
2667                                                        operands[2], operands[3],
2668                                                        operands[4], p));
2669   DONE;
2670 })
2671
2672 (define_expand "aarch64_sqdmlal2_laneq<mode>"
2673   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2674    (match_operand:<VWIDE> 1 "register_operand" "w")
2675    (match_operand:VQ_HSI 2 "register_operand" "w")
2676    (match_operand:<VCON> 3 "register_operand" "<vwx>")
2677    (match_operand:SI 4 "immediate_operand" "i")]
2678   "TARGET_SIMD"
2679 {
2680   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2681   aarch64_simd_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
2682   emit_insn (gen_aarch64_sqdmlal2_lane<mode>_internal (operands[0], operands[1],
2683                                                        operands[2], operands[3],
2684                                                        operands[4], p));
2685   DONE;
2686 })
2687
2688 (define_expand "aarch64_sqdmlsl2_lane<mode>"
2689   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2690    (match_operand:<VWIDE> 1 "register_operand" "w")
2691    (match_operand:VQ_HSI 2 "register_operand" "w")
2692    (match_operand:<VCON> 3 "register_operand" "<vwx>")
2693    (match_operand:SI 4 "immediate_operand" "i")]
2694   "TARGET_SIMD"
2695 {
2696   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2697   aarch64_simd_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode) / 2);
2698   emit_insn (gen_aarch64_sqdmlsl2_lane<mode>_internal (operands[0], operands[1],
2699                                                        operands[2], operands[3],
2700                                                        operands[4], p));
2701   DONE;
2702 })
2703
2704 (define_expand "aarch64_sqdmlsl2_laneq<mode>"
2705   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2706    (match_operand:<VWIDE> 1 "register_operand" "w")
2707    (match_operand:VQ_HSI 2 "register_operand" "w")
2708    (match_operand:<VCON> 3 "register_operand" "<vwx>")
2709    (match_operand:SI 4 "immediate_operand" "i")]
2710   "TARGET_SIMD"
2711 {
2712   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2713   aarch64_simd_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
2714   emit_insn (gen_aarch64_sqdmlsl2_lane<mode>_internal (operands[0], operands[1],
2715                                                        operands[2], operands[3],
2716                                                        operands[4], p));
2717   DONE;
2718 })
2719
2720 (define_insn "aarch64_sqdml<SBINQOPS:as>l2_n<mode>_internal"
2721   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2722         (SBINQOPS:<VWIDE>
2723           (match_operand:<VWIDE> 1 "register_operand" "0")
2724           (ss_ashift:<VWIDE>
2725             (mult:<VWIDE>
2726               (sign_extend:<VWIDE>
2727                 (vec_select:<VHALF>
2728                   (match_operand:VQ_HSI 2 "register_operand" "w")
2729                   (match_operand:VQ_HSI 4 "vect_par_cnst_hi_half" "")))
2730               (sign_extend:<VWIDE>
2731                 (vec_duplicate:<VHALF>
2732                   (match_operand:<VEL> 3 "register_operand" "w"))))
2733             (const_int 1))))]
2734   "TARGET_SIMD"
2735   "sqdml<SBINQOPS:as>l2\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[0]"
2736   [(set_attr "simd_type" "simd_sat_mlal")
2737    (set_attr "simd_mode" "<MODE>")]
2738 )
2739
2740 (define_expand "aarch64_sqdmlal2_n<mode>"
2741   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2742    (match_operand:<VWIDE> 1 "register_operand" "w")
2743    (match_operand:VQ_HSI 2 "register_operand" "w")
2744    (match_operand:<VEL> 3 "register_operand" "w")]
2745   "TARGET_SIMD"
2746 {
2747   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2748   emit_insn (gen_aarch64_sqdmlal2_n<mode>_internal (operands[0], operands[1],
2749                                                     operands[2], operands[3],
2750                                                     p));
2751   DONE;
2752 })
2753
2754 (define_expand "aarch64_sqdmlsl2_n<mode>"
2755   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2756    (match_operand:<VWIDE> 1 "register_operand" "w")
2757    (match_operand:VQ_HSI 2 "register_operand" "w")
2758    (match_operand:<VEL> 3 "register_operand" "w")]
2759   "TARGET_SIMD"
2760 {
2761   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2762   emit_insn (gen_aarch64_sqdmlsl2_n<mode>_internal (operands[0], operands[1],
2763                                                     operands[2], operands[3],
2764                                                     p));
2765   DONE;
2766 })
2767
2768 ;; vqdmull
2769
2770 (define_insn "aarch64_sqdmull<mode>"
2771   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2772         (ss_ashift:<VWIDE>
2773              (mult:<VWIDE>
2774                (sign_extend:<VWIDE>
2775                      (match_operand:VSD_HSI 1 "register_operand" "w"))
2776                (sign_extend:<VWIDE>
2777                      (match_operand:VSD_HSI 2 "register_operand" "w")))
2778              (const_int 1)))]
2779   "TARGET_SIMD"
2780   "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
2781   [(set_attr "simd_type" "simd_sat_mul")
2782    (set_attr "simd_mode" "<MODE>")]
2783 )
2784
2785 ;; vqdmull_lane
2786
2787 (define_insn "aarch64_sqdmull_lane<mode>_internal"
2788   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2789         (ss_ashift:<VWIDE>
2790              (mult:<VWIDE>
2791                (sign_extend:<VWIDE>
2792                  (match_operand:VD_HSI 1 "register_operand" "w"))
2793                (sign_extend:<VWIDE>
2794                  (vec_duplicate:VD_HSI
2795                    (vec_select:<VEL>
2796                      (match_operand:<VCON> 2 "register_operand" "<vwx>")
2797                      (parallel [(match_operand:SI 3 "immediate_operand" "i")])))
2798                ))
2799              (const_int 1)))]
2800   "TARGET_SIMD"
2801   "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]"
2802   [(set_attr "simd_type" "simd_sat_mul")
2803    (set_attr "simd_mode" "<MODE>")]
2804 )
2805
2806 (define_insn "aarch64_sqdmull_lane<mode>_internal"
2807   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2808         (ss_ashift:<VWIDE>
2809              (mult:<VWIDE>
2810                (sign_extend:<VWIDE>
2811                  (match_operand:SD_HSI 1 "register_operand" "w"))
2812                (sign_extend:<VWIDE>
2813                  (vec_select:<VEL>
2814                    (match_operand:<VCON> 2 "register_operand" "<vwx>")
2815                    (parallel [(match_operand:SI 3 "immediate_operand" "i")]))
2816                ))
2817              (const_int 1)))]
2818   "TARGET_SIMD"
2819   "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]"
2820   [(set_attr "simd_type" "simd_sat_mul")
2821    (set_attr "simd_mode" "<MODE>")]
2822 )
2823
2824 (define_expand "aarch64_sqdmull_lane<mode>"
2825   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2826    (match_operand:VSD_HSI 1 "register_operand" "w")
2827    (match_operand:<VCON> 2 "register_operand" "<vwx>")
2828    (match_operand:SI 3 "immediate_operand" "i")]
2829   "TARGET_SIMD"
2830 {
2831   aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCON>mode) / 2);
2832   emit_insn (gen_aarch64_sqdmull_lane<mode>_internal (operands[0], operands[1],
2833                                                       operands[2], operands[3]));
2834   DONE;
2835 })
2836
2837 (define_expand "aarch64_sqdmull_laneq<mode>"
2838   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2839    (match_operand:VD_HSI 1 "register_operand" "w")
2840    (match_operand:<VCON> 2 "register_operand" "<vwx>")
2841    (match_operand:SI 3 "immediate_operand" "i")]
2842   "TARGET_SIMD"
2843 {
2844   aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCON>mode));
2845   emit_insn (gen_aarch64_sqdmull_lane<mode>_internal
2846                (operands[0], operands[1], operands[2], operands[3]));
2847   DONE;
2848 })
2849
2850 ;; vqdmull_n
2851
2852 (define_insn "aarch64_sqdmull_n<mode>"
2853   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2854         (ss_ashift:<VWIDE>
2855              (mult:<VWIDE>
2856                (sign_extend:<VWIDE>
2857                  (match_operand:VD_HSI 1 "register_operand" "w"))
2858                (sign_extend:<VWIDE>
2859                  (vec_duplicate:VD_HSI
2860                    (match_operand:<VEL> 2 "register_operand" "w")))
2861                )
2862              (const_int 1)))]
2863   "TARGET_SIMD"
2864   "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[0]"
2865   [(set_attr "simd_type" "simd_sat_mul")
2866    (set_attr "simd_mode" "<MODE>")]
2867 )
2868
2869 ;; vqdmull2
2870
2871
2872
2873 (define_insn "aarch64_sqdmull2<mode>_internal"
2874   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2875         (ss_ashift:<VWIDE>
2876              (mult:<VWIDE>
2877                (sign_extend:<VWIDE>
2878                  (vec_select:<VHALF>
2879                    (match_operand:VQ_HSI 1 "register_operand" "w")
2880                    (match_operand:VQ_HSI 3 "vect_par_cnst_hi_half" "")))
2881                (sign_extend:<VWIDE>
2882                  (vec_select:<VHALF>
2883                    (match_operand:VQ_HSI 2 "register_operand" "w")
2884                    (match_dup 3)))
2885                )
2886              (const_int 1)))]
2887   "TARGET_SIMD"
2888   "sqdmull2\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
2889   [(set_attr "simd_type" "simd_sat_mul")
2890    (set_attr "simd_mode" "<MODE>")]
2891 )
2892
2893 (define_expand "aarch64_sqdmull2<mode>"
2894   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2895    (match_operand:VQ_HSI 1 "register_operand" "w")
2896    (match_operand:<VCON> 2 "register_operand" "w")]
2897   "TARGET_SIMD"
2898 {
2899   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2900   emit_insn (gen_aarch64_sqdmull2<mode>_internal (operands[0], operands[1],
2901                                                   operands[2], p));
2902   DONE;
2903 })
2904
2905 ;; vqdmull2_lane
2906
2907 (define_insn "aarch64_sqdmull2_lane<mode>_internal"
2908   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2909         (ss_ashift:<VWIDE>
2910              (mult:<VWIDE>
2911                (sign_extend:<VWIDE>
2912                  (vec_select:<VHALF>
2913                    (match_operand:VQ_HSI 1 "register_operand" "w")
2914                    (match_operand:VQ_HSI 4 "vect_par_cnst_hi_half" "")))
2915                (sign_extend:<VWIDE>
2916                  (vec_duplicate:<VHALF>
2917                    (vec_select:<VEL>
2918                      (match_operand:<VCON> 2 "register_operand" "<vwx>")
2919                      (parallel [(match_operand:SI 3 "immediate_operand" "i")])))
2920                ))
2921              (const_int 1)))]
2922   "TARGET_SIMD"
2923   "sqdmull2\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]"
2924   [(set_attr "simd_type" "simd_sat_mul")
2925    (set_attr "simd_mode" "<MODE>")]
2926 )
2927
2928 (define_expand "aarch64_sqdmull2_lane<mode>"
2929   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2930    (match_operand:VQ_HSI 1 "register_operand" "w")
2931    (match_operand:<VCON> 2 "register_operand" "<vwx>")
2932    (match_operand:SI 3 "immediate_operand" "i")]
2933   "TARGET_SIMD"
2934 {
2935   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2936   aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode) / 2);
2937   emit_insn (gen_aarch64_sqdmull2_lane<mode>_internal (operands[0], operands[1],
2938                                                        operands[2], operands[3],
2939                                                        p));
2940   DONE;
2941 })
2942
2943 (define_expand "aarch64_sqdmull2_laneq<mode>"
2944   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2945    (match_operand:VQ_HSI 1 "register_operand" "w")
2946    (match_operand:<VCON> 2 "register_operand" "<vwx>")
2947    (match_operand:SI 3 "immediate_operand" "i")]
2948   "TARGET_SIMD"
2949 {
2950   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2951   aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
2952   emit_insn (gen_aarch64_sqdmull2_lane<mode>_internal (operands[0], operands[1],
2953                                                        operands[2], operands[3],
2954                                                        p));
2955   DONE;
2956 })
2957
2958 ;; vqdmull2_n
2959
2960 (define_insn "aarch64_sqdmull2_n<mode>_internal"
2961   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2962         (ss_ashift:<VWIDE>
2963              (mult:<VWIDE>
2964                (sign_extend:<VWIDE>
2965                  (vec_select:<VHALF>
2966                    (match_operand:VQ_HSI 1 "register_operand" "w")
2967                    (match_operand:VQ_HSI 3 "vect_par_cnst_hi_half" "")))
2968                (sign_extend:<VWIDE>
2969                  (vec_duplicate:<VHALF>
2970                    (match_operand:<VEL> 2 "register_operand" "w")))
2971                )
2972              (const_int 1)))]
2973   "TARGET_SIMD"
2974   "sqdmull2\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[0]"
2975   [(set_attr "simd_type" "simd_sat_mul")
2976    (set_attr "simd_mode" "<MODE>")]
2977 )
2978
2979 (define_expand "aarch64_sqdmull2_n<mode>"
2980   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2981    (match_operand:VQ_HSI 1 "register_operand" "w")
2982    (match_operand:<VEL> 2 "register_operand" "w")]
2983   "TARGET_SIMD"
2984 {
2985   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2986   emit_insn (gen_aarch64_sqdmull2_n<mode>_internal (operands[0], operands[1],
2987                                                     operands[2], p));
2988   DONE;
2989 })
2990
2991 ;; vshl
2992
2993 (define_insn "aarch64_<sur>shl<mode>"
2994   [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
2995         (unspec:VSDQ_I_DI
2996           [(match_operand:VSDQ_I_DI 1 "register_operand" "w")
2997            (match_operand:VSDQ_I_DI 2 "register_operand" "w")]
2998          VSHL))]
2999   "TARGET_SIMD"
3000   "<sur>shl\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>";
3001   [(set_attr "simd_type" "simd_shift")
3002    (set_attr "simd_mode" "<MODE>")]
3003 )
3004
3005
3006 ;; vqshl
3007
3008 (define_insn "aarch64_<sur>q<r>shl<mode>"
3009   [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
3010         (unspec:VSDQ_I
3011           [(match_operand:VSDQ_I 1 "register_operand" "w")
3012            (match_operand:VSDQ_I 2 "register_operand" "w")]
3013          VQSHL))]
3014   "TARGET_SIMD"
3015   "<sur>q<r>shl\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>";
3016   [(set_attr "simd_type" "simd_sat_shift")
3017    (set_attr "simd_mode" "<MODE>")]
3018 )
3019
3020 ;; vshll_n
3021
3022 (define_insn "aarch64_<sur>shll_n<mode>"
3023   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3024         (unspec:<VWIDE> [(match_operand:VDW 1 "register_operand" "w")
3025                          (match_operand:SI 2 "immediate_operand" "i")]
3026                          VSHLL))]
3027   "TARGET_SIMD"
3028   "*
3029   int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
3030   aarch64_simd_const_bounds (operands[2], 0, bit_width + 1);
3031   if (INTVAL (operands[2]) == bit_width)
3032   {
3033     return \"shll\\t%0.<Vwtype>, %1.<Vtype>, %2\";
3034   }
3035   else {
3036     return \"<sur>shll\\t%0.<Vwtype>, %1.<Vtype>, %2\";
3037   }"
3038   [(set_attr "simd_type" "simd_shift_imm")
3039    (set_attr "simd_mode" "<MODE>")]
3040 )
3041
3042 ;; vshll_high_n
3043
3044 (define_insn "aarch64_<sur>shll2_n<mode>"
3045   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3046         (unspec:<VWIDE> [(match_operand:VQW 1 "register_operand" "w")
3047                          (match_operand:SI 2 "immediate_operand" "i")]
3048                          VSHLL))]
3049   "TARGET_SIMD"
3050   "*
3051   int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
3052   aarch64_simd_const_bounds (operands[2], 0, bit_width + 1);
3053   if (INTVAL (operands[2]) == bit_width)
3054   {
3055     return \"shll2\\t%0.<Vwtype>, %1.<Vtype>, %2\";
3056   }
3057   else {
3058     return \"<sur>shll2\\t%0.<Vwtype>, %1.<Vtype>, %2\";
3059   }"
3060   [(set_attr "simd_type" "simd_shift_imm")
3061    (set_attr "simd_mode" "<MODE>")]
3062 )
3063
3064 ;; vrshr_n
3065
3066 (define_insn "aarch64_<sur>shr_n<mode>"
3067   [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
3068         (unspec:VSDQ_I_DI [(match_operand:VSDQ_I_DI 1 "register_operand" "w")
3069                            (match_operand:SI 2 "immediate_operand" "i")]
3070                           VRSHR_N))]
3071   "TARGET_SIMD"
3072   "*
3073   int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
3074   aarch64_simd_const_bounds (operands[2], 1, bit_width + 1);
3075   return \"<sur>shr\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %2\";"
3076   [(set_attr "simd_type" "simd_shift_imm")
3077    (set_attr "simd_mode" "<MODE>")]
3078 )
3079
3080 ;; v(r)sra_n
3081
3082 (define_insn "aarch64_<sur>sra_n<mode>"
3083   [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
3084         (unspec:VSDQ_I_DI [(match_operand:VSDQ_I_DI 1 "register_operand" "0")
3085                        (match_operand:VSDQ_I_DI 2 "register_operand" "w")
3086                        (match_operand:SI 3 "immediate_operand" "i")]
3087                       VSRA))]
3088   "TARGET_SIMD"
3089   "*
3090   int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
3091   aarch64_simd_const_bounds (operands[3], 1, bit_width + 1);
3092   return \"<sur>sra\\t%<v>0<Vmtype>, %<v>2<Vmtype>, %3\";"
3093   [(set_attr "simd_type" "simd_shift_imm_acc")
3094    (set_attr "simd_mode" "<MODE>")]
3095 )
3096
3097 ;; vs<lr>i_n
3098
3099 (define_insn "aarch64_<sur>s<lr>i_n<mode>"
3100   [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
3101         (unspec:VSDQ_I_DI [(match_operand:VSDQ_I_DI 1 "register_operand" "0")
3102                        (match_operand:VSDQ_I_DI 2 "register_operand" "w")
3103                        (match_operand:SI 3 "immediate_operand" "i")]
3104                       VSLRI))]
3105   "TARGET_SIMD"
3106   "*
3107   int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
3108   aarch64_simd_const_bounds (operands[3], 1 - <VSLRI:offsetlr>,
3109                              bit_width - <VSLRI:offsetlr> + 1);
3110   return \"s<lr>i\\t%<v>0<Vmtype>, %<v>2<Vmtype>, %3\";"
3111   [(set_attr "simd_type" "simd_shift_imm")
3112    (set_attr "simd_mode" "<MODE>")]
3113 )
3114
3115 ;; vqshl(u)
3116
3117 (define_insn "aarch64_<sur>qshl<u>_n<mode>"
3118   [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
3119         (unspec:VSDQ_I [(match_operand:VSDQ_I 1 "register_operand" "w")
3120                        (match_operand:SI 2 "immediate_operand" "i")]
3121                       VQSHL_N))]
3122   "TARGET_SIMD"
3123   "*
3124   int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
3125   aarch64_simd_const_bounds (operands[2], 0, bit_width);
3126   return \"<sur>qshl<u>\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %2\";"
3127   [(set_attr "simd_type" "simd_sat_shift_imm")
3128    (set_attr "simd_mode" "<MODE>")]
3129 )
3130
3131
3132 ;; vq(r)shr(u)n_n
3133
3134 (define_insn "aarch64_<sur>q<r>shr<u>n_n<mode>"
3135   [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
3136         (unspec:<VNARROWQ> [(match_operand:VSQN_HSDI 1 "register_operand" "w")
3137                             (match_operand:SI 2 "immediate_operand" "i")]
3138                            VQSHRN_N))]
3139   "TARGET_SIMD"
3140   "*
3141   int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
3142   aarch64_simd_const_bounds (operands[2], 1, bit_width + 1);
3143   return \"<sur>q<r>shr<u>n\\t%<vn2>0<Vmntype>, %<v>1<Vmtype>, %2\";"
3144   [(set_attr "simd_type" "simd_sat_shiftn_imm")
3145    (set_attr "simd_mode" "<MODE>")]
3146 )
3147
3148
3149 ;; cm(eq|ge|gt|lt|le)
3150 ;; Note, we have constraints for Dz and Z as different expanders
3151 ;; have different ideas of what should be passed to this pattern.
3152
3153 (define_insn "aarch64_cm<optab><mode>"
3154   [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w,w")
3155         (neg:<V_cmp_result>
3156           (COMPARISONS:<V_cmp_result>
3157             (match_operand:VDQ 1 "register_operand" "w,w")
3158             (match_operand:VDQ 2 "aarch64_simd_reg_or_zero" "w,ZDz")
3159           )))]
3160   "TARGET_SIMD"
3161   "@
3162   cm<n_optab>\t%<v>0<Vmtype>, %<v><cmp_1><Vmtype>, %<v><cmp_2><Vmtype>
3163   cm<optab>\t%<v>0<Vmtype>, %<v>1<Vmtype>, #0"
3164   [(set_attr "simd_type" "simd_cmp")
3165    (set_attr "simd_mode" "<MODE>")]
3166 )
3167
3168 (define_insn_and_split "aarch64_cm<optab>di"
3169   [(set (match_operand:DI 0 "register_operand" "=w,w,r")
3170         (neg:DI
3171           (COMPARISONS:DI
3172             (match_operand:DI 1 "register_operand" "w,w,r")
3173             (match_operand:DI 2 "aarch64_simd_reg_or_zero" "w,ZDz,r")
3174           )))]
3175   "TARGET_SIMD"
3176   "@
3177   cm<n_optab>\t%d0, %d<cmp_1>, %d<cmp_2>
3178   cm<optab>\t%d0, %d1, #0
3179   #"
3180   "reload_completed
3181    /* We need to prevent the split from
3182       happening in the 'w' constraint cases.  */
3183    && GP_REGNUM_P (REGNO (operands[0]))
3184    && GP_REGNUM_P (REGNO (operands[1]))"
3185   [(set (reg:CC CC_REGNUM)
3186     (compare:CC
3187       (match_dup 1)
3188       (match_dup 2)))
3189   (set (match_dup 0)
3190     (neg:DI
3191       (COMPARISONS:DI
3192         (match_operand 3 "cc_register" "")
3193         (const_int 0))))]
3194   {
3195     enum machine_mode mode = SELECT_CC_MODE (<CMP>, operands[1], operands[2]);
3196     rtx cc_reg = aarch64_gen_compare_reg (<CMP>, operands[1], operands[2]);
3197     rtx comparison = gen_rtx_<CMP> (mode, operands[1], operands[2]);
3198     emit_insn (gen_cstoredi_neg (operands[0], comparison, cc_reg));
3199     DONE;
3200   }
3201   [(set_attr "simd_type" "simd_cmp")
3202    (set_attr "simd_mode" "DI")]
3203 )
3204
3205 ;; cm(hs|hi)
3206
3207 (define_insn "aarch64_cm<optab><mode>"
3208   [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w")
3209         (neg:<V_cmp_result>
3210           (UCOMPARISONS:<V_cmp_result>
3211             (match_operand:VDQ 1 "register_operand" "w")
3212             (match_operand:VDQ 2 "register_operand" "w")
3213           )))]
3214   "TARGET_SIMD"
3215   "cm<n_optab>\t%<v>0<Vmtype>, %<v><cmp_1><Vmtype>, %<v><cmp_2><Vmtype>"
3216   [(set_attr "simd_type" "simd_cmp")
3217    (set_attr "simd_mode" "<MODE>")]
3218 )
3219
3220 (define_insn_and_split "aarch64_cm<optab>di"
3221   [(set (match_operand:DI 0 "register_operand" "=w,r")
3222         (neg:DI
3223           (UCOMPARISONS:DI
3224             (match_operand:DI 1 "register_operand" "w,r")
3225             (match_operand:DI 2 "aarch64_simd_reg_or_zero" "w,r")
3226           )))]
3227   "TARGET_SIMD"
3228   "@
3229   cm<n_optab>\t%d0, %d<cmp_1>, %d<cmp_2>
3230   #"
3231   "reload_completed
3232    /* We need to prevent the split from
3233       happening in the 'w' constraint cases.  */
3234    && GP_REGNUM_P (REGNO (operands[0]))
3235    && GP_REGNUM_P (REGNO (operands[1]))"
3236   [(set (reg:CC CC_REGNUM)
3237     (compare:CC
3238       (match_dup 1)
3239       (match_dup 2)))
3240   (set (match_dup 0)
3241     (neg:DI
3242       (UCOMPARISONS:DI
3243         (match_operand 3 "cc_register" "")
3244         (const_int 0))))]
3245   {
3246     enum machine_mode mode = SELECT_CC_MODE (<CMP>, operands[1], operands[2]);
3247     rtx cc_reg = aarch64_gen_compare_reg (<CMP>, operands[1], operands[2]);
3248     rtx comparison = gen_rtx_<CMP> (mode, operands[1], operands[2]);
3249     emit_insn (gen_cstoredi_neg (operands[0], comparison, cc_reg));
3250     DONE;
3251   }
3252   [(set_attr "simd_type" "simd_cmp")
3253    (set_attr "simd_mode" "DI")]
3254 )
3255
3256 ;; cmtst
3257
3258 (define_insn "aarch64_cmtst<mode>"
3259   [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w")
3260         (neg:<V_cmp_result>
3261           (ne:<V_cmp_result>
3262             (and:VDQ
3263               (match_operand:VDQ 1 "register_operand" "w")
3264               (match_operand:VDQ 2 "register_operand" "w"))
3265             (vec_duplicate:<V_cmp_result> (const_int 0)))))]
3266   "TARGET_SIMD"
3267   "cmtst\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
3268   [(set_attr "simd_type" "simd_cmp")
3269    (set_attr "simd_mode" "<MODE>")]
3270 )
3271
3272 (define_insn_and_split "aarch64_cmtstdi"
3273   [(set (match_operand:DI 0 "register_operand" "=w,r")
3274         (neg:DI
3275           (ne:DI
3276             (and:DI
3277               (match_operand:DI 1 "register_operand" "w,r")
3278               (match_operand:DI 2 "register_operand" "w,r"))
3279             (const_int 0))))]
3280   "TARGET_SIMD"
3281   "@
3282   cmtst\t%d0, %d1, %d2
3283   #"
3284   "reload_completed
3285    /* We need to prevent the split from
3286       happening in the 'w' constraint cases.  */
3287    && GP_REGNUM_P (REGNO (operands[0]))
3288    && GP_REGNUM_P (REGNO (operands[1]))"
3289    [(set (reg:CC_NZ CC_REGNUM)
3290         (compare:CC_NZ
3291          (and:DI (match_dup 1)
3292                   (match_dup 2))
3293          (const_int 0)))
3294   (set (match_dup 0)
3295     (neg:DI
3296       (ne:DI
3297         (match_operand 3 "cc_register" "")
3298         (const_int 0))))]
3299   {
3300     rtx and_tree = gen_rtx_AND (DImode, operands[1], operands[2]);
3301     enum machine_mode mode = SELECT_CC_MODE (NE, and_tree, const0_rtx);
3302     rtx cc_reg = aarch64_gen_compare_reg (NE, and_tree, const0_rtx);
3303     rtx comparison = gen_rtx_NE (mode, and_tree, const0_rtx);
3304     emit_insn (gen_cstoredi_neg (operands[0], comparison, cc_reg));
3305     DONE;
3306   }
3307   [(set_attr "simd_type" "simd_cmp")
3308    (set_attr "simd_mode" "DI")]
3309 )
3310
3311 ;; fcm(eq|ge|gt|le|lt)
3312
3313 (define_insn "aarch64_cm<optab><mode>"
3314   [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w,w")
3315         (neg:<V_cmp_result>
3316           (COMPARISONS:<V_cmp_result>
3317             (match_operand:VALLF 1 "register_operand" "w,w")
3318             (match_operand:VALLF 2 "aarch64_simd_reg_or_zero" "w,YDz")
3319           )))]
3320   "TARGET_SIMD"
3321   "@
3322   fcm<n_optab>\t%<v>0<Vmtype>, %<v><cmp_1><Vmtype>, %<v><cmp_2><Vmtype>
3323   fcm<optab>\t%<v>0<Vmtype>, %<v>1<Vmtype>, 0"
3324   [(set_attr "simd_type" "simd_fcmp")
3325    (set_attr "simd_mode" "<MODE>")]
3326 )
3327
3328 ;; fac(ge|gt)
3329 ;; Note we can also handle what would be fac(le|lt) by
3330 ;; generating fac(ge|gt).
3331
3332 (define_insn "*aarch64_fac<optab><mode>"
3333   [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w")
3334         (neg:<V_cmp_result>
3335           (FAC_COMPARISONS:<V_cmp_result>
3336             (abs:VALLF (match_operand:VALLF 1 "register_operand" "w"))
3337             (abs:VALLF (match_operand:VALLF 2 "register_operand" "w"))
3338   )))]
3339   "TARGET_SIMD"
3340   "fac<n_optab>\t%<v>0<Vmtype>, %<v><cmp_1><Vmtype>, %<v><cmp_2><Vmtype>"
3341   [(set_attr "simd_type" "simd_fcmp")
3342    (set_attr "simd_mode" "<MODE>")]
3343 )
3344
3345 ;; addp
3346
3347 (define_insn "aarch64_addp<mode>"
3348   [(set (match_operand:VD_BHSI 0 "register_operand" "=w")
3349         (unspec:VD_BHSI
3350           [(match_operand:VD_BHSI 1 "register_operand" "w")
3351            (match_operand:VD_BHSI 2 "register_operand" "w")]
3352           UNSPEC_ADDP))]
3353   "TARGET_SIMD"
3354   "addp\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
3355   [(set_attr "simd_type" "simd_add")
3356    (set_attr "simd_mode" "<MODE>")]
3357 )
3358
3359 (define_insn "aarch64_addpdi"
3360   [(set (match_operand:DI 0 "register_operand" "=w")
3361         (unspec:DI
3362           [(match_operand:V2DI 1 "register_operand" "w")]
3363           UNSPEC_ADDP))]
3364   "TARGET_SIMD"
3365   "addp\t%d0, %1.2d"
3366   [(set_attr "simd_type" "simd_add")
3367    (set_attr "simd_mode" "DI")]
3368 )
3369
3370 ;; sqrt
3371
3372 (define_insn "sqrt<mode>2"
3373   [(set (match_operand:VDQF 0 "register_operand" "=w")
3374         (sqrt:VDQF (match_operand:VDQF 1 "register_operand" "w")))]
3375   "TARGET_SIMD"
3376   "fsqrt\\t%0.<Vtype>, %1.<Vtype>"
3377   [(set_attr "simd_type" "simd_fsqrt")
3378    (set_attr "simd_mode" "<MODE>")]
3379 )
3380
3381 ;; Patterns for vector struct loads and stores.
3382
3383 (define_insn "vec_load_lanesoi<mode>"
3384   [(set (match_operand:OI 0 "register_operand" "=w")
3385         (unspec:OI [(match_operand:OI 1 "aarch64_simd_struct_operand" "Utv")
3386                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3387                    UNSPEC_LD2))]
3388   "TARGET_SIMD"
3389   "ld2\\t{%S0.<Vtype> - %T0.<Vtype>}, %1"
3390   [(set_attr "simd_type" "simd_load2")
3391    (set_attr "simd_mode" "<MODE>")])
3392
3393 (define_insn "vec_store_lanesoi<mode>"
3394   [(set (match_operand:OI 0 "aarch64_simd_struct_operand" "=Utv")
3395         (unspec:OI [(match_operand:OI 1 "register_operand" "w")
3396                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3397                    UNSPEC_ST2))]
3398   "TARGET_SIMD"
3399   "st2\\t{%S1.<Vtype> - %T1.<Vtype>}, %0"
3400   [(set_attr "simd_type" "simd_store2")
3401    (set_attr "simd_mode" "<MODE>")])
3402
3403 (define_insn "vec_load_lanesci<mode>"
3404   [(set (match_operand:CI 0 "register_operand" "=w")
3405         (unspec:CI [(match_operand:CI 1 "aarch64_simd_struct_operand" "Utv")
3406                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3407                    UNSPEC_LD3))]
3408   "TARGET_SIMD"
3409   "ld3\\t{%S0.<Vtype> - %U0.<Vtype>}, %1"
3410   [(set_attr "simd_type" "simd_load3")
3411    (set_attr "simd_mode" "<MODE>")])
3412
3413 (define_insn "vec_store_lanesci<mode>"
3414   [(set (match_operand:CI 0 "aarch64_simd_struct_operand" "=Utv")
3415         (unspec:CI [(match_operand:CI 1 "register_operand" "w")
3416                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3417                    UNSPEC_ST3))]
3418   "TARGET_SIMD"
3419   "st3\\t{%S1.<Vtype> - %U1.<Vtype>}, %0"
3420   [(set_attr "simd_type" "simd_store3")
3421    (set_attr "simd_mode" "<MODE>")])
3422
3423 (define_insn "vec_load_lanesxi<mode>"
3424   [(set (match_operand:XI 0 "register_operand" "=w")
3425         (unspec:XI [(match_operand:XI 1 "aarch64_simd_struct_operand" "Utv")
3426                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3427                    UNSPEC_LD4))]
3428   "TARGET_SIMD"
3429   "ld4\\t{%S0.<Vtype> - %V0.<Vtype>}, %1"
3430   [(set_attr "simd_type" "simd_load4")
3431    (set_attr "simd_mode" "<MODE>")])
3432
3433 (define_insn "vec_store_lanesxi<mode>"
3434   [(set (match_operand:XI 0 "aarch64_simd_struct_operand" "=Utv")
3435         (unspec:XI [(match_operand:XI 1 "register_operand" "w")
3436                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3437                    UNSPEC_ST4))]
3438   "TARGET_SIMD"
3439   "st4\\t{%S1.<Vtype> - %V1.<Vtype>}, %0"
3440   [(set_attr "simd_type" "simd_store4")
3441    (set_attr "simd_mode" "<MODE>")])
3442
3443 ;; Reload patterns for AdvSIMD register list operands.
3444
3445 (define_expand "mov<mode>"
3446   [(set (match_operand:VSTRUCT 0 "aarch64_simd_nonimmediate_operand" "")
3447         (match_operand:VSTRUCT 1 "aarch64_simd_general_operand" ""))]
3448   "TARGET_SIMD"
3449 {
3450   if (can_create_pseudo_p ())
3451     {
3452       if (GET_CODE (operands[0]) != REG)
3453         operands[1] = force_reg (<MODE>mode, operands[1]);
3454     }
3455 })
3456
3457 (define_insn "*aarch64_mov<mode>"
3458   [(set (match_operand:VSTRUCT 0 "aarch64_simd_nonimmediate_operand" "=w,Utv,w")
3459         (match_operand:VSTRUCT 1 "aarch64_simd_general_operand" " w,w,Utv"))]
3460   "TARGET_SIMD
3461    && (register_operand (operands[0], <MODE>mode)
3462        || register_operand (operands[1], <MODE>mode))"
3463
3464 {
3465   switch (which_alternative)
3466     {
3467     case 0: return "#";
3468     case 1: return "st1\\t{%S1.16b - %<Vendreg>1.16b}, %0";
3469     case 2: return "ld1\\t{%S0.16b - %<Vendreg>0.16b}, %1";
3470     default: gcc_unreachable ();
3471     }
3472 }
3473   [(set_attr "simd_type" "simd_move,simd_store<nregs>,simd_load<nregs>")
3474    (set (attr "length") (symbol_ref "aarch64_simd_attr_length_move (insn)"))
3475    (set_attr "simd_mode" "<MODE>")])
3476
3477 (define_split
3478   [(set (match_operand:OI 0 "register_operand" "")
3479         (match_operand:OI 1 "register_operand" ""))]
3480   "TARGET_SIMD && reload_completed"
3481   [(set (match_dup 0) (match_dup 1))
3482    (set (match_dup 2) (match_dup 3))]
3483 {
3484   int rdest = REGNO (operands[0]);
3485   int rsrc = REGNO (operands[1]);
3486   rtx dest[2], src[2];
3487
3488   dest[0] = gen_rtx_REG (TFmode, rdest);
3489   src[0] = gen_rtx_REG (TFmode, rsrc);
3490   dest[1] = gen_rtx_REG (TFmode, rdest + 1);
3491   src[1] = gen_rtx_REG (TFmode, rsrc + 1);
3492
3493   aarch64_simd_disambiguate_copy (operands, dest, src, 2);
3494 })
3495
3496 (define_split
3497   [(set (match_operand:CI 0 "register_operand" "")
3498         (match_operand:CI 1 "register_operand" ""))]
3499   "TARGET_SIMD && reload_completed"
3500   [(set (match_dup 0) (match_dup 1))
3501    (set (match_dup 2) (match_dup 3))
3502    (set (match_dup 4) (match_dup 5))]
3503 {
3504   int rdest = REGNO (operands[0]);
3505   int rsrc = REGNO (operands[1]);
3506   rtx dest[3], src[3];
3507
3508   dest[0] = gen_rtx_REG (TFmode, rdest);
3509   src[0] = gen_rtx_REG (TFmode, rsrc);
3510   dest[1] = gen_rtx_REG (TFmode, rdest + 1);
3511   src[1] = gen_rtx_REG (TFmode, rsrc + 1);
3512   dest[2] = gen_rtx_REG (TFmode, rdest + 2);
3513   src[2] = gen_rtx_REG (TFmode, rsrc + 2);
3514
3515   aarch64_simd_disambiguate_copy (operands, dest, src, 3);
3516 })
3517
3518 (define_split
3519   [(set (match_operand:XI 0 "register_operand" "")
3520         (match_operand:XI 1 "register_operand" ""))]
3521   "TARGET_SIMD && reload_completed"
3522   [(set (match_dup 0) (match_dup 1))
3523    (set (match_dup 2) (match_dup 3))
3524    (set (match_dup 4) (match_dup 5))
3525    (set (match_dup 6) (match_dup 7))]
3526 {
3527   int rdest = REGNO (operands[0]);
3528   int rsrc = REGNO (operands[1]);
3529   rtx dest[4], src[4];
3530
3531   dest[0] = gen_rtx_REG (TFmode, rdest);
3532   src[0] = gen_rtx_REG (TFmode, rsrc);
3533   dest[1] = gen_rtx_REG (TFmode, rdest + 1);
3534   src[1] = gen_rtx_REG (TFmode, rsrc + 1);
3535   dest[2] = gen_rtx_REG (TFmode, rdest + 2);
3536   src[2] = gen_rtx_REG (TFmode, rsrc + 2);
3537   dest[3] = gen_rtx_REG (TFmode, rdest + 3);
3538   src[3] = gen_rtx_REG (TFmode, rsrc + 3);
3539
3540   aarch64_simd_disambiguate_copy (operands, dest, src, 4);
3541 })
3542
3543 (define_insn "aarch64_ld2<mode>_dreg"
3544   [(set (match_operand:OI 0 "register_operand" "=w")
3545         (subreg:OI
3546           (vec_concat:<VRL2>
3547             (vec_concat:<VDBL>
3548              (unspec:VD [(match_operand:TI 1 "aarch64_simd_struct_operand" "Utv")]
3549                         UNSPEC_LD2)
3550              (vec_duplicate:VD (const_int 0)))
3551             (vec_concat:<VDBL>
3552              (unspec:VD [(match_dup 1)]
3553                         UNSPEC_LD2)
3554              (vec_duplicate:VD (const_int 0)))) 0))]
3555   "TARGET_SIMD"
3556   "ld2\\t{%S0.<Vtype> - %T0.<Vtype>}, %1"
3557   [(set_attr "simd_type" "simd_load2")
3558    (set_attr "simd_mode" "<MODE>")])
3559
3560 (define_insn "aarch64_ld2<mode>_dreg"
3561   [(set (match_operand:OI 0 "register_operand" "=w")
3562         (subreg:OI
3563           (vec_concat:<VRL2>
3564             (vec_concat:<VDBL>
3565              (unspec:DX [(match_operand:TI 1 "aarch64_simd_struct_operand" "Utv")]
3566                         UNSPEC_LD2)
3567              (const_int 0))
3568             (vec_concat:<VDBL>
3569              (unspec:DX [(match_dup 1)]
3570                         UNSPEC_LD2)
3571              (const_int 0))) 0))]
3572   "TARGET_SIMD"
3573   "ld1\\t{%S0.1d - %T0.1d}, %1"
3574   [(set_attr "simd_type" "simd_load2")
3575    (set_attr "simd_mode" "<MODE>")])
3576
3577 (define_insn "aarch64_ld3<mode>_dreg"
3578   [(set (match_operand:CI 0 "register_operand" "=w")
3579         (subreg:CI
3580          (vec_concat:<VRL3>
3581           (vec_concat:<VRL2>
3582             (vec_concat:<VDBL>
3583              (unspec:VD [(match_operand:EI 1 "aarch64_simd_struct_operand" "Utv")]
3584                         UNSPEC_LD3)
3585              (vec_duplicate:VD (const_int 0)))
3586             (vec_concat:<VDBL>
3587              (unspec:VD [(match_dup 1)]
3588                         UNSPEC_LD3)
3589              (vec_duplicate:VD (const_int 0))))
3590           (vec_concat:<VDBL>
3591              (unspec:VD [(match_dup 1)]
3592                         UNSPEC_LD3)
3593              (vec_duplicate:VD (const_int 0)))) 0))]
3594   "TARGET_SIMD"
3595   "ld3\\t{%S0.<Vtype> - %U0.<Vtype>}, %1"
3596   [(set_attr "simd_type" "simd_load3")
3597    (set_attr "simd_mode" "<MODE>")])
3598
3599 (define_insn "aarch64_ld3<mode>_dreg"
3600   [(set (match_operand:CI 0 "register_operand" "=w")
3601         (subreg:CI
3602          (vec_concat:<VRL3>
3603           (vec_concat:<VRL2>
3604             (vec_concat:<VDBL>
3605              (unspec:DX [(match_operand:EI 1 "aarch64_simd_struct_operand" "Utv")]
3606                         UNSPEC_LD3)
3607              (const_int 0))
3608             (vec_concat:<VDBL>
3609              (unspec:DX [(match_dup 1)]
3610                         UNSPEC_LD3)
3611              (const_int 0)))
3612           (vec_concat:<VDBL>
3613              (unspec:DX [(match_dup 1)]
3614                         UNSPEC_LD3)
3615              (const_int 0))) 0))]
3616   "TARGET_SIMD"
3617   "ld1\\t{%S0.1d - %U0.1d}, %1"
3618   [(set_attr "simd_type" "simd_load3")
3619    (set_attr "simd_mode" "<MODE>")])
3620
3621 (define_insn "aarch64_ld4<mode>_dreg"
3622   [(set (match_operand:XI 0 "register_operand" "=w")
3623         (subreg:XI
3624          (vec_concat:<VRL4>
3625            (vec_concat:<VRL2>
3626              (vec_concat:<VDBL>
3627                (unspec:VD [(match_operand:OI 1 "aarch64_simd_struct_operand" "Utv")]
3628                           UNSPEC_LD4)
3629                (vec_duplicate:VD (const_int 0)))
3630               (vec_concat:<VDBL>
3631                 (unspec:VD [(match_dup 1)]
3632                         UNSPEC_LD4)
3633                 (vec_duplicate:VD (const_int 0))))
3634            (vec_concat:<VRL2>
3635              (vec_concat:<VDBL>
3636                (unspec:VD [(match_dup 1)]
3637                         UNSPEC_LD4)
3638                (vec_duplicate:VD (const_int 0)))
3639              (vec_concat:<VDBL>
3640                (unspec:VD [(match_dup 1)]
3641                         UNSPEC_LD4)
3642                (vec_duplicate:VD (const_int 0))))) 0))]
3643   "TARGET_SIMD"
3644   "ld4\\t{%S0.<Vtype> - %V0.<Vtype>}, %1"
3645   [(set_attr "simd_type" "simd_load4")
3646    (set_attr "simd_mode" "<MODE>")])
3647
3648 (define_insn "aarch64_ld4<mode>_dreg"
3649   [(set (match_operand:XI 0 "register_operand" "=w")
3650         (subreg:XI
3651          (vec_concat:<VRL4>
3652            (vec_concat:<VRL2>
3653              (vec_concat:<VDBL>
3654                (unspec:DX [(match_operand:OI 1 "aarch64_simd_struct_operand" "Utv")]
3655                           UNSPEC_LD4)
3656                (const_int 0))
3657               (vec_concat:<VDBL>
3658                 (unspec:DX [(match_dup 1)]
3659                         UNSPEC_LD4)
3660                 (const_int 0)))
3661            (vec_concat:<VRL2>
3662              (vec_concat:<VDBL>
3663                (unspec:DX [(match_dup 1)]
3664                         UNSPEC_LD4)
3665                (const_int 0))
3666              (vec_concat:<VDBL>
3667                (unspec:DX [(match_dup 1)]
3668                         UNSPEC_LD4)
3669                (const_int 0)))) 0))]
3670   "TARGET_SIMD"
3671   "ld1\\t{%S0.1d - %V0.1d}, %1"
3672   [(set_attr "simd_type" "simd_load4")
3673    (set_attr "simd_mode" "<MODE>")])
3674
3675 (define_expand "aarch64_ld<VSTRUCT:nregs><VDC:mode>"
3676  [(match_operand:VSTRUCT 0 "register_operand" "=w")
3677   (match_operand:DI 1 "register_operand" "r")
3678   (unspec:VDC [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3679   "TARGET_SIMD"
3680 {
3681   enum machine_mode mode = <VSTRUCT:VSTRUCT_DREG>mode;
3682   rtx mem = gen_rtx_MEM (mode, operands[1]);
3683
3684   emit_insn (gen_aarch64_ld<VSTRUCT:nregs><VDC:mode>_dreg (operands[0], mem));
3685   DONE;
3686 })
3687
3688 (define_expand "aarch64_ld<VSTRUCT:nregs><VQ:mode>"
3689  [(match_operand:VSTRUCT 0 "register_operand" "=w")
3690   (match_operand:DI 1 "register_operand" "r")
3691   (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3692   "TARGET_SIMD"
3693 {
3694   enum machine_mode mode = <VSTRUCT:MODE>mode;
3695   rtx mem = gen_rtx_MEM (mode, operands[1]);
3696
3697   emit_insn (gen_vec_load_lanes<VSTRUCT:mode><VQ:mode> (operands[0], mem));
3698   DONE;
3699 })
3700
3701 ;; Expanders for builtins to extract vector registers from large
3702 ;; opaque integer modes.
3703
3704 ;; D-register list.
3705
3706 (define_expand "aarch64_get_dreg<VSTRUCT:mode><VDC:mode>"
3707  [(match_operand:VDC 0 "register_operand" "=w")
3708   (match_operand:VSTRUCT 1 "register_operand" "w")
3709   (match_operand:SI 2 "immediate_operand" "i")]
3710   "TARGET_SIMD"
3711 {
3712   int part = INTVAL (operands[2]);
3713   rtx temp = gen_reg_rtx (<VDC:VDBL>mode);
3714   int offset = part * 16;
3715
3716   emit_move_insn (temp, gen_rtx_SUBREG (<VDC:VDBL>mode, operands[1], offset));
3717   emit_move_insn (operands[0], gen_lowpart (<VDC:MODE>mode, temp));
3718   DONE;
3719 })
3720
3721 ;; Q-register list.
3722
3723 (define_expand "aarch64_get_qreg<VSTRUCT:mode><VQ:mode>"
3724  [(match_operand:VQ 0 "register_operand" "=w")
3725   (match_operand:VSTRUCT 1 "register_operand" "w")
3726   (match_operand:SI 2 "immediate_operand" "i")]
3727   "TARGET_SIMD"
3728 {
3729   int part = INTVAL (operands[2]);
3730   int offset = part * 16;
3731
3732   emit_move_insn (operands[0],
3733                   gen_rtx_SUBREG (<VQ:MODE>mode, operands[1], offset));
3734   DONE;
3735 })
3736
3737 ;; Permuted-store expanders for neon intrinsics.
3738
3739 ;; Permute instructions
3740
3741 ;; vec_perm support
3742
3743 (define_expand "vec_perm_const<mode>"
3744   [(match_operand:VALL 0 "register_operand")
3745    (match_operand:VALL 1 "register_operand")
3746    (match_operand:VALL 2 "register_operand")
3747    (match_operand:<V_cmp_result> 3)]
3748   "TARGET_SIMD"
3749 {
3750   if (aarch64_expand_vec_perm_const (operands[0], operands[1],
3751                                      operands[2], operands[3]))
3752     DONE;
3753   else
3754     FAIL;
3755 })
3756
3757 (define_expand "vec_perm<mode>"
3758   [(match_operand:VB 0 "register_operand")
3759    (match_operand:VB 1 "register_operand")
3760    (match_operand:VB 2 "register_operand")
3761    (match_operand:VB 3 "register_operand")]
3762   "TARGET_SIMD"
3763 {
3764   aarch64_expand_vec_perm (operands[0], operands[1],
3765                            operands[2], operands[3]);
3766   DONE;
3767 })
3768
3769 (define_insn "aarch64_tbl1<mode>"
3770   [(set (match_operand:VB 0 "register_operand" "=w")
3771         (unspec:VB [(match_operand:V16QI 1 "register_operand" "w")
3772                     (match_operand:VB 2 "register_operand" "w")]
3773                    UNSPEC_TBL))]
3774   "TARGET_SIMD"
3775   "tbl\\t%0.<Vtype>, {%1.16b}, %2.<Vtype>"
3776   [(set_attr "simd_type" "simd_tbl")
3777    (set_attr "simd_mode" "<MODE>")]
3778 )
3779
3780 ;; Two source registers.
3781
3782 (define_insn "aarch64_tbl2v16qi"
3783   [(set (match_operand:V16QI 0 "register_operand" "=w")
3784         (unspec:V16QI [(match_operand:OI 1 "register_operand" "w")
3785                        (match_operand:V16QI 2 "register_operand" "w")]
3786                       UNSPEC_TBL))]
3787   "TARGET_SIMD"
3788   "tbl\\t%0.16b, {%S1.16b - %T1.16b}, %2.16b"
3789   [(set_attr "simd_type" "simd_tbl")
3790    (set_attr "simd_mode" "V16QI")]
3791 )
3792
3793 (define_insn_and_split "aarch64_combinev16qi"
3794   [(set (match_operand:OI 0 "register_operand" "=w")
3795         (unspec:OI [(match_operand:V16QI 1 "register_operand" "w")
3796                     (match_operand:V16QI 2 "register_operand" "w")]
3797                    UNSPEC_CONCAT))]
3798   "TARGET_SIMD"
3799   "#"
3800   "&& reload_completed"
3801   [(const_int 0)]
3802 {
3803   aarch64_split_combinev16qi (operands);
3804   DONE;
3805 })
3806
3807 (define_insn "aarch64_<PERMUTE:perm_insn><PERMUTE:perm_hilo><mode>"
3808   [(set (match_operand:VALL 0 "register_operand" "=w")
3809         (unspec:VALL [(match_operand:VALL 1 "register_operand" "w")
3810                       (match_operand:VALL 2 "register_operand" "w")]
3811                        PERMUTE))]
3812   "TARGET_SIMD"
3813   "<PERMUTE:perm_insn><PERMUTE:perm_hilo>\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
3814   [(set_attr "simd_type" "simd_<PERMUTE:perm_insn>")
3815    (set_attr "simd_mode" "<MODE>")]
3816 )
3817
3818 (define_insn "aarch64_st2<mode>_dreg"
3819   [(set (match_operand:TI 0 "aarch64_simd_struct_operand" "=Utv")
3820         (unspec:TI [(match_operand:OI 1 "register_operand" "w")
3821                     (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3822                    UNSPEC_ST2))]
3823   "TARGET_SIMD"
3824   "st2\\t{%S1.<Vtype> - %T1.<Vtype>}, %0"
3825   [(set_attr "simd_type" "simd_store2")
3826    (set_attr "simd_mode" "<MODE>")])
3827
3828 (define_insn "aarch64_st2<mode>_dreg"
3829   [(set (match_operand:TI 0 "aarch64_simd_struct_operand" "=Utv")
3830         (unspec:TI [(match_operand:OI 1 "register_operand" "w")
3831                     (unspec:DX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3832                    UNSPEC_ST2))]
3833   "TARGET_SIMD"
3834   "st1\\t{%S1.1d - %T1.1d}, %0"
3835   [(set_attr "simd_type" "simd_store2")
3836    (set_attr "simd_mode" "<MODE>")])
3837
3838 (define_insn "aarch64_st3<mode>_dreg"
3839   [(set (match_operand:EI 0 "aarch64_simd_struct_operand" "=Utv")
3840         (unspec:EI [(match_operand:CI 1 "register_operand" "w")
3841                     (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3842                    UNSPEC_ST3))]
3843   "TARGET_SIMD"
3844   "st3\\t{%S1.<Vtype> - %U1.<Vtype>}, %0"
3845   [(set_attr "simd_type" "simd_store3")
3846    (set_attr "simd_mode" "<MODE>")])
3847
3848 (define_insn "aarch64_st3<mode>_dreg"
3849   [(set (match_operand:EI 0 "aarch64_simd_struct_operand" "=Utv")
3850         (unspec:EI [(match_operand:CI 1 "register_operand" "w")
3851                     (unspec:DX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3852                    UNSPEC_ST3))]
3853   "TARGET_SIMD"
3854   "st1\\t{%S1.1d - %U1.1d}, %0"
3855   [(set_attr "simd_type" "simd_store3")
3856    (set_attr "simd_mode" "<MODE>")])
3857
3858 (define_insn "aarch64_st4<mode>_dreg"
3859   [(set (match_operand:OI 0 "aarch64_simd_struct_operand" "=Utv")
3860         (unspec:OI [(match_operand:XI 1 "register_operand" "w")
3861                     (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3862                    UNSPEC_ST4))]
3863   "TARGET_SIMD"
3864   "st4\\t{%S1.<Vtype> - %V1.<Vtype>}, %0"
3865   [(set_attr "simd_type" "simd_store4")
3866    (set_attr "simd_mode" "<MODE>")])
3867
3868 (define_insn "aarch64_st4<mode>_dreg"
3869   [(set (match_operand:OI 0 "aarch64_simd_struct_operand" "=Utv")
3870         (unspec:OI [(match_operand:XI 1 "register_operand" "w")
3871                     (unspec:DX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3872                    UNSPEC_ST4))]
3873   "TARGET_SIMD"
3874   "st1\\t{%S1.1d - %V1.1d}, %0"
3875   [(set_attr "simd_type" "simd_store4")
3876    (set_attr "simd_mode" "<MODE>")])
3877
3878 (define_expand "aarch64_st<VSTRUCT:nregs><VDC:mode>"
3879  [(match_operand:DI 0 "register_operand" "r")
3880   (match_operand:VSTRUCT 1 "register_operand" "w")
3881   (unspec:VDC [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3882   "TARGET_SIMD"
3883 {
3884   enum machine_mode mode = <VSTRUCT:VSTRUCT_DREG>mode;
3885   rtx mem = gen_rtx_MEM (mode, operands[0]);
3886
3887   emit_insn (gen_aarch64_st<VSTRUCT:nregs><VDC:mode>_dreg (mem, operands[1]));
3888   DONE;
3889 })
3890
3891 (define_expand "aarch64_st<VSTRUCT:nregs><VQ:mode>"
3892  [(match_operand:DI 0 "register_operand" "r")
3893   (match_operand:VSTRUCT 1 "register_operand" "w")
3894   (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3895   "TARGET_SIMD"
3896 {
3897   enum machine_mode mode = <VSTRUCT:MODE>mode;
3898   rtx mem = gen_rtx_MEM (mode, operands[0]);
3899
3900   emit_insn (gen_vec_store_lanes<VSTRUCT:mode><VQ:mode> (mem, operands[1]));
3901   DONE;
3902 })
3903
3904 ;; Expander for builtins to insert vector registers into large
3905 ;; opaque integer modes.
3906
3907 ;; Q-register list.  We don't need a D-reg inserter as we zero
3908 ;; extend them in arm_neon.h and insert the resulting Q-regs.
3909
3910 (define_expand "aarch64_set_qreg<VSTRUCT:mode><VQ:mode>"
3911  [(match_operand:VSTRUCT 0 "register_operand" "+w")
3912   (match_operand:VSTRUCT 1 "register_operand" "0")
3913   (match_operand:VQ 2 "register_operand" "w")
3914   (match_operand:SI 3 "immediate_operand" "i")]
3915   "TARGET_SIMD"
3916 {
3917   int part = INTVAL (operands[3]);
3918   int offset = part * 16;
3919
3920   emit_move_insn (operands[0], operands[1]);
3921   emit_move_insn (gen_rtx_SUBREG (<VQ:MODE>mode, operands[0], offset),
3922                   operands[2]);
3923   DONE;
3924 })
3925
3926 ;; Standard pattern name vec_init<mode>.
3927
3928 (define_expand "vec_init<mode>"
3929   [(match_operand:VALL 0 "register_operand" "")
3930    (match_operand 1 "" "")]
3931   "TARGET_SIMD"
3932 {
3933   aarch64_expand_vector_init (operands[0], operands[1]);
3934   DONE;
3935 })
3936
3937 (define_insn "*aarch64_simd_ld1r<mode>"
3938   [(set (match_operand:VALLDI 0 "register_operand" "=w")
3939         (vec_duplicate:VALLDI
3940           (match_operand:<VEL> 1 "aarch64_simd_struct_operand" "Utv")))]
3941   "TARGET_SIMD"
3942   "ld1r\\t{%0.<Vtype>}, %1"
3943   [(set_attr "simd_type" "simd_load1r")
3944    (set_attr "simd_mode" "<MODE>")])
3945
3946 (define_insn "aarch64_frecpe<mode>"
3947   [(set (match_operand:VDQF 0 "register_operand" "=w")
3948         (unspec:VDQF [(match_operand:VDQF 1 "register_operand" "w")]
3949                     UNSPEC_FRECPE))]
3950   "TARGET_SIMD"
3951   "frecpe\\t%0.<Vtype>, %1.<Vtype>"
3952   [(set_attr "simd_type" "simd_frecpe")
3953    (set_attr "simd_mode" "<MODE>")]
3954 )
3955
3956 (define_insn "aarch64_frecps<mode>"
3957   [(set (match_operand:VDQF 0 "register_operand" "=w")
3958         (unspec:VDQF [(match_operand:VDQF 1 "register_operand" "w")
3959                      (match_operand:VDQF 2 "register_operand" "w")]
3960                     UNSPEC_FRECPS))]
3961   "TARGET_SIMD"
3962   "frecps\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
3963   [(set_attr "simd_type" "simd_frecps")
3964    (set_attr "simd_mode" "<MODE>")]
3965 )
3966