gcc/config/aarch64/aarch64-simd.md

   1 ;; Machine description for AArch64 AdvSIMD architecture.
   2 ;; Copyright (C) 2011-2013 Free Software Foundation, Inc.
   3 ;; Contributed by ARM Ltd.
   4 ;;
   5 ;; This file is part of GCC.
   6 ;;
   7 ;; GCC is free software; you can redistribute it and/or modify it
   8 ;; under the terms of the GNU General Public License as published by
   9 ;; the Free Software Foundation; either version 3, or (at your option)
  10 ;; any later version.
  11 ;;
  12 ;; GCC is distributed in the hope that it will be useful, but
  13 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
  14 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15 ;; General Public License for more details.
  16 ;;
  17 ;; You should have received a copy of the GNU General Public License
  18 ;; along with GCC; see the file COPYING3.  If not see
  19 ;; <http://www.gnu.org/licenses/>.
  20
  21
  22 ; Main data types used by the insntructions
  23
  24 (define_attr "simd_mode" "unknown,none,V8QI,V16QI,V4HI,V8HI,V2SI,V4SI,V2DI,V2SF,V4SF,V2DF,OI,CI,XI,DI,DF,SI,SF,HI,QI"
  25   (const_string "unknown"))
  26
  27
  28 ; Classification of AdvSIMD instructions for scheduling purposes.
  29 ; Do not set this attribute and the "v8type" attribute together in
  30 ; any instruction pattern.
  31
  32 ; simd_abd              integer absolute difference and accumulate.
  33 ; simd_abdl             integer absolute difference and accumulate (long).
  34 ; simd_adal             integer add and accumulate (long).
  35 ; simd_add              integer addition/subtraction.
  36 ; simd_addl             integer addition/subtraction (long).
  37 ; simd_addlv            across lanes integer sum (long).
  38 ; simd_addn             integer addition/subtraction (narrow).
  39 ; simd_addn2            integer addition/subtraction (narrow, high).
  40 ; simd_addv             across lanes integer sum.
  41 ; simd_cls              count leading sign/zero bits.
  42 ; simd_cmp              compare / create mask.
  43 ; simd_cnt              population count.
  44 ; simd_dup              duplicate element.
  45 ; simd_dupgp            duplicate general purpose register.
  46 ; simd_ext              bitwise extract from pair.
  47 ; simd_fabd             floating absolute difference and accumulate.
  48 ; simd_fadd             floating point add/sub.
  49 ; simd_fcmp             floating point compare.
  50 ; simd_fcvti            floating point convert to integer.
  51 ; simd_fcvtl            floating-point convert upsize.
  52 ; simd_fcvtn            floating-point convert downsize (narrow).
  53 ; simd_fcvtn2           floating-point convert downsize (narrow, high).
  54 ; simd_fdiv             floating point division.
  55 ; simd_fminmax          floating point min/max.
  56 ; simd_fminmaxv         across lanes floating point min/max.
  57 ; simd_fmla             floating point multiply-add.
  58 ; simd_fmla_elt         floating point multiply-add (by element).
  59 ; simd_fmul             floating point multiply.
  60 ; simd_fmul_elt         floating point multiply (by element).
  61 ; simd_fnegabs          floating point neg/abs.
  62 ; simd_frecpe            floating point reciprocal estimate.
  63 ; simd_frecps            floating point reciprocal step.
  64 ; simd_frecpx            floating point reciprocal exponent.
  65 ; simd_frint            floating point round to integer.
  66 ; simd_fsqrt            floating point square root.
  67 ; simd_icvtf            integer convert to floating point.
  68 ; simd_ins              insert element.
  69 ; simd_insgp            insert general purpose register.
  70 ; simd_load1            load multiple structures to one register (LD1).
  71 ; simd_load1r           load single structure to all lanes of one register (LD1R).
  72 ; simd_load1s           load single structure to one lane of one register (LD1 [index]).
  73 ; simd_load2            load multiple structures to two registers (LD1, LD2).
  74 ; simd_load2r           load single structure to all lanes of two registers (LD1R, LD2R).
  75 ; simd_load2s           load single structure to one lane of two registers (LD2 [index]).
  76 ; simd_load3            load multiple structures to three registers (LD1, LD3).
  77 ; simd_load3r           load single structure to all lanes of three registers (LD3R).
  78 ; simd_load3s           load single structure to one lane of three registers (LD3 [index]).
  79 ; simd_load4            load multiple structures to four registers (LD1, LD2, LD4).
  80 ; simd_load4r           load single structure to all lanes of four registers (LD4R).
  81 ; simd_load4s           load single structure to one lane of four registers (LD4 [index]).
  82 ; simd_logic            logical operation.
  83 ; simd_logic_imm        logcial operation (immediate).
  84 ; simd_minmax           integer min/max.
  85 ; simd_minmaxv          across lanes integer min/max,
  86 ; simd_mla              integer multiply-accumulate.
  87 ; simd_mla_elt          integer multiply-accumulate (by element).
  88 ; simd_mlal             integer multiply-accumulate (long).
  89 ; simd_mlal_elt         integer multiply-accumulate (by element, long).
  90 ; simd_move             move register.
  91 ; simd_move_imm         move immediate.
  92 ; simd_movgp            move element to general purpose register.
  93 ; simd_mul              integer multiply.
  94 ; simd_mul_elt          integer multiply (by element).
  95 ; simd_mull             integer multiply (long).
  96 ; simd_mull_elt         integer multiply (by element, long).
  97 ; simd_negabs           integer negate/absolute.
  98 ; simd_rbit             bitwise reverse.
  99 ; simd_rcpe             integer reciprocal estimate.
 100 ; simd_rcps             integer reciprocal square root.
 101 ; simd_rev              element reverse.
 102 ; simd_sat_add          integer saturating addition/subtraction.
 103 ; simd_sat_mlal         integer saturating multiply-accumulate (long).
 104 ; simd_sat_mlal_elt     integer saturating multiply-accumulate (by element, long).
 105 ; simd_sat_mul          integer saturating multiply.
 106 ; simd_sat_mul_elt      integer saturating multiply (by element).
 107 ; simd_sat_mull         integer saturating multiply (long).
 108 ; simd_sat_mull_elt     integer saturating multiply (by element, long).
 109 ; simd_sat_negabs       integer saturating negate/absolute.
 110 ; simd_sat_shift        integer saturating shift.
 111 ; simd_sat_shift_imm    integer saturating shift (immediate).
 112 ; simd_sat_shiftn_imm   integer saturating shift (narrow, immediate).
 113 ; simd_sat_shiftn2_imm  integer saturating shift (narrow, high, immediate).
 114 ; simd_shift            shift register/vector.
 115 ; simd_shift_acc        shift accumulate.
 116 ; simd_shift_imm        shift immediate.
 117 ; simd_shift_imm_acc    shift immediate and accumualte.
 118 ; simd_shiftl           shift register/vector (long).
 119 ; simd_shiftl_imm       shift register/vector (long, immediate).
 120 ; simd_shiftn_imm       shift register/vector (narrow, immediate).
 121 ; simd_shiftn2_imm      shift register/vector (narrow, high, immediate).
 122 ; simd_store1           store multiple structures from one register (ST1).
 123 ; simd_store1s          store single structure from one lane of one register (ST1 [index]).
 124 ; simd_store2           store multiple structures from two registers (ST1, ST2).
 125 ; simd_store2s          store single structure from one lane of two registers (ST2 [index]).
 126 ; simd_store3           store multiple structures from three registers (ST1, ST3).
 127 ; simd_store3s          store single structure from one lane of three register (ST3 [index]).
 128 ; simd_store4           store multiple structures from four registers (ST1, ST2, ST4).
 129 ; simd_store4s          store single structure from one lane for four registers (ST4 [index]).
 130 ; simd_tbl              table lookup.
 131 ; simd_trn              transpose.
 132 ; simd_uzp              unzip.
 133 ; simd_zip              zip.
 134
 135 (define_attr "simd_type"
 136    "simd_abd,\
 137    simd_abdl,\
 138    simd_adal,\
 139    simd_add,\
 140    simd_addl,\
 141    simd_addlv,\
 142    simd_addn,\
 143    simd_addn2,\
 144    simd_addv,\
 145    simd_cls,\
 146    simd_cmp,\
 147    simd_cnt,\
 148    simd_dup,\
 149    simd_dupgp,\
 150    simd_ext,\
 151    simd_fabd,\
 152    simd_fadd,\
 153    simd_fcmp,\
 154    simd_fcvti,\
 155    simd_fcvtl,\
 156    simd_fcvtn,\
 157    simd_fcvtn2,\
 158    simd_fdiv,\
 159    simd_fminmax,\
 160    simd_fminmaxv,\
 161    simd_fmla,\
 162    simd_fmla_elt,\
 163    simd_fmul,\
 164    simd_fmul_elt,\
 165    simd_fnegabs,\
 166    simd_frecpe,\
 167    simd_frecps,\
 168    simd_frecpx,\
 169    simd_frint,\
 170    simd_fsqrt,\
 171    simd_icvtf,\
 172    simd_ins,\
 173    simd_insgp,\
 174    simd_load1,\
 175    simd_load1r,\
 176    simd_load1s,\
 177    simd_load2,\
 178    simd_load2r,\
 179    simd_load2s,\
 180    simd_load3,\
 181    simd_load3r,\
 182    simd_load3s,\
 183    simd_load4,\
 184    simd_load4r,\
 185    simd_load4s,\
 186    simd_logic,\
 187    simd_logic_imm,\
 188    simd_minmax,\
 189    simd_minmaxv,\
 190    simd_mla,\
 191    simd_mla_elt,\
 192    simd_mlal,\
 193    simd_mlal_elt,\
 194    simd_movgp,\
 195    simd_move,\
 196    simd_move_imm,\
 197    simd_mul,\
 198    simd_mul_elt,\
 199    simd_mull,\
 200    simd_mull_elt,\
 201    simd_negabs,\
 202    simd_rbit,\
 203    simd_rcpe,\
 204    simd_rcps,\
 205    simd_rev,\
 206    simd_sat_add,\
 207    simd_sat_mlal,\
 208    simd_sat_mlal_elt,\
 209    simd_sat_mul,\
 210    simd_sat_mul_elt,\
 211    simd_sat_mull,\
 212    simd_sat_mull_elt,\
 213    simd_sat_negabs,\
 214    simd_sat_shift,\
 215    simd_sat_shift_imm,\
 216    simd_sat_shiftn_imm,\
 217    simd_sat_shiftn2_imm,\
 218    simd_shift,\
 219    simd_shift_acc,\
 220    simd_shift_imm,\
 221    simd_shift_imm_acc,\
 222    simd_shiftl,\
 223    simd_shiftl_imm,\
 224    simd_shiftn_imm,\
 225    simd_shiftn2_imm,\
 226    simd_store1,\
 227    simd_store1s,\
 228    simd_store2,\
 229    simd_store2s,\
 230    simd_store3,\
 231    simd_store3s,\
 232    simd_store4,\
 233    simd_store4s,\
 234    simd_tbl,\
 235    simd_trn,\
 236    simd_uzp,\
 237    simd_zip,\
 238    none"
 239   (const_string "none"))
 240
 241
 242 ; The "neon_type" attribute is used by the AArch32 backend.  Below is a mapping
 243 ; from "simd_type" to "neon_type".
 244
 245 (define_attr "neon_type"
 246    "neon_int_1,neon_int_2,neon_int_3,neon_int_4,neon_int_5,neon_vqneg_vqabs,
 247    neon_vmov,neon_vaba,neon_vsma,neon_vaba_qqq,
 248    neon_mul_ddd_8_16_qdd_16_8_long_32_16_long,neon_mul_qqq_8_16_32_ddd_32,
 249    neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar,
 250    neon_mla_ddd_8_16_qdd_16_8_long_32_16_long,neon_mla_qqq_8_16,
 251    neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long,
 252    neon_mla_qqq_32_qqd_32_scalar,neon_mul_ddd_16_scalar_32_16_long_scalar,
 253    neon_mul_qqd_32_scalar,neon_mla_ddd_16_scalar_qdd_32_16_long_scalar,
 254    neon_shift_1,neon_shift_2,neon_shift_3,neon_vshl_ddd,
 255    neon_vqshl_vrshl_vqrshl_qqq,neon_vsra_vrsra,neon_fp_vadd_ddd_vabs_dd,
 256    neon_fp_vadd_qqq_vabs_qq,neon_fp_vsum,neon_fp_vmul_ddd,neon_fp_vmul_qqd,
 257    neon_fp_vmla_ddd,neon_fp_vmla_qqq,neon_fp_vmla_ddd_scalar,
 258    neon_fp_vmla_qqq_scalar,neon_fp_vrecps_vrsqrts_ddd,
 259    neon_fp_vrecps_vrsqrts_qqq,neon_bp_simple,neon_bp_2cycle,neon_bp_3cycle,
 260    neon_ldr,neon_str,neon_vld1_1_2_regs,neon_vld1_3_4_regs,
 261    neon_vld2_2_regs_vld1_vld2_all_lanes,neon_vld2_4_regs,neon_vld3_vld4,
 262    neon_vst1_1_2_regs_vst2_2_regs,neon_vst1_3_4_regs,
 263    neon_vst2_4_regs_vst3_vst4,neon_vst3_vst4,neon_vld1_vld2_lane,
 264    neon_vld3_vld4_lane,neon_vst1_vst2_lane,neon_vst3_vst4_lane,
 265    neon_vld3_vld4_all_lanes,neon_mcr,neon_mcr_2_mcrr,neon_mrc,neon_mrrc,
 266    neon_ldm_2,neon_stm_2,none,unknown"
 267   (cond [
 268           (eq_attr "simd_type" "simd_dup") (const_string "neon_bp_simple")
 269           (eq_attr "simd_type" "simd_movgp") (const_string "neon_bp_simple")
 270           (eq_attr "simd_type" "simd_add,simd_logic,simd_logic_imm") (const_string "neon_int_1")
 271           (eq_attr "simd_type" "simd_negabs,simd_addlv") (const_string "neon_int_3")
 272           (eq_attr "simd_type" "simd_addn,simd_addn2,simd_addl,simd_sat_add,simd_sat_negabs") (const_string "neon_int_4")
 273           (eq_attr "simd_type" "simd_move") (const_string "neon_vmov")
 274           (eq_attr "simd_type" "simd_ins") (const_string "neon_mcr")
 275           (and (eq_attr "simd_type" "simd_mul,simd_sat_mul") (eq_attr "simd_mode" "V8QI,V4HI")) (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long")
 276           (and (eq_attr "simd_type" "simd_mul,simd_sat_mul") (eq_attr "simd_mode" "V2SI,V8QI,V16QI,V2SI")) (const_string "neon_mul_qqq_8_16_32_ddd_32")
 277           (and (eq_attr "simd_type" "simd_mull,simd_sat_mull") (eq_attr "simd_mode" "V8QI,V16QI,V4HI,V8HI")) (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long")
 278           (and (eq_attr "simd_type" "simd_mull,simd_sat_mull") (eq_attr "simd_mode" "V2SI,V4SI,V2DI")) (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")
 279           (and (eq_attr "simd_type" "simd_mla,simd_sat_mlal") (eq_attr "simd_mode" "V8QI,V4HI")) (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
 280           (and (eq_attr "simd_type" "simd_mla,simd_sat_mlal") (eq_attr "simd_mode" "V2SI")) (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")
 281           (and (eq_attr "simd_type" "simd_mla,simd_sat_mlal") (eq_attr "simd_mode" "V16QI,V8HI")) (const_string "neon_mla_qqq_8_16")
 282           (and (eq_attr "simd_type" "simd_mla,simd_sat_mlal") (eq_attr "simd_mode" "V4SI")) (const_string "neon_mla_qqq_32_qqd_32_scalar")
 283           (and (eq_attr "simd_type" "simd_mlal") (eq_attr "simd_mode" "V8QI,V16QI,V4HI,V8HI")) (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
 284           (and (eq_attr "simd_type" "simd_mlal") (eq_attr "simd_mode" "V2SI,V4SI,V2DI")) (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")
 285           (and (eq_attr "simd_type" "simd_fmla") (eq_attr "simd_mode" "V2SF")) (const_string "neon_fp_vmla_ddd")
 286           (and (eq_attr "simd_type" "simd_fmla") (eq_attr "simd_mode" "V4SF,V2DF")) (const_string "neon_fp_vmla_qqq")
 287           (and (eq_attr "simd_type" "simd_fmla_elt") (eq_attr "simd_mode" "V2SF")) (const_string "neon_fp_vmla_ddd_scalar")
 288           (and (eq_attr "simd_type" "simd_fmla_elt") (eq_attr "simd_mode" "V4SF,V2DF")) (const_string "neon_fp_vmla_qqq_scalar")
 289           (and (eq_attr "simd_type" "simd_fmul,simd_fmul_elt,simd_fdiv,simd_fsqrt") (eq_attr "simd_mode" "V2SF")) (const_string "neon_fp_vmul_ddd")
 290           (and (eq_attr "simd_type" "simd_fmul,simd_fmul_elt,simd_fdiv,simd_fsqrt") (eq_attr "simd_mode" "V4SF,V2DF")) (const_string "neon_fp_vmul_qqd")
 291           (and (eq_attr "simd_type" "simd_fadd") (eq_attr "simd_mode" "V2SF")) (const_string "neon_fp_vadd_ddd_vabs_dd")
 292           (and (eq_attr "simd_type" "simd_fadd") (eq_attr "simd_mode" "V4SF,V2DF")) (const_string "neon_fp_vadd_qqq_vabs_qq")
 293           (and (eq_attr "simd_type" "simd_fnegabs,simd_fminmax,simd_fminmaxv") (eq_attr "simd_mode" "V2SF")) (const_string "neon_fp_vadd_ddd_vabs_dd")
 294           (and (eq_attr "simd_type" "simd_fnegabs,simd_fminmax,simd_fminmaxv") (eq_attr "simd_mode" "V4SF,V2DF")) (const_string "neon_fp_vadd_qqq_vabs_qq")
 295           (and (eq_attr "simd_type" "simd_shift,simd_shift_acc") (eq_attr "simd_mode" "V8QI,V4HI,V2SI")) (const_string "neon_vshl_ddd")
 296           (and (eq_attr "simd_type" "simd_shift,simd_shift_acc") (eq_attr "simd_mode" "V16QI,V8HI,V4SI,V2DI")) (const_string "neon_shift_3")
 297           (eq_attr "simd_type" "simd_minmax,simd_minmaxv") (const_string "neon_int_5")
 298           (eq_attr "simd_type" "simd_shiftn_imm,simd_shiftn2_imm,simd_shiftl_imm,") (const_string "neon_shift_1")
 299           (eq_attr "simd_type" "simd_load1,simd_load2") (const_string "neon_vld1_1_2_regs")
 300           (eq_attr "simd_type" "simd_load3,simd_load3") (const_string "neon_vld1_3_4_regs")
 301           (eq_attr "simd_type" "simd_load1r,simd_load2r,simd_load3r,simd_load4r") (const_string "neon_vld2_2_regs_vld1_vld2_all_lanes")
 302           (eq_attr "simd_type" "simd_load1s,simd_load2s") (const_string "neon_vld1_vld2_lane")
 303           (eq_attr "simd_type" "simd_load3s,simd_load4s") (const_string "neon_vld3_vld4_lane")
 304           (eq_attr "simd_type" "simd_store1,simd_store2") (const_string "neon_vst1_1_2_regs_vst2_2_regs")
 305           (eq_attr "simd_type" "simd_store3,simd_store4") (const_string "neon_vst1_3_4_regs")
 306           (eq_attr "simd_type" "simd_store1s,simd_store2s") (const_string "neon_vst1_vst2_lane")
 307           (eq_attr "simd_type" "simd_store3s,simd_store4s") (const_string "neon_vst3_vst4_lane")
 308           (and (eq_attr "simd_type" "simd_frecpe,simd_frecps") (eq_attr "simd_mode" "V2SF")) (const_string "neon_fp_vrecps_vrsqrts_ddd")
 309           (and (eq_attr "simd_type" "simd_frecpe,simd_frecps") (eq_attr "simd_mode" "V4SF,V2DF")) (const_string "neon_fp_vrecps_vrsqrts_qqq")
 310           (eq_attr "simd_type" "none") (const_string "none")
 311   ]
 312   (const_string "unknown")))
 313
 314
 315 (define_expand "mov<mode>"
 316   [(set (match_operand:VALL 0 "aarch64_simd_nonimmediate_operand" "")
 317         (match_operand:VALL 1 "aarch64_simd_general_operand" ""))]
 318   "TARGET_SIMD"
 319   "
 320     if (GET_CODE (operands[0]) == MEM)
 321       operands[1] = force_reg (<MODE>mode, operands[1]);
 322   "
 323 )
 324
 325 (define_expand "movmisalign<mode>"
 326   [(set (match_operand:VALL 0 "aarch64_simd_nonimmediate_operand" "")
 327         (match_operand:VALL 1 "aarch64_simd_general_operand" ""))]
 328   "TARGET_SIMD"
 329 {
 330   /* This pattern is not permitted to fail during expansion: if both arguments
 331      are non-registers (e.g. memory := constant, which can be created by the
 332      auto-vectorizer), force operand 1 into a register.  */
 333   if (!register_operand (operands[0], <MODE>mode)
 334       && !register_operand (operands[1], <MODE>mode))
 335     operands[1] = force_reg (<MODE>mode, operands[1]);
 336 })
 337
 338 (define_insn "aarch64_simd_dup<mode>"
 339   [(set (match_operand:VDQ 0 "register_operand" "=w")
 340         (vec_duplicate:VDQ (match_operand:<VEL> 1 "register_operand" "r")))]
 341   "TARGET_SIMD"
 342   "dup\\t%0.<Vtype>, %<vw>1"
 343   [(set_attr "simd_type" "simd_dupgp")
 344    (set_attr "simd_mode" "<MODE>")]
 345 )
 346
 347 (define_insn "aarch64_dup_lane<mode>"
 348   [(set (match_operand:VDQ_I 0 "register_operand" "=w")
 349         (vec_duplicate:VDQ_I
 350           (vec_select:<VEL>
 351             (match_operand:<VCON> 1 "register_operand" "w")
 352             (parallel [(match_operand:SI 2 "immediate_operand" "i")])
 353           )))]
 354   "TARGET_SIMD"
 355   "dup\\t%<v>0<Vmtype>, %1.<Vetype>[%2]"
 356   [(set_attr "simd_type" "simd_dup")
 357    (set_attr "simd_mode" "<MODE>")]
 358 )
 359
 360 (define_insn "aarch64_dup_lane<mode>"
 361   [(set (match_operand:SDQ_I 0 "register_operand" "=w")
 362         (vec_select:<VEL>
 363           (match_operand:<VCON> 1 "register_operand" "w")
 364           (parallel [(match_operand:SI 2 "immediate_operand" "i")])
 365         ))]
 366   "TARGET_SIMD"
 367   "dup\\t%<v>0<Vmtype>, %1.<Vetype>[%2]"
 368   [(set_attr "simd_type" "simd_dup")
 369    (set_attr "simd_mode" "<MODE>")]
 370 )
 371
 372 (define_insn "aarch64_simd_dup<mode>"
 373   [(set (match_operand:VDQF 0 "register_operand" "=w")
 374         (vec_duplicate:VDQF (match_operand:<VEL> 1 "register_operand" "w")))]
 375   "TARGET_SIMD"
 376   "dup\\t%0.<Vtype>, %1.<Vetype>[0]"
 377   [(set_attr "simd_type" "simd_dup")
 378    (set_attr "simd_mode" "<MODE>")]
 379 )
 380
 381 (define_insn "*aarch64_simd_mov<mode>"
 382   [(set (match_operand:VD 0 "aarch64_simd_nonimmediate_operand"
 383                 "=w, Utv,  w, ?r, ?w, ?r, w")
 384         (match_operand:VD 1 "aarch64_simd_general_operand"
 385                 "Utv,  w,  w,  w,  r,  r, Dn"))]
 386   "TARGET_SIMD
 387    && (register_operand (operands[0], <MODE>mode)
 388        || register_operand (operands[1], <MODE>mode))"
 389 {
 390    switch (which_alternative)
 391      {
 392      case 0: return "ld1\t{%0.<Vtype>}, %1";
 393      case 1: return "st1\t{%1.<Vtype>}, %0";
 394      case 2: return "orr\t%0.<Vbtype>, %1.<Vbtype>, %1.<Vbtype>";
 395      case 3: return "umov\t%0, %1.d[0]";
 396      case 4: return "ins\t%0.d[0], %1";
 397      case 5: return "mov\t%0, %1";
 398      case 6:
 399         return aarch64_output_simd_mov_immediate (&operands[1],
 400                                                   <MODE>mode, 64);
 401      default: gcc_unreachable ();
 402      }
 403 }
 404   [(set_attr "simd_type" "simd_load1,simd_store1,simd_move,simd_movgp,simd_insgp,simd_move,simd_move_imm")
 405    (set_attr "simd_mode" "<MODE>")]
 406 )
 407
 408 (define_insn "*aarch64_simd_mov<mode>"
 409   [(set (match_operand:VQ 0 "aarch64_simd_nonimmediate_operand"
 410                 "=w, Utv,  w, ?r, ?w, ?r, w")
 411         (match_operand:VQ 1 "aarch64_simd_general_operand"
 412                 "Utv,  w,  w,  w,  r,  r, Dn"))]
 413   "TARGET_SIMD
 414    && (register_operand (operands[0], <MODE>mode)
 415        || register_operand (operands[1], <MODE>mode))"
 416 {
 417   switch (which_alternative)
 418     {
 419     case 0: return "ld1\t{%0.<Vtype>}, %1";
 420     case 1: return "st1\t{%1.<Vtype>}, %0";
 421     case 2: return "orr\t%0.<Vbtype>, %1.<Vbtype>, %1.<Vbtype>";
 422     case 3: return "umov\t%0, %1.d[0]\;umov\t%H0, %1.d[1]";
 423     case 4: return "ins\t%0.d[0], %1\;ins\t%0.d[1], %H1";
 424     case 5: return "#";
 425     case 6:
 426         return aarch64_output_simd_mov_immediate (&operands[1],
 427                                                   <MODE>mode, 128);
 428     default: gcc_unreachable ();
 429     }
 430 }
 431   [(set_attr "simd_type" "simd_load1,simd_store1,simd_move,simd_movgp,simd_insgp,simd_move,simd_move_imm")
 432    (set_attr "simd_mode" "<MODE>")
 433    (set_attr "length" "4,4,4,8,8,8,4")]
 434 )
 435
 436 (define_split
 437   [(set (match_operand:VQ 0 "register_operand" "")
 438       (match_operand:VQ 1 "register_operand" ""))]
 439   "TARGET_SIMD && reload_completed
 440    && GP_REGNUM_P (REGNO (operands[0]))
 441    && GP_REGNUM_P (REGNO (operands[1]))"
 442   [(set (match_dup 0) (match_dup 1))
 443    (set (match_dup 2) (match_dup 3))]
 444 {
 445   int rdest = REGNO (operands[0]);
 446   int rsrc = REGNO (operands[1]);
 447   rtx dest[2], src[2];
 448
 449   dest[0] = gen_rtx_REG (DImode, rdest);
 450   src[0] = gen_rtx_REG (DImode, rsrc);
 451   dest[1] = gen_rtx_REG (DImode, rdest + 1);
 452   src[1] = gen_rtx_REG (DImode, rsrc + 1);
 453
 454   aarch64_simd_disambiguate_copy (operands, dest, src, 2);
 455 })
 456
 457 (define_insn "orn<mode>3"
 458  [(set (match_operand:VDQ 0 "register_operand" "=w")
 459        (ior:VDQ (not:VDQ (match_operand:VDQ 1 "register_operand" "w"))
 460                 (match_operand:VDQ 2 "register_operand" "w")))]
 461  "TARGET_SIMD"
 462  "orn\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>"
 463   [(set_attr "simd_type" "simd_logic")
 464    (set_attr "simd_mode" "<MODE>")]
 465 )
 466
 467 (define_insn "bic<mode>3"
 468  [(set (match_operand:VDQ 0 "register_operand" "=w")
 469        (and:VDQ (not:VDQ (match_operand:VDQ 1 "register_operand" "w"))
 470                 (match_operand:VDQ 2 "register_operand" "w")))]
 471  "TARGET_SIMD"
 472  "bic\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>"
 473   [(set_attr "simd_type" "simd_logic")
 474    (set_attr "simd_mode" "<MODE>")]
 475 )
 476
 477 (define_insn "add<mode>3"
 478   [(set (match_operand:VDQ 0 "register_operand" "=w")
 479         (plus:VDQ (match_operand:VDQ 1 "register_operand" "w")
 480                   (match_operand:VDQ 2 "register_operand" "w")))]
 481   "TARGET_SIMD"
 482   "add\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
 483   [(set_attr "simd_type" "simd_add")
 484    (set_attr "simd_mode" "<MODE>")]
 485 )
 486
 487 (define_insn "sub<mode>3"
 488   [(set (match_operand:VDQ 0 "register_operand" "=w")
 489         (minus:VDQ (match_operand:VDQ 1 "register_operand" "w")
 490                    (match_operand:VDQ 2 "register_operand" "w")))]
 491   "TARGET_SIMD"
 492   "sub\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
 493   [(set_attr "simd_type" "simd_add")
 494    (set_attr "simd_mode" "<MODE>")]
 495 )
 496
 497 (define_insn "mul<mode>3"
 498   [(set (match_operand:VDQM 0 "register_operand" "=w")
 499         (mult:VDQM (match_operand:VDQM 1 "register_operand" "w")
 500                    (match_operand:VDQM 2 "register_operand" "w")))]
 501   "TARGET_SIMD"
 502   "mul\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
 503   [(set_attr "simd_type" "simd_mul")
 504    (set_attr "simd_mode" "<MODE>")]
 505 )
 506
 507 (define_insn "neg<mode>2"
 508   [(set (match_operand:VDQ 0 "register_operand" "=w")
 509         (neg:VDQ (match_operand:VDQ 1 "register_operand" "w")))]
 510   "TARGET_SIMD"
 511   "neg\t%0.<Vtype>, %1.<Vtype>"
 512   [(set_attr "simd_type" "simd_negabs")
 513    (set_attr "simd_mode" "<MODE>")]
 514 )
 515
 516 (define_insn "abs<mode>2"
 517   [(set (match_operand:VDQ 0 "register_operand" "=w")
 518         (abs:VDQ (match_operand:VDQ 1 "register_operand" "w")))]
 519   "TARGET_SIMD"
 520   "abs\t%0.<Vtype>, %1.<Vtype>"
 521   [(set_attr "simd_type" "simd_negabs")
 522    (set_attr "simd_mode" "<MODE>")]
 523 )
 524
 525 (define_insn "abd<mode>_3"
 526   [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
 527         (abs:VDQ_BHSI (minus:VDQ_BHSI
 528                        (match_operand:VDQ_BHSI 1 "register_operand" "w")
 529                        (match_operand:VDQ_BHSI 2 "register_operand" "w"))))]
 530   "TARGET_SIMD"
 531   "sabd\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
 532   [(set_attr "simd_type" "simd_abd")
 533    (set_attr "simd_mode" "<MODE>")]
 534 )
 535
 536 (define_insn "aba<mode>_3"
 537   [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
 538         (plus:VDQ_BHSI (abs:VDQ_BHSI (minus:VDQ_BHSI
 539                          (match_operand:VDQ_BHSI 1 "register_operand" "w")
 540                          (match_operand:VDQ_BHSI 2 "register_operand" "w")))
 541                        (match_operand:VDQ_BHSI 3 "register_operand" "0")))]
 542   "TARGET_SIMD"
 543   "saba\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
 544   [(set_attr "simd_type" "simd_abd")
 545    (set_attr "simd_mode" "<MODE>")]
 546 )
 547
 548 (define_insn "fabd<mode>_3"
 549   [(set (match_operand:VDQF 0 "register_operand" "=w")
 550         (abs:VDQF (minus:VDQF
 551                    (match_operand:VDQF 1 "register_operand" "w")
 552                    (match_operand:VDQF 2 "register_operand" "w"))))]
 553   "TARGET_SIMD"
 554   "fabd\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
 555   [(set_attr "simd_type" "simd_fabd")
 556    (set_attr "simd_mode" "<MODE>")]
 557 )
 558
 559 (define_insn "and<mode>3"
 560   [(set (match_operand:VDQ 0 "register_operand" "=w")
 561         (and:VDQ (match_operand:VDQ 1 "register_operand" "w")
 562                  (match_operand:VDQ 2 "register_operand" "w")))]
 563   "TARGET_SIMD"
 564   "and\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>"
 565   [(set_attr "simd_type" "simd_logic")
 566    (set_attr "simd_mode" "<MODE>")]
 567 )
 568
 569 (define_insn "ior<mode>3"
 570   [(set (match_operand:VDQ 0 "register_operand" "=w")
 571         (ior:VDQ (match_operand:VDQ 1 "register_operand" "w")
 572                  (match_operand:VDQ 2 "register_operand" "w")))]
 573   "TARGET_SIMD"
 574   "orr\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>"
 575   [(set_attr "simd_type" "simd_logic")
 576    (set_attr "simd_mode" "<MODE>")]
 577 )
 578
 579 (define_insn "xor<mode>3"
 580   [(set (match_operand:VDQ 0 "register_operand" "=w")
 581         (xor:VDQ (match_operand:VDQ 1 "register_operand" "w")
 582                  (match_operand:VDQ 2 "register_operand" "w")))]
 583   "TARGET_SIMD"
 584   "eor\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>"
 585   [(set_attr "simd_type" "simd_logic")
 586    (set_attr "simd_mode" "<MODE>")]
 587 )
 588
 589 (define_insn "one_cmpl<mode>2"
 590   [(set (match_operand:VDQ 0 "register_operand" "=w")
 591         (not:VDQ (match_operand:VDQ 1 "register_operand" "w")))]
 592   "TARGET_SIMD"
 593   "not\t%0.<Vbtype>, %1.<Vbtype>"
 594   [(set_attr "simd_type" "simd_logic")
 595    (set_attr "simd_mode" "<MODE>")]
 596 )
 597
 598 (define_insn "aarch64_simd_vec_set<mode>"
 599   [(set (match_operand:VQ_S 0 "register_operand" "=w")
 600         (vec_merge:VQ_S
 601             (vec_duplicate:VQ_S
 602                 (match_operand:<VEL> 1 "register_operand" "r"))
 603             (match_operand:VQ_S 3 "register_operand" "0")
 604             (match_operand:SI 2 "immediate_operand" "i")))]
 605   "TARGET_SIMD"
 606   "ins\t%0.<Vetype>[%p2], %w1";
 607   [(set_attr "simd_type" "simd_insgp")
 608    (set_attr "simd_mode" "<MODE>")]
 609 )
 610
 611 (define_insn "aarch64_simd_lshr<mode>"
 612  [(set (match_operand:VDQ 0 "register_operand" "=w")
 613        (lshiftrt:VDQ (match_operand:VDQ 1 "register_operand" "w")
 614                      (match_operand:VDQ  2 "aarch64_simd_rshift_imm" "Dr")))]
 615  "TARGET_SIMD"
 616  "ushr\t%0.<Vtype>, %1.<Vtype>, %2"
 617   [(set_attr "simd_type" "simd_shift_imm")
 618    (set_attr "simd_mode" "<MODE>")]
 619 )
 620
 621 (define_insn "aarch64_simd_ashr<mode>"
 622  [(set (match_operand:VDQ 0 "register_operand" "=w")
 623        (ashiftrt:VDQ (match_operand:VDQ 1 "register_operand" "w")
 624                      (match_operand:VDQ  2 "aarch64_simd_rshift_imm" "Dr")))]
 625  "TARGET_SIMD"
 626  "sshr\t%0.<Vtype>, %1.<Vtype>, %2"
 627   [(set_attr "simd_type" "simd_shift_imm")
 628    (set_attr "simd_mode" "<MODE>")]
 629 )
 630
 631 (define_insn "aarch64_simd_imm_shl<mode>"
 632  [(set (match_operand:VDQ 0 "register_operand" "=w")
 633        (ashift:VDQ (match_operand:VDQ 1 "register_operand" "w")
 634                    (match_operand:VDQ  2 "aarch64_simd_lshift_imm" "Dl")))]
 635  "TARGET_SIMD"
 636   "shl\t%0.<Vtype>, %1.<Vtype>, %2"
 637   [(set_attr "simd_type" "simd_shift_imm")
 638    (set_attr "simd_mode" "<MODE>")]
 639 )
 640
 641 (define_insn "aarch64_simd_reg_sshl<mode>"
 642  [(set (match_operand:VDQ 0 "register_operand" "=w")
 643        (ashift:VDQ (match_operand:VDQ 1 "register_operand" "w")
 644                    (match_operand:VDQ 2 "register_operand" "w")))]
 645  "TARGET_SIMD"
 646  "sshl\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
 647   [(set_attr "simd_type" "simd_shift")
 648    (set_attr "simd_mode" "<MODE>")]
 649 )
 650
 651 (define_insn "aarch64_simd_reg_shl<mode>_unsigned"
 652  [(set (match_operand:VDQ 0 "register_operand" "=w")
 653        (unspec:VDQ [(match_operand:VDQ 1 "register_operand" "w")
 654                     (match_operand:VDQ 2 "register_operand" "w")]
 655                    UNSPEC_ASHIFT_UNSIGNED))]
 656  "TARGET_SIMD"
 657  "ushl\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
 658   [(set_attr "simd_type" "simd_shift")
 659    (set_attr "simd_mode" "<MODE>")]
 660 )
 661
 662 (define_insn "aarch64_simd_reg_shl<mode>_signed"
 663  [(set (match_operand:VDQ 0 "register_operand" "=w")
 664        (unspec:VDQ [(match_operand:VDQ 1 "register_operand" "w")
 665                     (match_operand:VDQ 2 "register_operand" "w")]
 666                    UNSPEC_ASHIFT_SIGNED))]
 667  "TARGET_SIMD"
 668  "sshl\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
 669   [(set_attr "simd_type" "simd_shift")
 670    (set_attr "simd_mode" "<MODE>")]
 671 )
 672
 673 (define_expand "ashl<mode>3"
 674   [(match_operand:VDQ 0 "register_operand" "")
 675    (match_operand:VDQ 1 "register_operand" "")
 676    (match_operand:SI  2 "general_operand" "")]
 677  "TARGET_SIMD"
 678 {
 679   int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
 680   int shift_amount;
 681
 682   if (CONST_INT_P (operands[2]))
 683     {
 684       shift_amount = INTVAL (operands[2]);
 685       if (shift_amount >= 0 && shift_amount < bit_width)
 686         {
 687           rtx tmp = aarch64_simd_gen_const_vector_dup (<MODE>mode,
 688                                                        shift_amount);
 689           emit_insn (gen_aarch64_simd_imm_shl<mode> (operands[0],
 690                                                      operands[1],
 691                                                      tmp));
 692           DONE;
 693         }
 694       else
 695         {
 696           operands[2] = force_reg (SImode, operands[2]);
 697         }
 698     }
 699   else if (MEM_P (operands[2]))
 700     {
 701       operands[2] = force_reg (SImode, operands[2]);
 702     }
 703
 704   if (REG_P (operands[2]))
 705     {
 706       rtx tmp = gen_reg_rtx (<MODE>mode);
 707       emit_insn (gen_aarch64_simd_dup<mode> (tmp,
 708                                              convert_to_mode (<VEL>mode,
 709                                                               operands[2],
 710                                                               0)));
 711       emit_insn (gen_aarch64_simd_reg_sshl<mode> (operands[0], operands[1],
 712                                                   tmp));
 713       DONE;
 714     }
 715   else
 716     FAIL;
 717 }
 718 )
 719
 720 (define_expand "lshr<mode>3"
 721   [(match_operand:VDQ 0 "register_operand" "")
 722    (match_operand:VDQ 1 "register_operand" "")
 723    (match_operand:SI  2 "general_operand" "")]
 724  "TARGET_SIMD"
 725 {
 726   int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
 727   int shift_amount;
 728
 729   if (CONST_INT_P (operands[2]))
 730     {
 731       shift_amount = INTVAL (operands[2]);
 732       if (shift_amount > 0 && shift_amount <= bit_width)
 733         {
 734           rtx tmp = aarch64_simd_gen_const_vector_dup (<MODE>mode,
 735                                                        shift_amount);
 736           emit_insn (gen_aarch64_simd_lshr<mode> (operands[0],
 737                                                   operands[1],
 738                                                   tmp));
 739           DONE;
 740         }
 741       else
 742         operands[2] = force_reg (SImode, operands[2]);
 743     }
 744   else if (MEM_P (operands[2]))
 745     {
 746       operands[2] = force_reg (SImode, operands[2]);
 747     }
 748
 749   if (REG_P (operands[2]))
 750     {
 751       rtx tmp = gen_reg_rtx (SImode);
 752       rtx tmp1 = gen_reg_rtx (<MODE>mode);
 753       emit_insn (gen_negsi2 (tmp, operands[2]));
 754       emit_insn (gen_aarch64_simd_dup<mode> (tmp1,
 755                                              convert_to_mode (<VEL>mode,
 756                                                               tmp, 0)));
 757       emit_insn (gen_aarch64_simd_reg_shl<mode>_unsigned (operands[0],
 758                                                           operands[1],
 759                                                           tmp1));
 760       DONE;
 761     }
 762   else
 763     FAIL;
 764 }
 765 )
 766
 767 (define_expand "ashr<mode>3"
 768   [(match_operand:VDQ 0 "register_operand" "")
 769    (match_operand:VDQ 1 "register_operand" "")
 770    (match_operand:SI  2 "general_operand" "")]
 771  "TARGET_SIMD"
 772 {
 773   int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
 774   int shift_amount;
 775
 776   if (CONST_INT_P (operands[2]))
 777     {
 778       shift_amount = INTVAL (operands[2]);
 779       if (shift_amount > 0 && shift_amount <= bit_width)
 780         {
 781           rtx tmp = aarch64_simd_gen_const_vector_dup (<MODE>mode,
 782                                                        shift_amount);
 783           emit_insn (gen_aarch64_simd_ashr<mode> (operands[0],
 784                                                   operands[1],
 785                                                   tmp));
 786           DONE;
 787         }
 788       else
 789         operands[2] = force_reg (SImode, operands[2]);
 790     }
 791   else if (MEM_P (operands[2]))
 792     {
 793       operands[2] = force_reg (SImode, operands[2]);
 794     }
 795
 796   if (REG_P (operands[2]))
 797     {
 798       rtx tmp = gen_reg_rtx (SImode);
 799       rtx tmp1 = gen_reg_rtx (<MODE>mode);
 800       emit_insn (gen_negsi2 (tmp, operands[2]));
 801       emit_insn (gen_aarch64_simd_dup<mode> (tmp1,
 802                                              convert_to_mode (<VEL>mode,
 803                                                               tmp, 0)));
 804       emit_insn (gen_aarch64_simd_reg_shl<mode>_signed (operands[0],
 805                                                         operands[1],
 806                                                         tmp1));
 807       DONE;
 808     }
 809   else
 810     FAIL;
 811 }
 812 )
 813
 814 (define_expand "vashl<mode>3"
 815  [(match_operand:VDQ 0 "register_operand" "")
 816   (match_operand:VDQ 1 "register_operand" "")
 817   (match_operand:VDQ 2 "register_operand" "")]
 818  "TARGET_SIMD"
 819 {
 820   emit_insn (gen_aarch64_simd_reg_sshl<mode> (operands[0], operands[1],
 821                                               operands[2]));
 822   DONE;
 823 })
 824
 825 ;; Using mode VQ_S as there is no V2DImode neg!
 826 ;; Negating individual lanes most certainly offsets the
 827 ;; gain from vectorization.
 828 (define_expand "vashr<mode>3"
 829  [(match_operand:VQ_S 0 "register_operand" "")
 830   (match_operand:VQ_S 1 "register_operand" "")
 831   (match_operand:VQ_S 2 "register_operand" "")]
 832  "TARGET_SIMD"
 833 {
 834   rtx neg = gen_reg_rtx (<MODE>mode);
 835   emit (gen_neg<mode>2 (neg, operands[2]));
 836   emit_insn (gen_aarch64_simd_reg_shl<mode>_signed (operands[0], operands[1],
 837                                                     neg));
 838   DONE;
 839 })
 840
 841 (define_expand "vlshr<mode>3"
 842  [(match_operand:VQ_S 0 "register_operand" "")
 843   (match_operand:VQ_S 1 "register_operand" "")
 844   (match_operand:VQ_S 2 "register_operand" "")]
 845  "TARGET_SIMD"
 846 {
 847   rtx neg = gen_reg_rtx (<MODE>mode);
 848   emit (gen_neg<mode>2 (neg, operands[2]));
 849   emit_insn (gen_aarch64_simd_reg_shl<mode>_unsigned (operands[0], operands[1],
 850                                                       neg));
 851   DONE;
 852 })
 853
 854 (define_expand "vec_set<mode>"
 855   [(match_operand:VQ_S 0 "register_operand" "+w")
 856    (match_operand:<VEL> 1 "register_operand" "r")
 857    (match_operand:SI 2 "immediate_operand" "")]
 858   "TARGET_SIMD"
 859   {
 860     HOST_WIDE_INT elem = (HOST_WIDE_INT) 1 << INTVAL (operands[2]);
 861     emit_insn (gen_aarch64_simd_vec_set<mode> (operands[0], operands[1],
 862                                             GEN_INT (elem), operands[0]));
 863     DONE;
 864   }
 865 )
 866
 867 (define_insn "aarch64_simd_vec_setv2di"
 868   [(set (match_operand:V2DI 0 "register_operand" "=w")
 869         (vec_merge:V2DI
 870             (vec_duplicate:V2DI
 871                 (match_operand:DI 1 "register_operand" "r"))
 872             (match_operand:V2DI 3 "register_operand" "0")
 873             (match_operand:SI 2 "immediate_operand" "i")))]
 874   "TARGET_SIMD"
 875   "ins\t%0.d[%p2], %1";
 876   [(set_attr "simd_type" "simd_insgp")
 877    (set_attr "simd_mode" "V2DI")]
 878 )
 879
 880 (define_expand "vec_setv2di"
 881   [(match_operand:V2DI 0 "register_operand" "+w")
 882    (match_operand:DI 1 "register_operand" "r")
 883    (match_operand:SI 2 "immediate_operand" "")]
 884   "TARGET_SIMD"
 885   {
 886     HOST_WIDE_INT elem = (HOST_WIDE_INT) 1 << INTVAL (operands[2]);
 887     emit_insn (gen_aarch64_simd_vec_setv2di (operands[0], operands[1],
 888                                           GEN_INT (elem), operands[0]));
 889     DONE;
 890   }
 891 )
 892
 893 (define_insn "aarch64_simd_vec_set<mode>"
 894   [(set (match_operand:VDQF 0 "register_operand" "=w")
 895         (vec_merge:VDQF
 896             (vec_duplicate:VDQF
 897                 (match_operand:<VEL> 1 "register_operand" "w"))
 898             (match_operand:VDQF 3 "register_operand" "0")
 899             (match_operand:SI 2 "immediate_operand" "i")))]
 900   "TARGET_SIMD"
 901   "ins\t%0.<Vetype>[%p2], %1.<Vetype>[0]";
 902   [(set_attr "simd_type" "simd_ins")
 903    (set_attr "simd_mode" "<MODE>")]
 904 )
 905
 906 (define_expand "vec_set<mode>"
 907   [(match_operand:VDQF 0 "register_operand" "+w")
 908    (match_operand:<VEL> 1 "register_operand" "w")
 909    (match_operand:SI 2 "immediate_operand" "")]
 910   "TARGET_SIMD"
 911   {
 912     HOST_WIDE_INT elem = (HOST_WIDE_INT) 1 << INTVAL (operands[2]);
 913     emit_insn (gen_aarch64_simd_vec_set<mode> (operands[0], operands[1],
 914                                           GEN_INT (elem), operands[0]));
 915     DONE;
 916   }
 917 )
 918
 919
 920 (define_insn "aarch64_mla<mode>"
 921  [(set (match_operand:VQ_S 0 "register_operand" "=w")
 922        (plus:VQ_S (mult:VQ_S (match_operand:VQ_S 2 "register_operand" "w")
 923                              (match_operand:VQ_S 3 "register_operand" "w"))
 924                   (match_operand:VQ_S 1 "register_operand" "0")))]
 925  "TARGET_SIMD"
 926  "mla\t%0.<Vtype>, %2.<Vtype>, %3.<Vtype>"
 927   [(set_attr "simd_type" "simd_mla")
 928    (set_attr "simd_mode" "<MODE>")]
 929 )
 930
 931 (define_insn "aarch64_mls<mode>"
 932  [(set (match_operand:VQ_S 0 "register_operand" "=w")
 933        (minus:VQ_S (match_operand:VQ_S 1 "register_operand" "0")
 934                    (mult:VQ_S (match_operand:VQ_S 2 "register_operand" "w")
 935                               (match_operand:VQ_S 3 "register_operand" "w"))))]
 936  "TARGET_SIMD"
 937  "mls\t%0.<Vtype>, %2.<Vtype>, %3.<Vtype>"
 938   [(set_attr "simd_type" "simd_mla")
 939    (set_attr "simd_mode" "<MODE>")]
 940 )
 941
 942 ;; Max/Min operations.
 943 (define_insn "<maxmin><mode>3"
 944  [(set (match_operand:VQ_S 0 "register_operand" "=w")
 945        (MAXMIN:VQ_S (match_operand:VQ_S 1 "register_operand" "w")
 946                     (match_operand:VQ_S 2 "register_operand" "w")))]
 947  "TARGET_SIMD"
 948  "<maxmin>\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
 949   [(set_attr "simd_type" "simd_minmax")
 950    (set_attr "simd_mode" "<MODE>")]
 951 )
 952
 953 ;; Move into low-half clearing high half to 0.
 954
 955 (define_insn "move_lo_quad_<mode>"
 956   [(set (match_operand:VQ 0 "register_operand" "=w")
 957         (vec_concat:VQ
 958           (match_operand:<VHALF> 1 "register_operand" "w")
 959           (vec_duplicate:<VHALF> (const_int 0))))]
 960   "TARGET_SIMD"
 961   "mov\\t%d0, %d1";
 962   [(set_attr "simd_type" "simd_dup")
 963    (set_attr "simd_mode" "<MODE>")]
 964 )
 965
 966 ;; Move into high-half.
 967
 968 (define_insn "aarch64_simd_move_hi_quad_<mode>"
 969   [(set (match_operand:VQ 0 "register_operand" "+w")
 970         (vec_concat:VQ
 971           (vec_select:<VHALF>
 972                 (match_dup 0)
 973                 (match_operand:VQ 2 "vect_par_cnst_lo_half" ""))
 974           (match_operand:<VHALF> 1 "register_operand" "w")))]
 975   "TARGET_SIMD"
 976   "ins\\t%0.d[1], %1.d[0]";
 977   [(set_attr "simd_type" "simd_ins")
 978    (set_attr "simd_mode" "<MODE>")]
 979 )
 980
 981 (define_expand "move_hi_quad_<mode>"
 982  [(match_operand:VQ 0 "register_operand" "")
 983   (match_operand:<VHALF> 1 "register_operand" "")]
 984  "TARGET_SIMD"
 985 {
 986   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
 987   emit_insn (gen_aarch64_simd_move_hi_quad_<mode> (operands[0],
 988                                                    operands[1], p));
 989   DONE;
 990 })
 991
 992 ;; Narrowing operations.
 993
 994 ;; For doubles.
 995 (define_insn "aarch64_simd_vec_pack_trunc_<mode>"
 996  [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
 997        (truncate:<VNARROWQ> (match_operand:VQN 1 "register_operand" "w")))]
 998  "TARGET_SIMD"
 999  "xtn\\t%0.<Vntype>, %1.<Vtype>"
1000   [(set_attr "simd_type" "simd_shiftn_imm")
1001    (set_attr "simd_mode" "<MODE>")]
1002 )
1003
1004 (define_expand "vec_pack_trunc_<mode>"
1005  [(match_operand:<VNARROWD> 0 "register_operand" "")
1006   (match_operand:VDN 1 "register_operand" "")
1007   (match_operand:VDN 2 "register_operand" "")]
1008  "TARGET_SIMD"
1009 {
1010   rtx tempreg = gen_reg_rtx (<VDBL>mode);
1011
1012   emit_insn (gen_move_lo_quad_<Vdbl> (tempreg, operands[1]));
1013   emit_insn (gen_move_hi_quad_<Vdbl> (tempreg, operands[2]));
1014   emit_insn (gen_aarch64_simd_vec_pack_trunc_<Vdbl> (operands[0], tempreg));
1015   DONE;
1016 })
1017
1018 ;; For quads.
1019
1020 (define_insn "vec_pack_trunc_<mode>"
1021  [(set (match_operand:<VNARROWQ2> 0 "register_operand" "+&w")
1022        (vec_concat:<VNARROWQ2>
1023          (truncate:<VNARROWQ> (match_operand:VQN 1 "register_operand" "w"))
1024          (truncate:<VNARROWQ> (match_operand:VQN 2 "register_operand" "w"))))]
1025  "TARGET_SIMD"
1026  "xtn\\t%0.<Vntype>, %1.<Vtype>\;xtn2\\t%0.<V2ntype>, %2.<Vtype>"
1027   [(set_attr "simd_type" "simd_shiftn2_imm")
1028    (set_attr "simd_mode" "<MODE>")
1029    (set_attr "length" "8")]
1030 )
1031
1032 ;; Widening operations.
1033
1034 (define_insn "aarch64_simd_vec_unpack<su>_lo_<mode>"
1035   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1036         (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1037                                (match_operand:VQW 1 "register_operand" "w")
1038                                (match_operand:VQW 2 "vect_par_cnst_lo_half" "")
1039                             )))]
1040   "TARGET_SIMD"
1041   "<su>shll %0.<Vwtype>, %1.<Vhalftype>, 0"
1042   [(set_attr "simd_type" "simd_shiftl_imm")
1043    (set_attr "simd_mode" "<MODE>")]
1044 )
1045
1046 (define_insn "aarch64_simd_vec_unpack<su>_hi_<mode>"
1047   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1048         (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1049                                (match_operand:VQW 1 "register_operand" "w")
1050                                (match_operand:VQW 2 "vect_par_cnst_hi_half" "")
1051                             )))]
1052   "TARGET_SIMD"
1053   "<su>shll2 %0.<Vwtype>, %1.<Vtype>, 0"
1054   [(set_attr "simd_type" "simd_shiftl_imm")
1055    (set_attr "simd_mode" "<MODE>")]
1056 )
1057
1058 (define_expand "vec_unpack<su>_hi_<mode>"
1059   [(match_operand:<VWIDE> 0 "register_operand" "")
1060    (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand"))]
1061   "TARGET_SIMD"
1062   {
1063     rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
1064     emit_insn (gen_aarch64_simd_vec_unpack<su>_hi_<mode> (operands[0],
1065                                                           operands[1], p));
1066     DONE;
1067   }
1068 )
1069
1070 (define_expand "vec_unpack<su>_lo_<mode>"
1071   [(match_operand:<VWIDE> 0 "register_operand" "")
1072    (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand" ""))]
1073   "TARGET_SIMD"
1074   {
1075     rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
1076     emit_insn (gen_aarch64_simd_vec_unpack<su>_lo_<mode> (operands[0],
1077                                                           operands[1], p));
1078     DONE;
1079   }
1080 )
1081
1082 ;; Widening arithmetic.
1083
1084 (define_insn "aarch64_simd_vec_<su>mult_lo_<mode>"
1085  [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1086        (mult:<VWIDE> (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1087                            (match_operand:VQW 1 "register_operand" "w")
1088                            (match_operand:VQW 3 "vect_par_cnst_lo_half" "")))
1089                      (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1090                            (match_operand:VQW 2 "register_operand" "w")
1091                            (match_dup 3)))))]
1092   "TARGET_SIMD"
1093   "<su>mull\\t%0.<Vwtype>, %1.<Vhalftype>, %2.<Vhalftype>"
1094   [(set_attr "simd_type" "simd_mull")
1095    (set_attr "simd_mode" "<MODE>")]
1096 )
1097
1098 (define_expand "vec_widen_<su>mult_lo_<mode>"
1099   [(match_operand:<VWIDE> 0 "register_operand" "")
1100    (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand" ""))
1101    (ANY_EXTEND:<VWIDE> (match_operand:VQW 2 "register_operand" ""))]
1102  "TARGET_SIMD"
1103  {
1104    rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
1105    emit_insn (gen_aarch64_simd_vec_<su>mult_lo_<mode> (operands[0],
1106                                                        operands[1],
1107                                                        operands[2], p));
1108    DONE;
1109  }
1110 )
1111
1112 (define_insn "aarch64_simd_vec_<su>mult_hi_<mode>"
1113  [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1114       (mult:<VWIDE> (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1115                             (match_operand:VQW 1 "register_operand" "w")
1116                             (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))
1117                     (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1118                             (match_operand:VQW 2 "register_operand" "w")
1119                             (match_dup 3)))))]
1120   "TARGET_SIMD"
1121   "<su>mull2\\t%0.<Vwtype>, %1.<Vtype>, %2.<Vtype>"
1122   [(set_attr "simd_type" "simd_mull")
1123    (set_attr "simd_mode" "<MODE>")]
1124 )
1125
1126 (define_expand "vec_widen_<su>mult_hi_<mode>"
1127   [(match_operand:<VWIDE> 0 "register_operand" "")
1128    (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand" ""))
1129    (ANY_EXTEND:<VWIDE> (match_operand:VQW 2 "register_operand" ""))]
1130  "TARGET_SIMD"
1131  {
1132    rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
1133    emit_insn (gen_aarch64_simd_vec_<su>mult_hi_<mode> (operands[0],
1134                                                        operands[1],
1135                                                        operands[2], p));
1136    DONE;
1137
1138  }
1139 )
1140
1141 ;; FP vector operations.
1142 ;; AArch64 AdvSIMD supports single-precision (32-bit) and
1143 ;; double-precision (64-bit) floating-point data types and arithmetic as
1144 ;; defined by the IEEE 754-2008 standard.  This makes them vectorizable
1145 ;; without the need for -ffast-math or -funsafe-math-optimizations.
1146 ;;
1147 ;; Floating-point operations can raise an exception.  Vectorizing such
1148 ;; operations are safe because of reasons explained below.
1149 ;;
1150 ;; ARMv8 permits an extension to enable trapped floating-point
1151 ;; exception handling, however this is an optional feature.  In the
1152 ;; event of a floating-point exception being raised by vectorised
1153 ;; code then:
1154 ;; 1.  If trapped floating-point exceptions are available, then a trap
1155 ;;     will be taken when any lane raises an enabled exception.  A trap
1156 ;;     handler may determine which lane raised the exception.
1157 ;; 2.  Alternatively a sticky exception flag is set in the
1158 ;;     floating-point status register (FPSR).  Software may explicitly
1159 ;;     test the exception flags, in which case the tests will either
1160 ;;     prevent vectorisation, allowing precise identification of the
1161 ;;     failing operation, or if tested outside of vectorisable regions
1162 ;;     then the specific operation and lane are not of interest.
1163
1164 ;; FP arithmetic operations.
1165
1166 (define_insn "add<mode>3"
1167  [(set (match_operand:VDQF 0 "register_operand" "=w")
1168        (plus:VDQF (match_operand:VDQF 1 "register_operand" "w")
1169                   (match_operand:VDQF 2 "register_operand" "w")))]
1170  "TARGET_SIMD"
1171  "fadd\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1172   [(set_attr "simd_type" "simd_fadd")
1173    (set_attr "simd_mode" "<MODE>")]
1174 )
1175
1176 (define_insn "sub<mode>3"
1177  [(set (match_operand:VDQF 0 "register_operand" "=w")
1178        (minus:VDQF (match_operand:VDQF 1 "register_operand" "w")
1179                    (match_operand:VDQF 2 "register_operand" "w")))]
1180  "TARGET_SIMD"
1181  "fsub\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1182   [(set_attr "simd_type" "simd_fadd")
1183    (set_attr "simd_mode" "<MODE>")]
1184 )
1185
1186 (define_insn "mul<mode>3"
1187  [(set (match_operand:VDQF 0 "register_operand" "=w")
1188        (mult:VDQF (match_operand:VDQF 1 "register_operand" "w")
1189                   (match_operand:VDQF 2 "register_operand" "w")))]
1190  "TARGET_SIMD"
1191  "fmul\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1192   [(set_attr "simd_type" "simd_fmul")
1193    (set_attr "simd_mode" "<MODE>")]
1194 )
1195
1196 (define_insn "div<mode>3"
1197  [(set (match_operand:VDQF 0 "register_operand" "=w")
1198        (div:VDQF (match_operand:VDQF 1 "register_operand" "w")
1199                  (match_operand:VDQF 2 "register_operand" "w")))]
1200  "TARGET_SIMD"
1201  "fdiv\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1202   [(set_attr "simd_type" "simd_fdiv")
1203    (set_attr "simd_mode" "<MODE>")]
1204 )
1205
1206 (define_insn "neg<mode>2"
1207  [(set (match_operand:VDQF 0 "register_operand" "=w")
1208        (neg:VDQF (match_operand:VDQF 1 "register_operand" "w")))]
1209  "TARGET_SIMD"
1210  "fneg\\t%0.<Vtype>, %1.<Vtype>"
1211   [(set_attr "simd_type" "simd_fnegabs")
1212    (set_attr "simd_mode" "<MODE>")]
1213 )
1214
1215 (define_insn "abs<mode>2"
1216  [(set (match_operand:VDQF 0 "register_operand" "=w")
1217        (abs:VDQF (match_operand:VDQF 1 "register_operand" "w")))]
1218  "TARGET_SIMD"
1219  "fabs\\t%0.<Vtype>, %1.<Vtype>"
1220   [(set_attr "simd_type" "simd_fnegabs")
1221    (set_attr "simd_mode" "<MODE>")]
1222 )
1223
1224 (define_insn "fma<mode>4"
1225   [(set (match_operand:VDQF 0 "register_operand" "=w")
1226        (fma:VDQF (match_operand:VDQF 1 "register_operand" "w")
1227                 (match_operand:VDQF 2 "register_operand" "w")
1228                 (match_operand:VDQF 3 "register_operand" "0")))]
1229   "TARGET_SIMD"
1230  "fmla\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1231   [(set_attr "simd_type" "simd_fmla")
1232    (set_attr "simd_mode" "<MODE>")]
1233 )
1234
1235 ;; Vector versions of the floating-point frint patterns.
1236 ;; Expands to btrunc, ceil, floor, nearbyint, rint, round.
1237 (define_insn "<frint_pattern><mode>2"
1238   [(set (match_operand:VDQF 0 "register_operand" "=w")
1239         (unspec:VDQF [(match_operand:VDQF 1 "register_operand" "w")]
1240                       FRINT))]
1241   "TARGET_SIMD"
1242   "frint<frint_suffix>\\t%0.<Vtype>, %1.<Vtype>"
1243   [(set_attr "simd_type" "simd_frint")
1244    (set_attr "simd_mode" "<MODE>")]
1245 )
1246
1247 ;; Vector versions of the fcvt standard patterns.
1248 ;; Expands to lbtrunc, lround, lceil, lfloor
1249 (define_insn "l<fcvt_pattern><su_optab><VDQF:mode><fcvt_target>2"
1250   [(set (match_operand:<FCVT_TARGET> 0 "register_operand" "=w")
1251         (FIXUORS:<FCVT_TARGET> (unspec:<FCVT_TARGET>
1252                                [(match_operand:VDQF 1 "register_operand" "w")]
1253                                FCVT)))]
1254   "TARGET_SIMD"
1255   "fcvt<frint_suffix><su>\\t%0.<Vtype>, %1.<Vtype>"
1256   [(set_attr "simd_type" "simd_fcvti")
1257    (set_attr "simd_mode" "<MODE>")]
1258 )
1259
1260 (define_expand "<optab><VDQF:mode><fcvt_target>2"
1261   [(set (match_operand:<FCVT_TARGET> 0 "register_operand")
1262         (FIXUORS:<FCVT_TARGET> (unspec:<FCVT_TARGET>
1263                                [(match_operand:VDQF 1 "register_operand")]
1264                                UNSPEC_FRINTZ)))]
1265   "TARGET_SIMD"
1266   {})
1267
1268 (define_expand "<fix_trunc_optab><VDQF:mode><fcvt_target>2"
1269   [(set (match_operand:<FCVT_TARGET> 0 "register_operand")
1270         (FIXUORS:<FCVT_TARGET> (unspec:<FCVT_TARGET>
1271                                [(match_operand:VDQF 1 "register_operand")]
1272                                UNSPEC_FRINTZ)))]
1273   "TARGET_SIMD"
1274   {})
1275
1276 (define_expand "ftrunc<VDQF:mode>2"
1277   [(set (match_operand:VDQF 0 "register_operand")
1278         (unspec:VDQF [(match_operand:VDQF 1 "register_operand")]
1279                       UNSPEC_FRINTZ))]
1280   "TARGET_SIMD"
1281   {})
1282
1283 (define_insn "<optab><fcvt_target><VDQF:mode>2"
1284   [(set (match_operand:VDQF 0 "register_operand" "=w")
1285         (FLOATUORS:VDQF
1286           (match_operand:<FCVT_TARGET> 1 "register_operand" "w")))]
1287   "TARGET_SIMD"
1288   "<su_optab>cvtf\\t%0.<Vtype>, %1.<Vtype>"
1289   [(set_attr "simd_type" "simd_icvtf")
1290    (set_attr "simd_mode" "<MODE>")]
1291 )
1292
1293 ;; Conversions between vectors of floats and doubles.
1294 ;; Contains a mix of patterns to match standard pattern names
1295 ;; and those for intrinsics.
1296
1297 ;; Float widening operations.
1298
1299 (define_insn "vec_unpacks_lo_v4sf"
1300   [(set (match_operand:V2DF 0 "register_operand" "=w")
1301         (float_extend:V2DF
1302           (vec_select:V2SF
1303             (match_operand:V4SF 1 "register_operand" "w")
1304             (parallel [(const_int 0) (const_int 1)])
1305           )))]
1306   "TARGET_SIMD"
1307   "fcvtl\\t%0.2d, %1.2s"
1308   [(set_attr "simd_type" "simd_fcvtl")
1309    (set_attr "simd_mode" "V2DF")]
1310 )
1311
1312 (define_insn "aarch64_float_extend_lo_v2df"
1313   [(set (match_operand:V2DF 0 "register_operand" "=w")
1314         (float_extend:V2DF
1315           (match_operand:V2SF 1 "register_operand" "w")))]
1316   "TARGET_SIMD"
1317   "fcvtl\\t%0.2d, %1.2s"
1318   [(set_attr "simd_type" "simd_fcvtl")
1319    (set_attr "simd_mode" "V2DF")]
1320 )
1321
1322 (define_insn "vec_unpacks_hi_v4sf"
1323   [(set (match_operand:V2DF 0 "register_operand" "=w")
1324         (float_extend:V2DF
1325           (vec_select:V2SF
1326             (match_operand:V4SF 1 "register_operand" "w")
1327             (parallel [(const_int 2) (const_int 3)])
1328           )))]
1329   "TARGET_SIMD"
1330   "fcvtl2\\t%0.2d, %1.4s"
1331   [(set_attr "simd_type" "simd_fcvtl")
1332    (set_attr "simd_mode" "V2DF")]
1333 )
1334
1335 ;; Float narrowing operations.
1336
1337 (define_insn "aarch64_float_truncate_lo_v2sf"
1338   [(set (match_operand:V2SF 0 "register_operand" "=w")
1339       (float_truncate:V2SF
1340         (match_operand:V2DF 1 "register_operand" "w")))]
1341   "TARGET_SIMD"
1342   "fcvtn\\t%0.2s, %1.2d"
1343   [(set_attr "simd_type" "simd_fcvtl")
1344    (set_attr "simd_mode" "V2SF")]
1345 )
1346
1347 (define_insn "aarch64_float_truncate_hi_v4sf"
1348   [(set (match_operand:V4SF 0 "register_operand" "=w")
1349     (vec_concat:V4SF
1350       (match_operand:V2SF 1 "register_operand" "0")
1351       (float_truncate:V2SF
1352         (match_operand:V2DF 2 "register_operand" "w"))))]
1353   "TARGET_SIMD"
1354   "fcvtn2\\t%0.4s, %2.2d"
1355   [(set_attr "simd_type" "simd_fcvtl")
1356    (set_attr "simd_mode" "V4SF")]
1357 )
1358
1359 (define_expand "vec_pack_trunc_v2df"
1360   [(set (match_operand:V4SF 0 "register_operand")
1361       (vec_concat:V4SF
1362         (float_truncate:V2SF
1363             (match_operand:V2DF 1 "register_operand"))
1364         (float_truncate:V2SF
1365             (match_operand:V2DF 2 "register_operand"))
1366           ))]
1367   "TARGET_SIMD"
1368   {
1369     rtx tmp = gen_reg_rtx (V2SFmode);
1370     emit_insn (gen_aarch64_float_truncate_lo_v2sf (tmp, operands[1]));
1371     emit_insn (gen_aarch64_float_truncate_hi_v4sf (operands[0],
1372                                                    tmp, operands[2]));
1373     DONE;
1374   }
1375 )
1376
1377 (define_expand "vec_pack_trunc_df"
1378   [(set (match_operand:V2SF 0 "register_operand")
1379       (vec_concat:V2SF
1380         (float_truncate:SF
1381             (match_operand:DF 1 "register_operand"))
1382         (float_truncate:SF
1383             (match_operand:DF 2 "register_operand"))
1384           ))]
1385   "TARGET_SIMD"
1386   {
1387     rtx tmp = gen_reg_rtx (V2SFmode);
1388     emit_insn (gen_move_lo_quad_v2df (tmp, operands[1]));
1389     emit_insn (gen_move_hi_quad_v2df (tmp, operands[2]));
1390     emit_insn (gen_aarch64_float_truncate_lo_v2sf (operands[0], tmp));
1391     DONE;
1392   }
1393 )
1394
1395 (define_insn "aarch64_vmls<mode>"
1396   [(set (match_operand:VDQF 0 "register_operand" "=w")
1397        (minus:VDQF (match_operand:VDQF 1 "register_operand" "0")
1398                    (mult:VDQF (match_operand:VDQF 2 "register_operand" "w")
1399                               (match_operand:VDQF 3 "register_operand" "w"))))]
1400   "TARGET_SIMD"
1401  "fmls\\t%0.<Vtype>, %2.<Vtype>, %3.<Vtype>"
1402   [(set_attr "simd_type" "simd_fmla")
1403    (set_attr "simd_mode" "<MODE>")]
1404 )
1405
1406 ;; FP Max/Min
1407 ;; Max/Min are introduced by idiom recognition by GCC's mid-end.  An
1408 ;; expression like:
1409 ;;      a = (b < c) ? b : c;
1410 ;; is idiom-matched as MIN_EXPR<b,c> only if -ffinite-math-only is enabled
1411 ;; either explicitly or indirectly via -ffast-math.
1412 ;;
1413 ;; MIN_EXPR and MAX_EXPR eventually map to 'smin' and 'smax' in RTL.
1414 ;; The 'smax' and 'smin' RTL standard pattern names do not specify which
1415 ;; operand will be returned when both operands are zero (i.e. they may not
1416 ;; honour signed zeroes), or when either operand is NaN.  Therefore GCC
1417 ;; only introduces MIN_EXPR/MAX_EXPR in fast math mode or when not honouring
1418 ;; NaNs.
1419
1420 (define_insn "smax<mode>3"
1421   [(set (match_operand:VDQF 0 "register_operand" "=w")
1422         (smax:VDQF (match_operand:VDQF 1 "register_operand" "w")
1423                    (match_operand:VDQF 2 "register_operand" "w")))]
1424   "TARGET_SIMD"
1425   "fmaxnm\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1426   [(set_attr "simd_type" "simd_fminmax")
1427    (set_attr "simd_mode" "<MODE>")]
1428 )
1429
1430 (define_insn "smin<mode>3"
1431   [(set (match_operand:VDQF 0 "register_operand" "=w")
1432         (smin:VDQF (match_operand:VDQF 1 "register_operand" "w")
1433                    (match_operand:VDQF 2 "register_operand" "w")))]
1434   "TARGET_SIMD"
1435   "fminnm\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1436   [(set_attr "simd_type" "simd_fminmax")
1437    (set_attr "simd_mode" "<MODE>")]
1438 )
1439
1440 ;; FP 'across lanes' max and min ops.
1441
1442 (define_insn "reduc_s<fmaxminv>_v4sf"
1443  [(set (match_operand:V4SF 0 "register_operand" "=w")
1444        (unspec:V4SF [(match_operand:V4SF 1 "register_operand" "w")]
1445                     FMAXMINV))]
1446  "TARGET_SIMD"
1447  "f<fmaxminv>nmv\\t%s0, %1.4s";
1448   [(set_attr "simd_type" "simd_fminmaxv")
1449    (set_attr "simd_mode" "V4SF")]
1450 )
1451
1452 (define_insn "reduc_s<fmaxminv>_<mode>"
1453  [(set (match_operand:V2F 0 "register_operand" "=w")
1454        (unspec:V2F [(match_operand:V2F 1 "register_operand" "w")]
1455                     FMAXMINV))]
1456  "TARGET_SIMD"
1457  "f<fmaxminv>nmp\\t%0.<Vtype>, %1.<Vtype>, %1.<Vtype>";
1458   [(set_attr "simd_type" "simd_fminmax")
1459    (set_attr "simd_mode" "<MODE>")]
1460 )
1461
1462 ;; FP 'across lanes' add.
1463
1464 (define_insn "aarch64_addpv4sf"
1465  [(set (match_operand:V4SF 0 "register_operand" "=w")
1466        (unspec:V4SF [(match_operand:V4SF 1 "register_operand" "w")]
1467                     UNSPEC_FADDV))]
1468  "TARGET_SIMD"
1469  "faddp\\t%0.4s, %1.4s, %1.4s"
1470   [(set_attr "simd_type" "simd_fadd")
1471    (set_attr "simd_mode" "V4SF")]
1472 )
1473
1474 (define_expand "reduc_uplus_v4sf"
1475  [(set (match_operand:V4SF 0 "register_operand" "=w")
1476        (match_operand:V4SF 1 "register_operand" "w"))]
1477  "TARGET_SIMD"
1478 {
1479   rtx tmp = gen_reg_rtx (V4SFmode);
1480   emit_insn (gen_aarch64_addpv4sf (tmp, operands[1]));
1481   emit_insn (gen_aarch64_addpv4sf (operands[0], tmp));
1482   DONE;
1483 })
1484
1485 (define_expand "reduc_splus_v4sf"
1486  [(set (match_operand:V4SF 0 "register_operand" "=w")
1487        (match_operand:V4SF 1 "register_operand" "w"))]
1488  "TARGET_SIMD"
1489 {
1490   rtx tmp = gen_reg_rtx (V4SFmode);
1491   emit_insn (gen_aarch64_addpv4sf (tmp, operands[1]));
1492   emit_insn (gen_aarch64_addpv4sf (operands[0], tmp));
1493   DONE;
1494 })
1495
1496 (define_expand "aarch64_addvv4sf"
1497  [(set (match_operand:V4SF 0 "register_operand" "=w")
1498         (unspec:V4SF [(match_operand:V4SF 1 "register_operand" "w")]
1499                      UNSPEC_FADDV))]
1500  "TARGET_SIMD"
1501 {
1502   emit_insn (gen_reduc_splus_v4sf (operands[0], operands[1]));
1503   DONE;
1504 })
1505
1506 (define_insn "aarch64_addv<mode>"
1507  [(set (match_operand:V2F 0 "register_operand" "=w")
1508        (unspec:V2F [(match_operand:V2F 1 "register_operand" "w")]
1509                     UNSPEC_FADDV))]
1510  "TARGET_SIMD"
1511  "faddp\\t%<Vetype>0, %1.<Vtype>"
1512   [(set_attr "simd_type" "simd_fadd")
1513    (set_attr "simd_mode" "<MODE>")]
1514 )
1515
1516 (define_expand "reduc_uplus_<mode>"
1517  [(set (match_operand:V2F 0 "register_operand" "=w")
1518        (unspec:V2F [(match_operand:V2F 1 "register_operand" "w")]
1519                     UNSPEC_FADDV))]
1520  "TARGET_SIMD"
1521  ""
1522 )
1523
1524 (define_expand "reduc_splus_<mode>"
1525  [(set (match_operand:V2F 0 "register_operand" "=w")
1526        (unspec:V2F [(match_operand:V2F 1 "register_operand" "w")]
1527                     UNSPEC_FADDV))]
1528  "TARGET_SIMD"
1529  ""
1530 )
1531
1532 ;; Reduction across lanes.
1533
1534 (define_insn "aarch64_addv<mode>"
1535  [(set (match_operand:VDQV 0 "register_operand" "=w")
1536        (unspec:VDQV [(match_operand:VDQV 1 "register_operand" "w")]
1537                     UNSPEC_ADDV))]
1538  "TARGET_SIMD"
1539  "addv\\t%<Vetype>0, %1.<Vtype>"
1540   [(set_attr "simd_type" "simd_addv")
1541    (set_attr "simd_mode" "<MODE>")]
1542 )
1543
1544 (define_expand "reduc_splus_<mode>"
1545  [(set (match_operand:VDQV 0 "register_operand" "=w")
1546        (unspec:VDQV [(match_operand:VDQV 1 "register_operand" "w")]
1547                     UNSPEC_ADDV))]
1548  "TARGET_SIMD"
1549  ""
1550 )
1551
1552 (define_expand "reduc_uplus_<mode>"
1553  [(set (match_operand:VDQV 0 "register_operand" "=w")
1554        (unspec:VDQV [(match_operand:VDQV 1 "register_operand" "w")]
1555                     UNSPEC_ADDV))]
1556  "TARGET_SIMD"
1557  ""
1558 )
1559
1560 (define_insn "aarch64_addvv2di"
1561  [(set (match_operand:V2DI 0 "register_operand" "=w")
1562        (unspec:V2DI [(match_operand:V2DI 1 "register_operand" "w")]
1563                     UNSPEC_ADDV))]
1564  "TARGET_SIMD"
1565  "addp\\t%d0, %1.2d"
1566   [(set_attr "simd_type" "simd_add")
1567    (set_attr "simd_mode" "V2DI")]
1568 )
1569
1570 (define_expand "reduc_uplus_v2di"
1571  [(set (match_operand:V2DI 0 "register_operand" "=w")
1572        (unspec:V2DI [(match_operand:V2DI 1 "register_operand" "w")]
1573                     UNSPEC_ADDV))]
1574  "TARGET_SIMD"
1575  ""
1576 )
1577
1578 (define_expand "reduc_splus_v2di"
1579  [(set (match_operand:V2DI 0 "register_operand" "=w")
1580        (unspec:V2DI [(match_operand:V2DI 1 "register_operand" "w")]
1581                     UNSPEC_ADDV))]
1582  "TARGET_SIMD"
1583  ""
1584 )
1585
1586 (define_insn "aarch64_addvv2si"
1587  [(set (match_operand:V2SI 0 "register_operand" "=w")
1588        (unspec:V2SI [(match_operand:V2SI 1 "register_operand" "w")]
1589                     UNSPEC_ADDV))]
1590  "TARGET_SIMD"
1591  "addp\\t%0.2s, %1.2s, %1.2s"
1592   [(set_attr "simd_type" "simd_add")
1593    (set_attr "simd_mode" "V2SI")]
1594 )
1595
1596 (define_expand "reduc_uplus_v2si"
1597  [(set (match_operand:V2SI 0 "register_operand" "=w")
1598        (unspec:V2SI [(match_operand:V2SI 1 "register_operand" "w")]
1599                     UNSPEC_ADDV))]
1600  "TARGET_SIMD"
1601  ""
1602 )
1603
1604 (define_expand "reduc_splus_v2si"
1605  [(set (match_operand:V2SI 0 "register_operand" "=w")
1606        (unspec:V2SI [(match_operand:V2SI 1 "register_operand" "w")]
1607                     UNSPEC_ADDV))]
1608  "TARGET_SIMD"
1609  ""
1610 )
1611
1612 (define_insn "reduc_<maxminv>_<mode>"
1613  [(set (match_operand:VDQV 0 "register_operand" "=w")
1614        (unspec:VDQV [(match_operand:VDQV 1 "register_operand" "w")]
1615                     MAXMINV))]
1616  "TARGET_SIMD"
1617  "<maxminv>v\\t%<Vetype>0, %1.<Vtype>"
1618   [(set_attr "simd_type" "simd_minmaxv")
1619    (set_attr "simd_mode" "<MODE>")]
1620 )
1621
1622 (define_insn "reduc_<maxminv>_v2si"
1623  [(set (match_operand:V2SI 0 "register_operand" "=w")
1624        (unspec:V2SI [(match_operand:V2SI 1 "register_operand" "w")]
1625                     MAXMINV))]
1626  "TARGET_SIMD"
1627  "<maxminv>p\\t%0.2s, %1.2s, %1.2s"
1628   [(set_attr "simd_type" "simd_minmax")
1629    (set_attr "simd_mode" "V2SI")]
1630 )
1631
1632 ;; aarch64_simd_bsl may compile to any of bsl/bif/bit depending on register
1633 ;; allocation.
1634 ;; Operand 1 is the mask, operands 2 and 3 are the bitfields from which
1635 ;; to select.
1636 ;;
1637 ;; Thus our BSL is of the form:
1638 ;;   op0 = bsl (mask, op2, op3)
1639 ;; We can use any of:
1640 ;;
1641 ;;   if (op0 = mask)
1642 ;;     bsl mask, op1, op2
1643 ;;   if (op0 = op1) (so 1-bits in mask choose bits from op2, else op0)
1644 ;;     bit op0, op2, mask
1645 ;;   if (op0 = op2) (so 0-bits in mask choose bits from op1, else op0)
1646 ;;     bif op0, op1, mask
1647
1648 (define_insn "aarch64_simd_bsl<mode>_internal"
1649   [(set (match_operand:VALL 0 "register_operand"                "=w,w,w")
1650         (ior:VALL
1651            (and:VALL
1652              (match_operand:<V_cmp_result> 1 "register_operand" " 0,w,w")
1653              (match_operand:VALL 2 "register_operand"           " w,w,0"))
1654            (and:VALL
1655              (not:<V_cmp_result>
1656                 (match_dup:<V_cmp_result> 1))
1657              (match_operand:VALL 3 "register_operand"           " w,0,w"))
1658         ))]
1659   "TARGET_SIMD"
1660   "@
1661   bsl\\t%0.<Vbtype>, %2.<Vbtype>, %3.<Vbtype>
1662   bit\\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>
1663   bif\\t%0.<Vbtype>, %3.<Vbtype>, %1.<Vbtype>"
1664 )
1665
1666 (define_expand "aarch64_simd_bsl<mode>"
1667   [(match_operand:VALL 0 "register_operand")
1668    (match_operand:<V_cmp_result> 1 "register_operand")
1669    (match_operand:VALL 2 "register_operand")
1670    (match_operand:VALL 3 "register_operand")]
1671  "TARGET_SIMD"
1672 {
1673   /* We can't alias operands together if they have different modes.  */
1674   operands[1] = gen_lowpart (<V_cmp_result>mode, operands[1]);
1675   emit_insn (gen_aarch64_simd_bsl<mode>_internal (operands[0], operands[1],
1676                                                   operands[2], operands[3]));
1677   DONE;
1678 })
1679
1680 (define_expand "aarch64_vcond_internal<mode>"
1681   [(set (match_operand:VDQ 0 "register_operand")
1682         (if_then_else:VDQ
1683           (match_operator 3 "comparison_operator"
1684             [(match_operand:VDQ 4 "register_operand")
1685              (match_operand:VDQ 5 "nonmemory_operand")])
1686           (match_operand:VDQ 1 "nonmemory_operand")
1687           (match_operand:VDQ 2 "nonmemory_operand")))]
1688   "TARGET_SIMD"
1689 {
1690   int inverse = 0, has_zero_imm_form = 0;
1691   rtx op1 = operands[1];
1692   rtx op2 = operands[2];
1693   rtx mask = gen_reg_rtx (<MODE>mode);
1694
1695   switch (GET_CODE (operands[3]))
1696     {
1697     case LE:
1698     case LT:
1699     case NE:
1700       inverse = 1;
1701       /* Fall through.  */
1702     case GE:
1703     case GT:
1704     case EQ:
1705       has_zero_imm_form = 1;
1706       break;
1707     case LEU:
1708     case LTU:
1709       inverse = 1;
1710       break;
1711     default:
1712       break;
1713     }
1714
1715   if (!REG_P (operands[5])
1716       && (operands[5] != CONST0_RTX (<MODE>mode) || !has_zero_imm_form))
1717     operands[5] = force_reg (<MODE>mode, operands[5]);
1718
1719   switch (GET_CODE (operands[3]))
1720     {
1721     case LT:
1722     case GE:
1723       emit_insn (gen_aarch64_cmge<mode> (mask, operands[4], operands[5]));
1724       break;
1725
1726     case LE:
1727     case GT:
1728       emit_insn (gen_aarch64_cmgt<mode> (mask, operands[4], operands[5]));
1729       break;
1730
1731     case LTU:
1732     case GEU:
1733       emit_insn (gen_aarch64_cmgeu<mode> (mask, operands[4], operands[5]));
1734       break;
1735
1736     case LEU:
1737     case GTU:
1738       emit_insn (gen_aarch64_cmgtu<mode> (mask, operands[4], operands[5]));
1739       break;
1740
1741     case NE:
1742     case EQ:
1743       emit_insn (gen_aarch64_cmeq<mode> (mask, operands[4], operands[5]));
1744       break;
1745
1746     default:
1747       gcc_unreachable ();
1748     }
1749
1750   if (inverse)
1751     {
1752       op1 = operands[2];
1753       op2 = operands[1];
1754     }
1755
1756     /* If we have (a = (b CMP c) ? -1 : 0);
1757        Then we can simply move the generated mask.  */
1758
1759     if (op1 == CONSTM1_RTX (<V_cmp_result>mode)
1760         && op2 == CONST0_RTX (<V_cmp_result>mode))
1761       emit_move_insn (operands[0], mask);
1762     else
1763       {
1764         if (!REG_P (op1))
1765           op1 = force_reg (<MODE>mode, op1);
1766         if (!REG_P (op2))
1767           op2 = force_reg (<MODE>mode, op2);
1768         emit_insn (gen_aarch64_simd_bsl<mode> (operands[0], mask,
1769                                                op1, op2));
1770       }
1771
1772   DONE;
1773 })
1774
1775 (define_expand "aarch64_vcond_internal<mode>"
1776   [(set (match_operand:VDQF 0 "register_operand")
1777         (if_then_else:VDQF
1778           (match_operator 3 "comparison_operator"
1779             [(match_operand:VDQF 4 "register_operand")
1780              (match_operand:VDQF 5 "nonmemory_operand")])
1781           (match_operand:VDQF 1 "nonmemory_operand")
1782           (match_operand:VDQF 2 "nonmemory_operand")))]
1783   "TARGET_SIMD"
1784 {
1785   int inverse = 0;
1786   int use_zero_form = 0;
1787   int swap_bsl_operands = 0;
1788   rtx op1 = operands[1];
1789   rtx op2 = operands[2];
1790   rtx mask = gen_reg_rtx (<V_cmp_result>mode);
1791   rtx tmp = gen_reg_rtx (<V_cmp_result>mode);
1792
1793   rtx (*base_comparison) (rtx, rtx, rtx);
1794   rtx (*complimentary_comparison) (rtx, rtx, rtx);
1795
1796   switch (GET_CODE (operands[3]))
1797     {
1798     case GE:
1799     case GT:
1800     case LE:
1801     case LT:
1802     case EQ:
1803       if (operands[5] == CONST0_RTX (<MODE>mode))
1804         {
1805           use_zero_form = 1;
1806           break;
1807         }
1808       /* Fall through.  */
1809     default:
1810       if (!REG_P (operands[5]))
1811         operands[5] = force_reg (<MODE>mode, operands[5]);
1812     }
1813
1814   switch (GET_CODE (operands[3]))
1815     {
1816     case LT:
1817     case UNLT:
1818       inverse = 1;
1819       /* Fall through.  */
1820     case GE:
1821     case UNGE:
1822     case ORDERED:
1823     case UNORDERED:
1824       base_comparison = gen_aarch64_cmge<mode>;
1825       complimentary_comparison = gen_aarch64_cmgt<mode>;
1826       break;
1827     case LE:
1828     case UNLE:
1829       inverse = 1;
1830       /* Fall through.  */
1831     case GT:
1832     case UNGT:
1833       base_comparison = gen_aarch64_cmgt<mode>;
1834       complimentary_comparison = gen_aarch64_cmge<mode>;
1835       break;
1836     case EQ:
1837     case NE:
1838     case UNEQ:
1839       base_comparison = gen_aarch64_cmeq<mode>;
1840       complimentary_comparison = gen_aarch64_cmeq<mode>;
1841       break;
1842     default:
1843       gcc_unreachable ();
1844     }
1845
1846   switch (GET_CODE (operands[3]))
1847     {
1848     case LT:
1849     case LE:
1850     case GT:
1851     case GE:
1852     case EQ:
1853       /* The easy case.  Here we emit one of FCMGE, FCMGT or FCMEQ.
1854          As a LT b <=> b GE a && a LE b <=> b GT a.  Our transformations are:
1855          a GE b -> a GE b
1856          a GT b -> a GT b
1857          a LE b -> b GE a
1858          a LT b -> b GT a
1859          a EQ b -> a EQ b
1860          Note that there also exist direct comparison against 0 forms,
1861          so catch those as a special case.  */
1862       if (use_zero_form)
1863         {
1864           inverse = 0;
1865           switch (GET_CODE (operands[3]))
1866             {
1867             case LT:
1868               base_comparison = gen_aarch64_cmlt<mode>;
1869               break;
1870             case LE:
1871               base_comparison = gen_aarch64_cmle<mode>;
1872               break;
1873             default:
1874               /* Do nothing, other zero form cases already have the correct
1875                  base_comparison.  */
1876               break;
1877             }
1878         }
1879
1880       if (!inverse)
1881         emit_insn (base_comparison (mask, operands[4], operands[5]));
1882       else
1883         emit_insn (complimentary_comparison (mask, operands[5], operands[4]));
1884       break;
1885     case UNLT:
1886     case UNLE:
1887     case UNGT:
1888     case UNGE:
1889     case NE:
1890       /* FCM returns false for lanes which are unordered, so if we use
1891          the inverse of the comparison we actually want to emit, then
1892          swap the operands to BSL, we will end up with the correct result.
1893          Note that a NE NaN and NaN NE b are true for all a, b.
1894
1895          Our transformations are:
1896          a GE b -> !(b GT a)
1897          a GT b -> !(b GE a)
1898          a LE b -> !(a GT b)
1899          a LT b -> !(a GE b)
1900          a NE b -> !(a EQ b)  */
1901
1902       if (inverse)
1903         emit_insn (base_comparison (mask, operands[4], operands[5]));
1904       else
1905         emit_insn (complimentary_comparison (mask, operands[5], operands[4]));
1906
1907       swap_bsl_operands = 1;
1908       break;
1909     case UNEQ:
1910       /* We check (a > b ||  b > a).  combining these comparisons give us
1911          true iff !(a != b && a ORDERED b), swapping the operands to BSL
1912          will then give us (a == b ||  a UNORDERED b) as intended.  */
1913
1914       emit_insn (gen_aarch64_cmgt<mode> (mask, operands[4], operands[5]));
1915       emit_insn (gen_aarch64_cmgt<mode> (tmp, operands[5], operands[4]));
1916       emit_insn (gen_ior<v_cmp_result>3 (mask, mask, tmp));
1917       swap_bsl_operands = 1;
1918       break;
1919     case UNORDERED:
1920        /* Operands are ORDERED iff (a > b || b >= a).
1921          Swapping the operands to BSL will give the UNORDERED case.  */
1922      swap_bsl_operands = 1;
1923      /* Fall through.  */
1924     case ORDERED:
1925       emit_insn (gen_aarch64_cmgt<mode> (tmp, operands[4], operands[5]));
1926       emit_insn (gen_aarch64_cmge<mode> (mask, operands[5], operands[4]));
1927       emit_insn (gen_ior<v_cmp_result>3 (mask, mask, tmp));
1928       break;
1929     default:
1930       gcc_unreachable ();
1931     }
1932
1933   if (swap_bsl_operands)
1934     {
1935       op1 = operands[2];
1936       op2 = operands[1];
1937     }
1938
1939     /* If we have (a = (b CMP c) ? -1 : 0);
1940        Then we can simply move the generated mask.  */
1941
1942     if (op1 == CONSTM1_RTX (<V_cmp_result>mode)
1943         && op2 == CONST0_RTX (<V_cmp_result>mode))
1944       emit_move_insn (operands[0], mask);
1945     else
1946       {
1947         if (!REG_P (op1))
1948           op1 = force_reg (<MODE>mode, op1);
1949         if (!REG_P (op2))
1950           op2 = force_reg (<MODE>mode, op2);
1951         emit_insn (gen_aarch64_simd_bsl<mode> (operands[0], mask,
1952                                                op1, op2));
1953       }
1954
1955   DONE;
1956 })
1957
1958 (define_expand "vcond<mode><mode>"
1959   [(set (match_operand:VALL 0 "register_operand")
1960         (if_then_else:VALL
1961           (match_operator 3 "comparison_operator"
1962             [(match_operand:VALL 4 "register_operand")
1963              (match_operand:VALL 5 "nonmemory_operand")])
1964           (match_operand:VALL 1 "nonmemory_operand")
1965           (match_operand:VALL 2 "nonmemory_operand")))]
1966   "TARGET_SIMD"
1967 {
1968   emit_insn (gen_aarch64_vcond_internal<mode> (operands[0], operands[1],
1969                                                operands[2], operands[3],
1970                                                operands[4], operands[5]));
1971   DONE;
1972 })
1973
1974 (define_expand "vcond<v_cmp_result><mode>"
1975   [(set (match_operand:<V_cmp_result> 0 "register_operand")
1976         (if_then_else:<V_cmp_result>
1977           (match_operator 3 "comparison_operator"
1978             [(match_operand:VDQF 4 "register_operand")
1979              (match_operand:VDQF 5 "nonmemory_operand")])
1980           (match_operand:<V_cmp_result> 1 "nonmemory_operand")
1981           (match_operand:<V_cmp_result> 2 "nonmemory_operand")))]
1982   "TARGET_SIMD"
1983 {
1984   emit_insn (gen_aarch64_vcond_internal<v_cmp_result> (
1985                                                 operands[0], operands[1],
1986                                                 operands[2], operands[3],
1987                                                 operands[4], operands[5]));
1988   DONE;
1989 })
1990
1991 (define_expand "vcondu<mode><mode>"
1992   [(set (match_operand:VDQ 0 "register_operand")
1993         (if_then_else:VDQ
1994           (match_operator 3 "comparison_operator"
1995             [(match_operand:VDQ 4 "register_operand")
1996              (match_operand:VDQ 5 "nonmemory_operand")])
1997           (match_operand:VDQ 1 "nonmemory_operand")
1998           (match_operand:VDQ 2 "nonmemory_operand")))]
1999   "TARGET_SIMD"
2000 {
2001   emit_insn (gen_aarch64_vcond_internal<mode> (operands[0], operands[1],
2002                                                operands[2], operands[3],
2003                                                operands[4], operands[5]));
2004   DONE;
2005 })
2006
2007 ;; Patterns for AArch64 SIMD Intrinsics.
2008
2009 (define_expand "aarch64_create<mode>"
2010   [(match_operand:VD_RE 0 "register_operand" "")
2011    (match_operand:DI 1 "general_operand" "")]
2012   "TARGET_SIMD"
2013 {
2014   rtx src = gen_lowpart (<MODE>mode, operands[1]);
2015   emit_move_insn (operands[0], src);
2016   DONE;
2017 })
2018
2019 (define_insn "aarch64_get_lane_signed<mode>"
2020   [(set (match_operand:<VEL> 0 "register_operand" "=r")
2021         (sign_extend:<VEL>
2022           (vec_select:<VEL>
2023             (match_operand:VQ_S 1 "register_operand" "w")
2024             (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2025   "TARGET_SIMD"
2026   "smov\\t%0, %1.<Vetype>[%2]"
2027   [(set_attr "simd_type" "simd_movgp")
2028    (set_attr "simd_mode" "<MODE>")]
2029 )
2030
2031 (define_insn "aarch64_get_lane_unsigned<mode>"
2032   [(set (match_operand:<VEL> 0 "register_operand" "=r")
2033         (zero_extend:<VEL>
2034           (vec_select:<VEL>
2035             (match_operand:VDQ 1 "register_operand" "w")
2036             (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2037   "TARGET_SIMD"
2038   "umov\\t%<vw>0, %1.<Vetype>[%2]"
2039   [(set_attr "simd_type" "simd_movgp")
2040    (set_attr "simd_mode" "<MODE>")]
2041 )
2042
2043 (define_insn "aarch64_get_lane<mode>"
2044   [(set (match_operand:<VEL> 0 "register_operand" "=w")
2045         (vec_select:<VEL>
2046             (match_operand:VDQF 1 "register_operand" "w")
2047             (parallel [(match_operand:SI 2 "immediate_operand" "i")])))]
2048   "TARGET_SIMD"
2049   "mov\\t%0.<Vetype>[0], %1.<Vetype>[%2]"
2050   [(set_attr "simd_type" "simd_ins")
2051    (set_attr "simd_mode" "<MODE>")]
2052 )
2053
2054 (define_expand "aarch64_get_lanedi"
2055   [(match_operand:DI 0 "register_operand" "=r")
2056    (match_operand:DI 1 "register_operand" "w")
2057    (match_operand:SI 2 "immediate_operand" "i")]
2058   "TARGET_SIMD"
2059 {
2060   aarch64_simd_lane_bounds (operands[2], 0, 1);
2061   emit_move_insn (operands[0], operands[1]);
2062   DONE;
2063 })
2064
2065 (define_expand "aarch64_reinterpretv8qi<mode>"
2066   [(match_operand:V8QI 0 "register_operand" "")
2067    (match_operand:VDC 1 "register_operand" "")]
2068   "TARGET_SIMD"
2069 {
2070   aarch64_simd_reinterpret (operands[0], operands[1]);
2071   DONE;
2072 })
2073
2074 (define_expand "aarch64_reinterpretv4hi<mode>"
2075   [(match_operand:V4HI 0 "register_operand" "")
2076    (match_operand:VDC 1 "register_operand" "")]
2077   "TARGET_SIMD"
2078 {
2079   aarch64_simd_reinterpret (operands[0], operands[1]);
2080   DONE;
2081 })
2082
2083 (define_expand "aarch64_reinterpretv2si<mode>"
2084   [(match_operand:V2SI 0 "register_operand" "")
2085    (match_operand:VDC 1 "register_operand" "")]
2086   "TARGET_SIMD"
2087 {
2088   aarch64_simd_reinterpret (operands[0], operands[1]);
2089   DONE;
2090 })
2091
2092 (define_expand "aarch64_reinterpretv2sf<mode>"
2093   [(match_operand:V2SF 0 "register_operand" "")
2094    (match_operand:VDC 1 "register_operand" "")]
2095   "TARGET_SIMD"
2096 {
2097   aarch64_simd_reinterpret (operands[0], operands[1]);
2098   DONE;
2099 })
2100
2101 (define_expand "aarch64_reinterpretdi<mode>"
2102   [(match_operand:DI 0 "register_operand" "")
2103    (match_operand:VD_RE 1 "register_operand" "")]
2104   "TARGET_SIMD"
2105 {
2106   aarch64_simd_reinterpret (operands[0], operands[1]);
2107   DONE;
2108 })
2109
2110 (define_expand "aarch64_reinterpretv16qi<mode>"
2111   [(match_operand:V16QI 0 "register_operand" "")
2112    (match_operand:VQ 1 "register_operand" "")]
2113   "TARGET_SIMD"
2114 {
2115   aarch64_simd_reinterpret (operands[0], operands[1]);
2116   DONE;
2117 })
2118
2119 (define_expand "aarch64_reinterpretv8hi<mode>"
2120   [(match_operand:V8HI 0 "register_operand" "")
2121    (match_operand:VQ 1 "register_operand" "")]
2122   "TARGET_SIMD"
2123 {
2124   aarch64_simd_reinterpret (operands[0], operands[1]);
2125   DONE;
2126 })
2127
2128 (define_expand "aarch64_reinterpretv4si<mode>"
2129   [(match_operand:V4SI 0 "register_operand" "")
2130    (match_operand:VQ 1 "register_operand" "")]
2131   "TARGET_SIMD"
2132 {
2133   aarch64_simd_reinterpret (operands[0], operands[1]);
2134   DONE;
2135 })
2136
2137 (define_expand "aarch64_reinterpretv4sf<mode>"
2138   [(match_operand:V4SF 0 "register_operand" "")
2139    (match_operand:VQ 1 "register_operand" "")]
2140   "TARGET_SIMD"
2141 {
2142   aarch64_simd_reinterpret (operands[0], operands[1]);
2143   DONE;
2144 })
2145
2146 (define_expand "aarch64_reinterpretv2di<mode>"
2147   [(match_operand:V2DI 0 "register_operand" "")
2148    (match_operand:VQ 1 "register_operand" "")]
2149   "TARGET_SIMD"
2150 {
2151   aarch64_simd_reinterpret (operands[0], operands[1]);
2152   DONE;
2153 })
2154
2155 (define_expand "aarch64_reinterpretv2df<mode>"
2156   [(match_operand:V2DF 0 "register_operand" "")
2157    (match_operand:VQ 1 "register_operand" "")]
2158   "TARGET_SIMD"
2159 {
2160   aarch64_simd_reinterpret (operands[0], operands[1]);
2161   DONE;
2162 })
2163
2164 ;; In this insn, operand 1 should be low, and operand 2 the high part of the
2165 ;; dest vector.
2166
2167 (define_insn "*aarch64_combinez<mode>"
2168   [(set (match_operand:<VDBL> 0 "register_operand" "=&w")
2169         (vec_concat:<VDBL>
2170            (match_operand:VDIC 1 "register_operand" "w")
2171            (match_operand:VDIC 2 "aarch64_simd_imm_zero" "Dz")))]
2172   "TARGET_SIMD"
2173   "mov\\t%0.8b, %1.8b"
2174   [(set_attr "simd_type" "simd_move")
2175    (set_attr "simd_mode" "<MODE>")]
2176 )
2177
2178 (define_insn "aarch64_combine<mode>"
2179   [(set (match_operand:<VDBL> 0 "register_operand" "=&w")
2180         (vec_concat:<VDBL> (match_operand:VDC 1 "register_operand" "w")
2181                            (match_operand:VDC 2 "register_operand" "w")))]
2182   "TARGET_SIMD"
2183   "mov\\t%0.d[0], %1.d[0]\;ins\\t%0.d[1], %2.d[0]"
2184   [(set_attr "simd_type" "simd_ins")
2185    (set_attr "simd_mode" "<MODE>")]
2186 )
2187
2188 ;; <su><addsub>l<q>.
2189
2190 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>l2<mode>_internal"
2191  [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2192        (ADDSUB:<VWIDE> (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
2193                            (match_operand:VQW 1 "register_operand" "w")
2194                            (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))
2195                        (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
2196                            (match_operand:VQW 2 "register_operand" "w")
2197                            (match_dup 3)))))]
2198   "TARGET_SIMD"
2199   "<ANY_EXTEND:su><ADDSUB:optab>l2 %0.<Vwtype>, %1.<Vtype>, %2.<Vtype>"
2200   [(set_attr "simd_type" "simd_addl")
2201    (set_attr "simd_mode" "<MODE>")]
2202 )
2203
2204 (define_expand "aarch64_saddl2<mode>"
2205   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2206    (match_operand:VQW 1 "register_operand" "w")
2207    (match_operand:VQW 2 "register_operand" "w")]
2208   "TARGET_SIMD"
2209 {
2210   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2211   emit_insn (gen_aarch64_saddl2<mode>_internal (operands[0], operands[1],
2212                                                 operands[2], p));
2213   DONE;
2214 })
2215
2216 (define_expand "aarch64_uaddl2<mode>"
2217   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2218    (match_operand:VQW 1 "register_operand" "w")
2219    (match_operand:VQW 2 "register_operand" "w")]
2220   "TARGET_SIMD"
2221 {
2222   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2223   emit_insn (gen_aarch64_uaddl2<mode>_internal (operands[0], operands[1],
2224                                                 operands[2], p));
2225   DONE;
2226 })
2227
2228 (define_expand "aarch64_ssubl2<mode>"
2229   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2230    (match_operand:VQW 1 "register_operand" "w")
2231    (match_operand:VQW 2 "register_operand" "w")]
2232   "TARGET_SIMD"
2233 {
2234   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2235   emit_insn (gen_aarch64_ssubl2<mode>_internal (operands[0], operands[1],
2236                                                 operands[2], p));
2237   DONE;
2238 })
2239
2240 (define_expand "aarch64_usubl2<mode>"
2241   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2242    (match_operand:VQW 1 "register_operand" "w")
2243    (match_operand:VQW 2 "register_operand" "w")]
2244   "TARGET_SIMD"
2245 {
2246   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2247   emit_insn (gen_aarch64_usubl2<mode>_internal (operands[0], operands[1],
2248                                                 operands[2], p));
2249   DONE;
2250 })
2251
2252 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>l<mode>"
2253  [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2254        (ADDSUB:<VWIDE> (ANY_EXTEND:<VWIDE>
2255                            (match_operand:VDW 1 "register_operand" "w"))
2256                        (ANY_EXTEND:<VWIDE>
2257                            (match_operand:VDW 2 "register_operand" "w"))))]
2258   "TARGET_SIMD"
2259   "<ANY_EXTEND:su><ADDSUB:optab>l %0.<Vwtype>, %1.<Vtype>, %2.<Vtype>"
2260   [(set_attr "simd_type" "simd_addl")
2261    (set_attr "simd_mode" "<MODE>")]
2262 )
2263
2264 ;; <su><addsub>w<q>.
2265
2266 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>w<mode>"
2267   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2268         (ADDSUB:<VWIDE> (match_operand:<VWIDE> 1 "register_operand" "w")
2269                         (ANY_EXTEND:<VWIDE>
2270                           (match_operand:VDW 2 "register_operand" "w"))))]
2271   "TARGET_SIMD"
2272   "<ANY_EXTEND:su><ADDSUB:optab>w\\t%0.<Vwtype>, %1.<Vwtype>, %2.<Vtype>"
2273   [(set_attr "simd_type" "simd_addl")
2274    (set_attr "simd_mode" "<MODE>")]
2275 )
2276
2277 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>w2<mode>_internal"
2278   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2279         (ADDSUB:<VWIDE> (match_operand:<VWIDE> 1 "register_operand" "w")
2280                         (ANY_EXTEND:<VWIDE>
2281                           (vec_select:<VHALF>
2282                            (match_operand:VQW 2 "register_operand" "w")
2283                            (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))))]
2284   "TARGET_SIMD"
2285   "<ANY_EXTEND:su><ADDSUB:optab>w2\\t%0.<Vwtype>, %1.<Vwtype>, %2.<Vtype>"
2286   [(set_attr "simd_type" "simd_addl")
2287    (set_attr "simd_mode" "<MODE>")]
2288 )
2289
2290 (define_expand "aarch64_saddw2<mode>"
2291   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2292    (match_operand:<VWIDE> 1 "register_operand" "w")
2293    (match_operand:VQW 2 "register_operand" "w")]
2294   "TARGET_SIMD"
2295 {
2296   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2297   emit_insn (gen_aarch64_saddw2<mode>_internal (operands[0], operands[1],
2298                                                 operands[2], p));
2299   DONE;
2300 })
2301
2302 (define_expand "aarch64_uaddw2<mode>"
2303   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2304    (match_operand:<VWIDE> 1 "register_operand" "w")
2305    (match_operand:VQW 2 "register_operand" "w")]
2306   "TARGET_SIMD"
2307 {
2308   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2309   emit_insn (gen_aarch64_uaddw2<mode>_internal (operands[0], operands[1],
2310                                                 operands[2], p));
2311   DONE;
2312 })
2313
2314
2315 (define_expand "aarch64_ssubw2<mode>"
2316   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2317    (match_operand:<VWIDE> 1 "register_operand" "w")
2318    (match_operand:VQW 2 "register_operand" "w")]
2319   "TARGET_SIMD"
2320 {
2321   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2322   emit_insn (gen_aarch64_ssubw2<mode>_internal (operands[0], operands[1],
2323                                                 operands[2], p));
2324   DONE;
2325 })
2326
2327 (define_expand "aarch64_usubw2<mode>"
2328   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2329    (match_operand:<VWIDE> 1 "register_operand" "w")
2330    (match_operand:VQW 2 "register_operand" "w")]
2331   "TARGET_SIMD"
2332 {
2333   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2334   emit_insn (gen_aarch64_usubw2<mode>_internal (operands[0], operands[1],
2335                                                 operands[2], p));
2336   DONE;
2337 })
2338
2339 ;; <su><r>h<addsub>.
2340
2341 (define_insn "aarch64_<sur>h<addsub><mode>"
2342   [(set (match_operand:VQ_S 0 "register_operand" "=w")
2343         (unspec:VQ_S [(match_operand:VQ_S 1 "register_operand" "w")
2344                       (match_operand:VQ_S 2 "register_operand" "w")]
2345                      HADDSUB))]
2346   "TARGET_SIMD"
2347   "<sur>h<addsub>\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
2348   [(set_attr "simd_type" "simd_add")
2349    (set_attr "simd_mode" "<MODE>")]
2350 )
2351
2352 ;; <r><addsub>hn<q>.
2353
2354 (define_insn "aarch64_<sur><addsub>hn<mode>"
2355   [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
2356         (unspec:<VNARROWQ> [(match_operand:VQN 1 "register_operand" "w")
2357                             (match_operand:VQN 2 "register_operand" "w")]
2358                            ADDSUBHN))]
2359   "TARGET_SIMD"
2360   "<sur><addsub>hn\\t%0.<Vntype>, %1.<Vtype>, %2.<Vtype>"
2361   [(set_attr "simd_type" "simd_addn")
2362    (set_attr "simd_mode" "<MODE>")]
2363 )
2364
2365 (define_insn "aarch64_<sur><addsub>hn2<mode>"
2366   [(set (match_operand:<VNARROWQ2> 0 "register_operand" "=w")
2367         (unspec:<VNARROWQ2> [(match_operand:<VNARROWQ> 1 "register_operand" "0")
2368                              (match_operand:VQN 2 "register_operand" "w")
2369                              (match_operand:VQN 3 "register_operand" "w")]
2370                             ADDSUBHN2))]
2371   "TARGET_SIMD"
2372   "<sur><addsub>hn2\\t%0.<V2ntype>, %2.<Vtype>, %3.<Vtype>"
2373   [(set_attr "simd_type" "simd_addn2")
2374    (set_attr "simd_mode" "<MODE>")]
2375 )
2376
2377 ;; pmul.
2378
2379 (define_insn "aarch64_pmul<mode>"
2380   [(set (match_operand:VB 0 "register_operand" "=w")
2381         (unspec:VB [(match_operand:VB 1 "register_operand" "w")
2382                     (match_operand:VB 2 "register_operand" "w")]
2383                    UNSPEC_PMUL))]
2384  "TARGET_SIMD"
2385  "pmul\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
2386   [(set_attr "simd_type" "simd_mul")
2387    (set_attr "simd_mode" "<MODE>")]
2388 )
2389
2390 ;; <su>q<addsub>
2391
2392 (define_insn "aarch64_<su_optab><optab><mode>"
2393   [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
2394         (BINQOPS:VSDQ_I (match_operand:VSDQ_I 1 "register_operand" "w")
2395                           (match_operand:VSDQ_I 2 "register_operand" "w")))]
2396   "TARGET_SIMD"
2397   "<su_optab><optab>\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
2398   [(set_attr "simd_type" "simd_add")
2399    (set_attr "simd_mode" "<MODE>")]
2400 )
2401
2402 ;; suqadd and usqadd
2403
2404 (define_insn "aarch64_<sur>qadd<mode>"
2405   [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
2406         (unspec:VSDQ_I [(match_operand:VSDQ_I 1 "register_operand" "0")
2407                         (match_operand:VSDQ_I 2 "register_operand" "w")]
2408                        USSUQADD))]
2409   "TARGET_SIMD"
2410   "<sur>qadd\\t%<v>0<Vmtype>, %<v>2<Vmtype>"
2411   [(set_attr "simd_type" "simd_sat_add")
2412    (set_attr "simd_mode" "<MODE>")]
2413 )
2414
2415 ;; sqmovun
2416
2417 (define_insn "aarch64_sqmovun<mode>"
2418   [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
2419         (unspec:<VNARROWQ> [(match_operand:VSQN_HSDI 1 "register_operand" "w")]
2420                             UNSPEC_SQXTUN))]
2421    "TARGET_SIMD"
2422    "sqxtun\\t%<vn2>0<Vmntype>, %<v>1<Vmtype>"
2423    [(set_attr "simd_type" "simd_sat_shiftn_imm")
2424     (set_attr "simd_mode" "<MODE>")]
2425  )
2426
2427 ;; sqmovn and uqmovn
2428
2429 (define_insn "aarch64_<sur>qmovn<mode>"
2430   [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
2431         (unspec:<VNARROWQ> [(match_operand:VSQN_HSDI 1 "register_operand" "w")]
2432                             SUQMOVN))]
2433   "TARGET_SIMD"
2434   "<sur>qxtn\\t%<vn2>0<Vmntype>, %<v>1<Vmtype>"
2435    [(set_attr "simd_type" "simd_sat_shiftn_imm")
2436     (set_attr "simd_mode" "<MODE>")]
2437  )
2438
2439 ;; <su>q<absneg>
2440
2441 (define_insn "aarch64_s<optab><mode>"
2442   [(set (match_operand:VSDQ_I_BHSI 0 "register_operand" "=w")
2443         (UNQOPS:VSDQ_I_BHSI
2444           (match_operand:VSDQ_I_BHSI 1 "register_operand" "w")))]
2445   "TARGET_SIMD"
2446   "s<optab>\\t%<v>0<Vmtype>, %<v>1<Vmtype>"
2447   [(set_attr "simd_type" "simd_sat_negabs")
2448    (set_attr "simd_mode" "<MODE>")]
2449 )
2450
2451 ;; sq<r>dmulh.
2452
2453 (define_insn "aarch64_sq<r>dmulh<mode>"
2454   [(set (match_operand:VSDQ_HSI 0 "register_operand" "=w")
2455         (unspec:VSDQ_HSI
2456           [(match_operand:VSDQ_HSI 1 "register_operand" "w")
2457            (match_operand:VSDQ_HSI 2 "register_operand" "w")]
2458          VQDMULH))]
2459   "TARGET_SIMD"
2460   "sq<r>dmulh\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
2461   [(set_attr "simd_type" "simd_sat_mul")
2462    (set_attr "simd_mode" "<MODE>")]
2463 )
2464
2465 ;; sq<r>dmulh_lane
2466
2467 (define_insn "aarch64_sq<r>dmulh_lane<mode>"
2468   [(set (match_operand:VDQHS 0 "register_operand" "=w")
2469         (unspec:VDQHS
2470           [(match_operand:VDQHS 1 "register_operand" "w")
2471            (vec_select:<VEL>
2472              (match_operand:<VCOND> 2 "register_operand" "<vwx>")
2473              (parallel [(match_operand:SI 3 "immediate_operand" "i")]))]
2474          VQDMULH))]
2475   "TARGET_SIMD"
2476   "*
2477    aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCOND>mode));
2478    return \"sq<r>dmulh\\t%0.<Vtype>, %1.<Vtype>, %2.<Vetype>[%3]\";"
2479   [(set_attr "simd_type" "simd_sat_mul")
2480    (set_attr "simd_mode" "<MODE>")]
2481 )
2482
2483 (define_insn "aarch64_sq<r>dmulh_laneq<mode>"
2484   [(set (match_operand:VDQHS 0 "register_operand" "=w")
2485         (unspec:VDQHS
2486           [(match_operand:VDQHS 1 "register_operand" "w")
2487            (vec_select:<VEL>
2488              (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
2489              (parallel [(match_operand:SI 3 "immediate_operand" "i")]))]
2490          VQDMULH))]
2491   "TARGET_SIMD"
2492   "*
2493    aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCONQ>mode));
2494    return \"sq<r>dmulh\\t%0.<Vtype>, %1.<Vtype>, %2.<Vetype>[%3]\";"
2495   [(set_attr "simd_type" "simd_sat_mul")
2496    (set_attr "simd_mode" "<MODE>")]
2497 )
2498
2499 (define_insn "aarch64_sq<r>dmulh_lane<mode>"
2500   [(set (match_operand:SD_HSI 0 "register_operand" "=w")
2501         (unspec:SD_HSI
2502           [(match_operand:SD_HSI 1 "register_operand" "w")
2503            (vec_select:<VEL>
2504              (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
2505              (parallel [(match_operand:SI 3 "immediate_operand" "i")]))]
2506          VQDMULH))]
2507   "TARGET_SIMD"
2508   "*
2509    aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCONQ>mode));
2510    return \"sq<r>dmulh\\t%<v>0, %<v>1, %2.<v>[%3]\";"
2511   [(set_attr "simd_type" "simd_sat_mul")
2512    (set_attr "simd_mode" "<MODE>")]
2513 )
2514
2515 ;; vqdml[sa]l
2516
2517 (define_insn "aarch64_sqdml<SBINQOPS:as>l<mode>"
2518   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2519         (SBINQOPS:<VWIDE>
2520           (match_operand:<VWIDE> 1 "register_operand" "0")
2521           (ss_ashift:<VWIDE>
2522               (mult:<VWIDE>
2523                 (sign_extend:<VWIDE>
2524                       (match_operand:VSD_HSI 2 "register_operand" "w"))
2525                 (sign_extend:<VWIDE>
2526                       (match_operand:VSD_HSI 3 "register_operand" "w")))
2527               (const_int 1))))]
2528   "TARGET_SIMD"
2529   "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %<v>3<Vmtype>"
2530   [(set_attr "simd_type" "simd_sat_mlal")
2531    (set_attr "simd_mode" "<MODE>")]
2532 )
2533
2534 ;; vqdml[sa]l_lane
2535
2536 (define_insn "aarch64_sqdml<SBINQOPS:as>l_lane<mode>_internal"
2537   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2538         (SBINQOPS:<VWIDE>
2539           (match_operand:<VWIDE> 1 "register_operand" "0")
2540           (ss_ashift:<VWIDE>
2541             (mult:<VWIDE>
2542               (sign_extend:<VWIDE>
2543                 (match_operand:VD_HSI 2 "register_operand" "w"))
2544               (sign_extend:<VWIDE>
2545                 (vec_duplicate:VD_HSI
2546                   (vec_select:<VEL>
2547                     (match_operand:<VCON> 3 "register_operand" "<vwx>")
2548                     (parallel [(match_operand:SI 4 "immediate_operand" "i")])))
2549               ))
2550             (const_int 1))))]
2551   "TARGET_SIMD"
2552   "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]"
2553   [(set_attr "simd_type" "simd_sat_mlal")
2554    (set_attr "simd_mode" "<MODE>")]
2555 )
2556
2557 (define_insn "aarch64_sqdml<SBINQOPS:as>l_lane<mode>_internal"
2558   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2559         (SBINQOPS:<VWIDE>
2560           (match_operand:<VWIDE> 1 "register_operand" "0")
2561           (ss_ashift:<VWIDE>
2562             (mult:<VWIDE>
2563               (sign_extend:<VWIDE>
2564                 (match_operand:SD_HSI 2 "register_operand" "w"))
2565               (sign_extend:<VWIDE>
2566                 (vec_select:<VEL>
2567                   (match_operand:<VCON> 3 "register_operand" "<vwx>")
2568                   (parallel [(match_operand:SI 4 "immediate_operand" "i")])))
2569               )
2570             (const_int 1))))]
2571   "TARGET_SIMD"
2572   "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]"
2573   [(set_attr "simd_type" "simd_sat_mlal")
2574    (set_attr "simd_mode" "<MODE>")]
2575 )
2576
2577 (define_expand "aarch64_sqdmlal_lane<mode>"
2578   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2579    (match_operand:<VWIDE> 1 "register_operand" "0")
2580    (match_operand:VSD_HSI 2 "register_operand" "w")
2581    (match_operand:<VCON> 3 "register_operand" "<vwx>")
2582    (match_operand:SI 4 "immediate_operand" "i")]
2583   "TARGET_SIMD"
2584 {
2585   aarch64_simd_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<VCON>mode) / 2);
2586   emit_insn (gen_aarch64_sqdmlal_lane<mode>_internal (operands[0], operands[1],
2587                                                       operands[2], operands[3],
2588                                                       operands[4]));
2589   DONE;
2590 })
2591
2592 (define_expand "aarch64_sqdmlal_laneq<mode>"
2593   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2594    (match_operand:<VWIDE> 1 "register_operand" "0")
2595    (match_operand:VSD_HSI 2 "register_operand" "w")
2596    (match_operand:<VCON> 3 "register_operand" "<vwx>")
2597    (match_operand:SI 4 "immediate_operand" "i")]
2598   "TARGET_SIMD"
2599 {
2600   aarch64_simd_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<VCON>mode));
2601   emit_insn (gen_aarch64_sqdmlal_lane<mode>_internal (operands[0], operands[1],
2602                                                       operands[2], operands[3],
2603                                                       operands[4]));
2604   DONE;
2605 })
2606
2607 (define_expand "aarch64_sqdmlsl_lane<mode>"
2608   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2609    (match_operand:<VWIDE> 1 "register_operand" "0")
2610    (match_operand:VSD_HSI 2 "register_operand" "w")
2611    (match_operand:<VCON> 3 "register_operand" "<vwx>")
2612    (match_operand:SI 4 "immediate_operand" "i")]
2613   "TARGET_SIMD"
2614 {
2615   aarch64_simd_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<VCON>mode) / 2);
2616   emit_insn (gen_aarch64_sqdmlsl_lane<mode>_internal (operands[0], operands[1],
2617                                                       operands[2], operands[3],
2618                                                       operands[4]));
2619   DONE;
2620 })
2621
2622 (define_expand "aarch64_sqdmlsl_laneq<mode>"
2623   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2624    (match_operand:<VWIDE> 1 "register_operand" "0")
2625    (match_operand:VSD_HSI 2 "register_operand" "w")
2626    (match_operand:<VCON> 3 "register_operand" "<vwx>")
2627    (match_operand:SI 4 "immediate_operand" "i")]
2628   "TARGET_SIMD"
2629 {
2630   aarch64_simd_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<VCON>mode));
2631   emit_insn (gen_aarch64_sqdmlsl_lane<mode>_internal (operands[0], operands[1],
2632                                                       operands[2], operands[3],
2633                                                       operands[4]));
2634   DONE;
2635 })
2636
2637 ;; vqdml[sa]l_n
2638
2639 (define_insn "aarch64_sqdml<SBINQOPS:as>l_n<mode>"
2640   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2641         (SBINQOPS:<VWIDE>
2642           (match_operand:<VWIDE> 1 "register_operand" "0")
2643           (ss_ashift:<VWIDE>
2644               (mult:<VWIDE>
2645                 (sign_extend:<VWIDE>
2646                       (match_operand:VD_HSI 2 "register_operand" "w"))
2647                 (sign_extend:<VWIDE>
2648                   (vec_duplicate:VD_HSI
2649                     (match_operand:<VEL> 3 "register_operand" "w"))))
2650               (const_int 1))))]
2651   "TARGET_SIMD"
2652   "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[0]"
2653   [(set_attr "simd_type" "simd_sat_mlal")
2654    (set_attr "simd_mode" "<MODE>")]
2655 )
2656
2657 ;; sqdml[as]l2
2658
2659 (define_insn "aarch64_sqdml<SBINQOPS:as>l2<mode>_internal"
2660   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2661         (SBINQOPS:<VWIDE>
2662          (match_operand:<VWIDE> 1 "register_operand" "0")
2663          (ss_ashift:<VWIDE>
2664              (mult:<VWIDE>
2665                (sign_extend:<VWIDE>
2666                  (vec_select:<VHALF>
2667                      (match_operand:VQ_HSI 2 "register_operand" "w")
2668                      (match_operand:VQ_HSI 4 "vect_par_cnst_hi_half" "")))
2669                (sign_extend:<VWIDE>
2670                  (vec_select:<VHALF>
2671                      (match_operand:VQ_HSI 3 "register_operand" "w")
2672                      (match_dup 4))))
2673              (const_int 1))))]
2674   "TARGET_SIMD"
2675   "sqdml<SBINQOPS:as>l2\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %<v>3<Vmtype>"
2676   [(set_attr "simd_type" "simd_sat_mlal")
2677    (set_attr "simd_mode" "<MODE>")]
2678 )
2679
2680 (define_expand "aarch64_sqdmlal2<mode>"
2681   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2682    (match_operand:<VWIDE> 1 "register_operand" "w")
2683    (match_operand:VQ_HSI 2 "register_operand" "w")
2684    (match_operand:VQ_HSI 3 "register_operand" "w")]
2685   "TARGET_SIMD"
2686 {
2687   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2688   emit_insn (gen_aarch64_sqdmlal2<mode>_internal (operands[0], operands[1],
2689                                                   operands[2], operands[3], p));
2690   DONE;
2691 })
2692
2693 (define_expand "aarch64_sqdmlsl2<mode>"
2694   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2695    (match_operand:<VWIDE> 1 "register_operand" "w")
2696    (match_operand:VQ_HSI 2 "register_operand" "w")
2697    (match_operand:VQ_HSI 3 "register_operand" "w")]
2698   "TARGET_SIMD"
2699 {
2700   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2701   emit_insn (gen_aarch64_sqdmlsl2<mode>_internal (operands[0], operands[1],
2702                                                   operands[2], operands[3], p));
2703   DONE;
2704 })
2705
2706 ;; vqdml[sa]l2_lane
2707
2708 (define_insn "aarch64_sqdml<SBINQOPS:as>l2_lane<mode>_internal"
2709   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2710         (SBINQOPS:<VWIDE>
2711           (match_operand:<VWIDE> 1 "register_operand" "0")
2712           (ss_ashift:<VWIDE>
2713               (mult:<VWIDE>
2714                 (sign_extend:<VWIDE>
2715                   (vec_select:<VHALF>
2716                     (match_operand:VQ_HSI 2 "register_operand" "w")
2717                     (match_operand:VQ_HSI 5 "vect_par_cnst_hi_half" "")))
2718                 (sign_extend:<VWIDE>
2719                   (vec_duplicate:<VHALF>
2720                     (vec_select:<VEL>
2721                       (match_operand:<VCON> 3 "register_operand" "<vwx>")
2722                       (parallel [(match_operand:SI 4 "immediate_operand" "i")])
2723                     ))))
2724               (const_int 1))))]
2725   "TARGET_SIMD"
2726   "sqdml<SBINQOPS:as>l2\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]"
2727   [(set_attr "simd_type" "simd_sat_mlal")
2728    (set_attr "simd_mode" "<MODE>")]
2729 )
2730
2731 (define_expand "aarch64_sqdmlal2_lane<mode>"
2732   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2733    (match_operand:<VWIDE> 1 "register_operand" "w")
2734    (match_operand:VQ_HSI 2 "register_operand" "w")
2735    (match_operand:<VCON> 3 "register_operand" "<vwx>")
2736    (match_operand:SI 4 "immediate_operand" "i")]
2737   "TARGET_SIMD"
2738 {
2739   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2740   aarch64_simd_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode) / 2);
2741   emit_insn (gen_aarch64_sqdmlal2_lane<mode>_internal (operands[0], operands[1],
2742                                                        operands[2], operands[3],
2743                                                        operands[4], p));
2744   DONE;
2745 })
2746
2747 (define_expand "aarch64_sqdmlal2_laneq<mode>"
2748   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2749    (match_operand:<VWIDE> 1 "register_operand" "w")
2750    (match_operand:VQ_HSI 2 "register_operand" "w")
2751    (match_operand:<VCON> 3 "register_operand" "<vwx>")
2752    (match_operand:SI 4 "immediate_operand" "i")]
2753   "TARGET_SIMD"
2754 {
2755   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2756   aarch64_simd_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
2757   emit_insn (gen_aarch64_sqdmlal2_lane<mode>_internal (operands[0], operands[1],
2758                                                        operands[2], operands[3],
2759                                                        operands[4], p));
2760   DONE;
2761 })
2762
2763 (define_expand "aarch64_sqdmlsl2_lane<mode>"
2764   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2765    (match_operand:<VWIDE> 1 "register_operand" "w")
2766    (match_operand:VQ_HSI 2 "register_operand" "w")
2767    (match_operand:<VCON> 3 "register_operand" "<vwx>")
2768    (match_operand:SI 4 "immediate_operand" "i")]
2769   "TARGET_SIMD"
2770 {
2771   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2772   aarch64_simd_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode) / 2);
2773   emit_insn (gen_aarch64_sqdmlsl2_lane<mode>_internal (operands[0], operands[1],
2774                                                        operands[2], operands[3],
2775                                                        operands[4], p));
2776   DONE;
2777 })
2778
2779 (define_expand "aarch64_sqdmlsl2_laneq<mode>"
2780   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2781    (match_operand:<VWIDE> 1 "register_operand" "w")
2782    (match_operand:VQ_HSI 2 "register_operand" "w")
2783    (match_operand:<VCON> 3 "register_operand" "<vwx>")
2784    (match_operand:SI 4 "immediate_operand" "i")]
2785   "TARGET_SIMD"
2786 {
2787   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2788   aarch64_simd_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
2789   emit_insn (gen_aarch64_sqdmlsl2_lane<mode>_internal (operands[0], operands[1],
2790                                                        operands[2], operands[3],
2791                                                        operands[4], p));
2792   DONE;
2793 })
2794
2795 (define_insn "aarch64_sqdml<SBINQOPS:as>l2_n<mode>_internal"
2796   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2797         (SBINQOPS:<VWIDE>
2798           (match_operand:<VWIDE> 1 "register_operand" "0")
2799           (ss_ashift:<VWIDE>
2800             (mult:<VWIDE>
2801               (sign_extend:<VWIDE>
2802                 (vec_select:<VHALF>
2803                   (match_operand:VQ_HSI 2 "register_operand" "w")
2804                   (match_operand:VQ_HSI 4 "vect_par_cnst_hi_half" "")))
2805               (sign_extend:<VWIDE>
2806                 (vec_duplicate:<VHALF>
2807                   (match_operand:<VEL> 3 "register_operand" "w"))))
2808             (const_int 1))))]
2809   "TARGET_SIMD"
2810   "sqdml<SBINQOPS:as>l2\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[0]"
2811   [(set_attr "simd_type" "simd_sat_mlal")
2812    (set_attr "simd_mode" "<MODE>")]
2813 )
2814
2815 (define_expand "aarch64_sqdmlal2_n<mode>"
2816   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2817    (match_operand:<VWIDE> 1 "register_operand" "w")
2818    (match_operand:VQ_HSI 2 "register_operand" "w")
2819    (match_operand:<VEL> 3 "register_operand" "w")]
2820   "TARGET_SIMD"
2821 {
2822   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2823   emit_insn (gen_aarch64_sqdmlal2_n<mode>_internal (operands[0], operands[1],
2824                                                     operands[2], operands[3],
2825                                                     p));
2826   DONE;
2827 })
2828
2829 (define_expand "aarch64_sqdmlsl2_n<mode>"
2830   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2831    (match_operand:<VWIDE> 1 "register_operand" "w")
2832    (match_operand:VQ_HSI 2 "register_operand" "w")
2833    (match_operand:<VEL> 3 "register_operand" "w")]
2834   "TARGET_SIMD"
2835 {
2836   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2837   emit_insn (gen_aarch64_sqdmlsl2_n<mode>_internal (operands[0], operands[1],
2838                                                     operands[2], operands[3],
2839                                                     p));
2840   DONE;
2841 })
2842
2843 ;; vqdmull
2844
2845 (define_insn "aarch64_sqdmull<mode>"
2846   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2847         (ss_ashift:<VWIDE>
2848              (mult:<VWIDE>
2849                (sign_extend:<VWIDE>
2850                      (match_operand:VSD_HSI 1 "register_operand" "w"))
2851                (sign_extend:<VWIDE>
2852                      (match_operand:VSD_HSI 2 "register_operand" "w")))
2853              (const_int 1)))]
2854   "TARGET_SIMD"
2855   "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
2856   [(set_attr "simd_type" "simd_sat_mul")
2857    (set_attr "simd_mode" "<MODE>")]
2858 )
2859
2860 ;; vqdmull_lane
2861
2862 (define_insn "aarch64_sqdmull_lane<mode>_internal"
2863   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2864         (ss_ashift:<VWIDE>
2865              (mult:<VWIDE>
2866                (sign_extend:<VWIDE>
2867                  (match_operand:VD_HSI 1 "register_operand" "w"))
2868                (sign_extend:<VWIDE>
2869                  (vec_duplicate:VD_HSI
2870                    (vec_select:<VEL>
2871                      (match_operand:<VCON> 2 "register_operand" "<vwx>")
2872                      (parallel [(match_operand:SI 3 "immediate_operand" "i")])))
2873                ))
2874              (const_int 1)))]
2875   "TARGET_SIMD"
2876   "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]"
2877   [(set_attr "simd_type" "simd_sat_mul")
2878    (set_attr "simd_mode" "<MODE>")]
2879 )
2880
2881 (define_insn "aarch64_sqdmull_lane<mode>_internal"
2882   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2883         (ss_ashift:<VWIDE>
2884              (mult:<VWIDE>
2885                (sign_extend:<VWIDE>
2886                  (match_operand:SD_HSI 1 "register_operand" "w"))
2887                (sign_extend:<VWIDE>
2888                  (vec_select:<VEL>
2889                    (match_operand:<VCON> 2 "register_operand" "<vwx>")
2890                    (parallel [(match_operand:SI 3 "immediate_operand" "i")]))
2891                ))
2892              (const_int 1)))]
2893   "TARGET_SIMD"
2894   "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]"
2895   [(set_attr "simd_type" "simd_sat_mul")
2896    (set_attr "simd_mode" "<MODE>")]
2897 )
2898
2899 (define_expand "aarch64_sqdmull_lane<mode>"
2900   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2901    (match_operand:VSD_HSI 1 "register_operand" "w")
2902    (match_operand:<VCON> 2 "register_operand" "<vwx>")
2903    (match_operand:SI 3 "immediate_operand" "i")]
2904   "TARGET_SIMD"
2905 {
2906   aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCON>mode) / 2);
2907   emit_insn (gen_aarch64_sqdmull_lane<mode>_internal (operands[0], operands[1],
2908                                                       operands[2], operands[3]));
2909   DONE;
2910 })
2911
2912 (define_expand "aarch64_sqdmull_laneq<mode>"
2913   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2914    (match_operand:VD_HSI 1 "register_operand" "w")
2915    (match_operand:<VCON> 2 "register_operand" "<vwx>")
2916    (match_operand:SI 3 "immediate_operand" "i")]
2917   "TARGET_SIMD"
2918 {
2919   aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCON>mode));
2920   emit_insn (gen_aarch64_sqdmull_lane<mode>_internal
2921                (operands[0], operands[1], operands[2], operands[3]));
2922   DONE;
2923 })
2924
2925 ;; vqdmull_n
2926
2927 (define_insn "aarch64_sqdmull_n<mode>"
2928   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2929         (ss_ashift:<VWIDE>
2930              (mult:<VWIDE>
2931                (sign_extend:<VWIDE>
2932                  (match_operand:VD_HSI 1 "register_operand" "w"))
2933                (sign_extend:<VWIDE>
2934                  (vec_duplicate:VD_HSI
2935                    (match_operand:<VEL> 2 "register_operand" "w")))
2936                )
2937              (const_int 1)))]
2938   "TARGET_SIMD"
2939   "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[0]"
2940   [(set_attr "simd_type" "simd_sat_mul")
2941    (set_attr "simd_mode" "<MODE>")]
2942 )
2943
2944 ;; vqdmull2
2945
2946
2947
2948 (define_insn "aarch64_sqdmull2<mode>_internal"
2949   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2950         (ss_ashift:<VWIDE>
2951              (mult:<VWIDE>
2952                (sign_extend:<VWIDE>
2953                  (vec_select:<VHALF>
2954                    (match_operand:VQ_HSI 1 "register_operand" "w")
2955                    (match_operand:VQ_HSI 3 "vect_par_cnst_hi_half" "")))
2956                (sign_extend:<VWIDE>
2957                  (vec_select:<VHALF>
2958                    (match_operand:VQ_HSI 2 "register_operand" "w")
2959                    (match_dup 3)))
2960                )
2961              (const_int 1)))]
2962   "TARGET_SIMD"
2963   "sqdmull2\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
2964   [(set_attr "simd_type" "simd_sat_mul")
2965    (set_attr "simd_mode" "<MODE>")]
2966 )
2967
2968 (define_expand "aarch64_sqdmull2<mode>"
2969   [(match_operand:<VWIDE> 0 "register_operand" "=w")
2970    (match_operand:VQ_HSI 1 "register_operand" "w")
2971    (match_operand:<VCON> 2 "register_operand" "w")]
2972   "TARGET_SIMD"
2973 {
2974   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2975   emit_insn (gen_aarch64_sqdmull2<mode>_internal (operands[0], operands[1],
2976                                                   operands[2], p));
2977   DONE;
2978 })
2979
2980 ;; vqdmull2_lane
2981
2982 (define_insn "aarch64_sqdmull2_lane<mode>_internal"
2983   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2984         (ss_ashift:<VWIDE>
2985              (mult:<VWIDE>
2986                (sign_extend:<VWIDE>
2987                  (vec_select:<VHALF>
2988                    (match_operand:VQ_HSI 1 "register_operand" "w")
2989                    (match_operand:VQ_HSI 4 "vect_par_cnst_hi_half" "")))
2990                (sign_extend:<VWIDE>
2991                  (vec_duplicate:<VHALF>
2992                    (vec_select:<VEL>
2993                      (match_operand:<VCON> 2 "register_operand" "<vwx>")
2994                      (parallel [(match_operand:SI 3 "immediate_operand" "i")])))
2995                ))
2996              (const_int 1)))]
2997   "TARGET_SIMD"
2998   "sqdmull2\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]"
2999   [(set_attr "simd_type" "simd_sat_mul")
3000    (set_attr "simd_mode" "<MODE>")]
3001 )
3002
3003 (define_expand "aarch64_sqdmull2_lane<mode>"
3004   [(match_operand:<VWIDE> 0 "register_operand" "=w")
3005    (match_operand:VQ_HSI 1 "register_operand" "w")
3006    (match_operand:<VCON> 2 "register_operand" "<vwx>")
3007    (match_operand:SI 3 "immediate_operand" "i")]
3008   "TARGET_SIMD"
3009 {
3010   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3011   aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode) / 2);
3012   emit_insn (gen_aarch64_sqdmull2_lane<mode>_internal (operands[0], operands[1],
3013                                                        operands[2], operands[3],
3014                                                        p));
3015   DONE;
3016 })
3017
3018 (define_expand "aarch64_sqdmull2_laneq<mode>"
3019   [(match_operand:<VWIDE> 0 "register_operand" "=w")
3020    (match_operand:VQ_HSI 1 "register_operand" "w")
3021    (match_operand:<VCON> 2 "register_operand" "<vwx>")
3022    (match_operand:SI 3 "immediate_operand" "i")]
3023   "TARGET_SIMD"
3024 {
3025   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3026   aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3027   emit_insn (gen_aarch64_sqdmull2_lane<mode>_internal (operands[0], operands[1],
3028                                                        operands[2], operands[3],
3029                                                        p));
3030   DONE;
3031 })
3032
3033 ;; vqdmull2_n
3034
3035 (define_insn "aarch64_sqdmull2_n<mode>_internal"
3036   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3037         (ss_ashift:<VWIDE>
3038              (mult:<VWIDE>
3039                (sign_extend:<VWIDE>
3040                  (vec_select:<VHALF>
3041                    (match_operand:VQ_HSI 1 "register_operand" "w")
3042                    (match_operand:VQ_HSI 3 "vect_par_cnst_hi_half" "")))
3043                (sign_extend:<VWIDE>
3044                  (vec_duplicate:<VHALF>
3045                    (match_operand:<VEL> 2 "register_operand" "w")))
3046                )
3047              (const_int 1)))]
3048   "TARGET_SIMD"
3049   "sqdmull2\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[0]"
3050   [(set_attr "simd_type" "simd_sat_mul")
3051    (set_attr "simd_mode" "<MODE>")]
3052 )
3053
3054 (define_expand "aarch64_sqdmull2_n<mode>"
3055   [(match_operand:<VWIDE> 0 "register_operand" "=w")
3056    (match_operand:VQ_HSI 1 "register_operand" "w")
3057    (match_operand:<VEL> 2 "register_operand" "w")]
3058   "TARGET_SIMD"
3059 {
3060   rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3061   emit_insn (gen_aarch64_sqdmull2_n<mode>_internal (operands[0], operands[1],
3062                                                     operands[2], p));
3063   DONE;
3064 })
3065
3066 ;; vshl
3067
3068 (define_insn "aarch64_<sur>shl<mode>"
3069   [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
3070         (unspec:VSDQ_I_DI
3071           [(match_operand:VSDQ_I_DI 1 "register_operand" "w")
3072            (match_operand:VSDQ_I_DI 2 "register_operand" "w")]
3073          VSHL))]
3074   "TARGET_SIMD"
3075   "<sur>shl\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>";
3076   [(set_attr "simd_type" "simd_shift")
3077    (set_attr "simd_mode" "<MODE>")]
3078 )
3079
3080
3081 ;; vqshl
3082
3083 (define_insn "aarch64_<sur>q<r>shl<mode>"
3084   [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
3085         (unspec:VSDQ_I
3086           [(match_operand:VSDQ_I 1 "register_operand" "w")
3087            (match_operand:VSDQ_I 2 "register_operand" "w")]
3088          VQSHL))]
3089   "TARGET_SIMD"
3090   "<sur>q<r>shl\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>";
3091   [(set_attr "simd_type" "simd_sat_shift")
3092    (set_attr "simd_mode" "<MODE>")]
3093 )
3094
3095 ;; vshll_n
3096
3097 (define_insn "aarch64_<sur>shll_n<mode>"
3098   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3099         (unspec:<VWIDE> [(match_operand:VDW 1 "register_operand" "w")
3100                          (match_operand:SI 2 "immediate_operand" "i")]
3101                          VSHLL))]
3102   "TARGET_SIMD"
3103   "*
3104   int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
3105   aarch64_simd_const_bounds (operands[2], 0, bit_width + 1);
3106   if (INTVAL (operands[2]) == bit_width)
3107   {
3108     return \"shll\\t%0.<Vwtype>, %1.<Vtype>, %2\";
3109   }
3110   else {
3111     return \"<sur>shll\\t%0.<Vwtype>, %1.<Vtype>, %2\";
3112   }"
3113   [(set_attr "simd_type" "simd_shift_imm")
3114    (set_attr "simd_mode" "<MODE>")]
3115 )
3116
3117 ;; vshll_high_n
3118
3119 (define_insn "aarch64_<sur>shll2_n<mode>"
3120   [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3121         (unspec:<VWIDE> [(match_operand:VQW 1 "register_operand" "w")
3122                          (match_operand:SI 2 "immediate_operand" "i")]
3123                          VSHLL))]
3124   "TARGET_SIMD"
3125   "*
3126   int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
3127   aarch64_simd_const_bounds (operands[2], 0, bit_width + 1);
3128   if (INTVAL (operands[2]) == bit_width)
3129   {
3130     return \"shll2\\t%0.<Vwtype>, %1.<Vtype>, %2\";
3131   }
3132   else {
3133     return \"<sur>shll2\\t%0.<Vwtype>, %1.<Vtype>, %2\";
3134   }"
3135   [(set_attr "simd_type" "simd_shift_imm")
3136    (set_attr "simd_mode" "<MODE>")]
3137 )
3138
3139 ;; vrshr_n
3140
3141 (define_insn "aarch64_<sur>shr_n<mode>"
3142   [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
3143         (unspec:VSDQ_I_DI [(match_operand:VSDQ_I_DI 1 "register_operand" "w")
3144                            (match_operand:SI 2 "immediate_operand" "i")]
3145                           VRSHR_N))]
3146   "TARGET_SIMD"
3147   "*
3148   int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
3149   aarch64_simd_const_bounds (operands[2], 1, bit_width + 1);
3150   return \"<sur>shr\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %2\";"
3151   [(set_attr "simd_type" "simd_shift_imm")
3152    (set_attr "simd_mode" "<MODE>")]
3153 )
3154
3155 ;; v(r)sra_n
3156
3157 (define_insn "aarch64_<sur>sra_n<mode>"
3158   [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
3159         (unspec:VSDQ_I_DI [(match_operand:VSDQ_I_DI 1 "register_operand" "0")
3160                        (match_operand:VSDQ_I_DI 2 "register_operand" "w")
3161                        (match_operand:SI 3 "immediate_operand" "i")]
3162                       VSRA))]
3163   "TARGET_SIMD"
3164   "*
3165   int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
3166   aarch64_simd_const_bounds (operands[3], 1, bit_width + 1);
3167   return \"<sur>sra\\t%<v>0<Vmtype>, %<v>2<Vmtype>, %3\";"
3168   [(set_attr "simd_type" "simd_shift_imm_acc")
3169    (set_attr "simd_mode" "<MODE>")]
3170 )
3171
3172 ;; vs<lr>i_n
3173
3174 (define_insn "aarch64_<sur>s<lr>i_n<mode>"
3175   [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
3176         (unspec:VSDQ_I_DI [(match_operand:VSDQ_I_DI 1 "register_operand" "0")
3177                        (match_operand:VSDQ_I_DI 2 "register_operand" "w")
3178                        (match_operand:SI 3 "immediate_operand" "i")]
3179                       VSLRI))]
3180   "TARGET_SIMD"
3181   "*
3182   int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
3183   aarch64_simd_const_bounds (operands[3], 1 - <VSLRI:offsetlr>,
3184                              bit_width - <VSLRI:offsetlr> + 1);
3185   return \"s<lr>i\\t%<v>0<Vmtype>, %<v>2<Vmtype>, %3\";"
3186   [(set_attr "simd_type" "simd_shift_imm")
3187    (set_attr "simd_mode" "<MODE>")]
3188 )
3189
3190 ;; vqshl(u)
3191
3192 (define_insn "aarch64_<sur>qshl<u>_n<mode>"
3193   [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
3194         (unspec:VSDQ_I [(match_operand:VSDQ_I 1 "register_operand" "w")
3195                        (match_operand:SI 2 "immediate_operand" "i")]
3196                       VQSHL_N))]
3197   "TARGET_SIMD"
3198   "*
3199   int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
3200   aarch64_simd_const_bounds (operands[2], 0, bit_width);
3201   return \"<sur>qshl<u>\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %2\";"
3202   [(set_attr "simd_type" "simd_sat_shift_imm")
3203    (set_attr "simd_mode" "<MODE>")]
3204 )
3205
3206
3207 ;; vq(r)shr(u)n_n
3208
3209 (define_insn "aarch64_<sur>q<r>shr<u>n_n<mode>"
3210   [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
3211         (unspec:<VNARROWQ> [(match_operand:VSQN_HSDI 1 "register_operand" "w")
3212                             (match_operand:SI 2 "immediate_operand" "i")]
3213                            VQSHRN_N))]
3214   "TARGET_SIMD"
3215   "*
3216   int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
3217   aarch64_simd_const_bounds (operands[2], 1, bit_width + 1);
3218   return \"<sur>q<r>shr<u>n\\t%<vn2>0<Vmntype>, %<v>1<Vmtype>, %2\";"
3219   [(set_attr "simd_type" "simd_sat_shiftn_imm")
3220    (set_attr "simd_mode" "<MODE>")]
3221 )
3222
3223
3224 ;; cm(eq|ge|gt|lt|le)
3225 ;; Note, we have constraints for Dz and Z as different expanders
3226 ;; have different ideas of what should be passed to this pattern.
3227
3228 (define_insn "aarch64_cm<optab><mode>"
3229   [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w,w")
3230         (neg:<V_cmp_result>
3231           (COMPARISONS:<V_cmp_result>
3232             (match_operand:VDQ 1 "register_operand" "w,w")
3233             (match_operand:VDQ 2 "aarch64_simd_reg_or_zero" "w,ZDz")
3234           )))]
3235   "TARGET_SIMD"
3236   "@
3237   cm<n_optab>\t%<v>0<Vmtype>, %<v><cmp_1><Vmtype>, %<v><cmp_2><Vmtype>
3238   cm<optab>\t%<v>0<Vmtype>, %<v>1<Vmtype>, #0"
3239   [(set_attr "simd_type" "simd_cmp")
3240    (set_attr "simd_mode" "<MODE>")]
3241 )
3242
3243 (define_insn_and_split "aarch64_cm<optab>di"
3244   [(set (match_operand:DI 0 "register_operand" "=w,w,r")
3245         (neg:DI
3246           (COMPARISONS:DI
3247             (match_operand:DI 1 "register_operand" "w,w,r")
3248             (match_operand:DI 2 "aarch64_simd_reg_or_zero" "w,ZDz,r")
3249           )))]
3250   "TARGET_SIMD"
3251   "@
3252   cm<n_optab>\t%d0, %d<cmp_1>, %d<cmp_2>
3253   cm<optab>\t%d0, %d1, #0
3254   #"
3255   "reload_completed
3256    /* We need to prevent the split from
3257       happening in the 'w' constraint cases.  */
3258    && GP_REGNUM_P (REGNO (operands[0]))
3259    && GP_REGNUM_P (REGNO (operands[1]))"
3260   [(set (reg:CC CC_REGNUM)
3261     (compare:CC
3262       (match_dup 1)
3263       (match_dup 2)))
3264   (set (match_dup 0)
3265     (neg:DI
3266       (COMPARISONS:DI
3267         (match_operand 3 "cc_register" "")
3268         (const_int 0))))]
3269   {
3270     enum machine_mode mode = SELECT_CC_MODE (<CMP>, operands[1], operands[2]);
3271     rtx cc_reg = aarch64_gen_compare_reg (<CMP>, operands[1], operands[2]);
3272     rtx comparison = gen_rtx_<CMP> (mode, operands[1], operands[2]);
3273     emit_insn (gen_cstoredi_neg (operands[0], comparison, cc_reg));
3274     DONE;
3275   }
3276   [(set_attr "simd_type" "simd_cmp")
3277    (set_attr "simd_mode" "DI")]
3278 )
3279
3280 ;; cm(hs|hi)
3281
3282 (define_insn "aarch64_cm<optab><mode>"
3283   [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w")
3284         (neg:<V_cmp_result>
3285           (UCOMPARISONS:<V_cmp_result>
3286             (match_operand:VDQ 1 "register_operand" "w")
3287             (match_operand:VDQ 2 "register_operand" "w")
3288           )))]
3289   "TARGET_SIMD"
3290   "cm<n_optab>\t%<v>0<Vmtype>, %<v><cmp_1><Vmtype>, %<v><cmp_2><Vmtype>"
3291   [(set_attr "simd_type" "simd_cmp")
3292    (set_attr "simd_mode" "<MODE>")]
3293 )
3294
3295 (define_insn_and_split "aarch64_cm<optab>di"
3296   [(set (match_operand:DI 0 "register_operand" "=w,r")
3297         (neg:DI
3298           (UCOMPARISONS:DI
3299             (match_operand:DI 1 "register_operand" "w,r")
3300             (match_operand:DI 2 "aarch64_simd_reg_or_zero" "w,r")
3301           )))]
3302   "TARGET_SIMD"
3303   "@
3304   cm<n_optab>\t%d0, %d<cmp_1>, %d<cmp_2>
3305   #"
3306   "reload_completed
3307    /* We need to prevent the split from
3308       happening in the 'w' constraint cases.  */
3309    && GP_REGNUM_P (REGNO (operands[0]))
3310    && GP_REGNUM_P (REGNO (operands[1]))"
3311   [(set (reg:CC CC_REGNUM)
3312     (compare:CC
3313       (match_dup 1)
3314       (match_dup 2)))
3315   (set (match_dup 0)
3316     (neg:DI
3317       (UCOMPARISONS:DI
3318         (match_operand 3 "cc_register" "")
3319         (const_int 0))))]
3320   {
3321     enum machine_mode mode = SELECT_CC_MODE (<CMP>, operands[1], operands[2]);
3322     rtx cc_reg = aarch64_gen_compare_reg (<CMP>, operands[1], operands[2]);
3323     rtx comparison = gen_rtx_<CMP> (mode, operands[1], operands[2]);
3324     emit_insn (gen_cstoredi_neg (operands[0], comparison, cc_reg));
3325     DONE;
3326   }
3327   [(set_attr "simd_type" "simd_cmp")
3328    (set_attr "simd_mode" "DI")]
3329 )
3330
3331 ;; cmtst
3332
3333 (define_insn "aarch64_cmtst<mode>"
3334   [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w")
3335         (neg:<V_cmp_result>
3336           (ne:<V_cmp_result>
3337             (and:VDQ
3338               (match_operand:VDQ 1 "register_operand" "w")
3339               (match_operand:VDQ 2 "register_operand" "w"))
3340             (vec_duplicate:<V_cmp_result> (const_int 0)))))]
3341   "TARGET_SIMD"
3342   "cmtst\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
3343   [(set_attr "simd_type" "simd_cmp")
3344    (set_attr "simd_mode" "<MODE>")]
3345 )
3346
3347 (define_insn_and_split "aarch64_cmtstdi"
3348   [(set (match_operand:DI 0 "register_operand" "=w,r")
3349         (neg:DI
3350           (ne:DI
3351             (and:DI
3352               (match_operand:DI 1 "register_operand" "w,r")
3353               (match_operand:DI 2 "register_operand" "w,r"))
3354             (const_int 0))))]
3355   "TARGET_SIMD"
3356   "@
3357   cmtst\t%d0, %d1, %d2
3358   #"
3359   "reload_completed
3360    /* We need to prevent the split from
3361       happening in the 'w' constraint cases.  */
3362    && GP_REGNUM_P (REGNO (operands[0]))
3363    && GP_REGNUM_P (REGNO (operands[1]))"
3364    [(set (reg:CC_NZ CC_REGNUM)
3365         (compare:CC_NZ
3366          (and:DI (match_dup 1)
3367                   (match_dup 2))
3368          (const_int 0)))
3369   (set (match_dup 0)
3370     (neg:DI
3371       (ne:DI
3372         (match_operand 3 "cc_register" "")
3373         (const_int 0))))]
3374   {
3375     rtx and_tree = gen_rtx_AND (DImode, operands[1], operands[2]);
3376     enum machine_mode mode = SELECT_CC_MODE (NE, and_tree, const0_rtx);
3377     rtx cc_reg = aarch64_gen_compare_reg (NE, and_tree, const0_rtx);
3378     rtx comparison = gen_rtx_NE (mode, and_tree, const0_rtx);
3379     emit_insn (gen_cstoredi_neg (operands[0], comparison, cc_reg));
3380     DONE;
3381   }
3382   [(set_attr "simd_type" "simd_cmp")
3383    (set_attr "simd_mode" "DI")]
3384 )
3385
3386 ;; fcm(eq|ge|gt|le|lt)
3387
3388 (define_insn "aarch64_cm<optab><mode>"
3389   [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w,w")
3390         (neg:<V_cmp_result>
3391           (COMPARISONS:<V_cmp_result>
3392             (match_operand:VALLF 1 "register_operand" "w,w")
3393             (match_operand:VALLF 2 "aarch64_simd_reg_or_zero" "w,YDz")
3394           )))]
3395   "TARGET_SIMD"
3396   "@
3397   fcm<n_optab>\t%<v>0<Vmtype>, %<v><cmp_1><Vmtype>, %<v><cmp_2><Vmtype>
3398   fcm<optab>\t%<v>0<Vmtype>, %<v>1<Vmtype>, 0"
3399   [(set_attr "simd_type" "simd_fcmp")
3400    (set_attr "simd_mode" "<MODE>")]
3401 )
3402
3403 ;; fac(ge|gt)
3404 ;; Note we can also handle what would be fac(le|lt) by
3405 ;; generating fac(ge|gt).
3406
3407 (define_insn "*aarch64_fac<optab><mode>"
3408   [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w")
3409         (neg:<V_cmp_result>
3410           (FAC_COMPARISONS:<V_cmp_result>
3411             (abs:VALLF (match_operand:VALLF 1 "register_operand" "w"))
3412             (abs:VALLF (match_operand:VALLF 2 "register_operand" "w"))
3413   )))]
3414   "TARGET_SIMD"
3415   "fac<n_optab>\t%<v>0<Vmtype>, %<v><cmp_1><Vmtype>, %<v><cmp_2><Vmtype>"
3416   [(set_attr "simd_type" "simd_fcmp")
3417    (set_attr "simd_mode" "<MODE>")]
3418 )
3419
3420 ;; addp
3421
3422 (define_insn "aarch64_addp<mode>"
3423   [(set (match_operand:VD_BHSI 0 "register_operand" "=w")
3424         (unspec:VD_BHSI
3425           [(match_operand:VD_BHSI 1 "register_operand" "w")
3426            (match_operand:VD_BHSI 2 "register_operand" "w")]
3427           UNSPEC_ADDP))]
3428   "TARGET_SIMD"
3429   "addp\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
3430   [(set_attr "simd_type" "simd_add")
3431    (set_attr "simd_mode" "<MODE>")]
3432 )
3433
3434 (define_insn "aarch64_addpdi"
3435   [(set (match_operand:DI 0 "register_operand" "=w")
3436         (unspec:DI
3437           [(match_operand:V2DI 1 "register_operand" "w")]
3438           UNSPEC_ADDP))]
3439   "TARGET_SIMD"
3440   "addp\t%d0, %1.2d"
3441   [(set_attr "simd_type" "simd_add")
3442    (set_attr "simd_mode" "DI")]
3443 )
3444
3445 (define_insn "aarch64_<fmaxmin><mode>"
3446   [(set (match_operand:VDQF 0 "register_operand" "=w")
3447         (unspec:VDQF [(match_operand:VDQF 1 "register_operand" "w")
3448                       (match_operand:VDQF 2 "register_operand" "w")]
3449                       FMAXMIN))]
3450   "TARGET_SIMD"
3451   "<fmaxmin>\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
3452   [(set_attr "simd_type" "simd_fminmax")
3453    (set_attr "simd_mode" "<MODE>")]
3454 )
3455
3456 ;; sqrt
3457
3458 (define_insn "sqrt<mode>2"
3459   [(set (match_operand:VDQF 0 "register_operand" "=w")
3460         (sqrt:VDQF (match_operand:VDQF 1 "register_operand" "w")))]
3461   "TARGET_SIMD"
3462   "fsqrt\\t%0.<Vtype>, %1.<Vtype>"
3463   [(set_attr "simd_type" "simd_fsqrt")
3464    (set_attr "simd_mode" "<MODE>")]
3465 )
3466
3467 ;; Patterns for vector struct loads and stores.
3468
3469 (define_insn "vec_load_lanesoi<mode>"
3470   [(set (match_operand:OI 0 "register_operand" "=w")
3471         (unspec:OI [(match_operand:OI 1 "aarch64_simd_struct_operand" "Utv")
3472                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3473                    UNSPEC_LD2))]
3474   "TARGET_SIMD"
3475   "ld2\\t{%S0.<Vtype> - %T0.<Vtype>}, %1"
3476   [(set_attr "simd_type" "simd_load2")
3477    (set_attr "simd_mode" "<MODE>")])
3478
3479 (define_insn "vec_store_lanesoi<mode>"
3480   [(set (match_operand:OI 0 "aarch64_simd_struct_operand" "=Utv")
3481         (unspec:OI [(match_operand:OI 1 "register_operand" "w")
3482                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3483                    UNSPEC_ST2))]
3484   "TARGET_SIMD"
3485   "st2\\t{%S1.<Vtype> - %T1.<Vtype>}, %0"
3486   [(set_attr "simd_type" "simd_store2")
3487    (set_attr "simd_mode" "<MODE>")])
3488
3489 (define_insn "vec_load_lanesci<mode>"
3490   [(set (match_operand:CI 0 "register_operand" "=w")
3491         (unspec:CI [(match_operand:CI 1 "aarch64_simd_struct_operand" "Utv")
3492                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3493                    UNSPEC_LD3))]
3494   "TARGET_SIMD"
3495   "ld3\\t{%S0.<Vtype> - %U0.<Vtype>}, %1"
3496   [(set_attr "simd_type" "simd_load3")
3497    (set_attr "simd_mode" "<MODE>")])
3498
3499 (define_insn "vec_store_lanesci<mode>"
3500   [(set (match_operand:CI 0 "aarch64_simd_struct_operand" "=Utv")
3501         (unspec:CI [(match_operand:CI 1 "register_operand" "w")
3502                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3503                    UNSPEC_ST3))]
3504   "TARGET_SIMD"
3505   "st3\\t{%S1.<Vtype> - %U1.<Vtype>}, %0"
3506   [(set_attr "simd_type" "simd_store3")
3507    (set_attr "simd_mode" "<MODE>")])
3508
3509 (define_insn "vec_load_lanesxi<mode>"
3510   [(set (match_operand:XI 0 "register_operand" "=w")
3511         (unspec:XI [(match_operand:XI 1 "aarch64_simd_struct_operand" "Utv")
3512                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3513                    UNSPEC_LD4))]
3514   "TARGET_SIMD"
3515   "ld4\\t{%S0.<Vtype> - %V0.<Vtype>}, %1"
3516   [(set_attr "simd_type" "simd_load4")
3517    (set_attr "simd_mode" "<MODE>")])
3518
3519 (define_insn "vec_store_lanesxi<mode>"
3520   [(set (match_operand:XI 0 "aarch64_simd_struct_operand" "=Utv")
3521         (unspec:XI [(match_operand:XI 1 "register_operand" "w")
3522                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3523                    UNSPEC_ST4))]
3524   "TARGET_SIMD"
3525   "st4\\t{%S1.<Vtype> - %V1.<Vtype>}, %0"
3526   [(set_attr "simd_type" "simd_store4")
3527    (set_attr "simd_mode" "<MODE>")])
3528
3529 ;; Reload patterns for AdvSIMD register list operands.
3530
3531 (define_expand "mov<mode>"
3532   [(set (match_operand:VSTRUCT 0 "aarch64_simd_nonimmediate_operand" "")
3533         (match_operand:VSTRUCT 1 "aarch64_simd_general_operand" ""))]
3534   "TARGET_SIMD"
3535 {
3536   if (can_create_pseudo_p ())
3537     {
3538       if (GET_CODE (operands[0]) != REG)
3539         operands[1] = force_reg (<MODE>mode, operands[1]);
3540     }
3541 })
3542
3543 (define_insn "*aarch64_mov<mode>"
3544   [(set (match_operand:VSTRUCT 0 "aarch64_simd_nonimmediate_operand" "=w,Utv,w")
3545         (match_operand:VSTRUCT 1 "aarch64_simd_general_operand" " w,w,Utv"))]
3546   "TARGET_SIMD
3547    && (register_operand (operands[0], <MODE>mode)
3548        || register_operand (operands[1], <MODE>mode))"
3549
3550 {
3551   switch (which_alternative)
3552     {
3553     case 0: return "#";
3554     case 1: return "st1\\t{%S1.16b - %<Vendreg>1.16b}, %0";
3555     case 2: return "ld1\\t{%S0.16b - %<Vendreg>0.16b}, %1";
3556     default: gcc_unreachable ();
3557     }
3558 }
3559   [(set_attr "simd_type" "simd_move,simd_store<nregs>,simd_load<nregs>")
3560    (set (attr "length") (symbol_ref "aarch64_simd_attr_length_move (insn)"))
3561    (set_attr "simd_mode" "<MODE>")])
3562
3563 (define_split
3564   [(set (match_operand:OI 0 "register_operand" "")
3565         (match_operand:OI 1 "register_operand" ""))]
3566   "TARGET_SIMD && reload_completed"
3567   [(set (match_dup 0) (match_dup 1))
3568    (set (match_dup 2) (match_dup 3))]
3569 {
3570   int rdest = REGNO (operands[0]);
3571   int rsrc = REGNO (operands[1]);
3572   rtx dest[2], src[2];
3573
3574   dest[0] = gen_rtx_REG (TFmode, rdest);
3575   src[0] = gen_rtx_REG (TFmode, rsrc);
3576   dest[1] = gen_rtx_REG (TFmode, rdest + 1);
3577   src[1] = gen_rtx_REG (TFmode, rsrc + 1);
3578
3579   aarch64_simd_disambiguate_copy (operands, dest, src, 2);
3580 })
3581
3582 (define_split
3583   [(set (match_operand:CI 0 "register_operand" "")
3584         (match_operand:CI 1 "register_operand" ""))]
3585   "TARGET_SIMD && reload_completed"
3586   [(set (match_dup 0) (match_dup 1))
3587    (set (match_dup 2) (match_dup 3))
3588    (set (match_dup 4) (match_dup 5))]
3589 {
3590   int rdest = REGNO (operands[0]);
3591   int rsrc = REGNO (operands[1]);
3592   rtx dest[3], src[3];
3593
3594   dest[0] = gen_rtx_REG (TFmode, rdest);
3595   src[0] = gen_rtx_REG (TFmode, rsrc);
3596   dest[1] = gen_rtx_REG (TFmode, rdest + 1);
3597   src[1] = gen_rtx_REG (TFmode, rsrc + 1);
3598   dest[2] = gen_rtx_REG (TFmode, rdest + 2);
3599   src[2] = gen_rtx_REG (TFmode, rsrc + 2);
3600
3601   aarch64_simd_disambiguate_copy (operands, dest, src, 3);
3602 })
3603
3604 (define_split
3605   [(set (match_operand:XI 0 "register_operand" "")
3606         (match_operand:XI 1 "register_operand" ""))]
3607   "TARGET_SIMD && reload_completed"
3608   [(set (match_dup 0) (match_dup 1))
3609    (set (match_dup 2) (match_dup 3))
3610    (set (match_dup 4) (match_dup 5))
3611    (set (match_dup 6) (match_dup 7))]
3612 {
3613   int rdest = REGNO (operands[0]);
3614   int rsrc = REGNO (operands[1]);
3615   rtx dest[4], src[4];
3616
3617   dest[0] = gen_rtx_REG (TFmode, rdest);
3618   src[0] = gen_rtx_REG (TFmode, rsrc);
3619   dest[1] = gen_rtx_REG (TFmode, rdest + 1);
3620   src[1] = gen_rtx_REG (TFmode, rsrc + 1);
3621   dest[2] = gen_rtx_REG (TFmode, rdest + 2);
3622   src[2] = gen_rtx_REG (TFmode, rsrc + 2);
3623   dest[3] = gen_rtx_REG (TFmode, rdest + 3);
3624   src[3] = gen_rtx_REG (TFmode, rsrc + 3);
3625
3626   aarch64_simd_disambiguate_copy (operands, dest, src, 4);
3627 })
3628
3629 (define_insn "aarch64_ld2<mode>_dreg"
3630   [(set (match_operand:OI 0 "register_operand" "=w")
3631         (subreg:OI
3632           (vec_concat:<VRL2>
3633             (vec_concat:<VDBL>
3634              (unspec:VD [(match_operand:TI 1 "aarch64_simd_struct_operand" "Utv")]
3635                         UNSPEC_LD2)
3636              (vec_duplicate:VD (const_int 0)))
3637             (vec_concat:<VDBL>
3638              (unspec:VD [(match_dup 1)]
3639                         UNSPEC_LD2)
3640              (vec_duplicate:VD (const_int 0)))) 0))]
3641   "TARGET_SIMD"
3642   "ld2\\t{%S0.<Vtype> - %T0.<Vtype>}, %1"
3643   [(set_attr "simd_type" "simd_load2")
3644    (set_attr "simd_mode" "<MODE>")])
3645
3646 (define_insn "aarch64_ld2<mode>_dreg"
3647   [(set (match_operand:OI 0 "register_operand" "=w")
3648         (subreg:OI
3649           (vec_concat:<VRL2>
3650             (vec_concat:<VDBL>
3651              (unspec:DX [(match_operand:TI 1 "aarch64_simd_struct_operand" "Utv")]
3652                         UNSPEC_LD2)
3653              (const_int 0))
3654             (vec_concat:<VDBL>
3655              (unspec:DX [(match_dup 1)]
3656                         UNSPEC_LD2)
3657              (const_int 0))) 0))]
3658   "TARGET_SIMD"
3659   "ld1\\t{%S0.1d - %T0.1d}, %1"
3660   [(set_attr "simd_type" "simd_load2")
3661    (set_attr "simd_mode" "<MODE>")])
3662
3663 (define_insn "aarch64_ld3<mode>_dreg"
3664   [(set (match_operand:CI 0 "register_operand" "=w")
3665         (subreg:CI
3666          (vec_concat:<VRL3>
3667           (vec_concat:<VRL2>
3668             (vec_concat:<VDBL>
3669              (unspec:VD [(match_operand:EI 1 "aarch64_simd_struct_operand" "Utv")]
3670                         UNSPEC_LD3)
3671              (vec_duplicate:VD (const_int 0)))
3672             (vec_concat:<VDBL>
3673              (unspec:VD [(match_dup 1)]
3674                         UNSPEC_LD3)
3675              (vec_duplicate:VD (const_int 0))))
3676           (vec_concat:<VDBL>
3677              (unspec:VD [(match_dup 1)]
3678                         UNSPEC_LD3)
3679              (vec_duplicate:VD (const_int 0)))) 0))]
3680   "TARGET_SIMD"
3681   "ld3\\t{%S0.<Vtype> - %U0.<Vtype>}, %1"
3682   [(set_attr "simd_type" "simd_load3")
3683    (set_attr "simd_mode" "<MODE>")])
3684
3685 (define_insn "aarch64_ld3<mode>_dreg"
3686   [(set (match_operand:CI 0 "register_operand" "=w")
3687         (subreg:CI
3688          (vec_concat:<VRL3>
3689           (vec_concat:<VRL2>
3690             (vec_concat:<VDBL>
3691              (unspec:DX [(match_operand:EI 1 "aarch64_simd_struct_operand" "Utv")]
3692                         UNSPEC_LD3)
3693              (const_int 0))
3694             (vec_concat:<VDBL>
3695              (unspec:DX [(match_dup 1)]
3696                         UNSPEC_LD3)
3697              (const_int 0)))
3698           (vec_concat:<VDBL>
3699              (unspec:DX [(match_dup 1)]
3700                         UNSPEC_LD3)
3701              (const_int 0))) 0))]
3702   "TARGET_SIMD"
3703   "ld1\\t{%S0.1d - %U0.1d}, %1"
3704   [(set_attr "simd_type" "simd_load3")
3705    (set_attr "simd_mode" "<MODE>")])
3706
3707 (define_insn "aarch64_ld4<mode>_dreg"
3708   [(set (match_operand:XI 0 "register_operand" "=w")
3709         (subreg:XI
3710          (vec_concat:<VRL4>
3711            (vec_concat:<VRL2>
3712              (vec_concat:<VDBL>
3713                (unspec:VD [(match_operand:OI 1 "aarch64_simd_struct_operand" "Utv")]
3714                           UNSPEC_LD4)
3715                (vec_duplicate:VD (const_int 0)))
3716               (vec_concat:<VDBL>
3717                 (unspec:VD [(match_dup 1)]
3718                         UNSPEC_LD4)
3719                 (vec_duplicate:VD (const_int 0))))
3720            (vec_concat:<VRL2>
3721              (vec_concat:<VDBL>
3722                (unspec:VD [(match_dup 1)]
3723                         UNSPEC_LD4)
3724                (vec_duplicate:VD (const_int 0)))
3725              (vec_concat:<VDBL>
3726                (unspec:VD [(match_dup 1)]
3727                         UNSPEC_LD4)
3728                (vec_duplicate:VD (const_int 0))))) 0))]
3729   "TARGET_SIMD"
3730   "ld4\\t{%S0.<Vtype> - %V0.<Vtype>}, %1"
3731   [(set_attr "simd_type" "simd_load4")
3732    (set_attr "simd_mode" "<MODE>")])
3733
3734 (define_insn "aarch64_ld4<mode>_dreg"
3735   [(set (match_operand:XI 0 "register_operand" "=w")
3736         (subreg:XI
3737          (vec_concat:<VRL4>
3738            (vec_concat:<VRL2>
3739              (vec_concat:<VDBL>
3740                (unspec:DX [(match_operand:OI 1 "aarch64_simd_struct_operand" "Utv")]
3741                           UNSPEC_LD4)
3742                (const_int 0))
3743               (vec_concat:<VDBL>
3744                 (unspec:DX [(match_dup 1)]
3745                         UNSPEC_LD4)
3746                 (const_int 0)))
3747            (vec_concat:<VRL2>
3748              (vec_concat:<VDBL>
3749                (unspec:DX [(match_dup 1)]
3750                         UNSPEC_LD4)
3751                (const_int 0))
3752              (vec_concat:<VDBL>
3753                (unspec:DX [(match_dup 1)]
3754                         UNSPEC_LD4)
3755                (const_int 0)))) 0))]
3756   "TARGET_SIMD"
3757   "ld1\\t{%S0.1d - %V0.1d}, %1"
3758   [(set_attr "simd_type" "simd_load4")
3759    (set_attr "simd_mode" "<MODE>")])
3760
3761 (define_expand "aarch64_ld<VSTRUCT:nregs><VDC:mode>"
3762  [(match_operand:VSTRUCT 0 "register_operand" "=w")
3763   (match_operand:DI 1 "register_operand" "r")
3764   (unspec:VDC [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3765   "TARGET_SIMD"
3766 {
3767   enum machine_mode mode = <VSTRUCT:VSTRUCT_DREG>mode;
3768   rtx mem = gen_rtx_MEM (mode, operands[1]);
3769
3770   emit_insn (gen_aarch64_ld<VSTRUCT:nregs><VDC:mode>_dreg (operands[0], mem));
3771   DONE;
3772 })
3773
3774 (define_expand "aarch64_ld<VSTRUCT:nregs><VQ:mode>"
3775  [(match_operand:VSTRUCT 0 "register_operand" "=w")
3776   (match_operand:DI 1 "register_operand" "r")
3777   (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3778   "TARGET_SIMD"
3779 {
3780   enum machine_mode mode = <VSTRUCT:MODE>mode;
3781   rtx mem = gen_rtx_MEM (mode, operands[1]);
3782
3783   emit_insn (gen_vec_load_lanes<VSTRUCT:mode><VQ:mode> (operands[0], mem));
3784   DONE;
3785 })
3786
3787 ;; Expanders for builtins to extract vector registers from large
3788 ;; opaque integer modes.
3789
3790 ;; D-register list.
3791
3792 (define_expand "aarch64_get_dreg<VSTRUCT:mode><VDC:mode>"
3793  [(match_operand:VDC 0 "register_operand" "=w")
3794   (match_operand:VSTRUCT 1 "register_operand" "w")
3795   (match_operand:SI 2 "immediate_operand" "i")]
3796   "TARGET_SIMD"
3797 {
3798   int part = INTVAL (operands[2]);
3799   rtx temp = gen_reg_rtx (<VDC:VDBL>mode);
3800   int offset = part * 16;
3801
3802   emit_move_insn (temp, gen_rtx_SUBREG (<VDC:VDBL>mode, operands[1], offset));
3803   emit_move_insn (operands[0], gen_lowpart (<VDC:MODE>mode, temp));
3804   DONE;
3805 })
3806
3807 ;; Q-register list.
3808
3809 (define_expand "aarch64_get_qreg<VSTRUCT:mode><VQ:mode>"
3810  [(match_operand:VQ 0 "register_operand" "=w")
3811   (match_operand:VSTRUCT 1 "register_operand" "w")
3812   (match_operand:SI 2 "immediate_operand" "i")]
3813   "TARGET_SIMD"
3814 {
3815   int part = INTVAL (operands[2]);
3816   int offset = part * 16;
3817
3818   emit_move_insn (operands[0],
3819                   gen_rtx_SUBREG (<VQ:MODE>mode, operands[1], offset));
3820   DONE;
3821 })
3822
3823 ;; Permuted-store expanders for neon intrinsics.
3824
3825 ;; Permute instructions
3826
3827 ;; vec_perm support
3828
3829 (define_expand "vec_perm_const<mode>"
3830   [(match_operand:VALL 0 "register_operand")
3831    (match_operand:VALL 1 "register_operand")
3832    (match_operand:VALL 2 "register_operand")
3833    (match_operand:<V_cmp_result> 3)]
3834   "TARGET_SIMD"
3835 {
3836   if (aarch64_expand_vec_perm_const (operands[0], operands[1],
3837                                      operands[2], operands[3]))
3838     DONE;
3839   else
3840     FAIL;
3841 })
3842
3843 (define_expand "vec_perm<mode>"
3844   [(match_operand:VB 0 "register_operand")
3845    (match_operand:VB 1 "register_operand")
3846    (match_operand:VB 2 "register_operand")
3847    (match_operand:VB 3 "register_operand")]
3848   "TARGET_SIMD"
3849 {
3850   aarch64_expand_vec_perm (operands[0], operands[1],
3851                            operands[2], operands[3]);
3852   DONE;
3853 })
3854
3855 (define_insn "aarch64_tbl1<mode>"
3856   [(set (match_operand:VB 0 "register_operand" "=w")
3857         (unspec:VB [(match_operand:V16QI 1 "register_operand" "w")
3858                     (match_operand:VB 2 "register_operand" "w")]
3859                    UNSPEC_TBL))]
3860   "TARGET_SIMD"
3861   "tbl\\t%0.<Vtype>, {%1.16b}, %2.<Vtype>"
3862   [(set_attr "simd_type" "simd_tbl")
3863    (set_attr "simd_mode" "<MODE>")]
3864 )
3865
3866 ;; Two source registers.
3867
3868 (define_insn "aarch64_tbl2v16qi"
3869   [(set (match_operand:V16QI 0 "register_operand" "=w")
3870         (unspec:V16QI [(match_operand:OI 1 "register_operand" "w")
3871                        (match_operand:V16QI 2 "register_operand" "w")]
3872                       UNSPEC_TBL))]
3873   "TARGET_SIMD"
3874   "tbl\\t%0.16b, {%S1.16b - %T1.16b}, %2.16b"
3875   [(set_attr "simd_type" "simd_tbl")
3876    (set_attr "simd_mode" "V16QI")]
3877 )
3878
3879 (define_insn_and_split "aarch64_combinev16qi"
3880   [(set (match_operand:OI 0 "register_operand" "=w")
3881         (unspec:OI [(match_operand:V16QI 1 "register_operand" "w")
3882                     (match_operand:V16QI 2 "register_operand" "w")]
3883                    UNSPEC_CONCAT))]
3884   "TARGET_SIMD"
3885   "#"
3886   "&& reload_completed"
3887   [(const_int 0)]
3888 {
3889   aarch64_split_combinev16qi (operands);
3890   DONE;
3891 })
3892
3893 (define_insn "aarch64_<PERMUTE:perm_insn><PERMUTE:perm_hilo><mode>"
3894   [(set (match_operand:VALL 0 "register_operand" "=w")
3895         (unspec:VALL [(match_operand:VALL 1 "register_operand" "w")
3896                       (match_operand:VALL 2 "register_operand" "w")]
3897                        PERMUTE))]
3898   "TARGET_SIMD"
3899   "<PERMUTE:perm_insn><PERMUTE:perm_hilo>\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
3900   [(set_attr "simd_type" "simd_<PERMUTE:perm_insn>")
3901    (set_attr "simd_mode" "<MODE>")]
3902 )
3903
3904 (define_insn "aarch64_st2<mode>_dreg"
3905   [(set (match_operand:TI 0 "aarch64_simd_struct_operand" "=Utv")
3906         (unspec:TI [(match_operand:OI 1 "register_operand" "w")
3907                     (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3908                    UNSPEC_ST2))]
3909   "TARGET_SIMD"
3910   "st2\\t{%S1.<Vtype> - %T1.<Vtype>}, %0"
3911   [(set_attr "simd_type" "simd_store2")
3912    (set_attr "simd_mode" "<MODE>")])
3913
3914 (define_insn "aarch64_st2<mode>_dreg"
3915   [(set (match_operand:TI 0 "aarch64_simd_struct_operand" "=Utv")
3916         (unspec:TI [(match_operand:OI 1 "register_operand" "w")
3917                     (unspec:DX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3918                    UNSPEC_ST2))]
3919   "TARGET_SIMD"
3920   "st1\\t{%S1.1d - %T1.1d}, %0"
3921   [(set_attr "simd_type" "simd_store2")
3922    (set_attr "simd_mode" "<MODE>")])
3923
3924 (define_insn "aarch64_st3<mode>_dreg"
3925   [(set (match_operand:EI 0 "aarch64_simd_struct_operand" "=Utv")
3926         (unspec:EI [(match_operand:CI 1 "register_operand" "w")
3927                     (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3928                    UNSPEC_ST3))]
3929   "TARGET_SIMD"
3930   "st3\\t{%S1.<Vtype> - %U1.<Vtype>}, %0"
3931   [(set_attr "simd_type" "simd_store3")
3932    (set_attr "simd_mode" "<MODE>")])
3933
3934 (define_insn "aarch64_st3<mode>_dreg"
3935   [(set (match_operand:EI 0 "aarch64_simd_struct_operand" "=Utv")
3936         (unspec:EI [(match_operand:CI 1 "register_operand" "w")
3937                     (unspec:DX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3938                    UNSPEC_ST3))]
3939   "TARGET_SIMD"
3940   "st1\\t{%S1.1d - %U1.1d}, %0"
3941   [(set_attr "simd_type" "simd_store3")
3942    (set_attr "simd_mode" "<MODE>")])
3943
3944 (define_insn "aarch64_st4<mode>_dreg"
3945   [(set (match_operand:OI 0 "aarch64_simd_struct_operand" "=Utv")
3946         (unspec:OI [(match_operand:XI 1 "register_operand" "w")
3947                     (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3948                    UNSPEC_ST4))]
3949   "TARGET_SIMD"
3950   "st4\\t{%S1.<Vtype> - %V1.<Vtype>}, %0"
3951   [(set_attr "simd_type" "simd_store4")
3952    (set_attr "simd_mode" "<MODE>")])
3953
3954 (define_insn "aarch64_st4<mode>_dreg"
3955   [(set (match_operand:OI 0 "aarch64_simd_struct_operand" "=Utv")
3956         (unspec:OI [(match_operand:XI 1 "register_operand" "w")
3957                     (unspec:DX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3958                    UNSPEC_ST4))]
3959   "TARGET_SIMD"
3960   "st1\\t{%S1.1d - %V1.1d}, %0"
3961   [(set_attr "simd_type" "simd_store4")
3962    (set_attr "simd_mode" "<MODE>")])
3963
3964 (define_expand "aarch64_st<VSTRUCT:nregs><VDC:mode>"
3965  [(match_operand:DI 0 "register_operand" "r")
3966   (match_operand:VSTRUCT 1 "register_operand" "w")
3967   (unspec:VDC [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3968   "TARGET_SIMD"
3969 {
3970   enum machine_mode mode = <VSTRUCT:VSTRUCT_DREG>mode;
3971   rtx mem = gen_rtx_MEM (mode, operands[0]);
3972
3973   emit_insn (gen_aarch64_st<VSTRUCT:nregs><VDC:mode>_dreg (mem, operands[1]));
3974   DONE;
3975 })
3976
3977 (define_expand "aarch64_st<VSTRUCT:nregs><VQ:mode>"
3978  [(match_operand:DI 0 "register_operand" "r")
3979   (match_operand:VSTRUCT 1 "register_operand" "w")
3980   (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3981   "TARGET_SIMD"
3982 {
3983   enum machine_mode mode = <VSTRUCT:MODE>mode;
3984   rtx mem = gen_rtx_MEM (mode, operands[0]);
3985
3986   emit_insn (gen_vec_store_lanes<VSTRUCT:mode><VQ:mode> (mem, operands[1]));
3987   DONE;
3988 })
3989
3990 ;; Expander for builtins to insert vector registers into large
3991 ;; opaque integer modes.
3992
3993 ;; Q-register list.  We don't need a D-reg inserter as we zero
3994 ;; extend them in arm_neon.h and insert the resulting Q-regs.
3995
3996 (define_expand "aarch64_set_qreg<VSTRUCT:mode><VQ:mode>"
3997  [(match_operand:VSTRUCT 0 "register_operand" "+w")
3998   (match_operand:VSTRUCT 1 "register_operand" "0")
3999   (match_operand:VQ 2 "register_operand" "w")
4000   (match_operand:SI 3 "immediate_operand" "i")]
4001   "TARGET_SIMD"
4002 {
4003   int part = INTVAL (operands[3]);
4004   int offset = part * 16;
4005
4006   emit_move_insn (operands[0], operands[1]);
4007   emit_move_insn (gen_rtx_SUBREG (<VQ:MODE>mode, operands[0], offset),
4008                   operands[2]);
4009   DONE;
4010 })
4011
4012 ;; Standard pattern name vec_init<mode>.
4013
4014 (define_expand "vec_init<mode>"
4015   [(match_operand:VALL 0 "register_operand" "")
4016    (match_operand 1 "" "")]
4017   "TARGET_SIMD"
4018 {
4019   aarch64_expand_vector_init (operands[0], operands[1]);
4020   DONE;
4021 })
4022
4023 (define_insn "*aarch64_simd_ld1r<mode>"
4024   [(set (match_operand:VALLDI 0 "register_operand" "=w")
4025         (vec_duplicate:VALLDI
4026           (match_operand:<VEL> 1 "aarch64_simd_struct_operand" "Utv")))]
4027   "TARGET_SIMD"
4028   "ld1r\\t{%0.<Vtype>}, %1"
4029   [(set_attr "simd_type" "simd_load1r")
4030    (set_attr "simd_mode" "<MODE>")])
4031
4032 (define_insn "aarch64_frecpe<mode>"
4033   [(set (match_operand:VDQF 0 "register_operand" "=w")
4034         (unspec:VDQF [(match_operand:VDQF 1 "register_operand" "w")]
4035                     UNSPEC_FRECPE))]
4036   "TARGET_SIMD"
4037   "frecpe\\t%0.<Vtype>, %1.<Vtype>"
4038   [(set_attr "simd_type" "simd_frecpe")
4039    (set_attr "simd_mode" "<MODE>")]
4040 )
4041
4042 (define_insn "aarch64_frecps<mode>"
4043   [(set (match_operand:VDQF 0 "register_operand" "=w")
4044         (unspec:VDQF [(match_operand:VDQF 1 "register_operand" "w")
4045                      (match_operand:VDQF 2 "register_operand" "w")]
4046                     UNSPEC_FRECPS))]
4047   "TARGET_SIMD"
4048   "frecps\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
4049   [(set_attr "simd_type" "simd_frecps")
4050    (set_attr "simd_mode" "<MODE>")]
4051 )
4052