gcc/config/arm/neon.md

   1 ;; ARM NEON coprocessor Machine Description
   2 ;; Copyright (C) 2006-2013 Free Software Foundation, Inc.
   3 ;; Written by CodeSourcery.
   4 ;;
   5 ;; This file is part of GCC.
   6 ;;
   7 ;; GCC is free software; you can redistribute it and/or modify it
   8 ;; under the terms of the GNU General Public License as published by
   9 ;; the Free Software Foundation; either version 3, or (at your option)
  10 ;; any later version.
  11 ;;
  12 ;; GCC is distributed in the hope that it will be useful, but
  13 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
  14 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15 ;; General Public License for more details.
  16 ;;
  17 ;; You should have received a copy of the GNU General Public License
  18 ;; along with GCC; see the file COPYING3.  If not see
  19 ;; <http://www.gnu.org/licenses/>.
  20
  21
  22 ;; Attribute used to permit string comparisons against <VQH_mnem> in
  23 ;; type attribute definitions.
  24 (define_attr "vqh_mnem" "vadd,vmin,vmax" (const_string "vadd"))
  25
  26 (define_insn "*neon_mov<mode>"
  27   [(set (match_operand:VDX 0 "nonimmediate_operand"
  28           "=w,Un,w, w,  ?r,?w,?r,?r, ?Us")
  29         (match_operand:VDX 1 "general_operand"
  30           " w,w, Dn,Uni, w, r, r, Usi,r"))]
  31   "TARGET_NEON
  32    && (register_operand (operands[0], <MODE>mode)
  33        || register_operand (operands[1], <MODE>mode))"
  34 {
  35   if (which_alternative == 2)
  36     {
  37       int width, is_valid;
  38       static char templ[40];
  39
  40       is_valid = neon_immediate_valid_for_move (operands[1], <MODE>mode,
  41         &operands[1], &width);
  42
  43       gcc_assert (is_valid != 0);
  44
  45       if (width == 0)
  46         return "vmov.f32\t%P0, %1  @ <mode>";
  47       else
  48         sprintf (templ, "vmov.i%d\t%%P0, %%x1  @ <mode>", width);
  49
  50       return templ;
  51     }
  52
  53   switch (which_alternative)
  54     {
  55     case 0: return "vmov\t%P0, %P1  @ <mode>";
  56     case 1: case 3: return output_move_neon (operands);
  57     case 2: gcc_unreachable ();
  58     case 4: return "vmov\t%Q0, %R0, %P1  @ <mode>";
  59     case 5: return "vmov\t%P0, %Q1, %R1  @ <mode>";
  60     default: return output_move_double (operands, true, NULL);
  61     }
  62 }
  63  [(set_attr "type" "neon_move<q>,neon_store1_1reg,neon_move<q>,\
  64                     neon_load1_1reg, neon_to_gp<q>,neon_from_gp<q>,mov_reg,\
  65                     neon_load1_2reg, neon_store1_2reg")
  66   (set_attr "length" "4,4,4,4,4,4,8,8,8")
  67   (set_attr "arm_pool_range"     "*,*,*,1020,*,*,*,1020,*")
  68   (set_attr "thumb2_pool_range"     "*,*,*,1018,*,*,*,1018,*")
  69   (set_attr "neg_pool_range" "*,*,*,1004,*,*,*,1004,*")])
  70
  71 (define_insn "*neon_mov<mode>"
  72   [(set (match_operand:VQXMOV 0 "nonimmediate_operand"
  73           "=w,Un,w, w,  ?r,?w,?r,?r,  ?Us")
  74         (match_operand:VQXMOV 1 "general_operand"
  75           " w,w, Dn,Uni, w, r, r, Usi, r"))]
  76   "TARGET_NEON
  77    && (register_operand (operands[0], <MODE>mode)
  78        || register_operand (operands[1], <MODE>mode))"
  79 {
  80   if (which_alternative == 2)
  81     {
  82       int width, is_valid;
  83       static char templ[40];
  84
  85       is_valid = neon_immediate_valid_for_move (operands[1], <MODE>mode,
  86         &operands[1], &width);
  87
  88       gcc_assert (is_valid != 0);
  89
  90       if (width == 0)
  91         return "vmov.f32\t%q0, %1  @ <mode>";
  92       else
  93         sprintf (templ, "vmov.i%d\t%%q0, %%1  @ <mode>", width);
  94
  95       return templ;
  96     }
  97
  98   switch (which_alternative)
  99     {
 100     case 0: return "vmov\t%q0, %q1  @ <mode>";
 101     case 1: case 3: return output_move_neon (operands);
 102     case 2: gcc_unreachable ();
 103     case 4: return "vmov\t%Q0, %R0, %e1  @ <mode>\;vmov\t%J0, %K0, %f1";
 104     case 5: return "vmov\t%e0, %Q1, %R1  @ <mode>\;vmov\t%f0, %J1, %K1";
 105     default: return output_move_quad (operands);
 106     }
 107 }
 108   [(set_attr "type" "neon_move_q,neon_store2_2reg_q,neon_move_q,\
 109                      neon_load2_2reg_q,neon_to_gp_q,neon_from_gp_q,\
 110                      mov_reg,neon_load1_4reg,neon_store1_4reg")
 111    (set_attr "length" "4,8,4,8,8,8,16,8,16")
 112    (set_attr "arm_pool_range" "*,*,*,1020,*,*,*,1020,*")
 113    (set_attr "thumb2_pool_range" "*,*,*,1018,*,*,*,1018,*")
 114    (set_attr "neg_pool_range" "*,*,*,996,*,*,*,996,*")])
 115
 116 (define_expand "movti"
 117   [(set (match_operand:TI 0 "nonimmediate_operand" "")
 118         (match_operand:TI 1 "general_operand" ""))]
 119   "TARGET_NEON"
 120 {
 121   if (can_create_pseudo_p ())
 122     {
 123       if (!REG_P (operands[0]))
 124         operands[1] = force_reg (TImode, operands[1]);
 125     }
 126 })
 127
 128 (define_expand "mov<mode>"
 129   [(set (match_operand:VSTRUCT 0 "nonimmediate_operand" "")
 130         (match_operand:VSTRUCT 1 "general_operand" ""))]
 131   "TARGET_NEON"
 132 {
 133   if (can_create_pseudo_p ())
 134     {
 135       if (!REG_P (operands[0]))
 136         operands[1] = force_reg (<MODE>mode, operands[1]);
 137     }
 138 })
 139
 140 (define_insn "*neon_mov<mode>"
 141   [(set (match_operand:VSTRUCT 0 "nonimmediate_operand" "=w,Ut,w")
 142         (match_operand:VSTRUCT 1 "general_operand"      " w,w, Ut"))]
 143   "TARGET_NEON
 144    && (register_operand (operands[0], <MODE>mode)
 145        || register_operand (operands[1], <MODE>mode))"
 146 {
 147   switch (which_alternative)
 148     {
 149     case 0: return "#";
 150     case 1: case 2: return output_move_neon (operands);
 151     default: gcc_unreachable ();
 152     }
 153 }
 154   [(set_attr "type" "neon_move_q,neon_store2_2reg_q,neon_load2_2reg_q")
 155    (set (attr "length") (symbol_ref "arm_attr_length_move_neon (insn)"))])
 156
 157 (define_split
 158   [(set (match_operand:EI 0 "s_register_operand" "")
 159         (match_operand:EI 1 "s_register_operand" ""))]
 160   "TARGET_NEON && reload_completed"
 161   [(set (match_dup 0) (match_dup 1))
 162    (set (match_dup 2) (match_dup 3))]
 163 {
 164   int rdest = REGNO (operands[0]);
 165   int rsrc = REGNO (operands[1]);
 166   rtx dest[2], src[2];
 167
 168   dest[0] = gen_rtx_REG (TImode, rdest);
 169   src[0] = gen_rtx_REG (TImode, rsrc);
 170   dest[1] = gen_rtx_REG (DImode, rdest + 4);
 171   src[1] = gen_rtx_REG (DImode, rsrc + 4);
 172
 173   neon_disambiguate_copy (operands, dest, src, 2);
 174 })
 175
 176 (define_split
 177   [(set (match_operand:OI 0 "s_register_operand" "")
 178         (match_operand:OI 1 "s_register_operand" ""))]
 179   "TARGET_NEON && reload_completed"
 180   [(set (match_dup 0) (match_dup 1))
 181    (set (match_dup 2) (match_dup 3))]
 182 {
 183   int rdest = REGNO (operands[0]);
 184   int rsrc = REGNO (operands[1]);
 185   rtx dest[2], src[2];
 186
 187   dest[0] = gen_rtx_REG (TImode, rdest);
 188   src[0] = gen_rtx_REG (TImode, rsrc);
 189   dest[1] = gen_rtx_REG (TImode, rdest + 4);
 190   src[1] = gen_rtx_REG (TImode, rsrc + 4);
 191
 192   neon_disambiguate_copy (operands, dest, src, 2);
 193 })
 194
 195 (define_split
 196   [(set (match_operand:CI 0 "s_register_operand" "")
 197         (match_operand:CI 1 "s_register_operand" ""))]
 198   "TARGET_NEON && reload_completed"
 199   [(set (match_dup 0) (match_dup 1))
 200    (set (match_dup 2) (match_dup 3))
 201    (set (match_dup 4) (match_dup 5))]
 202 {
 203   int rdest = REGNO (operands[0]);
 204   int rsrc = REGNO (operands[1]);
 205   rtx dest[3], src[3];
 206
 207   dest[0] = gen_rtx_REG (TImode, rdest);
 208   src[0] = gen_rtx_REG (TImode, rsrc);
 209   dest[1] = gen_rtx_REG (TImode, rdest + 4);
 210   src[1] = gen_rtx_REG (TImode, rsrc + 4);
 211   dest[2] = gen_rtx_REG (TImode, rdest + 8);
 212   src[2] = gen_rtx_REG (TImode, rsrc + 8);
 213
 214   neon_disambiguate_copy (operands, dest, src, 3);
 215 })
 216
 217 (define_split
 218   [(set (match_operand:XI 0 "s_register_operand" "")
 219         (match_operand:XI 1 "s_register_operand" ""))]
 220   "TARGET_NEON && reload_completed"
 221   [(set (match_dup 0) (match_dup 1))
 222    (set (match_dup 2) (match_dup 3))
 223    (set (match_dup 4) (match_dup 5))
 224    (set (match_dup 6) (match_dup 7))]
 225 {
 226   int rdest = REGNO (operands[0]);
 227   int rsrc = REGNO (operands[1]);
 228   rtx dest[4], src[4];
 229
 230   dest[0] = gen_rtx_REG (TImode, rdest);
 231   src[0] = gen_rtx_REG (TImode, rsrc);
 232   dest[1] = gen_rtx_REG (TImode, rdest + 4);
 233   src[1] = gen_rtx_REG (TImode, rsrc + 4);
 234   dest[2] = gen_rtx_REG (TImode, rdest + 8);
 235   src[2] = gen_rtx_REG (TImode, rsrc + 8);
 236   dest[3] = gen_rtx_REG (TImode, rdest + 12);
 237   src[3] = gen_rtx_REG (TImode, rsrc + 12);
 238
 239   neon_disambiguate_copy (operands, dest, src, 4);
 240 })
 241
 242 (define_expand "movmisalign<mode>"
 243   [(set (match_operand:VDQX 0 "neon_perm_struct_or_reg_operand")
 244         (unspec:VDQX [(match_operand:VDQX 1 "neon_perm_struct_or_reg_operand")]
 245                      UNSPEC_MISALIGNED_ACCESS))]
 246   "TARGET_NEON && !BYTES_BIG_ENDIAN && unaligned_access"
 247 {
 248   /* This pattern is not permitted to fail during expansion: if both arguments
 249      are non-registers (e.g. memory := constant, which can be created by the
 250      auto-vectorizer), force operand 1 into a register.  */
 251   if (!s_register_operand (operands[0], <MODE>mode)
 252       && !s_register_operand (operands[1], <MODE>mode))
 253     operands[1] = force_reg (<MODE>mode, operands[1]);
 254 })
 255
 256 (define_insn "*movmisalign<mode>_neon_store"
 257   [(set (match_operand:VDX 0 "neon_permissive_struct_operand"   "=Um")
 258         (unspec:VDX [(match_operand:VDX 1 "s_register_operand" " w")]
 259                     UNSPEC_MISALIGNED_ACCESS))]
 260   "TARGET_NEON && !BYTES_BIG_ENDIAN && unaligned_access"
 261   "vst1.<V_sz_elem>\t{%P1}, %A0"
 262   [(set_attr "type" "neon_store1_1reg<q>")])
 263
 264 (define_insn "*movmisalign<mode>_neon_load"
 265   [(set (match_operand:VDX 0 "s_register_operand"                       "=w")
 266         (unspec:VDX [(match_operand:VDX 1 "neon_permissive_struct_operand"
 267                                                                         " Um")]
 268                     UNSPEC_MISALIGNED_ACCESS))]
 269   "TARGET_NEON && !BYTES_BIG_ENDIAN && unaligned_access"
 270   "vld1.<V_sz_elem>\t{%P0}, %A1"
 271   [(set_attr "type" "neon_load1_1reg<q>")])
 272
 273 (define_insn "*movmisalign<mode>_neon_store"
 274   [(set (match_operand:VQX 0 "neon_permissive_struct_operand"  "=Um")
 275         (unspec:VQX [(match_operand:VQX 1 "s_register_operand" " w")]
 276                     UNSPEC_MISALIGNED_ACCESS))]
 277   "TARGET_NEON && !BYTES_BIG_ENDIAN && unaligned_access"
 278   "vst1.<V_sz_elem>\t{%q1}, %A0"
 279   [(set_attr "type" "neon_store1_1reg<q>")])
 280
 281 (define_insn "*movmisalign<mode>_neon_load"
 282   [(set (match_operand:VQX 0 "s_register_operand"                       "=w")
 283         (unspec:VQX [(match_operand:VQX 1 "neon_permissive_struct_operand"
 284                                                                         " Um")]
 285                     UNSPEC_MISALIGNED_ACCESS))]
 286   "TARGET_NEON && !BYTES_BIG_ENDIAN && unaligned_access"
 287   "vld1.<V_sz_elem>\t{%q0}, %A1"
 288   [(set_attr "type" "neon_store1_1reg<q>")])
 289
 290 (define_insn "vec_set<mode>_internal"
 291   [(set (match_operand:VD 0 "s_register_operand" "=w,w")
 292         (vec_merge:VD
 293           (vec_duplicate:VD
 294             (match_operand:<V_elem> 1 "nonimmediate_operand" "Um,r"))
 295           (match_operand:VD 3 "s_register_operand" "0,0")
 296           (match_operand:SI 2 "immediate_operand" "i,i")))]
 297   "TARGET_NEON"
 298 {
 299   int elt = ffs ((int) INTVAL (operands[2])) - 1;
 300   if (BYTES_BIG_ENDIAN)
 301     elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
 302   operands[2] = GEN_INT (elt);
 303
 304   if (which_alternative == 0)
 305     return "vld1.<V_sz_elem>\t{%P0[%c2]}, %A1";
 306   else
 307     return "vmov.<V_sz_elem>\t%P0[%c2], %1";
 308 }
 309   [(set_attr "type" "neon_load1_all_lanes<q>,neon_from_gp<q>")])
 310
 311 (define_insn "vec_set<mode>_internal"
 312   [(set (match_operand:VQ 0 "s_register_operand" "=w,w")
 313         (vec_merge:VQ
 314           (vec_duplicate:VQ
 315             (match_operand:<V_elem> 1 "nonimmediate_operand" "Um,r"))
 316           (match_operand:VQ 3 "s_register_operand" "0,0")
 317           (match_operand:SI 2 "immediate_operand" "i,i")))]
 318   "TARGET_NEON"
 319 {
 320   HOST_WIDE_INT elem = ffs ((int) INTVAL (operands[2])) - 1;
 321   int half_elts = GET_MODE_NUNITS (<MODE>mode) / 2;
 322   int elt = elem % half_elts;
 323   int hi = (elem / half_elts) * 2;
 324   int regno = REGNO (operands[0]);
 325
 326   if (BYTES_BIG_ENDIAN)
 327     elt = half_elts - 1 - elt;
 328
 329   operands[0] = gen_rtx_REG (<V_HALF>mode, regno + hi);
 330   operands[2] = GEN_INT (elt);
 331
 332   if (which_alternative == 0)
 333     return "vld1.<V_sz_elem>\t{%P0[%c2]}, %A1";
 334   else
 335     return "vmov.<V_sz_elem>\t%P0[%c2], %1";
 336 }
 337   [(set_attr "type" "neon_load1_all_lanes<q>,neon_from_gp<q>")]
 338 )
 339
 340 (define_insn "vec_setv2di_internal"
 341   [(set (match_operand:V2DI 0 "s_register_operand" "=w,w")
 342         (vec_merge:V2DI
 343           (vec_duplicate:V2DI
 344             (match_operand:DI 1 "nonimmediate_operand" "Um,r"))
 345           (match_operand:V2DI 3 "s_register_operand" "0,0")
 346           (match_operand:SI 2 "immediate_operand" "i,i")))]
 347   "TARGET_NEON"
 348 {
 349   HOST_WIDE_INT elem = ffs ((int) INTVAL (operands[2])) - 1;
 350   int regno = REGNO (operands[0]) + 2 * elem;
 351
 352   operands[0] = gen_rtx_REG (DImode, regno);
 353
 354   if (which_alternative == 0)
 355     return "vld1.64\t%P0, %A1";
 356   else
 357     return "vmov\t%P0, %Q1, %R1";
 358 }
 359   [(set_attr "type" "neon_load1_all_lanes_q,neon_from_gp_q")]
 360 )
 361
 362 (define_expand "vec_set<mode>"
 363   [(match_operand:VDQ 0 "s_register_operand" "")
 364    (match_operand:<V_elem> 1 "s_register_operand" "")
 365    (match_operand:SI 2 "immediate_operand" "")]
 366   "TARGET_NEON"
 367 {
 368   HOST_WIDE_INT elem = (HOST_WIDE_INT) 1 << INTVAL (operands[2]);
 369   emit_insn (gen_vec_set<mode>_internal (operands[0], operands[1],
 370                                          GEN_INT (elem), operands[0]));
 371   DONE;
 372 })
 373
 374 (define_insn "vec_extract<mode>"
 375   [(set (match_operand:<V_elem> 0 "nonimmediate_operand" "=Um,r")
 376         (vec_select:<V_elem>
 377           (match_operand:VD 1 "s_register_operand" "w,w")
 378           (parallel [(match_operand:SI 2 "immediate_operand" "i,i")])))]
 379   "TARGET_NEON"
 380 {
 381   if (BYTES_BIG_ENDIAN)
 382     {
 383       int elt = INTVAL (operands[2]);
 384       elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
 385       operands[2] = GEN_INT (elt);
 386     }
 387
 388   if (which_alternative == 0)
 389     return "vst1.<V_sz_elem>\t{%P1[%c2]}, %A0";
 390   else
 391     return "vmov.<V_uf_sclr>\t%0, %P1[%c2]";
 392 }
 393   [(set_attr "type" "neon_store1_one_lane<q>,neon_to_gp<q>")]
 394 )
 395
 396 (define_insn "vec_extract<mode>"
 397   [(set (match_operand:<V_elem> 0 "nonimmediate_operand" "=Um,r")
 398         (vec_select:<V_elem>
 399           (match_operand:VQ 1 "s_register_operand" "w,w")
 400           (parallel [(match_operand:SI 2 "immediate_operand" "i,i")])))]
 401   "TARGET_NEON"
 402 {
 403   int half_elts = GET_MODE_NUNITS (<MODE>mode) / 2;
 404   int elt = INTVAL (operands[2]) % half_elts;
 405   int hi = (INTVAL (operands[2]) / half_elts) * 2;
 406   int regno = REGNO (operands[1]);
 407
 408   if (BYTES_BIG_ENDIAN)
 409     elt = half_elts - 1 - elt;
 410
 411   operands[1] = gen_rtx_REG (<V_HALF>mode, regno + hi);
 412   operands[2] = GEN_INT (elt);
 413
 414   if (which_alternative == 0)
 415     return "vst1.<V_sz_elem>\t{%P1[%c2]}, %A0";
 416   else
 417     return "vmov.<V_uf_sclr>\t%0, %P1[%c2]";
 418 }
 419   [(set_attr "type" "neon_store1_one_lane<q>,neon_to_gp<q>")]
 420 )
 421
 422 (define_insn "vec_extractv2di"
 423   [(set (match_operand:DI 0 "nonimmediate_operand" "=Um,r")
 424         (vec_select:DI
 425           (match_operand:V2DI 1 "s_register_operand" "w,w")
 426           (parallel [(match_operand:SI 2 "immediate_operand" "i,i")])))]
 427   "TARGET_NEON"
 428 {
 429   int regno = REGNO (operands[1]) + 2 * INTVAL (operands[2]);
 430
 431   operands[1] = gen_rtx_REG (DImode, regno);
 432
 433   if (which_alternative == 0)
 434     return "vst1.64\t{%P1}, %A0  @ v2di";
 435   else
 436     return "vmov\t%Q0, %R0, %P1  @ v2di";
 437 }
 438   [(set_attr "type" "neon_store1_one_lane_q,neon_to_gp_q")]
 439 )
 440
 441 (define_expand "vec_init<mode>"
 442   [(match_operand:VDQ 0 "s_register_operand" "")
 443    (match_operand 1 "" "")]
 444   "TARGET_NEON"
 445 {
 446   neon_expand_vector_init (operands[0], operands[1]);
 447   DONE;
 448 })
 449
 450 ;; Doubleword and quadword arithmetic.
 451
 452 ;; NOTE: some other instructions also support 64-bit integer
 453 ;; element size, which we could potentially use for "long long" operations.
 454
 455 (define_insn "*add<mode>3_neon"
 456   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
 457         (plus:VDQ (match_operand:VDQ 1 "s_register_operand" "w")
 458                   (match_operand:VDQ 2 "s_register_operand" "w")))]
 459   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
 460   "vadd.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 461   [(set (attr "type")
 462       (if_then_else (match_test "<Is_float_mode>")
 463                     (const_string "neon_fp_addsub_s<q>")
 464                     (const_string "neon_add<q>")))]
 465 )
 466
 467 (define_insn "adddi3_neon"
 468   [(set (match_operand:DI 0 "s_register_operand" "=w,?&r,?&r,?w,?&r,?&r,?&r")
 469         (plus:DI (match_operand:DI 1 "s_register_operand" "%w,0,0,w,r,0,r")
 470                  (match_operand:DI 2 "arm_adddi_operand"     "w,r,0,w,r,Dd,Dd")))
 471    (clobber (reg:CC CC_REGNUM))]
 472   "TARGET_NEON"
 473 {
 474   switch (which_alternative)
 475     {
 476     case 0: /* fall through */
 477     case 3: return "vadd.i64\t%P0, %P1, %P2";
 478     case 1: return "#";
 479     case 2: return "#";
 480     case 4: return "#";
 481     case 5: return "#";
 482     case 6: return "#";
 483     default: gcc_unreachable ();
 484     }
 485 }
 486   [(set_attr "type" "neon_add,multiple,multiple,neon_add,\
 487                      multiple,multiple,multiple")
 488    (set_attr "conds" "*,clob,clob,*,clob,clob,clob")
 489    (set_attr "length" "*,8,8,*,8,8,8")
 490    (set_attr "arch" "neon_for_64bits,*,*,avoid_neon_for_64bits,*,*,*")]
 491 )
 492
 493 (define_insn "*sub<mode>3_neon"
 494   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
 495         (minus:VDQ (match_operand:VDQ 1 "s_register_operand" "w")
 496                    (match_operand:VDQ 2 "s_register_operand" "w")))]
 497   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
 498   "vsub.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 499   [(set (attr "type")
 500       (if_then_else (match_test "<Is_float_mode>")
 501                     (const_string "neon_fp_addsub_s<q>")
 502                     (const_string "neon_sub<q>")))]
 503 )
 504
 505 (define_insn "subdi3_neon"
 506   [(set (match_operand:DI 0 "s_register_operand" "=w,?&r,?&r,?&r,?w")
 507         (minus:DI (match_operand:DI 1 "s_register_operand" "w,0,r,0,w")
 508                   (match_operand:DI 2 "s_register_operand" "w,r,0,0,w")))
 509    (clobber (reg:CC CC_REGNUM))]
 510   "TARGET_NEON"
 511 {
 512   switch (which_alternative)
 513     {
 514     case 0: /* fall through */
 515     case 4: return "vsub.i64\t%P0, %P1, %P2";
 516     case 1: /* fall through */
 517     case 2: /* fall through */
 518     case 3: return  "subs\\t%Q0, %Q1, %Q2\;sbc\\t%R0, %R1, %R2";
 519     default: gcc_unreachable ();
 520     }
 521 }
 522   [(set_attr "type" "neon_sub,multiple,multiple,multiple,neon_sub")
 523    (set_attr "conds" "*,clob,clob,clob,*")
 524    (set_attr "length" "*,8,8,8,*")
 525    (set_attr "arch" "neon_for_64bits,*,*,*,avoid_neon_for_64bits")]
 526 )
 527
 528 (define_insn "*mul<mode>3_neon"
 529   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
 530         (mult:VDQW (match_operand:VDQW 1 "s_register_operand" "w")
 531                    (match_operand:VDQW 2 "s_register_operand" "w")))]
 532   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
 533   "vmul.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 534   [(set (attr "type")
 535       (if_then_else (match_test "<Is_float_mode>")
 536                     (const_string "neon_fp_mul_s<q>")
 537                     (const_string "neon_mul_<V_elem_ch><q>")))]
 538 )
 539
 540 (define_insn "mul<mode>3add<mode>_neon"
 541   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
 542         (plus:VDQW (mult:VDQW (match_operand:VDQW 2 "s_register_operand" "w")
 543                             (match_operand:VDQW 3 "s_register_operand" "w"))
 544                   (match_operand:VDQW 1 "s_register_operand" "0")))]
 545   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
 546   "vmla.<V_if_elem>\t%<V_reg>0, %<V_reg>2, %<V_reg>3"
 547   [(set (attr "type")
 548       (if_then_else (match_test "<Is_float_mode>")
 549                     (const_string "neon_fp_mla_s<q>")
 550                     (const_string "neon_mla_<V_elem_ch><q>")))]
 551 )
 552
 553 (define_insn "mul<mode>3neg<mode>add<mode>_neon"
 554   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
 555         (minus:VDQW (match_operand:VDQW 1 "s_register_operand" "0")
 556                     (mult:VDQW (match_operand:VDQW 2 "s_register_operand" "w")
 557                                (match_operand:VDQW 3 "s_register_operand" "w"))))]
 558   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
 559   "vmls.<V_if_elem>\t%<V_reg>0, %<V_reg>2, %<V_reg>3"
 560   [(set (attr "type")
 561       (if_then_else (match_test "<Is_float_mode>")
 562                     (const_string "neon_fp_mla_s<q>")
 563                     (const_string "neon_mla_<V_elem_ch><q>")))]
 564 )
 565
 566 ;; Fused multiply-accumulate
 567 ;; We define each insn twice here:
 568 ;;    1: with flag_unsafe_math_optimizations for the widening multiply phase
 569 ;;       to be able to use when converting to FMA.
 570 ;;    2: without flag_unsafe_math_optimizations for the intrinsics to use.
 571 (define_insn "fma<VCVTF:mode>4"
 572   [(set (match_operand:VCVTF 0 "register_operand" "=w")
 573         (fma:VCVTF (match_operand:VCVTF 1 "register_operand" "w")
 574                  (match_operand:VCVTF 2 "register_operand" "w")
 575                  (match_operand:VCVTF 3 "register_operand" "0")))]
 576   "TARGET_NEON && TARGET_FMA && flag_unsafe_math_optimizations"
 577   "vfma%?.<V_if_elem>\\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 578   [(set_attr "type" "neon_fp_mla_s<q>")]
 579 )
 580
 581 (define_insn "fma<VCVTF:mode>4_intrinsic"
 582   [(set (match_operand:VCVTF 0 "register_operand" "=w")
 583         (fma:VCVTF (match_operand:VCVTF 1 "register_operand" "w")
 584                  (match_operand:VCVTF 2 "register_operand" "w")
 585                  (match_operand:VCVTF 3 "register_operand" "0")))]
 586   "TARGET_NEON && TARGET_FMA"
 587   "vfma%?.<V_if_elem>\\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 588   [(set_attr "type" "neon_fp_mla_s<q>")]
 589 )
 590
 591 (define_insn "*fmsub<VCVTF:mode>4"
 592   [(set (match_operand:VCVTF 0 "register_operand" "=w")
 593         (fma:VCVTF (neg:VCVTF (match_operand:VCVTF 1 "register_operand" "w"))
 594                    (match_operand:VCVTF 2 "register_operand" "w")
 595                    (match_operand:VCVTF 3 "register_operand" "0")))]
 596   "TARGET_NEON && TARGET_FMA && flag_unsafe_math_optimizations"
 597   "vfms%?.<V_if_elem>\\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 598   [(set_attr "type" "neon_fp_mla_s<q>")]
 599 )
 600
 601 (define_insn "fmsub<VCVTF:mode>4_intrinsic"
 602   [(set (match_operand:VCVTF 0 "register_operand" "=w")
 603         (fma:VCVTF (neg:VCVTF (match_operand:VCVTF 1 "register_operand" "w"))
 604                    (match_operand:VCVTF 2 "register_operand" "w")
 605                    (match_operand:VCVTF 3 "register_operand" "0")))]
 606   "TARGET_NEON && TARGET_FMA"
 607   "vfms%?.<V_if_elem>\\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 608   [(set_attr "type" "neon_fp_mla_s<q>")]
 609 )
 610
 611 (define_insn "neon_vrint<NEON_VRINT:nvrint_variant><VCVTF:mode>"
 612   [(set (match_operand:VCVTF 0 "s_register_operand" "=w")
 613         (unspec:VCVTF [(match_operand:VCVTF 1
 614                          "s_register_operand" "w")]
 615                 NEON_VRINT))]
 616   "TARGET_NEON && TARGET_FPU_ARMV8"
 617   "vrint<nvrint_variant>%?.f32\\t%<V_reg>0, %<V_reg>1"
 618   [(set_attr "type" "neon_fp_round_<V_elem_ch><q>")]
 619 )
 620
 621 (define_insn "ior<mode>3"
 622   [(set (match_operand:VDQ 0 "s_register_operand" "=w,w")
 623         (ior:VDQ (match_operand:VDQ 1 "s_register_operand" "w,0")
 624                  (match_operand:VDQ 2 "neon_logic_op2" "w,Dl")))]
 625   "TARGET_NEON"
 626 {
 627   switch (which_alternative)
 628     {
 629     case 0: return "vorr\t%<V_reg>0, %<V_reg>1, %<V_reg>2";
 630     case 1: return neon_output_logic_immediate ("vorr", &operands[2],
 631                      <MODE>mode, 0, VALID_NEON_QREG_MODE (<MODE>mode));
 632     default: gcc_unreachable ();
 633     }
 634 }
 635   [(set_attr "type" "neon_logic<q>")]
 636 )
 637
 638 ;; The concrete forms of the Neon immediate-logic instructions are vbic and
 639 ;; vorr. We support the pseudo-instruction vand instead, because that
 640 ;; corresponds to the canonical form the middle-end expects to use for
 641 ;; immediate bitwise-ANDs.
 642
 643 (define_insn "and<mode>3"
 644   [(set (match_operand:VDQ 0 "s_register_operand" "=w,w")
 645         (and:VDQ (match_operand:VDQ 1 "s_register_operand" "w,0")
 646                  (match_operand:VDQ 2 "neon_inv_logic_op2" "w,DL")))]
 647   "TARGET_NEON"
 648 {
 649   switch (which_alternative)
 650     {
 651     case 0: return "vand\t%<V_reg>0, %<V_reg>1, %<V_reg>2";
 652     case 1: return neon_output_logic_immediate ("vand", &operands[2],
 653                      <MODE>mode, 1, VALID_NEON_QREG_MODE (<MODE>mode));
 654     default: gcc_unreachable ();
 655     }
 656 }
 657   [(set_attr "type" "neon_logic<q>")]
 658 )
 659
 660 (define_insn "orn<mode>3_neon"
 661   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
 662         (ior:VDQ (not:VDQ (match_operand:VDQ 2 "s_register_operand" "w"))
 663                  (match_operand:VDQ 1 "s_register_operand" "w")))]
 664   "TARGET_NEON"
 665   "vorn\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 666   [(set_attr "type" "neon_logic<q>")]
 667 )
 668
 669 ;; TODO: investigate whether we should disable
 670 ;; this and bicdi3_neon for the A8 in line with the other
 671 ;; changes above.
 672 (define_insn_and_split "orndi3_neon"
 673   [(set (match_operand:DI 0 "s_register_operand" "=w,?&r,?&r,?&r")
 674         (ior:DI (not:DI (match_operand:DI 2 "s_register_operand" "w,0,0,r"))
 675                 (match_operand:DI 1 "s_register_operand" "w,r,r,0")))]
 676   "TARGET_NEON"
 677   "@
 678    vorn\t%P0, %P1, %P2
 679    #
 680    #
 681    #"
 682   "reload_completed &&
 683    (TARGET_NEON && !(IS_VFP_REGNUM (REGNO (operands[0]))))"
 684   [(set (match_dup 0) (ior:SI (not:SI (match_dup 2)) (match_dup 1)))
 685    (set (match_dup 3) (ior:SI (not:SI (match_dup 4)) (match_dup 5)))]
 686   "
 687   {
 688     if (TARGET_THUMB2)
 689       {
 690         operands[3] = gen_highpart (SImode, operands[0]);
 691         operands[0] = gen_lowpart (SImode, operands[0]);
 692         operands[4] = gen_highpart (SImode, operands[2]);
 693         operands[2] = gen_lowpart (SImode, operands[2]);
 694         operands[5] = gen_highpart (SImode, operands[1]);
 695         operands[1] = gen_lowpart (SImode, operands[1]);
 696       }
 697     else
 698       {
 699         emit_insn (gen_one_cmpldi2 (operands[0], operands[2]));
 700         emit_insn (gen_iordi3 (operands[0], operands[1], operands[0]));
 701         DONE;
 702       }
 703   }"
 704   [(set_attr "type" "neon_logic,multiple,multiple,multiple")
 705    (set_attr "length" "*,16,8,8")
 706    (set_attr "arch" "any,a,t2,t2")]
 707 )
 708
 709 (define_insn "bic<mode>3_neon"
 710   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
 711         (and:VDQ (not:VDQ (match_operand:VDQ 2 "s_register_operand" "w"))
 712                  (match_operand:VDQ 1 "s_register_operand" "w")))]
 713   "TARGET_NEON"
 714   "vbic\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 715   [(set_attr "type" "neon_logic<q>")]
 716 )
 717
 718 ;; Compare to *anddi_notdi_di.
 719 (define_insn "bicdi3_neon"
 720   [(set (match_operand:DI 0 "s_register_operand" "=w,?=&r,?&r")
 721         (and:DI (not:DI (match_operand:DI 2 "s_register_operand" "w,r,0"))
 722                 (match_operand:DI 1 "s_register_operand" "w,0,r")))]
 723   "TARGET_NEON"
 724   "@
 725    vbic\t%P0, %P1, %P2
 726    #
 727    #"
 728   [(set_attr "type" "neon_logic,multiple,multiple")
 729    (set_attr "length" "*,8,8")]
 730 )
 731
 732 (define_insn "xor<mode>3"
 733   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
 734         (xor:VDQ (match_operand:VDQ 1 "s_register_operand" "w")
 735                  (match_operand:VDQ 2 "s_register_operand" "w")))]
 736   "TARGET_NEON"
 737   "veor\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 738   [(set_attr "type" "neon_logic<q>")]
 739 )
 740
 741 (define_insn "one_cmpl<mode>2"
 742   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
 743         (not:VDQ (match_operand:VDQ 1 "s_register_operand" "w")))]
 744   "TARGET_NEON"
 745   "vmvn\t%<V_reg>0, %<V_reg>1"
 746   [(set_attr "type" "neon_move<q>")]
 747 )
 748
 749 (define_insn "abs<mode>2"
 750   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
 751         (abs:VDQW (match_operand:VDQW 1 "s_register_operand" "w")))]
 752   "TARGET_NEON"
 753   "vabs.<V_s_elem>\t%<V_reg>0, %<V_reg>1"
 754   [(set (attr "type")
 755       (if_then_else (match_test "<Is_float_mode>")
 756                     (const_string "neon_fp_abs_s<q>")
 757                     (const_string "neon_abs<q>")))]
 758 )
 759
 760 (define_insn "neg<mode>2"
 761   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
 762         (neg:VDQW (match_operand:VDQW 1 "s_register_operand" "w")))]
 763   "TARGET_NEON"
 764   "vneg.<V_s_elem>\t%<V_reg>0, %<V_reg>1"
 765   [(set (attr "type")
 766       (if_then_else (match_test "<Is_float_mode>")
 767                     (const_string "neon_fp_neg_s<q>")
 768                     (const_string "neon_neg<q>")))]
 769 )
 770
 771 (define_insn "negdi2_neon"
 772   [(set (match_operand:DI 0 "s_register_operand"         "=&w, w,r,&r")
 773         (neg:DI (match_operand:DI 1 "s_register_operand" "  w, w,0, r")))
 774    (clobber (match_scratch:DI 2                          "= X,&w,X, X"))
 775    (clobber (reg:CC CC_REGNUM))]
 776   "TARGET_NEON"
 777   "#"
 778   [(set_attr "length" "8")
 779    (set_attr "type" "multiple")]
 780 )
 781
 782 ; Split negdi2_neon for vfp registers
 783 (define_split
 784   [(set (match_operand:DI 0 "s_register_operand" "")
 785         (neg:DI (match_operand:DI 1 "s_register_operand" "")))
 786    (clobber (match_scratch:DI 2 ""))
 787    (clobber (reg:CC CC_REGNUM))]
 788   "TARGET_NEON && reload_completed && IS_VFP_REGNUM (REGNO (operands[0]))"
 789   [(set (match_dup 2) (const_int 0))
 790    (parallel [(set (match_dup 0) (minus:DI (match_dup 2) (match_dup 1)))
 791               (clobber (reg:CC CC_REGNUM))])]
 792   {
 793     if (!REG_P (operands[2]))
 794       operands[2] = operands[0];
 795   }
 796 )
 797
 798 ; Split negdi2_neon for core registers
 799 (define_split
 800   [(set (match_operand:DI 0 "s_register_operand" "")
 801         (neg:DI (match_operand:DI 1 "s_register_operand" "")))
 802    (clobber (match_scratch:DI 2 ""))
 803    (clobber (reg:CC CC_REGNUM))]
 804   "TARGET_32BIT && reload_completed
 805    && arm_general_register_operand (operands[0], DImode)"
 806   [(parallel [(set (match_dup 0) (neg:DI (match_dup 1)))
 807               (clobber (reg:CC CC_REGNUM))])]
 808   ""
 809 )
 810
 811 (define_insn "*umin<mode>3_neon"
 812   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
 813         (umin:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w")
 814                     (match_operand:VDQIW 2 "s_register_operand" "w")))]
 815   "TARGET_NEON"
 816   "vmin.<V_u_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 817   [(set_attr "type" "neon_minmax<q>")]
 818 )
 819
 820 (define_insn "*umax<mode>3_neon"
 821   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
 822         (umax:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w")
 823                     (match_operand:VDQIW 2 "s_register_operand" "w")))]
 824   "TARGET_NEON"
 825   "vmax.<V_u_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 826   [(set_attr "type" "neon_minmax<q>")]
 827 )
 828
 829 (define_insn "*smin<mode>3_neon"
 830   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
 831         (smin:VDQW (match_operand:VDQW 1 "s_register_operand" "w")
 832                    (match_operand:VDQW 2 "s_register_operand" "w")))]
 833   "TARGET_NEON"
 834   "vmin.<V_s_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 835   [(set (attr "type")
 836       (if_then_else (match_test "<Is_float_mode>")
 837                     (const_string "neon_fp_minmax_s<q>")
 838                     (const_string "neon_minmax<q>")))]
 839 )
 840
 841 (define_insn "*smax<mode>3_neon"
 842   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
 843         (smax:VDQW (match_operand:VDQW 1 "s_register_operand" "w")
 844                    (match_operand:VDQW 2 "s_register_operand" "w")))]
 845   "TARGET_NEON"
 846   "vmax.<V_s_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 847   [(set (attr "type")
 848       (if_then_else (match_test "<Is_float_mode>")
 849                     (const_string "neon_fp_minmax_s<q>")
 850                     (const_string "neon_minmax<q>")))]
 851 )
 852
 853 ; TODO: V2DI shifts are current disabled because there are bugs in the
 854 ; generic vectorizer code.  It ends up creating a V2DI constructor with
 855 ; SImode elements.
 856
 857 (define_insn "vashl<mode>3"
 858   [(set (match_operand:VDQIW 0 "s_register_operand" "=w,w")
 859         (ashift:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w,w")
 860                       (match_operand:VDQIW 2 "imm_lshift_or_reg_neon" "w,Dn")))]
 861   "TARGET_NEON"
 862   {
 863     switch (which_alternative)
 864       {
 865         case 0: return "vshl.<V_s_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2";
 866         case 1: return neon_output_shift_immediate ("vshl", 'i', &operands[2],
 867                                                     <MODE>mode,
 868                                                     VALID_NEON_QREG_MODE (<MODE>mode),
 869                                                     true);
 870         default: gcc_unreachable ();
 871       }
 872   }
 873   [(set_attr "type" "neon_shift_reg<q>, neon_shift_imm<q>")]
 874 )
 875
 876 (define_insn "vashr<mode>3_imm"
 877   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
 878         (ashiftrt:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w")
 879                         (match_operand:VDQIW 2 "imm_for_neon_rshift_operand" "Dn")))]
 880   "TARGET_NEON"
 881   {
 882     return neon_output_shift_immediate ("vshr", 's', &operands[2],
 883                                         <MODE>mode, VALID_NEON_QREG_MODE (<MODE>mode),
 884                                         false);
 885   }
 886   [(set_attr "type" "neon_shift_imm<q>")]
 887 )
 888
 889 (define_insn "vlshr<mode>3_imm"
 890   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
 891         (lshiftrt:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w")
 892                         (match_operand:VDQIW 2 "imm_for_neon_rshift_operand" "Dn")))]
 893   "TARGET_NEON"
 894   {
 895     return neon_output_shift_immediate ("vshr", 'u', &operands[2],
 896                                         <MODE>mode, VALID_NEON_QREG_MODE (<MODE>mode),
 897                                         false);
 898   }
 899   [(set_attr "type" "neon_shift_imm<q>")]
 900 )
 901
 902 ; Used for implementing logical shift-right, which is a left-shift by a negative
 903 ; amount, with signed operands. This is essentially the same as ashl<mode>3
 904 ; above, but using an unspec in case GCC tries anything tricky with negative
 905 ; shift amounts.
 906
 907 (define_insn "ashl<mode>3_signed"
 908   [(set (match_operand:VDQI 0 "s_register_operand" "=w")
 909         (unspec:VDQI [(match_operand:VDQI 1 "s_register_operand" "w")
 910                       (match_operand:VDQI 2 "s_register_operand" "w")]
 911                      UNSPEC_ASHIFT_SIGNED))]
 912   "TARGET_NEON"
 913   "vshl.<V_s_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 914   [(set_attr "type" "neon_shift_reg<q>")]
 915 )
 916
 917 ; Used for implementing logical shift-right, which is a left-shift by a negative
 918 ; amount, with unsigned operands.
 919
 920 (define_insn "ashl<mode>3_unsigned"
 921   [(set (match_operand:VDQI 0 "s_register_operand" "=w")
 922         (unspec:VDQI [(match_operand:VDQI 1 "s_register_operand" "w")
 923                       (match_operand:VDQI 2 "s_register_operand" "w")]
 924                      UNSPEC_ASHIFT_UNSIGNED))]
 925   "TARGET_NEON"
 926   "vshl.<V_u_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
 927   [(set_attr "type" "neon_shift_reg<q>")]
 928 )
 929
 930 (define_expand "vashr<mode>3"
 931   [(set (match_operand:VDQIW 0 "s_register_operand" "")
 932         (ashiftrt:VDQIW (match_operand:VDQIW 1 "s_register_operand" "")
 933                         (match_operand:VDQIW 2 "imm_rshift_or_reg_neon" "")))]
 934   "TARGET_NEON"
 935 {
 936   if (s_register_operand (operands[2], <MODE>mode))
 937     {
 938       rtx neg = gen_reg_rtx (<MODE>mode);
 939       emit_insn (gen_neg<mode>2 (neg, operands[2]));
 940       emit_insn (gen_ashl<mode>3_signed (operands[0], operands[1], neg));
 941     }
 942   else
 943     emit_insn (gen_vashr<mode>3_imm (operands[0], operands[1], operands[2]));
 944   DONE;
 945 })
 946
 947 (define_expand "vlshr<mode>3"
 948   [(set (match_operand:VDQIW 0 "s_register_operand" "")
 949         (lshiftrt:VDQIW (match_operand:VDQIW 1 "s_register_operand" "")
 950                         (match_operand:VDQIW 2 "imm_rshift_or_reg_neon" "")))]
 951   "TARGET_NEON"
 952 {
 953   if (s_register_operand (operands[2], <MODE>mode))
 954     {
 955       rtx neg = gen_reg_rtx (<MODE>mode);
 956       emit_insn (gen_neg<mode>2 (neg, operands[2]));
 957       emit_insn (gen_ashl<mode>3_unsigned (operands[0], operands[1], neg));
 958     }
 959   else
 960     emit_insn (gen_vlshr<mode>3_imm (operands[0], operands[1], operands[2]));
 961   DONE;
 962 })
 963
 964 ;; 64-bit shifts
 965
 966 ;; This pattern loads a 32-bit shift count into a 64-bit NEON register,
 967 ;; leaving the upper half uninitalized.  This is OK since the shift
 968 ;; instruction only looks at the low 8 bits anyway.  To avoid confusing
 969 ;; data flow analysis however, we pretend the full register is set
 970 ;; using an unspec.
 971 (define_insn "neon_load_count"
 972   [(set (match_operand:DI 0 "s_register_operand" "=w,w")
 973         (unspec:DI [(match_operand:SI 1 "nonimmediate_operand" "Um,r")]
 974                    UNSPEC_LOAD_COUNT))]
 975   "TARGET_NEON"
 976   "@
 977    vld1.32\t{%P0[0]}, %A1
 978    vmov.32\t%P0[0], %1"
 979   [(set_attr "type" "neon_load1_1reg,neon_from_gp")]
 980 )
 981
 982 (define_insn "ashldi3_neon_noclobber"
 983   [(set (match_operand:DI 0 "s_register_operand"            "=w,w")
 984         (ashift:DI (match_operand:DI 1 "s_register_operand" " w,w")
 985                    (match_operand:DI 2 "reg_or_int_operand" " i,w")))]
 986   "TARGET_NEON && reload_completed
 987    && (!CONST_INT_P (operands[2])
 988        || (INTVAL (operands[2]) >= 0 && INTVAL (operands[2]) < 64))"
 989   "@
 990    vshl.u64\t%P0, %P1, %2
 991    vshl.u64\t%P0, %P1, %P2"
 992   [(set_attr "type" "neon_shift_imm, neon_shift_reg")]
 993 )
 994
 995 (define_insn_and_split "ashldi3_neon"
 996   [(set (match_operand:DI 0 "s_register_operand"            "= w, w,?&r,?r, ?w,w")
 997         (ashift:DI (match_operand:DI 1 "s_register_operand" " 0w, w, 0r, r, 0w,w")
 998                    (match_operand:SI 2 "general_operand"    "rUm, i,  r, i,rUm,i")))
 999    (clobber (match_scratch:SI 3                             "= X, X,?&r, X,  X,X"))
1000    (clobber (match_scratch:SI 4                             "= X, X,?&r, X,  X,X"))
1001    (clobber (match_scratch:DI 5                             "=&w, X,  X, X, &w,X"))
1002    (clobber (reg:CC_C CC_REGNUM))]
1003   "TARGET_NEON"
1004   "#"
1005   "TARGET_NEON && reload_completed"
1006   [(const_int 0)]
1007   "
1008   {
1009     if (IS_VFP_REGNUM (REGNO (operands[0])))
1010       {
1011         if (CONST_INT_P (operands[2]))
1012           {
1013             if (INTVAL (operands[2]) < 1)
1014               {
1015                 emit_insn (gen_movdi (operands[0], operands[1]));
1016                 DONE;
1017               }
1018             else if (INTVAL (operands[2]) > 63)
1019               operands[2] = gen_rtx_CONST_INT (VOIDmode, 63);
1020           }
1021         else
1022           {
1023             emit_insn (gen_neon_load_count (operands[5], operands[2]));
1024             operands[2] = operands[5];
1025           }
1026
1027         /* Ditch the unnecessary clobbers.  */
1028         emit_insn (gen_ashldi3_neon_noclobber (operands[0], operands[1],
1029                                                operands[2]));
1030       }
1031     else
1032       {
1033         if (CONST_INT_P (operands[2]) && INTVAL (operands[2]) == 1)
1034           /* This clobbers CC.  */
1035           emit_insn (gen_arm_ashldi3_1bit (operands[0], operands[1]));
1036         else
1037           arm_emit_coreregs_64bit_shift (ASHIFT, operands[0], operands[1],
1038                                          operands[2], operands[3], operands[4]);
1039       }
1040     DONE;
1041   }"
1042   [(set_attr "arch" "neon_for_64bits,neon_for_64bits,*,*,avoid_neon_for_64bits,avoid_neon_for_64bits")
1043    (set_attr "opt" "*,*,speed,speed,*,*")
1044    (set_attr "type" "multiple")]
1045 )
1046
1047 ; The shift amount needs to be negated for right-shifts
1048 (define_insn "signed_shift_di3_neon"
1049   [(set (match_operand:DI 0 "s_register_operand"             "=w")
1050         (unspec:DI [(match_operand:DI 1 "s_register_operand" " w")
1051                     (match_operand:DI 2 "s_register_operand" " w")]
1052                    UNSPEC_ASHIFT_SIGNED))]
1053   "TARGET_NEON && reload_completed"
1054   "vshl.s64\t%P0, %P1, %P2"
1055   [(set_attr "type" "neon_shift_reg")]
1056 )
1057
1058 ; The shift amount needs to be negated for right-shifts
1059 (define_insn "unsigned_shift_di3_neon"
1060   [(set (match_operand:DI 0 "s_register_operand"             "=w")
1061         (unspec:DI [(match_operand:DI 1 "s_register_operand" " w")
1062                     (match_operand:DI 2 "s_register_operand" " w")]
1063                    UNSPEC_ASHIFT_UNSIGNED))]
1064   "TARGET_NEON && reload_completed"
1065   "vshl.u64\t%P0, %P1, %P2"
1066   [(set_attr "type" "neon_shift_reg")]
1067 )
1068
1069 (define_insn "ashrdi3_neon_imm_noclobber"
1070   [(set (match_operand:DI 0 "s_register_operand"              "=w")
1071         (ashiftrt:DI (match_operand:DI 1 "s_register_operand" " w")
1072                      (match_operand:DI 2 "const_int_operand"  " i")))]
1073   "TARGET_NEON && reload_completed
1074    && INTVAL (operands[2]) > 0 && INTVAL (operands[2]) <= 64"
1075   "vshr.s64\t%P0, %P1, %2"
1076   [(set_attr "type" "neon_shift_imm")]
1077 )
1078
1079 (define_insn "lshrdi3_neon_imm_noclobber"
1080   [(set (match_operand:DI 0 "s_register_operand"              "=w")
1081         (lshiftrt:DI (match_operand:DI 1 "s_register_operand" " w")
1082                      (match_operand:DI 2 "const_int_operand"  " i")))]
1083   "TARGET_NEON && reload_completed
1084    && INTVAL (operands[2]) > 0 && INTVAL (operands[2]) <= 64"
1085   "vshr.u64\t%P0, %P1, %2"
1086   [(set_attr "type" "neon_shift_imm")]
1087 )
1088
1089 ;; ashrdi3_neon
1090 ;; lshrdi3_neon
1091 (define_insn_and_split "<shift>di3_neon"
1092   [(set (match_operand:DI 0 "s_register_operand"             "= w, w,?&r,?r,?w,?w")
1093         (rshifts:DI (match_operand:DI 1 "s_register_operand" " 0w, w, 0r, r,0w, w")
1094                     (match_operand:SI 2 "reg_or_int_operand" "  r, i,  r, i, r, i")))
1095    (clobber (match_scratch:SI 3                              "=2r, X, &r, X,2r, X"))
1096    (clobber (match_scratch:SI 4                              "= X, X, &r, X, X, X"))
1097    (clobber (match_scratch:DI 5                              "=&w, X,  X, X,&w, X"))
1098    (clobber (reg:CC CC_REGNUM))]
1099   "TARGET_NEON"
1100   "#"
1101   "TARGET_NEON && reload_completed"
1102   [(const_int 0)]
1103   "
1104   {
1105     if (IS_VFP_REGNUM (REGNO (operands[0])))
1106       {
1107         if (CONST_INT_P (operands[2]))
1108           {
1109             if (INTVAL (operands[2]) < 1)
1110               {
1111                 emit_insn (gen_movdi (operands[0], operands[1]));
1112                 DONE;
1113               }
1114             else if (INTVAL (operands[2]) > 64)
1115               operands[2] = gen_rtx_CONST_INT (VOIDmode, 64);
1116
1117             /* Ditch the unnecessary clobbers.  */
1118             emit_insn (gen_<shift>di3_neon_imm_noclobber (operands[0],
1119                                                           operands[1],
1120                                                           operands[2]));
1121           }
1122         else
1123           {
1124             /* We must use a negative left-shift.  */
1125             emit_insn (gen_negsi2 (operands[3], operands[2]));
1126             emit_insn (gen_neon_load_count (operands[5], operands[3]));
1127             emit_insn (gen_<shifttype>_shift_di3_neon (operands[0], operands[1],
1128                                                        operands[5]));
1129           }
1130       }
1131     else
1132       {
1133         if (CONST_INT_P (operands[2]) && INTVAL (operands[2]) == 1)
1134           /* This clobbers CC.  */
1135           emit_insn (gen_arm_<shift>di3_1bit (operands[0], operands[1]));
1136         else
1137           /* This clobbers CC (ASHIFTRT by register only).  */
1138           arm_emit_coreregs_64bit_shift (<CODE>, operands[0], operands[1],
1139                                          operands[2], operands[3], operands[4]);
1140       }
1141
1142     DONE;
1143   }"
1144   [(set_attr "arch" "neon_for_64bits,neon_for_64bits,*,*,avoid_neon_for_64bits,avoid_neon_for_64bits")
1145    (set_attr "opt" "*,*,speed,speed,*,*")
1146    (set_attr "type" "multiple")]
1147 )
1148
1149 ;; Widening operations
1150
1151 (define_insn "widen_ssum<mode>3"
1152   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
1153         (plus:<V_widen> (sign_extend:<V_widen>
1154                           (match_operand:VW 1 "s_register_operand" "%w"))
1155                         (match_operand:<V_widen> 2 "s_register_operand" "w")))]
1156   "TARGET_NEON"
1157   "vaddw.<V_s_elem>\t%q0, %q2, %P1"
1158   [(set_attr "type" "neon_add_widen")]
1159 )
1160
1161 (define_insn "widen_usum<mode>3"
1162   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
1163         (plus:<V_widen> (zero_extend:<V_widen>
1164                           (match_operand:VW 1 "s_register_operand" "%w"))
1165                         (match_operand:<V_widen> 2 "s_register_operand" "w")))]
1166   "TARGET_NEON"
1167   "vaddw.<V_u_elem>\t%q0, %q2, %P1"
1168   [(set_attr "type" "neon_add_widen")]
1169 )
1170
1171 ;; VEXT can be used to synthesize coarse whole-vector shifts with 8-bit
1172 ;; shift-count granularity. That's good enough for the middle-end's current
1173 ;; needs.
1174
1175 ;; Note that it's not safe to perform such an operation in big-endian mode,
1176 ;; due to element-ordering issues.
1177
1178 (define_expand "vec_shr_<mode>"
1179   [(match_operand:VDQ 0 "s_register_operand" "")
1180    (match_operand:VDQ 1 "s_register_operand" "")
1181    (match_operand:SI 2 "const_multiple_of_8_operand" "")]
1182   "TARGET_NEON && !BYTES_BIG_ENDIAN"
1183 {
1184   rtx zero_reg;
1185   HOST_WIDE_INT num_bits = INTVAL (operands[2]);
1186   const int width = GET_MODE_BITSIZE (<MODE>mode);
1187   const enum machine_mode bvecmode = (width == 128) ? V16QImode : V8QImode;
1188   rtx (*gen_ext) (rtx, rtx, rtx, rtx) =
1189     (width == 128) ? gen_neon_vextv16qi : gen_neon_vextv8qi;
1190
1191   if (num_bits == width)
1192     {
1193       emit_move_insn (operands[0], operands[1]);
1194       DONE;
1195     }
1196
1197   zero_reg = force_reg (bvecmode, CONST0_RTX (bvecmode));
1198   operands[0] = gen_lowpart (bvecmode, operands[0]);
1199   operands[1] = gen_lowpart (bvecmode, operands[1]);
1200
1201   emit_insn (gen_ext (operands[0], operands[1], zero_reg,
1202                       GEN_INT (num_bits / BITS_PER_UNIT)));
1203   DONE;
1204 })
1205
1206 (define_expand "vec_shl_<mode>"
1207   [(match_operand:VDQ 0 "s_register_operand" "")
1208    (match_operand:VDQ 1 "s_register_operand" "")
1209    (match_operand:SI 2 "const_multiple_of_8_operand" "")]
1210   "TARGET_NEON && !BYTES_BIG_ENDIAN"
1211 {
1212   rtx zero_reg;
1213   HOST_WIDE_INT num_bits = INTVAL (operands[2]);
1214   const int width = GET_MODE_BITSIZE (<MODE>mode);
1215   const enum machine_mode bvecmode = (width == 128) ? V16QImode : V8QImode;
1216   rtx (*gen_ext) (rtx, rtx, rtx, rtx) =
1217     (width == 128) ? gen_neon_vextv16qi : gen_neon_vextv8qi;
1218
1219   if (num_bits == 0)
1220     {
1221       emit_move_insn (operands[0], CONST0_RTX (<MODE>mode));
1222       DONE;
1223     }
1224
1225   num_bits = width - num_bits;
1226
1227   zero_reg = force_reg (bvecmode, CONST0_RTX (bvecmode));
1228   operands[0] = gen_lowpart (bvecmode, operands[0]);
1229   operands[1] = gen_lowpart (bvecmode, operands[1]);
1230
1231   emit_insn (gen_ext (operands[0], zero_reg, operands[1],
1232                       GEN_INT (num_bits / BITS_PER_UNIT)));
1233   DONE;
1234 })
1235
1236 ;; Helpers for quad-word reduction operations
1237
1238 ; Add (or smin, smax...) the low N/2 elements of the N-element vector
1239 ; operand[1] to the high N/2 elements of same. Put the result in operand[0], an
1240 ; N/2-element vector.
1241
1242 (define_insn "quad_halves_<code>v4si"
1243   [(set (match_operand:V2SI 0 "s_register_operand" "=w")
1244         (vqh_ops:V2SI
1245           (vec_select:V2SI (match_operand:V4SI 1 "s_register_operand" "w")
1246                            (parallel [(const_int 0) (const_int 1)]))
1247           (vec_select:V2SI (match_dup 1)
1248                            (parallel [(const_int 2) (const_int 3)]))))]
1249   "TARGET_NEON"
1250   "<VQH_mnem>.<VQH_sign>32\t%P0, %e1, %f1"
1251   [(set_attr "vqh_mnem" "<VQH_mnem>")
1252    (set_attr "type" "neon_reduc_<VQH_type>_q")]
1253 )
1254
1255 (define_insn "quad_halves_<code>v4sf"
1256   [(set (match_operand:V2SF 0 "s_register_operand" "=w")
1257         (vqhs_ops:V2SF
1258           (vec_select:V2SF (match_operand:V4SF 1 "s_register_operand" "w")
1259                            (parallel [(const_int 0) (const_int 1)]))
1260           (vec_select:V2SF (match_dup 1)
1261                            (parallel [(const_int 2) (const_int 3)]))))]
1262   "TARGET_NEON && flag_unsafe_math_optimizations"
1263   "<VQH_mnem>.f32\t%P0, %e1, %f1"
1264   [(set_attr "vqh_mnem" "<VQH_mnem>")
1265    (set_attr "type" "neon_fp_reduc_<VQH_type>_s_q")]
1266 )
1267
1268 (define_insn "quad_halves_<code>v8hi"
1269   [(set (match_operand:V4HI 0 "s_register_operand" "+w")
1270         (vqh_ops:V4HI
1271           (vec_select:V4HI (match_operand:V8HI 1 "s_register_operand" "w")
1272                            (parallel [(const_int 0) (const_int 1)
1273                                       (const_int 2) (const_int 3)]))
1274           (vec_select:V4HI (match_dup 1)
1275                            (parallel [(const_int 4) (const_int 5)
1276                                       (const_int 6) (const_int 7)]))))]
1277   "TARGET_NEON"
1278   "<VQH_mnem>.<VQH_sign>16\t%P0, %e1, %f1"
1279   [(set_attr "vqh_mnem" "<VQH_mnem>")
1280    (set_attr "type" "neon_reduc_<VQH_type>_q")]
1281 )
1282
1283 (define_insn "quad_halves_<code>v16qi"
1284   [(set (match_operand:V8QI 0 "s_register_operand" "+w")
1285         (vqh_ops:V8QI
1286           (vec_select:V8QI (match_operand:V16QI 1 "s_register_operand" "w")
1287                            (parallel [(const_int 0) (const_int 1)
1288                                       (const_int 2) (const_int 3)
1289                                       (const_int 4) (const_int 5)
1290                                       (const_int 6) (const_int 7)]))
1291           (vec_select:V8QI (match_dup 1)
1292                            (parallel [(const_int 8) (const_int 9)
1293                                       (const_int 10) (const_int 11)
1294                                       (const_int 12) (const_int 13)
1295                                       (const_int 14) (const_int 15)]))))]
1296   "TARGET_NEON"
1297   "<VQH_mnem>.<VQH_sign>8\t%P0, %e1, %f1"
1298   [(set_attr "vqh_mnem" "<VQH_mnem>")
1299    (set_attr "type" "neon_reduc_<VQH_type>_q")]
1300 )
1301
1302 (define_expand "move_hi_quad_<mode>"
1303  [(match_operand:ANY128 0 "s_register_operand" "")
1304   (match_operand:<V_HALF> 1 "s_register_operand" "")]
1305  "TARGET_NEON"
1306 {
1307   emit_move_insn (simplify_gen_subreg (<V_HALF>mode, operands[0], <MODE>mode,
1308                                        GET_MODE_SIZE (<V_HALF>mode)),
1309                   operands[1]);
1310   DONE;
1311 })
1312
1313 (define_expand "move_lo_quad_<mode>"
1314  [(match_operand:ANY128 0 "s_register_operand" "")
1315   (match_operand:<V_HALF> 1 "s_register_operand" "")]
1316  "TARGET_NEON"
1317 {
1318   emit_move_insn (simplify_gen_subreg (<V_HALF>mode, operands[0],
1319                                        <MODE>mode, 0),
1320                   operands[1]);
1321   DONE;
1322 })
1323
1324 ;; Reduction operations
1325
1326 (define_expand "reduc_splus_<mode>"
1327   [(match_operand:VD 0 "s_register_operand" "")
1328    (match_operand:VD 1 "s_register_operand" "")]
1329   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
1330 {
1331   neon_pairwise_reduce (operands[0], operands[1], <MODE>mode,
1332                         &gen_neon_vpadd_internal<mode>);
1333   DONE;
1334 })
1335
1336 (define_expand "reduc_splus_<mode>"
1337   [(match_operand:VQ 0 "s_register_operand" "")
1338    (match_operand:VQ 1 "s_register_operand" "")]
1339   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)
1340    && !BYTES_BIG_ENDIAN"
1341 {
1342   rtx step1 = gen_reg_rtx (<V_HALF>mode);
1343   rtx res_d = gen_reg_rtx (<V_HALF>mode);
1344
1345   emit_insn (gen_quad_halves_plus<mode> (step1, operands[1]));
1346   emit_insn (gen_reduc_splus_<V_half> (res_d, step1));
1347   emit_insn (gen_move_lo_quad_<mode> (operands[0], res_d));
1348
1349   DONE;
1350 })
1351
1352 (define_insn "reduc_splus_v2di"
1353   [(set (match_operand:V2DI 0 "s_register_operand" "=w")
1354         (unspec:V2DI [(match_operand:V2DI 1 "s_register_operand" "w")]
1355                      UNSPEC_VPADD))]
1356   "TARGET_NEON && !BYTES_BIG_ENDIAN"
1357   "vadd.i64\t%e0, %e1, %f1"
1358   [(set_attr "type" "neon_add_q")]
1359 )
1360
1361 ;; NEON does not distinguish between signed and unsigned addition except on
1362 ;; widening operations.
1363 (define_expand "reduc_uplus_<mode>"
1364   [(match_operand:VDQI 0 "s_register_operand" "")
1365    (match_operand:VDQI 1 "s_register_operand" "")]
1366   "TARGET_NEON && (<Is_d_reg> || !BYTES_BIG_ENDIAN)"
1367 {
1368   emit_insn (gen_reduc_splus_<mode> (operands[0], operands[1]));
1369   DONE;
1370 })
1371
1372 (define_expand "reduc_smin_<mode>"
1373   [(match_operand:VD 0 "s_register_operand" "")
1374    (match_operand:VD 1 "s_register_operand" "")]
1375   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
1376 {
1377   neon_pairwise_reduce (operands[0], operands[1], <MODE>mode,
1378                         &gen_neon_vpsmin<mode>);
1379   DONE;
1380 })
1381
1382 (define_expand "reduc_smin_<mode>"
1383   [(match_operand:VQ 0 "s_register_operand" "")
1384    (match_operand:VQ 1 "s_register_operand" "")]
1385   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)
1386    && !BYTES_BIG_ENDIAN"
1387 {
1388   rtx step1 = gen_reg_rtx (<V_HALF>mode);
1389   rtx res_d = gen_reg_rtx (<V_HALF>mode);
1390
1391   emit_insn (gen_quad_halves_smin<mode> (step1, operands[1]));
1392   emit_insn (gen_reduc_smin_<V_half> (res_d, step1));
1393   emit_insn (gen_move_lo_quad_<mode> (operands[0], res_d));
1394
1395   DONE;
1396 })
1397
1398 (define_expand "reduc_smax_<mode>"
1399   [(match_operand:VD 0 "s_register_operand" "")
1400    (match_operand:VD 1 "s_register_operand" "")]
1401   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
1402 {
1403   neon_pairwise_reduce (operands[0], operands[1], <MODE>mode,
1404                         &gen_neon_vpsmax<mode>);
1405   DONE;
1406 })
1407
1408 (define_expand "reduc_smax_<mode>"
1409   [(match_operand:VQ 0 "s_register_operand" "")
1410    (match_operand:VQ 1 "s_register_operand" "")]
1411   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)
1412    && !BYTES_BIG_ENDIAN"
1413 {
1414   rtx step1 = gen_reg_rtx (<V_HALF>mode);
1415   rtx res_d = gen_reg_rtx (<V_HALF>mode);
1416
1417   emit_insn (gen_quad_halves_smax<mode> (step1, operands[1]));
1418   emit_insn (gen_reduc_smax_<V_half> (res_d, step1));
1419   emit_insn (gen_move_lo_quad_<mode> (operands[0], res_d));
1420
1421   DONE;
1422 })
1423
1424 (define_expand "reduc_umin_<mode>"
1425   [(match_operand:VDI 0 "s_register_operand" "")
1426    (match_operand:VDI 1 "s_register_operand" "")]
1427   "TARGET_NEON"
1428 {
1429   neon_pairwise_reduce (operands[0], operands[1], <MODE>mode,
1430                         &gen_neon_vpumin<mode>);
1431   DONE;
1432 })
1433
1434 (define_expand "reduc_umin_<mode>"
1435   [(match_operand:VQI 0 "s_register_operand" "")
1436    (match_operand:VQI 1 "s_register_operand" "")]
1437   "TARGET_NEON && !BYTES_BIG_ENDIAN"
1438 {
1439   rtx step1 = gen_reg_rtx (<V_HALF>mode);
1440   rtx res_d = gen_reg_rtx (<V_HALF>mode);
1441
1442   emit_insn (gen_quad_halves_umin<mode> (step1, operands[1]));
1443   emit_insn (gen_reduc_umin_<V_half> (res_d, step1));
1444   emit_insn (gen_move_lo_quad_<mode> (operands[0], res_d));
1445
1446   DONE;
1447 })
1448
1449 (define_expand "reduc_umax_<mode>"
1450   [(match_operand:VDI 0 "s_register_operand" "")
1451    (match_operand:VDI 1 "s_register_operand" "")]
1452   "TARGET_NEON"
1453 {
1454   neon_pairwise_reduce (operands[0], operands[1], <MODE>mode,
1455                         &gen_neon_vpumax<mode>);
1456   DONE;
1457 })
1458
1459 (define_expand "reduc_umax_<mode>"
1460   [(match_operand:VQI 0 "s_register_operand" "")
1461    (match_operand:VQI 1 "s_register_operand" "")]
1462   "TARGET_NEON && !BYTES_BIG_ENDIAN"
1463 {
1464   rtx step1 = gen_reg_rtx (<V_HALF>mode);
1465   rtx res_d = gen_reg_rtx (<V_HALF>mode);
1466
1467   emit_insn (gen_quad_halves_umax<mode> (step1, operands[1]));
1468   emit_insn (gen_reduc_umax_<V_half> (res_d, step1));
1469   emit_insn (gen_move_lo_quad_<mode> (operands[0], res_d));
1470
1471   DONE;
1472 })
1473
1474 (define_insn "neon_vpadd_internal<mode>"
1475   [(set (match_operand:VD 0 "s_register_operand" "=w")
1476         (unspec:VD [(match_operand:VD 1 "s_register_operand" "w")
1477                     (match_operand:VD 2 "s_register_operand" "w")]
1478                    UNSPEC_VPADD))]
1479   "TARGET_NEON"
1480   "vpadd.<V_if_elem>\t%P0, %P1, %P2"
1481   ;; Assume this schedules like vadd.
1482   [(set (attr "type")
1483       (if_then_else (match_test "<Is_float_mode>")
1484                     (const_string "neon_fp_reduc_add_s<q>")
1485                     (const_string "neon_reduc_add<q>")))]
1486 )
1487
1488 (define_insn "neon_vpsmin<mode>"
1489   [(set (match_operand:VD 0 "s_register_operand" "=w")
1490         (unspec:VD [(match_operand:VD 1 "s_register_operand" "w")
1491                     (match_operand:VD 2 "s_register_operand" "w")]
1492                    UNSPEC_VPSMIN))]
1493   "TARGET_NEON"
1494   "vpmin.<V_s_elem>\t%P0, %P1, %P2"
1495   [(set (attr "type")
1496       (if_then_else (match_test "<Is_float_mode>")
1497                     (const_string "neon_fp_reduc_minmax_s<q>")
1498                     (const_string "neon_reduc_minmax<q>")))]
1499 )
1500
1501 (define_insn "neon_vpsmax<mode>"
1502   [(set (match_operand:VD 0 "s_register_operand" "=w")
1503         (unspec:VD [(match_operand:VD 1 "s_register_operand" "w")
1504                     (match_operand:VD 2 "s_register_operand" "w")]
1505                    UNSPEC_VPSMAX))]
1506   "TARGET_NEON"
1507   "vpmax.<V_s_elem>\t%P0, %P1, %P2"
1508   [(set (attr "type")
1509       (if_then_else (match_test "<Is_float_mode>")
1510                     (const_string "neon_fp_reduc_minmax_s<q>")
1511                     (const_string "neon_reduc_minmax<q>")))]
1512 )
1513
1514 (define_insn "neon_vpumin<mode>"
1515   [(set (match_operand:VDI 0 "s_register_operand" "=w")
1516         (unspec:VDI [(match_operand:VDI 1 "s_register_operand" "w")
1517                      (match_operand:VDI 2 "s_register_operand" "w")]
1518                    UNSPEC_VPUMIN))]
1519   "TARGET_NEON"
1520   "vpmin.<V_u_elem>\t%P0, %P1, %P2"
1521   [(set_attr "type" "neon_reduc_minmax<q>")]
1522 )
1523
1524 (define_insn "neon_vpumax<mode>"
1525   [(set (match_operand:VDI 0 "s_register_operand" "=w")
1526         (unspec:VDI [(match_operand:VDI 1 "s_register_operand" "w")
1527                      (match_operand:VDI 2 "s_register_operand" "w")]
1528                    UNSPEC_VPUMAX))]
1529   "TARGET_NEON"
1530   "vpmax.<V_u_elem>\t%P0, %P1, %P2"
1531   [(set_attr "type" "neon_reduc_minmax<q>")]
1532 )
1533
1534 ;; Saturating arithmetic
1535
1536 ; NOTE: Neon supports many more saturating variants of instructions than the
1537 ; following, but these are all GCC currently understands.
1538 ; FIXME: Actually, GCC doesn't know how to create saturating add/sub by itself
1539 ; yet either, although these patterns may be used by intrinsics when they're
1540 ; added.
1541
1542 (define_insn "*ss_add<mode>_neon"
1543   [(set (match_operand:VD 0 "s_register_operand" "=w")
1544        (ss_plus:VD (match_operand:VD 1 "s_register_operand" "w")
1545                    (match_operand:VD 2 "s_register_operand" "w")))]
1546   "TARGET_NEON"
1547   "vqadd.<V_s_elem>\t%P0, %P1, %P2"
1548   [(set_attr "type" "neon_qadd<q>")]
1549 )
1550
1551 (define_insn "*us_add<mode>_neon"
1552   [(set (match_operand:VD 0 "s_register_operand" "=w")
1553        (us_plus:VD (match_operand:VD 1 "s_register_operand" "w")
1554                    (match_operand:VD 2 "s_register_operand" "w")))]
1555   "TARGET_NEON"
1556   "vqadd.<V_u_elem>\t%P0, %P1, %P2"
1557   [(set_attr "type" "neon_qadd<q>")]
1558 )
1559
1560 (define_insn "*ss_sub<mode>_neon"
1561   [(set (match_operand:VD 0 "s_register_operand" "=w")
1562        (ss_minus:VD (match_operand:VD 1 "s_register_operand" "w")
1563                     (match_operand:VD 2 "s_register_operand" "w")))]
1564   "TARGET_NEON"
1565   "vqsub.<V_s_elem>\t%P0, %P1, %P2"
1566   [(set_attr "type" "neon_qsub<q>")]
1567 )
1568
1569 (define_insn "*us_sub<mode>_neon"
1570   [(set (match_operand:VD 0 "s_register_operand" "=w")
1571        (us_minus:VD (match_operand:VD 1 "s_register_operand" "w")
1572                     (match_operand:VD 2 "s_register_operand" "w")))]
1573   "TARGET_NEON"
1574   "vqsub.<V_u_elem>\t%P0, %P1, %P2"
1575   [(set_attr "type" "neon_qsub<q>")]
1576 )
1577
1578 ;; Conditional instructions.  These are comparisons with conditional moves for
1579 ;; vectors.  They perform the assignment:
1580 ;;
1581 ;;     Vop0 = (Vop4 <op3> Vop5) ? Vop1 : Vop2;
1582 ;;
1583 ;; where op3 is <, <=, ==, !=, >= or >.  Operations are performed
1584 ;; element-wise.
1585
1586 (define_expand "vcond<mode><mode>"
1587   [(set (match_operand:VDQW 0 "s_register_operand" "")
1588         (if_then_else:VDQW
1589           (match_operator 3 "comparison_operator"
1590             [(match_operand:VDQW 4 "s_register_operand" "")
1591              (match_operand:VDQW 5 "nonmemory_operand" "")])
1592           (match_operand:VDQW 1 "s_register_operand" "")
1593           (match_operand:VDQW 2 "s_register_operand" "")))]
1594   "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
1595 {
1596   HOST_WIDE_INT magic_word = (<MODE>mode == V2SFmode || <MODE>mode == V4SFmode)
1597                              ? 3 : 1;
1598   rtx magic_rtx = GEN_INT (magic_word);
1599   int inverse = 0;
1600   int use_zero_form = 0;
1601   int swap_bsl_operands = 0;
1602   rtx mask = gen_reg_rtx (<V_cmp_result>mode);
1603   rtx tmp = gen_reg_rtx (<V_cmp_result>mode);
1604
1605   rtx (*base_comparison) (rtx, rtx, rtx, rtx);
1606   rtx (*complimentary_comparison) (rtx, rtx, rtx, rtx);
1607
1608   switch (GET_CODE (operands[3]))
1609     {
1610     case GE:
1611     case GT:
1612     case LE:
1613     case LT:
1614     case EQ:
1615       if (operands[5] == CONST0_RTX (<MODE>mode))
1616         {
1617           use_zero_form = 1;
1618           break;
1619         }
1620       /* Fall through.  */
1621     default:
1622       if (!REG_P (operands[5]))
1623         operands[5] = force_reg (<MODE>mode, operands[5]);
1624     }
1625
1626   switch (GET_CODE (operands[3]))
1627     {
1628     case LT:
1629     case UNLT:
1630       inverse = 1;
1631       /* Fall through.  */
1632     case GE:
1633     case UNGE:
1634     case ORDERED:
1635     case UNORDERED:
1636       base_comparison = gen_neon_vcge<mode>;
1637       complimentary_comparison = gen_neon_vcgt<mode>;
1638       break;
1639     case LE:
1640     case UNLE:
1641       inverse = 1;
1642       /* Fall through.  */
1643     case GT:
1644     case UNGT:
1645       base_comparison = gen_neon_vcgt<mode>;
1646       complimentary_comparison = gen_neon_vcge<mode>;
1647       break;
1648     case EQ:
1649     case NE:
1650     case UNEQ:
1651       base_comparison = gen_neon_vceq<mode>;
1652       complimentary_comparison = gen_neon_vceq<mode>;
1653       break;
1654     default:
1655       gcc_unreachable ();
1656     }
1657
1658   switch (GET_CODE (operands[3]))
1659     {
1660     case LT:
1661     case LE:
1662     case GT:
1663     case GE:
1664     case EQ:
1665       /* The easy case.  Here we emit one of vcge, vcgt or vceq.
1666          As a LT b <=> b GE a && a LE b <=> b GT a.  Our transformations are:
1667          a GE b -> a GE b
1668          a GT b -> a GT b
1669          a LE b -> b GE a
1670          a LT b -> b GT a
1671          a EQ b -> a EQ b
1672          Note that there also exist direct comparison against 0 forms,
1673          so catch those as a special case.  */
1674       if (use_zero_form)
1675         {
1676           inverse = 0;
1677           switch (GET_CODE (operands[3]))
1678             {
1679             case LT:
1680               base_comparison = gen_neon_vclt<mode>;
1681               break;
1682             case LE:
1683               base_comparison = gen_neon_vcle<mode>;
1684               break;
1685             default:
1686               /* Do nothing, other zero form cases already have the correct
1687                  base_comparison.  */
1688               break;
1689             }
1690         }
1691
1692       if (!inverse)
1693         emit_insn (base_comparison (mask, operands[4], operands[5], magic_rtx));
1694       else
1695         emit_insn (complimentary_comparison (mask, operands[5], operands[4], magic_rtx));
1696       break;
1697     case UNLT:
1698     case UNLE:
1699     case UNGT:
1700     case UNGE:
1701     case NE:
1702       /* Vector compare returns false for lanes which are unordered, so if we use
1703          the inverse of the comparison we actually want to emit, then
1704          swap the operands to BSL, we will end up with the correct result.
1705          Note that a NE NaN and NaN NE b are true for all a, b.
1706
1707          Our transformations are:
1708          a GE b -> !(b GT a)
1709          a GT b -> !(b GE a)
1710          a LE b -> !(a GT b)
1711          a LT b -> !(a GE b)
1712          a NE b -> !(a EQ b)  */
1713
1714       if (inverse)
1715         emit_insn (base_comparison (mask, operands[4], operands[5], magic_rtx));
1716       else
1717         emit_insn (complimentary_comparison (mask, operands[5], operands[4], magic_rtx));
1718
1719       swap_bsl_operands = 1;
1720       break;
1721     case UNEQ:
1722       /* We check (a > b ||  b > a).  combining these comparisons give us
1723          true iff !(a != b && a ORDERED b), swapping the operands to BSL
1724          will then give us (a == b ||  a UNORDERED b) as intended.  */
1725
1726       emit_insn (gen_neon_vcgt<mode> (mask, operands[4], operands[5], magic_rtx));
1727       emit_insn (gen_neon_vcgt<mode> (tmp, operands[5], operands[4], magic_rtx));
1728       emit_insn (gen_ior<v_cmp_result>3 (mask, mask, tmp));
1729       swap_bsl_operands = 1;
1730       break;
1731     case UNORDERED:
1732        /* Operands are ORDERED iff (a > b || b >= a).
1733          Swapping the operands to BSL will give the UNORDERED case.  */
1734      swap_bsl_operands = 1;
1735      /* Fall through.  */
1736     case ORDERED:
1737       emit_insn (gen_neon_vcgt<mode> (tmp, operands[4], operands[5], magic_rtx));
1738       emit_insn (gen_neon_vcge<mode> (mask, operands[5], operands[4], magic_rtx));
1739       emit_insn (gen_ior<v_cmp_result>3 (mask, mask, tmp));
1740       break;
1741     default:
1742       gcc_unreachable ();
1743     }
1744
1745   if (swap_bsl_operands)
1746     emit_insn (gen_neon_vbsl<mode> (operands[0], mask, operands[2],
1747                                     operands[1]));
1748   else
1749     emit_insn (gen_neon_vbsl<mode> (operands[0], mask, operands[1],
1750                                     operands[2]));
1751   DONE;
1752 })
1753
1754 (define_expand "vcondu<mode><mode>"
1755   [(set (match_operand:VDQIW 0 "s_register_operand" "")
1756         (if_then_else:VDQIW
1757           (match_operator 3 "arm_comparison_operator"
1758             [(match_operand:VDQIW 4 "s_register_operand" "")
1759              (match_operand:VDQIW 5 "s_register_operand" "")])
1760           (match_operand:VDQIW 1 "s_register_operand" "")
1761           (match_operand:VDQIW 2 "s_register_operand" "")))]
1762   "TARGET_NEON"
1763 {
1764   rtx mask;
1765   int inverse = 0, immediate_zero = 0;
1766
1767   mask = gen_reg_rtx (<V_cmp_result>mode);
1768
1769   if (operands[5] == CONST0_RTX (<MODE>mode))
1770     immediate_zero = 1;
1771   else if (!REG_P (operands[5]))
1772     operands[5] = force_reg (<MODE>mode, operands[5]);
1773
1774   switch (GET_CODE (operands[3]))
1775     {
1776     case GEU:
1777       emit_insn (gen_neon_vcge<mode> (mask, operands[4], operands[5],
1778                                       const0_rtx));
1779       break;
1780
1781     case GTU:
1782       emit_insn (gen_neon_vcgt<mode> (mask, operands[4], operands[5],
1783                                       const0_rtx));
1784       break;
1785
1786     case EQ:
1787       emit_insn (gen_neon_vceq<mode> (mask, operands[4], operands[5],
1788                                       const0_rtx));
1789       break;
1790
1791     case LEU:
1792       if (immediate_zero)
1793         emit_insn (gen_neon_vcle<mode> (mask, operands[4], operands[5],
1794                                         const0_rtx));
1795       else
1796         emit_insn (gen_neon_vcge<mode> (mask, operands[5], operands[4],
1797                                         const0_rtx));
1798       break;
1799
1800     case LTU:
1801       if (immediate_zero)
1802         emit_insn (gen_neon_vclt<mode> (mask, operands[4], operands[5],
1803                                         const0_rtx));
1804       else
1805         emit_insn (gen_neon_vcgt<mode> (mask, operands[5], operands[4],
1806                                         const0_rtx));
1807       break;
1808
1809     case NE:
1810       emit_insn (gen_neon_vceq<mode> (mask, operands[4], operands[5],
1811                                       const0_rtx));
1812       inverse = 1;
1813       break;
1814
1815     default:
1816       gcc_unreachable ();
1817     }
1818
1819   if (inverse)
1820     emit_insn (gen_neon_vbsl<mode> (operands[0], mask, operands[2],
1821                                     operands[1]));
1822   else
1823     emit_insn (gen_neon_vbsl<mode> (operands[0], mask, operands[1],
1824                                     operands[2]));
1825
1826   DONE;
1827 })
1828
1829 ;; Patterns for builtins.
1830
1831 ; good for plain vadd, vaddq.
1832
1833 (define_expand "neon_vadd<mode>"
1834   [(match_operand:VDQX 0 "s_register_operand" "=w")
1835    (match_operand:VDQX 1 "s_register_operand" "w")
1836    (match_operand:VDQX 2 "s_register_operand" "w")
1837    (match_operand:SI 3 "immediate_operand" "i")]
1838   "TARGET_NEON"
1839 {
1840   if (!<Is_float_mode> || flag_unsafe_math_optimizations)
1841     emit_insn (gen_add<mode>3 (operands[0], operands[1], operands[2]));
1842   else
1843     emit_insn (gen_neon_vadd<mode>_unspec (operands[0], operands[1],
1844                                            operands[2]));
1845   DONE;
1846 })
1847
1848 ; Note that NEON operations don't support the full IEEE 754 standard: in
1849 ; particular, denormal values are flushed to zero.  This means that GCC cannot
1850 ; use those instructions for autovectorization, etc. unless
1851 ; -funsafe-math-optimizations is in effect (in which case flush-to-zero
1852 ; behaviour is permissible).  Intrinsic operations (provided by the arm_neon.h
1853 ; header) must work in either case: if -funsafe-math-optimizations is given,
1854 ; intrinsics expand to "canonical" RTL where possible, otherwise intrinsics
1855 ; expand to unspecs (which may potentially limit the extent to which they might
1856 ; be optimized by generic code).
1857
1858 ; Used for intrinsics when flag_unsafe_math_optimizations is false.
1859
1860 (define_insn "neon_vadd<mode>_unspec"
1861   [(set (match_operand:VDQX 0 "s_register_operand" "=w")
1862         (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand" "w")
1863                       (match_operand:VDQX 2 "s_register_operand" "w")]
1864                      UNSPEC_VADD))]
1865   "TARGET_NEON"
1866   "vadd.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1867   [(set (attr "type")
1868       (if_then_else (match_test "<Is_float_mode>")
1869                     (const_string "neon_fp_addsub_s<q>")
1870                     (const_string "neon_add<q>")))]
1871 )
1872
1873 ; operand 3 represents in bits:
1874 ;  bit 0: signed (vs unsigned).
1875 ;  bit 1: rounding (vs none).
1876
1877 (define_insn "neon_vaddl<mode>"
1878   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
1879         (unspec:<V_widen> [(match_operand:VDI 1 "s_register_operand" "w")
1880                            (match_operand:VDI 2 "s_register_operand" "w")
1881                            (match_operand:SI 3 "immediate_operand" "i")]
1882                           UNSPEC_VADDL))]
1883   "TARGET_NEON"
1884   "vaddl.%T3%#<V_sz_elem>\t%q0, %P1, %P2"
1885   [(set_attr "type" "neon_add_long")]
1886 )
1887
1888 (define_insn "neon_vaddw<mode>"
1889   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
1890         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "w")
1891                            (match_operand:VDI 2 "s_register_operand" "w")
1892                            (match_operand:SI 3 "immediate_operand" "i")]
1893                           UNSPEC_VADDW))]
1894   "TARGET_NEON"
1895   "vaddw.%T3%#<V_sz_elem>\t%q0, %q1, %P2"
1896   [(set_attr "type" "neon_add_widen")]
1897 )
1898
1899 ; vhadd and vrhadd.
1900
1901 (define_insn "neon_vhadd<mode>"
1902   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
1903         (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
1904                        (match_operand:VDQIW 2 "s_register_operand" "w")
1905                        (match_operand:SI 3 "immediate_operand" "i")]
1906                       UNSPEC_VHADD))]
1907   "TARGET_NEON"
1908   "v%O3hadd.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1909   [(set_attr "type" "neon_add_halve_q")]
1910 )
1911
1912 (define_insn "neon_vqadd<mode>"
1913   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
1914         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
1915                        (match_operand:VDQIX 2 "s_register_operand" "w")
1916                        (match_operand:SI 3 "immediate_operand" "i")]
1917                      UNSPEC_VQADD))]
1918   "TARGET_NEON"
1919   "vqadd.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1920   [(set_attr "type" "neon_qadd<q>")]
1921 )
1922
1923 (define_insn "neon_vaddhn<mode>"
1924   [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
1925         (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
1926                             (match_operand:VN 2 "s_register_operand" "w")
1927                             (match_operand:SI 3 "immediate_operand" "i")]
1928                            UNSPEC_VADDHN))]
1929   "TARGET_NEON"
1930   "v%O3addhn.<V_if_elem>\t%P0, %q1, %q2"
1931   [(set_attr "type" "neon_add_halve_narrow_q")]
1932 )
1933
1934 ;; We cannot replace this unspec with mul<mode>3 because of the odd
1935 ;; polynomial multiplication case that can specified by operand 3.
1936 (define_insn "neon_vmul<mode>"
1937   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
1938         (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "w")
1939                       (match_operand:VDQW 2 "s_register_operand" "w")
1940                       (match_operand:SI 3 "immediate_operand" "i")]
1941                      UNSPEC_VMUL))]
1942   "TARGET_NEON"
1943   "vmul.%F3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1944   [(set (attr "type")
1945       (if_then_else (match_test "<Is_float_mode>")
1946                     (const_string "neon_fp_mul_s<q>")
1947                     (const_string "neon_mul_<V_elem_ch><q>")))]
1948 )
1949
1950 (define_expand "neon_vmla<mode>"
1951   [(match_operand:VDQW 0 "s_register_operand" "=w")
1952    (match_operand:VDQW 1 "s_register_operand" "0")
1953    (match_operand:VDQW 2 "s_register_operand" "w")
1954    (match_operand:VDQW 3 "s_register_operand" "w")
1955    (match_operand:SI 4 "immediate_operand" "i")]
1956   "TARGET_NEON"
1957 {
1958   if (!<Is_float_mode> || flag_unsafe_math_optimizations)
1959     emit_insn (gen_mul<mode>3add<mode>_neon (operands[0], operands[1],
1960                                              operands[2], operands[3]));
1961   else
1962     emit_insn (gen_neon_vmla<mode>_unspec (operands[0], operands[1],
1963                                            operands[2], operands[3]));
1964   DONE;
1965 })
1966
1967 (define_expand "neon_vfma<VCVTF:mode>"
1968   [(match_operand:VCVTF 0 "s_register_operand")
1969    (match_operand:VCVTF 1 "s_register_operand")
1970    (match_operand:VCVTF 2 "s_register_operand")
1971    (match_operand:VCVTF 3 "s_register_operand")
1972    (match_operand:SI 4 "immediate_operand")]
1973   "TARGET_NEON && TARGET_FMA"
1974 {
1975   emit_insn (gen_fma<mode>4_intrinsic (operands[0], operands[2], operands[3],
1976                                        operands[1]));
1977   DONE;
1978 })
1979
1980 (define_expand "neon_vfms<VCVTF:mode>"
1981   [(match_operand:VCVTF 0 "s_register_operand")
1982    (match_operand:VCVTF 1 "s_register_operand")
1983    (match_operand:VCVTF 2 "s_register_operand")
1984    (match_operand:VCVTF 3 "s_register_operand")
1985    (match_operand:SI 4 "immediate_operand")]
1986   "TARGET_NEON && TARGET_FMA"
1987 {
1988   emit_insn (gen_fmsub<mode>4_intrinsic (operands[0], operands[2], operands[3],
1989                                          operands[1]));
1990   DONE;
1991 })
1992
1993 ; Used for intrinsics when flag_unsafe_math_optimizations is false.
1994
1995 (define_insn "neon_vmla<mode>_unspec"
1996   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
1997         (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "0")
1998                       (match_operand:VDQW 2 "s_register_operand" "w")
1999                       (match_operand:VDQW 3 "s_register_operand" "w")]
2000                     UNSPEC_VMLA))]
2001   "TARGET_NEON"
2002   "vmla.<V_if_elem>\t%<V_reg>0, %<V_reg>2, %<V_reg>3"
2003   [(set (attr "type")
2004       (if_then_else (match_test "<Is_float_mode>")
2005                     (const_string "neon_fp_mla_s<q>")
2006                     (const_string "neon_mla_<V_elem_ch><q>")))]
2007 )
2008
2009 (define_insn "neon_vmlal<mode>"
2010   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2011         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
2012                            (match_operand:VW 2 "s_register_operand" "w")
2013                            (match_operand:VW 3 "s_register_operand" "w")
2014                            (match_operand:SI 4 "immediate_operand" "i")]
2015                           UNSPEC_VMLAL))]
2016   "TARGET_NEON"
2017   "vmlal.%T4%#<V_sz_elem>\t%q0, %P2, %P3"
2018   [(set_attr "type" "neon_mla_<V_elem_ch>_long")]
2019 )
2020
2021 (define_expand "neon_vmls<mode>"
2022   [(match_operand:VDQW 0 "s_register_operand" "=w")
2023    (match_operand:VDQW 1 "s_register_operand" "0")
2024    (match_operand:VDQW 2 "s_register_operand" "w")
2025    (match_operand:VDQW 3 "s_register_operand" "w")
2026    (match_operand:SI 4 "immediate_operand" "i")]
2027   "TARGET_NEON"
2028 {
2029   if (!<Is_float_mode> || flag_unsafe_math_optimizations)
2030     emit_insn (gen_mul<mode>3neg<mode>add<mode>_neon (operands[0],
2031                  operands[1], operands[2], operands[3]));
2032   else
2033     emit_insn (gen_neon_vmls<mode>_unspec (operands[0], operands[1],
2034                                            operands[2], operands[3]));
2035   DONE;
2036 })
2037
2038 ; Used for intrinsics when flag_unsafe_math_optimizations is false.
2039
2040 (define_insn "neon_vmls<mode>_unspec"
2041   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
2042         (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "0")
2043                       (match_operand:VDQW 2 "s_register_operand" "w")
2044                       (match_operand:VDQW 3 "s_register_operand" "w")]
2045                     UNSPEC_VMLS))]
2046   "TARGET_NEON"
2047   "vmls.<V_if_elem>\t%<V_reg>0, %<V_reg>2, %<V_reg>3"
2048   [(set (attr "type")
2049       (if_then_else (match_test "<Is_float_mode>")
2050                     (const_string "neon_fp_mla_s<q>")
2051                     (const_string "neon_mla_<V_elem_ch><q>")))]
2052 )
2053
2054 (define_insn "neon_vmlsl<mode>"
2055   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2056         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
2057                            (match_operand:VW 2 "s_register_operand" "w")
2058                            (match_operand:VW 3 "s_register_operand" "w")
2059                            (match_operand:SI 4 "immediate_operand" "i")]
2060                           UNSPEC_VMLSL))]
2061   "TARGET_NEON"
2062   "vmlsl.%T4%#<V_sz_elem>\t%q0, %P2, %P3"
2063   [(set_attr "type" "neon_mla_<V_elem_ch>_long")]
2064 )
2065
2066 (define_insn "neon_vqdmulh<mode>"
2067   [(set (match_operand:VMDQI 0 "s_register_operand" "=w")
2068         (unspec:VMDQI [(match_operand:VMDQI 1 "s_register_operand" "w")
2069                        (match_operand:VMDQI 2 "s_register_operand" "w")
2070                        (match_operand:SI 3 "immediate_operand" "i")]
2071                       UNSPEC_VQDMULH))]
2072   "TARGET_NEON"
2073   "vq%O3dmulh.<V_s_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2074   [(set_attr "type" "neon_sat_mul_<V_elem_ch><q>")]
2075 )
2076
2077 (define_insn "neon_vqdmlal<mode>"
2078   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2079         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
2080                            (match_operand:VMDI 2 "s_register_operand" "w")
2081                            (match_operand:VMDI 3 "s_register_operand" "w")
2082                            (match_operand:SI 4 "immediate_operand" "i")]
2083                           UNSPEC_VQDMLAL))]
2084   "TARGET_NEON"
2085   "vqdmlal.<V_s_elem>\t%q0, %P2, %P3"
2086   [(set_attr "type" "neon_sat_mla_<V_elem_ch>_long")]
2087 )
2088
2089 (define_insn "neon_vqdmlsl<mode>"
2090   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2091         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
2092                            (match_operand:VMDI 2 "s_register_operand" "w")
2093                            (match_operand:VMDI 3 "s_register_operand" "w")
2094                            (match_operand:SI 4 "immediate_operand" "i")]
2095                           UNSPEC_VQDMLSL))]
2096   "TARGET_NEON"
2097   "vqdmlsl.<V_s_elem>\t%q0, %P2, %P3"
2098   [(set_attr "type" "neon_sat_mla_<V_elem_ch>_long")]
2099 )
2100
2101 (define_insn "neon_vmull<mode>"
2102   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2103         (unspec:<V_widen> [(match_operand:VW 1 "s_register_operand" "w")
2104                            (match_operand:VW 2 "s_register_operand" "w")
2105                            (match_operand:SI 3 "immediate_operand" "i")]
2106                           UNSPEC_VMULL))]
2107   "TARGET_NEON"
2108   "vmull.%T3%#<V_sz_elem>\t%q0, %P1, %P2"
2109   [(set_attr "type" "neon_mul_<V_elem_ch>_long")]
2110 )
2111
2112 (define_insn "neon_vqdmull<mode>"
2113   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2114         (unspec:<V_widen> [(match_operand:VMDI 1 "s_register_operand" "w")
2115                            (match_operand:VMDI 2 "s_register_operand" "w")
2116                            (match_operand:SI 3 "immediate_operand" "i")]
2117                           UNSPEC_VQDMULL))]
2118   "TARGET_NEON"
2119   "vqdmull.<V_s_elem>\t%q0, %P1, %P2"
2120   [(set_attr "type" "neon_sat_mul_<V_elem_ch>_long")]
2121 )
2122
2123 (define_expand "neon_vsub<mode>"
2124   [(match_operand:VDQX 0 "s_register_operand" "=w")
2125    (match_operand:VDQX 1 "s_register_operand" "w")
2126    (match_operand:VDQX 2 "s_register_operand" "w")
2127    (match_operand:SI 3 "immediate_operand" "i")]
2128   "TARGET_NEON"
2129 {
2130   if (!<Is_float_mode> || flag_unsafe_math_optimizations)
2131     emit_insn (gen_sub<mode>3 (operands[0], operands[1], operands[2]));
2132   else
2133     emit_insn (gen_neon_vsub<mode>_unspec (operands[0], operands[1],
2134                                            operands[2]));
2135   DONE;
2136 })
2137
2138 ; Used for intrinsics when flag_unsafe_math_optimizations is false.
2139
2140 (define_insn "neon_vsub<mode>_unspec"
2141   [(set (match_operand:VDQX 0 "s_register_operand" "=w")
2142         (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand" "w")
2143                       (match_operand:VDQX 2 "s_register_operand" "w")]
2144                      UNSPEC_VSUB))]
2145   "TARGET_NEON"
2146   "vsub.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2147   [(set (attr "type")
2148       (if_then_else (match_test "<Is_float_mode>")
2149                     (const_string "neon_fp_addsub_s<q>")
2150                     (const_string "neon_sub<q>")))]
2151 )
2152
2153 (define_insn "neon_vsubl<mode>"
2154   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2155         (unspec:<V_widen> [(match_operand:VDI 1 "s_register_operand" "w")
2156                            (match_operand:VDI 2 "s_register_operand" "w")
2157                            (match_operand:SI 3 "immediate_operand" "i")]
2158                           UNSPEC_VSUBL))]
2159   "TARGET_NEON"
2160   "vsubl.%T3%#<V_sz_elem>\t%q0, %P1, %P2"
2161   [(set_attr "type" "neon_sub_long")]
2162 )
2163
2164 (define_insn "neon_vsubw<mode>"
2165   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2166         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "w")
2167                            (match_operand:VDI 2 "s_register_operand" "w")
2168                            (match_operand:SI 3 "immediate_operand" "i")]
2169                           UNSPEC_VSUBW))]
2170   "TARGET_NEON"
2171   "vsubw.%T3%#<V_sz_elem>\t%q0, %q1, %P2"
2172   [(set_attr "type" "neon_sub_widen")]
2173 )
2174
2175 (define_insn "neon_vqsub<mode>"
2176   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
2177         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
2178                        (match_operand:VDQIX 2 "s_register_operand" "w")
2179                        (match_operand:SI 3 "immediate_operand" "i")]
2180                       UNSPEC_VQSUB))]
2181   "TARGET_NEON"
2182   "vqsub.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2183   [(set_attr "type" "neon_qsub<q>")]
2184 )
2185
2186 (define_insn "neon_vhsub<mode>"
2187   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2188         (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
2189                        (match_operand:VDQIW 2 "s_register_operand" "w")
2190                        (match_operand:SI 3 "immediate_operand" "i")]
2191                       UNSPEC_VHSUB))]
2192   "TARGET_NEON"
2193   "vhsub.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2194   [(set_attr "type" "neon_sub_halve<q>")]
2195 )
2196
2197 (define_insn "neon_vsubhn<mode>"
2198   [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
2199         (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
2200                             (match_operand:VN 2 "s_register_operand" "w")
2201                             (match_operand:SI 3 "immediate_operand" "i")]
2202                            UNSPEC_VSUBHN))]
2203   "TARGET_NEON"
2204   "v%O3subhn.<V_if_elem>\t%P0, %q1, %q2"
2205   [(set_attr "type" "neon_sub_halve_narrow_q")]
2206 )
2207
2208 (define_insn "neon_vceq<mode>"
2209   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w,w")
2210         (unspec:<V_cmp_result>
2211           [(match_operand:VDQW 1 "s_register_operand" "w,w")
2212            (match_operand:VDQW 2 "reg_or_zero_operand" "w,Dz")
2213            (match_operand:SI 3 "immediate_operand" "i,i")]
2214           UNSPEC_VCEQ))]
2215   "TARGET_NEON"
2216   "@
2217   vceq.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2
2218   vceq.<V_if_elem>\t%<V_reg>0, %<V_reg>1, #0"
2219   [(set (attr "type")
2220       (if_then_else (match_test "<Is_float_mode>")
2221                     (const_string "neon_fp_compare_s<q>")
2222                     (if_then_else (match_operand 2 "zero_operand")
2223                       (const_string "neon_compare_zero<q>")
2224                       (const_string "neon_compare<q>"))))]
2225 )
2226
2227 (define_insn "neon_vcge<mode>"
2228   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w,w")
2229         (unspec:<V_cmp_result>
2230           [(match_operand:VDQW 1 "s_register_operand" "w,w")
2231            (match_operand:VDQW 2 "reg_or_zero_operand" "w,Dz")
2232            (match_operand:SI 3 "immediate_operand" "i,i")]
2233           UNSPEC_VCGE))]
2234   "TARGET_NEON"
2235   "@
2236   vcge.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2
2237   vcge.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, #0"
2238   [(set (attr "type")
2239      (if_then_else (match_test "<Is_float_mode>")
2240                    (const_string "neon_fp_compare_s<q>")
2241                     (if_then_else (match_operand 2 "zero_operand")
2242                       (const_string "neon_compare_zero<q>")
2243                       (const_string "neon_compare<q>"))))]
2244 )
2245
2246 (define_insn "neon_vcgeu<mode>"
2247   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w")
2248         (unspec:<V_cmp_result>
2249           [(match_operand:VDQIW 1 "s_register_operand" "w")
2250            (match_operand:VDQIW 2 "s_register_operand" "w")
2251            (match_operand:SI 3 "immediate_operand" "i")]
2252           UNSPEC_VCGEU))]
2253   "TARGET_NEON"
2254   "vcge.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2255   [(set_attr "type" "neon_compare<q>")]
2256 )
2257
2258 (define_insn "neon_vcgt<mode>"
2259   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w,w")
2260         (unspec:<V_cmp_result>
2261           [(match_operand:VDQW 1 "s_register_operand" "w,w")
2262            (match_operand:VDQW 2 "reg_or_zero_operand" "w,Dz")
2263            (match_operand:SI 3 "immediate_operand" "i,i")]
2264           UNSPEC_VCGT))]
2265   "TARGET_NEON"
2266   "@
2267   vcgt.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2
2268   vcgt.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, #0"
2269   [(set (attr "type")
2270      (if_then_else (match_test "<Is_float_mode>")
2271                    (const_string "neon_fp_compare_s<q>")
2272                     (if_then_else (match_operand 2 "zero_operand")
2273                       (const_string "neon_compare_zero<q>")
2274                       (const_string "neon_compare<q>"))))]
2275 )
2276
2277 (define_insn "neon_vcgtu<mode>"
2278   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w")
2279         (unspec:<V_cmp_result>
2280           [(match_operand:VDQIW 1 "s_register_operand" "w")
2281            (match_operand:VDQIW 2 "s_register_operand" "w")
2282            (match_operand:SI 3 "immediate_operand" "i")]
2283           UNSPEC_VCGTU))]
2284   "TARGET_NEON"
2285   "vcgt.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2286   [(set_attr "type" "neon_compare<q>")]
2287 )
2288
2289 ;; VCLE and VCLT only support comparisons with immediate zero (register
2290 ;; variants are VCGE and VCGT with operands reversed).
2291
2292 (define_insn "neon_vcle<mode>"
2293   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w")
2294         (unspec:<V_cmp_result>
2295           [(match_operand:VDQW 1 "s_register_operand" "w")
2296            (match_operand:VDQW 2 "zero_operand" "Dz")
2297            (match_operand:SI 3 "immediate_operand" "i")]
2298           UNSPEC_VCLE))]
2299   "TARGET_NEON"
2300   "vcle.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, #0"
2301   [(set (attr "type")
2302       (if_then_else (match_test "<Is_float_mode>")
2303                     (const_string "neon_fp_compare_s<q>")
2304                     (if_then_else (match_operand 2 "zero_operand")
2305                       (const_string "neon_compare_zero<q>")
2306                       (const_string "neon_compare<q>"))))]
2307 )
2308
2309 (define_insn "neon_vclt<mode>"
2310   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w")
2311         (unspec:<V_cmp_result>
2312           [(match_operand:VDQW 1 "s_register_operand" "w")
2313            (match_operand:VDQW 2 "zero_operand" "Dz")
2314            (match_operand:SI 3 "immediate_operand" "i")]
2315           UNSPEC_VCLT))]
2316   "TARGET_NEON"
2317   "vclt.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, #0"
2318   [(set (attr "type")
2319       (if_then_else (match_test "<Is_float_mode>")
2320                     (const_string "neon_fp_compare_s<q>")
2321                     (if_then_else (match_operand 2 "zero_operand")
2322                       (const_string "neon_compare_zero<q>")
2323                       (const_string "neon_compare<q>"))))]
2324 )
2325
2326 (define_insn "neon_vcage<mode>"
2327   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w")
2328         (unspec:<V_cmp_result> [(match_operand:VCVTF 1 "s_register_operand" "w")
2329                                 (match_operand:VCVTF 2 "s_register_operand" "w")
2330                                 (match_operand:SI 3 "immediate_operand" "i")]
2331                                UNSPEC_VCAGE))]
2332   "TARGET_NEON"
2333   "vacge.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2334   [(set_attr "type" "neon_fp_compare_s<q>")]
2335 )
2336
2337 (define_insn "neon_vcagt<mode>"
2338   [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w")
2339         (unspec:<V_cmp_result> [(match_operand:VCVTF 1 "s_register_operand" "w")
2340                                 (match_operand:VCVTF 2 "s_register_operand" "w")
2341                                 (match_operand:SI 3 "immediate_operand" "i")]
2342                                UNSPEC_VCAGT))]
2343   "TARGET_NEON"
2344   "vacgt.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2345   [(set_attr "type" "neon_fp_compare_s<q>")]
2346 )
2347
2348 (define_insn "neon_vtst<mode>"
2349   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2350         (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
2351                        (match_operand:VDQIW 2 "s_register_operand" "w")
2352                        (match_operand:SI 3 "immediate_operand" "i")]
2353                       UNSPEC_VTST))]
2354   "TARGET_NEON"
2355   "vtst.<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2356   [(set_attr "type" "neon_tst<q>")]
2357 )
2358
2359 (define_insn "neon_vabd<mode>"
2360   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
2361         (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "w")
2362                       (match_operand:VDQW 2 "s_register_operand" "w")
2363                       (match_operand:SI 3 "immediate_operand" "i")]
2364                      UNSPEC_VABD))]
2365   "TARGET_NEON"
2366   "vabd.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2367   [(set (attr "type")
2368      (if_then_else (match_test "<Is_float_mode>")
2369                    (const_string "neon_fp_abd_s<q>")
2370                    (const_string "neon_abd<q>")))]
2371 )
2372
2373 (define_insn "neon_vabdl<mode>"
2374   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2375         (unspec:<V_widen> [(match_operand:VW 1 "s_register_operand" "w")
2376                            (match_operand:VW 2 "s_register_operand" "w")
2377                            (match_operand:SI 3 "immediate_operand" "i")]
2378                           UNSPEC_VABDL))]
2379   "TARGET_NEON"
2380   "vabdl.%T3%#<V_sz_elem>\t%q0, %P1, %P2"
2381   [(set_attr "type" "neon_abd_long")]
2382 )
2383
2384 (define_insn "neon_vaba<mode>"
2385   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2386         (plus:VDQIW (unspec:VDQIW [(match_operand:VDQIW 2 "s_register_operand" "w")
2387                                    (match_operand:VDQIW 3 "s_register_operand" "w")
2388                                    (match_operand:SI 4 "immediate_operand" "i")]
2389                                   UNSPEC_VABD)
2390                     (match_operand:VDQIW 1 "s_register_operand" "0")))]
2391   "TARGET_NEON"
2392   "vaba.%T4%#<V_sz_elem>\t%<V_reg>0, %<V_reg>2, %<V_reg>3"
2393   [(set_attr "type" "neon_arith_acc<q>")]
2394 )
2395
2396 (define_insn "neon_vabal<mode>"
2397   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2398         (plus:<V_widen> (unspec:<V_widen> [(match_operand:VW 2 "s_register_operand" "w")
2399                                            (match_operand:VW 3 "s_register_operand" "w")
2400                                            (match_operand:SI 4 "immediate_operand" "i")]
2401                                            UNSPEC_VABDL)
2402                          (match_operand:<V_widen> 1 "s_register_operand" "0")))]
2403   "TARGET_NEON"
2404   "vabal.%T4%#<V_sz_elem>\t%q0, %P2, %P3"
2405   [(set_attr "type" "neon_arith_acc<q>")]
2406 )
2407
2408 (define_insn "neon_vmax<mode>"
2409   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
2410         (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "w")
2411                       (match_operand:VDQW 2 "s_register_operand" "w")
2412                       (match_operand:SI 3 "immediate_operand" "i")]
2413                      UNSPEC_VMAX))]
2414   "TARGET_NEON"
2415   "vmax.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2416   [(set (attr "type")
2417     (if_then_else (match_test "<Is_float_mode>")
2418                   (const_string "neon_fp_minmax_s<q>")
2419                   (const_string "neon_minmax<q>")))]
2420 )
2421
2422 (define_insn "neon_vmin<mode>"
2423   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
2424         (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "w")
2425                       (match_operand:VDQW 2 "s_register_operand" "w")
2426                       (match_operand:SI 3 "immediate_operand" "i")]
2427                      UNSPEC_VMIN))]
2428   "TARGET_NEON"
2429   "vmin.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2430   [(set (attr "type")
2431     (if_then_else (match_test "<Is_float_mode>")
2432                   (const_string "neon_fp_minmax_s<q>")
2433                   (const_string "neon_minmax<q>")))]
2434 )
2435
2436 (define_expand "neon_vpadd<mode>"
2437   [(match_operand:VD 0 "s_register_operand" "=w")
2438    (match_operand:VD 1 "s_register_operand" "w")
2439    (match_operand:VD 2 "s_register_operand" "w")
2440    (match_operand:SI 3 "immediate_operand" "i")]
2441   "TARGET_NEON"
2442 {
2443   emit_insn (gen_neon_vpadd_internal<mode> (operands[0], operands[1],
2444                                             operands[2]));
2445   DONE;
2446 })
2447
2448 (define_insn "neon_vpaddl<mode>"
2449   [(set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
2450         (unspec:<V_double_width> [(match_operand:VDQIW 1 "s_register_operand" "w")
2451                                   (match_operand:SI 2 "immediate_operand" "i")]
2452                                  UNSPEC_VPADDL))]
2453   "TARGET_NEON"
2454   "vpaddl.%T2%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1"
2455   [(set_attr "type" "neon_reduc_add_long")]
2456 )
2457
2458 (define_insn "neon_vpadal<mode>"
2459   [(set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
2460         (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
2461                                   (match_operand:VDQIW 2 "s_register_operand" "w")
2462                                   (match_operand:SI 3 "immediate_operand" "i")]
2463                                  UNSPEC_VPADAL))]
2464   "TARGET_NEON"
2465   "vpadal.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>2"
2466   [(set_attr "type" "neon_reduc_add_acc")]
2467 )
2468
2469 (define_insn "neon_vpmax<mode>"
2470   [(set (match_operand:VD 0 "s_register_operand" "=w")
2471         (unspec:VD [(match_operand:VD 1 "s_register_operand" "w")
2472                     (match_operand:VD 2 "s_register_operand" "w")
2473                     (match_operand:SI 3 "immediate_operand" "i")]
2474                    UNSPEC_VPMAX))]
2475   "TARGET_NEON"
2476   "vpmax.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2477   [(set (attr "type")
2478     (if_then_else (match_test "<Is_float_mode>")
2479                   (const_string "neon_fp_reduc_minmax_s<q>")
2480                   (const_string "neon_reduc_minmax<q>")))]
2481 )
2482
2483 (define_insn "neon_vpmin<mode>"
2484   [(set (match_operand:VD 0 "s_register_operand" "=w")
2485         (unspec:VD [(match_operand:VD 1 "s_register_operand" "w")
2486                     (match_operand:VD 2 "s_register_operand" "w")
2487                     (match_operand:SI 3 "immediate_operand" "i")]
2488                    UNSPEC_VPMIN))]
2489   "TARGET_NEON"
2490   "vpmin.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2491   [(set (attr "type")
2492     (if_then_else (match_test "<Is_float_mode>")
2493                   (const_string "neon_fp_reduc_minmax_s<q>")
2494                   (const_string "neon_reduc_minmax<q>")))]
2495 )
2496
2497 (define_insn "neon_vrecps<mode>"
2498   [(set (match_operand:VCVTF 0 "s_register_operand" "=w")
2499         (unspec:VCVTF [(match_operand:VCVTF 1 "s_register_operand" "w")
2500                        (match_operand:VCVTF 2 "s_register_operand" "w")
2501                        (match_operand:SI 3 "immediate_operand" "i")]
2502                       UNSPEC_VRECPS))]
2503   "TARGET_NEON"
2504   "vrecps.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2505   [(set_attr "type" "neon_fp_recps_s<q>")]
2506 )
2507
2508 (define_insn "neon_vrsqrts<mode>"
2509   [(set (match_operand:VCVTF 0 "s_register_operand" "=w")
2510         (unspec:VCVTF [(match_operand:VCVTF 1 "s_register_operand" "w")
2511                        (match_operand:VCVTF 2 "s_register_operand" "w")
2512                        (match_operand:SI 3 "immediate_operand" "i")]
2513                       UNSPEC_VRSQRTS))]
2514   "TARGET_NEON"
2515   "vrsqrts.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2516   [(set_attr "type" "neon_fp_rsqrts_s<q>")]
2517 )
2518
2519 (define_expand "neon_vabs<mode>"
2520   [(match_operand:VDQW 0 "s_register_operand" "")
2521    (match_operand:VDQW 1 "s_register_operand" "")
2522    (match_operand:SI 2 "immediate_operand" "")]
2523   "TARGET_NEON"
2524 {
2525   emit_insn (gen_abs<mode>2 (operands[0], operands[1]));
2526   DONE;
2527 })
2528
2529 (define_insn "neon_vqabs<mode>"
2530   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2531         (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
2532                        (match_operand:SI 2 "immediate_operand" "i")]
2533                       UNSPEC_VQABS))]
2534   "TARGET_NEON"
2535   "vqabs.<V_s_elem>\t%<V_reg>0, %<V_reg>1"
2536   [(set_attr "type" "neon_qabs<q>")]
2537 )
2538
2539 (define_expand "neon_vneg<mode>"
2540   [(match_operand:VDQW 0 "s_register_operand" "")
2541    (match_operand:VDQW 1 "s_register_operand" "")
2542    (match_operand:SI 2 "immediate_operand" "")]
2543   "TARGET_NEON"
2544 {
2545   emit_insn (gen_neg<mode>2 (operands[0], operands[1]));
2546   DONE;
2547 })
2548
2549 (define_insn "neon_vqneg<mode>"
2550   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2551         (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
2552                        (match_operand:SI 2 "immediate_operand" "i")]
2553                       UNSPEC_VQNEG))]
2554   "TARGET_NEON"
2555   "vqneg.<V_s_elem>\t%<V_reg>0, %<V_reg>1"
2556   [(set_attr "type" "neon_qneg<q>")]
2557 )
2558
2559 (define_insn "neon_vcls<mode>"
2560   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2561         (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
2562                        (match_operand:SI 2 "immediate_operand" "i")]
2563                       UNSPEC_VCLS))]
2564   "TARGET_NEON"
2565   "vcls.<V_s_elem>\t%<V_reg>0, %<V_reg>1"
2566   [(set_attr "type" "neon_cls<q>")]
2567 )
2568
2569 (define_insn "clz<mode>2"
2570   [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2571         (clz:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w")))]
2572   "TARGET_NEON"
2573   "vclz.<V_if_elem>\t%<V_reg>0, %<V_reg>1"
2574   [(set_attr "type" "neon_cnt<q>")]
2575 )
2576
2577 (define_expand "neon_vclz<mode>"
2578   [(match_operand:VDQIW 0 "s_register_operand" "")
2579    (match_operand:VDQIW 1 "s_register_operand" "")
2580    (match_operand:SI 2 "immediate_operand" "")]
2581   "TARGET_NEON"
2582 {
2583   emit_insn (gen_clz<mode>2 (operands[0], operands[1]));
2584   DONE;
2585 })
2586
2587 (define_insn "popcount<mode>2"
2588   [(set (match_operand:VE 0 "s_register_operand" "=w")
2589         (popcount:VE (match_operand:VE 1 "s_register_operand" "w")))]
2590   "TARGET_NEON"
2591   "vcnt.<V_sz_elem>\t%<V_reg>0, %<V_reg>1"
2592   [(set_attr "type" "neon_cnt<q>")]
2593 )
2594
2595 (define_expand "neon_vcnt<mode>"
2596   [(match_operand:VE 0 "s_register_operand" "=w")
2597    (match_operand:VE 1 "s_register_operand" "w")
2598    (match_operand:SI 2 "immediate_operand" "i")]
2599   "TARGET_NEON"
2600 {
2601   emit_insn (gen_popcount<mode>2 (operands[0], operands[1]));
2602   DONE;
2603 })
2604
2605 (define_insn "neon_vrecpe<mode>"
2606   [(set (match_operand:V32 0 "s_register_operand" "=w")
2607         (unspec:V32 [(match_operand:V32 1 "s_register_operand" "w")
2608                      (match_operand:SI 2 "immediate_operand" "i")]
2609                     UNSPEC_VRECPE))]
2610   "TARGET_NEON"
2611   "vrecpe.<V_u_elem>\t%<V_reg>0, %<V_reg>1"
2612   [(set_attr "type" "neon_fp_recpe_s<q>")]
2613 )
2614
2615 (define_insn "neon_vrsqrte<mode>"
2616   [(set (match_operand:V32 0 "s_register_operand" "=w")
2617         (unspec:V32 [(match_operand:V32 1 "s_register_operand" "w")
2618                      (match_operand:SI 2 "immediate_operand" "i")]
2619                     UNSPEC_VRSQRTE))]
2620   "TARGET_NEON"
2621   "vrsqrte.<V_u_elem>\t%<V_reg>0, %<V_reg>1"
2622   [(set_attr "type" "neon_fp_rsqrte_s<q>")]
2623 )
2624
2625 (define_expand "neon_vmvn<mode>"
2626   [(match_operand:VDQIW 0 "s_register_operand" "")
2627    (match_operand:VDQIW 1 "s_register_operand" "")
2628    (match_operand:SI 2 "immediate_operand" "")]
2629   "TARGET_NEON"
2630 {
2631   emit_insn (gen_one_cmpl<mode>2 (operands[0], operands[1]));
2632   DONE;
2633 })
2634
2635 (define_insn "neon_vget_lane<mode>_sext_internal"
2636   [(set (match_operand:SI 0 "s_register_operand" "=r")
2637         (sign_extend:SI
2638           (vec_select:<V_elem>
2639             (match_operand:VD 1 "s_register_operand" "w")
2640             (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2641   "TARGET_NEON"
2642 {
2643   if (BYTES_BIG_ENDIAN)
2644     {
2645       int elt = INTVAL (operands[2]);
2646       elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
2647       operands[2] = GEN_INT (elt);
2648     }
2649   return "vmov.s<V_sz_elem>\t%0, %P1[%c2]";
2650 }
2651   [(set_attr "type" "neon_to_gp")]
2652 )
2653
2654 (define_insn "neon_vget_lane<mode>_zext_internal"
2655   [(set (match_operand:SI 0 "s_register_operand" "=r")
2656         (zero_extend:SI
2657           (vec_select:<V_elem>
2658             (match_operand:VD 1 "s_register_operand" "w")
2659             (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2660   "TARGET_NEON"
2661 {
2662   if (BYTES_BIG_ENDIAN)
2663     {
2664       int elt = INTVAL (operands[2]);
2665       elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
2666       operands[2] = GEN_INT (elt);
2667     }
2668   return "vmov.u<V_sz_elem>\t%0, %P1[%c2]";
2669 }
2670   [(set_attr "type" "neon_to_gp")]
2671 )
2672
2673 (define_insn "neon_vget_lane<mode>_sext_internal"
2674   [(set (match_operand:SI 0 "s_register_operand" "=r")
2675         (sign_extend:SI
2676           (vec_select:<V_elem>
2677             (match_operand:VQ 1 "s_register_operand" "w")
2678             (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2679   "TARGET_NEON"
2680 {
2681   rtx ops[3];
2682   int regno = REGNO (operands[1]);
2683   unsigned int halfelts = GET_MODE_NUNITS (<MODE>mode) / 2;
2684   unsigned int elt = INTVAL (operands[2]);
2685   unsigned int elt_adj = elt % halfelts;
2686
2687   if (BYTES_BIG_ENDIAN)
2688     elt_adj = halfelts - 1 - elt_adj;
2689
2690   ops[0] = operands[0];
2691   ops[1] = gen_rtx_REG (<V_HALF>mode, regno + 2 * (elt / halfelts));
2692   ops[2] = GEN_INT (elt_adj);
2693   output_asm_insn ("vmov.s<V_sz_elem>\t%0, %P1[%c2]", ops);
2694
2695   return "";
2696 }
2697   [(set_attr "type" "neon_to_gp_q")]
2698 )
2699
2700 (define_insn "neon_vget_lane<mode>_zext_internal"
2701   [(set (match_operand:SI 0 "s_register_operand" "=r")
2702         (zero_extend:SI
2703           (vec_select:<V_elem>
2704             (match_operand:VQ 1 "s_register_operand" "w")
2705             (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2706   "TARGET_NEON"
2707 {
2708   rtx ops[3];
2709   int regno = REGNO (operands[1]);
2710   unsigned int halfelts = GET_MODE_NUNITS (<MODE>mode) / 2;
2711   unsigned int elt = INTVAL (operands[2]);
2712   unsigned int elt_adj = elt % halfelts;
2713
2714   if (BYTES_BIG_ENDIAN)
2715     elt_adj = halfelts - 1 - elt_adj;
2716
2717   ops[0] = operands[0];
2718   ops[1] = gen_rtx_REG (<V_HALF>mode, regno + 2 * (elt / halfelts));
2719   ops[2] = GEN_INT (elt_adj);
2720   output_asm_insn ("vmov.u<V_sz_elem>\t%0, %P1[%c2]", ops);
2721
2722   return "";
2723 }
2724   [(set_attr "type" "neon_to_gp_q")]
2725 )
2726
2727 (define_expand "neon_vget_lane<mode>"
2728   [(match_operand:<V_ext> 0 "s_register_operand" "")
2729    (match_operand:VDQW 1 "s_register_operand" "")
2730    (match_operand:SI 2 "immediate_operand" "")
2731    (match_operand:SI 3 "immediate_operand" "")]
2732   "TARGET_NEON"
2733 {
2734   HOST_WIDE_INT magic = INTVAL (operands[3]);
2735   rtx insn;
2736
2737   neon_lane_bounds (operands[2], 0, GET_MODE_NUNITS (<MODE>mode));
2738
2739   if (BYTES_BIG_ENDIAN)
2740     {
2741       /* The intrinsics are defined in terms of a model where the
2742          element ordering in memory is vldm order, whereas the generic
2743          RTL is defined in terms of a model where the element ordering
2744          in memory is array order.  Convert the lane number to conform
2745          to this model.  */
2746       unsigned int elt = INTVAL (operands[2]);
2747       unsigned int reg_nelts
2748         = 64 / GET_MODE_BITSIZE (GET_MODE_INNER (<MODE>mode));
2749       elt ^= reg_nelts - 1;
2750       operands[2] = GEN_INT (elt);
2751     }
2752
2753   if ((magic & 3) == 3 || GET_MODE_BITSIZE (GET_MODE_INNER (<MODE>mode)) == 32)
2754     insn = gen_vec_extract<mode> (operands[0], operands[1], operands[2]);
2755   else
2756     {
2757       if ((magic & 1) != 0)
2758         insn = gen_neon_vget_lane<mode>_sext_internal (operands[0], operands[1],
2759                                                        operands[2]);
2760       else
2761         insn = gen_neon_vget_lane<mode>_zext_internal (operands[0], operands[1],
2762                                                        operands[2]);
2763     }
2764   emit_insn (insn);
2765   DONE;
2766 })
2767
2768 ; Operand 3 (info word) is ignored because it does nothing useful with 64-bit
2769 ; elements.
2770
2771 (define_expand "neon_vget_lanedi"
2772   [(match_operand:DI 0 "s_register_operand" "=r")
2773    (match_operand:DI 1 "s_register_operand" "w")
2774    (match_operand:SI 2 "immediate_operand" "i")
2775    (match_operand:SI 3 "immediate_operand" "i")]
2776   "TARGET_NEON"
2777 {
2778   neon_lane_bounds (operands[2], 0, 1);
2779   emit_move_insn (operands[0], operands[1]);
2780   DONE;
2781 })
2782
2783 (define_expand "neon_vget_lanev2di"
2784   [(match_operand:DI 0 "s_register_operand" "")
2785    (match_operand:V2DI 1 "s_register_operand" "")
2786    (match_operand:SI 2 "immediate_operand" "")
2787    (match_operand:SI 3 "immediate_operand" "")]
2788   "TARGET_NEON"
2789 {
2790   switch (INTVAL (operands[2]))
2791     {
2792     case 0:
2793       emit_move_insn (operands[0], gen_lowpart (DImode, operands[1]));
2794       break;
2795     case 1:
2796       emit_move_insn (operands[0], gen_highpart (DImode, operands[1]));
2797       break;
2798     default:
2799       neon_lane_bounds (operands[2], 0, 1);
2800       FAIL;
2801     }
2802   DONE;
2803 })
2804
2805 (define_expand "neon_vset_lane<mode>"
2806   [(match_operand:VDQ 0 "s_register_operand" "=w")
2807    (match_operand:<V_elem> 1 "s_register_operand" "r")
2808    (match_operand:VDQ 2 "s_register_operand" "0")
2809    (match_operand:SI 3 "immediate_operand" "i")]
2810   "TARGET_NEON"
2811 {
2812   unsigned int elt = INTVAL (operands[3]);
2813   neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
2814
2815   if (BYTES_BIG_ENDIAN)
2816     {
2817       unsigned int reg_nelts
2818         = 64 / GET_MODE_BITSIZE (GET_MODE_INNER (<MODE>mode));
2819       elt ^= reg_nelts - 1;
2820     }
2821
2822   emit_insn (gen_vec_set<mode>_internal (operands[0], operands[1],
2823                                          GEN_INT (1 << elt), operands[2]));
2824   DONE;
2825 })
2826
2827 ; See neon_vget_lanedi comment for reasons operands 2 & 3 are ignored.
2828
2829 (define_expand "neon_vset_lanedi"
2830   [(match_operand:DI 0 "s_register_operand" "=w")
2831    (match_operand:DI 1 "s_register_operand" "r")
2832    (match_operand:DI 2 "s_register_operand" "0")
2833    (match_operand:SI 3 "immediate_operand" "i")]
2834   "TARGET_NEON"
2835 {
2836   neon_lane_bounds (operands[3], 0, 1);
2837   emit_move_insn (operands[0], operands[1]);
2838   DONE;
2839 })
2840
2841 (define_expand "neon_vcreate<mode>"
2842   [(match_operand:VDX 0 "s_register_operand" "")
2843    (match_operand:DI 1 "general_operand" "")]
2844   "TARGET_NEON"
2845 {
2846   rtx src = gen_lowpart (<MODE>mode, operands[1]);
2847   emit_move_insn (operands[0], src);
2848   DONE;
2849 })
2850
2851 (define_insn "neon_vdup_n<mode>"
2852   [(set (match_operand:VX 0 "s_register_operand" "=w")
2853         (vec_duplicate:VX (match_operand:<V_elem> 1 "s_register_operand" "r")))]
2854   "TARGET_NEON"
2855   "vdup.<V_sz_elem>\t%<V_reg>0, %1"
2856   [(set_attr "type" "neon_from_gp<q>")]
2857 )
2858
2859 (define_insn "neon_vdup_n<mode>"
2860   [(set (match_operand:V32 0 "s_register_operand" "=w,w")
2861         (vec_duplicate:V32 (match_operand:<V_elem> 1 "s_register_operand" "r,t")))]
2862   "TARGET_NEON"
2863   "@
2864   vdup.<V_sz_elem>\t%<V_reg>0, %1
2865   vdup.<V_sz_elem>\t%<V_reg>0, %y1"
2866   [(set_attr "type" "neon_from_gp<q>,neon_dup<q>")]
2867 )
2868
2869 (define_expand "neon_vdup_ndi"
2870   [(match_operand:DI 0 "s_register_operand" "=w")
2871    (match_operand:DI 1 "s_register_operand" "r")]
2872   "TARGET_NEON"
2873 {
2874   emit_move_insn (operands[0], operands[1]);
2875   DONE;
2876 }
2877 )
2878
2879 (define_insn "neon_vdup_nv2di"
2880   [(set (match_operand:V2DI 0 "s_register_operand" "=w,w")
2881         (vec_duplicate:V2DI (match_operand:DI 1 "s_register_operand" "r,w")))]
2882   "TARGET_NEON"
2883   "@
2884   vmov\t%e0, %Q1, %R1\;vmov\t%f0, %Q1, %R1
2885   vmov\t%e0, %P1\;vmov\t%f0, %P1"
2886   [(set_attr "length" "8")
2887    (set_attr "type" "multiple")]
2888 )
2889
2890 (define_insn "neon_vdup_lane<mode>_internal"
2891   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
2892         (vec_duplicate:VDQW
2893           (vec_select:<V_elem>
2894             (match_operand:<V_double_vector_mode> 1 "s_register_operand" "w")
2895             (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2896   "TARGET_NEON"
2897 {
2898   if (BYTES_BIG_ENDIAN)
2899     {
2900       int elt = INTVAL (operands[2]);
2901       elt = GET_MODE_NUNITS (<V_double_vector_mode>mode) - 1 - elt;
2902       operands[2] = GEN_INT (elt);
2903     }
2904   if (<Is_d_reg>)
2905     return "vdup.<V_sz_elem>\t%P0, %P1[%c2]";
2906   else
2907     return "vdup.<V_sz_elem>\t%q0, %P1[%c2]";
2908 }
2909   [(set_attr "type" "neon_dup<q>")]
2910 )
2911
2912 (define_expand "neon_vdup_lane<mode>"
2913   [(match_operand:VDQW 0 "s_register_operand" "=w")
2914    (match_operand:<V_double_vector_mode> 1 "s_register_operand" "w")
2915    (match_operand:SI 2 "immediate_operand" "i")]
2916   "TARGET_NEON"
2917 {
2918   neon_lane_bounds (operands[2], 0, GET_MODE_NUNITS (<V_double_vector_mode>mode));
2919   if (BYTES_BIG_ENDIAN)
2920     {
2921       unsigned int elt = INTVAL (operands[2]);
2922       unsigned int reg_nelts
2923         = 64 / GET_MODE_BITSIZE (GET_MODE_INNER (<V_double_vector_mode>mode));
2924       elt ^= reg_nelts - 1;
2925       operands[2] = GEN_INT (elt);
2926     }
2927     emit_insn (gen_neon_vdup_lane<mode>_internal (operands[0], operands[1],
2928                                                   operands[2]));
2929     DONE;
2930 })
2931
2932 ; Scalar index is ignored, since only zero is valid here.
2933 (define_expand "neon_vdup_lanedi"
2934   [(match_operand:DI 0 "s_register_operand" "=w")
2935    (match_operand:DI 1 "s_register_operand" "w")
2936    (match_operand:SI 2 "immediate_operand" "i")]
2937   "TARGET_NEON"
2938 {
2939   neon_lane_bounds (operands[2], 0, 1);
2940   emit_move_insn (operands[0], operands[1]);
2941   DONE;
2942 })
2943
2944 ; Likewise for v2di, as the DImode second operand has only a single element.
2945 (define_expand "neon_vdup_lanev2di"
2946   [(match_operand:V2DI 0 "s_register_operand" "=w")
2947    (match_operand:DI 1 "s_register_operand" "w")
2948    (match_operand:SI 2 "immediate_operand" "i")]
2949   "TARGET_NEON"
2950 {
2951   neon_lane_bounds (operands[2], 0, 1);
2952   emit_insn (gen_neon_vdup_nv2di (operands[0], operands[1]));
2953   DONE;
2954 })
2955
2956 ; Disabled before reload because we don't want combine doing something silly,
2957 ; but used by the post-reload expansion of neon_vcombine.
2958 (define_insn "*neon_vswp<mode>"
2959   [(set (match_operand:VDQX 0 "s_register_operand" "+w")
2960         (match_operand:VDQX 1 "s_register_operand" "+w"))
2961    (set (match_dup 1) (match_dup 0))]
2962   "TARGET_NEON && reload_completed"
2963   "vswp\t%<V_reg>0, %<V_reg>1"
2964   [(set_attr "type" "neon_permute<q>")]
2965 )
2966
2967 ;; In this insn, operand 1 should be low, and operand 2 the high part of the
2968 ;; dest vector.
2969 ;; FIXME: A different implementation of this builtin could make it much
2970 ;; more likely that we wouldn't actually need to output anything (we could make
2971 ;; it so that the reg allocator puts things in the right places magically
2972 ;; instead). Lack of subregs for vectors makes that tricky though, I think.
2973
2974 (define_insn_and_split "neon_vcombine<mode>"
2975   [(set (match_operand:<V_DOUBLE> 0 "s_register_operand" "=w")
2976         (vec_concat:<V_DOUBLE>
2977           (match_operand:VDX 1 "s_register_operand" "w")
2978           (match_operand:VDX 2 "s_register_operand" "w")))]
2979   "TARGET_NEON"
2980   "#"
2981   "&& reload_completed"
2982   [(const_int 0)]
2983 {
2984   neon_split_vcombine (operands);
2985   DONE;
2986 }
2987 [(set_attr "type" "multiple")]
2988 )
2989
2990 (define_expand "neon_vget_high<mode>"
2991   [(match_operand:<V_HALF> 0 "s_register_operand")
2992    (match_operand:VQX 1 "s_register_operand")]
2993   "TARGET_NEON"
2994 {
2995   emit_move_insn (operands[0],
2996                   simplify_gen_subreg (<V_HALF>mode, operands[1], <MODE>mode,
2997                                        GET_MODE_SIZE (<V_HALF>mode)));
2998   DONE;
2999 })
3000
3001 (define_expand "neon_vget_low<mode>"
3002   [(match_operand:<V_HALF> 0 "s_register_operand")
3003    (match_operand:VQX 1 "s_register_operand")]
3004   "TARGET_NEON"
3005 {
3006   emit_move_insn (operands[0],
3007                   simplify_gen_subreg (<V_HALF>mode, operands[1],
3008                                        <MODE>mode, 0));
3009   DONE;
3010 })
3011
3012 (define_insn "float<mode><V_cvtto>2"
3013   [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3014         (float:<V_CVTTO> (match_operand:VCVTI 1 "s_register_operand" "w")))]
3015   "TARGET_NEON && !flag_rounding_math"
3016   "vcvt.f32.s32\t%<V_reg>0, %<V_reg>1"
3017   [(set_attr "type" "neon_int_to_fp_<V_elem_ch><q>")]
3018 )
3019
3020 (define_insn "floatuns<mode><V_cvtto>2"
3021   [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3022         (unsigned_float:<V_CVTTO> (match_operand:VCVTI 1 "s_register_operand" "w")))]
3023   "TARGET_NEON && !flag_rounding_math"
3024   "vcvt.f32.u32\t%<V_reg>0, %<V_reg>1"
3025   [(set_attr "type" "neon_int_to_fp_<V_elem_ch><q>")]
3026 )
3027
3028 (define_insn "fix_trunc<mode><V_cvtto>2"
3029   [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3030         (fix:<V_CVTTO> (match_operand:VCVTF 1 "s_register_operand" "w")))]
3031   "TARGET_NEON"
3032   "vcvt.s32.f32\t%<V_reg>0, %<V_reg>1"
3033   [(set_attr "type" "neon_fp_to_int_<V_elem_ch><q>")]
3034 )
3035
3036 (define_insn "fixuns_trunc<mode><V_cvtto>2"
3037   [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3038         (unsigned_fix:<V_CVTTO> (match_operand:VCVTF 1 "s_register_operand" "w")))]
3039   "TARGET_NEON"
3040   "vcvt.u32.f32\t%<V_reg>0, %<V_reg>1"
3041   [(set_attr "type" "neon_fp_to_int_<V_elem_ch><q>")]
3042 )
3043
3044 (define_insn "neon_vcvt<mode>"
3045   [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3046         (unspec:<V_CVTTO> [(match_operand:VCVTF 1 "s_register_operand" "w")
3047                            (match_operand:SI 2 "immediate_operand" "i")]
3048                           UNSPEC_VCVT))]
3049   "TARGET_NEON"
3050   "vcvt.%T2%#32.f32\t%<V_reg>0, %<V_reg>1"
3051   [(set_attr "type" "neon_fp_to_int_<V_elem_ch><q>")]
3052 )
3053
3054 (define_insn "neon_vcvt<mode>"
3055   [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3056         (unspec:<V_CVTTO> [(match_operand:VCVTI 1 "s_register_operand" "w")
3057                            (match_operand:SI 2 "immediate_operand" "i")]
3058                           UNSPEC_VCVT))]
3059   "TARGET_NEON"
3060   "vcvt.f32.%T2%#32\t%<V_reg>0, %<V_reg>1"
3061   [(set_attr "type" "neon_int_to_fp_<V_elem_ch><q>")]
3062 )
3063
3064 (define_insn "neon_vcvtv4sfv4hf"
3065   [(set (match_operand:V4SF 0 "s_register_operand" "=w")
3066         (unspec:V4SF [(match_operand:V4HF 1 "s_register_operand" "w")]
3067                           UNSPEC_VCVT))]
3068   "TARGET_NEON && TARGET_FP16"
3069   "vcvt.f32.f16\t%q0, %P1"
3070   [(set_attr "type" "neon_fp_cvt_widen_h")]
3071 )
3072
3073 (define_insn "neon_vcvtv4hfv4sf"
3074   [(set (match_operand:V4HF 0 "s_register_operand" "=w")
3075         (unspec:V4HF [(match_operand:V4SF 1 "s_register_operand" "w")]
3076                           UNSPEC_VCVT))]
3077   "TARGET_NEON && TARGET_FP16"
3078   "vcvt.f16.f32\t%P0, %q1"
3079   [(set_attr "type" "neon_fp_cvt_narrow_s_q")]
3080 )
3081
3082 (define_insn "neon_vcvt_n<mode>"
3083   [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3084         (unspec:<V_CVTTO> [(match_operand:VCVTF 1 "s_register_operand" "w")
3085                            (match_operand:SI 2 "immediate_operand" "i")
3086                            (match_operand:SI 3 "immediate_operand" "i")]
3087                           UNSPEC_VCVT_N))]
3088   "TARGET_NEON"
3089 {
3090   neon_const_bounds (operands[2], 1, 33);
3091   return "vcvt.%T3%#32.f32\t%<V_reg>0, %<V_reg>1, %2";
3092 }
3093   [(set_attr "type" "neon_fp_to_int_<V_elem_ch><q>")]
3094 )
3095
3096 (define_insn "neon_vcvt_n<mode>"
3097   [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3098         (unspec:<V_CVTTO> [(match_operand:VCVTI 1 "s_register_operand" "w")
3099                            (match_operand:SI 2 "immediate_operand" "i")
3100                            (match_operand:SI 3 "immediate_operand" "i")]
3101                           UNSPEC_VCVT_N))]
3102   "TARGET_NEON"
3103 {
3104   neon_const_bounds (operands[2], 1, 33);
3105   return "vcvt.f32.%T3%#32\t%<V_reg>0, %<V_reg>1, %2";
3106 }
3107   [(set_attr "type" "neon_int_to_fp_<V_elem_ch><q>")]
3108 )
3109
3110 (define_insn "neon_vmovn<mode>"
3111   [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
3112         (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
3113                             (match_operand:SI 2 "immediate_operand" "i")]
3114                            UNSPEC_VMOVN))]
3115   "TARGET_NEON"
3116   "vmovn.<V_if_elem>\t%P0, %q1"
3117   [(set_attr "type" "neon_shift_imm_narrow_q")]
3118 )
3119
3120 (define_insn "neon_vqmovn<mode>"
3121   [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
3122         (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
3123                             (match_operand:SI 2 "immediate_operand" "i")]
3124                            UNSPEC_VQMOVN))]
3125   "TARGET_NEON"
3126   "vqmovn.%T2%#<V_sz_elem>\t%P0, %q1"
3127   [(set_attr "type" "neon_sat_shift_imm_narrow_q")]
3128 )
3129
3130 (define_insn "neon_vqmovun<mode>"
3131   [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
3132         (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
3133                             (match_operand:SI 2 "immediate_operand" "i")]
3134                            UNSPEC_VQMOVUN))]
3135   "TARGET_NEON"
3136   "vqmovun.<V_s_elem>\t%P0, %q1"
3137   [(set_attr "type" "neon_sat_shift_imm_narrow_q")]
3138 )
3139
3140 (define_insn "neon_vmovl<mode>"
3141   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3142         (unspec:<V_widen> [(match_operand:VW 1 "s_register_operand" "w")
3143                            (match_operand:SI 2 "immediate_operand" "i")]
3144                           UNSPEC_VMOVL))]
3145   "TARGET_NEON"
3146   "vmovl.%T2%#<V_sz_elem>\t%q0, %P1"
3147   [(set_attr "type" "neon_shift_imm_long")]
3148 )
3149
3150 (define_insn "neon_vmul_lane<mode>"
3151   [(set (match_operand:VMD 0 "s_register_operand" "=w")
3152         (unspec:VMD [(match_operand:VMD 1 "s_register_operand" "w")
3153                      (match_operand:VMD 2 "s_register_operand"
3154                                         "<scalar_mul_constraint>")
3155                      (match_operand:SI 3 "immediate_operand" "i")
3156                      (match_operand:SI 4 "immediate_operand" "i")]
3157                     UNSPEC_VMUL_LANE))]
3158   "TARGET_NEON"
3159 {
3160   neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3161   return "vmul.<V_if_elem>\t%P0, %P1, %P2[%c3]";
3162 }
3163   [(set (attr "type")
3164      (if_then_else (match_test "<Is_float_mode>")
3165                    (const_string "neon_fp_mul_s_scalar<q>")
3166                    (const_string "neon_mul_<V_elem_ch>_scalar<q>")))]
3167 )
3168
3169 (define_insn "neon_vmul_lane<mode>"
3170   [(set (match_operand:VMQ 0 "s_register_operand" "=w")
3171         (unspec:VMQ [(match_operand:VMQ 1 "s_register_operand" "w")
3172                      (match_operand:<V_HALF> 2 "s_register_operand"
3173                                              "<scalar_mul_constraint>")
3174                      (match_operand:SI 3 "immediate_operand" "i")
3175                      (match_operand:SI 4 "immediate_operand" "i")]
3176                     UNSPEC_VMUL_LANE))]
3177   "TARGET_NEON"
3178 {
3179   neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<V_HALF>mode));
3180   return "vmul.<V_if_elem>\t%q0, %q1, %P2[%c3]";
3181 }
3182   [(set (attr "type")
3183      (if_then_else (match_test "<Is_float_mode>")
3184                    (const_string "neon_fp_mul_s_scalar<q>")
3185                    (const_string "neon_mul_<V_elem_ch>_scalar<q>")))]
3186 )
3187
3188 (define_insn "neon_vmull_lane<mode>"
3189   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3190         (unspec:<V_widen> [(match_operand:VMDI 1 "s_register_operand" "w")
3191                            (match_operand:VMDI 2 "s_register_operand"
3192                                                "<scalar_mul_constraint>")
3193                            (match_operand:SI 3 "immediate_operand" "i")
3194                            (match_operand:SI 4 "immediate_operand" "i")]
3195                           UNSPEC_VMULL_LANE))]
3196   "TARGET_NEON"
3197 {
3198   neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3199   return "vmull.%T4%#<V_sz_elem>\t%q0, %P1, %P2[%c3]";
3200 }
3201   [(set_attr "type" "neon_mul_<V_elem_ch>_scalar_long")]
3202 )
3203
3204 (define_insn "neon_vqdmull_lane<mode>"
3205   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3206         (unspec:<V_widen> [(match_operand:VMDI 1 "s_register_operand" "w")
3207                            (match_operand:VMDI 2 "s_register_operand"
3208                                                "<scalar_mul_constraint>")
3209                            (match_operand:SI 3 "immediate_operand" "i")
3210                            (match_operand:SI 4 "immediate_operand" "i")]
3211                           UNSPEC_VQDMULL_LANE))]
3212   "TARGET_NEON"
3213 {
3214   neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3215   return "vqdmull.<V_s_elem>\t%q0, %P1, %P2[%c3]";
3216 }
3217   [(set_attr "type" "neon_sat_mul_<V_elem_ch>_scalar_long")]
3218 )
3219
3220 (define_insn "neon_vqdmulh_lane<mode>"
3221   [(set (match_operand:VMQI 0 "s_register_operand" "=w")
3222         (unspec:VMQI [(match_operand:VMQI 1 "s_register_operand" "w")
3223                       (match_operand:<V_HALF> 2 "s_register_operand"
3224                                               "<scalar_mul_constraint>")
3225                       (match_operand:SI 3 "immediate_operand" "i")
3226                       (match_operand:SI 4 "immediate_operand" "i")]
3227                       UNSPEC_VQDMULH_LANE))]
3228   "TARGET_NEON"
3229 {
3230   neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3231   return "vq%O4dmulh.%T4%#<V_sz_elem>\t%q0, %q1, %P2[%c3]";
3232 }
3233   [(set_attr "type" "neon_sat_mul_<V_elem_ch>_scalar_q")]
3234 )
3235
3236 (define_insn "neon_vqdmulh_lane<mode>"
3237   [(set (match_operand:VMDI 0 "s_register_operand" "=w")
3238         (unspec:VMDI [(match_operand:VMDI 1 "s_register_operand" "w")
3239                       (match_operand:VMDI 2 "s_register_operand"
3240                                           "<scalar_mul_constraint>")
3241                       (match_operand:SI 3 "immediate_operand" "i")
3242                       (match_operand:SI 4 "immediate_operand" "i")]
3243                       UNSPEC_VQDMULH_LANE))]
3244   "TARGET_NEON"
3245 {
3246   neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3247   return "vq%O4dmulh.%T4%#<V_sz_elem>\t%P0, %P1, %P2[%c3]";
3248 }
3249   [(set_attr "type" "neon_sat_mul_<V_elem_ch>_scalar_q")]
3250 )
3251
3252 (define_insn "neon_vmla_lane<mode>"
3253   [(set (match_operand:VMD 0 "s_register_operand" "=w")
3254         (unspec:VMD [(match_operand:VMD 1 "s_register_operand" "0")
3255                      (match_operand:VMD 2 "s_register_operand" "w")
3256                      (match_operand:VMD 3 "s_register_operand"
3257                                         "<scalar_mul_constraint>")
3258                      (match_operand:SI 4 "immediate_operand" "i")
3259                      (match_operand:SI 5 "immediate_operand" "i")]
3260                      UNSPEC_VMLA_LANE))]
3261   "TARGET_NEON"
3262 {
3263   neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3264   return "vmla.<V_if_elem>\t%P0, %P2, %P3[%c4]";
3265 }
3266   [(set (attr "type")
3267      (if_then_else (match_test "<Is_float_mode>")
3268                    (const_string "neon_fp_mla_s_scalar<q>")
3269                    (const_string "neon_mla_<V_elem_ch>_scalar<q>")))]
3270 )
3271
3272 (define_insn "neon_vmla_lane<mode>"
3273   [(set (match_operand:VMQ 0 "s_register_operand" "=w")
3274         (unspec:VMQ [(match_operand:VMQ 1 "s_register_operand" "0")
3275                      (match_operand:VMQ 2 "s_register_operand" "w")
3276                      (match_operand:<V_HALF> 3 "s_register_operand"
3277                                              "<scalar_mul_constraint>")
3278                      (match_operand:SI 4 "immediate_operand" "i")
3279                      (match_operand:SI 5 "immediate_operand" "i")]
3280                      UNSPEC_VMLA_LANE))]
3281   "TARGET_NEON"
3282 {
3283   neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3284   return "vmla.<V_if_elem>\t%q0, %q2, %P3[%c4]";
3285 }
3286   [(set (attr "type")
3287      (if_then_else (match_test "<Is_float_mode>")
3288                    (const_string "neon_fp_mla_s_scalar<q>")
3289                    (const_string "neon_mla_<V_elem_ch>_scalar<q>")))]
3290 )
3291
3292 (define_insn "neon_vmlal_lane<mode>"
3293   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3294         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
3295                            (match_operand:VMDI 2 "s_register_operand" "w")
3296                            (match_operand:VMDI 3 "s_register_operand"
3297                                                "<scalar_mul_constraint>")
3298                            (match_operand:SI 4 "immediate_operand" "i")
3299                            (match_operand:SI 5 "immediate_operand" "i")]
3300                           UNSPEC_VMLAL_LANE))]
3301   "TARGET_NEON"
3302 {
3303   neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3304   return "vmlal.%T5%#<V_sz_elem>\t%q0, %P2, %P3[%c4]";
3305 }
3306   [(set_attr "type" "neon_mla_<V_elem_ch>_scalar_long")]
3307 )
3308
3309 (define_insn "neon_vqdmlal_lane<mode>"
3310   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3311         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
3312                            (match_operand:VMDI 2 "s_register_operand" "w")
3313                            (match_operand:VMDI 3 "s_register_operand"
3314                                                "<scalar_mul_constraint>")
3315                            (match_operand:SI 4 "immediate_operand" "i")
3316                            (match_operand:SI 5 "immediate_operand" "i")]
3317                           UNSPEC_VQDMLAL_LANE))]
3318   "TARGET_NEON"
3319 {
3320   neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3321   return "vqdmlal.<V_s_elem>\t%q0, %P2, %P3[%c4]";
3322 }
3323   [(set_attr "type" "neon_sat_mla_<V_elem_ch>_scalar_long")]
3324 )
3325
3326 (define_insn "neon_vmls_lane<mode>"
3327   [(set (match_operand:VMD 0 "s_register_operand" "=w")
3328         (unspec:VMD [(match_operand:VMD 1 "s_register_operand" "0")
3329                      (match_operand:VMD 2 "s_register_operand" "w")
3330                      (match_operand:VMD 3 "s_register_operand"
3331                                         "<scalar_mul_constraint>")
3332                      (match_operand:SI 4 "immediate_operand" "i")
3333                      (match_operand:SI 5 "immediate_operand" "i")]
3334                     UNSPEC_VMLS_LANE))]
3335   "TARGET_NEON"
3336 {
3337   neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3338   return "vmls.<V_if_elem>\t%P0, %P2, %P3[%c4]";
3339 }
3340   [(set (attr "type")
3341      (if_then_else (match_test "<Is_float_mode>")
3342                    (const_string "neon_fp_mla_s_scalar<q>")
3343                    (const_string "neon_mla_<V_elem_ch>_scalar<q>")))]
3344 )
3345
3346 (define_insn "neon_vmls_lane<mode>"
3347   [(set (match_operand:VMQ 0 "s_register_operand" "=w")
3348         (unspec:VMQ [(match_operand:VMQ 1 "s_register_operand" "0")
3349                      (match_operand:VMQ 2 "s_register_operand" "w")
3350                      (match_operand:<V_HALF> 3 "s_register_operand"
3351                                              "<scalar_mul_constraint>")
3352                      (match_operand:SI 4 "immediate_operand" "i")
3353                      (match_operand:SI 5 "immediate_operand" "i")]
3354                     UNSPEC_VMLS_LANE))]
3355   "TARGET_NEON"
3356 {
3357   neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3358   return "vmls.<V_if_elem>\t%q0, %q2, %P3[%c4]";
3359 }
3360   [(set (attr "type")
3361      (if_then_else (match_test "<Is_float_mode>")
3362                    (const_string "neon_fp_mla_s_scalar<q>")
3363                    (const_string "neon_mla_<V_elem_ch>_scalar<q>")))]
3364 )
3365
3366 (define_insn "neon_vmlsl_lane<mode>"
3367   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3368         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
3369                            (match_operand:VMDI 2 "s_register_operand" "w")
3370                            (match_operand:VMDI 3 "s_register_operand"
3371                                                "<scalar_mul_constraint>")
3372                            (match_operand:SI 4 "immediate_operand" "i")
3373                            (match_operand:SI 5 "immediate_operand" "i")]
3374                           UNSPEC_VMLSL_LANE))]
3375   "TARGET_NEON"
3376 {
3377   neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3378   return "vmlsl.%T5%#<V_sz_elem>\t%q0, %P2, %P3[%c4]";
3379 }
3380   [(set_attr "type" "neon_mla_<V_elem_ch>_scalar_long")]
3381 )
3382
3383 (define_insn "neon_vqdmlsl_lane<mode>"
3384   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3385         (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
3386                            (match_operand:VMDI 2 "s_register_operand" "w")
3387                            (match_operand:VMDI 3 "s_register_operand"
3388                                                "<scalar_mul_constraint>")
3389                            (match_operand:SI 4 "immediate_operand" "i")
3390                            (match_operand:SI 5 "immediate_operand" "i")]
3391                           UNSPEC_VQDMLSL_LANE))]
3392   "TARGET_NEON"
3393 {
3394   neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3395   return "vqdmlsl.<V_s_elem>\t%q0, %P2, %P3[%c4]";
3396 }
3397   [(set_attr "type" "neon_sat_mla_<V_elem_ch>_scalar_long")]
3398 )
3399
3400 ; FIXME: For the "_n" multiply/multiply-accumulate insns, we copy a value in a
3401 ; core register into a temp register, then use a scalar taken from that. This
3402 ; isn't an optimal solution if e.g. the scalar has just been read from memory
3403 ; or extracted from another vector. The latter case it's currently better to
3404 ; use the "_lane" variant, and the former case can probably be implemented
3405 ; using vld1_lane, but that hasn't been done yet.
3406
3407 (define_expand "neon_vmul_n<mode>"
3408   [(match_operand:VMD 0 "s_register_operand" "")
3409    (match_operand:VMD 1 "s_register_operand" "")
3410    (match_operand:<V_elem> 2 "s_register_operand" "")
3411    (match_operand:SI 3 "immediate_operand" "")]
3412   "TARGET_NEON"
3413 {
3414   rtx tmp = gen_reg_rtx (<MODE>mode);
3415   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[2], tmp, const0_rtx));
3416   emit_insn (gen_neon_vmul_lane<mode> (operands[0], operands[1], tmp,
3417                                        const0_rtx, const0_rtx));
3418   DONE;
3419 })
3420
3421 (define_expand "neon_vmul_n<mode>"
3422   [(match_operand:VMQ 0 "s_register_operand" "")
3423    (match_operand:VMQ 1 "s_register_operand" "")
3424    (match_operand:<V_elem> 2 "s_register_operand" "")
3425    (match_operand:SI 3 "immediate_operand" "")]
3426   "TARGET_NEON"
3427 {
3428   rtx tmp = gen_reg_rtx (<V_HALF>mode);
3429   emit_insn (gen_neon_vset_lane<V_half> (tmp, operands[2], tmp, const0_rtx));
3430   emit_insn (gen_neon_vmul_lane<mode> (operands[0], operands[1], tmp,
3431                                        const0_rtx, const0_rtx));
3432   DONE;
3433 })
3434
3435 (define_expand "neon_vmull_n<mode>"
3436   [(match_operand:<V_widen> 0 "s_register_operand" "")
3437    (match_operand:VMDI 1 "s_register_operand" "")
3438    (match_operand:<V_elem> 2 "s_register_operand" "")
3439    (match_operand:SI 3 "immediate_operand" "")]
3440   "TARGET_NEON"
3441 {
3442   rtx tmp = gen_reg_rtx (<MODE>mode);
3443   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[2], tmp, const0_rtx));
3444   emit_insn (gen_neon_vmull_lane<mode> (operands[0], operands[1], tmp,
3445                                         const0_rtx, operands[3]));
3446   DONE;
3447 })
3448
3449 (define_expand "neon_vqdmull_n<mode>"
3450   [(match_operand:<V_widen> 0 "s_register_operand" "")
3451    (match_operand:VMDI 1 "s_register_operand" "")
3452    (match_operand:<V_elem> 2 "s_register_operand" "")
3453    (match_operand:SI 3 "immediate_operand" "")]
3454   "TARGET_NEON"
3455 {
3456   rtx tmp = gen_reg_rtx (<MODE>mode);
3457   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[2], tmp, const0_rtx));
3458   emit_insn (gen_neon_vqdmull_lane<mode> (operands[0], operands[1], tmp,
3459                                           const0_rtx, const0_rtx));
3460   DONE;
3461 })
3462
3463 (define_expand "neon_vqdmulh_n<mode>"
3464   [(match_operand:VMDI 0 "s_register_operand" "")
3465    (match_operand:VMDI 1 "s_register_operand" "")
3466    (match_operand:<V_elem> 2 "s_register_operand" "")
3467    (match_operand:SI 3 "immediate_operand" "")]
3468   "TARGET_NEON"
3469 {
3470   rtx tmp = gen_reg_rtx (<MODE>mode);
3471   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[2], tmp, const0_rtx));
3472   emit_insn (gen_neon_vqdmulh_lane<mode> (operands[0], operands[1], tmp,
3473                                           const0_rtx, operands[3]));
3474   DONE;
3475 })
3476
3477 (define_expand "neon_vqdmulh_n<mode>"
3478   [(match_operand:VMQI 0 "s_register_operand" "")
3479    (match_operand:VMQI 1 "s_register_operand" "")
3480    (match_operand:<V_elem> 2 "s_register_operand" "")
3481    (match_operand:SI 3 "immediate_operand" "")]
3482   "TARGET_NEON"
3483 {
3484   rtx tmp = gen_reg_rtx (<V_HALF>mode);
3485   emit_insn (gen_neon_vset_lane<V_half> (tmp, operands[2], tmp, const0_rtx));
3486   emit_insn (gen_neon_vqdmulh_lane<mode> (operands[0], operands[1], tmp,
3487                                           const0_rtx, operands[3]));
3488   DONE;
3489 })
3490
3491 (define_expand "neon_vmla_n<mode>"
3492   [(match_operand:VMD 0 "s_register_operand" "")
3493    (match_operand:VMD 1 "s_register_operand" "")
3494    (match_operand:VMD 2 "s_register_operand" "")
3495    (match_operand:<V_elem> 3 "s_register_operand" "")
3496    (match_operand:SI 4 "immediate_operand" "")]
3497   "TARGET_NEON"
3498 {
3499   rtx tmp = gen_reg_rtx (<MODE>mode);
3500   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3501   emit_insn (gen_neon_vmla_lane<mode> (operands[0], operands[1], operands[2],
3502                                        tmp, const0_rtx, operands[4]));
3503   DONE;
3504 })
3505
3506 (define_expand "neon_vmla_n<mode>"
3507   [(match_operand:VMQ 0 "s_register_operand" "")
3508    (match_operand:VMQ 1 "s_register_operand" "")
3509    (match_operand:VMQ 2 "s_register_operand" "")
3510    (match_operand:<V_elem> 3 "s_register_operand" "")
3511    (match_operand:SI 4 "immediate_operand" "")]
3512   "TARGET_NEON"
3513 {
3514   rtx tmp = gen_reg_rtx (<V_HALF>mode);
3515   emit_insn (gen_neon_vset_lane<V_half> (tmp, operands[3], tmp, const0_rtx));
3516   emit_insn (gen_neon_vmla_lane<mode> (operands[0], operands[1], operands[2],
3517                                        tmp, const0_rtx, operands[4]));
3518   DONE;
3519 })
3520
3521 (define_expand "neon_vmlal_n<mode>"
3522   [(match_operand:<V_widen> 0 "s_register_operand" "")
3523    (match_operand:<V_widen> 1 "s_register_operand" "")
3524    (match_operand:VMDI 2 "s_register_operand" "")
3525    (match_operand:<V_elem> 3 "s_register_operand" "")
3526    (match_operand:SI 4 "immediate_operand" "")]
3527   "TARGET_NEON"
3528 {
3529   rtx tmp = gen_reg_rtx (<MODE>mode);
3530   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3531   emit_insn (gen_neon_vmlal_lane<mode> (operands[0], operands[1], operands[2],
3532                                         tmp, const0_rtx, operands[4]));
3533   DONE;
3534 })
3535
3536 (define_expand "neon_vqdmlal_n<mode>"
3537   [(match_operand:<V_widen> 0 "s_register_operand" "")
3538    (match_operand:<V_widen> 1 "s_register_operand" "")
3539    (match_operand:VMDI 2 "s_register_operand" "")
3540    (match_operand:<V_elem> 3 "s_register_operand" "")
3541    (match_operand:SI 4 "immediate_operand" "")]
3542   "TARGET_NEON"
3543 {
3544   rtx tmp = gen_reg_rtx (<MODE>mode);
3545   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3546   emit_insn (gen_neon_vqdmlal_lane<mode> (operands[0], operands[1], operands[2],
3547                                           tmp, const0_rtx, operands[4]));
3548   DONE;
3549 })
3550
3551 (define_expand "neon_vmls_n<mode>"
3552   [(match_operand:VMD 0 "s_register_operand" "")
3553    (match_operand:VMD 1 "s_register_operand" "")
3554    (match_operand:VMD 2 "s_register_operand" "")
3555    (match_operand:<V_elem> 3 "s_register_operand" "")
3556    (match_operand:SI 4 "immediate_operand" "")]
3557   "TARGET_NEON"
3558 {
3559   rtx tmp = gen_reg_rtx (<MODE>mode);
3560   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3561   emit_insn (gen_neon_vmls_lane<mode> (operands[0], operands[1], operands[2],
3562                                        tmp, const0_rtx, operands[4]));
3563   DONE;
3564 })
3565
3566 (define_expand "neon_vmls_n<mode>"
3567   [(match_operand:VMQ 0 "s_register_operand" "")
3568    (match_operand:VMQ 1 "s_register_operand" "")
3569    (match_operand:VMQ 2 "s_register_operand" "")
3570    (match_operand:<V_elem> 3 "s_register_operand" "")
3571    (match_operand:SI 4 "immediate_operand" "")]
3572   "TARGET_NEON"
3573 {
3574   rtx tmp = gen_reg_rtx (<V_HALF>mode);
3575   emit_insn (gen_neon_vset_lane<V_half> (tmp, operands[3], tmp, const0_rtx));
3576   emit_insn (gen_neon_vmls_lane<mode> (operands[0], operands[1], operands[2],
3577                                        tmp, const0_rtx, operands[4]));
3578   DONE;
3579 })
3580
3581 (define_expand "neon_vmlsl_n<mode>"
3582   [(match_operand:<V_widen> 0 "s_register_operand" "")
3583    (match_operand:<V_widen> 1 "s_register_operand" "")
3584    (match_operand:VMDI 2 "s_register_operand" "")
3585    (match_operand:<V_elem> 3 "s_register_operand" "")
3586    (match_operand:SI 4 "immediate_operand" "")]
3587   "TARGET_NEON"
3588 {
3589   rtx tmp = gen_reg_rtx (<MODE>mode);
3590   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3591   emit_insn (gen_neon_vmlsl_lane<mode> (operands[0], operands[1], operands[2],
3592                                         tmp, const0_rtx, operands[4]));
3593   DONE;
3594 })
3595
3596 (define_expand "neon_vqdmlsl_n<mode>"
3597   [(match_operand:<V_widen> 0 "s_register_operand" "")
3598    (match_operand:<V_widen> 1 "s_register_operand" "")
3599    (match_operand:VMDI 2 "s_register_operand" "")
3600    (match_operand:<V_elem> 3 "s_register_operand" "")
3601    (match_operand:SI 4 "immediate_operand" "")]
3602   "TARGET_NEON"
3603 {
3604   rtx tmp = gen_reg_rtx (<MODE>mode);
3605   emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3606   emit_insn (gen_neon_vqdmlsl_lane<mode> (operands[0], operands[1], operands[2],
3607                                           tmp, const0_rtx, operands[4]));
3608   DONE;
3609 })
3610
3611 (define_insn "neon_vext<mode>"
3612   [(set (match_operand:VDQX 0 "s_register_operand" "=w")
3613         (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand" "w")
3614                       (match_operand:VDQX 2 "s_register_operand" "w")
3615                       (match_operand:SI 3 "immediate_operand" "i")]
3616                      UNSPEC_VEXT))]
3617   "TARGET_NEON"
3618 {
3619   neon_const_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3620   return "vext.<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2, %3";
3621 }
3622   [(set_attr "type" "neon_ext<q>")]
3623 )
3624
3625 (define_insn "neon_vrev64<mode>"
3626   [(set (match_operand:VDQ 0 "s_register_operand" "=w")
3627         (unspec:VDQ [(match_operand:VDQ 1 "s_register_operand" "w")
3628                      (match_operand:SI 2 "immediate_operand" "i")]
3629                     UNSPEC_VREV64))]
3630   "TARGET_NEON"
3631   "vrev64.<V_sz_elem>\t%<V_reg>0, %<V_reg>1"
3632   [(set_attr "type" "neon_rev<q>")]
3633 )
3634
3635 (define_insn "neon_vrev32<mode>"
3636   [(set (match_operand:VX 0 "s_register_operand" "=w")
3637         (unspec:VX [(match_operand:VX 1 "s_register_operand" "w")
3638                     (match_operand:SI 2 "immediate_operand" "i")]
3639                    UNSPEC_VREV32))]
3640   "TARGET_NEON"
3641   "vrev32.<V_sz_elem>\t%<V_reg>0, %<V_reg>1"
3642   [(set_attr "type" "neon_rev<q>")]
3643 )
3644
3645 (define_insn "neon_vrev16<mode>"
3646   [(set (match_operand:VE 0 "s_register_operand" "=w")
3647         (unspec:VE [(match_operand:VE 1 "s_register_operand" "w")
3648                     (match_operand:SI 2 "immediate_operand" "i")]
3649                    UNSPEC_VREV16))]
3650   "TARGET_NEON"
3651   "vrev16.<V_sz_elem>\t%<V_reg>0, %<V_reg>1"
3652   [(set_attr "type" "neon_rev<q>")]
3653 )
3654
3655 ; vbsl_* intrinsics may compile to any of vbsl/vbif/vbit depending on register
3656 ; allocation. For an intrinsic of form:
3657 ;   rD = vbsl_* (rS, rN, rM)
3658 ; We can use any of:
3659 ;   vbsl rS, rN, rM  (if D = S)
3660 ;   vbit rD, rN, rS  (if D = M, so 1-bits in rS choose bits from rN, else rM)
3661 ;   vbif rD, rM, rS  (if D = N, so 0-bits in rS choose bits from rM, else rN)
3662
3663 (define_insn "neon_vbsl<mode>_internal"
3664   [(set (match_operand:VDQX 0 "s_register_operand"               "=w,w,w")
3665         (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand" " 0,w,w")
3666                       (match_operand:VDQX 2 "s_register_operand" " w,w,0")
3667                       (match_operand:VDQX 3 "s_register_operand" " w,0,w")]
3668                      UNSPEC_VBSL))]
3669   "TARGET_NEON"
3670   "@
3671   vbsl\t%<V_reg>0, %<V_reg>2, %<V_reg>3
3672   vbit\t%<V_reg>0, %<V_reg>2, %<V_reg>1
3673   vbif\t%<V_reg>0, %<V_reg>3, %<V_reg>1"
3674   [(set_attr "type" "neon_bsl<q>")]
3675 )
3676
3677 (define_expand "neon_vbsl<mode>"
3678   [(set (match_operand:VDQX 0 "s_register_operand" "")
3679         (unspec:VDQX [(match_operand:<V_cmp_result> 1 "s_register_operand" "")
3680                       (match_operand:VDQX 2 "s_register_operand" "")
3681                       (match_operand:VDQX 3 "s_register_operand" "")]
3682                      UNSPEC_VBSL))]
3683   "TARGET_NEON"
3684 {
3685   /* We can't alias operands together if they have different modes.  */
3686   operands[1] = gen_lowpart (<MODE>mode, operands[1]);
3687 })
3688
3689 (define_insn "neon_vshl<mode>"
3690   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3691         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
3692                        (match_operand:VDQIX 2 "s_register_operand" "w")
3693                        (match_operand:SI 3 "immediate_operand" "i")]
3694                       UNSPEC_VSHL))]
3695   "TARGET_NEON"
3696   "v%O3shl.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
3697   [(set_attr "type" "neon_shift_imm<q>")]
3698 )
3699
3700 (define_insn "neon_vqshl<mode>"
3701   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3702         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
3703                        (match_operand:VDQIX 2 "s_register_operand" "w")
3704                        (match_operand:SI 3 "immediate_operand" "i")]
3705                       UNSPEC_VQSHL))]
3706   "TARGET_NEON"
3707   "vq%O3shl.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
3708   [(set_attr "type" "neon_sat_shift_imm<q>")]
3709 )
3710
3711 (define_insn "neon_vshr_n<mode>"
3712   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3713         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
3714                        (match_operand:SI 2 "immediate_operand" "i")
3715                        (match_operand:SI 3 "immediate_operand" "i")]
3716                       UNSPEC_VSHR_N))]
3717   "TARGET_NEON"
3718 {
3719   neon_const_bounds (operands[2], 1, neon_element_bits (<MODE>mode) + 1);
3720   return "v%O3shr.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %2";
3721 }
3722   [(set_attr "type" "neon_shift_imm<q>")]
3723 )
3724
3725 (define_insn "neon_vshrn_n<mode>"
3726   [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
3727         (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
3728                             (match_operand:SI 2 "immediate_operand" "i")
3729                             (match_operand:SI 3 "immediate_operand" "i")]
3730                            UNSPEC_VSHRN_N))]
3731   "TARGET_NEON"
3732 {
3733   neon_const_bounds (operands[2], 1, neon_element_bits (<MODE>mode) / 2 + 1);
3734   return "v%O3shrn.<V_if_elem>\t%P0, %q1, %2";
3735 }
3736   [(set_attr "type" "neon_shift_imm_narrow_q")]
3737 )
3738
3739 (define_insn "neon_vqshrn_n<mode>"
3740   [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
3741         (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
3742                             (match_operand:SI 2 "immediate_operand" "i")
3743                             (match_operand:SI 3 "immediate_operand" "i")]
3744                            UNSPEC_VQSHRN_N))]
3745   "TARGET_NEON"
3746 {
3747   neon_const_bounds (operands[2], 1, neon_element_bits (<MODE>mode) / 2 + 1);
3748   return "vq%O3shrn.%T3%#<V_sz_elem>\t%P0, %q1, %2";
3749 }
3750   [(set_attr "type" "neon_sat_shift_imm_narrow_q")]
3751 )
3752
3753 (define_insn "neon_vqshrun_n<mode>"
3754   [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
3755         (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
3756                             (match_operand:SI 2 "immediate_operand" "i")
3757                             (match_operand:SI 3 "immediate_operand" "i")]
3758                            UNSPEC_VQSHRUN_N))]
3759   "TARGET_NEON"
3760 {
3761   neon_const_bounds (operands[2], 1, neon_element_bits (<MODE>mode) / 2 + 1);
3762   return "vq%O3shrun.%T3%#<V_sz_elem>\t%P0, %q1, %2";
3763 }
3764   [(set_attr "type" "neon_sat_shift_imm_narrow_q")]
3765 )
3766
3767 (define_insn "neon_vshl_n<mode>"
3768   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3769         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
3770                        (match_operand:SI 2 "immediate_operand" "i")
3771                        (match_operand:SI 3 "immediate_operand" "i")]
3772                       UNSPEC_VSHL_N))]
3773   "TARGET_NEON"
3774 {
3775   neon_const_bounds (operands[2], 0, neon_element_bits (<MODE>mode));
3776   return "vshl.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %2";
3777 }
3778   [(set_attr "type" "neon_shift_imm<q>")]
3779 )
3780
3781 (define_insn "neon_vqshl_n<mode>"
3782   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3783         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
3784                        (match_operand:SI 2 "immediate_operand" "i")
3785                        (match_operand:SI 3 "immediate_operand" "i")]
3786                       UNSPEC_VQSHL_N))]
3787   "TARGET_NEON"
3788 {
3789   neon_const_bounds (operands[2], 0, neon_element_bits (<MODE>mode));
3790   return "vqshl.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %2";
3791 }
3792   [(set_attr "type" "neon_sat_shift_imm<q>")]
3793 )
3794
3795 (define_insn "neon_vqshlu_n<mode>"
3796   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3797         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
3798                        (match_operand:SI 2 "immediate_operand" "i")
3799                        (match_operand:SI 3 "immediate_operand" "i")]
3800                       UNSPEC_VQSHLU_N))]
3801   "TARGET_NEON"
3802 {
3803   neon_const_bounds (operands[2], 0, neon_element_bits (<MODE>mode));
3804   return "vqshlu.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %2";
3805 }
3806   [(set_attr "type" "neon_sat_shift_imm<q>")]
3807 )
3808
3809 (define_insn "neon_vshll_n<mode>"
3810   [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3811         (unspec:<V_widen> [(match_operand:VW 1 "s_register_operand" "w")
3812                            (match_operand:SI 2 "immediate_operand" "i")
3813                            (match_operand:SI 3 "immediate_operand" "i")]
3814                           UNSPEC_VSHLL_N))]
3815   "TARGET_NEON"
3816 {
3817   /* The boundaries are: 0 < imm <= size.  */
3818   neon_const_bounds (operands[2], 0, neon_element_bits (<MODE>mode) + 1);
3819   return "vshll.%T3%#<V_sz_elem>\t%q0, %P1, %2";
3820 }
3821   [(set_attr "type" "neon_shift_imm_long")]
3822 )
3823
3824 (define_insn "neon_vsra_n<mode>"
3825   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3826         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "0")
3827                        (match_operand:VDQIX 2 "s_register_operand" "w")
3828                        (match_operand:SI 3 "immediate_operand" "i")
3829                        (match_operand:SI 4 "immediate_operand" "i")]
3830                       UNSPEC_VSRA_N))]
3831   "TARGET_NEON"
3832 {
3833   neon_const_bounds (operands[3], 1, neon_element_bits (<MODE>mode) + 1);
3834   return "v%O4sra.%T4%#<V_sz_elem>\t%<V_reg>0, %<V_reg>2, %3";
3835 }
3836   [(set_attr "type" "neon_shift_acc<q>")]
3837 )
3838
3839 (define_insn "neon_vsri_n<mode>"
3840   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3841         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "0")
3842                        (match_operand:VDQIX 2 "s_register_operand" "w")
3843                        (match_operand:SI 3 "immediate_operand" "i")]
3844                       UNSPEC_VSRI))]
3845   "TARGET_NEON"
3846 {
3847   neon_const_bounds (operands[3], 1, neon_element_bits (<MODE>mode) + 1);
3848   return "vsri.<V_sz_elem>\t%<V_reg>0, %<V_reg>2, %3";
3849 }
3850   [(set_attr "type" "neon_shift_reg<q>")]
3851 )
3852
3853 (define_insn "neon_vsli_n<mode>"
3854   [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3855         (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "0")
3856                        (match_operand:VDQIX 2 "s_register_operand" "w")
3857                        (match_operand:SI 3 "immediate_operand" "i")]
3858                       UNSPEC_VSLI))]
3859   "TARGET_NEON"
3860 {
3861   neon_const_bounds (operands[3], 0, neon_element_bits (<MODE>mode));
3862   return "vsli.<V_sz_elem>\t%<V_reg>0, %<V_reg>2, %3";
3863 }
3864   [(set_attr "type" "neon_shift_reg<q>")]
3865 )
3866
3867 (define_insn "neon_vtbl1v8qi"
3868   [(set (match_operand:V8QI 0 "s_register_operand" "=w")
3869         (unspec:V8QI [(match_operand:V8QI 1 "s_register_operand" "w")
3870                       (match_operand:V8QI 2 "s_register_operand" "w")]
3871                      UNSPEC_VTBL))]
3872   "TARGET_NEON"
3873   "vtbl.8\t%P0, {%P1}, %P2"
3874   [(set_attr "type" "neon_tbl1")]
3875 )
3876
3877 (define_insn "neon_vtbl2v8qi"
3878   [(set (match_operand:V8QI 0 "s_register_operand" "=w")
3879         (unspec:V8QI [(match_operand:TI 1 "s_register_operand" "w")
3880                       (match_operand:V8QI 2 "s_register_operand" "w")]
3881                      UNSPEC_VTBL))]
3882   "TARGET_NEON"
3883 {
3884   rtx ops[4];
3885   int tabbase = REGNO (operands[1]);
3886
3887   ops[0] = operands[0];
3888   ops[1] = gen_rtx_REG (V8QImode, tabbase);
3889   ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
3890   ops[3] = operands[2];
3891   output_asm_insn ("vtbl.8\t%P0, {%P1, %P2}, %P3", ops);
3892
3893   return "";
3894 }
3895   [(set_attr "type" "neon_tbl2")]
3896 )
3897
3898 (define_insn "neon_vtbl3v8qi"
3899   [(set (match_operand:V8QI 0 "s_register_operand" "=w")
3900         (unspec:V8QI [(match_operand:EI 1 "s_register_operand" "w")
3901                       (match_operand:V8QI 2 "s_register_operand" "w")]
3902                      UNSPEC_VTBL))]
3903   "TARGET_NEON"
3904 {
3905   rtx ops[5];
3906   int tabbase = REGNO (operands[1]);
3907
3908   ops[0] = operands[0];
3909   ops[1] = gen_rtx_REG (V8QImode, tabbase);
3910   ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
3911   ops[3] = gen_rtx_REG (V8QImode, tabbase + 4);
3912   ops[4] = operands[2];
3913   output_asm_insn ("vtbl.8\t%P0, {%P1, %P2, %P3}, %P4", ops);
3914
3915   return "";
3916 }
3917   [(set_attr "type" "neon_tbl3")]
3918 )
3919
3920 (define_insn "neon_vtbl4v8qi"
3921   [(set (match_operand:V8QI 0 "s_register_operand" "=w")
3922         (unspec:V8QI [(match_operand:OI 1 "s_register_operand" "w")
3923                       (match_operand:V8QI 2 "s_register_operand" "w")]
3924                      UNSPEC_VTBL))]
3925   "TARGET_NEON"
3926 {
3927   rtx ops[6];
3928   int tabbase = REGNO (operands[1]);
3929
3930   ops[0] = operands[0];
3931   ops[1] = gen_rtx_REG (V8QImode, tabbase);
3932   ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
3933   ops[3] = gen_rtx_REG (V8QImode, tabbase + 4);
3934   ops[4] = gen_rtx_REG (V8QImode, tabbase + 6);
3935   ops[5] = operands[2];
3936   output_asm_insn ("vtbl.8\t%P0, {%P1, %P2, %P3, %P4}, %P5", ops);
3937
3938   return "";
3939 }
3940   [(set_attr "type" "neon_tbl4")]
3941 )
3942
3943 ;; These three are used by the vec_perm infrastructure for V16QImode.
3944 (define_insn_and_split "neon_vtbl1v16qi"
3945   [(set (match_operand:V16QI 0 "s_register_operand" "=&w")
3946         (unspec:V16QI [(match_operand:V16QI 1 "s_register_operand" "w")
3947                        (match_operand:V16QI 2 "s_register_operand" "w")]
3948                       UNSPEC_VTBL))]
3949   "TARGET_NEON"
3950   "#"
3951   "&& reload_completed"
3952   [(const_int 0)]
3953 {
3954   rtx op0, op1, op2, part0, part2;
3955   unsigned ofs;
3956
3957   op0 = operands[0];
3958   op1 = gen_lowpart (TImode, operands[1]);
3959   op2 = operands[2];
3960
3961   ofs = subreg_lowpart_offset (V8QImode, V16QImode);
3962   part0 = simplify_subreg (V8QImode, op0, V16QImode, ofs);
3963   part2 = simplify_subreg (V8QImode, op2, V16QImode, ofs);
3964   emit_insn (gen_neon_vtbl2v8qi (part0, op1, part2));
3965
3966   ofs = subreg_highpart_offset (V8QImode, V16QImode);
3967   part0 = simplify_subreg (V8QImode, op0, V16QImode, ofs);
3968   part2 = simplify_subreg (V8QImode, op2, V16QImode, ofs);
3969   emit_insn (gen_neon_vtbl2v8qi (part0, op1, part2));
3970   DONE;
3971 }
3972   [(set_attr "type" "multiple")]
3973 )
3974
3975 (define_insn_and_split "neon_vtbl2v16qi"
3976   [(set (match_operand:V16QI 0 "s_register_operand" "=&w")
3977         (unspec:V16QI [(match_operand:OI 1 "s_register_operand" "w")
3978                        (match_operand:V16QI 2 "s_register_operand" "w")]
3979                       UNSPEC_VTBL))]
3980   "TARGET_NEON"
3981   "#"
3982   "&& reload_completed"
3983   [(const_int 0)]
3984 {
3985   rtx op0, op1, op2, part0, part2;
3986   unsigned ofs;
3987
3988   op0 = operands[0];
3989   op1 = operands[1];
3990   op2 = operands[2];
3991
3992   ofs = subreg_lowpart_offset (V8QImode, V16QImode);
3993   part0 = simplify_subreg (V8QImode, op0, V16QImode, ofs);
3994   part2 = simplify_subreg (V8QImode, op2, V16QImode, ofs);
3995   emit_insn (gen_neon_vtbl2v8qi (part0, op1, part2));
3996
3997   ofs = subreg_highpart_offset (V8QImode, V16QImode);
3998   part0 = simplify_subreg (V8QImode, op0, V16QImode, ofs);
3999   part2 = simplify_subreg (V8QImode, op2, V16QImode, ofs);
4000   emit_insn (gen_neon_vtbl2v8qi (part0, op1, part2));
4001   DONE;
4002 }
4003   [(set_attr "type" "multiple")]
4004 )
4005
4006 ;; ??? Logically we should extend the regular neon_vcombine pattern to
4007 ;; handle quad-word input modes, producing octa-word output modes.  But
4008 ;; that requires us to add support for octa-word vector modes in moves.
4009 ;; That seems overkill for this one use in vec_perm.
4010 (define_insn_and_split "neon_vcombinev16qi"
4011   [(set (match_operand:OI 0 "s_register_operand" "=w")
4012         (unspec:OI [(match_operand:V16QI 1 "s_register_operand" "w")
4013                     (match_operand:V16QI 2 "s_register_operand" "w")]
4014                    UNSPEC_VCONCAT))]
4015   "TARGET_NEON"
4016   "#"
4017   "&& reload_completed"
4018   [(const_int 0)]
4019 {
4020   neon_split_vcombine (operands);
4021   DONE;
4022 }
4023 [(set_attr "type" "multiple")]
4024 )
4025
4026 (define_insn "neon_vtbx1v8qi"
4027   [(set (match_operand:V8QI 0 "s_register_operand" "=w")
4028         (unspec:V8QI [(match_operand:V8QI 1 "s_register_operand" "0")
4029                       (match_operand:V8QI 2 "s_register_operand" "w")
4030                       (match_operand:V8QI 3 "s_register_operand" "w")]
4031                      UNSPEC_VTBX))]
4032   "TARGET_NEON"
4033   "vtbx.8\t%P0, {%P2}, %P3"
4034   [(set_attr "type" "neon_tbl1")]
4035 )
4036
4037 (define_insn "neon_vtbx2v8qi"
4038   [(set (match_operand:V8QI 0 "s_register_operand" "=w")
4039         (unspec:V8QI [(match_operand:V8QI 1 "s_register_operand" "0")
4040                       (match_operand:TI 2 "s_register_operand" "w")
4041                       (match_operand:V8QI 3 "s_register_operand" "w")]
4042                      UNSPEC_VTBX))]
4043   "TARGET_NEON"
4044 {
4045   rtx ops[4];
4046   int tabbase = REGNO (operands[2]);
4047
4048   ops[0] = operands[0];
4049   ops[1] = gen_rtx_REG (V8QImode, tabbase);
4050   ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
4051   ops[3] = operands[3];
4052   output_asm_insn ("vtbx.8\t%P0, {%P1, %P2}, %P3", ops);
4053
4054   return "";
4055 }
4056   [(set_attr "type" "neon_tbl2")]
4057 )
4058
4059 (define_insn "neon_vtbx3v8qi"
4060   [(set (match_operand:V8QI 0 "s_register_operand" "=w")
4061         (unspec:V8QI [(match_operand:V8QI 1 "s_register_operand" "0")
4062                       (match_operand:EI 2 "s_register_operand" "w")
4063                       (match_operand:V8QI 3 "s_register_operand" "w")]
4064                      UNSPEC_VTBX))]
4065   "TARGET_NEON"
4066 {
4067   rtx ops[5];
4068   int tabbase = REGNO (operands[2]);
4069
4070   ops[0] = operands[0];
4071   ops[1] = gen_rtx_REG (V8QImode, tabbase);
4072   ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
4073   ops[3] = gen_rtx_REG (V8QImode, tabbase + 4);
4074   ops[4] = operands[3];
4075   output_asm_insn ("vtbx.8\t%P0, {%P1, %P2, %P3}, %P4", ops);
4076
4077   return "";
4078 }
4079   [(set_attr "type" "neon_tbl3")]
4080 )
4081
4082 (define_insn "neon_vtbx4v8qi"
4083   [(set (match_operand:V8QI 0 "s_register_operand" "=w")
4084         (unspec:V8QI [(match_operand:V8QI 1 "s_register_operand" "0")
4085                       (match_operand:OI 2 "s_register_operand" "w")
4086                       (match_operand:V8QI 3 "s_register_operand" "w")]
4087                      UNSPEC_VTBX))]
4088   "TARGET_NEON"
4089 {
4090   rtx ops[6];
4091   int tabbase = REGNO (operands[2]);
4092
4093   ops[0] = operands[0];
4094   ops[1] = gen_rtx_REG (V8QImode, tabbase);
4095   ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
4096   ops[3] = gen_rtx_REG (V8QImode, tabbase + 4);
4097   ops[4] = gen_rtx_REG (V8QImode, tabbase + 6);
4098   ops[5] = operands[3];
4099   output_asm_insn ("vtbx.8\t%P0, {%P1, %P2, %P3, %P4}, %P5", ops);
4100
4101   return "";
4102 }
4103   [(set_attr "type" "neon_tbl4")]
4104 )
4105
4106 (define_expand "neon_vtrn<mode>_internal"
4107   [(parallel
4108     [(set (match_operand:VDQW 0 "s_register_operand" "")
4109           (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "")
4110                         (match_operand:VDQW 2 "s_register_operand" "")]
4111            UNSPEC_VTRN1))
4112      (set (match_operand:VDQW 3 "s_register_operand" "")
4113           (unspec:VDQW [(match_dup 1) (match_dup 2)] UNSPEC_VTRN2))])]
4114   "TARGET_NEON"
4115   ""
4116 )
4117
4118 ;; Note: Different operand numbering to handle tied registers correctly.
4119 (define_insn "*neon_vtrn<mode>_insn"
4120   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
4121         (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "0")
4122                       (match_operand:VDQW 3 "s_register_operand" "2")]
4123                      UNSPEC_VTRN1))
4124    (set (match_operand:VDQW 2 "s_register_operand" "=w")
4125          (unspec:VDQW [(match_dup 1) (match_dup 3)]
4126                      UNSPEC_VTRN2))]
4127   "TARGET_NEON"
4128   "vtrn.<V_sz_elem>\t%<V_reg>0, %<V_reg>2"
4129   [(set_attr "type" "neon_permute<q>")]
4130 )
4131
4132 (define_expand "neon_vtrn<mode>"
4133   [(match_operand:SI 0 "s_register_operand" "r")
4134    (match_operand:VDQW 1 "s_register_operand" "w")
4135    (match_operand:VDQW 2 "s_register_operand" "w")]
4136   "TARGET_NEON"
4137 {
4138   neon_emit_pair_result_insn (<MODE>mode, gen_neon_vtrn<mode>_internal,
4139                               operands[0], operands[1], operands[2]);
4140   DONE;
4141 })
4142
4143 (define_expand "neon_vzip<mode>_internal"
4144   [(parallel
4145     [(set (match_operand:VDQW 0 "s_register_operand" "")
4146           (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "")
4147                         (match_operand:VDQW 2 "s_register_operand" "")]
4148                        UNSPEC_VZIP1))
4149     (set (match_operand:VDQW 3 "s_register_operand" "")
4150          (unspec:VDQW [(match_dup 1) (match_dup 2)] UNSPEC_VZIP2))])]
4151   "TARGET_NEON"
4152   ""
4153 )
4154
4155 ;; Note: Different operand numbering to handle tied registers correctly.
4156 (define_insn "*neon_vzip<mode>_insn"
4157   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
4158         (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "0")
4159                       (match_operand:VDQW 3 "s_register_operand" "2")]
4160                      UNSPEC_VZIP1))
4161    (set (match_operand:VDQW 2 "s_register_operand" "=w")
4162         (unspec:VDQW [(match_dup 1) (match_dup 3)]
4163                      UNSPEC_VZIP2))]
4164   "TARGET_NEON"
4165   "vzip.<V_sz_elem>\t%<V_reg>0, %<V_reg>2"
4166   [(set_attr "type" "neon_zip<q>")]
4167 )
4168
4169 (define_expand "neon_vzip<mode>"
4170   [(match_operand:SI 0 "s_register_operand" "r")
4171    (match_operand:VDQW 1 "s_register_operand" "w")
4172    (match_operand:VDQW 2 "s_register_operand" "w")]
4173   "TARGET_NEON"
4174 {
4175   neon_emit_pair_result_insn (<MODE>mode, gen_neon_vzip<mode>_internal,
4176                               operands[0], operands[1], operands[2]);
4177   DONE;
4178 })
4179
4180 (define_expand "neon_vuzp<mode>_internal"
4181   [(parallel
4182     [(set (match_operand:VDQW 0 "s_register_operand" "")
4183           (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "")
4184                         (match_operand:VDQW 2 "s_register_operand" "")]
4185            UNSPEC_VUZP1))
4186      (set (match_operand:VDQW 3 "s_register_operand" "")
4187           (unspec:VDQW [(match_dup 1) (match_dup 2)] UNSPEC_VUZP2))])]
4188   "TARGET_NEON"
4189   ""
4190 )
4191
4192 ;; Note: Different operand numbering to handle tied registers correctly.
4193 (define_insn "*neon_vuzp<mode>_insn"
4194   [(set (match_operand:VDQW 0 "s_register_operand" "=w")
4195         (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "0")
4196                       (match_operand:VDQW 3 "s_register_operand" "2")]
4197                      UNSPEC_VUZP1))
4198    (set (match_operand:VDQW 2 "s_register_operand" "=w")
4199         (unspec:VDQW [(match_dup 1) (match_dup 3)]
4200                      UNSPEC_VUZP2))]
4201   "TARGET_NEON"
4202   "vuzp.<V_sz_elem>\t%<V_reg>0, %<V_reg>2"
4203   [(set_attr "type" "neon_zip<q>")]
4204 )
4205
4206 (define_expand "neon_vuzp<mode>"
4207   [(match_operand:SI 0 "s_register_operand" "r")
4208    (match_operand:VDQW 1 "s_register_operand" "w")
4209    (match_operand:VDQW 2 "s_register_operand" "w")]
4210   "TARGET_NEON"
4211 {
4212   neon_emit_pair_result_insn (<MODE>mode, gen_neon_vuzp<mode>_internal,
4213                               operands[0], operands[1], operands[2]);
4214   DONE;
4215 })
4216
4217 (define_expand "neon_vreinterpretv8qi<mode>"
4218   [(match_operand:V8QI 0 "s_register_operand" "")
4219    (match_operand:VDX 1 "s_register_operand" "")]
4220   "TARGET_NEON"
4221 {
4222   neon_reinterpret (operands[0], operands[1]);
4223   DONE;
4224 })
4225
4226 (define_expand "neon_vreinterpretv4hi<mode>"
4227   [(match_operand:V4HI 0 "s_register_operand" "")
4228    (match_operand:VDX 1 "s_register_operand" "")]
4229   "TARGET_NEON"
4230 {
4231   neon_reinterpret (operands[0], operands[1]);
4232   DONE;
4233 })
4234
4235 (define_expand "neon_vreinterpretv2si<mode>"
4236   [(match_operand:V2SI 0 "s_register_operand" "")
4237    (match_operand:VDX 1 "s_register_operand" "")]
4238   "TARGET_NEON"
4239 {
4240   neon_reinterpret (operands[0], operands[1]);
4241   DONE;
4242 })
4243
4244 (define_expand "neon_vreinterpretv2sf<mode>"
4245   [(match_operand:V2SF 0 "s_register_operand" "")
4246    (match_operand:VDX 1 "s_register_operand" "")]
4247   "TARGET_NEON"
4248 {
4249   neon_reinterpret (operands[0], operands[1]);
4250   DONE;
4251 })
4252
4253 (define_expand "neon_vreinterpretdi<mode>"
4254   [(match_operand:DI 0 "s_register_operand" "")
4255    (match_operand:VDX 1 "s_register_operand" "")]
4256   "TARGET_NEON"
4257 {
4258   neon_reinterpret (operands[0], operands[1]);
4259   DONE;
4260 })
4261
4262 (define_expand "neon_vreinterpretv16qi<mode>"
4263   [(match_operand:V16QI 0 "s_register_operand" "")
4264    (match_operand:VQX 1 "s_register_operand" "")]
4265   "TARGET_NEON"
4266 {
4267   neon_reinterpret (operands[0], operands[1]);
4268   DONE;
4269 })
4270
4271 (define_expand "neon_vreinterpretv8hi<mode>"
4272   [(match_operand:V8HI 0 "s_register_operand" "")
4273    (match_operand:VQX 1 "s_register_operand" "")]
4274   "TARGET_NEON"
4275 {
4276   neon_reinterpret (operands[0], operands[1]);
4277   DONE;
4278 })
4279
4280 (define_expand "neon_vreinterpretv4si<mode>"
4281   [(match_operand:V4SI 0 "s_register_operand" "")
4282    (match_operand:VQX 1 "s_register_operand" "")]
4283   "TARGET_NEON"
4284 {
4285   neon_reinterpret (operands[0], operands[1]);
4286   DONE;
4287 })
4288
4289 (define_expand "neon_vreinterpretv4sf<mode>"
4290   [(match_operand:V4SF 0 "s_register_operand" "")
4291    (match_operand:VQX 1 "s_register_operand" "")]
4292   "TARGET_NEON"
4293 {
4294   neon_reinterpret (operands[0], operands[1]);
4295   DONE;
4296 })
4297
4298 (define_expand "neon_vreinterpretv2di<mode>"
4299   [(match_operand:V2DI 0 "s_register_operand" "")
4300    (match_operand:VQX 1 "s_register_operand" "")]
4301   "TARGET_NEON"
4302 {
4303   neon_reinterpret (operands[0], operands[1]);
4304   DONE;
4305 })
4306
4307 (define_expand "vec_load_lanes<mode><mode>"
4308   [(set (match_operand:VDQX 0 "s_register_operand")
4309         (unspec:VDQX [(match_operand:VDQX 1 "neon_struct_operand")]
4310                      UNSPEC_VLD1))]
4311   "TARGET_NEON")
4312
4313 (define_insn "neon_vld1<mode>"
4314   [(set (match_operand:VDQX 0 "s_register_operand" "=w")
4315         (unspec:VDQX [(match_operand:VDQX 1 "neon_struct_operand" "Um")]
4316                     UNSPEC_VLD1))]
4317   "TARGET_NEON"
4318   "vld1.<V_sz_elem>\t%h0, %A1"
4319   [(set_attr "type" "neon_load1_1reg<q>")]
4320 )
4321
4322 (define_insn "neon_vld1_lane<mode>"
4323   [(set (match_operand:VDX 0 "s_register_operand" "=w")
4324         (unspec:VDX [(match_operand:<V_elem> 1 "neon_struct_operand" "Um")
4325                      (match_operand:VDX 2 "s_register_operand" "0")
4326                      (match_operand:SI 3 "immediate_operand" "i")]
4327                     UNSPEC_VLD1_LANE))]
4328   "TARGET_NEON"
4329 {
4330   HOST_WIDE_INT lane = INTVAL (operands[3]);
4331   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4332   if (lane < 0 || lane >= max)
4333     error ("lane out of range");
4334   if (max == 1)
4335     return "vld1.<V_sz_elem>\t%P0, %A1";
4336   else
4337     return "vld1.<V_sz_elem>\t{%P0[%c3]}, %A1";
4338 }
4339   [(set_attr "type" "neon_load1_one_lane<q>")]
4340 )
4341
4342 (define_insn "neon_vld1_lane<mode>"
4343   [(set (match_operand:VQX 0 "s_register_operand" "=w")
4344         (unspec:VQX [(match_operand:<V_elem> 1 "neon_struct_operand" "Um")
4345                      (match_operand:VQX 2 "s_register_operand" "0")
4346                      (match_operand:SI 3 "immediate_operand" "i")]
4347                     UNSPEC_VLD1_LANE))]
4348   "TARGET_NEON"
4349 {
4350   HOST_WIDE_INT lane = INTVAL (operands[3]);
4351   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4352   int regno = REGNO (operands[0]);
4353   if (lane < 0 || lane >= max)
4354     error ("lane out of range");
4355   else if (lane >= max / 2)
4356     {
4357       lane -= max / 2;
4358       regno += 2;
4359       operands[3] = GEN_INT (lane);
4360     }
4361   operands[0] = gen_rtx_REG (<V_HALF>mode, regno);
4362   if (max == 2)
4363     return "vld1.<V_sz_elem>\t%P0, %A1";
4364   else
4365     return "vld1.<V_sz_elem>\t{%P0[%c3]}, %A1";
4366 }
4367   [(set_attr "type" "neon_load1_one_lane<q>")]
4368 )
4369
4370 (define_insn "neon_vld1_dup<mode>"
4371   [(set (match_operand:VD 0 "s_register_operand" "=w")
4372         (vec_duplicate:VD (match_operand:<V_elem> 1 "neon_struct_operand" "Um")))]
4373   "TARGET_NEON"
4374   "vld1.<V_sz_elem>\t{%P0[]}, %A1"
4375   [(set_attr "type" "neon_load1_all_lanes<q>")]
4376 )
4377
4378 ;; Special case for DImode.  Treat it exactly like a simple load.
4379 (define_expand "neon_vld1_dupdi"
4380   [(set (match_operand:DI 0 "s_register_operand" "")
4381         (unspec:DI [(match_operand:DI 1 "neon_struct_operand" "")]
4382                    UNSPEC_VLD1))]
4383   "TARGET_NEON"
4384   ""
4385 )
4386
4387 (define_insn "neon_vld1_dup<mode>"
4388   [(set (match_operand:VQ 0 "s_register_operand" "=w")
4389         (vec_duplicate:VQ (match_operand:<V_elem> 1 "neon_struct_operand" "Um")))]
4390   "TARGET_NEON"
4391 {
4392   return "vld1.<V_sz_elem>\t{%e0[], %f0[]}, %A1";
4393 }
4394   [(set_attr "type" "neon_load1_all_lanes<q>")]
4395 )
4396
4397 (define_insn_and_split "neon_vld1_dupv2di"
4398    [(set (match_operand:V2DI 0 "s_register_operand" "=w")
4399     (vec_duplicate:V2DI (match_operand:DI 1 "neon_struct_operand" "Um")))]
4400    "TARGET_NEON"
4401    "#"
4402    "&& reload_completed"
4403    [(const_int 0)]
4404    {
4405     rtx tmprtx = gen_lowpart (DImode, operands[0]);
4406     emit_insn (gen_neon_vld1_dupdi (tmprtx, operands[1]));
4407     emit_move_insn (gen_highpart (DImode, operands[0]), tmprtx );
4408     DONE;
4409     }
4410   [(set_attr "length" "8")
4411    (set_attr "type" "neon_load1_all_lanes_q")]
4412 )
4413
4414 (define_expand "vec_store_lanes<mode><mode>"
4415   [(set (match_operand:VDQX 0 "neon_struct_operand")
4416         (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand")]
4417                      UNSPEC_VST1))]
4418   "TARGET_NEON")
4419
4420 (define_insn "neon_vst1<mode>"
4421   [(set (match_operand:VDQX 0 "neon_struct_operand" "=Um")
4422         (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand" "w")]
4423                      UNSPEC_VST1))]
4424   "TARGET_NEON"
4425   "vst1.<V_sz_elem>\t%h1, %A0"
4426   [(set_attr "type" "neon_store1_1reg<q>")])
4427
4428 (define_insn "neon_vst1_lane<mode>"
4429   [(set (match_operand:<V_elem> 0 "neon_struct_operand" "=Um")
4430         (unspec:<V_elem>
4431           [(match_operand:VDX 1 "s_register_operand" "w")
4432            (match_operand:SI 2 "immediate_operand" "i")]
4433           UNSPEC_VST1_LANE))]
4434   "TARGET_NEON"
4435 {
4436   HOST_WIDE_INT lane = INTVAL (operands[2]);
4437   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4438   if (lane < 0 || lane >= max)
4439     error ("lane out of range");
4440   if (max == 1)
4441     return "vst1.<V_sz_elem>\t{%P1}, %A0";
4442   else
4443     return "vst1.<V_sz_elem>\t{%P1[%c2]}, %A0";
4444 }
4445   [(set_attr "type" "neon_store1_one_lane<q>")]
4446 )
4447
4448 (define_insn "neon_vst1_lane<mode>"
4449   [(set (match_operand:<V_elem> 0 "neon_struct_operand" "=Um")
4450         (unspec:<V_elem>
4451           [(match_operand:VQX 1 "s_register_operand" "w")
4452            (match_operand:SI 2 "immediate_operand" "i")]
4453           UNSPEC_VST1_LANE))]
4454   "TARGET_NEON"
4455 {
4456   HOST_WIDE_INT lane = INTVAL (operands[2]);
4457   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4458   int regno = REGNO (operands[1]);
4459   if (lane < 0 || lane >= max)
4460     error ("lane out of range");
4461   else if (lane >= max / 2)
4462     {
4463       lane -= max / 2;
4464       regno += 2;
4465       operands[2] = GEN_INT (lane);
4466     }
4467   operands[1] = gen_rtx_REG (<V_HALF>mode, regno);
4468   if (max == 2)
4469     return "vst1.<V_sz_elem>\t{%P1}, %A0";
4470   else
4471     return "vst1.<V_sz_elem>\t{%P1[%c2]}, %A0";
4472 }
4473   [(set_attr "type" "neon_store1_one_lane<q>")]
4474 )
4475
4476 (define_expand "vec_load_lanesti<mode>"
4477   [(set (match_operand:TI 0 "s_register_operand")
4478         (unspec:TI [(match_operand:TI 1 "neon_struct_operand")
4479                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4480                    UNSPEC_VLD2))]
4481   "TARGET_NEON")
4482
4483 (define_insn "neon_vld2<mode>"
4484   [(set (match_operand:TI 0 "s_register_operand" "=w")
4485         (unspec:TI [(match_operand:TI 1 "neon_struct_operand" "Um")
4486                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4487                    UNSPEC_VLD2))]
4488   "TARGET_NEON"
4489 {
4490   if (<V_sz_elem> == 64)
4491     return "vld1.64\t%h0, %A1";
4492   else
4493     return "vld2.<V_sz_elem>\t%h0, %A1";
4494 }
4495   [(set (attr "type")
4496       (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
4497                     (const_string "neon_load1_2reg<q>")
4498                     (const_string "neon_load2_2reg<q>")))]
4499 )
4500
4501 (define_expand "vec_load_lanesoi<mode>"
4502   [(set (match_operand:OI 0 "s_register_operand")
4503         (unspec:OI [(match_operand:OI 1 "neon_struct_operand")
4504                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4505                    UNSPEC_VLD2))]
4506   "TARGET_NEON")
4507
4508 (define_insn "neon_vld2<mode>"
4509   [(set (match_operand:OI 0 "s_register_operand" "=w")
4510         (unspec:OI [(match_operand:OI 1 "neon_struct_operand" "Um")
4511                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4512                    UNSPEC_VLD2))]
4513   "TARGET_NEON"
4514   "vld2.<V_sz_elem>\t%h0, %A1"
4515   [(set_attr "type" "neon_load2_2reg_q")])
4516
4517 (define_insn "neon_vld2_lane<mode>"
4518   [(set (match_operand:TI 0 "s_register_operand" "=w")
4519         (unspec:TI [(match_operand:<V_two_elem> 1 "neon_struct_operand" "Um")
4520                     (match_operand:TI 2 "s_register_operand" "0")
4521                     (match_operand:SI 3 "immediate_operand" "i")
4522                     (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4523                    UNSPEC_VLD2_LANE))]
4524   "TARGET_NEON"
4525 {
4526   HOST_WIDE_INT lane = INTVAL (operands[3]);
4527   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4528   int regno = REGNO (operands[0]);
4529   rtx ops[4];
4530   if (lane < 0 || lane >= max)
4531     error ("lane out of range");
4532   ops[0] = gen_rtx_REG (DImode, regno);
4533   ops[1] = gen_rtx_REG (DImode, regno + 2);
4534   ops[2] = operands[1];
4535   ops[3] = operands[3];
4536   output_asm_insn ("vld2.<V_sz_elem>\t{%P0[%c3], %P1[%c3]}, %A2", ops);
4537   return "";
4538 }
4539   [(set_attr "type" "neon_load2_one_lane<q>")]
4540 )
4541
4542 (define_insn "neon_vld2_lane<mode>"
4543   [(set (match_operand:OI 0 "s_register_operand" "=w")
4544         (unspec:OI [(match_operand:<V_two_elem> 1 "neon_struct_operand" "Um")
4545                     (match_operand:OI 2 "s_register_operand" "0")
4546                     (match_operand:SI 3 "immediate_operand" "i")
4547                     (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4548                    UNSPEC_VLD2_LANE))]
4549   "TARGET_NEON"
4550 {
4551   HOST_WIDE_INT lane = INTVAL (operands[3]);
4552   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4553   int regno = REGNO (operands[0]);
4554   rtx ops[4];
4555   if (lane < 0 || lane >= max)
4556     error ("lane out of range");
4557   else if (lane >= max / 2)
4558     {
4559       lane -= max / 2;
4560       regno += 2;
4561     }
4562   ops[0] = gen_rtx_REG (DImode, regno);
4563   ops[1] = gen_rtx_REG (DImode, regno + 4);
4564   ops[2] = operands[1];
4565   ops[3] = GEN_INT (lane);
4566   output_asm_insn ("vld2.<V_sz_elem>\t{%P0[%c3], %P1[%c3]}, %A2", ops);
4567   return "";
4568 }
4569   [(set_attr "type" "neon_load2_one_lane<q>")]
4570 )
4571
4572 (define_insn "neon_vld2_dup<mode>"
4573   [(set (match_operand:TI 0 "s_register_operand" "=w")
4574         (unspec:TI [(match_operand:<V_two_elem> 1 "neon_struct_operand" "Um")
4575                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4576                    UNSPEC_VLD2_DUP))]
4577   "TARGET_NEON"
4578 {
4579   if (GET_MODE_NUNITS (<MODE>mode) > 1)
4580     return "vld2.<V_sz_elem>\t{%e0[], %f0[]}, %A1";
4581   else
4582     return "vld1.<V_sz_elem>\t%h0, %A1";
4583 }
4584   [(set (attr "type")
4585       (if_then_else (gt (const_string "<V_mode_nunits>") (const_string "1"))
4586                     (const_string "neon_load2_all_lanes<q>")
4587                     (const_string "neon_load1_1reg<q>")))]
4588 )
4589
4590 (define_expand "vec_store_lanesti<mode>"
4591   [(set (match_operand:TI 0 "neon_struct_operand")
4592         (unspec:TI [(match_operand:TI 1 "s_register_operand")
4593                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4594                    UNSPEC_VST2))]
4595   "TARGET_NEON")
4596
4597 (define_insn "neon_vst2<mode>"
4598   [(set (match_operand:TI 0 "neon_struct_operand" "=Um")
4599         (unspec:TI [(match_operand:TI 1 "s_register_operand" "w")
4600                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4601                    UNSPEC_VST2))]
4602   "TARGET_NEON"
4603 {
4604   if (<V_sz_elem> == 64)
4605     return "vst1.64\t%h1, %A0";
4606   else
4607     return "vst2.<V_sz_elem>\t%h1, %A0";
4608 }
4609   [(set (attr "type")
4610       (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
4611                     (const_string "neon_store1_2reg<q>")
4612                     (const_string "neon_store2_one_lane<q>")))]
4613 )
4614
4615 (define_expand "vec_store_lanesoi<mode>"
4616   [(set (match_operand:OI 0 "neon_struct_operand")
4617         (unspec:OI [(match_operand:OI 1 "s_register_operand")
4618                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4619                    UNSPEC_VST2))]
4620   "TARGET_NEON")
4621
4622 (define_insn "neon_vst2<mode>"
4623   [(set (match_operand:OI 0 "neon_struct_operand" "=Um")
4624         (unspec:OI [(match_operand:OI 1 "s_register_operand" "w")
4625                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4626                    UNSPEC_VST2))]
4627   "TARGET_NEON"
4628   "vst2.<V_sz_elem>\t%h1, %A0"
4629   [(set_attr "type" "neon_store2_4reg<q>")]
4630 )
4631
4632 (define_insn "neon_vst2_lane<mode>"
4633   [(set (match_operand:<V_two_elem> 0 "neon_struct_operand" "=Um")
4634         (unspec:<V_two_elem>
4635           [(match_operand:TI 1 "s_register_operand" "w")
4636            (match_operand:SI 2 "immediate_operand" "i")
4637            (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4638           UNSPEC_VST2_LANE))]
4639   "TARGET_NEON"
4640 {
4641   HOST_WIDE_INT lane = INTVAL (operands[2]);
4642   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4643   int regno = REGNO (operands[1]);
4644   rtx ops[4];
4645   if (lane < 0 || lane >= max)
4646     error ("lane out of range");
4647   ops[0] = operands[0];
4648   ops[1] = gen_rtx_REG (DImode, regno);
4649   ops[2] = gen_rtx_REG (DImode, regno + 2);
4650   ops[3] = operands[2];
4651   output_asm_insn ("vst2.<V_sz_elem>\t{%P1[%c3], %P2[%c3]}, %A0", ops);
4652   return "";
4653 }
4654   [(set_attr "type" "neon_store2_one_lane<q>")]
4655 )
4656
4657 (define_insn "neon_vst2_lane<mode>"
4658   [(set (match_operand:<V_two_elem> 0 "neon_struct_operand" "=Um")
4659         (unspec:<V_two_elem>
4660            [(match_operand:OI 1 "s_register_operand" "w")
4661             (match_operand:SI 2 "immediate_operand" "i")
4662             (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4663            UNSPEC_VST2_LANE))]
4664   "TARGET_NEON"
4665 {
4666   HOST_WIDE_INT lane = INTVAL (operands[2]);
4667   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4668   int regno = REGNO (operands[1]);
4669   rtx ops[4];
4670   if (lane < 0 || lane >= max)
4671     error ("lane out of range");
4672   else if (lane >= max / 2)
4673     {
4674       lane -= max / 2;
4675       regno += 2;
4676     }
4677   ops[0] = operands[0];
4678   ops[1] = gen_rtx_REG (DImode, regno);
4679   ops[2] = gen_rtx_REG (DImode, regno + 4);
4680   ops[3] = GEN_INT (lane);
4681   output_asm_insn ("vst2.<V_sz_elem>\t{%P1[%c3], %P2[%c3]}, %A0", ops);
4682   return "";
4683 }
4684   [(set_attr "type" "neon_store2_one_lane<q>")]
4685 )
4686
4687 (define_expand "vec_load_lanesei<mode>"
4688   [(set (match_operand:EI 0 "s_register_operand")
4689         (unspec:EI [(match_operand:EI 1 "neon_struct_operand")
4690                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4691                    UNSPEC_VLD3))]
4692   "TARGET_NEON")
4693
4694 (define_insn "neon_vld3<mode>"
4695   [(set (match_operand:EI 0 "s_register_operand" "=w")
4696         (unspec:EI [(match_operand:EI 1 "neon_struct_operand" "Um")
4697                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4698                    UNSPEC_VLD3))]
4699   "TARGET_NEON"
4700 {
4701   if (<V_sz_elem> == 64)
4702     return "vld1.64\t%h0, %A1";
4703   else
4704     return "vld3.<V_sz_elem>\t%h0, %A1";
4705 }
4706   [(set (attr "type")
4707       (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
4708                     (const_string "neon_load1_3reg<q>")
4709                     (const_string "neon_load3_3reg<q>")))]
4710 )
4711
4712 (define_expand "vec_load_lanesci<mode>"
4713   [(match_operand:CI 0 "s_register_operand")
4714    (match_operand:CI 1 "neon_struct_operand")
4715    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4716   "TARGET_NEON"
4717 {
4718   emit_insn (gen_neon_vld3<mode> (operands[0], operands[1]));
4719   DONE;
4720 })
4721
4722 (define_expand "neon_vld3<mode>"
4723   [(match_operand:CI 0 "s_register_operand")
4724    (match_operand:CI 1 "neon_struct_operand")
4725    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4726   "TARGET_NEON"
4727 {
4728   rtx mem;
4729
4730   mem = adjust_address (operands[1], EImode, 0);
4731   emit_insn (gen_neon_vld3qa<mode> (operands[0], mem));
4732   mem = adjust_address (mem, EImode, GET_MODE_SIZE (EImode));
4733   emit_insn (gen_neon_vld3qb<mode> (operands[0], mem, operands[0]));
4734   DONE;
4735 })
4736
4737 (define_insn "neon_vld3qa<mode>"
4738   [(set (match_operand:CI 0 "s_register_operand" "=w")
4739         (unspec:CI [(match_operand:EI 1 "neon_struct_operand" "Um")
4740                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4741                    UNSPEC_VLD3A))]
4742   "TARGET_NEON"
4743 {
4744   int regno = REGNO (operands[0]);
4745   rtx ops[4];
4746   ops[0] = gen_rtx_REG (DImode, regno);
4747   ops[1] = gen_rtx_REG (DImode, regno + 4);
4748   ops[2] = gen_rtx_REG (DImode, regno + 8);
4749   ops[3] = operands[1];
4750   output_asm_insn ("vld3.<V_sz_elem>\t{%P0, %P1, %P2}, %A3", ops);
4751   return "";
4752 }
4753   [(set_attr "type" "neon_load3_3reg<q>")]
4754 )
4755
4756 (define_insn "neon_vld3qb<mode>"
4757   [(set (match_operand:CI 0 "s_register_operand" "=w")
4758         (unspec:CI [(match_operand:EI 1 "neon_struct_operand" "Um")
4759                     (match_operand:CI 2 "s_register_operand" "0")
4760                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4761                    UNSPEC_VLD3B))]
4762   "TARGET_NEON"
4763 {
4764   int regno = REGNO (operands[0]);
4765   rtx ops[4];
4766   ops[0] = gen_rtx_REG (DImode, regno + 2);
4767   ops[1] = gen_rtx_REG (DImode, regno + 6);
4768   ops[2] = gen_rtx_REG (DImode, regno + 10);
4769   ops[3] = operands[1];
4770   output_asm_insn ("vld3.<V_sz_elem>\t{%P0, %P1, %P2}, %A3", ops);
4771   return "";
4772 }
4773   [(set_attr "type" "neon_load3_3reg<q>")]
4774 )
4775
4776 (define_insn "neon_vld3_lane<mode>"
4777   [(set (match_operand:EI 0 "s_register_operand" "=w")
4778         (unspec:EI [(match_operand:<V_three_elem> 1 "neon_struct_operand" "Um")
4779                     (match_operand:EI 2 "s_register_operand" "0")
4780                     (match_operand:SI 3 "immediate_operand" "i")
4781                     (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4782                    UNSPEC_VLD3_LANE))]
4783   "TARGET_NEON"
4784 {
4785   HOST_WIDE_INT lane = INTVAL (operands[3]);
4786   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4787   int regno = REGNO (operands[0]);
4788   rtx ops[5];
4789   if (lane < 0 || lane >= max)
4790     error ("lane out of range");
4791   ops[0] = gen_rtx_REG (DImode, regno);
4792   ops[1] = gen_rtx_REG (DImode, regno + 2);
4793   ops[2] = gen_rtx_REG (DImode, regno + 4);
4794   ops[3] = operands[1];
4795   ops[4] = operands[3];
4796   output_asm_insn ("vld3.<V_sz_elem>\t{%P0[%c4], %P1[%c4], %P2[%c4]}, %3",
4797                    ops);
4798   return "";
4799 }
4800   [(set_attr "type" "neon_load3_one_lane<q>")]
4801 )
4802
4803 (define_insn "neon_vld3_lane<mode>"
4804   [(set (match_operand:CI 0 "s_register_operand" "=w")
4805         (unspec:CI [(match_operand:<V_three_elem> 1 "neon_struct_operand" "Um")
4806                     (match_operand:CI 2 "s_register_operand" "0")
4807                     (match_operand:SI 3 "immediate_operand" "i")
4808                     (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4809                    UNSPEC_VLD3_LANE))]
4810   "TARGET_NEON"
4811 {
4812   HOST_WIDE_INT lane = INTVAL (operands[3]);
4813   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4814   int regno = REGNO (operands[0]);
4815   rtx ops[5];
4816   if (lane < 0 || lane >= max)
4817     error ("lane out of range");
4818   else if (lane >= max / 2)
4819     {
4820       lane -= max / 2;
4821       regno += 2;
4822     }
4823   ops[0] = gen_rtx_REG (DImode, regno);
4824   ops[1] = gen_rtx_REG (DImode, regno + 4);
4825   ops[2] = gen_rtx_REG (DImode, regno + 8);
4826   ops[3] = operands[1];
4827   ops[4] = GEN_INT (lane);
4828   output_asm_insn ("vld3.<V_sz_elem>\t{%P0[%c4], %P1[%c4], %P2[%c4]}, %3",
4829                    ops);
4830   return "";
4831 }
4832   [(set_attr "type" "neon_load3_one_lane<q>")]
4833 )
4834
4835 (define_insn "neon_vld3_dup<mode>"
4836   [(set (match_operand:EI 0 "s_register_operand" "=w")
4837         (unspec:EI [(match_operand:<V_three_elem> 1 "neon_struct_operand" "Um")
4838                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4839                    UNSPEC_VLD3_DUP))]
4840   "TARGET_NEON"
4841 {
4842   if (GET_MODE_NUNITS (<MODE>mode) > 1)
4843     {
4844       int regno = REGNO (operands[0]);
4845       rtx ops[4];
4846       ops[0] = gen_rtx_REG (DImode, regno);
4847       ops[1] = gen_rtx_REG (DImode, regno + 2);
4848       ops[2] = gen_rtx_REG (DImode, regno + 4);
4849       ops[3] = operands[1];
4850       output_asm_insn ("vld3.<V_sz_elem>\t{%P0[], %P1[], %P2[]}, %3", ops);
4851       return "";
4852     }
4853   else
4854     return "vld1.<V_sz_elem>\t%h0, %A1";
4855 }
4856   [(set (attr "type")
4857       (if_then_else (gt (const_string "<V_mode_nunits>") (const_string "1"))
4858                     (const_string "neon_load3_all_lanes<q>")
4859                     (const_string "neon_load1_1reg<q>")))])
4860
4861 (define_expand "vec_store_lanesei<mode>"
4862   [(set (match_operand:EI 0 "neon_struct_operand")
4863         (unspec:EI [(match_operand:EI 1 "s_register_operand")
4864                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4865                    UNSPEC_VST3))]
4866   "TARGET_NEON")
4867
4868 (define_insn "neon_vst3<mode>"
4869   [(set (match_operand:EI 0 "neon_struct_operand" "=Um")
4870         (unspec:EI [(match_operand:EI 1 "s_register_operand" "w")
4871                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4872                    UNSPEC_VST3))]
4873   "TARGET_NEON"
4874 {
4875   if (<V_sz_elem> == 64)
4876     return "vst1.64\t%h1, %A0";
4877   else
4878     return "vst3.<V_sz_elem>\t%h1, %A0";
4879 }
4880   [(set (attr "type")
4881       (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
4882                     (const_string "neon_store1_3reg<q>")
4883                     (const_string "neon_store3_one_lane<q>")))])
4884
4885 (define_expand "vec_store_lanesci<mode>"
4886   [(match_operand:CI 0 "neon_struct_operand")
4887    (match_operand:CI 1 "s_register_operand")
4888    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4889   "TARGET_NEON"
4890 {
4891   emit_insn (gen_neon_vst3<mode> (operands[0], operands[1]));
4892   DONE;
4893 })
4894
4895 (define_expand "neon_vst3<mode>"
4896   [(match_operand:CI 0 "neon_struct_operand")
4897    (match_operand:CI 1 "s_register_operand")
4898    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4899   "TARGET_NEON"
4900 {
4901   rtx mem;
4902
4903   mem = adjust_address (operands[0], EImode, 0);
4904   emit_insn (gen_neon_vst3qa<mode> (mem, operands[1]));
4905   mem = adjust_address (mem, EImode, GET_MODE_SIZE (EImode));
4906   emit_insn (gen_neon_vst3qb<mode> (mem, operands[1]));
4907   DONE;
4908 })
4909
4910 (define_insn "neon_vst3qa<mode>"
4911   [(set (match_operand:EI 0 "neon_struct_operand" "=Um")
4912         (unspec:EI [(match_operand:CI 1 "s_register_operand" "w")
4913                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4914                    UNSPEC_VST3A))]
4915   "TARGET_NEON"
4916 {
4917   int regno = REGNO (operands[1]);
4918   rtx ops[4];
4919   ops[0] = operands[0];
4920   ops[1] = gen_rtx_REG (DImode, regno);
4921   ops[2] = gen_rtx_REG (DImode, regno + 4);
4922   ops[3] = gen_rtx_REG (DImode, regno + 8);
4923   output_asm_insn ("vst3.<V_sz_elem>\t{%P1, %P2, %P3}, %A0", ops);
4924   return "";
4925 }
4926   [(set_attr "type" "neon_store3_3reg<q>")]
4927 )
4928
4929 (define_insn "neon_vst3qb<mode>"
4930   [(set (match_operand:EI 0 "neon_struct_operand" "=Um")
4931         (unspec:EI [(match_operand:CI 1 "s_register_operand" "w")
4932                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4933                    UNSPEC_VST3B))]
4934   "TARGET_NEON"
4935 {
4936   int regno = REGNO (operands[1]);
4937   rtx ops[4];
4938   ops[0] = operands[0];
4939   ops[1] = gen_rtx_REG (DImode, regno + 2);
4940   ops[2] = gen_rtx_REG (DImode, regno + 6);
4941   ops[3] = gen_rtx_REG (DImode, regno + 10);
4942   output_asm_insn ("vst3.<V_sz_elem>\t{%P1, %P2, %P3}, %A0", ops);
4943   return "";
4944 }
4945   [(set_attr "type" "neon_store3_3reg<q>")]
4946 )
4947
4948 (define_insn "neon_vst3_lane<mode>"
4949   [(set (match_operand:<V_three_elem> 0 "neon_struct_operand" "=Um")
4950         (unspec:<V_three_elem>
4951            [(match_operand:EI 1 "s_register_operand" "w")
4952             (match_operand:SI 2 "immediate_operand" "i")
4953             (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4954            UNSPEC_VST3_LANE))]
4955   "TARGET_NEON"
4956 {
4957   HOST_WIDE_INT lane = INTVAL (operands[2]);
4958   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4959   int regno = REGNO (operands[1]);
4960   rtx ops[5];
4961   if (lane < 0 || lane >= max)
4962     error ("lane out of range");
4963   ops[0] = operands[0];
4964   ops[1] = gen_rtx_REG (DImode, regno);
4965   ops[2] = gen_rtx_REG (DImode, regno + 2);
4966   ops[3] = gen_rtx_REG (DImode, regno + 4);
4967   ops[4] = operands[2];
4968   output_asm_insn ("vst3.<V_sz_elem>\t{%P1[%c4], %P2[%c4], %P3[%c4]}, %0",
4969                    ops);
4970   return "";
4971 }
4972   [(set_attr "type" "neon_store3_one_lane<q>")]
4973 )
4974
4975 (define_insn "neon_vst3_lane<mode>"
4976   [(set (match_operand:<V_three_elem> 0 "neon_struct_operand" "=Um")
4977         (unspec:<V_three_elem>
4978            [(match_operand:CI 1 "s_register_operand" "w")
4979             (match_operand:SI 2 "immediate_operand" "i")
4980             (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4981            UNSPEC_VST3_LANE))]
4982   "TARGET_NEON"
4983 {
4984   HOST_WIDE_INT lane = INTVAL (operands[2]);
4985   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4986   int regno = REGNO (operands[1]);
4987   rtx ops[5];
4988   if (lane < 0 || lane >= max)
4989     error ("lane out of range");
4990   else if (lane >= max / 2)
4991     {
4992       lane -= max / 2;
4993       regno += 2;
4994     }
4995   ops[0] = operands[0];
4996   ops[1] = gen_rtx_REG (DImode, regno);
4997   ops[2] = gen_rtx_REG (DImode, regno + 4);
4998   ops[3] = gen_rtx_REG (DImode, regno + 8);
4999   ops[4] = GEN_INT (lane);
5000   output_asm_insn ("vst3.<V_sz_elem>\t{%P1[%c4], %P2[%c4], %P3[%c4]}, %0",
5001                    ops);
5002   return "";
5003 }
5004   [(set_attr "type" "neon_store3_one_lane<q>")]
5005 )
5006
5007 (define_expand "vec_load_lanesoi<mode>"
5008   [(set (match_operand:OI 0 "s_register_operand")
5009         (unspec:OI [(match_operand:OI 1 "neon_struct_operand")
5010                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5011                    UNSPEC_VLD4))]
5012   "TARGET_NEON")
5013
5014 (define_insn "neon_vld4<mode>"
5015   [(set (match_operand:OI 0 "s_register_operand" "=w")
5016         (unspec:OI [(match_operand:OI 1 "neon_struct_operand" "Um")
5017                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5018                    UNSPEC_VLD4))]
5019   "TARGET_NEON"
5020 {
5021   if (<V_sz_elem> == 64)
5022     return "vld1.64\t%h0, %A1";
5023   else
5024     return "vld4.<V_sz_elem>\t%h0, %A1";
5025 }
5026   [(set (attr "type")
5027       (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
5028                     (const_string "neon_load1_4reg<q>")
5029                     (const_string "neon_load4_4reg<q>")))]
5030 )
5031
5032 (define_expand "vec_load_lanesxi<mode>"
5033   [(match_operand:XI 0 "s_register_operand")
5034    (match_operand:XI 1 "neon_struct_operand")
5035    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5036   "TARGET_NEON"
5037 {
5038   emit_insn (gen_neon_vld4<mode> (operands[0], operands[1]));
5039   DONE;
5040 })
5041
5042 (define_expand "neon_vld4<mode>"
5043   [(match_operand:XI 0 "s_register_operand")
5044    (match_operand:XI 1 "neon_struct_operand")
5045    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5046   "TARGET_NEON"
5047 {
5048   rtx mem;
5049
5050   mem = adjust_address (operands[1], OImode, 0);
5051   emit_insn (gen_neon_vld4qa<mode> (operands[0], mem));
5052   mem = adjust_address (mem, OImode, GET_MODE_SIZE (OImode));
5053   emit_insn (gen_neon_vld4qb<mode> (operands[0], mem, operands[0]));
5054   DONE;
5055 })
5056
5057 (define_insn "neon_vld4qa<mode>"
5058   [(set (match_operand:XI 0 "s_register_operand" "=w")
5059         (unspec:XI [(match_operand:OI 1 "neon_struct_operand" "Um")
5060                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5061                    UNSPEC_VLD4A))]
5062   "TARGET_NEON"
5063 {
5064   int regno = REGNO (operands[0]);
5065   rtx ops[5];
5066   ops[0] = gen_rtx_REG (DImode, regno);
5067   ops[1] = gen_rtx_REG (DImode, regno + 4);
5068   ops[2] = gen_rtx_REG (DImode, regno + 8);
5069   ops[3] = gen_rtx_REG (DImode, regno + 12);
5070   ops[4] = operands[1];
5071   output_asm_insn ("vld4.<V_sz_elem>\t{%P0, %P1, %P2, %P3}, %A4", ops);
5072   return "";
5073 }
5074   [(set_attr "type" "neon_load4_4reg<q>")]
5075 )
5076
5077 (define_insn "neon_vld4qb<mode>"
5078   [(set (match_operand:XI 0 "s_register_operand" "=w")
5079         (unspec:XI [(match_operand:OI 1 "neon_struct_operand" "Um")
5080                     (match_operand:XI 2 "s_register_operand" "0")
5081                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5082                    UNSPEC_VLD4B))]
5083   "TARGET_NEON"
5084 {
5085   int regno = REGNO (operands[0]);
5086   rtx ops[5];
5087   ops[0] = gen_rtx_REG (DImode, regno + 2);
5088   ops[1] = gen_rtx_REG (DImode, regno + 6);
5089   ops[2] = gen_rtx_REG (DImode, regno + 10);
5090   ops[3] = gen_rtx_REG (DImode, regno + 14);
5091   ops[4] = operands[1];
5092   output_asm_insn ("vld4.<V_sz_elem>\t{%P0, %P1, %P2, %P3}, %A4", ops);
5093   return "";
5094 }
5095   [(set_attr "type" "neon_load4_4reg<q>")]
5096 )
5097
5098 (define_insn "neon_vld4_lane<mode>"
5099   [(set (match_operand:OI 0 "s_register_operand" "=w")
5100         (unspec:OI [(match_operand:<V_four_elem> 1 "neon_struct_operand" "Um")
5101                     (match_operand:OI 2 "s_register_operand" "0")
5102                     (match_operand:SI 3 "immediate_operand" "i")
5103                     (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5104                    UNSPEC_VLD4_LANE))]
5105   "TARGET_NEON"
5106 {
5107   HOST_WIDE_INT lane = INTVAL (operands[3]);
5108   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
5109   int regno = REGNO (operands[0]);
5110   rtx ops[6];
5111   if (lane < 0 || lane >= max)
5112     error ("lane out of range");
5113   ops[0] = gen_rtx_REG (DImode, regno);
5114   ops[1] = gen_rtx_REG (DImode, regno + 2);
5115   ops[2] = gen_rtx_REG (DImode, regno + 4);
5116   ops[3] = gen_rtx_REG (DImode, regno + 6);
5117   ops[4] = operands[1];
5118   ops[5] = operands[3];
5119   output_asm_insn ("vld4.<V_sz_elem>\t{%P0[%c5], %P1[%c5], %P2[%c5], %P3[%c5]}, %A4",
5120                    ops);
5121   return "";
5122 }
5123   [(set_attr "type" "neon_load4_one_lane<q>")]
5124 )
5125
5126 (define_insn "neon_vld4_lane<mode>"
5127   [(set (match_operand:XI 0 "s_register_operand" "=w")
5128         (unspec:XI [(match_operand:<V_four_elem> 1 "neon_struct_operand" "Um")
5129                     (match_operand:XI 2 "s_register_operand" "0")
5130                     (match_operand:SI 3 "immediate_operand" "i")
5131                     (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5132                    UNSPEC_VLD4_LANE))]
5133   "TARGET_NEON"
5134 {
5135   HOST_WIDE_INT lane = INTVAL (operands[3]);
5136   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
5137   int regno = REGNO (operands[0]);
5138   rtx ops[6];
5139   if (lane < 0 || lane >= max)
5140     error ("lane out of range");
5141   else if (lane >= max / 2)
5142     {
5143       lane -= max / 2;
5144       regno += 2;
5145     }
5146   ops[0] = gen_rtx_REG (DImode, regno);
5147   ops[1] = gen_rtx_REG (DImode, regno + 4);
5148   ops[2] = gen_rtx_REG (DImode, regno + 8);
5149   ops[3] = gen_rtx_REG (DImode, regno + 12);
5150   ops[4] = operands[1];
5151   ops[5] = GEN_INT (lane);
5152   output_asm_insn ("vld4.<V_sz_elem>\t{%P0[%c5], %P1[%c5], %P2[%c5], %P3[%c5]}, %A4",
5153                    ops);
5154   return "";
5155 }
5156   [(set_attr "type" "neon_load4_one_lane<q>")]
5157 )
5158
5159 (define_insn "neon_vld4_dup<mode>"
5160   [(set (match_operand:OI 0 "s_register_operand" "=w")
5161         (unspec:OI [(match_operand:<V_four_elem> 1 "neon_struct_operand" "Um")
5162                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5163                    UNSPEC_VLD4_DUP))]
5164   "TARGET_NEON"
5165 {
5166   if (GET_MODE_NUNITS (<MODE>mode) > 1)
5167     {
5168       int regno = REGNO (operands[0]);
5169       rtx ops[5];
5170       ops[0] = gen_rtx_REG (DImode, regno);
5171       ops[1] = gen_rtx_REG (DImode, regno + 2);
5172       ops[2] = gen_rtx_REG (DImode, regno + 4);
5173       ops[3] = gen_rtx_REG (DImode, regno + 6);
5174       ops[4] = operands[1];
5175       output_asm_insn ("vld4.<V_sz_elem>\t{%P0[], %P1[], %P2[], %P3[]}, %A4",
5176                        ops);
5177       return "";
5178     }
5179   else
5180     return "vld1.<V_sz_elem>\t%h0, %A1";
5181 }
5182   [(set (attr "type")
5183       (if_then_else (gt (const_string "<V_mode_nunits>") (const_string "1"))
5184                     (const_string "neon_load4_all_lanes<q>")
5185                     (const_string "neon_load1_1reg<q>")))]
5186 )
5187
5188 (define_expand "vec_store_lanesoi<mode>"
5189   [(set (match_operand:OI 0 "neon_struct_operand")
5190         (unspec:OI [(match_operand:OI 1 "s_register_operand")
5191                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5192                    UNSPEC_VST4))]
5193   "TARGET_NEON")
5194
5195 (define_insn "neon_vst4<mode>"
5196   [(set (match_operand:OI 0 "neon_struct_operand" "=Um")
5197         (unspec:OI [(match_operand:OI 1 "s_register_operand" "w")
5198                     (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5199                    UNSPEC_VST4))]
5200   "TARGET_NEON"
5201 {
5202   if (<V_sz_elem> == 64)
5203     return "vst1.64\t%h1, %A0";
5204   else
5205     return "vst4.<V_sz_elem>\t%h1, %A0";
5206 }
5207   [(set (attr "type")
5208       (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
5209                     (const_string "neon_store1_4reg<q>")
5210                     (const_string "neon_store4_4reg<q>")))]
5211 )
5212
5213 (define_expand "vec_store_lanesxi<mode>"
5214   [(match_operand:XI 0 "neon_struct_operand")
5215    (match_operand:XI 1 "s_register_operand")
5216    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5217   "TARGET_NEON"
5218 {
5219   emit_insn (gen_neon_vst4<mode> (operands[0], operands[1]));
5220   DONE;
5221 })
5222
5223 (define_expand "neon_vst4<mode>"
5224   [(match_operand:XI 0 "neon_struct_operand")
5225    (match_operand:XI 1 "s_register_operand")
5226    (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5227   "TARGET_NEON"
5228 {
5229   rtx mem;
5230
5231   mem = adjust_address (operands[0], OImode, 0);
5232   emit_insn (gen_neon_vst4qa<mode> (mem, operands[1]));
5233   mem = adjust_address (mem, OImode, GET_MODE_SIZE (OImode));
5234   emit_insn (gen_neon_vst4qb<mode> (mem, operands[1]));
5235   DONE;
5236 })
5237
5238 (define_insn "neon_vst4qa<mode>"
5239   [(set (match_operand:OI 0 "neon_struct_operand" "=Um")
5240         (unspec:OI [(match_operand:XI 1 "s_register_operand" "w")
5241                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5242                    UNSPEC_VST4A))]
5243   "TARGET_NEON"
5244 {
5245   int regno = REGNO (operands[1]);
5246   rtx ops[5];
5247   ops[0] = operands[0];
5248   ops[1] = gen_rtx_REG (DImode, regno);
5249   ops[2] = gen_rtx_REG (DImode, regno + 4);
5250   ops[3] = gen_rtx_REG (DImode, regno + 8);
5251   ops[4] = gen_rtx_REG (DImode, regno + 12);
5252   output_asm_insn ("vst4.<V_sz_elem>\t{%P1, %P2, %P3, %P4}, %A0", ops);
5253   return "";
5254 }
5255   [(set_attr "type" "neon_store4_4reg<q>")]
5256 )
5257
5258 (define_insn "neon_vst4qb<mode>"
5259   [(set (match_operand:OI 0 "neon_struct_operand" "=Um")
5260         (unspec:OI [(match_operand:XI 1 "s_register_operand" "w")
5261                     (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5262                    UNSPEC_VST4B))]
5263   "TARGET_NEON"
5264 {
5265   int regno = REGNO (operands[1]);
5266   rtx ops[5];
5267   ops[0] = operands[0];
5268   ops[1] = gen_rtx_REG (DImode, regno + 2);
5269   ops[2] = gen_rtx_REG (DImode, regno + 6);
5270   ops[3] = gen_rtx_REG (DImode, regno + 10);
5271   ops[4] = gen_rtx_REG (DImode, regno + 14);
5272   output_asm_insn ("vst4.<V_sz_elem>\t{%P1, %P2, %P3, %P4}, %A0", ops);
5273   return "";
5274 }
5275   [(set_attr "type" "neon_store4_4reg<q>")]
5276 )
5277
5278 (define_insn "neon_vst4_lane<mode>"
5279   [(set (match_operand:<V_four_elem> 0 "neon_struct_operand" "=Um")
5280         (unspec:<V_four_elem>
5281            [(match_operand:OI 1 "s_register_operand" "w")
5282             (match_operand:SI 2 "immediate_operand" "i")
5283             (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5284            UNSPEC_VST4_LANE))]
5285   "TARGET_NEON"
5286 {
5287   HOST_WIDE_INT lane = INTVAL (operands[2]);
5288   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
5289   int regno = REGNO (operands[1]);
5290   rtx ops[6];
5291   if (lane < 0 || lane >= max)
5292     error ("lane out of range");
5293   ops[0] = operands[0];
5294   ops[1] = gen_rtx_REG (DImode, regno);
5295   ops[2] = gen_rtx_REG (DImode, regno + 2);
5296   ops[3] = gen_rtx_REG (DImode, regno + 4);
5297   ops[4] = gen_rtx_REG (DImode, regno + 6);
5298   ops[5] = operands[2];
5299   output_asm_insn ("vst4.<V_sz_elem>\t{%P1[%c5], %P2[%c5], %P3[%c5], %P4[%c5]}, %A0",
5300                    ops);
5301   return "";
5302 }
5303   [(set_attr "type" "neon_store4_one_lane<q>")]
5304 )
5305
5306 (define_insn "neon_vst4_lane<mode>"
5307   [(set (match_operand:<V_four_elem> 0 "neon_struct_operand" "=Um")
5308         (unspec:<V_four_elem>
5309            [(match_operand:XI 1 "s_register_operand" "w")
5310             (match_operand:SI 2 "immediate_operand" "i")
5311             (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5312            UNSPEC_VST4_LANE))]
5313   "TARGET_NEON"
5314 {
5315   HOST_WIDE_INT lane = INTVAL (operands[2]);
5316   HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
5317   int regno = REGNO (operands[1]);
5318   rtx ops[6];
5319   if (lane < 0 || lane >= max)
5320     error ("lane out of range");
5321   else if (lane >= max / 2)
5322     {
5323       lane -= max / 2;
5324       regno += 2;
5325     }
5326   ops[0] = operands[0];
5327   ops[1] = gen_rtx_REG (DImode, regno);
5328   ops[2] = gen_rtx_REG (DImode, regno + 4);
5329   ops[3] = gen_rtx_REG (DImode, regno + 8);
5330   ops[4] = gen_rtx_REG (DImode, regno + 12);
5331   ops[5] = GEN_INT (lane);
5332   output_asm_insn ("vst4.<V_sz_elem>\t{%P1[%c5], %P2[%c5], %P3[%c5], %P4[%c5]}, %A0",
5333                    ops);
5334   return "";
5335 }
5336   [(set_attr "type" "neon_store4_4reg<q>")]
5337 )
5338
5339 (define_expand "neon_vand<mode>"
5340   [(match_operand:VDQX 0 "s_register_operand" "")
5341    (match_operand:VDQX 1 "s_register_operand" "")
5342    (match_operand:VDQX 2 "neon_inv_logic_op2" "")
5343    (match_operand:SI 3 "immediate_operand" "")]
5344   "TARGET_NEON"
5345 {
5346   emit_insn (gen_and<mode>3 (operands[0], operands[1], operands[2]));
5347   DONE;
5348 })
5349
5350 (define_expand "neon_vorr<mode>"
5351   [(match_operand:VDQX 0 "s_register_operand" "")
5352    (match_operand:VDQX 1 "s_register_operand" "")
5353    (match_operand:VDQX 2 "neon_logic_op2" "")
5354    (match_operand:SI 3 "immediate_operand" "")]
5355   "TARGET_NEON"
5356 {
5357   emit_insn (gen_ior<mode>3 (operands[0], operands[1], operands[2]));
5358   DONE;
5359 })
5360
5361 (define_expand "neon_veor<mode>"
5362   [(match_operand:VDQX 0 "s_register_operand" "")
5363    (match_operand:VDQX 1 "s_register_operand" "")
5364    (match_operand:VDQX 2 "s_register_operand" "")
5365    (match_operand:SI 3 "immediate_operand" "")]
5366   "TARGET_NEON"
5367 {
5368   emit_insn (gen_xor<mode>3 (operands[0], operands[1], operands[2]));
5369   DONE;
5370 })
5371
5372 (define_expand "neon_vbic<mode>"
5373   [(match_operand:VDQX 0 "s_register_operand" "")
5374    (match_operand:VDQX 1 "s_register_operand" "")
5375    (match_operand:VDQX 2 "neon_logic_op2" "")
5376    (match_operand:SI 3 "immediate_operand" "")]
5377   "TARGET_NEON"
5378 {
5379   emit_insn (gen_bic<mode>3_neon (operands[0], operands[1], operands[2]));
5380   DONE;
5381 })
5382
5383 (define_expand "neon_vorn<mode>"
5384   [(match_operand:VDQX 0 "s_register_operand" "")
5385    (match_operand:VDQX 1 "s_register_operand" "")
5386    (match_operand:VDQX 2 "neon_inv_logic_op2" "")
5387    (match_operand:SI 3 "immediate_operand" "")]
5388   "TARGET_NEON"
5389 {
5390   emit_insn (gen_orn<mode>3_neon (operands[0], operands[1], operands[2]));
5391   DONE;
5392 })
5393
5394 (define_insn "neon_vec_unpack<US>_lo_<mode>"
5395   [(set (match_operand:<V_unpack> 0 "register_operand" "=w")
5396         (SE:<V_unpack> (vec_select:<V_HALF>
5397                           (match_operand:VU 1 "register_operand" "w")
5398                           (match_operand:VU 2 "vect_par_constant_low" ""))))]
5399   "TARGET_NEON && !BYTES_BIG_ENDIAN"
5400   "vmovl.<US><V_sz_elem> %q0, %e1"
5401   [(set_attr "type" "neon_shift_imm_long")]
5402 )
5403
5404 (define_insn "neon_vec_unpack<US>_hi_<mode>"
5405   [(set (match_operand:<V_unpack> 0 "register_operand" "=w")
5406         (SE:<V_unpack> (vec_select:<V_HALF>
5407                           (match_operand:VU 1 "register_operand" "w")
5408                           (match_operand:VU 2 "vect_par_constant_high" ""))))]
5409   "TARGET_NEON && !BYTES_BIG_ENDIAN"
5410   "vmovl.<US><V_sz_elem> %q0, %f1"
5411   [(set_attr "type" "neon_shift_imm_long")]
5412 )
5413
5414 (define_expand "vec_unpack<US>_hi_<mode>"
5415   [(match_operand:<V_unpack> 0 "register_operand" "")
5416    (SE:<V_unpack> (match_operand:VU 1 "register_operand"))]
5417  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5418   {
5419    rtvec v = rtvec_alloc (<V_mode_nunits>/2)  ;
5420    rtx t1;
5421    int i;
5422    for (i = 0; i < (<V_mode_nunits>/2); i++)
5423      RTVEC_ELT (v, i) = GEN_INT ((<V_mode_nunits>/2) + i);
5424
5425    t1 = gen_rtx_PARALLEL (<MODE>mode, v);
5426    emit_insn (gen_neon_vec_unpack<US>_hi_<mode> (operands[0],
5427                                                  operands[1],
5428                                                  t1));
5429    DONE;
5430   }
5431 )
5432
5433 (define_expand "vec_unpack<US>_lo_<mode>"
5434   [(match_operand:<V_unpack> 0 "register_operand" "")
5435    (SE:<V_unpack> (match_operand:VU 1 "register_operand" ""))]
5436  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5437   {
5438    rtvec v = rtvec_alloc (<V_mode_nunits>/2)  ;
5439    rtx t1;
5440    int i;
5441    for (i = 0; i < (<V_mode_nunits>/2) ; i++)
5442      RTVEC_ELT (v, i) = GEN_INT (i);
5443    t1 = gen_rtx_PARALLEL (<MODE>mode, v);
5444    emit_insn (gen_neon_vec_unpack<US>_lo_<mode> (operands[0],
5445                                                  operands[1],
5446                                                  t1));
5447    DONE;
5448   }
5449 )
5450
5451 (define_insn "neon_vec_<US>mult_lo_<mode>"
5452  [(set (match_operand:<V_unpack> 0 "register_operand" "=w")
5453        (mult:<V_unpack> (SE:<V_unpack> (vec_select:<V_HALF>
5454                            (match_operand:VU 1 "register_operand" "w")
5455                            (match_operand:VU 2 "vect_par_constant_low" "")))
5456                         (SE:<V_unpack> (vec_select:<V_HALF>
5457                            (match_operand:VU 3 "register_operand" "w")
5458                            (match_dup 2)))))]
5459   "TARGET_NEON && !BYTES_BIG_ENDIAN"
5460   "vmull.<US><V_sz_elem> %q0, %e1, %e3"
5461   [(set_attr "type" "neon_mul_<V_elem_ch>_long")]
5462 )
5463
5464 (define_expand "vec_widen_<US>mult_lo_<mode>"
5465   [(match_operand:<V_unpack> 0 "register_operand" "")
5466    (SE:<V_unpack> (match_operand:VU 1 "register_operand" ""))
5467    (SE:<V_unpack> (match_operand:VU 2 "register_operand" ""))]
5468  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5469  {
5470    rtvec v = rtvec_alloc (<V_mode_nunits>/2)  ;
5471    rtx t1;
5472    int i;
5473    for (i = 0; i < (<V_mode_nunits>/2) ; i++)
5474      RTVEC_ELT (v, i) = GEN_INT (i);
5475    t1 = gen_rtx_PARALLEL (<MODE>mode, v);
5476
5477    emit_insn (gen_neon_vec_<US>mult_lo_<mode> (operands[0],
5478                                                operands[1],
5479                                                t1,
5480                                                operands[2]));
5481    DONE;
5482  }
5483 )
5484
5485 (define_insn "neon_vec_<US>mult_hi_<mode>"
5486  [(set (match_operand:<V_unpack> 0 "register_operand" "=w")
5487       (mult:<V_unpack> (SE:<V_unpack> (vec_select:<V_HALF>
5488                             (match_operand:VU 1 "register_operand" "w")
5489                             (match_operand:VU 2 "vect_par_constant_high" "")))
5490                        (SE:<V_unpack> (vec_select:<V_HALF>
5491                             (match_operand:VU 3 "register_operand" "w")
5492                             (match_dup 2)))))]
5493   "TARGET_NEON && !BYTES_BIG_ENDIAN"
5494   "vmull.<US><V_sz_elem> %q0, %f1, %f3"
5495   [(set_attr "type" "neon_mul_<V_elem_ch>_long")]
5496 )
5497
5498 (define_expand "vec_widen_<US>mult_hi_<mode>"
5499   [(match_operand:<V_unpack> 0 "register_operand" "")
5500    (SE:<V_unpack> (match_operand:VU 1 "register_operand" ""))
5501    (SE:<V_unpack> (match_operand:VU 2 "register_operand" ""))]
5502  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5503  {
5504    rtvec v = rtvec_alloc (<V_mode_nunits>/2)  ;
5505    rtx t1;
5506    int i;
5507    for (i = 0; i < (<V_mode_nunits>/2) ; i++)
5508      RTVEC_ELT (v, i) = GEN_INT (<V_mode_nunits>/2 + i);
5509    t1 = gen_rtx_PARALLEL (<MODE>mode, v);
5510
5511    emit_insn (gen_neon_vec_<US>mult_hi_<mode> (operands[0],
5512                                                operands[1],
5513                                                t1,
5514                                                operands[2]));
5515    DONE;
5516
5517  }
5518 )
5519
5520 (define_insn "neon_vec_<US>shiftl_<mode>"
5521  [(set (match_operand:<V_widen> 0 "register_operand" "=w")
5522        (SE:<V_widen> (ashift:VW (match_operand:VW 1 "register_operand" "w")
5523        (match_operand:<V_innermode> 2 "const_neon_scalar_shift_amount_operand" ""))))]
5524   "TARGET_NEON"
5525 {
5526   return "vshll.<US><V_sz_elem> %q0, %P1, %2";
5527 }
5528   [(set_attr "type" "neon_shift_imm_long")]
5529 )
5530
5531 (define_expand "vec_widen_<US>shiftl_lo_<mode>"
5532   [(match_operand:<V_unpack> 0 "register_operand" "")
5533    (SE:<V_unpack> (match_operand:VU 1 "register_operand" ""))
5534    (match_operand:SI 2 "immediate_operand" "i")]
5535  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5536  {
5537   emit_insn (gen_neon_vec_<US>shiftl_<V_half> (operands[0],
5538                 simplify_gen_subreg (<V_HALF>mode, operands[1], <MODE>mode, 0),
5539                 operands[2]));
5540    DONE;
5541  }
5542 )
5543
5544 (define_expand "vec_widen_<US>shiftl_hi_<mode>"
5545   [(match_operand:<V_unpack> 0 "register_operand" "")
5546    (SE:<V_unpack> (match_operand:VU 1 "register_operand" ""))
5547    (match_operand:SI 2 "immediate_operand" "i")]
5548  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5549  {
5550   emit_insn (gen_neon_vec_<US>shiftl_<V_half> (operands[0],
5551                 simplify_gen_subreg (<V_HALF>mode, operands[1], <MODE>mode,
5552                                      GET_MODE_SIZE (<V_HALF>mode)),
5553                 operands[2]));
5554    DONE;
5555  }
5556 )
5557
5558 ;; Vectorize for non-neon-quad case
5559 (define_insn "neon_unpack<US>_<mode>"
5560  [(set (match_operand:<V_widen> 0 "register_operand" "=w")
5561        (SE:<V_widen> (match_operand:VDI 1 "register_operand" "w")))]
5562  "TARGET_NEON"
5563  "vmovl.<US><V_sz_elem> %q0, %P1"
5564   [(set_attr "type" "neon_move")]
5565 )
5566
5567 (define_expand "vec_unpack<US>_lo_<mode>"
5568  [(match_operand:<V_double_width> 0 "register_operand" "")
5569   (SE:<V_double_width>(match_operand:VDI 1 "register_operand"))]
5570  "TARGET_NEON"
5571 {
5572   rtx tmpreg = gen_reg_rtx (<V_widen>mode);
5573   emit_insn (gen_neon_unpack<US>_<mode> (tmpreg, operands[1]));
5574   emit_insn (gen_neon_vget_low<V_widen_l> (operands[0], tmpreg));
5575
5576   DONE;
5577 }
5578 )
5579
5580 (define_expand "vec_unpack<US>_hi_<mode>"
5581  [(match_operand:<V_double_width> 0 "register_operand" "")
5582   (SE:<V_double_width>(match_operand:VDI 1 "register_operand"))]
5583  "TARGET_NEON"
5584 {
5585   rtx tmpreg = gen_reg_rtx (<V_widen>mode);
5586   emit_insn (gen_neon_unpack<US>_<mode> (tmpreg, operands[1]));
5587   emit_insn (gen_neon_vget_high<V_widen_l> (operands[0], tmpreg));
5588
5589   DONE;
5590 }
5591 )
5592
5593 (define_insn "neon_vec_<US>mult_<mode>"
5594  [(set (match_operand:<V_widen> 0 "register_operand" "=w")
5595        (mult:<V_widen> (SE:<V_widen>
5596                            (match_operand:VDI 1 "register_operand" "w"))
5597                        (SE:<V_widen>
5598                            (match_operand:VDI 2 "register_operand" "w"))))]
5599   "TARGET_NEON"
5600   "vmull.<US><V_sz_elem> %q0, %P1, %P2"
5601   [(set_attr "type" "neon_mul_<V_elem_ch>_long")]
5602 )
5603
5604 (define_expand "vec_widen_<US>mult_hi_<mode>"
5605   [(match_operand:<V_double_width> 0 "register_operand" "")
5606    (SE:<V_double_width> (match_operand:VDI 1 "register_operand" ""))
5607    (SE:<V_double_width> (match_operand:VDI 2 "register_operand" ""))]
5608  "TARGET_NEON"
5609  {
5610    rtx tmpreg = gen_reg_rtx (<V_widen>mode);
5611    emit_insn (gen_neon_vec_<US>mult_<mode> (tmpreg, operands[1], operands[2]));
5612    emit_insn (gen_neon_vget_high<V_widen_l> (operands[0], tmpreg));
5613
5614    DONE;
5615
5616  }
5617 )
5618
5619 (define_expand "vec_widen_<US>mult_lo_<mode>"
5620   [(match_operand:<V_double_width> 0 "register_operand" "")
5621    (SE:<V_double_width> (match_operand:VDI 1 "register_operand" ""))
5622    (SE:<V_double_width> (match_operand:VDI 2 "register_operand" ""))]
5623  "TARGET_NEON"
5624  {
5625    rtx tmpreg = gen_reg_rtx (<V_widen>mode);
5626    emit_insn (gen_neon_vec_<US>mult_<mode> (tmpreg, operands[1], operands[2]));
5627    emit_insn (gen_neon_vget_low<V_widen_l> (operands[0], tmpreg));
5628
5629    DONE;
5630
5631  }
5632 )
5633
5634 (define_expand "vec_widen_<US>shiftl_hi_<mode>"
5635  [(match_operand:<V_double_width> 0 "register_operand" "")
5636    (SE:<V_double_width> (match_operand:VDI 1 "register_operand" ""))
5637    (match_operand:SI 2 "immediate_operand" "i")]
5638  "TARGET_NEON"
5639  {
5640    rtx tmpreg = gen_reg_rtx (<V_widen>mode);
5641    emit_insn (gen_neon_vec_<US>shiftl_<mode> (tmpreg, operands[1], operands[2]));
5642    emit_insn (gen_neon_vget_high<V_widen_l> (operands[0], tmpreg));
5643
5644    DONE;
5645  }
5646 )
5647
5648 (define_expand "vec_widen_<US>shiftl_lo_<mode>"
5649   [(match_operand:<V_double_width> 0 "register_operand" "")
5650    (SE:<V_double_width> (match_operand:VDI 1 "register_operand" ""))
5651    (match_operand:SI 2 "immediate_operand" "i")]
5652  "TARGET_NEON"
5653  {
5654    rtx tmpreg = gen_reg_rtx (<V_widen>mode);
5655    emit_insn (gen_neon_vec_<US>shiftl_<mode> (tmpreg, operands[1], operands[2]));
5656    emit_insn (gen_neon_vget_low<V_widen_l> (operands[0], tmpreg));
5657
5658    DONE;
5659  }
5660 )
5661
5662 ; FIXME: These instruction patterns can't be used safely in big-endian mode
5663 ; because the ordering of vector elements in Q registers is different from what
5664 ; the semantics of the instructions require.
5665
5666 (define_insn "vec_pack_trunc_<mode>"
5667  [(set (match_operand:<V_narrow_pack> 0 "register_operand" "=&w")
5668        (vec_concat:<V_narrow_pack>
5669                 (truncate:<V_narrow>
5670                         (match_operand:VN 1 "register_operand" "w"))
5671                 (truncate:<V_narrow>
5672                         (match_operand:VN 2 "register_operand" "w"))))]
5673  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5674  "vmovn.i<V_sz_elem>\t%e0, %q1\;vmovn.i<V_sz_elem>\t%f0, %q2"
5675  [(set_attr "type" "multiple")
5676   (set_attr "length" "8")]
5677 )
5678
5679 ;; For the non-quad case.
5680 (define_insn "neon_vec_pack_trunc_<mode>"
5681  [(set (match_operand:<V_narrow> 0 "register_operand" "=w")
5682        (truncate:<V_narrow> (match_operand:VN 1 "register_operand" "w")))]
5683  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5684  "vmovn.i<V_sz_elem>\t%P0, %q1"
5685  [(set_attr "type" "neon_move_narrow_q")]
5686 )
5687
5688 (define_expand "vec_pack_trunc_<mode>"
5689  [(match_operand:<V_narrow_pack> 0 "register_operand" "")
5690   (match_operand:VSHFT 1 "register_operand" "")
5691   (match_operand:VSHFT 2 "register_operand")]
5692  "TARGET_NEON && !BYTES_BIG_ENDIAN"
5693 {
5694   rtx tempreg = gen_reg_rtx (<V_DOUBLE>mode);
5695
5696   emit_insn (gen_move_lo_quad_<V_double> (tempreg, operands[1]));
5697   emit_insn (gen_move_hi_quad_<V_double> (tempreg, operands[2]));
5698   emit_insn (gen_neon_vec_pack_trunc_<V_double> (operands[0], tempreg));
5699   DONE;
5700 })
5701
5702 (define_insn "neon_vabd<mode>_2"
5703  [(set (match_operand:VDQ 0 "s_register_operand" "=w")
5704        (abs:VDQ (minus:VDQ (match_operand:VDQ 1 "s_register_operand" "w")
5705                            (match_operand:VDQ 2 "s_register_operand" "w"))))]
5706  "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
5707  "vabd.<V_s_elem> %<V_reg>0, %<V_reg>1, %<V_reg>2"
5708  [(set (attr "type")
5709        (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
5710                      (const_string "neon_fp_abd_s<q>")
5711                      (const_string "neon_abd<q>")))]
5712 )
5713
5714 (define_insn "neon_vabd<mode>_3"
5715  [(set (match_operand:VDQ 0 "s_register_operand" "=w")
5716        (abs:VDQ (unspec:VDQ [(match_operand:VDQ 1 "s_register_operand" "w")
5717                              (match_operand:VDQ 2 "s_register_operand" "w")]
5718                  UNSPEC_VSUB)))]
5719  "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
5720  "vabd.<V_if_elem> %<V_reg>0, %<V_reg>1, %<V_reg>2"
5721  [(set (attr "type")
5722        (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
5723                      (const_string "neon_fp_abd_s<q>")
5724                      (const_string "neon_abd<q>")))]
5725 )
5726
5727 ;; Copy from core-to-neon regs, then extend, not vice-versa
5728
5729 (define_split
5730   [(set (match_operand:DI 0 "s_register_operand" "")
5731         (sign_extend:DI (match_operand:SI 1 "s_register_operand" "")))]
5732   "TARGET_NEON && reload_completed && IS_VFP_REGNUM (REGNO (operands[0]))"
5733   [(set (match_dup 2) (vec_duplicate:V2SI (match_dup 1)))
5734    (set (match_dup 0) (ashiftrt:DI (match_dup 0) (const_int 32)))]
5735   {
5736     operands[2] = gen_rtx_REG (V2SImode, REGNO (operands[0]));
5737   })
5738
5739 (define_split
5740   [(set (match_operand:DI 0 "s_register_operand" "")
5741         (sign_extend:DI (match_operand:HI 1 "s_register_operand" "")))]
5742   "TARGET_NEON && reload_completed && IS_VFP_REGNUM (REGNO (operands[0]))"
5743   [(set (match_dup 2) (vec_duplicate:V4HI (match_dup 1)))
5744    (set (match_dup 0) (ashiftrt:DI (match_dup 0) (const_int 48)))]
5745   {
5746     operands[2] = gen_rtx_REG (V4HImode, REGNO (operands[0]));
5747   })
5748
5749 (define_split
5750   [(set (match_operand:DI 0 "s_register_operand" "")
5751         (sign_extend:DI (match_operand:QI 1 "s_register_operand" "")))]
5752   "TARGET_NEON && reload_completed && IS_VFP_REGNUM (REGNO (operands[0]))"
5753   [(set (match_dup 2) (vec_duplicate:V8QI (match_dup 1)))
5754    (set (match_dup 0) (ashiftrt:DI (match_dup 0) (const_int 56)))]
5755   {
5756     operands[2] = gen_rtx_REG (V8QImode, REGNO (operands[0]));
5757   })
5758
5759 (define_split
5760   [(set (match_operand:DI 0 "s_register_operand" "")
5761         (zero_extend:DI (match_operand:SI 1 "s_register_operand" "")))]
5762   "TARGET_NEON && reload_completed && IS_VFP_REGNUM (REGNO (operands[0]))"
5763   [(set (match_dup 2) (vec_duplicate:V2SI (match_dup 1)))
5764    (set (match_dup 0) (lshiftrt:DI (match_dup 0) (const_int 32)))]
5765   {
5766     operands[2] = gen_rtx_REG (V2SImode, REGNO (operands[0]));
5767   })
5768
5769 (define_split
5770   [(set (match_operand:DI 0 "s_register_operand" "")
5771         (zero_extend:DI (match_operand:HI 1 "s_register_operand" "")))]
5772   "TARGET_NEON && reload_completed && IS_VFP_REGNUM (REGNO (operands[0]))"
5773   [(set (match_dup 2) (vec_duplicate:V4HI (match_dup 1)))
5774    (set (match_dup 0) (lshiftrt:DI (match_dup 0) (const_int 48)))]
5775   {
5776     operands[2] = gen_rtx_REG (V4HImode, REGNO (operands[0]));
5777   })
5778
5779 (define_split
5780   [(set (match_operand:DI 0 "s_register_operand" "")
5781         (zero_extend:DI (match_operand:QI 1 "s_register_operand" "")))]
5782   "TARGET_NEON && reload_completed && IS_VFP_REGNUM (REGNO (operands[0]))"
5783   [(set (match_dup 2) (vec_duplicate:V8QI (match_dup 1)))
5784    (set (match_dup 0) (lshiftrt:DI (match_dup 0) (const_int 56)))]
5785   {
5786     operands[2] = gen_rtx_REG (V8QImode, REGNO (operands[0]));
5787   })