| blob | 3bb74a2d3ee8dc7ade1ec8135de6f0443c53c2b1 |
1 ;; ARM NEON coprocessor Machine Description
2 ;; Copyright (C) 2006, 2007, 2008, 2009 Free Software Foundation, Inc.
3 ;; Written by CodeSourcery.
4 ;;
5 ;; This file is part of GCC.
6 ;;
7 ;; GCC is free software; you can redistribute it and/or modify it
8 ;; under the terms of the GNU General Public License as published by
9 ;; the Free Software Foundation; either version 3, or (at your option)
10 ;; any later version.
11 ;;
12 ;; GCC is distributed in the hope that it will be useful, but
13 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
14 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 ;; General Public License for more details.
16 ;;
17 ;; You should have received a copy of the GNU General Public License
18 ;; along with GCC; see the file COPYING3. If not see
19 ;; <http://www.gnu.org/licenses/>.
21 ;; Constants for unspecs.
22 (define_constants
23 [(UNSPEC_ASHIFT_SIGNED 65)
24 (UNSPEC_ASHIFT_UNSIGNED 66)
25 (UNSPEC_VABD 69)
26 (UNSPEC_VABDL 70)
27 (UNSPEC_VADD 72)
28 (UNSPEC_VADDHN 73)
29 (UNSPEC_VADDL 74)
30 (UNSPEC_VADDW 75)
31 (UNSPEC_VBSL 78)
32 (UNSPEC_VCAGE 79)
33 (UNSPEC_VCAGT 80)
34 (UNSPEC_VCEQ 81)
35 (UNSPEC_VCGE 82)
36 (UNSPEC_VCGT 83)
37 (UNSPEC_VCLS 84)
38 (UNSPEC_VCVT 88)
39 (UNSPEC_VCVT_N 89)
40 (UNSPEC_VEXT 93)
41 (UNSPEC_VHADD 97)
42 (UNSPEC_VHSUB 98)
43 (UNSPEC_VLD1 99)
44 (UNSPEC_VLD1_DUP 100)
45 (UNSPEC_VLD1_LANE 101)
46 (UNSPEC_VLD2 102)
47 (UNSPEC_VLD2_DUP 103)
48 (UNSPEC_VLD2_LANE 104)
49 (UNSPEC_VLD3 105)
50 (UNSPEC_VLD3A 106)
51 (UNSPEC_VLD3B 107)
52 (UNSPEC_VLD3_DUP 108)
53 (UNSPEC_VLD3_LANE 109)
54 (UNSPEC_VLD4 110)
55 (UNSPEC_VLD4A 111)
56 (UNSPEC_VLD4B 112)
57 (UNSPEC_VLD4_DUP 113)
58 (UNSPEC_VLD4_LANE 114)
59 (UNSPEC_VMAX 115)
60 (UNSPEC_VMIN 116)
61 (UNSPEC_VMLA 117)
62 (UNSPEC_VMLAL 118)
63 (UNSPEC_VMLA_LANE 119)
64 (UNSPEC_VMLAL_LANE 120)
65 (UNSPEC_VMLS 121)
66 (UNSPEC_VMLSL 122)
67 (UNSPEC_VMLS_LANE 123)
68 (UNSPEC_VMLSL_LANE 124)
69 (UNSPEC_VMOVL 125)
70 (UNSPEC_VMOVN 126)
71 (UNSPEC_VMUL 127)
72 (UNSPEC_VMULL 128)
73 (UNSPEC_VMUL_LANE 129)
74 (UNSPEC_VMULL_LANE 130)
75 (UNSPEC_VPADAL 135)
76 (UNSPEC_VPADD 136)
77 (UNSPEC_VPADDL 137)
78 (UNSPEC_VPMAX 138)
79 (UNSPEC_VPMIN 139)
80 (UNSPEC_VPSMAX 140)
81 (UNSPEC_VPSMIN 141)
82 (UNSPEC_VPUMAX 142)
83 (UNSPEC_VPUMIN 143)
84 (UNSPEC_VQABS 144)
85 (UNSPEC_VQADD 145)
86 (UNSPEC_VQDMLAL 146)
87 (UNSPEC_VQDMLAL_LANE 147)
88 (UNSPEC_VQDMLSL 148)
89 (UNSPEC_VQDMLSL_LANE 149)
90 (UNSPEC_VQDMULH 150)
91 (UNSPEC_VQDMULH_LANE 151)
92 (UNSPEC_VQDMULL 152)
93 (UNSPEC_VQDMULL_LANE 153)
94 (UNSPEC_VQMOVN 154)
95 (UNSPEC_VQMOVUN 155)
96 (UNSPEC_VQNEG 156)
97 (UNSPEC_VQSHL 157)
98 (UNSPEC_VQSHL_N 158)
99 (UNSPEC_VQSHLU_N 159)
100 (UNSPEC_VQSHRN_N 160)
101 (UNSPEC_VQSHRUN_N 161)
102 (UNSPEC_VQSUB 162)
103 (UNSPEC_VRECPE 163)
104 (UNSPEC_VRECPS 164)
105 (UNSPEC_VREV16 165)
106 (UNSPEC_VREV32 166)
107 (UNSPEC_VREV64 167)
108 (UNSPEC_VRSQRTE 168)
109 (UNSPEC_VRSQRTS 169)
110 (UNSPEC_VSHL 171)
111 (UNSPEC_VSHLL_N 172)
112 (UNSPEC_VSHL_N 173)
113 (UNSPEC_VSHR_N 174)
114 (UNSPEC_VSHRN_N 175)
115 (UNSPEC_VSLI 176)
116 (UNSPEC_VSRA_N 177)
117 (UNSPEC_VSRI 178)
118 (UNSPEC_VST1 179)
119 (UNSPEC_VST1_LANE 180)
120 (UNSPEC_VST2 181)
121 (UNSPEC_VST2_LANE 182)
122 (UNSPEC_VST3 183)
123 (UNSPEC_VST3A 184)
124 (UNSPEC_VST3B 185)
125 (UNSPEC_VST3_LANE 186)
126 (UNSPEC_VST4 187)
127 (UNSPEC_VST4A 188)
128 (UNSPEC_VST4B 189)
129 (UNSPEC_VST4_LANE 190)
130 (UNSPEC_VSTRUCTDUMMY 191)
131 (UNSPEC_VSUB 192)
132 (UNSPEC_VSUBHN 193)
133 (UNSPEC_VSUBL 194)
134 (UNSPEC_VSUBW 195)
135 (UNSPEC_VTBL 196)
136 (UNSPEC_VTBX 197)
137 (UNSPEC_VTRN1 198)
138 (UNSPEC_VTRN2 199)
139 (UNSPEC_VTST 200)
140 (UNSPEC_VUZP1 201)
141 (UNSPEC_VUZP2 202)
142 (UNSPEC_VZIP1 203)
143 (UNSPEC_VZIP2 204)
144 (UNSPEC_VCLE 206)
145 (UNSPEC_VCLT 207)])
148 ;; Attribute used to permit string comparisons against <VQH_mnem> in
149 ;; neon_type attribute definitions.
150 (define_attr "vqh_mnem" "vadd,vmin,vmax" (const_string "vadd"))
152 (define_insn "*neon_mov<mode>"
153 [(set (match_operand:VD 0 "nonimmediate_operand"
154 "=w,Uv,w, w, ?r,?w,?r,?r, ?Us")
155 (match_operand:VD 1 "general_operand"
156 " w,w, Dn,Uvi, w, r, r, Usi,r"))]
157 "TARGET_NEON
158 && (register_operand (operands[0], <MODE>mode)
159 || register_operand (operands[1], <MODE>mode))"
160 {
161 if (which_alternative == 2)
162 {
163 int width, is_valid;
164 static char templ[40];
166 is_valid = neon_immediate_valid_for_move (operands[1], <MODE>mode,
167 &operands[1], &width);
169 gcc_assert (is_valid != 0);
171 if (width == 0)
172 return "vmov.f32\t%P0, %1 @ <mode>";
173 else
174 sprintf (templ, "vmov.i%d\t%%P0, %%1 @ <mode>", width);
176 return templ;
177 }
179 /* FIXME: If the memory layout is changed in big-endian mode, output_move_vfp
180 below must be changed to output_move_neon (which will use the
181 element/structure loads/stores), and the constraint changed to 'Um' instead
182 of 'Uv'. */
184 switch (which_alternative)
185 {
186 case 0: return "vmov\t%P0, %P1 @ <mode>";
187 case 1: case 3: return output_move_vfp (operands);
188 case 2: gcc_unreachable ();
189 case 4: return "vmov\t%Q0, %R0, %P1 @ <mode>";
190 case 5: return "vmov\t%P0, %Q1, %R1 @ <mode>";
191 default: return output_move_double (operands);
192 }
193 }
194 [(set_attr "neon_type" "neon_int_1,*,neon_vmov,*,neon_mrrc,neon_mcr_2_mcrr,*,*,*")
195 (set_attr "type" "*,f_stored,*,f_loadd,*,*,alu,load2,store2")
196 (set_attr "insn" "*,*,*,*,*,*,mov,*,*")
197 (set_attr "length" "4,4,4,4,4,4,8,8,8")
198 (set_attr "pool_range" "*,*,*,1020,*,*,*,1020,*")
199 (set_attr "neg_pool_range" "*,*,*,1008,*,*,*,1008,*")])
201 (define_insn "*neon_mov<mode>"
202 [(set (match_operand:VQXMOV 0 "nonimmediate_operand"
203 "=w,Un,w, w, ?r,?w,?r,?r, ?Us")
204 (match_operand:VQXMOV 1 "general_operand"
205 " w,w, Dn,Uni, w, r, r, Usi, r"))]
206 "TARGET_NEON
207 && (register_operand (operands[0], <MODE>mode)
208 || register_operand (operands[1], <MODE>mode))"
209 {
210 if (which_alternative == 2)
211 {
212 int width, is_valid;
213 static char templ[40];
215 is_valid = neon_immediate_valid_for_move (operands[1], <MODE>mode,
216 &operands[1], &width);
218 gcc_assert (is_valid != 0);
220 if (width == 0)
221 return "vmov.f32\t%q0, %1 @ <mode>";
222 else
223 sprintf (templ, "vmov.i%d\t%%q0, %%1 @ <mode>", width);
225 return templ;
226 }
228 switch (which_alternative)
229 {
230 case 0: return "vmov\t%q0, %q1 @ <mode>";
231 case 1: case 3: return output_move_neon (operands);
232 case 2: gcc_unreachable ();
233 case 4: return "vmov\t%Q0, %R0, %e1 @ <mode>\;vmov\t%J0, %K0, %f1";
234 case 5: return "vmov\t%e0, %Q1, %R1 @ <mode>\;vmov\t%f0, %J1, %K1";
235 default: return output_move_quad (operands);
236 }
237 }
238 [(set_attr "neon_type" "neon_int_1,neon_stm_2,neon_vmov,neon_ldm_2,\
239 neon_mrrc,neon_mcr_2_mcrr,*,*,*")
240 (set_attr "type" "*,*,*,*,*,*,alu,load4,store4")
241 (set_attr "insn" "*,*,*,*,*,*,mov,*,*")
242 (set_attr "length" "4,8,4,8,8,8,16,8,16")
243 (set_attr "pool_range" "*,*,*,1020,*,*,*,1020,*")
244 (set_attr "neg_pool_range" "*,*,*,1008,*,*,*,1008,*")])
246 (define_expand "movti"
247 [(set (match_operand:TI 0 "nonimmediate_operand" "")
248 (match_operand:TI 1 "general_operand" ""))]
249 "TARGET_NEON"
250 {
251 if (can_create_pseudo_p ())
252 {
253 if (GET_CODE (operands[0]) != REG)
254 operands[1] = force_reg (TImode, operands[1]);
255 }
256 })
258 (define_expand "mov<mode>"
259 [(set (match_operand:VSTRUCT 0 "nonimmediate_operand" "")
260 (match_operand:VSTRUCT 1 "general_operand" ""))]
261 "TARGET_NEON"
262 {
263 if (can_create_pseudo_p ())
264 {
265 if (GET_CODE (operands[0]) != REG)
266 operands[1] = force_reg (<MODE>mode, operands[1]);
267 }
268 })
270 (define_insn "*neon_mov<mode>"
271 [(set (match_operand:VSTRUCT 0 "nonimmediate_operand" "=w,Ut,w")
272 (match_operand:VSTRUCT 1 "general_operand" " w,w, Ut"))]
273 "TARGET_NEON
274 && (register_operand (operands[0], <MODE>mode)
275 || register_operand (operands[1], <MODE>mode))"
276 {
277 switch (which_alternative)
278 {
279 case 0: return "#";
280 case 1: case 2: return output_move_neon (operands);
281 default: gcc_unreachable ();
282 }
283 }
284 [(set_attr "neon_type" "neon_int_1,neon_stm_2,neon_ldm_2")
285 (set (attr "length") (symbol_ref "arm_attr_length_move_neon (insn)"))])
287 (define_split
288 [(set (match_operand:EI 0 "s_register_operand" "")
289 (match_operand:EI 1 "s_register_operand" ""))]
290 "TARGET_NEON && reload_completed"
291 [(set (match_dup 0) (match_dup 1))
292 (set (match_dup 2) (match_dup 3))]
293 {
294 int rdest = REGNO (operands[0]);
295 int rsrc = REGNO (operands[1]);
296 rtx dest[2], src[2];
298 dest[0] = gen_rtx_REG (TImode, rdest);
299 src[0] = gen_rtx_REG (TImode, rsrc);
300 dest[1] = gen_rtx_REG (DImode, rdest + 4);
301 src[1] = gen_rtx_REG (DImode, rsrc + 4);
303 neon_disambiguate_copy (operands, dest, src, 2);
304 })
306 (define_split
307 [(set (match_operand:OI 0 "s_register_operand" "")
308 (match_operand:OI 1 "s_register_operand" ""))]
309 "TARGET_NEON && reload_completed"
310 [(set (match_dup 0) (match_dup 1))
311 (set (match_dup 2) (match_dup 3))]
312 {
313 int rdest = REGNO (operands[0]);
314 int rsrc = REGNO (operands[1]);
315 rtx dest[2], src[2];
317 dest[0] = gen_rtx_REG (TImode, rdest);
318 src[0] = gen_rtx_REG (TImode, rsrc);
319 dest[1] = gen_rtx_REG (TImode, rdest + 4);
320 src[1] = gen_rtx_REG (TImode, rsrc + 4);
322 neon_disambiguate_copy (operands, dest, src, 2);
323 })
325 (define_split
326 [(set (match_operand:CI 0 "s_register_operand" "")
327 (match_operand:CI 1 "s_register_operand" ""))]
328 "TARGET_NEON && reload_completed"
329 [(set (match_dup 0) (match_dup 1))
330 (set (match_dup 2) (match_dup 3))
331 (set (match_dup 4) (match_dup 5))]
332 {
333 int rdest = REGNO (operands[0]);
334 int rsrc = REGNO (operands[1]);
335 rtx dest[3], src[3];
337 dest[0] = gen_rtx_REG (TImode, rdest);
338 src[0] = gen_rtx_REG (TImode, rsrc);
339 dest[1] = gen_rtx_REG (TImode, rdest + 4);
340 src[1] = gen_rtx_REG (TImode, rsrc + 4);
341 dest[2] = gen_rtx_REG (TImode, rdest + 8);
342 src[2] = gen_rtx_REG (TImode, rsrc + 8);
344 neon_disambiguate_copy (operands, dest, src, 3);
345 })
347 (define_split
348 [(set (match_operand:XI 0 "s_register_operand" "")
349 (match_operand:XI 1 "s_register_operand" ""))]
350 "TARGET_NEON && reload_completed"
351 [(set (match_dup 0) (match_dup 1))
352 (set (match_dup 2) (match_dup 3))
353 (set (match_dup 4) (match_dup 5))
354 (set (match_dup 6) (match_dup 7))]
355 {
356 int rdest = REGNO (operands[0]);
357 int rsrc = REGNO (operands[1]);
358 rtx dest[4], src[4];
360 dest[0] = gen_rtx_REG (TImode, rdest);
361 src[0] = gen_rtx_REG (TImode, rsrc);
362 dest[1] = gen_rtx_REG (TImode, rdest + 4);
363 src[1] = gen_rtx_REG (TImode, rsrc + 4);
364 dest[2] = gen_rtx_REG (TImode, rdest + 8);
365 src[2] = gen_rtx_REG (TImode, rsrc + 8);
366 dest[3] = gen_rtx_REG (TImode, rdest + 12);
367 src[3] = gen_rtx_REG (TImode, rsrc + 12);
369 neon_disambiguate_copy (operands, dest, src, 4);
370 })
372 (define_insn "vec_set<mode>_internal"
373 [(set (match_operand:VD 0 "s_register_operand" "=w")
374 (vec_merge:VD
375 (vec_duplicate:VD
376 (match_operand:<V_elem> 1 "s_register_operand" "r"))
377 (match_operand:VD 3 "s_register_operand" "0")
378 (match_operand:SI 2 "immediate_operand" "i")))]
379 "TARGET_NEON"
380 {
381 int elt = ffs ((int) INTVAL (operands[2]) - 1);
382 if (BYTES_BIG_ENDIAN)
383 elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
384 operands[2] = GEN_INT (elt);
386 return "vmov%?.<V_sz_elem>\t%P0[%c2], %1";
387 }
388 [(set_attr "predicable" "yes")
389 (set_attr "neon_type" "neon_mcr")])
391 (define_insn "vec_set<mode>_internal"
392 [(set (match_operand:VQ 0 "s_register_operand" "=w")
393 (vec_merge:VQ
394 (vec_duplicate:VQ
395 (match_operand:<V_elem> 1 "s_register_operand" "r"))
396 (match_operand:VQ 3 "s_register_operand" "0")
397 (match_operand:SI 2 "immediate_operand" "i")))]
398 "TARGET_NEON"
399 {
400 HOST_WIDE_INT elem = ffs ((int) INTVAL (operands[2])) - 1;
401 int half_elts = GET_MODE_NUNITS (<MODE>mode) / 2;
402 int elt = elem % half_elts;
403 int hi = (elem / half_elts) * 2;
404 int regno = REGNO (operands[0]);
406 if (BYTES_BIG_ENDIAN)
407 elt = half_elts - 1 - elt;
409 operands[0] = gen_rtx_REG (<V_HALF>mode, regno + hi);
410 operands[2] = GEN_INT (elt);
412 return "vmov%?.<V_sz_elem>\t%P0[%c2], %1";
413 }
414 [(set_attr "predicable" "yes")
415 (set_attr "neon_type" "neon_mcr")]
416 )
418 (define_insn "vec_setv2di_internal"
419 [(set (match_operand:V2DI 0 "s_register_operand" "=w")
420 (vec_merge:V2DI
421 (vec_duplicate:V2DI
422 (match_operand:DI 1 "s_register_operand" "r"))
423 (match_operand:V2DI 3 "s_register_operand" "0")
424 (match_operand:SI 2 "immediate_operand" "i")))]
425 "TARGET_NEON"
426 {
427 HOST_WIDE_INT elem = ffs ((int) INTVAL (operands[2])) - 1;
428 int regno = REGNO (operands[0]) + 2 * elem;
430 operands[0] = gen_rtx_REG (DImode, regno);
432 return "vmov%?\t%P0, %Q1, %R1";
433 }
434 [(set_attr "predicable" "yes")
435 (set_attr "neon_type" "neon_mcr_2_mcrr")]
436 )
438 (define_expand "vec_set<mode>"
439 [(match_operand:VDQ 0 "s_register_operand" "")
440 (match_operand:<V_elem> 1 "s_register_operand" "")
441 (match_operand:SI 2 "immediate_operand" "")]
442 "TARGET_NEON"
443 {
444 HOST_WIDE_INT elem = (HOST_WIDE_INT) 1 << INTVAL (operands[2]);
445 emit_insn (gen_vec_set<mode>_internal (operands[0], operands[1],
446 GEN_INT (elem), operands[0]));
447 DONE;
448 })
450 (define_insn "vec_extract<mode>"
451 [(set (match_operand:<V_elem> 0 "s_register_operand" "=r")
452 (vec_select:<V_elem>
453 (match_operand:VD 1 "s_register_operand" "w")
454 (parallel [(match_operand:SI 2 "immediate_operand" "i")])))]
455 "TARGET_NEON"
456 {
457 if (BYTES_BIG_ENDIAN)
458 {
459 int elt = INTVAL (operands[2]);
460 elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
461 operands[2] = GEN_INT (elt);
462 }
463 return "vmov%?.<V_uf_sclr>\t%0, %P1[%c2]";
464 }
465 [(set_attr "predicable" "yes")
466 (set_attr "neon_type" "neon_bp_simple")]
467 )
469 (define_insn "vec_extract<mode>"
470 [(set (match_operand:<V_elem> 0 "s_register_operand" "=r")
471 (vec_select:<V_elem>
472 (match_operand:VQ 1 "s_register_operand" "w")
473 (parallel [(match_operand:SI 2 "immediate_operand" "i")])))]
474 "TARGET_NEON"
475 {
476 int half_elts = GET_MODE_NUNITS (<MODE>mode) / 2;
477 int elt = INTVAL (operands[2]) % half_elts;
478 int hi = (INTVAL (operands[2]) / half_elts) * 2;
479 int regno = REGNO (operands[1]);
481 if (BYTES_BIG_ENDIAN)
482 elt = half_elts - 1 - elt;
484 operands[1] = gen_rtx_REG (<V_HALF>mode, regno + hi);
485 operands[2] = GEN_INT (elt);
487 return "vmov%?.<V_uf_sclr>\t%0, %P1[%c2]";
488 }
489 [(set_attr "predicable" "yes")
490 (set_attr "neon_type" "neon_bp_simple")]
491 )
493 (define_insn "vec_extractv2di"
494 [(set (match_operand:DI 0 "s_register_operand" "=r")
495 (vec_select:DI
496 (match_operand:V2DI 1 "s_register_operand" "w")
497 (parallel [(match_operand:SI 2 "immediate_operand" "i")])))]
498 "TARGET_NEON"
499 {
500 int regno = REGNO (operands[1]) + 2 * INTVAL (operands[2]);
502 operands[1] = gen_rtx_REG (DImode, regno);
504 return "vmov%?\t%Q0, %R0, %P1 @ v2di";
505 }
506 [(set_attr "predicable" "yes")
507 (set_attr "neon_type" "neon_int_1")]
508 )
510 (define_expand "vec_init<mode>"
511 [(match_operand:VDQ 0 "s_register_operand" "")
512 (match_operand 1 "" "")]
513 "TARGET_NEON"
514 {
515 neon_expand_vector_init (operands[0], operands[1]);
516 DONE;
517 })
519 ;; Doubleword and quadword arithmetic.
521 ;; NOTE: some other instructions also support 64-bit integer
522 ;; element size, which we could potentially use for "long long" operations.
524 (define_insn "*add<mode>3_neon"
525 [(set (match_operand:VDQ 0 "s_register_operand" "=w")
526 (plus:VDQ (match_operand:VDQ 1 "s_register_operand" "w")
527 (match_operand:VDQ 2 "s_register_operand" "w")))]
528 "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
529 "vadd.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
530 [(set (attr "neon_type")
531 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
532 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
533 (const_string "neon_fp_vadd_ddd_vabs_dd")
534 (const_string "neon_fp_vadd_qqq_vabs_qq"))
535 (const_string "neon_int_1")))]
536 )
538 (define_insn "adddi3_neon"
539 [(set (match_operand:DI 0 "s_register_operand" "=w,?&r,?&r")
540 (plus:DI (match_operand:DI 1 "s_register_operand" "%w,0,0")
541 (match_operand:DI 2 "s_register_operand" "w,r,0")))
542 (clobber (reg:CC CC_REGNUM))]
543 "TARGET_NEON"
544 {
545 switch (which_alternative)
546 {
547 case 0: return "vadd.i64\t%P0, %P1, %P2";
548 case 1: return "#";
549 case 2: return "#";
550 default: gcc_unreachable ();
551 }
552 }
553 [(set_attr "neon_type" "neon_int_1,*,*")
554 (set_attr "conds" "*,clob,clob")
555 (set_attr "length" "*,8,8")]
556 )
558 (define_insn "*sub<mode>3_neon"
559 [(set (match_operand:VDQ 0 "s_register_operand" "=w")
560 (minus:VDQ (match_operand:VDQ 1 "s_register_operand" "w")
561 (match_operand:VDQ 2 "s_register_operand" "w")))]
562 "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
563 "vsub.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
564 [(set (attr "neon_type")
565 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
566 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
567 (const_string "neon_fp_vadd_ddd_vabs_dd")
568 (const_string "neon_fp_vadd_qqq_vabs_qq"))
569 (const_string "neon_int_2")))]
570 )
572 (define_insn "subdi3_neon"
573 [(set (match_operand:DI 0 "s_register_operand" "=w,?&r,?&r,?&r")
574 (minus:DI (match_operand:DI 1 "s_register_operand" "w,0,r,0")
575 (match_operand:DI 2 "s_register_operand" "w,r,0,0")))
576 (clobber (reg:CC CC_REGNUM))]
577 "TARGET_NEON"
578 {
579 switch (which_alternative)
580 {
581 case 0: return "vsub.i64\t%P0, %P1, %P2";
582 case 1: /* fall through */
583 case 2: /* fall through */
584 case 3: return "subs\\t%Q0, %Q1, %Q2\;sbc\\t%R0, %R1, %R2";
585 default: gcc_unreachable ();
586 }
587 }
588 [(set_attr "neon_type" "neon_int_2,*,*,*")
589 (set_attr "conds" "*,clob,clob,clob")
590 (set_attr "length" "*,8,8,8")]
591 )
593 (define_insn "*mul<mode>3_neon"
594 [(set (match_operand:VDQ 0 "s_register_operand" "=w")
595 (mult:VDQ (match_operand:VDQ 1 "s_register_operand" "w")
596 (match_operand:VDQ 2 "s_register_operand" "w")))]
597 "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
598 "vmul.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
599 [(set (attr "neon_type")
600 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
601 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
602 (const_string "neon_fp_vadd_ddd_vabs_dd")
603 (const_string "neon_fp_vadd_qqq_vabs_qq"))
604 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
605 (if_then_else
606 (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0))
607 (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long")
608 (const_string "neon_mul_qqq_8_16_32_ddd_32"))
609 (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0))
610 (const_string "neon_mul_qqq_8_16_32_ddd_32")
611 (const_string "neon_mul_qqq_8_16_32_ddd_32")))))]
612 )
614 (define_insn "mul<mode>3add<mode>_neon"
615 [(set (match_operand:VDQ 0 "s_register_operand" "=w")
616 (plus:VDQ (mult:VDQ (match_operand:VDQ 2 "s_register_operand" "w")
617 (match_operand:VDQ 3 "s_register_operand" "w"))
618 (match_operand:VDQ 1 "s_register_operand" "0")))]
619 "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
620 "vmla.<V_if_elem>\t%<V_reg>0, %<V_reg>2, %<V_reg>3"
621 [(set (attr "neon_type")
622 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
623 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
624 (const_string "neon_fp_vmla_ddd")
625 (const_string "neon_fp_vmla_qqq"))
626 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
627 (if_then_else
628 (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0))
629 (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
630 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long"))
631 (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0))
632 (const_string "neon_mla_qqq_8_16")
633 (const_string "neon_mla_qqq_32_qqd_32_scalar")))))]
634 )
636 (define_insn "mul<mode>3neg<mode>add<mode>_neon"
637 [(set (match_operand:VDQ 0 "s_register_operand" "=w")
638 (minus:VDQ (match_operand:VDQ 1 "s_register_operand" "0")
639 (mult:VDQ (match_operand:VDQ 2 "s_register_operand" "w")
640 (match_operand:VDQ 3 "s_register_operand" "w"))))]
641 "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
642 "vmls.<V_if_elem>\t%<V_reg>0, %<V_reg>2, %<V_reg>3"
643 [(set (attr "neon_type")
644 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
645 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
646 (const_string "neon_fp_vmla_ddd")
647 (const_string "neon_fp_vmla_qqq"))
648 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
649 (if_then_else
650 (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0))
651 (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
652 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long"))
653 (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0))
654 (const_string "neon_mla_qqq_8_16")
655 (const_string "neon_mla_qqq_32_qqd_32_scalar")))))]
656 )
658 (define_insn "ior<mode>3"
659 [(set (match_operand:VDQ 0 "s_register_operand" "=w,w")
660 (ior:VDQ (match_operand:VDQ 1 "s_register_operand" "w,0")
661 (match_operand:VDQ 2 "neon_logic_op2" "w,Dl")))]
662 "TARGET_NEON"
663 {
664 switch (which_alternative)
665 {
666 case 0: return "vorr\t%<V_reg>0, %<V_reg>1, %<V_reg>2";
667 case 1: return neon_output_logic_immediate ("vorr", &operands[2],
668 <MODE>mode, 0, VALID_NEON_QREG_MODE (<MODE>mode));
669 default: gcc_unreachable ();
670 }
671 }
672 [(set_attr "neon_type" "neon_int_1")]
673 )
675 (define_insn "iordi3_neon"
676 [(set (match_operand:DI 0 "s_register_operand" "=w,w,?&r,?&r")
677 (ior:DI (match_operand:DI 1 "s_register_operand" "%w,0,0,r")
678 (match_operand:DI 2 "neon_logic_op2" "w,Dl,r,r")))]
679 "TARGET_NEON"
680 {
681 switch (which_alternative)
682 {
683 case 0: return "vorr\t%P0, %P1, %P2";
684 case 1: return neon_output_logic_immediate ("vorr", &operands[2],
685 DImode, 0, VALID_NEON_QREG_MODE (DImode));
686 case 2: return "#";
687 case 3: return "#";
688 default: gcc_unreachable ();
689 }
690 }
691 [(set_attr "neon_type" "neon_int_1,neon_int_1,*,*")
692 (set_attr "length" "*,*,8,8")]
693 )
695 ;; The concrete forms of the Neon immediate-logic instructions are vbic and
696 ;; vorr. We support the pseudo-instruction vand instead, because that
697 ;; corresponds to the canonical form the middle-end expects to use for
698 ;; immediate bitwise-ANDs.
700 (define_insn "and<mode>3"
701 [(set (match_operand:VDQ 0 "s_register_operand" "=w,w")
702 (and:VDQ (match_operand:VDQ 1 "s_register_operand" "w,0")
703 (match_operand:VDQ 2 "neon_inv_logic_op2" "w,DL")))]
704 "TARGET_NEON"
705 {
706 switch (which_alternative)
707 {
708 case 0: return "vand\t%<V_reg>0, %<V_reg>1, %<V_reg>2";
709 case 1: return neon_output_logic_immediate ("vand", &operands[2],
710 <MODE>mode, 1, VALID_NEON_QREG_MODE (<MODE>mode));
711 default: gcc_unreachable ();
712 }
713 }
714 [(set_attr "neon_type" "neon_int_1")]
715 )
717 (define_insn "anddi3_neon"
718 [(set (match_operand:DI 0 "s_register_operand" "=w,w,?&r,?&r")
719 (and:DI (match_operand:DI 1 "s_register_operand" "%w,0,0,r")
720 (match_operand:DI 2 "neon_inv_logic_op2" "w,DL,r,r")))]
721 "TARGET_NEON"
722 {
723 switch (which_alternative)
724 {
725 case 0: return "vand\t%P0, %P1, %P2";
726 case 1: return neon_output_logic_immediate ("vand", &operands[2],
727 DImode, 1, VALID_NEON_QREG_MODE (DImode));
728 case 2: return "#";
729 case 3: return "#";
730 default: gcc_unreachable ();
731 }
732 }
733 [(set_attr "neon_type" "neon_int_1,neon_int_1,*,*")
734 (set_attr "length" "*,*,8,8")]
735 )
737 (define_insn "orn<mode>3_neon"
738 [(set (match_operand:VDQ 0 "s_register_operand" "=w")
739 (ior:VDQ (match_operand:VDQ 1 "s_register_operand" "w")
740 (not:VDQ (match_operand:VDQ 2 "s_register_operand" "w"))))]
741 "TARGET_NEON"
742 "vorn\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
743 [(set_attr "neon_type" "neon_int_1")]
744 )
746 (define_insn "orndi3_neon"
747 [(set (match_operand:DI 0 "s_register_operand" "=w,?=&r,?&r")
748 (ior:DI (match_operand:DI 1 "s_register_operand" "w,r,0")
749 (not:DI (match_operand:DI 2 "s_register_operand" "w,0,r"))))]
750 "TARGET_NEON"
751 "@
752 vorn\t%P0, %P1, %P2
753 #
754 #"
755 [(set_attr "neon_type" "neon_int_1,*,*")
756 (set_attr "length" "*,8,8")]
757 )
759 (define_insn "bic<mode>3_neon"
760 [(set (match_operand:VDQ 0 "s_register_operand" "=w")
761 (and:VDQ (match_operand:VDQ 1 "s_register_operand" "w")
762 (not:VDQ (match_operand:VDQ 2 "s_register_operand" "w"))))]
763 "TARGET_NEON"
764 "vbic\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
765 [(set_attr "neon_type" "neon_int_1")]
766 )
768 ;; Compare to *anddi_notdi_di.
769 (define_insn "bicdi3_neon"
770 [(set (match_operand:DI 0 "s_register_operand" "=w,?=&r,?&r")
771 (and:DI (not:DI (match_operand:DI 2 "s_register_operand" "w,r,0"))
772 (match_operand:DI 1 "s_register_operand" "w,0,r")))]
773 "TARGET_NEON"
774 "@
775 vbic\t%P0, %P1, %P2
776 #
777 #"
778 [(set_attr "neon_type" "neon_int_1,*,*")
779 (set_attr "length" "*,8,8")]
780 )
782 (define_insn "xor<mode>3"
783 [(set (match_operand:VDQ 0 "s_register_operand" "=w")
784 (xor:VDQ (match_operand:VDQ 1 "s_register_operand" "w")
785 (match_operand:VDQ 2 "s_register_operand" "w")))]
786 "TARGET_NEON"
787 "veor\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
788 [(set_attr "neon_type" "neon_int_1")]
789 )
791 (define_insn "xordi3_neon"
792 [(set (match_operand:DI 0 "s_register_operand" "=w,?&r,?&r")
793 (xor:DI (match_operand:DI 1 "s_register_operand" "%w,0,r")
794 (match_operand:DI 2 "s_register_operand" "w,r,r")))]
795 "TARGET_NEON"
796 "@
797 veor\t%P0, %P1, %P2
798 #
799 #"
800 [(set_attr "neon_type" "neon_int_1,*,*")
801 (set_attr "length" "*,8,8")]
802 )
804 (define_insn "one_cmpl<mode>2"
805 [(set (match_operand:VDQ 0 "s_register_operand" "=w")
806 (not:VDQ (match_operand:VDQ 1 "s_register_operand" "w")))]
807 "TARGET_NEON"
808 "vmvn\t%<V_reg>0, %<V_reg>1"
809 [(set_attr "neon_type" "neon_int_1")]
810 )
812 (define_insn "abs<mode>2"
813 [(set (match_operand:VDQW 0 "s_register_operand" "=w")
814 (abs:VDQW (match_operand:VDQW 1 "s_register_operand" "w")))]
815 "TARGET_NEON"
816 "vabs.<V_s_elem>\t%<V_reg>0, %<V_reg>1"
817 [(set (attr "neon_type")
818 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
819 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
820 (const_string "neon_fp_vadd_ddd_vabs_dd")
821 (const_string "neon_fp_vadd_qqq_vabs_qq"))
822 (const_string "neon_int_3")))]
823 )
825 (define_insn "neg<mode>2"
826 [(set (match_operand:VDQW 0 "s_register_operand" "=w")
827 (neg:VDQW (match_operand:VDQW 1 "s_register_operand" "w")))]
828 "TARGET_NEON"
829 "vneg.<V_s_elem>\t%<V_reg>0, %<V_reg>1"
830 [(set (attr "neon_type")
831 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
832 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
833 (const_string "neon_fp_vadd_ddd_vabs_dd")
834 (const_string "neon_fp_vadd_qqq_vabs_qq"))
835 (const_string "neon_int_3")))]
836 )
838 (define_insn "*umin<mode>3_neon"
839 [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
840 (umin:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w")
841 (match_operand:VDQIW 2 "s_register_operand" "w")))]
842 "TARGET_NEON"
843 "vmin.<V_u_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
844 [(set_attr "neon_type" "neon_int_5")]
845 )
847 (define_insn "*umax<mode>3_neon"
848 [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
849 (umax:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w")
850 (match_operand:VDQIW 2 "s_register_operand" "w")))]
851 "TARGET_NEON"
852 "vmax.<V_u_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
853 [(set_attr "neon_type" "neon_int_5")]
854 )
856 (define_insn "*smin<mode>3_neon"
857 [(set (match_operand:VDQW 0 "s_register_operand" "=w")
858 (smin:VDQW (match_operand:VDQW 1 "s_register_operand" "w")
859 (match_operand:VDQW 2 "s_register_operand" "w")))]
860 "TARGET_NEON"
861 "vmin.<V_s_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
862 [(set (attr "neon_type")
863 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
864 (const_string "neon_fp_vadd_ddd_vabs_dd")
865 (const_string "neon_int_5")))]
866 )
868 (define_insn "*smax<mode>3_neon"
869 [(set (match_operand:VDQW 0 "s_register_operand" "=w")
870 (smax:VDQW (match_operand:VDQW 1 "s_register_operand" "w")
871 (match_operand:VDQW 2 "s_register_operand" "w")))]
872 "TARGET_NEON"
873 "vmax.<V_s_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
874 [(set (attr "neon_type")
875 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
876 (const_string "neon_fp_vadd_ddd_vabs_dd")
877 (const_string "neon_int_5")))]
878 )
880 ; TODO: V2DI shifts are current disabled because there are bugs in the
881 ; generic vectorizer code. It ends up creating a V2DI constructor with
882 ; SImode elements.
884 (define_insn "vashl<mode>3"
885 [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
886 (ashift:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w")
887 (match_operand:VDQIW 2 "s_register_operand" "w")))]
888 "TARGET_NEON"
889 "vshl.<V_s_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
890 [(set (attr "neon_type")
891 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
892 (const_string "neon_vshl_ddd")
893 (const_string "neon_shift_3")))]
894 )
896 ; Used for implementing logical shift-right, which is a left-shift by a negative
897 ; amount, with signed operands. This is essentially the same as ashl<mode>3
898 ; above, but using an unspec in case GCC tries anything tricky with negative
899 ; shift amounts.
901 (define_insn "ashl<mode>3_signed"
902 [(set (match_operand:VDQI 0 "s_register_operand" "=w")
903 (unspec:VDQI [(match_operand:VDQI 1 "s_register_operand" "w")
904 (match_operand:VDQI 2 "s_register_operand" "w")]
905 UNSPEC_ASHIFT_SIGNED))]
906 "TARGET_NEON"
907 "vshl.<V_s_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
908 [(set (attr "neon_type")
909 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
910 (const_string "neon_vshl_ddd")
911 (const_string "neon_shift_3")))]
912 )
914 ; Used for implementing logical shift-right, which is a left-shift by a negative
915 ; amount, with unsigned operands.
917 (define_insn "ashl<mode>3_unsigned"
918 [(set (match_operand:VDQI 0 "s_register_operand" "=w")
919 (unspec:VDQI [(match_operand:VDQI 1 "s_register_operand" "w")
920 (match_operand:VDQI 2 "s_register_operand" "w")]
921 UNSPEC_ASHIFT_UNSIGNED))]
922 "TARGET_NEON"
923 "vshl.<V_u_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
924 [(set (attr "neon_type")
925 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
926 (const_string "neon_vshl_ddd")
927 (const_string "neon_shift_3")))]
928 )
930 (define_expand "vashr<mode>3"
931 [(set (match_operand:VDQIW 0 "s_register_operand" "")
932 (ashiftrt:VDQIW (match_operand:VDQIW 1 "s_register_operand" "")
933 (match_operand:VDQIW 2 "s_register_operand" "")))]
934 "TARGET_NEON"
935 {
936 rtx neg = gen_reg_rtx (<MODE>mode);
938 emit_insn (gen_neg<mode>2 (neg, operands[2]));
939 emit_insn (gen_ashl<mode>3_signed (operands[0], operands[1], neg));
941 DONE;
942 })
944 (define_expand "vlshr<mode>3"
945 [(set (match_operand:VDQIW 0 "s_register_operand" "")
946 (lshiftrt:VDQIW (match_operand:VDQIW 1 "s_register_operand" "")
947 (match_operand:VDQIW 2 "s_register_operand" "")))]
948 "TARGET_NEON"
949 {
950 rtx neg = gen_reg_rtx (<MODE>mode);
952 emit_insn (gen_neg<mode>2 (neg, operands[2]));
953 emit_insn (gen_ashl<mode>3_unsigned (operands[0], operands[1], neg));
955 DONE;
956 })
958 ;; Widening operations
960 (define_insn "widen_ssum<mode>3"
961 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
962 (plus:<V_widen> (sign_extend:<V_widen>
963 (match_operand:VW 1 "s_register_operand" "%w"))
964 (match_operand:<V_widen> 2 "s_register_operand" "w")))]
965 "TARGET_NEON"
966 "vaddw.<V_s_elem>\t%q0, %q2, %P1"
967 [(set_attr "neon_type" "neon_int_3")]
968 )
970 (define_insn "widen_usum<mode>3"
971 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
972 (plus:<V_widen> (zero_extend:<V_widen>
973 (match_operand:VW 1 "s_register_operand" "%w"))
974 (match_operand:<V_widen> 2 "s_register_operand" "w")))]
975 "TARGET_NEON"
976 "vaddw.<V_u_elem>\t%q0, %q2, %P1"
977 [(set_attr "neon_type" "neon_int_3")]
978 )
980 ;; VEXT can be used to synthesize coarse whole-vector shifts with 8-bit
981 ;; shift-count granularity. That's good enough for the middle-end's current
982 ;; needs.
984 (define_expand "vec_shr_<mode>"
985 [(match_operand:VDQ 0 "s_register_operand" "")
986 (match_operand:VDQ 1 "s_register_operand" "")
987 (match_operand:SI 2 "const_multiple_of_8_operand" "")]
988 "TARGET_NEON"
989 {
990 rtx zero_reg;
991 HOST_WIDE_INT num_bits = INTVAL (operands[2]);
992 const int width = GET_MODE_BITSIZE (<MODE>mode);
993 const enum machine_mode bvecmode = (width == 128) ? V16QImode : V8QImode;
994 rtx (*gen_ext) (rtx, rtx, rtx, rtx) =
995 (width == 128) ? gen_neon_vextv16qi : gen_neon_vextv8qi;
997 if (num_bits == width)
998 {
999 emit_move_insn (operands[0], operands[1]);
1000 DONE;
1001 }
1003 zero_reg = force_reg (bvecmode, CONST0_RTX (bvecmode));
1004 operands[0] = gen_lowpart (bvecmode, operands[0]);
1005 operands[1] = gen_lowpart (bvecmode, operands[1]);
1007 emit_insn (gen_ext (operands[0], operands[1], zero_reg,
1008 GEN_INT (num_bits / BITS_PER_UNIT)));
1009 DONE;
1010 })
1012 (define_expand "vec_shl_<mode>"
1013 [(match_operand:VDQ 0 "s_register_operand" "")
1014 (match_operand:VDQ 1 "s_register_operand" "")
1015 (match_operand:SI 2 "const_multiple_of_8_operand" "")]
1016 "TARGET_NEON"
1017 {
1018 rtx zero_reg;
1019 HOST_WIDE_INT num_bits = INTVAL (operands[2]);
1020 const int width = GET_MODE_BITSIZE (<MODE>mode);
1021 const enum machine_mode bvecmode = (width == 128) ? V16QImode : V8QImode;
1022 rtx (*gen_ext) (rtx, rtx, rtx, rtx) =
1023 (width == 128) ? gen_neon_vextv16qi : gen_neon_vextv8qi;
1025 if (num_bits == 0)
1026 {
1027 emit_move_insn (operands[0], CONST0_RTX (<MODE>mode));
1028 DONE;
1029 }
1031 num_bits = width - num_bits;
1033 zero_reg = force_reg (bvecmode, CONST0_RTX (bvecmode));
1034 operands[0] = gen_lowpart (bvecmode, operands[0]);
1035 operands[1] = gen_lowpart (bvecmode, operands[1]);
1037 emit_insn (gen_ext (operands[0], zero_reg, operands[1],
1038 GEN_INT (num_bits / BITS_PER_UNIT)));
1039 DONE;
1040 })
1042 ;; Helpers for quad-word reduction operations
1044 ; Add (or smin, smax...) the low N/2 elements of the N-element vector
1045 ; operand[1] to the high N/2 elements of same. Put the result in operand[0], an
1046 ; N/2-element vector.
1048 (define_insn "quad_halves_<code>v4si"
1049 [(set (match_operand:V2SI 0 "s_register_operand" "=w")
1050 (vqh_ops:V2SI
1051 (vec_select:V2SI (match_operand:V4SI 1 "s_register_operand" "w")
1052 (parallel [(const_int 0) (const_int 1)]))
1053 (vec_select:V2SI (match_dup 1)
1054 (parallel [(const_int 2) (const_int 3)]))))]
1055 "TARGET_NEON"
1056 "<VQH_mnem>.<VQH_sign>32\t%P0, %e1, %f1"
1057 [(set_attr "vqh_mnem" "<VQH_mnem>")
1058 (set (attr "neon_type")
1059 (if_then_else (eq_attr "vqh_mnem" "vadd")
1060 (const_string "neon_int_1") (const_string "neon_int_5")))]
1061 )
1063 (define_insn "quad_halves_<code>v4sf"
1064 [(set (match_operand:V2SF 0 "s_register_operand" "=w")
1065 (vqhs_ops:V2SF
1066 (vec_select:V2SF (match_operand:V4SF 1 "s_register_operand" "w")
1067 (parallel [(const_int 0) (const_int 1)]))
1068 (vec_select:V2SF (match_dup 1)
1069 (parallel [(const_int 2) (const_int 3)]))))]
1070 "TARGET_NEON && flag_unsafe_math_optimizations"
1071 "<VQH_mnem>.f32\t%P0, %e1, %f1"
1072 [(set_attr "vqh_mnem" "<VQH_mnem>")
1073 (set (attr "neon_type")
1074 (if_then_else (eq_attr "vqh_mnem" "vadd")
1075 (const_string "neon_int_1") (const_string "neon_int_5")))]
1076 )
1078 (define_insn "quad_halves_<code>v8hi"
1079 [(set (match_operand:V4HI 0 "s_register_operand" "+w")
1080 (vqh_ops:V4HI
1081 (vec_select:V4HI (match_operand:V8HI 1 "s_register_operand" "w")
1082 (parallel [(const_int 0) (const_int 1)
1083 (const_int 2) (const_int 3)]))
1084 (vec_select:V4HI (match_dup 1)
1085 (parallel [(const_int 4) (const_int 5)
1086 (const_int 6) (const_int 7)]))))]
1087 "TARGET_NEON"
1088 "<VQH_mnem>.<VQH_sign>16\t%P0, %e1, %f1"
1089 [(set_attr "vqh_mnem" "<VQH_mnem>")
1090 (set (attr "neon_type")
1091 (if_then_else (eq_attr "vqh_mnem" "vadd")
1092 (const_string "neon_int_1") (const_string "neon_int_5")))]
1093 )
1095 (define_insn "quad_halves_<code>v16qi"
1096 [(set (match_operand:V8QI 0 "s_register_operand" "+w")
1097 (vqh_ops:V8QI
1098 (vec_select:V8QI (match_operand:V16QI 1 "s_register_operand" "w")
1099 (parallel [(const_int 0) (const_int 1)
1100 (const_int 2) (const_int 3)
1101 (const_int 4) (const_int 5)
1102 (const_int 6) (const_int 7)]))
1103 (vec_select:V8QI (match_dup 1)
1104 (parallel [(const_int 8) (const_int 9)
1105 (const_int 10) (const_int 11)
1106 (const_int 12) (const_int 13)
1107 (const_int 14) (const_int 15)]))))]
1108 "TARGET_NEON"
1109 "<VQH_mnem>.<VQH_sign>8\t%P0, %e1, %f1"
1110 [(set_attr "vqh_mnem" "<VQH_mnem>")
1111 (set (attr "neon_type")
1112 (if_then_else (eq_attr "vqh_mnem" "vadd")
1113 (const_string "neon_int_1") (const_string "neon_int_5")))]
1114 )
1116 ; FIXME: We wouldn't need the following insns if we could write subregs of
1117 ; vector registers. Make an attempt at removing unnecessary moves, though
1118 ; we're really at the mercy of the register allocator.
1120 (define_insn "neon_move_lo_quad_<mode>"
1121 [(set (match_operand:ANY128 0 "s_register_operand" "+w")
1122 (vec_concat:ANY128
1123 (match_operand:<V_HALF> 1 "s_register_operand" "w")
1124 (vec_select:<V_HALF>
1125 (match_dup 0)
1126 (match_operand:ANY128 2 "vect_par_constant_high" ""))))]
1127 "TARGET_NEON"
1128 {
1129 int dest = REGNO (operands[0]);
1130 int src = REGNO (operands[1]);
1132 if (dest != src)
1133 return "vmov\t%e0, %P1";
1134 else
1135 return "";
1136 }
1137 [(set_attr "neon_type" "neon_bp_simple")]
1138 )
1140 (define_insn "neon_move_hi_quad_<mode>"
1141 [(set (match_operand:ANY128 0 "s_register_operand" "+w")
1142 (vec_concat:ANY128
1143 (match_operand:<V_HALF> 1 "s_register_operand" "w")
1144 (vec_select:<V_HALF>
1145 (match_dup 0)
1146 (match_operand:ANY128 2 "vect_par_constant_low" ""))))]
1147 "TARGET_NEON"
1148 {
1149 int dest = REGNO (operands[0]);
1150 int src = REGNO (operands[1]);
1152 if (dest != src)
1153 return "vmov\t%f0, %P1";
1154 else
1155 return "";
1156 }
1157 [(set_attr "neon_type" "neon_bp_simple")]
1158 )
1160 (define_expand "move_hi_quad_<mode>"
1161 [(match_operand:ANY128 0 "s_register_operand" "")
1162 (match_operand:<V_HALF> 1 "s_register_operand" "")]
1163 "TARGET_NEON"
1164 {
1165 rtvec v = rtvec_alloc (<V_mode_nunits>/2);
1166 rtx t1;
1167 int i;
1169 for (i=0; i < (<V_mode_nunits>/2); i++)
1170 RTVEC_ELT (v, i) = GEN_INT (i);
1172 t1 = gen_rtx_PARALLEL (<MODE>mode, v);
1173 emit_insn (gen_neon_move_hi_quad_<mode> (operands[0], operands[1], t1));
1175 DONE;
1176 })
1178 (define_expand "move_lo_quad_<mode>"
1179 [(match_operand:ANY128 0 "s_register_operand" "")
1180 (match_operand:<V_HALF> 1 "s_register_operand" "")]
1181 "TARGET_NEON"
1182 {
1183 rtvec v = rtvec_alloc (<V_mode_nunits>/2);
1184 rtx t1;
1185 int i;
1187 for (i=0; i < (<V_mode_nunits>/2); i++)
1188 RTVEC_ELT (v, i) = GEN_INT ((<V_mode_nunits>/2) + i);
1190 t1 = gen_rtx_PARALLEL (<MODE>mode, v);
1191 emit_insn (gen_neon_move_lo_quad_<mode> (operands[0], operands[1], t1));
1193 DONE;
1194 })
1196 ;; Reduction operations
1198 (define_expand "reduc_splus_<mode>"
1199 [(match_operand:VD 0 "s_register_operand" "")
1200 (match_operand:VD 1 "s_register_operand" "")]
1201 "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
1202 {
1203 neon_pairwise_reduce (operands[0], operands[1], <MODE>mode,
1204 &gen_neon_vpadd_internal<mode>);
1205 DONE;
1206 })
1208 (define_expand "reduc_splus_<mode>"
1209 [(match_operand:VQ 0 "s_register_operand" "")
1210 (match_operand:VQ 1 "s_register_operand" "")]
1211 "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
1212 {
1213 rtx step1 = gen_reg_rtx (<V_HALF>mode);
1214 rtx res_d = gen_reg_rtx (<V_HALF>mode);
1216 emit_insn (gen_quad_halves_plus<mode> (step1, operands[1]));
1217 emit_insn (gen_reduc_splus_<V_half> (res_d, step1));
1218 emit_insn (gen_move_lo_quad_<mode> (operands[0], res_d));
1220 DONE;
1221 })
1223 (define_insn "reduc_splus_v2di"
1224 [(set (match_operand:V2DI 0 "s_register_operand" "=w")
1225 (unspec:V2DI [(match_operand:V2DI 1 "s_register_operand" "w")]
1226 UNSPEC_VPADD))]
1227 "TARGET_NEON"
1228 "vadd.i64\t%e0, %e1, %f1"
1229 [(set_attr "neon_type" "neon_int_1")]
1230 )
1232 ;; NEON does not distinguish between signed and unsigned addition except on
1233 ;; widening operations.
1234 (define_expand "reduc_uplus_<mode>"
1235 [(match_operand:VDQI 0 "s_register_operand" "")
1236 (match_operand:VDQI 1 "s_register_operand" "")]
1237 "TARGET_NEON"
1238 {
1239 emit_insn (gen_reduc_splus_<mode> (operands[0], operands[1]));
1240 DONE;
1241 })
1243 (define_expand "reduc_smin_<mode>"
1244 [(match_operand:VD 0 "s_register_operand" "")
1245 (match_operand:VD 1 "s_register_operand" "")]
1246 "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
1247 {
1248 neon_pairwise_reduce (operands[0], operands[1], <MODE>mode,
1249 &gen_neon_vpsmin<mode>);
1250 DONE;
1251 })
1253 (define_expand "reduc_smin_<mode>"
1254 [(match_operand:VQ 0 "s_register_operand" "")
1255 (match_operand:VQ 1 "s_register_operand" "")]
1256 "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
1257 {
1258 rtx step1 = gen_reg_rtx (<V_HALF>mode);
1259 rtx res_d = gen_reg_rtx (<V_HALF>mode);
1261 emit_insn (gen_quad_halves_smin<mode> (step1, operands[1]));
1262 emit_insn (gen_reduc_smin_<V_half> (res_d, step1));
1263 emit_insn (gen_move_lo_quad_<mode> (operands[0], res_d));
1265 DONE;
1266 })
1268 (define_expand "reduc_smax_<mode>"
1269 [(match_operand:VD 0 "s_register_operand" "")
1270 (match_operand:VD 1 "s_register_operand" "")]
1271 "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
1272 {
1273 neon_pairwise_reduce (operands[0], operands[1], <MODE>mode,
1274 &gen_neon_vpsmax<mode>);
1275 DONE;
1276 })
1278 (define_expand "reduc_smax_<mode>"
1279 [(match_operand:VQ 0 "s_register_operand" "")
1280 (match_operand:VQ 1 "s_register_operand" "")]
1281 "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
1282 {
1283 rtx step1 = gen_reg_rtx (<V_HALF>mode);
1284 rtx res_d = gen_reg_rtx (<V_HALF>mode);
1286 emit_insn (gen_quad_halves_smax<mode> (step1, operands[1]));
1287 emit_insn (gen_reduc_smax_<V_half> (res_d, step1));
1288 emit_insn (gen_move_lo_quad_<mode> (operands[0], res_d));
1290 DONE;
1291 })
1293 (define_expand "reduc_umin_<mode>"
1294 [(match_operand:VDI 0 "s_register_operand" "")
1295 (match_operand:VDI 1 "s_register_operand" "")]
1296 "TARGET_NEON"
1297 {
1298 neon_pairwise_reduce (operands[0], operands[1], <MODE>mode,
1299 &gen_neon_vpumin<mode>);
1300 DONE;
1301 })
1303 (define_expand "reduc_umin_<mode>"
1304 [(match_operand:VQI 0 "s_register_operand" "")
1305 (match_operand:VQI 1 "s_register_operand" "")]
1306 "TARGET_NEON"
1307 {
1308 rtx step1 = gen_reg_rtx (<V_HALF>mode);
1309 rtx res_d = gen_reg_rtx (<V_HALF>mode);
1311 emit_insn (gen_quad_halves_umin<mode> (step1, operands[1]));
1312 emit_insn (gen_reduc_umin_<V_half> (res_d, step1));
1313 emit_insn (gen_move_lo_quad_<mode> (operands[0], res_d));
1315 DONE;
1316 })
1318 (define_expand "reduc_umax_<mode>"
1319 [(match_operand:VDI 0 "s_register_operand" "")
1320 (match_operand:VDI 1 "s_register_operand" "")]
1321 "TARGET_NEON"
1322 {
1323 neon_pairwise_reduce (operands[0], operands[1], <MODE>mode,
1324 &gen_neon_vpumax<mode>);
1325 DONE;
1326 })
1328 (define_expand "reduc_umax_<mode>"
1329 [(match_operand:VQI 0 "s_register_operand" "")
1330 (match_operand:VQI 1 "s_register_operand" "")]
1331 "TARGET_NEON"
1332 {
1333 rtx step1 = gen_reg_rtx (<V_HALF>mode);
1334 rtx res_d = gen_reg_rtx (<V_HALF>mode);
1336 emit_insn (gen_quad_halves_umax<mode> (step1, operands[1]));
1337 emit_insn (gen_reduc_umax_<V_half> (res_d, step1));
1338 emit_insn (gen_move_lo_quad_<mode> (operands[0], res_d));
1340 DONE;
1341 })
1343 (define_insn "neon_vpadd_internal<mode>"
1344 [(set (match_operand:VD 0 "s_register_operand" "=w")
1345 (unspec:VD [(match_operand:VD 1 "s_register_operand" "w")
1346 (match_operand:VD 2 "s_register_operand" "w")]
1347 UNSPEC_VPADD))]
1348 "TARGET_NEON"
1349 "vpadd.<V_if_elem>\t%P0, %P1, %P2"
1350 ;; Assume this schedules like vadd.
1351 [(set (attr "neon_type")
1352 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
1353 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
1354 (const_string "neon_fp_vadd_ddd_vabs_dd")
1355 (const_string "neon_fp_vadd_qqq_vabs_qq"))
1356 (const_string "neon_int_1")))]
1357 )
1359 (define_insn "neon_vpsmin<mode>"
1360 [(set (match_operand:VD 0 "s_register_operand" "=w")
1361 (unspec:VD [(match_operand:VD 1 "s_register_operand" "w")
1362 (match_operand:VD 2 "s_register_operand" "w")]
1363 UNSPEC_VPSMIN))]
1364 "TARGET_NEON"
1365 "vpmin.<V_s_elem>\t%P0, %P1, %P2"
1366 ;; Assume this schedules like vmin.
1367 [(set (attr "neon_type")
1368 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
1369 (const_string "neon_fp_vadd_ddd_vabs_dd")
1370 (const_string "neon_int_5")))]
1371 )
1373 (define_insn "neon_vpsmax<mode>"
1374 [(set (match_operand:VD 0 "s_register_operand" "=w")
1375 (unspec:VD [(match_operand:VD 1 "s_register_operand" "w")
1376 (match_operand:VD 2 "s_register_operand" "w")]
1377 UNSPEC_VPSMAX))]
1378 "TARGET_NEON"
1379 "vpmax.<V_s_elem>\t%P0, %P1, %P2"
1380 ;; Assume this schedules like vmax.
1381 [(set (attr "neon_type")
1382 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
1383 (const_string "neon_fp_vadd_ddd_vabs_dd")
1384 (const_string "neon_int_5")))]
1385 )
1387 (define_insn "neon_vpumin<mode>"
1388 [(set (match_operand:VDI 0 "s_register_operand" "=w")
1389 (unspec:VDI [(match_operand:VDI 1 "s_register_operand" "w")
1390 (match_operand:VDI 2 "s_register_operand" "w")]
1391 UNSPEC_VPUMIN))]
1392 "TARGET_NEON"
1393 "vpmin.<V_u_elem>\t%P0, %P1, %P2"
1394 ;; Assume this schedules like umin.
1395 [(set_attr "neon_type" "neon_int_5")]
1396 )
1398 (define_insn "neon_vpumax<mode>"
1399 [(set (match_operand:VDI 0 "s_register_operand" "=w")
1400 (unspec:VDI [(match_operand:VDI 1 "s_register_operand" "w")
1401 (match_operand:VDI 2 "s_register_operand" "w")]
1402 UNSPEC_VPUMAX))]
1403 "TARGET_NEON"
1404 "vpmax.<V_u_elem>\t%P0, %P1, %P2"
1405 ;; Assume this schedules like umax.
1406 [(set_attr "neon_type" "neon_int_5")]
1407 )
1409 ;; Saturating arithmetic
1411 ; NOTE: Neon supports many more saturating variants of instructions than the
1412 ; following, but these are all GCC currently understands.
1413 ; FIXME: Actually, GCC doesn't know how to create saturating add/sub by itself
1414 ; yet either, although these patterns may be used by intrinsics when they're
1415 ; added.
1417 (define_insn "*ss_add<mode>_neon"
1418 [(set (match_operand:VD 0 "s_register_operand" "=w")
1419 (ss_plus:VD (match_operand:VD 1 "s_register_operand" "w")
1420 (match_operand:VD 2 "s_register_operand" "w")))]
1421 "TARGET_NEON"
1422 "vqadd.<V_s_elem>\t%P0, %P1, %P2"
1423 [(set_attr "neon_type" "neon_int_4")]
1424 )
1426 (define_insn "*us_add<mode>_neon"
1427 [(set (match_operand:VD 0 "s_register_operand" "=w")
1428 (us_plus:VD (match_operand:VD 1 "s_register_operand" "w")
1429 (match_operand:VD 2 "s_register_operand" "w")))]
1430 "TARGET_NEON"
1431 "vqadd.<V_u_elem>\t%P0, %P1, %P2"
1432 [(set_attr "neon_type" "neon_int_4")]
1433 )
1435 (define_insn "*ss_sub<mode>_neon"
1436 [(set (match_operand:VD 0 "s_register_operand" "=w")
1437 (ss_minus:VD (match_operand:VD 1 "s_register_operand" "w")
1438 (match_operand:VD 2 "s_register_operand" "w")))]
1439 "TARGET_NEON"
1440 "vqsub.<V_s_elem>\t%P0, %P1, %P2"
1441 [(set_attr "neon_type" "neon_int_5")]
1442 )
1444 (define_insn "*us_sub<mode>_neon"
1445 [(set (match_operand:VD 0 "s_register_operand" "=w")
1446 (us_minus:VD (match_operand:VD 1 "s_register_operand" "w")
1447 (match_operand:VD 2 "s_register_operand" "w")))]
1448 "TARGET_NEON"
1449 "vqsub.<V_u_elem>\t%P0, %P1, %P2"
1450 [(set_attr "neon_type" "neon_int_5")]
1451 )
1453 ;; Conditional instructions. These are comparisons with conditional moves for
1454 ;; vectors. They perform the assignment:
1455 ;;
1456 ;; Vop0 = (Vop4 <op3> Vop5) ? Vop1 : Vop2;
1457 ;;
1458 ;; where op3 is <, <=, ==, !=, >= or >. Operations are performed
1459 ;; element-wise.
1461 (define_expand "vcond<mode>"
1462 [(set (match_operand:VDQW 0 "s_register_operand" "")
1463 (if_then_else:VDQW
1464 (match_operator 3 "arm_comparison_operator"
1465 [(match_operand:VDQW 4 "s_register_operand" "")
1466 (match_operand:VDQW 5 "nonmemory_operand" "")])
1467 (match_operand:VDQW 1 "s_register_operand" "")
1468 (match_operand:VDQW 2 "s_register_operand" "")))]
1469 "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
1470 {
1471 rtx mask;
1472 int inverse = 0, immediate_zero = 0;
1473 /* See the description of "magic" bits in the 'T' case of
1474 arm_print_operand. */
1475 HOST_WIDE_INT magic_word = (<MODE>mode == V2SFmode || <MODE>mode == V4SFmode)
1476 ? 3 : 1;
1477 rtx magic_rtx = GEN_INT (magic_word);
1479 mask = gen_reg_rtx (<V_cmp_result>mode);
1481 if (operands[5] == CONST0_RTX (<MODE>mode))
1482 immediate_zero = 1;
1483 else if (!REG_P (operands[5]))
1484 operands[5] = force_reg (<MODE>mode, operands[5]);
1486 switch (GET_CODE (operands[3]))
1487 {
1488 case GE:
1489 emit_insn (gen_neon_vcge<mode> (mask, operands[4], operands[5],
1490 magic_rtx));
1491 break;
1493 case GT:
1494 emit_insn (gen_neon_vcgt<mode> (mask, operands[4], operands[5],
1495 magic_rtx));
1496 break;
1498 case EQ:
1499 emit_insn (gen_neon_vceq<mode> (mask, operands[4], operands[5],
1500 magic_rtx));
1501 break;
1503 case LE:
1504 if (immediate_zero)
1505 emit_insn (gen_neon_vcle<mode> (mask, operands[4], operands[5],
1506 magic_rtx));
1507 else
1508 emit_insn (gen_neon_vcge<mode> (mask, operands[5], operands[4],
1509 magic_rtx));
1510 break;
1512 case LT:
1513 if (immediate_zero)
1514 emit_insn (gen_neon_vclt<mode> (mask, operands[4], operands[5],
1515 magic_rtx));
1516 else
1517 emit_insn (gen_neon_vcgt<mode> (mask, operands[5], operands[4],
1518 magic_rtx));
1519 break;
1521 case NE:
1522 emit_insn (gen_neon_vceq<mode> (mask, operands[4], operands[5],
1523 magic_rtx));
1524 inverse = 1;
1525 break;
1527 default:
1528 gcc_unreachable ();
1529 }
1531 if (inverse)
1532 emit_insn (gen_neon_vbsl<mode> (operands[0], mask, operands[2],
1533 operands[1]));
1534 else
1535 emit_insn (gen_neon_vbsl<mode> (operands[0], mask, operands[1],
1536 operands[2]));
1538 DONE;
1539 })
1541 (define_expand "vcondu<mode>"
1542 [(set (match_operand:VDQIW 0 "s_register_operand" "")
1543 (if_then_else:VDQIW
1544 (match_operator 3 "arm_comparison_operator"
1545 [(match_operand:VDQIW 4 "s_register_operand" "")
1546 (match_operand:VDQIW 5 "s_register_operand" "")])
1547 (match_operand:VDQIW 1 "s_register_operand" "")
1548 (match_operand:VDQIW 2 "s_register_operand" "")))]
1549 "TARGET_NEON"
1550 {
1551 rtx mask;
1552 int inverse = 0, immediate_zero = 0;
1554 mask = gen_reg_rtx (<V_cmp_result>mode);
1556 if (operands[5] == CONST0_RTX (<MODE>mode))
1557 immediate_zero = 1;
1558 else if (!REG_P (operands[5]))
1559 operands[5] = force_reg (<MODE>mode, operands[5]);
1561 switch (GET_CODE (operands[3]))
1562 {
1563 case GEU:
1564 emit_insn (gen_neon_vcge<mode> (mask, operands[4], operands[5],
1565 const0_rtx));
1566 break;
1568 case GTU:
1569 emit_insn (gen_neon_vcgt<mode> (mask, operands[4], operands[5],
1570 const0_rtx));
1571 break;
1573 case EQ:
1574 emit_insn (gen_neon_vceq<mode> (mask, operands[4], operands[5],
1575 const0_rtx));
1576 break;
1578 case LEU:
1579 if (immediate_zero)
1580 emit_insn (gen_neon_vcle<mode> (mask, operands[4], operands[5],
1581 const0_rtx));
1582 else
1583 emit_insn (gen_neon_vcge<mode> (mask, operands[5], operands[4],
1584 const0_rtx));
1585 break;
1587 case LTU:
1588 if (immediate_zero)
1589 emit_insn (gen_neon_vclt<mode> (mask, operands[4], operands[5],
1590 const0_rtx));
1591 else
1592 emit_insn (gen_neon_vcgt<mode> (mask, operands[5], operands[4],
1593 const0_rtx));
1594 break;
1596 case NE:
1597 emit_insn (gen_neon_vceq<mode> (mask, operands[4], operands[5],
1598 const0_rtx));
1599 inverse = 1;
1600 break;
1602 default:
1603 gcc_unreachable ();
1604 }
1606 if (inverse)
1607 emit_insn (gen_neon_vbsl<mode> (operands[0], mask, operands[2],
1608 operands[1]));
1609 else
1610 emit_insn (gen_neon_vbsl<mode> (operands[0], mask, operands[1],
1611 operands[2]));
1613 DONE;
1614 })
1616 ;; Patterns for builtins.
1618 ; good for plain vadd, vaddq.
1620 (define_expand "neon_vadd<mode>"
1621 [(match_operand:VDQX 0 "s_register_operand" "=w")
1622 (match_operand:VDQX 1 "s_register_operand" "w")
1623 (match_operand:VDQX 2 "s_register_operand" "w")
1624 (match_operand:SI 3 "immediate_operand" "i")]
1625 "TARGET_NEON"
1626 {
1627 if (!<Is_float_mode> || flag_unsafe_math_optimizations)
1628 emit_insn (gen_add<mode>3 (operands[0], operands[1], operands[2]));
1629 else
1630 emit_insn (gen_neon_vadd<mode>_unspec (operands[0], operands[1],
1631 operands[2]));
1632 DONE;
1633 })
1635 ; Note that NEON operations don't support the full IEEE 754 standard: in
1636 ; particular, denormal values are flushed to zero. This means that GCC cannot
1637 ; use those instructions for autovectorization, etc. unless
1638 ; -funsafe-math-optimizations is in effect (in which case flush-to-zero
1639 ; behaviour is permissible). Intrinsic operations (provided by the arm_neon.h
1640 ; header) must work in either case: if -funsafe-math-optimizations is given,
1641 ; intrinsics expand to "canonical" RTL where possible, otherwise intrinsics
1642 ; expand to unspecs (which may potentially limit the extent to which they might
1643 ; be optimized by generic code).
1645 ; Used for intrinsics when flag_unsafe_math_optimizations is false.
1647 (define_insn "neon_vadd<mode>_unspec"
1648 [(set (match_operand:VDQX 0 "s_register_operand" "=w")
1649 (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand" "w")
1650 (match_operand:VDQX 2 "s_register_operand" "w")]
1651 UNSPEC_VADD))]
1652 "TARGET_NEON"
1653 "vadd.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1654 [(set (attr "neon_type")
1655 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
1656 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
1657 (const_string "neon_fp_vadd_ddd_vabs_dd")
1658 (const_string "neon_fp_vadd_qqq_vabs_qq"))
1659 (const_string "neon_int_1")))]
1660 )
1662 ; operand 3 represents in bits:
1663 ; bit 0: signed (vs unsigned).
1664 ; bit 1: rounding (vs none).
1666 (define_insn "neon_vaddl<mode>"
1667 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
1668 (unspec:<V_widen> [(match_operand:VDI 1 "s_register_operand" "w")
1669 (match_operand:VDI 2 "s_register_operand" "w")
1670 (match_operand:SI 3 "immediate_operand" "i")]
1671 UNSPEC_VADDL))]
1672 "TARGET_NEON"
1673 "vaddl.%T3%#<V_sz_elem>\t%q0, %P1, %P2"
1674 [(set_attr "neon_type" "neon_int_3")]
1675 )
1677 (define_insn "neon_vaddw<mode>"
1678 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
1679 (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "w")
1680 (match_operand:VDI 2 "s_register_operand" "w")
1681 (match_operand:SI 3 "immediate_operand" "i")]
1682 UNSPEC_VADDW))]
1683 "TARGET_NEON"
1684 "vaddw.%T3%#<V_sz_elem>\t%q0, %q1, %P2"
1685 [(set_attr "neon_type" "neon_int_2")]
1686 )
1688 ; vhadd and vrhadd.
1690 (define_insn "neon_vhadd<mode>"
1691 [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
1692 (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
1693 (match_operand:VDQIW 2 "s_register_operand" "w")
1694 (match_operand:SI 3 "immediate_operand" "i")]
1695 UNSPEC_VHADD))]
1696 "TARGET_NEON"
1697 "v%O3hadd.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1698 [(set_attr "neon_type" "neon_int_4")]
1699 )
1701 (define_insn "neon_vqadd<mode>"
1702 [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
1703 (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
1704 (match_operand:VDQIX 2 "s_register_operand" "w")
1705 (match_operand:SI 3 "immediate_operand" "i")]
1706 UNSPEC_VQADD))]
1707 "TARGET_NEON"
1708 "vqadd.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1709 [(set_attr "neon_type" "neon_int_4")]
1710 )
1712 (define_insn "neon_vaddhn<mode>"
1713 [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
1714 (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
1715 (match_operand:VN 2 "s_register_operand" "w")
1716 (match_operand:SI 3 "immediate_operand" "i")]
1717 UNSPEC_VADDHN))]
1718 "TARGET_NEON"
1719 "v%O3addhn.<V_if_elem>\t%P0, %q1, %q2"
1720 [(set_attr "neon_type" "neon_int_4")]
1721 )
1723 ;; We cannot replace this unspec with mul<mode>3 because of the odd
1724 ;; polynomial multiplication case that can specified by operand 3.
1725 (define_insn "neon_vmul<mode>"
1726 [(set (match_operand:VDQW 0 "s_register_operand" "=w")
1727 (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "w")
1728 (match_operand:VDQW 2 "s_register_operand" "w")
1729 (match_operand:SI 3 "immediate_operand" "i")]
1730 UNSPEC_VMUL))]
1731 "TARGET_NEON"
1732 "vmul.%F3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1733 [(set (attr "neon_type")
1734 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
1735 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
1736 (const_string "neon_fp_vadd_ddd_vabs_dd")
1737 (const_string "neon_fp_vadd_qqq_vabs_qq"))
1738 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
1739 (if_then_else
1740 (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0))
1741 (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long")
1742 (const_string "neon_mul_qqq_8_16_32_ddd_32"))
1743 (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0))
1744 (const_string "neon_mul_qqq_8_16_32_ddd_32")
1745 (const_string "neon_mul_qqq_8_16_32_ddd_32")))))]
1746 )
1748 (define_expand "neon_vmla<mode>"
1749 [(match_operand:VDQW 0 "s_register_operand" "=w")
1750 (match_operand:VDQW 1 "s_register_operand" "0")
1751 (match_operand:VDQW 2 "s_register_operand" "w")
1752 (match_operand:VDQW 3 "s_register_operand" "w")
1753 (match_operand:SI 4 "immediate_operand" "i")]
1754 "TARGET_NEON"
1755 {
1756 if (!<Is_float_mode> || flag_unsafe_math_optimizations)
1757 emit_insn (gen_mul<mode>3add<mode>_neon (operands[0], operands[1],
1758 operands[2], operands[3]));
1759 else
1760 emit_insn (gen_neon_vmla<mode>_unspec (operands[0], operands[1],
1761 operands[2], operands[3]));
1762 DONE;
1763 })
1765 ; Used for intrinsics when flag_unsafe_math_optimizations is false.
1767 (define_insn "neon_vmla<mode>_unspec"
1768 [(set (match_operand:VDQ 0 "s_register_operand" "=w")
1769 (unspec:VDQ [(match_operand:VDQ 1 "s_register_operand" "0")
1770 (match_operand:VDQ 2 "s_register_operand" "w")
1771 (match_operand:VDQ 3 "s_register_operand" "w")]
1772 UNSPEC_VMLA))]
1773 "TARGET_NEON"
1774 "vmla.<V_if_elem>\t%<V_reg>0, %<V_reg>2, %<V_reg>3"
1775 [(set (attr "neon_type")
1776 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
1777 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
1778 (const_string "neon_fp_vmla_ddd")
1779 (const_string "neon_fp_vmla_qqq"))
1780 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
1781 (if_then_else
1782 (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0))
1783 (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
1784 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long"))
1785 (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0))
1786 (const_string "neon_mla_qqq_8_16")
1787 (const_string "neon_mla_qqq_32_qqd_32_scalar")))))]
1788 )
1790 (define_insn "neon_vmlal<mode>"
1791 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
1792 (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
1793 (match_operand:VW 2 "s_register_operand" "w")
1794 (match_operand:VW 3 "s_register_operand" "w")
1795 (match_operand:SI 4 "immediate_operand" "i")]
1796 UNSPEC_VMLAL))]
1797 "TARGET_NEON"
1798 "vmlal.%T4%#<V_sz_elem>\t%q0, %P2, %P3"
1799 [(set (attr "neon_type")
1800 (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0))
1801 (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
1802 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
1803 )
1805 (define_expand "neon_vmls<mode>"
1806 [(match_operand:VDQW 0 "s_register_operand" "=w")
1807 (match_operand:VDQW 1 "s_register_operand" "0")
1808 (match_operand:VDQW 2 "s_register_operand" "w")
1809 (match_operand:VDQW 3 "s_register_operand" "w")
1810 (match_operand:SI 4 "immediate_operand" "i")]
1811 "TARGET_NEON"
1812 {
1813 if (!<Is_float_mode> || flag_unsafe_math_optimizations)
1814 emit_insn (gen_mul<mode>3neg<mode>add<mode>_neon (operands[0],
1815 operands[1], operands[2], operands[3]));
1816 else
1817 emit_insn (gen_neon_vmls<mode>_unspec (operands[0], operands[1],
1818 operands[2], operands[3]));
1819 DONE;
1820 })
1822 ; Used for intrinsics when flag_unsafe_math_optimizations is false.
1824 (define_insn "neon_vmls<mode>_unspec"
1825 [(set (match_operand:VDQ 0 "s_register_operand" "=w")
1826 (unspec:VDQ [(match_operand:VDQ 1 "s_register_operand" "0")
1827 (match_operand:VDQ 2 "s_register_operand" "w")
1828 (match_operand:VDQ 3 "s_register_operand" "w")]
1829 UNSPEC_VMLS))]
1830 "TARGET_NEON"
1831 "vmls.<V_if_elem>\t%<V_reg>0, %<V_reg>2, %<V_reg>3"
1832 [(set (attr "neon_type")
1833 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
1834 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
1835 (const_string "neon_fp_vmla_ddd")
1836 (const_string "neon_fp_vmla_qqq"))
1837 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
1838 (if_then_else
1839 (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0))
1840 (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
1841 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long"))
1842 (if_then_else
1843 (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0))
1844 (const_string "neon_mla_qqq_8_16")
1845 (const_string "neon_mla_qqq_32_qqd_32_scalar")))))]
1846 )
1848 (define_insn "neon_vmlsl<mode>"
1849 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
1850 (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
1851 (match_operand:VW 2 "s_register_operand" "w")
1852 (match_operand:VW 3 "s_register_operand" "w")
1853 (match_operand:SI 4 "immediate_operand" "i")]
1854 UNSPEC_VMLSL))]
1855 "TARGET_NEON"
1856 "vmlsl.%T4%#<V_sz_elem>\t%q0, %P2, %P3"
1857 [(set (attr "neon_type")
1858 (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0))
1859 (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
1860 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
1861 )
1863 (define_insn "neon_vqdmulh<mode>"
1864 [(set (match_operand:VMDQI 0 "s_register_operand" "=w")
1865 (unspec:VMDQI [(match_operand:VMDQI 1 "s_register_operand" "w")
1866 (match_operand:VMDQI 2 "s_register_operand" "w")
1867 (match_operand:SI 3 "immediate_operand" "i")]
1868 UNSPEC_VQDMULH))]
1869 "TARGET_NEON"
1870 "vq%O3dmulh.<V_s_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1871 [(set (attr "neon_type")
1872 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
1873 (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0))
1874 (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long")
1875 (const_string "neon_mul_qqq_8_16_32_ddd_32"))
1876 (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0))
1877 (const_string "neon_mul_qqq_8_16_32_ddd_32")
1878 (const_string "neon_mul_qqq_8_16_32_ddd_32"))))]
1879 )
1881 (define_insn "neon_vqdmlal<mode>"
1882 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
1883 (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
1884 (match_operand:VMDI 2 "s_register_operand" "w")
1885 (match_operand:VMDI 3 "s_register_operand" "w")
1886 (match_operand:SI 4 "immediate_operand" "i")]
1887 UNSPEC_VQDMLAL))]
1888 "TARGET_NEON"
1889 "vqdmlal.<V_s_elem>\t%q0, %P2, %P3"
1890 [(set (attr "neon_type")
1891 (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0))
1892 (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
1893 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
1894 )
1896 (define_insn "neon_vqdmlsl<mode>"
1897 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
1898 (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
1899 (match_operand:VMDI 2 "s_register_operand" "w")
1900 (match_operand:VMDI 3 "s_register_operand" "w")
1901 (match_operand:SI 4 "immediate_operand" "i")]
1902 UNSPEC_VQDMLSL))]
1903 "TARGET_NEON"
1904 "vqdmlsl.<V_s_elem>\t%q0, %P2, %P3"
1905 [(set (attr "neon_type")
1906 (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0))
1907 (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
1908 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
1909 )
1911 (define_insn "neon_vmull<mode>"
1912 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
1913 (unspec:<V_widen> [(match_operand:VW 1 "s_register_operand" "w")
1914 (match_operand:VW 2 "s_register_operand" "w")
1915 (match_operand:SI 3 "immediate_operand" "i")]
1916 UNSPEC_VMULL))]
1917 "TARGET_NEON"
1918 "vmull.%T3%#<V_sz_elem>\t%q0, %P1, %P2"
1919 [(set (attr "neon_type")
1920 (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0))
1921 (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long")
1922 (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")))]
1923 )
1925 (define_insn "neon_vqdmull<mode>"
1926 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
1927 (unspec:<V_widen> [(match_operand:VMDI 1 "s_register_operand" "w")
1928 (match_operand:VMDI 2 "s_register_operand" "w")
1929 (match_operand:SI 3 "immediate_operand" "i")]
1930 UNSPEC_VQDMULL))]
1931 "TARGET_NEON"
1932 "vqdmull.<V_s_elem>\t%q0, %P1, %P2"
1933 [(set (attr "neon_type")
1934 (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0))
1935 (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long")
1936 (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")))]
1937 )
1939 (define_expand "neon_vsub<mode>"
1940 [(match_operand:VDQX 0 "s_register_operand" "=w")
1941 (match_operand:VDQX 1 "s_register_operand" "w")
1942 (match_operand:VDQX 2 "s_register_operand" "w")
1943 (match_operand:SI 3 "immediate_operand" "i")]
1944 "TARGET_NEON"
1945 {
1946 if (!<Is_float_mode> || flag_unsafe_math_optimizations)
1947 emit_insn (gen_sub<mode>3 (operands[0], operands[1], operands[2]));
1948 else
1949 emit_insn (gen_neon_vsub<mode>_unspec (operands[0], operands[1],
1950 operands[2]));
1951 DONE;
1952 })
1954 ; Used for intrinsics when flag_unsafe_math_optimizations is false.
1956 (define_insn "neon_vsub<mode>_unspec"
1957 [(set (match_operand:VDQX 0 "s_register_operand" "=w")
1958 (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand" "w")
1959 (match_operand:VDQX 2 "s_register_operand" "w")]
1960 UNSPEC_VSUB))]
1961 "TARGET_NEON"
1962 "vsub.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1963 [(set (attr "neon_type")
1964 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
1965 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
1966 (const_string "neon_fp_vadd_ddd_vabs_dd")
1967 (const_string "neon_fp_vadd_qqq_vabs_qq"))
1968 (const_string "neon_int_2")))]
1969 )
1971 (define_insn "neon_vsubl<mode>"
1972 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
1973 (unspec:<V_widen> [(match_operand:VDI 1 "s_register_operand" "w")
1974 (match_operand:VDI 2 "s_register_operand" "w")
1975 (match_operand:SI 3 "immediate_operand" "i")]
1976 UNSPEC_VSUBL))]
1977 "TARGET_NEON"
1978 "vsubl.%T3%#<V_sz_elem>\t%q0, %P1, %P2"
1979 [(set_attr "neon_type" "neon_int_2")]
1980 )
1982 (define_insn "neon_vsubw<mode>"
1983 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
1984 (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "w")
1985 (match_operand:VDI 2 "s_register_operand" "w")
1986 (match_operand:SI 3 "immediate_operand" "i")]
1987 UNSPEC_VSUBW))]
1988 "TARGET_NEON"
1989 "vsubw.%T3%#<V_sz_elem>\t%q0, %q1, %P2"
1990 [(set_attr "neon_type" "neon_int_2")]
1991 )
1993 (define_insn "neon_vqsub<mode>"
1994 [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
1995 (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
1996 (match_operand:VDQIX 2 "s_register_operand" "w")
1997 (match_operand:SI 3 "immediate_operand" "i")]
1998 UNSPEC_VQSUB))]
1999 "TARGET_NEON"
2000 "vqsub.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2001 [(set_attr "neon_type" "neon_int_5")]
2002 )
2004 (define_insn "neon_vhsub<mode>"
2005 [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2006 (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
2007 (match_operand:VDQIW 2 "s_register_operand" "w")
2008 (match_operand:SI 3 "immediate_operand" "i")]
2009 UNSPEC_VHSUB))]
2010 "TARGET_NEON"
2011 "vhsub.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2012 [(set_attr "neon_type" "neon_int_5")]
2013 )
2015 (define_insn "neon_vsubhn<mode>"
2016 [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
2017 (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
2018 (match_operand:VN 2 "s_register_operand" "w")
2019 (match_operand:SI 3 "immediate_operand" "i")]
2020 UNSPEC_VSUBHN))]
2021 "TARGET_NEON"
2022 "v%O3subhn.<V_if_elem>\t%P0, %q1, %q2"
2023 [(set_attr "neon_type" "neon_int_4")]
2024 )
2026 (define_insn "neon_vceq<mode>"
2027 [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w,w")
2028 (unspec:<V_cmp_result>
2029 [(match_operand:VDQW 1 "s_register_operand" "w,w")
2030 (match_operand:VDQW 2 "nonmemory_operand" "w,Dz")
2031 (match_operand:SI 3 "immediate_operand" "i,i")]
2032 UNSPEC_VCEQ))]
2033 "TARGET_NEON"
2034 "@
2035 vceq.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2
2036 vceq.<V_if_elem>\t%<V_reg>0, %<V_reg>1, #0"
2037 [(set (attr "neon_type")
2038 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
2039 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
2040 (const_string "neon_fp_vadd_ddd_vabs_dd")
2041 (const_string "neon_fp_vadd_qqq_vabs_qq"))
2042 (const_string "neon_int_5")))]
2043 )
2045 (define_insn "neon_vcge<mode>"
2046 [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w,w")
2047 (unspec:<V_cmp_result>
2048 [(match_operand:VDQW 1 "s_register_operand" "w,w")
2049 (match_operand:VDQW 2 "nonmemory_operand" "w,Dz")
2050 (match_operand:SI 3 "immediate_operand" "i,i")]
2051 UNSPEC_VCGE))]
2052 "TARGET_NEON"
2053 "@
2054 vcge.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2
2055 vcge.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, #0"
2056 [(set (attr "neon_type")
2057 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
2058 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
2059 (const_string "neon_fp_vadd_ddd_vabs_dd")
2060 (const_string "neon_fp_vadd_qqq_vabs_qq"))
2061 (const_string "neon_int_5")))]
2062 )
2064 (define_insn "neon_vcgt<mode>"
2065 [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w,w")
2066 (unspec:<V_cmp_result>
2067 [(match_operand:VDQW 1 "s_register_operand" "w,w")
2068 (match_operand:VDQW 2 "nonmemory_operand" "w,Dz")
2069 (match_operand:SI 3 "immediate_operand" "i,i")]
2070 UNSPEC_VCGT))]
2071 "TARGET_NEON"
2072 "@
2073 vcgt.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2
2074 vcgt.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, #0"
2075 [(set (attr "neon_type")
2076 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
2077 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
2078 (const_string "neon_fp_vadd_ddd_vabs_dd")
2079 (const_string "neon_fp_vadd_qqq_vabs_qq"))
2080 (const_string "neon_int_5")))]
2081 )
2083 ;; VCLE and VCLT only support comparisons with immediate zero (register
2084 ;; variants are VCGE and VCGT with operands reversed).
2086 (define_insn "neon_vcle<mode>"
2087 [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w")
2088 (unspec:<V_cmp_result>
2089 [(match_operand:VDQW 1 "s_register_operand" "w")
2090 (match_operand:VDQW 2 "nonmemory_operand" "Dz")
2091 (match_operand:SI 3 "immediate_operand" "i")]
2092 UNSPEC_VCLE))]
2093 "TARGET_NEON"
2094 "vcle.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, #0"
2095 [(set (attr "neon_type")
2096 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
2097 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
2098 (const_string "neon_fp_vadd_ddd_vabs_dd")
2099 (const_string "neon_fp_vadd_qqq_vabs_qq"))
2100 (const_string "neon_int_5")))]
2101 )
2103 (define_insn "neon_vclt<mode>"
2104 [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w")
2105 (unspec:<V_cmp_result>
2106 [(match_operand:VDQW 1 "s_register_operand" "w")
2107 (match_operand:VDQW 2 "nonmemory_operand" "Dz")
2108 (match_operand:SI 3 "immediate_operand" "i")]
2109 UNSPEC_VCLT))]
2110 "TARGET_NEON"
2111 "vclt.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, #0"
2112 [(set (attr "neon_type")
2113 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
2114 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
2115 (const_string "neon_fp_vadd_ddd_vabs_dd")
2116 (const_string "neon_fp_vadd_qqq_vabs_qq"))
2117 (const_string "neon_int_5")))]
2118 )
2120 (define_insn "neon_vcage<mode>"
2121 [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w")
2122 (unspec:<V_cmp_result> [(match_operand:VCVTF 1 "s_register_operand" "w")
2123 (match_operand:VCVTF 2 "s_register_operand" "w")
2124 (match_operand:SI 3 "immediate_operand" "i")]
2125 UNSPEC_VCAGE))]
2126 "TARGET_NEON"
2127 "vacge.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2128 [(set (attr "neon_type")
2129 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
2130 (const_string "neon_fp_vadd_ddd_vabs_dd")
2131 (const_string "neon_fp_vadd_qqq_vabs_qq")))]
2132 )
2134 (define_insn "neon_vcagt<mode>"
2135 [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w")
2136 (unspec:<V_cmp_result> [(match_operand:VCVTF 1 "s_register_operand" "w")
2137 (match_operand:VCVTF 2 "s_register_operand" "w")
2138 (match_operand:SI 3 "immediate_operand" "i")]
2139 UNSPEC_VCAGT))]
2140 "TARGET_NEON"
2141 "vacgt.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2142 [(set (attr "neon_type")
2143 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
2144 (const_string "neon_fp_vadd_ddd_vabs_dd")
2145 (const_string "neon_fp_vadd_qqq_vabs_qq")))]
2146 )
2148 (define_insn "neon_vtst<mode>"
2149 [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2150 (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
2151 (match_operand:VDQIW 2 "s_register_operand" "w")
2152 (match_operand:SI 3 "immediate_operand" "i")]
2153 UNSPEC_VTST))]
2154 "TARGET_NEON"
2155 "vtst.<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2156 [(set_attr "neon_type" "neon_int_4")]
2157 )
2159 (define_insn "neon_vabd<mode>"
2160 [(set (match_operand:VDQW 0 "s_register_operand" "=w")
2161 (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "w")
2162 (match_operand:VDQW 2 "s_register_operand" "w")
2163 (match_operand:SI 3 "immediate_operand" "i")]
2164 UNSPEC_VABD))]
2165 "TARGET_NEON"
2166 "vabd.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2167 [(set (attr "neon_type")
2168 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
2169 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
2170 (const_string "neon_fp_vadd_ddd_vabs_dd")
2171 (const_string "neon_fp_vadd_qqq_vabs_qq"))
2172 (const_string "neon_int_5")))]
2173 )
2175 (define_insn "neon_vabdl<mode>"
2176 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2177 (unspec:<V_widen> [(match_operand:VW 1 "s_register_operand" "w")
2178 (match_operand:VW 2 "s_register_operand" "w")
2179 (match_operand:SI 3 "immediate_operand" "i")]
2180 UNSPEC_VABDL))]
2181 "TARGET_NEON"
2182 "vabdl.%T3%#<V_sz_elem>\t%q0, %P1, %P2"
2183 [(set_attr "neon_type" "neon_int_5")]
2184 )
2186 (define_insn "neon_vaba<mode>"
2187 [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2188 (plus:VDQIW (match_operand:VDQIW 1 "s_register_operand" "0")
2189 (unspec:VDQIW [(match_operand:VDQIW 2 "s_register_operand" "w")
2190 (match_operand:VDQIW 3 "s_register_operand" "w")
2191 (match_operand:SI 4 "immediate_operand" "i")]
2192 UNSPEC_VABD)))]
2193 "TARGET_NEON"
2194 "vaba.%T4%#<V_sz_elem>\t%<V_reg>0, %<V_reg>2, %<V_reg>3"
2195 [(set (attr "neon_type")
2196 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
2197 (const_string "neon_vaba") (const_string "neon_vaba_qqq")))]
2198 )
2200 (define_insn "neon_vabal<mode>"
2201 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2202 (plus:<V_widen> (match_operand:<V_widen> 1 "s_register_operand" "0")
2203 (unspec:<V_widen> [(match_operand:VW 2 "s_register_operand" "w")
2204 (match_operand:VW 3 "s_register_operand" "w")
2205 (match_operand:SI 4 "immediate_operand" "i")]
2206 UNSPEC_VABDL)))]
2207 "TARGET_NEON"
2208 "vabal.%T4%#<V_sz_elem>\t%q0, %P2, %P3"
2209 [(set_attr "neon_type" "neon_vaba")]
2210 )
2212 (define_insn "neon_vmax<mode>"
2213 [(set (match_operand:VDQW 0 "s_register_operand" "=w")
2214 (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "w")
2215 (match_operand:VDQW 2 "s_register_operand" "w")
2216 (match_operand:SI 3 "immediate_operand" "i")]
2217 UNSPEC_VMAX))]
2218 "TARGET_NEON"
2219 "vmax.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2220 [(set (attr "neon_type")
2221 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
2222 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
2223 (const_string "neon_fp_vadd_ddd_vabs_dd")
2224 (const_string "neon_fp_vadd_qqq_vabs_qq"))
2225 (const_string "neon_int_5")))]
2226 )
2228 (define_insn "neon_vmin<mode>"
2229 [(set (match_operand:VDQW 0 "s_register_operand" "=w")
2230 (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "w")
2231 (match_operand:VDQW 2 "s_register_operand" "w")
2232 (match_operand:SI 3 "immediate_operand" "i")]
2233 UNSPEC_VMIN))]
2234 "TARGET_NEON"
2235 "vmin.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2236 [(set (attr "neon_type")
2237 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
2238 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
2239 (const_string "neon_fp_vadd_ddd_vabs_dd")
2240 (const_string "neon_fp_vadd_qqq_vabs_qq"))
2241 (const_string "neon_int_5")))]
2242 )
2244 (define_expand "neon_vpadd<mode>"
2245 [(match_operand:VD 0 "s_register_operand" "=w")
2246 (match_operand:VD 1 "s_register_operand" "w")
2247 (match_operand:VD 2 "s_register_operand" "w")
2248 (match_operand:SI 3 "immediate_operand" "i")]
2249 "TARGET_NEON"
2250 {
2251 emit_insn (gen_neon_vpadd_internal<mode> (operands[0], operands[1],
2252 operands[2]));
2253 DONE;
2254 })
2256 (define_insn "neon_vpaddl<mode>"
2257 [(set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
2258 (unspec:<V_double_width> [(match_operand:VDQIW 1 "s_register_operand" "w")
2259 (match_operand:SI 2 "immediate_operand" "i")]
2260 UNSPEC_VPADDL))]
2261 "TARGET_NEON"
2262 "vpaddl.%T2%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1"
2263 ;; Assume this schedules like vaddl.
2264 [(set_attr "neon_type" "neon_int_3")]
2265 )
2267 (define_insn "neon_vpadal<mode>"
2268 [(set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
2269 (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
2270 (match_operand:VDQIW 2 "s_register_operand" "w")
2271 (match_operand:SI 3 "immediate_operand" "i")]
2272 UNSPEC_VPADAL))]
2273 "TARGET_NEON"
2274 "vpadal.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>2"
2275 ;; Assume this schedules like vpadd.
2276 [(set_attr "neon_type" "neon_int_1")]
2277 )
2279 (define_insn "neon_vpmax<mode>"
2280 [(set (match_operand:VD 0 "s_register_operand" "=w")
2281 (unspec:VD [(match_operand:VD 1 "s_register_operand" "w")
2282 (match_operand:VD 2 "s_register_operand" "w")
2283 (match_operand:SI 3 "immediate_operand" "i")]
2284 UNSPEC_VPMAX))]
2285 "TARGET_NEON"
2286 "vpmax.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2287 ;; Assume this schedules like vmax.
2288 [(set (attr "neon_type")
2289 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
2290 (const_string "neon_fp_vadd_ddd_vabs_dd")
2291 (const_string "neon_int_5")))]
2292 )
2294 (define_insn "neon_vpmin<mode>"
2295 [(set (match_operand:VD 0 "s_register_operand" "=w")
2296 (unspec:VD [(match_operand:VD 1 "s_register_operand" "w")
2297 (match_operand:VD 2 "s_register_operand" "w")
2298 (match_operand:SI 3 "immediate_operand" "i")]
2299 UNSPEC_VPMIN))]
2300 "TARGET_NEON"
2301 "vpmin.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2302 ;; Assume this schedules like vmin.
2303 [(set (attr "neon_type")
2304 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
2305 (const_string "neon_fp_vadd_ddd_vabs_dd")
2306 (const_string "neon_int_5")))]
2307 )
2309 (define_insn "neon_vrecps<mode>"
2310 [(set (match_operand:VCVTF 0 "s_register_operand" "=w")
2311 (unspec:VCVTF [(match_operand:VCVTF 1 "s_register_operand" "w")
2312 (match_operand:VCVTF 2 "s_register_operand" "w")
2313 (match_operand:SI 3 "immediate_operand" "i")]
2314 UNSPEC_VRECPS))]
2315 "TARGET_NEON"
2316 "vrecps.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2317 [(set (attr "neon_type")
2318 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
2319 (const_string "neon_fp_vrecps_vrsqrts_ddd")
2320 (const_string "neon_fp_vrecps_vrsqrts_qqq")))]
2321 )
2323 (define_insn "neon_vrsqrts<mode>"
2324 [(set (match_operand:VCVTF 0 "s_register_operand" "=w")
2325 (unspec:VCVTF [(match_operand:VCVTF 1 "s_register_operand" "w")
2326 (match_operand:VCVTF 2 "s_register_operand" "w")
2327 (match_operand:SI 3 "immediate_operand" "i")]
2328 UNSPEC_VRSQRTS))]
2329 "TARGET_NEON"
2330 "vrsqrts.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2331 [(set (attr "neon_type")
2332 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
2333 (const_string "neon_fp_vrecps_vrsqrts_ddd")
2334 (const_string "neon_fp_vrecps_vrsqrts_qqq")))]
2335 )
2337 (define_expand "neon_vabs<mode>"
2338 [(match_operand:VDQW 0 "s_register_operand" "")
2339 (match_operand:VDQW 1 "s_register_operand" "")
2340 (match_operand:SI 2 "immediate_operand" "")]
2341 "TARGET_NEON"
2342 {
2343 emit_insn (gen_abs<mode>2 (operands[0], operands[1]));
2344 DONE;
2345 })
2347 (define_insn "neon_vqabs<mode>"
2348 [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2349 (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
2350 (match_operand:SI 2 "immediate_operand" "i")]
2351 UNSPEC_VQABS))]
2352 "TARGET_NEON"
2353 "vqabs.<V_s_elem>\t%<V_reg>0, %<V_reg>1"
2354 [(set_attr "neon_type" "neon_vqneg_vqabs")]
2355 )
2357 (define_expand "neon_vneg<mode>"
2358 [(match_operand:VDQW 0 "s_register_operand" "")
2359 (match_operand:VDQW 1 "s_register_operand" "")
2360 (match_operand:SI 2 "immediate_operand" "")]
2361 "TARGET_NEON"
2362 {
2363 emit_insn (gen_neg<mode>2 (operands[0], operands[1]));
2364 DONE;
2365 })
2367 (define_insn "neon_vqneg<mode>"
2368 [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2369 (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
2370 (match_operand:SI 2 "immediate_operand" "i")]
2371 UNSPEC_VQNEG))]
2372 "TARGET_NEON"
2373 "vqneg.<V_s_elem>\t%<V_reg>0, %<V_reg>1"
2374 [(set_attr "neon_type" "neon_vqneg_vqabs")]
2375 )
2377 (define_insn "neon_vcls<mode>"
2378 [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2379 (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
2380 (match_operand:SI 2 "immediate_operand" "i")]
2381 UNSPEC_VCLS))]
2382 "TARGET_NEON"
2383 "vcls.<V_s_elem>\t%<V_reg>0, %<V_reg>1"
2384 [(set_attr "neon_type" "neon_int_1")]
2385 )
2387 (define_insn "clz<mode>2"
2388 [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2389 (clz:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w")))]
2390 "TARGET_NEON"
2391 "vclz.<V_if_elem>\t%<V_reg>0, %<V_reg>1"
2392 [(set_attr "neon_type" "neon_int_1")]
2393 )
2395 (define_expand "neon_vclz<mode>"
2396 [(match_operand:VDQIW 0 "s_register_operand" "")
2397 (match_operand:VDQIW 1 "s_register_operand" "")
2398 (match_operand:SI 2 "immediate_operand" "")]
2399 "TARGET_NEON"
2400 {
2401 emit_insn (gen_clz<mode>2 (operands[0], operands[1]));
2402 DONE;
2403 })
2405 (define_insn "popcount<mode>2"
2406 [(set (match_operand:VE 0 "s_register_operand" "=w")
2407 (popcount:VE (match_operand:VE 1 "s_register_operand" "w")))]
2408 "TARGET_NEON"
2409 "vcnt.<V_sz_elem>\t%<V_reg>0, %<V_reg>1"
2410 [(set_attr "neon_type" "neon_int_1")]
2411 )
2413 (define_expand "neon_vcnt<mode>"
2414 [(match_operand:VE 0 "s_register_operand" "=w")
2415 (match_operand:VE 1 "s_register_operand" "w")
2416 (match_operand:SI 2 "immediate_operand" "i")]
2417 "TARGET_NEON"
2418 {
2419 emit_insn (gen_popcount<mode>2 (operands[0], operands[1]));
2420 DONE;
2421 })
2423 (define_insn "neon_vrecpe<mode>"
2424 [(set (match_operand:V32 0 "s_register_operand" "=w")
2425 (unspec:V32 [(match_operand:V32 1 "s_register_operand" "w")
2426 (match_operand:SI 2 "immediate_operand" "i")]
2427 UNSPEC_VRECPE))]
2428 "TARGET_NEON"
2429 "vrecpe.<V_u_elem>\t%<V_reg>0, %<V_reg>1"
2430 [(set (attr "neon_type")
2431 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
2432 (const_string "neon_fp_vadd_ddd_vabs_dd")
2433 (const_string "neon_fp_vadd_qqq_vabs_qq")))]
2434 )
2436 (define_insn "neon_vrsqrte<mode>"
2437 [(set (match_operand:V32 0 "s_register_operand" "=w")
2438 (unspec:V32 [(match_operand:V32 1 "s_register_operand" "w")
2439 (match_operand:SI 2 "immediate_operand" "i")]
2440 UNSPEC_VRSQRTE))]
2441 "TARGET_NEON"
2442 "vrsqrte.<V_u_elem>\t%<V_reg>0, %<V_reg>1"
2443 [(set (attr "neon_type")
2444 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
2445 (const_string "neon_fp_vadd_ddd_vabs_dd")
2446 (const_string "neon_fp_vadd_qqq_vabs_qq")))]
2447 )
2449 (define_expand "neon_vmvn<mode>"
2450 [(match_operand:VDQIW 0 "s_register_operand" "")
2451 (match_operand:VDQIW 1 "s_register_operand" "")
2452 (match_operand:SI 2 "immediate_operand" "")]
2453 "TARGET_NEON"
2454 {
2455 emit_insn (gen_one_cmpl<mode>2 (operands[0], operands[1]));
2456 DONE;
2457 })
2459 (define_insn "neon_vget_lane<mode>_sext_internal"
2460 [(set (match_operand:SI 0 "s_register_operand" "=r")
2461 (sign_extend:SI
2462 (vec_select:<V_elem>
2463 (match_operand:VD 1 "s_register_operand" "w")
2464 (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2465 "TARGET_NEON"
2466 {
2467 if (BYTES_BIG_ENDIAN)
2468 {
2469 int elt = INTVAL (operands[2]);
2470 elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
2471 operands[2] = GEN_INT (elt);
2472 }
2473 return "vmov%?.s<V_sz_elem>\t%0, %P1[%c2]";
2474 }
2475 [(set_attr "predicable" "yes")
2476 (set_attr "neon_type" "neon_bp_simple")]
2477 )
2479 (define_insn "neon_vget_lane<mode>_zext_internal"
2480 [(set (match_operand:SI 0 "s_register_operand" "=r")
2481 (zero_extend:SI
2482 (vec_select:<V_elem>
2483 (match_operand:VD 1 "s_register_operand" "w")
2484 (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2485 "TARGET_NEON"
2486 {
2487 if (BYTES_BIG_ENDIAN)
2488 {
2489 int elt = INTVAL (operands[2]);
2490 elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
2491 operands[2] = GEN_INT (elt);
2492 }
2493 return "vmov%?.u<V_sz_elem>\t%0, %P1[%c2]";
2494 }
2495 [(set_attr "predicable" "yes")
2496 (set_attr "neon_type" "neon_bp_simple")]
2497 )
2499 (define_insn "neon_vget_lane<mode>_sext_internal"
2500 [(set (match_operand:SI 0 "s_register_operand" "=r")
2501 (sign_extend:SI
2502 (vec_select:<V_elem>
2503 (match_operand:VQ 1 "s_register_operand" "w")
2504 (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2505 "TARGET_NEON"
2506 {
2507 rtx ops[3];
2508 int regno = REGNO (operands[1]);
2509 unsigned int halfelts = GET_MODE_NUNITS (<MODE>mode) / 2;
2510 unsigned int elt = INTVAL (operands[2]);
2511 unsigned int elt_adj = elt % halfelts;
2513 if (BYTES_BIG_ENDIAN)
2514 elt_adj = halfelts - 1 - elt_adj;
2516 ops[0] = operands[0];
2517 ops[1] = gen_rtx_REG (<V_HALF>mode, regno + 2 * (elt / halfelts));
2518 ops[2] = GEN_INT (elt_adj);
2519 output_asm_insn ("vmov%?.s<V_sz_elem>\t%0, %P1[%c2]", ops);
2521 return "";
2522 }
2523 [(set_attr "predicable" "yes")
2524 (set_attr "neon_type" "neon_bp_simple")]
2525 )
2527 (define_insn "neon_vget_lane<mode>_zext_internal"
2528 [(set (match_operand:SI 0 "s_register_operand" "=r")
2529 (zero_extend:SI
2530 (vec_select:<V_elem>
2531 (match_operand:VQ 1 "s_register_operand" "w")
2532 (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2533 "TARGET_NEON"
2534 {
2535 rtx ops[3];
2536 int regno = REGNO (operands[1]);
2537 unsigned int halfelts = GET_MODE_NUNITS (<MODE>mode) / 2;
2538 unsigned int elt = INTVAL (operands[2]);
2539 unsigned int elt_adj = elt % halfelts;
2541 if (BYTES_BIG_ENDIAN)
2542 elt_adj = halfelts - 1 - elt_adj;
2544 ops[0] = operands[0];
2545 ops[1] = gen_rtx_REG (<V_HALF>mode, regno + 2 * (elt / halfelts));
2546 ops[2] = GEN_INT (elt_adj);
2547 output_asm_insn ("vmov%?.u<V_sz_elem>\t%0, %P1[%c2]", ops);
2549 return "";
2550 }
2551 [(set_attr "predicable" "yes")
2552 (set_attr "neon_type" "neon_bp_simple")]
2553 )
2555 (define_expand "neon_vget_lane<mode>"
2556 [(match_operand:<V_ext> 0 "s_register_operand" "")
2557 (match_operand:VDQW 1 "s_register_operand" "")
2558 (match_operand:SI 2 "immediate_operand" "")
2559 (match_operand:SI 3 "immediate_operand" "")]
2560 "TARGET_NEON"
2561 {
2562 HOST_WIDE_INT magic = INTVAL (operands[3]);
2563 rtx insn;
2565 neon_lane_bounds (operands[2], 0, GET_MODE_NUNITS (<MODE>mode));
2567 if (BYTES_BIG_ENDIAN)
2568 {
2569 /* The intrinsics are defined in terms of a model where the
2570 element ordering in memory is vldm order, whereas the generic
2571 RTL is defined in terms of a model where the element ordering
2572 in memory is array order. Convert the lane number to conform
2573 to this model. */
2574 unsigned int elt = INTVAL (operands[2]);
2575 unsigned int reg_nelts
2576 = 64 / GET_MODE_BITSIZE (GET_MODE_INNER (<MODE>mode));
2577 elt ^= reg_nelts - 1;
2578 operands[2] = GEN_INT (elt);
2579 }
2581 if ((magic & 3) == 3 || GET_MODE_BITSIZE (GET_MODE_INNER (<MODE>mode)) == 32)
2582 insn = gen_vec_extract<mode> (operands[0], operands[1], operands[2]);
2583 else
2584 {
2585 if ((magic & 1) != 0)
2586 insn = gen_neon_vget_lane<mode>_sext_internal (operands[0], operands[1],
2587 operands[2]);
2588 else
2589 insn = gen_neon_vget_lane<mode>_zext_internal (operands[0], operands[1],
2590 operands[2]);
2591 }
2592 emit_insn (insn);
2593 DONE;
2594 })
2596 ; Operand 3 (info word) is ignored because it does nothing useful with 64-bit
2597 ; elements.
2599 (define_expand "neon_vget_lanedi"
2600 [(match_operand:DI 0 "s_register_operand" "=r")
2601 (match_operand:DI 1 "s_register_operand" "w")
2602 (match_operand:SI 2 "immediate_operand" "i")
2603 (match_operand:SI 3 "immediate_operand" "i")]
2604 "TARGET_NEON"
2605 {
2606 neon_lane_bounds (operands[2], 0, 1);
2607 emit_move_insn (operands[0], operands[1]);
2608 DONE;
2609 })
2611 (define_expand "neon_vget_lanev2di"
2612 [(match_operand:DI 0 "s_register_operand" "=r")
2613 (match_operand:V2DI 1 "s_register_operand" "w")
2614 (match_operand:SI 2 "immediate_operand" "i")
2615 (match_operand:SI 3 "immediate_operand" "i")]
2616 "TARGET_NEON"
2617 {
2618 neon_lane_bounds (operands[2], 0, 2);
2619 emit_insn (gen_vec_extractv2di (operands[0], operands[1], operands[2]));
2620 DONE;
2621 })
2623 (define_expand "neon_vset_lane<mode>"
2624 [(match_operand:VDQ 0 "s_register_operand" "=w")
2625 (match_operand:<V_elem> 1 "s_register_operand" "r")
2626 (match_operand:VDQ 2 "s_register_operand" "0")
2627 (match_operand:SI 3 "immediate_operand" "i")]
2628 "TARGET_NEON"
2629 {
2630 unsigned int elt = INTVAL (operands[3]);
2631 neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
2633 if (BYTES_BIG_ENDIAN)
2634 {
2635 unsigned int reg_nelts
2636 = 64 / GET_MODE_BITSIZE (GET_MODE_INNER (<MODE>mode));
2637 elt ^= reg_nelts - 1;
2638 }
2640 emit_insn (gen_vec_set<mode>_internal (operands[0], operands[1],
2641 GEN_INT (1 << elt), operands[2]));
2642 DONE;
2643 })
2645 ; See neon_vget_lanedi comment for reasons operands 2 & 3 are ignored.
2647 (define_expand "neon_vset_lanedi"
2648 [(match_operand:DI 0 "s_register_operand" "=w")
2649 (match_operand:DI 1 "s_register_operand" "r")
2650 (match_operand:DI 2 "s_register_operand" "0")
2651 (match_operand:SI 3 "immediate_operand" "i")]
2652 "TARGET_NEON"
2653 {
2654 neon_lane_bounds (operands[3], 0, 1);
2655 emit_move_insn (operands[0], operands[1]);
2656 DONE;
2657 })
2659 (define_expand "neon_vcreate<mode>"
2660 [(match_operand:VDX 0 "s_register_operand" "")
2661 (match_operand:DI 1 "general_operand" "")]
2662 "TARGET_NEON"
2663 {
2664 rtx src = gen_lowpart (<MODE>mode, operands[1]);
2665 emit_move_insn (operands[0], src);
2666 DONE;
2667 })
2669 (define_insn "neon_vdup_n<mode>"
2670 [(set (match_operand:VX 0 "s_register_operand" "=w")
2671 (vec_duplicate:VX (match_operand:<V_elem> 1 "s_register_operand" "r")))]
2672 "TARGET_NEON"
2673 "vdup%?.<V_sz_elem>\t%<V_reg>0, %1"
2674 ;; Assume this schedules like vmov.
2675 [(set_attr "predicable" "yes")
2676 (set_attr "neon_type" "neon_bp_simple")]
2677 )
2679 (define_insn "neon_vdup_n<mode>"
2680 [(set (match_operand:V32 0 "s_register_operand" "=w,w")
2681 (vec_duplicate:V32 (match_operand:<V_elem> 1 "s_register_operand" "r,t")))]
2682 "TARGET_NEON"
2683 "@
2684 vdup%?.<V_sz_elem>\t%<V_reg>0, %1
2685 vdup%?.<V_sz_elem>\t%<V_reg>0, %y1"
2686 ;; Assume this schedules like vmov.
2687 [(set_attr "predicable" "yes")
2688 (set_attr "neon_type" "neon_bp_simple")]
2689 )
2691 (define_expand "neon_vdup_ndi"
2692 [(match_operand:DI 0 "s_register_operand" "=w")
2693 (match_operand:DI 1 "s_register_operand" "r")]
2694 "TARGET_NEON"
2695 {
2696 emit_move_insn (operands[0], operands[1]);
2697 DONE;
2698 }
2699 )
2701 (define_insn "neon_vdup_nv2di"
2702 [(set (match_operand:V2DI 0 "s_register_operand" "=w,w")
2703 (vec_duplicate:V2DI (match_operand:DI 1 "s_register_operand" "r,w")))]
2704 "TARGET_NEON"
2705 "@
2706 vmov%?\t%e0, %Q1, %R1\;vmov%?\t%f0, %Q1, %R1
2707 vmov%?\t%e0, %P1\;vmov%?\t%f0, %P1"
2708 [(set_attr "predicable" "yes")
2709 (set_attr "length" "8")
2710 (set_attr "neon_type" "neon_bp_simple")]
2711 )
2713 (define_insn "neon_vdup_lane<mode>_internal"
2714 [(set (match_operand:VDQW 0 "s_register_operand" "=w")
2715 (vec_duplicate:VDQW
2716 (vec_select:<V_elem>
2717 (match_operand:<V_double_vector_mode> 1 "s_register_operand" "w")
2718 (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2719 "TARGET_NEON"
2720 {
2721 if (BYTES_BIG_ENDIAN)
2722 {
2723 int elt = INTVAL (operands[2]);
2724 elt = GET_MODE_NUNITS (<V_double_vector_mode>mode) - 1 - elt;
2725 operands[2] = GEN_INT (elt);
2726 }
2727 if (<Is_d_reg>)
2728 return "vdup.<V_sz_elem>\t%P0, %P1[%c2]";
2729 else
2730 return "vdup.<V_sz_elem>\t%q0, %P1[%c2]";
2731 }
2732 ;; Assume this schedules like vmov.
2733 [(set_attr "neon_type" "neon_bp_simple")]
2734 )
2736 (define_expand "neon_vdup_lane<mode>"
2737 [(match_operand:VDQW 0 "s_register_operand" "=w")
2738 (match_operand:<V_double_vector_mode> 1 "s_register_operand" "w")
2739 (match_operand:SI 2 "immediate_operand" "i")]
2740 "TARGET_NEON"
2741 {
2742 neon_lane_bounds (operands[2], 0, GET_MODE_NUNITS (<V_double_vector_mode>mode));
2743 if (BYTES_BIG_ENDIAN)
2744 {
2745 unsigned int elt = INTVAL (operands[2]);
2746 unsigned int reg_nelts
2747 = 64 / GET_MODE_BITSIZE (GET_MODE_INNER (<V_double_vector_mode>mode));
2748 elt ^= reg_nelts - 1;
2749 operands[2] = GEN_INT (elt);
2750 }
2751 emit_insn (gen_neon_vdup_lane<mode>_internal (operands[0], operands[1],
2752 operands[2]));
2753 DONE;
2754 })
2756 ; Scalar index is ignored, since only zero is valid here.
2757 (define_expand "neon_vdup_lanedi"
2758 [(match_operand:DI 0 "s_register_operand" "=w")
2759 (match_operand:DI 1 "s_register_operand" "w")
2760 (match_operand:SI 2 "immediate_operand" "i")]
2761 "TARGET_NEON"
2762 {
2763 neon_lane_bounds (operands[2], 0, 1);
2764 emit_move_insn (operands[0], operands[1]);
2765 DONE;
2766 })
2768 ; Likewise for v2di, as the DImode second operand has only a single element.
2769 (define_expand "neon_vdup_lanev2di"
2770 [(match_operand:V2DI 0 "s_register_operand" "=w")
2771 (match_operand:DI 1 "s_register_operand" "w")
2772 (match_operand:SI 2 "immediate_operand" "i")]
2773 "TARGET_NEON"
2774 {
2775 neon_lane_bounds (operands[2], 0, 1);
2776 emit_insn (gen_neon_vdup_nv2di (operands[0], operands[1]));
2777 DONE;
2778 })
2780 ;; In this insn, operand 1 should be low, and operand 2 the high part of the
2781 ;; dest vector.
2782 ;; FIXME: A different implementation of this builtin could make it much
2783 ;; more likely that we wouldn't actually need to output anything (we could make
2784 ;; it so that the reg allocator puts things in the right places magically
2785 ;; instead). Lack of subregs for vectors makes that tricky though, I think.
2787 (define_insn "neon_vcombine<mode>"
2788 [(set (match_operand:<V_DOUBLE> 0 "s_register_operand" "=w")
2789 (vec_concat:<V_DOUBLE> (match_operand:VDX 1 "s_register_operand" "w")
2790 (match_operand:VDX 2 "s_register_operand" "w")))]
2791 "TARGET_NEON"
2792 {
2793 int dest = REGNO (operands[0]);
2794 int src1 = REGNO (operands[1]);
2795 int src2 = REGNO (operands[2]);
2796 rtx destlo;
2798 if (src1 == dest && src2 == dest + 2)
2799 return "";
2800 else if (src2 == dest && src1 == dest + 2)
2801 /* Special case of reversed high/low parts. */
2802 return "vswp\t%P1, %P2";
2804 destlo = gen_rtx_REG (<MODE>mode, dest);
2806 if (!reg_overlap_mentioned_p (operands[2], destlo))
2807 {
2808 /* Try to avoid unnecessary moves if part of the result is in the right
2809 place already. */
2810 if (src1 != dest)
2811 output_asm_insn ("vmov\t%e0, %P1", operands);
2812 if (src2 != dest + 2)
2813 output_asm_insn ("vmov\t%f0, %P2", operands);
2814 }
2815 else
2816 {
2817 if (src2 != dest + 2)
2818 output_asm_insn ("vmov\t%f0, %P2", operands);
2819 if (src1 != dest)
2820 output_asm_insn ("vmov\t%e0, %P1", operands);
2821 }
2823 return "";
2824 }
2825 ;; We set the neon_type attribute based on the vmov instructions above.
2826 [(set_attr "length" "8")
2827 (set_attr "neon_type" "neon_bp_simple")]
2828 )
2830 (define_insn "neon_vget_highv16qi"
2831 [(set (match_operand:V8QI 0 "s_register_operand" "=w")
2832 (vec_select:V8QI (match_operand:V16QI 1 "s_register_operand" "w")
2833 (parallel [(const_int 8) (const_int 9)
2834 (const_int 10) (const_int 11)
2835 (const_int 12) (const_int 13)
2836 (const_int 14) (const_int 15)])))]
2837 "TARGET_NEON"
2838 {
2839 int dest = REGNO (operands[0]);
2840 int src = REGNO (operands[1]);
2842 if (dest != src + 2)
2843 return "vmov\t%P0, %f1";
2844 else
2845 return "";
2846 }
2847 [(set_attr "neon_type" "neon_bp_simple")]
2848 )
2850 (define_insn "neon_vget_highv8hi"
2851 [(set (match_operand:V4HI 0 "s_register_operand" "=w")
2852 (vec_select:V4HI (match_operand:V8HI 1 "s_register_operand" "w")
2853 (parallel [(const_int 4) (const_int 5)
2854 (const_int 6) (const_int 7)])))]
2855 "TARGET_NEON"
2856 {
2857 int dest = REGNO (operands[0]);
2858 int src = REGNO (operands[1]);
2860 if (dest != src + 2)
2861 return "vmov\t%P0, %f1";
2862 else
2863 return "";
2864 }
2865 [(set_attr "neon_type" "neon_bp_simple")]
2866 )
2868 (define_insn "neon_vget_highv4si"
2869 [(set (match_operand:V2SI 0 "s_register_operand" "=w")
2870 (vec_select:V2SI (match_operand:V4SI 1 "s_register_operand" "w")
2871 (parallel [(const_int 2) (const_int 3)])))]
2872 "TARGET_NEON"
2873 {
2874 int dest = REGNO (operands[0]);
2875 int src = REGNO (operands[1]);
2877 if (dest != src + 2)
2878 return "vmov\t%P0, %f1";
2879 else
2880 return "";
2881 }
2882 [(set_attr "neon_type" "neon_bp_simple")]
2883 )
2885 (define_insn "neon_vget_highv4sf"
2886 [(set (match_operand:V2SF 0 "s_register_operand" "=w")
2887 (vec_select:V2SF (match_operand:V4SF 1 "s_register_operand" "w")
2888 (parallel [(const_int 2) (const_int 3)])))]
2889 "TARGET_NEON"
2890 {
2891 int dest = REGNO (operands[0]);
2892 int src = REGNO (operands[1]);
2894 if (dest != src + 2)
2895 return "vmov\t%P0, %f1";
2896 else
2897 return "";
2898 }
2899 [(set_attr "neon_type" "neon_bp_simple")]
2900 )
2902 (define_insn "neon_vget_highv2di"
2903 [(set (match_operand:DI 0 "s_register_operand" "=w")
2904 (vec_select:DI (match_operand:V2DI 1 "s_register_operand" "w")
2905 (parallel [(const_int 1)])))]
2906 "TARGET_NEON"
2907 {
2908 int dest = REGNO (operands[0]);
2909 int src = REGNO (operands[1]);
2911 if (dest != src + 2)
2912 return "vmov\t%P0, %f1";
2913 else
2914 return "";
2915 }
2916 [(set_attr "neon_type" "neon_bp_simple")]
2917 )
2919 (define_insn "neon_vget_lowv16qi"
2920 [(set (match_operand:V8QI 0 "s_register_operand" "=w")
2921 (vec_select:V8QI (match_operand:V16QI 1 "s_register_operand" "w")
2922 (parallel [(const_int 0) (const_int 1)
2923 (const_int 2) (const_int 3)
2924 (const_int 4) (const_int 5)
2925 (const_int 6) (const_int 7)])))]
2926 "TARGET_NEON"
2927 {
2928 int dest = REGNO (operands[0]);
2929 int src = REGNO (operands[1]);
2931 if (dest != src)
2932 return "vmov\t%P0, %e1";
2933 else
2934 return "";
2935 }
2936 [(set_attr "neon_type" "neon_bp_simple")]
2937 )
2939 (define_insn "neon_vget_lowv8hi"
2940 [(set (match_operand:V4HI 0 "s_register_operand" "=w")
2941 (vec_select:V4HI (match_operand:V8HI 1 "s_register_operand" "w")
2942 (parallel [(const_int 0) (const_int 1)
2943 (const_int 2) (const_int 3)])))]
2944 "TARGET_NEON"
2945 {
2946 int dest = REGNO (operands[0]);
2947 int src = REGNO (operands[1]);
2949 if (dest != src)
2950 return "vmov\t%P0, %e1";
2951 else
2952 return "";
2953 }
2954 [(set_attr "neon_type" "neon_bp_simple")]
2955 )
2957 (define_insn "neon_vget_lowv4si"
2958 [(set (match_operand:V2SI 0 "s_register_operand" "=w")
2959 (vec_select:V2SI (match_operand:V4SI 1 "s_register_operand" "w")
2960 (parallel [(const_int 0) (const_int 1)])))]
2961 "TARGET_NEON"
2962 {
2963 int dest = REGNO (operands[0]);
2964 int src = REGNO (operands[1]);
2966 if (dest != src)
2967 return "vmov\t%P0, %e1";
2968 else
2969 return "";
2970 }
2971 [(set_attr "neon_type" "neon_bp_simple")]
2972 )
2974 (define_insn "neon_vget_lowv4sf"
2975 [(set (match_operand:V2SF 0 "s_register_operand" "=w")
2976 (vec_select:V2SF (match_operand:V4SF 1 "s_register_operand" "w")
2977 (parallel [(const_int 0) (const_int 1)])))]
2978 "TARGET_NEON"
2979 {
2980 int dest = REGNO (operands[0]);
2981 int src = REGNO (operands[1]);
2983 if (dest != src)
2984 return "vmov\t%P0, %e1";
2985 else
2986 return "";
2987 }
2988 [(set_attr "neon_type" "neon_bp_simple")]
2989 )
2991 (define_insn "neon_vget_lowv2di"
2992 [(set (match_operand:DI 0 "s_register_operand" "=w")
2993 (vec_select:DI (match_operand:V2DI 1 "s_register_operand" "w")
2994 (parallel [(const_int 0)])))]
2995 "TARGET_NEON"
2996 {
2997 int dest = REGNO (operands[0]);
2998 int src = REGNO (operands[1]);
3000 if (dest != src)
3001 return "vmov\t%P0, %e1";
3002 else
3003 return "";
3004 }
3005 [(set_attr "neon_type" "neon_bp_simple")]
3006 )
3008 (define_insn "neon_vcvt<mode>"
3009 [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3010 (unspec:<V_CVTTO> [(match_operand:VCVTF 1 "s_register_operand" "w")
3011 (match_operand:SI 2 "immediate_operand" "i")]
3012 UNSPEC_VCVT))]
3013 "TARGET_NEON"
3014 "vcvt.%T2%#32.f32\t%<V_reg>0, %<V_reg>1"
3015 [(set (attr "neon_type")
3016 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
3017 (const_string "neon_fp_vadd_ddd_vabs_dd")
3018 (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3019 )
3021 (define_insn "neon_vcvt<mode>"
3022 [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3023 (unspec:<V_CVTTO> [(match_operand:VCVTI 1 "s_register_operand" "w")
3024 (match_operand:SI 2 "immediate_operand" "i")]
3025 UNSPEC_VCVT))]
3026 "TARGET_NEON"
3027 "vcvt.f32.%T2%#32\t%<V_reg>0, %<V_reg>1"
3028 [(set (attr "neon_type")
3029 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
3030 (const_string "neon_fp_vadd_ddd_vabs_dd")
3031 (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3032 )
3034 (define_insn "neon_vcvt_n<mode>"
3035 [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3036 (unspec:<V_CVTTO> [(match_operand:VCVTF 1 "s_register_operand" "w")
3037 (match_operand:SI 2 "immediate_operand" "i")
3038 (match_operand:SI 3 "immediate_operand" "i")]
3039 UNSPEC_VCVT_N))]
3040 "TARGET_NEON"
3041 {
3042 neon_const_bounds (operands[2], 1, 33);
3043 return "vcvt.%T3%#32.f32\t%<V_reg>0, %<V_reg>1, %2";
3044 }
3045 [(set (attr "neon_type")
3046 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
3047 (const_string "neon_fp_vadd_ddd_vabs_dd")
3048 (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3049 )
3051 (define_insn "neon_vcvt_n<mode>"
3052 [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3053 (unspec:<V_CVTTO> [(match_operand:VCVTI 1 "s_register_operand" "w")
3054 (match_operand:SI 2 "immediate_operand" "i")
3055 (match_operand:SI 3 "immediate_operand" "i")]
3056 UNSPEC_VCVT_N))]
3057 "TARGET_NEON"
3058 {
3059 neon_const_bounds (operands[2], 1, 33);
3060 return "vcvt.f32.%T3%#32\t%<V_reg>0, %<V_reg>1, %2";
3061 }
3062 [(set (attr "neon_type")
3063 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
3064 (const_string "neon_fp_vadd_ddd_vabs_dd")
3065 (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3066 )
3068 (define_insn "neon_vmovn<mode>"
3069 [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
3070 (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
3071 (match_operand:SI 2 "immediate_operand" "i")]
3072 UNSPEC_VMOVN))]
3073 "TARGET_NEON"
3074 "vmovn.<V_if_elem>\t%P0, %q1"
3075 [(set_attr "neon_type" "neon_bp_simple")]
3076 )
3078 (define_insn "neon_vqmovn<mode>"
3079 [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
3080 (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
3081 (match_operand:SI 2 "immediate_operand" "i")]
3082 UNSPEC_VQMOVN))]
3083 "TARGET_NEON"
3084 "vqmovn.%T2%#<V_sz_elem>\t%P0, %q1"
3085 [(set_attr "neon_type" "neon_shift_2")]
3086 )
3088 (define_insn "neon_vqmovun<mode>"
3089 [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
3090 (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
3091 (match_operand:SI 2 "immediate_operand" "i")]
3092 UNSPEC_VQMOVUN))]
3093 "TARGET_NEON"
3094 "vqmovun.<V_s_elem>\t%P0, %q1"
3095 [(set_attr "neon_type" "neon_shift_2")]
3096 )
3098 (define_insn "neon_vmovl<mode>"
3099 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3100 (unspec:<V_widen> [(match_operand:VW 1 "s_register_operand" "w")
3101 (match_operand:SI 2 "immediate_operand" "i")]
3102 UNSPEC_VMOVL))]
3103 "TARGET_NEON"
3104 "vmovl.%T2%#<V_sz_elem>\t%q0, %P1"
3105 [(set_attr "neon_type" "neon_shift_1")]
3106 )
3108 (define_insn "neon_vmul_lane<mode>"
3109 [(set (match_operand:VMD 0 "s_register_operand" "=w")
3110 (unspec:VMD [(match_operand:VMD 1 "s_register_operand" "w")
3111 (match_operand:VMD 2 "s_register_operand"
3112 "<scalar_mul_constraint>")
3113 (match_operand:SI 3 "immediate_operand" "i")
3114 (match_operand:SI 4 "immediate_operand" "i")]
3115 UNSPEC_VMUL_LANE))]
3116 "TARGET_NEON"
3117 {
3118 neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3119 return "vmul.<V_if_elem>\t%P0, %P1, %P2[%c3]";
3120 }
3121 [(set (attr "neon_type")
3122 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
3123 (const_string "neon_fp_vmul_ddd")
3124 (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0))
3125 (const_string "neon_mul_ddd_16_scalar_32_16_long_scalar")
3126 (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar"))))]
3127 )
3129 (define_insn "neon_vmul_lane<mode>"
3130 [(set (match_operand:VMQ 0 "s_register_operand" "=w")
3131 (unspec:VMQ [(match_operand:VMQ 1 "s_register_operand" "w")
3132 (match_operand:<V_HALF> 2 "s_register_operand"
3133 "<scalar_mul_constraint>")
3134 (match_operand:SI 3 "immediate_operand" "i")
3135 (match_operand:SI 4 "immediate_operand" "i")]
3136 UNSPEC_VMUL_LANE))]
3137 "TARGET_NEON"
3138 {
3139 neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<V_HALF>mode));
3140 return "vmul.<V_if_elem>\t%q0, %q1, %P2[%c3]";
3141 }
3142 [(set (attr "neon_type")
3143 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
3144 (const_string "neon_fp_vmul_qqd")
3145 (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0))
3146 (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")
3147 (const_string "neon_mul_qqd_32_scalar"))))]
3148 )
3150 (define_insn "neon_vmull_lane<mode>"
3151 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3152 (unspec:<V_widen> [(match_operand:VMDI 1 "s_register_operand" "w")
3153 (match_operand:VMDI 2 "s_register_operand"
3154 "<scalar_mul_constraint>")
3155 (match_operand:SI 3 "immediate_operand" "i")
3156 (match_operand:SI 4 "immediate_operand" "i")]
3157 UNSPEC_VMULL_LANE))]
3158 "TARGET_NEON"
3159 {
3160 neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3161 return "vmull.%T4%#<V_sz_elem>\t%q0, %P1, %P2[%c3]";
3162 }
3163 [(set (attr "neon_type")
3164 (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0))
3165 (const_string "neon_mul_ddd_16_scalar_32_16_long_scalar")
3166 (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")))]
3167 )
3169 (define_insn "neon_vqdmull_lane<mode>"
3170 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3171 (unspec:<V_widen> [(match_operand:VMDI 1 "s_register_operand" "w")
3172 (match_operand:VMDI 2 "s_register_operand"
3173 "<scalar_mul_constraint>")
3174 (match_operand:SI 3 "immediate_operand" "i")
3175 (match_operand:SI 4 "immediate_operand" "i")]
3176 UNSPEC_VQDMULL_LANE))]
3177 "TARGET_NEON"
3178 {
3179 neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3180 return "vqdmull.<V_s_elem>\t%q0, %P1, %P2[%c3]";
3181 }
3182 [(set (attr "neon_type")
3183 (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0))
3184 (const_string "neon_mul_ddd_16_scalar_32_16_long_scalar")
3185 (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")))]
3186 )
3188 (define_insn "neon_vqdmulh_lane<mode>"
3189 [(set (match_operand:VMQI 0 "s_register_operand" "=w")
3190 (unspec:VMQI [(match_operand:VMQI 1 "s_register_operand" "w")
3191 (match_operand:<V_HALF> 2 "s_register_operand"
3192 "<scalar_mul_constraint>")
3193 (match_operand:SI 3 "immediate_operand" "i")
3194 (match_operand:SI 4 "immediate_operand" "i")]
3195 UNSPEC_VQDMULH_LANE))]
3196 "TARGET_NEON"
3197 {
3198 neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3199 return "vq%O4dmulh.%T4%#<V_sz_elem>\t%q0, %q1, %P2[%c3]";
3200 }
3201 [(set (attr "neon_type")
3202 (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0))
3203 (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")
3204 (const_string "neon_mul_qqd_32_scalar")))]
3205 )
3207 (define_insn "neon_vqdmulh_lane<mode>"
3208 [(set (match_operand:VMDI 0 "s_register_operand" "=w")
3209 (unspec:VMDI [(match_operand:VMDI 1 "s_register_operand" "w")
3210 (match_operand:VMDI 2 "s_register_operand"
3211 "<scalar_mul_constraint>")
3212 (match_operand:SI 3 "immediate_operand" "i")
3213 (match_operand:SI 4 "immediate_operand" "i")]
3214 UNSPEC_VQDMULH_LANE))]
3215 "TARGET_NEON"
3216 {
3217 neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3218 return "vq%O4dmulh.%T4%#<V_sz_elem>\t%P0, %P1, %P2[%c3]";
3219 }
3220 [(set (attr "neon_type")
3221 (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0))
3222 (const_string "neon_mul_ddd_16_scalar_32_16_long_scalar")
3223 (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")))]
3224 )
3226 (define_insn "neon_vmla_lane<mode>"
3227 [(set (match_operand:VMD 0 "s_register_operand" "=w")
3228 (unspec:VMD [(match_operand:VMD 1 "s_register_operand" "0")
3229 (match_operand:VMD 2 "s_register_operand" "w")
3230 (match_operand:VMD 3 "s_register_operand"
3231 "<scalar_mul_constraint>")
3232 (match_operand:SI 4 "immediate_operand" "i")
3233 (match_operand:SI 5 "immediate_operand" "i")]
3234 UNSPEC_VMLA_LANE))]
3235 "TARGET_NEON"
3236 {
3237 neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3238 return "vmla.<V_if_elem>\t%P0, %P2, %P3[%c4]";
3239 }
3240 [(set (attr "neon_type")
3241 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
3242 (const_string "neon_fp_vmla_ddd_scalar")
3243 (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0))
3244 (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar")
3245 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long"))))]
3246 )
3248 (define_insn "neon_vmla_lane<mode>"
3249 [(set (match_operand:VMQ 0 "s_register_operand" "=w")
3250 (unspec:VMQ [(match_operand:VMQ 1 "s_register_operand" "0")
3251 (match_operand:VMQ 2 "s_register_operand" "w")
3252 (match_operand:<V_HALF> 3 "s_register_operand"
3253 "<scalar_mul_constraint>")
3254 (match_operand:SI 4 "immediate_operand" "i")
3255 (match_operand:SI 5 "immediate_operand" "i")]
3256 UNSPEC_VMLA_LANE))]
3257 "TARGET_NEON"
3258 {
3259 neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3260 return "vmla.<V_if_elem>\t%q0, %q2, %P3[%c4]";
3261 }
3262 [(set (attr "neon_type")
3263 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
3264 (const_string "neon_fp_vmla_qqq_scalar")
3265 (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0))
3266 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")
3267 (const_string "neon_mla_qqq_32_qqd_32_scalar"))))]
3268 )
3270 (define_insn "neon_vmlal_lane<mode>"
3271 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3272 (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
3273 (match_operand:VMDI 2 "s_register_operand" "w")
3274 (match_operand:VMDI 3 "s_register_operand"
3275 "<scalar_mul_constraint>")
3276 (match_operand:SI 4 "immediate_operand" "i")
3277 (match_operand:SI 5 "immediate_operand" "i")]
3278 UNSPEC_VMLAL_LANE))]
3279 "TARGET_NEON"
3280 {
3281 neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3282 return "vmlal.%T5%#<V_sz_elem>\t%q0, %P2, %P3[%c4]";
3283 }
3284 [(set (attr "neon_type")
3285 (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0))
3286 (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar")
3287 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
3288 )
3290 (define_insn "neon_vqdmlal_lane<mode>"
3291 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3292 (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
3293 (match_operand:VMDI 2 "s_register_operand" "w")
3294 (match_operand:VMDI 3 "s_register_operand"
3295 "<scalar_mul_constraint>")
3296 (match_operand:SI 4 "immediate_operand" "i")
3297 (match_operand:SI 5 "immediate_operand" "i")]
3298 UNSPEC_VQDMLAL_LANE))]
3299 "TARGET_NEON"
3300 {
3301 neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3302 return "vqdmlal.<V_s_elem>\t%q0, %P2, %P3[%c4]";
3303 }
3304 [(set (attr "neon_type")
3305 (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0))
3306 (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar")
3307 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
3308 )
3310 (define_insn "neon_vmls_lane<mode>"
3311 [(set (match_operand:VMD 0 "s_register_operand" "=w")
3312 (unspec:VMD [(match_operand:VMD 1 "s_register_operand" "0")
3313 (match_operand:VMD 2 "s_register_operand" "w")
3314 (match_operand:VMD 3 "s_register_operand"
3315 "<scalar_mul_constraint>")
3316 (match_operand:SI 4 "immediate_operand" "i")
3317 (match_operand:SI 5 "immediate_operand" "i")]
3318 UNSPEC_VMLS_LANE))]
3319 "TARGET_NEON"
3320 {
3321 neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3322 return "vmls.<V_if_elem>\t%P0, %P2, %P3[%c4]";
3323 }
3324 [(set (attr "neon_type")
3325 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
3326 (const_string "neon_fp_vmla_ddd_scalar")
3327 (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0))
3328 (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar")
3329 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long"))))]
3330 )
3332 (define_insn "neon_vmls_lane<mode>"
3333 [(set (match_operand:VMQ 0 "s_register_operand" "=w")
3334 (unspec:VMQ [(match_operand:VMQ 1 "s_register_operand" "0")
3335 (match_operand:VMQ 2 "s_register_operand" "w")
3336 (match_operand:<V_HALF> 3 "s_register_operand"
3337 "<scalar_mul_constraint>")
3338 (match_operand:SI 4 "immediate_operand" "i")
3339 (match_operand:SI 5 "immediate_operand" "i")]
3340 UNSPEC_VMLS_LANE))]
3341 "TARGET_NEON"
3342 {
3343 neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3344 return "vmls.<V_if_elem>\t%q0, %q2, %P3[%c4]";
3345 }
3346 [(set (attr "neon_type")
3347 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
3348 (const_string "neon_fp_vmla_qqq_scalar")
3349 (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0))
3350 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")
3351 (const_string "neon_mla_qqq_32_qqd_32_scalar"))))]
3352 )
3354 (define_insn "neon_vmlsl_lane<mode>"
3355 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3356 (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
3357 (match_operand:VMDI 2 "s_register_operand" "w")
3358 (match_operand:VMDI 3 "s_register_operand"
3359 "<scalar_mul_constraint>")
3360 (match_operand:SI 4 "immediate_operand" "i")
3361 (match_operand:SI 5 "immediate_operand" "i")]
3362 UNSPEC_VMLSL_LANE))]
3363 "TARGET_NEON"
3364 {
3365 neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3366 return "vmlsl.%T5%#<V_sz_elem>\t%q0, %P2, %P3[%c4]";
3367 }
3368 [(set (attr "neon_type")
3369 (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0))
3370 (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar")
3371 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
3372 )
3374 (define_insn "neon_vqdmlsl_lane<mode>"
3375 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3376 (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
3377 (match_operand:VMDI 2 "s_register_operand" "w")
3378 (match_operand:VMDI 3 "s_register_operand"
3379 "<scalar_mul_constraint>")
3380 (match_operand:SI 4 "immediate_operand" "i")
3381 (match_operand:SI 5 "immediate_operand" "i")]
3382 UNSPEC_VQDMLSL_LANE))]
3383 "TARGET_NEON"
3384 {
3385 neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3386 return "vqdmlsl.<V_s_elem>\t%q0, %P2, %P3[%c4]";
3387 }
3388 [(set (attr "neon_type")
3389 (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0))
3390 (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar")
3391 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
3392 )
3394 ; FIXME: For the "_n" multiply/multiply-accumulate insns, we copy a value in a
3395 ; core register into a temp register, then use a scalar taken from that. This
3396 ; isn't an optimal solution if e.g. the scalar has just been read from memory
3397 ; or extracted from another vector. The latter case it's currently better to
3398 ; use the "_lane" variant, and the former case can probably be implemented
3399 ; using vld1_lane, but that hasn't been done yet.
3401 (define_expand "neon_vmul_n<mode>"
3402 [(match_operand:VMD 0 "s_register_operand" "")
3403 (match_operand:VMD 1 "s_register_operand" "")
3404 (match_operand:<V_elem> 2 "s_register_operand" "")
3405 (match_operand:SI 3 "immediate_operand" "")]
3406 "TARGET_NEON"
3407 {
3408 rtx tmp = gen_reg_rtx (<MODE>mode);
3409 emit_insn (gen_neon_vset_lane<mode> (tmp, operands[2], tmp, const0_rtx));
3410 emit_insn (gen_neon_vmul_lane<mode> (operands[0], operands[1], tmp,
3411 const0_rtx, const0_rtx));
3412 DONE;
3413 })
3415 (define_expand "neon_vmul_n<mode>"
3416 [(match_operand:VMQ 0 "s_register_operand" "")
3417 (match_operand:VMQ 1 "s_register_operand" "")
3418 (match_operand:<V_elem> 2 "s_register_operand" "")
3419 (match_operand:SI 3 "immediate_operand" "")]
3420 "TARGET_NEON"
3421 {
3422 rtx tmp = gen_reg_rtx (<V_HALF>mode);
3423 emit_insn (gen_neon_vset_lane<V_half> (tmp, operands[2], tmp, const0_rtx));
3424 emit_insn (gen_neon_vmul_lane<mode> (operands[0], operands[1], tmp,
3425 const0_rtx, const0_rtx));
3426 DONE;
3427 })
3429 (define_expand "neon_vmull_n<mode>"
3430 [(match_operand:<V_widen> 0 "s_register_operand" "")
3431 (match_operand:VMDI 1 "s_register_operand" "")
3432 (match_operand:<V_elem> 2 "s_register_operand" "")
3433 (match_operand:SI 3 "immediate_operand" "")]
3434 "TARGET_NEON"
3435 {
3436 rtx tmp = gen_reg_rtx (<MODE>mode);
3437 emit_insn (gen_neon_vset_lane<mode> (tmp, operands[2], tmp, const0_rtx));
3438 emit_insn (gen_neon_vmull_lane<mode> (operands[0], operands[1], tmp,
3439 const0_rtx, operands[3]));
3440 DONE;
3441 })
3443 (define_expand "neon_vqdmull_n<mode>"
3444 [(match_operand:<V_widen> 0 "s_register_operand" "")
3445 (match_operand:VMDI 1 "s_register_operand" "")
3446 (match_operand:<V_elem> 2 "s_register_operand" "")
3447 (match_operand:SI 3 "immediate_operand" "")]
3448 "TARGET_NEON"
3449 {
3450 rtx tmp = gen_reg_rtx (<MODE>mode);
3451 emit_insn (gen_neon_vset_lane<mode> (tmp, operands[2], tmp, const0_rtx));
3452 emit_insn (gen_neon_vqdmull_lane<mode> (operands[0], operands[1], tmp,
3453 const0_rtx, const0_rtx));
3454 DONE;
3455 })
3457 (define_expand "neon_vqdmulh_n<mode>"
3458 [(match_operand:VMDI 0 "s_register_operand" "")
3459 (match_operand:VMDI 1 "s_register_operand" "")
3460 (match_operand:<V_elem> 2 "s_register_operand" "")
3461 (match_operand:SI 3 "immediate_operand" "")]
3462 "TARGET_NEON"
3463 {
3464 rtx tmp = gen_reg_rtx (<MODE>mode);
3465 emit_insn (gen_neon_vset_lane<mode> (tmp, operands[2], tmp, const0_rtx));
3466 emit_insn (gen_neon_vqdmulh_lane<mode> (operands[0], operands[1], tmp,
3467 const0_rtx, operands[3]));
3468 DONE;
3469 })
3471 (define_expand "neon_vqdmulh_n<mode>"
3472 [(match_operand:VMQI 0 "s_register_operand" "")
3473 (match_operand:VMQI 1 "s_register_operand" "")
3474 (match_operand:<V_elem> 2 "s_register_operand" "")
3475 (match_operand:SI 3 "immediate_operand" "")]
3476 "TARGET_NEON"
3477 {
3478 rtx tmp = gen_reg_rtx (<V_HALF>mode);
3479 emit_insn (gen_neon_vset_lane<V_half> (tmp, operands[2], tmp, const0_rtx));
3480 emit_insn (gen_neon_vqdmulh_lane<mode> (operands[0], operands[1], tmp,
3481 const0_rtx, operands[3]));
3482 DONE;
3483 })
3485 (define_expand "neon_vmla_n<mode>"
3486 [(match_operand:VMD 0 "s_register_operand" "")
3487 (match_operand:VMD 1 "s_register_operand" "")
3488 (match_operand:VMD 2 "s_register_operand" "")
3489 (match_operand:<V_elem> 3 "s_register_operand" "")
3490 (match_operand:SI 4 "immediate_operand" "")]
3491 "TARGET_NEON"
3492 {
3493 rtx tmp = gen_reg_rtx (<MODE>mode);
3494 emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3495 emit_insn (gen_neon_vmla_lane<mode> (operands[0], operands[1], operands[2],
3496 tmp, const0_rtx, operands[4]));
3497 DONE;
3498 })
3500 (define_expand "neon_vmla_n<mode>"
3501 [(match_operand:VMQ 0 "s_register_operand" "")
3502 (match_operand:VMQ 1 "s_register_operand" "")
3503 (match_operand:VMQ 2 "s_register_operand" "")
3504 (match_operand:<V_elem> 3 "s_register_operand" "")
3505 (match_operand:SI 4 "immediate_operand" "")]
3506 "TARGET_NEON"
3507 {
3508 rtx tmp = gen_reg_rtx (<V_HALF>mode);
3509 emit_insn (gen_neon_vset_lane<V_half> (tmp, operands[3], tmp, const0_rtx));
3510 emit_insn (gen_neon_vmla_lane<mode> (operands[0], operands[1], operands[2],
3511 tmp, const0_rtx, operands[4]));
3512 DONE;
3513 })
3515 (define_expand "neon_vmlal_n<mode>"
3516 [(match_operand:<V_widen> 0 "s_register_operand" "")
3517 (match_operand:<V_widen> 1 "s_register_operand" "")
3518 (match_operand:VMDI 2 "s_register_operand" "")
3519 (match_operand:<V_elem> 3 "s_register_operand" "")
3520 (match_operand:SI 4 "immediate_operand" "")]
3521 "TARGET_NEON"
3522 {
3523 rtx tmp = gen_reg_rtx (<MODE>mode);
3524 emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3525 emit_insn (gen_neon_vmlal_lane<mode> (operands[0], operands[1], operands[2],
3526 tmp, const0_rtx, operands[4]));
3527 DONE;
3528 })
3530 (define_expand "neon_vqdmlal_n<mode>"
3531 [(match_operand:<V_widen> 0 "s_register_operand" "")
3532 (match_operand:<V_widen> 1 "s_register_operand" "")
3533 (match_operand:VMDI 2 "s_register_operand" "")
3534 (match_operand:<V_elem> 3 "s_register_operand" "")
3535 (match_operand:SI 4 "immediate_operand" "")]
3536 "TARGET_NEON"
3537 {
3538 rtx tmp = gen_reg_rtx (<MODE>mode);
3539 emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3540 emit_insn (gen_neon_vqdmlal_lane<mode> (operands[0], operands[1], operands[2],
3541 tmp, const0_rtx, operands[4]));
3542 DONE;
3543 })
3545 (define_expand "neon_vmls_n<mode>"
3546 [(match_operand:VMD 0 "s_register_operand" "")
3547 (match_operand:VMD 1 "s_register_operand" "")
3548 (match_operand:VMD 2 "s_register_operand" "")
3549 (match_operand:<V_elem> 3 "s_register_operand" "")
3550 (match_operand:SI 4 "immediate_operand" "")]
3551 "TARGET_NEON"
3552 {
3553 rtx tmp = gen_reg_rtx (<MODE>mode);
3554 emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3555 emit_insn (gen_neon_vmls_lane<mode> (operands[0], operands[1], operands[2],
3556 tmp, const0_rtx, operands[4]));
3557 DONE;
3558 })
3560 (define_expand "neon_vmls_n<mode>"
3561 [(match_operand:VMQ 0 "s_register_operand" "")
3562 (match_operand:VMQ 1 "s_register_operand" "")
3563 (match_operand:VMQ 2 "s_register_operand" "")
3564 (match_operand:<V_elem> 3 "s_register_operand" "")
3565 (match_operand:SI 4 "immediate_operand" "")]
3566 "TARGET_NEON"
3567 {
3568 rtx tmp = gen_reg_rtx (<V_HALF>mode);
3569 emit_insn (gen_neon_vset_lane<V_half> (tmp, operands[3], tmp, const0_rtx));
3570 emit_insn (gen_neon_vmls_lane<mode> (operands[0], operands[1], operands[2],
3571 tmp, const0_rtx, operands[4]));
3572 DONE;
3573 })
3575 (define_expand "neon_vmlsl_n<mode>"
3576 [(match_operand:<V_widen> 0 "s_register_operand" "")
3577 (match_operand:<V_widen> 1 "s_register_operand" "")
3578 (match_operand:VMDI 2 "s_register_operand" "")
3579 (match_operand:<V_elem> 3 "s_register_operand" "")
3580 (match_operand:SI 4 "immediate_operand" "")]
3581 "TARGET_NEON"
3582 {
3583 rtx tmp = gen_reg_rtx (<MODE>mode);
3584 emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3585 emit_insn (gen_neon_vmlsl_lane<mode> (operands[0], operands[1], operands[2],
3586 tmp, const0_rtx, operands[4]));
3587 DONE;
3588 })
3590 (define_expand "neon_vqdmlsl_n<mode>"
3591 [(match_operand:<V_widen> 0 "s_register_operand" "")
3592 (match_operand:<V_widen> 1 "s_register_operand" "")
3593 (match_operand:VMDI 2 "s_register_operand" "")
3594 (match_operand:<V_elem> 3 "s_register_operand" "")
3595 (match_operand:SI 4 "immediate_operand" "")]
3596 "TARGET_NEON"
3597 {
3598 rtx tmp = gen_reg_rtx (<MODE>mode);
3599 emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3600 emit_insn (gen_neon_vqdmlsl_lane<mode> (operands[0], operands[1], operands[2],
3601 tmp, const0_rtx, operands[4]));
3602 DONE;
3603 })
3605 (define_insn "neon_vext<mode>"
3606 [(set (match_operand:VDQX 0 "s_register_operand" "=w")
3607 (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand" "w")
3608 (match_operand:VDQX 2 "s_register_operand" "w")
3609 (match_operand:SI 3 "immediate_operand" "i")]
3610 UNSPEC_VEXT))]
3611 "TARGET_NEON"
3612 {
3613 neon_const_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3614 return "vext.<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2, %3";
3615 }
3616 [(set (attr "neon_type")
3617 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
3618 (const_string "neon_bp_simple")
3619 (const_string "neon_bp_2cycle")))]
3620 )
3622 (define_insn "neon_vrev64<mode>"
3623 [(set (match_operand:VDQ 0 "s_register_operand" "=w")
3624 (unspec:VDQ [(match_operand:VDQ 1 "s_register_operand" "w")
3625 (match_operand:SI 2 "immediate_operand" "i")]
3626 UNSPEC_VREV64))]
3627 "TARGET_NEON"
3628 "vrev64.<V_sz_elem>\t%<V_reg>0, %<V_reg>1"
3629 [(set_attr "neon_type" "neon_bp_simple")]
3630 )
3632 (define_insn "neon_vrev32<mode>"
3633 [(set (match_operand:VX 0 "s_register_operand" "=w")
3634 (unspec:VX [(match_operand:VX 1 "s_register_operand" "w")
3635 (match_operand:SI 2 "immediate_operand" "i")]
3636 UNSPEC_VREV32))]
3637 "TARGET_NEON"
3638 "vrev32.<V_sz_elem>\t%<V_reg>0, %<V_reg>1"
3639 [(set_attr "neon_type" "neon_bp_simple")]
3640 )
3642 (define_insn "neon_vrev16<mode>"
3643 [(set (match_operand:VE 0 "s_register_operand" "=w")
3644 (unspec:VE [(match_operand:VE 1 "s_register_operand" "w")
3645 (match_operand:SI 2 "immediate_operand" "i")]
3646 UNSPEC_VREV16))]
3647 "TARGET_NEON"
3648 "vrev16.<V_sz_elem>\t%<V_reg>0, %<V_reg>1"
3649 [(set_attr "neon_type" "neon_bp_simple")]
3650 )
3652 ; vbsl_* intrinsics may compile to any of vbsl/vbif/vbit depending on register
3653 ; allocation. For an intrinsic of form:
3654 ; rD = vbsl_* (rS, rN, rM)
3655 ; We can use any of:
3656 ; vbsl rS, rN, rM (if D = S)
3657 ; vbit rD, rN, rS (if D = M, so 1-bits in rS choose bits from rN, else rM)
3658 ; vbif rD, rM, rS (if D = N, so 0-bits in rS choose bits from rM, else rN)
3660 (define_insn "neon_vbsl<mode>_internal"
3661 [(set (match_operand:VDQX 0 "s_register_operand" "=w,w,w")
3662 (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand" " 0,w,w")
3663 (match_operand:VDQX 2 "s_register_operand" " w,w,0")
3664 (match_operand:VDQX 3 "s_register_operand" " w,0,w")]
3665 UNSPEC_VBSL))]
3666 "TARGET_NEON"
3667 "@
3668 vbsl\t%<V_reg>0, %<V_reg>2, %<V_reg>3
3669 vbit\t%<V_reg>0, %<V_reg>2, %<V_reg>1
3670 vbif\t%<V_reg>0, %<V_reg>3, %<V_reg>1"
3671 [(set_attr "neon_type" "neon_int_1")]
3672 )
3674 (define_expand "neon_vbsl<mode>"
3675 [(set (match_operand:VDQX 0 "s_register_operand" "")
3676 (unspec:VDQX [(match_operand:<V_cmp_result> 1 "s_register_operand" "")
3677 (match_operand:VDQX 2 "s_register_operand" "")
3678 (match_operand:VDQX 3 "s_register_operand" "")]
3679 UNSPEC_VBSL))]
3680 "TARGET_NEON"
3681 {
3682 /* We can't alias operands together if they have different modes. */
3683 operands[1] = gen_lowpart (<MODE>mode, operands[1]);
3684 })
3686 (define_insn "neon_vshl<mode>"
3687 [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3688 (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
3689 (match_operand:VDQIX 2 "s_register_operand" "w")
3690 (match_operand:SI 3 "immediate_operand" "i")]
3691 UNSPEC_VSHL))]
3692 "TARGET_NEON"
3693 "v%O3shl.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
3694 [(set (attr "neon_type")
3695 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
3696 (const_string "neon_vshl_ddd")
3697 (const_string "neon_shift_3")))]
3698 )
3700 (define_insn "neon_vqshl<mode>"
3701 [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3702 (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
3703 (match_operand:VDQIX 2 "s_register_operand" "w")
3704 (match_operand:SI 3 "immediate_operand" "i")]
3705 UNSPEC_VQSHL))]
3706 "TARGET_NEON"
3707 "vq%O3shl.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
3708 [(set (attr "neon_type")
3709 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
3710 (const_string "neon_shift_2")
3711 (const_string "neon_vqshl_vrshl_vqrshl_qqq")))]
3712 )
3714 (define_insn "neon_vshr_n<mode>"
3715 [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3716 (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
3717 (match_operand:SI 2 "immediate_operand" "i")
3718 (match_operand:SI 3 "immediate_operand" "i")]
3719 UNSPEC_VSHR_N))]
3720 "TARGET_NEON"
3721 {
3722 neon_const_bounds (operands[2], 1, neon_element_bits (<MODE>mode) + 1);
3723 return "v%O3shr.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %2";
3724 }
3725 [(set_attr "neon_type" "neon_shift_1")]
3726 )
3728 (define_insn "neon_vshrn_n<mode>"
3729 [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
3730 (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
3731 (match_operand:SI 2 "immediate_operand" "i")
3732 (match_operand:SI 3 "immediate_operand" "i")]
3733 UNSPEC_VSHRN_N))]
3734 "TARGET_NEON"
3735 {
3736 neon_const_bounds (operands[2], 1, neon_element_bits (<MODE>mode) / 2 + 1);
3737 return "v%O3shrn.<V_if_elem>\t%P0, %q1, %2";
3738 }
3739 [(set_attr "neon_type" "neon_shift_1")]
3740 )
3742 (define_insn "neon_vqshrn_n<mode>"
3743 [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
3744 (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
3745 (match_operand:SI 2 "immediate_operand" "i")
3746 (match_operand:SI 3 "immediate_operand" "i")]
3747 UNSPEC_VQSHRN_N))]
3748 "TARGET_NEON"
3749 {
3750 neon_const_bounds (operands[2], 1, neon_element_bits (<MODE>mode) / 2 + 1);
3751 return "vq%O3shrn.%T3%#<V_sz_elem>\t%P0, %q1, %2";
3752 }
3753 [(set_attr "neon_type" "neon_shift_2")]
3754 )
3756 (define_insn "neon_vqshrun_n<mode>"
3757 [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
3758 (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
3759 (match_operand:SI 2 "immediate_operand" "i")
3760 (match_operand:SI 3 "immediate_operand" "i")]
3761 UNSPEC_VQSHRUN_N))]
3762 "TARGET_NEON"
3763 {
3764 neon_const_bounds (operands[2], 1, neon_element_bits (<MODE>mode) / 2 + 1);
3765 return "vq%O3shrun.%T3%#<V_sz_elem>\t%P0, %q1, %2";
3766 }
3767 [(set_attr "neon_type" "neon_shift_2")]
3768 )
3770 (define_insn "neon_vshl_n<mode>"
3771 [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3772 (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
3773 (match_operand:SI 2 "immediate_operand" "i")
3774 (match_operand:SI 3 "immediate_operand" "i")]
3775 UNSPEC_VSHL_N))]
3776 "TARGET_NEON"
3777 {
3778 neon_const_bounds (operands[2], 0, neon_element_bits (<MODE>mode));
3779 return "vshl.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %2";
3780 }
3781 [(set_attr "neon_type" "neon_shift_1")]
3782 )
3784 (define_insn "neon_vqshl_n<mode>"
3785 [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3786 (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
3787 (match_operand:SI 2 "immediate_operand" "i")
3788 (match_operand:SI 3 "immediate_operand" "i")]
3789 UNSPEC_VQSHL_N))]
3790 "TARGET_NEON"
3791 {
3792 neon_const_bounds (operands[2], 0, neon_element_bits (<MODE>mode));
3793 return "vqshl.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %2";
3794 }
3795 [(set_attr "neon_type" "neon_shift_2")]
3796 )
3798 (define_insn "neon_vqshlu_n<mode>"
3799 [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3800 (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
3801 (match_operand:SI 2 "immediate_operand" "i")
3802 (match_operand:SI 3 "immediate_operand" "i")]
3803 UNSPEC_VQSHLU_N))]
3804 "TARGET_NEON"
3805 {
3806 neon_const_bounds (operands[2], 0, neon_element_bits (<MODE>mode));
3807 return "vqshlu.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %2";
3808 }
3809 [(set_attr "neon_type" "neon_shift_2")]
3810 )
3812 (define_insn "neon_vshll_n<mode>"
3813 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3814 (unspec:<V_widen> [(match_operand:VW 1 "s_register_operand" "w")
3815 (match_operand:SI 2 "immediate_operand" "i")
3816 (match_operand:SI 3 "immediate_operand" "i")]
3817 UNSPEC_VSHLL_N))]
3818 "TARGET_NEON"
3819 {
3820 /* The boundaries are: 0 < imm <= size. */
3821 neon_const_bounds (operands[2], 0, neon_element_bits (<MODE>mode) + 1);
3822 return "vshll.%T3%#<V_sz_elem>\t%q0, %P1, %2";
3823 }
3824 [(set_attr "neon_type" "neon_shift_1")]
3825 )
3827 (define_insn "neon_vsra_n<mode>"
3828 [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3829 (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "0")
3830 (match_operand:VDQIX 2 "s_register_operand" "w")
3831 (match_operand:SI 3 "immediate_operand" "i")
3832 (match_operand:SI 4 "immediate_operand" "i")]
3833 UNSPEC_VSRA_N))]
3834 "TARGET_NEON"
3835 {
3836 neon_const_bounds (operands[3], 1, neon_element_bits (<MODE>mode) + 1);
3837 return "v%O4sra.%T4%#<V_sz_elem>\t%<V_reg>0, %<V_reg>2, %3";
3838 }
3839 [(set_attr "neon_type" "neon_vsra_vrsra")]
3840 )
3842 (define_insn "neon_vsri_n<mode>"
3843 [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3844 (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "0")
3845 (match_operand:VDQIX 2 "s_register_operand" "w")
3846 (match_operand:SI 3 "immediate_operand" "i")]
3847 UNSPEC_VSRI))]
3848 "TARGET_NEON"
3849 {
3850 neon_const_bounds (operands[3], 1, neon_element_bits (<MODE>mode) + 1);
3851 return "vsri.<V_sz_elem>\t%<V_reg>0, %<V_reg>2, %3";
3852 }
3853 [(set (attr "neon_type")
3854 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
3855 (const_string "neon_shift_1")
3856 (const_string "neon_shift_3")))]
3857 )
3859 (define_insn "neon_vsli_n<mode>"
3860 [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3861 (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "0")
3862 (match_operand:VDQIX 2 "s_register_operand" "w")
3863 (match_operand:SI 3 "immediate_operand" "i")]
3864 UNSPEC_VSLI))]
3865 "TARGET_NEON"
3866 {
3867 neon_const_bounds (operands[3], 0, neon_element_bits (<MODE>mode));
3868 return "vsli.<V_sz_elem>\t%<V_reg>0, %<V_reg>2, %3";
3869 }
3870 [(set (attr "neon_type")
3871 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
3872 (const_string "neon_shift_1")
3873 (const_string "neon_shift_3")))]
3874 )
3876 (define_insn "neon_vtbl1v8qi"
3877 [(set (match_operand:V8QI 0 "s_register_operand" "=w")
3878 (unspec:V8QI [(match_operand:V8QI 1 "s_register_operand" "w")
3879 (match_operand:V8QI 2 "s_register_operand" "w")]
3880 UNSPEC_VTBL))]
3881 "TARGET_NEON"
3882 "vtbl.8\t%P0, {%P1}, %P2"
3883 [(set_attr "neon_type" "neon_bp_2cycle")]
3884 )
3886 (define_insn "neon_vtbl2v8qi"
3887 [(set (match_operand:V8QI 0 "s_register_operand" "=w")
3888 (unspec:V8QI [(match_operand:TI 1 "s_register_operand" "w")
3889 (match_operand:V8QI 2 "s_register_operand" "w")]
3890 UNSPEC_VTBL))]
3891 "TARGET_NEON"
3892 {
3893 rtx ops[4];
3894 int tabbase = REGNO (operands[1]);
3896 ops[0] = operands[0];
3897 ops[1] = gen_rtx_REG (V8QImode, tabbase);
3898 ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
3899 ops[3] = operands[2];
3900 output_asm_insn ("vtbl.8\t%P0, {%P1, %P2}, %P3", ops);
3902 return "";
3903 }
3904 [(set_attr "neon_type" "neon_bp_2cycle")]
3905 )
3907 (define_insn "neon_vtbl3v8qi"
3908 [(set (match_operand:V8QI 0 "s_register_operand" "=w")
3909 (unspec:V8QI [(match_operand:EI 1 "s_register_operand" "w")
3910 (match_operand:V8QI 2 "s_register_operand" "w")]
3911 UNSPEC_VTBL))]
3912 "TARGET_NEON"
3913 {
3914 rtx ops[5];
3915 int tabbase = REGNO (operands[1]);
3917 ops[0] = operands[0];
3918 ops[1] = gen_rtx_REG (V8QImode, tabbase);
3919 ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
3920 ops[3] = gen_rtx_REG (V8QImode, tabbase + 4);
3921 ops[4] = operands[2];
3922 output_asm_insn ("vtbl.8\t%P0, {%P1, %P2, %P3}, %P4", ops);
3924 return "";
3925 }
3926 [(set_attr "neon_type" "neon_bp_3cycle")]
3927 )
3929 (define_insn "neon_vtbl4v8qi"
3930 [(set (match_operand:V8QI 0 "s_register_operand" "=w")
3931 (unspec:V8QI [(match_operand:OI 1 "s_register_operand" "w")
3932 (match_operand:V8QI 2 "s_register_operand" "w")]
3933 UNSPEC_VTBL))]
3934 "TARGET_NEON"
3935 {
3936 rtx ops[6];
3937 int tabbase = REGNO (operands[1]);
3939 ops[0] = operands[0];
3940 ops[1] = gen_rtx_REG (V8QImode, tabbase);
3941 ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
3942 ops[3] = gen_rtx_REG (V8QImode, tabbase + 4);
3943 ops[4] = gen_rtx_REG (V8QImode, tabbase + 6);
3944 ops[5] = operands[2];
3945 output_asm_insn ("vtbl.8\t%P0, {%P1, %P2, %P3, %P4}, %P5", ops);
3947 return "";
3948 }
3949 [(set_attr "neon_type" "neon_bp_3cycle")]
3950 )
3952 (define_insn "neon_vtbx1v8qi"
3953 [(set (match_operand:V8QI 0 "s_register_operand" "=w")
3954 (unspec:V8QI [(match_operand:V8QI 1 "s_register_operand" "0")
3955 (match_operand:V8QI 2 "s_register_operand" "w")
3956 (match_operand:V8QI 3 "s_register_operand" "w")]
3957 UNSPEC_VTBX))]
3958 "TARGET_NEON"
3959 "vtbx.8\t%P0, {%P2}, %P3"
3960 [(set_attr "neon_type" "neon_bp_2cycle")]
3961 )
3963 (define_insn "neon_vtbx2v8qi"
3964 [(set (match_operand:V8QI 0 "s_register_operand" "=w")
3965 (unspec:V8QI [(match_operand:V8QI 1 "s_register_operand" "0")
3966 (match_operand:TI 2 "s_register_operand" "w")
3967 (match_operand:V8QI 3 "s_register_operand" "w")]
3968 UNSPEC_VTBX))]
3969 "TARGET_NEON"
3970 {
3971 rtx ops[4];
3972 int tabbase = REGNO (operands[2]);
3974 ops[0] = operands[0];
3975 ops[1] = gen_rtx_REG (V8QImode, tabbase);
3976 ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
3977 ops[3] = operands[3];
3978 output_asm_insn ("vtbx.8\t%P0, {%P1, %P2}, %P3", ops);
3980 return "";
3981 }
3982 [(set_attr "neon_type" "neon_bp_2cycle")]
3983 )
3985 (define_insn "neon_vtbx3v8qi"
3986 [(set (match_operand:V8QI 0 "s_register_operand" "=w")
3987 (unspec:V8QI [(match_operand:V8QI 1 "s_register_operand" "0")
3988 (match_operand:EI 2 "s_register_operand" "w")
3989 (match_operand:V8QI 3 "s_register_operand" "w")]
3990 UNSPEC_VTBX))]
3991 "TARGET_NEON"
3992 {
3993 rtx ops[5];
3994 int tabbase = REGNO (operands[2]);
3996 ops[0] = operands[0];
3997 ops[1] = gen_rtx_REG (V8QImode, tabbase);
3998 ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
3999 ops[3] = gen_rtx_REG (V8QImode, tabbase + 4);
4000 ops[4] = operands[3];
4001 output_asm_insn ("vtbx.8\t%P0, {%P1, %P2, %P3}, %P4", ops);
4003 return "";
4004 }
4005 [(set_attr "neon_type" "neon_bp_3cycle")]
4006 )
4008 (define_insn "neon_vtbx4v8qi"
4009 [(set (match_operand:V8QI 0 "s_register_operand" "=w")
4010 (unspec:V8QI [(match_operand:V8QI 1 "s_register_operand" "0")
4011 (match_operand:OI 2 "s_register_operand" "w")
4012 (match_operand:V8QI 3 "s_register_operand" "w")]
4013 UNSPEC_VTBX))]
4014 "TARGET_NEON"
4015 {
4016 rtx ops[6];
4017 int tabbase = REGNO (operands[2]);
4019 ops[0] = operands[0];
4020 ops[1] = gen_rtx_REG (V8QImode, tabbase);
4021 ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
4022 ops[3] = gen_rtx_REG (V8QImode, tabbase + 4);
4023 ops[4] = gen_rtx_REG (V8QImode, tabbase + 6);
4024 ops[5] = operands[3];
4025 output_asm_insn ("vtbx.8\t%P0, {%P1, %P2, %P3, %P4}, %P5", ops);
4027 return "";
4028 }
4029 [(set_attr "neon_type" "neon_bp_3cycle")]
4030 )
4032 (define_insn "neon_vtrn<mode>_internal"
4033 [(set (match_operand:VDQW 0 "s_register_operand" "=w")
4034 (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "0")]
4035 UNSPEC_VTRN1))
4036 (set (match_operand:VDQW 2 "s_register_operand" "=w")
4037 (unspec:VDQW [(match_operand:VDQW 3 "s_register_operand" "2")]
4038 UNSPEC_VTRN2))]
4039 "TARGET_NEON"
4040 "vtrn.<V_sz_elem>\t%<V_reg>0, %<V_reg>2"
4041 [(set (attr "neon_type")
4042 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
4043 (const_string "neon_bp_simple")
4044 (const_string "neon_bp_3cycle")))]
4045 )
4047 (define_expand "neon_vtrn<mode>"
4048 [(match_operand:SI 0 "s_register_operand" "r")
4049 (match_operand:VDQW 1 "s_register_operand" "w")
4050 (match_operand:VDQW 2 "s_register_operand" "w")]
4051 "TARGET_NEON"
4052 {
4053 neon_emit_pair_result_insn (<MODE>mode, gen_neon_vtrn<mode>_internal,
4054 operands[0], operands[1], operands[2]);
4055 DONE;
4056 })
4058 (define_insn "neon_vzip<mode>_internal"
4059 [(set (match_operand:VDQW 0 "s_register_operand" "=w")
4060 (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "0")]
4061 UNSPEC_VZIP1))
4062 (set (match_operand:VDQW 2 "s_register_operand" "=w")
4063 (unspec:VDQW [(match_operand:VDQW 3 "s_register_operand" "2")]
4064 UNSPEC_VZIP2))]
4065 "TARGET_NEON"
4066 "vzip.<V_sz_elem>\t%<V_reg>0, %<V_reg>2"
4067 [(set (attr "neon_type")
4068 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
4069 (const_string "neon_bp_simple")
4070 (const_string "neon_bp_3cycle")))]
4071 )
4073 (define_expand "neon_vzip<mode>"
4074 [(match_operand:SI 0 "s_register_operand" "r")
4075 (match_operand:VDQW 1 "s_register_operand" "w")
4076 (match_operand:VDQW 2 "s_register_operand" "w")]
4077 "TARGET_NEON"
4078 {
4079 neon_emit_pair_result_insn (<MODE>mode, gen_neon_vzip<mode>_internal,
4080 operands[0], operands[1], operands[2]);
4081 DONE;
4082 })
4084 (define_insn "neon_vuzp<mode>_internal"
4085 [(set (match_operand:VDQW 0 "s_register_operand" "=w")
4086 (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "0")]
4087 UNSPEC_VUZP1))
4088 (set (match_operand:VDQW 2 "s_register_operand" "=w")
4089 (unspec:VDQW [(match_operand:VDQW 3 "s_register_operand" "2")]
4090 UNSPEC_VUZP2))]
4091 "TARGET_NEON"
4092 "vuzp.<V_sz_elem>\t%<V_reg>0, %<V_reg>2"
4093 [(set (attr "neon_type")
4094 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
4095 (const_string "neon_bp_simple")
4096 (const_string "neon_bp_3cycle")))]
4097 )
4099 (define_expand "neon_vuzp<mode>"
4100 [(match_operand:SI 0 "s_register_operand" "r")
4101 (match_operand:VDQW 1 "s_register_operand" "w")
4102 (match_operand:VDQW 2 "s_register_operand" "w")]
4103 "TARGET_NEON"
4104 {
4105 neon_emit_pair_result_insn (<MODE>mode, gen_neon_vuzp<mode>_internal,
4106 operands[0], operands[1], operands[2]);
4107 DONE;
4108 })
4110 (define_expand "neon_vreinterpretv8qi<mode>"
4111 [(match_operand:V8QI 0 "s_register_operand" "")
4112 (match_operand:VDX 1 "s_register_operand" "")]
4113 "TARGET_NEON"
4114 {
4115 neon_reinterpret (operands[0], operands[1]);
4116 DONE;
4117 })
4119 (define_expand "neon_vreinterpretv4hi<mode>"
4120 [(match_operand:V4HI 0 "s_register_operand" "")
4121 (match_operand:VDX 1 "s_register_operand" "")]
4122 "TARGET_NEON"
4123 {
4124 neon_reinterpret (operands[0], operands[1]);
4125 DONE;
4126 })
4128 (define_expand "neon_vreinterpretv2si<mode>"
4129 [(match_operand:V2SI 0 "s_register_operand" "")
4130 (match_operand:VDX 1 "s_register_operand" "")]
4131 "TARGET_NEON"
4132 {
4133 neon_reinterpret (operands[0], operands[1]);
4134 DONE;
4135 })
4137 (define_expand "neon_vreinterpretv2sf<mode>"
4138 [(match_operand:V2SF 0 "s_register_operand" "")
4139 (match_operand:VDX 1 "s_register_operand" "")]
4140 "TARGET_NEON"
4141 {
4142 neon_reinterpret (operands[0], operands[1]);
4143 DONE;
4144 })
4146 (define_expand "neon_vreinterpretdi<mode>"
4147 [(match_operand:DI 0 "s_register_operand" "")
4148 (match_operand:VDX 1 "s_register_operand" "")]
4149 "TARGET_NEON"
4150 {
4151 neon_reinterpret (operands[0], operands[1]);
4152 DONE;
4153 })
4155 (define_expand "neon_vreinterpretv16qi<mode>"
4156 [(match_operand:V16QI 0 "s_register_operand" "")
4157 (match_operand:VQX 1 "s_register_operand" "")]
4158 "TARGET_NEON"
4159 {
4160 neon_reinterpret (operands[0], operands[1]);
4161 DONE;
4162 })
4164 (define_expand "neon_vreinterpretv8hi<mode>"
4165 [(match_operand:V8HI 0 "s_register_operand" "")
4166 (match_operand:VQX 1 "s_register_operand" "")]
4167 "TARGET_NEON"
4168 {
4169 neon_reinterpret (operands[0], operands[1]);
4170 DONE;
4171 })
4173 (define_expand "neon_vreinterpretv4si<mode>"
4174 [(match_operand:V4SI 0 "s_register_operand" "")
4175 (match_operand:VQX 1 "s_register_operand" "")]
4176 "TARGET_NEON"
4177 {
4178 neon_reinterpret (operands[0], operands[1]);
4179 DONE;
4180 })
4182 (define_expand "neon_vreinterpretv4sf<mode>"
4183 [(match_operand:V4SF 0 "s_register_operand" "")
4184 (match_operand:VQX 1 "s_register_operand" "")]
4185 "TARGET_NEON"
4186 {
4187 neon_reinterpret (operands[0], operands[1]);
4188 DONE;
4189 })
4191 (define_expand "neon_vreinterpretv2di<mode>"
4192 [(match_operand:V2DI 0 "s_register_operand" "")
4193 (match_operand:VQX 1 "s_register_operand" "")]
4194 "TARGET_NEON"
4195 {
4196 neon_reinterpret (operands[0], operands[1]);
4197 DONE;
4198 })
4200 (define_insn "neon_vld1<mode>"
4201 [(set (match_operand:VDQX 0 "s_register_operand" "=w")
4202 (unspec:VDQX [(mem:VDQX (match_operand:SI 1 "s_register_operand" "r"))]
4203 UNSPEC_VLD1))]
4204 "TARGET_NEON"
4205 "vld1.<V_sz_elem>\t%h0, [%1]"
4206 [(set_attr "neon_type" "neon_vld1_1_2_regs")]
4207 )
4209 (define_insn "neon_vld1_lane<mode>"
4210 [(set (match_operand:VDX 0 "s_register_operand" "=w")
4211 (unspec:VDX [(mem:<V_elem> (match_operand:SI 1 "s_register_operand" "r"))
4212 (match_operand:VDX 2 "s_register_operand" "0")
4213 (match_operand:SI 3 "immediate_operand" "i")]
4214 UNSPEC_VLD1_LANE))]
4215 "TARGET_NEON"
4216 {
4217 HOST_WIDE_INT lane = INTVAL (operands[3]);
4218 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4219 if (lane < 0 || lane >= max)
4220 error ("lane out of range");
4221 if (max == 1)
4222 return "vld1.<V_sz_elem>\t%P0, [%1]";
4223 else
4224 return "vld1.<V_sz_elem>\t{%P0[%c3]}, [%1]";
4225 }
4226 [(set (attr "neon_type")
4227 (if_then_else (eq (const_string "<V_mode_nunits>") (const_int 2))
4228 (const_string "neon_vld1_1_2_regs")
4229 (const_string "neon_vld1_vld2_lane")))]
4230 )
4232 (define_insn "neon_vld1_lane<mode>"
4233 [(set (match_operand:VQX 0 "s_register_operand" "=w")
4234 (unspec:VQX [(mem:<V_elem> (match_operand:SI 1 "s_register_operand" "r"))
4235 (match_operand:VQX 2 "s_register_operand" "0")
4236 (match_operand:SI 3 "immediate_operand" "i")]
4237 UNSPEC_VLD1_LANE))]
4238 "TARGET_NEON"
4239 {
4240 HOST_WIDE_INT lane = INTVAL (operands[3]);
4241 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4242 int regno = REGNO (operands[0]);
4243 if (lane < 0 || lane >= max)
4244 error ("lane out of range");
4245 else if (lane >= max / 2)
4246 {
4247 lane -= max / 2;
4248 regno += 2;
4249 operands[3] = GEN_INT (lane);
4250 }
4251 operands[0] = gen_rtx_REG (<V_HALF>mode, regno);
4252 if (max == 2)
4253 return "vld1.<V_sz_elem>\t%P0, [%1]";
4254 else
4255 return "vld1.<V_sz_elem>\t{%P0[%c3]}, [%1]";
4256 }
4257 [(set (attr "neon_type")
4258 (if_then_else (eq (const_string "<V_mode_nunits>") (const_int 2))
4259 (const_string "neon_vld1_1_2_regs")
4260 (const_string "neon_vld1_vld2_lane")))]
4261 )
4263 (define_insn "neon_vld1_dup<mode>"
4264 [(set (match_operand:VDX 0 "s_register_operand" "=w")
4265 (unspec:VDX [(mem:<V_elem> (match_operand:SI 1 "s_register_operand" "r"))]
4266 UNSPEC_VLD1_DUP))]
4267 "TARGET_NEON"
4268 {
4269 if (GET_MODE_NUNITS (<MODE>mode) > 1)
4270 return "vld1.<V_sz_elem>\t{%P0[]}, [%1]";
4271 else
4272 return "vld1.<V_sz_elem>\t%h0, [%1]";
4273 }
4274 [(set (attr "neon_type")
4275 (if_then_else (gt (const_string "<V_mode_nunits>") (const_string "1"))
4276 (const_string "neon_vld2_2_regs_vld1_vld2_all_lanes")
4277 (const_string "neon_vld1_1_2_regs")))]
4278 )
4280 (define_insn "neon_vld1_dup<mode>"
4281 [(set (match_operand:VQX 0 "s_register_operand" "=w")
4282 (unspec:VQX [(mem:<V_elem> (match_operand:SI 1 "s_register_operand" "r"))]
4283 UNSPEC_VLD1_DUP))]
4284 "TARGET_NEON"
4285 {
4286 if (GET_MODE_NUNITS (<MODE>mode) > 2)
4287 return "vld1.<V_sz_elem>\t{%e0[], %f0[]}, [%1]";
4288 else
4289 return "vld1.<V_sz_elem>\t%h0, [%1]";
4290 }
4291 [(set (attr "neon_type")
4292 (if_then_else (gt (const_string "<V_mode_nunits>") (const_string "1"))
4293 (const_string "neon_vld2_2_regs_vld1_vld2_all_lanes")
4294 (const_string "neon_vld1_1_2_regs")))]
4295 )
4297 (define_insn "neon_vst1<mode>"
4298 [(set (mem:VDQX (match_operand:SI 0 "s_register_operand" "r"))
4299 (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand" "w")]
4300 UNSPEC_VST1))]
4301 "TARGET_NEON"
4302 "vst1.<V_sz_elem>\t%h1, [%0]"
4303 [(set_attr "neon_type" "neon_vst1_1_2_regs_vst2_2_regs")])
4305 (define_insn "neon_vst1_lane<mode>"
4306 [(set (mem:<V_elem> (match_operand:SI 0 "s_register_operand" "r"))
4307 (vec_select:<V_elem>
4308 (match_operand:VDX 1 "s_register_operand" "w")
4309 (parallel [(match_operand:SI 2 "neon_lane_number" "i")])))]
4310 "TARGET_NEON"
4311 {
4312 HOST_WIDE_INT lane = INTVAL (operands[2]);
4313 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4314 if (lane < 0 || lane >= max)
4315 error ("lane out of range");
4316 if (max == 1)
4317 return "vst1.<V_sz_elem>\t{%P1}, [%0]";
4318 else
4319 return "vst1.<V_sz_elem>\t{%P1[%c2]}, [%0]";
4320 }
4321 [(set (attr "neon_type")
4322 (if_then_else (eq (const_string "<V_mode_nunits>") (const_int 1))
4323 (const_string "neon_vst1_1_2_regs_vst2_2_regs")
4324 (const_string "neon_vst1_vst2_lane")))])
4326 (define_insn "neon_vst1_lane<mode>"
4327 [(set (mem:<V_elem> (match_operand:SI 0 "s_register_operand" "r"))
4328 (vec_select:<V_elem>
4329 (match_operand:VQX 1 "s_register_operand" "w")
4330 (parallel [(match_operand:SI 2 "neon_lane_number" "i")])))]
4331 "TARGET_NEON"
4332 {
4333 HOST_WIDE_INT lane = INTVAL (operands[2]);
4334 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4335 int regno = REGNO (operands[1]);
4336 if (lane < 0 || lane >= max)
4337 error ("lane out of range");
4338 else if (lane >= max / 2)
4339 {
4340 lane -= max / 2;
4341 regno += 2;
4342 operands[2] = GEN_INT (lane);
4343 }
4344 operands[1] = gen_rtx_REG (<V_HALF>mode, regno);
4345 if (max == 2)
4346 return "vst1.<V_sz_elem>\t{%P1}, [%0]";
4347 else
4348 return "vst1.<V_sz_elem>\t{%P1[%c2]}, [%0]";
4349 }
4350 [(set_attr "neon_type" "neon_vst1_vst2_lane")]
4351 )
4353 (define_insn "neon_vld2<mode>"
4354 [(set (match_operand:TI 0 "s_register_operand" "=w")
4355 (unspec:TI [(mem:TI (match_operand:SI 1 "s_register_operand" "r"))
4356 (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4357 UNSPEC_VLD2))]
4358 "TARGET_NEON"
4359 {
4360 if (<V_sz_elem> == 64)
4361 return "vld1.64\t%h0, [%1]";
4362 else
4363 return "vld2.<V_sz_elem>\t%h0, [%1]";
4364 }
4365 [(set (attr "neon_type")
4366 (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
4367 (const_string "neon_vld1_1_2_regs")
4368 (const_string "neon_vld2_2_regs_vld1_vld2_all_lanes")))]
4369 )
4371 (define_insn "neon_vld2<mode>"
4372 [(set (match_operand:OI 0 "s_register_operand" "=w")
4373 (unspec:OI [(mem:OI (match_operand:SI 1 "s_register_operand" "r"))
4374 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4375 UNSPEC_VLD2))]
4376 "TARGET_NEON"
4377 "vld2.<V_sz_elem>\t%h0, [%1]"
4378 [(set_attr "neon_type" "neon_vld2_2_regs_vld1_vld2_all_lanes")])
4380 (define_insn "neon_vld2_lane<mode>"
4381 [(set (match_operand:TI 0 "s_register_operand" "=w")
4382 (unspec:TI [(mem:<V_two_elem> (match_operand:SI 1 "s_register_operand" "r"))
4383 (match_operand:TI 2 "s_register_operand" "0")
4384 (match_operand:SI 3 "immediate_operand" "i")
4385 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4386 UNSPEC_VLD2_LANE))]
4387 "TARGET_NEON"
4388 {
4389 HOST_WIDE_INT lane = INTVAL (operands[3]);
4390 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4391 int regno = REGNO (operands[0]);
4392 rtx ops[4];
4393 if (lane < 0 || lane >= max)
4394 error ("lane out of range");
4395 ops[0] = gen_rtx_REG (DImode, regno);
4396 ops[1] = gen_rtx_REG (DImode, regno + 2);
4397 ops[2] = operands[1];
4398 ops[3] = operands[3];
4399 output_asm_insn ("vld2.<V_sz_elem>\t{%P0[%c3], %P1[%c3]}, [%2]", ops);
4400 return "";
4401 }
4402 [(set_attr "neon_type" "neon_vld1_vld2_lane")]
4403 )
4405 (define_insn "neon_vld2_lane<mode>"
4406 [(set (match_operand:OI 0 "s_register_operand" "=w")
4407 (unspec:OI [(mem:<V_two_elem> (match_operand:SI 1 "s_register_operand" "r"))
4408 (match_operand:OI 2 "s_register_operand" "0")
4409 (match_operand:SI 3 "immediate_operand" "i")
4410 (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4411 UNSPEC_VLD2_LANE))]
4412 "TARGET_NEON"
4413 {
4414 HOST_WIDE_INT lane = INTVAL (operands[3]);
4415 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4416 int regno = REGNO (operands[0]);
4417 rtx ops[4];
4418 if (lane < 0 || lane >= max)
4419 error ("lane out of range");
4420 else if (lane >= max / 2)
4421 {
4422 lane -= max / 2;
4423 regno += 2;
4424 }
4425 ops[0] = gen_rtx_REG (DImode, regno);
4426 ops[1] = gen_rtx_REG (DImode, regno + 4);
4427 ops[2] = operands[1];
4428 ops[3] = GEN_INT (lane);
4429 output_asm_insn ("vld2.<V_sz_elem>\t{%P0[%c3], %P1[%c3]}, [%2]", ops);
4430 return "";
4431 }
4432 [(set_attr "neon_type" "neon_vld1_vld2_lane")]
4433 )
4435 (define_insn "neon_vld2_dup<mode>"
4436 [(set (match_operand:TI 0 "s_register_operand" "=w")
4437 (unspec:TI [(mem:<V_two_elem> (match_operand:SI 1 "s_register_operand" "r"))
4438 (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4439 UNSPEC_VLD2_DUP))]
4440 "TARGET_NEON"
4441 {
4442 if (GET_MODE_NUNITS (<MODE>mode) > 1)
4443 return "vld2.<V_sz_elem>\t{%e0[], %f0[]}, [%1]";
4444 else
4445 return "vld1.<V_sz_elem>\t%h0, [%1]";
4446 }
4447 [(set (attr "neon_type")
4448 (if_then_else (gt (const_string "<V_mode_nunits>") (const_string "1"))
4449 (const_string "neon_vld2_2_regs_vld1_vld2_all_lanes")
4450 (const_string "neon_vld1_1_2_regs")))]
4451 )
4453 (define_insn "neon_vst2<mode>"
4454 [(set (mem:TI (match_operand:SI 0 "s_register_operand" "r"))
4455 (unspec:TI [(match_operand:TI 1 "s_register_operand" "w")
4456 (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4457 UNSPEC_VST2))]
4458 "TARGET_NEON"
4459 {
4460 if (<V_sz_elem> == 64)
4461 return "vst1.64\t%h1, [%0]";
4462 else
4463 return "vst2.<V_sz_elem>\t%h1, [%0]";
4464 }
4465 [(set (attr "neon_type")
4466 (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
4467 (const_string "neon_vst1_1_2_regs_vst2_2_regs")
4468 (const_string "neon_vst1_1_2_regs_vst2_2_regs")))]
4469 )
4471 (define_insn "neon_vst2<mode>"
4472 [(set (mem:OI (match_operand:SI 0 "s_register_operand" "r"))
4473 (unspec:OI [(match_operand:OI 1 "s_register_operand" "w")
4474 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4475 UNSPEC_VST2))]
4476 "TARGET_NEON"
4477 "vst2.<V_sz_elem>\t%h1, [%0]"
4478 [(set_attr "neon_type" "neon_vst1_1_2_regs_vst2_2_regs")]
4479 )
4481 (define_insn "neon_vst2_lane<mode>"
4482 [(set (mem:<V_two_elem> (match_operand:SI 0 "s_register_operand" "r"))
4483 (unspec:<V_two_elem>
4484 [(match_operand:TI 1 "s_register_operand" "w")
4485 (match_operand:SI 2 "immediate_operand" "i")
4486 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4487 UNSPEC_VST2_LANE))]
4488 "TARGET_NEON"
4489 {
4490 HOST_WIDE_INT lane = INTVAL (operands[2]);
4491 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4492 int regno = REGNO (operands[1]);
4493 rtx ops[4];
4494 if (lane < 0 || lane >= max)
4495 error ("lane out of range");
4496 ops[0] = operands[0];
4497 ops[1] = gen_rtx_REG (DImode, regno);
4498 ops[2] = gen_rtx_REG (DImode, regno + 2);
4499 ops[3] = operands[2];
4500 output_asm_insn ("vst2.<V_sz_elem>\t{%P1[%c3], %P2[%c3]}, [%0]", ops);
4501 return "";
4502 }
4503 [(set_attr "neon_type" "neon_vst1_vst2_lane")]
4504 )
4506 (define_insn "neon_vst2_lane<mode>"
4507 [(set (mem:<V_two_elem> (match_operand:SI 0 "s_register_operand" "r"))
4508 (unspec:<V_two_elem>
4509 [(match_operand:OI 1 "s_register_operand" "w")
4510 (match_operand:SI 2 "immediate_operand" "i")
4511 (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4512 UNSPEC_VST2_LANE))]
4513 "TARGET_NEON"
4514 {
4515 HOST_WIDE_INT lane = INTVAL (operands[2]);
4516 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4517 int regno = REGNO (operands[1]);
4518 rtx ops[4];
4519 if (lane < 0 || lane >= max)
4520 error ("lane out of range");
4521 else if (lane >= max / 2)
4522 {
4523 lane -= max / 2;
4524 regno += 2;
4525 }
4526 ops[0] = operands[0];
4527 ops[1] = gen_rtx_REG (DImode, regno);
4528 ops[2] = gen_rtx_REG (DImode, regno + 4);
4529 ops[3] = GEN_INT (lane);
4530 output_asm_insn ("vst2.<V_sz_elem>\t{%P1[%c3], %P2[%c3]}, [%0]", ops);
4531 return "";
4532 }
4533 [(set_attr "neon_type" "neon_vst1_vst2_lane")]
4534 )
4536 (define_insn "neon_vld3<mode>"
4537 [(set (match_operand:EI 0 "s_register_operand" "=w")
4538 (unspec:EI [(mem:EI (match_operand:SI 1 "s_register_operand" "r"))
4539 (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4540 UNSPEC_VLD3))]
4541 "TARGET_NEON"
4542 {
4543 if (<V_sz_elem> == 64)
4544 return "vld1.64\t%h0, [%1]";
4545 else
4546 return "vld3.<V_sz_elem>\t%h0, [%1]";
4547 }
4548 [(set (attr "neon_type")
4549 (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
4550 (const_string "neon_vld1_1_2_regs")
4551 (const_string "neon_vld3_vld4")))]
4552 )
4554 (define_expand "neon_vld3<mode>"
4555 [(match_operand:CI 0 "s_register_operand" "=w")
4556 (match_operand:SI 1 "s_register_operand" "+r")
4557 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4558 "TARGET_NEON"
4559 {
4560 emit_insn (gen_neon_vld3qa<mode> (operands[0], operands[0],
4561 operands[1], operands[1]));
4562 emit_insn (gen_neon_vld3qb<mode> (operands[0], operands[0],
4563 operands[1], operands[1]));
4564 DONE;
4565 })
4567 (define_insn "neon_vld3qa<mode>"
4568 [(set (match_operand:CI 0 "s_register_operand" "=w")
4569 (unspec:CI [(mem:CI (match_operand:SI 3 "s_register_operand" "2"))
4570 (match_operand:CI 1 "s_register_operand" "0")
4571 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4572 UNSPEC_VLD3A))
4573 (set (match_operand:SI 2 "s_register_operand" "=r")
4574 (plus:SI (match_dup 3)
4575 (const_int 24)))]
4576 "TARGET_NEON"
4577 {
4578 int regno = REGNO (operands[0]);
4579 rtx ops[4];
4580 ops[0] = gen_rtx_REG (DImode, regno);
4581 ops[1] = gen_rtx_REG (DImode, regno + 4);
4582 ops[2] = gen_rtx_REG (DImode, regno + 8);
4583 ops[3] = operands[2];
4584 output_asm_insn ("vld3.<V_sz_elem>\t{%P0, %P1, %P2}, [%3]!", ops);
4585 return "";
4586 }
4587 [(set_attr "neon_type" "neon_vld3_vld4")]
4588 )
4590 (define_insn "neon_vld3qb<mode>"
4591 [(set (match_operand:CI 0 "s_register_operand" "=w")
4592 (unspec:CI [(mem:CI (match_operand:SI 3 "s_register_operand" "2"))
4593 (match_operand:CI 1 "s_register_operand" "0")
4594 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4595 UNSPEC_VLD3B))
4596 (set (match_operand:SI 2 "s_register_operand" "=r")
4597 (plus:SI (match_dup 3)
4598 (const_int 24)))]
4599 "TARGET_NEON"
4600 {
4601 int regno = REGNO (operands[0]);
4602 rtx ops[4];
4603 ops[0] = gen_rtx_REG (DImode, regno + 2);
4604 ops[1] = gen_rtx_REG (DImode, regno + 6);
4605 ops[2] = gen_rtx_REG (DImode, regno + 10);
4606 ops[3] = operands[2];
4607 output_asm_insn ("vld3.<V_sz_elem>\t{%P0, %P1, %P2}, [%3]!", ops);
4608 return "";
4609 }
4610 [(set_attr "neon_type" "neon_vld3_vld4")]
4611 )
4613 (define_insn "neon_vld3_lane<mode>"
4614 [(set (match_operand:EI 0 "s_register_operand" "=w")
4615 (unspec:EI [(mem:<V_three_elem> (match_operand:SI 1 "s_register_operand" "r"))
4616 (match_operand:EI 2 "s_register_operand" "0")
4617 (match_operand:SI 3 "immediate_operand" "i")
4618 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4619 UNSPEC_VLD3_LANE))]
4620 "TARGET_NEON"
4621 {
4622 HOST_WIDE_INT lane = INTVAL (operands[3]);
4623 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4624 int regno = REGNO (operands[0]);
4625 rtx ops[5];
4626 if (lane < 0 || lane >= max)
4627 error ("lane out of range");
4628 ops[0] = gen_rtx_REG (DImode, regno);
4629 ops[1] = gen_rtx_REG (DImode, regno + 2);
4630 ops[2] = gen_rtx_REG (DImode, regno + 4);
4631 ops[3] = operands[1];
4632 ops[4] = operands[3];
4633 output_asm_insn ("vld3.<V_sz_elem>\t{%P0[%c4], %P1[%c4], %P2[%c4]}, [%3]",
4634 ops);
4635 return "";
4636 }
4637 [(set_attr "neon_type" "neon_vld3_vld4_lane")]
4638 )
4640 (define_insn "neon_vld3_lane<mode>"
4641 [(set (match_operand:CI 0 "s_register_operand" "=w")
4642 (unspec:CI [(mem:<V_three_elem> (match_operand:SI 1 "s_register_operand" "r"))
4643 (match_operand:CI 2 "s_register_operand" "0")
4644 (match_operand:SI 3 "immediate_operand" "i")
4645 (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4646 UNSPEC_VLD3_LANE))]
4647 "TARGET_NEON"
4648 {
4649 HOST_WIDE_INT lane = INTVAL (operands[3]);
4650 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4651 int regno = REGNO (operands[0]);
4652 rtx ops[5];
4653 if (lane < 0 || lane >= max)
4654 error ("lane out of range");
4655 else if (lane >= max / 2)
4656 {
4657 lane -= max / 2;
4658 regno += 2;
4659 }
4660 ops[0] = gen_rtx_REG (DImode, regno);
4661 ops[1] = gen_rtx_REG (DImode, regno + 4);
4662 ops[2] = gen_rtx_REG (DImode, regno + 8);
4663 ops[3] = operands[1];
4664 ops[4] = GEN_INT (lane);
4665 output_asm_insn ("vld3.<V_sz_elem>\t{%P0[%c4], %P1[%c4], %P2[%c4]}, [%3]",
4666 ops);
4667 return "";
4668 }
4669 [(set_attr "neon_type" "neon_vld3_vld4_lane")]
4670 )
4672 (define_insn "neon_vld3_dup<mode>"
4673 [(set (match_operand:EI 0 "s_register_operand" "=w")
4674 (unspec:EI [(mem:<V_three_elem> (match_operand:SI 1 "s_register_operand" "r"))
4675 (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4676 UNSPEC_VLD3_DUP))]
4677 "TARGET_NEON"
4678 {
4679 if (GET_MODE_NUNITS (<MODE>mode) > 1)
4680 {
4681 int regno = REGNO (operands[0]);
4682 rtx ops[4];
4683 ops[0] = gen_rtx_REG (DImode, regno);
4684 ops[1] = gen_rtx_REG (DImode, regno + 2);
4685 ops[2] = gen_rtx_REG (DImode, regno + 4);
4686 ops[3] = operands[1];
4687 output_asm_insn ("vld3.<V_sz_elem>\t{%P0[], %P1[], %P2[]}, [%3]", ops);
4688 return "";
4689 }
4690 else
4691 return "vld1.<V_sz_elem>\t%h0, [%1]";
4692 }
4693 [(set (attr "neon_type")
4694 (if_then_else (gt (const_string "<V_mode_nunits>") (const_string "1"))
4695 (const_string "neon_vld3_vld4_all_lanes")
4696 (const_string "neon_vld1_1_2_regs")))])
4698 (define_insn "neon_vst3<mode>"
4699 [(set (mem:EI (match_operand:SI 0 "s_register_operand" "r"))
4700 (unspec:EI [(match_operand:EI 1 "s_register_operand" "w")
4701 (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4702 UNSPEC_VST3))]
4703 "TARGET_NEON"
4704 {
4705 if (<V_sz_elem> == 64)
4706 return "vst1.64\t%h1, [%0]";
4707 else
4708 return "vst3.<V_sz_elem>\t%h1, [%0]";
4709 }
4710 [(set (attr "neon_type")
4711 (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
4712 (const_string "neon_vst1_1_2_regs_vst2_2_regs")
4713 (const_string "neon_vst2_4_regs_vst3_vst4")))])
4715 (define_expand "neon_vst3<mode>"
4716 [(match_operand:SI 0 "s_register_operand" "+r")
4717 (match_operand:CI 1 "s_register_operand" "w")
4718 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4719 "TARGET_NEON"
4720 {
4721 emit_insn (gen_neon_vst3qa<mode> (operands[0], operands[0], operands[1]));
4722 emit_insn (gen_neon_vst3qb<mode> (operands[0], operands[0], operands[1]));
4723 DONE;
4724 })
4726 (define_insn "neon_vst3qa<mode>"
4727 [(set (mem:EI (match_operand:SI 1 "s_register_operand" "0"))
4728 (unspec:EI [(match_operand:CI 2 "s_register_operand" "w")
4729 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4730 UNSPEC_VST3A))
4731 (set (match_operand:SI 0 "s_register_operand" "=r")
4732 (plus:SI (match_dup 1)
4733 (const_int 24)))]
4734 "TARGET_NEON"
4735 {
4736 int regno = REGNO (operands[2]);
4737 rtx ops[4];
4738 ops[0] = operands[0];
4739 ops[1] = gen_rtx_REG (DImode, regno);
4740 ops[2] = gen_rtx_REG (DImode, regno + 4);
4741 ops[3] = gen_rtx_REG (DImode, regno + 8);
4742 output_asm_insn ("vst3.<V_sz_elem>\t{%P1, %P2, %P3}, [%0]!", ops);
4743 return "";
4744 }
4745 [(set_attr "neon_type" "neon_vst2_4_regs_vst3_vst4")]
4746 )
4748 (define_insn "neon_vst3qb<mode>"
4749 [(set (mem:EI (match_operand:SI 1 "s_register_operand" "0"))
4750 (unspec:EI [(match_operand:CI 2 "s_register_operand" "w")
4751 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4752 UNSPEC_VST3B))
4753 (set (match_operand:SI 0 "s_register_operand" "=r")
4754 (plus:SI (match_dup 1)
4755 (const_int 24)))]
4756 "TARGET_NEON"
4757 {
4758 int regno = REGNO (operands[2]);
4759 rtx ops[4];
4760 ops[0] = operands[0];
4761 ops[1] = gen_rtx_REG (DImode, regno + 2);
4762 ops[2] = gen_rtx_REG (DImode, regno + 6);
4763 ops[3] = gen_rtx_REG (DImode, regno + 10);
4764 output_asm_insn ("vst3.<V_sz_elem>\t{%P1, %P2, %P3}, [%0]!", ops);
4765 return "";
4766 }
4767 [(set_attr "neon_type" "neon_vst2_4_regs_vst3_vst4")]
4768 )
4770 (define_insn "neon_vst3_lane<mode>"
4771 [(set (mem:<V_three_elem> (match_operand:SI 0 "s_register_operand" "r"))
4772 (unspec:<V_three_elem>
4773 [(match_operand:EI 1 "s_register_operand" "w")
4774 (match_operand:SI 2 "immediate_operand" "i")
4775 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4776 UNSPEC_VST3_LANE))]
4777 "TARGET_NEON"
4778 {
4779 HOST_WIDE_INT lane = INTVAL (operands[2]);
4780 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4781 int regno = REGNO (operands[1]);
4782 rtx ops[5];
4783 if (lane < 0 || lane >= max)
4784 error ("lane out of range");
4785 ops[0] = operands[0];
4786 ops[1] = gen_rtx_REG (DImode, regno);
4787 ops[2] = gen_rtx_REG (DImode, regno + 2);
4788 ops[3] = gen_rtx_REG (DImode, regno + 4);
4789 ops[4] = operands[2];
4790 output_asm_insn ("vst3.<V_sz_elem>\t{%P1[%c4], %P2[%c4], %P3[%c4]}, [%0]",
4791 ops);
4792 return "";
4793 }
4794 [(set_attr "neon_type" "neon_vst3_vst4_lane")]
4795 )
4797 (define_insn "neon_vst3_lane<mode>"
4798 [(set (mem:<V_three_elem> (match_operand:SI 0 "s_register_operand" "r"))
4799 (unspec:<V_three_elem>
4800 [(match_operand:CI 1 "s_register_operand" "w")
4801 (match_operand:SI 2 "immediate_operand" "i")
4802 (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4803 UNSPEC_VST3_LANE))]
4804 "TARGET_NEON"
4805 {
4806 HOST_WIDE_INT lane = INTVAL (operands[2]);
4807 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4808 int regno = REGNO (operands[1]);
4809 rtx ops[5];
4810 if (lane < 0 || lane >= max)
4811 error ("lane out of range");
4812 else if (lane >= max / 2)
4813 {
4814 lane -= max / 2;
4815 regno += 2;
4816 }
4817 ops[0] = operands[0];
4818 ops[1] = gen_rtx_REG (DImode, regno);
4819 ops[2] = gen_rtx_REG (DImode, regno + 4);
4820 ops[3] = gen_rtx_REG (DImode, regno + 8);
4821 ops[4] = GEN_INT (lane);
4822 output_asm_insn ("vst3.<V_sz_elem>\t{%P1[%c4], %P2[%c4], %P3[%c4]}, [%0]",
4823 ops);
4824 return "";
4825 }
4826 [(set_attr "neon_type" "neon_vst3_vst4_lane")])
4828 (define_insn "neon_vld4<mode>"
4829 [(set (match_operand:OI 0 "s_register_operand" "=w")
4830 (unspec:OI [(mem:OI (match_operand:SI 1 "s_register_operand" "r"))
4831 (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4832 UNSPEC_VLD4))]
4833 "TARGET_NEON"
4834 {
4835 if (<V_sz_elem> == 64)
4836 return "vld1.64\t%h0, [%1]";
4837 else
4838 return "vld4.<V_sz_elem>\t%h0, [%1]";
4839 }
4840 [(set (attr "neon_type")
4841 (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
4842 (const_string "neon_vld1_1_2_regs")
4843 (const_string "neon_vld3_vld4")))]
4844 )
4846 (define_expand "neon_vld4<mode>"
4847 [(match_operand:XI 0 "s_register_operand" "=w")
4848 (match_operand:SI 1 "s_register_operand" "+r")
4849 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4850 "TARGET_NEON"
4851 {
4852 emit_insn (gen_neon_vld4qa<mode> (operands[0], operands[0],
4853 operands[1], operands[1]));
4854 emit_insn (gen_neon_vld4qb<mode> (operands[0], operands[0],
4855 operands[1], operands[1]));
4856 DONE;
4857 })
4859 (define_insn "neon_vld4qa<mode>"
4860 [(set (match_operand:XI 0 "s_register_operand" "=w")
4861 (unspec:XI [(mem:XI (match_operand:SI 3 "s_register_operand" "2"))
4862 (match_operand:XI 1 "s_register_operand" "0")
4863 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4864 UNSPEC_VLD4A))
4865 (set (match_operand:SI 2 "s_register_operand" "=r")
4866 (plus:SI (match_dup 3)
4867 (const_int 32)))]
4868 "TARGET_NEON"
4869 {
4870 int regno = REGNO (operands[0]);
4871 rtx ops[5];
4872 ops[0] = gen_rtx_REG (DImode, regno);
4873 ops[1] = gen_rtx_REG (DImode, regno + 4);
4874 ops[2] = gen_rtx_REG (DImode, regno + 8);
4875 ops[3] = gen_rtx_REG (DImode, regno + 12);
4876 ops[4] = operands[2];
4877 output_asm_insn ("vld4.<V_sz_elem>\t{%P0, %P1, %P2, %P3}, [%4]!", ops);
4878 return "";
4879 }
4880 [(set_attr "neon_type" "neon_vld3_vld4")]
4881 )
4883 (define_insn "neon_vld4qb<mode>"
4884 [(set (match_operand:XI 0 "s_register_operand" "=w")
4885 (unspec:XI [(mem:XI (match_operand:SI 3 "s_register_operand" "2"))
4886 (match_operand:XI 1 "s_register_operand" "0")
4887 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4888 UNSPEC_VLD4B))
4889 (set (match_operand:SI 2 "s_register_operand" "=r")
4890 (plus:SI (match_dup 3)
4891 (const_int 32)))]
4892 "TARGET_NEON"
4893 {
4894 int regno = REGNO (operands[0]);
4895 rtx ops[5];
4896 ops[0] = gen_rtx_REG (DImode, regno + 2);
4897 ops[1] = gen_rtx_REG (DImode, regno + 6);
4898 ops[2] = gen_rtx_REG (DImode, regno + 10);
4899 ops[3] = gen_rtx_REG (DImode, regno + 14);
4900 ops[4] = operands[2];
4901 output_asm_insn ("vld4.<V_sz_elem>\t{%P0, %P1, %P2, %P3}, [%4]!", ops);
4902 return "";
4903 }
4904 [(set_attr "neon_type" "neon_vld3_vld4")]
4905 )
4907 (define_insn "neon_vld4_lane<mode>"
4908 [(set (match_operand:OI 0 "s_register_operand" "=w")
4909 (unspec:OI [(mem:<V_four_elem> (match_operand:SI 1 "s_register_operand" "r"))
4910 (match_operand:OI 2 "s_register_operand" "0")
4911 (match_operand:SI 3 "immediate_operand" "i")
4912 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4913 UNSPEC_VLD4_LANE))]
4914 "TARGET_NEON"
4915 {
4916 HOST_WIDE_INT lane = INTVAL (operands[3]);
4917 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4918 int regno = REGNO (operands[0]);
4919 rtx ops[6];
4920 if (lane < 0 || lane >= max)
4921 error ("lane out of range");
4922 ops[0] = gen_rtx_REG (DImode, regno);
4923 ops[1] = gen_rtx_REG (DImode, regno + 2);
4924 ops[2] = gen_rtx_REG (DImode, regno + 4);
4925 ops[3] = gen_rtx_REG (DImode, regno + 6);
4926 ops[4] = operands[1];
4927 ops[5] = operands[3];
4928 output_asm_insn ("vld4.<V_sz_elem>\t{%P0[%c5], %P1[%c5], %P2[%c5], %P3[%c5]}, [%4]",
4929 ops);
4930 return "";
4931 }
4932 [(set_attr "neon_type" "neon_vld3_vld4_lane")]
4933 )
4935 (define_insn "neon_vld4_lane<mode>"
4936 [(set (match_operand:XI 0 "s_register_operand" "=w")
4937 (unspec:XI [(mem:<V_four_elem> (match_operand:SI 1 "s_register_operand" "r"))
4938 (match_operand:XI 2 "s_register_operand" "0")
4939 (match_operand:SI 3 "immediate_operand" "i")
4940 (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4941 UNSPEC_VLD4_LANE))]
4942 "TARGET_NEON"
4943 {
4944 HOST_WIDE_INT lane = INTVAL (operands[3]);
4945 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4946 int regno = REGNO (operands[0]);
4947 rtx ops[6];
4948 if (lane < 0 || lane >= max)
4949 error ("lane out of range");
4950 else if (lane >= max / 2)
4951 {
4952 lane -= max / 2;
4953 regno += 2;
4954 }
4955 ops[0] = gen_rtx_REG (DImode, regno);
4956 ops[1] = gen_rtx_REG (DImode, regno + 4);
4957 ops[2] = gen_rtx_REG (DImode, regno + 8);
4958 ops[3] = gen_rtx_REG (DImode, regno + 12);
4959 ops[4] = operands[1];
4960 ops[5] = GEN_INT (lane);
4961 output_asm_insn ("vld4.<V_sz_elem>\t{%P0[%c5], %P1[%c5], %P2[%c5], %P3[%c5]}, [%4]",
4962 ops);
4963 return "";
4964 }
4965 [(set_attr "neon_type" "neon_vld3_vld4_lane")]
4966 )
4968 (define_insn "neon_vld4_dup<mode>"
4969 [(set (match_operand:OI 0 "s_register_operand" "=w")
4970 (unspec:OI [(mem:<V_four_elem> (match_operand:SI 1 "s_register_operand" "r"))
4971 (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4972 UNSPEC_VLD4_DUP))]
4973 "TARGET_NEON"
4974 {
4975 if (GET_MODE_NUNITS (<MODE>mode) > 1)
4976 {
4977 int regno = REGNO (operands[0]);
4978 rtx ops[5];
4979 ops[0] = gen_rtx_REG (DImode, regno);
4980 ops[1] = gen_rtx_REG (DImode, regno + 2);
4981 ops[2] = gen_rtx_REG (DImode, regno + 4);
4982 ops[3] = gen_rtx_REG (DImode, regno + 6);
4983 ops[4] = operands[1];
4984 output_asm_insn ("vld4.<V_sz_elem>\t{%P0[], %P1[], %P2[], %P3[]}, [%4]",
4985 ops);
4986 return "";
4987 }
4988 else
4989 return "vld1.<V_sz_elem>\t%h0, [%1]";
4990 }
4991 [(set (attr "neon_type")
4992 (if_then_else (gt (const_string "<V_mode_nunits>") (const_string "1"))
4993 (const_string "neon_vld3_vld4_all_lanes")
4994 (const_string "neon_vld1_1_2_regs")))]
4995 )
4997 (define_insn "neon_vst4<mode>"
4998 [(set (mem:OI (match_operand:SI 0 "s_register_operand" "r"))
4999 (unspec:OI [(match_operand:OI 1 "s_register_operand" "w")
5000 (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5001 UNSPEC_VST4))]
5002 "TARGET_NEON"
5003 {
5004 if (<V_sz_elem> == 64)
5005 return "vst1.64\t%h1, [%0]";
5006 else
5007 return "vst4.<V_sz_elem>\t%h1, [%0]";
5008 }
5009 [(set (attr "neon_type")
5010 (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
5011 (const_string "neon_vst1_1_2_regs_vst2_2_regs")
5012 (const_string "neon_vst2_4_regs_vst3_vst4")))]
5013 )
5015 (define_expand "neon_vst4<mode>"
5016 [(match_operand:SI 0 "s_register_operand" "+r")
5017 (match_operand:XI 1 "s_register_operand" "w")
5018 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5019 "TARGET_NEON"
5020 {
5021 emit_insn (gen_neon_vst4qa<mode> (operands[0], operands[0], operands[1]));
5022 emit_insn (gen_neon_vst4qb<mode> (operands[0], operands[0], operands[1]));
5023 DONE;
5024 })
5026 (define_insn "neon_vst4qa<mode>"
5027 [(set (mem:OI (match_operand:SI 1 "s_register_operand" "0"))
5028 (unspec:OI [(match_operand:XI 2 "s_register_operand" "w")
5029 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5030 UNSPEC_VST4A))
5031 (set (match_operand:SI 0 "s_register_operand" "=r")
5032 (plus:SI (match_dup 1)
5033 (const_int 32)))]
5034 "TARGET_NEON"
5035 {
5036 int regno = REGNO (operands[2]);
5037 rtx ops[5];
5038 ops[0] = operands[0];
5039 ops[1] = gen_rtx_REG (DImode, regno);
5040 ops[2] = gen_rtx_REG (DImode, regno + 4);
5041 ops[3] = gen_rtx_REG (DImode, regno + 8);
5042 ops[4] = gen_rtx_REG (DImode, regno + 12);
5043 output_asm_insn ("vst4.<V_sz_elem>\t{%P1, %P2, %P3, %P4}, [%0]!", ops);
5044 return "";
5045 }
5046 [(set_attr "neon_type" "neon_vst2_4_regs_vst3_vst4")]
5047 )
5049 (define_insn "neon_vst4qb<mode>"
5050 [(set (mem:OI (match_operand:SI 1 "s_register_operand" "0"))
5051 (unspec:OI [(match_operand:XI 2 "s_register_operand" "w")
5052 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5053 UNSPEC_VST4B))
5054 (set (match_operand:SI 0 "s_register_operand" "=r")
5055 (plus:SI (match_dup 1)
5056 (const_int 32)))]
5057 "TARGET_NEON"
5058 {
5059 int regno = REGNO (operands[2]);
5060 rtx ops[5];
5061 ops[0] = operands[0];
5062 ops[1] = gen_rtx_REG (DImode, regno + 2);
5063 ops[2] = gen_rtx_REG (DImode, regno + 6);
5064 ops[3] = gen_rtx_REG (DImode, regno + 10);
5065 ops[4] = gen_rtx_REG (DImode, regno + 14);
5066 output_asm_insn ("vst4.<V_sz_elem>\t{%P1, %P2, %P3, %P4}, [%0]!", ops);
5067 return "";
5068 }
5069 [(set_attr "neon_type" "neon_vst2_4_regs_vst3_vst4")]
5070 )
5072 (define_insn "neon_vst4_lane<mode>"
5073 [(set (mem:<V_four_elem> (match_operand:SI 0 "s_register_operand" "r"))
5074 (unspec:<V_four_elem>
5075 [(match_operand:OI 1 "s_register_operand" "w")
5076 (match_operand:SI 2 "immediate_operand" "i")
5077 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5078 UNSPEC_VST4_LANE))]
5079 "TARGET_NEON"
5080 {
5081 HOST_WIDE_INT lane = INTVAL (operands[2]);
5082 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
5083 int regno = REGNO (operands[1]);
5084 rtx ops[6];
5085 if (lane < 0 || lane >= max)
5086 error ("lane out of range");
5087 ops[0] = operands[0];
5088 ops[1] = gen_rtx_REG (DImode, regno);
5089 ops[2] = gen_rtx_REG (DImode, regno + 2);
5090 ops[3] = gen_rtx_REG (DImode, regno + 4);
5091 ops[4] = gen_rtx_REG (DImode, regno + 6);
5092 ops[5] = operands[2];
5093 output_asm_insn ("vst4.<V_sz_elem>\t{%P1[%c5], %P2[%c5], %P3[%c5], %P4[%c5]}, [%0]",
5094 ops);
5095 return "";
5096 }
5097 [(set_attr "neon_type" "neon_vst3_vst4_lane")]
5098 )
5100 (define_insn "neon_vst4_lane<mode>"
5101 [(set (mem:<V_four_elem> (match_operand:SI 0 "s_register_operand" "r"))
5102 (unspec:<V_four_elem>
5103 [(match_operand:XI 1 "s_register_operand" "w")
5104 (match_operand:SI 2 "immediate_operand" "i")
5105 (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5106 UNSPEC_VST4_LANE))]
5107 "TARGET_NEON"
5108 {
5109 HOST_WIDE_INT lane = INTVAL (operands[2]);
5110 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
5111 int regno = REGNO (operands[1]);
5112 rtx ops[6];
5113 if (lane < 0 || lane >= max)
5114 error ("lane out of range");
5115 else if (lane >= max / 2)
5116 {
5117 lane -= max / 2;
5118 regno += 2;
5119 }
5120 ops[0] = operands[0];
5121 ops[1] = gen_rtx_REG (DImode, regno);
5122 ops[2] = gen_rtx_REG (DImode, regno + 4);
5123 ops[3] = gen_rtx_REG (DImode, regno + 8);
5124 ops[4] = gen_rtx_REG (DImode, regno + 12);
5125 ops[5] = GEN_INT (lane);
5126 output_asm_insn ("vst4.<V_sz_elem>\t{%P1[%c5], %P2[%c5], %P3[%c5], %P4[%c5]}, [%0]",
5127 ops);
5128 return "";
5129 }
5130 [(set_attr "neon_type" "neon_vst3_vst4_lane")]
5131 )
5133 (define_expand "neon_vand<mode>"
5134 [(match_operand:VDQX 0 "s_register_operand" "")
5135 (match_operand:VDQX 1 "s_register_operand" "")
5136 (match_operand:VDQX 2 "neon_inv_logic_op2" "")
5137 (match_operand:SI 3 "immediate_operand" "")]
5138 "TARGET_NEON"
5139 {
5140 emit_insn (gen_and<mode>3<V_suf64> (operands[0], operands[1], operands[2]));
5141 DONE;
5142 })
5144 (define_expand "neon_vorr<mode>"
5145 [(match_operand:VDQX 0 "s_register_operand" "")
5146 (match_operand:VDQX 1 "s_register_operand" "")
5147 (match_operand:VDQX 2 "neon_logic_op2" "")
5148 (match_operand:SI 3 "immediate_operand" "")]
5149 "TARGET_NEON"
5150 {
5151 emit_insn (gen_ior<mode>3<V_suf64> (operands[0], operands[1], operands[2]));
5152 DONE;
5153 })
5155 (define_expand "neon_veor<mode>"
5156 [(match_operand:VDQX 0 "s_register_operand" "")
5157 (match_operand:VDQX 1 "s_register_operand" "")
5158 (match_operand:VDQX 2 "s_register_operand" "")
5159 (match_operand:SI 3 "immediate_operand" "")]
5160 "TARGET_NEON"
5161 {
5162 emit_insn (gen_xor<mode>3<V_suf64> (operands[0], operands[1], operands[2]));
5163 DONE;
5164 })
5166 (define_expand "neon_vbic<mode>"
5167 [(match_operand:VDQX 0 "s_register_operand" "")
5168 (match_operand:VDQX 1 "s_register_operand" "")
5169 (match_operand:VDQX 2 "neon_logic_op2" "")
5170 (match_operand:SI 3 "immediate_operand" "")]
5171 "TARGET_NEON"
5172 {
5173 emit_insn (gen_bic<mode>3_neon (operands[0], operands[1], operands[2]));
5174 DONE;
5175 })
5177 (define_expand "neon_vorn<mode>"
5178 [(match_operand:VDQX 0 "s_register_operand" "")
5179 (match_operand:VDQX 1 "s_register_operand" "")
5180 (match_operand:VDQX 2 "neon_inv_logic_op2" "")
5181 (match_operand:SI 3 "immediate_operand" "")]
5182 "TARGET_NEON"
5183 {
5184 emit_insn (gen_orn<mode>3_neon (operands[0], operands[1], operands[2]));
5185 DONE;
5186 })
5188 (define_insn "neon_vec_unpack<US>_lo_<mode>"
5189 [(set (match_operand:<V_unpack> 0 "register_operand" "=w")
5190 (SE:<V_unpack> (vec_select:<V_HALF>
5191 (match_operand:VU 1 "register_operand" "w")
5192 (match_operand:VU 2 "vect_par_constant_low" ""))))]
5193 "TARGET_NEON"
5194 "vmovl.<US><V_sz_elem> %q0, %e1"
5195 [(set_attr "neon_type" "neon_shift_1")]
5196 )
5198 (define_insn "neon_vec_unpack<US>_hi_<mode>"
5199 [(set (match_operand:<V_unpack> 0 "register_operand" "=w")
5200 (SE:<V_unpack> (vec_select:<V_HALF>
5201 (match_operand:VU 1 "register_operand" "w")
5202 (match_operand:VU 2 "vect_par_constant_high" ""))))]
5203 "TARGET_NEON"
5204 "vmovl.<US><V_sz_elem> %q0, %f1"
5205 [(set_attr "neon_type" "neon_shift_1")]
5206 )
5208 (define_expand "vec_unpack<US>_hi_<mode>"
5209 [(match_operand:<V_unpack> 0 "register_operand" "")
5210 (SE:<V_unpack> (match_operand:VU 1 "register_operand"))]
5211 "TARGET_NEON"
5212 {
5213 rtvec v = rtvec_alloc (<V_mode_nunits>/2) ;
5214 rtx t1;
5215 int i;
5216 for (i = 0; i < (<V_mode_nunits>/2); i++)
5217 RTVEC_ELT (v, i) = GEN_INT ((<V_mode_nunits>/2) + i);
5219 t1 = gen_rtx_PARALLEL (<MODE>mode, v);
5220 emit_insn (gen_neon_vec_unpack<US>_hi_<mode> (operands[0],
5221 operands[1],
5222 t1));
5223 DONE;
5224 }
5225 )
5227 (define_expand "vec_unpack<US>_lo_<mode>"
5228 [(match_operand:<V_unpack> 0 "register_operand" "")
5229 (SE:<V_unpack> (match_operand:VU 1 "register_operand" ""))]
5230 "TARGET_NEON"
5231 {
5232 rtvec v = rtvec_alloc (<V_mode_nunits>/2) ;
5233 rtx t1;
5234 int i;
5235 for (i = 0; i < (<V_mode_nunits>/2) ; i++)
5236 RTVEC_ELT (v, i) = GEN_INT (i);
5237 t1 = gen_rtx_PARALLEL (<MODE>mode, v);
5238 emit_insn (gen_neon_vec_unpack<US>_lo_<mode> (operands[0],
5239 operands[1],
5240 t1));
5241 DONE;
5242 }
5243 )
5245 (define_insn "neon_vec_<US>mult_lo_<mode>"
5246 [(set (match_operand:<V_unpack> 0 "register_operand" "=w")
5247 (mult:<V_unpack> (SE:<V_unpack> (vec_select:<V_HALF>
5248 (match_operand:VU 1 "register_operand" "w")
5249 (match_operand:VU 2 "vect_par_constant_low" "")))
5250 (SE:<V_unpack> (vec_select:<V_HALF>
5251 (match_operand:VU 3 "register_operand" "w")
5252 (match_dup 2)))))]
5253 "TARGET_NEON"
5254 "vmull.<US><V_sz_elem> %q0, %e1, %e3"
5255 [(set_attr "neon_type" "neon_shift_1")]
5256 )
5258 (define_expand "vec_widen_<US>mult_lo_<mode>"
5259 [(match_operand:<V_unpack> 0 "register_operand" "")
5260 (SE:<V_unpack> (match_operand:VU 1 "register_operand" ""))
5261 (SE:<V_unpack> (match_operand:VU 2 "register_operand" ""))]
5262 "TARGET_NEON"
5263 {
5264 rtvec v = rtvec_alloc (<V_mode_nunits>/2) ;
5265 rtx t1;
5266 int i;
5267 for (i = 0; i < (<V_mode_nunits>/2) ; i++)
5268 RTVEC_ELT (v, i) = GEN_INT (i);
5269 t1 = gen_rtx_PARALLEL (<MODE>mode, v);
5271 emit_insn (gen_neon_vec_<US>mult_lo_<mode> (operands[0],
5272 operands[1],
5273 t1,
5274 operands[2]));
5275 DONE;
5276 }
5277 )
5279 (define_insn "neon_vec_<US>mult_hi_<mode>"
5280 [(set (match_operand:<V_unpack> 0 "register_operand" "=w")
5281 (mult:<V_unpack> (SE:<V_unpack> (vec_select:<V_HALF>
5282 (match_operand:VU 1 "register_operand" "w")
5283 (match_operand:VU 2 "vect_par_constant_high" "")))
5284 (SE:<V_unpack> (vec_select:<V_HALF>
5285 (match_operand:VU 3 "register_operand" "w")
5286 (match_dup 2)))))]
5287 "TARGET_NEON"
5288 "vmull.<US><V_sz_elem> %q0, %f1, %f3"
5289 [(set_attr "neon_type" "neon_shift_1")]
5290 )
5292 (define_expand "vec_widen_<US>mult_hi_<mode>"
5293 [(match_operand:<V_unpack> 0 "register_operand" "")
5294 (SE:<V_unpack> (match_operand:VU 1 "register_operand" ""))
5295 (SE:<V_unpack> (match_operand:VU 2 "register_operand" ""))]
5296 "TARGET_NEON"
5297 {
5298 rtvec v = rtvec_alloc (<V_mode_nunits>/2) ;
5299 rtx t1;
5300 int i;
5301 for (i = 0; i < (<V_mode_nunits>/2) ; i++)
5302 RTVEC_ELT (v, i) = GEN_INT (<V_mode_nunits>/2 + i);
5303 t1 = gen_rtx_PARALLEL (<MODE>mode, v);
5305 emit_insn (gen_neon_vec_<US>mult_hi_<mode> (operands[0],
5306 operands[1],
5307 t1,
5308 operands[2]));
5309 DONE;
5311 }
5312 )
5314 ;; Vectorize for non-neon-quad case
5315 (define_insn "neon_unpack<US>_<mode>"
5316 [(set (match_operand:<V_widen> 0 "register_operand" "=w")
5317 (SE:<V_widen> (match_operand:VDI 1 "register_operand" "")))]
5318 "TARGET_NEON"
5319 "vmovl.<US><V_sz_elem> %q0, %1"
5320 [(set_attr "neon_type" "neon_shift_1")]
5321 )
5323 (define_expand "vec_unpack<US>_lo_<mode>"
5324 [(match_operand:<V_double_width> 0 "register_operand" "")
5325 (SE:<V_double_width>(match_operand:VDI 1 "register_operand"))]
5326 "TARGET_NEON"
5327 {
5328 rtx tmpreg = gen_reg_rtx (<V_widen>mode);
5329 emit_insn (gen_neon_unpack<US>_<mode> (tmpreg, operands[1]));
5330 emit_insn (gen_neon_vget_low<V_widen_l> (operands[0], tmpreg));
5332 DONE;
5333 }
5334 )
5336 (define_expand "vec_unpack<US>_hi_<mode>"
5337 [(match_operand:<V_double_width> 0 "register_operand" "")
5338 (SE:<V_double_width>(match_operand:VDI 1 "register_operand"))]
5339 "TARGET_NEON"
5340 {
5341 rtx tmpreg = gen_reg_rtx (<V_widen>mode);
5342 emit_insn (gen_neon_unpack<US>_<mode> (tmpreg, operands[1]));
5343 emit_insn (gen_neon_vget_high<V_widen_l> (operands[0], tmpreg));
5345 DONE;
5346 }
5347 )
5349 (define_insn "neon_vec_<US>mult_<mode>"
5350 [(set (match_operand:<V_widen> 0 "register_operand" "=w")
5351 (mult:<V_widen> (SE:<V_widen>
5352 (match_operand:VDI 1 "register_operand" "w"))
5353 (SE:<V_widen>
5354 (match_operand:VDI 2 "register_operand" "w"))))]
5355 "TARGET_NEON"
5356 "vmull.<US><V_sz_elem> %q0, %1, %2"
5357 [(set_attr "neon_type" "neon_shift_1")]
5358 )
5360 (define_expand "vec_widen_<US>mult_hi_<mode>"
5361 [(match_operand:<V_double_width> 0 "register_operand" "")
5362 (SE:<V_double_width> (match_operand:VDI 1 "register_operand" ""))
5363 (SE:<V_double_width> (match_operand:VDI 2 "register_operand" ""))]
5364 "TARGET_NEON"
5365 {
5366 rtx tmpreg = gen_reg_rtx (<V_widen>mode);
5367 emit_insn (gen_neon_vec_<US>mult_<mode> (tmpreg, operands[1], operands[2]));
5368 emit_insn (gen_neon_vget_high<V_widen_l> (operands[0], tmpreg));
5370 DONE;
5372 }
5373 )
5375 (define_expand "vec_widen_<US>mult_lo_<mode>"
5376 [(match_operand:<V_double_width> 0 "register_operand" "")
5377 (SE:<V_double_width> (match_operand:VDI 1 "register_operand" ""))
5378 (SE:<V_double_width> (match_operand:VDI 2 "register_operand" ""))]
5379 "TARGET_NEON"
5380 {
5381 rtx tmpreg = gen_reg_rtx (<V_widen>mode);
5382 emit_insn (gen_neon_vec_<US>mult_<mode> (tmpreg, operands[1], operands[2]));
5383 emit_insn (gen_neon_vget_low<V_widen_l> (operands[0], tmpreg));
5385 DONE;
5387 }
5388 )
5390 (define_insn "vec_pack_trunc_<mode>"
5391 [(set (match_operand:<V_narrow_pack> 0 "register_operand" "=&w")
5392 (vec_concat:<V_narrow_pack>
5393 (truncate:<V_narrow>
5394 (match_operand:VN 1 "register_operand" "w"))
5395 (truncate:<V_narrow>
5396 (match_operand:VN 2 "register_operand" "w"))))]
5397 "TARGET_NEON"
5398 "vmovn.i<V_sz_elem>\t%e0, %q1\n\tvmovn.i<V_sz_elem>\t%f0, %q2"
5399 [(set_attr "neon_type" "neon_shift_1")]
5400 )
5402 ;; For the non-quad case.
5403 (define_insn "neon_vec_pack_trunc_<mode>"
5404 [(set (match_operand:<V_narrow> 0 "register_operand" "=w")
5405 (truncate:<V_narrow> (match_operand:VN 1 "register_operand" "")))]
5406 "TARGET_NEON"
5407 "vmovn.i<V_sz_elem>\t%0, %q1"
5408 [(set_attr "neon_type" "neon_shift_1")]
5409 )
5411 (define_expand "vec_pack_trunc_<mode>"
5412 [(match_operand:<V_narrow_pack> 0 "register_operand" "")
5413 (match_operand:VSHFT 1 "register_operand" "")
5414 (match_operand:VSHFT 2 "register_operand")]
5415 "TARGET_NEON"
5416 {
5417 rtx tempreg = gen_reg_rtx (<V_DOUBLE>mode);
5419 emit_insn (gen_move_lo_quad_<V_double> (tempreg, operands[1]));
5420 emit_insn (gen_move_hi_quad_<V_double> (tempreg, operands[2]));
5421 emit_insn (gen_neon_vec_pack_trunc_<V_double> (operands[0], tempreg));
5422 DONE;
5423 })