| blob | 87ae2faa09be7f6ca3463076b04c44594a6717a0 |
1 ;; ARM NEON coprocessor Machine Description
2 ;; Copyright (C) 2006, 2007, 2008, 2009, 2010 Free Software Foundation, Inc.
3 ;; Written by CodeSourcery.
4 ;;
5 ;; This file is part of GCC.
6 ;;
7 ;; GCC is free software; you can redistribute it and/or modify it
8 ;; under the terms of the GNU General Public License as published by
9 ;; the Free Software Foundation; either version 3, or (at your option)
10 ;; any later version.
11 ;;
12 ;; GCC is distributed in the hope that it will be useful, but
13 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
14 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 ;; General Public License for more details.
16 ;;
17 ;; You should have received a copy of the GNU General Public License
18 ;; along with GCC; see the file COPYING3. If not see
19 ;; <http://www.gnu.org/licenses/>.
21 ;; Constants for unspecs.
22 (define_constants
23 [(UNSPEC_ASHIFT_SIGNED 65)
24 (UNSPEC_ASHIFT_UNSIGNED 66)
25 (UNSPEC_VABD 69)
26 (UNSPEC_VABDL 70)
27 (UNSPEC_VADD 72)
28 (UNSPEC_VADDHN 73)
29 (UNSPEC_VADDL 74)
30 (UNSPEC_VADDW 75)
31 (UNSPEC_VBSL 78)
32 (UNSPEC_VCAGE 79)
33 (UNSPEC_VCAGT 80)
34 (UNSPEC_VCEQ 81)
35 (UNSPEC_VCGE 82)
36 (UNSPEC_VCGT 83)
37 (UNSPEC_VCLS 84)
38 (UNSPEC_VCVT 88)
39 (UNSPEC_VCVT_N 89)
40 (UNSPEC_VEXT 93)
41 (UNSPEC_VHADD 97)
42 (UNSPEC_VHSUB 98)
43 (UNSPEC_VLD1 99)
44 (UNSPEC_VLD1_DUP 100)
45 (UNSPEC_VLD1_LANE 101)
46 (UNSPEC_VLD2 102)
47 (UNSPEC_VLD2_DUP 103)
48 (UNSPEC_VLD2_LANE 104)
49 (UNSPEC_VLD3 105)
50 (UNSPEC_VLD3A 106)
51 (UNSPEC_VLD3B 107)
52 (UNSPEC_VLD3_DUP 108)
53 (UNSPEC_VLD3_LANE 109)
54 (UNSPEC_VLD4 110)
55 (UNSPEC_VLD4A 111)
56 (UNSPEC_VLD4B 112)
57 (UNSPEC_VLD4_DUP 113)
58 (UNSPEC_VLD4_LANE 114)
59 (UNSPEC_VMAX 115)
60 (UNSPEC_VMIN 116)
61 (UNSPEC_VMLA 117)
62 (UNSPEC_VMLAL 118)
63 (UNSPEC_VMLA_LANE 119)
64 (UNSPEC_VMLAL_LANE 120)
65 (UNSPEC_VMLS 121)
66 (UNSPEC_VMLSL 122)
67 (UNSPEC_VMLS_LANE 123)
68 (UNSPEC_VMLSL_LANE 124)
69 (UNSPEC_VMOVL 125)
70 (UNSPEC_VMOVN 126)
71 (UNSPEC_VMUL 127)
72 (UNSPEC_VMULL 128)
73 (UNSPEC_VMUL_LANE 129)
74 (UNSPEC_VMULL_LANE 130)
75 (UNSPEC_VPADAL 135)
76 (UNSPEC_VPADD 136)
77 (UNSPEC_VPADDL 137)
78 (UNSPEC_VPMAX 138)
79 (UNSPEC_VPMIN 139)
80 (UNSPEC_VPSMAX 140)
81 (UNSPEC_VPSMIN 141)
82 (UNSPEC_VPUMAX 142)
83 (UNSPEC_VPUMIN 143)
84 (UNSPEC_VQABS 144)
85 (UNSPEC_VQADD 145)
86 (UNSPEC_VQDMLAL 146)
87 (UNSPEC_VQDMLAL_LANE 147)
88 (UNSPEC_VQDMLSL 148)
89 (UNSPEC_VQDMLSL_LANE 149)
90 (UNSPEC_VQDMULH 150)
91 (UNSPEC_VQDMULH_LANE 151)
92 (UNSPEC_VQDMULL 152)
93 (UNSPEC_VQDMULL_LANE 153)
94 (UNSPEC_VQMOVN 154)
95 (UNSPEC_VQMOVUN 155)
96 (UNSPEC_VQNEG 156)
97 (UNSPEC_VQSHL 157)
98 (UNSPEC_VQSHL_N 158)
99 (UNSPEC_VQSHLU_N 159)
100 (UNSPEC_VQSHRN_N 160)
101 (UNSPEC_VQSHRUN_N 161)
102 (UNSPEC_VQSUB 162)
103 (UNSPEC_VRECPE 163)
104 (UNSPEC_VRECPS 164)
105 (UNSPEC_VREV16 165)
106 (UNSPEC_VREV32 166)
107 (UNSPEC_VREV64 167)
108 (UNSPEC_VRSQRTE 168)
109 (UNSPEC_VRSQRTS 169)
110 (UNSPEC_VSHL 171)
111 (UNSPEC_VSHLL_N 172)
112 (UNSPEC_VSHL_N 173)
113 (UNSPEC_VSHR_N 174)
114 (UNSPEC_VSHRN_N 175)
115 (UNSPEC_VSLI 176)
116 (UNSPEC_VSRA_N 177)
117 (UNSPEC_VSRI 178)
118 (UNSPEC_VST1 179)
119 (UNSPEC_VST1_LANE 180)
120 (UNSPEC_VST2 181)
121 (UNSPEC_VST2_LANE 182)
122 (UNSPEC_VST3 183)
123 (UNSPEC_VST3A 184)
124 (UNSPEC_VST3B 185)
125 (UNSPEC_VST3_LANE 186)
126 (UNSPEC_VST4 187)
127 (UNSPEC_VST4A 188)
128 (UNSPEC_VST4B 189)
129 (UNSPEC_VST4_LANE 190)
130 (UNSPEC_VSTRUCTDUMMY 191)
131 (UNSPEC_VSUB 192)
132 (UNSPEC_VSUBHN 193)
133 (UNSPEC_VSUBL 194)
134 (UNSPEC_VSUBW 195)
135 (UNSPEC_VTBL 196)
136 (UNSPEC_VTBX 197)
137 (UNSPEC_VTRN1 198)
138 (UNSPEC_VTRN2 199)
139 (UNSPEC_VTST 200)
140 (UNSPEC_VUZP1 201)
141 (UNSPEC_VUZP2 202)
142 (UNSPEC_VZIP1 203)
143 (UNSPEC_VZIP2 204)
144 (UNSPEC_MISALIGNED_ACCESS 205)
145 (UNSPEC_VCLE 206)
146 (UNSPEC_VCLT 207)])
149 ;; Attribute used to permit string comparisons against <VQH_mnem> in
150 ;; neon_type attribute definitions.
151 (define_attr "vqh_mnem" "vadd,vmin,vmax" (const_string "vadd"))
153 (define_insn "*neon_mov<mode>"
154 [(set (match_operand:VD 0 "nonimmediate_operand"
155 "=w,Uv,w, w, ?r,?w,?r,?r, ?Us")
156 (match_operand:VD 1 "general_operand"
157 " w,w, Dn,Uvi, w, r, r, Usi,r"))]
158 "TARGET_NEON
159 && (register_operand (operands[0], <MODE>mode)
160 || register_operand (operands[1], <MODE>mode))"
161 {
162 if (which_alternative == 2)
163 {
164 int width, is_valid;
165 static char templ[40];
167 is_valid = neon_immediate_valid_for_move (operands[1], <MODE>mode,
168 &operands[1], &width);
170 gcc_assert (is_valid != 0);
172 if (width == 0)
173 return "vmov.f32\t%P0, %1 @ <mode>";
174 else
175 sprintf (templ, "vmov.i%d\t%%P0, %%1 @ <mode>", width);
177 return templ;
178 }
180 /* FIXME: If the memory layout is changed in big-endian mode, output_move_vfp
181 below must be changed to output_move_neon (which will use the
182 element/structure loads/stores), and the constraint changed to 'Um' instead
183 of 'Uv'. */
185 switch (which_alternative)
186 {
187 case 0: return "vmov\t%P0, %P1 @ <mode>";
188 case 1: case 3: return output_move_vfp (operands);
189 case 2: gcc_unreachable ();
190 case 4: return "vmov\t%Q0, %R0, %P1 @ <mode>";
191 case 5: return "vmov\t%P0, %Q1, %R1 @ <mode>";
192 default: return output_move_double (operands);
193 }
194 }
195 [(set_attr "neon_type" "neon_int_1,*,neon_vmov,*,neon_mrrc,neon_mcr_2_mcrr,*,*,*")
196 (set_attr "type" "*,f_stored,*,f_loadd,*,*,alu,load2,store2")
197 (set_attr "insn" "*,*,*,*,*,*,mov,*,*")
198 (set_attr "length" "4,4,4,4,4,4,8,8,8")
199 (set_attr "pool_range" "*,*,*,1020,*,*,*,1020,*")
200 (set_attr "neg_pool_range" "*,*,*,1008,*,*,*,1008,*")])
202 (define_insn "*neon_mov<mode>"
203 [(set (match_operand:VQXMOV 0 "nonimmediate_operand"
204 "=w,Un,w, w, ?r,?w,?r,?r, ?Us")
205 (match_operand:VQXMOV 1 "general_operand"
206 " w,w, Dn,Uni, w, r, r, Usi, r"))]
207 "TARGET_NEON
208 && (register_operand (operands[0], <MODE>mode)
209 || register_operand (operands[1], <MODE>mode))"
210 {
211 if (which_alternative == 2)
212 {
213 int width, is_valid;
214 static char templ[40];
216 is_valid = neon_immediate_valid_for_move (operands[1], <MODE>mode,
217 &operands[1], &width);
219 gcc_assert (is_valid != 0);
221 if (width == 0)
222 return "vmov.f32\t%q0, %1 @ <mode>";
223 else
224 sprintf (templ, "vmov.i%d\t%%q0, %%1 @ <mode>", width);
226 return templ;
227 }
229 switch (which_alternative)
230 {
231 case 0: return "vmov\t%q0, %q1 @ <mode>";
232 case 1: case 3: return output_move_neon (operands);
233 case 2: gcc_unreachable ();
234 case 4: return "vmov\t%Q0, %R0, %e1 @ <mode>\;vmov\t%J0, %K0, %f1";
235 case 5: return "vmov\t%e0, %Q1, %R1 @ <mode>\;vmov\t%f0, %J1, %K1";
236 default: return output_move_quad (operands);
237 }
238 }
239 [(set_attr "neon_type" "neon_int_1,neon_stm_2,neon_vmov,neon_ldm_2,\
240 neon_mrrc,neon_mcr_2_mcrr,*,*,*")
241 (set_attr "type" "*,*,*,*,*,*,alu,load4,store4")
242 (set_attr "insn" "*,*,*,*,*,*,mov,*,*")
243 (set_attr "length" "4,8,4,8,8,8,16,8,16")
244 (set_attr "pool_range" "*,*,*,1020,*,*,*,1020,*")
245 (set_attr "neg_pool_range" "*,*,*,1008,*,*,*,1008,*")])
247 (define_expand "movti"
248 [(set (match_operand:TI 0 "nonimmediate_operand" "")
249 (match_operand:TI 1 "general_operand" ""))]
250 "TARGET_NEON"
251 {
252 if (can_create_pseudo_p ())
253 {
254 if (GET_CODE (operands[0]) != REG)
255 operands[1] = force_reg (TImode, operands[1]);
256 }
257 })
259 (define_expand "mov<mode>"
260 [(set (match_operand:VSTRUCT 0 "nonimmediate_operand" "")
261 (match_operand:VSTRUCT 1 "general_operand" ""))]
262 "TARGET_NEON"
263 {
264 if (can_create_pseudo_p ())
265 {
266 if (GET_CODE (operands[0]) != REG)
267 operands[1] = force_reg (<MODE>mode, operands[1]);
268 }
269 })
271 (define_insn "*neon_mov<mode>"
272 [(set (match_operand:VSTRUCT 0 "nonimmediate_operand" "=w,Ut,w")
273 (match_operand:VSTRUCT 1 "general_operand" " w,w, Ut"))]
274 "TARGET_NEON
275 && (register_operand (operands[0], <MODE>mode)
276 || register_operand (operands[1], <MODE>mode))"
277 {
278 switch (which_alternative)
279 {
280 case 0: return "#";
281 case 1: case 2: return output_move_neon (operands);
282 default: gcc_unreachable ();
283 }
284 }
285 [(set_attr "neon_type" "neon_int_1,neon_stm_2,neon_ldm_2")
286 (set (attr "length") (symbol_ref "arm_attr_length_move_neon (insn)"))])
288 (define_split
289 [(set (match_operand:EI 0 "s_register_operand" "")
290 (match_operand:EI 1 "s_register_operand" ""))]
291 "TARGET_NEON && reload_completed"
292 [(set (match_dup 0) (match_dup 1))
293 (set (match_dup 2) (match_dup 3))]
294 {
295 int rdest = REGNO (operands[0]);
296 int rsrc = REGNO (operands[1]);
297 rtx dest[2], src[2];
299 dest[0] = gen_rtx_REG (TImode, rdest);
300 src[0] = gen_rtx_REG (TImode, rsrc);
301 dest[1] = gen_rtx_REG (DImode, rdest + 4);
302 src[1] = gen_rtx_REG (DImode, rsrc + 4);
304 neon_disambiguate_copy (operands, dest, src, 2);
305 })
307 (define_split
308 [(set (match_operand:OI 0 "s_register_operand" "")
309 (match_operand:OI 1 "s_register_operand" ""))]
310 "TARGET_NEON && reload_completed"
311 [(set (match_dup 0) (match_dup 1))
312 (set (match_dup 2) (match_dup 3))]
313 {
314 int rdest = REGNO (operands[0]);
315 int rsrc = REGNO (operands[1]);
316 rtx dest[2], src[2];
318 dest[0] = gen_rtx_REG (TImode, rdest);
319 src[0] = gen_rtx_REG (TImode, rsrc);
320 dest[1] = gen_rtx_REG (TImode, rdest + 4);
321 src[1] = gen_rtx_REG (TImode, rsrc + 4);
323 neon_disambiguate_copy (operands, dest, src, 2);
324 })
326 (define_split
327 [(set (match_operand:CI 0 "s_register_operand" "")
328 (match_operand:CI 1 "s_register_operand" ""))]
329 "TARGET_NEON && reload_completed"
330 [(set (match_dup 0) (match_dup 1))
331 (set (match_dup 2) (match_dup 3))
332 (set (match_dup 4) (match_dup 5))]
333 {
334 int rdest = REGNO (operands[0]);
335 int rsrc = REGNO (operands[1]);
336 rtx dest[3], src[3];
338 dest[0] = gen_rtx_REG (TImode, rdest);
339 src[0] = gen_rtx_REG (TImode, rsrc);
340 dest[1] = gen_rtx_REG (TImode, rdest + 4);
341 src[1] = gen_rtx_REG (TImode, rsrc + 4);
342 dest[2] = gen_rtx_REG (TImode, rdest + 8);
343 src[2] = gen_rtx_REG (TImode, rsrc + 8);
345 neon_disambiguate_copy (operands, dest, src, 3);
346 })
348 (define_split
349 [(set (match_operand:XI 0 "s_register_operand" "")
350 (match_operand:XI 1 "s_register_operand" ""))]
351 "TARGET_NEON && reload_completed"
352 [(set (match_dup 0) (match_dup 1))
353 (set (match_dup 2) (match_dup 3))
354 (set (match_dup 4) (match_dup 5))
355 (set (match_dup 6) (match_dup 7))]
356 {
357 int rdest = REGNO (operands[0]);
358 int rsrc = REGNO (operands[1]);
359 rtx dest[4], src[4];
361 dest[0] = gen_rtx_REG (TImode, rdest);
362 src[0] = gen_rtx_REG (TImode, rsrc);
363 dest[1] = gen_rtx_REG (TImode, rdest + 4);
364 src[1] = gen_rtx_REG (TImode, rsrc + 4);
365 dest[2] = gen_rtx_REG (TImode, rdest + 8);
366 src[2] = gen_rtx_REG (TImode, rsrc + 8);
367 dest[3] = gen_rtx_REG (TImode, rdest + 12);
368 src[3] = gen_rtx_REG (TImode, rsrc + 12);
370 neon_disambiguate_copy (operands, dest, src, 4);
371 })
373 (define_expand "movmisalign<mode>"
374 [(set (match_operand:VDQX 0 "nonimmediate_operand" "")
375 (unspec:VDQX [(match_operand:VDQX 1 "general_operand" "")]
376 UNSPEC_MISALIGNED_ACCESS))]
377 "TARGET_NEON && !BYTES_BIG_ENDIAN"
378 {
379 /* This pattern is not permitted to fail during expansion: if both arguments
380 are non-registers (e.g. memory := constant, which can be created by the
381 auto-vectorizer), force operand 1 into a register. */
382 if (!s_register_operand (operands[0], <MODE>mode)
383 && !s_register_operand (operands[1], <MODE>mode))
384 operands[1] = force_reg (<MODE>mode, operands[1]);
385 })
387 (define_insn "*movmisalign<mode>_neon_store"
388 [(set (match_operand:VDX 0 "memory_operand" "=Um")
389 (unspec:VDX [(match_operand:VDX 1 "s_register_operand" " w")]
390 UNSPEC_MISALIGNED_ACCESS))]
391 "TARGET_NEON && !BYTES_BIG_ENDIAN"
392 "vst1.<V_sz_elem>\t{%P1}, %A0"
393 [(set_attr "neon_type" "neon_vst1_1_2_regs_vst2_2_regs")])
395 (define_insn "*movmisalign<mode>_neon_load"
396 [(set (match_operand:VDX 0 "s_register_operand" "=w")
397 (unspec:VDX [(match_operand:VDX 1 "memory_operand" " Um")]
398 UNSPEC_MISALIGNED_ACCESS))]
399 "TARGET_NEON && !BYTES_BIG_ENDIAN"
400 "vld1.<V_sz_elem>\t{%P0}, %A1"
401 [(set_attr "neon_type" "neon_vld1_1_2_regs")])
403 (define_insn "*movmisalign<mode>_neon_store"
404 [(set (match_operand:VQX 0 "memory_operand" "=Um")
405 (unspec:VQX [(match_operand:VQX 1 "s_register_operand" " w")]
406 UNSPEC_MISALIGNED_ACCESS))]
407 "TARGET_NEON && !BYTES_BIG_ENDIAN"
408 "vst1.<V_sz_elem>\t{%q1}, %A0"
409 [(set_attr "neon_type" "neon_vst1_1_2_regs_vst2_2_regs")])
411 (define_insn "*movmisalign<mode>_neon_load"
412 [(set (match_operand:VQX 0 "s_register_operand" "=w")
413 (unspec:VQX [(match_operand:VQX 1 "memory_operand" " Um")]
414 UNSPEC_MISALIGNED_ACCESS))]
415 "TARGET_NEON && !BYTES_BIG_ENDIAN"
416 "vld1.<V_sz_elem>\t{%q0}, %A1"
417 [(set_attr "neon_type" "neon_vld1_1_2_regs")])
419 (define_insn "vec_set<mode>_internal"
420 [(set (match_operand:VD 0 "s_register_operand" "=w")
421 (vec_merge:VD
422 (vec_duplicate:VD
423 (match_operand:<V_elem> 1 "s_register_operand" "r"))
424 (match_operand:VD 3 "s_register_operand" "0")
425 (match_operand:SI 2 "immediate_operand" "i")))]
426 "TARGET_NEON"
427 {
428 int elt = ffs ((int) INTVAL (operands[2]) - 1);
429 if (BYTES_BIG_ENDIAN)
430 elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
431 operands[2] = GEN_INT (elt);
433 return "vmov%?.<V_sz_elem>\t%P0[%c2], %1";
434 }
435 [(set_attr "predicable" "yes")
436 (set_attr "neon_type" "neon_mcr")])
438 (define_insn "vec_set<mode>_internal"
439 [(set (match_operand:VQ 0 "s_register_operand" "=w")
440 (vec_merge:VQ
441 (vec_duplicate:VQ
442 (match_operand:<V_elem> 1 "s_register_operand" "r"))
443 (match_operand:VQ 3 "s_register_operand" "0")
444 (match_operand:SI 2 "immediate_operand" "i")))]
445 "TARGET_NEON"
446 {
447 HOST_WIDE_INT elem = ffs ((int) INTVAL (operands[2])) - 1;
448 int half_elts = GET_MODE_NUNITS (<MODE>mode) / 2;
449 int elt = elem % half_elts;
450 int hi = (elem / half_elts) * 2;
451 int regno = REGNO (operands[0]);
453 if (BYTES_BIG_ENDIAN)
454 elt = half_elts - 1 - elt;
456 operands[0] = gen_rtx_REG (<V_HALF>mode, regno + hi);
457 operands[2] = GEN_INT (elt);
459 return "vmov%?.<V_sz_elem>\t%P0[%c2], %1";
460 }
461 [(set_attr "predicable" "yes")
462 (set_attr "neon_type" "neon_mcr")]
463 )
465 (define_insn "vec_setv2di_internal"
466 [(set (match_operand:V2DI 0 "s_register_operand" "=w")
467 (vec_merge:V2DI
468 (vec_duplicate:V2DI
469 (match_operand:DI 1 "s_register_operand" "r"))
470 (match_operand:V2DI 3 "s_register_operand" "0")
471 (match_operand:SI 2 "immediate_operand" "i")))]
472 "TARGET_NEON"
473 {
474 HOST_WIDE_INT elem = ffs ((int) INTVAL (operands[2])) - 1;
475 int regno = REGNO (operands[0]) + 2 * elem;
477 operands[0] = gen_rtx_REG (DImode, regno);
479 return "vmov%?\t%P0, %Q1, %R1";
480 }
481 [(set_attr "predicable" "yes")
482 (set_attr "neon_type" "neon_mcr_2_mcrr")]
483 )
485 (define_expand "vec_set<mode>"
486 [(match_operand:VDQ 0 "s_register_operand" "")
487 (match_operand:<V_elem> 1 "s_register_operand" "")
488 (match_operand:SI 2 "immediate_operand" "")]
489 "TARGET_NEON"
490 {
491 HOST_WIDE_INT elem = (HOST_WIDE_INT) 1 << INTVAL (operands[2]);
492 emit_insn (gen_vec_set<mode>_internal (operands[0], operands[1],
493 GEN_INT (elem), operands[0]));
494 DONE;
495 })
497 (define_insn "vec_extract<mode>"
498 [(set (match_operand:<V_elem> 0 "s_register_operand" "=r")
499 (vec_select:<V_elem>
500 (match_operand:VD 1 "s_register_operand" "w")
501 (parallel [(match_operand:SI 2 "immediate_operand" "i")])))]
502 "TARGET_NEON"
503 {
504 if (BYTES_BIG_ENDIAN)
505 {
506 int elt = INTVAL (operands[2]);
507 elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
508 operands[2] = GEN_INT (elt);
509 }
510 return "vmov%?.<V_uf_sclr>\t%0, %P1[%c2]";
511 }
512 [(set_attr "predicable" "yes")
513 (set_attr "neon_type" "neon_bp_simple")]
514 )
516 (define_insn "vec_extract<mode>"
517 [(set (match_operand:<V_elem> 0 "s_register_operand" "=r")
518 (vec_select:<V_elem>
519 (match_operand:VQ 1 "s_register_operand" "w")
520 (parallel [(match_operand:SI 2 "immediate_operand" "i")])))]
521 "TARGET_NEON"
522 {
523 int half_elts = GET_MODE_NUNITS (<MODE>mode) / 2;
524 int elt = INTVAL (operands[2]) % half_elts;
525 int hi = (INTVAL (operands[2]) / half_elts) * 2;
526 int regno = REGNO (operands[1]);
528 if (BYTES_BIG_ENDIAN)
529 elt = half_elts - 1 - elt;
531 operands[1] = gen_rtx_REG (<V_HALF>mode, regno + hi);
532 operands[2] = GEN_INT (elt);
534 return "vmov%?.<V_uf_sclr>\t%0, %P1[%c2]";
535 }
536 [(set_attr "predicable" "yes")
537 (set_attr "neon_type" "neon_bp_simple")]
538 )
540 (define_insn "vec_extractv2di"
541 [(set (match_operand:DI 0 "s_register_operand" "=r")
542 (vec_select:DI
543 (match_operand:V2DI 1 "s_register_operand" "w")
544 (parallel [(match_operand:SI 2 "immediate_operand" "i")])))]
545 "TARGET_NEON"
546 {
547 int regno = REGNO (operands[1]) + 2 * INTVAL (operands[2]);
549 operands[1] = gen_rtx_REG (DImode, regno);
551 return "vmov%?\t%Q0, %R0, %P1 @ v2di";
552 }
553 [(set_attr "predicable" "yes")
554 (set_attr "neon_type" "neon_int_1")]
555 )
557 (define_expand "vec_init<mode>"
558 [(match_operand:VDQ 0 "s_register_operand" "")
559 (match_operand 1 "" "")]
560 "TARGET_NEON"
561 {
562 neon_expand_vector_init (operands[0], operands[1]);
563 DONE;
564 })
566 ;; Doubleword and quadword arithmetic.
568 ;; NOTE: some other instructions also support 64-bit integer
569 ;; element size, which we could potentially use for "long long" operations.
571 (define_insn "*add<mode>3_neon"
572 [(set (match_operand:VDQ 0 "s_register_operand" "=w")
573 (plus:VDQ (match_operand:VDQ 1 "s_register_operand" "w")
574 (match_operand:VDQ 2 "s_register_operand" "w")))]
575 "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
576 "vadd.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
577 [(set (attr "neon_type")
578 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
579 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
580 (const_string "neon_fp_vadd_ddd_vabs_dd")
581 (const_string "neon_fp_vadd_qqq_vabs_qq"))
582 (const_string "neon_int_1")))]
583 )
585 (define_insn "adddi3_neon"
586 [(set (match_operand:DI 0 "s_register_operand" "=w,?&r,?&r,?w")
587 (plus:DI (match_operand:DI 1 "s_register_operand" "%w,0,0,w")
588 (match_operand:DI 2 "s_register_operand" "w,r,0,w")))
589 (clobber (reg:CC CC_REGNUM))]
590 "TARGET_NEON"
591 {
592 switch (which_alternative)
593 {
594 case 0: /* fall through */
595 case 3: return "vadd.i64\t%P0, %P1, %P2";
596 case 1: return "#";
597 case 2: return "#";
598 default: gcc_unreachable ();
599 }
600 }
601 [(set_attr "neon_type" "neon_int_1,*,*,neon_int_1")
602 (set_attr "conds" "*,clob,clob,*")
603 (set_attr "length" "*,8,8,*")
604 (set_attr "arch" "nota8,*,*,onlya8")]
605 )
607 (define_insn "*sub<mode>3_neon"
608 [(set (match_operand:VDQ 0 "s_register_operand" "=w")
609 (minus:VDQ (match_operand:VDQ 1 "s_register_operand" "w")
610 (match_operand:VDQ 2 "s_register_operand" "w")))]
611 "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
612 "vsub.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
613 [(set (attr "neon_type")
614 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
615 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
616 (const_string "neon_fp_vadd_ddd_vabs_dd")
617 (const_string "neon_fp_vadd_qqq_vabs_qq"))
618 (const_string "neon_int_2")))]
619 )
621 (define_insn "subdi3_neon"
622 [(set (match_operand:DI 0 "s_register_operand" "=w,?&r,?&r,?&r,?w")
623 (minus:DI (match_operand:DI 1 "s_register_operand" "w,0,r,0,w")
624 (match_operand:DI 2 "s_register_operand" "w,r,0,0,w")))
625 (clobber (reg:CC CC_REGNUM))]
626 "TARGET_NEON"
627 {
628 switch (which_alternative)
629 {
630 case 0: /* fall through */
631 case 4: return "vsub.i64\t%P0, %P1, %P2";
632 case 1: /* fall through */
633 case 2: /* fall through */
634 case 3: return "subs\\t%Q0, %Q1, %Q2\;sbc\\t%R0, %R1, %R2";
635 default: gcc_unreachable ();
636 }
637 }
638 [(set_attr "neon_type" "neon_int_2,*,*,*,neon_int_2")
639 (set_attr "conds" "*,clob,clob,clob,*")
640 (set_attr "length" "*,8,8,8,*")
641 (set_attr "arch" "nota8,*,*,*,onlya8")]
642 )
644 (define_insn "*mul<mode>3_neon"
645 [(set (match_operand:VDQ 0 "s_register_operand" "=w")
646 (mult:VDQ (match_operand:VDQ 1 "s_register_operand" "w")
647 (match_operand:VDQ 2 "s_register_operand" "w")))]
648 "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
649 "vmul.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
650 [(set (attr "neon_type")
651 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
652 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
653 (const_string "neon_fp_vadd_ddd_vabs_dd")
654 (const_string "neon_fp_vadd_qqq_vabs_qq"))
655 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
656 (if_then_else
657 (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0))
658 (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long")
659 (const_string "neon_mul_qqq_8_16_32_ddd_32"))
660 (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0))
661 (const_string "neon_mul_qqq_8_16_32_ddd_32")
662 (const_string "neon_mul_qqq_8_16_32_ddd_32")))))]
663 )
665 (define_insn "mul<mode>3add<mode>_neon"
666 [(set (match_operand:VDQ 0 "s_register_operand" "=w")
667 (plus:VDQ (mult:VDQ (match_operand:VDQ 2 "s_register_operand" "w")
668 (match_operand:VDQ 3 "s_register_operand" "w"))
669 (match_operand:VDQ 1 "s_register_operand" "0")))]
670 "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
671 "vmla.<V_if_elem>\t%<V_reg>0, %<V_reg>2, %<V_reg>3"
672 [(set (attr "neon_type")
673 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
674 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
675 (const_string "neon_fp_vmla_ddd")
676 (const_string "neon_fp_vmla_qqq"))
677 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
678 (if_then_else
679 (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0))
680 (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
681 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long"))
682 (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0))
683 (const_string "neon_mla_qqq_8_16")
684 (const_string "neon_mla_qqq_32_qqd_32_scalar")))))]
685 )
687 (define_insn "mul<mode>3neg<mode>add<mode>_neon"
688 [(set (match_operand:VDQ 0 "s_register_operand" "=w")
689 (minus:VDQ (match_operand:VDQ 1 "s_register_operand" "0")
690 (mult:VDQ (match_operand:VDQ 2 "s_register_operand" "w")
691 (match_operand:VDQ 3 "s_register_operand" "w"))))]
692 "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
693 "vmls.<V_if_elem>\t%<V_reg>0, %<V_reg>2, %<V_reg>3"
694 [(set (attr "neon_type")
695 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
696 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
697 (const_string "neon_fp_vmla_ddd")
698 (const_string "neon_fp_vmla_qqq"))
699 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
700 (if_then_else
701 (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0))
702 (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
703 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long"))
704 (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0))
705 (const_string "neon_mla_qqq_8_16")
706 (const_string "neon_mla_qqq_32_qqd_32_scalar")))))]
707 )
709 (define_insn "ior<mode>3"
710 [(set (match_operand:VDQ 0 "s_register_operand" "=w,w")
711 (ior:VDQ (match_operand:VDQ 1 "s_register_operand" "w,0")
712 (match_operand:VDQ 2 "neon_logic_op2" "w,Dl")))]
713 "TARGET_NEON"
714 {
715 switch (which_alternative)
716 {
717 case 0: return "vorr\t%<V_reg>0, %<V_reg>1, %<V_reg>2";
718 case 1: return neon_output_logic_immediate ("vorr", &operands[2],
719 <MODE>mode, 0, VALID_NEON_QREG_MODE (<MODE>mode));
720 default: gcc_unreachable ();
721 }
722 }
723 [(set_attr "neon_type" "neon_int_1")]
724 )
726 (define_insn "iordi3_neon"
727 [(set (match_operand:DI 0 "s_register_operand" "=w,w,?&r,?&r,?w,?w")
728 (ior:DI (match_operand:DI 1 "s_register_operand" "%w,0,0,r,w,0")
729 (match_operand:DI 2 "neon_logic_op2" "w,Dl,r,r,w,Dl")))]
730 "TARGET_NEON"
731 {
732 switch (which_alternative)
733 {
734 case 0: /* fall through */
735 case 4: return "vorr\t%P0, %P1, %P2";
736 case 1: /* fall through */
737 case 5: return neon_output_logic_immediate ("vorr", &operands[2],
738 DImode, 0, VALID_NEON_QREG_MODE (DImode));
739 case 2: return "#";
740 case 3: return "#";
741 default: gcc_unreachable ();
742 }
743 }
744 [(set_attr "neon_type" "neon_int_1,neon_int_1,*,*,neon_int_1,neon_int_1")
745 (set_attr "length" "*,*,8,8,*,*")
746 (set_attr "arch" "nota8,nota8,*,*,onlya8,onlya8")]
747 )
749 ;; The concrete forms of the Neon immediate-logic instructions are vbic and
750 ;; vorr. We support the pseudo-instruction vand instead, because that
751 ;; corresponds to the canonical form the middle-end expects to use for
752 ;; immediate bitwise-ANDs.
754 (define_insn "and<mode>3"
755 [(set (match_operand:VDQ 0 "s_register_operand" "=w,w")
756 (and:VDQ (match_operand:VDQ 1 "s_register_operand" "w,0")
757 (match_operand:VDQ 2 "neon_inv_logic_op2" "w,DL")))]
758 "TARGET_NEON"
759 {
760 switch (which_alternative)
761 {
762 case 0: return "vand\t%<V_reg>0, %<V_reg>1, %<V_reg>2";
763 case 1: return neon_output_logic_immediate ("vand", &operands[2],
764 <MODE>mode, 1, VALID_NEON_QREG_MODE (<MODE>mode));
765 default: gcc_unreachable ();
766 }
767 }
768 [(set_attr "neon_type" "neon_int_1")]
769 )
771 (define_insn "anddi3_neon"
772 [(set (match_operand:DI 0 "s_register_operand" "=w,w,?&r,?&r,?w,?w")
773 (and:DI (match_operand:DI 1 "s_register_operand" "%w,0,0,r,w,0")
774 (match_operand:DI 2 "neon_inv_logic_op2" "w,DL,r,r,w,DL")))]
775 "TARGET_NEON"
776 {
777 switch (which_alternative)
778 {
779 case 0: /* fall through */
780 case 4: return "vand\t%P0, %P1, %P2";
781 case 1: /* fall through */
782 case 5: return neon_output_logic_immediate ("vand", &operands[2],
783 DImode, 1, VALID_NEON_QREG_MODE (DImode));
784 case 2: return "#";
785 case 3: return "#";
786 default: gcc_unreachable ();
787 }
788 }
789 [(set_attr "neon_type" "neon_int_1,neon_int_1,*,*,neon_int_1,neon_int_1")
790 (set_attr "length" "*,*,8,8,*,*")
791 (set_attr "arch" "nota8,nota8,*,*,onlya8,onlya8")]
792 )
794 (define_insn "orn<mode>3_neon"
795 [(set (match_operand:VDQ 0 "s_register_operand" "=w")
796 (ior:VDQ (match_operand:VDQ 1 "s_register_operand" "w")
797 (not:VDQ (match_operand:VDQ 2 "s_register_operand" "w"))))]
798 "TARGET_NEON"
799 "vorn\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
800 [(set_attr "neon_type" "neon_int_1")]
801 )
803 (define_insn "orndi3_neon"
804 [(set (match_operand:DI 0 "s_register_operand" "=w,?=&r,?&r")
805 (ior:DI (match_operand:DI 1 "s_register_operand" "w,r,0")
806 (not:DI (match_operand:DI 2 "s_register_operand" "w,0,r"))))]
807 "TARGET_NEON"
808 "@
809 vorn\t%P0, %P1, %P2
810 #
811 #"
812 [(set_attr "neon_type" "neon_int_1,*,*")
813 (set_attr "length" "*,8,8")]
814 )
816 (define_insn "bic<mode>3_neon"
817 [(set (match_operand:VDQ 0 "s_register_operand" "=w")
818 (and:VDQ (match_operand:VDQ 1 "s_register_operand" "w")
819 (not:VDQ (match_operand:VDQ 2 "s_register_operand" "w"))))]
820 "TARGET_NEON"
821 "vbic\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
822 [(set_attr "neon_type" "neon_int_1")]
823 )
825 ;; Compare to *anddi_notdi_di.
826 (define_insn "bicdi3_neon"
827 [(set (match_operand:DI 0 "s_register_operand" "=w,?=&r,?&r")
828 (and:DI (not:DI (match_operand:DI 2 "s_register_operand" "w,r,0"))
829 (match_operand:DI 1 "s_register_operand" "w,0,r")))]
830 "TARGET_NEON"
831 "@
832 vbic\t%P0, %P1, %P2
833 #
834 #"
835 [(set_attr "neon_type" "neon_int_1,*,*")
836 (set_attr "length" "*,8,8")]
837 )
839 (define_insn "xor<mode>3"
840 [(set (match_operand:VDQ 0 "s_register_operand" "=w")
841 (xor:VDQ (match_operand:VDQ 1 "s_register_operand" "w")
842 (match_operand:VDQ 2 "s_register_operand" "w")))]
843 "TARGET_NEON"
844 "veor\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
845 [(set_attr "neon_type" "neon_int_1")]
846 )
848 (define_insn "xordi3_neon"
849 [(set (match_operand:DI 0 "s_register_operand" "=w,?&r,?&r,?w")
850 (xor:DI (match_operand:DI 1 "s_register_operand" "%w,0,r,w")
851 (match_operand:DI 2 "s_register_operand" "w,r,r,w")))]
852 "TARGET_NEON"
853 "@
854 veor\t%P0, %P1, %P2
855 #
856 #
857 veor\t%P0, %P1, %P2"
858 [(set_attr "neon_type" "neon_int_1,*,*,neon_int_1")
859 (set_attr "length" "*,8,8,*")
860 (set_attr "arch" "nota8,*,*,onlya8")]
861 )
863 (define_insn "one_cmpl<mode>2"
864 [(set (match_operand:VDQ 0 "s_register_operand" "=w")
865 (not:VDQ (match_operand:VDQ 1 "s_register_operand" "w")))]
866 "TARGET_NEON"
867 "vmvn\t%<V_reg>0, %<V_reg>1"
868 [(set_attr "neon_type" "neon_int_1")]
869 )
871 (define_insn "abs<mode>2"
872 [(set (match_operand:VDQW 0 "s_register_operand" "=w")
873 (abs:VDQW (match_operand:VDQW 1 "s_register_operand" "w")))]
874 "TARGET_NEON"
875 "vabs.<V_s_elem>\t%<V_reg>0, %<V_reg>1"
876 [(set (attr "neon_type")
877 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
878 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
879 (const_string "neon_fp_vadd_ddd_vabs_dd")
880 (const_string "neon_fp_vadd_qqq_vabs_qq"))
881 (const_string "neon_int_3")))]
882 )
884 (define_insn "neg<mode>2"
885 [(set (match_operand:VDQW 0 "s_register_operand" "=w")
886 (neg:VDQW (match_operand:VDQW 1 "s_register_operand" "w")))]
887 "TARGET_NEON"
888 "vneg.<V_s_elem>\t%<V_reg>0, %<V_reg>1"
889 [(set (attr "neon_type")
890 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
891 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
892 (const_string "neon_fp_vadd_ddd_vabs_dd")
893 (const_string "neon_fp_vadd_qqq_vabs_qq"))
894 (const_string "neon_int_3")))]
895 )
897 (define_insn "*umin<mode>3_neon"
898 [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
899 (umin:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w")
900 (match_operand:VDQIW 2 "s_register_operand" "w")))]
901 "TARGET_NEON"
902 "vmin.<V_u_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
903 [(set_attr "neon_type" "neon_int_5")]
904 )
906 (define_insn "*umax<mode>3_neon"
907 [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
908 (umax:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w")
909 (match_operand:VDQIW 2 "s_register_operand" "w")))]
910 "TARGET_NEON"
911 "vmax.<V_u_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
912 [(set_attr "neon_type" "neon_int_5")]
913 )
915 (define_insn "*smin<mode>3_neon"
916 [(set (match_operand:VDQW 0 "s_register_operand" "=w")
917 (smin:VDQW (match_operand:VDQW 1 "s_register_operand" "w")
918 (match_operand:VDQW 2 "s_register_operand" "w")))]
919 "TARGET_NEON"
920 "vmin.<V_s_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
921 [(set (attr "neon_type")
922 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
923 (const_string "neon_fp_vadd_ddd_vabs_dd")
924 (const_string "neon_int_5")))]
925 )
927 (define_insn "*smax<mode>3_neon"
928 [(set (match_operand:VDQW 0 "s_register_operand" "=w")
929 (smax:VDQW (match_operand:VDQW 1 "s_register_operand" "w")
930 (match_operand:VDQW 2 "s_register_operand" "w")))]
931 "TARGET_NEON"
932 "vmax.<V_s_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
933 [(set (attr "neon_type")
934 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
935 (const_string "neon_fp_vadd_ddd_vabs_dd")
936 (const_string "neon_int_5")))]
937 )
939 ; TODO: V2DI shifts are current disabled because there are bugs in the
940 ; generic vectorizer code. It ends up creating a V2DI constructor with
941 ; SImode elements.
943 (define_insn "vashl<mode>3"
944 [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
945 (ashift:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w")
946 (match_operand:VDQIW 2 "s_register_operand" "w")))]
947 "TARGET_NEON"
948 "vshl.<V_s_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
949 [(set (attr "neon_type")
950 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
951 (const_string "neon_vshl_ddd")
952 (const_string "neon_shift_3")))]
953 )
955 ; Used for implementing logical shift-right, which is a left-shift by a negative
956 ; amount, with signed operands. This is essentially the same as ashl<mode>3
957 ; above, but using an unspec in case GCC tries anything tricky with negative
958 ; shift amounts.
960 (define_insn "ashl<mode>3_signed"
961 [(set (match_operand:VDQI 0 "s_register_operand" "=w")
962 (unspec:VDQI [(match_operand:VDQI 1 "s_register_operand" "w")
963 (match_operand:VDQI 2 "s_register_operand" "w")]
964 UNSPEC_ASHIFT_SIGNED))]
965 "TARGET_NEON"
966 "vshl.<V_s_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
967 [(set (attr "neon_type")
968 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
969 (const_string "neon_vshl_ddd")
970 (const_string "neon_shift_3")))]
971 )
973 ; Used for implementing logical shift-right, which is a left-shift by a negative
974 ; amount, with unsigned operands.
976 (define_insn "ashl<mode>3_unsigned"
977 [(set (match_operand:VDQI 0 "s_register_operand" "=w")
978 (unspec:VDQI [(match_operand:VDQI 1 "s_register_operand" "w")
979 (match_operand:VDQI 2 "s_register_operand" "w")]
980 UNSPEC_ASHIFT_UNSIGNED))]
981 "TARGET_NEON"
982 "vshl.<V_u_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
983 [(set (attr "neon_type")
984 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
985 (const_string "neon_vshl_ddd")
986 (const_string "neon_shift_3")))]
987 )
989 (define_expand "vashr<mode>3"
990 [(set (match_operand:VDQIW 0 "s_register_operand" "")
991 (ashiftrt:VDQIW (match_operand:VDQIW 1 "s_register_operand" "")
992 (match_operand:VDQIW 2 "s_register_operand" "")))]
993 "TARGET_NEON"
994 {
995 rtx neg = gen_reg_rtx (<MODE>mode);
997 emit_insn (gen_neg<mode>2 (neg, operands[2]));
998 emit_insn (gen_ashl<mode>3_signed (operands[0], operands[1], neg));
1000 DONE;
1001 })
1003 (define_expand "vlshr<mode>3"
1004 [(set (match_operand:VDQIW 0 "s_register_operand" "")
1005 (lshiftrt:VDQIW (match_operand:VDQIW 1 "s_register_operand" "")
1006 (match_operand:VDQIW 2 "s_register_operand" "")))]
1007 "TARGET_NEON"
1008 {
1009 rtx neg = gen_reg_rtx (<MODE>mode);
1011 emit_insn (gen_neg<mode>2 (neg, operands[2]));
1012 emit_insn (gen_ashl<mode>3_unsigned (operands[0], operands[1], neg));
1014 DONE;
1015 })
1017 ;; Widening operations
1019 (define_insn "widen_ssum<mode>3"
1020 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
1021 (plus:<V_widen> (sign_extend:<V_widen>
1022 (match_operand:VW 1 "s_register_operand" "%w"))
1023 (match_operand:<V_widen> 2 "s_register_operand" "w")))]
1024 "TARGET_NEON"
1025 "vaddw.<V_s_elem>\t%q0, %q2, %P1"
1026 [(set_attr "neon_type" "neon_int_3")]
1027 )
1029 (define_insn "widen_usum<mode>3"
1030 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
1031 (plus:<V_widen> (zero_extend:<V_widen>
1032 (match_operand:VW 1 "s_register_operand" "%w"))
1033 (match_operand:<V_widen> 2 "s_register_operand" "w")))]
1034 "TARGET_NEON"
1035 "vaddw.<V_u_elem>\t%q0, %q2, %P1"
1036 [(set_attr "neon_type" "neon_int_3")]
1037 )
1039 ;; VEXT can be used to synthesize coarse whole-vector shifts with 8-bit
1040 ;; shift-count granularity. That's good enough for the middle-end's current
1041 ;; needs.
1043 (define_expand "vec_shr_<mode>"
1044 [(match_operand:VDQ 0 "s_register_operand" "")
1045 (match_operand:VDQ 1 "s_register_operand" "")
1046 (match_operand:SI 2 "const_multiple_of_8_operand" "")]
1047 "TARGET_NEON"
1048 {
1049 rtx zero_reg;
1050 HOST_WIDE_INT num_bits = INTVAL (operands[2]);
1051 const int width = GET_MODE_BITSIZE (<MODE>mode);
1052 const enum machine_mode bvecmode = (width == 128) ? V16QImode : V8QImode;
1053 rtx (*gen_ext) (rtx, rtx, rtx, rtx) =
1054 (width == 128) ? gen_neon_vextv16qi : gen_neon_vextv8qi;
1056 if (num_bits == width)
1057 {
1058 emit_move_insn (operands[0], operands[1]);
1059 DONE;
1060 }
1062 zero_reg = force_reg (bvecmode, CONST0_RTX (bvecmode));
1063 operands[0] = gen_lowpart (bvecmode, operands[0]);
1064 operands[1] = gen_lowpart (bvecmode, operands[1]);
1066 emit_insn (gen_ext (operands[0], operands[1], zero_reg,
1067 GEN_INT (num_bits / BITS_PER_UNIT)));
1068 DONE;
1069 })
1071 (define_expand "vec_shl_<mode>"
1072 [(match_operand:VDQ 0 "s_register_operand" "")
1073 (match_operand:VDQ 1 "s_register_operand" "")
1074 (match_operand:SI 2 "const_multiple_of_8_operand" "")]
1075 "TARGET_NEON"
1076 {
1077 rtx zero_reg;
1078 HOST_WIDE_INT num_bits = INTVAL (operands[2]);
1079 const int width = GET_MODE_BITSIZE (<MODE>mode);
1080 const enum machine_mode bvecmode = (width == 128) ? V16QImode : V8QImode;
1081 rtx (*gen_ext) (rtx, rtx, rtx, rtx) =
1082 (width == 128) ? gen_neon_vextv16qi : gen_neon_vextv8qi;
1084 if (num_bits == 0)
1085 {
1086 emit_move_insn (operands[0], CONST0_RTX (<MODE>mode));
1087 DONE;
1088 }
1090 num_bits = width - num_bits;
1092 zero_reg = force_reg (bvecmode, CONST0_RTX (bvecmode));
1093 operands[0] = gen_lowpart (bvecmode, operands[0]);
1094 operands[1] = gen_lowpart (bvecmode, operands[1]);
1096 emit_insn (gen_ext (operands[0], zero_reg, operands[1],
1097 GEN_INT (num_bits / BITS_PER_UNIT)));
1098 DONE;
1099 })
1101 ;; Helpers for quad-word reduction operations
1103 ; Add (or smin, smax...) the low N/2 elements of the N-element vector
1104 ; operand[1] to the high N/2 elements of same. Put the result in operand[0], an
1105 ; N/2-element vector.
1107 (define_insn "quad_halves_<code>v4si"
1108 [(set (match_operand:V2SI 0 "s_register_operand" "=w")
1109 (vqh_ops:V2SI
1110 (vec_select:V2SI (match_operand:V4SI 1 "s_register_operand" "w")
1111 (parallel [(const_int 0) (const_int 1)]))
1112 (vec_select:V2SI (match_dup 1)
1113 (parallel [(const_int 2) (const_int 3)]))))]
1114 "TARGET_NEON"
1115 "<VQH_mnem>.<VQH_sign>32\t%P0, %e1, %f1"
1116 [(set_attr "vqh_mnem" "<VQH_mnem>")
1117 (set (attr "neon_type")
1118 (if_then_else (eq_attr "vqh_mnem" "vadd")
1119 (const_string "neon_int_1") (const_string "neon_int_5")))]
1120 )
1122 (define_insn "quad_halves_<code>v4sf"
1123 [(set (match_operand:V2SF 0 "s_register_operand" "=w")
1124 (vqhs_ops:V2SF
1125 (vec_select:V2SF (match_operand:V4SF 1 "s_register_operand" "w")
1126 (parallel [(const_int 0) (const_int 1)]))
1127 (vec_select:V2SF (match_dup 1)
1128 (parallel [(const_int 2) (const_int 3)]))))]
1129 "TARGET_NEON && flag_unsafe_math_optimizations"
1130 "<VQH_mnem>.f32\t%P0, %e1, %f1"
1131 [(set_attr "vqh_mnem" "<VQH_mnem>")
1132 (set (attr "neon_type")
1133 (if_then_else (eq_attr "vqh_mnem" "vadd")
1134 (const_string "neon_int_1") (const_string "neon_int_5")))]
1135 )
1137 (define_insn "quad_halves_<code>v8hi"
1138 [(set (match_operand:V4HI 0 "s_register_operand" "+w")
1139 (vqh_ops:V4HI
1140 (vec_select:V4HI (match_operand:V8HI 1 "s_register_operand" "w")
1141 (parallel [(const_int 0) (const_int 1)
1142 (const_int 2) (const_int 3)]))
1143 (vec_select:V4HI (match_dup 1)
1144 (parallel [(const_int 4) (const_int 5)
1145 (const_int 6) (const_int 7)]))))]
1146 "TARGET_NEON"
1147 "<VQH_mnem>.<VQH_sign>16\t%P0, %e1, %f1"
1148 [(set_attr "vqh_mnem" "<VQH_mnem>")
1149 (set (attr "neon_type")
1150 (if_then_else (eq_attr "vqh_mnem" "vadd")
1151 (const_string "neon_int_1") (const_string "neon_int_5")))]
1152 )
1154 (define_insn "quad_halves_<code>v16qi"
1155 [(set (match_operand:V8QI 0 "s_register_operand" "+w")
1156 (vqh_ops:V8QI
1157 (vec_select:V8QI (match_operand:V16QI 1 "s_register_operand" "w")
1158 (parallel [(const_int 0) (const_int 1)
1159 (const_int 2) (const_int 3)
1160 (const_int 4) (const_int 5)
1161 (const_int 6) (const_int 7)]))
1162 (vec_select:V8QI (match_dup 1)
1163 (parallel [(const_int 8) (const_int 9)
1164 (const_int 10) (const_int 11)
1165 (const_int 12) (const_int 13)
1166 (const_int 14) (const_int 15)]))))]
1167 "TARGET_NEON"
1168 "<VQH_mnem>.<VQH_sign>8\t%P0, %e1, %f1"
1169 [(set_attr "vqh_mnem" "<VQH_mnem>")
1170 (set (attr "neon_type")
1171 (if_then_else (eq_attr "vqh_mnem" "vadd")
1172 (const_string "neon_int_1") (const_string "neon_int_5")))]
1173 )
1175 ; FIXME: We wouldn't need the following insns if we could write subregs of
1176 ; vector registers. Make an attempt at removing unnecessary moves, though
1177 ; we're really at the mercy of the register allocator.
1179 (define_insn "neon_move_lo_quad_<mode>"
1180 [(set (match_operand:ANY128 0 "s_register_operand" "+w")
1181 (vec_concat:ANY128
1182 (match_operand:<V_HALF> 1 "s_register_operand" "w")
1183 (vec_select:<V_HALF>
1184 (match_dup 0)
1185 (match_operand:ANY128 2 "vect_par_constant_high" ""))))]
1186 "TARGET_NEON"
1187 {
1188 int dest = REGNO (operands[0]);
1189 int src = REGNO (operands[1]);
1191 if (dest != src)
1192 return "vmov\t%e0, %P1";
1193 else
1194 return "";
1195 }
1196 [(set_attr "neon_type" "neon_bp_simple")]
1197 )
1199 (define_insn "neon_move_hi_quad_<mode>"
1200 [(set (match_operand:ANY128 0 "s_register_operand" "+w")
1201 (vec_concat:ANY128
1202 (vec_select:<V_HALF>
1203 (match_dup 0)
1204 (match_operand:ANY128 2 "vect_par_constant_low" ""))
1205 (match_operand:<V_HALF> 1 "s_register_operand" "w")))]
1207 "TARGET_NEON"
1208 {
1209 int dest = REGNO (operands[0]);
1210 int src = REGNO (operands[1]);
1212 if (dest != src)
1213 return "vmov\t%f0, %P1";
1214 else
1215 return "";
1216 }
1217 [(set_attr "neon_type" "neon_bp_simple")]
1218 )
1220 (define_expand "move_hi_quad_<mode>"
1221 [(match_operand:ANY128 0 "s_register_operand" "")
1222 (match_operand:<V_HALF> 1 "s_register_operand" "")]
1223 "TARGET_NEON"
1224 {
1225 rtvec v = rtvec_alloc (<V_mode_nunits>/2);
1226 rtx t1;
1227 int i;
1229 for (i=0; i < (<V_mode_nunits>/2); i++)
1230 RTVEC_ELT (v, i) = GEN_INT (i);
1232 t1 = gen_rtx_PARALLEL (<MODE>mode, v);
1233 emit_insn (gen_neon_move_hi_quad_<mode> (operands[0], operands[1], t1));
1235 DONE;
1236 })
1238 (define_expand "move_lo_quad_<mode>"
1239 [(match_operand:ANY128 0 "s_register_operand" "")
1240 (match_operand:<V_HALF> 1 "s_register_operand" "")]
1241 "TARGET_NEON"
1242 {
1243 rtvec v = rtvec_alloc (<V_mode_nunits>/2);
1244 rtx t1;
1245 int i;
1247 for (i=0; i < (<V_mode_nunits>/2); i++)
1248 RTVEC_ELT (v, i) = GEN_INT ((<V_mode_nunits>/2) + i);
1250 t1 = gen_rtx_PARALLEL (<MODE>mode, v);
1251 emit_insn (gen_neon_move_lo_quad_<mode> (operands[0], operands[1], t1));
1253 DONE;
1254 })
1256 ;; Reduction operations
1258 (define_expand "reduc_splus_<mode>"
1259 [(match_operand:VD 0 "s_register_operand" "")
1260 (match_operand:VD 1 "s_register_operand" "")]
1261 "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
1262 {
1263 neon_pairwise_reduce (operands[0], operands[1], <MODE>mode,
1264 &gen_neon_vpadd_internal<mode>);
1265 DONE;
1266 })
1268 (define_expand "reduc_splus_<mode>"
1269 [(match_operand:VQ 0 "s_register_operand" "")
1270 (match_operand:VQ 1 "s_register_operand" "")]
1271 "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
1272 {
1273 rtx step1 = gen_reg_rtx (<V_HALF>mode);
1274 rtx res_d = gen_reg_rtx (<V_HALF>mode);
1276 emit_insn (gen_quad_halves_plus<mode> (step1, operands[1]));
1277 emit_insn (gen_reduc_splus_<V_half> (res_d, step1));
1278 emit_insn (gen_move_lo_quad_<mode> (operands[0], res_d));
1280 DONE;
1281 })
1283 (define_insn "reduc_splus_v2di"
1284 [(set (match_operand:V2DI 0 "s_register_operand" "=w")
1285 (unspec:V2DI [(match_operand:V2DI 1 "s_register_operand" "w")]
1286 UNSPEC_VPADD))]
1287 "TARGET_NEON"
1288 "vadd.i64\t%e0, %e1, %f1"
1289 [(set_attr "neon_type" "neon_int_1")]
1290 )
1292 ;; NEON does not distinguish between signed and unsigned addition except on
1293 ;; widening operations.
1294 (define_expand "reduc_uplus_<mode>"
1295 [(match_operand:VDQI 0 "s_register_operand" "")
1296 (match_operand:VDQI 1 "s_register_operand" "")]
1297 "TARGET_NEON"
1298 {
1299 emit_insn (gen_reduc_splus_<mode> (operands[0], operands[1]));
1300 DONE;
1301 })
1303 (define_expand "reduc_smin_<mode>"
1304 [(match_operand:VD 0 "s_register_operand" "")
1305 (match_operand:VD 1 "s_register_operand" "")]
1306 "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
1307 {
1308 neon_pairwise_reduce (operands[0], operands[1], <MODE>mode,
1309 &gen_neon_vpsmin<mode>);
1310 DONE;
1311 })
1313 (define_expand "reduc_smin_<mode>"
1314 [(match_operand:VQ 0 "s_register_operand" "")
1315 (match_operand:VQ 1 "s_register_operand" "")]
1316 "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
1317 {
1318 rtx step1 = gen_reg_rtx (<V_HALF>mode);
1319 rtx res_d = gen_reg_rtx (<V_HALF>mode);
1321 emit_insn (gen_quad_halves_smin<mode> (step1, operands[1]));
1322 emit_insn (gen_reduc_smin_<V_half> (res_d, step1));
1323 emit_insn (gen_move_lo_quad_<mode> (operands[0], res_d));
1325 DONE;
1326 })
1328 (define_expand "reduc_smax_<mode>"
1329 [(match_operand:VD 0 "s_register_operand" "")
1330 (match_operand:VD 1 "s_register_operand" "")]
1331 "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
1332 {
1333 neon_pairwise_reduce (operands[0], operands[1], <MODE>mode,
1334 &gen_neon_vpsmax<mode>);
1335 DONE;
1336 })
1338 (define_expand "reduc_smax_<mode>"
1339 [(match_operand:VQ 0 "s_register_operand" "")
1340 (match_operand:VQ 1 "s_register_operand" "")]
1341 "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
1342 {
1343 rtx step1 = gen_reg_rtx (<V_HALF>mode);
1344 rtx res_d = gen_reg_rtx (<V_HALF>mode);
1346 emit_insn (gen_quad_halves_smax<mode> (step1, operands[1]));
1347 emit_insn (gen_reduc_smax_<V_half> (res_d, step1));
1348 emit_insn (gen_move_lo_quad_<mode> (operands[0], res_d));
1350 DONE;
1351 })
1353 (define_expand "reduc_umin_<mode>"
1354 [(match_operand:VDI 0 "s_register_operand" "")
1355 (match_operand:VDI 1 "s_register_operand" "")]
1356 "TARGET_NEON"
1357 {
1358 neon_pairwise_reduce (operands[0], operands[1], <MODE>mode,
1359 &gen_neon_vpumin<mode>);
1360 DONE;
1361 })
1363 (define_expand "reduc_umin_<mode>"
1364 [(match_operand:VQI 0 "s_register_operand" "")
1365 (match_operand:VQI 1 "s_register_operand" "")]
1366 "TARGET_NEON"
1367 {
1368 rtx step1 = gen_reg_rtx (<V_HALF>mode);
1369 rtx res_d = gen_reg_rtx (<V_HALF>mode);
1371 emit_insn (gen_quad_halves_umin<mode> (step1, operands[1]));
1372 emit_insn (gen_reduc_umin_<V_half> (res_d, step1));
1373 emit_insn (gen_move_lo_quad_<mode> (operands[0], res_d));
1375 DONE;
1376 })
1378 (define_expand "reduc_umax_<mode>"
1379 [(match_operand:VDI 0 "s_register_operand" "")
1380 (match_operand:VDI 1 "s_register_operand" "")]
1381 "TARGET_NEON"
1382 {
1383 neon_pairwise_reduce (operands[0], operands[1], <MODE>mode,
1384 &gen_neon_vpumax<mode>);
1385 DONE;
1386 })
1388 (define_expand "reduc_umax_<mode>"
1389 [(match_operand:VQI 0 "s_register_operand" "")
1390 (match_operand:VQI 1 "s_register_operand" "")]
1391 "TARGET_NEON"
1392 {
1393 rtx step1 = gen_reg_rtx (<V_HALF>mode);
1394 rtx res_d = gen_reg_rtx (<V_HALF>mode);
1396 emit_insn (gen_quad_halves_umax<mode> (step1, operands[1]));
1397 emit_insn (gen_reduc_umax_<V_half> (res_d, step1));
1398 emit_insn (gen_move_lo_quad_<mode> (operands[0], res_d));
1400 DONE;
1401 })
1403 (define_insn "neon_vpadd_internal<mode>"
1404 [(set (match_operand:VD 0 "s_register_operand" "=w")
1405 (unspec:VD [(match_operand:VD 1 "s_register_operand" "w")
1406 (match_operand:VD 2 "s_register_operand" "w")]
1407 UNSPEC_VPADD))]
1408 "TARGET_NEON"
1409 "vpadd.<V_if_elem>\t%P0, %P1, %P2"
1410 ;; Assume this schedules like vadd.
1411 [(set (attr "neon_type")
1412 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
1413 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
1414 (const_string "neon_fp_vadd_ddd_vabs_dd")
1415 (const_string "neon_fp_vadd_qqq_vabs_qq"))
1416 (const_string "neon_int_1")))]
1417 )
1419 (define_insn "neon_vpsmin<mode>"
1420 [(set (match_operand:VD 0 "s_register_operand" "=w")
1421 (unspec:VD [(match_operand:VD 1 "s_register_operand" "w")
1422 (match_operand:VD 2 "s_register_operand" "w")]
1423 UNSPEC_VPSMIN))]
1424 "TARGET_NEON"
1425 "vpmin.<V_s_elem>\t%P0, %P1, %P2"
1426 ;; Assume this schedules like vmin.
1427 [(set (attr "neon_type")
1428 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
1429 (const_string "neon_fp_vadd_ddd_vabs_dd")
1430 (const_string "neon_int_5")))]
1431 )
1433 (define_insn "neon_vpsmax<mode>"
1434 [(set (match_operand:VD 0 "s_register_operand" "=w")
1435 (unspec:VD [(match_operand:VD 1 "s_register_operand" "w")
1436 (match_operand:VD 2 "s_register_operand" "w")]
1437 UNSPEC_VPSMAX))]
1438 "TARGET_NEON"
1439 "vpmax.<V_s_elem>\t%P0, %P1, %P2"
1440 ;; Assume this schedules like vmax.
1441 [(set (attr "neon_type")
1442 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
1443 (const_string "neon_fp_vadd_ddd_vabs_dd")
1444 (const_string "neon_int_5")))]
1445 )
1447 (define_insn "neon_vpumin<mode>"
1448 [(set (match_operand:VDI 0 "s_register_operand" "=w")
1449 (unspec:VDI [(match_operand:VDI 1 "s_register_operand" "w")
1450 (match_operand:VDI 2 "s_register_operand" "w")]
1451 UNSPEC_VPUMIN))]
1452 "TARGET_NEON"
1453 "vpmin.<V_u_elem>\t%P0, %P1, %P2"
1454 ;; Assume this schedules like umin.
1455 [(set_attr "neon_type" "neon_int_5")]
1456 )
1458 (define_insn "neon_vpumax<mode>"
1459 [(set (match_operand:VDI 0 "s_register_operand" "=w")
1460 (unspec:VDI [(match_operand:VDI 1 "s_register_operand" "w")
1461 (match_operand:VDI 2 "s_register_operand" "w")]
1462 UNSPEC_VPUMAX))]
1463 "TARGET_NEON"
1464 "vpmax.<V_u_elem>\t%P0, %P1, %P2"
1465 ;; Assume this schedules like umax.
1466 [(set_attr "neon_type" "neon_int_5")]
1467 )
1469 ;; Saturating arithmetic
1471 ; NOTE: Neon supports many more saturating variants of instructions than the
1472 ; following, but these are all GCC currently understands.
1473 ; FIXME: Actually, GCC doesn't know how to create saturating add/sub by itself
1474 ; yet either, although these patterns may be used by intrinsics when they're
1475 ; added.
1477 (define_insn "*ss_add<mode>_neon"
1478 [(set (match_operand:VD 0 "s_register_operand" "=w")
1479 (ss_plus:VD (match_operand:VD 1 "s_register_operand" "w")
1480 (match_operand:VD 2 "s_register_operand" "w")))]
1481 "TARGET_NEON"
1482 "vqadd.<V_s_elem>\t%P0, %P1, %P2"
1483 [(set_attr "neon_type" "neon_int_4")]
1484 )
1486 (define_insn "*us_add<mode>_neon"
1487 [(set (match_operand:VD 0 "s_register_operand" "=w")
1488 (us_plus:VD (match_operand:VD 1 "s_register_operand" "w")
1489 (match_operand:VD 2 "s_register_operand" "w")))]
1490 "TARGET_NEON"
1491 "vqadd.<V_u_elem>\t%P0, %P1, %P2"
1492 [(set_attr "neon_type" "neon_int_4")]
1493 )
1495 (define_insn "*ss_sub<mode>_neon"
1496 [(set (match_operand:VD 0 "s_register_operand" "=w")
1497 (ss_minus:VD (match_operand:VD 1 "s_register_operand" "w")
1498 (match_operand:VD 2 "s_register_operand" "w")))]
1499 "TARGET_NEON"
1500 "vqsub.<V_s_elem>\t%P0, %P1, %P2"
1501 [(set_attr "neon_type" "neon_int_5")]
1502 )
1504 (define_insn "*us_sub<mode>_neon"
1505 [(set (match_operand:VD 0 "s_register_operand" "=w")
1506 (us_minus:VD (match_operand:VD 1 "s_register_operand" "w")
1507 (match_operand:VD 2 "s_register_operand" "w")))]
1508 "TARGET_NEON"
1509 "vqsub.<V_u_elem>\t%P0, %P1, %P2"
1510 [(set_attr "neon_type" "neon_int_5")]
1511 )
1513 ;; Conditional instructions. These are comparisons with conditional moves for
1514 ;; vectors. They perform the assignment:
1515 ;;
1516 ;; Vop0 = (Vop4 <op3> Vop5) ? Vop1 : Vop2;
1517 ;;
1518 ;; where op3 is <, <=, ==, !=, >= or >. Operations are performed
1519 ;; element-wise.
1521 (define_expand "vcond<mode>"
1522 [(set (match_operand:VDQW 0 "s_register_operand" "")
1523 (if_then_else:VDQW
1524 (match_operator 3 "arm_comparison_operator"
1525 [(match_operand:VDQW 4 "s_register_operand" "")
1526 (match_operand:VDQW 5 "nonmemory_operand" "")])
1527 (match_operand:VDQW 1 "s_register_operand" "")
1528 (match_operand:VDQW 2 "s_register_operand" "")))]
1529 "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
1530 {
1531 rtx mask;
1532 int inverse = 0, immediate_zero = 0;
1533 /* See the description of "magic" bits in the 'T' case of
1534 arm_print_operand. */
1535 HOST_WIDE_INT magic_word = (<MODE>mode == V2SFmode || <MODE>mode == V4SFmode)
1536 ? 3 : 1;
1537 rtx magic_rtx = GEN_INT (magic_word);
1539 mask = gen_reg_rtx (<V_cmp_result>mode);
1541 if (operands[5] == CONST0_RTX (<MODE>mode))
1542 immediate_zero = 1;
1543 else if (!REG_P (operands[5]))
1544 operands[5] = force_reg (<MODE>mode, operands[5]);
1546 switch (GET_CODE (operands[3]))
1547 {
1548 case GE:
1549 emit_insn (gen_neon_vcge<mode> (mask, operands[4], operands[5],
1550 magic_rtx));
1551 break;
1553 case GT:
1554 emit_insn (gen_neon_vcgt<mode> (mask, operands[4], operands[5],
1555 magic_rtx));
1556 break;
1558 case EQ:
1559 emit_insn (gen_neon_vceq<mode> (mask, operands[4], operands[5],
1560 magic_rtx));
1561 break;
1563 case LE:
1564 if (immediate_zero)
1565 emit_insn (gen_neon_vcle<mode> (mask, operands[4], operands[5],
1566 magic_rtx));
1567 else
1568 emit_insn (gen_neon_vcge<mode> (mask, operands[5], operands[4],
1569 magic_rtx));
1570 break;
1572 case LT:
1573 if (immediate_zero)
1574 emit_insn (gen_neon_vclt<mode> (mask, operands[4], operands[5],
1575 magic_rtx));
1576 else
1577 emit_insn (gen_neon_vcgt<mode> (mask, operands[5], operands[4],
1578 magic_rtx));
1579 break;
1581 case NE:
1582 emit_insn (gen_neon_vceq<mode> (mask, operands[4], operands[5],
1583 magic_rtx));
1584 inverse = 1;
1585 break;
1587 default:
1588 gcc_unreachable ();
1589 }
1591 if (inverse)
1592 emit_insn (gen_neon_vbsl<mode> (operands[0], mask, operands[2],
1593 operands[1]));
1594 else
1595 emit_insn (gen_neon_vbsl<mode> (operands[0], mask, operands[1],
1596 operands[2]));
1598 DONE;
1599 })
1601 (define_expand "vcondu<mode>"
1602 [(set (match_operand:VDQIW 0 "s_register_operand" "")
1603 (if_then_else:VDQIW
1604 (match_operator 3 "arm_comparison_operator"
1605 [(match_operand:VDQIW 4 "s_register_operand" "")
1606 (match_operand:VDQIW 5 "s_register_operand" "")])
1607 (match_operand:VDQIW 1 "s_register_operand" "")
1608 (match_operand:VDQIW 2 "s_register_operand" "")))]
1609 "TARGET_NEON"
1610 {
1611 rtx mask;
1612 int inverse = 0, immediate_zero = 0;
1614 mask = gen_reg_rtx (<V_cmp_result>mode);
1616 if (operands[5] == CONST0_RTX (<MODE>mode))
1617 immediate_zero = 1;
1618 else if (!REG_P (operands[5]))
1619 operands[5] = force_reg (<MODE>mode, operands[5]);
1621 switch (GET_CODE (operands[3]))
1622 {
1623 case GEU:
1624 emit_insn (gen_neon_vcge<mode> (mask, operands[4], operands[5],
1625 const0_rtx));
1626 break;
1628 case GTU:
1629 emit_insn (gen_neon_vcgt<mode> (mask, operands[4], operands[5],
1630 const0_rtx));
1631 break;
1633 case EQ:
1634 emit_insn (gen_neon_vceq<mode> (mask, operands[4], operands[5],
1635 const0_rtx));
1636 break;
1638 case LEU:
1639 if (immediate_zero)
1640 emit_insn (gen_neon_vcle<mode> (mask, operands[4], operands[5],
1641 const0_rtx));
1642 else
1643 emit_insn (gen_neon_vcge<mode> (mask, operands[5], operands[4],
1644 const0_rtx));
1645 break;
1647 case LTU:
1648 if (immediate_zero)
1649 emit_insn (gen_neon_vclt<mode> (mask, operands[4], operands[5],
1650 const0_rtx));
1651 else
1652 emit_insn (gen_neon_vcgt<mode> (mask, operands[5], operands[4],
1653 const0_rtx));
1654 break;
1656 case NE:
1657 emit_insn (gen_neon_vceq<mode> (mask, operands[4], operands[5],
1658 const0_rtx));
1659 inverse = 1;
1660 break;
1662 default:
1663 gcc_unreachable ();
1664 }
1666 if (inverse)
1667 emit_insn (gen_neon_vbsl<mode> (operands[0], mask, operands[2],
1668 operands[1]));
1669 else
1670 emit_insn (gen_neon_vbsl<mode> (operands[0], mask, operands[1],
1671 operands[2]));
1673 DONE;
1674 })
1676 ;; Patterns for builtins.
1678 ; good for plain vadd, vaddq.
1680 (define_expand "neon_vadd<mode>"
1681 [(match_operand:VDQX 0 "s_register_operand" "=w")
1682 (match_operand:VDQX 1 "s_register_operand" "w")
1683 (match_operand:VDQX 2 "s_register_operand" "w")
1684 (match_operand:SI 3 "immediate_operand" "i")]
1685 "TARGET_NEON"
1686 {
1687 if (!<Is_float_mode> || flag_unsafe_math_optimizations)
1688 emit_insn (gen_add<mode>3 (operands[0], operands[1], operands[2]));
1689 else
1690 emit_insn (gen_neon_vadd<mode>_unspec (operands[0], operands[1],
1691 operands[2]));
1692 DONE;
1693 })
1695 ; Note that NEON operations don't support the full IEEE 754 standard: in
1696 ; particular, denormal values are flushed to zero. This means that GCC cannot
1697 ; use those instructions for autovectorization, etc. unless
1698 ; -funsafe-math-optimizations is in effect (in which case flush-to-zero
1699 ; behaviour is permissible). Intrinsic operations (provided by the arm_neon.h
1700 ; header) must work in either case: if -funsafe-math-optimizations is given,
1701 ; intrinsics expand to "canonical" RTL where possible, otherwise intrinsics
1702 ; expand to unspecs (which may potentially limit the extent to which they might
1703 ; be optimized by generic code).
1705 ; Used for intrinsics when flag_unsafe_math_optimizations is false.
1707 (define_insn "neon_vadd<mode>_unspec"
1708 [(set (match_operand:VDQX 0 "s_register_operand" "=w")
1709 (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand" "w")
1710 (match_operand:VDQX 2 "s_register_operand" "w")]
1711 UNSPEC_VADD))]
1712 "TARGET_NEON"
1713 "vadd.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1714 [(set (attr "neon_type")
1715 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
1716 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
1717 (const_string "neon_fp_vadd_ddd_vabs_dd")
1718 (const_string "neon_fp_vadd_qqq_vabs_qq"))
1719 (const_string "neon_int_1")))]
1720 )
1722 ; operand 3 represents in bits:
1723 ; bit 0: signed (vs unsigned).
1724 ; bit 1: rounding (vs none).
1726 (define_insn "neon_vaddl<mode>"
1727 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
1728 (unspec:<V_widen> [(match_operand:VDI 1 "s_register_operand" "w")
1729 (match_operand:VDI 2 "s_register_operand" "w")
1730 (match_operand:SI 3 "immediate_operand" "i")]
1731 UNSPEC_VADDL))]
1732 "TARGET_NEON"
1733 "vaddl.%T3%#<V_sz_elem>\t%q0, %P1, %P2"
1734 [(set_attr "neon_type" "neon_int_3")]
1735 )
1737 (define_insn "neon_vaddw<mode>"
1738 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
1739 (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "w")
1740 (match_operand:VDI 2 "s_register_operand" "w")
1741 (match_operand:SI 3 "immediate_operand" "i")]
1742 UNSPEC_VADDW))]
1743 "TARGET_NEON"
1744 "vaddw.%T3%#<V_sz_elem>\t%q0, %q1, %P2"
1745 [(set_attr "neon_type" "neon_int_2")]
1746 )
1748 ; vhadd and vrhadd.
1750 (define_insn "neon_vhadd<mode>"
1751 [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
1752 (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
1753 (match_operand:VDQIW 2 "s_register_operand" "w")
1754 (match_operand:SI 3 "immediate_operand" "i")]
1755 UNSPEC_VHADD))]
1756 "TARGET_NEON"
1757 "v%O3hadd.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1758 [(set_attr "neon_type" "neon_int_4")]
1759 )
1761 (define_insn "neon_vqadd<mode>"
1762 [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
1763 (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
1764 (match_operand:VDQIX 2 "s_register_operand" "w")
1765 (match_operand:SI 3 "immediate_operand" "i")]
1766 UNSPEC_VQADD))]
1767 "TARGET_NEON"
1768 "vqadd.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1769 [(set_attr "neon_type" "neon_int_4")]
1770 )
1772 (define_insn "neon_vaddhn<mode>"
1773 [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
1774 (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
1775 (match_operand:VN 2 "s_register_operand" "w")
1776 (match_operand:SI 3 "immediate_operand" "i")]
1777 UNSPEC_VADDHN))]
1778 "TARGET_NEON"
1779 "v%O3addhn.<V_if_elem>\t%P0, %q1, %q2"
1780 [(set_attr "neon_type" "neon_int_4")]
1781 )
1783 ;; We cannot replace this unspec with mul<mode>3 because of the odd
1784 ;; polynomial multiplication case that can specified by operand 3.
1785 (define_insn "neon_vmul<mode>"
1786 [(set (match_operand:VDQW 0 "s_register_operand" "=w")
1787 (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "w")
1788 (match_operand:VDQW 2 "s_register_operand" "w")
1789 (match_operand:SI 3 "immediate_operand" "i")]
1790 UNSPEC_VMUL))]
1791 "TARGET_NEON"
1792 "vmul.%F3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1793 [(set (attr "neon_type")
1794 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
1795 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
1796 (const_string "neon_fp_vadd_ddd_vabs_dd")
1797 (const_string "neon_fp_vadd_qqq_vabs_qq"))
1798 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
1799 (if_then_else
1800 (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0))
1801 (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long")
1802 (const_string "neon_mul_qqq_8_16_32_ddd_32"))
1803 (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0))
1804 (const_string "neon_mul_qqq_8_16_32_ddd_32")
1805 (const_string "neon_mul_qqq_8_16_32_ddd_32")))))]
1806 )
1808 (define_expand "neon_vmla<mode>"
1809 [(match_operand:VDQW 0 "s_register_operand" "=w")
1810 (match_operand:VDQW 1 "s_register_operand" "0")
1811 (match_operand:VDQW 2 "s_register_operand" "w")
1812 (match_operand:VDQW 3 "s_register_operand" "w")
1813 (match_operand:SI 4 "immediate_operand" "i")]
1814 "TARGET_NEON"
1815 {
1816 if (!<Is_float_mode> || flag_unsafe_math_optimizations)
1817 emit_insn (gen_mul<mode>3add<mode>_neon (operands[0], operands[1],
1818 operands[2], operands[3]));
1819 else
1820 emit_insn (gen_neon_vmla<mode>_unspec (operands[0], operands[1],
1821 operands[2], operands[3]));
1822 DONE;
1823 })
1825 ; Used for intrinsics when flag_unsafe_math_optimizations is false.
1827 (define_insn "neon_vmla<mode>_unspec"
1828 [(set (match_operand:VDQ 0 "s_register_operand" "=w")
1829 (unspec:VDQ [(match_operand:VDQ 1 "s_register_operand" "0")
1830 (match_operand:VDQ 2 "s_register_operand" "w")
1831 (match_operand:VDQ 3 "s_register_operand" "w")]
1832 UNSPEC_VMLA))]
1833 "TARGET_NEON"
1834 "vmla.<V_if_elem>\t%<V_reg>0, %<V_reg>2, %<V_reg>3"
1835 [(set (attr "neon_type")
1836 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
1837 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
1838 (const_string "neon_fp_vmla_ddd")
1839 (const_string "neon_fp_vmla_qqq"))
1840 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
1841 (if_then_else
1842 (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0))
1843 (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
1844 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long"))
1845 (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0))
1846 (const_string "neon_mla_qqq_8_16")
1847 (const_string "neon_mla_qqq_32_qqd_32_scalar")))))]
1848 )
1850 (define_insn "neon_vmlal<mode>"
1851 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
1852 (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
1853 (match_operand:VW 2 "s_register_operand" "w")
1854 (match_operand:VW 3 "s_register_operand" "w")
1855 (match_operand:SI 4 "immediate_operand" "i")]
1856 UNSPEC_VMLAL))]
1857 "TARGET_NEON"
1858 "vmlal.%T4%#<V_sz_elem>\t%q0, %P2, %P3"
1859 [(set (attr "neon_type")
1860 (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0))
1861 (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
1862 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
1863 )
1865 (define_expand "neon_vmls<mode>"
1866 [(match_operand:VDQW 0 "s_register_operand" "=w")
1867 (match_operand:VDQW 1 "s_register_operand" "0")
1868 (match_operand:VDQW 2 "s_register_operand" "w")
1869 (match_operand:VDQW 3 "s_register_operand" "w")
1870 (match_operand:SI 4 "immediate_operand" "i")]
1871 "TARGET_NEON"
1872 {
1873 if (!<Is_float_mode> || flag_unsafe_math_optimizations)
1874 emit_insn (gen_mul<mode>3neg<mode>add<mode>_neon (operands[0],
1875 operands[1], operands[2], operands[3]));
1876 else
1877 emit_insn (gen_neon_vmls<mode>_unspec (operands[0], operands[1],
1878 operands[2], operands[3]));
1879 DONE;
1880 })
1882 ; Used for intrinsics when flag_unsafe_math_optimizations is false.
1884 (define_insn "neon_vmls<mode>_unspec"
1885 [(set (match_operand:VDQ 0 "s_register_operand" "=w")
1886 (unspec:VDQ [(match_operand:VDQ 1 "s_register_operand" "0")
1887 (match_operand:VDQ 2 "s_register_operand" "w")
1888 (match_operand:VDQ 3 "s_register_operand" "w")]
1889 UNSPEC_VMLS))]
1890 "TARGET_NEON"
1891 "vmls.<V_if_elem>\t%<V_reg>0, %<V_reg>2, %<V_reg>3"
1892 [(set (attr "neon_type")
1893 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
1894 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
1895 (const_string "neon_fp_vmla_ddd")
1896 (const_string "neon_fp_vmla_qqq"))
1897 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
1898 (if_then_else
1899 (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0))
1900 (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
1901 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long"))
1902 (if_then_else
1903 (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0))
1904 (const_string "neon_mla_qqq_8_16")
1905 (const_string "neon_mla_qqq_32_qqd_32_scalar")))))]
1906 )
1908 (define_insn "neon_vmlsl<mode>"
1909 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
1910 (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
1911 (match_operand:VW 2 "s_register_operand" "w")
1912 (match_operand:VW 3 "s_register_operand" "w")
1913 (match_operand:SI 4 "immediate_operand" "i")]
1914 UNSPEC_VMLSL))]
1915 "TARGET_NEON"
1916 "vmlsl.%T4%#<V_sz_elem>\t%q0, %P2, %P3"
1917 [(set (attr "neon_type")
1918 (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0))
1919 (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
1920 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
1921 )
1923 (define_insn "neon_vqdmulh<mode>"
1924 [(set (match_operand:VMDQI 0 "s_register_operand" "=w")
1925 (unspec:VMDQI [(match_operand:VMDQI 1 "s_register_operand" "w")
1926 (match_operand:VMDQI 2 "s_register_operand" "w")
1927 (match_operand:SI 3 "immediate_operand" "i")]
1928 UNSPEC_VQDMULH))]
1929 "TARGET_NEON"
1930 "vq%O3dmulh.<V_s_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1931 [(set (attr "neon_type")
1932 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
1933 (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0))
1934 (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long")
1935 (const_string "neon_mul_qqq_8_16_32_ddd_32"))
1936 (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0))
1937 (const_string "neon_mul_qqq_8_16_32_ddd_32")
1938 (const_string "neon_mul_qqq_8_16_32_ddd_32"))))]
1939 )
1941 (define_insn "neon_vqdmlal<mode>"
1942 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
1943 (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
1944 (match_operand:VMDI 2 "s_register_operand" "w")
1945 (match_operand:VMDI 3 "s_register_operand" "w")
1946 (match_operand:SI 4 "immediate_operand" "i")]
1947 UNSPEC_VQDMLAL))]
1948 "TARGET_NEON"
1949 "vqdmlal.<V_s_elem>\t%q0, %P2, %P3"
1950 [(set (attr "neon_type")
1951 (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0))
1952 (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
1953 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
1954 )
1956 (define_insn "neon_vqdmlsl<mode>"
1957 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
1958 (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
1959 (match_operand:VMDI 2 "s_register_operand" "w")
1960 (match_operand:VMDI 3 "s_register_operand" "w")
1961 (match_operand:SI 4 "immediate_operand" "i")]
1962 UNSPEC_VQDMLSL))]
1963 "TARGET_NEON"
1964 "vqdmlsl.<V_s_elem>\t%q0, %P2, %P3"
1965 [(set (attr "neon_type")
1966 (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0))
1967 (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
1968 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
1969 )
1971 (define_insn "neon_vmull<mode>"
1972 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
1973 (unspec:<V_widen> [(match_operand:VW 1 "s_register_operand" "w")
1974 (match_operand:VW 2 "s_register_operand" "w")
1975 (match_operand:SI 3 "immediate_operand" "i")]
1976 UNSPEC_VMULL))]
1977 "TARGET_NEON"
1978 "vmull.%T3%#<V_sz_elem>\t%q0, %P1, %P2"
1979 [(set (attr "neon_type")
1980 (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0))
1981 (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long")
1982 (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")))]
1983 )
1985 (define_insn "neon_vqdmull<mode>"
1986 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
1987 (unspec:<V_widen> [(match_operand:VMDI 1 "s_register_operand" "w")
1988 (match_operand:VMDI 2 "s_register_operand" "w")
1989 (match_operand:SI 3 "immediate_operand" "i")]
1990 UNSPEC_VQDMULL))]
1991 "TARGET_NEON"
1992 "vqdmull.<V_s_elem>\t%q0, %P1, %P2"
1993 [(set (attr "neon_type")
1994 (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0))
1995 (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long")
1996 (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")))]
1997 )
1999 (define_expand "neon_vsub<mode>"
2000 [(match_operand:VDQX 0 "s_register_operand" "=w")
2001 (match_operand:VDQX 1 "s_register_operand" "w")
2002 (match_operand:VDQX 2 "s_register_operand" "w")
2003 (match_operand:SI 3 "immediate_operand" "i")]
2004 "TARGET_NEON"
2005 {
2006 if (!<Is_float_mode> || flag_unsafe_math_optimizations)
2007 emit_insn (gen_sub<mode>3 (operands[0], operands[1], operands[2]));
2008 else
2009 emit_insn (gen_neon_vsub<mode>_unspec (operands[0], operands[1],
2010 operands[2]));
2011 DONE;
2012 })
2014 ; Used for intrinsics when flag_unsafe_math_optimizations is false.
2016 (define_insn "neon_vsub<mode>_unspec"
2017 [(set (match_operand:VDQX 0 "s_register_operand" "=w")
2018 (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand" "w")
2019 (match_operand:VDQX 2 "s_register_operand" "w")]
2020 UNSPEC_VSUB))]
2021 "TARGET_NEON"
2022 "vsub.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2023 [(set (attr "neon_type")
2024 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
2025 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
2026 (const_string "neon_fp_vadd_ddd_vabs_dd")
2027 (const_string "neon_fp_vadd_qqq_vabs_qq"))
2028 (const_string "neon_int_2")))]
2029 )
2031 (define_insn "neon_vsubl<mode>"
2032 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2033 (unspec:<V_widen> [(match_operand:VDI 1 "s_register_operand" "w")
2034 (match_operand:VDI 2 "s_register_operand" "w")
2035 (match_operand:SI 3 "immediate_operand" "i")]
2036 UNSPEC_VSUBL))]
2037 "TARGET_NEON"
2038 "vsubl.%T3%#<V_sz_elem>\t%q0, %P1, %P2"
2039 [(set_attr "neon_type" "neon_int_2")]
2040 )
2042 (define_insn "neon_vsubw<mode>"
2043 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2044 (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "w")
2045 (match_operand:VDI 2 "s_register_operand" "w")
2046 (match_operand:SI 3 "immediate_operand" "i")]
2047 UNSPEC_VSUBW))]
2048 "TARGET_NEON"
2049 "vsubw.%T3%#<V_sz_elem>\t%q0, %q1, %P2"
2050 [(set_attr "neon_type" "neon_int_2")]
2051 )
2053 (define_insn "neon_vqsub<mode>"
2054 [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
2055 (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
2056 (match_operand:VDQIX 2 "s_register_operand" "w")
2057 (match_operand:SI 3 "immediate_operand" "i")]
2058 UNSPEC_VQSUB))]
2059 "TARGET_NEON"
2060 "vqsub.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2061 [(set_attr "neon_type" "neon_int_5")]
2062 )
2064 (define_insn "neon_vhsub<mode>"
2065 [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2066 (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
2067 (match_operand:VDQIW 2 "s_register_operand" "w")
2068 (match_operand:SI 3 "immediate_operand" "i")]
2069 UNSPEC_VHSUB))]
2070 "TARGET_NEON"
2071 "vhsub.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2072 [(set_attr "neon_type" "neon_int_5")]
2073 )
2075 (define_insn "neon_vsubhn<mode>"
2076 [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
2077 (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
2078 (match_operand:VN 2 "s_register_operand" "w")
2079 (match_operand:SI 3 "immediate_operand" "i")]
2080 UNSPEC_VSUBHN))]
2081 "TARGET_NEON"
2082 "v%O3subhn.<V_if_elem>\t%P0, %q1, %q2"
2083 [(set_attr "neon_type" "neon_int_4")]
2084 )
2086 (define_insn "neon_vceq<mode>"
2087 [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w,w")
2088 (unspec:<V_cmp_result>
2089 [(match_operand:VDQW 1 "s_register_operand" "w,w")
2090 (match_operand:VDQW 2 "nonmemory_operand" "w,Dz")
2091 (match_operand:SI 3 "immediate_operand" "i,i")]
2092 UNSPEC_VCEQ))]
2093 "TARGET_NEON"
2094 "@
2095 vceq.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2
2096 vceq.<V_if_elem>\t%<V_reg>0, %<V_reg>1, #0"
2097 [(set (attr "neon_type")
2098 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
2099 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
2100 (const_string "neon_fp_vadd_ddd_vabs_dd")
2101 (const_string "neon_fp_vadd_qqq_vabs_qq"))
2102 (const_string "neon_int_5")))]
2103 )
2105 (define_insn "neon_vcge<mode>"
2106 [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w,w")
2107 (unspec:<V_cmp_result>
2108 [(match_operand:VDQW 1 "s_register_operand" "w,w")
2109 (match_operand:VDQW 2 "nonmemory_operand" "w,Dz")
2110 (match_operand:SI 3 "immediate_operand" "i,i")]
2111 UNSPEC_VCGE))]
2112 "TARGET_NEON"
2113 "@
2114 vcge.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2
2115 vcge.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, #0"
2116 [(set (attr "neon_type")
2117 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
2118 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
2119 (const_string "neon_fp_vadd_ddd_vabs_dd")
2120 (const_string "neon_fp_vadd_qqq_vabs_qq"))
2121 (const_string "neon_int_5")))]
2122 )
2124 (define_insn "neon_vcgt<mode>"
2125 [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w,w")
2126 (unspec:<V_cmp_result>
2127 [(match_operand:VDQW 1 "s_register_operand" "w,w")
2128 (match_operand:VDQW 2 "nonmemory_operand" "w,Dz")
2129 (match_operand:SI 3 "immediate_operand" "i,i")]
2130 UNSPEC_VCGT))]
2131 "TARGET_NEON"
2132 "@
2133 vcgt.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2
2134 vcgt.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, #0"
2135 [(set (attr "neon_type")
2136 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
2137 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
2138 (const_string "neon_fp_vadd_ddd_vabs_dd")
2139 (const_string "neon_fp_vadd_qqq_vabs_qq"))
2140 (const_string "neon_int_5")))]
2141 )
2143 ;; VCLE and VCLT only support comparisons with immediate zero (register
2144 ;; variants are VCGE and VCGT with operands reversed).
2146 (define_insn "neon_vcle<mode>"
2147 [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w")
2148 (unspec:<V_cmp_result>
2149 [(match_operand:VDQW 1 "s_register_operand" "w")
2150 (match_operand:VDQW 2 "nonmemory_operand" "Dz")
2151 (match_operand:SI 3 "immediate_operand" "i")]
2152 UNSPEC_VCLE))]
2153 "TARGET_NEON"
2154 "vcle.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, #0"
2155 [(set (attr "neon_type")
2156 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
2157 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
2158 (const_string "neon_fp_vadd_ddd_vabs_dd")
2159 (const_string "neon_fp_vadd_qqq_vabs_qq"))
2160 (const_string "neon_int_5")))]
2161 )
2163 (define_insn "neon_vclt<mode>"
2164 [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w")
2165 (unspec:<V_cmp_result>
2166 [(match_operand:VDQW 1 "s_register_operand" "w")
2167 (match_operand:VDQW 2 "nonmemory_operand" "Dz")
2168 (match_operand:SI 3 "immediate_operand" "i")]
2169 UNSPEC_VCLT))]
2170 "TARGET_NEON"
2171 "vclt.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, #0"
2172 [(set (attr "neon_type")
2173 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
2174 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
2175 (const_string "neon_fp_vadd_ddd_vabs_dd")
2176 (const_string "neon_fp_vadd_qqq_vabs_qq"))
2177 (const_string "neon_int_5")))]
2178 )
2180 (define_insn "neon_vcage<mode>"
2181 [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w")
2182 (unspec:<V_cmp_result> [(match_operand:VCVTF 1 "s_register_operand" "w")
2183 (match_operand:VCVTF 2 "s_register_operand" "w")
2184 (match_operand:SI 3 "immediate_operand" "i")]
2185 UNSPEC_VCAGE))]
2186 "TARGET_NEON"
2187 "vacge.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2188 [(set (attr "neon_type")
2189 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
2190 (const_string "neon_fp_vadd_ddd_vabs_dd")
2191 (const_string "neon_fp_vadd_qqq_vabs_qq")))]
2192 )
2194 (define_insn "neon_vcagt<mode>"
2195 [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w")
2196 (unspec:<V_cmp_result> [(match_operand:VCVTF 1 "s_register_operand" "w")
2197 (match_operand:VCVTF 2 "s_register_operand" "w")
2198 (match_operand:SI 3 "immediate_operand" "i")]
2199 UNSPEC_VCAGT))]
2200 "TARGET_NEON"
2201 "vacgt.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2202 [(set (attr "neon_type")
2203 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
2204 (const_string "neon_fp_vadd_ddd_vabs_dd")
2205 (const_string "neon_fp_vadd_qqq_vabs_qq")))]
2206 )
2208 (define_insn "neon_vtst<mode>"
2209 [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2210 (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
2211 (match_operand:VDQIW 2 "s_register_operand" "w")
2212 (match_operand:SI 3 "immediate_operand" "i")]
2213 UNSPEC_VTST))]
2214 "TARGET_NEON"
2215 "vtst.<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2216 [(set_attr "neon_type" "neon_int_4")]
2217 )
2219 (define_insn "neon_vabd<mode>"
2220 [(set (match_operand:VDQW 0 "s_register_operand" "=w")
2221 (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "w")
2222 (match_operand:VDQW 2 "s_register_operand" "w")
2223 (match_operand:SI 3 "immediate_operand" "i")]
2224 UNSPEC_VABD))]
2225 "TARGET_NEON"
2226 "vabd.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2227 [(set (attr "neon_type")
2228 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
2229 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
2230 (const_string "neon_fp_vadd_ddd_vabs_dd")
2231 (const_string "neon_fp_vadd_qqq_vabs_qq"))
2232 (const_string "neon_int_5")))]
2233 )
2235 (define_insn "neon_vabdl<mode>"
2236 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2237 (unspec:<V_widen> [(match_operand:VW 1 "s_register_operand" "w")
2238 (match_operand:VW 2 "s_register_operand" "w")
2239 (match_operand:SI 3 "immediate_operand" "i")]
2240 UNSPEC_VABDL))]
2241 "TARGET_NEON"
2242 "vabdl.%T3%#<V_sz_elem>\t%q0, %P1, %P2"
2243 [(set_attr "neon_type" "neon_int_5")]
2244 )
2246 (define_insn "neon_vaba<mode>"
2247 [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2248 (plus:VDQIW (match_operand:VDQIW 1 "s_register_operand" "0")
2249 (unspec:VDQIW [(match_operand:VDQIW 2 "s_register_operand" "w")
2250 (match_operand:VDQIW 3 "s_register_operand" "w")
2251 (match_operand:SI 4 "immediate_operand" "i")]
2252 UNSPEC_VABD)))]
2253 "TARGET_NEON"
2254 "vaba.%T4%#<V_sz_elem>\t%<V_reg>0, %<V_reg>2, %<V_reg>3"
2255 [(set (attr "neon_type")
2256 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
2257 (const_string "neon_vaba") (const_string "neon_vaba_qqq")))]
2258 )
2260 (define_insn "neon_vabal<mode>"
2261 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2262 (plus:<V_widen> (match_operand:<V_widen> 1 "s_register_operand" "0")
2263 (unspec:<V_widen> [(match_operand:VW 2 "s_register_operand" "w")
2264 (match_operand:VW 3 "s_register_operand" "w")
2265 (match_operand:SI 4 "immediate_operand" "i")]
2266 UNSPEC_VABDL)))]
2267 "TARGET_NEON"
2268 "vabal.%T4%#<V_sz_elem>\t%q0, %P2, %P3"
2269 [(set_attr "neon_type" "neon_vaba")]
2270 )
2272 (define_insn "neon_vmax<mode>"
2273 [(set (match_operand:VDQW 0 "s_register_operand" "=w")
2274 (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "w")
2275 (match_operand:VDQW 2 "s_register_operand" "w")
2276 (match_operand:SI 3 "immediate_operand" "i")]
2277 UNSPEC_VMAX))]
2278 "TARGET_NEON"
2279 "vmax.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2280 [(set (attr "neon_type")
2281 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
2282 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
2283 (const_string "neon_fp_vadd_ddd_vabs_dd")
2284 (const_string "neon_fp_vadd_qqq_vabs_qq"))
2285 (const_string "neon_int_5")))]
2286 )
2288 (define_insn "neon_vmin<mode>"
2289 [(set (match_operand:VDQW 0 "s_register_operand" "=w")
2290 (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "w")
2291 (match_operand:VDQW 2 "s_register_operand" "w")
2292 (match_operand:SI 3 "immediate_operand" "i")]
2293 UNSPEC_VMIN))]
2294 "TARGET_NEON"
2295 "vmin.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2296 [(set (attr "neon_type")
2297 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
2298 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
2299 (const_string "neon_fp_vadd_ddd_vabs_dd")
2300 (const_string "neon_fp_vadd_qqq_vabs_qq"))
2301 (const_string "neon_int_5")))]
2302 )
2304 (define_expand "neon_vpadd<mode>"
2305 [(match_operand:VD 0 "s_register_operand" "=w")
2306 (match_operand:VD 1 "s_register_operand" "w")
2307 (match_operand:VD 2 "s_register_operand" "w")
2308 (match_operand:SI 3 "immediate_operand" "i")]
2309 "TARGET_NEON"
2310 {
2311 emit_insn (gen_neon_vpadd_internal<mode> (operands[0], operands[1],
2312 operands[2]));
2313 DONE;
2314 })
2316 (define_insn "neon_vpaddl<mode>"
2317 [(set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
2318 (unspec:<V_double_width> [(match_operand:VDQIW 1 "s_register_operand" "w")
2319 (match_operand:SI 2 "immediate_operand" "i")]
2320 UNSPEC_VPADDL))]
2321 "TARGET_NEON"
2322 "vpaddl.%T2%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1"
2323 ;; Assume this schedules like vaddl.
2324 [(set_attr "neon_type" "neon_int_3")]
2325 )
2327 (define_insn "neon_vpadal<mode>"
2328 [(set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
2329 (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
2330 (match_operand:VDQIW 2 "s_register_operand" "w")
2331 (match_operand:SI 3 "immediate_operand" "i")]
2332 UNSPEC_VPADAL))]
2333 "TARGET_NEON"
2334 "vpadal.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>2"
2335 ;; Assume this schedules like vpadd.
2336 [(set_attr "neon_type" "neon_int_1")]
2337 )
2339 (define_insn "neon_vpmax<mode>"
2340 [(set (match_operand:VD 0 "s_register_operand" "=w")
2341 (unspec:VD [(match_operand:VD 1 "s_register_operand" "w")
2342 (match_operand:VD 2 "s_register_operand" "w")
2343 (match_operand:SI 3 "immediate_operand" "i")]
2344 UNSPEC_VPMAX))]
2345 "TARGET_NEON"
2346 "vpmax.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2347 ;; Assume this schedules like vmax.
2348 [(set (attr "neon_type")
2349 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
2350 (const_string "neon_fp_vadd_ddd_vabs_dd")
2351 (const_string "neon_int_5")))]
2352 )
2354 (define_insn "neon_vpmin<mode>"
2355 [(set (match_operand:VD 0 "s_register_operand" "=w")
2356 (unspec:VD [(match_operand:VD 1 "s_register_operand" "w")
2357 (match_operand:VD 2 "s_register_operand" "w")
2358 (match_operand:SI 3 "immediate_operand" "i")]
2359 UNSPEC_VPMIN))]
2360 "TARGET_NEON"
2361 "vpmin.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2362 ;; Assume this schedules like vmin.
2363 [(set (attr "neon_type")
2364 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
2365 (const_string "neon_fp_vadd_ddd_vabs_dd")
2366 (const_string "neon_int_5")))]
2367 )
2369 (define_insn "neon_vrecps<mode>"
2370 [(set (match_operand:VCVTF 0 "s_register_operand" "=w")
2371 (unspec:VCVTF [(match_operand:VCVTF 1 "s_register_operand" "w")
2372 (match_operand:VCVTF 2 "s_register_operand" "w")
2373 (match_operand:SI 3 "immediate_operand" "i")]
2374 UNSPEC_VRECPS))]
2375 "TARGET_NEON"
2376 "vrecps.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2377 [(set (attr "neon_type")
2378 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
2379 (const_string "neon_fp_vrecps_vrsqrts_ddd")
2380 (const_string "neon_fp_vrecps_vrsqrts_qqq")))]
2381 )
2383 (define_insn "neon_vrsqrts<mode>"
2384 [(set (match_operand:VCVTF 0 "s_register_operand" "=w")
2385 (unspec:VCVTF [(match_operand:VCVTF 1 "s_register_operand" "w")
2386 (match_operand:VCVTF 2 "s_register_operand" "w")
2387 (match_operand:SI 3 "immediate_operand" "i")]
2388 UNSPEC_VRSQRTS))]
2389 "TARGET_NEON"
2390 "vrsqrts.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2391 [(set (attr "neon_type")
2392 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
2393 (const_string "neon_fp_vrecps_vrsqrts_ddd")
2394 (const_string "neon_fp_vrecps_vrsqrts_qqq")))]
2395 )
2397 (define_expand "neon_vabs<mode>"
2398 [(match_operand:VDQW 0 "s_register_operand" "")
2399 (match_operand:VDQW 1 "s_register_operand" "")
2400 (match_operand:SI 2 "immediate_operand" "")]
2401 "TARGET_NEON"
2402 {
2403 emit_insn (gen_abs<mode>2 (operands[0], operands[1]));
2404 DONE;
2405 })
2407 (define_insn "neon_vqabs<mode>"
2408 [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2409 (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
2410 (match_operand:SI 2 "immediate_operand" "i")]
2411 UNSPEC_VQABS))]
2412 "TARGET_NEON"
2413 "vqabs.<V_s_elem>\t%<V_reg>0, %<V_reg>1"
2414 [(set_attr "neon_type" "neon_vqneg_vqabs")]
2415 )
2417 (define_expand "neon_vneg<mode>"
2418 [(match_operand:VDQW 0 "s_register_operand" "")
2419 (match_operand:VDQW 1 "s_register_operand" "")
2420 (match_operand:SI 2 "immediate_operand" "")]
2421 "TARGET_NEON"
2422 {
2423 emit_insn (gen_neg<mode>2 (operands[0], operands[1]));
2424 DONE;
2425 })
2427 (define_insn "neon_vqneg<mode>"
2428 [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2429 (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
2430 (match_operand:SI 2 "immediate_operand" "i")]
2431 UNSPEC_VQNEG))]
2432 "TARGET_NEON"
2433 "vqneg.<V_s_elem>\t%<V_reg>0, %<V_reg>1"
2434 [(set_attr "neon_type" "neon_vqneg_vqabs")]
2435 )
2437 (define_insn "neon_vcls<mode>"
2438 [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2439 (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
2440 (match_operand:SI 2 "immediate_operand" "i")]
2441 UNSPEC_VCLS))]
2442 "TARGET_NEON"
2443 "vcls.<V_s_elem>\t%<V_reg>0, %<V_reg>1"
2444 [(set_attr "neon_type" "neon_int_1")]
2445 )
2447 (define_insn "clz<mode>2"
2448 [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2449 (clz:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w")))]
2450 "TARGET_NEON"
2451 "vclz.<V_if_elem>\t%<V_reg>0, %<V_reg>1"
2452 [(set_attr "neon_type" "neon_int_1")]
2453 )
2455 (define_expand "neon_vclz<mode>"
2456 [(match_operand:VDQIW 0 "s_register_operand" "")
2457 (match_operand:VDQIW 1 "s_register_operand" "")
2458 (match_operand:SI 2 "immediate_operand" "")]
2459 "TARGET_NEON"
2460 {
2461 emit_insn (gen_clz<mode>2 (operands[0], operands[1]));
2462 DONE;
2463 })
2465 (define_insn "popcount<mode>2"
2466 [(set (match_operand:VE 0 "s_register_operand" "=w")
2467 (popcount:VE (match_operand:VE 1 "s_register_operand" "w")))]
2468 "TARGET_NEON"
2469 "vcnt.<V_sz_elem>\t%<V_reg>0, %<V_reg>1"
2470 [(set_attr "neon_type" "neon_int_1")]
2471 )
2473 (define_expand "neon_vcnt<mode>"
2474 [(match_operand:VE 0 "s_register_operand" "=w")
2475 (match_operand:VE 1 "s_register_operand" "w")
2476 (match_operand:SI 2 "immediate_operand" "i")]
2477 "TARGET_NEON"
2478 {
2479 emit_insn (gen_popcount<mode>2 (operands[0], operands[1]));
2480 DONE;
2481 })
2483 (define_insn "neon_vrecpe<mode>"
2484 [(set (match_operand:V32 0 "s_register_operand" "=w")
2485 (unspec:V32 [(match_operand:V32 1 "s_register_operand" "w")
2486 (match_operand:SI 2 "immediate_operand" "i")]
2487 UNSPEC_VRECPE))]
2488 "TARGET_NEON"
2489 "vrecpe.<V_u_elem>\t%<V_reg>0, %<V_reg>1"
2490 [(set (attr "neon_type")
2491 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
2492 (const_string "neon_fp_vadd_ddd_vabs_dd")
2493 (const_string "neon_fp_vadd_qqq_vabs_qq")))]
2494 )
2496 (define_insn "neon_vrsqrte<mode>"
2497 [(set (match_operand:V32 0 "s_register_operand" "=w")
2498 (unspec:V32 [(match_operand:V32 1 "s_register_operand" "w")
2499 (match_operand:SI 2 "immediate_operand" "i")]
2500 UNSPEC_VRSQRTE))]
2501 "TARGET_NEON"
2502 "vrsqrte.<V_u_elem>\t%<V_reg>0, %<V_reg>1"
2503 [(set (attr "neon_type")
2504 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
2505 (const_string "neon_fp_vadd_ddd_vabs_dd")
2506 (const_string "neon_fp_vadd_qqq_vabs_qq")))]
2507 )
2509 (define_expand "neon_vmvn<mode>"
2510 [(match_operand:VDQIW 0 "s_register_operand" "")
2511 (match_operand:VDQIW 1 "s_register_operand" "")
2512 (match_operand:SI 2 "immediate_operand" "")]
2513 "TARGET_NEON"
2514 {
2515 emit_insn (gen_one_cmpl<mode>2 (operands[0], operands[1]));
2516 DONE;
2517 })
2519 (define_insn "neon_vget_lane<mode>_sext_internal"
2520 [(set (match_operand:SI 0 "s_register_operand" "=r")
2521 (sign_extend:SI
2522 (vec_select:<V_elem>
2523 (match_operand:VD 1 "s_register_operand" "w")
2524 (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2525 "TARGET_NEON"
2526 {
2527 if (BYTES_BIG_ENDIAN)
2528 {
2529 int elt = INTVAL (operands[2]);
2530 elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
2531 operands[2] = GEN_INT (elt);
2532 }
2533 return "vmov%?.s<V_sz_elem>\t%0, %P1[%c2]";
2534 }
2535 [(set_attr "predicable" "yes")
2536 (set_attr "neon_type" "neon_bp_simple")]
2537 )
2539 (define_insn "neon_vget_lane<mode>_zext_internal"
2540 [(set (match_operand:SI 0 "s_register_operand" "=r")
2541 (zero_extend:SI
2542 (vec_select:<V_elem>
2543 (match_operand:VD 1 "s_register_operand" "w")
2544 (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2545 "TARGET_NEON"
2546 {
2547 if (BYTES_BIG_ENDIAN)
2548 {
2549 int elt = INTVAL (operands[2]);
2550 elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
2551 operands[2] = GEN_INT (elt);
2552 }
2553 return "vmov%?.u<V_sz_elem>\t%0, %P1[%c2]";
2554 }
2555 [(set_attr "predicable" "yes")
2556 (set_attr "neon_type" "neon_bp_simple")]
2557 )
2559 (define_insn "neon_vget_lane<mode>_sext_internal"
2560 [(set (match_operand:SI 0 "s_register_operand" "=r")
2561 (sign_extend:SI
2562 (vec_select:<V_elem>
2563 (match_operand:VQ 1 "s_register_operand" "w")
2564 (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2565 "TARGET_NEON"
2566 {
2567 rtx ops[3];
2568 int regno = REGNO (operands[1]);
2569 unsigned int halfelts = GET_MODE_NUNITS (<MODE>mode) / 2;
2570 unsigned int elt = INTVAL (operands[2]);
2571 unsigned int elt_adj = elt % halfelts;
2573 if (BYTES_BIG_ENDIAN)
2574 elt_adj = halfelts - 1 - elt_adj;
2576 ops[0] = operands[0];
2577 ops[1] = gen_rtx_REG (<V_HALF>mode, regno + 2 * (elt / halfelts));
2578 ops[2] = GEN_INT (elt_adj);
2579 output_asm_insn ("vmov%?.s<V_sz_elem>\t%0, %P1[%c2]", ops);
2581 return "";
2582 }
2583 [(set_attr "predicable" "yes")
2584 (set_attr "neon_type" "neon_bp_simple")]
2585 )
2587 (define_insn "neon_vget_lane<mode>_zext_internal"
2588 [(set (match_operand:SI 0 "s_register_operand" "=r")
2589 (zero_extend:SI
2590 (vec_select:<V_elem>
2591 (match_operand:VQ 1 "s_register_operand" "w")
2592 (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2593 "TARGET_NEON"
2594 {
2595 rtx ops[3];
2596 int regno = REGNO (operands[1]);
2597 unsigned int halfelts = GET_MODE_NUNITS (<MODE>mode) / 2;
2598 unsigned int elt = INTVAL (operands[2]);
2599 unsigned int elt_adj = elt % halfelts;
2601 if (BYTES_BIG_ENDIAN)
2602 elt_adj = halfelts - 1 - elt_adj;
2604 ops[0] = operands[0];
2605 ops[1] = gen_rtx_REG (<V_HALF>mode, regno + 2 * (elt / halfelts));
2606 ops[2] = GEN_INT (elt_adj);
2607 output_asm_insn ("vmov%?.u<V_sz_elem>\t%0, %P1[%c2]", ops);
2609 return "";
2610 }
2611 [(set_attr "predicable" "yes")
2612 (set_attr "neon_type" "neon_bp_simple")]
2613 )
2615 (define_expand "neon_vget_lane<mode>"
2616 [(match_operand:<V_ext> 0 "s_register_operand" "")
2617 (match_operand:VDQW 1 "s_register_operand" "")
2618 (match_operand:SI 2 "immediate_operand" "")
2619 (match_operand:SI 3 "immediate_operand" "")]
2620 "TARGET_NEON"
2621 {
2622 HOST_WIDE_INT magic = INTVAL (operands[3]);
2623 rtx insn;
2625 neon_lane_bounds (operands[2], 0, GET_MODE_NUNITS (<MODE>mode));
2627 if (BYTES_BIG_ENDIAN)
2628 {
2629 /* The intrinsics are defined in terms of a model where the
2630 element ordering in memory is vldm order, whereas the generic
2631 RTL is defined in terms of a model where the element ordering
2632 in memory is array order. Convert the lane number to conform
2633 to this model. */
2634 unsigned int elt = INTVAL (operands[2]);
2635 unsigned int reg_nelts
2636 = 64 / GET_MODE_BITSIZE (GET_MODE_INNER (<MODE>mode));
2637 elt ^= reg_nelts - 1;
2638 operands[2] = GEN_INT (elt);
2639 }
2641 if ((magic & 3) == 3 || GET_MODE_BITSIZE (GET_MODE_INNER (<MODE>mode)) == 32)
2642 insn = gen_vec_extract<mode> (operands[0], operands[1], operands[2]);
2643 else
2644 {
2645 if ((magic & 1) != 0)
2646 insn = gen_neon_vget_lane<mode>_sext_internal (operands[0], operands[1],
2647 operands[2]);
2648 else
2649 insn = gen_neon_vget_lane<mode>_zext_internal (operands[0], operands[1],
2650 operands[2]);
2651 }
2652 emit_insn (insn);
2653 DONE;
2654 })
2656 ; Operand 3 (info word) is ignored because it does nothing useful with 64-bit
2657 ; elements.
2659 (define_expand "neon_vget_lanedi"
2660 [(match_operand:DI 0 "s_register_operand" "=r")
2661 (match_operand:DI 1 "s_register_operand" "w")
2662 (match_operand:SI 2 "immediate_operand" "i")
2663 (match_operand:SI 3 "immediate_operand" "i")]
2664 "TARGET_NEON"
2665 {
2666 neon_lane_bounds (operands[2], 0, 1);
2667 emit_move_insn (operands[0], operands[1]);
2668 DONE;
2669 })
2671 (define_expand "neon_vget_lanev2di"
2672 [(match_operand:DI 0 "s_register_operand" "=r")
2673 (match_operand:V2DI 1 "s_register_operand" "w")
2674 (match_operand:SI 2 "immediate_operand" "i")
2675 (match_operand:SI 3 "immediate_operand" "i")]
2676 "TARGET_NEON"
2677 {
2678 neon_lane_bounds (operands[2], 0, 2);
2679 emit_insn (gen_vec_extractv2di (operands[0], operands[1], operands[2]));
2680 DONE;
2681 })
2683 (define_expand "neon_vset_lane<mode>"
2684 [(match_operand:VDQ 0 "s_register_operand" "=w")
2685 (match_operand:<V_elem> 1 "s_register_operand" "r")
2686 (match_operand:VDQ 2 "s_register_operand" "0")
2687 (match_operand:SI 3 "immediate_operand" "i")]
2688 "TARGET_NEON"
2689 {
2690 unsigned int elt = INTVAL (operands[3]);
2691 neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
2693 if (BYTES_BIG_ENDIAN)
2694 {
2695 unsigned int reg_nelts
2696 = 64 / GET_MODE_BITSIZE (GET_MODE_INNER (<MODE>mode));
2697 elt ^= reg_nelts - 1;
2698 }
2700 emit_insn (gen_vec_set<mode>_internal (operands[0], operands[1],
2701 GEN_INT (1 << elt), operands[2]));
2702 DONE;
2703 })
2705 ; See neon_vget_lanedi comment for reasons operands 2 & 3 are ignored.
2707 (define_expand "neon_vset_lanedi"
2708 [(match_operand:DI 0 "s_register_operand" "=w")
2709 (match_operand:DI 1 "s_register_operand" "r")
2710 (match_operand:DI 2 "s_register_operand" "0")
2711 (match_operand:SI 3 "immediate_operand" "i")]
2712 "TARGET_NEON"
2713 {
2714 neon_lane_bounds (operands[3], 0, 1);
2715 emit_move_insn (operands[0], operands[1]);
2716 DONE;
2717 })
2719 (define_expand "neon_vcreate<mode>"
2720 [(match_operand:VDX 0 "s_register_operand" "")
2721 (match_operand:DI 1 "general_operand" "")]
2722 "TARGET_NEON"
2723 {
2724 rtx src = gen_lowpart (<MODE>mode, operands[1]);
2725 emit_move_insn (operands[0], src);
2726 DONE;
2727 })
2729 (define_insn "neon_vdup_n<mode>"
2730 [(set (match_operand:VX 0 "s_register_operand" "=w")
2731 (vec_duplicate:VX (match_operand:<V_elem> 1 "s_register_operand" "r")))]
2732 "TARGET_NEON"
2733 "vdup%?.<V_sz_elem>\t%<V_reg>0, %1"
2734 ;; Assume this schedules like vmov.
2735 [(set_attr "predicable" "yes")
2736 (set_attr "neon_type" "neon_bp_simple")]
2737 )
2739 (define_insn "neon_vdup_n<mode>"
2740 [(set (match_operand:V32 0 "s_register_operand" "=w,w")
2741 (vec_duplicate:V32 (match_operand:<V_elem> 1 "s_register_operand" "r,t")))]
2742 "TARGET_NEON"
2743 "@
2744 vdup%?.<V_sz_elem>\t%<V_reg>0, %1
2745 vdup%?.<V_sz_elem>\t%<V_reg>0, %y1"
2746 ;; Assume this schedules like vmov.
2747 [(set_attr "predicable" "yes")
2748 (set_attr "neon_type" "neon_bp_simple")]
2749 )
2751 (define_expand "neon_vdup_ndi"
2752 [(match_operand:DI 0 "s_register_operand" "=w")
2753 (match_operand:DI 1 "s_register_operand" "r")]
2754 "TARGET_NEON"
2755 {
2756 emit_move_insn (operands[0], operands[1]);
2757 DONE;
2758 }
2759 )
2761 (define_insn "neon_vdup_nv2di"
2762 [(set (match_operand:V2DI 0 "s_register_operand" "=w,w")
2763 (vec_duplicate:V2DI (match_operand:DI 1 "s_register_operand" "r,w")))]
2764 "TARGET_NEON"
2765 "@
2766 vmov%?\t%e0, %Q1, %R1\;vmov%?\t%f0, %Q1, %R1
2767 vmov%?\t%e0, %P1\;vmov%?\t%f0, %P1"
2768 [(set_attr "predicable" "yes")
2769 (set_attr "length" "8")
2770 (set_attr "neon_type" "neon_bp_simple")]
2771 )
2773 (define_insn "neon_vdup_lane<mode>_internal"
2774 [(set (match_operand:VDQW 0 "s_register_operand" "=w")
2775 (vec_duplicate:VDQW
2776 (vec_select:<V_elem>
2777 (match_operand:<V_double_vector_mode> 1 "s_register_operand" "w")
2778 (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2779 "TARGET_NEON"
2780 {
2781 if (BYTES_BIG_ENDIAN)
2782 {
2783 int elt = INTVAL (operands[2]);
2784 elt = GET_MODE_NUNITS (<V_double_vector_mode>mode) - 1 - elt;
2785 operands[2] = GEN_INT (elt);
2786 }
2787 if (<Is_d_reg>)
2788 return "vdup.<V_sz_elem>\t%P0, %P1[%c2]";
2789 else
2790 return "vdup.<V_sz_elem>\t%q0, %P1[%c2]";
2791 }
2792 ;; Assume this schedules like vmov.
2793 [(set_attr "neon_type" "neon_bp_simple")]
2794 )
2796 (define_expand "neon_vdup_lane<mode>"
2797 [(match_operand:VDQW 0 "s_register_operand" "=w")
2798 (match_operand:<V_double_vector_mode> 1 "s_register_operand" "w")
2799 (match_operand:SI 2 "immediate_operand" "i")]
2800 "TARGET_NEON"
2801 {
2802 neon_lane_bounds (operands[2], 0, GET_MODE_NUNITS (<V_double_vector_mode>mode));
2803 if (BYTES_BIG_ENDIAN)
2804 {
2805 unsigned int elt = INTVAL (operands[2]);
2806 unsigned int reg_nelts
2807 = 64 / GET_MODE_BITSIZE (GET_MODE_INNER (<V_double_vector_mode>mode));
2808 elt ^= reg_nelts - 1;
2809 operands[2] = GEN_INT (elt);
2810 }
2811 emit_insn (gen_neon_vdup_lane<mode>_internal (operands[0], operands[1],
2812 operands[2]));
2813 DONE;
2814 })
2816 ; Scalar index is ignored, since only zero is valid here.
2817 (define_expand "neon_vdup_lanedi"
2818 [(match_operand:DI 0 "s_register_operand" "=w")
2819 (match_operand:DI 1 "s_register_operand" "w")
2820 (match_operand:SI 2 "immediate_operand" "i")]
2821 "TARGET_NEON"
2822 {
2823 neon_lane_bounds (operands[2], 0, 1);
2824 emit_move_insn (operands[0], operands[1]);
2825 DONE;
2826 })
2828 ; Likewise for v2di, as the DImode second operand has only a single element.
2829 (define_expand "neon_vdup_lanev2di"
2830 [(match_operand:V2DI 0 "s_register_operand" "=w")
2831 (match_operand:DI 1 "s_register_operand" "w")
2832 (match_operand:SI 2 "immediate_operand" "i")]
2833 "TARGET_NEON"
2834 {
2835 neon_lane_bounds (operands[2], 0, 1);
2836 emit_insn (gen_neon_vdup_nv2di (operands[0], operands[1]));
2837 DONE;
2838 })
2840 ;; In this insn, operand 1 should be low, and operand 2 the high part of the
2841 ;; dest vector.
2842 ;; FIXME: A different implementation of this builtin could make it much
2843 ;; more likely that we wouldn't actually need to output anything (we could make
2844 ;; it so that the reg allocator puts things in the right places magically
2845 ;; instead). Lack of subregs for vectors makes that tricky though, I think.
2847 (define_insn "neon_vcombine<mode>"
2848 [(set (match_operand:<V_DOUBLE> 0 "s_register_operand" "=w")
2849 (vec_concat:<V_DOUBLE> (match_operand:VDX 1 "s_register_operand" "w")
2850 (match_operand:VDX 2 "s_register_operand" "w")))]
2851 "TARGET_NEON"
2852 {
2853 int dest = REGNO (operands[0]);
2854 int src1 = REGNO (operands[1]);
2855 int src2 = REGNO (operands[2]);
2856 rtx destlo;
2858 if (src1 == dest && src2 == dest + 2)
2859 return "";
2860 else if (src2 == dest && src1 == dest + 2)
2861 /* Special case of reversed high/low parts. */
2862 return "vswp\t%P1, %P2";
2864 destlo = gen_rtx_REG (<MODE>mode, dest);
2866 if (!reg_overlap_mentioned_p (operands[2], destlo))
2867 {
2868 /* Try to avoid unnecessary moves if part of the result is in the right
2869 place already. */
2870 if (src1 != dest)
2871 output_asm_insn ("vmov\t%e0, %P1", operands);
2872 if (src2 != dest + 2)
2873 output_asm_insn ("vmov\t%f0, %P2", operands);
2874 }
2875 else
2876 {
2877 if (src2 != dest + 2)
2878 output_asm_insn ("vmov\t%f0, %P2", operands);
2879 if (src1 != dest)
2880 output_asm_insn ("vmov\t%e0, %P1", operands);
2881 }
2883 return "";
2884 }
2885 ;; We set the neon_type attribute based on the vmov instructions above.
2886 [(set_attr "length" "8")
2887 (set_attr "neon_type" "neon_bp_simple")]
2888 )
2890 (define_insn "neon_vget_highv16qi"
2891 [(set (match_operand:V8QI 0 "s_register_operand" "=w")
2892 (vec_select:V8QI (match_operand:V16QI 1 "s_register_operand" "w")
2893 (parallel [(const_int 8) (const_int 9)
2894 (const_int 10) (const_int 11)
2895 (const_int 12) (const_int 13)
2896 (const_int 14) (const_int 15)])))]
2897 "TARGET_NEON"
2898 {
2899 int dest = REGNO (operands[0]);
2900 int src = REGNO (operands[1]);
2902 if (dest != src + 2)
2903 return "vmov\t%P0, %f1";
2904 else
2905 return "";
2906 }
2907 [(set_attr "neon_type" "neon_bp_simple")]
2908 )
2910 (define_insn "neon_vget_highv8hi"
2911 [(set (match_operand:V4HI 0 "s_register_operand" "=w")
2912 (vec_select:V4HI (match_operand:V8HI 1 "s_register_operand" "w")
2913 (parallel [(const_int 4) (const_int 5)
2914 (const_int 6) (const_int 7)])))]
2915 "TARGET_NEON"
2916 {
2917 int dest = REGNO (operands[0]);
2918 int src = REGNO (operands[1]);
2920 if (dest != src + 2)
2921 return "vmov\t%P0, %f1";
2922 else
2923 return "";
2924 }
2925 [(set_attr "neon_type" "neon_bp_simple")]
2926 )
2928 (define_insn "neon_vget_highv4si"
2929 [(set (match_operand:V2SI 0 "s_register_operand" "=w")
2930 (vec_select:V2SI (match_operand:V4SI 1 "s_register_operand" "w")
2931 (parallel [(const_int 2) (const_int 3)])))]
2932 "TARGET_NEON"
2933 {
2934 int dest = REGNO (operands[0]);
2935 int src = REGNO (operands[1]);
2937 if (dest != src + 2)
2938 return "vmov\t%P0, %f1";
2939 else
2940 return "";
2941 }
2942 [(set_attr "neon_type" "neon_bp_simple")]
2943 )
2945 (define_insn "neon_vget_highv4sf"
2946 [(set (match_operand:V2SF 0 "s_register_operand" "=w")
2947 (vec_select:V2SF (match_operand:V4SF 1 "s_register_operand" "w")
2948 (parallel [(const_int 2) (const_int 3)])))]
2949 "TARGET_NEON"
2950 {
2951 int dest = REGNO (operands[0]);
2952 int src = REGNO (operands[1]);
2954 if (dest != src + 2)
2955 return "vmov\t%P0, %f1";
2956 else
2957 return "";
2958 }
2959 [(set_attr "neon_type" "neon_bp_simple")]
2960 )
2962 (define_insn "neon_vget_highv2di"
2963 [(set (match_operand:DI 0 "s_register_operand" "=w")
2964 (vec_select:DI (match_operand:V2DI 1 "s_register_operand" "w")
2965 (parallel [(const_int 1)])))]
2966 "TARGET_NEON"
2967 {
2968 int dest = REGNO (operands[0]);
2969 int src = REGNO (operands[1]);
2971 if (dest != src + 2)
2972 return "vmov\t%P0, %f1";
2973 else
2974 return "";
2975 }
2976 [(set_attr "neon_type" "neon_bp_simple")]
2977 )
2979 (define_insn "neon_vget_lowv16qi"
2980 [(set (match_operand:V8QI 0 "s_register_operand" "=w")
2981 (vec_select:V8QI (match_operand:V16QI 1 "s_register_operand" "w")
2982 (parallel [(const_int 0) (const_int 1)
2983 (const_int 2) (const_int 3)
2984 (const_int 4) (const_int 5)
2985 (const_int 6) (const_int 7)])))]
2986 "TARGET_NEON"
2987 {
2988 int dest = REGNO (operands[0]);
2989 int src = REGNO (operands[1]);
2991 if (dest != src)
2992 return "vmov\t%P0, %e1";
2993 else
2994 return "";
2995 }
2996 [(set_attr "neon_type" "neon_bp_simple")]
2997 )
2999 (define_insn "neon_vget_lowv8hi"
3000 [(set (match_operand:V4HI 0 "s_register_operand" "=w")
3001 (vec_select:V4HI (match_operand:V8HI 1 "s_register_operand" "w")
3002 (parallel [(const_int 0) (const_int 1)
3003 (const_int 2) (const_int 3)])))]
3004 "TARGET_NEON"
3005 {
3006 int dest = REGNO (operands[0]);
3007 int src = REGNO (operands[1]);
3009 if (dest != src)
3010 return "vmov\t%P0, %e1";
3011 else
3012 return "";
3013 }
3014 [(set_attr "neon_type" "neon_bp_simple")]
3015 )
3017 (define_insn "neon_vget_lowv4si"
3018 [(set (match_operand:V2SI 0 "s_register_operand" "=w")
3019 (vec_select:V2SI (match_operand:V4SI 1 "s_register_operand" "w")
3020 (parallel [(const_int 0) (const_int 1)])))]
3021 "TARGET_NEON"
3022 {
3023 int dest = REGNO (operands[0]);
3024 int src = REGNO (operands[1]);
3026 if (dest != src)
3027 return "vmov\t%P0, %e1";
3028 else
3029 return "";
3030 }
3031 [(set_attr "neon_type" "neon_bp_simple")]
3032 )
3034 (define_insn "neon_vget_lowv4sf"
3035 [(set (match_operand:V2SF 0 "s_register_operand" "=w")
3036 (vec_select:V2SF (match_operand:V4SF 1 "s_register_operand" "w")
3037 (parallel [(const_int 0) (const_int 1)])))]
3038 "TARGET_NEON"
3039 {
3040 int dest = REGNO (operands[0]);
3041 int src = REGNO (operands[1]);
3043 if (dest != src)
3044 return "vmov\t%P0, %e1";
3045 else
3046 return "";
3047 }
3048 [(set_attr "neon_type" "neon_bp_simple")]
3049 )
3051 (define_insn "neon_vget_lowv2di"
3052 [(set (match_operand:DI 0 "s_register_operand" "=w")
3053 (vec_select:DI (match_operand:V2DI 1 "s_register_operand" "w")
3054 (parallel [(const_int 0)])))]
3055 "TARGET_NEON"
3056 {
3057 int dest = REGNO (operands[0]);
3058 int src = REGNO (operands[1]);
3060 if (dest != src)
3061 return "vmov\t%P0, %e1";
3062 else
3063 return "";
3064 }
3065 [(set_attr "neon_type" "neon_bp_simple")]
3066 )
3068 (define_insn "neon_vcvt<mode>"
3069 [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3070 (unspec:<V_CVTTO> [(match_operand:VCVTF 1 "s_register_operand" "w")
3071 (match_operand:SI 2 "immediate_operand" "i")]
3072 UNSPEC_VCVT))]
3073 "TARGET_NEON"
3074 "vcvt.%T2%#32.f32\t%<V_reg>0, %<V_reg>1"
3075 [(set (attr "neon_type")
3076 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
3077 (const_string "neon_fp_vadd_ddd_vabs_dd")
3078 (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3079 )
3081 (define_insn "neon_vcvt<mode>"
3082 [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3083 (unspec:<V_CVTTO> [(match_operand:VCVTI 1 "s_register_operand" "w")
3084 (match_operand:SI 2 "immediate_operand" "i")]
3085 UNSPEC_VCVT))]
3086 "TARGET_NEON"
3087 "vcvt.f32.%T2%#32\t%<V_reg>0, %<V_reg>1"
3088 [(set (attr "neon_type")
3089 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
3090 (const_string "neon_fp_vadd_ddd_vabs_dd")
3091 (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3092 )
3094 (define_insn "neon_vcvt_n<mode>"
3095 [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3096 (unspec:<V_CVTTO> [(match_operand:VCVTF 1 "s_register_operand" "w")
3097 (match_operand:SI 2 "immediate_operand" "i")
3098 (match_operand:SI 3 "immediate_operand" "i")]
3099 UNSPEC_VCVT_N))]
3100 "TARGET_NEON"
3101 {
3102 neon_const_bounds (operands[2], 1, 33);
3103 return "vcvt.%T3%#32.f32\t%<V_reg>0, %<V_reg>1, %2";
3104 }
3105 [(set (attr "neon_type")
3106 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
3107 (const_string "neon_fp_vadd_ddd_vabs_dd")
3108 (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3109 )
3111 (define_insn "neon_vcvt_n<mode>"
3112 [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3113 (unspec:<V_CVTTO> [(match_operand:VCVTI 1 "s_register_operand" "w")
3114 (match_operand:SI 2 "immediate_operand" "i")
3115 (match_operand:SI 3 "immediate_operand" "i")]
3116 UNSPEC_VCVT_N))]
3117 "TARGET_NEON"
3118 {
3119 neon_const_bounds (operands[2], 1, 33);
3120 return "vcvt.f32.%T3%#32\t%<V_reg>0, %<V_reg>1, %2";
3121 }
3122 [(set (attr "neon_type")
3123 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
3124 (const_string "neon_fp_vadd_ddd_vabs_dd")
3125 (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3126 )
3128 (define_insn "neon_vmovn<mode>"
3129 [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
3130 (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
3131 (match_operand:SI 2 "immediate_operand" "i")]
3132 UNSPEC_VMOVN))]
3133 "TARGET_NEON"
3134 "vmovn.<V_if_elem>\t%P0, %q1"
3135 [(set_attr "neon_type" "neon_bp_simple")]
3136 )
3138 (define_insn "neon_vqmovn<mode>"
3139 [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
3140 (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
3141 (match_operand:SI 2 "immediate_operand" "i")]
3142 UNSPEC_VQMOVN))]
3143 "TARGET_NEON"
3144 "vqmovn.%T2%#<V_sz_elem>\t%P0, %q1"
3145 [(set_attr "neon_type" "neon_shift_2")]
3146 )
3148 (define_insn "neon_vqmovun<mode>"
3149 [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
3150 (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
3151 (match_operand:SI 2 "immediate_operand" "i")]
3152 UNSPEC_VQMOVUN))]
3153 "TARGET_NEON"
3154 "vqmovun.<V_s_elem>\t%P0, %q1"
3155 [(set_attr "neon_type" "neon_shift_2")]
3156 )
3158 (define_insn "neon_vmovl<mode>"
3159 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3160 (unspec:<V_widen> [(match_operand:VW 1 "s_register_operand" "w")
3161 (match_operand:SI 2 "immediate_operand" "i")]
3162 UNSPEC_VMOVL))]
3163 "TARGET_NEON"
3164 "vmovl.%T2%#<V_sz_elem>\t%q0, %P1"
3165 [(set_attr "neon_type" "neon_shift_1")]
3166 )
3168 (define_insn "neon_vmul_lane<mode>"
3169 [(set (match_operand:VMD 0 "s_register_operand" "=w")
3170 (unspec:VMD [(match_operand:VMD 1 "s_register_operand" "w")
3171 (match_operand:VMD 2 "s_register_operand"
3172 "<scalar_mul_constraint>")
3173 (match_operand:SI 3 "immediate_operand" "i")
3174 (match_operand:SI 4 "immediate_operand" "i")]
3175 UNSPEC_VMUL_LANE))]
3176 "TARGET_NEON"
3177 {
3178 neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3179 return "vmul.<V_if_elem>\t%P0, %P1, %P2[%c3]";
3180 }
3181 [(set (attr "neon_type")
3182 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
3183 (const_string "neon_fp_vmul_ddd")
3184 (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0))
3185 (const_string "neon_mul_ddd_16_scalar_32_16_long_scalar")
3186 (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar"))))]
3187 )
3189 (define_insn "neon_vmul_lane<mode>"
3190 [(set (match_operand:VMQ 0 "s_register_operand" "=w")
3191 (unspec:VMQ [(match_operand:VMQ 1 "s_register_operand" "w")
3192 (match_operand:<V_HALF> 2 "s_register_operand"
3193 "<scalar_mul_constraint>")
3194 (match_operand:SI 3 "immediate_operand" "i")
3195 (match_operand:SI 4 "immediate_operand" "i")]
3196 UNSPEC_VMUL_LANE))]
3197 "TARGET_NEON"
3198 {
3199 neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<V_HALF>mode));
3200 return "vmul.<V_if_elem>\t%q0, %q1, %P2[%c3]";
3201 }
3202 [(set (attr "neon_type")
3203 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
3204 (const_string "neon_fp_vmul_qqd")
3205 (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0))
3206 (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")
3207 (const_string "neon_mul_qqd_32_scalar"))))]
3208 )
3210 (define_insn "neon_vmull_lane<mode>"
3211 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3212 (unspec:<V_widen> [(match_operand:VMDI 1 "s_register_operand" "w")
3213 (match_operand:VMDI 2 "s_register_operand"
3214 "<scalar_mul_constraint>")
3215 (match_operand:SI 3 "immediate_operand" "i")
3216 (match_operand:SI 4 "immediate_operand" "i")]
3217 UNSPEC_VMULL_LANE))]
3218 "TARGET_NEON"
3219 {
3220 neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3221 return "vmull.%T4%#<V_sz_elem>\t%q0, %P1, %P2[%c3]";
3222 }
3223 [(set (attr "neon_type")
3224 (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0))
3225 (const_string "neon_mul_ddd_16_scalar_32_16_long_scalar")
3226 (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")))]
3227 )
3229 (define_insn "neon_vqdmull_lane<mode>"
3230 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3231 (unspec:<V_widen> [(match_operand:VMDI 1 "s_register_operand" "w")
3232 (match_operand:VMDI 2 "s_register_operand"
3233 "<scalar_mul_constraint>")
3234 (match_operand:SI 3 "immediate_operand" "i")
3235 (match_operand:SI 4 "immediate_operand" "i")]
3236 UNSPEC_VQDMULL_LANE))]
3237 "TARGET_NEON"
3238 {
3239 neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3240 return "vqdmull.<V_s_elem>\t%q0, %P1, %P2[%c3]";
3241 }
3242 [(set (attr "neon_type")
3243 (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0))
3244 (const_string "neon_mul_ddd_16_scalar_32_16_long_scalar")
3245 (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")))]
3246 )
3248 (define_insn "neon_vqdmulh_lane<mode>"
3249 [(set (match_operand:VMQI 0 "s_register_operand" "=w")
3250 (unspec:VMQI [(match_operand:VMQI 1 "s_register_operand" "w")
3251 (match_operand:<V_HALF> 2 "s_register_operand"
3252 "<scalar_mul_constraint>")
3253 (match_operand:SI 3 "immediate_operand" "i")
3254 (match_operand:SI 4 "immediate_operand" "i")]
3255 UNSPEC_VQDMULH_LANE))]
3256 "TARGET_NEON"
3257 {
3258 neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3259 return "vq%O4dmulh.%T4%#<V_sz_elem>\t%q0, %q1, %P2[%c3]";
3260 }
3261 [(set (attr "neon_type")
3262 (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0))
3263 (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")
3264 (const_string "neon_mul_qqd_32_scalar")))]
3265 )
3267 (define_insn "neon_vqdmulh_lane<mode>"
3268 [(set (match_operand:VMDI 0 "s_register_operand" "=w")
3269 (unspec:VMDI [(match_operand:VMDI 1 "s_register_operand" "w")
3270 (match_operand:VMDI 2 "s_register_operand"
3271 "<scalar_mul_constraint>")
3272 (match_operand:SI 3 "immediate_operand" "i")
3273 (match_operand:SI 4 "immediate_operand" "i")]
3274 UNSPEC_VQDMULH_LANE))]
3275 "TARGET_NEON"
3276 {
3277 neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3278 return "vq%O4dmulh.%T4%#<V_sz_elem>\t%P0, %P1, %P2[%c3]";
3279 }
3280 [(set (attr "neon_type")
3281 (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0))
3282 (const_string "neon_mul_ddd_16_scalar_32_16_long_scalar")
3283 (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")))]
3284 )
3286 (define_insn "neon_vmla_lane<mode>"
3287 [(set (match_operand:VMD 0 "s_register_operand" "=w")
3288 (unspec:VMD [(match_operand:VMD 1 "s_register_operand" "0")
3289 (match_operand:VMD 2 "s_register_operand" "w")
3290 (match_operand:VMD 3 "s_register_operand"
3291 "<scalar_mul_constraint>")
3292 (match_operand:SI 4 "immediate_operand" "i")
3293 (match_operand:SI 5 "immediate_operand" "i")]
3294 UNSPEC_VMLA_LANE))]
3295 "TARGET_NEON"
3296 {
3297 neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3298 return "vmla.<V_if_elem>\t%P0, %P2, %P3[%c4]";
3299 }
3300 [(set (attr "neon_type")
3301 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
3302 (const_string "neon_fp_vmla_ddd_scalar")
3303 (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0))
3304 (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar")
3305 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long"))))]
3306 )
3308 (define_insn "neon_vmla_lane<mode>"
3309 [(set (match_operand:VMQ 0 "s_register_operand" "=w")
3310 (unspec:VMQ [(match_operand:VMQ 1 "s_register_operand" "0")
3311 (match_operand:VMQ 2 "s_register_operand" "w")
3312 (match_operand:<V_HALF> 3 "s_register_operand"
3313 "<scalar_mul_constraint>")
3314 (match_operand:SI 4 "immediate_operand" "i")
3315 (match_operand:SI 5 "immediate_operand" "i")]
3316 UNSPEC_VMLA_LANE))]
3317 "TARGET_NEON"
3318 {
3319 neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3320 return "vmla.<V_if_elem>\t%q0, %q2, %P3[%c4]";
3321 }
3322 [(set (attr "neon_type")
3323 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
3324 (const_string "neon_fp_vmla_qqq_scalar")
3325 (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0))
3326 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")
3327 (const_string "neon_mla_qqq_32_qqd_32_scalar"))))]
3328 )
3330 (define_insn "neon_vmlal_lane<mode>"
3331 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3332 (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
3333 (match_operand:VMDI 2 "s_register_operand" "w")
3334 (match_operand:VMDI 3 "s_register_operand"
3335 "<scalar_mul_constraint>")
3336 (match_operand:SI 4 "immediate_operand" "i")
3337 (match_operand:SI 5 "immediate_operand" "i")]
3338 UNSPEC_VMLAL_LANE))]
3339 "TARGET_NEON"
3340 {
3341 neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3342 return "vmlal.%T5%#<V_sz_elem>\t%q0, %P2, %P3[%c4]";
3343 }
3344 [(set (attr "neon_type")
3345 (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0))
3346 (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar")
3347 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
3348 )
3350 (define_insn "neon_vqdmlal_lane<mode>"
3351 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3352 (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
3353 (match_operand:VMDI 2 "s_register_operand" "w")
3354 (match_operand:VMDI 3 "s_register_operand"
3355 "<scalar_mul_constraint>")
3356 (match_operand:SI 4 "immediate_operand" "i")
3357 (match_operand:SI 5 "immediate_operand" "i")]
3358 UNSPEC_VQDMLAL_LANE))]
3359 "TARGET_NEON"
3360 {
3361 neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3362 return "vqdmlal.<V_s_elem>\t%q0, %P2, %P3[%c4]";
3363 }
3364 [(set (attr "neon_type")
3365 (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0))
3366 (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar")
3367 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
3368 )
3370 (define_insn "neon_vmls_lane<mode>"
3371 [(set (match_operand:VMD 0 "s_register_operand" "=w")
3372 (unspec:VMD [(match_operand:VMD 1 "s_register_operand" "0")
3373 (match_operand:VMD 2 "s_register_operand" "w")
3374 (match_operand:VMD 3 "s_register_operand"
3375 "<scalar_mul_constraint>")
3376 (match_operand:SI 4 "immediate_operand" "i")
3377 (match_operand:SI 5 "immediate_operand" "i")]
3378 UNSPEC_VMLS_LANE))]
3379 "TARGET_NEON"
3380 {
3381 neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3382 return "vmls.<V_if_elem>\t%P0, %P2, %P3[%c4]";
3383 }
3384 [(set (attr "neon_type")
3385 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
3386 (const_string "neon_fp_vmla_ddd_scalar")
3387 (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0))
3388 (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar")
3389 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long"))))]
3390 )
3392 (define_insn "neon_vmls_lane<mode>"
3393 [(set (match_operand:VMQ 0 "s_register_operand" "=w")
3394 (unspec:VMQ [(match_operand:VMQ 1 "s_register_operand" "0")
3395 (match_operand:VMQ 2 "s_register_operand" "w")
3396 (match_operand:<V_HALF> 3 "s_register_operand"
3397 "<scalar_mul_constraint>")
3398 (match_operand:SI 4 "immediate_operand" "i")
3399 (match_operand:SI 5 "immediate_operand" "i")]
3400 UNSPEC_VMLS_LANE))]
3401 "TARGET_NEON"
3402 {
3403 neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3404 return "vmls.<V_if_elem>\t%q0, %q2, %P3[%c4]";
3405 }
3406 [(set (attr "neon_type")
3407 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
3408 (const_string "neon_fp_vmla_qqq_scalar")
3409 (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0))
3410 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")
3411 (const_string "neon_mla_qqq_32_qqd_32_scalar"))))]
3412 )
3414 (define_insn "neon_vmlsl_lane<mode>"
3415 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3416 (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
3417 (match_operand:VMDI 2 "s_register_operand" "w")
3418 (match_operand:VMDI 3 "s_register_operand"
3419 "<scalar_mul_constraint>")
3420 (match_operand:SI 4 "immediate_operand" "i")
3421 (match_operand:SI 5 "immediate_operand" "i")]
3422 UNSPEC_VMLSL_LANE))]
3423 "TARGET_NEON"
3424 {
3425 neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3426 return "vmlsl.%T5%#<V_sz_elem>\t%q0, %P2, %P3[%c4]";
3427 }
3428 [(set (attr "neon_type")
3429 (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0))
3430 (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar")
3431 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
3432 )
3434 (define_insn "neon_vqdmlsl_lane<mode>"
3435 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3436 (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
3437 (match_operand:VMDI 2 "s_register_operand" "w")
3438 (match_operand:VMDI 3 "s_register_operand"
3439 "<scalar_mul_constraint>")
3440 (match_operand:SI 4 "immediate_operand" "i")
3441 (match_operand:SI 5 "immediate_operand" "i")]
3442 UNSPEC_VQDMLSL_LANE))]
3443 "TARGET_NEON"
3444 {
3445 neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3446 return "vqdmlsl.<V_s_elem>\t%q0, %P2, %P3[%c4]";
3447 }
3448 [(set (attr "neon_type")
3449 (if_then_else (ne (symbol_ref "<Scalar_mul_8_16>") (const_int 0))
3450 (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar")
3451 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
3452 )
3454 ; FIXME: For the "_n" multiply/multiply-accumulate insns, we copy a value in a
3455 ; core register into a temp register, then use a scalar taken from that. This
3456 ; isn't an optimal solution if e.g. the scalar has just been read from memory
3457 ; or extracted from another vector. The latter case it's currently better to
3458 ; use the "_lane" variant, and the former case can probably be implemented
3459 ; using vld1_lane, but that hasn't been done yet.
3461 (define_expand "neon_vmul_n<mode>"
3462 [(match_operand:VMD 0 "s_register_operand" "")
3463 (match_operand:VMD 1 "s_register_operand" "")
3464 (match_operand:<V_elem> 2 "s_register_operand" "")
3465 (match_operand:SI 3 "immediate_operand" "")]
3466 "TARGET_NEON"
3467 {
3468 rtx tmp = gen_reg_rtx (<MODE>mode);
3469 emit_insn (gen_neon_vset_lane<mode> (tmp, operands[2], tmp, const0_rtx));
3470 emit_insn (gen_neon_vmul_lane<mode> (operands[0], operands[1], tmp,
3471 const0_rtx, const0_rtx));
3472 DONE;
3473 })
3475 (define_expand "neon_vmul_n<mode>"
3476 [(match_operand:VMQ 0 "s_register_operand" "")
3477 (match_operand:VMQ 1 "s_register_operand" "")
3478 (match_operand:<V_elem> 2 "s_register_operand" "")
3479 (match_operand:SI 3 "immediate_operand" "")]
3480 "TARGET_NEON"
3481 {
3482 rtx tmp = gen_reg_rtx (<V_HALF>mode);
3483 emit_insn (gen_neon_vset_lane<V_half> (tmp, operands[2], tmp, const0_rtx));
3484 emit_insn (gen_neon_vmul_lane<mode> (operands[0], operands[1], tmp,
3485 const0_rtx, const0_rtx));
3486 DONE;
3487 })
3489 (define_expand "neon_vmull_n<mode>"
3490 [(match_operand:<V_widen> 0 "s_register_operand" "")
3491 (match_operand:VMDI 1 "s_register_operand" "")
3492 (match_operand:<V_elem> 2 "s_register_operand" "")
3493 (match_operand:SI 3 "immediate_operand" "")]
3494 "TARGET_NEON"
3495 {
3496 rtx tmp = gen_reg_rtx (<MODE>mode);
3497 emit_insn (gen_neon_vset_lane<mode> (tmp, operands[2], tmp, const0_rtx));
3498 emit_insn (gen_neon_vmull_lane<mode> (operands[0], operands[1], tmp,
3499 const0_rtx, operands[3]));
3500 DONE;
3501 })
3503 (define_expand "neon_vqdmull_n<mode>"
3504 [(match_operand:<V_widen> 0 "s_register_operand" "")
3505 (match_operand:VMDI 1 "s_register_operand" "")
3506 (match_operand:<V_elem> 2 "s_register_operand" "")
3507 (match_operand:SI 3 "immediate_operand" "")]
3508 "TARGET_NEON"
3509 {
3510 rtx tmp = gen_reg_rtx (<MODE>mode);
3511 emit_insn (gen_neon_vset_lane<mode> (tmp, operands[2], tmp, const0_rtx));
3512 emit_insn (gen_neon_vqdmull_lane<mode> (operands[0], operands[1], tmp,
3513 const0_rtx, const0_rtx));
3514 DONE;
3515 })
3517 (define_expand "neon_vqdmulh_n<mode>"
3518 [(match_operand:VMDI 0 "s_register_operand" "")
3519 (match_operand:VMDI 1 "s_register_operand" "")
3520 (match_operand:<V_elem> 2 "s_register_operand" "")
3521 (match_operand:SI 3 "immediate_operand" "")]
3522 "TARGET_NEON"
3523 {
3524 rtx tmp = gen_reg_rtx (<MODE>mode);
3525 emit_insn (gen_neon_vset_lane<mode> (tmp, operands[2], tmp, const0_rtx));
3526 emit_insn (gen_neon_vqdmulh_lane<mode> (operands[0], operands[1], tmp,
3527 const0_rtx, operands[3]));
3528 DONE;
3529 })
3531 (define_expand "neon_vqdmulh_n<mode>"
3532 [(match_operand:VMQI 0 "s_register_operand" "")
3533 (match_operand:VMQI 1 "s_register_operand" "")
3534 (match_operand:<V_elem> 2 "s_register_operand" "")
3535 (match_operand:SI 3 "immediate_operand" "")]
3536 "TARGET_NEON"
3537 {
3538 rtx tmp = gen_reg_rtx (<V_HALF>mode);
3539 emit_insn (gen_neon_vset_lane<V_half> (tmp, operands[2], tmp, const0_rtx));
3540 emit_insn (gen_neon_vqdmulh_lane<mode> (operands[0], operands[1], tmp,
3541 const0_rtx, operands[3]));
3542 DONE;
3543 })
3545 (define_expand "neon_vmla_n<mode>"
3546 [(match_operand:VMD 0 "s_register_operand" "")
3547 (match_operand:VMD 1 "s_register_operand" "")
3548 (match_operand:VMD 2 "s_register_operand" "")
3549 (match_operand:<V_elem> 3 "s_register_operand" "")
3550 (match_operand:SI 4 "immediate_operand" "")]
3551 "TARGET_NEON"
3552 {
3553 rtx tmp = gen_reg_rtx (<MODE>mode);
3554 emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3555 emit_insn (gen_neon_vmla_lane<mode> (operands[0], operands[1], operands[2],
3556 tmp, const0_rtx, operands[4]));
3557 DONE;
3558 })
3560 (define_expand "neon_vmla_n<mode>"
3561 [(match_operand:VMQ 0 "s_register_operand" "")
3562 (match_operand:VMQ 1 "s_register_operand" "")
3563 (match_operand:VMQ 2 "s_register_operand" "")
3564 (match_operand:<V_elem> 3 "s_register_operand" "")
3565 (match_operand:SI 4 "immediate_operand" "")]
3566 "TARGET_NEON"
3567 {
3568 rtx tmp = gen_reg_rtx (<V_HALF>mode);
3569 emit_insn (gen_neon_vset_lane<V_half> (tmp, operands[3], tmp, const0_rtx));
3570 emit_insn (gen_neon_vmla_lane<mode> (operands[0], operands[1], operands[2],
3571 tmp, const0_rtx, operands[4]));
3572 DONE;
3573 })
3575 (define_expand "neon_vmlal_n<mode>"
3576 [(match_operand:<V_widen> 0 "s_register_operand" "")
3577 (match_operand:<V_widen> 1 "s_register_operand" "")
3578 (match_operand:VMDI 2 "s_register_operand" "")
3579 (match_operand:<V_elem> 3 "s_register_operand" "")
3580 (match_operand:SI 4 "immediate_operand" "")]
3581 "TARGET_NEON"
3582 {
3583 rtx tmp = gen_reg_rtx (<MODE>mode);
3584 emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3585 emit_insn (gen_neon_vmlal_lane<mode> (operands[0], operands[1], operands[2],
3586 tmp, const0_rtx, operands[4]));
3587 DONE;
3588 })
3590 (define_expand "neon_vqdmlal_n<mode>"
3591 [(match_operand:<V_widen> 0 "s_register_operand" "")
3592 (match_operand:<V_widen> 1 "s_register_operand" "")
3593 (match_operand:VMDI 2 "s_register_operand" "")
3594 (match_operand:<V_elem> 3 "s_register_operand" "")
3595 (match_operand:SI 4 "immediate_operand" "")]
3596 "TARGET_NEON"
3597 {
3598 rtx tmp = gen_reg_rtx (<MODE>mode);
3599 emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3600 emit_insn (gen_neon_vqdmlal_lane<mode> (operands[0], operands[1], operands[2],
3601 tmp, const0_rtx, operands[4]));
3602 DONE;
3603 })
3605 (define_expand "neon_vmls_n<mode>"
3606 [(match_operand:VMD 0 "s_register_operand" "")
3607 (match_operand:VMD 1 "s_register_operand" "")
3608 (match_operand:VMD 2 "s_register_operand" "")
3609 (match_operand:<V_elem> 3 "s_register_operand" "")
3610 (match_operand:SI 4 "immediate_operand" "")]
3611 "TARGET_NEON"
3612 {
3613 rtx tmp = gen_reg_rtx (<MODE>mode);
3614 emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3615 emit_insn (gen_neon_vmls_lane<mode> (operands[0], operands[1], operands[2],
3616 tmp, const0_rtx, operands[4]));
3617 DONE;
3618 })
3620 (define_expand "neon_vmls_n<mode>"
3621 [(match_operand:VMQ 0 "s_register_operand" "")
3622 (match_operand:VMQ 1 "s_register_operand" "")
3623 (match_operand:VMQ 2 "s_register_operand" "")
3624 (match_operand:<V_elem> 3 "s_register_operand" "")
3625 (match_operand:SI 4 "immediate_operand" "")]
3626 "TARGET_NEON"
3627 {
3628 rtx tmp = gen_reg_rtx (<V_HALF>mode);
3629 emit_insn (gen_neon_vset_lane<V_half> (tmp, operands[3], tmp, const0_rtx));
3630 emit_insn (gen_neon_vmls_lane<mode> (operands[0], operands[1], operands[2],
3631 tmp, const0_rtx, operands[4]));
3632 DONE;
3633 })
3635 (define_expand "neon_vmlsl_n<mode>"
3636 [(match_operand:<V_widen> 0 "s_register_operand" "")
3637 (match_operand:<V_widen> 1 "s_register_operand" "")
3638 (match_operand:VMDI 2 "s_register_operand" "")
3639 (match_operand:<V_elem> 3 "s_register_operand" "")
3640 (match_operand:SI 4 "immediate_operand" "")]
3641 "TARGET_NEON"
3642 {
3643 rtx tmp = gen_reg_rtx (<MODE>mode);
3644 emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3645 emit_insn (gen_neon_vmlsl_lane<mode> (operands[0], operands[1], operands[2],
3646 tmp, const0_rtx, operands[4]));
3647 DONE;
3648 })
3650 (define_expand "neon_vqdmlsl_n<mode>"
3651 [(match_operand:<V_widen> 0 "s_register_operand" "")
3652 (match_operand:<V_widen> 1 "s_register_operand" "")
3653 (match_operand:VMDI 2 "s_register_operand" "")
3654 (match_operand:<V_elem> 3 "s_register_operand" "")
3655 (match_operand:SI 4 "immediate_operand" "")]
3656 "TARGET_NEON"
3657 {
3658 rtx tmp = gen_reg_rtx (<MODE>mode);
3659 emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3660 emit_insn (gen_neon_vqdmlsl_lane<mode> (operands[0], operands[1], operands[2],
3661 tmp, const0_rtx, operands[4]));
3662 DONE;
3663 })
3665 (define_insn "neon_vext<mode>"
3666 [(set (match_operand:VDQX 0 "s_register_operand" "=w")
3667 (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand" "w")
3668 (match_operand:VDQX 2 "s_register_operand" "w")
3669 (match_operand:SI 3 "immediate_operand" "i")]
3670 UNSPEC_VEXT))]
3671 "TARGET_NEON"
3672 {
3673 neon_const_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3674 return "vext.<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2, %3";
3675 }
3676 [(set (attr "neon_type")
3677 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
3678 (const_string "neon_bp_simple")
3679 (const_string "neon_bp_2cycle")))]
3680 )
3682 (define_insn "neon_vrev64<mode>"
3683 [(set (match_operand:VDQ 0 "s_register_operand" "=w")
3684 (unspec:VDQ [(match_operand:VDQ 1 "s_register_operand" "w")
3685 (match_operand:SI 2 "immediate_operand" "i")]
3686 UNSPEC_VREV64))]
3687 "TARGET_NEON"
3688 "vrev64.<V_sz_elem>\t%<V_reg>0, %<V_reg>1"
3689 [(set_attr "neon_type" "neon_bp_simple")]
3690 )
3692 (define_insn "neon_vrev32<mode>"
3693 [(set (match_operand:VX 0 "s_register_operand" "=w")
3694 (unspec:VX [(match_operand:VX 1 "s_register_operand" "w")
3695 (match_operand:SI 2 "immediate_operand" "i")]
3696 UNSPEC_VREV32))]
3697 "TARGET_NEON"
3698 "vrev32.<V_sz_elem>\t%<V_reg>0, %<V_reg>1"
3699 [(set_attr "neon_type" "neon_bp_simple")]
3700 )
3702 (define_insn "neon_vrev16<mode>"
3703 [(set (match_operand:VE 0 "s_register_operand" "=w")
3704 (unspec:VE [(match_operand:VE 1 "s_register_operand" "w")
3705 (match_operand:SI 2 "immediate_operand" "i")]
3706 UNSPEC_VREV16))]
3707 "TARGET_NEON"
3708 "vrev16.<V_sz_elem>\t%<V_reg>0, %<V_reg>1"
3709 [(set_attr "neon_type" "neon_bp_simple")]
3710 )
3712 ; vbsl_* intrinsics may compile to any of vbsl/vbif/vbit depending on register
3713 ; allocation. For an intrinsic of form:
3714 ; rD = vbsl_* (rS, rN, rM)
3715 ; We can use any of:
3716 ; vbsl rS, rN, rM (if D = S)
3717 ; vbit rD, rN, rS (if D = M, so 1-bits in rS choose bits from rN, else rM)
3718 ; vbif rD, rM, rS (if D = N, so 0-bits in rS choose bits from rM, else rN)
3720 (define_insn "neon_vbsl<mode>_internal"
3721 [(set (match_operand:VDQX 0 "s_register_operand" "=w,w,w")
3722 (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand" " 0,w,w")
3723 (match_operand:VDQX 2 "s_register_operand" " w,w,0")
3724 (match_operand:VDQX 3 "s_register_operand" " w,0,w")]
3725 UNSPEC_VBSL))]
3726 "TARGET_NEON"
3727 "@
3728 vbsl\t%<V_reg>0, %<V_reg>2, %<V_reg>3
3729 vbit\t%<V_reg>0, %<V_reg>2, %<V_reg>1
3730 vbif\t%<V_reg>0, %<V_reg>3, %<V_reg>1"
3731 [(set_attr "neon_type" "neon_int_1")]
3732 )
3734 (define_expand "neon_vbsl<mode>"
3735 [(set (match_operand:VDQX 0 "s_register_operand" "")
3736 (unspec:VDQX [(match_operand:<V_cmp_result> 1 "s_register_operand" "")
3737 (match_operand:VDQX 2 "s_register_operand" "")
3738 (match_operand:VDQX 3 "s_register_operand" "")]
3739 UNSPEC_VBSL))]
3740 "TARGET_NEON"
3741 {
3742 /* We can't alias operands together if they have different modes. */
3743 operands[1] = gen_lowpart (<MODE>mode, operands[1]);
3744 })
3746 (define_insn "neon_vshl<mode>"
3747 [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3748 (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
3749 (match_operand:VDQIX 2 "s_register_operand" "w")
3750 (match_operand:SI 3 "immediate_operand" "i")]
3751 UNSPEC_VSHL))]
3752 "TARGET_NEON"
3753 "v%O3shl.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
3754 [(set (attr "neon_type")
3755 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
3756 (const_string "neon_vshl_ddd")
3757 (const_string "neon_shift_3")))]
3758 )
3760 (define_insn "neon_vqshl<mode>"
3761 [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3762 (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
3763 (match_operand:VDQIX 2 "s_register_operand" "w")
3764 (match_operand:SI 3 "immediate_operand" "i")]
3765 UNSPEC_VQSHL))]
3766 "TARGET_NEON"
3767 "vq%O3shl.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
3768 [(set (attr "neon_type")
3769 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
3770 (const_string "neon_shift_2")
3771 (const_string "neon_vqshl_vrshl_vqrshl_qqq")))]
3772 )
3774 (define_insn "neon_vshr_n<mode>"
3775 [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3776 (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
3777 (match_operand:SI 2 "immediate_operand" "i")
3778 (match_operand:SI 3 "immediate_operand" "i")]
3779 UNSPEC_VSHR_N))]
3780 "TARGET_NEON"
3781 {
3782 neon_const_bounds (operands[2], 1, neon_element_bits (<MODE>mode) + 1);
3783 return "v%O3shr.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %2";
3784 }
3785 [(set_attr "neon_type" "neon_shift_1")]
3786 )
3788 (define_insn "neon_vshrn_n<mode>"
3789 [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
3790 (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
3791 (match_operand:SI 2 "immediate_operand" "i")
3792 (match_operand:SI 3 "immediate_operand" "i")]
3793 UNSPEC_VSHRN_N))]
3794 "TARGET_NEON"
3795 {
3796 neon_const_bounds (operands[2], 1, neon_element_bits (<MODE>mode) / 2 + 1);
3797 return "v%O3shrn.<V_if_elem>\t%P0, %q1, %2";
3798 }
3799 [(set_attr "neon_type" "neon_shift_1")]
3800 )
3802 (define_insn "neon_vqshrn_n<mode>"
3803 [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
3804 (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
3805 (match_operand:SI 2 "immediate_operand" "i")
3806 (match_operand:SI 3 "immediate_operand" "i")]
3807 UNSPEC_VQSHRN_N))]
3808 "TARGET_NEON"
3809 {
3810 neon_const_bounds (operands[2], 1, neon_element_bits (<MODE>mode) / 2 + 1);
3811 return "vq%O3shrn.%T3%#<V_sz_elem>\t%P0, %q1, %2";
3812 }
3813 [(set_attr "neon_type" "neon_shift_2")]
3814 )
3816 (define_insn "neon_vqshrun_n<mode>"
3817 [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
3818 (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
3819 (match_operand:SI 2 "immediate_operand" "i")
3820 (match_operand:SI 3 "immediate_operand" "i")]
3821 UNSPEC_VQSHRUN_N))]
3822 "TARGET_NEON"
3823 {
3824 neon_const_bounds (operands[2], 1, neon_element_bits (<MODE>mode) / 2 + 1);
3825 return "vq%O3shrun.%T3%#<V_sz_elem>\t%P0, %q1, %2";
3826 }
3827 [(set_attr "neon_type" "neon_shift_2")]
3828 )
3830 (define_insn "neon_vshl_n<mode>"
3831 [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3832 (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
3833 (match_operand:SI 2 "immediate_operand" "i")
3834 (match_operand:SI 3 "immediate_operand" "i")]
3835 UNSPEC_VSHL_N))]
3836 "TARGET_NEON"
3837 {
3838 neon_const_bounds (operands[2], 0, neon_element_bits (<MODE>mode));
3839 return "vshl.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %2";
3840 }
3841 [(set_attr "neon_type" "neon_shift_1")]
3842 )
3844 (define_insn "neon_vqshl_n<mode>"
3845 [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3846 (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
3847 (match_operand:SI 2 "immediate_operand" "i")
3848 (match_operand:SI 3 "immediate_operand" "i")]
3849 UNSPEC_VQSHL_N))]
3850 "TARGET_NEON"
3851 {
3852 neon_const_bounds (operands[2], 0, neon_element_bits (<MODE>mode));
3853 return "vqshl.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %2";
3854 }
3855 [(set_attr "neon_type" "neon_shift_2")]
3856 )
3858 (define_insn "neon_vqshlu_n<mode>"
3859 [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3860 (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
3861 (match_operand:SI 2 "immediate_operand" "i")
3862 (match_operand:SI 3 "immediate_operand" "i")]
3863 UNSPEC_VQSHLU_N))]
3864 "TARGET_NEON"
3865 {
3866 neon_const_bounds (operands[2], 0, neon_element_bits (<MODE>mode));
3867 return "vqshlu.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %2";
3868 }
3869 [(set_attr "neon_type" "neon_shift_2")]
3870 )
3872 (define_insn "neon_vshll_n<mode>"
3873 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3874 (unspec:<V_widen> [(match_operand:VW 1 "s_register_operand" "w")
3875 (match_operand:SI 2 "immediate_operand" "i")
3876 (match_operand:SI 3 "immediate_operand" "i")]
3877 UNSPEC_VSHLL_N))]
3878 "TARGET_NEON"
3879 {
3880 /* The boundaries are: 0 < imm <= size. */
3881 neon_const_bounds (operands[2], 0, neon_element_bits (<MODE>mode) + 1);
3882 return "vshll.%T3%#<V_sz_elem>\t%q0, %P1, %2";
3883 }
3884 [(set_attr "neon_type" "neon_shift_1")]
3885 )
3887 (define_insn "neon_vsra_n<mode>"
3888 [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3889 (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "0")
3890 (match_operand:VDQIX 2 "s_register_operand" "w")
3891 (match_operand:SI 3 "immediate_operand" "i")
3892 (match_operand:SI 4 "immediate_operand" "i")]
3893 UNSPEC_VSRA_N))]
3894 "TARGET_NEON"
3895 {
3896 neon_const_bounds (operands[3], 1, neon_element_bits (<MODE>mode) + 1);
3897 return "v%O4sra.%T4%#<V_sz_elem>\t%<V_reg>0, %<V_reg>2, %3";
3898 }
3899 [(set_attr "neon_type" "neon_vsra_vrsra")]
3900 )
3902 (define_insn "neon_vsri_n<mode>"
3903 [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3904 (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "0")
3905 (match_operand:VDQIX 2 "s_register_operand" "w")
3906 (match_operand:SI 3 "immediate_operand" "i")]
3907 UNSPEC_VSRI))]
3908 "TARGET_NEON"
3909 {
3910 neon_const_bounds (operands[3], 1, neon_element_bits (<MODE>mode) + 1);
3911 return "vsri.<V_sz_elem>\t%<V_reg>0, %<V_reg>2, %3";
3912 }
3913 [(set (attr "neon_type")
3914 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
3915 (const_string "neon_shift_1")
3916 (const_string "neon_shift_3")))]
3917 )
3919 (define_insn "neon_vsli_n<mode>"
3920 [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3921 (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "0")
3922 (match_operand:VDQIX 2 "s_register_operand" "w")
3923 (match_operand:SI 3 "immediate_operand" "i")]
3924 UNSPEC_VSLI))]
3925 "TARGET_NEON"
3926 {
3927 neon_const_bounds (operands[3], 0, neon_element_bits (<MODE>mode));
3928 return "vsli.<V_sz_elem>\t%<V_reg>0, %<V_reg>2, %3";
3929 }
3930 [(set (attr "neon_type")
3931 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
3932 (const_string "neon_shift_1")
3933 (const_string "neon_shift_3")))]
3934 )
3936 (define_insn "neon_vtbl1v8qi"
3937 [(set (match_operand:V8QI 0 "s_register_operand" "=w")
3938 (unspec:V8QI [(match_operand:V8QI 1 "s_register_operand" "w")
3939 (match_operand:V8QI 2 "s_register_operand" "w")]
3940 UNSPEC_VTBL))]
3941 "TARGET_NEON"
3942 "vtbl.8\t%P0, {%P1}, %P2"
3943 [(set_attr "neon_type" "neon_bp_2cycle")]
3944 )
3946 (define_insn "neon_vtbl2v8qi"
3947 [(set (match_operand:V8QI 0 "s_register_operand" "=w")
3948 (unspec:V8QI [(match_operand:TI 1 "s_register_operand" "w")
3949 (match_operand:V8QI 2 "s_register_operand" "w")]
3950 UNSPEC_VTBL))]
3951 "TARGET_NEON"
3952 {
3953 rtx ops[4];
3954 int tabbase = REGNO (operands[1]);
3956 ops[0] = operands[0];
3957 ops[1] = gen_rtx_REG (V8QImode, tabbase);
3958 ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
3959 ops[3] = operands[2];
3960 output_asm_insn ("vtbl.8\t%P0, {%P1, %P2}, %P3", ops);
3962 return "";
3963 }
3964 [(set_attr "neon_type" "neon_bp_2cycle")]
3965 )
3967 (define_insn "neon_vtbl3v8qi"
3968 [(set (match_operand:V8QI 0 "s_register_operand" "=w")
3969 (unspec:V8QI [(match_operand:EI 1 "s_register_operand" "w")
3970 (match_operand:V8QI 2 "s_register_operand" "w")]
3971 UNSPEC_VTBL))]
3972 "TARGET_NEON"
3973 {
3974 rtx ops[5];
3975 int tabbase = REGNO (operands[1]);
3977 ops[0] = operands[0];
3978 ops[1] = gen_rtx_REG (V8QImode, tabbase);
3979 ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
3980 ops[3] = gen_rtx_REG (V8QImode, tabbase + 4);
3981 ops[4] = operands[2];
3982 output_asm_insn ("vtbl.8\t%P0, {%P1, %P2, %P3}, %P4", ops);
3984 return "";
3985 }
3986 [(set_attr "neon_type" "neon_bp_3cycle")]
3987 )
3989 (define_insn "neon_vtbl4v8qi"
3990 [(set (match_operand:V8QI 0 "s_register_operand" "=w")
3991 (unspec:V8QI [(match_operand:OI 1 "s_register_operand" "w")
3992 (match_operand:V8QI 2 "s_register_operand" "w")]
3993 UNSPEC_VTBL))]
3994 "TARGET_NEON"
3995 {
3996 rtx ops[6];
3997 int tabbase = REGNO (operands[1]);
3999 ops[0] = operands[0];
4000 ops[1] = gen_rtx_REG (V8QImode, tabbase);
4001 ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
4002 ops[3] = gen_rtx_REG (V8QImode, tabbase + 4);
4003 ops[4] = gen_rtx_REG (V8QImode, tabbase + 6);
4004 ops[5] = operands[2];
4005 output_asm_insn ("vtbl.8\t%P0, {%P1, %P2, %P3, %P4}, %P5", ops);
4007 return "";
4008 }
4009 [(set_attr "neon_type" "neon_bp_3cycle")]
4010 )
4012 (define_insn "neon_vtbx1v8qi"
4013 [(set (match_operand:V8QI 0 "s_register_operand" "=w")
4014 (unspec:V8QI [(match_operand:V8QI 1 "s_register_operand" "0")
4015 (match_operand:V8QI 2 "s_register_operand" "w")
4016 (match_operand:V8QI 3 "s_register_operand" "w")]
4017 UNSPEC_VTBX))]
4018 "TARGET_NEON"
4019 "vtbx.8\t%P0, {%P2}, %P3"
4020 [(set_attr "neon_type" "neon_bp_2cycle")]
4021 )
4023 (define_insn "neon_vtbx2v8qi"
4024 [(set (match_operand:V8QI 0 "s_register_operand" "=w")
4025 (unspec:V8QI [(match_operand:V8QI 1 "s_register_operand" "0")
4026 (match_operand:TI 2 "s_register_operand" "w")
4027 (match_operand:V8QI 3 "s_register_operand" "w")]
4028 UNSPEC_VTBX))]
4029 "TARGET_NEON"
4030 {
4031 rtx ops[4];
4032 int tabbase = REGNO (operands[2]);
4034 ops[0] = operands[0];
4035 ops[1] = gen_rtx_REG (V8QImode, tabbase);
4036 ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
4037 ops[3] = operands[3];
4038 output_asm_insn ("vtbx.8\t%P0, {%P1, %P2}, %P3", ops);
4040 return "";
4041 }
4042 [(set_attr "neon_type" "neon_bp_2cycle")]
4043 )
4045 (define_insn "neon_vtbx3v8qi"
4046 [(set (match_operand:V8QI 0 "s_register_operand" "=w")
4047 (unspec:V8QI [(match_operand:V8QI 1 "s_register_operand" "0")
4048 (match_operand:EI 2 "s_register_operand" "w")
4049 (match_operand:V8QI 3 "s_register_operand" "w")]
4050 UNSPEC_VTBX))]
4051 "TARGET_NEON"
4052 {
4053 rtx ops[5];
4054 int tabbase = REGNO (operands[2]);
4056 ops[0] = operands[0];
4057 ops[1] = gen_rtx_REG (V8QImode, tabbase);
4058 ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
4059 ops[3] = gen_rtx_REG (V8QImode, tabbase + 4);
4060 ops[4] = operands[3];
4061 output_asm_insn ("vtbx.8\t%P0, {%P1, %P2, %P3}, %P4", ops);
4063 return "";
4064 }
4065 [(set_attr "neon_type" "neon_bp_3cycle")]
4066 )
4068 (define_insn "neon_vtbx4v8qi"
4069 [(set (match_operand:V8QI 0 "s_register_operand" "=w")
4070 (unspec:V8QI [(match_operand:V8QI 1 "s_register_operand" "0")
4071 (match_operand:OI 2 "s_register_operand" "w")
4072 (match_operand:V8QI 3 "s_register_operand" "w")]
4073 UNSPEC_VTBX))]
4074 "TARGET_NEON"
4075 {
4076 rtx ops[6];
4077 int tabbase = REGNO (operands[2]);
4079 ops[0] = operands[0];
4080 ops[1] = gen_rtx_REG (V8QImode, tabbase);
4081 ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
4082 ops[3] = gen_rtx_REG (V8QImode, tabbase + 4);
4083 ops[4] = gen_rtx_REG (V8QImode, tabbase + 6);
4084 ops[5] = operands[3];
4085 output_asm_insn ("vtbx.8\t%P0, {%P1, %P2, %P3, %P4}, %P5", ops);
4087 return "";
4088 }
4089 [(set_attr "neon_type" "neon_bp_3cycle")]
4090 )
4092 (define_insn "neon_vtrn<mode>_internal"
4093 [(set (match_operand:VDQW 0 "s_register_operand" "=w")
4094 (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "0")]
4095 UNSPEC_VTRN1))
4096 (set (match_operand:VDQW 2 "s_register_operand" "=w")
4097 (unspec:VDQW [(match_operand:VDQW 3 "s_register_operand" "2")]
4098 UNSPEC_VTRN2))]
4099 "TARGET_NEON"
4100 "vtrn.<V_sz_elem>\t%<V_reg>0, %<V_reg>2"
4101 [(set (attr "neon_type")
4102 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
4103 (const_string "neon_bp_simple")
4104 (const_string "neon_bp_3cycle")))]
4105 )
4107 (define_expand "neon_vtrn<mode>"
4108 [(match_operand:SI 0 "s_register_operand" "r")
4109 (match_operand:VDQW 1 "s_register_operand" "w")
4110 (match_operand:VDQW 2 "s_register_operand" "w")]
4111 "TARGET_NEON"
4112 {
4113 neon_emit_pair_result_insn (<MODE>mode, gen_neon_vtrn<mode>_internal,
4114 operands[0], operands[1], operands[2]);
4115 DONE;
4116 })
4118 (define_insn "neon_vzip<mode>_internal"
4119 [(set (match_operand:VDQW 0 "s_register_operand" "=w")
4120 (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "0")]
4121 UNSPEC_VZIP1))
4122 (set (match_operand:VDQW 2 "s_register_operand" "=w")
4123 (unspec:VDQW [(match_operand:VDQW 3 "s_register_operand" "2")]
4124 UNSPEC_VZIP2))]
4125 "TARGET_NEON"
4126 "vzip.<V_sz_elem>\t%<V_reg>0, %<V_reg>2"
4127 [(set (attr "neon_type")
4128 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
4129 (const_string "neon_bp_simple")
4130 (const_string "neon_bp_3cycle")))]
4131 )
4133 (define_expand "neon_vzip<mode>"
4134 [(match_operand:SI 0 "s_register_operand" "r")
4135 (match_operand:VDQW 1 "s_register_operand" "w")
4136 (match_operand:VDQW 2 "s_register_operand" "w")]
4137 "TARGET_NEON"
4138 {
4139 neon_emit_pair_result_insn (<MODE>mode, gen_neon_vzip<mode>_internal,
4140 operands[0], operands[1], operands[2]);
4141 DONE;
4142 })
4144 (define_insn "neon_vuzp<mode>_internal"
4145 [(set (match_operand:VDQW 0 "s_register_operand" "=w")
4146 (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "0")]
4147 UNSPEC_VUZP1))
4148 (set (match_operand:VDQW 2 "s_register_operand" "=w")
4149 (unspec:VDQW [(match_operand:VDQW 3 "s_register_operand" "2")]
4150 UNSPEC_VUZP2))]
4151 "TARGET_NEON"
4152 "vuzp.<V_sz_elem>\t%<V_reg>0, %<V_reg>2"
4153 [(set (attr "neon_type")
4154 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
4155 (const_string "neon_bp_simple")
4156 (const_string "neon_bp_3cycle")))]
4157 )
4159 (define_expand "neon_vuzp<mode>"
4160 [(match_operand:SI 0 "s_register_operand" "r")
4161 (match_operand:VDQW 1 "s_register_operand" "w")
4162 (match_operand:VDQW 2 "s_register_operand" "w")]
4163 "TARGET_NEON"
4164 {
4165 neon_emit_pair_result_insn (<MODE>mode, gen_neon_vuzp<mode>_internal,
4166 operands[0], operands[1], operands[2]);
4167 DONE;
4168 })
4170 (define_expand "neon_vreinterpretv8qi<mode>"
4171 [(match_operand:V8QI 0 "s_register_operand" "")
4172 (match_operand:VDX 1 "s_register_operand" "")]
4173 "TARGET_NEON"
4174 {
4175 neon_reinterpret (operands[0], operands[1]);
4176 DONE;
4177 })
4179 (define_expand "neon_vreinterpretv4hi<mode>"
4180 [(match_operand:V4HI 0 "s_register_operand" "")
4181 (match_operand:VDX 1 "s_register_operand" "")]
4182 "TARGET_NEON"
4183 {
4184 neon_reinterpret (operands[0], operands[1]);
4185 DONE;
4186 })
4188 (define_expand "neon_vreinterpretv2si<mode>"
4189 [(match_operand:V2SI 0 "s_register_operand" "")
4190 (match_operand:VDX 1 "s_register_operand" "")]
4191 "TARGET_NEON"
4192 {
4193 neon_reinterpret (operands[0], operands[1]);
4194 DONE;
4195 })
4197 (define_expand "neon_vreinterpretv2sf<mode>"
4198 [(match_operand:V2SF 0 "s_register_operand" "")
4199 (match_operand:VDX 1 "s_register_operand" "")]
4200 "TARGET_NEON"
4201 {
4202 neon_reinterpret (operands[0], operands[1]);
4203 DONE;
4204 })
4206 (define_expand "neon_vreinterpretdi<mode>"
4207 [(match_operand:DI 0 "s_register_operand" "")
4208 (match_operand:VDX 1 "s_register_operand" "")]
4209 "TARGET_NEON"
4210 {
4211 neon_reinterpret (operands[0], operands[1]);
4212 DONE;
4213 })
4215 (define_expand "neon_vreinterpretv16qi<mode>"
4216 [(match_operand:V16QI 0 "s_register_operand" "")
4217 (match_operand:VQX 1 "s_register_operand" "")]
4218 "TARGET_NEON"
4219 {
4220 neon_reinterpret (operands[0], operands[1]);
4221 DONE;
4222 })
4224 (define_expand "neon_vreinterpretv8hi<mode>"
4225 [(match_operand:V8HI 0 "s_register_operand" "")
4226 (match_operand:VQX 1 "s_register_operand" "")]
4227 "TARGET_NEON"
4228 {
4229 neon_reinterpret (operands[0], operands[1]);
4230 DONE;
4231 })
4233 (define_expand "neon_vreinterpretv4si<mode>"
4234 [(match_operand:V4SI 0 "s_register_operand" "")
4235 (match_operand:VQX 1 "s_register_operand" "")]
4236 "TARGET_NEON"
4237 {
4238 neon_reinterpret (operands[0], operands[1]);
4239 DONE;
4240 })
4242 (define_expand "neon_vreinterpretv4sf<mode>"
4243 [(match_operand:V4SF 0 "s_register_operand" "")
4244 (match_operand:VQX 1 "s_register_operand" "")]
4245 "TARGET_NEON"
4246 {
4247 neon_reinterpret (operands[0], operands[1]);
4248 DONE;
4249 })
4251 (define_expand "neon_vreinterpretv2di<mode>"
4252 [(match_operand:V2DI 0 "s_register_operand" "")
4253 (match_operand:VQX 1 "s_register_operand" "")]
4254 "TARGET_NEON"
4255 {
4256 neon_reinterpret (operands[0], operands[1]);
4257 DONE;
4258 })
4260 (define_insn "neon_vld1<mode>"
4261 [(set (match_operand:VDQX 0 "s_register_operand" "=w")
4262 (unspec:VDQX [(mem:VDQX (match_operand:SI 1 "s_register_operand" "r"))]
4263 UNSPEC_VLD1))]
4264 "TARGET_NEON"
4265 "vld1.<V_sz_elem>\t%h0, [%1]"
4266 [(set_attr "neon_type" "neon_vld1_1_2_regs")]
4267 )
4269 (define_insn "neon_vld1_lane<mode>"
4270 [(set (match_operand:VDX 0 "s_register_operand" "=w")
4271 (unspec:VDX [(mem:<V_elem> (match_operand:SI 1 "s_register_operand" "r"))
4272 (match_operand:VDX 2 "s_register_operand" "0")
4273 (match_operand:SI 3 "immediate_operand" "i")]
4274 UNSPEC_VLD1_LANE))]
4275 "TARGET_NEON"
4276 {
4277 HOST_WIDE_INT lane = INTVAL (operands[3]);
4278 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4279 if (lane < 0 || lane >= max)
4280 error ("lane out of range");
4281 if (max == 1)
4282 return "vld1.<V_sz_elem>\t%P0, [%1]";
4283 else
4284 return "vld1.<V_sz_elem>\t{%P0[%c3]}, [%1]";
4285 }
4286 [(set (attr "neon_type")
4287 (if_then_else (eq (const_string "<V_mode_nunits>") (const_int 2))
4288 (const_string "neon_vld1_1_2_regs")
4289 (const_string "neon_vld1_vld2_lane")))]
4290 )
4292 (define_insn "neon_vld1_lane<mode>"
4293 [(set (match_operand:VQX 0 "s_register_operand" "=w")
4294 (unspec:VQX [(mem:<V_elem> (match_operand:SI 1 "s_register_operand" "r"))
4295 (match_operand:VQX 2 "s_register_operand" "0")
4296 (match_operand:SI 3 "immediate_operand" "i")]
4297 UNSPEC_VLD1_LANE))]
4298 "TARGET_NEON"
4299 {
4300 HOST_WIDE_INT lane = INTVAL (operands[3]);
4301 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4302 int regno = REGNO (operands[0]);
4303 if (lane < 0 || lane >= max)
4304 error ("lane out of range");
4305 else if (lane >= max / 2)
4306 {
4307 lane -= max / 2;
4308 regno += 2;
4309 operands[3] = GEN_INT (lane);
4310 }
4311 operands[0] = gen_rtx_REG (<V_HALF>mode, regno);
4312 if (max == 2)
4313 return "vld1.<V_sz_elem>\t%P0, [%1]";
4314 else
4315 return "vld1.<V_sz_elem>\t{%P0[%c3]}, [%1]";
4316 }
4317 [(set (attr "neon_type")
4318 (if_then_else (eq (const_string "<V_mode_nunits>") (const_int 2))
4319 (const_string "neon_vld1_1_2_regs")
4320 (const_string "neon_vld1_vld2_lane")))]
4321 )
4323 (define_insn "neon_vld1_dup<mode>"
4324 [(set (match_operand:VDX 0 "s_register_operand" "=w")
4325 (unspec:VDX [(mem:<V_elem> (match_operand:SI 1 "s_register_operand" "r"))]
4326 UNSPEC_VLD1_DUP))]
4327 "TARGET_NEON"
4328 {
4329 if (GET_MODE_NUNITS (<MODE>mode) > 1)
4330 return "vld1.<V_sz_elem>\t{%P0[]}, [%1]";
4331 else
4332 return "vld1.<V_sz_elem>\t%h0, [%1]";
4333 }
4334 [(set (attr "neon_type")
4335 (if_then_else (gt (const_string "<V_mode_nunits>") (const_string "1"))
4336 (const_string "neon_vld2_2_regs_vld1_vld2_all_lanes")
4337 (const_string "neon_vld1_1_2_regs")))]
4338 )
4340 (define_insn "neon_vld1_dup<mode>"
4341 [(set (match_operand:VQX 0 "s_register_operand" "=w")
4342 (unspec:VQX [(mem:<V_elem> (match_operand:SI 1 "s_register_operand" "r"))]
4343 UNSPEC_VLD1_DUP))]
4344 "TARGET_NEON"
4345 {
4346 if (GET_MODE_NUNITS (<MODE>mode) > 2)
4347 return "vld1.<V_sz_elem>\t{%e0[], %f0[]}, [%1]";
4348 else
4349 return "vld1.<V_sz_elem>\t%h0, [%1]";
4350 }
4351 [(set (attr "neon_type")
4352 (if_then_else (gt (const_string "<V_mode_nunits>") (const_string "1"))
4353 (const_string "neon_vld2_2_regs_vld1_vld2_all_lanes")
4354 (const_string "neon_vld1_1_2_regs")))]
4355 )
4357 (define_insn "neon_vst1<mode>"
4358 [(set (mem:VDQX (match_operand:SI 0 "s_register_operand" "r"))
4359 (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand" "w")]
4360 UNSPEC_VST1))]
4361 "TARGET_NEON"
4362 "vst1.<V_sz_elem>\t%h1, [%0]"
4363 [(set_attr "neon_type" "neon_vst1_1_2_regs_vst2_2_regs")])
4365 (define_insn "neon_vst1_lane<mode>"
4366 [(set (mem:<V_elem> (match_operand:SI 0 "s_register_operand" "r"))
4367 (vec_select:<V_elem>
4368 (match_operand:VDX 1 "s_register_operand" "w")
4369 (parallel [(match_operand:SI 2 "neon_lane_number" "i")])))]
4370 "TARGET_NEON"
4371 {
4372 HOST_WIDE_INT lane = INTVAL (operands[2]);
4373 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4374 if (lane < 0 || lane >= max)
4375 error ("lane out of range");
4376 if (max == 1)
4377 return "vst1.<V_sz_elem>\t{%P1}, [%0]";
4378 else
4379 return "vst1.<V_sz_elem>\t{%P1[%c2]}, [%0]";
4380 }
4381 [(set (attr "neon_type")
4382 (if_then_else (eq (const_string "<V_mode_nunits>") (const_int 1))
4383 (const_string "neon_vst1_1_2_regs_vst2_2_regs")
4384 (const_string "neon_vst1_vst2_lane")))])
4386 (define_insn "neon_vst1_lane<mode>"
4387 [(set (mem:<V_elem> (match_operand:SI 0 "s_register_operand" "r"))
4388 (vec_select:<V_elem>
4389 (match_operand:VQX 1 "s_register_operand" "w")
4390 (parallel [(match_operand:SI 2 "neon_lane_number" "i")])))]
4391 "TARGET_NEON"
4392 {
4393 HOST_WIDE_INT lane = INTVAL (operands[2]);
4394 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4395 int regno = REGNO (operands[1]);
4396 if (lane < 0 || lane >= max)
4397 error ("lane out of range");
4398 else if (lane >= max / 2)
4399 {
4400 lane -= max / 2;
4401 regno += 2;
4402 operands[2] = GEN_INT (lane);
4403 }
4404 operands[1] = gen_rtx_REG (<V_HALF>mode, regno);
4405 if (max == 2)
4406 return "vst1.<V_sz_elem>\t{%P1}, [%0]";
4407 else
4408 return "vst1.<V_sz_elem>\t{%P1[%c2]}, [%0]";
4409 }
4410 [(set_attr "neon_type" "neon_vst1_vst2_lane")]
4411 )
4413 (define_insn "neon_vld2<mode>"
4414 [(set (match_operand:TI 0 "s_register_operand" "=w")
4415 (unspec:TI [(mem:TI (match_operand:SI 1 "s_register_operand" "r"))
4416 (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4417 UNSPEC_VLD2))]
4418 "TARGET_NEON"
4419 {
4420 if (<V_sz_elem> == 64)
4421 return "vld1.64\t%h0, [%1]";
4422 else
4423 return "vld2.<V_sz_elem>\t%h0, [%1]";
4424 }
4425 [(set (attr "neon_type")
4426 (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
4427 (const_string "neon_vld1_1_2_regs")
4428 (const_string "neon_vld2_2_regs_vld1_vld2_all_lanes")))]
4429 )
4431 (define_insn "neon_vld2<mode>"
4432 [(set (match_operand:OI 0 "s_register_operand" "=w")
4433 (unspec:OI [(mem:OI (match_operand:SI 1 "s_register_operand" "r"))
4434 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4435 UNSPEC_VLD2))]
4436 "TARGET_NEON"
4437 "vld2.<V_sz_elem>\t%h0, [%1]"
4438 [(set_attr "neon_type" "neon_vld2_2_regs_vld1_vld2_all_lanes")])
4440 (define_insn "neon_vld2_lane<mode>"
4441 [(set (match_operand:TI 0 "s_register_operand" "=w")
4442 (unspec:TI [(mem:<V_two_elem> (match_operand:SI 1 "s_register_operand" "r"))
4443 (match_operand:TI 2 "s_register_operand" "0")
4444 (match_operand:SI 3 "immediate_operand" "i")
4445 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4446 UNSPEC_VLD2_LANE))]
4447 "TARGET_NEON"
4448 {
4449 HOST_WIDE_INT lane = INTVAL (operands[3]);
4450 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4451 int regno = REGNO (operands[0]);
4452 rtx ops[4];
4453 if (lane < 0 || lane >= max)
4454 error ("lane out of range");
4455 ops[0] = gen_rtx_REG (DImode, regno);
4456 ops[1] = gen_rtx_REG (DImode, regno + 2);
4457 ops[2] = operands[1];
4458 ops[3] = operands[3];
4459 output_asm_insn ("vld2.<V_sz_elem>\t{%P0[%c3], %P1[%c3]}, [%2]", ops);
4460 return "";
4461 }
4462 [(set_attr "neon_type" "neon_vld1_vld2_lane")]
4463 )
4465 (define_insn "neon_vld2_lane<mode>"
4466 [(set (match_operand:OI 0 "s_register_operand" "=w")
4467 (unspec:OI [(mem:<V_two_elem> (match_operand:SI 1 "s_register_operand" "r"))
4468 (match_operand:OI 2 "s_register_operand" "0")
4469 (match_operand:SI 3 "immediate_operand" "i")
4470 (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4471 UNSPEC_VLD2_LANE))]
4472 "TARGET_NEON"
4473 {
4474 HOST_WIDE_INT lane = INTVAL (operands[3]);
4475 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4476 int regno = REGNO (operands[0]);
4477 rtx ops[4];
4478 if (lane < 0 || lane >= max)
4479 error ("lane out of range");
4480 else if (lane >= max / 2)
4481 {
4482 lane -= max / 2;
4483 regno += 2;
4484 }
4485 ops[0] = gen_rtx_REG (DImode, regno);
4486 ops[1] = gen_rtx_REG (DImode, regno + 4);
4487 ops[2] = operands[1];
4488 ops[3] = GEN_INT (lane);
4489 output_asm_insn ("vld2.<V_sz_elem>\t{%P0[%c3], %P1[%c3]}, [%2]", ops);
4490 return "";
4491 }
4492 [(set_attr "neon_type" "neon_vld1_vld2_lane")]
4493 )
4495 (define_insn "neon_vld2_dup<mode>"
4496 [(set (match_operand:TI 0 "s_register_operand" "=w")
4497 (unspec:TI [(mem:<V_two_elem> (match_operand:SI 1 "s_register_operand" "r"))
4498 (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4499 UNSPEC_VLD2_DUP))]
4500 "TARGET_NEON"
4501 {
4502 if (GET_MODE_NUNITS (<MODE>mode) > 1)
4503 return "vld2.<V_sz_elem>\t{%e0[], %f0[]}, [%1]";
4504 else
4505 return "vld1.<V_sz_elem>\t%h0, [%1]";
4506 }
4507 [(set (attr "neon_type")
4508 (if_then_else (gt (const_string "<V_mode_nunits>") (const_string "1"))
4509 (const_string "neon_vld2_2_regs_vld1_vld2_all_lanes")
4510 (const_string "neon_vld1_1_2_regs")))]
4511 )
4513 (define_insn "neon_vst2<mode>"
4514 [(set (mem:TI (match_operand:SI 0 "s_register_operand" "r"))
4515 (unspec:TI [(match_operand:TI 1 "s_register_operand" "w")
4516 (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4517 UNSPEC_VST2))]
4518 "TARGET_NEON"
4519 {
4520 if (<V_sz_elem> == 64)
4521 return "vst1.64\t%h1, [%0]";
4522 else
4523 return "vst2.<V_sz_elem>\t%h1, [%0]";
4524 }
4525 [(set (attr "neon_type")
4526 (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
4527 (const_string "neon_vst1_1_2_regs_vst2_2_regs")
4528 (const_string "neon_vst1_1_2_regs_vst2_2_regs")))]
4529 )
4531 (define_insn "neon_vst2<mode>"
4532 [(set (mem:OI (match_operand:SI 0 "s_register_operand" "r"))
4533 (unspec:OI [(match_operand:OI 1 "s_register_operand" "w")
4534 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4535 UNSPEC_VST2))]
4536 "TARGET_NEON"
4537 "vst2.<V_sz_elem>\t%h1, [%0]"
4538 [(set_attr "neon_type" "neon_vst1_1_2_regs_vst2_2_regs")]
4539 )
4541 (define_insn "neon_vst2_lane<mode>"
4542 [(set (mem:<V_two_elem> (match_operand:SI 0 "s_register_operand" "r"))
4543 (unspec:<V_two_elem>
4544 [(match_operand:TI 1 "s_register_operand" "w")
4545 (match_operand:SI 2 "immediate_operand" "i")
4546 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4547 UNSPEC_VST2_LANE))]
4548 "TARGET_NEON"
4549 {
4550 HOST_WIDE_INT lane = INTVAL (operands[2]);
4551 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4552 int regno = REGNO (operands[1]);
4553 rtx ops[4];
4554 if (lane < 0 || lane >= max)
4555 error ("lane out of range");
4556 ops[0] = operands[0];
4557 ops[1] = gen_rtx_REG (DImode, regno);
4558 ops[2] = gen_rtx_REG (DImode, regno + 2);
4559 ops[3] = operands[2];
4560 output_asm_insn ("vst2.<V_sz_elem>\t{%P1[%c3], %P2[%c3]}, [%0]", ops);
4561 return "";
4562 }
4563 [(set_attr "neon_type" "neon_vst1_vst2_lane")]
4564 )
4566 (define_insn "neon_vst2_lane<mode>"
4567 [(set (mem:<V_two_elem> (match_operand:SI 0 "s_register_operand" "r"))
4568 (unspec:<V_two_elem>
4569 [(match_operand:OI 1 "s_register_operand" "w")
4570 (match_operand:SI 2 "immediate_operand" "i")
4571 (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4572 UNSPEC_VST2_LANE))]
4573 "TARGET_NEON"
4574 {
4575 HOST_WIDE_INT lane = INTVAL (operands[2]);
4576 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4577 int regno = REGNO (operands[1]);
4578 rtx ops[4];
4579 if (lane < 0 || lane >= max)
4580 error ("lane out of range");
4581 else if (lane >= max / 2)
4582 {
4583 lane -= max / 2;
4584 regno += 2;
4585 }
4586 ops[0] = operands[0];
4587 ops[1] = gen_rtx_REG (DImode, regno);
4588 ops[2] = gen_rtx_REG (DImode, regno + 4);
4589 ops[3] = GEN_INT (lane);
4590 output_asm_insn ("vst2.<V_sz_elem>\t{%P1[%c3], %P2[%c3]}, [%0]", ops);
4591 return "";
4592 }
4593 [(set_attr "neon_type" "neon_vst1_vst2_lane")]
4594 )
4596 (define_insn "neon_vld3<mode>"
4597 [(set (match_operand:EI 0 "s_register_operand" "=w")
4598 (unspec:EI [(mem:EI (match_operand:SI 1 "s_register_operand" "r"))
4599 (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4600 UNSPEC_VLD3))]
4601 "TARGET_NEON"
4602 {
4603 if (<V_sz_elem> == 64)
4604 return "vld1.64\t%h0, [%1]";
4605 else
4606 return "vld3.<V_sz_elem>\t%h0, [%1]";
4607 }
4608 [(set (attr "neon_type")
4609 (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
4610 (const_string "neon_vld1_1_2_regs")
4611 (const_string "neon_vld3_vld4")))]
4612 )
4614 (define_expand "neon_vld3<mode>"
4615 [(match_operand:CI 0 "s_register_operand" "=w")
4616 (match_operand:SI 1 "s_register_operand" "+r")
4617 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4618 "TARGET_NEON"
4619 {
4620 emit_insn (gen_neon_vld3qa<mode> (operands[0], operands[0],
4621 operands[1], operands[1]));
4622 emit_insn (gen_neon_vld3qb<mode> (operands[0], operands[0],
4623 operands[1], operands[1]));
4624 DONE;
4625 })
4627 (define_insn "neon_vld3qa<mode>"
4628 [(set (match_operand:CI 0 "s_register_operand" "=w")
4629 (unspec:CI [(mem:CI (match_operand:SI 3 "s_register_operand" "2"))
4630 (match_operand:CI 1 "s_register_operand" "0")
4631 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4632 UNSPEC_VLD3A))
4633 (set (match_operand:SI 2 "s_register_operand" "=r")
4634 (plus:SI (match_dup 3)
4635 (const_int 24)))]
4636 "TARGET_NEON"
4637 {
4638 int regno = REGNO (operands[0]);
4639 rtx ops[4];
4640 ops[0] = gen_rtx_REG (DImode, regno);
4641 ops[1] = gen_rtx_REG (DImode, regno + 4);
4642 ops[2] = gen_rtx_REG (DImode, regno + 8);
4643 ops[3] = operands[2];
4644 output_asm_insn ("vld3.<V_sz_elem>\t{%P0, %P1, %P2}, [%3]!", ops);
4645 return "";
4646 }
4647 [(set_attr "neon_type" "neon_vld3_vld4")]
4648 )
4650 (define_insn "neon_vld3qb<mode>"
4651 [(set (match_operand:CI 0 "s_register_operand" "=w")
4652 (unspec:CI [(mem:CI (match_operand:SI 3 "s_register_operand" "2"))
4653 (match_operand:CI 1 "s_register_operand" "0")
4654 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4655 UNSPEC_VLD3B))
4656 (set (match_operand:SI 2 "s_register_operand" "=r")
4657 (plus:SI (match_dup 3)
4658 (const_int 24)))]
4659 "TARGET_NEON"
4660 {
4661 int regno = REGNO (operands[0]);
4662 rtx ops[4];
4663 ops[0] = gen_rtx_REG (DImode, regno + 2);
4664 ops[1] = gen_rtx_REG (DImode, regno + 6);
4665 ops[2] = gen_rtx_REG (DImode, regno + 10);
4666 ops[3] = operands[2];
4667 output_asm_insn ("vld3.<V_sz_elem>\t{%P0, %P1, %P2}, [%3]!", ops);
4668 return "";
4669 }
4670 [(set_attr "neon_type" "neon_vld3_vld4")]
4671 )
4673 (define_insn "neon_vld3_lane<mode>"
4674 [(set (match_operand:EI 0 "s_register_operand" "=w")
4675 (unspec:EI [(mem:<V_three_elem> (match_operand:SI 1 "s_register_operand" "r"))
4676 (match_operand:EI 2 "s_register_operand" "0")
4677 (match_operand:SI 3 "immediate_operand" "i")
4678 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4679 UNSPEC_VLD3_LANE))]
4680 "TARGET_NEON"
4681 {
4682 HOST_WIDE_INT lane = INTVAL (operands[3]);
4683 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4684 int regno = REGNO (operands[0]);
4685 rtx ops[5];
4686 if (lane < 0 || lane >= max)
4687 error ("lane out of range");
4688 ops[0] = gen_rtx_REG (DImode, regno);
4689 ops[1] = gen_rtx_REG (DImode, regno + 2);
4690 ops[2] = gen_rtx_REG (DImode, regno + 4);
4691 ops[3] = operands[1];
4692 ops[4] = operands[3];
4693 output_asm_insn ("vld3.<V_sz_elem>\t{%P0[%c4], %P1[%c4], %P2[%c4]}, [%3]",
4694 ops);
4695 return "";
4696 }
4697 [(set_attr "neon_type" "neon_vld3_vld4_lane")]
4698 )
4700 (define_insn "neon_vld3_lane<mode>"
4701 [(set (match_operand:CI 0 "s_register_operand" "=w")
4702 (unspec:CI [(mem:<V_three_elem> (match_operand:SI 1 "s_register_operand" "r"))
4703 (match_operand:CI 2 "s_register_operand" "0")
4704 (match_operand:SI 3 "immediate_operand" "i")
4705 (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4706 UNSPEC_VLD3_LANE))]
4707 "TARGET_NEON"
4708 {
4709 HOST_WIDE_INT lane = INTVAL (operands[3]);
4710 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4711 int regno = REGNO (operands[0]);
4712 rtx ops[5];
4713 if (lane < 0 || lane >= max)
4714 error ("lane out of range");
4715 else if (lane >= max / 2)
4716 {
4717 lane -= max / 2;
4718 regno += 2;
4719 }
4720 ops[0] = gen_rtx_REG (DImode, regno);
4721 ops[1] = gen_rtx_REG (DImode, regno + 4);
4722 ops[2] = gen_rtx_REG (DImode, regno + 8);
4723 ops[3] = operands[1];
4724 ops[4] = GEN_INT (lane);
4725 output_asm_insn ("vld3.<V_sz_elem>\t{%P0[%c4], %P1[%c4], %P2[%c4]}, [%3]",
4726 ops);
4727 return "";
4728 }
4729 [(set_attr "neon_type" "neon_vld3_vld4_lane")]
4730 )
4732 (define_insn "neon_vld3_dup<mode>"
4733 [(set (match_operand:EI 0 "s_register_operand" "=w")
4734 (unspec:EI [(mem:<V_three_elem> (match_operand:SI 1 "s_register_operand" "r"))
4735 (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4736 UNSPEC_VLD3_DUP))]
4737 "TARGET_NEON"
4738 {
4739 if (GET_MODE_NUNITS (<MODE>mode) > 1)
4740 {
4741 int regno = REGNO (operands[0]);
4742 rtx ops[4];
4743 ops[0] = gen_rtx_REG (DImode, regno);
4744 ops[1] = gen_rtx_REG (DImode, regno + 2);
4745 ops[2] = gen_rtx_REG (DImode, regno + 4);
4746 ops[3] = operands[1];
4747 output_asm_insn ("vld3.<V_sz_elem>\t{%P0[], %P1[], %P2[]}, [%3]", ops);
4748 return "";
4749 }
4750 else
4751 return "vld1.<V_sz_elem>\t%h0, [%1]";
4752 }
4753 [(set (attr "neon_type")
4754 (if_then_else (gt (const_string "<V_mode_nunits>") (const_string "1"))
4755 (const_string "neon_vld3_vld4_all_lanes")
4756 (const_string "neon_vld1_1_2_regs")))])
4758 (define_insn "neon_vst3<mode>"
4759 [(set (mem:EI (match_operand:SI 0 "s_register_operand" "r"))
4760 (unspec:EI [(match_operand:EI 1 "s_register_operand" "w")
4761 (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4762 UNSPEC_VST3))]
4763 "TARGET_NEON"
4764 {
4765 if (<V_sz_elem> == 64)
4766 return "vst1.64\t%h1, [%0]";
4767 else
4768 return "vst3.<V_sz_elem>\t%h1, [%0]";
4769 }
4770 [(set (attr "neon_type")
4771 (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
4772 (const_string "neon_vst1_1_2_regs_vst2_2_regs")
4773 (const_string "neon_vst2_4_regs_vst3_vst4")))])
4775 (define_expand "neon_vst3<mode>"
4776 [(match_operand:SI 0 "s_register_operand" "+r")
4777 (match_operand:CI 1 "s_register_operand" "w")
4778 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4779 "TARGET_NEON"
4780 {
4781 emit_insn (gen_neon_vst3qa<mode> (operands[0], operands[0], operands[1]));
4782 emit_insn (gen_neon_vst3qb<mode> (operands[0], operands[0], operands[1]));
4783 DONE;
4784 })
4786 (define_insn "neon_vst3qa<mode>"
4787 [(set (mem:EI (match_operand:SI 1 "s_register_operand" "0"))
4788 (unspec:EI [(match_operand:CI 2 "s_register_operand" "w")
4789 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4790 UNSPEC_VST3A))
4791 (set (match_operand:SI 0 "s_register_operand" "=r")
4792 (plus:SI (match_dup 1)
4793 (const_int 24)))]
4794 "TARGET_NEON"
4795 {
4796 int regno = REGNO (operands[2]);
4797 rtx ops[4];
4798 ops[0] = operands[0];
4799 ops[1] = gen_rtx_REG (DImode, regno);
4800 ops[2] = gen_rtx_REG (DImode, regno + 4);
4801 ops[3] = gen_rtx_REG (DImode, regno + 8);
4802 output_asm_insn ("vst3.<V_sz_elem>\t{%P1, %P2, %P3}, [%0]!", ops);
4803 return "";
4804 }
4805 [(set_attr "neon_type" "neon_vst2_4_regs_vst3_vst4")]
4806 )
4808 (define_insn "neon_vst3qb<mode>"
4809 [(set (mem:EI (match_operand:SI 1 "s_register_operand" "0"))
4810 (unspec:EI [(match_operand:CI 2 "s_register_operand" "w")
4811 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4812 UNSPEC_VST3B))
4813 (set (match_operand:SI 0 "s_register_operand" "=r")
4814 (plus:SI (match_dup 1)
4815 (const_int 24)))]
4816 "TARGET_NEON"
4817 {
4818 int regno = REGNO (operands[2]);
4819 rtx ops[4];
4820 ops[0] = operands[0];
4821 ops[1] = gen_rtx_REG (DImode, regno + 2);
4822 ops[2] = gen_rtx_REG (DImode, regno + 6);
4823 ops[3] = gen_rtx_REG (DImode, regno + 10);
4824 output_asm_insn ("vst3.<V_sz_elem>\t{%P1, %P2, %P3}, [%0]!", ops);
4825 return "";
4826 }
4827 [(set_attr "neon_type" "neon_vst2_4_regs_vst3_vst4")]
4828 )
4830 (define_insn "neon_vst3_lane<mode>"
4831 [(set (mem:<V_three_elem> (match_operand:SI 0 "s_register_operand" "r"))
4832 (unspec:<V_three_elem>
4833 [(match_operand:EI 1 "s_register_operand" "w")
4834 (match_operand:SI 2 "immediate_operand" "i")
4835 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4836 UNSPEC_VST3_LANE))]
4837 "TARGET_NEON"
4838 {
4839 HOST_WIDE_INT lane = INTVAL (operands[2]);
4840 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4841 int regno = REGNO (operands[1]);
4842 rtx ops[5];
4843 if (lane < 0 || lane >= max)
4844 error ("lane out of range");
4845 ops[0] = operands[0];
4846 ops[1] = gen_rtx_REG (DImode, regno);
4847 ops[2] = gen_rtx_REG (DImode, regno + 2);
4848 ops[3] = gen_rtx_REG (DImode, regno + 4);
4849 ops[4] = operands[2];
4850 output_asm_insn ("vst3.<V_sz_elem>\t{%P1[%c4], %P2[%c4], %P3[%c4]}, [%0]",
4851 ops);
4852 return "";
4853 }
4854 [(set_attr "neon_type" "neon_vst3_vst4_lane")]
4855 )
4857 (define_insn "neon_vst3_lane<mode>"
4858 [(set (mem:<V_three_elem> (match_operand:SI 0 "s_register_operand" "r"))
4859 (unspec:<V_three_elem>
4860 [(match_operand:CI 1 "s_register_operand" "w")
4861 (match_operand:SI 2 "immediate_operand" "i")
4862 (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4863 UNSPEC_VST3_LANE))]
4864 "TARGET_NEON"
4865 {
4866 HOST_WIDE_INT lane = INTVAL (operands[2]);
4867 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4868 int regno = REGNO (operands[1]);
4869 rtx ops[5];
4870 if (lane < 0 || lane >= max)
4871 error ("lane out of range");
4872 else if (lane >= max / 2)
4873 {
4874 lane -= max / 2;
4875 regno += 2;
4876 }
4877 ops[0] = operands[0];
4878 ops[1] = gen_rtx_REG (DImode, regno);
4879 ops[2] = gen_rtx_REG (DImode, regno + 4);
4880 ops[3] = gen_rtx_REG (DImode, regno + 8);
4881 ops[4] = GEN_INT (lane);
4882 output_asm_insn ("vst3.<V_sz_elem>\t{%P1[%c4], %P2[%c4], %P3[%c4]}, [%0]",
4883 ops);
4884 return "";
4885 }
4886 [(set_attr "neon_type" "neon_vst3_vst4_lane")])
4888 (define_insn "neon_vld4<mode>"
4889 [(set (match_operand:OI 0 "s_register_operand" "=w")
4890 (unspec:OI [(mem:OI (match_operand:SI 1 "s_register_operand" "r"))
4891 (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4892 UNSPEC_VLD4))]
4893 "TARGET_NEON"
4894 {
4895 if (<V_sz_elem> == 64)
4896 return "vld1.64\t%h0, [%1]";
4897 else
4898 return "vld4.<V_sz_elem>\t%h0, [%1]";
4899 }
4900 [(set (attr "neon_type")
4901 (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
4902 (const_string "neon_vld1_1_2_regs")
4903 (const_string "neon_vld3_vld4")))]
4904 )
4906 (define_expand "neon_vld4<mode>"
4907 [(match_operand:XI 0 "s_register_operand" "=w")
4908 (match_operand:SI 1 "s_register_operand" "+r")
4909 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4910 "TARGET_NEON"
4911 {
4912 emit_insn (gen_neon_vld4qa<mode> (operands[0], operands[0],
4913 operands[1], operands[1]));
4914 emit_insn (gen_neon_vld4qb<mode> (operands[0], operands[0],
4915 operands[1], operands[1]));
4916 DONE;
4917 })
4919 (define_insn "neon_vld4qa<mode>"
4920 [(set (match_operand:XI 0 "s_register_operand" "=w")
4921 (unspec:XI [(mem:XI (match_operand:SI 3 "s_register_operand" "2"))
4922 (match_operand:XI 1 "s_register_operand" "0")
4923 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4924 UNSPEC_VLD4A))
4925 (set (match_operand:SI 2 "s_register_operand" "=r")
4926 (plus:SI (match_dup 3)
4927 (const_int 32)))]
4928 "TARGET_NEON"
4929 {
4930 int regno = REGNO (operands[0]);
4931 rtx ops[5];
4932 ops[0] = gen_rtx_REG (DImode, regno);
4933 ops[1] = gen_rtx_REG (DImode, regno + 4);
4934 ops[2] = gen_rtx_REG (DImode, regno + 8);
4935 ops[3] = gen_rtx_REG (DImode, regno + 12);
4936 ops[4] = operands[2];
4937 output_asm_insn ("vld4.<V_sz_elem>\t{%P0, %P1, %P2, %P3}, [%4]!", ops);
4938 return "";
4939 }
4940 [(set_attr "neon_type" "neon_vld3_vld4")]
4941 )
4943 (define_insn "neon_vld4qb<mode>"
4944 [(set (match_operand:XI 0 "s_register_operand" "=w")
4945 (unspec:XI [(mem:XI (match_operand:SI 3 "s_register_operand" "2"))
4946 (match_operand:XI 1 "s_register_operand" "0")
4947 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4948 UNSPEC_VLD4B))
4949 (set (match_operand:SI 2 "s_register_operand" "=r")
4950 (plus:SI (match_dup 3)
4951 (const_int 32)))]
4952 "TARGET_NEON"
4953 {
4954 int regno = REGNO (operands[0]);
4955 rtx ops[5];
4956 ops[0] = gen_rtx_REG (DImode, regno + 2);
4957 ops[1] = gen_rtx_REG (DImode, regno + 6);
4958 ops[2] = gen_rtx_REG (DImode, regno + 10);
4959 ops[3] = gen_rtx_REG (DImode, regno + 14);
4960 ops[4] = operands[2];
4961 output_asm_insn ("vld4.<V_sz_elem>\t{%P0, %P1, %P2, %P3}, [%4]!", ops);
4962 return "";
4963 }
4964 [(set_attr "neon_type" "neon_vld3_vld4")]
4965 )
4967 (define_insn "neon_vld4_lane<mode>"
4968 [(set (match_operand:OI 0 "s_register_operand" "=w")
4969 (unspec:OI [(mem:<V_four_elem> (match_operand:SI 1 "s_register_operand" "r"))
4970 (match_operand:OI 2 "s_register_operand" "0")
4971 (match_operand:SI 3 "immediate_operand" "i")
4972 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4973 UNSPEC_VLD4_LANE))]
4974 "TARGET_NEON"
4975 {
4976 HOST_WIDE_INT lane = INTVAL (operands[3]);
4977 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4978 int regno = REGNO (operands[0]);
4979 rtx ops[6];
4980 if (lane < 0 || lane >= max)
4981 error ("lane out of range");
4982 ops[0] = gen_rtx_REG (DImode, regno);
4983 ops[1] = gen_rtx_REG (DImode, regno + 2);
4984 ops[2] = gen_rtx_REG (DImode, regno + 4);
4985 ops[3] = gen_rtx_REG (DImode, regno + 6);
4986 ops[4] = operands[1];
4987 ops[5] = operands[3];
4988 output_asm_insn ("vld4.<V_sz_elem>\t{%P0[%c5], %P1[%c5], %P2[%c5], %P3[%c5]}, [%4]",
4989 ops);
4990 return "";
4991 }
4992 [(set_attr "neon_type" "neon_vld3_vld4_lane")]
4993 )
4995 (define_insn "neon_vld4_lane<mode>"
4996 [(set (match_operand:XI 0 "s_register_operand" "=w")
4997 (unspec:XI [(mem:<V_four_elem> (match_operand:SI 1 "s_register_operand" "r"))
4998 (match_operand:XI 2 "s_register_operand" "0")
4999 (match_operand:SI 3 "immediate_operand" "i")
5000 (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5001 UNSPEC_VLD4_LANE))]
5002 "TARGET_NEON"
5003 {
5004 HOST_WIDE_INT lane = INTVAL (operands[3]);
5005 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
5006 int regno = REGNO (operands[0]);
5007 rtx ops[6];
5008 if (lane < 0 || lane >= max)
5009 error ("lane out of range");
5010 else if (lane >= max / 2)
5011 {
5012 lane -= max / 2;
5013 regno += 2;
5014 }
5015 ops[0] = gen_rtx_REG (DImode, regno);
5016 ops[1] = gen_rtx_REG (DImode, regno + 4);
5017 ops[2] = gen_rtx_REG (DImode, regno + 8);
5018 ops[3] = gen_rtx_REG (DImode, regno + 12);
5019 ops[4] = operands[1];
5020 ops[5] = GEN_INT (lane);
5021 output_asm_insn ("vld4.<V_sz_elem>\t{%P0[%c5], %P1[%c5], %P2[%c5], %P3[%c5]}, [%4]",
5022 ops);
5023 return "";
5024 }
5025 [(set_attr "neon_type" "neon_vld3_vld4_lane")]
5026 )
5028 (define_insn "neon_vld4_dup<mode>"
5029 [(set (match_operand:OI 0 "s_register_operand" "=w")
5030 (unspec:OI [(mem:<V_four_elem> (match_operand:SI 1 "s_register_operand" "r"))
5031 (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5032 UNSPEC_VLD4_DUP))]
5033 "TARGET_NEON"
5034 {
5035 if (GET_MODE_NUNITS (<MODE>mode) > 1)
5036 {
5037 int regno = REGNO (operands[0]);
5038 rtx ops[5];
5039 ops[0] = gen_rtx_REG (DImode, regno);
5040 ops[1] = gen_rtx_REG (DImode, regno + 2);
5041 ops[2] = gen_rtx_REG (DImode, regno + 4);
5042 ops[3] = gen_rtx_REG (DImode, regno + 6);
5043 ops[4] = operands[1];
5044 output_asm_insn ("vld4.<V_sz_elem>\t{%P0[], %P1[], %P2[], %P3[]}, [%4]",
5045 ops);
5046 return "";
5047 }
5048 else
5049 return "vld1.<V_sz_elem>\t%h0, [%1]";
5050 }
5051 [(set (attr "neon_type")
5052 (if_then_else (gt (const_string "<V_mode_nunits>") (const_string "1"))
5053 (const_string "neon_vld3_vld4_all_lanes")
5054 (const_string "neon_vld1_1_2_regs")))]
5055 )
5057 (define_insn "neon_vst4<mode>"
5058 [(set (mem:OI (match_operand:SI 0 "s_register_operand" "r"))
5059 (unspec:OI [(match_operand:OI 1 "s_register_operand" "w")
5060 (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5061 UNSPEC_VST4))]
5062 "TARGET_NEON"
5063 {
5064 if (<V_sz_elem> == 64)
5065 return "vst1.64\t%h1, [%0]";
5066 else
5067 return "vst4.<V_sz_elem>\t%h1, [%0]";
5068 }
5069 [(set (attr "neon_type")
5070 (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
5071 (const_string "neon_vst1_1_2_regs_vst2_2_regs")
5072 (const_string "neon_vst2_4_regs_vst3_vst4")))]
5073 )
5075 (define_expand "neon_vst4<mode>"
5076 [(match_operand:SI 0 "s_register_operand" "+r")
5077 (match_operand:XI 1 "s_register_operand" "w")
5078 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5079 "TARGET_NEON"
5080 {
5081 emit_insn (gen_neon_vst4qa<mode> (operands[0], operands[0], operands[1]));
5082 emit_insn (gen_neon_vst4qb<mode> (operands[0], operands[0], operands[1]));
5083 DONE;
5084 })
5086 (define_insn "neon_vst4qa<mode>"
5087 [(set (mem:OI (match_operand:SI 1 "s_register_operand" "0"))
5088 (unspec:OI [(match_operand:XI 2 "s_register_operand" "w")
5089 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5090 UNSPEC_VST4A))
5091 (set (match_operand:SI 0 "s_register_operand" "=r")
5092 (plus:SI (match_dup 1)
5093 (const_int 32)))]
5094 "TARGET_NEON"
5095 {
5096 int regno = REGNO (operands[2]);
5097 rtx ops[5];
5098 ops[0] = operands[0];
5099 ops[1] = gen_rtx_REG (DImode, regno);
5100 ops[2] = gen_rtx_REG (DImode, regno + 4);
5101 ops[3] = gen_rtx_REG (DImode, regno + 8);
5102 ops[4] = gen_rtx_REG (DImode, regno + 12);
5103 output_asm_insn ("vst4.<V_sz_elem>\t{%P1, %P2, %P3, %P4}, [%0]!", ops);
5104 return "";
5105 }
5106 [(set_attr "neon_type" "neon_vst2_4_regs_vst3_vst4")]
5107 )
5109 (define_insn "neon_vst4qb<mode>"
5110 [(set (mem:OI (match_operand:SI 1 "s_register_operand" "0"))
5111 (unspec:OI [(match_operand:XI 2 "s_register_operand" "w")
5112 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5113 UNSPEC_VST4B))
5114 (set (match_operand:SI 0 "s_register_operand" "=r")
5115 (plus:SI (match_dup 1)
5116 (const_int 32)))]
5117 "TARGET_NEON"
5118 {
5119 int regno = REGNO (operands[2]);
5120 rtx ops[5];
5121 ops[0] = operands[0];
5122 ops[1] = gen_rtx_REG (DImode, regno + 2);
5123 ops[2] = gen_rtx_REG (DImode, regno + 6);
5124 ops[3] = gen_rtx_REG (DImode, regno + 10);
5125 ops[4] = gen_rtx_REG (DImode, regno + 14);
5126 output_asm_insn ("vst4.<V_sz_elem>\t{%P1, %P2, %P3, %P4}, [%0]!", ops);
5127 return "";
5128 }
5129 [(set_attr "neon_type" "neon_vst2_4_regs_vst3_vst4")]
5130 )
5132 (define_insn "neon_vst4_lane<mode>"
5133 [(set (mem:<V_four_elem> (match_operand:SI 0 "s_register_operand" "r"))
5134 (unspec:<V_four_elem>
5135 [(match_operand:OI 1 "s_register_operand" "w")
5136 (match_operand:SI 2 "immediate_operand" "i")
5137 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5138 UNSPEC_VST4_LANE))]
5139 "TARGET_NEON"
5140 {
5141 HOST_WIDE_INT lane = INTVAL (operands[2]);
5142 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
5143 int regno = REGNO (operands[1]);
5144 rtx ops[6];
5145 if (lane < 0 || lane >= max)
5146 error ("lane out of range");
5147 ops[0] = operands[0];
5148 ops[1] = gen_rtx_REG (DImode, regno);
5149 ops[2] = gen_rtx_REG (DImode, regno + 2);
5150 ops[3] = gen_rtx_REG (DImode, regno + 4);
5151 ops[4] = gen_rtx_REG (DImode, regno + 6);
5152 ops[5] = operands[2];
5153 output_asm_insn ("vst4.<V_sz_elem>\t{%P1[%c5], %P2[%c5], %P3[%c5], %P4[%c5]}, [%0]",
5154 ops);
5155 return "";
5156 }
5157 [(set_attr "neon_type" "neon_vst3_vst4_lane")]
5158 )
5160 (define_insn "neon_vst4_lane<mode>"
5161 [(set (mem:<V_four_elem> (match_operand:SI 0 "s_register_operand" "r"))
5162 (unspec:<V_four_elem>
5163 [(match_operand:XI 1 "s_register_operand" "w")
5164 (match_operand:SI 2 "immediate_operand" "i")
5165 (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5166 UNSPEC_VST4_LANE))]
5167 "TARGET_NEON"
5168 {
5169 HOST_WIDE_INT lane = INTVAL (operands[2]);
5170 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
5171 int regno = REGNO (operands[1]);
5172 rtx ops[6];
5173 if (lane < 0 || lane >= max)
5174 error ("lane out of range");
5175 else if (lane >= max / 2)
5176 {
5177 lane -= max / 2;
5178 regno += 2;
5179 }
5180 ops[0] = operands[0];
5181 ops[1] = gen_rtx_REG (DImode, regno);
5182 ops[2] = gen_rtx_REG (DImode, regno + 4);
5183 ops[3] = gen_rtx_REG (DImode, regno + 8);
5184 ops[4] = gen_rtx_REG (DImode, regno + 12);
5185 ops[5] = GEN_INT (lane);
5186 output_asm_insn ("vst4.<V_sz_elem>\t{%P1[%c5], %P2[%c5], %P3[%c5], %P4[%c5]}, [%0]",
5187 ops);
5188 return "";
5189 }
5190 [(set_attr "neon_type" "neon_vst3_vst4_lane")]
5191 )
5193 (define_expand "neon_vand<mode>"
5194 [(match_operand:VDQX 0 "s_register_operand" "")
5195 (match_operand:VDQX 1 "s_register_operand" "")
5196 (match_operand:VDQX 2 "neon_inv_logic_op2" "")
5197 (match_operand:SI 3 "immediate_operand" "")]
5198 "TARGET_NEON"
5199 {
5200 emit_insn (gen_and<mode>3<V_suf64> (operands[0], operands[1], operands[2]));
5201 DONE;
5202 })
5204 (define_expand "neon_vorr<mode>"
5205 [(match_operand:VDQX 0 "s_register_operand" "")
5206 (match_operand:VDQX 1 "s_register_operand" "")
5207 (match_operand:VDQX 2 "neon_logic_op2" "")
5208 (match_operand:SI 3 "immediate_operand" "")]
5209 "TARGET_NEON"
5210 {
5211 emit_insn (gen_ior<mode>3<V_suf64> (operands[0], operands[1], operands[2]));
5212 DONE;
5213 })
5215 (define_expand "neon_veor<mode>"
5216 [(match_operand:VDQX 0 "s_register_operand" "")
5217 (match_operand:VDQX 1 "s_register_operand" "")
5218 (match_operand:VDQX 2 "s_register_operand" "")
5219 (match_operand:SI 3 "immediate_operand" "")]
5220 "TARGET_NEON"
5221 {
5222 emit_insn (gen_xor<mode>3<V_suf64> (operands[0], operands[1], operands[2]));
5223 DONE;
5224 })
5226 (define_expand "neon_vbic<mode>"
5227 [(match_operand:VDQX 0 "s_register_operand" "")
5228 (match_operand:VDQX 1 "s_register_operand" "")
5229 (match_operand:VDQX 2 "neon_logic_op2" "")
5230 (match_operand:SI 3 "immediate_operand" "")]
5231 "TARGET_NEON"
5232 {
5233 emit_insn (gen_bic<mode>3_neon (operands[0], operands[1], operands[2]));
5234 DONE;
5235 })
5237 (define_expand "neon_vorn<mode>"
5238 [(match_operand:VDQX 0 "s_register_operand" "")
5239 (match_operand:VDQX 1 "s_register_operand" "")
5240 (match_operand:VDQX 2 "neon_inv_logic_op2" "")
5241 (match_operand:SI 3 "immediate_operand" "")]
5242 "TARGET_NEON"
5243 {
5244 emit_insn (gen_orn<mode>3_neon (operands[0], operands[1], operands[2]));
5245 DONE;
5246 })
5248 (define_insn "neon_vec_unpack<US>_lo_<mode>"
5249 [(set (match_operand:<V_unpack> 0 "register_operand" "=w")
5250 (SE:<V_unpack> (vec_select:<V_HALF>
5251 (match_operand:VU 1 "register_operand" "w")
5252 (match_operand:VU 2 "vect_par_constant_low" ""))))]
5253 "TARGET_NEON"
5254 "vmovl.<US><V_sz_elem> %q0, %e1"
5255 [(set_attr "neon_type" "neon_shift_1")]
5256 )
5258 (define_insn "neon_vec_unpack<US>_hi_<mode>"
5259 [(set (match_operand:<V_unpack> 0 "register_operand" "=w")
5260 (SE:<V_unpack> (vec_select:<V_HALF>
5261 (match_operand:VU 1 "register_operand" "w")
5262 (match_operand:VU 2 "vect_par_constant_high" ""))))]
5263 "TARGET_NEON"
5264 "vmovl.<US><V_sz_elem> %q0, %f1"
5265 [(set_attr "neon_type" "neon_shift_1")]
5266 )
5268 (define_expand "vec_unpack<US>_hi_<mode>"
5269 [(match_operand:<V_unpack> 0 "register_operand" "")
5270 (SE:<V_unpack> (match_operand:VU 1 "register_operand"))]
5271 "TARGET_NEON"
5272 {
5273 rtvec v = rtvec_alloc (<V_mode_nunits>/2) ;
5274 rtx t1;
5275 int i;
5276 for (i = 0; i < (<V_mode_nunits>/2); i++)
5277 RTVEC_ELT (v, i) = GEN_INT ((<V_mode_nunits>/2) + i);
5279 t1 = gen_rtx_PARALLEL (<MODE>mode, v);
5280 emit_insn (gen_neon_vec_unpack<US>_hi_<mode> (operands[0],
5281 operands[1],
5282 t1));
5283 DONE;
5284 }
5285 )
5287 (define_expand "vec_unpack<US>_lo_<mode>"
5288 [(match_operand:<V_unpack> 0 "register_operand" "")
5289 (SE:<V_unpack> (match_operand:VU 1 "register_operand" ""))]
5290 "TARGET_NEON"
5291 {
5292 rtvec v = rtvec_alloc (<V_mode_nunits>/2) ;
5293 rtx t1;
5294 int i;
5295 for (i = 0; i < (<V_mode_nunits>/2) ; i++)
5296 RTVEC_ELT (v, i) = GEN_INT (i);
5297 t1 = gen_rtx_PARALLEL (<MODE>mode, v);
5298 emit_insn (gen_neon_vec_unpack<US>_lo_<mode> (operands[0],
5299 operands[1],
5300 t1));
5301 DONE;
5302 }
5303 )
5305 (define_insn "neon_vec_<US>mult_lo_<mode>"
5306 [(set (match_operand:<V_unpack> 0 "register_operand" "=w")
5307 (mult:<V_unpack> (SE:<V_unpack> (vec_select:<V_HALF>
5308 (match_operand:VU 1 "register_operand" "w")
5309 (match_operand:VU 2 "vect_par_constant_low" "")))
5310 (SE:<V_unpack> (vec_select:<V_HALF>
5311 (match_operand:VU 3 "register_operand" "w")
5312 (match_dup 2)))))]
5313 "TARGET_NEON"
5314 "vmull.<US><V_sz_elem> %q0, %e1, %e3"
5315 [(set_attr "neon_type" "neon_shift_1")]
5316 )
5318 (define_expand "vec_widen_<US>mult_lo_<mode>"
5319 [(match_operand:<V_unpack> 0 "register_operand" "")
5320 (SE:<V_unpack> (match_operand:VU 1 "register_operand" ""))
5321 (SE:<V_unpack> (match_operand:VU 2 "register_operand" ""))]
5322 "TARGET_NEON"
5323 {
5324 rtvec v = rtvec_alloc (<V_mode_nunits>/2) ;
5325 rtx t1;
5326 int i;
5327 for (i = 0; i < (<V_mode_nunits>/2) ; i++)
5328 RTVEC_ELT (v, i) = GEN_INT (i);
5329 t1 = gen_rtx_PARALLEL (<MODE>mode, v);
5331 emit_insn (gen_neon_vec_<US>mult_lo_<mode> (operands[0],
5332 operands[1],
5333 t1,
5334 operands[2]));
5335 DONE;
5336 }
5337 )
5339 (define_insn "neon_vec_<US>mult_hi_<mode>"
5340 [(set (match_operand:<V_unpack> 0 "register_operand" "=w")
5341 (mult:<V_unpack> (SE:<V_unpack> (vec_select:<V_HALF>
5342 (match_operand:VU 1 "register_operand" "w")
5343 (match_operand:VU 2 "vect_par_constant_high" "")))
5344 (SE:<V_unpack> (vec_select:<V_HALF>
5345 (match_operand:VU 3 "register_operand" "w")
5346 (match_dup 2)))))]
5347 "TARGET_NEON"
5348 "vmull.<US><V_sz_elem> %q0, %f1, %f3"
5349 [(set_attr "neon_type" "neon_shift_1")]
5350 )
5352 (define_expand "vec_widen_<US>mult_hi_<mode>"
5353 [(match_operand:<V_unpack> 0 "register_operand" "")
5354 (SE:<V_unpack> (match_operand:VU 1 "register_operand" ""))
5355 (SE:<V_unpack> (match_operand:VU 2 "register_operand" ""))]
5356 "TARGET_NEON"
5357 {
5358 rtvec v = rtvec_alloc (<V_mode_nunits>/2) ;
5359 rtx t1;
5360 int i;
5361 for (i = 0; i < (<V_mode_nunits>/2) ; i++)
5362 RTVEC_ELT (v, i) = GEN_INT (<V_mode_nunits>/2 + i);
5363 t1 = gen_rtx_PARALLEL (<MODE>mode, v);
5365 emit_insn (gen_neon_vec_<US>mult_hi_<mode> (operands[0],
5366 operands[1],
5367 t1,
5368 operands[2]));
5369 DONE;
5371 }
5372 )
5374 ;; Vectorize for non-neon-quad case
5375 (define_insn "neon_unpack<US>_<mode>"
5376 [(set (match_operand:<V_widen> 0 "register_operand" "=w")
5377 (SE:<V_widen> (match_operand:VDI 1 "register_operand" "w")))]
5378 "TARGET_NEON"
5379 "vmovl.<US><V_sz_elem> %q0, %P1"
5380 [(set_attr "neon_type" "neon_shift_1")]
5381 )
5383 (define_expand "vec_unpack<US>_lo_<mode>"
5384 [(match_operand:<V_double_width> 0 "register_operand" "")
5385 (SE:<V_double_width>(match_operand:VDI 1 "register_operand"))]
5386 "TARGET_NEON"
5387 {
5388 rtx tmpreg = gen_reg_rtx (<V_widen>mode);
5389 emit_insn (gen_neon_unpack<US>_<mode> (tmpreg, operands[1]));
5390 emit_insn (gen_neon_vget_low<V_widen_l> (operands[0], tmpreg));
5392 DONE;
5393 }
5394 )
5396 (define_expand "vec_unpack<US>_hi_<mode>"
5397 [(match_operand:<V_double_width> 0 "register_operand" "")
5398 (SE:<V_double_width>(match_operand:VDI 1 "register_operand"))]
5399 "TARGET_NEON"
5400 {
5401 rtx tmpreg = gen_reg_rtx (<V_widen>mode);
5402 emit_insn (gen_neon_unpack<US>_<mode> (tmpreg, operands[1]));
5403 emit_insn (gen_neon_vget_high<V_widen_l> (operands[0], tmpreg));
5405 DONE;
5406 }
5407 )
5409 (define_insn "neon_vec_<US>mult_<mode>"
5410 [(set (match_operand:<V_widen> 0 "register_operand" "=w")
5411 (mult:<V_widen> (SE:<V_widen>
5412 (match_operand:VDI 1 "register_operand" "w"))
5413 (SE:<V_widen>
5414 (match_operand:VDI 2 "register_operand" "w"))))]
5415 "TARGET_NEON"
5416 "vmull.<US><V_sz_elem> %q0, %P1, %P2"
5417 [(set_attr "neon_type" "neon_shift_1")]
5418 )
5420 (define_expand "vec_widen_<US>mult_hi_<mode>"
5421 [(match_operand:<V_double_width> 0 "register_operand" "")
5422 (SE:<V_double_width> (match_operand:VDI 1 "register_operand" ""))
5423 (SE:<V_double_width> (match_operand:VDI 2 "register_operand" ""))]
5424 "TARGET_NEON"
5425 {
5426 rtx tmpreg = gen_reg_rtx (<V_widen>mode);
5427 emit_insn (gen_neon_vec_<US>mult_<mode> (tmpreg, operands[1], operands[2]));
5428 emit_insn (gen_neon_vget_high<V_widen_l> (operands[0], tmpreg));
5430 DONE;
5432 }
5433 )
5435 (define_expand "vec_widen_<US>mult_lo_<mode>"
5436 [(match_operand:<V_double_width> 0 "register_operand" "")
5437 (SE:<V_double_width> (match_operand:VDI 1 "register_operand" ""))
5438 (SE:<V_double_width> (match_operand:VDI 2 "register_operand" ""))]
5439 "TARGET_NEON"
5440 {
5441 rtx tmpreg = gen_reg_rtx (<V_widen>mode);
5442 emit_insn (gen_neon_vec_<US>mult_<mode> (tmpreg, operands[1], operands[2]));
5443 emit_insn (gen_neon_vget_low<V_widen_l> (operands[0], tmpreg));
5445 DONE;
5447 }
5448 )
5450 (define_insn "vec_pack_trunc_<mode>"
5451 [(set (match_operand:<V_narrow_pack> 0 "register_operand" "=&w")
5452 (vec_concat:<V_narrow_pack>
5453 (truncate:<V_narrow>
5454 (match_operand:VN 1 "register_operand" "w"))
5455 (truncate:<V_narrow>
5456 (match_operand:VN 2 "register_operand" "w"))))]
5457 "TARGET_NEON"
5458 "vmovn.i<V_sz_elem>\t%e0, %q1\n\tvmovn.i<V_sz_elem>\t%f0, %q2"
5459 [(set_attr "neon_type" "neon_shift_1")]
5460 )
5462 ;; For the non-quad case.
5463 (define_insn "neon_vec_pack_trunc_<mode>"
5464 [(set (match_operand:<V_narrow> 0 "register_operand" "=w")
5465 (truncate:<V_narrow> (match_operand:VN 1 "register_operand" "w")))]
5466 "TARGET_NEON"
5467 "vmovn.i<V_sz_elem>\t%P0, %q1"
5468 [(set_attr "neon_type" "neon_shift_1")]
5469 )
5471 (define_expand "vec_pack_trunc_<mode>"
5472 [(match_operand:<V_narrow_pack> 0 "register_operand" "")
5473 (match_operand:VSHFT 1 "register_operand" "")
5474 (match_operand:VSHFT 2 "register_operand")]
5475 "TARGET_NEON"
5476 {
5477 rtx tempreg = gen_reg_rtx (<V_DOUBLE>mode);
5479 emit_insn (gen_move_lo_quad_<V_double> (tempreg, operands[1]));
5480 emit_insn (gen_move_hi_quad_<V_double> (tempreg, operands[2]));
5481 emit_insn (gen_neon_vec_pack_trunc_<V_double> (operands[0], tempreg));
5482 DONE;
5483 })