| blob | ef196e4b6fb39c0d2fd9ebfee76abab8369b1e92 |
1 ;; Machine description for AArch64 AdvSIMD architecture.
2 ;; Copyright (C) 2011-2014 Free Software Foundation, Inc.
3 ;; Contributed by ARM Ltd.
4 ;;
5 ;; This file is part of GCC.
6 ;;
7 ;; GCC is free software; you can redistribute it and/or modify it
8 ;; under the terms of the GNU General Public License as published by
9 ;; the Free Software Foundation; either version 3, or (at your option)
10 ;; any later version.
11 ;;
12 ;; GCC is distributed in the hope that it will be useful, but
13 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
14 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 ;; General Public License for more details.
16 ;;
17 ;; You should have received a copy of the GNU General Public License
18 ;; along with GCC; see the file COPYING3. If not see
19 ;; <http://www.gnu.org/licenses/>.
21 (define_expand "mov<mode>"
22 [(set (match_operand:VALL 0 "nonimmediate_operand" "")
23 (match_operand:VALL 1 "general_operand" ""))]
24 "TARGET_SIMD"
25 "
26 if (GET_CODE (operands[0]) == MEM)
27 operands[1] = force_reg (<MODE>mode, operands[1]);
28 "
29 )
31 (define_expand "movmisalign<mode>"
32 [(set (match_operand:VALL 0 "nonimmediate_operand" "")
33 (match_operand:VALL 1 "general_operand" ""))]
34 "TARGET_SIMD"
35 {
36 /* This pattern is not permitted to fail during expansion: if both arguments
37 are non-registers (e.g. memory := constant, which can be created by the
38 auto-vectorizer), force operand 1 into a register. */
39 if (!register_operand (operands[0], <MODE>mode)
40 && !register_operand (operands[1], <MODE>mode))
41 operands[1] = force_reg (<MODE>mode, operands[1]);
42 })
44 (define_insn "aarch64_simd_dup<mode>"
45 [(set (match_operand:VDQ 0 "register_operand" "=w, w")
46 (vec_duplicate:VDQ (match_operand:<VEL> 1 "register_operand" "r, w")))]
47 "TARGET_SIMD"
48 "@
49 dup\\t%0.<Vtype>, %<vw>1
50 dup\\t%0.<Vtype>, %1.<Vetype>[0]"
51 [(set_attr "type" "neon_from_gp<q>, neon_dup<q>")]
52 )
54 (define_insn "aarch64_simd_dup<mode>"
55 [(set (match_operand:VDQF 0 "register_operand" "=w")
56 (vec_duplicate:VDQF (match_operand:<VEL> 1 "register_operand" "w")))]
57 "TARGET_SIMD"
58 "dup\\t%0.<Vtype>, %1.<Vetype>[0]"
59 [(set_attr "type" "neon_dup<q>")]
60 )
62 (define_insn "aarch64_dup_lane<mode>"
63 [(set (match_operand:VALL 0 "register_operand" "=w")
64 (vec_duplicate:VALL
65 (vec_select:<VEL>
66 (match_operand:VALL 1 "register_operand" "w")
67 (parallel [(match_operand:SI 2 "immediate_operand" "i")])
68 )))]
69 "TARGET_SIMD"
70 {
71 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
72 return "dup\\t%0.<Vtype>, %1.<Vetype>[%2]";
73 }
74 [(set_attr "type" "neon_dup<q>")]
75 )
77 (define_insn "aarch64_dup_lane_<vswap_width_name><mode>"
78 [(set (match_operand:VALL 0 "register_operand" "=w")
79 (vec_duplicate:VALL
80 (vec_select:<VEL>
81 (match_operand:<VSWAP_WIDTH> 1 "register_operand" "w")
82 (parallel [(match_operand:SI 2 "immediate_operand" "i")])
83 )))]
84 "TARGET_SIMD"
85 {
86 operands[2] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
87 INTVAL (operands[2])));
88 return "dup\\t%0.<Vtype>, %1.<Vetype>[%2]";
89 }
90 [(set_attr "type" "neon_dup<q>")]
91 )
93 (define_insn "*aarch64_simd_mov<mode>"
94 [(set (match_operand:VD 0 "nonimmediate_operand"
95 "=w, m, w, ?r, ?w, ?r, w")
96 (match_operand:VD 1 "general_operand"
97 "m, w, w, w, r, r, Dn"))]
98 "TARGET_SIMD
99 && (register_operand (operands[0], <MODE>mode)
100 || register_operand (operands[1], <MODE>mode))"
101 {
102 switch (which_alternative)
103 {
104 case 0: return "ldr\\t%d0, %1";
105 case 1: return "str\\t%d1, %0";
106 case 2: return "orr\t%0.<Vbtype>, %1.<Vbtype>, %1.<Vbtype>";
107 case 3: return "umov\t%0, %1.d[0]";
108 case 4: return "ins\t%0.d[0], %1";
109 case 5: return "mov\t%0, %1";
110 case 6:
111 return aarch64_output_simd_mov_immediate (operands[1],
112 <MODE>mode, 64);
113 default: gcc_unreachable ();
114 }
115 }
116 [(set_attr "type" "neon_load1_1reg<q>, neon_store1_1reg<q>,\
117 neon_logic<q>, neon_to_gp<q>, neon_from_gp<q>,\
118 mov_reg, neon_move<q>")]
119 )
121 (define_insn "*aarch64_simd_mov<mode>"
122 [(set (match_operand:VQ 0 "nonimmediate_operand"
123 "=w, m, w, ?r, ?w, ?r, w")
124 (match_operand:VQ 1 "general_operand"
125 "m, w, w, w, r, r, Dn"))]
126 "TARGET_SIMD
127 && (register_operand (operands[0], <MODE>mode)
128 || register_operand (operands[1], <MODE>mode))"
129 {
130 switch (which_alternative)
131 {
132 case 0:
133 return "ldr\\t%q0, %1";
134 case 1:
135 return "str\\t%q1, %0";
136 case 2:
137 return "orr\t%0.<Vbtype>, %1.<Vbtype>, %1.<Vbtype>";
138 case 3:
139 case 4:
140 case 5:
141 return "#";
142 case 6:
143 return aarch64_output_simd_mov_immediate (operands[1], <MODE>mode, 128);
144 default:
145 gcc_unreachable ();
146 }
147 }
148 [(set_attr "type" "neon_load1_1reg<q>, neon_store1_1reg<q>,\
149 neon_logic<q>, multiple, multiple, multiple,\
150 neon_move<q>")
151 (set_attr "length" "4,4,4,8,8,8,4")]
152 )
154 (define_split
155 [(set (match_operand:VQ 0 "register_operand" "")
156 (match_operand:VQ 1 "register_operand" ""))]
157 "TARGET_SIMD && reload_completed
158 && GP_REGNUM_P (REGNO (operands[0]))
159 && GP_REGNUM_P (REGNO (operands[1]))"
160 [(set (match_dup 0) (match_dup 1))
161 (set (match_dup 2) (match_dup 3))]
162 {
163 int rdest = REGNO (operands[0]);
164 int rsrc = REGNO (operands[1]);
165 rtx dest[2], src[2];
167 dest[0] = gen_rtx_REG (DImode, rdest);
168 src[0] = gen_rtx_REG (DImode, rsrc);
169 dest[1] = gen_rtx_REG (DImode, rdest + 1);
170 src[1] = gen_rtx_REG (DImode, rsrc + 1);
172 aarch64_simd_disambiguate_copy (operands, dest, src, 2);
173 })
175 (define_split
176 [(set (match_operand:VQ 0 "register_operand" "")
177 (match_operand:VQ 1 "register_operand" ""))]
178 "TARGET_SIMD && reload_completed
179 && ((FP_REGNUM_P (REGNO (operands[0])) && GP_REGNUM_P (REGNO (operands[1])))
180 || (GP_REGNUM_P (REGNO (operands[0])) && FP_REGNUM_P (REGNO (operands[1]))))"
181 [(const_int 0)]
182 {
183 aarch64_split_simd_move (operands[0], operands[1]);
184 DONE;
185 })
187 (define_expand "aarch64_split_simd_mov<mode>"
188 [(set (match_operand:VQ 0)
189 (match_operand:VQ 1))]
190 "TARGET_SIMD"
191 {
192 rtx dst = operands[0];
193 rtx src = operands[1];
195 if (GP_REGNUM_P (REGNO (src)))
196 {
197 rtx src_low_part = gen_lowpart (<VHALF>mode, src);
198 rtx src_high_part = gen_highpart (<VHALF>mode, src);
200 emit_insn
201 (gen_move_lo_quad_<mode> (dst, src_low_part));
202 emit_insn
203 (gen_move_hi_quad_<mode> (dst, src_high_part));
204 }
206 else
207 {
208 rtx dst_low_part = gen_lowpart (<VHALF>mode, dst);
209 rtx dst_high_part = gen_highpart (<VHALF>mode, dst);
210 rtx lo = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
211 rtx hi = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
213 emit_insn
214 (gen_aarch64_simd_mov_from_<mode>low (dst_low_part, src, lo));
215 emit_insn
216 (gen_aarch64_simd_mov_from_<mode>high (dst_high_part, src, hi));
217 }
218 DONE;
219 }
220 )
222 (define_insn "aarch64_simd_mov_from_<mode>low"
223 [(set (match_operand:<VHALF> 0 "register_operand" "=r")
224 (vec_select:<VHALF>
225 (match_operand:VQ 1 "register_operand" "w")
226 (match_operand:VQ 2 "vect_par_cnst_lo_half" "")))]
227 "TARGET_SIMD && reload_completed"
228 "umov\t%0, %1.d[0]"
229 [(set_attr "type" "neon_to_gp<q>")
230 (set_attr "length" "4")
231 ])
233 (define_insn "aarch64_simd_mov_from_<mode>high"
234 [(set (match_operand:<VHALF> 0 "register_operand" "=r")
235 (vec_select:<VHALF>
236 (match_operand:VQ 1 "register_operand" "w")
237 (match_operand:VQ 2 "vect_par_cnst_hi_half" "")))]
238 "TARGET_SIMD && reload_completed"
239 "umov\t%0, %1.d[1]"
240 [(set_attr "type" "neon_to_gp<q>")
241 (set_attr "length" "4")
242 ])
244 (define_insn "orn<mode>3"
245 [(set (match_operand:VDQ 0 "register_operand" "=w")
246 (ior:VDQ (not:VDQ (match_operand:VDQ 1 "register_operand" "w"))
247 (match_operand:VDQ 2 "register_operand" "w")))]
248 "TARGET_SIMD"
249 "orn\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>"
250 [(set_attr "type" "neon_logic<q>")]
251 )
253 (define_insn "bic<mode>3"
254 [(set (match_operand:VDQ 0 "register_operand" "=w")
255 (and:VDQ (not:VDQ (match_operand:VDQ 1 "register_operand" "w"))
256 (match_operand:VDQ 2 "register_operand" "w")))]
257 "TARGET_SIMD"
258 "bic\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>"
259 [(set_attr "type" "neon_logic<q>")]
260 )
262 (define_insn "add<mode>3"
263 [(set (match_operand:VDQ 0 "register_operand" "=w")
264 (plus:VDQ (match_operand:VDQ 1 "register_operand" "w")
265 (match_operand:VDQ 2 "register_operand" "w")))]
266 "TARGET_SIMD"
267 "add\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
268 [(set_attr "type" "neon_add<q>")]
269 )
271 (define_insn "sub<mode>3"
272 [(set (match_operand:VDQ 0 "register_operand" "=w")
273 (minus:VDQ (match_operand:VDQ 1 "register_operand" "w")
274 (match_operand:VDQ 2 "register_operand" "w")))]
275 "TARGET_SIMD"
276 "sub\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
277 [(set_attr "type" "neon_sub<q>")]
278 )
280 (define_insn "mul<mode>3"
281 [(set (match_operand:VDQM 0 "register_operand" "=w")
282 (mult:VDQM (match_operand:VDQM 1 "register_operand" "w")
283 (match_operand:VDQM 2 "register_operand" "w")))]
284 "TARGET_SIMD"
285 "mul\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
286 [(set_attr "type" "neon_mul_<Vetype><q>")]
287 )
289 (define_insn "bswap<mode>"
290 [(set (match_operand:VDQHSD 0 "register_operand" "=w")
291 (bswap:VDQHSD (match_operand:VDQHSD 1 "register_operand" "w")))]
292 "TARGET_SIMD"
293 "rev<Vrevsuff>\\t%0.<Vbtype>, %1.<Vbtype>"
294 [(set_attr "type" "neon_rev<q>")]
295 )
297 (define_insn "aarch64_rbit<mode>"
298 [(set (match_operand:VB 0 "register_operand" "=w")
299 (unspec:VB [(match_operand:VB 1 "register_operand" "w")]
300 UNSPEC_RBIT))]
301 "TARGET_SIMD"
302 "rbit\\t%0.<Vbtype>, %1.<Vbtype>"
303 [(set_attr "type" "neon_rbit")]
304 )
306 (define_insn "*aarch64_mul3_elt<mode>"
307 [(set (match_operand:VMUL 0 "register_operand" "=w")
308 (mult:VMUL
309 (vec_duplicate:VMUL
310 (vec_select:<VEL>
311 (match_operand:VMUL 1 "register_operand" "<h_con>")
312 (parallel [(match_operand:SI 2 "immediate_operand")])))
313 (match_operand:VMUL 3 "register_operand" "w")))]
314 "TARGET_SIMD"
315 {
316 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
317 return "<f>mul\\t%0.<Vtype>, %3.<Vtype>, %1.<Vetype>[%2]";
318 }
319 [(set_attr "type" "neon<fp>_mul_<Vetype>_scalar<q>")]
320 )
322 (define_insn "*aarch64_mul3_elt_<vswap_width_name><mode>"
323 [(set (match_operand:VMUL_CHANGE_NLANES 0 "register_operand" "=w")
324 (mult:VMUL_CHANGE_NLANES
325 (vec_duplicate:VMUL_CHANGE_NLANES
326 (vec_select:<VEL>
327 (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
328 (parallel [(match_operand:SI 2 "immediate_operand")])))
329 (match_operand:VMUL_CHANGE_NLANES 3 "register_operand" "w")))]
330 "TARGET_SIMD"
331 {
332 operands[2] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
333 INTVAL (operands[2])));
334 return "<f>mul\\t%0.<Vtype>, %3.<Vtype>, %1.<Vetype>[%2]";
335 }
336 [(set_attr "type" "neon<fp>_mul_<Vetype>_scalar<q>")]
337 )
339 (define_insn "*aarch64_mul3_elt_to_128df"
340 [(set (match_operand:V2DF 0 "register_operand" "=w")
341 (mult:V2DF
342 (vec_duplicate:V2DF
343 (match_operand:DF 2 "register_operand" "w"))
344 (match_operand:V2DF 1 "register_operand" "w")))]
345 "TARGET_SIMD"
346 "fmul\\t%0.2d, %1.2d, %2.d[0]"
347 [(set_attr "type" "neon_fp_mul_d_scalar_q")]
348 )
350 (define_insn "*aarch64_mul3_elt_to_64v2df"
351 [(set (match_operand:DF 0 "register_operand" "=w")
352 (mult:DF
353 (vec_select:DF
354 (match_operand:V2DF 1 "register_operand" "w")
355 (parallel [(match_operand:SI 2 "immediate_operand")]))
356 (match_operand:DF 3 "register_operand" "w")))]
357 "TARGET_SIMD"
358 {
359 operands[2] = GEN_INT (ENDIAN_LANE_N (V2DFmode, INTVAL (operands[2])));
360 return "fmul\\t%0.2d, %3.2d, %1.d[%2]";
361 }
362 [(set_attr "type" "neon_fp_mul_d_scalar_q")]
363 )
365 (define_insn "neg<mode>2"
366 [(set (match_operand:VDQ 0 "register_operand" "=w")
367 (neg:VDQ (match_operand:VDQ 1 "register_operand" "w")))]
368 "TARGET_SIMD"
369 "neg\t%0.<Vtype>, %1.<Vtype>"
370 [(set_attr "type" "neon_neg<q>")]
371 )
373 (define_insn "abs<mode>2"
374 [(set (match_operand:VDQ 0 "register_operand" "=w")
375 (abs:VDQ (match_operand:VDQ 1 "register_operand" "w")))]
376 "TARGET_SIMD"
377 "abs\t%0.<Vtype>, %1.<Vtype>"
378 [(set_attr "type" "neon_abs<q>")]
379 )
381 (define_insn "abd<mode>_3"
382 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
383 (abs:VDQ_BHSI (minus:VDQ_BHSI
384 (match_operand:VDQ_BHSI 1 "register_operand" "w")
385 (match_operand:VDQ_BHSI 2 "register_operand" "w"))))]
386 "TARGET_SIMD"
387 "sabd\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
388 [(set_attr "type" "neon_abd<q>")]
389 )
391 (define_insn "aba<mode>_3"
392 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
393 (plus:VDQ_BHSI (abs:VDQ_BHSI (minus:VDQ_BHSI
394 (match_operand:VDQ_BHSI 1 "register_operand" "w")
395 (match_operand:VDQ_BHSI 2 "register_operand" "w")))
396 (match_operand:VDQ_BHSI 3 "register_operand" "0")))]
397 "TARGET_SIMD"
398 "saba\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
399 [(set_attr "type" "neon_arith_acc<q>")]
400 )
402 (define_insn "fabd<mode>_3"
403 [(set (match_operand:VDQF 0 "register_operand" "=w")
404 (abs:VDQF (minus:VDQF
405 (match_operand:VDQF 1 "register_operand" "w")
406 (match_operand:VDQF 2 "register_operand" "w"))))]
407 "TARGET_SIMD"
408 "fabd\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
409 [(set_attr "type" "neon_fp_abd_<Vetype><q>")]
410 )
412 (define_insn "*fabd_scalar<mode>3"
413 [(set (match_operand:GPF 0 "register_operand" "=w")
414 (abs:GPF (minus:GPF
415 (match_operand:GPF 1 "register_operand" "w")
416 (match_operand:GPF 2 "register_operand" "w"))))]
417 "TARGET_SIMD"
418 "fabd\t%<s>0, %<s>1, %<s>2"
419 [(set_attr "type" "neon_fp_abd_<Vetype><q>")]
420 )
422 (define_insn "and<mode>3"
423 [(set (match_operand:VDQ 0 "register_operand" "=w")
424 (and:VDQ (match_operand:VDQ 1 "register_operand" "w")
425 (match_operand:VDQ 2 "register_operand" "w")))]
426 "TARGET_SIMD"
427 "and\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>"
428 [(set_attr "type" "neon_logic<q>")]
429 )
431 (define_insn "ior<mode>3"
432 [(set (match_operand:VDQ 0 "register_operand" "=w")
433 (ior:VDQ (match_operand:VDQ 1 "register_operand" "w")
434 (match_operand:VDQ 2 "register_operand" "w")))]
435 "TARGET_SIMD"
436 "orr\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>"
437 [(set_attr "type" "neon_logic<q>")]
438 )
440 (define_insn "xor<mode>3"
441 [(set (match_operand:VDQ 0 "register_operand" "=w")
442 (xor:VDQ (match_operand:VDQ 1 "register_operand" "w")
443 (match_operand:VDQ 2 "register_operand" "w")))]
444 "TARGET_SIMD"
445 "eor\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>"
446 [(set_attr "type" "neon_logic<q>")]
447 )
449 (define_insn "one_cmpl<mode>2"
450 [(set (match_operand:VDQ 0 "register_operand" "=w")
451 (not:VDQ (match_operand:VDQ 1 "register_operand" "w")))]
452 "TARGET_SIMD"
453 "not\t%0.<Vbtype>, %1.<Vbtype>"
454 [(set_attr "type" "neon_logic<q>")]
455 )
457 (define_insn "aarch64_simd_vec_set<mode>"
458 [(set (match_operand:VQ_S 0 "register_operand" "=w,w")
459 (vec_merge:VQ_S
460 (vec_duplicate:VQ_S
461 (match_operand:<VEL> 1 "register_operand" "r,w"))
462 (match_operand:VQ_S 3 "register_operand" "0,0")
463 (match_operand:SI 2 "immediate_operand" "i,i")))]
464 "TARGET_SIMD"
465 {
466 int elt = ENDIAN_LANE_N (<MODE>mode, exact_log2 (INTVAL (operands[2])));
467 operands[2] = GEN_INT ((HOST_WIDE_INT) 1 << elt);
468 switch (which_alternative)
469 {
470 case 0:
471 return "ins\\t%0.<Vetype>[%p2], %w1";
472 case 1:
473 return "ins\\t%0.<Vetype>[%p2], %1.<Vetype>[0]";
474 default:
475 gcc_unreachable ();
476 }
477 }
478 [(set_attr "type" "neon_from_gp<q>, neon_ins<q>")]
479 )
481 (define_insn "aarch64_simd_lshr<mode>"
482 [(set (match_operand:VDQ 0 "register_operand" "=w")
483 (lshiftrt:VDQ (match_operand:VDQ 1 "register_operand" "w")
484 (match_operand:VDQ 2 "aarch64_simd_rshift_imm" "Dr")))]
485 "TARGET_SIMD"
486 "ushr\t%0.<Vtype>, %1.<Vtype>, %2"
487 [(set_attr "type" "neon_shift_imm<q>")]
488 )
490 (define_insn "aarch64_simd_ashr<mode>"
491 [(set (match_operand:VDQ 0 "register_operand" "=w")
492 (ashiftrt:VDQ (match_operand:VDQ 1 "register_operand" "w")
493 (match_operand:VDQ 2 "aarch64_simd_rshift_imm" "Dr")))]
494 "TARGET_SIMD"
495 "sshr\t%0.<Vtype>, %1.<Vtype>, %2"
496 [(set_attr "type" "neon_shift_imm<q>")]
497 )
499 (define_insn "aarch64_simd_imm_shl<mode>"
500 [(set (match_operand:VDQ 0 "register_operand" "=w")
501 (ashift:VDQ (match_operand:VDQ 1 "register_operand" "w")
502 (match_operand:VDQ 2 "aarch64_simd_lshift_imm" "Dl")))]
503 "TARGET_SIMD"
504 "shl\t%0.<Vtype>, %1.<Vtype>, %2"
505 [(set_attr "type" "neon_shift_imm<q>")]
506 )
508 (define_insn "aarch64_simd_reg_sshl<mode>"
509 [(set (match_operand:VDQ 0 "register_operand" "=w")
510 (ashift:VDQ (match_operand:VDQ 1 "register_operand" "w")
511 (match_operand:VDQ 2 "register_operand" "w")))]
512 "TARGET_SIMD"
513 "sshl\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
514 [(set_attr "type" "neon_shift_reg<q>")]
515 )
517 (define_insn "aarch64_simd_reg_shl<mode>_unsigned"
518 [(set (match_operand:VDQ 0 "register_operand" "=w")
519 (unspec:VDQ [(match_operand:VDQ 1 "register_operand" "w")
520 (match_operand:VDQ 2 "register_operand" "w")]
521 UNSPEC_ASHIFT_UNSIGNED))]
522 "TARGET_SIMD"
523 "ushl\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
524 [(set_attr "type" "neon_shift_reg<q>")]
525 )
527 (define_insn "aarch64_simd_reg_shl<mode>_signed"
528 [(set (match_operand:VDQ 0 "register_operand" "=w")
529 (unspec:VDQ [(match_operand:VDQ 1 "register_operand" "w")
530 (match_operand:VDQ 2 "register_operand" "w")]
531 UNSPEC_ASHIFT_SIGNED))]
532 "TARGET_SIMD"
533 "sshl\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
534 [(set_attr "type" "neon_shift_reg<q>")]
535 )
537 (define_expand "ashl<mode>3"
538 [(match_operand:VDQ 0 "register_operand" "")
539 (match_operand:VDQ 1 "register_operand" "")
540 (match_operand:SI 2 "general_operand" "")]
541 "TARGET_SIMD"
542 {
543 int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
544 int shift_amount;
546 if (CONST_INT_P (operands[2]))
547 {
548 shift_amount = INTVAL (operands[2]);
549 if (shift_amount >= 0 && shift_amount < bit_width)
550 {
551 rtx tmp = aarch64_simd_gen_const_vector_dup (<MODE>mode,
552 shift_amount);
553 emit_insn (gen_aarch64_simd_imm_shl<mode> (operands[0],
554 operands[1],
555 tmp));
556 DONE;
557 }
558 else
559 {
560 operands[2] = force_reg (SImode, operands[2]);
561 }
562 }
563 else if (MEM_P (operands[2]))
564 {
565 operands[2] = force_reg (SImode, operands[2]);
566 }
568 if (REG_P (operands[2]))
569 {
570 rtx tmp = gen_reg_rtx (<MODE>mode);
571 emit_insn (gen_aarch64_simd_dup<mode> (tmp,
572 convert_to_mode (<VEL>mode,
573 operands[2],
574 0)));
575 emit_insn (gen_aarch64_simd_reg_sshl<mode> (operands[0], operands[1],
576 tmp));
577 DONE;
578 }
579 else
580 FAIL;
581 }
582 )
584 (define_expand "lshr<mode>3"
585 [(match_operand:VDQ 0 "register_operand" "")
586 (match_operand:VDQ 1 "register_operand" "")
587 (match_operand:SI 2 "general_operand" "")]
588 "TARGET_SIMD"
589 {
590 int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
591 int shift_amount;
593 if (CONST_INT_P (operands[2]))
594 {
595 shift_amount = INTVAL (operands[2]);
596 if (shift_amount > 0 && shift_amount <= bit_width)
597 {
598 rtx tmp = aarch64_simd_gen_const_vector_dup (<MODE>mode,
599 shift_amount);
600 emit_insn (gen_aarch64_simd_lshr<mode> (operands[0],
601 operands[1],
602 tmp));
603 DONE;
604 }
605 else
606 operands[2] = force_reg (SImode, operands[2]);
607 }
608 else if (MEM_P (operands[2]))
609 {
610 operands[2] = force_reg (SImode, operands[2]);
611 }
613 if (REG_P (operands[2]))
614 {
615 rtx tmp = gen_reg_rtx (SImode);
616 rtx tmp1 = gen_reg_rtx (<MODE>mode);
617 emit_insn (gen_negsi2 (tmp, operands[2]));
618 emit_insn (gen_aarch64_simd_dup<mode> (tmp1,
619 convert_to_mode (<VEL>mode,
620 tmp, 0)));
621 emit_insn (gen_aarch64_simd_reg_shl<mode>_unsigned (operands[0],
622 operands[1],
623 tmp1));
624 DONE;
625 }
626 else
627 FAIL;
628 }
629 )
631 (define_expand "ashr<mode>3"
632 [(match_operand:VDQ 0 "register_operand" "")
633 (match_operand:VDQ 1 "register_operand" "")
634 (match_operand:SI 2 "general_operand" "")]
635 "TARGET_SIMD"
636 {
637 int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
638 int shift_amount;
640 if (CONST_INT_P (operands[2]))
641 {
642 shift_amount = INTVAL (operands[2]);
643 if (shift_amount > 0 && shift_amount <= bit_width)
644 {
645 rtx tmp = aarch64_simd_gen_const_vector_dup (<MODE>mode,
646 shift_amount);
647 emit_insn (gen_aarch64_simd_ashr<mode> (operands[0],
648 operands[1],
649 tmp));
650 DONE;
651 }
652 else
653 operands[2] = force_reg (SImode, operands[2]);
654 }
655 else if (MEM_P (operands[2]))
656 {
657 operands[2] = force_reg (SImode, operands[2]);
658 }
660 if (REG_P (operands[2]))
661 {
662 rtx tmp = gen_reg_rtx (SImode);
663 rtx tmp1 = gen_reg_rtx (<MODE>mode);
664 emit_insn (gen_negsi2 (tmp, operands[2]));
665 emit_insn (gen_aarch64_simd_dup<mode> (tmp1,
666 convert_to_mode (<VEL>mode,
667 tmp, 0)));
668 emit_insn (gen_aarch64_simd_reg_shl<mode>_signed (operands[0],
669 operands[1],
670 tmp1));
671 DONE;
672 }
673 else
674 FAIL;
675 }
676 )
678 (define_expand "vashl<mode>3"
679 [(match_operand:VDQ 0 "register_operand" "")
680 (match_operand:VDQ 1 "register_operand" "")
681 (match_operand:VDQ 2 "register_operand" "")]
682 "TARGET_SIMD"
683 {
684 emit_insn (gen_aarch64_simd_reg_sshl<mode> (operands[0], operands[1],
685 operands[2]));
686 DONE;
687 })
689 ;; Using mode VQ_S as there is no V2DImode neg!
690 ;; Negating individual lanes most certainly offsets the
691 ;; gain from vectorization.
692 (define_expand "vashr<mode>3"
693 [(match_operand:VQ_S 0 "register_operand" "")
694 (match_operand:VQ_S 1 "register_operand" "")
695 (match_operand:VQ_S 2 "register_operand" "")]
696 "TARGET_SIMD"
697 {
698 rtx neg = gen_reg_rtx (<MODE>mode);
699 emit (gen_neg<mode>2 (neg, operands[2]));
700 emit_insn (gen_aarch64_simd_reg_shl<mode>_signed (operands[0], operands[1],
701 neg));
702 DONE;
703 })
705 ;; DI vector shift
706 (define_expand "aarch64_ashr_simddi"
707 [(match_operand:DI 0 "register_operand" "=w")
708 (match_operand:DI 1 "register_operand" "w")
709 (match_operand:SI 2 "aarch64_shift_imm64_di" "")]
710 "TARGET_SIMD"
711 {
712 if (INTVAL (operands[2]) == 64)
713 emit_insn (gen_aarch64_sshr_simddi (operands[0], operands[1]));
714 else
715 emit_insn (gen_ashrdi3 (operands[0], operands[1], operands[2]));
716 DONE;
717 }
718 )
720 ;; SIMD shift by 64. This pattern is a special case as standard pattern does
721 ;; not handle NEON shifts by 64.
722 (define_insn "aarch64_sshr_simddi"
723 [(set (match_operand:DI 0 "register_operand" "=w")
724 (unspec:DI
725 [(match_operand:DI 1 "register_operand" "w")] UNSPEC_SSHR64))]
726 "TARGET_SIMD"
727 "sshr\t%d0, %d1, 64"
728 [(set_attr "type" "neon_shift_imm")]
729 )
731 (define_expand "vlshr<mode>3"
732 [(match_operand:VQ_S 0 "register_operand" "")
733 (match_operand:VQ_S 1 "register_operand" "")
734 (match_operand:VQ_S 2 "register_operand" "")]
735 "TARGET_SIMD"
736 {
737 rtx neg = gen_reg_rtx (<MODE>mode);
738 emit (gen_neg<mode>2 (neg, operands[2]));
739 emit_insn (gen_aarch64_simd_reg_shl<mode>_unsigned (operands[0], operands[1],
740 neg));
741 DONE;
742 })
744 (define_expand "aarch64_lshr_simddi"
745 [(match_operand:DI 0 "register_operand" "=w")
746 (match_operand:DI 1 "register_operand" "w")
747 (match_operand:SI 2 "aarch64_shift_imm64_di" "")]
748 "TARGET_SIMD"
749 {
750 if (INTVAL (operands[2]) == 64)
751 emit_insn (gen_aarch64_ushr_simddi (operands[0], operands[1]));
752 else
753 emit_insn (gen_lshrdi3 (operands[0], operands[1], operands[2]));
754 DONE;
755 }
756 )
758 ;; SIMD shift by 64. This pattern is a special case as standard pattern does
759 ;; not handle NEON shifts by 64.
760 (define_insn "aarch64_ushr_simddi"
761 [(set (match_operand:DI 0 "register_operand" "=w")
762 (unspec:DI
763 [(match_operand:DI 1 "register_operand" "w")] UNSPEC_USHR64))]
764 "TARGET_SIMD"
765 "ushr\t%d0, %d1, 64"
766 [(set_attr "type" "neon_shift_imm")]
767 )
769 (define_expand "vec_set<mode>"
770 [(match_operand:VQ_S 0 "register_operand")
771 (match_operand:<VEL> 1 "register_operand")
772 (match_operand:SI 2 "immediate_operand")]
773 "TARGET_SIMD"
774 {
775 HOST_WIDE_INT elem = (HOST_WIDE_INT) 1 << INTVAL (operands[2]);
776 emit_insn (gen_aarch64_simd_vec_set<mode> (operands[0], operands[1],
777 GEN_INT (elem), operands[0]));
778 DONE;
779 }
780 )
782 (define_insn "aarch64_simd_vec_setv2di"
783 [(set (match_operand:V2DI 0 "register_operand" "=w,w")
784 (vec_merge:V2DI
785 (vec_duplicate:V2DI
786 (match_operand:DI 1 "register_operand" "r,w"))
787 (match_operand:V2DI 3 "register_operand" "0,0")
788 (match_operand:SI 2 "immediate_operand" "i,i")))]
789 "TARGET_SIMD"
790 {
791 int elt = ENDIAN_LANE_N (V2DImode, exact_log2 (INTVAL (operands[2])));
792 operands[2] = GEN_INT ((HOST_WIDE_INT) 1 << elt);
793 switch (which_alternative)
794 {
795 case 0:
796 return "ins\\t%0.d[%p2], %1";
797 case 1:
798 return "ins\\t%0.d[%p2], %1.d[0]";
799 default:
800 gcc_unreachable ();
801 }
802 }
803 [(set_attr "type" "neon_from_gp, neon_ins_q")]
804 )
806 (define_expand "vec_setv2di"
807 [(match_operand:V2DI 0 "register_operand")
808 (match_operand:DI 1 "register_operand")
809 (match_operand:SI 2 "immediate_operand")]
810 "TARGET_SIMD"
811 {
812 HOST_WIDE_INT elem = (HOST_WIDE_INT) 1 << INTVAL (operands[2]);
813 emit_insn (gen_aarch64_simd_vec_setv2di (operands[0], operands[1],
814 GEN_INT (elem), operands[0]));
815 DONE;
816 }
817 )
819 (define_insn "aarch64_simd_vec_set<mode>"
820 [(set (match_operand:VDQF 0 "register_operand" "=w")
821 (vec_merge:VDQF
822 (vec_duplicate:VDQF
823 (match_operand:<VEL> 1 "register_operand" "w"))
824 (match_operand:VDQF 3 "register_operand" "0")
825 (match_operand:SI 2 "immediate_operand" "i")))]
826 "TARGET_SIMD"
827 {
828 int elt = ENDIAN_LANE_N (<MODE>mode, exact_log2 (INTVAL (operands[2])));
830 operands[2] = GEN_INT ((HOST_WIDE_INT)1 << elt);
831 return "ins\t%0.<Vetype>[%p2], %1.<Vetype>[0]";
832 }
833 [(set_attr "type" "neon_ins<q>")]
834 )
836 (define_expand "vec_set<mode>"
837 [(match_operand:VDQF 0 "register_operand" "+w")
838 (match_operand:<VEL> 1 "register_operand" "w")
839 (match_operand:SI 2 "immediate_operand" "")]
840 "TARGET_SIMD"
841 {
842 HOST_WIDE_INT elem = (HOST_WIDE_INT) 1 << INTVAL (operands[2]);
843 emit_insn (gen_aarch64_simd_vec_set<mode> (operands[0], operands[1],
844 GEN_INT (elem), operands[0]));
845 DONE;
846 }
847 )
850 (define_insn "aarch64_mla<mode>"
851 [(set (match_operand:VQ_S 0 "register_operand" "=w")
852 (plus:VQ_S (mult:VQ_S (match_operand:VQ_S 2 "register_operand" "w")
853 (match_operand:VQ_S 3 "register_operand" "w"))
854 (match_operand:VQ_S 1 "register_operand" "0")))]
855 "TARGET_SIMD"
856 "mla\t%0.<Vtype>, %2.<Vtype>, %3.<Vtype>"
857 [(set_attr "type" "neon_mla_<Vetype><q>")]
858 )
860 (define_insn "*aarch64_mla_elt<mode>"
861 [(set (match_operand:VDQHS 0 "register_operand" "=w")
862 (plus:VDQHS
863 (mult:VDQHS
864 (vec_duplicate:VDQHS
865 (vec_select:<VEL>
866 (match_operand:VDQHS 1 "register_operand" "<h_con>")
867 (parallel [(match_operand:SI 2 "immediate_operand")])))
868 (match_operand:VDQHS 3 "register_operand" "w"))
869 (match_operand:VDQHS 4 "register_operand" "0")))]
870 "TARGET_SIMD"
871 {
872 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
873 return "mla\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
874 }
875 [(set_attr "type" "neon_mla_<Vetype>_scalar<q>")]
876 )
878 (define_insn "*aarch64_mla_elt_<vswap_width_name><mode>"
879 [(set (match_operand:VDQHS 0 "register_operand" "=w")
880 (plus:VDQHS
881 (mult:VDQHS
882 (vec_duplicate:VDQHS
883 (vec_select:<VEL>
884 (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
885 (parallel [(match_operand:SI 2 "immediate_operand")])))
886 (match_operand:VDQHS 3 "register_operand" "w"))
887 (match_operand:VDQHS 4 "register_operand" "0")))]
888 "TARGET_SIMD"
889 {
890 operands[2] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
891 INTVAL (operands[2])));
892 return "mla\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
893 }
894 [(set_attr "type" "neon_mla_<Vetype>_scalar<q>")]
895 )
897 (define_insn "aarch64_mls<mode>"
898 [(set (match_operand:VQ_S 0 "register_operand" "=w")
899 (minus:VQ_S (match_operand:VQ_S 1 "register_operand" "0")
900 (mult:VQ_S (match_operand:VQ_S 2 "register_operand" "w")
901 (match_operand:VQ_S 3 "register_operand" "w"))))]
902 "TARGET_SIMD"
903 "mls\t%0.<Vtype>, %2.<Vtype>, %3.<Vtype>"
904 [(set_attr "type" "neon_mla_<Vetype><q>")]
905 )
907 (define_insn "*aarch64_mls_elt<mode>"
908 [(set (match_operand:VDQHS 0 "register_operand" "=w")
909 (minus:VDQHS
910 (match_operand:VDQHS 4 "register_operand" "0")
911 (mult:VDQHS
912 (vec_duplicate:VDQHS
913 (vec_select:<VEL>
914 (match_operand:VDQHS 1 "register_operand" "<h_con>")
915 (parallel [(match_operand:SI 2 "immediate_operand")])))
916 (match_operand:VDQHS 3 "register_operand" "w"))))]
917 "TARGET_SIMD"
918 {
919 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
920 return "mls\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
921 }
922 [(set_attr "type" "neon_mla_<Vetype>_scalar<q>")]
923 )
925 (define_insn "*aarch64_mls_elt_<vswap_width_name><mode>"
926 [(set (match_operand:VDQHS 0 "register_operand" "=w")
927 (minus:VDQHS
928 (match_operand:VDQHS 4 "register_operand" "0")
929 (mult:VDQHS
930 (vec_duplicate:VDQHS
931 (vec_select:<VEL>
932 (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
933 (parallel [(match_operand:SI 2 "immediate_operand")])))
934 (match_operand:VDQHS 3 "register_operand" "w"))))]
935 "TARGET_SIMD"
936 {
937 operands[2] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
938 INTVAL (operands[2])));
939 return "mls\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
940 }
941 [(set_attr "type" "neon_mla_<Vetype>_scalar<q>")]
942 )
944 ;; Max/Min operations.
945 (define_insn "<su><maxmin><mode>3"
946 [(set (match_operand:VQ_S 0 "register_operand" "=w")
947 (MAXMIN:VQ_S (match_operand:VQ_S 1 "register_operand" "w")
948 (match_operand:VQ_S 2 "register_operand" "w")))]
949 "TARGET_SIMD"
950 "<su><maxmin>\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
951 [(set_attr "type" "neon_minmax<q>")]
952 )
954 ;; vec_concat gives a new vector with the low elements from operand 1, and
955 ;; the high elements from operand 2. That is to say, given op1 = { a, b }
956 ;; op2 = { c, d }, vec_concat (op1, op2) = { a, b, c, d }.
957 ;; What that means, is that the RTL descriptions of the below patterns
958 ;; need to change depending on endianness.
960 ;; Move to the low architectural bits of the register.
961 ;; On little-endian this is { operand, zeroes }
962 ;; On big-endian this is { zeroes, operand }
964 (define_insn "move_lo_quad_internal_<mode>"
965 [(set (match_operand:VQ_NO2E 0 "register_operand" "=w,w,w")
966 (vec_concat:VQ_NO2E
967 (match_operand:<VHALF> 1 "register_operand" "w,r,r")
968 (vec_duplicate:<VHALF> (const_int 0))))]
969 "TARGET_SIMD && !BYTES_BIG_ENDIAN"
970 "@
971 dup\\t%d0, %1.d[0]
972 fmov\\t%d0, %1
973 dup\\t%d0, %1"
974 [(set_attr "type" "neon_dup<q>,f_mcr,neon_dup<q>")
975 (set_attr "simd" "yes,*,yes")
976 (set_attr "fp" "*,yes,*")
977 (set_attr "length" "4")]
978 )
980 (define_insn "move_lo_quad_internal_<mode>"
981 [(set (match_operand:VQ_2E 0 "register_operand" "=w,w,w")
982 (vec_concat:VQ_2E
983 (match_operand:<VHALF> 1 "register_operand" "w,r,r")
984 (const_int 0)))]
985 "TARGET_SIMD && !BYTES_BIG_ENDIAN"
986 "@
987 dup\\t%d0, %1.d[0]
988 fmov\\t%d0, %1
989 dup\\t%d0, %1"
990 [(set_attr "type" "neon_dup<q>,f_mcr,neon_dup<q>")
991 (set_attr "simd" "yes,*,yes")
992 (set_attr "fp" "*,yes,*")
993 (set_attr "length" "4")]
994 )
996 (define_insn "move_lo_quad_internal_be_<mode>"
997 [(set (match_operand:VQ_NO2E 0 "register_operand" "=w,w,w")
998 (vec_concat:VQ_NO2E
999 (vec_duplicate:<VHALF> (const_int 0))
1000 (match_operand:<VHALF> 1 "register_operand" "w,r,r")))]
1001 "TARGET_SIMD && BYTES_BIG_ENDIAN"
1002 "@
1003 dup\\t%d0, %1.d[0]
1004 fmov\\t%d0, %1
1005 dup\\t%d0, %1"
1006 [(set_attr "type" "neon_dup<q>,f_mcr,neon_dup<q>")
1007 (set_attr "simd" "yes,*,yes")
1008 (set_attr "fp" "*,yes,*")
1009 (set_attr "length" "4")]
1010 )
1012 (define_insn "move_lo_quad_internal_be_<mode>"
1013 [(set (match_operand:VQ_2E 0 "register_operand" "=w,w,w")
1014 (vec_concat:VQ_2E
1015 (const_int 0)
1016 (match_operand:<VHALF> 1 "register_operand" "w,r,r")))]
1017 "TARGET_SIMD && BYTES_BIG_ENDIAN"
1018 "@
1019 dup\\t%d0, %1.d[0]
1020 fmov\\t%d0, %1
1021 dup\\t%d0, %1"
1022 [(set_attr "type" "neon_dup<q>,f_mcr,neon_dup<q>")
1023 (set_attr "simd" "yes,*,yes")
1024 (set_attr "fp" "*,yes,*")
1025 (set_attr "length" "4")]
1026 )
1028 (define_expand "move_lo_quad_<mode>"
1029 [(match_operand:VQ 0 "register_operand")
1030 (match_operand:VQ 1 "register_operand")]
1031 "TARGET_SIMD"
1032 {
1033 if (BYTES_BIG_ENDIAN)
1034 emit_insn (gen_move_lo_quad_internal_be_<mode> (operands[0], operands[1]));
1035 else
1036 emit_insn (gen_move_lo_quad_internal_<mode> (operands[0], operands[1]));
1037 DONE;
1038 }
1039 )
1041 ;; Move operand1 to the high architectural bits of the register, keeping
1042 ;; the low architectural bits of operand2.
1043 ;; For little-endian this is { operand2, operand1 }
1044 ;; For big-endian this is { operand1, operand2 }
1046 (define_insn "aarch64_simd_move_hi_quad_<mode>"
1047 [(set (match_operand:VQ 0 "register_operand" "+w,w")
1048 (vec_concat:VQ
1049 (vec_select:<VHALF>
1050 (match_dup 0)
1051 (match_operand:VQ 2 "vect_par_cnst_lo_half" ""))
1052 (match_operand:<VHALF> 1 "register_operand" "w,r")))]
1053 "TARGET_SIMD && !BYTES_BIG_ENDIAN"
1054 "@
1055 ins\\t%0.d[1], %1.d[0]
1056 ins\\t%0.d[1], %1"
1057 [(set_attr "type" "neon_ins")]
1058 )
1060 (define_insn "aarch64_simd_move_hi_quad_be_<mode>"
1061 [(set (match_operand:VQ 0 "register_operand" "+w,w")
1062 (vec_concat:VQ
1063 (match_operand:<VHALF> 1 "register_operand" "w,r")
1064 (vec_select:<VHALF>
1065 (match_dup 0)
1066 (match_operand:VQ 2 "vect_par_cnst_lo_half" ""))))]
1067 "TARGET_SIMD && BYTES_BIG_ENDIAN"
1068 "@
1069 ins\\t%0.d[1], %1.d[0]
1070 ins\\t%0.d[1], %1"
1071 [(set_attr "type" "neon_ins")]
1072 )
1074 (define_expand "move_hi_quad_<mode>"
1075 [(match_operand:VQ 0 "register_operand" "")
1076 (match_operand:<VHALF> 1 "register_operand" "")]
1077 "TARGET_SIMD"
1078 {
1079 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
1080 if (BYTES_BIG_ENDIAN)
1081 emit_insn (gen_aarch64_simd_move_hi_quad_be_<mode> (operands[0],
1082 operands[1], p));
1083 else
1084 emit_insn (gen_aarch64_simd_move_hi_quad_<mode> (operands[0],
1085 operands[1], p));
1086 DONE;
1087 })
1089 ;; Narrowing operations.
1091 ;; For doubles.
1092 (define_insn "aarch64_simd_vec_pack_trunc_<mode>"
1093 [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
1094 (truncate:<VNARROWQ> (match_operand:VQN 1 "register_operand" "w")))]
1095 "TARGET_SIMD"
1096 "xtn\\t%0.<Vntype>, %1.<Vtype>"
1097 [(set_attr "type" "neon_shift_imm_narrow_q")]
1098 )
1100 (define_expand "vec_pack_trunc_<mode>"
1101 [(match_operand:<VNARROWD> 0 "register_operand" "")
1102 (match_operand:VDN 1 "register_operand" "")
1103 (match_operand:VDN 2 "register_operand" "")]
1104 "TARGET_SIMD"
1105 {
1106 rtx tempreg = gen_reg_rtx (<VDBL>mode);
1107 int lo = BYTES_BIG_ENDIAN ? 2 : 1;
1108 int hi = BYTES_BIG_ENDIAN ? 1 : 2;
1110 emit_insn (gen_move_lo_quad_<Vdbl> (tempreg, operands[lo]));
1111 emit_insn (gen_move_hi_quad_<Vdbl> (tempreg, operands[hi]));
1112 emit_insn (gen_aarch64_simd_vec_pack_trunc_<Vdbl> (operands[0], tempreg));
1113 DONE;
1114 })
1116 ;; For quads.
1118 (define_insn "vec_pack_trunc_<mode>"
1119 [(set (match_operand:<VNARROWQ2> 0 "register_operand" "=&w")
1120 (vec_concat:<VNARROWQ2>
1121 (truncate:<VNARROWQ> (match_operand:VQN 1 "register_operand" "w"))
1122 (truncate:<VNARROWQ> (match_operand:VQN 2 "register_operand" "w"))))]
1123 "TARGET_SIMD"
1124 {
1125 if (BYTES_BIG_ENDIAN)
1126 return "xtn\\t%0.<Vntype>, %2.<Vtype>\;xtn2\\t%0.<V2ntype>, %1.<Vtype>";
1127 else
1128 return "xtn\\t%0.<Vntype>, %1.<Vtype>\;xtn2\\t%0.<V2ntype>, %2.<Vtype>";
1129 }
1130 [(set_attr "type" "multiple")
1131 (set_attr "length" "8")]
1132 )
1134 ;; Widening operations.
1136 (define_insn "aarch64_simd_vec_unpack<su>_lo_<mode>"
1137 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1138 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1139 (match_operand:VQW 1 "register_operand" "w")
1140 (match_operand:VQW 2 "vect_par_cnst_lo_half" "")
1141 )))]
1142 "TARGET_SIMD"
1143 "<su>shll %0.<Vwtype>, %1.<Vhalftype>, 0"
1144 [(set_attr "type" "neon_shift_imm_long")]
1145 )
1147 (define_insn "aarch64_simd_vec_unpack<su>_hi_<mode>"
1148 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1149 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1150 (match_operand:VQW 1 "register_operand" "w")
1151 (match_operand:VQW 2 "vect_par_cnst_hi_half" "")
1152 )))]
1153 "TARGET_SIMD"
1154 "<su>shll2 %0.<Vwtype>, %1.<Vtype>, 0"
1155 [(set_attr "type" "neon_shift_imm_long")]
1156 )
1158 (define_expand "vec_unpack<su>_hi_<mode>"
1159 [(match_operand:<VWIDE> 0 "register_operand" "")
1160 (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand"))]
1161 "TARGET_SIMD"
1162 {
1163 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
1164 emit_insn (gen_aarch64_simd_vec_unpack<su>_hi_<mode> (operands[0],
1165 operands[1], p));
1166 DONE;
1167 }
1168 )
1170 (define_expand "vec_unpack<su>_lo_<mode>"
1171 [(match_operand:<VWIDE> 0 "register_operand" "")
1172 (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand" ""))]
1173 "TARGET_SIMD"
1174 {
1175 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
1176 emit_insn (gen_aarch64_simd_vec_unpack<su>_lo_<mode> (operands[0],
1177 operands[1], p));
1178 DONE;
1179 }
1180 )
1182 ;; Widening arithmetic.
1184 (define_insn "*aarch64_<su>mlal_lo<mode>"
1185 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1186 (plus:<VWIDE>
1187 (mult:<VWIDE>
1188 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1189 (match_operand:VQW 2 "register_operand" "w")
1190 (match_operand:VQW 3 "vect_par_cnst_lo_half" "")))
1191 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1192 (match_operand:VQW 4 "register_operand" "w")
1193 (match_dup 3))))
1194 (match_operand:<VWIDE> 1 "register_operand" "0")))]
1195 "TARGET_SIMD"
1196 "<su>mlal\t%0.<Vwtype>, %2.<Vhalftype>, %4.<Vhalftype>"
1197 [(set_attr "type" "neon_mla_<Vetype>_long")]
1198 )
1200 (define_insn "*aarch64_<su>mlal_hi<mode>"
1201 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1202 (plus:<VWIDE>
1203 (mult:<VWIDE>
1204 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1205 (match_operand:VQW 2 "register_operand" "w")
1206 (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))
1207 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1208 (match_operand:VQW 4 "register_operand" "w")
1209 (match_dup 3))))
1210 (match_operand:<VWIDE> 1 "register_operand" "0")))]
1211 "TARGET_SIMD"
1212 "<su>mlal2\t%0.<Vwtype>, %2.<Vtype>, %4.<Vtype>"
1213 [(set_attr "type" "neon_mla_<Vetype>_long")]
1214 )
1216 (define_insn "*aarch64_<su>mlsl_lo<mode>"
1217 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1218 (minus:<VWIDE>
1219 (match_operand:<VWIDE> 1 "register_operand" "0")
1220 (mult:<VWIDE>
1221 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1222 (match_operand:VQW 2 "register_operand" "w")
1223 (match_operand:VQW 3 "vect_par_cnst_lo_half" "")))
1224 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1225 (match_operand:VQW 4 "register_operand" "w")
1226 (match_dup 3))))))]
1227 "TARGET_SIMD"
1228 "<su>mlsl\t%0.<Vwtype>, %2.<Vhalftype>, %4.<Vhalftype>"
1229 [(set_attr "type" "neon_mla_<Vetype>_long")]
1230 )
1232 (define_insn "*aarch64_<su>mlsl_hi<mode>"
1233 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1234 (minus:<VWIDE>
1235 (match_operand:<VWIDE> 1 "register_operand" "0")
1236 (mult:<VWIDE>
1237 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1238 (match_operand:VQW 2 "register_operand" "w")
1239 (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))
1240 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1241 (match_operand:VQW 4 "register_operand" "w")
1242 (match_dup 3))))))]
1243 "TARGET_SIMD"
1244 "<su>mlsl2\t%0.<Vwtype>, %2.<Vtype>, %4.<Vtype>"
1245 [(set_attr "type" "neon_mla_<Vetype>_long")]
1246 )
1248 (define_insn "*aarch64_<su>mlal<mode>"
1249 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1250 (plus:<VWIDE>
1251 (mult:<VWIDE>
1252 (ANY_EXTEND:<VWIDE>
1253 (match_operand:VDW 1 "register_operand" "w"))
1254 (ANY_EXTEND:<VWIDE>
1255 (match_operand:VDW 2 "register_operand" "w")))
1256 (match_operand:<VWIDE> 3 "register_operand" "0")))]
1257 "TARGET_SIMD"
1258 "<su>mlal\t%0.<Vwtype>, %1.<Vtype>, %2.<Vtype>"
1259 [(set_attr "type" "neon_mla_<Vetype>_long")]
1260 )
1262 (define_insn "*aarch64_<su>mlsl<mode>"
1263 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1264 (minus:<VWIDE>
1265 (match_operand:<VWIDE> 1 "register_operand" "0")
1266 (mult:<VWIDE>
1267 (ANY_EXTEND:<VWIDE>
1268 (match_operand:VDW 2 "register_operand" "w"))
1269 (ANY_EXTEND:<VWIDE>
1270 (match_operand:VDW 3 "register_operand" "w")))))]
1271 "TARGET_SIMD"
1272 "<su>mlsl\t%0.<Vwtype>, %2.<Vtype>, %3.<Vtype>"
1273 [(set_attr "type" "neon_mla_<Vetype>_long")]
1274 )
1276 (define_insn "aarch64_simd_vec_<su>mult_lo_<mode>"
1277 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1278 (mult:<VWIDE> (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1279 (match_operand:VQW 1 "register_operand" "w")
1280 (match_operand:VQW 3 "vect_par_cnst_lo_half" "")))
1281 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1282 (match_operand:VQW 2 "register_operand" "w")
1283 (match_dup 3)))))]
1284 "TARGET_SIMD"
1285 "<su>mull\\t%0.<Vwtype>, %1.<Vhalftype>, %2.<Vhalftype>"
1286 [(set_attr "type" "neon_mul_<Vetype>_long")]
1287 )
1289 (define_expand "vec_widen_<su>mult_lo_<mode>"
1290 [(match_operand:<VWIDE> 0 "register_operand" "")
1291 (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand" ""))
1292 (ANY_EXTEND:<VWIDE> (match_operand:VQW 2 "register_operand" ""))]
1293 "TARGET_SIMD"
1294 {
1295 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
1296 emit_insn (gen_aarch64_simd_vec_<su>mult_lo_<mode> (operands[0],
1297 operands[1],
1298 operands[2], p));
1299 DONE;
1300 }
1301 )
1303 (define_insn "aarch64_simd_vec_<su>mult_hi_<mode>"
1304 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1305 (mult:<VWIDE> (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1306 (match_operand:VQW 1 "register_operand" "w")
1307 (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))
1308 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1309 (match_operand:VQW 2 "register_operand" "w")
1310 (match_dup 3)))))]
1311 "TARGET_SIMD"
1312 "<su>mull2\\t%0.<Vwtype>, %1.<Vtype>, %2.<Vtype>"
1313 [(set_attr "type" "neon_mul_<Vetype>_long")]
1314 )
1316 (define_expand "vec_widen_<su>mult_hi_<mode>"
1317 [(match_operand:<VWIDE> 0 "register_operand" "")
1318 (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand" ""))
1319 (ANY_EXTEND:<VWIDE> (match_operand:VQW 2 "register_operand" ""))]
1320 "TARGET_SIMD"
1321 {
1322 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
1323 emit_insn (gen_aarch64_simd_vec_<su>mult_hi_<mode> (operands[0],
1324 operands[1],
1325 operands[2], p));
1326 DONE;
1328 }
1329 )
1331 ;; FP vector operations.
1332 ;; AArch64 AdvSIMD supports single-precision (32-bit) and
1333 ;; double-precision (64-bit) floating-point data types and arithmetic as
1334 ;; defined by the IEEE 754-2008 standard. This makes them vectorizable
1335 ;; without the need for -ffast-math or -funsafe-math-optimizations.
1336 ;;
1337 ;; Floating-point operations can raise an exception. Vectorizing such
1338 ;; operations are safe because of reasons explained below.
1339 ;;
1340 ;; ARMv8 permits an extension to enable trapped floating-point
1341 ;; exception handling, however this is an optional feature. In the
1342 ;; event of a floating-point exception being raised by vectorised
1343 ;; code then:
1344 ;; 1. If trapped floating-point exceptions are available, then a trap
1345 ;; will be taken when any lane raises an enabled exception. A trap
1346 ;; handler may determine which lane raised the exception.
1347 ;; 2. Alternatively a sticky exception flag is set in the
1348 ;; floating-point status register (FPSR). Software may explicitly
1349 ;; test the exception flags, in which case the tests will either
1350 ;; prevent vectorisation, allowing precise identification of the
1351 ;; failing operation, or if tested outside of vectorisable regions
1352 ;; then the specific operation and lane are not of interest.
1354 ;; FP arithmetic operations.
1356 (define_insn "add<mode>3"
1357 [(set (match_operand:VDQF 0 "register_operand" "=w")
1358 (plus:VDQF (match_operand:VDQF 1 "register_operand" "w")
1359 (match_operand:VDQF 2 "register_operand" "w")))]
1360 "TARGET_SIMD"
1361 "fadd\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1362 [(set_attr "type" "neon_fp_addsub_<Vetype><q>")]
1363 )
1365 (define_insn "sub<mode>3"
1366 [(set (match_operand:VDQF 0 "register_operand" "=w")
1367 (minus:VDQF (match_operand:VDQF 1 "register_operand" "w")
1368 (match_operand:VDQF 2 "register_operand" "w")))]
1369 "TARGET_SIMD"
1370 "fsub\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1371 [(set_attr "type" "neon_fp_addsub_<Vetype><q>")]
1372 )
1374 (define_insn "mul<mode>3"
1375 [(set (match_operand:VDQF 0 "register_operand" "=w")
1376 (mult:VDQF (match_operand:VDQF 1 "register_operand" "w")
1377 (match_operand:VDQF 2 "register_operand" "w")))]
1378 "TARGET_SIMD"
1379 "fmul\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1380 [(set_attr "type" "neon_fp_mul_<Vetype><q>")]
1381 )
1383 (define_insn "div<mode>3"
1384 [(set (match_operand:VDQF 0 "register_operand" "=w")
1385 (div:VDQF (match_operand:VDQF 1 "register_operand" "w")
1386 (match_operand:VDQF 2 "register_operand" "w")))]
1387 "TARGET_SIMD"
1388 "fdiv\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1389 [(set_attr "type" "neon_fp_div_<Vetype><q>")]
1390 )
1392 (define_insn "neg<mode>2"
1393 [(set (match_operand:VDQF 0 "register_operand" "=w")
1394 (neg:VDQF (match_operand:VDQF 1 "register_operand" "w")))]
1395 "TARGET_SIMD"
1396 "fneg\\t%0.<Vtype>, %1.<Vtype>"
1397 [(set_attr "type" "neon_fp_neg_<Vetype><q>")]
1398 )
1400 (define_insn "abs<mode>2"
1401 [(set (match_operand:VDQF 0 "register_operand" "=w")
1402 (abs:VDQF (match_operand:VDQF 1 "register_operand" "w")))]
1403 "TARGET_SIMD"
1404 "fabs\\t%0.<Vtype>, %1.<Vtype>"
1405 [(set_attr "type" "neon_fp_abs_<Vetype><q>")]
1406 )
1408 (define_insn "fma<mode>4"
1409 [(set (match_operand:VDQF 0 "register_operand" "=w")
1410 (fma:VDQF (match_operand:VDQF 1 "register_operand" "w")
1411 (match_operand:VDQF 2 "register_operand" "w")
1412 (match_operand:VDQF 3 "register_operand" "0")))]
1413 "TARGET_SIMD"
1414 "fmla\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1415 [(set_attr "type" "neon_fp_mla_<Vetype><q>")]
1416 )
1418 (define_insn "*aarch64_fma4_elt<mode>"
1419 [(set (match_operand:VDQF 0 "register_operand" "=w")
1420 (fma:VDQF
1421 (vec_duplicate:VDQF
1422 (vec_select:<VEL>
1423 (match_operand:VDQF 1 "register_operand" "<h_con>")
1424 (parallel [(match_operand:SI 2 "immediate_operand")])))
1425 (match_operand:VDQF 3 "register_operand" "w")
1426 (match_operand:VDQF 4 "register_operand" "0")))]
1427 "TARGET_SIMD"
1428 {
1429 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
1430 return "fmla\\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
1431 }
1432 [(set_attr "type" "neon_fp_mla_<Vetype>_scalar<q>")]
1433 )
1435 (define_insn "*aarch64_fma4_elt_<vswap_width_name><mode>"
1436 [(set (match_operand:VDQSF 0 "register_operand" "=w")
1437 (fma:VDQSF
1438 (vec_duplicate:VDQSF
1439 (vec_select:<VEL>
1440 (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
1441 (parallel [(match_operand:SI 2 "immediate_operand")])))
1442 (match_operand:VDQSF 3 "register_operand" "w")
1443 (match_operand:VDQSF 4 "register_operand" "0")))]
1444 "TARGET_SIMD"
1445 {
1446 operands[2] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
1447 INTVAL (operands[2])));
1448 return "fmla\\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
1449 }
1450 [(set_attr "type" "neon_fp_mla_<Vetype>_scalar<q>")]
1451 )
1453 (define_insn "*aarch64_fma4_elt_to_128df"
1454 [(set (match_operand:V2DF 0 "register_operand" "=w")
1455 (fma:V2DF
1456 (vec_duplicate:V2DF
1457 (match_operand:DF 1 "register_operand" "w"))
1458 (match_operand:V2DF 2 "register_operand" "w")
1459 (match_operand:V2DF 3 "register_operand" "0")))]
1460 "TARGET_SIMD"
1461 "fmla\\t%0.2d, %2.2d, %1.2d[0]"
1462 [(set_attr "type" "neon_fp_mla_d_scalar_q")]
1463 )
1465 (define_insn "*aarch64_fma4_elt_to_64v2df"
1466 [(set (match_operand:DF 0 "register_operand" "=w")
1467 (fma:DF
1468 (vec_select:DF
1469 (match_operand:V2DF 1 "register_operand" "w")
1470 (parallel [(match_operand:SI 2 "immediate_operand")]))
1471 (match_operand:DF 3 "register_operand" "w")
1472 (match_operand:DF 4 "register_operand" "0")))]
1473 "TARGET_SIMD"
1474 {
1475 operands[2] = GEN_INT (ENDIAN_LANE_N (V2DFmode, INTVAL (operands[2])));
1476 return "fmla\\t%0.2d, %3.2d, %1.2d[%2]";
1477 }
1478 [(set_attr "type" "neon_fp_mla_d_scalar_q")]
1479 )
1481 (define_insn "fnma<mode>4"
1482 [(set (match_operand:VDQF 0 "register_operand" "=w")
1483 (fma:VDQF
1484 (match_operand:VDQF 1 "register_operand" "w")
1485 (neg:VDQF
1486 (match_operand:VDQF 2 "register_operand" "w"))
1487 (match_operand:VDQF 3 "register_operand" "0")))]
1488 "TARGET_SIMD"
1489 "fmls\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1490 [(set_attr "type" "neon_fp_mla_<Vetype><q>")]
1491 )
1493 (define_insn "*aarch64_fnma4_elt<mode>"
1494 [(set (match_operand:VDQF 0 "register_operand" "=w")
1495 (fma:VDQF
1496 (neg:VDQF
1497 (match_operand:VDQF 3 "register_operand" "w"))
1498 (vec_duplicate:VDQF
1499 (vec_select:<VEL>
1500 (match_operand:VDQF 1 "register_operand" "<h_con>")
1501 (parallel [(match_operand:SI 2 "immediate_operand")])))
1502 (match_operand:VDQF 4 "register_operand" "0")))]
1503 "TARGET_SIMD"
1504 {
1505 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
1506 return "fmls\\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
1507 }
1508 [(set_attr "type" "neon_fp_mla_<Vetype>_scalar<q>")]
1509 )
1511 (define_insn "*aarch64_fnma4_elt_<vswap_width_name><mode>"
1512 [(set (match_operand:VDQSF 0 "register_operand" "=w")
1513 (fma:VDQSF
1514 (neg:VDQSF
1515 (match_operand:VDQSF 3 "register_operand" "w"))
1516 (vec_duplicate:VDQSF
1517 (vec_select:<VEL>
1518 (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
1519 (parallel [(match_operand:SI 2 "immediate_operand")])))
1520 (match_operand:VDQSF 4 "register_operand" "0")))]
1521 "TARGET_SIMD"
1522 {
1523 operands[2] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
1524 INTVAL (operands[2])));
1525 return "fmls\\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
1526 }
1527 [(set_attr "type" "neon_fp_mla_<Vetype>_scalar<q>")]
1528 )
1530 (define_insn "*aarch64_fnma4_elt_to_128df"
1531 [(set (match_operand:V2DF 0 "register_operand" "=w")
1532 (fma:V2DF
1533 (neg:V2DF
1534 (match_operand:V2DF 2 "register_operand" "w"))
1535 (vec_duplicate:V2DF
1536 (match_operand:DF 1 "register_operand" "w"))
1537 (match_operand:V2DF 3 "register_operand" "0")))]
1538 "TARGET_SIMD"
1539 "fmls\\t%0.2d, %2.2d, %1.2d[0]"
1540 [(set_attr "type" "neon_fp_mla_d_scalar_q")]
1541 )
1543 (define_insn "*aarch64_fnma4_elt_to_64v2df"
1544 [(set (match_operand:DF 0 "register_operand" "=w")
1545 (fma:DF
1546 (vec_select:DF
1547 (match_operand:V2DF 1 "register_operand" "w")
1548 (parallel [(match_operand:SI 2 "immediate_operand")]))
1549 (neg:DF
1550 (match_operand:DF 3 "register_operand" "w"))
1551 (match_operand:DF 4 "register_operand" "0")))]
1552 "TARGET_SIMD"
1553 {
1554 operands[2] = GEN_INT (ENDIAN_LANE_N (V2DFmode, INTVAL (operands[2])));
1555 return "fmls\\t%0.2d, %3.2d, %1.2d[%2]";
1556 }
1557 [(set_attr "type" "neon_fp_mla_d_scalar_q")]
1558 )
1560 ;; Vector versions of the floating-point frint patterns.
1561 ;; Expands to btrunc, ceil, floor, nearbyint, rint, round, frintn.
1562 (define_insn "<frint_pattern><mode>2"
1563 [(set (match_operand:VDQF 0 "register_operand" "=w")
1564 (unspec:VDQF [(match_operand:VDQF 1 "register_operand" "w")]
1565 FRINT))]
1566 "TARGET_SIMD"
1567 "frint<frint_suffix>\\t%0.<Vtype>, %1.<Vtype>"
1568 [(set_attr "type" "neon_fp_round_<Vetype><q>")]
1569 )
1571 ;; Vector versions of the fcvt standard patterns.
1572 ;; Expands to lbtrunc, lround, lceil, lfloor
1573 (define_insn "l<fcvt_pattern><su_optab><VDQF:mode><fcvt_target>2"
1574 [(set (match_operand:<FCVT_TARGET> 0 "register_operand" "=w")
1575 (FIXUORS:<FCVT_TARGET> (unspec:<FCVT_TARGET>
1576 [(match_operand:VDQF 1 "register_operand" "w")]
1577 FCVT)))]
1578 "TARGET_SIMD"
1579 "fcvt<frint_suffix><su>\\t%0.<Vtype>, %1.<Vtype>"
1580 [(set_attr "type" "neon_fp_to_int_<Vetype><q>")]
1581 )
1583 (define_expand "<optab><VDQF:mode><fcvt_target>2"
1584 [(set (match_operand:<FCVT_TARGET> 0 "register_operand")
1585 (FIXUORS:<FCVT_TARGET> (unspec:<FCVT_TARGET>
1586 [(match_operand:VDQF 1 "register_operand")]
1587 UNSPEC_FRINTZ)))]
1588 "TARGET_SIMD"
1589 {})
1591 (define_expand "<fix_trunc_optab><VDQF:mode><fcvt_target>2"
1592 [(set (match_operand:<FCVT_TARGET> 0 "register_operand")
1593 (FIXUORS:<FCVT_TARGET> (unspec:<FCVT_TARGET>
1594 [(match_operand:VDQF 1 "register_operand")]
1595 UNSPEC_FRINTZ)))]
1596 "TARGET_SIMD"
1597 {})
1599 (define_expand "ftrunc<VDQF:mode>2"
1600 [(set (match_operand:VDQF 0 "register_operand")
1601 (unspec:VDQF [(match_operand:VDQF 1 "register_operand")]
1602 UNSPEC_FRINTZ))]
1603 "TARGET_SIMD"
1604 {})
1606 (define_insn "<optab><fcvt_target><VDQF:mode>2"
1607 [(set (match_operand:VDQF 0 "register_operand" "=w")
1608 (FLOATUORS:VDQF
1609 (match_operand:<FCVT_TARGET> 1 "register_operand" "w")))]
1610 "TARGET_SIMD"
1611 "<su_optab>cvtf\\t%0.<Vtype>, %1.<Vtype>"
1612 [(set_attr "type" "neon_int_to_fp_<Vetype><q>")]
1613 )
1615 ;; Conversions between vectors of floats and doubles.
1616 ;; Contains a mix of patterns to match standard pattern names
1617 ;; and those for intrinsics.
1619 ;; Float widening operations.
1621 (define_insn "vec_unpacks_lo_v4sf"
1622 [(set (match_operand:V2DF 0 "register_operand" "=w")
1623 (float_extend:V2DF
1624 (vec_select:V2SF
1625 (match_operand:V4SF 1 "register_operand" "w")
1626 (parallel [(const_int 0) (const_int 1)])
1627 )))]
1628 "TARGET_SIMD"
1629 "fcvtl\\t%0.2d, %1.2s"
1630 [(set_attr "type" "neon_fp_cvt_widen_s")]
1631 )
1633 (define_insn "aarch64_float_extend_lo_v2df"
1634 [(set (match_operand:V2DF 0 "register_operand" "=w")
1635 (float_extend:V2DF
1636 (match_operand:V2SF 1 "register_operand" "w")))]
1637 "TARGET_SIMD"
1638 "fcvtl\\t%0.2d, %1.2s"
1639 [(set_attr "type" "neon_fp_cvt_widen_s")]
1640 )
1642 (define_insn "vec_unpacks_hi_v4sf"
1643 [(set (match_operand:V2DF 0 "register_operand" "=w")
1644 (float_extend:V2DF
1645 (vec_select:V2SF
1646 (match_operand:V4SF 1 "register_operand" "w")
1647 (parallel [(const_int 2) (const_int 3)])
1648 )))]
1649 "TARGET_SIMD"
1650 "fcvtl2\\t%0.2d, %1.4s"
1651 [(set_attr "type" "neon_fp_cvt_widen_s")]
1652 )
1654 ;; Float narrowing operations.
1656 (define_insn "aarch64_float_truncate_lo_v2sf"
1657 [(set (match_operand:V2SF 0 "register_operand" "=w")
1658 (float_truncate:V2SF
1659 (match_operand:V2DF 1 "register_operand" "w")))]
1660 "TARGET_SIMD"
1661 "fcvtn\\t%0.2s, %1.2d"
1662 [(set_attr "type" "neon_fp_cvt_narrow_d_q")]
1663 )
1665 (define_insn "aarch64_float_truncate_hi_v4sf"
1666 [(set (match_operand:V4SF 0 "register_operand" "=w")
1667 (vec_concat:V4SF
1668 (match_operand:V2SF 1 "register_operand" "0")
1669 (float_truncate:V2SF
1670 (match_operand:V2DF 2 "register_operand" "w"))))]
1671 "TARGET_SIMD"
1672 "fcvtn2\\t%0.4s, %2.2d"
1673 [(set_attr "type" "neon_fp_cvt_narrow_d_q")]
1674 )
1676 (define_expand "vec_pack_trunc_v2df"
1677 [(set (match_operand:V4SF 0 "register_operand")
1678 (vec_concat:V4SF
1679 (float_truncate:V2SF
1680 (match_operand:V2DF 1 "register_operand"))
1681 (float_truncate:V2SF
1682 (match_operand:V2DF 2 "register_operand"))
1683 ))]
1684 "TARGET_SIMD"
1685 {
1686 rtx tmp = gen_reg_rtx (V2SFmode);
1687 int lo = BYTES_BIG_ENDIAN ? 2 : 1;
1688 int hi = BYTES_BIG_ENDIAN ? 1 : 2;
1690 emit_insn (gen_aarch64_float_truncate_lo_v2sf (tmp, operands[lo]));
1691 emit_insn (gen_aarch64_float_truncate_hi_v4sf (operands[0],
1692 tmp, operands[hi]));
1693 DONE;
1694 }
1695 )
1697 (define_expand "vec_pack_trunc_df"
1698 [(set (match_operand:V2SF 0 "register_operand")
1699 (vec_concat:V2SF
1700 (float_truncate:SF
1701 (match_operand:DF 1 "register_operand"))
1702 (float_truncate:SF
1703 (match_operand:DF 2 "register_operand"))
1704 ))]
1705 "TARGET_SIMD"
1706 {
1707 rtx tmp = gen_reg_rtx (V2SFmode);
1708 int lo = BYTES_BIG_ENDIAN ? 2 : 1;
1709 int hi = BYTES_BIG_ENDIAN ? 1 : 2;
1711 emit_insn (gen_move_lo_quad_v2df (tmp, operands[lo]));
1712 emit_insn (gen_move_hi_quad_v2df (tmp, operands[hi]));
1713 emit_insn (gen_aarch64_float_truncate_lo_v2sf (operands[0], tmp));
1714 DONE;
1715 }
1716 )
1718 (define_insn "aarch64_vmls<mode>"
1719 [(set (match_operand:VDQF 0 "register_operand" "=w")
1720 (minus:VDQF (match_operand:VDQF 1 "register_operand" "0")
1721 (mult:VDQF (match_operand:VDQF 2 "register_operand" "w")
1722 (match_operand:VDQF 3 "register_operand" "w"))))]
1723 "TARGET_SIMD"
1724 "fmls\\t%0.<Vtype>, %2.<Vtype>, %3.<Vtype>"
1725 [(set_attr "type" "neon_fp_mla_<Vetype>_scalar<q>")]
1726 )
1728 ;; FP Max/Min
1729 ;; Max/Min are introduced by idiom recognition by GCC's mid-end. An
1730 ;; expression like:
1731 ;; a = (b < c) ? b : c;
1732 ;; is idiom-matched as MIN_EXPR<b,c> only if -ffinite-math-only is enabled
1733 ;; either explicitly or indirectly via -ffast-math.
1734 ;;
1735 ;; MIN_EXPR and MAX_EXPR eventually map to 'smin' and 'smax' in RTL.
1736 ;; The 'smax' and 'smin' RTL standard pattern names do not specify which
1737 ;; operand will be returned when both operands are zero (i.e. they may not
1738 ;; honour signed zeroes), or when either operand is NaN. Therefore GCC
1739 ;; only introduces MIN_EXPR/MAX_EXPR in fast math mode or when not honouring
1740 ;; NaNs.
1742 (define_insn "<su><maxmin><mode>3"
1743 [(set (match_operand:VDQF 0 "register_operand" "=w")
1744 (FMAXMIN:VDQF (match_operand:VDQF 1 "register_operand" "w")
1745 (match_operand:VDQF 2 "register_operand" "w")))]
1746 "TARGET_SIMD"
1747 "f<maxmin>nm\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1748 [(set_attr "type" "neon_fp_minmax_<Vetype><q>")]
1749 )
1751 (define_insn "<maxmin_uns><mode>3"
1752 [(set (match_operand:VDQF 0 "register_operand" "=w")
1753 (unspec:VDQF [(match_operand:VDQF 1 "register_operand" "w")
1754 (match_operand:VDQF 2 "register_operand" "w")]
1755 FMAXMIN_UNS))]
1756 "TARGET_SIMD"
1757 "<maxmin_uns_op>\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1758 [(set_attr "type" "neon_fp_minmax_<Vetype><q>")]
1759 )
1761 ;; 'across lanes' add.
1763 (define_expand "reduc_plus_scal_<mode>"
1764 [(match_operand:<VEL> 0 "register_operand" "=w")
1765 (unspec:VDQ [(match_operand:VDQ 1 "register_operand" "w")]
1766 UNSPEC_ADDV)]
1767 "TARGET_SIMD"
1768 {
1769 rtx elt = GEN_INT (ENDIAN_LANE_N (<MODE>mode, 0));
1770 rtx scratch = gen_reg_rtx (<MODE>mode);
1771 emit_insn (gen_aarch64_reduc_plus_internal<mode> (scratch, operands[1]));
1772 emit_insn (gen_aarch64_get_lane<mode> (operands[0], scratch, elt));
1773 DONE;
1774 }
1775 )
1777 (define_expand "reduc_plus_scal_<mode>"
1778 [(match_operand:<VEL> 0 "register_operand" "=w")
1779 (match_operand:V2F 1 "register_operand" "w")]
1780 "TARGET_SIMD"
1781 {
1782 rtx elt = GEN_INT (ENDIAN_LANE_N (<MODE>mode, 0));
1783 rtx scratch = gen_reg_rtx (<MODE>mode);
1784 emit_insn (gen_aarch64_reduc_plus_internal<mode> (scratch, operands[1]));
1785 emit_insn (gen_aarch64_get_lane<mode> (operands[0], scratch, elt));
1786 DONE;
1787 }
1788 )
1790 (define_insn "aarch64_reduc_plus_internal<mode>"
1791 [(set (match_operand:VDQV 0 "register_operand" "=w")
1792 (unspec:VDQV [(match_operand:VDQV 1 "register_operand" "w")]
1793 UNSPEC_ADDV))]
1794 "TARGET_SIMD"
1795 "add<VDQV:vp>\\t%<Vetype>0, %1.<Vtype>"
1796 [(set_attr "type" "neon_reduc_add<q>")]
1797 )
1799 (define_insn "aarch64_reduc_plus_internalv2si"
1800 [(set (match_operand:V2SI 0 "register_operand" "=w")
1801 (unspec:V2SI [(match_operand:V2SI 1 "register_operand" "w")]
1802 UNSPEC_ADDV))]
1803 "TARGET_SIMD"
1804 "addp\\t%0.2s, %1.2s, %1.2s"
1805 [(set_attr "type" "neon_reduc_add")]
1806 )
1808 (define_insn "aarch64_reduc_plus_internal<mode>"
1809 [(set (match_operand:V2F 0 "register_operand" "=w")
1810 (unspec:V2F [(match_operand:V2F 1 "register_operand" "w")]
1811 UNSPEC_FADDV))]
1812 "TARGET_SIMD"
1813 "faddp\\t%<Vetype>0, %1.<Vtype>"
1814 [(set_attr "type" "neon_fp_reduc_add_<Vetype><q>")]
1815 )
1817 (define_insn "aarch64_addpv4sf"
1818 [(set (match_operand:V4SF 0 "register_operand" "=w")
1819 (unspec:V4SF [(match_operand:V4SF 1 "register_operand" "w")]
1820 UNSPEC_FADDV))]
1821 "TARGET_SIMD"
1822 "faddp\\t%0.4s, %1.4s, %1.4s"
1823 [(set_attr "type" "neon_fp_reduc_add_s_q")]
1824 )
1826 (define_expand "reduc_plus_scal_v4sf"
1827 [(set (match_operand:SF 0 "register_operand")
1828 (unspec:V4SF [(match_operand:V4SF 1 "register_operand")]
1829 UNSPEC_FADDV))]
1830 "TARGET_SIMD"
1831 {
1832 rtx elt = GEN_INT (ENDIAN_LANE_N (V4SFmode, 0));
1833 rtx scratch = gen_reg_rtx (V4SFmode);
1834 emit_insn (gen_aarch64_addpv4sf (scratch, operands[1]));
1835 emit_insn (gen_aarch64_addpv4sf (scratch, scratch));
1836 emit_insn (gen_aarch64_get_lanev4sf (operands[0], scratch, elt));
1837 DONE;
1838 })
1840 (define_insn "clz<mode>2"
1841 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
1842 (clz:VDQ_BHSI (match_operand:VDQ_BHSI 1 "register_operand" "w")))]
1843 "TARGET_SIMD"
1844 "clz\\t%0.<Vtype>, %1.<Vtype>"
1845 [(set_attr "type" "neon_cls<q>")]
1846 )
1848 ;; 'across lanes' max and min ops.
1850 ;; Template for outputting a scalar, so we can create __builtins which can be
1851 ;; gimple_fold'd to the REDUC_(MAX|MIN)_EXPR tree code. (This is FP smax/smin).
1852 (define_expand "reduc_<maxmin_uns>_scal_<mode>"
1853 [(match_operand:<VEL> 0 "register_operand")
1854 (unspec:VDQF [(match_operand:VDQF 1 "register_operand")]
1855 FMAXMINV)]
1856 "TARGET_SIMD"
1857 {
1858 rtx elt = GEN_INT (ENDIAN_LANE_N (<MODE>mode, 0));
1859 rtx scratch = gen_reg_rtx (<MODE>mode);
1860 emit_insn (gen_aarch64_reduc_<maxmin_uns>_internal<mode> (scratch,
1861 operands[1]));
1862 emit_insn (gen_aarch64_get_lane<mode> (operands[0], scratch, elt));
1863 DONE;
1864 }
1865 )
1867 ;; Likewise for integer cases, signed and unsigned.
1868 (define_expand "reduc_<maxmin_uns>_scal_<mode>"
1869 [(match_operand:<VEL> 0 "register_operand")
1870 (unspec:VDQ_BHSI [(match_operand:VDQ_BHSI 1 "register_operand")]
1871 MAXMINV)]
1872 "TARGET_SIMD"
1873 {
1874 rtx elt = GEN_INT (ENDIAN_LANE_N (<MODE>mode, 0));
1875 rtx scratch = gen_reg_rtx (<MODE>mode);
1876 emit_insn (gen_aarch64_reduc_<maxmin_uns>_internal<mode> (scratch,
1877 operands[1]));
1878 emit_insn (gen_aarch64_get_lane<mode> (operands[0], scratch, elt));
1879 DONE;
1880 }
1881 )
1883 (define_insn "aarch64_reduc_<maxmin_uns>_internal<mode>"
1884 [(set (match_operand:VDQV_S 0 "register_operand" "=w")
1885 (unspec:VDQV_S [(match_operand:VDQV_S 1 "register_operand" "w")]
1886 MAXMINV))]
1887 "TARGET_SIMD"
1888 "<maxmin_uns_op>v\\t%<Vetype>0, %1.<Vtype>"
1889 [(set_attr "type" "neon_reduc_minmax<q>")]
1890 )
1892 (define_insn "aarch64_reduc_<maxmin_uns>_internalv2si"
1893 [(set (match_operand:V2SI 0 "register_operand" "=w")
1894 (unspec:V2SI [(match_operand:V2SI 1 "register_operand" "w")]
1895 MAXMINV))]
1896 "TARGET_SIMD"
1897 "<maxmin_uns_op>p\\t%0.2s, %1.2s, %1.2s"
1898 [(set_attr "type" "neon_reduc_minmax")]
1899 )
1901 (define_insn "aarch64_reduc_<maxmin_uns>_internal<mode>"
1902 [(set (match_operand:VDQF 0 "register_operand" "=w")
1903 (unspec:VDQF [(match_operand:VDQF 1 "register_operand" "w")]
1904 FMAXMINV))]
1905 "TARGET_SIMD"
1906 "<maxmin_uns_op><vp>\\t%<Vetype>0, %1.<Vtype>"
1907 [(set_attr "type" "neon_fp_reduc_minmax_<Vetype><q>")]
1908 )
1910 ;; aarch64_simd_bsl may compile to any of bsl/bif/bit depending on register
1911 ;; allocation.
1912 ;; Operand 1 is the mask, operands 2 and 3 are the bitfields from which
1913 ;; to select.
1914 ;;
1915 ;; Thus our BSL is of the form:
1916 ;; op0 = bsl (mask, op2, op3)
1917 ;; We can use any of:
1918 ;;
1919 ;; if (op0 = mask)
1920 ;; bsl mask, op1, op2
1921 ;; if (op0 = op1) (so 1-bits in mask choose bits from op2, else op0)
1922 ;; bit op0, op2, mask
1923 ;; if (op0 = op2) (so 0-bits in mask choose bits from op1, else op0)
1924 ;; bif op0, op1, mask
1926 (define_insn "aarch64_simd_bsl<mode>_internal"
1927 [(set (match_operand:VALLDIF 0 "register_operand" "=w,w,w")
1928 (ior:VALLDIF
1929 (and:VALLDIF
1930 (match_operand:<V_cmp_result> 1 "register_operand" " 0,w,w")
1931 (match_operand:VALLDIF 2 "register_operand" " w,w,0"))
1932 (and:VALLDIF
1933 (not:<V_cmp_result>
1934 (match_dup:<V_cmp_result> 1))
1935 (match_operand:VALLDIF 3 "register_operand" " w,0,w"))
1936 ))]
1937 "TARGET_SIMD"
1938 "@
1939 bsl\\t%0.<Vbtype>, %2.<Vbtype>, %3.<Vbtype>
1940 bit\\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>
1941 bif\\t%0.<Vbtype>, %3.<Vbtype>, %1.<Vbtype>"
1942 [(set_attr "type" "neon_bsl<q>")]
1943 )
1945 (define_expand "aarch64_simd_bsl<mode>"
1946 [(match_operand:VALLDIF 0 "register_operand")
1947 (match_operand:<V_cmp_result> 1 "register_operand")
1948 (match_operand:VALLDIF 2 "register_operand")
1949 (match_operand:VALLDIF 3 "register_operand")]
1950 "TARGET_SIMD"
1951 {
1952 /* We can't alias operands together if they have different modes. */
1953 operands[1] = gen_lowpart (<V_cmp_result>mode, operands[1]);
1954 emit_insn (gen_aarch64_simd_bsl<mode>_internal (operands[0], operands[1],
1955 operands[2], operands[3]));
1956 DONE;
1957 })
1959 (define_expand "aarch64_vcond_internal<mode><mode>"
1960 [(set (match_operand:VDQ 0 "register_operand")
1961 (if_then_else:VDQ
1962 (match_operator 3 "comparison_operator"
1963 [(match_operand:VDQ 4 "register_operand")
1964 (match_operand:VDQ 5 "nonmemory_operand")])
1965 (match_operand:VDQ 1 "nonmemory_operand")
1966 (match_operand:VDQ 2 "nonmemory_operand")))]
1967 "TARGET_SIMD"
1968 {
1969 rtx op1 = operands[1];
1970 rtx op2 = operands[2];
1971 rtx mask = gen_reg_rtx (<MODE>mode);
1972 enum rtx_code code = GET_CODE (operands[3]);
1974 /* Switching OP1 and OP2 is necessary for NE (to output a cmeq insn),
1975 and desirable for other comparisons if it results in FOO ? -1 : 0
1976 (this allows direct use of the comparison result without a bsl). */
1977 if (code == NE
1978 || (code != EQ
1979 && op1 == CONST0_RTX (<V_cmp_result>mode)
1980 && op2 == CONSTM1_RTX (<V_cmp_result>mode)))
1981 {
1982 op1 = operands[2];
1983 op2 = operands[1];
1984 switch (code)
1985 {
1986 case LE: code = GT; break;
1987 case LT: code = GE; break;
1988 case GE: code = LT; break;
1989 case GT: code = LE; break;
1990 /* No case EQ. */
1991 case NE: code = EQ; break;
1992 case LTU: code = GEU; break;
1993 case LEU: code = GTU; break;
1994 case GTU: code = LEU; break;
1995 case GEU: code = LTU; break;
1996 default: gcc_unreachable ();
1997 }
1998 }
2000 /* Make sure we can handle the last operand. */
2001 switch (code)
2002 {
2003 case NE:
2004 /* Normalized to EQ above. */
2005 gcc_unreachable ();
2007 case LE:
2008 case LT:
2009 case GE:
2010 case GT:
2011 case EQ:
2012 /* These instructions have a form taking an immediate zero. */
2013 if (operands[5] == CONST0_RTX (<MODE>mode))
2014 break;
2015 /* Fall through, as may need to load into register. */
2016 default:
2017 if (!REG_P (operands[5]))
2018 operands[5] = force_reg (<MODE>mode, operands[5]);
2019 break;
2020 }
2022 switch (code)
2023 {
2024 case LT:
2025 emit_insn (gen_aarch64_cmlt<mode> (mask, operands[4], operands[5]));
2026 break;
2028 case GE:
2029 emit_insn (gen_aarch64_cmge<mode> (mask, operands[4], operands[5]));
2030 break;
2032 case LE:
2033 emit_insn (gen_aarch64_cmle<mode> (mask, operands[4], operands[5]));
2034 break;
2036 case GT:
2037 emit_insn (gen_aarch64_cmgt<mode> (mask, operands[4], operands[5]));
2038 break;
2040 case LTU:
2041 emit_insn (gen_aarch64_cmgtu<mode> (mask, operands[5], operands[4]));
2042 break;
2044 case GEU:
2045 emit_insn (gen_aarch64_cmgeu<mode> (mask, operands[4], operands[5]));
2046 break;
2048 case LEU:
2049 emit_insn (gen_aarch64_cmgeu<mode> (mask, operands[5], operands[4]));
2050 break;
2052 case GTU:
2053 emit_insn (gen_aarch64_cmgtu<mode> (mask, operands[4], operands[5]));
2054 break;
2056 /* NE has been normalized to EQ above. */
2057 case EQ:
2058 emit_insn (gen_aarch64_cmeq<mode> (mask, operands[4], operands[5]));
2059 break;
2061 default:
2062 gcc_unreachable ();
2063 }
2065 /* If we have (a = (b CMP c) ? -1 : 0);
2066 Then we can simply move the generated mask. */
2068 if (op1 == CONSTM1_RTX (<V_cmp_result>mode)
2069 && op2 == CONST0_RTX (<V_cmp_result>mode))
2070 emit_move_insn (operands[0], mask);
2071 else
2072 {
2073 if (!REG_P (op1))
2074 op1 = force_reg (<MODE>mode, op1);
2075 if (!REG_P (op2))
2076 op2 = force_reg (<MODE>mode, op2);
2077 emit_insn (gen_aarch64_simd_bsl<mode> (operands[0], mask,
2078 op1, op2));
2079 }
2081 DONE;
2082 })
2084 (define_expand "aarch64_vcond_internal<VDQF_COND:mode><VDQF:mode>"
2085 [(set (match_operand:VDQF_COND 0 "register_operand")
2086 (if_then_else:VDQF
2087 (match_operator 3 "comparison_operator"
2088 [(match_operand:VDQF 4 "register_operand")
2089 (match_operand:VDQF 5 "nonmemory_operand")])
2090 (match_operand:VDQF_COND 1 "nonmemory_operand")
2091 (match_operand:VDQF_COND 2 "nonmemory_operand")))]
2092 "TARGET_SIMD"
2093 {
2094 int inverse = 0;
2095 int use_zero_form = 0;
2096 int swap_bsl_operands = 0;
2097 rtx op1 = operands[1];
2098 rtx op2 = operands[2];
2099 rtx mask = gen_reg_rtx (<VDQF_COND:V_cmp_result>mode);
2100 rtx tmp = gen_reg_rtx (<VDQF_COND:V_cmp_result>mode);
2102 rtx (*base_comparison) (rtx, rtx, rtx);
2103 rtx (*complimentary_comparison) (rtx, rtx, rtx);
2105 switch (GET_CODE (operands[3]))
2106 {
2107 case GE:
2108 case GT:
2109 case LE:
2110 case LT:
2111 case EQ:
2112 if (operands[5] == CONST0_RTX (<MODE>mode))
2113 {
2114 use_zero_form = 1;
2115 break;
2116 }
2117 /* Fall through. */
2118 default:
2119 if (!REG_P (operands[5]))
2120 operands[5] = force_reg (<VDQF:MODE>mode, operands[5]);
2121 }
2123 switch (GET_CODE (operands[3]))
2124 {
2125 case LT:
2126 case UNLT:
2127 inverse = 1;
2128 /* Fall through. */
2129 case GE:
2130 case UNGE:
2131 case ORDERED:
2132 case UNORDERED:
2133 base_comparison = gen_aarch64_cmge<VDQF:mode>;
2134 complimentary_comparison = gen_aarch64_cmgt<VDQF:mode>;
2135 break;
2136 case LE:
2137 case UNLE:
2138 inverse = 1;
2139 /* Fall through. */
2140 case GT:
2141 case UNGT:
2142 base_comparison = gen_aarch64_cmgt<VDQF:mode>;
2143 complimentary_comparison = gen_aarch64_cmge<VDQF:mode>;
2144 break;
2145 case EQ:
2146 case NE:
2147 case UNEQ:
2148 base_comparison = gen_aarch64_cmeq<VDQF:mode>;
2149 complimentary_comparison = gen_aarch64_cmeq<VDQF:mode>;
2150 break;
2151 default:
2152 gcc_unreachable ();
2153 }
2155 switch (GET_CODE (operands[3]))
2156 {
2157 case LT:
2158 case LE:
2159 case GT:
2160 case GE:
2161 case EQ:
2162 /* The easy case. Here we emit one of FCMGE, FCMGT or FCMEQ.
2163 As a LT b <=> b GE a && a LE b <=> b GT a. Our transformations are:
2164 a GE b -> a GE b
2165 a GT b -> a GT b
2166 a LE b -> b GE a
2167 a LT b -> b GT a
2168 a EQ b -> a EQ b
2169 Note that there also exist direct comparison against 0 forms,
2170 so catch those as a special case. */
2171 if (use_zero_form)
2172 {
2173 inverse = 0;
2174 switch (GET_CODE (operands[3]))
2175 {
2176 case LT:
2177 base_comparison = gen_aarch64_cmlt<VDQF:mode>;
2178 break;
2179 case LE:
2180 base_comparison = gen_aarch64_cmle<VDQF:mode>;
2181 break;
2182 default:
2183 /* Do nothing, other zero form cases already have the correct
2184 base_comparison. */
2185 break;
2186 }
2187 }
2189 if (!inverse)
2190 emit_insn (base_comparison (mask, operands[4], operands[5]));
2191 else
2192 emit_insn (complimentary_comparison (mask, operands[5], operands[4]));
2193 break;
2194 case UNLT:
2195 case UNLE:
2196 case UNGT:
2197 case UNGE:
2198 case NE:
2199 /* FCM returns false for lanes which are unordered, so if we use
2200 the inverse of the comparison we actually want to emit, then
2201 swap the operands to BSL, we will end up with the correct result.
2202 Note that a NE NaN and NaN NE b are true for all a, b.
2204 Our transformations are:
2205 a GE b -> !(b GT a)
2206 a GT b -> !(b GE a)
2207 a LE b -> !(a GT b)
2208 a LT b -> !(a GE b)
2209 a NE b -> !(a EQ b) */
2211 if (inverse)
2212 emit_insn (base_comparison (mask, operands[4], operands[5]));
2213 else
2214 emit_insn (complimentary_comparison (mask, operands[5], operands[4]));
2216 swap_bsl_operands = 1;
2217 break;
2218 case UNEQ:
2219 /* We check (a > b || b > a). combining these comparisons give us
2220 true iff !(a != b && a ORDERED b), swapping the operands to BSL
2221 will then give us (a == b || a UNORDERED b) as intended. */
2223 emit_insn (gen_aarch64_cmgt<VDQF:mode> (mask, operands[4], operands[5]));
2224 emit_insn (gen_aarch64_cmgt<VDQF:mode> (tmp, operands[5], operands[4]));
2225 emit_insn (gen_ior<VDQF_COND:v_cmp_result>3 (mask, mask, tmp));
2226 swap_bsl_operands = 1;
2227 break;
2228 case UNORDERED:
2229 /* Operands are ORDERED iff (a > b || b >= a).
2230 Swapping the operands to BSL will give the UNORDERED case. */
2231 swap_bsl_operands = 1;
2232 /* Fall through. */
2233 case ORDERED:
2234 emit_insn (gen_aarch64_cmgt<VDQF:mode> (tmp, operands[4], operands[5]));
2235 emit_insn (gen_aarch64_cmge<VDQF:mode> (mask, operands[5], operands[4]));
2236 emit_insn (gen_ior<VDQF_COND:v_cmp_result>3 (mask, mask, tmp));
2237 break;
2238 default:
2239 gcc_unreachable ();
2240 }
2242 if (swap_bsl_operands)
2243 {
2244 op1 = operands[2];
2245 op2 = operands[1];
2246 }
2248 /* If we have (a = (b CMP c) ? -1 : 0);
2249 Then we can simply move the generated mask. */
2251 if (op1 == CONSTM1_RTX (<VDQF_COND:V_cmp_result>mode)
2252 && op2 == CONST0_RTX (<VDQF_COND:V_cmp_result>mode))
2253 emit_move_insn (operands[0], mask);
2254 else
2255 {
2256 if (!REG_P (op1))
2257 op1 = force_reg (<VDQF_COND:MODE>mode, op1);
2258 if (!REG_P (op2))
2259 op2 = force_reg (<VDQF_COND:MODE>mode, op2);
2260 emit_insn (gen_aarch64_simd_bsl<VDQF_COND:mode> (operands[0], mask,
2261 op1, op2));
2262 }
2264 DONE;
2265 })
2267 (define_expand "vcond<mode><mode>"
2268 [(set (match_operand:VALL 0 "register_operand")
2269 (if_then_else:VALL
2270 (match_operator 3 "comparison_operator"
2271 [(match_operand:VALL 4 "register_operand")
2272 (match_operand:VALL 5 "nonmemory_operand")])
2273 (match_operand:VALL 1 "nonmemory_operand")
2274 (match_operand:VALL 2 "nonmemory_operand")))]
2275 "TARGET_SIMD"
2276 {
2277 emit_insn (gen_aarch64_vcond_internal<mode><mode> (operands[0], operands[1],
2278 operands[2], operands[3],
2279 operands[4], operands[5]));
2280 DONE;
2281 })
2283 (define_expand "vcond<v_cmp_result><mode>"
2284 [(set (match_operand:<V_cmp_result> 0 "register_operand")
2285 (if_then_else:<V_cmp_result>
2286 (match_operator 3 "comparison_operator"
2287 [(match_operand:VDQF 4 "register_operand")
2288 (match_operand:VDQF 5 "nonmemory_operand")])
2289 (match_operand:<V_cmp_result> 1 "nonmemory_operand")
2290 (match_operand:<V_cmp_result> 2 "nonmemory_operand")))]
2291 "TARGET_SIMD"
2292 {
2293 emit_insn (gen_aarch64_vcond_internal<v_cmp_result><mode> (
2294 operands[0], operands[1],
2295 operands[2], operands[3],
2296 operands[4], operands[5]));
2297 DONE;
2298 })
2300 (define_expand "vcondu<mode><mode>"
2301 [(set (match_operand:VDQ 0 "register_operand")
2302 (if_then_else:VDQ
2303 (match_operator 3 "comparison_operator"
2304 [(match_operand:VDQ 4 "register_operand")
2305 (match_operand:VDQ 5 "nonmemory_operand")])
2306 (match_operand:VDQ 1 "nonmemory_operand")
2307 (match_operand:VDQ 2 "nonmemory_operand")))]
2308 "TARGET_SIMD"
2309 {
2310 emit_insn (gen_aarch64_vcond_internal<mode><mode> (operands[0], operands[1],
2311 operands[2], operands[3],
2312 operands[4], operands[5]));
2313 DONE;
2314 })
2316 ;; Patterns for AArch64 SIMD Intrinsics.
2318 (define_expand "aarch64_create<mode>"
2319 [(match_operand:VD1 0 "register_operand" "")
2320 (match_operand:DI 1 "general_operand" "")]
2321 "TARGET_SIMD"
2322 {
2323 rtx src = gen_lowpart (<MODE>mode, operands[1]);
2324 emit_move_insn (operands[0], src);
2325 DONE;
2326 })
2328 ;; Lane extraction with sign extension to general purpose register.
2329 (define_insn "*aarch64_get_lane_extend<GPI:mode><VDQQH:mode>"
2330 [(set (match_operand:GPI 0 "register_operand" "=r")
2331 (sign_extend:GPI
2332 (vec_select:<VEL>
2333 (match_operand:VDQQH 1 "register_operand" "w")
2334 (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2335 "TARGET_SIMD"
2336 {
2337 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
2338 return "smov\\t%<GPI:w>0, %1.<VDQQH:Vetype>[%2]";
2339 }
2340 [(set_attr "type" "neon_to_gp<q>")]
2341 )
2343 (define_insn "*aarch64_get_lane_zero_extendsi<mode>"
2344 [(set (match_operand:SI 0 "register_operand" "=r")
2345 (zero_extend:SI
2346 (vec_select:<VEL>
2347 (match_operand:VDQQH 1 "register_operand" "w")
2348 (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2349 "TARGET_SIMD"
2350 {
2351 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
2352 return "umov\\t%w0, %1.<Vetype>[%2]";
2353 }
2354 [(set_attr "type" "neon_to_gp<q>")]
2355 )
2357 (define_expand "aarch64_be_checked_get_lane<mode>"
2358 [(match_operand:<VEL> 0 "aarch64_simd_nonimmediate_operand")
2359 (match_operand:VALL 1 "register_operand")
2360 (match_operand:SI 2 "immediate_operand")]
2361 "TARGET_SIMD"
2362 {
2363 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
2364 emit_insn (gen_aarch64_get_lane<mode> (operands[0],
2365 operands[1],
2366 operands[2]));
2367 DONE;
2368 }
2369 )
2371 ;; Lane extraction of a value, neither sign nor zero extension
2372 ;; is guaranteed so upper bits should be considered undefined.
2373 (define_insn "aarch64_get_lane<mode>"
2374 [(set (match_operand:<VEL> 0 "aarch64_simd_nonimmediate_operand" "=r, w, Utv")
2375 (vec_select:<VEL>
2376 (match_operand:VALL 1 "register_operand" "w, w, w")
2377 (parallel [(match_operand:SI 2 "immediate_operand" "i, i, i")])))]
2378 "TARGET_SIMD"
2379 {
2380 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
2381 switch (which_alternative)
2382 {
2383 case 0:
2384 return "umov\\t%<vwcore>0, %1.<Vetype>[%2]";
2385 case 1:
2386 return "dup\\t%<Vetype>0, %1.<Vetype>[%2]";
2387 case 2:
2388 return "st1\\t{%1.<Vetype>}[%2], %0";
2389 default:
2390 gcc_unreachable ();
2391 }
2392 }
2393 [(set_attr "type" "neon_to_gp<q>, neon_dup<q>, neon_store1_one_lane<q>")]
2394 )
2396 (define_expand "aarch64_get_lanedi"
2397 [(match_operand:DI 0 "register_operand")
2398 (match_operand:DI 1 "register_operand")
2399 (match_operand:SI 2 "immediate_operand")]
2400 "TARGET_SIMD"
2401 {
2402 aarch64_simd_lane_bounds (operands[2], 0, 1);
2403 emit_move_insn (operands[0], operands[1]);
2404 DONE;
2405 })
2407 ;; In this insn, operand 1 should be low, and operand 2 the high part of the
2408 ;; dest vector.
2410 (define_insn "*aarch64_combinez<mode>"
2411 [(set (match_operand:<VDBL> 0 "register_operand" "=&w")
2412 (vec_concat:<VDBL>
2413 (match_operand:VDIC 1 "register_operand" "w")
2414 (match_operand:VDIC 2 "aarch64_simd_imm_zero" "Dz")))]
2415 "TARGET_SIMD && !BYTES_BIG_ENDIAN"
2416 "mov\\t%0.8b, %1.8b"
2417 [(set_attr "type" "neon_move<q>")]
2418 )
2420 (define_insn "*aarch64_combinez_be<mode>"
2421 [(set (match_operand:<VDBL> 0 "register_operand" "=&w")
2422 (vec_concat:<VDBL>
2423 (match_operand:VDIC 2 "aarch64_simd_imm_zero" "Dz")
2424 (match_operand:VDIC 1 "register_operand" "w")))]
2425 "TARGET_SIMD && BYTES_BIG_ENDIAN"
2426 "mov\\t%0.8b, %1.8b"
2427 [(set_attr "type" "neon_move<q>")]
2428 )
2430 (define_expand "aarch64_combine<mode>"
2431 [(match_operand:<VDBL> 0 "register_operand")
2432 (match_operand:VDC 1 "register_operand")
2433 (match_operand:VDC 2 "register_operand")]
2434 "TARGET_SIMD"
2435 {
2436 rtx op1, op2;
2437 if (BYTES_BIG_ENDIAN)
2438 {
2439 op1 = operands[2];
2440 op2 = operands[1];
2441 }
2442 else
2443 {
2444 op1 = operands[1];
2445 op2 = operands[2];
2446 }
2447 emit_insn (gen_aarch64_combine_internal<mode> (operands[0], op1, op2));
2448 DONE;
2449 }
2450 )
2452 (define_insn_and_split "aarch64_combine_internal<mode>"
2453 [(set (match_operand:<VDBL> 0 "register_operand" "=&w")
2454 (vec_concat:<VDBL> (match_operand:VDC 1 "register_operand" "w")
2455 (match_operand:VDC 2 "register_operand" "w")))]
2456 "TARGET_SIMD"
2457 "#"
2458 "&& reload_completed"
2459 [(const_int 0)]
2460 {
2461 if (BYTES_BIG_ENDIAN)
2462 aarch64_split_simd_combine (operands[0], operands[2], operands[1]);
2463 else
2464 aarch64_split_simd_combine (operands[0], operands[1], operands[2]);
2465 DONE;
2466 }
2467 [(set_attr "type" "multiple")]
2468 )
2470 (define_expand "aarch64_simd_combine<mode>"
2471 [(match_operand:<VDBL> 0 "register_operand")
2472 (match_operand:VDC 1 "register_operand")
2473 (match_operand:VDC 2 "register_operand")]
2474 "TARGET_SIMD"
2475 {
2476 emit_insn (gen_move_lo_quad_<Vdbl> (operands[0], operands[1]));
2477 emit_insn (gen_move_hi_quad_<Vdbl> (operands[0], operands[2]));
2478 DONE;
2479 }
2480 [(set_attr "type" "multiple")]
2481 )
2483 ;; <su><addsub>l<q>.
2485 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>l<mode>_hi_internal"
2486 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2487 (ADDSUB:<VWIDE> (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
2488 (match_operand:VQW 1 "register_operand" "w")
2489 (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))
2490 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
2491 (match_operand:VQW 2 "register_operand" "w")
2492 (match_dup 3)))))]
2493 "TARGET_SIMD"
2494 "<ANY_EXTEND:su><ADDSUB:optab>l2\t%0.<Vwtype>, %1.<Vtype>, %2.<Vtype>"
2495 [(set_attr "type" "neon_<ADDSUB:optab>_long")]
2496 )
2498 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>l<mode>_lo_internal"
2499 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2500 (ADDSUB:<VWIDE> (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
2501 (match_operand:VQW 1 "register_operand" "w")
2502 (match_operand:VQW 3 "vect_par_cnst_lo_half" "")))
2503 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
2504 (match_operand:VQW 2 "register_operand" "w")
2505 (match_dup 3)))))]
2506 "TARGET_SIMD"
2507 "<ANY_EXTEND:su><ADDSUB:optab>l\t%0.<Vwtype>, %1.<Vhalftype>, %2.<Vhalftype>"
2508 [(set_attr "type" "neon_<ADDSUB:optab>_long")]
2509 )
2512 (define_expand "aarch64_saddl2<mode>"
2513 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2514 (match_operand:VQW 1 "register_operand" "w")
2515 (match_operand:VQW 2 "register_operand" "w")]
2516 "TARGET_SIMD"
2517 {
2518 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2519 emit_insn (gen_aarch64_saddl<mode>_hi_internal (operands[0], operands[1],
2520 operands[2], p));
2521 DONE;
2522 })
2524 (define_expand "aarch64_uaddl2<mode>"
2525 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2526 (match_operand:VQW 1 "register_operand" "w")
2527 (match_operand:VQW 2 "register_operand" "w")]
2528 "TARGET_SIMD"
2529 {
2530 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2531 emit_insn (gen_aarch64_uaddl<mode>_hi_internal (operands[0], operands[1],
2532 operands[2], p));
2533 DONE;
2534 })
2536 (define_expand "aarch64_ssubl2<mode>"
2537 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2538 (match_operand:VQW 1 "register_operand" "w")
2539 (match_operand:VQW 2 "register_operand" "w")]
2540 "TARGET_SIMD"
2541 {
2542 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2543 emit_insn (gen_aarch64_ssubl<mode>_hi_internal (operands[0], operands[1],
2544 operands[2], p));
2545 DONE;
2546 })
2548 (define_expand "aarch64_usubl2<mode>"
2549 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2550 (match_operand:VQW 1 "register_operand" "w")
2551 (match_operand:VQW 2 "register_operand" "w")]
2552 "TARGET_SIMD"
2553 {
2554 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2555 emit_insn (gen_aarch64_usubl<mode>_hi_internal (operands[0], operands[1],
2556 operands[2], p));
2557 DONE;
2558 })
2560 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>l<mode>"
2561 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2562 (ADDSUB:<VWIDE> (ANY_EXTEND:<VWIDE>
2563 (match_operand:VDW 1 "register_operand" "w"))
2564 (ANY_EXTEND:<VWIDE>
2565 (match_operand:VDW 2 "register_operand" "w"))))]
2566 "TARGET_SIMD"
2567 "<ANY_EXTEND:su><ADDSUB:optab>l %0.<Vwtype>, %1.<Vtype>, %2.<Vtype>"
2568 [(set_attr "type" "neon_<ADDSUB:optab>_long")]
2569 )
2571 ;; <su><addsub>w<q>.
2573 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>w<mode>"
2574 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2575 (ADDSUB:<VWIDE> (match_operand:<VWIDE> 1 "register_operand" "w")
2576 (ANY_EXTEND:<VWIDE>
2577 (match_operand:VDW 2 "register_operand" "w"))))]
2578 "TARGET_SIMD"
2579 "<ANY_EXTEND:su><ADDSUB:optab>w\\t%0.<Vwtype>, %1.<Vwtype>, %2.<Vtype>"
2580 [(set_attr "type" "neon_<ADDSUB:optab>_widen")]
2581 )
2583 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>w2<mode>_internal"
2584 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2585 (ADDSUB:<VWIDE> (match_operand:<VWIDE> 1 "register_operand" "w")
2586 (ANY_EXTEND:<VWIDE>
2587 (vec_select:<VHALF>
2588 (match_operand:VQW 2 "register_operand" "w")
2589 (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))))]
2590 "TARGET_SIMD"
2591 "<ANY_EXTEND:su><ADDSUB:optab>w2\\t%0.<Vwtype>, %1.<Vwtype>, %2.<Vtype>"
2592 [(set_attr "type" "neon_<ADDSUB:optab>_widen")]
2593 )
2595 (define_expand "aarch64_saddw2<mode>"
2596 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2597 (match_operand:<VWIDE> 1 "register_operand" "w")
2598 (match_operand:VQW 2 "register_operand" "w")]
2599 "TARGET_SIMD"
2600 {
2601 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2602 emit_insn (gen_aarch64_saddw2<mode>_internal (operands[0], operands[1],
2603 operands[2], p));
2604 DONE;
2605 })
2607 (define_expand "aarch64_uaddw2<mode>"
2608 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2609 (match_operand:<VWIDE> 1 "register_operand" "w")
2610 (match_operand:VQW 2 "register_operand" "w")]
2611 "TARGET_SIMD"
2612 {
2613 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2614 emit_insn (gen_aarch64_uaddw2<mode>_internal (operands[0], operands[1],
2615 operands[2], p));
2616 DONE;
2617 })
2620 (define_expand "aarch64_ssubw2<mode>"
2621 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2622 (match_operand:<VWIDE> 1 "register_operand" "w")
2623 (match_operand:VQW 2 "register_operand" "w")]
2624 "TARGET_SIMD"
2625 {
2626 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2627 emit_insn (gen_aarch64_ssubw2<mode>_internal (operands[0], operands[1],
2628 operands[2], p));
2629 DONE;
2630 })
2632 (define_expand "aarch64_usubw2<mode>"
2633 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2634 (match_operand:<VWIDE> 1 "register_operand" "w")
2635 (match_operand:VQW 2 "register_operand" "w")]
2636 "TARGET_SIMD"
2637 {
2638 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2639 emit_insn (gen_aarch64_usubw2<mode>_internal (operands[0], operands[1],
2640 operands[2], p));
2641 DONE;
2642 })
2644 ;; <su><r>h<addsub>.
2646 (define_insn "aarch64_<sur>h<addsub><mode>"
2647 [(set (match_operand:VQ_S 0 "register_operand" "=w")
2648 (unspec:VQ_S [(match_operand:VQ_S 1 "register_operand" "w")
2649 (match_operand:VQ_S 2 "register_operand" "w")]
2650 HADDSUB))]
2651 "TARGET_SIMD"
2652 "<sur>h<addsub>\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
2653 [(set_attr "type" "neon_<addsub>_halve<q>")]
2654 )
2656 ;; <r><addsub>hn<q>.
2658 (define_insn "aarch64_<sur><addsub>hn<mode>"
2659 [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
2660 (unspec:<VNARROWQ> [(match_operand:VQN 1 "register_operand" "w")
2661 (match_operand:VQN 2 "register_operand" "w")]
2662 ADDSUBHN))]
2663 "TARGET_SIMD"
2664 "<sur><addsub>hn\\t%0.<Vntype>, %1.<Vtype>, %2.<Vtype>"
2665 [(set_attr "type" "neon_<addsub>_halve_narrow_q")]
2666 )
2668 (define_insn "aarch64_<sur><addsub>hn2<mode>"
2669 [(set (match_operand:<VNARROWQ2> 0 "register_operand" "=w")
2670 (unspec:<VNARROWQ2> [(match_operand:<VNARROWQ> 1 "register_operand" "0")
2671 (match_operand:VQN 2 "register_operand" "w")
2672 (match_operand:VQN 3 "register_operand" "w")]
2673 ADDSUBHN2))]
2674 "TARGET_SIMD"
2675 "<sur><addsub>hn2\\t%0.<V2ntype>, %2.<Vtype>, %3.<Vtype>"
2676 [(set_attr "type" "neon_<addsub>_halve_narrow_q")]
2677 )
2679 ;; pmul.
2681 (define_insn "aarch64_pmul<mode>"
2682 [(set (match_operand:VB 0 "register_operand" "=w")
2683 (unspec:VB [(match_operand:VB 1 "register_operand" "w")
2684 (match_operand:VB 2 "register_operand" "w")]
2685 UNSPEC_PMUL))]
2686 "TARGET_SIMD"
2687 "pmul\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
2688 [(set_attr "type" "neon_mul_<Vetype><q>")]
2689 )
2691 ;; <su>q<addsub>
2693 (define_insn "aarch64_<su_optab><optab><mode>"
2694 [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
2695 (BINQOPS:VSDQ_I (match_operand:VSDQ_I 1 "register_operand" "w")
2696 (match_operand:VSDQ_I 2 "register_operand" "w")))]
2697 "TARGET_SIMD"
2698 "<su_optab><optab>\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
2699 [(set_attr "type" "neon_<optab><q>")]
2700 )
2702 ;; suqadd and usqadd
2704 (define_insn "aarch64_<sur>qadd<mode>"
2705 [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
2706 (unspec:VSDQ_I [(match_operand:VSDQ_I 1 "register_operand" "0")
2707 (match_operand:VSDQ_I 2 "register_operand" "w")]
2708 USSUQADD))]
2709 "TARGET_SIMD"
2710 "<sur>qadd\\t%<v>0<Vmtype>, %<v>2<Vmtype>"
2711 [(set_attr "type" "neon_qadd<q>")]
2712 )
2714 ;; sqmovun
2716 (define_insn "aarch64_sqmovun<mode>"
2717 [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
2718 (unspec:<VNARROWQ> [(match_operand:VSQN_HSDI 1 "register_operand" "w")]
2719 UNSPEC_SQXTUN))]
2720 "TARGET_SIMD"
2721 "sqxtun\\t%<vn2>0<Vmntype>, %<v>1<Vmtype>"
2722 [(set_attr "type" "neon_sat_shift_imm_narrow_q")]
2723 )
2725 ;; sqmovn and uqmovn
2727 (define_insn "aarch64_<sur>qmovn<mode>"
2728 [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
2729 (unspec:<VNARROWQ> [(match_operand:VSQN_HSDI 1 "register_operand" "w")]
2730 SUQMOVN))]
2731 "TARGET_SIMD"
2732 "<sur>qxtn\\t%<vn2>0<Vmntype>, %<v>1<Vmtype>"
2733 [(set_attr "type" "neon_sat_shift_imm_narrow_q")]
2734 )
2736 ;; <su>q<absneg>
2738 (define_insn "aarch64_s<optab><mode>"
2739 [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
2740 (UNQOPS:VSDQ_I
2741 (match_operand:VSDQ_I 1 "register_operand" "w")))]
2742 "TARGET_SIMD"
2743 "s<optab>\\t%<v>0<Vmtype>, %<v>1<Vmtype>"
2744 [(set_attr "type" "neon_<optab><q>")]
2745 )
2747 ;; sq<r>dmulh.
2749 (define_insn "aarch64_sq<r>dmulh<mode>"
2750 [(set (match_operand:VSDQ_HSI 0 "register_operand" "=w")
2751 (unspec:VSDQ_HSI
2752 [(match_operand:VSDQ_HSI 1 "register_operand" "w")
2753 (match_operand:VSDQ_HSI 2 "register_operand" "w")]
2754 VQDMULH))]
2755 "TARGET_SIMD"
2756 "sq<r>dmulh\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
2757 [(set_attr "type" "neon_sat_mul_<Vetype><q>")]
2758 )
2760 ;; sq<r>dmulh_lane
2762 (define_expand "aarch64_sqdmulh_lane<mode>"
2763 [(match_operand:VDQHS 0 "register_operand" "")
2764 (match_operand:VDQHS 1 "register_operand" "")
2765 (match_operand:<VCOND> 2 "register_operand" "")
2766 (match_operand:SI 3 "immediate_operand" "")]
2767 "TARGET_SIMD"
2768 {
2769 aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCOND>mode));
2770 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
2771 emit_insn (gen_aarch64_sqdmulh_lane<mode>_internal (operands[0],
2772 operands[1],
2773 operands[2],
2774 operands[3]));
2775 DONE;
2776 }
2777 )
2779 (define_expand "aarch64_sqrdmulh_lane<mode>"
2780 [(match_operand:VDQHS 0 "register_operand" "")
2781 (match_operand:VDQHS 1 "register_operand" "")
2782 (match_operand:<VCOND> 2 "register_operand" "")
2783 (match_operand:SI 3 "immediate_operand" "")]
2784 "TARGET_SIMD"
2785 {
2786 aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCOND>mode));
2787 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
2788 emit_insn (gen_aarch64_sqrdmulh_lane<mode>_internal (operands[0],
2789 operands[1],
2790 operands[2],
2791 operands[3]));
2792 DONE;
2793 }
2794 )
2796 (define_insn "aarch64_sq<r>dmulh_lane<mode>_internal"
2797 [(set (match_operand:VDQHS 0 "register_operand" "=w")
2798 (unspec:VDQHS
2799 [(match_operand:VDQHS 1 "register_operand" "w")
2800 (vec_select:<VEL>
2801 (match_operand:<VCOND> 2 "register_operand" "<vwx>")
2802 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))]
2803 VQDMULH))]
2804 "TARGET_SIMD"
2805 "*
2806 aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCOND>mode));
2807 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
2808 return \"sq<r>dmulh\\t%0.<Vtype>, %1.<Vtype>, %2.<Vetype>[%3]\";"
2809 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar<q>")]
2810 )
2812 (define_expand "aarch64_sqdmulh_laneq<mode>"
2813 [(match_operand:VSDQ_HSI 0 "register_operand" "")
2814 (match_operand:VSDQ_HSI 1 "register_operand" "")
2815 (match_operand:<VCONQ> 2 "register_operand" "")
2816 (match_operand:SI 3 "immediate_operand" "")]
2817 "TARGET_SIMD"
2818 {
2819 aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCONQ>mode));
2820 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[3])));
2821 emit_insn (gen_aarch64_sqdmulh_laneq<mode>_internal (operands[0],
2822 operands[1],
2823 operands[2],
2824 operands[3]));
2825 DONE;
2826 }
2827 )
2829 (define_expand "aarch64_sqrdmulh_laneq<mode>"
2830 [(match_operand:VSDQ_HSI 0 "register_operand" "")
2831 (match_operand:VSDQ_HSI 1 "register_operand" "")
2832 (match_operand:<VCONQ> 2 "register_operand" "")
2833 (match_operand:SI 3 "immediate_operand" "")]
2834 "TARGET_SIMD"
2835 {
2836 aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCONQ>mode));
2837 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[3])));
2838 emit_insn (gen_aarch64_sqrdmulh_laneq<mode>_internal (operands[0],
2839 operands[1],
2840 operands[2],
2841 operands[3]));
2842 DONE;
2843 }
2844 )
2846 (define_insn "aarch64_sq<r>dmulh_laneq<mode>_internal"
2847 [(set (match_operand:VDQHS 0 "register_operand" "=w")
2848 (unspec:VDQHS
2849 [(match_operand:VDQHS 1 "register_operand" "w")
2850 (vec_select:<VEL>
2851 (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
2852 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))]
2853 VQDMULH))]
2854 "TARGET_SIMD"
2855 "*
2856 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[3])));
2857 return \"sq<r>dmulh\\t%0.<Vtype>, %1.<Vtype>, %2.<Vetype>[%3]\";"
2858 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar<q>")]
2859 )
2861 (define_expand "aarch64_sqdmulh_lane<mode>"
2862 [(match_operand:SD_HSI 0 "register_operand" "")
2863 (match_operand:SD_HSI 1 "register_operand" "")
2864 (match_operand:<VCOND> 2 "register_operand" "")
2865 (match_operand:SI 3 "immediate_operand" "")]
2866 "TARGET_SIMD"
2867 {
2868 aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCOND>mode));
2869 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
2870 emit_insn (gen_aarch64_sqdmulh_lane<mode>_internal (operands[0],
2871 operands[1],
2872 operands[2],
2873 operands[3]));
2874 DONE;
2875 }
2876 )
2878 (define_expand "aarch64_sqrdmulh_lane<mode>"
2879 [(match_operand:SD_HSI 0 "register_operand" "")
2880 (match_operand:SD_HSI 1 "register_operand" "")
2881 (match_operand:<VCOND> 2 "register_operand" "")
2882 (match_operand:SI 3 "immediate_operand" "")]
2883 "TARGET_SIMD"
2884 {
2885 aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCOND>mode));
2886 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
2887 emit_insn (gen_aarch64_sqrdmulh_lane<mode>_internal (operands[0],
2888 operands[1],
2889 operands[2],
2890 operands[3]));
2891 DONE;
2892 }
2893 )
2895 (define_insn "aarch64_sq<r>dmulh_lane<mode>_internal"
2896 [(set (match_operand:SD_HSI 0 "register_operand" "=w")
2897 (unspec:SD_HSI
2898 [(match_operand:SD_HSI 1 "register_operand" "w")
2899 (vec_select:<VEL>
2900 (match_operand:<VCOND> 2 "register_operand" "<vwx>")
2901 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))]
2902 VQDMULH))]
2903 "TARGET_SIMD"
2904 "*
2905 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
2906 return \"sq<r>dmulh\\t%<v>0, %<v>1, %2.<v>[%3]\";"
2907 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar<q>")]
2908 )
2910 (define_insn "aarch64_sq<r>dmulh_laneq<mode>_internal"
2911 [(set (match_operand:SD_HSI 0 "register_operand" "=w")
2912 (unspec:SD_HSI
2913 [(match_operand:SD_HSI 1 "register_operand" "w")
2914 (vec_select:<VEL>
2915 (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
2916 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))]
2917 VQDMULH))]
2918 "TARGET_SIMD"
2919 "*
2920 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[3])));
2921 return \"sq<r>dmulh\\t%<v>0, %<v>1, %2.<v>[%3]\";"
2922 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar<q>")]
2923 )
2925 ;; vqdml[sa]l
2927 (define_insn "aarch64_sqdml<SBINQOPS:as>l<mode>"
2928 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2929 (SBINQOPS:<VWIDE>
2930 (match_operand:<VWIDE> 1 "register_operand" "0")
2931 (ss_ashift:<VWIDE>
2932 (mult:<VWIDE>
2933 (sign_extend:<VWIDE>
2934 (match_operand:VSD_HSI 2 "register_operand" "w"))
2935 (sign_extend:<VWIDE>
2936 (match_operand:VSD_HSI 3 "register_operand" "w")))
2937 (const_int 1))))]
2938 "TARGET_SIMD"
2939 "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %<v>3<Vmtype>"
2940 [(set_attr "type" "neon_sat_mla_<Vetype>_long")]
2941 )
2943 ;; vqdml[sa]l_lane
2945 (define_insn "aarch64_sqdml<SBINQOPS:as>l_lane<mode>_internal"
2946 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2947 (SBINQOPS:<VWIDE>
2948 (match_operand:<VWIDE> 1 "register_operand" "0")
2949 (ss_ashift:<VWIDE>
2950 (mult:<VWIDE>
2951 (sign_extend:<VWIDE>
2952 (match_operand:VD_HSI 2 "register_operand" "w"))
2953 (sign_extend:<VWIDE>
2954 (vec_duplicate:VD_HSI
2955 (vec_select:<VEL>
2956 (match_operand:<VCOND> 3 "register_operand" "<vwx>")
2957 (parallel [(match_operand:SI 4 "immediate_operand" "i")])))
2958 ))
2959 (const_int 1))))]
2960 "TARGET_SIMD"
2961 {
2962 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[4])));
2963 return
2964 "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
2965 }
2966 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
2967 )
2969 (define_insn "aarch64_sqdml<SBINQOPS:as>l_laneq<mode>_internal"
2970 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2971 (SBINQOPS:<VWIDE>
2972 (match_operand:<VWIDE> 1 "register_operand" "0")
2973 (ss_ashift:<VWIDE>
2974 (mult:<VWIDE>
2975 (sign_extend:<VWIDE>
2976 (match_operand:VD_HSI 2 "register_operand" "w"))
2977 (sign_extend:<VWIDE>
2978 (vec_duplicate:VD_HSI
2979 (vec_select:<VEL>
2980 (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
2981 (parallel [(match_operand:SI 4 "immediate_operand" "i")])))
2982 ))
2983 (const_int 1))))]
2984 "TARGET_SIMD"
2985 {
2986 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[4])));
2987 return
2988 "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
2989 }
2990 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
2991 )
2993 (define_insn "aarch64_sqdml<SBINQOPS:as>l_lane<mode>_internal"
2994 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2995 (SBINQOPS:<VWIDE>
2996 (match_operand:<VWIDE> 1 "register_operand" "0")
2997 (ss_ashift:<VWIDE>
2998 (mult:<VWIDE>
2999 (sign_extend:<VWIDE>
3000 (match_operand:SD_HSI 2 "register_operand" "w"))
3001 (sign_extend:<VWIDE>
3002 (vec_select:<VEL>
3003 (match_operand:<VCOND> 3 "register_operand" "<vwx>")
3004 (parallel [(match_operand:SI 4 "immediate_operand" "i")])))
3005 )
3006 (const_int 1))))]
3007 "TARGET_SIMD"
3008 {
3009 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[4])));
3010 return
3011 "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
3012 }
3013 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3014 )
3016 (define_insn "aarch64_sqdml<SBINQOPS:as>l_laneq<mode>_internal"
3017 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3018 (SBINQOPS:<VWIDE>
3019 (match_operand:<VWIDE> 1 "register_operand" "0")
3020 (ss_ashift:<VWIDE>
3021 (mult:<VWIDE>
3022 (sign_extend:<VWIDE>
3023 (match_operand:SD_HSI 2 "register_operand" "w"))
3024 (sign_extend:<VWIDE>
3025 (vec_select:<VEL>
3026 (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
3027 (parallel [(match_operand:SI 4 "immediate_operand" "i")])))
3028 )
3029 (const_int 1))))]
3030 "TARGET_SIMD"
3031 {
3032 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[4])));
3033 return
3034 "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
3035 }
3036 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3037 )
3039 (define_expand "aarch64_sqdmlal_lane<mode>"
3040 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3041 (match_operand:<VWIDE> 1 "register_operand" "0")
3042 (match_operand:VSD_HSI 2 "register_operand" "w")
3043 (match_operand:<VCOND> 3 "register_operand" "<vwx>")
3044 (match_operand:SI 4 "immediate_operand" "i")]
3045 "TARGET_SIMD"
3046 {
3047 aarch64_simd_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<VCOND>mode));
3048 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[4])));
3049 emit_insn (gen_aarch64_sqdmlal_lane<mode>_internal (operands[0], operands[1],
3050 operands[2], operands[3],
3051 operands[4]));
3052 DONE;
3053 })
3055 (define_expand "aarch64_sqdmlal_laneq<mode>"
3056 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3057 (match_operand:<VWIDE> 1 "register_operand" "0")
3058 (match_operand:VSD_HSI 2 "register_operand" "w")
3059 (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
3060 (match_operand:SI 4 "immediate_operand" "i")]
3061 "TARGET_SIMD"
3062 {
3063 aarch64_simd_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<VCONQ>mode));
3064 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[4])));
3065 emit_insn (gen_aarch64_sqdmlal_laneq<mode>_internal (operands[0], operands[1],
3066 operands[2], operands[3],
3067 operands[4]));
3068 DONE;
3069 })
3071 (define_expand "aarch64_sqdmlsl_lane<mode>"
3072 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3073 (match_operand:<VWIDE> 1 "register_operand" "0")
3074 (match_operand:VSD_HSI 2 "register_operand" "w")
3075 (match_operand:<VCOND> 3 "register_operand" "<vwx>")
3076 (match_operand:SI 4 "immediate_operand" "i")]
3077 "TARGET_SIMD"
3078 {
3079 aarch64_simd_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<VCOND>mode));
3080 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[4])));
3081 emit_insn (gen_aarch64_sqdmlsl_lane<mode>_internal (operands[0], operands[1],
3082 operands[2], operands[3],
3083 operands[4]));
3084 DONE;
3085 })
3087 (define_expand "aarch64_sqdmlsl_laneq<mode>"
3088 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3089 (match_operand:<VWIDE> 1 "register_operand" "0")
3090 (match_operand:VSD_HSI 2 "register_operand" "w")
3091 (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
3092 (match_operand:SI 4 "immediate_operand" "i")]
3093 "TARGET_SIMD"
3094 {
3095 aarch64_simd_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<VCONQ>mode));
3096 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[4])));
3097 emit_insn (gen_aarch64_sqdmlsl_laneq<mode>_internal (operands[0], operands[1],
3098 operands[2], operands[3],
3099 operands[4]));
3100 DONE;
3101 })
3103 ;; vqdml[sa]l_n
3105 (define_insn "aarch64_sqdml<SBINQOPS:as>l_n<mode>"
3106 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3107 (SBINQOPS:<VWIDE>
3108 (match_operand:<VWIDE> 1 "register_operand" "0")
3109 (ss_ashift:<VWIDE>
3110 (mult:<VWIDE>
3111 (sign_extend:<VWIDE>
3112 (match_operand:VD_HSI 2 "register_operand" "w"))
3113 (sign_extend:<VWIDE>
3114 (vec_duplicate:VD_HSI
3115 (match_operand:<VEL> 3 "register_operand" "<vwx>"))))
3116 (const_int 1))))]
3117 "TARGET_SIMD"
3118 "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[0]"
3119 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3120 )
3122 ;; sqdml[as]l2
3124 (define_insn "aarch64_sqdml<SBINQOPS:as>l2<mode>_internal"
3125 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3126 (SBINQOPS:<VWIDE>
3127 (match_operand:<VWIDE> 1 "register_operand" "0")
3128 (ss_ashift:<VWIDE>
3129 (mult:<VWIDE>
3130 (sign_extend:<VWIDE>
3131 (vec_select:<VHALF>
3132 (match_operand:VQ_HSI 2 "register_operand" "w")
3133 (match_operand:VQ_HSI 4 "vect_par_cnst_hi_half" "")))
3134 (sign_extend:<VWIDE>
3135 (vec_select:<VHALF>
3136 (match_operand:VQ_HSI 3 "register_operand" "w")
3137 (match_dup 4))))
3138 (const_int 1))))]
3139 "TARGET_SIMD"
3140 "sqdml<SBINQOPS:as>l2\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %<v>3<Vmtype>"
3141 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3142 )
3144 (define_expand "aarch64_sqdmlal2<mode>"
3145 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3146 (match_operand:<VWIDE> 1 "register_operand" "w")
3147 (match_operand:VQ_HSI 2 "register_operand" "w")
3148 (match_operand:VQ_HSI 3 "register_operand" "w")]
3149 "TARGET_SIMD"
3150 {
3151 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3152 emit_insn (gen_aarch64_sqdmlal2<mode>_internal (operands[0], operands[1],
3153 operands[2], operands[3], p));
3154 DONE;
3155 })
3157 (define_expand "aarch64_sqdmlsl2<mode>"
3158 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3159 (match_operand:<VWIDE> 1 "register_operand" "w")
3160 (match_operand:VQ_HSI 2 "register_operand" "w")
3161 (match_operand:VQ_HSI 3 "register_operand" "w")]
3162 "TARGET_SIMD"
3163 {
3164 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3165 emit_insn (gen_aarch64_sqdmlsl2<mode>_internal (operands[0], operands[1],
3166 operands[2], operands[3], p));
3167 DONE;
3168 })
3170 ;; vqdml[sa]l2_lane
3172 (define_insn "aarch64_sqdml<SBINQOPS:as>l2_lane<mode>_internal"
3173 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3174 (SBINQOPS:<VWIDE>
3175 (match_operand:<VWIDE> 1 "register_operand" "0")
3176 (ss_ashift:<VWIDE>
3177 (mult:<VWIDE>
3178 (sign_extend:<VWIDE>
3179 (vec_select:<VHALF>
3180 (match_operand:VQ_HSI 2 "register_operand" "w")
3181 (match_operand:VQ_HSI 5 "vect_par_cnst_hi_half" "")))
3182 (sign_extend:<VWIDE>
3183 (vec_duplicate:<VHALF>
3184 (vec_select:<VEL>
3185 (match_operand:<VCOND> 3 "register_operand" "<vwx>")
3186 (parallel [(match_operand:SI 4 "immediate_operand" "i")])
3187 ))))
3188 (const_int 1))))]
3189 "TARGET_SIMD"
3190 {
3191 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[4])));
3192 return
3193 "sqdml<SBINQOPS:as>l2\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
3194 }
3195 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3196 )
3198 (define_insn "aarch64_sqdml<SBINQOPS:as>l2_laneq<mode>_internal"
3199 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3200 (SBINQOPS:<VWIDE>
3201 (match_operand:<VWIDE> 1 "register_operand" "0")
3202 (ss_ashift:<VWIDE>
3203 (mult:<VWIDE>
3204 (sign_extend:<VWIDE>
3205 (vec_select:<VHALF>
3206 (match_operand:VQ_HSI 2 "register_operand" "w")
3207 (match_operand:VQ_HSI 5 "vect_par_cnst_hi_half" "")))
3208 (sign_extend:<VWIDE>
3209 (vec_duplicate:<VHALF>
3210 (vec_select:<VEL>
3211 (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
3212 (parallel [(match_operand:SI 4 "immediate_operand" "i")])
3213 ))))
3214 (const_int 1))))]
3215 "TARGET_SIMD"
3216 {
3217 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[4])));
3218 return
3219 "sqdml<SBINQOPS:as>l2\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
3220 }
3221 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3222 )
3224 (define_expand "aarch64_sqdmlal2_lane<mode>"
3225 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3226 (match_operand:<VWIDE> 1 "register_operand" "w")
3227 (match_operand:VQ_HSI 2 "register_operand" "w")
3228 (match_operand:<VCOND> 3 "register_operand" "<vwx>")
3229 (match_operand:SI 4 "immediate_operand" "i")]
3230 "TARGET_SIMD"
3231 {
3232 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3233 aarch64_simd_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<VCOND>mode));
3234 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[4])));
3235 emit_insn (gen_aarch64_sqdmlal2_lane<mode>_internal (operands[0], operands[1],
3236 operands[2], operands[3],
3237 operands[4], p));
3238 DONE;
3239 })
3241 (define_expand "aarch64_sqdmlal2_laneq<mode>"
3242 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3243 (match_operand:<VWIDE> 1 "register_operand" "w")
3244 (match_operand:VQ_HSI 2 "register_operand" "w")
3245 (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
3246 (match_operand:SI 4 "immediate_operand" "i")]
3247 "TARGET_SIMD"
3248 {
3249 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3250 aarch64_simd_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<VCONQ>mode));
3251 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[4])));
3252 emit_insn (gen_aarch64_sqdmlal2_laneq<mode>_internal (operands[0], operands[1],
3253 operands[2], operands[3],
3254 operands[4], p));
3255 DONE;
3256 })
3258 (define_expand "aarch64_sqdmlsl2_lane<mode>"
3259 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3260 (match_operand:<VWIDE> 1 "register_operand" "w")
3261 (match_operand:VQ_HSI 2 "register_operand" "w")
3262 (match_operand:<VCOND> 3 "register_operand" "<vwx>")
3263 (match_operand:SI 4 "immediate_operand" "i")]
3264 "TARGET_SIMD"
3265 {
3266 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3267 aarch64_simd_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<VCOND>mode));
3268 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[4])));
3269 emit_insn (gen_aarch64_sqdmlsl2_lane<mode>_internal (operands[0], operands[1],
3270 operands[2], operands[3],
3271 operands[4], p));
3272 DONE;
3273 })
3275 (define_expand "aarch64_sqdmlsl2_laneq<mode>"
3276 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3277 (match_operand:<VWIDE> 1 "register_operand" "w")
3278 (match_operand:VQ_HSI 2 "register_operand" "w")
3279 (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
3280 (match_operand:SI 4 "immediate_operand" "i")]
3281 "TARGET_SIMD"
3282 {
3283 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3284 aarch64_simd_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<VCONQ>mode));
3285 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[4])));
3286 emit_insn (gen_aarch64_sqdmlsl2_laneq<mode>_internal (operands[0], operands[1],
3287 operands[2], operands[3],
3288 operands[4], p));
3289 DONE;
3290 })
3292 (define_insn "aarch64_sqdml<SBINQOPS:as>l2_n<mode>_internal"
3293 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3294 (SBINQOPS:<VWIDE>
3295 (match_operand:<VWIDE> 1 "register_operand" "0")
3296 (ss_ashift:<VWIDE>
3297 (mult:<VWIDE>
3298 (sign_extend:<VWIDE>
3299 (vec_select:<VHALF>
3300 (match_operand:VQ_HSI 2 "register_operand" "w")
3301 (match_operand:VQ_HSI 4 "vect_par_cnst_hi_half" "")))
3302 (sign_extend:<VWIDE>
3303 (vec_duplicate:<VHALF>
3304 (match_operand:<VEL> 3 "register_operand" "<vwx>"))))
3305 (const_int 1))))]
3306 "TARGET_SIMD"
3307 "sqdml<SBINQOPS:as>l2\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[0]"
3308 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3309 )
3311 (define_expand "aarch64_sqdmlal2_n<mode>"
3312 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3313 (match_operand:<VWIDE> 1 "register_operand" "w")
3314 (match_operand:VQ_HSI 2 "register_operand" "w")
3315 (match_operand:<VEL> 3 "register_operand" "w")]
3316 "TARGET_SIMD"
3317 {
3318 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3319 emit_insn (gen_aarch64_sqdmlal2_n<mode>_internal (operands[0], operands[1],
3320 operands[2], operands[3],
3321 p));
3322 DONE;
3323 })
3325 (define_expand "aarch64_sqdmlsl2_n<mode>"
3326 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3327 (match_operand:<VWIDE> 1 "register_operand" "w")
3328 (match_operand:VQ_HSI 2 "register_operand" "w")
3329 (match_operand:<VEL> 3 "register_operand" "w")]
3330 "TARGET_SIMD"
3331 {
3332 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3333 emit_insn (gen_aarch64_sqdmlsl2_n<mode>_internal (operands[0], operands[1],
3334 operands[2], operands[3],
3335 p));
3336 DONE;
3337 })
3339 ;; vqdmull
3341 (define_insn "aarch64_sqdmull<mode>"
3342 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3343 (ss_ashift:<VWIDE>
3344 (mult:<VWIDE>
3345 (sign_extend:<VWIDE>
3346 (match_operand:VSD_HSI 1 "register_operand" "w"))
3347 (sign_extend:<VWIDE>
3348 (match_operand:VSD_HSI 2 "register_operand" "w")))
3349 (const_int 1)))]
3350 "TARGET_SIMD"
3351 "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
3352 [(set_attr "type" "neon_sat_mul_<Vetype>_long")]
3353 )
3355 ;; vqdmull_lane
3357 (define_insn "aarch64_sqdmull_lane<mode>_internal"
3358 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3359 (ss_ashift:<VWIDE>
3360 (mult:<VWIDE>
3361 (sign_extend:<VWIDE>
3362 (match_operand:VD_HSI 1 "register_operand" "w"))
3363 (sign_extend:<VWIDE>
3364 (vec_duplicate:VD_HSI
3365 (vec_select:<VEL>
3366 (match_operand:<VCOND> 2 "register_operand" "<vwx>")
3367 (parallel [(match_operand:SI 3 "immediate_operand" "i")])))
3368 ))
3369 (const_int 1)))]
3370 "TARGET_SIMD"
3371 {
3372 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
3373 return "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3374 }
3375 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3376 )
3378 (define_insn "aarch64_sqdmull_laneq<mode>_internal"
3379 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3380 (ss_ashift:<VWIDE>
3381 (mult:<VWIDE>
3382 (sign_extend:<VWIDE>
3383 (match_operand:VD_HSI 1 "register_operand" "w"))
3384 (sign_extend:<VWIDE>
3385 (vec_duplicate:VD_HSI
3386 (vec_select:<VEL>
3387 (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
3388 (parallel [(match_operand:SI 3 "immediate_operand" "i")])))
3389 ))
3390 (const_int 1)))]
3391 "TARGET_SIMD"
3392 {
3393 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[3])));
3394 return "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3395 }
3396 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3397 )
3399 (define_insn "aarch64_sqdmull_lane<mode>_internal"
3400 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3401 (ss_ashift:<VWIDE>
3402 (mult:<VWIDE>
3403 (sign_extend:<VWIDE>
3404 (match_operand:SD_HSI 1 "register_operand" "w"))
3405 (sign_extend:<VWIDE>
3406 (vec_select:<VEL>
3407 (match_operand:<VCOND> 2 "register_operand" "<vwx>")
3408 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))
3409 ))
3410 (const_int 1)))]
3411 "TARGET_SIMD"
3412 {
3413 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
3414 return "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3415 }
3416 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3417 )
3419 (define_insn "aarch64_sqdmull_laneq<mode>_internal"
3420 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3421 (ss_ashift:<VWIDE>
3422 (mult:<VWIDE>
3423 (sign_extend:<VWIDE>
3424 (match_operand:SD_HSI 1 "register_operand" "w"))
3425 (sign_extend:<VWIDE>
3426 (vec_select:<VEL>
3427 (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
3428 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))
3429 ))
3430 (const_int 1)))]
3431 "TARGET_SIMD"
3432 {
3433 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[3])));
3434 return "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3435 }
3436 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3437 )
3439 (define_expand "aarch64_sqdmull_lane<mode>"
3440 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3441 (match_operand:VSD_HSI 1 "register_operand" "w")
3442 (match_operand:<VCOND> 2 "register_operand" "<vwx>")
3443 (match_operand:SI 3 "immediate_operand" "i")]
3444 "TARGET_SIMD"
3445 {
3446 aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCOND>mode));
3447 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
3448 emit_insn (gen_aarch64_sqdmull_lane<mode>_internal (operands[0], operands[1],
3449 operands[2], operands[3]));
3450 DONE;
3451 })
3453 (define_expand "aarch64_sqdmull_laneq<mode>"
3454 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3455 (match_operand:VSD_HSI 1 "register_operand" "w")
3456 (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
3457 (match_operand:SI 3 "immediate_operand" "i")]
3458 "TARGET_SIMD"
3459 {
3460 aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCONQ>mode));
3461 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[3])));
3462 emit_insn (gen_aarch64_sqdmull_laneq<mode>_internal
3463 (operands[0], operands[1], operands[2], operands[3]));
3464 DONE;
3465 })
3467 ;; vqdmull_n
3469 (define_insn "aarch64_sqdmull_n<mode>"
3470 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3471 (ss_ashift:<VWIDE>
3472 (mult:<VWIDE>
3473 (sign_extend:<VWIDE>
3474 (match_operand:VD_HSI 1 "register_operand" "w"))
3475 (sign_extend:<VWIDE>
3476 (vec_duplicate:VD_HSI
3477 (match_operand:<VEL> 2 "register_operand" "<vwx>")))
3478 )
3479 (const_int 1)))]
3480 "TARGET_SIMD"
3481 "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[0]"
3482 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3483 )
3485 ;; vqdmull2
3489 (define_insn "aarch64_sqdmull2<mode>_internal"
3490 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3491 (ss_ashift:<VWIDE>
3492 (mult:<VWIDE>
3493 (sign_extend:<VWIDE>
3494 (vec_select:<VHALF>
3495 (match_operand:VQ_HSI 1 "register_operand" "w")
3496 (match_operand:VQ_HSI 3 "vect_par_cnst_hi_half" "")))
3497 (sign_extend:<VWIDE>
3498 (vec_select:<VHALF>
3499 (match_operand:VQ_HSI 2 "register_operand" "w")
3500 (match_dup 3)))
3501 )
3502 (const_int 1)))]
3503 "TARGET_SIMD"
3504 "sqdmull2\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
3505 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3506 )
3508 (define_expand "aarch64_sqdmull2<mode>"
3509 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3510 (match_operand:VQ_HSI 1 "register_operand" "w")
3511 (match_operand:VQ_HSI 2 "register_operand" "w")]
3512 "TARGET_SIMD"
3513 {
3514 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3515 emit_insn (gen_aarch64_sqdmull2<mode>_internal (operands[0], operands[1],
3516 operands[2], p));
3517 DONE;
3518 })
3520 ;; vqdmull2_lane
3522 (define_insn "aarch64_sqdmull2_lane<mode>_internal"
3523 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3524 (ss_ashift:<VWIDE>
3525 (mult:<VWIDE>
3526 (sign_extend:<VWIDE>
3527 (vec_select:<VHALF>
3528 (match_operand:VQ_HSI 1 "register_operand" "w")
3529 (match_operand:VQ_HSI 4 "vect_par_cnst_hi_half" "")))
3530 (sign_extend:<VWIDE>
3531 (vec_duplicate:<VHALF>
3532 (vec_select:<VEL>
3533 (match_operand:<VCOND> 2 "register_operand" "<vwx>")
3534 (parallel [(match_operand:SI 3 "immediate_operand" "i")])))
3535 ))
3536 (const_int 1)))]
3537 "TARGET_SIMD"
3538 {
3539 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
3540 return "sqdmull2\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3541 }
3542 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3543 )
3545 (define_insn "aarch64_sqdmull2_laneq<mode>_internal"
3546 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3547 (ss_ashift:<VWIDE>
3548 (mult:<VWIDE>
3549 (sign_extend:<VWIDE>
3550 (vec_select:<VHALF>
3551 (match_operand:VQ_HSI 1 "register_operand" "w")
3552 (match_operand:VQ_HSI 4 "vect_par_cnst_hi_half" "")))
3553 (sign_extend:<VWIDE>
3554 (vec_duplicate:<VHALF>
3555 (vec_select:<VEL>
3556 (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
3557 (parallel [(match_operand:SI 3 "immediate_operand" "i")])))
3558 ))
3559 (const_int 1)))]
3560 "TARGET_SIMD"
3561 {
3562 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[3])));
3563 return "sqdmull2\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3564 }
3565 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3566 )
3568 (define_expand "aarch64_sqdmull2_lane<mode>"
3569 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3570 (match_operand:VQ_HSI 1 "register_operand" "w")
3571 (match_operand:<VCOND> 2 "register_operand" "<vwx>")
3572 (match_operand:SI 3 "immediate_operand" "i")]
3573 "TARGET_SIMD"
3574 {
3575 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3576 aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCOND>mode));
3577 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
3578 emit_insn (gen_aarch64_sqdmull2_lane<mode>_internal (operands[0], operands[1],
3579 operands[2], operands[3],
3580 p));
3581 DONE;
3582 })
3584 (define_expand "aarch64_sqdmull2_laneq<mode>"
3585 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3586 (match_operand:VQ_HSI 1 "register_operand" "w")
3587 (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
3588 (match_operand:SI 3 "immediate_operand" "i")]
3589 "TARGET_SIMD"
3590 {
3591 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3592 aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCONQ>mode));
3593 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[3])));
3594 emit_insn (gen_aarch64_sqdmull2_laneq<mode>_internal (operands[0], operands[1],
3595 operands[2], operands[3],
3596 p));
3597 DONE;
3598 })
3600 ;; vqdmull2_n
3602 (define_insn "aarch64_sqdmull2_n<mode>_internal"
3603 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3604 (ss_ashift:<VWIDE>
3605 (mult:<VWIDE>
3606 (sign_extend:<VWIDE>
3607 (vec_select:<VHALF>
3608 (match_operand:VQ_HSI 1 "register_operand" "w")
3609 (match_operand:VQ_HSI 3 "vect_par_cnst_hi_half" "")))
3610 (sign_extend:<VWIDE>
3611 (vec_duplicate:<VHALF>
3612 (match_operand:<VEL> 2 "register_operand" "<vwx>")))
3613 )
3614 (const_int 1)))]
3615 "TARGET_SIMD"
3616 "sqdmull2\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[0]"
3617 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3618 )
3620 (define_expand "aarch64_sqdmull2_n<mode>"
3621 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3622 (match_operand:VQ_HSI 1 "register_operand" "w")
3623 (match_operand:<VEL> 2 "register_operand" "w")]
3624 "TARGET_SIMD"
3625 {
3626 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3627 emit_insn (gen_aarch64_sqdmull2_n<mode>_internal (operands[0], operands[1],
3628 operands[2], p));
3629 DONE;
3630 })
3632 ;; vshl
3634 (define_insn "aarch64_<sur>shl<mode>"
3635 [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
3636 (unspec:VSDQ_I_DI
3637 [(match_operand:VSDQ_I_DI 1 "register_operand" "w")
3638 (match_operand:VSDQ_I_DI 2 "register_operand" "w")]
3639 VSHL))]
3640 "TARGET_SIMD"
3641 "<sur>shl\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>";
3642 [(set_attr "type" "neon_shift_reg<q>")]
3643 )
3646 ;; vqshl
3648 (define_insn "aarch64_<sur>q<r>shl<mode>"
3649 [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
3650 (unspec:VSDQ_I
3651 [(match_operand:VSDQ_I 1 "register_operand" "w")
3652 (match_operand:VSDQ_I 2 "register_operand" "w")]
3653 VQSHL))]
3654 "TARGET_SIMD"
3655 "<sur>q<r>shl\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>";
3656 [(set_attr "type" "neon_sat_shift_reg<q>")]
3657 )
3659 ;; vshll_n
3661 (define_insn "aarch64_<sur>shll_n<mode>"
3662 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3663 (unspec:<VWIDE> [(match_operand:VDW 1 "register_operand" "w")
3664 (match_operand:SI 2
3665 "aarch64_simd_shift_imm_bitsize_<ve_mode>" "i")]
3666 VSHLL))]
3667 "TARGET_SIMD"
3668 "*
3669 int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
3670 if (INTVAL (operands[2]) == bit_width)
3671 {
3672 return \"shll\\t%0.<Vwtype>, %1.<Vtype>, %2\";
3673 }
3674 else {
3675 return \"<sur>shll\\t%0.<Vwtype>, %1.<Vtype>, %2\";
3676 }"
3677 [(set_attr "type" "neon_shift_imm_long")]
3678 )
3680 ;; vshll_high_n
3682 (define_insn "aarch64_<sur>shll2_n<mode>"
3683 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3684 (unspec:<VWIDE> [(match_operand:VQW 1 "register_operand" "w")
3685 (match_operand:SI 2 "immediate_operand" "i")]
3686 VSHLL))]
3687 "TARGET_SIMD"
3688 "*
3689 int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
3690 if (INTVAL (operands[2]) == bit_width)
3691 {
3692 return \"shll2\\t%0.<Vwtype>, %1.<Vtype>, %2\";
3693 }
3694 else {
3695 return \"<sur>shll2\\t%0.<Vwtype>, %1.<Vtype>, %2\";
3696 }"
3697 [(set_attr "type" "neon_shift_imm_long")]
3698 )
3700 ;; vrshr_n
3702 (define_insn "aarch64_<sur>shr_n<mode>"
3703 [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
3704 (unspec:VSDQ_I_DI [(match_operand:VSDQ_I_DI 1 "register_operand" "w")
3705 (match_operand:SI 2
3706 "aarch64_simd_shift_imm_offset_<ve_mode>" "i")]
3707 VRSHR_N))]
3708 "TARGET_SIMD"
3709 "<sur>shr\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %2"
3710 [(set_attr "type" "neon_sat_shift_imm<q>")]
3711 )
3713 ;; v(r)sra_n
3715 (define_insn "aarch64_<sur>sra_n<mode>"
3716 [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
3717 (unspec:VSDQ_I_DI [(match_operand:VSDQ_I_DI 1 "register_operand" "0")
3718 (match_operand:VSDQ_I_DI 2 "register_operand" "w")
3719 (match_operand:SI 3
3720 "aarch64_simd_shift_imm_offset_<ve_mode>" "i")]
3721 VSRA))]
3722 "TARGET_SIMD"
3723 "<sur>sra\\t%<v>0<Vmtype>, %<v>2<Vmtype>, %3"
3724 [(set_attr "type" "neon_shift_acc<q>")]
3725 )
3727 ;; vs<lr>i_n
3729 (define_insn "aarch64_<sur>s<lr>i_n<mode>"
3730 [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
3731 (unspec:VSDQ_I_DI [(match_operand:VSDQ_I_DI 1 "register_operand" "0")
3732 (match_operand:VSDQ_I_DI 2 "register_operand" "w")
3733 (match_operand:SI 3
3734 "aarch64_simd_shift_imm_<offsetlr><ve_mode>" "i")]
3735 VSLRI))]
3736 "TARGET_SIMD"
3737 "s<lr>i\\t%<v>0<Vmtype>, %<v>2<Vmtype>, %3"
3738 [(set_attr "type" "neon_shift_imm<q>")]
3739 )
3741 ;; vqshl(u)
3743 (define_insn "aarch64_<sur>qshl<u>_n<mode>"
3744 [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
3745 (unspec:VSDQ_I [(match_operand:VSDQ_I 1 "register_operand" "w")
3746 (match_operand:SI 2
3747 "aarch64_simd_shift_imm_<ve_mode>" "i")]
3748 VQSHL_N))]
3749 "TARGET_SIMD"
3750 "<sur>qshl<u>\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %2"
3751 [(set_attr "type" "neon_sat_shift_imm<q>")]
3752 )
3755 ;; vq(r)shr(u)n_n
3757 (define_insn "aarch64_<sur>q<r>shr<u>n_n<mode>"
3758 [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
3759 (unspec:<VNARROWQ> [(match_operand:VSQN_HSDI 1 "register_operand" "w")
3760 (match_operand:SI 2
3761 "aarch64_simd_shift_imm_offset_<ve_mode>" "i")]
3762 VQSHRN_N))]
3763 "TARGET_SIMD"
3764 "<sur>q<r>shr<u>n\\t%<vn2>0<Vmntype>, %<v>1<Vmtype>, %2"
3765 [(set_attr "type" "neon_sat_shift_imm_narrow_q")]
3766 )
3769 ;; cm(eq|ge|gt|lt|le)
3770 ;; Note, we have constraints for Dz and Z as different expanders
3771 ;; have different ideas of what should be passed to this pattern.
3773 (define_insn "aarch64_cm<optab><mode>"
3774 [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w,w")
3775 (neg:<V_cmp_result>
3776 (COMPARISONS:<V_cmp_result>
3777 (match_operand:VDQ 1 "register_operand" "w,w")
3778 (match_operand:VDQ 2 "aarch64_simd_reg_or_zero" "w,ZDz")
3779 )))]
3780 "TARGET_SIMD"
3781 "@
3782 cm<n_optab>\t%<v>0<Vmtype>, %<v><cmp_1><Vmtype>, %<v><cmp_2><Vmtype>
3783 cm<optab>\t%<v>0<Vmtype>, %<v>1<Vmtype>, #0"
3784 [(set_attr "type" "neon_compare<q>, neon_compare_zero<q>")]
3785 )
3787 (define_insn_and_split "aarch64_cm<optab>di"
3788 [(set (match_operand:DI 0 "register_operand" "=w,w,r")
3789 (neg:DI
3790 (COMPARISONS:DI
3791 (match_operand:DI 1 "register_operand" "w,w,r")
3792 (match_operand:DI 2 "aarch64_simd_reg_or_zero" "w,ZDz,r")
3793 )))
3794 (clobber (reg:CC CC_REGNUM))]
3795 "TARGET_SIMD"
3796 "#"
3797 "reload_completed"
3798 [(set (match_operand:DI 0 "register_operand")
3799 (neg:DI
3800 (COMPARISONS:DI
3801 (match_operand:DI 1 "register_operand")
3802 (match_operand:DI 2 "aarch64_simd_reg_or_zero")
3803 )))]
3804 {
3805 /* If we are in the general purpose register file,
3806 we split to a sequence of comparison and store. */
3807 if (GP_REGNUM_P (REGNO (operands[0]))
3808 && GP_REGNUM_P (REGNO (operands[1])))
3809 {
3810 machine_mode mode = SELECT_CC_MODE (<CMP>, operands[1], operands[2]);
3811 rtx cc_reg = aarch64_gen_compare_reg (<CMP>, operands[1], operands[2]);
3812 rtx comparison = gen_rtx_<CMP> (mode, operands[1], operands[2]);
3813 emit_insn (gen_cstoredi_neg (operands[0], comparison, cc_reg));
3814 DONE;
3815 }
3816 /* Otherwise, we expand to a similar pattern which does not
3817 clobber CC_REGNUM. */
3818 }
3819 [(set_attr "type" "neon_compare, neon_compare_zero, multiple")]
3820 )
3822 (define_insn "*aarch64_cm<optab>di"
3823 [(set (match_operand:DI 0 "register_operand" "=w,w")
3824 (neg:DI
3825 (COMPARISONS:DI
3826 (match_operand:DI 1 "register_operand" "w,w")
3827 (match_operand:DI 2 "aarch64_simd_reg_or_zero" "w,ZDz")
3828 )))]
3829 "TARGET_SIMD && reload_completed"
3830 "@
3831 cm<n_optab>\t%d0, %d<cmp_1>, %d<cmp_2>
3832 cm<optab>\t%d0, %d1, #0"
3833 [(set_attr "type" "neon_compare, neon_compare_zero")]
3834 )
3836 ;; cm(hs|hi)
3838 (define_insn "aarch64_cm<optab><mode>"
3839 [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w")
3840 (neg:<V_cmp_result>
3841 (UCOMPARISONS:<V_cmp_result>
3842 (match_operand:VDQ 1 "register_operand" "w")
3843 (match_operand:VDQ 2 "register_operand" "w")
3844 )))]
3845 "TARGET_SIMD"
3846 "cm<n_optab>\t%<v>0<Vmtype>, %<v><cmp_1><Vmtype>, %<v><cmp_2><Vmtype>"
3847 [(set_attr "type" "neon_compare<q>")]
3848 )
3850 (define_insn_and_split "aarch64_cm<optab>di"
3851 [(set (match_operand:DI 0 "register_operand" "=w,r")
3852 (neg:DI
3853 (UCOMPARISONS:DI
3854 (match_operand:DI 1 "register_operand" "w,r")
3855 (match_operand:DI 2 "aarch64_simd_reg_or_zero" "w,r")
3856 )))
3857 (clobber (reg:CC CC_REGNUM))]
3858 "TARGET_SIMD"
3859 "#"
3860 "reload_completed"
3861 [(set (match_operand:DI 0 "register_operand")
3862 (neg:DI
3863 (UCOMPARISONS:DI
3864 (match_operand:DI 1 "register_operand")
3865 (match_operand:DI 2 "aarch64_simd_reg_or_zero")
3866 )))]
3867 {
3868 /* If we are in the general purpose register file,
3869 we split to a sequence of comparison and store. */
3870 if (GP_REGNUM_P (REGNO (operands[0]))
3871 && GP_REGNUM_P (REGNO (operands[1])))
3872 {
3873 machine_mode mode = CCmode;
3874 rtx cc_reg = aarch64_gen_compare_reg (<CMP>, operands[1], operands[2]);
3875 rtx comparison = gen_rtx_<CMP> (mode, operands[1], operands[2]);
3876 emit_insn (gen_cstoredi_neg (operands[0], comparison, cc_reg));
3877 DONE;
3878 }
3879 /* Otherwise, we expand to a similar pattern which does not
3880 clobber CC_REGNUM. */
3881 }
3882 [(set_attr "type" "neon_compare,multiple")]
3883 )
3885 (define_insn "*aarch64_cm<optab>di"
3886 [(set (match_operand:DI 0 "register_operand" "=w")
3887 (neg:DI
3888 (UCOMPARISONS:DI
3889 (match_operand:DI 1 "register_operand" "w")
3890 (match_operand:DI 2 "aarch64_simd_reg_or_zero" "w")
3891 )))]
3892 "TARGET_SIMD && reload_completed"
3893 "cm<n_optab>\t%d0, %d<cmp_1>, %d<cmp_2>"
3894 [(set_attr "type" "neon_compare")]
3895 )
3897 ;; cmtst
3899 ;; Although neg (ne (and x y) 0) is the natural way of expressing a cmtst,
3900 ;; we don't have any insns using ne, and aarch64_vcond_internal outputs
3901 ;; not (neg (eq (and x y) 0))
3902 ;; which is rewritten by simplify_rtx as
3903 ;; plus (eq (and x y) 0) -1.
3905 (define_insn "aarch64_cmtst<mode>"
3906 [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w")
3907 (plus:<V_cmp_result>
3908 (eq:<V_cmp_result>
3909 (and:VDQ
3910 (match_operand:VDQ 1 "register_operand" "w")
3911 (match_operand:VDQ 2 "register_operand" "w"))
3912 (match_operand:VDQ 3 "aarch64_simd_imm_zero"))
3913 (match_operand:<V_cmp_result> 4 "aarch64_simd_imm_minus_one")))
3914 ]
3915 "TARGET_SIMD"
3916 "cmtst\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
3917 [(set_attr "type" "neon_tst<q>")]
3918 )
3920 (define_insn_and_split "aarch64_cmtstdi"
3921 [(set (match_operand:DI 0 "register_operand" "=w,r")
3922 (neg:DI
3923 (ne:DI
3924 (and:DI
3925 (match_operand:DI 1 "register_operand" "w,r")
3926 (match_operand:DI 2 "register_operand" "w,r"))
3927 (const_int 0))))
3928 (clobber (reg:CC CC_REGNUM))]
3929 "TARGET_SIMD"
3930 "#"
3931 "reload_completed"
3932 [(set (match_operand:DI 0 "register_operand")
3933 (neg:DI
3934 (ne:DI
3935 (and:DI
3936 (match_operand:DI 1 "register_operand")
3937 (match_operand:DI 2 "register_operand"))
3938 (const_int 0))))]
3939 {
3940 /* If we are in the general purpose register file,
3941 we split to a sequence of comparison and store. */
3942 if (GP_REGNUM_P (REGNO (operands[0]))
3943 && GP_REGNUM_P (REGNO (operands[1])))
3944 {
3945 rtx and_tree = gen_rtx_AND (DImode, operands[1], operands[2]);
3946 machine_mode mode = SELECT_CC_MODE (NE, and_tree, const0_rtx);
3947 rtx cc_reg = aarch64_gen_compare_reg (NE, and_tree, const0_rtx);
3948 rtx comparison = gen_rtx_NE (mode, and_tree, const0_rtx);
3949 emit_insn (gen_cstoredi_neg (operands[0], comparison, cc_reg));
3950 DONE;
3951 }
3952 /* Otherwise, we expand to a similar pattern which does not
3953 clobber CC_REGNUM. */
3954 }
3955 [(set_attr "type" "neon_tst,multiple")]
3956 )
3958 (define_insn "*aarch64_cmtstdi"
3959 [(set (match_operand:DI 0 "register_operand" "=w")
3960 (neg:DI
3961 (ne:DI
3962 (and:DI
3963 (match_operand:DI 1 "register_operand" "w")
3964 (match_operand:DI 2 "register_operand" "w"))
3965 (const_int 0))))]
3966 "TARGET_SIMD"
3967 "cmtst\t%d0, %d1, %d2"
3968 [(set_attr "type" "neon_tst")]
3969 )
3971 ;; fcm(eq|ge|gt|le|lt)
3973 (define_insn "aarch64_cm<optab><mode>"
3974 [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w,w")
3975 (neg:<V_cmp_result>
3976 (COMPARISONS:<V_cmp_result>
3977 (match_operand:VALLF 1 "register_operand" "w,w")
3978 (match_operand:VALLF 2 "aarch64_simd_reg_or_zero" "w,YDz")
3979 )))]
3980 "TARGET_SIMD"
3981 "@
3982 fcm<n_optab>\t%<v>0<Vmtype>, %<v><cmp_1><Vmtype>, %<v><cmp_2><Vmtype>
3983 fcm<optab>\t%<v>0<Vmtype>, %<v>1<Vmtype>, 0"
3984 [(set_attr "type" "neon_fp_compare_<Vetype><q>")]
3985 )
3987 ;; fac(ge|gt)
3988 ;; Note we can also handle what would be fac(le|lt) by
3989 ;; generating fac(ge|gt).
3991 (define_insn "*aarch64_fac<optab><mode>"
3992 [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w")
3993 (neg:<V_cmp_result>
3994 (FAC_COMPARISONS:<V_cmp_result>
3995 (abs:VALLF (match_operand:VALLF 1 "register_operand" "w"))
3996 (abs:VALLF (match_operand:VALLF 2 "register_operand" "w"))
3997 )))]
3998 "TARGET_SIMD"
3999 "fac<n_optab>\t%<v>0<Vmtype>, %<v><cmp_1><Vmtype>, %<v><cmp_2><Vmtype>"
4000 [(set_attr "type" "neon_fp_compare_<Vetype><q>")]
4001 )
4003 ;; addp
4005 (define_insn "aarch64_addp<mode>"
4006 [(set (match_operand:VD_BHSI 0 "register_operand" "=w")
4007 (unspec:VD_BHSI
4008 [(match_operand:VD_BHSI 1 "register_operand" "w")
4009 (match_operand:VD_BHSI 2 "register_operand" "w")]
4010 UNSPEC_ADDP))]
4011 "TARGET_SIMD"
4012 "addp\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
4013 [(set_attr "type" "neon_reduc_add<q>")]
4014 )
4016 (define_insn "aarch64_addpdi"
4017 [(set (match_operand:DI 0 "register_operand" "=w")
4018 (unspec:DI
4019 [(match_operand:V2DI 1 "register_operand" "w")]
4020 UNSPEC_ADDP))]
4021 "TARGET_SIMD"
4022 "addp\t%d0, %1.2d"
4023 [(set_attr "type" "neon_reduc_add")]
4024 )
4026 ;; sqrt
4028 (define_insn "sqrt<mode>2"
4029 [(set (match_operand:VDQF 0 "register_operand" "=w")
4030 (sqrt:VDQF (match_operand:VDQF 1 "register_operand" "w")))]
4031 "TARGET_SIMD"
4032 "fsqrt\\t%0.<Vtype>, %1.<Vtype>"
4033 [(set_attr "type" "neon_fp_sqrt_<Vetype><q>")]
4034 )
4036 ;; Patterns for vector struct loads and stores.
4038 (define_insn "vec_load_lanesoi<mode>"
4039 [(set (match_operand:OI 0 "register_operand" "=w")
4040 (unspec:OI [(match_operand:OI 1 "aarch64_simd_struct_operand" "Utv")
4041 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4042 UNSPEC_LD2))]
4043 "TARGET_SIMD"
4044 "ld2\\t{%S0.<Vtype> - %T0.<Vtype>}, %1"
4045 [(set_attr "type" "neon_load2_2reg<q>")]
4046 )
4048 (define_insn "aarch64_simd_ld2r<mode>"
4049 [(set (match_operand:OI 0 "register_operand" "=w")
4050 (unspec:OI [(match_operand:<V_TWO_ELEM> 1 "aarch64_simd_struct_operand" "Utv")
4051 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY) ]
4052 UNSPEC_LD2_DUP))]
4053 "TARGET_SIMD"
4054 "ld2r\\t{%S0.<Vtype> - %T0.<Vtype>}, %1"
4055 [(set_attr "type" "neon_load2_all_lanes<q>")]
4056 )
4058 (define_insn "aarch64_vec_load_lanesoi_lane<mode>"
4059 [(set (match_operand:OI 0 "register_operand" "=w")
4060 (unspec:OI [(match_operand:<V_TWO_ELEM> 1 "aarch64_simd_struct_operand" "Utv")
4061 (match_operand:OI 2 "register_operand" "0")
4062 (match_operand:SI 3 "immediate_operand" "i")
4063 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY) ]
4064 UNSPEC_LD2_LANE))]
4065 "TARGET_SIMD"
4066 "ld2\\t{%S0.<Vetype> - %T0.<Vetype>}[%3], %1"
4067 [(set_attr "type" "neon_load2_one_lane")]
4068 )
4070 (define_insn "vec_store_lanesoi<mode>"
4071 [(set (match_operand:OI 0 "aarch64_simd_struct_operand" "=Utv")
4072 (unspec:OI [(match_operand:OI 1 "register_operand" "w")
4073 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4074 UNSPEC_ST2))]
4075 "TARGET_SIMD"
4076 "st2\\t{%S1.<Vtype> - %T1.<Vtype>}, %0"
4077 [(set_attr "type" "neon_store2_2reg<q>")]
4078 )
4080 (define_insn "vec_store_lanesoi_lane<mode>"
4081 [(set (match_operand:<V_TWO_ELEM> 0 "aarch64_simd_struct_operand" "=Utv")
4082 (unspec:<V_TWO_ELEM> [(match_operand:OI 1 "register_operand" "w")
4083 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)
4084 (match_operand:SI 2 "immediate_operand" "i")]
4085 UNSPEC_ST2_LANE))]
4086 "TARGET_SIMD"
4087 "st2\\t{%S1.<Vetype> - %T1.<Vetype>}[%2], %0"
4088 [(set_attr "type" "neon_store3_one_lane<q>")]
4089 )
4091 (define_insn "vec_load_lanesci<mode>"
4092 [(set (match_operand:CI 0 "register_operand" "=w")
4093 (unspec:CI [(match_operand:CI 1 "aarch64_simd_struct_operand" "Utv")
4094 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4095 UNSPEC_LD3))]
4096 "TARGET_SIMD"
4097 "ld3\\t{%S0.<Vtype> - %U0.<Vtype>}, %1"
4098 [(set_attr "type" "neon_load3_3reg<q>")]
4099 )
4101 (define_insn "aarch64_simd_ld3r<mode>"
4102 [(set (match_operand:CI 0 "register_operand" "=w")
4103 (unspec:CI [(match_operand:<V_THREE_ELEM> 1 "aarch64_simd_struct_operand" "Utv")
4104 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY) ]
4105 UNSPEC_LD3_DUP))]
4106 "TARGET_SIMD"
4107 "ld3r\\t{%S0.<Vtype> - %U0.<Vtype>}, %1"
4108 [(set_attr "type" "neon_load3_all_lanes<q>")]
4109 )
4111 (define_insn "aarch64_vec_load_lanesci_lane<mode>"
4112 [(set (match_operand:CI 0 "register_operand" "=w")
4113 (unspec:CI [(match_operand:<V_THREE_ELEM> 1 "aarch64_simd_struct_operand" "Utv")
4114 (match_operand:CI 2 "register_operand" "0")
4115 (match_operand:SI 3 "immediate_operand" "i")
4116 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4117 UNSPEC_LD3_LANE))]
4118 "TARGET_SIMD"
4119 "ld3\\t{%S0.<Vetype> - %U0.<Vetype>}[%3], %1"
4120 [(set_attr "type" "neon_load3_one_lane")]
4121 )
4123 (define_insn "vec_store_lanesci<mode>"
4124 [(set (match_operand:CI 0 "aarch64_simd_struct_operand" "=Utv")
4125 (unspec:CI [(match_operand:CI 1 "register_operand" "w")
4126 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4127 UNSPEC_ST3))]
4128 "TARGET_SIMD"
4129 "st3\\t{%S1.<Vtype> - %U1.<Vtype>}, %0"
4130 [(set_attr "type" "neon_store3_3reg<q>")]
4131 )
4133 (define_insn "vec_store_lanesci_lane<mode>"
4134 [(set (match_operand:<V_THREE_ELEM> 0 "aarch64_simd_struct_operand" "=Utv")
4135 (unspec:<V_THREE_ELEM> [(match_operand:CI 1 "register_operand" "w")
4136 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)
4137 (match_operand:SI 2 "immediate_operand" "i")]
4138 UNSPEC_ST3_LANE))]
4139 "TARGET_SIMD"
4140 "st3\\t{%S1.<Vetype> - %U1.<Vetype>}[%2], %0"
4141 [(set_attr "type" "neon_store3_one_lane<q>")]
4142 )
4144 (define_insn "vec_load_lanesxi<mode>"
4145 [(set (match_operand:XI 0 "register_operand" "=w")
4146 (unspec:XI [(match_operand:XI 1 "aarch64_simd_struct_operand" "Utv")
4147 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4148 UNSPEC_LD4))]
4149 "TARGET_SIMD"
4150 "ld4\\t{%S0.<Vtype> - %V0.<Vtype>}, %1"
4151 [(set_attr "type" "neon_load4_4reg<q>")]
4152 )
4154 (define_insn "aarch64_simd_ld4r<mode>"
4155 [(set (match_operand:XI 0 "register_operand" "=w")
4156 (unspec:XI [(match_operand:<V_FOUR_ELEM> 1 "aarch64_simd_struct_operand" "Utv")
4157 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY) ]
4158 UNSPEC_LD4_DUP))]
4159 "TARGET_SIMD"
4160 "ld4r\\t{%S0.<Vtype> - %V0.<Vtype>}, %1"
4161 [(set_attr "type" "neon_load4_all_lanes<q>")]
4162 )
4164 (define_insn "aarch64_vec_load_lanesxi_lane<mode>"
4165 [(set (match_operand:XI 0 "register_operand" "=w")
4166 (unspec:XI [(match_operand:<V_FOUR_ELEM> 1 "aarch64_simd_struct_operand" "Utv")
4167 (match_operand:XI 2 "register_operand" "0")
4168 (match_operand:SI 3 "immediate_operand" "i")
4169 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4170 UNSPEC_LD4_LANE))]
4171 "TARGET_SIMD"
4172 "ld4\\t{%S0.<Vetype> - %V0.<Vetype>}[%3], %1"
4173 [(set_attr "type" "neon_load4_one_lane")]
4174 )
4176 (define_insn "vec_store_lanesxi<mode>"
4177 [(set (match_operand:XI 0 "aarch64_simd_struct_operand" "=Utv")
4178 (unspec:XI [(match_operand:XI 1 "register_operand" "w")
4179 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4180 UNSPEC_ST4))]
4181 "TARGET_SIMD"
4182 "st4\\t{%S1.<Vtype> - %V1.<Vtype>}, %0"
4183 [(set_attr "type" "neon_store4_4reg<q>")]
4184 )
4186 (define_insn "vec_store_lanesxi_lane<mode>"
4187 [(set (match_operand:<V_FOUR_ELEM> 0 "aarch64_simd_struct_operand" "=Utv")
4188 (unspec:<V_FOUR_ELEM> [(match_operand:XI 1 "register_operand" "w")
4189 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)
4190 (match_operand:SI 2 "immediate_operand" "i")]
4191 UNSPEC_ST4_LANE))]
4192 "TARGET_SIMD"
4193 "st4\\t{%S1.<Vetype> - %V1.<Vetype>}[%2], %0"
4194 [(set_attr "type" "neon_store4_one_lane<q>")]
4195 )
4197 ;; Reload patterns for AdvSIMD register list operands.
4199 (define_expand "mov<mode>"
4200 [(set (match_operand:VSTRUCT 0 "aarch64_simd_nonimmediate_operand" "")
4201 (match_operand:VSTRUCT 1 "aarch64_simd_general_operand" ""))]
4202 "TARGET_SIMD"
4203 {
4204 if (can_create_pseudo_p ())
4205 {
4206 if (GET_CODE (operands[0]) != REG)
4207 operands[1] = force_reg (<MODE>mode, operands[1]);
4208 }
4209 })
4211 (define_insn "*aarch64_mov<mode>"
4212 [(set (match_operand:VSTRUCT 0 "aarch64_simd_nonimmediate_operand" "=w,Utv,w")
4213 (match_operand:VSTRUCT 1 "aarch64_simd_general_operand" " w,w,Utv"))]
4214 "TARGET_SIMD
4215 && (register_operand (operands[0], <MODE>mode)
4216 || register_operand (operands[1], <MODE>mode))"
4218 {
4219 switch (which_alternative)
4220 {
4221 case 0: return "#";
4222 case 1: return "st1\\t{%S1.16b - %<Vendreg>1.16b}, %0";
4223 case 2: return "ld1\\t{%S0.16b - %<Vendreg>0.16b}, %1";
4224 default: gcc_unreachable ();
4225 }
4226 }
4227 [(set_attr "type" "neon_move,neon_store<nregs>_<nregs>reg_q,\
4228 neon_load<nregs>_<nregs>reg_q")
4229 (set (attr "length") (symbol_ref "aarch64_simd_attr_length_move (insn)"))]
4230 )
4232 (define_insn "aarch64_be_ld1<mode>"
4233 [(set (match_operand:VALLDI 0 "register_operand" "=w")
4234 (unspec:VALLDI [(match_operand:VALLDI 1 "aarch64_simd_struct_operand" "Utv")]
4235 UNSPEC_LD1))]
4236 "TARGET_SIMD"
4237 "ld1\\t{%0<Vmtype>}, %1"
4238 [(set_attr "type" "neon_load1_1reg<q>")]
4239 )
4241 (define_insn "aarch64_be_st1<mode>"
4242 [(set (match_operand:VALLDI 0 "aarch64_simd_struct_operand" "=Utv")
4243 (unspec:VALLDI [(match_operand:VALLDI 1 "register_operand" "w")]
4244 UNSPEC_ST1))]
4245 "TARGET_SIMD"
4246 "st1\\t{%1<Vmtype>}, %0"
4247 [(set_attr "type" "neon_store1_1reg<q>")]
4248 )
4250 (define_split
4251 [(set (match_operand:OI 0 "register_operand" "")
4252 (match_operand:OI 1 "register_operand" ""))]
4253 "TARGET_SIMD && reload_completed"
4254 [(set (match_dup 0) (match_dup 1))
4255 (set (match_dup 2) (match_dup 3))]
4256 {
4257 int rdest = REGNO (operands[0]);
4258 int rsrc = REGNO (operands[1]);
4259 rtx dest[2], src[2];
4261 dest[0] = gen_rtx_REG (TFmode, rdest);
4262 src[0] = gen_rtx_REG (TFmode, rsrc);
4263 dest[1] = gen_rtx_REG (TFmode, rdest + 1);
4264 src[1] = gen_rtx_REG (TFmode, rsrc + 1);
4266 aarch64_simd_disambiguate_copy (operands, dest, src, 2);
4267 })
4269 (define_split
4270 [(set (match_operand:CI 0 "register_operand" "")
4271 (match_operand:CI 1 "register_operand" ""))]
4272 "TARGET_SIMD && reload_completed"
4273 [(set (match_dup 0) (match_dup 1))
4274 (set (match_dup 2) (match_dup 3))
4275 (set (match_dup 4) (match_dup 5))]
4276 {
4277 int rdest = REGNO (operands[0]);
4278 int rsrc = REGNO (operands[1]);
4279 rtx dest[3], src[3];
4281 dest[0] = gen_rtx_REG (TFmode, rdest);
4282 src[0] = gen_rtx_REG (TFmode, rsrc);
4283 dest[1] = gen_rtx_REG (TFmode, rdest + 1);
4284 src[1] = gen_rtx_REG (TFmode, rsrc + 1);
4285 dest[2] = gen_rtx_REG (TFmode, rdest + 2);
4286 src[2] = gen_rtx_REG (TFmode, rsrc + 2);
4288 aarch64_simd_disambiguate_copy (operands, dest, src, 3);
4289 })
4291 (define_split
4292 [(set (match_operand:XI 0 "register_operand" "")
4293 (match_operand:XI 1 "register_operand" ""))]
4294 "TARGET_SIMD && reload_completed"
4295 [(set (match_dup 0) (match_dup 1))
4296 (set (match_dup 2) (match_dup 3))
4297 (set (match_dup 4) (match_dup 5))
4298 (set (match_dup 6) (match_dup 7))]
4299 {
4300 int rdest = REGNO (operands[0]);
4301 int rsrc = REGNO (operands[1]);
4302 rtx dest[4], src[4];
4304 dest[0] = gen_rtx_REG (TFmode, rdest);
4305 src[0] = gen_rtx_REG (TFmode, rsrc);
4306 dest[1] = gen_rtx_REG (TFmode, rdest + 1);
4307 src[1] = gen_rtx_REG (TFmode, rsrc + 1);
4308 dest[2] = gen_rtx_REG (TFmode, rdest + 2);
4309 src[2] = gen_rtx_REG (TFmode, rsrc + 2);
4310 dest[3] = gen_rtx_REG (TFmode, rdest + 3);
4311 src[3] = gen_rtx_REG (TFmode, rsrc + 3);
4313 aarch64_simd_disambiguate_copy (operands, dest, src, 4);
4314 })
4316 (define_expand "aarch64_ld2r<mode>"
4317 [(match_operand:OI 0 "register_operand" "=w")
4318 (match_operand:DI 1 "register_operand" "w")
4319 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4320 "TARGET_SIMD"
4321 {
4322 machine_mode mode = <V_TWO_ELEM>mode;
4323 rtx mem = gen_rtx_MEM (mode, operands[1]);
4325 emit_insn (gen_aarch64_simd_ld2r<mode> (operands[0], mem));
4326 DONE;
4327 })
4329 (define_expand "aarch64_ld3r<mode>"
4330 [(match_operand:CI 0 "register_operand" "=w")
4331 (match_operand:DI 1 "register_operand" "w")
4332 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4333 "TARGET_SIMD"
4334 {
4335 machine_mode mode = <V_THREE_ELEM>mode;
4336 rtx mem = gen_rtx_MEM (mode, operands[1]);
4338 emit_insn (gen_aarch64_simd_ld3r<mode> (operands[0], mem));
4339 DONE;
4340 })
4342 (define_expand "aarch64_ld4r<mode>"
4343 [(match_operand:XI 0 "register_operand" "=w")
4344 (match_operand:DI 1 "register_operand" "w")
4345 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4346 "TARGET_SIMD"
4347 {
4348 machine_mode mode = <V_FOUR_ELEM>mode;
4349 rtx mem = gen_rtx_MEM (mode, operands[1]);
4351 emit_insn (gen_aarch64_simd_ld4r<mode> (operands[0],mem));
4352 DONE;
4353 })
4355 (define_insn "aarch64_ld2<mode>_dreg"
4356 [(set (match_operand:OI 0 "register_operand" "=w")
4357 (subreg:OI
4358 (vec_concat:<VRL2>
4359 (vec_concat:<VDBL>
4360 (unspec:VD [(match_operand:TI 1 "aarch64_simd_struct_operand" "Utv")]
4361 UNSPEC_LD2)
4362 (vec_duplicate:VD (const_int 0)))
4363 (vec_concat:<VDBL>
4364 (unspec:VD [(match_dup 1)]
4365 UNSPEC_LD2)
4366 (vec_duplicate:VD (const_int 0)))) 0))]
4367 "TARGET_SIMD"
4368 "ld2\\t{%S0.<Vtype> - %T0.<Vtype>}, %1"
4369 [(set_attr "type" "neon_load2_2reg<q>")]
4370 )
4372 (define_insn "aarch64_ld2<mode>_dreg"
4373 [(set (match_operand:OI 0 "register_operand" "=w")
4374 (subreg:OI
4375 (vec_concat:<VRL2>
4376 (vec_concat:<VDBL>
4377 (unspec:DX [(match_operand:TI 1 "aarch64_simd_struct_operand" "Utv")]
4378 UNSPEC_LD2)
4379 (const_int 0))
4380 (vec_concat:<VDBL>
4381 (unspec:DX [(match_dup 1)]
4382 UNSPEC_LD2)
4383 (const_int 0))) 0))]
4384 "TARGET_SIMD"
4385 "ld1\\t{%S0.1d - %T0.1d}, %1"
4386 [(set_attr "type" "neon_load1_2reg<q>")]
4387 )
4389 (define_insn "aarch64_ld3<mode>_dreg"
4390 [(set (match_operand:CI 0 "register_operand" "=w")
4391 (subreg:CI
4392 (vec_concat:<VRL3>
4393 (vec_concat:<VRL2>
4394 (vec_concat:<VDBL>
4395 (unspec:VD [(match_operand:EI 1 "aarch64_simd_struct_operand" "Utv")]
4396 UNSPEC_LD3)
4397 (vec_duplicate:VD (const_int 0)))
4398 (vec_concat:<VDBL>
4399 (unspec:VD [(match_dup 1)]
4400 UNSPEC_LD3)
4401 (vec_duplicate:VD (const_int 0))))
4402 (vec_concat:<VDBL>
4403 (unspec:VD [(match_dup 1)]
4404 UNSPEC_LD3)
4405 (vec_duplicate:VD (const_int 0)))) 0))]
4406 "TARGET_SIMD"
4407 "ld3\\t{%S0.<Vtype> - %U0.<Vtype>}, %1"
4408 [(set_attr "type" "neon_load3_3reg<q>")]
4409 )
4411 (define_insn "aarch64_ld3<mode>_dreg"
4412 [(set (match_operand:CI 0 "register_operand" "=w")
4413 (subreg:CI
4414 (vec_concat:<VRL3>
4415 (vec_concat:<VRL2>
4416 (vec_concat:<VDBL>
4417 (unspec:DX [(match_operand:EI 1 "aarch64_simd_struct_operand" "Utv")]
4418 UNSPEC_LD3)
4419 (const_int 0))
4420 (vec_concat:<VDBL>
4421 (unspec:DX [(match_dup 1)]
4422 UNSPEC_LD3)
4423 (const_int 0)))
4424 (vec_concat:<VDBL>
4425 (unspec:DX [(match_dup 1)]
4426 UNSPEC_LD3)
4427 (const_int 0))) 0))]
4428 "TARGET_SIMD"
4429 "ld1\\t{%S0.1d - %U0.1d}, %1"
4430 [(set_attr "type" "neon_load1_3reg<q>")]
4431 )
4433 (define_insn "aarch64_ld4<mode>_dreg"
4434 [(set (match_operand:XI 0 "register_operand" "=w")
4435 (subreg:XI
4436 (vec_concat:<VRL4>
4437 (vec_concat:<VRL2>
4438 (vec_concat:<VDBL>
4439 (unspec:VD [(match_operand:OI 1 "aarch64_simd_struct_operand" "Utv")]
4440 UNSPEC_LD4)
4441 (vec_duplicate:VD (const_int 0)))
4442 (vec_concat:<VDBL>
4443 (unspec:VD [(match_dup 1)]
4444 UNSPEC_LD4)
4445 (vec_duplicate:VD (const_int 0))))
4446 (vec_concat:<VRL2>
4447 (vec_concat:<VDBL>
4448 (unspec:VD [(match_dup 1)]
4449 UNSPEC_LD4)
4450 (vec_duplicate:VD (const_int 0)))
4451 (vec_concat:<VDBL>
4452 (unspec:VD [(match_dup 1)]
4453 UNSPEC_LD4)
4454 (vec_duplicate:VD (const_int 0))))) 0))]
4455 "TARGET_SIMD"
4456 "ld4\\t{%S0.<Vtype> - %V0.<Vtype>}, %1"
4457 [(set_attr "type" "neon_load4_4reg<q>")]
4458 )
4460 (define_insn "aarch64_ld4<mode>_dreg"
4461 [(set (match_operand:XI 0 "register_operand" "=w")
4462 (subreg:XI
4463 (vec_concat:<VRL4>
4464 (vec_concat:<VRL2>
4465 (vec_concat:<VDBL>
4466 (unspec:DX [(match_operand:OI 1 "aarch64_simd_struct_operand" "Utv")]
4467 UNSPEC_LD4)
4468 (const_int 0))
4469 (vec_concat:<VDBL>
4470 (unspec:DX [(match_dup 1)]
4471 UNSPEC_LD4)
4472 (const_int 0)))
4473 (vec_concat:<VRL2>
4474 (vec_concat:<VDBL>
4475 (unspec:DX [(match_dup 1)]
4476 UNSPEC_LD4)
4477 (const_int 0))
4478 (vec_concat:<VDBL>
4479 (unspec:DX [(match_dup 1)]
4480 UNSPEC_LD4)
4481 (const_int 0)))) 0))]
4482 "TARGET_SIMD"
4483 "ld1\\t{%S0.1d - %V0.1d}, %1"
4484 [(set_attr "type" "neon_load1_4reg<q>")]
4485 )
4487 (define_expand "aarch64_ld<VSTRUCT:nregs><VDC:mode>"
4488 [(match_operand:VSTRUCT 0 "register_operand" "=w")
4489 (match_operand:DI 1 "register_operand" "r")
4490 (unspec:VDC [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4491 "TARGET_SIMD"
4492 {
4493 machine_mode mode = <VSTRUCT:VSTRUCT_DREG>mode;
4494 rtx mem = gen_rtx_MEM (mode, operands[1]);
4496 emit_insn (gen_aarch64_ld<VSTRUCT:nregs><VDC:mode>_dreg (operands[0], mem));
4497 DONE;
4498 })
4500 (define_expand "aarch64_ld1<VALL:mode>"
4501 [(match_operand:VALL 0 "register_operand")
4502 (match_operand:DI 1 "register_operand")]
4503 "TARGET_SIMD"
4504 {
4505 machine_mode mode = <VALL:MODE>mode;
4506 rtx mem = gen_rtx_MEM (mode, operands[1]);
4508 if (BYTES_BIG_ENDIAN)
4509 emit_insn (gen_aarch64_be_ld1<VALL:mode> (operands[0], mem));
4510 else
4511 emit_move_insn (operands[0], mem);
4512 DONE;
4513 })
4515 (define_expand "aarch64_ld<VSTRUCT:nregs><VQ:mode>"
4516 [(match_operand:VSTRUCT 0 "register_operand" "=w")
4517 (match_operand:DI 1 "register_operand" "r")
4518 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4519 "TARGET_SIMD"
4520 {
4521 machine_mode mode = <VSTRUCT:MODE>mode;
4522 rtx mem = gen_rtx_MEM (mode, operands[1]);
4524 emit_insn (gen_vec_load_lanes<VSTRUCT:mode><VQ:mode> (operands[0], mem));
4525 DONE;
4526 })
4528 (define_expand "aarch64_ld2_lane<mode>"
4529 [(match_operand:OI 0 "register_operand" "=w")
4530 (match_operand:DI 1 "register_operand" "w")
4531 (match_operand:OI 2 "register_operand" "0")
4532 (match_operand:SI 3 "immediate_operand" "i")
4533 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4534 "TARGET_SIMD"
4535 {
4536 machine_mode mode = <V_TWO_ELEM>mode;
4537 rtx mem = gen_rtx_MEM (mode, operands[1]);
4539 aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCONQ>mode));
4540 emit_insn (gen_aarch64_vec_load_lanesoi_lane<mode> (operands[0],
4541 mem,
4542 operands[2],
4543 operands[3]));
4544 DONE;
4545 })
4547 (define_expand "aarch64_ld3_lane<mode>"
4548 [(match_operand:CI 0 "register_operand" "=w")
4549 (match_operand:DI 1 "register_operand" "w")
4550 (match_operand:CI 2 "register_operand" "0")
4551 (match_operand:SI 3 "immediate_operand" "i")
4552 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4553 "TARGET_SIMD"
4554 {
4555 machine_mode mode = <V_THREE_ELEM>mode;
4556 rtx mem = gen_rtx_MEM (mode, operands[1]);
4558 aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCONQ>mode));
4559 emit_insn (gen_aarch64_vec_load_lanesci_lane<mode> (operands[0],
4560 mem,
4561 operands[2],
4562 operands[3]));
4563 DONE;
4564 })
4566 (define_expand "aarch64_ld4_lane<mode>"
4567 [(match_operand:XI 0 "register_operand" "=w")
4568 (match_operand:DI 1 "register_operand" "w")
4569 (match_operand:XI 2 "register_operand" "0")
4570 (match_operand:SI 3 "immediate_operand" "i")
4571 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4572 "TARGET_SIMD"
4573 {
4574 machine_mode mode = <V_FOUR_ELEM>mode;
4575 rtx mem = gen_rtx_MEM (mode, operands[1]);
4577 aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCONQ>mode));
4578 emit_insn (gen_aarch64_vec_load_lanesxi_lane<mode> (operands[0],
4579 mem,
4580 operands[2],
4581 operands[3]));
4582 DONE;
4583 })
4587 ;; Expanders for builtins to extract vector registers from large
4588 ;; opaque integer modes.
4590 ;; D-register list.
4592 (define_expand "aarch64_get_dreg<VSTRUCT:mode><VDC:mode>"
4593 [(match_operand:VDC 0 "register_operand" "=w")
4594 (match_operand:VSTRUCT 1 "register_operand" "w")
4595 (match_operand:SI 2 "immediate_operand" "i")]
4596 "TARGET_SIMD"
4597 {
4598 int part = INTVAL (operands[2]);
4599 rtx temp = gen_reg_rtx (<VDC:VDBL>mode);
4600 int offset = part * 16;
4602 emit_move_insn (temp, gen_rtx_SUBREG (<VDC:VDBL>mode, operands[1], offset));
4603 emit_move_insn (operands[0], gen_lowpart (<VDC:MODE>mode, temp));
4604 DONE;
4605 })
4607 ;; Q-register list.
4609 (define_expand "aarch64_get_qreg<VSTRUCT:mode><VQ:mode>"
4610 [(match_operand:VQ 0 "register_operand" "=w")
4611 (match_operand:VSTRUCT 1 "register_operand" "w")
4612 (match_operand:SI 2 "immediate_operand" "i")]
4613 "TARGET_SIMD"
4614 {
4615 int part = INTVAL (operands[2]);
4616 int offset = part * 16;
4618 emit_move_insn (operands[0],
4619 gen_rtx_SUBREG (<VQ:MODE>mode, operands[1], offset));
4620 DONE;
4621 })
4623 ;; Permuted-store expanders for neon intrinsics.
4625 ;; Permute instructions
4627 ;; vec_perm support
4629 (define_expand "vec_perm_const<mode>"
4630 [(match_operand:VALL 0 "register_operand")
4631 (match_operand:VALL 1 "register_operand")
4632 (match_operand:VALL 2 "register_operand")
4633 (match_operand:<V_cmp_result> 3)]
4634 "TARGET_SIMD"
4635 {
4636 if (aarch64_expand_vec_perm_const (operands[0], operands[1],
4637 operands[2], operands[3]))
4638 DONE;
4639 else
4640 FAIL;
4641 })
4643 (define_expand "vec_perm<mode>"
4644 [(match_operand:VB 0 "register_operand")
4645 (match_operand:VB 1 "register_operand")
4646 (match_operand:VB 2 "register_operand")
4647 (match_operand:VB 3 "register_operand")]
4648 "TARGET_SIMD"
4649 {
4650 aarch64_expand_vec_perm (operands[0], operands[1],
4651 operands[2], operands[3]);
4652 DONE;
4653 })
4655 (define_insn "aarch64_tbl1<mode>"
4656 [(set (match_operand:VB 0 "register_operand" "=w")
4657 (unspec:VB [(match_operand:V16QI 1 "register_operand" "w")
4658 (match_operand:VB 2 "register_operand" "w")]
4659 UNSPEC_TBL))]
4660 "TARGET_SIMD"
4661 "tbl\\t%0.<Vtype>, {%1.16b}, %2.<Vtype>"
4662 [(set_attr "type" "neon_tbl1<q>")]
4663 )
4665 ;; Two source registers.
4667 (define_insn "aarch64_tbl2v16qi"
4668 [(set (match_operand:V16QI 0 "register_operand" "=w")
4669 (unspec:V16QI [(match_operand:OI 1 "register_operand" "w")
4670 (match_operand:V16QI 2 "register_operand" "w")]
4671 UNSPEC_TBL))]
4672 "TARGET_SIMD"
4673 "tbl\\t%0.16b, {%S1.16b - %T1.16b}, %2.16b"
4674 [(set_attr "type" "neon_tbl2_q")]
4675 )
4677 (define_insn_and_split "aarch64_combinev16qi"
4678 [(set (match_operand:OI 0 "register_operand" "=w")
4679 (unspec:OI [(match_operand:V16QI 1 "register_operand" "w")
4680 (match_operand:V16QI 2 "register_operand" "w")]
4681 UNSPEC_CONCAT))]
4682 "TARGET_SIMD"
4683 "#"
4684 "&& reload_completed"
4685 [(const_int 0)]
4686 {
4687 aarch64_split_combinev16qi (operands);
4688 DONE;
4689 }
4690 [(set_attr "type" "multiple")]
4691 )
4693 (define_insn "aarch64_<PERMUTE:perm_insn><PERMUTE:perm_hilo><mode>"
4694 [(set (match_operand:VALL 0 "register_operand" "=w")
4695 (unspec:VALL [(match_operand:VALL 1 "register_operand" "w")
4696 (match_operand:VALL 2 "register_operand" "w")]
4697 PERMUTE))]
4698 "TARGET_SIMD"
4699 "<PERMUTE:perm_insn><PERMUTE:perm_hilo>\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
4700 [(set_attr "type" "neon_permute<q>")]
4701 )
4703 ;; Note immediate (third) operand is lane index not byte index.
4704 (define_insn "aarch64_ext<mode>"
4705 [(set (match_operand:VALL 0 "register_operand" "=w")
4706 (unspec:VALL [(match_operand:VALL 1 "register_operand" "w")
4707 (match_operand:VALL 2 "register_operand" "w")
4708 (match_operand:SI 3 "immediate_operand" "i")]
4709 UNSPEC_EXT))]
4710 "TARGET_SIMD"
4711 {
4712 operands[3] = GEN_INT (INTVAL (operands[3])
4713 * GET_MODE_SIZE (GET_MODE_INNER (<MODE>mode)));
4714 return "ext\\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>, #%3";
4715 }
4716 [(set_attr "type" "neon_ext<q>")]
4717 )
4719 ;; This exists solely to check the arguments to the corresponding __builtin.
4720 ;; Used where we want an error for out-of-range indices which would otherwise
4721 ;; be silently wrapped (e.g. the mask to a __builtin_shuffle).
4722 (define_expand "aarch64_im_lane_boundsi"
4723 [(match_operand:SI 0 "immediate_operand" "i")
4724 (match_operand:SI 1 "immediate_operand" "i")]
4725 "TARGET_SIMD"
4726 {
4727 aarch64_simd_lane_bounds (operands[0], 0, INTVAL (operands[1]));
4728 DONE;
4729 }
4730 )
4732 (define_insn "aarch64_rev<REVERSE:rev_op><mode>"
4733 [(set (match_operand:VALL 0 "register_operand" "=w")
4734 (unspec:VALL [(match_operand:VALL 1 "register_operand" "w")]
4735 REVERSE))]
4736 "TARGET_SIMD"
4737 "rev<REVERSE:rev_op>\\t%0.<Vtype>, %1.<Vtype>"
4738 [(set_attr "type" "neon_rev<q>")]
4739 )
4741 (define_insn "aarch64_st2<mode>_dreg"
4742 [(set (match_operand:TI 0 "aarch64_simd_struct_operand" "=Utv")
4743 (unspec:TI [(match_operand:OI 1 "register_operand" "w")
4744 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4745 UNSPEC_ST2))]
4746 "TARGET_SIMD"
4747 "st2\\t{%S1.<Vtype> - %T1.<Vtype>}, %0"
4748 [(set_attr "type" "neon_store2_2reg")]
4749 )
4751 (define_insn "aarch64_st2<mode>_dreg"
4752 [(set (match_operand:TI 0 "aarch64_simd_struct_operand" "=Utv")
4753 (unspec:TI [(match_operand:OI 1 "register_operand" "w")
4754 (unspec:DX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4755 UNSPEC_ST2))]
4756 "TARGET_SIMD"
4757 "st1\\t{%S1.1d - %T1.1d}, %0"
4758 [(set_attr "type" "neon_store1_2reg")]
4759 )
4761 (define_insn "aarch64_st3<mode>_dreg"
4762 [(set (match_operand:EI 0 "aarch64_simd_struct_operand" "=Utv")
4763 (unspec:EI [(match_operand:CI 1 "register_operand" "w")
4764 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4765 UNSPEC_ST3))]
4766 "TARGET_SIMD"
4767 "st3\\t{%S1.<Vtype> - %U1.<Vtype>}, %0"
4768 [(set_attr "type" "neon_store3_3reg")]
4769 )
4771 (define_insn "aarch64_st3<mode>_dreg"
4772 [(set (match_operand:EI 0 "aarch64_simd_struct_operand" "=Utv")
4773 (unspec:EI [(match_operand:CI 1 "register_operand" "w")
4774 (unspec:DX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4775 UNSPEC_ST3))]
4776 "TARGET_SIMD"
4777 "st1\\t{%S1.1d - %U1.1d}, %0"
4778 [(set_attr "type" "neon_store1_3reg")]
4779 )
4781 (define_insn "aarch64_st4<mode>_dreg"
4782 [(set (match_operand:OI 0 "aarch64_simd_struct_operand" "=Utv")
4783 (unspec:OI [(match_operand:XI 1 "register_operand" "w")
4784 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4785 UNSPEC_ST4))]
4786 "TARGET_SIMD"
4787 "st4\\t{%S1.<Vtype> - %V1.<Vtype>}, %0"
4788 [(set_attr "type" "neon_store4_4reg")]
4789 )
4791 (define_insn "aarch64_st4<mode>_dreg"
4792 [(set (match_operand:OI 0 "aarch64_simd_struct_operand" "=Utv")
4793 (unspec:OI [(match_operand:XI 1 "register_operand" "w")
4794 (unspec:DX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4795 UNSPEC_ST4))]
4796 "TARGET_SIMD"
4797 "st1\\t{%S1.1d - %V1.1d}, %0"
4798 [(set_attr "type" "neon_store1_4reg")]
4799 )
4801 (define_expand "aarch64_st<VSTRUCT:nregs><VDC:mode>"
4802 [(match_operand:DI 0 "register_operand" "r")
4803 (match_operand:VSTRUCT 1 "register_operand" "w")
4804 (unspec:VDC [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4805 "TARGET_SIMD"
4806 {
4807 machine_mode mode = <VSTRUCT:VSTRUCT_DREG>mode;
4808 rtx mem = gen_rtx_MEM (mode, operands[0]);
4810 emit_insn (gen_aarch64_st<VSTRUCT:nregs><VDC:mode>_dreg (mem, operands[1]));
4811 DONE;
4812 })
4814 (define_expand "aarch64_st<VSTRUCT:nregs><VQ:mode>"
4815 [(match_operand:DI 0 "register_operand" "r")
4816 (match_operand:VSTRUCT 1 "register_operand" "w")
4817 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4818 "TARGET_SIMD"
4819 {
4820 machine_mode mode = <VSTRUCT:MODE>mode;
4821 rtx mem = gen_rtx_MEM (mode, operands[0]);
4823 emit_insn (gen_vec_store_lanes<VSTRUCT:mode><VQ:mode> (mem, operands[1]));
4824 DONE;
4825 })
4827 (define_expand "aarch64_st2_lane<VQ:mode>"
4828 [(match_operand:DI 0 "register_operand" "r")
4829 (match_operand:OI 1 "register_operand" "w")
4830 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)
4831 (match_operand:SI 2 "immediate_operand")]
4832 "TARGET_SIMD"
4833 {
4834 machine_mode mode = <V_TWO_ELEM>mode;
4835 rtx mem = gen_rtx_MEM (mode, operands[0]);
4836 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
4838 emit_insn (gen_vec_store_lanesoi_lane<VQ:mode> (mem,
4839 operands[1],
4840 operands[2]));
4841 DONE;
4842 })
4844 (define_expand "aarch64_st3_lane<VQ:mode>"
4845 [(match_operand:DI 0 "register_operand" "r")
4846 (match_operand:CI 1 "register_operand" "w")
4847 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)
4848 (match_operand:SI 2 "immediate_operand")]
4849 "TARGET_SIMD"
4850 {
4851 machine_mode mode = <V_THREE_ELEM>mode;
4852 rtx mem = gen_rtx_MEM (mode, operands[0]);
4853 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
4855 emit_insn (gen_vec_store_lanesci_lane<VQ:mode> (mem,
4856 operands[1],
4857 operands[2]));
4858 DONE;
4859 })
4861 (define_expand "aarch64_st4_lane<VQ:mode>"
4862 [(match_operand:DI 0 "register_operand" "r")
4863 (match_operand:XI 1 "register_operand" "w")
4864 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)
4865 (match_operand:SI 2 "immediate_operand")]
4866 "TARGET_SIMD"
4867 {
4868 machine_mode mode = <V_FOUR_ELEM>mode;
4869 rtx mem = gen_rtx_MEM (mode, operands[0]);
4870 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
4872 emit_insn (gen_vec_store_lanesxi_lane<VQ:mode> (mem,
4873 operands[1],
4874 operands[2]));
4875 DONE;
4876 })
4878 (define_expand "aarch64_st1<VALL:mode>"
4879 [(match_operand:DI 0 "register_operand")
4880 (match_operand:VALL 1 "register_operand")]
4881 "TARGET_SIMD"
4882 {
4883 machine_mode mode = <VALL:MODE>mode;
4884 rtx mem = gen_rtx_MEM (mode, operands[0]);
4886 if (BYTES_BIG_ENDIAN)
4887 emit_insn (gen_aarch64_be_st1<VALL:mode> (mem, operands[1]));
4888 else
4889 emit_move_insn (mem, operands[1]);
4890 DONE;
4891 })
4893 ;; Expander for builtins to insert vector registers into large
4894 ;; opaque integer modes.
4896 ;; Q-register list. We don't need a D-reg inserter as we zero
4897 ;; extend them in arm_neon.h and insert the resulting Q-regs.
4899 (define_expand "aarch64_set_qreg<VSTRUCT:mode><VQ:mode>"
4900 [(match_operand:VSTRUCT 0 "register_operand" "+w")
4901 (match_operand:VSTRUCT 1 "register_operand" "0")
4902 (match_operand:VQ 2 "register_operand" "w")
4903 (match_operand:SI 3 "immediate_operand" "i")]
4904 "TARGET_SIMD"
4905 {
4906 int part = INTVAL (operands[3]);
4907 int offset = part * 16;
4909 emit_move_insn (operands[0], operands[1]);
4910 emit_move_insn (gen_rtx_SUBREG (<VQ:MODE>mode, operands[0], offset),
4911 operands[2]);
4912 DONE;
4913 })
4915 ;; Standard pattern name vec_init<mode>.
4917 (define_expand "vec_init<mode>"
4918 [(match_operand:VALL 0 "register_operand" "")
4919 (match_operand 1 "" "")]
4920 "TARGET_SIMD"
4921 {
4922 aarch64_expand_vector_init (operands[0], operands[1]);
4923 DONE;
4924 })
4926 (define_insn "*aarch64_simd_ld1r<mode>"
4927 [(set (match_operand:VALLDI 0 "register_operand" "=w")
4928 (vec_duplicate:VALLDI
4929 (match_operand:<VEL> 1 "aarch64_simd_struct_operand" "Utv")))]
4930 "TARGET_SIMD"
4931 "ld1r\\t{%0.<Vtype>}, %1"
4932 [(set_attr "type" "neon_load1_all_lanes")]
4933 )
4935 (define_insn "aarch64_frecpe<mode>"
4936 [(set (match_operand:VDQF 0 "register_operand" "=w")
4937 (unspec:VDQF [(match_operand:VDQF 1 "register_operand" "w")]
4938 UNSPEC_FRECPE))]
4939 "TARGET_SIMD"
4940 "frecpe\\t%0.<Vtype>, %1.<Vtype>"
4941 [(set_attr "type" "neon_fp_recpe_<Vetype><q>")]
4942 )
4944 (define_insn "aarch64_frecp<FRECP:frecp_suffix><mode>"
4945 [(set (match_operand:GPF 0 "register_operand" "=w")
4946 (unspec:GPF [(match_operand:GPF 1 "register_operand" "w")]
4947 FRECP))]
4948 "TARGET_SIMD"
4949 "frecp<FRECP:frecp_suffix>\\t%<s>0, %<s>1"
4950 [(set_attr "type" "neon_fp_recp<FRECP:frecp_suffix>_<GPF:Vetype><GPF:q>")]
4951 )
4953 (define_insn "aarch64_frecps<mode>"
4954 [(set (match_operand:VALLF 0 "register_operand" "=w")
4955 (unspec:VALLF [(match_operand:VALLF 1 "register_operand" "w")
4956 (match_operand:VALLF 2 "register_operand" "w")]
4957 UNSPEC_FRECPS))]
4958 "TARGET_SIMD"
4959 "frecps\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
4960 [(set_attr "type" "neon_fp_recps_<Vetype><q>")]
4961 )
4963 ;; Standard pattern name vec_extract<mode>.
4965 (define_expand "vec_extract<mode>"
4966 [(match_operand:<VEL> 0 "aarch64_simd_nonimmediate_operand" "")
4967 (match_operand:VALL 1 "register_operand" "")
4968 (match_operand:SI 2 "immediate_operand" "")]
4969 "TARGET_SIMD"
4970 {
4971 emit_insn
4972 (gen_aarch64_get_lane<mode> (operands[0], operands[1], operands[2]));
4973 DONE;
4974 })
4976 ;; aes
4978 (define_insn "aarch64_crypto_aes<aes_op>v16qi"
4979 [(set (match_operand:V16QI 0 "register_operand" "=w")
4980 (unspec:V16QI [(match_operand:V16QI 1 "register_operand" "0")
4981 (match_operand:V16QI 2 "register_operand" "w")]
4982 CRYPTO_AES))]
4983 "TARGET_SIMD && TARGET_CRYPTO"
4984 "aes<aes_op>\\t%0.16b, %2.16b"
4985 [(set_attr "type" "crypto_aese")]
4986 )
4988 (define_insn "aarch64_crypto_aes<aesmc_op>v16qi"
4989 [(set (match_operand:V16QI 0 "register_operand" "=w")
4990 (unspec:V16QI [(match_operand:V16QI 1 "register_operand" "w")]
4991 CRYPTO_AESMC))]
4992 "TARGET_SIMD && TARGET_CRYPTO"
4993 "aes<aesmc_op>\\t%0.16b, %1.16b"
4994 [(set_attr "type" "crypto_aesmc")]
4995 )
4997 ;; sha1
4999 (define_insn "aarch64_crypto_sha1hsi"
5000 [(set (match_operand:SI 0 "register_operand" "=w")
5001 (unspec:SI [(match_operand:SI 1
5002 "register_operand" "w")]
5003 UNSPEC_SHA1H))]
5004 "TARGET_SIMD && TARGET_CRYPTO"
5005 "sha1h\\t%s0, %s1"
5006 [(set_attr "type" "crypto_sha1_fast")]
5007 )
5009 (define_insn "aarch64_crypto_sha1su1v4si"
5010 [(set (match_operand:V4SI 0 "register_operand" "=w")
5011 (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
5012 (match_operand:V4SI 2 "register_operand" "w")]
5013 UNSPEC_SHA1SU1))]
5014 "TARGET_SIMD && TARGET_CRYPTO"
5015 "sha1su1\\t%0.4s, %2.4s"
5016 [(set_attr "type" "crypto_sha1_fast")]
5017 )
5019 (define_insn "aarch64_crypto_sha1<sha1_op>v4si"
5020 [(set (match_operand:V4SI 0 "register_operand" "=w")
5021 (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
5022 (match_operand:SI 2 "register_operand" "w")
5023 (match_operand:V4SI 3 "register_operand" "w")]
5024 CRYPTO_SHA1))]
5025 "TARGET_SIMD && TARGET_CRYPTO"
5026 "sha1<sha1_op>\\t%q0, %s2, %3.4s"
5027 [(set_attr "type" "crypto_sha1_slow")]
5028 )
5030 (define_insn "aarch64_crypto_sha1su0v4si"
5031 [(set (match_operand:V4SI 0 "register_operand" "=w")
5032 (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
5033 (match_operand:V4SI 2 "register_operand" "w")
5034 (match_operand:V4SI 3 "register_operand" "w")]
5035 UNSPEC_SHA1SU0))]
5036 "TARGET_SIMD && TARGET_CRYPTO"
5037 "sha1su0\\t%0.4s, %2.4s, %3.4s"
5038 [(set_attr "type" "crypto_sha1_xor")]
5039 )
5041 ;; sha256
5043 (define_insn "aarch64_crypto_sha256h<sha256_op>v4si"
5044 [(set (match_operand:V4SI 0 "register_operand" "=w")
5045 (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
5046 (match_operand:V4SI 2 "register_operand" "w")
5047 (match_operand:V4SI 3 "register_operand" "w")]
5048 CRYPTO_SHA256))]
5049 "TARGET_SIMD && TARGET_CRYPTO"
5050 "sha256h<sha256_op>\\t%q0, %q2, %3.4s"
5051 [(set_attr "type" "crypto_sha256_slow")]
5052 )
5054 (define_insn "aarch64_crypto_sha256su0v4si"
5055 [(set (match_operand:V4SI 0 "register_operand" "=w")
5056 (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
5057 (match_operand:V4SI 2 "register_operand" "w")]
5058 UNSPEC_SHA256SU0))]
5059 "TARGET_SIMD &&TARGET_CRYPTO"
5060 "sha256su0\\t%0.4s, %2.4s"
5061 [(set_attr "type" "crypto_sha256_fast")]
5062 )
5064 (define_insn "aarch64_crypto_sha256su1v4si"
5065 [(set (match_operand:V4SI 0 "register_operand" "=w")
5066 (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
5067 (match_operand:V4SI 2 "register_operand" "w")
5068 (match_operand:V4SI 3 "register_operand" "w")]
5069 UNSPEC_SHA256SU1))]
5070 "TARGET_SIMD &&TARGET_CRYPTO"
5071 "sha256su1\\t%0.4s, %2.4s, %3.4s"
5072 [(set_attr "type" "crypto_sha256_slow")]
5073 )
5075 ;; pmull
5077 (define_insn "aarch64_crypto_pmulldi"
5078 [(set (match_operand:TI 0 "register_operand" "=w")
5079 (unspec:TI [(match_operand:DI 1 "register_operand" "w")
5080 (match_operand:DI 2 "register_operand" "w")]
5081 UNSPEC_PMULL))]
5082 "TARGET_SIMD && TARGET_CRYPTO"
5083 "pmull\\t%0.1q, %1.1d, %2.1d"
5084 [(set_attr "type" "neon_mul_d_long")]
5085 )
5087 (define_insn "aarch64_crypto_pmullv2di"
5088 [(set (match_operand:TI 0 "register_operand" "=w")
5089 (unspec:TI [(match_operand:V2DI 1 "register_operand" "w")
5090 (match_operand:V2DI 2 "register_operand" "w")]
5091 UNSPEC_PMULL2))]
5092 "TARGET_SIMD && TARGET_CRYPTO"
5093 "pmull2\\t%0.1q, %1.2d, %2.2d"
5094 [(set_attr "type" "neon_mul_d_long")]
5095 )