| blob | a94c6fdabdc2bcdc8e8fcdfec70f41e87f752a17 |
1 ;; Machine description for AArch64 AdvSIMD architecture.
2 ;; Copyright (C) 2011-2017 Free Software Foundation, Inc.
3 ;; Contributed by ARM Ltd.
4 ;;
5 ;; This file is part of GCC.
6 ;;
7 ;; GCC is free software; you can redistribute it and/or modify it
8 ;; under the terms of the GNU General Public License as published by
9 ;; the Free Software Foundation; either version 3, or (at your option)
10 ;; any later version.
11 ;;
12 ;; GCC is distributed in the hope that it will be useful, but
13 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
14 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 ;; General Public License for more details.
16 ;;
17 ;; You should have received a copy of the GNU General Public License
18 ;; along with GCC; see the file COPYING3. If not see
19 ;; <http://www.gnu.org/licenses/>.
21 (define_expand "mov<mode>"
22 [(set (match_operand:VALL_F16 0 "nonimmediate_operand" "")
23 (match_operand:VALL_F16 1 "general_operand" ""))]
24 "TARGET_SIMD"
25 "
26 if (GET_CODE (operands[0]) == MEM
27 && !(aarch64_simd_imm_zero (operands[1], <MODE>mode)
28 && aarch64_legitimate_address_p (<MODE>mode, operands[0],
29 PARALLEL, 1)))
30 operands[1] = force_reg (<MODE>mode, operands[1]);
31 "
32 )
34 (define_expand "movmisalign<mode>"
35 [(set (match_operand:VALL 0 "nonimmediate_operand" "")
36 (match_operand:VALL 1 "general_operand" ""))]
37 "TARGET_SIMD"
38 {
39 /* This pattern is not permitted to fail during expansion: if both arguments
40 are non-registers (e.g. memory := constant, which can be created by the
41 auto-vectorizer), force operand 1 into a register. */
42 if (!register_operand (operands[0], <MODE>mode)
43 && !register_operand (operands[1], <MODE>mode))
44 operands[1] = force_reg (<MODE>mode, operands[1]);
45 })
47 (define_insn "aarch64_simd_dup<mode>"
48 [(set (match_operand:VDQ_I 0 "register_operand" "=w, w")
49 (vec_duplicate:VDQ_I
50 (match_operand:<VEL> 1 "register_operand" "w,?r")))]
51 "TARGET_SIMD"
52 "@
53 dup\\t%0.<Vtype>, %1.<Vetype>[0]
54 dup\\t%0.<Vtype>, %<vw>1"
55 [(set_attr "type" "neon_dup<q>, neon_from_gp<q>")]
56 )
58 (define_insn "aarch64_simd_dup<mode>"
59 [(set (match_operand:VDQF_F16 0 "register_operand" "=w")
60 (vec_duplicate:VDQF_F16
61 (match_operand:<VEL> 1 "register_operand" "w")))]
62 "TARGET_SIMD"
63 "dup\\t%0.<Vtype>, %1.<Vetype>[0]"
64 [(set_attr "type" "neon_dup<q>")]
65 )
67 (define_insn "aarch64_dup_lane<mode>"
68 [(set (match_operand:VALL_F16 0 "register_operand" "=w")
69 (vec_duplicate:VALL_F16
70 (vec_select:<VEL>
71 (match_operand:VALL_F16 1 "register_operand" "w")
72 (parallel [(match_operand:SI 2 "immediate_operand" "i")])
73 )))]
74 "TARGET_SIMD"
75 {
76 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
77 return "dup\\t%0.<Vtype>, %1.<Vetype>[%2]";
78 }
79 [(set_attr "type" "neon_dup<q>")]
80 )
82 (define_insn "aarch64_dup_lane_<vswap_width_name><mode>"
83 [(set (match_operand:VALL_F16_NO_V2Q 0 "register_operand" "=w")
84 (vec_duplicate:VALL_F16_NO_V2Q
85 (vec_select:<VEL>
86 (match_operand:<VSWAP_WIDTH> 1 "register_operand" "w")
87 (parallel [(match_operand:SI 2 "immediate_operand" "i")])
88 )))]
89 "TARGET_SIMD"
90 {
91 operands[2] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
92 INTVAL (operands[2])));
93 return "dup\\t%0.<Vtype>, %1.<Vetype>[%2]";
94 }
95 [(set_attr "type" "neon_dup<q>")]
96 )
98 (define_insn "*aarch64_simd_mov<mode>"
99 [(set (match_operand:VD 0 "nonimmediate_operand"
100 "=w, m, m, w, ?r, ?w, ?r, w")
101 (match_operand:VD 1 "general_operand"
102 "m, Dz, w, w, w, r, r, Dn"))]
103 "TARGET_SIMD
104 && (register_operand (operands[0], <MODE>mode)
105 || aarch64_simd_reg_or_zero (operands[1], <MODE>mode))"
106 {
107 switch (which_alternative)
108 {
109 case 0: return "ldr\t%d0, %1";
110 case 1: return "str\txzr, %0";
111 case 2: return "str\t%d1, %0";
112 case 3: return "mov\t%0.<Vbtype>, %1.<Vbtype>";
113 case 4: return "umov\t%0, %1.d[0]";
114 case 5: return "fmov\t%d0, %1";
115 case 6: return "mov\t%0, %1";
116 case 7:
117 return aarch64_output_simd_mov_immediate (operands[1],
118 <MODE>mode, 64);
119 default: gcc_unreachable ();
120 }
121 }
122 [(set_attr "type" "neon_load1_1reg<q>, neon_stp, neon_store1_1reg<q>,\
123 neon_logic<q>, neon_to_gp<q>, f_mcr,\
124 mov_reg, neon_move<q>")]
125 )
127 (define_insn "*aarch64_simd_mov<mode>"
128 [(set (match_operand:VQ 0 "nonimmediate_operand"
129 "=w, Ump, m, w, ?r, ?w, ?r, w")
130 (match_operand:VQ 1 "general_operand"
131 "m, Dz, w, w, w, r, r, Dn"))]
132 "TARGET_SIMD
133 && (register_operand (operands[0], <MODE>mode)
134 || aarch64_simd_reg_or_zero (operands[1], <MODE>mode))"
135 {
136 switch (which_alternative)
137 {
138 case 0:
139 return "ldr\t%q0, %1";
140 case 1:
141 return "stp\txzr, xzr, %0";
142 case 2:
143 return "str\t%q1, %0";
144 case 3:
145 return "mov\t%0.<Vbtype>, %1.<Vbtype>";
146 case 4:
147 case 5:
148 case 6:
149 return "#";
150 case 7:
151 return aarch64_output_simd_mov_immediate (operands[1], <MODE>mode, 128);
152 default:
153 gcc_unreachable ();
154 }
155 }
156 [(set_attr "type" "neon_load1_1reg<q>, neon_store1_1reg<q>,\
157 neon_stp, neon_logic<q>, multiple, multiple,\
158 multiple, neon_move<q>")
159 (set_attr "length" "4,4,4,4,8,8,8,4")]
160 )
162 ;; When storing lane zero we can use the normal STR and its more permissive
163 ;; addressing modes.
165 (define_insn "aarch64_store_lane0<mode>"
166 [(set (match_operand:<VEL> 0 "memory_operand" "=m")
167 (vec_select:<VEL> (match_operand:VALL_F16 1 "register_operand" "w")
168 (parallel [(match_operand 2 "const_int_operand" "n")])))]
169 "TARGET_SIMD
170 && ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])) == 0"
171 "str\\t%<Vetype>1, %0"
172 [(set_attr "type" "neon_store1_1reg<q>")]
173 )
175 (define_insn "load_pair<mode>"
176 [(set (match_operand:VD 0 "register_operand" "=w")
177 (match_operand:VD 1 "aarch64_mem_pair_operand" "Ump"))
178 (set (match_operand:VD 2 "register_operand" "=w")
179 (match_operand:VD 3 "memory_operand" "m"))]
180 "TARGET_SIMD
181 && rtx_equal_p (XEXP (operands[3], 0),
182 plus_constant (Pmode,
183 XEXP (operands[1], 0),
184 GET_MODE_SIZE (<MODE>mode)))"
185 "ldp\\t%d0, %d2, %1"
186 [(set_attr "type" "neon_ldp")]
187 )
189 (define_insn "store_pair<mode>"
190 [(set (match_operand:VD 0 "aarch64_mem_pair_operand" "=Ump")
191 (match_operand:VD 1 "register_operand" "w"))
192 (set (match_operand:VD 2 "memory_operand" "=m")
193 (match_operand:VD 3 "register_operand" "w"))]
194 "TARGET_SIMD
195 && rtx_equal_p (XEXP (operands[2], 0),
196 plus_constant (Pmode,
197 XEXP (operands[0], 0),
198 GET_MODE_SIZE (<MODE>mode)))"
199 "stp\\t%d1, %d3, %0"
200 [(set_attr "type" "neon_stp")]
201 )
203 (define_split
204 [(set (match_operand:VQ 0 "register_operand" "")
205 (match_operand:VQ 1 "register_operand" ""))]
206 "TARGET_SIMD && reload_completed
207 && GP_REGNUM_P (REGNO (operands[0]))
208 && GP_REGNUM_P (REGNO (operands[1]))"
209 [(const_int 0)]
210 {
211 aarch64_simd_emit_reg_reg_move (operands, DImode, 2);
212 DONE;
213 })
215 (define_split
216 [(set (match_operand:VQ 0 "register_operand" "")
217 (match_operand:VQ 1 "register_operand" ""))]
218 "TARGET_SIMD && reload_completed
219 && ((FP_REGNUM_P (REGNO (operands[0])) && GP_REGNUM_P (REGNO (operands[1])))
220 || (GP_REGNUM_P (REGNO (operands[0])) && FP_REGNUM_P (REGNO (operands[1]))))"
221 [(const_int 0)]
222 {
223 aarch64_split_simd_move (operands[0], operands[1]);
224 DONE;
225 })
227 (define_expand "aarch64_split_simd_mov<mode>"
228 [(set (match_operand:VQ 0)
229 (match_operand:VQ 1))]
230 "TARGET_SIMD"
231 {
232 rtx dst = operands[0];
233 rtx src = operands[1];
235 if (GP_REGNUM_P (REGNO (src)))
236 {
237 rtx src_low_part = gen_lowpart (<VHALF>mode, src);
238 rtx src_high_part = gen_highpart (<VHALF>mode, src);
240 emit_insn
241 (gen_move_lo_quad_<mode> (dst, src_low_part));
242 emit_insn
243 (gen_move_hi_quad_<mode> (dst, src_high_part));
244 }
246 else
247 {
248 rtx dst_low_part = gen_lowpart (<VHALF>mode, dst);
249 rtx dst_high_part = gen_highpart (<VHALF>mode, dst);
250 rtx lo = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
251 rtx hi = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
253 emit_insn
254 (gen_aarch64_simd_mov_from_<mode>low (dst_low_part, src, lo));
255 emit_insn
256 (gen_aarch64_simd_mov_from_<mode>high (dst_high_part, src, hi));
257 }
258 DONE;
259 }
260 )
262 (define_insn "aarch64_simd_mov_from_<mode>low"
263 [(set (match_operand:<VHALF> 0 "register_operand" "=r")
264 (vec_select:<VHALF>
265 (match_operand:VQ 1 "register_operand" "w")
266 (match_operand:VQ 2 "vect_par_cnst_lo_half" "")))]
267 "TARGET_SIMD && reload_completed"
268 "umov\t%0, %1.d[0]"
269 [(set_attr "type" "neon_to_gp<q>")
270 (set_attr "length" "4")
271 ])
273 (define_insn "aarch64_simd_mov_from_<mode>high"
274 [(set (match_operand:<VHALF> 0 "register_operand" "=r")
275 (vec_select:<VHALF>
276 (match_operand:VQ 1 "register_operand" "w")
277 (match_operand:VQ 2 "vect_par_cnst_hi_half" "")))]
278 "TARGET_SIMD && reload_completed"
279 "umov\t%0, %1.d[1]"
280 [(set_attr "type" "neon_to_gp<q>")
281 (set_attr "length" "4")
282 ])
284 (define_insn "orn<mode>3"
285 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
286 (ior:VDQ_I (not:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w"))
287 (match_operand:VDQ_I 2 "register_operand" "w")))]
288 "TARGET_SIMD"
289 "orn\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>"
290 [(set_attr "type" "neon_logic<q>")]
291 )
293 (define_insn "bic<mode>3"
294 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
295 (and:VDQ_I (not:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w"))
296 (match_operand:VDQ_I 2 "register_operand" "w")))]
297 "TARGET_SIMD"
298 "bic\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>"
299 [(set_attr "type" "neon_logic<q>")]
300 )
302 (define_insn "add<mode>3"
303 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
304 (plus:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
305 (match_operand:VDQ_I 2 "register_operand" "w")))]
306 "TARGET_SIMD"
307 "add\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
308 [(set_attr "type" "neon_add<q>")]
309 )
311 (define_insn "sub<mode>3"
312 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
313 (minus:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
314 (match_operand:VDQ_I 2 "register_operand" "w")))]
315 "TARGET_SIMD"
316 "sub\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
317 [(set_attr "type" "neon_sub<q>")]
318 )
320 (define_insn "mul<mode>3"
321 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
322 (mult:VDQ_BHSI (match_operand:VDQ_BHSI 1 "register_operand" "w")
323 (match_operand:VDQ_BHSI 2 "register_operand" "w")))]
324 "TARGET_SIMD"
325 "mul\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
326 [(set_attr "type" "neon_mul_<Vetype><q>")]
327 )
329 (define_insn "bswap<mode>2"
330 [(set (match_operand:VDQHSD 0 "register_operand" "=w")
331 (bswap:VDQHSD (match_operand:VDQHSD 1 "register_operand" "w")))]
332 "TARGET_SIMD"
333 "rev<Vrevsuff>\\t%0.<Vbtype>, %1.<Vbtype>"
334 [(set_attr "type" "neon_rev<q>")]
335 )
337 (define_insn "aarch64_rbit<mode>"
338 [(set (match_operand:VB 0 "register_operand" "=w")
339 (unspec:VB [(match_operand:VB 1 "register_operand" "w")]
340 UNSPEC_RBIT))]
341 "TARGET_SIMD"
342 "rbit\\t%0.<Vbtype>, %1.<Vbtype>"
343 [(set_attr "type" "neon_rbit")]
344 )
346 (define_expand "ctz<mode>2"
347 [(set (match_operand:VS 0 "register_operand")
348 (ctz:VS (match_operand:VS 1 "register_operand")))]
349 "TARGET_SIMD"
350 {
351 emit_insn (gen_bswap<mode>2 (operands[0], operands[1]));
352 rtx op0_castsi2qi = simplify_gen_subreg(<VS:VSI2QI>mode, operands[0],
353 <MODE>mode, 0);
354 emit_insn (gen_aarch64_rbit<VS:vsi2qi> (op0_castsi2qi, op0_castsi2qi));
355 emit_insn (gen_clz<mode>2 (operands[0], operands[0]));
356 DONE;
357 }
358 )
360 (define_expand "xorsign<mode>3"
361 [(match_operand:VHSDF 0 "register_operand")
362 (match_operand:VHSDF 1 "register_operand")
363 (match_operand:VHSDF 2 "register_operand")]
364 "TARGET_SIMD"
365 {
367 machine_mode imode = <V_INT_EQUIV>mode;
368 rtx v_bitmask = gen_reg_rtx (imode);
369 rtx op1x = gen_reg_rtx (imode);
370 rtx op2x = gen_reg_rtx (imode);
372 rtx arg1 = lowpart_subreg (imode, operands[1], <MODE>mode);
373 rtx arg2 = lowpart_subreg (imode, operands[2], <MODE>mode);
375 int bits = GET_MODE_UNIT_BITSIZE (<MODE>mode) - 1;
377 emit_move_insn (v_bitmask,
378 aarch64_simd_gen_const_vector_dup (<V_INT_EQUIV>mode,
379 HOST_WIDE_INT_M1U << bits));
381 emit_insn (gen_and<v_int_equiv>3 (op2x, v_bitmask, arg2));
382 emit_insn (gen_xor<v_int_equiv>3 (op1x, arg1, op2x));
383 emit_move_insn (operands[0],
384 lowpart_subreg (<MODE>mode, op1x, imode));
385 DONE;
386 }
387 )
389 (define_expand "copysign<mode>3"
390 [(match_operand:VHSDF 0 "register_operand")
391 (match_operand:VHSDF 1 "register_operand")
392 (match_operand:VHSDF 2 "register_operand")]
393 "TARGET_FLOAT && TARGET_SIMD"
394 {
395 rtx v_bitmask = gen_reg_rtx (<V_INT_EQUIV>mode);
396 int bits = GET_MODE_UNIT_BITSIZE (<MODE>mode) - 1;
398 emit_move_insn (v_bitmask,
399 aarch64_simd_gen_const_vector_dup (<V_INT_EQUIV>mode,
400 HOST_WIDE_INT_M1U << bits));
401 emit_insn (gen_aarch64_simd_bsl<mode> (operands[0], v_bitmask,
402 operands[2], operands[1]));
403 DONE;
404 }
405 )
407 (define_insn "*aarch64_mul3_elt<mode>"
408 [(set (match_operand:VMUL 0 "register_operand" "=w")
409 (mult:VMUL
410 (vec_duplicate:VMUL
411 (vec_select:<VEL>
412 (match_operand:VMUL 1 "register_operand" "<h_con>")
413 (parallel [(match_operand:SI 2 "immediate_operand")])))
414 (match_operand:VMUL 3 "register_operand" "w")))]
415 "TARGET_SIMD"
416 {
417 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
418 return "<f>mul\\t%0.<Vtype>, %3.<Vtype>, %1.<Vetype>[%2]";
419 }
420 [(set_attr "type" "neon<fp>_mul_<stype>_scalar<q>")]
421 )
423 (define_insn "*aarch64_mul3_elt_<vswap_width_name><mode>"
424 [(set (match_operand:VMUL_CHANGE_NLANES 0 "register_operand" "=w")
425 (mult:VMUL_CHANGE_NLANES
426 (vec_duplicate:VMUL_CHANGE_NLANES
427 (vec_select:<VEL>
428 (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
429 (parallel [(match_operand:SI 2 "immediate_operand")])))
430 (match_operand:VMUL_CHANGE_NLANES 3 "register_operand" "w")))]
431 "TARGET_SIMD"
432 {
433 operands[2] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
434 INTVAL (operands[2])));
435 return "<f>mul\\t%0.<Vtype>, %3.<Vtype>, %1.<Vetype>[%2]";
436 }
437 [(set_attr "type" "neon<fp>_mul_<Vetype>_scalar<q>")]
438 )
440 (define_insn "*aarch64_mul3_elt_from_dup<mode>"
441 [(set (match_operand:VMUL 0 "register_operand" "=w")
442 (mult:VMUL
443 (vec_duplicate:VMUL
444 (match_operand:<VEL> 1 "register_operand" "<h_con>"))
445 (match_operand:VMUL 2 "register_operand" "w")))]
446 "TARGET_SIMD"
447 "<f>mul\t%0.<Vtype>, %2.<Vtype>, %1.<Vetype>[0]";
448 [(set_attr "type" "neon<fp>_mul_<stype>_scalar<q>")]
449 )
451 (define_insn "aarch64_rsqrte<mode>"
452 [(set (match_operand:VHSDF_HSDF 0 "register_operand" "=w")
453 (unspec:VHSDF_HSDF [(match_operand:VHSDF_HSDF 1 "register_operand" "w")]
454 UNSPEC_RSQRTE))]
455 "TARGET_SIMD"
456 "frsqrte\\t%<v>0<Vmtype>, %<v>1<Vmtype>"
457 [(set_attr "type" "neon_fp_rsqrte_<stype><q>")])
459 (define_insn "aarch64_rsqrts<mode>"
460 [(set (match_operand:VHSDF_HSDF 0 "register_operand" "=w")
461 (unspec:VHSDF_HSDF [(match_operand:VHSDF_HSDF 1 "register_operand" "w")
462 (match_operand:VHSDF_HSDF 2 "register_operand" "w")]
463 UNSPEC_RSQRTS))]
464 "TARGET_SIMD"
465 "frsqrts\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
466 [(set_attr "type" "neon_fp_rsqrts_<stype><q>")])
468 (define_expand "rsqrt<mode>2"
469 [(set (match_operand:VALLF 0 "register_operand" "=w")
470 (unspec:VALLF [(match_operand:VALLF 1 "register_operand" "w")]
471 UNSPEC_RSQRT))]
472 "TARGET_SIMD"
473 {
474 aarch64_emit_approx_sqrt (operands[0], operands[1], true);
475 DONE;
476 })
478 (define_insn "*aarch64_mul3_elt_to_64v2df"
479 [(set (match_operand:DF 0 "register_operand" "=w")
480 (mult:DF
481 (vec_select:DF
482 (match_operand:V2DF 1 "register_operand" "w")
483 (parallel [(match_operand:SI 2 "immediate_operand")]))
484 (match_operand:DF 3 "register_operand" "w")))]
485 "TARGET_SIMD"
486 {
487 operands[2] = GEN_INT (ENDIAN_LANE_N (V2DFmode, INTVAL (operands[2])));
488 return "fmul\\t%0.2d, %3.2d, %1.d[%2]";
489 }
490 [(set_attr "type" "neon_fp_mul_d_scalar_q")]
491 )
493 (define_insn "neg<mode>2"
494 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
495 (neg:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")))]
496 "TARGET_SIMD"
497 "neg\t%0.<Vtype>, %1.<Vtype>"
498 [(set_attr "type" "neon_neg<q>")]
499 )
501 (define_insn "abs<mode>2"
502 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
503 (abs:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")))]
504 "TARGET_SIMD"
505 "abs\t%0.<Vtype>, %1.<Vtype>"
506 [(set_attr "type" "neon_abs<q>")]
507 )
509 ;; The intrinsic version of integer ABS must not be allowed to
510 ;; combine with any operation with an integerated ABS step, such
511 ;; as SABD.
512 (define_insn "aarch64_abs<mode>"
513 [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
514 (unspec:VSDQ_I_DI
515 [(match_operand:VSDQ_I_DI 1 "register_operand" "w")]
516 UNSPEC_ABS))]
517 "TARGET_SIMD"
518 "abs\t%<v>0<Vmtype>, %<v>1<Vmtype>"
519 [(set_attr "type" "neon_abs<q>")]
520 )
522 (define_insn "abd<mode>_3"
523 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
524 (abs:VDQ_BHSI (minus:VDQ_BHSI
525 (match_operand:VDQ_BHSI 1 "register_operand" "w")
526 (match_operand:VDQ_BHSI 2 "register_operand" "w"))))]
527 "TARGET_SIMD"
528 "sabd\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
529 [(set_attr "type" "neon_abd<q>")]
530 )
532 (define_insn "aba<mode>_3"
533 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
534 (plus:VDQ_BHSI (abs:VDQ_BHSI (minus:VDQ_BHSI
535 (match_operand:VDQ_BHSI 1 "register_operand" "w")
536 (match_operand:VDQ_BHSI 2 "register_operand" "w")))
537 (match_operand:VDQ_BHSI 3 "register_operand" "0")))]
538 "TARGET_SIMD"
539 "saba\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
540 [(set_attr "type" "neon_arith_acc<q>")]
541 )
543 (define_insn "fabd<mode>3"
544 [(set (match_operand:VHSDF_HSDF 0 "register_operand" "=w")
545 (abs:VHSDF_HSDF
546 (minus:VHSDF_HSDF
547 (match_operand:VHSDF_HSDF 1 "register_operand" "w")
548 (match_operand:VHSDF_HSDF 2 "register_operand" "w"))))]
549 "TARGET_SIMD"
550 "fabd\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
551 [(set_attr "type" "neon_fp_abd_<stype><q>")]
552 )
554 (define_insn "and<mode>3"
555 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
556 (and:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
557 (match_operand:VDQ_I 2 "register_operand" "w")))]
558 "TARGET_SIMD"
559 "and\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>"
560 [(set_attr "type" "neon_logic<q>")]
561 )
563 (define_insn "ior<mode>3"
564 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
565 (ior:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
566 (match_operand:VDQ_I 2 "register_operand" "w")))]
567 "TARGET_SIMD"
568 "orr\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>"
569 [(set_attr "type" "neon_logic<q>")]
570 )
572 (define_insn "xor<mode>3"
573 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
574 (xor:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
575 (match_operand:VDQ_I 2 "register_operand" "w")))]
576 "TARGET_SIMD"
577 "eor\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>"
578 [(set_attr "type" "neon_logic<q>")]
579 )
581 (define_insn "one_cmpl<mode>2"
582 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
583 (not:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")))]
584 "TARGET_SIMD"
585 "not\t%0.<Vbtype>, %1.<Vbtype>"
586 [(set_attr "type" "neon_logic<q>")]
587 )
589 (define_insn "aarch64_simd_vec_set<mode>"
590 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w,w,w")
591 (vec_merge:VDQ_BHSI
592 (vec_duplicate:VDQ_BHSI
593 (match_operand:<VEL> 1 "aarch64_simd_general_operand" "r,w,Utv"))
594 (match_operand:VDQ_BHSI 3 "register_operand" "0,0,0")
595 (match_operand:SI 2 "immediate_operand" "i,i,i")))]
596 "TARGET_SIMD"
597 {
598 int elt = ENDIAN_LANE_N (<MODE>mode, exact_log2 (INTVAL (operands[2])));
599 operands[2] = GEN_INT ((HOST_WIDE_INT) 1 << elt);
600 switch (which_alternative)
601 {
602 case 0:
603 return "ins\\t%0.<Vetype>[%p2], %w1";
604 case 1:
605 return "ins\\t%0.<Vetype>[%p2], %1.<Vetype>[0]";
606 case 2:
607 return "ld1\\t{%0.<Vetype>}[%p2], %1";
608 default:
609 gcc_unreachable ();
610 }
611 }
612 [(set_attr "type" "neon_from_gp<q>, neon_ins<q>, neon_load1_one_lane<q>")]
613 )
615 (define_insn "*aarch64_simd_vec_copy_lane<mode>"
616 [(set (match_operand:VALL_F16 0 "register_operand" "=w")
617 (vec_merge:VALL_F16
618 (vec_duplicate:VALL_F16
619 (vec_select:<VEL>
620 (match_operand:VALL_F16 3 "register_operand" "w")
621 (parallel
622 [(match_operand:SI 4 "immediate_operand" "i")])))
623 (match_operand:VALL_F16 1 "register_operand" "0")
624 (match_operand:SI 2 "immediate_operand" "i")))]
625 "TARGET_SIMD"
626 {
627 int elt = ENDIAN_LANE_N (<MODE>mode, exact_log2 (INTVAL (operands[2])));
628 operands[2] = GEN_INT (HOST_WIDE_INT_1 << elt);
629 operands[4] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[4])));
631 return "ins\t%0.<Vetype>[%p2], %3.<Vetype>[%4]";
632 }
633 [(set_attr "type" "neon_ins<q>")]
634 )
636 (define_insn "*aarch64_simd_vec_copy_lane_<vswap_width_name><mode>"
637 [(set (match_operand:VALL_F16_NO_V2Q 0 "register_operand" "=w")
638 (vec_merge:VALL_F16_NO_V2Q
639 (vec_duplicate:VALL_F16_NO_V2Q
640 (vec_select:<VEL>
641 (match_operand:<VSWAP_WIDTH> 3 "register_operand" "w")
642 (parallel
643 [(match_operand:SI 4 "immediate_operand" "i")])))
644 (match_operand:VALL_F16_NO_V2Q 1 "register_operand" "0")
645 (match_operand:SI 2 "immediate_operand" "i")))]
646 "TARGET_SIMD"
647 {
648 int elt = ENDIAN_LANE_N (<MODE>mode, exact_log2 (INTVAL (operands[2])));
649 operands[2] = GEN_INT (HOST_WIDE_INT_1 << elt);
650 operands[4] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
651 INTVAL (operands[4])));
653 return "ins\t%0.<Vetype>[%p2], %3.<Vetype>[%4]";
654 }
655 [(set_attr "type" "neon_ins<q>")]
656 )
658 (define_insn "aarch64_simd_lshr<mode>"
659 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
660 (lshiftrt:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
661 (match_operand:VDQ_I 2 "aarch64_simd_rshift_imm" "Dr")))]
662 "TARGET_SIMD"
663 "ushr\t%0.<Vtype>, %1.<Vtype>, %2"
664 [(set_attr "type" "neon_shift_imm<q>")]
665 )
667 (define_insn "aarch64_simd_ashr<mode>"
668 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
669 (ashiftrt:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
670 (match_operand:VDQ_I 2 "aarch64_simd_rshift_imm" "Dr")))]
671 "TARGET_SIMD"
672 "sshr\t%0.<Vtype>, %1.<Vtype>, %2"
673 [(set_attr "type" "neon_shift_imm<q>")]
674 )
676 (define_insn "aarch64_simd_imm_shl<mode>"
677 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
678 (ashift:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
679 (match_operand:VDQ_I 2 "aarch64_simd_lshift_imm" "Dl")))]
680 "TARGET_SIMD"
681 "shl\t%0.<Vtype>, %1.<Vtype>, %2"
682 [(set_attr "type" "neon_shift_imm<q>")]
683 )
685 (define_insn "aarch64_simd_reg_sshl<mode>"
686 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
687 (ashift:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
688 (match_operand:VDQ_I 2 "register_operand" "w")))]
689 "TARGET_SIMD"
690 "sshl\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
691 [(set_attr "type" "neon_shift_reg<q>")]
692 )
694 (define_insn "aarch64_simd_reg_shl<mode>_unsigned"
695 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
696 (unspec:VDQ_I [(match_operand:VDQ_I 1 "register_operand" "w")
697 (match_operand:VDQ_I 2 "register_operand" "w")]
698 UNSPEC_ASHIFT_UNSIGNED))]
699 "TARGET_SIMD"
700 "ushl\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
701 [(set_attr "type" "neon_shift_reg<q>")]
702 )
704 (define_insn "aarch64_simd_reg_shl<mode>_signed"
705 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
706 (unspec:VDQ_I [(match_operand:VDQ_I 1 "register_operand" "w")
707 (match_operand:VDQ_I 2 "register_operand" "w")]
708 UNSPEC_ASHIFT_SIGNED))]
709 "TARGET_SIMD"
710 "sshl\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
711 [(set_attr "type" "neon_shift_reg<q>")]
712 )
714 (define_expand "ashl<mode>3"
715 [(match_operand:VDQ_I 0 "register_operand" "")
716 (match_operand:VDQ_I 1 "register_operand" "")
717 (match_operand:SI 2 "general_operand" "")]
718 "TARGET_SIMD"
719 {
720 int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
721 int shift_amount;
723 if (CONST_INT_P (operands[2]))
724 {
725 shift_amount = INTVAL (operands[2]);
726 if (shift_amount >= 0 && shift_amount < bit_width)
727 {
728 rtx tmp = aarch64_simd_gen_const_vector_dup (<MODE>mode,
729 shift_amount);
730 emit_insn (gen_aarch64_simd_imm_shl<mode> (operands[0],
731 operands[1],
732 tmp));
733 DONE;
734 }
735 else
736 {
737 operands[2] = force_reg (SImode, operands[2]);
738 }
739 }
740 else if (MEM_P (operands[2]))
741 {
742 operands[2] = force_reg (SImode, operands[2]);
743 }
745 if (REG_P (operands[2]))
746 {
747 rtx tmp = gen_reg_rtx (<MODE>mode);
748 emit_insn (gen_aarch64_simd_dup<mode> (tmp,
749 convert_to_mode (<VEL>mode,
750 operands[2],
751 0)));
752 emit_insn (gen_aarch64_simd_reg_sshl<mode> (operands[0], operands[1],
753 tmp));
754 DONE;
755 }
756 else
757 FAIL;
758 }
759 )
761 (define_expand "lshr<mode>3"
762 [(match_operand:VDQ_I 0 "register_operand" "")
763 (match_operand:VDQ_I 1 "register_operand" "")
764 (match_operand:SI 2 "general_operand" "")]
765 "TARGET_SIMD"
766 {
767 int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
768 int shift_amount;
770 if (CONST_INT_P (operands[2]))
771 {
772 shift_amount = INTVAL (operands[2]);
773 if (shift_amount > 0 && shift_amount <= bit_width)
774 {
775 rtx tmp = aarch64_simd_gen_const_vector_dup (<MODE>mode,
776 shift_amount);
777 emit_insn (gen_aarch64_simd_lshr<mode> (operands[0],
778 operands[1],
779 tmp));
780 DONE;
781 }
782 else
783 operands[2] = force_reg (SImode, operands[2]);
784 }
785 else if (MEM_P (operands[2]))
786 {
787 operands[2] = force_reg (SImode, operands[2]);
788 }
790 if (REG_P (operands[2]))
791 {
792 rtx tmp = gen_reg_rtx (SImode);
793 rtx tmp1 = gen_reg_rtx (<MODE>mode);
794 emit_insn (gen_negsi2 (tmp, operands[2]));
795 emit_insn (gen_aarch64_simd_dup<mode> (tmp1,
796 convert_to_mode (<VEL>mode,
797 tmp, 0)));
798 emit_insn (gen_aarch64_simd_reg_shl<mode>_unsigned (operands[0],
799 operands[1],
800 tmp1));
801 DONE;
802 }
803 else
804 FAIL;
805 }
806 )
808 (define_expand "ashr<mode>3"
809 [(match_operand:VDQ_I 0 "register_operand" "")
810 (match_operand:VDQ_I 1 "register_operand" "")
811 (match_operand:SI 2 "general_operand" "")]
812 "TARGET_SIMD"
813 {
814 int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
815 int shift_amount;
817 if (CONST_INT_P (operands[2]))
818 {
819 shift_amount = INTVAL (operands[2]);
820 if (shift_amount > 0 && shift_amount <= bit_width)
821 {
822 rtx tmp = aarch64_simd_gen_const_vector_dup (<MODE>mode,
823 shift_amount);
824 emit_insn (gen_aarch64_simd_ashr<mode> (operands[0],
825 operands[1],
826 tmp));
827 DONE;
828 }
829 else
830 operands[2] = force_reg (SImode, operands[2]);
831 }
832 else if (MEM_P (operands[2]))
833 {
834 operands[2] = force_reg (SImode, operands[2]);
835 }
837 if (REG_P (operands[2]))
838 {
839 rtx tmp = gen_reg_rtx (SImode);
840 rtx tmp1 = gen_reg_rtx (<MODE>mode);
841 emit_insn (gen_negsi2 (tmp, operands[2]));
842 emit_insn (gen_aarch64_simd_dup<mode> (tmp1,
843 convert_to_mode (<VEL>mode,
844 tmp, 0)));
845 emit_insn (gen_aarch64_simd_reg_shl<mode>_signed (operands[0],
846 operands[1],
847 tmp1));
848 DONE;
849 }
850 else
851 FAIL;
852 }
853 )
855 (define_expand "vashl<mode>3"
856 [(match_operand:VDQ_I 0 "register_operand" "")
857 (match_operand:VDQ_I 1 "register_operand" "")
858 (match_operand:VDQ_I 2 "register_operand" "")]
859 "TARGET_SIMD"
860 {
861 emit_insn (gen_aarch64_simd_reg_sshl<mode> (operands[0], operands[1],
862 operands[2]));
863 DONE;
864 })
866 ;; Using mode VDQ_BHSI as there is no V2DImode neg!
867 ;; Negating individual lanes most certainly offsets the
868 ;; gain from vectorization.
869 (define_expand "vashr<mode>3"
870 [(match_operand:VDQ_BHSI 0 "register_operand" "")
871 (match_operand:VDQ_BHSI 1 "register_operand" "")
872 (match_operand:VDQ_BHSI 2 "register_operand" "")]
873 "TARGET_SIMD"
874 {
875 rtx neg = gen_reg_rtx (<MODE>mode);
876 emit (gen_neg<mode>2 (neg, operands[2]));
877 emit_insn (gen_aarch64_simd_reg_shl<mode>_signed (operands[0], operands[1],
878 neg));
879 DONE;
880 })
882 ;; DI vector shift
883 (define_expand "aarch64_ashr_simddi"
884 [(match_operand:DI 0 "register_operand" "=w")
885 (match_operand:DI 1 "register_operand" "w")
886 (match_operand:SI 2 "aarch64_shift_imm64_di" "")]
887 "TARGET_SIMD"
888 {
889 /* An arithmetic shift right by 64 fills the result with copies of the sign
890 bit, just like asr by 63 - however the standard pattern does not handle
891 a shift by 64. */
892 if (INTVAL (operands[2]) == 64)
893 operands[2] = GEN_INT (63);
894 emit_insn (gen_ashrdi3 (operands[0], operands[1], operands[2]));
895 DONE;
896 }
897 )
899 (define_expand "vlshr<mode>3"
900 [(match_operand:VDQ_BHSI 0 "register_operand" "")
901 (match_operand:VDQ_BHSI 1 "register_operand" "")
902 (match_operand:VDQ_BHSI 2 "register_operand" "")]
903 "TARGET_SIMD"
904 {
905 rtx neg = gen_reg_rtx (<MODE>mode);
906 emit (gen_neg<mode>2 (neg, operands[2]));
907 emit_insn (gen_aarch64_simd_reg_shl<mode>_unsigned (operands[0], operands[1],
908 neg));
909 DONE;
910 })
912 (define_expand "aarch64_lshr_simddi"
913 [(match_operand:DI 0 "register_operand" "=w")
914 (match_operand:DI 1 "register_operand" "w")
915 (match_operand:SI 2 "aarch64_shift_imm64_di" "")]
916 "TARGET_SIMD"
917 {
918 if (INTVAL (operands[2]) == 64)
919 emit_move_insn (operands[0], const0_rtx);
920 else
921 emit_insn (gen_lshrdi3 (operands[0], operands[1], operands[2]));
922 DONE;
923 }
924 )
926 (define_expand "vec_set<mode>"
927 [(match_operand:VDQ_BHSI 0 "register_operand")
928 (match_operand:<VEL> 1 "register_operand")
929 (match_operand:SI 2 "immediate_operand")]
930 "TARGET_SIMD"
931 {
932 HOST_WIDE_INT elem = (HOST_WIDE_INT) 1 << INTVAL (operands[2]);
933 emit_insn (gen_aarch64_simd_vec_set<mode> (operands[0], operands[1],
934 GEN_INT (elem), operands[0]));
935 DONE;
936 }
937 )
939 ;; For 64-bit modes we use ushl/r, as this does not require a SIMD zero.
940 (define_insn "vec_shr_<mode>"
941 [(set (match_operand:VD 0 "register_operand" "=w")
942 (unspec:VD [(match_operand:VD 1 "register_operand" "w")
943 (match_operand:SI 2 "immediate_operand" "i")]
944 UNSPEC_VEC_SHR))]
945 "TARGET_SIMD"
946 {
947 if (BYTES_BIG_ENDIAN)
948 return "shl %d0, %d1, %2";
949 else
950 return "ushr %d0, %d1, %2";
951 }
952 [(set_attr "type" "neon_shift_imm")]
953 )
955 (define_insn "aarch64_simd_vec_setv2di"
956 [(set (match_operand:V2DI 0 "register_operand" "=w,w")
957 (vec_merge:V2DI
958 (vec_duplicate:V2DI
959 (match_operand:DI 1 "register_operand" "r,w"))
960 (match_operand:V2DI 3 "register_operand" "0,0")
961 (match_operand:SI 2 "immediate_operand" "i,i")))]
962 "TARGET_SIMD"
963 {
964 int elt = ENDIAN_LANE_N (V2DImode, exact_log2 (INTVAL (operands[2])));
965 operands[2] = GEN_INT ((HOST_WIDE_INT) 1 << elt);
966 switch (which_alternative)
967 {
968 case 0:
969 return "ins\\t%0.d[%p2], %1";
970 case 1:
971 return "ins\\t%0.d[%p2], %1.d[0]";
972 default:
973 gcc_unreachable ();
974 }
975 }
976 [(set_attr "type" "neon_from_gp, neon_ins_q")]
977 )
979 (define_expand "vec_setv2di"
980 [(match_operand:V2DI 0 "register_operand")
981 (match_operand:DI 1 "register_operand")
982 (match_operand:SI 2 "immediate_operand")]
983 "TARGET_SIMD"
984 {
985 HOST_WIDE_INT elem = (HOST_WIDE_INT) 1 << INTVAL (operands[2]);
986 emit_insn (gen_aarch64_simd_vec_setv2di (operands[0], operands[1],
987 GEN_INT (elem), operands[0]));
988 DONE;
989 }
990 )
992 (define_insn "aarch64_simd_vec_set<mode>"
993 [(set (match_operand:VDQF_F16 0 "register_operand" "=w")
994 (vec_merge:VDQF_F16
995 (vec_duplicate:VDQF_F16
996 (match_operand:<VEL> 1 "register_operand" "w"))
997 (match_operand:VDQF_F16 3 "register_operand" "0")
998 (match_operand:SI 2 "immediate_operand" "i")))]
999 "TARGET_SIMD"
1000 {
1001 int elt = ENDIAN_LANE_N (<MODE>mode, exact_log2 (INTVAL (operands[2])));
1003 operands[2] = GEN_INT ((HOST_WIDE_INT)1 << elt);
1004 return "ins\t%0.<Vetype>[%p2], %1.<Vetype>[0]";
1005 }
1006 [(set_attr "type" "neon_ins<q>")]
1007 )
1009 (define_expand "vec_set<mode>"
1010 [(match_operand:VDQF_F16 0 "register_operand" "+w")
1011 (match_operand:<VEL> 1 "register_operand" "w")
1012 (match_operand:SI 2 "immediate_operand" "")]
1013 "TARGET_SIMD"
1014 {
1015 HOST_WIDE_INT elem = (HOST_WIDE_INT) 1 << INTVAL (operands[2]);
1016 emit_insn (gen_aarch64_simd_vec_set<mode> (operands[0], operands[1],
1017 GEN_INT (elem), operands[0]));
1018 DONE;
1019 }
1020 )
1023 (define_insn "aarch64_mla<mode>"
1024 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
1025 (plus:VDQ_BHSI (mult:VDQ_BHSI
1026 (match_operand:VDQ_BHSI 2 "register_operand" "w")
1027 (match_operand:VDQ_BHSI 3 "register_operand" "w"))
1028 (match_operand:VDQ_BHSI 1 "register_operand" "0")))]
1029 "TARGET_SIMD"
1030 "mla\t%0.<Vtype>, %2.<Vtype>, %3.<Vtype>"
1031 [(set_attr "type" "neon_mla_<Vetype><q>")]
1032 )
1034 (define_insn "*aarch64_mla_elt<mode>"
1035 [(set (match_operand:VDQHS 0 "register_operand" "=w")
1036 (plus:VDQHS
1037 (mult:VDQHS
1038 (vec_duplicate:VDQHS
1039 (vec_select:<VEL>
1040 (match_operand:VDQHS 1 "register_operand" "<h_con>")
1041 (parallel [(match_operand:SI 2 "immediate_operand")])))
1042 (match_operand:VDQHS 3 "register_operand" "w"))
1043 (match_operand:VDQHS 4 "register_operand" "0")))]
1044 "TARGET_SIMD"
1045 {
1046 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
1047 return "mla\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
1048 }
1049 [(set_attr "type" "neon_mla_<Vetype>_scalar<q>")]
1050 )
1052 (define_insn "*aarch64_mla_elt_<vswap_width_name><mode>"
1053 [(set (match_operand:VDQHS 0 "register_operand" "=w")
1054 (plus:VDQHS
1055 (mult:VDQHS
1056 (vec_duplicate:VDQHS
1057 (vec_select:<VEL>
1058 (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
1059 (parallel [(match_operand:SI 2 "immediate_operand")])))
1060 (match_operand:VDQHS 3 "register_operand" "w"))
1061 (match_operand:VDQHS 4 "register_operand" "0")))]
1062 "TARGET_SIMD"
1063 {
1064 operands[2] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
1065 INTVAL (operands[2])));
1066 return "mla\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
1067 }
1068 [(set_attr "type" "neon_mla_<Vetype>_scalar<q>")]
1069 )
1071 (define_insn "*aarch64_mla_elt_merge<mode>"
1072 [(set (match_operand:VDQHS 0 "register_operand" "=w")
1073 (plus:VDQHS
1074 (mult:VDQHS (vec_duplicate:VDQHS
1075 (match_operand:<VEL> 1 "register_operand" "w"))
1076 (match_operand:VDQHS 2 "register_operand" "w"))
1077 (match_operand:VDQHS 3 "register_operand" "0")))]
1078 "TARGET_SIMD"
1079 "mla\t%0.<Vtype>, %2.<Vtype>, %1.<Vetype>[0]"
1080 [(set_attr "type" "neon_mla_<Vetype>_scalar<q>")]
1081 )
1083 (define_insn "aarch64_mls<mode>"
1084 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
1085 (minus:VDQ_BHSI (match_operand:VDQ_BHSI 1 "register_operand" "0")
1086 (mult:VDQ_BHSI (match_operand:VDQ_BHSI 2 "register_operand" "w")
1087 (match_operand:VDQ_BHSI 3 "register_operand" "w"))))]
1088 "TARGET_SIMD"
1089 "mls\t%0.<Vtype>, %2.<Vtype>, %3.<Vtype>"
1090 [(set_attr "type" "neon_mla_<Vetype><q>")]
1091 )
1093 (define_insn "*aarch64_mls_elt<mode>"
1094 [(set (match_operand:VDQHS 0 "register_operand" "=w")
1095 (minus:VDQHS
1096 (match_operand:VDQHS 4 "register_operand" "0")
1097 (mult:VDQHS
1098 (vec_duplicate:VDQHS
1099 (vec_select:<VEL>
1100 (match_operand:VDQHS 1 "register_operand" "<h_con>")
1101 (parallel [(match_operand:SI 2 "immediate_operand")])))
1102 (match_operand:VDQHS 3 "register_operand" "w"))))]
1103 "TARGET_SIMD"
1104 {
1105 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
1106 return "mls\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
1107 }
1108 [(set_attr "type" "neon_mla_<Vetype>_scalar<q>")]
1109 )
1111 (define_insn "*aarch64_mls_elt_<vswap_width_name><mode>"
1112 [(set (match_operand:VDQHS 0 "register_operand" "=w")
1113 (minus:VDQHS
1114 (match_operand:VDQHS 4 "register_operand" "0")
1115 (mult:VDQHS
1116 (vec_duplicate:VDQHS
1117 (vec_select:<VEL>
1118 (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
1119 (parallel [(match_operand:SI 2 "immediate_operand")])))
1120 (match_operand:VDQHS 3 "register_operand" "w"))))]
1121 "TARGET_SIMD"
1122 {
1123 operands[2] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
1124 INTVAL (operands[2])));
1125 return "mls\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
1126 }
1127 [(set_attr "type" "neon_mla_<Vetype>_scalar<q>")]
1128 )
1130 (define_insn "*aarch64_mls_elt_merge<mode>"
1131 [(set (match_operand:VDQHS 0 "register_operand" "=w")
1132 (minus:VDQHS
1133 (match_operand:VDQHS 1 "register_operand" "0")
1134 (mult:VDQHS (vec_duplicate:VDQHS
1135 (match_operand:<VEL> 2 "register_operand" "w"))
1136 (match_operand:VDQHS 3 "register_operand" "w"))))]
1137 "TARGET_SIMD"
1138 "mls\t%0.<Vtype>, %3.<Vtype>, %2.<Vetype>[0]"
1139 [(set_attr "type" "neon_mla_<Vetype>_scalar<q>")]
1140 )
1142 ;; Max/Min operations.
1143 (define_insn "<su><maxmin><mode>3"
1144 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
1145 (MAXMIN:VDQ_BHSI (match_operand:VDQ_BHSI 1 "register_operand" "w")
1146 (match_operand:VDQ_BHSI 2 "register_operand" "w")))]
1147 "TARGET_SIMD"
1148 "<su><maxmin>\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1149 [(set_attr "type" "neon_minmax<q>")]
1150 )
1152 (define_expand "<su><maxmin>v2di3"
1153 [(set (match_operand:V2DI 0 "register_operand" "")
1154 (MAXMIN:V2DI (match_operand:V2DI 1 "register_operand" "")
1155 (match_operand:V2DI 2 "register_operand" "")))]
1156 "TARGET_SIMD"
1157 {
1158 enum rtx_code cmp_operator;
1159 rtx cmp_fmt;
1161 switch (<CODE>)
1162 {
1163 case UMIN:
1164 cmp_operator = LTU;
1165 break;
1166 case SMIN:
1167 cmp_operator = LT;
1168 break;
1169 case UMAX:
1170 cmp_operator = GTU;
1171 break;
1172 case SMAX:
1173 cmp_operator = GT;
1174 break;
1175 default:
1176 gcc_unreachable ();
1177 }
1179 cmp_fmt = gen_rtx_fmt_ee (cmp_operator, V2DImode, operands[1], operands[2]);
1180 emit_insn (gen_vcondv2div2di (operands[0], operands[1],
1181 operands[2], cmp_fmt, operands[1], operands[2]));
1182 DONE;
1183 })
1185 ;; Pairwise Integer Max/Min operations.
1186 (define_insn "aarch64_<maxmin_uns>p<mode>"
1187 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
1188 (unspec:VDQ_BHSI [(match_operand:VDQ_BHSI 1 "register_operand" "w")
1189 (match_operand:VDQ_BHSI 2 "register_operand" "w")]
1190 MAXMINV))]
1191 "TARGET_SIMD"
1192 "<maxmin_uns_op>p\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1193 [(set_attr "type" "neon_minmax<q>")]
1194 )
1196 ;; Pairwise FP Max/Min operations.
1197 (define_insn "aarch64_<maxmin_uns>p<mode>"
1198 [(set (match_operand:VHSDF 0 "register_operand" "=w")
1199 (unspec:VHSDF [(match_operand:VHSDF 1 "register_operand" "w")
1200 (match_operand:VHSDF 2 "register_operand" "w")]
1201 FMAXMINV))]
1202 "TARGET_SIMD"
1203 "<maxmin_uns_op>p\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1204 [(set_attr "type" "neon_minmax<q>")]
1205 )
1207 ;; vec_concat gives a new vector with the low elements from operand 1, and
1208 ;; the high elements from operand 2. That is to say, given op1 = { a, b }
1209 ;; op2 = { c, d }, vec_concat (op1, op2) = { a, b, c, d }.
1210 ;; What that means, is that the RTL descriptions of the below patterns
1211 ;; need to change depending on endianness.
1213 ;; Move to the low architectural bits of the register.
1214 ;; On little-endian this is { operand, zeroes }
1215 ;; On big-endian this is { zeroes, operand }
1217 (define_insn "move_lo_quad_internal_<mode>"
1218 [(set (match_operand:VQ_NO2E 0 "register_operand" "=w,w,w")
1219 (vec_concat:VQ_NO2E
1220 (match_operand:<VHALF> 1 "register_operand" "w,r,r")
1221 (vec_duplicate:<VHALF> (const_int 0))))]
1222 "TARGET_SIMD && !BYTES_BIG_ENDIAN"
1223 "@
1224 dup\\t%d0, %1.d[0]
1225 fmov\\t%d0, %1
1226 dup\\t%d0, %1"
1227 [(set_attr "type" "neon_dup<q>,f_mcr,neon_dup<q>")
1228 (set_attr "simd" "yes,*,yes")
1229 (set_attr "fp" "*,yes,*")
1230 (set_attr "length" "4")]
1231 )
1233 (define_insn "move_lo_quad_internal_<mode>"
1234 [(set (match_operand:VQ_2E 0 "register_operand" "=w,w,w")
1235 (vec_concat:VQ_2E
1236 (match_operand:<VHALF> 1 "register_operand" "w,r,r")
1237 (const_int 0)))]
1238 "TARGET_SIMD && !BYTES_BIG_ENDIAN"
1239 "@
1240 dup\\t%d0, %1.d[0]
1241 fmov\\t%d0, %1
1242 dup\\t%d0, %1"
1243 [(set_attr "type" "neon_dup<q>,f_mcr,neon_dup<q>")
1244 (set_attr "simd" "yes,*,yes")
1245 (set_attr "fp" "*,yes,*")
1246 (set_attr "length" "4")]
1247 )
1249 (define_insn "move_lo_quad_internal_be_<mode>"
1250 [(set (match_operand:VQ_NO2E 0 "register_operand" "=w,w,w")
1251 (vec_concat:VQ_NO2E
1252 (vec_duplicate:<VHALF> (const_int 0))
1253 (match_operand:<VHALF> 1 "register_operand" "w,r,r")))]
1254 "TARGET_SIMD && BYTES_BIG_ENDIAN"
1255 "@
1256 dup\\t%d0, %1.d[0]
1257 fmov\\t%d0, %1
1258 dup\\t%d0, %1"
1259 [(set_attr "type" "neon_dup<q>,f_mcr,neon_dup<q>")
1260 (set_attr "simd" "yes,*,yes")
1261 (set_attr "fp" "*,yes,*")
1262 (set_attr "length" "4")]
1263 )
1265 (define_insn "move_lo_quad_internal_be_<mode>"
1266 [(set (match_operand:VQ_2E 0 "register_operand" "=w,w,w")
1267 (vec_concat:VQ_2E
1268 (const_int 0)
1269 (match_operand:<VHALF> 1 "register_operand" "w,r,r")))]
1270 "TARGET_SIMD && BYTES_BIG_ENDIAN"
1271 "@
1272 dup\\t%d0, %1.d[0]
1273 fmov\\t%d0, %1
1274 dup\\t%d0, %1"
1275 [(set_attr "type" "neon_dup<q>,f_mcr,neon_dup<q>")
1276 (set_attr "simd" "yes,*,yes")
1277 (set_attr "fp" "*,yes,*")
1278 (set_attr "length" "4")]
1279 )
1281 (define_expand "move_lo_quad_<mode>"
1282 [(match_operand:VQ 0 "register_operand")
1283 (match_operand:VQ 1 "register_operand")]
1284 "TARGET_SIMD"
1285 {
1286 if (BYTES_BIG_ENDIAN)
1287 emit_insn (gen_move_lo_quad_internal_be_<mode> (operands[0], operands[1]));
1288 else
1289 emit_insn (gen_move_lo_quad_internal_<mode> (operands[0], operands[1]));
1290 DONE;
1291 }
1292 )
1294 ;; Move operand1 to the high architectural bits of the register, keeping
1295 ;; the low architectural bits of operand2.
1296 ;; For little-endian this is { operand2, operand1 }
1297 ;; For big-endian this is { operand1, operand2 }
1299 (define_insn "aarch64_simd_move_hi_quad_<mode>"
1300 [(set (match_operand:VQ 0 "register_operand" "+w,w")
1301 (vec_concat:VQ
1302 (vec_select:<VHALF>
1303 (match_dup 0)
1304 (match_operand:VQ 2 "vect_par_cnst_lo_half" ""))
1305 (match_operand:<VHALF> 1 "register_operand" "w,r")))]
1306 "TARGET_SIMD && !BYTES_BIG_ENDIAN"
1307 "@
1308 ins\\t%0.d[1], %1.d[0]
1309 ins\\t%0.d[1], %1"
1310 [(set_attr "type" "neon_ins")]
1311 )
1313 (define_insn "aarch64_simd_move_hi_quad_be_<mode>"
1314 [(set (match_operand:VQ 0 "register_operand" "+w,w")
1315 (vec_concat:VQ
1316 (match_operand:<VHALF> 1 "register_operand" "w,r")
1317 (vec_select:<VHALF>
1318 (match_dup 0)
1319 (match_operand:VQ 2 "vect_par_cnst_lo_half" ""))))]
1320 "TARGET_SIMD && BYTES_BIG_ENDIAN"
1321 "@
1322 ins\\t%0.d[1], %1.d[0]
1323 ins\\t%0.d[1], %1"
1324 [(set_attr "type" "neon_ins")]
1325 )
1327 (define_expand "move_hi_quad_<mode>"
1328 [(match_operand:VQ 0 "register_operand" "")
1329 (match_operand:<VHALF> 1 "register_operand" "")]
1330 "TARGET_SIMD"
1331 {
1332 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
1333 if (BYTES_BIG_ENDIAN)
1334 emit_insn (gen_aarch64_simd_move_hi_quad_be_<mode> (operands[0],
1335 operands[1], p));
1336 else
1337 emit_insn (gen_aarch64_simd_move_hi_quad_<mode> (operands[0],
1338 operands[1], p));
1339 DONE;
1340 })
1342 ;; Narrowing operations.
1344 ;; For doubles.
1345 (define_insn "aarch64_simd_vec_pack_trunc_<mode>"
1346 [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
1347 (truncate:<VNARROWQ> (match_operand:VQN 1 "register_operand" "w")))]
1348 "TARGET_SIMD"
1349 "xtn\\t%0.<Vntype>, %1.<Vtype>"
1350 [(set_attr "type" "neon_shift_imm_narrow_q")]
1351 )
1353 (define_expand "vec_pack_trunc_<mode>"
1354 [(match_operand:<VNARROWD> 0 "register_operand" "")
1355 (match_operand:VDN 1 "register_operand" "")
1356 (match_operand:VDN 2 "register_operand" "")]
1357 "TARGET_SIMD"
1358 {
1359 rtx tempreg = gen_reg_rtx (<VDBL>mode);
1360 int lo = BYTES_BIG_ENDIAN ? 2 : 1;
1361 int hi = BYTES_BIG_ENDIAN ? 1 : 2;
1363 emit_insn (gen_move_lo_quad_<Vdbl> (tempreg, operands[lo]));
1364 emit_insn (gen_move_hi_quad_<Vdbl> (tempreg, operands[hi]));
1365 emit_insn (gen_aarch64_simd_vec_pack_trunc_<Vdbl> (operands[0], tempreg));
1366 DONE;
1367 })
1369 ;; For quads.
1371 (define_insn "vec_pack_trunc_<mode>"
1372 [(set (match_operand:<VNARROWQ2> 0 "register_operand" "=&w")
1373 (vec_concat:<VNARROWQ2>
1374 (truncate:<VNARROWQ> (match_operand:VQN 1 "register_operand" "w"))
1375 (truncate:<VNARROWQ> (match_operand:VQN 2 "register_operand" "w"))))]
1376 "TARGET_SIMD"
1377 {
1378 if (BYTES_BIG_ENDIAN)
1379 return "xtn\\t%0.<Vntype>, %2.<Vtype>\;xtn2\\t%0.<V2ntype>, %1.<Vtype>";
1380 else
1381 return "xtn\\t%0.<Vntype>, %1.<Vtype>\;xtn2\\t%0.<V2ntype>, %2.<Vtype>";
1382 }
1383 [(set_attr "type" "multiple")
1384 (set_attr "length" "8")]
1385 )
1387 ;; Widening operations.
1389 (define_insn "aarch64_simd_vec_unpack<su>_lo_<mode>"
1390 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1391 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1392 (match_operand:VQW 1 "register_operand" "w")
1393 (match_operand:VQW 2 "vect_par_cnst_lo_half" "")
1394 )))]
1395 "TARGET_SIMD"
1396 "<su>shll\t%0.<Vwtype>, %1.<Vhalftype>, 0"
1397 [(set_attr "type" "neon_shift_imm_long")]
1398 )
1400 (define_insn "aarch64_simd_vec_unpack<su>_hi_<mode>"
1401 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1402 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1403 (match_operand:VQW 1 "register_operand" "w")
1404 (match_operand:VQW 2 "vect_par_cnst_hi_half" "")
1405 )))]
1406 "TARGET_SIMD"
1407 "<su>shll2\t%0.<Vwtype>, %1.<Vtype>, 0"
1408 [(set_attr "type" "neon_shift_imm_long")]
1409 )
1411 (define_expand "vec_unpack<su>_hi_<mode>"
1412 [(match_operand:<VWIDE> 0 "register_operand" "")
1413 (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand"))]
1414 "TARGET_SIMD"
1415 {
1416 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
1417 emit_insn (gen_aarch64_simd_vec_unpack<su>_hi_<mode> (operands[0],
1418 operands[1], p));
1419 DONE;
1420 }
1421 )
1423 (define_expand "vec_unpack<su>_lo_<mode>"
1424 [(match_operand:<VWIDE> 0 "register_operand" "")
1425 (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand" ""))]
1426 "TARGET_SIMD"
1427 {
1428 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
1429 emit_insn (gen_aarch64_simd_vec_unpack<su>_lo_<mode> (operands[0],
1430 operands[1], p));
1431 DONE;
1432 }
1433 )
1435 ;; Widening arithmetic.
1437 (define_insn "*aarch64_<su>mlal_lo<mode>"
1438 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1439 (plus:<VWIDE>
1440 (mult:<VWIDE>
1441 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1442 (match_operand:VQW 2 "register_operand" "w")
1443 (match_operand:VQW 3 "vect_par_cnst_lo_half" "")))
1444 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1445 (match_operand:VQW 4 "register_operand" "w")
1446 (match_dup 3))))
1447 (match_operand:<VWIDE> 1 "register_operand" "0")))]
1448 "TARGET_SIMD"
1449 "<su>mlal\t%0.<Vwtype>, %2.<Vhalftype>, %4.<Vhalftype>"
1450 [(set_attr "type" "neon_mla_<Vetype>_long")]
1451 )
1453 (define_insn "*aarch64_<su>mlal_hi<mode>"
1454 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1455 (plus:<VWIDE>
1456 (mult:<VWIDE>
1457 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1458 (match_operand:VQW 2 "register_operand" "w")
1459 (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))
1460 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1461 (match_operand:VQW 4 "register_operand" "w")
1462 (match_dup 3))))
1463 (match_operand:<VWIDE> 1 "register_operand" "0")))]
1464 "TARGET_SIMD"
1465 "<su>mlal2\t%0.<Vwtype>, %2.<Vtype>, %4.<Vtype>"
1466 [(set_attr "type" "neon_mla_<Vetype>_long")]
1467 )
1469 (define_insn "*aarch64_<su>mlsl_lo<mode>"
1470 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1471 (minus:<VWIDE>
1472 (match_operand:<VWIDE> 1 "register_operand" "0")
1473 (mult:<VWIDE>
1474 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1475 (match_operand:VQW 2 "register_operand" "w")
1476 (match_operand:VQW 3 "vect_par_cnst_lo_half" "")))
1477 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1478 (match_operand:VQW 4 "register_operand" "w")
1479 (match_dup 3))))))]
1480 "TARGET_SIMD"
1481 "<su>mlsl\t%0.<Vwtype>, %2.<Vhalftype>, %4.<Vhalftype>"
1482 [(set_attr "type" "neon_mla_<Vetype>_long")]
1483 )
1485 (define_insn "*aarch64_<su>mlsl_hi<mode>"
1486 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1487 (minus:<VWIDE>
1488 (match_operand:<VWIDE> 1 "register_operand" "0")
1489 (mult:<VWIDE>
1490 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1491 (match_operand:VQW 2 "register_operand" "w")
1492 (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))
1493 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1494 (match_operand:VQW 4 "register_operand" "w")
1495 (match_dup 3))))))]
1496 "TARGET_SIMD"
1497 "<su>mlsl2\t%0.<Vwtype>, %2.<Vtype>, %4.<Vtype>"
1498 [(set_attr "type" "neon_mla_<Vetype>_long")]
1499 )
1501 (define_insn "*aarch64_<su>mlal<mode>"
1502 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1503 (plus:<VWIDE>
1504 (mult:<VWIDE>
1505 (ANY_EXTEND:<VWIDE>
1506 (match_operand:VD_BHSI 1 "register_operand" "w"))
1507 (ANY_EXTEND:<VWIDE>
1508 (match_operand:VD_BHSI 2 "register_operand" "w")))
1509 (match_operand:<VWIDE> 3 "register_operand" "0")))]
1510 "TARGET_SIMD"
1511 "<su>mlal\t%0.<Vwtype>, %1.<Vtype>, %2.<Vtype>"
1512 [(set_attr "type" "neon_mla_<Vetype>_long")]
1513 )
1515 (define_insn "*aarch64_<su>mlsl<mode>"
1516 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1517 (minus:<VWIDE>
1518 (match_operand:<VWIDE> 1 "register_operand" "0")
1519 (mult:<VWIDE>
1520 (ANY_EXTEND:<VWIDE>
1521 (match_operand:VD_BHSI 2 "register_operand" "w"))
1522 (ANY_EXTEND:<VWIDE>
1523 (match_operand:VD_BHSI 3 "register_operand" "w")))))]
1524 "TARGET_SIMD"
1525 "<su>mlsl\t%0.<Vwtype>, %2.<Vtype>, %3.<Vtype>"
1526 [(set_attr "type" "neon_mla_<Vetype>_long")]
1527 )
1529 (define_insn "aarch64_simd_vec_<su>mult_lo_<mode>"
1530 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1531 (mult:<VWIDE> (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1532 (match_operand:VQW 1 "register_operand" "w")
1533 (match_operand:VQW 3 "vect_par_cnst_lo_half" "")))
1534 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1535 (match_operand:VQW 2 "register_operand" "w")
1536 (match_dup 3)))))]
1537 "TARGET_SIMD"
1538 "<su>mull\\t%0.<Vwtype>, %1.<Vhalftype>, %2.<Vhalftype>"
1539 [(set_attr "type" "neon_mul_<Vetype>_long")]
1540 )
1542 (define_expand "vec_widen_<su>mult_lo_<mode>"
1543 [(match_operand:<VWIDE> 0 "register_operand" "")
1544 (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand" ""))
1545 (ANY_EXTEND:<VWIDE> (match_operand:VQW 2 "register_operand" ""))]
1546 "TARGET_SIMD"
1547 {
1548 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
1549 emit_insn (gen_aarch64_simd_vec_<su>mult_lo_<mode> (operands[0],
1550 operands[1],
1551 operands[2], p));
1552 DONE;
1553 }
1554 )
1556 (define_insn "aarch64_simd_vec_<su>mult_hi_<mode>"
1557 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1558 (mult:<VWIDE> (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1559 (match_operand:VQW 1 "register_operand" "w")
1560 (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))
1561 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1562 (match_operand:VQW 2 "register_operand" "w")
1563 (match_dup 3)))))]
1564 "TARGET_SIMD"
1565 "<su>mull2\\t%0.<Vwtype>, %1.<Vtype>, %2.<Vtype>"
1566 [(set_attr "type" "neon_mul_<Vetype>_long")]
1567 )
1569 (define_expand "vec_widen_<su>mult_hi_<mode>"
1570 [(match_operand:<VWIDE> 0 "register_operand" "")
1571 (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand" ""))
1572 (ANY_EXTEND:<VWIDE> (match_operand:VQW 2 "register_operand" ""))]
1573 "TARGET_SIMD"
1574 {
1575 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
1576 emit_insn (gen_aarch64_simd_vec_<su>mult_hi_<mode> (operands[0],
1577 operands[1],
1578 operands[2], p));
1579 DONE;
1581 }
1582 )
1584 ;; FP vector operations.
1585 ;; AArch64 AdvSIMD supports single-precision (32-bit) and
1586 ;; double-precision (64-bit) floating-point data types and arithmetic as
1587 ;; defined by the IEEE 754-2008 standard. This makes them vectorizable
1588 ;; without the need for -ffast-math or -funsafe-math-optimizations.
1589 ;;
1590 ;; Floating-point operations can raise an exception. Vectorizing such
1591 ;; operations are safe because of reasons explained below.
1592 ;;
1593 ;; ARMv8 permits an extension to enable trapped floating-point
1594 ;; exception handling, however this is an optional feature. In the
1595 ;; event of a floating-point exception being raised by vectorised
1596 ;; code then:
1597 ;; 1. If trapped floating-point exceptions are available, then a trap
1598 ;; will be taken when any lane raises an enabled exception. A trap
1599 ;; handler may determine which lane raised the exception.
1600 ;; 2. Alternatively a sticky exception flag is set in the
1601 ;; floating-point status register (FPSR). Software may explicitly
1602 ;; test the exception flags, in which case the tests will either
1603 ;; prevent vectorisation, allowing precise identification of the
1604 ;; failing operation, or if tested outside of vectorisable regions
1605 ;; then the specific operation and lane are not of interest.
1607 ;; FP arithmetic operations.
1609 (define_insn "add<mode>3"
1610 [(set (match_operand:VHSDF 0 "register_operand" "=w")
1611 (plus:VHSDF (match_operand:VHSDF 1 "register_operand" "w")
1612 (match_operand:VHSDF 2 "register_operand" "w")))]
1613 "TARGET_SIMD"
1614 "fadd\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1615 [(set_attr "type" "neon_fp_addsub_<stype><q>")]
1616 )
1618 (define_insn "sub<mode>3"
1619 [(set (match_operand:VHSDF 0 "register_operand" "=w")
1620 (minus:VHSDF (match_operand:VHSDF 1 "register_operand" "w")
1621 (match_operand:VHSDF 2 "register_operand" "w")))]
1622 "TARGET_SIMD"
1623 "fsub\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1624 [(set_attr "type" "neon_fp_addsub_<stype><q>")]
1625 )
1627 (define_insn "mul<mode>3"
1628 [(set (match_operand:VHSDF 0 "register_operand" "=w")
1629 (mult:VHSDF (match_operand:VHSDF 1 "register_operand" "w")
1630 (match_operand:VHSDF 2 "register_operand" "w")))]
1631 "TARGET_SIMD"
1632 "fmul\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1633 [(set_attr "type" "neon_fp_mul_<stype><q>")]
1634 )
1636 (define_expand "div<mode>3"
1637 [(set (match_operand:VHSDF 0 "register_operand" "=w")
1638 (div:VHSDF (match_operand:VHSDF 1 "register_operand" "w")
1639 (match_operand:VHSDF 2 "register_operand" "w")))]
1640 "TARGET_SIMD"
1641 {
1642 if (aarch64_emit_approx_div (operands[0], operands[1], operands[2]))
1643 DONE;
1645 operands[1] = force_reg (<MODE>mode, operands[1]);
1646 })
1648 (define_insn "*div<mode>3"
1649 [(set (match_operand:VHSDF 0 "register_operand" "=w")
1650 (div:VHSDF (match_operand:VHSDF 1 "register_operand" "w")
1651 (match_operand:VHSDF 2 "register_operand" "w")))]
1652 "TARGET_SIMD"
1653 "fdiv\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1654 [(set_attr "type" "neon_fp_div_<stype><q>")]
1655 )
1657 (define_insn "neg<mode>2"
1658 [(set (match_operand:VHSDF 0 "register_operand" "=w")
1659 (neg:VHSDF (match_operand:VHSDF 1 "register_operand" "w")))]
1660 "TARGET_SIMD"
1661 "fneg\\t%0.<Vtype>, %1.<Vtype>"
1662 [(set_attr "type" "neon_fp_neg_<stype><q>")]
1663 )
1665 (define_insn "abs<mode>2"
1666 [(set (match_operand:VHSDF 0 "register_operand" "=w")
1667 (abs:VHSDF (match_operand:VHSDF 1 "register_operand" "w")))]
1668 "TARGET_SIMD"
1669 "fabs\\t%0.<Vtype>, %1.<Vtype>"
1670 [(set_attr "type" "neon_fp_abs_<stype><q>")]
1671 )
1673 (define_insn "fma<mode>4"
1674 [(set (match_operand:VHSDF 0 "register_operand" "=w")
1675 (fma:VHSDF (match_operand:VHSDF 1 "register_operand" "w")
1676 (match_operand:VHSDF 2 "register_operand" "w")
1677 (match_operand:VHSDF 3 "register_operand" "0")))]
1678 "TARGET_SIMD"
1679 "fmla\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1680 [(set_attr "type" "neon_fp_mla_<stype><q>")]
1681 )
1683 (define_insn "*aarch64_fma4_elt<mode>"
1684 [(set (match_operand:VDQF 0 "register_operand" "=w")
1685 (fma:VDQF
1686 (vec_duplicate:VDQF
1687 (vec_select:<VEL>
1688 (match_operand:VDQF 1 "register_operand" "<h_con>")
1689 (parallel [(match_operand:SI 2 "immediate_operand")])))
1690 (match_operand:VDQF 3 "register_operand" "w")
1691 (match_operand:VDQF 4 "register_operand" "0")))]
1692 "TARGET_SIMD"
1693 {
1694 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
1695 return "fmla\\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
1696 }
1697 [(set_attr "type" "neon_fp_mla_<Vetype>_scalar<q>")]
1698 )
1700 (define_insn "*aarch64_fma4_elt_<vswap_width_name><mode>"
1701 [(set (match_operand:VDQSF 0 "register_operand" "=w")
1702 (fma:VDQSF
1703 (vec_duplicate:VDQSF
1704 (vec_select:<VEL>
1705 (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
1706 (parallel [(match_operand:SI 2 "immediate_operand")])))
1707 (match_operand:VDQSF 3 "register_operand" "w")
1708 (match_operand:VDQSF 4 "register_operand" "0")))]
1709 "TARGET_SIMD"
1710 {
1711 operands[2] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
1712 INTVAL (operands[2])));
1713 return "fmla\\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
1714 }
1715 [(set_attr "type" "neon_fp_mla_<Vetype>_scalar<q>")]
1716 )
1718 (define_insn "*aarch64_fma4_elt_from_dup<mode>"
1719 [(set (match_operand:VMUL 0 "register_operand" "=w")
1720 (fma:VMUL
1721 (vec_duplicate:VMUL
1722 (match_operand:<VEL> 1 "register_operand" "<h_con>"))
1723 (match_operand:VMUL 2 "register_operand" "w")
1724 (match_operand:VMUL 3 "register_operand" "0")))]
1725 "TARGET_SIMD"
1726 "fmla\t%0.<Vtype>, %2.<Vtype>, %1.<Vetype>[0]"
1727 [(set_attr "type" "neon<fp>_mla_<stype>_scalar<q>")]
1728 )
1730 (define_insn "*aarch64_fma4_elt_to_64v2df"
1731 [(set (match_operand:DF 0 "register_operand" "=w")
1732 (fma:DF
1733 (vec_select:DF
1734 (match_operand:V2DF 1 "register_operand" "w")
1735 (parallel [(match_operand:SI 2 "immediate_operand")]))
1736 (match_operand:DF 3 "register_operand" "w")
1737 (match_operand:DF 4 "register_operand" "0")))]
1738 "TARGET_SIMD"
1739 {
1740 operands[2] = GEN_INT (ENDIAN_LANE_N (V2DFmode, INTVAL (operands[2])));
1741 return "fmla\\t%0.2d, %3.2d, %1.2d[%2]";
1742 }
1743 [(set_attr "type" "neon_fp_mla_d_scalar_q")]
1744 )
1746 (define_insn "fnma<mode>4"
1747 [(set (match_operand:VHSDF 0 "register_operand" "=w")
1748 (fma:VHSDF
1749 (match_operand:VHSDF 1 "register_operand" "w")
1750 (neg:VHSDF
1751 (match_operand:VHSDF 2 "register_operand" "w"))
1752 (match_operand:VHSDF 3 "register_operand" "0")))]
1753 "TARGET_SIMD"
1754 "fmls\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1755 [(set_attr "type" "neon_fp_mla_<stype><q>")]
1756 )
1758 (define_insn "*aarch64_fnma4_elt<mode>"
1759 [(set (match_operand:VDQF 0 "register_operand" "=w")
1760 (fma:VDQF
1761 (neg:VDQF
1762 (match_operand:VDQF 3 "register_operand" "w"))
1763 (vec_duplicate:VDQF
1764 (vec_select:<VEL>
1765 (match_operand:VDQF 1 "register_operand" "<h_con>")
1766 (parallel [(match_operand:SI 2 "immediate_operand")])))
1767 (match_operand:VDQF 4 "register_operand" "0")))]
1768 "TARGET_SIMD"
1769 {
1770 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
1771 return "fmls\\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
1772 }
1773 [(set_attr "type" "neon_fp_mla_<Vetype>_scalar<q>")]
1774 )
1776 (define_insn "*aarch64_fnma4_elt_<vswap_width_name><mode>"
1777 [(set (match_operand:VDQSF 0 "register_operand" "=w")
1778 (fma:VDQSF
1779 (neg:VDQSF
1780 (match_operand:VDQSF 3 "register_operand" "w"))
1781 (vec_duplicate:VDQSF
1782 (vec_select:<VEL>
1783 (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
1784 (parallel [(match_operand:SI 2 "immediate_operand")])))
1785 (match_operand:VDQSF 4 "register_operand" "0")))]
1786 "TARGET_SIMD"
1787 {
1788 operands[2] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
1789 INTVAL (operands[2])));
1790 return "fmls\\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
1791 }
1792 [(set_attr "type" "neon_fp_mla_<Vetype>_scalar<q>")]
1793 )
1795 (define_insn "*aarch64_fnma4_elt_from_dup<mode>"
1796 [(set (match_operand:VMUL 0 "register_operand" "=w")
1797 (fma:VMUL
1798 (neg:VMUL
1799 (match_operand:VMUL 2 "register_operand" "w"))
1800 (vec_duplicate:VMUL
1801 (match_operand:<VEL> 1 "register_operand" "<h_con>"))
1802 (match_operand:VMUL 3 "register_operand" "0")))]
1803 "TARGET_SIMD"
1804 "fmls\t%0.<Vtype>, %2.<Vtype>, %1.<Vetype>[0]"
1805 [(set_attr "type" "neon<fp>_mla_<stype>_scalar<q>")]
1806 )
1808 (define_insn "*aarch64_fnma4_elt_to_64v2df"
1809 [(set (match_operand:DF 0 "register_operand" "=w")
1810 (fma:DF
1811 (vec_select:DF
1812 (match_operand:V2DF 1 "register_operand" "w")
1813 (parallel [(match_operand:SI 2 "immediate_operand")]))
1814 (neg:DF
1815 (match_operand:DF 3 "register_operand" "w"))
1816 (match_operand:DF 4 "register_operand" "0")))]
1817 "TARGET_SIMD"
1818 {
1819 operands[2] = GEN_INT (ENDIAN_LANE_N (V2DFmode, INTVAL (operands[2])));
1820 return "fmls\\t%0.2d, %3.2d, %1.2d[%2]";
1821 }
1822 [(set_attr "type" "neon_fp_mla_d_scalar_q")]
1823 )
1825 ;; Vector versions of the floating-point frint patterns.
1826 ;; Expands to btrunc, ceil, floor, nearbyint, rint, round, frintn.
1827 (define_insn "<frint_pattern><mode>2"
1828 [(set (match_operand:VHSDF 0 "register_operand" "=w")
1829 (unspec:VHSDF [(match_operand:VHSDF 1 "register_operand" "w")]
1830 FRINT))]
1831 "TARGET_SIMD"
1832 "frint<frint_suffix>\\t%0.<Vtype>, %1.<Vtype>"
1833 [(set_attr "type" "neon_fp_round_<stype><q>")]
1834 )
1836 ;; Vector versions of the fcvt standard patterns.
1837 ;; Expands to lbtrunc, lround, lceil, lfloor
1838 (define_insn "l<fcvt_pattern><su_optab><VHSDF:mode><fcvt_target>2"
1839 [(set (match_operand:<FCVT_TARGET> 0 "register_operand" "=w")
1840 (FIXUORS:<FCVT_TARGET> (unspec:<FCVT_TARGET>
1841 [(match_operand:VHSDF 1 "register_operand" "w")]
1842 FCVT)))]
1843 "TARGET_SIMD"
1844 "fcvt<frint_suffix><su>\\t%0.<Vtype>, %1.<Vtype>"
1845 [(set_attr "type" "neon_fp_to_int_<stype><q>")]
1846 )
1848 ;; HF Scalar variants of related SIMD instructions.
1849 (define_insn "l<fcvt_pattern><su_optab>hfhi2"
1850 [(set (match_operand:HI 0 "register_operand" "=w")
1851 (FIXUORS:HI (unspec:HF [(match_operand:HF 1 "register_operand" "w")]
1852 FCVT)))]
1853 "TARGET_SIMD_F16INST"
1854 "fcvt<frint_suffix><su>\t%h0, %h1"
1855 [(set_attr "type" "neon_fp_to_int_s")]
1856 )
1858 (define_insn "<optab>_trunchfhi2"
1859 [(set (match_operand:HI 0 "register_operand" "=w")
1860 (FIXUORS:HI (match_operand:HF 1 "register_operand" "w")))]
1861 "TARGET_SIMD_F16INST"
1862 "fcvtz<su>\t%h0, %h1"
1863 [(set_attr "type" "neon_fp_to_int_s")]
1864 )
1866 (define_insn "<optab>hihf2"
1867 [(set (match_operand:HF 0 "register_operand" "=w")
1868 (FLOATUORS:HF (match_operand:HI 1 "register_operand" "w")))]
1869 "TARGET_SIMD_F16INST"
1870 "<su_optab>cvtf\t%h0, %h1"
1871 [(set_attr "type" "neon_int_to_fp_s")]
1872 )
1874 (define_insn "*aarch64_fcvt<su_optab><VDQF:mode><fcvt_target>2_mult"
1875 [(set (match_operand:<FCVT_TARGET> 0 "register_operand" "=w")
1876 (FIXUORS:<FCVT_TARGET> (unspec:<FCVT_TARGET>
1877 [(mult:VDQF
1878 (match_operand:VDQF 1 "register_operand" "w")
1879 (match_operand:VDQF 2 "aarch64_fp_vec_pow2" ""))]
1880 UNSPEC_FRINTZ)))]
1881 "TARGET_SIMD
1882 && IN_RANGE (aarch64_vec_fpconst_pow_of_2 (operands[2]), 1,
1883 GET_MODE_BITSIZE (GET_MODE_INNER (<VDQF:MODE>mode)))"
1884 {
1885 int fbits = aarch64_vec_fpconst_pow_of_2 (operands[2]);
1886 char buf[64];
1887 snprintf (buf, 64, "fcvtz<su>\\t%%0.<Vtype>, %%1.<Vtype>, #%d", fbits);
1888 output_asm_insn (buf, operands);
1889 return "";
1890 }
1891 [(set_attr "type" "neon_fp_to_int_<Vetype><q>")]
1892 )
1894 (define_expand "<optab><VHSDF:mode><fcvt_target>2"
1895 [(set (match_operand:<FCVT_TARGET> 0 "register_operand")
1896 (FIXUORS:<FCVT_TARGET> (unspec:<FCVT_TARGET>
1897 [(match_operand:VHSDF 1 "register_operand")]
1898 UNSPEC_FRINTZ)))]
1899 "TARGET_SIMD"
1900 {})
1902 (define_expand "<fix_trunc_optab><VHSDF:mode><fcvt_target>2"
1903 [(set (match_operand:<FCVT_TARGET> 0 "register_operand")
1904 (FIXUORS:<FCVT_TARGET> (unspec:<FCVT_TARGET>
1905 [(match_operand:VHSDF 1 "register_operand")]
1906 UNSPEC_FRINTZ)))]
1907 "TARGET_SIMD"
1908 {})
1910 (define_expand "ftrunc<VHSDF:mode>2"
1911 [(set (match_operand:VHSDF 0 "register_operand")
1912 (unspec:VHSDF [(match_operand:VHSDF 1 "register_operand")]
1913 UNSPEC_FRINTZ))]
1914 "TARGET_SIMD"
1915 {})
1917 (define_insn "<optab><fcvt_target><VHSDF:mode>2"
1918 [(set (match_operand:VHSDF 0 "register_operand" "=w")
1919 (FLOATUORS:VHSDF
1920 (match_operand:<FCVT_TARGET> 1 "register_operand" "w")))]
1921 "TARGET_SIMD"
1922 "<su_optab>cvtf\\t%0.<Vtype>, %1.<Vtype>"
1923 [(set_attr "type" "neon_int_to_fp_<stype><q>")]
1924 )
1926 ;; Conversions between vectors of floats and doubles.
1927 ;; Contains a mix of patterns to match standard pattern names
1928 ;; and those for intrinsics.
1930 ;; Float widening operations.
1932 (define_insn "aarch64_simd_vec_unpacks_lo_<mode>"
1933 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1934 (float_extend:<VWIDE> (vec_select:<VHALF>
1935 (match_operand:VQ_HSF 1 "register_operand" "w")
1936 (match_operand:VQ_HSF 2 "vect_par_cnst_lo_half" "")
1937 )))]
1938 "TARGET_SIMD"
1939 "fcvtl\\t%0.<Vwtype>, %1.<Vhalftype>"
1940 [(set_attr "type" "neon_fp_cvt_widen_s")]
1941 )
1943 ;; Convert between fixed-point and floating-point (vector modes)
1945 (define_insn "<FCVT_F2FIXED:fcvt_fixed_insn><VHSDF:mode>3"
1946 [(set (match_operand:<VHSDF:FCVT_TARGET> 0 "register_operand" "=w")
1947 (unspec:<VHSDF:FCVT_TARGET>
1948 [(match_operand:VHSDF 1 "register_operand" "w")
1949 (match_operand:SI 2 "immediate_operand" "i")]
1950 FCVT_F2FIXED))]
1951 "TARGET_SIMD"
1952 "<FCVT_F2FIXED:fcvt_fixed_insn>\t%<v>0<Vmtype>, %<v>1<Vmtype>, #%2"
1953 [(set_attr "type" "neon_fp_to_int_<VHSDF:stype><q>")]
1954 )
1956 (define_insn "<FCVT_FIXED2F:fcvt_fixed_insn><VDQ_HSDI:mode>3"
1957 [(set (match_operand:<VDQ_HSDI:FCVT_TARGET> 0 "register_operand" "=w")
1958 (unspec:<VDQ_HSDI:FCVT_TARGET>
1959 [(match_operand:VDQ_HSDI 1 "register_operand" "w")
1960 (match_operand:SI 2 "immediate_operand" "i")]
1961 FCVT_FIXED2F))]
1962 "TARGET_SIMD"
1963 "<FCVT_FIXED2F:fcvt_fixed_insn>\t%<v>0<Vmtype>, %<v>1<Vmtype>, #%2"
1964 [(set_attr "type" "neon_int_to_fp_<VDQ_HSDI:stype><q>")]
1965 )
1967 ;; ??? Note that the vectorizer usage of the vec_unpacks_[lo/hi] patterns
1968 ;; is inconsistent with vector ordering elsewhere in the compiler, in that
1969 ;; the meaning of HI and LO changes depending on the target endianness.
1970 ;; While elsewhere we map the higher numbered elements of a vector to
1971 ;; the lower architectural lanes of the vector, for these patterns we want
1972 ;; to always treat "hi" as referring to the higher architectural lanes.
1973 ;; Consequently, while the patterns below look inconsistent with our
1974 ;; other big-endian patterns their behavior is as required.
1976 (define_expand "vec_unpacks_lo_<mode>"
1977 [(match_operand:<VWIDE> 0 "register_operand" "")
1978 (match_operand:VQ_HSF 1 "register_operand" "")]
1979 "TARGET_SIMD"
1980 {
1981 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
1982 emit_insn (gen_aarch64_simd_vec_unpacks_lo_<mode> (operands[0],
1983 operands[1], p));
1984 DONE;
1985 }
1986 )
1988 (define_insn "aarch64_simd_vec_unpacks_hi_<mode>"
1989 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1990 (float_extend:<VWIDE> (vec_select:<VHALF>
1991 (match_operand:VQ_HSF 1 "register_operand" "w")
1992 (match_operand:VQ_HSF 2 "vect_par_cnst_hi_half" "")
1993 )))]
1994 "TARGET_SIMD"
1995 "fcvtl2\\t%0.<Vwtype>, %1.<Vtype>"
1996 [(set_attr "type" "neon_fp_cvt_widen_s")]
1997 )
1999 (define_expand "vec_unpacks_hi_<mode>"
2000 [(match_operand:<VWIDE> 0 "register_operand" "")
2001 (match_operand:VQ_HSF 1 "register_operand" "")]
2002 "TARGET_SIMD"
2003 {
2004 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2005 emit_insn (gen_aarch64_simd_vec_unpacks_lo_<mode> (operands[0],
2006 operands[1], p));
2007 DONE;
2008 }
2009 )
2010 (define_insn "aarch64_float_extend_lo_<Vwide>"
2011 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2012 (float_extend:<VWIDE>
2013 (match_operand:VDF 1 "register_operand" "w")))]
2014 "TARGET_SIMD"
2015 "fcvtl\\t%0<Vmwtype>, %1<Vmtype>"
2016 [(set_attr "type" "neon_fp_cvt_widen_s")]
2017 )
2019 ;; Float narrowing operations.
2021 (define_insn "aarch64_float_truncate_lo_<mode>"
2022 [(set (match_operand:VDF 0 "register_operand" "=w")
2023 (float_truncate:VDF
2024 (match_operand:<VWIDE> 1 "register_operand" "w")))]
2025 "TARGET_SIMD"
2026 "fcvtn\\t%0.<Vtype>, %1<Vmwtype>"
2027 [(set_attr "type" "neon_fp_cvt_narrow_d_q")]
2028 )
2030 (define_insn "aarch64_float_truncate_hi_<Vdbl>_le"
2031 [(set (match_operand:<VDBL> 0 "register_operand" "=w")
2032 (vec_concat:<VDBL>
2033 (match_operand:VDF 1 "register_operand" "0")
2034 (float_truncate:VDF
2035 (match_operand:<VWIDE> 2 "register_operand" "w"))))]
2036 "TARGET_SIMD && !BYTES_BIG_ENDIAN"
2037 "fcvtn2\\t%0.<Vdtype>, %2<Vmwtype>"
2038 [(set_attr "type" "neon_fp_cvt_narrow_d_q")]
2039 )
2041 (define_insn "aarch64_float_truncate_hi_<Vdbl>_be"
2042 [(set (match_operand:<VDBL> 0 "register_operand" "=w")
2043 (vec_concat:<VDBL>
2044 (float_truncate:VDF
2045 (match_operand:<VWIDE> 2 "register_operand" "w"))
2046 (match_operand:VDF 1 "register_operand" "0")))]
2047 "TARGET_SIMD && BYTES_BIG_ENDIAN"
2048 "fcvtn2\\t%0.<Vdtype>, %2<Vmwtype>"
2049 [(set_attr "type" "neon_fp_cvt_narrow_d_q")]
2050 )
2052 (define_expand "aarch64_float_truncate_hi_<Vdbl>"
2053 [(match_operand:<VDBL> 0 "register_operand" "=w")
2054 (match_operand:VDF 1 "register_operand" "0")
2055 (match_operand:<VWIDE> 2 "register_operand" "w")]
2056 "TARGET_SIMD"
2057 {
2058 rtx (*gen) (rtx, rtx, rtx) = BYTES_BIG_ENDIAN
2059 ? gen_aarch64_float_truncate_hi_<Vdbl>_be
2060 : gen_aarch64_float_truncate_hi_<Vdbl>_le;
2061 emit_insn (gen (operands[0], operands[1], operands[2]));
2062 DONE;
2063 }
2064 )
2066 (define_expand "vec_pack_trunc_v2df"
2067 [(set (match_operand:V4SF 0 "register_operand")
2068 (vec_concat:V4SF
2069 (float_truncate:V2SF
2070 (match_operand:V2DF 1 "register_operand"))
2071 (float_truncate:V2SF
2072 (match_operand:V2DF 2 "register_operand"))
2073 ))]
2074 "TARGET_SIMD"
2075 {
2076 rtx tmp = gen_reg_rtx (V2SFmode);
2077 int lo = BYTES_BIG_ENDIAN ? 2 : 1;
2078 int hi = BYTES_BIG_ENDIAN ? 1 : 2;
2080 emit_insn (gen_aarch64_float_truncate_lo_v2sf (tmp, operands[lo]));
2081 emit_insn (gen_aarch64_float_truncate_hi_v4sf (operands[0],
2082 tmp, operands[hi]));
2083 DONE;
2084 }
2085 )
2087 (define_expand "vec_pack_trunc_df"
2088 [(set (match_operand:V2SF 0 "register_operand")
2089 (vec_concat:V2SF
2090 (float_truncate:SF
2091 (match_operand:DF 1 "register_operand"))
2092 (float_truncate:SF
2093 (match_operand:DF 2 "register_operand"))
2094 ))]
2095 "TARGET_SIMD"
2096 {
2097 rtx tmp = gen_reg_rtx (V2SFmode);
2098 int lo = BYTES_BIG_ENDIAN ? 2 : 1;
2099 int hi = BYTES_BIG_ENDIAN ? 1 : 2;
2101 emit_insn (gen_move_lo_quad_v2df (tmp, operands[lo]));
2102 emit_insn (gen_move_hi_quad_v2df (tmp, operands[hi]));
2103 emit_insn (gen_aarch64_float_truncate_lo_v2sf (operands[0], tmp));
2104 DONE;
2105 }
2106 )
2108 ;; FP Max/Min
2109 ;; Max/Min are introduced by idiom recognition by GCC's mid-end. An
2110 ;; expression like:
2111 ;; a = (b < c) ? b : c;
2112 ;; is idiom-matched as MIN_EXPR<b,c> only if -ffinite-math-only is enabled
2113 ;; either explicitly or indirectly via -ffast-math.
2114 ;;
2115 ;; MIN_EXPR and MAX_EXPR eventually map to 'smin' and 'smax' in RTL.
2116 ;; The 'smax' and 'smin' RTL standard pattern names do not specify which
2117 ;; operand will be returned when both operands are zero (i.e. they may not
2118 ;; honour signed zeroes), or when either operand is NaN. Therefore GCC
2119 ;; only introduces MIN_EXPR/MAX_EXPR in fast math mode or when not honouring
2120 ;; NaNs.
2122 (define_insn "<su><maxmin><mode>3"
2123 [(set (match_operand:VHSDF 0 "register_operand" "=w")
2124 (FMAXMIN:VHSDF (match_operand:VHSDF 1 "register_operand" "w")
2125 (match_operand:VHSDF 2 "register_operand" "w")))]
2126 "TARGET_SIMD"
2127 "f<maxmin>nm\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
2128 [(set_attr "type" "neon_fp_minmax_<stype><q>")]
2129 )
2131 ;; Vector forms for fmax, fmin, fmaxnm, fminnm.
2132 ;; fmaxnm and fminnm are used for the fmax<mode>3 standard pattern names,
2133 ;; which implement the IEEE fmax ()/fmin () functions.
2134 (define_insn "<maxmin_uns><mode>3"
2135 [(set (match_operand:VHSDF 0 "register_operand" "=w")
2136 (unspec:VHSDF [(match_operand:VHSDF 1 "register_operand" "w")
2137 (match_operand:VHSDF 2 "register_operand" "w")]
2138 FMAXMIN_UNS))]
2139 "TARGET_SIMD"
2140 "<maxmin_uns_op>\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
2141 [(set_attr "type" "neon_fp_minmax_<stype><q>")]
2142 )
2144 ;; 'across lanes' add.
2146 (define_expand "reduc_plus_scal_<mode>"
2147 [(match_operand:<VEL> 0 "register_operand" "=w")
2148 (unspec:VDQ_I [(match_operand:VDQ_I 1 "register_operand" "w")]
2149 UNSPEC_ADDV)]
2150 "TARGET_SIMD"
2151 {
2152 rtx elt = GEN_INT (ENDIAN_LANE_N (<MODE>mode, 0));
2153 rtx scratch = gen_reg_rtx (<MODE>mode);
2154 emit_insn (gen_aarch64_reduc_plus_internal<mode> (scratch, operands[1]));
2155 emit_insn (gen_aarch64_get_lane<mode> (operands[0], scratch, elt));
2156 DONE;
2157 }
2158 )
2160 (define_insn "aarch64_faddp<mode>"
2161 [(set (match_operand:VHSDF 0 "register_operand" "=w")
2162 (unspec:VHSDF [(match_operand:VHSDF 1 "register_operand" "w")
2163 (match_operand:VHSDF 2 "register_operand" "w")]
2164 UNSPEC_FADDV))]
2165 "TARGET_SIMD"
2166 "faddp\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
2167 [(set_attr "type" "neon_fp_reduc_add_<stype><q>")]
2168 )
2170 (define_insn "aarch64_reduc_plus_internal<mode>"
2171 [(set (match_operand:VDQV 0 "register_operand" "=w")
2172 (unspec:VDQV [(match_operand:VDQV 1 "register_operand" "w")]
2173 UNSPEC_ADDV))]
2174 "TARGET_SIMD"
2175 "add<VDQV:vp>\\t%<Vetype>0, %1.<Vtype>"
2176 [(set_attr "type" "neon_reduc_add<q>")]
2177 )
2179 (define_insn "aarch64_reduc_plus_internalv2si"
2180 [(set (match_operand:V2SI 0 "register_operand" "=w")
2181 (unspec:V2SI [(match_operand:V2SI 1 "register_operand" "w")]
2182 UNSPEC_ADDV))]
2183 "TARGET_SIMD"
2184 "addp\\t%0.2s, %1.2s, %1.2s"
2185 [(set_attr "type" "neon_reduc_add")]
2186 )
2188 (define_insn "reduc_plus_scal_<mode>"
2189 [(set (match_operand:<VEL> 0 "register_operand" "=w")
2190 (unspec:<VEL> [(match_operand:V2F 1 "register_operand" "w")]
2191 UNSPEC_FADDV))]
2192 "TARGET_SIMD"
2193 "faddp\\t%<Vetype>0, %1.<Vtype>"
2194 [(set_attr "type" "neon_fp_reduc_add_<Vetype><q>")]
2195 )
2197 (define_expand "reduc_plus_scal_v4sf"
2198 [(set (match_operand:SF 0 "register_operand")
2199 (unspec:V4SF [(match_operand:V4SF 1 "register_operand")]
2200 UNSPEC_FADDV))]
2201 "TARGET_SIMD"
2202 {
2203 rtx elt = GEN_INT (ENDIAN_LANE_N (V4SFmode, 0));
2204 rtx scratch = gen_reg_rtx (V4SFmode);
2205 emit_insn (gen_aarch64_faddpv4sf (scratch, operands[1], operands[1]));
2206 emit_insn (gen_aarch64_faddpv4sf (scratch, scratch, scratch));
2207 emit_insn (gen_aarch64_get_lanev4sf (operands[0], scratch, elt));
2208 DONE;
2209 })
2211 (define_insn "clrsb<mode>2"
2212 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
2213 (clrsb:VDQ_BHSI (match_operand:VDQ_BHSI 1 "register_operand" "w")))]
2214 "TARGET_SIMD"
2215 "cls\\t%0.<Vtype>, %1.<Vtype>"
2216 [(set_attr "type" "neon_cls<q>")]
2217 )
2219 (define_insn "clz<mode>2"
2220 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
2221 (clz:VDQ_BHSI (match_operand:VDQ_BHSI 1 "register_operand" "w")))]
2222 "TARGET_SIMD"
2223 "clz\\t%0.<Vtype>, %1.<Vtype>"
2224 [(set_attr "type" "neon_cls<q>")]
2225 )
2227 (define_insn "popcount<mode>2"
2228 [(set (match_operand:VB 0 "register_operand" "=w")
2229 (popcount:VB (match_operand:VB 1 "register_operand" "w")))]
2230 "TARGET_SIMD"
2231 "cnt\\t%0.<Vbtype>, %1.<Vbtype>"
2232 [(set_attr "type" "neon_cnt<q>")]
2233 )
2235 ;; 'across lanes' max and min ops.
2237 ;; Template for outputting a scalar, so we can create __builtins which can be
2238 ;; gimple_fold'd to the REDUC_(MAX|MIN)_EXPR tree code. (This is FP smax/smin).
2239 (define_expand "reduc_<maxmin_uns>_scal_<mode>"
2240 [(match_operand:<VEL> 0 "register_operand")
2241 (unspec:VHSDF [(match_operand:VHSDF 1 "register_operand")]
2242 FMAXMINV)]
2243 "TARGET_SIMD"
2244 {
2245 rtx elt = GEN_INT (ENDIAN_LANE_N (<MODE>mode, 0));
2246 rtx scratch = gen_reg_rtx (<MODE>mode);
2247 emit_insn (gen_aarch64_reduc_<maxmin_uns>_internal<mode> (scratch,
2248 operands[1]));
2249 emit_insn (gen_aarch64_get_lane<mode> (operands[0], scratch, elt));
2250 DONE;
2251 }
2252 )
2254 ;; Likewise for integer cases, signed and unsigned.
2255 (define_expand "reduc_<maxmin_uns>_scal_<mode>"
2256 [(match_operand:<VEL> 0 "register_operand")
2257 (unspec:VDQ_BHSI [(match_operand:VDQ_BHSI 1 "register_operand")]
2258 MAXMINV)]
2259 "TARGET_SIMD"
2260 {
2261 rtx elt = GEN_INT (ENDIAN_LANE_N (<MODE>mode, 0));
2262 rtx scratch = gen_reg_rtx (<MODE>mode);
2263 emit_insn (gen_aarch64_reduc_<maxmin_uns>_internal<mode> (scratch,
2264 operands[1]));
2265 emit_insn (gen_aarch64_get_lane<mode> (operands[0], scratch, elt));
2266 DONE;
2267 }
2268 )
2270 (define_insn "aarch64_reduc_<maxmin_uns>_internal<mode>"
2271 [(set (match_operand:VDQV_S 0 "register_operand" "=w")
2272 (unspec:VDQV_S [(match_operand:VDQV_S 1 "register_operand" "w")]
2273 MAXMINV))]
2274 "TARGET_SIMD"
2275 "<maxmin_uns_op>v\\t%<Vetype>0, %1.<Vtype>"
2276 [(set_attr "type" "neon_reduc_minmax<q>")]
2277 )
2279 (define_insn "aarch64_reduc_<maxmin_uns>_internalv2si"
2280 [(set (match_operand:V2SI 0 "register_operand" "=w")
2281 (unspec:V2SI [(match_operand:V2SI 1 "register_operand" "w")]
2282 MAXMINV))]
2283 "TARGET_SIMD"
2284 "<maxmin_uns_op>p\\t%0.2s, %1.2s, %1.2s"
2285 [(set_attr "type" "neon_reduc_minmax")]
2286 )
2288 (define_insn "aarch64_reduc_<maxmin_uns>_internal<mode>"
2289 [(set (match_operand:VHSDF 0 "register_operand" "=w")
2290 (unspec:VHSDF [(match_operand:VHSDF 1 "register_operand" "w")]
2291 FMAXMINV))]
2292 "TARGET_SIMD"
2293 "<maxmin_uns_op><vp>\\t%<Vetype>0, %1.<Vtype>"
2294 [(set_attr "type" "neon_fp_reduc_minmax_<stype><q>")]
2295 )
2297 ;; aarch64_simd_bsl may compile to any of bsl/bif/bit depending on register
2298 ;; allocation.
2299 ;; Operand 1 is the mask, operands 2 and 3 are the bitfields from which
2300 ;; to select.
2301 ;;
2302 ;; Thus our BSL is of the form:
2303 ;; op0 = bsl (mask, op2, op3)
2304 ;; We can use any of:
2305 ;;
2306 ;; if (op0 = mask)
2307 ;; bsl mask, op1, op2
2308 ;; if (op0 = op1) (so 1-bits in mask choose bits from op2, else op0)
2309 ;; bit op0, op2, mask
2310 ;; if (op0 = op2) (so 0-bits in mask choose bits from op1, else op0)
2311 ;; bif op0, op1, mask
2312 ;;
2313 ;; This pattern is expanded to by the aarch64_simd_bsl<mode> expander.
2314 ;; Some forms of straight-line code may generate the equivalent form
2315 ;; in *aarch64_simd_bsl<mode>_alt.
2317 (define_insn "aarch64_simd_bsl<mode>_internal"
2318 [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w,w,w")
2319 (xor:VSDQ_I_DI
2320 (and:VSDQ_I_DI
2321 (xor:VSDQ_I_DI
2322 (match_operand:<V_INT_EQUIV> 3 "register_operand" "w,0,w")
2323 (match_operand:VSDQ_I_DI 2 "register_operand" "w,w,0"))
2324 (match_operand:VSDQ_I_DI 1 "register_operand" "0,w,w"))
2325 (match_dup:<V_INT_EQUIV> 3)
2326 ))]
2327 "TARGET_SIMD"
2328 "@
2329 bsl\\t%0.<Vbtype>, %2.<Vbtype>, %3.<Vbtype>
2330 bit\\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>
2331 bif\\t%0.<Vbtype>, %3.<Vbtype>, %1.<Vbtype>"
2332 [(set_attr "type" "neon_bsl<q>")]
2333 )
2335 ;; We need this form in addition to the above pattern to match the case
2336 ;; when combine tries merging three insns such that the second operand of
2337 ;; the outer XOR matches the second operand of the inner XOR rather than
2338 ;; the first. The two are equivalent but since recog doesn't try all
2339 ;; permutations of commutative operations, we have to have a separate pattern.
2341 (define_insn "*aarch64_simd_bsl<mode>_alt"
2342 [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w,w,w")
2343 (xor:VSDQ_I_DI
2344 (and:VSDQ_I_DI
2345 (xor:VSDQ_I_DI
2346 (match_operand:VSDQ_I_DI 3 "register_operand" "w,w,0")
2347 (match_operand:VSDQ_I_DI 2 "register_operand" "w,0,w"))
2348 (match_operand:VSDQ_I_DI 1 "register_operand" "0,w,w"))
2349 (match_dup:VSDQ_I_DI 2)))]
2350 "TARGET_SIMD"
2351 "@
2352 bsl\\t%0.<Vbtype>, %3.<Vbtype>, %2.<Vbtype>
2353 bit\\t%0.<Vbtype>, %3.<Vbtype>, %1.<Vbtype>
2354 bif\\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>"
2355 [(set_attr "type" "neon_bsl<q>")]
2356 )
2358 (define_expand "aarch64_simd_bsl<mode>"
2359 [(match_operand:VALLDIF 0 "register_operand")
2360 (match_operand:<V_INT_EQUIV> 1 "register_operand")
2361 (match_operand:VALLDIF 2 "register_operand")
2362 (match_operand:VALLDIF 3 "register_operand")]
2363 "TARGET_SIMD"
2364 {
2365 /* We can't alias operands together if they have different modes. */
2366 rtx tmp = operands[0];
2367 if (FLOAT_MODE_P (<MODE>mode))
2368 {
2369 operands[2] = gen_lowpart (<V_INT_EQUIV>mode, operands[2]);
2370 operands[3] = gen_lowpart (<V_INT_EQUIV>mode, operands[3]);
2371 tmp = gen_reg_rtx (<V_INT_EQUIV>mode);
2372 }
2373 operands[1] = gen_lowpart (<V_INT_EQUIV>mode, operands[1]);
2374 emit_insn (gen_aarch64_simd_bsl<v_int_equiv>_internal (tmp,
2375 operands[1],
2376 operands[2],
2377 operands[3]));
2378 if (tmp != operands[0])
2379 emit_move_insn (operands[0], gen_lowpart (<MODE>mode, tmp));
2381 DONE;
2382 })
2384 (define_expand "vcond_mask_<mode><v_int_equiv>"
2385 [(match_operand:VALLDI 0 "register_operand")
2386 (match_operand:VALLDI 1 "nonmemory_operand")
2387 (match_operand:VALLDI 2 "nonmemory_operand")
2388 (match_operand:<V_INT_EQUIV> 3 "register_operand")]
2389 "TARGET_SIMD"
2390 {
2391 /* If we have (a = (P) ? -1 : 0);
2392 Then we can simply move the generated mask (result must be int). */
2393 if (operands[1] == CONSTM1_RTX (<MODE>mode)
2394 && operands[2] == CONST0_RTX (<MODE>mode))
2395 emit_move_insn (operands[0], operands[3]);
2396 /* Similarly, (a = (P) ? 0 : -1) is just inverting the generated mask. */
2397 else if (operands[1] == CONST0_RTX (<MODE>mode)
2398 && operands[2] == CONSTM1_RTX (<MODE>mode))
2399 emit_insn (gen_one_cmpl<v_int_equiv>2 (operands[0], operands[3]));
2400 else
2401 {
2402 if (!REG_P (operands[1]))
2403 operands[1] = force_reg (<MODE>mode, operands[1]);
2404 if (!REG_P (operands[2]))
2405 operands[2] = force_reg (<MODE>mode, operands[2]);
2406 emit_insn (gen_aarch64_simd_bsl<mode> (operands[0], operands[3],
2407 operands[1], operands[2]));
2408 }
2410 DONE;
2411 })
2413 ;; Patterns comparing two vectors to produce a mask.
2415 (define_expand "vec_cmp<mode><mode>"
2416 [(set (match_operand:VSDQ_I_DI 0 "register_operand")
2417 (match_operator 1 "comparison_operator"
2418 [(match_operand:VSDQ_I_DI 2 "register_operand")
2419 (match_operand:VSDQ_I_DI 3 "nonmemory_operand")]))]
2420 "TARGET_SIMD"
2421 {
2422 rtx mask = operands[0];
2423 enum rtx_code code = GET_CODE (operands[1]);
2425 switch (code)
2426 {
2427 case NE:
2428 case LE:
2429 case LT:
2430 case GE:
2431 case GT:
2432 case EQ:
2433 if (operands[3] == CONST0_RTX (<MODE>mode))
2434 break;
2436 /* Fall through. */
2437 default:
2438 if (!REG_P (operands[3]))
2439 operands[3] = force_reg (<MODE>mode, operands[3]);
2441 break;
2442 }
2444 switch (code)
2445 {
2446 case LT:
2447 emit_insn (gen_aarch64_cmlt<mode> (mask, operands[2], operands[3]));
2448 break;
2450 case GE:
2451 emit_insn (gen_aarch64_cmge<mode> (mask, operands[2], operands[3]));
2452 break;
2454 case LE:
2455 emit_insn (gen_aarch64_cmle<mode> (mask, operands[2], operands[3]));
2456 break;
2458 case GT:
2459 emit_insn (gen_aarch64_cmgt<mode> (mask, operands[2], operands[3]));
2460 break;
2462 case LTU:
2463 emit_insn (gen_aarch64_cmgtu<mode> (mask, operands[3], operands[2]));
2464 break;
2466 case GEU:
2467 emit_insn (gen_aarch64_cmgeu<mode> (mask, operands[2], operands[3]));
2468 break;
2470 case LEU:
2471 emit_insn (gen_aarch64_cmgeu<mode> (mask, operands[3], operands[2]));
2472 break;
2474 case GTU:
2475 emit_insn (gen_aarch64_cmgtu<mode> (mask, operands[2], operands[3]));
2476 break;
2478 case NE:
2479 /* Handle NE as !EQ. */
2480 emit_insn (gen_aarch64_cmeq<mode> (mask, operands[2], operands[3]));
2481 emit_insn (gen_one_cmpl<v_int_equiv>2 (mask, mask));
2482 break;
2484 case EQ:
2485 emit_insn (gen_aarch64_cmeq<mode> (mask, operands[2], operands[3]));
2486 break;
2488 default:
2489 gcc_unreachable ();
2490 }
2492 DONE;
2493 })
2495 (define_expand "vec_cmp<mode><v_int_equiv>"
2496 [(set (match_operand:<V_INT_EQUIV> 0 "register_operand")
2497 (match_operator 1 "comparison_operator"
2498 [(match_operand:VDQF 2 "register_operand")
2499 (match_operand:VDQF 3 "nonmemory_operand")]))]
2500 "TARGET_SIMD"
2501 {
2502 int use_zero_form = 0;
2503 enum rtx_code code = GET_CODE (operands[1]);
2504 rtx tmp = gen_reg_rtx (<V_INT_EQUIV>mode);
2506 rtx (*comparison) (rtx, rtx, rtx) = NULL;
2508 switch (code)
2509 {
2510 case LE:
2511 case LT:
2512 case GE:
2513 case GT:
2514 case EQ:
2515 if (operands[3] == CONST0_RTX (<MODE>mode))
2516 {
2517 use_zero_form = 1;
2518 break;
2519 }
2520 /* Fall through. */
2521 default:
2522 if (!REG_P (operands[3]))
2523 operands[3] = force_reg (<MODE>mode, operands[3]);
2525 break;
2526 }
2528 switch (code)
2529 {
2530 case LT:
2531 if (use_zero_form)
2532 {
2533 comparison = gen_aarch64_cmlt<mode>;
2534 break;
2535 }
2536 /* Fall through. */
2537 case UNGE:
2538 std::swap (operands[2], operands[3]);
2539 /* Fall through. */
2540 case UNLE:
2541 case GT:
2542 comparison = gen_aarch64_cmgt<mode>;
2543 break;
2544 case LE:
2545 if (use_zero_form)
2546 {
2547 comparison = gen_aarch64_cmle<mode>;
2548 break;
2549 }
2550 /* Fall through. */
2551 case UNGT:
2552 std::swap (operands[2], operands[3]);
2553 /* Fall through. */
2554 case UNLT:
2555 case GE:
2556 comparison = gen_aarch64_cmge<mode>;
2557 break;
2558 case NE:
2559 case EQ:
2560 comparison = gen_aarch64_cmeq<mode>;
2561 break;
2562 case UNEQ:
2563 case ORDERED:
2564 case UNORDERED:
2565 break;
2566 default:
2567 gcc_unreachable ();
2568 }
2570 switch (code)
2571 {
2572 case UNGE:
2573 case UNGT:
2574 case UNLE:
2575 case UNLT:
2576 case NE:
2577 /* FCM returns false for lanes which are unordered, so if we use
2578 the inverse of the comparison we actually want to emit, then
2579 invert the result, we will end up with the correct result.
2580 Note that a NE NaN and NaN NE b are true for all a, b.
2582 Our transformations are:
2583 a UNGE b -> !(b GT a)
2584 a UNGT b -> !(b GE a)
2585 a UNLE b -> !(a GT b)
2586 a UNLT b -> !(a GE b)
2587 a NE b -> !(a EQ b) */
2588 gcc_assert (comparison != NULL);
2589 emit_insn (comparison (operands[0], operands[2], operands[3]));
2590 emit_insn (gen_one_cmpl<v_int_equiv>2 (operands[0], operands[0]));
2591 break;
2593 case LT:
2594 case LE:
2595 case GT:
2596 case GE:
2597 case EQ:
2598 /* The easy case. Here we emit one of FCMGE, FCMGT or FCMEQ.
2599 As a LT b <=> b GE a && a LE b <=> b GT a. Our transformations are:
2600 a GE b -> a GE b
2601 a GT b -> a GT b
2602 a LE b -> b GE a
2603 a LT b -> b GT a
2604 a EQ b -> a EQ b */
2605 gcc_assert (comparison != NULL);
2606 emit_insn (comparison (operands[0], operands[2], operands[3]));
2607 break;
2609 case UNEQ:
2610 /* We first check (a > b || b > a) which is !UNEQ, inverting
2611 this result will then give us (a == b || a UNORDERED b). */
2612 emit_insn (gen_aarch64_cmgt<mode> (operands[0],
2613 operands[2], operands[3]));
2614 emit_insn (gen_aarch64_cmgt<mode> (tmp, operands[3], operands[2]));
2615 emit_insn (gen_ior<v_int_equiv>3 (operands[0], operands[0], tmp));
2616 emit_insn (gen_one_cmpl<v_int_equiv>2 (operands[0], operands[0]));
2617 break;
2619 case UNORDERED:
2620 /* Operands are ORDERED iff (a > b || b >= a), so we can compute
2621 UNORDERED as !ORDERED. */
2622 emit_insn (gen_aarch64_cmgt<mode> (tmp, operands[2], operands[3]));
2623 emit_insn (gen_aarch64_cmge<mode> (operands[0],
2624 operands[3], operands[2]));
2625 emit_insn (gen_ior<v_int_equiv>3 (operands[0], operands[0], tmp));
2626 emit_insn (gen_one_cmpl<v_int_equiv>2 (operands[0], operands[0]));
2627 break;
2629 case ORDERED:
2630 emit_insn (gen_aarch64_cmgt<mode> (tmp, operands[2], operands[3]));
2631 emit_insn (gen_aarch64_cmge<mode> (operands[0],
2632 operands[3], operands[2]));
2633 emit_insn (gen_ior<v_int_equiv>3 (operands[0], operands[0], tmp));
2634 break;
2636 default:
2637 gcc_unreachable ();
2638 }
2640 DONE;
2641 })
2643 (define_expand "vec_cmpu<mode><mode>"
2644 [(set (match_operand:VSDQ_I_DI 0 "register_operand")
2645 (match_operator 1 "comparison_operator"
2646 [(match_operand:VSDQ_I_DI 2 "register_operand")
2647 (match_operand:VSDQ_I_DI 3 "nonmemory_operand")]))]
2648 "TARGET_SIMD"
2649 {
2650 emit_insn (gen_vec_cmp<mode><mode> (operands[0], operands[1],
2651 operands[2], operands[3]));
2652 DONE;
2653 })
2655 (define_expand "vcond<mode><mode>"
2656 [(set (match_operand:VALLDI 0 "register_operand")
2657 (if_then_else:VALLDI
2658 (match_operator 3 "comparison_operator"
2659 [(match_operand:VALLDI 4 "register_operand")
2660 (match_operand:VALLDI 5 "nonmemory_operand")])
2661 (match_operand:VALLDI 1 "nonmemory_operand")
2662 (match_operand:VALLDI 2 "nonmemory_operand")))]
2663 "TARGET_SIMD"
2664 {
2665 rtx mask = gen_reg_rtx (<V_INT_EQUIV>mode);
2666 enum rtx_code code = GET_CODE (operands[3]);
2668 /* NE is handled as !EQ in vec_cmp patterns, we can explicitly invert
2669 it as well as switch operands 1/2 in order to avoid the additional
2670 NOT instruction. */
2671 if (code == NE)
2672 {
2673 operands[3] = gen_rtx_fmt_ee (EQ, GET_MODE (operands[3]),
2674 operands[4], operands[5]);
2675 std::swap (operands[1], operands[2]);
2676 }
2677 emit_insn (gen_vec_cmp<mode><v_int_equiv> (mask, operands[3],
2678 operands[4], operands[5]));
2679 emit_insn (gen_vcond_mask_<mode><v_int_equiv> (operands[0], operands[1],
2680 operands[2], mask));
2682 DONE;
2683 })
2685 (define_expand "vcond<v_cmp_mixed><mode>"
2686 [(set (match_operand:<V_cmp_mixed> 0 "register_operand")
2687 (if_then_else:<V_cmp_mixed>
2688 (match_operator 3 "comparison_operator"
2689 [(match_operand:VDQF_COND 4 "register_operand")
2690 (match_operand:VDQF_COND 5 "nonmemory_operand")])
2691 (match_operand:<V_cmp_mixed> 1 "nonmemory_operand")
2692 (match_operand:<V_cmp_mixed> 2 "nonmemory_operand")))]
2693 "TARGET_SIMD"
2694 {
2695 rtx mask = gen_reg_rtx (<V_INT_EQUIV>mode);
2696 enum rtx_code code = GET_CODE (operands[3]);
2698 /* NE is handled as !EQ in vec_cmp patterns, we can explicitly invert
2699 it as well as switch operands 1/2 in order to avoid the additional
2700 NOT instruction. */
2701 if (code == NE)
2702 {
2703 operands[3] = gen_rtx_fmt_ee (EQ, GET_MODE (operands[3]),
2704 operands[4], operands[5]);
2705 std::swap (operands[1], operands[2]);
2706 }
2707 emit_insn (gen_vec_cmp<mode><v_int_equiv> (mask, operands[3],
2708 operands[4], operands[5]));
2709 emit_insn (gen_vcond_mask_<v_cmp_mixed><v_int_equiv> (
2710 operands[0], operands[1],
2711 operands[2], mask));
2713 DONE;
2714 })
2716 (define_expand "vcondu<mode><mode>"
2717 [(set (match_operand:VSDQ_I_DI 0 "register_operand")
2718 (if_then_else:VSDQ_I_DI
2719 (match_operator 3 "comparison_operator"
2720 [(match_operand:VSDQ_I_DI 4 "register_operand")
2721 (match_operand:VSDQ_I_DI 5 "nonmemory_operand")])
2722 (match_operand:VSDQ_I_DI 1 "nonmemory_operand")
2723 (match_operand:VSDQ_I_DI 2 "nonmemory_operand")))]
2724 "TARGET_SIMD"
2725 {
2726 rtx mask = gen_reg_rtx (<MODE>mode);
2727 enum rtx_code code = GET_CODE (operands[3]);
2729 /* NE is handled as !EQ in vec_cmp patterns, we can explicitly invert
2730 it as well as switch operands 1/2 in order to avoid the additional
2731 NOT instruction. */
2732 if (code == NE)
2733 {
2734 operands[3] = gen_rtx_fmt_ee (EQ, GET_MODE (operands[3]),
2735 operands[4], operands[5]);
2736 std::swap (operands[1], operands[2]);
2737 }
2738 emit_insn (gen_vec_cmp<mode><mode> (mask, operands[3],
2739 operands[4], operands[5]));
2740 emit_insn (gen_vcond_mask_<mode><v_int_equiv> (operands[0], operands[1],
2741 operands[2], mask));
2742 DONE;
2743 })
2745 (define_expand "vcondu<mode><v_cmp_mixed>"
2746 [(set (match_operand:VDQF 0 "register_operand")
2747 (if_then_else:VDQF
2748 (match_operator 3 "comparison_operator"
2749 [(match_operand:<V_cmp_mixed> 4 "register_operand")
2750 (match_operand:<V_cmp_mixed> 5 "nonmemory_operand")])
2751 (match_operand:VDQF 1 "nonmemory_operand")
2752 (match_operand:VDQF 2 "nonmemory_operand")))]
2753 "TARGET_SIMD"
2754 {
2755 rtx mask = gen_reg_rtx (<V_INT_EQUIV>mode);
2756 enum rtx_code code = GET_CODE (operands[3]);
2758 /* NE is handled as !EQ in vec_cmp patterns, we can explicitly invert
2759 it as well as switch operands 1/2 in order to avoid the additional
2760 NOT instruction. */
2761 if (code == NE)
2762 {
2763 operands[3] = gen_rtx_fmt_ee (EQ, GET_MODE (operands[3]),
2764 operands[4], operands[5]);
2765 std::swap (operands[1], operands[2]);
2766 }
2767 emit_insn (gen_vec_cmp<v_cmp_mixed><v_cmp_mixed> (
2768 mask, operands[3],
2769 operands[4], operands[5]));
2770 emit_insn (gen_vcond_mask_<mode><v_int_equiv> (operands[0], operands[1],
2771 operands[2], mask));
2772 DONE;
2773 })
2775 ;; Patterns for AArch64 SIMD Intrinsics.
2777 ;; Lane extraction with sign extension to general purpose register.
2778 (define_insn "*aarch64_get_lane_extend<GPI:mode><VDQQH:mode>"
2779 [(set (match_operand:GPI 0 "register_operand" "=r")
2780 (sign_extend:GPI
2781 (vec_select:<VEL>
2782 (match_operand:VDQQH 1 "register_operand" "w")
2783 (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2784 "TARGET_SIMD"
2785 {
2786 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
2787 return "smov\\t%<GPI:w>0, %1.<VDQQH:Vetype>[%2]";
2788 }
2789 [(set_attr "type" "neon_to_gp<q>")]
2790 )
2792 (define_insn "*aarch64_get_lane_zero_extendsi<mode>"
2793 [(set (match_operand:SI 0 "register_operand" "=r")
2794 (zero_extend:SI
2795 (vec_select:<VEL>
2796 (match_operand:VDQQH 1 "register_operand" "w")
2797 (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2798 "TARGET_SIMD"
2799 {
2800 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
2801 return "umov\\t%w0, %1.<Vetype>[%2]";
2802 }
2803 [(set_attr "type" "neon_to_gp<q>")]
2804 )
2806 ;; Lane extraction of a value, neither sign nor zero extension
2807 ;; is guaranteed so upper bits should be considered undefined.
2808 ;; RTL uses GCC vector extension indices throughout so flip only for assembly.
2809 (define_insn "aarch64_get_lane<mode>"
2810 [(set (match_operand:<VEL> 0 "aarch64_simd_nonimmediate_operand" "=r, w, Utv")
2811 (vec_select:<VEL>
2812 (match_operand:VALL_F16 1 "register_operand" "w, w, w")
2813 (parallel [(match_operand:SI 2 "immediate_operand" "i, i, i")])))]
2814 "TARGET_SIMD"
2815 {
2816 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
2817 switch (which_alternative)
2818 {
2819 case 0:
2820 return "umov\\t%<vwcore>0, %1.<Vetype>[%2]";
2821 case 1:
2822 return "dup\\t%<Vetype>0, %1.<Vetype>[%2]";
2823 case 2:
2824 return "st1\\t{%1.<Vetype>}[%2], %0";
2825 default:
2826 gcc_unreachable ();
2827 }
2828 }
2829 [(set_attr "type" "neon_to_gp<q>, neon_dup<q>, neon_store1_one_lane<q>")]
2830 )
2832 ;; In this insn, operand 1 should be low, and operand 2 the high part of the
2833 ;; dest vector.
2835 (define_insn "*aarch64_combinez<mode>"
2836 [(set (match_operand:<VDBL> 0 "register_operand" "=w,w,w")
2837 (vec_concat:<VDBL>
2838 (match_operand:VD_BHSI 1 "general_operand" "w,?r,m")
2839 (match_operand:VD_BHSI 2 "aarch64_simd_imm_zero" "Dz,Dz,Dz")))]
2840 "TARGET_SIMD && !BYTES_BIG_ENDIAN"
2841 "@
2842 mov\\t%0.8b, %1.8b
2843 fmov\t%d0, %1
2844 ldr\\t%d0, %1"
2845 [(set_attr "type" "neon_move<q>, neon_from_gp, neon_load1_1reg")
2846 (set_attr "simd" "yes,*,yes")
2847 (set_attr "fp" "*,yes,*")]
2848 )
2850 (define_insn "*aarch64_combinez_be<mode>"
2851 [(set (match_operand:<VDBL> 0 "register_operand" "=w,w,w")
2852 (vec_concat:<VDBL>
2853 (match_operand:VD_BHSI 2 "aarch64_simd_imm_zero" "Dz,Dz,Dz")
2854 (match_operand:VD_BHSI 1 "general_operand" "w,?r,m")))]
2855 "TARGET_SIMD && BYTES_BIG_ENDIAN"
2856 "@
2857 mov\\t%0.8b, %1.8b
2858 fmov\t%d0, %1
2859 ldr\\t%d0, %1"
2860 [(set_attr "type" "neon_move<q>, neon_from_gp, neon_load1_1reg")
2861 (set_attr "simd" "yes,*,yes")
2862 (set_attr "fp" "*,yes,*")]
2863 )
2865 (define_expand "aarch64_combine<mode>"
2866 [(match_operand:<VDBL> 0 "register_operand")
2867 (match_operand:VDC 1 "register_operand")
2868 (match_operand:VDC 2 "register_operand")]
2869 "TARGET_SIMD"
2870 {
2871 aarch64_split_simd_combine (operands[0], operands[1], operands[2]);
2873 DONE;
2874 }
2875 )
2877 (define_expand "aarch64_simd_combine<mode>"
2878 [(match_operand:<VDBL> 0 "register_operand")
2879 (match_operand:VDC 1 "register_operand")
2880 (match_operand:VDC 2 "register_operand")]
2881 "TARGET_SIMD"
2882 {
2883 emit_insn (gen_move_lo_quad_<Vdbl> (operands[0], operands[1]));
2884 emit_insn (gen_move_hi_quad_<Vdbl> (operands[0], operands[2]));
2885 DONE;
2886 }
2887 [(set_attr "type" "multiple")]
2888 )
2890 ;; <su><addsub>l<q>.
2892 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>l<mode>_hi_internal"
2893 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2894 (ADDSUB:<VWIDE> (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
2895 (match_operand:VQW 1 "register_operand" "w")
2896 (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))
2897 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
2898 (match_operand:VQW 2 "register_operand" "w")
2899 (match_dup 3)))))]
2900 "TARGET_SIMD"
2901 "<ANY_EXTEND:su><ADDSUB:optab>l2\t%0.<Vwtype>, %1.<Vtype>, %2.<Vtype>"
2902 [(set_attr "type" "neon_<ADDSUB:optab>_long")]
2903 )
2905 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>l<mode>_lo_internal"
2906 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2907 (ADDSUB:<VWIDE> (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
2908 (match_operand:VQW 1 "register_operand" "w")
2909 (match_operand:VQW 3 "vect_par_cnst_lo_half" "")))
2910 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
2911 (match_operand:VQW 2 "register_operand" "w")
2912 (match_dup 3)))))]
2913 "TARGET_SIMD"
2914 "<ANY_EXTEND:su><ADDSUB:optab>l\t%0.<Vwtype>, %1.<Vhalftype>, %2.<Vhalftype>"
2915 [(set_attr "type" "neon_<ADDSUB:optab>_long")]
2916 )
2919 (define_expand "aarch64_saddl2<mode>"
2920 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2921 (match_operand:VQW 1 "register_operand" "w")
2922 (match_operand:VQW 2 "register_operand" "w")]
2923 "TARGET_SIMD"
2924 {
2925 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2926 emit_insn (gen_aarch64_saddl<mode>_hi_internal (operands[0], operands[1],
2927 operands[2], p));
2928 DONE;
2929 })
2931 (define_expand "aarch64_uaddl2<mode>"
2932 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2933 (match_operand:VQW 1 "register_operand" "w")
2934 (match_operand:VQW 2 "register_operand" "w")]
2935 "TARGET_SIMD"
2936 {
2937 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2938 emit_insn (gen_aarch64_uaddl<mode>_hi_internal (operands[0], operands[1],
2939 operands[2], p));
2940 DONE;
2941 })
2943 (define_expand "aarch64_ssubl2<mode>"
2944 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2945 (match_operand:VQW 1 "register_operand" "w")
2946 (match_operand:VQW 2 "register_operand" "w")]
2947 "TARGET_SIMD"
2948 {
2949 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2950 emit_insn (gen_aarch64_ssubl<mode>_hi_internal (operands[0], operands[1],
2951 operands[2], p));
2952 DONE;
2953 })
2955 (define_expand "aarch64_usubl2<mode>"
2956 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2957 (match_operand:VQW 1 "register_operand" "w")
2958 (match_operand:VQW 2 "register_operand" "w")]
2959 "TARGET_SIMD"
2960 {
2961 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2962 emit_insn (gen_aarch64_usubl<mode>_hi_internal (operands[0], operands[1],
2963 operands[2], p));
2964 DONE;
2965 })
2967 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>l<mode>"
2968 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2969 (ADDSUB:<VWIDE> (ANY_EXTEND:<VWIDE>
2970 (match_operand:VD_BHSI 1 "register_operand" "w"))
2971 (ANY_EXTEND:<VWIDE>
2972 (match_operand:VD_BHSI 2 "register_operand" "w"))))]
2973 "TARGET_SIMD"
2974 "<ANY_EXTEND:su><ADDSUB:optab>l\t%0.<Vwtype>, %1.<Vtype>, %2.<Vtype>"
2975 [(set_attr "type" "neon_<ADDSUB:optab>_long")]
2976 )
2978 ;; <su><addsub>w<q>.
2980 (define_expand "widen_ssum<mode>3"
2981 [(set (match_operand:<VDBLW> 0 "register_operand" "")
2982 (plus:<VDBLW> (sign_extend:<VDBLW>
2983 (match_operand:VQW 1 "register_operand" ""))
2984 (match_operand:<VDBLW> 2 "register_operand" "")))]
2985 "TARGET_SIMD"
2986 {
2987 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
2988 rtx temp = gen_reg_rtx (GET_MODE (operands[0]));
2990 emit_insn (gen_aarch64_saddw<mode>_internal (temp, operands[2],
2991 operands[1], p));
2992 emit_insn (gen_aarch64_saddw2<mode> (operands[0], temp, operands[1]));
2993 DONE;
2994 }
2995 )
2997 (define_expand "widen_ssum<mode>3"
2998 [(set (match_operand:<VWIDE> 0 "register_operand" "")
2999 (plus:<VWIDE> (sign_extend:<VWIDE>
3000 (match_operand:VD_BHSI 1 "register_operand" ""))
3001 (match_operand:<VWIDE> 2 "register_operand" "")))]
3002 "TARGET_SIMD"
3003 {
3004 emit_insn (gen_aarch64_saddw<mode> (operands[0], operands[2], operands[1]));
3005 DONE;
3006 })
3008 (define_expand "widen_usum<mode>3"
3009 [(set (match_operand:<VDBLW> 0 "register_operand" "")
3010 (plus:<VDBLW> (zero_extend:<VDBLW>
3011 (match_operand:VQW 1 "register_operand" ""))
3012 (match_operand:<VDBLW> 2 "register_operand" "")))]
3013 "TARGET_SIMD"
3014 {
3015 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
3016 rtx temp = gen_reg_rtx (GET_MODE (operands[0]));
3018 emit_insn (gen_aarch64_uaddw<mode>_internal (temp, operands[2],
3019 operands[1], p));
3020 emit_insn (gen_aarch64_uaddw2<mode> (operands[0], temp, operands[1]));
3021 DONE;
3022 }
3023 )
3025 (define_expand "widen_usum<mode>3"
3026 [(set (match_operand:<VWIDE> 0 "register_operand" "")
3027 (plus:<VWIDE> (zero_extend:<VWIDE>
3028 (match_operand:VD_BHSI 1 "register_operand" ""))
3029 (match_operand:<VWIDE> 2 "register_operand" "")))]
3030 "TARGET_SIMD"
3031 {
3032 emit_insn (gen_aarch64_uaddw<mode> (operands[0], operands[2], operands[1]));
3033 DONE;
3034 })
3036 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>w<mode>"
3037 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3038 (ADDSUB:<VWIDE> (match_operand:<VWIDE> 1 "register_operand" "w")
3039 (ANY_EXTEND:<VWIDE>
3040 (match_operand:VD_BHSI 2 "register_operand" "w"))))]
3041 "TARGET_SIMD"
3042 "<ANY_EXTEND:su><ADDSUB:optab>w\\t%0.<Vwtype>, %1.<Vwtype>, %2.<Vtype>"
3043 [(set_attr "type" "neon_<ADDSUB:optab>_widen")]
3044 )
3046 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>w<mode>_internal"
3047 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3048 (ADDSUB:<VWIDE> (match_operand:<VWIDE> 1 "register_operand" "w")
3049 (ANY_EXTEND:<VWIDE>
3050 (vec_select:<VHALF>
3051 (match_operand:VQW 2 "register_operand" "w")
3052 (match_operand:VQW 3 "vect_par_cnst_lo_half" "")))))]
3053 "TARGET_SIMD"
3054 "<ANY_EXTEND:su><ADDSUB:optab>w\\t%0.<Vwtype>, %1.<Vwtype>, %2.<Vhalftype>"
3055 [(set_attr "type" "neon_<ADDSUB:optab>_widen")]
3056 )
3058 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>w2<mode>_internal"
3059 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3060 (ADDSUB:<VWIDE> (match_operand:<VWIDE> 1 "register_operand" "w")
3061 (ANY_EXTEND:<VWIDE>
3062 (vec_select:<VHALF>
3063 (match_operand:VQW 2 "register_operand" "w")
3064 (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))))]
3065 "TARGET_SIMD"
3066 "<ANY_EXTEND:su><ADDSUB:optab>w2\\t%0.<Vwtype>, %1.<Vwtype>, %2.<Vtype>"
3067 [(set_attr "type" "neon_<ADDSUB:optab>_widen")]
3068 )
3070 (define_expand "aarch64_saddw2<mode>"
3071 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3072 (match_operand:<VWIDE> 1 "register_operand" "w")
3073 (match_operand:VQW 2 "register_operand" "w")]
3074 "TARGET_SIMD"
3075 {
3076 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3077 emit_insn (gen_aarch64_saddw2<mode>_internal (operands[0], operands[1],
3078 operands[2], p));
3079 DONE;
3080 })
3082 (define_expand "aarch64_uaddw2<mode>"
3083 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3084 (match_operand:<VWIDE> 1 "register_operand" "w")
3085 (match_operand:VQW 2 "register_operand" "w")]
3086 "TARGET_SIMD"
3087 {
3088 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3089 emit_insn (gen_aarch64_uaddw2<mode>_internal (operands[0], operands[1],
3090 operands[2], p));
3091 DONE;
3092 })
3095 (define_expand "aarch64_ssubw2<mode>"
3096 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3097 (match_operand:<VWIDE> 1 "register_operand" "w")
3098 (match_operand:VQW 2 "register_operand" "w")]
3099 "TARGET_SIMD"
3100 {
3101 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3102 emit_insn (gen_aarch64_ssubw2<mode>_internal (operands[0], operands[1],
3103 operands[2], p));
3104 DONE;
3105 })
3107 (define_expand "aarch64_usubw2<mode>"
3108 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3109 (match_operand:<VWIDE> 1 "register_operand" "w")
3110 (match_operand:VQW 2 "register_operand" "w")]
3111 "TARGET_SIMD"
3112 {
3113 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3114 emit_insn (gen_aarch64_usubw2<mode>_internal (operands[0], operands[1],
3115 operands[2], p));
3116 DONE;
3117 })
3119 ;; <su><r>h<addsub>.
3121 (define_insn "aarch64_<sur>h<addsub><mode>"
3122 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
3123 (unspec:VDQ_BHSI [(match_operand:VDQ_BHSI 1 "register_operand" "w")
3124 (match_operand:VDQ_BHSI 2 "register_operand" "w")]
3125 HADDSUB))]
3126 "TARGET_SIMD"
3127 "<sur>h<addsub>\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
3128 [(set_attr "type" "neon_<addsub>_halve<q>")]
3129 )
3131 ;; <r><addsub>hn<q>.
3133 (define_insn "aarch64_<sur><addsub>hn<mode>"
3134 [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
3135 (unspec:<VNARROWQ> [(match_operand:VQN 1 "register_operand" "w")
3136 (match_operand:VQN 2 "register_operand" "w")]
3137 ADDSUBHN))]
3138 "TARGET_SIMD"
3139 "<sur><addsub>hn\\t%0.<Vntype>, %1.<Vtype>, %2.<Vtype>"
3140 [(set_attr "type" "neon_<addsub>_halve_narrow_q")]
3141 )
3143 (define_insn "aarch64_<sur><addsub>hn2<mode>"
3144 [(set (match_operand:<VNARROWQ2> 0 "register_operand" "=w")
3145 (unspec:<VNARROWQ2> [(match_operand:<VNARROWQ> 1 "register_operand" "0")
3146 (match_operand:VQN 2 "register_operand" "w")
3147 (match_operand:VQN 3 "register_operand" "w")]
3148 ADDSUBHN2))]
3149 "TARGET_SIMD"
3150 "<sur><addsub>hn2\\t%0.<V2ntype>, %2.<Vtype>, %3.<Vtype>"
3151 [(set_attr "type" "neon_<addsub>_halve_narrow_q")]
3152 )
3154 ;; pmul.
3156 (define_insn "aarch64_pmul<mode>"
3157 [(set (match_operand:VB 0 "register_operand" "=w")
3158 (unspec:VB [(match_operand:VB 1 "register_operand" "w")
3159 (match_operand:VB 2 "register_operand" "w")]
3160 UNSPEC_PMUL))]
3161 "TARGET_SIMD"
3162 "pmul\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
3163 [(set_attr "type" "neon_mul_<Vetype><q>")]
3164 )
3166 ;; fmulx.
3168 (define_insn "aarch64_fmulx<mode>"
3169 [(set (match_operand:VHSDF_HSDF 0 "register_operand" "=w")
3170 (unspec:VHSDF_HSDF
3171 [(match_operand:VHSDF_HSDF 1 "register_operand" "w")
3172 (match_operand:VHSDF_HSDF 2 "register_operand" "w")]
3173 UNSPEC_FMULX))]
3174 "TARGET_SIMD"
3175 "fmulx\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
3176 [(set_attr "type" "neon_fp_mul_<stype>")]
3177 )
3179 ;; vmulxq_lane_f32, and vmulx_laneq_f32
3181 (define_insn "*aarch64_mulx_elt_<vswap_width_name><mode>"
3182 [(set (match_operand:VDQSF 0 "register_operand" "=w")
3183 (unspec:VDQSF
3184 [(match_operand:VDQSF 1 "register_operand" "w")
3185 (vec_duplicate:VDQSF
3186 (vec_select:<VEL>
3187 (match_operand:<VSWAP_WIDTH> 2 "register_operand" "w")
3188 (parallel [(match_operand:SI 3 "immediate_operand" "i")])))]
3189 UNSPEC_FMULX))]
3190 "TARGET_SIMD"
3191 {
3192 operands[3] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
3193 INTVAL (operands[3])));
3194 return "fmulx\t%<v>0<Vmtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3195 }
3196 [(set_attr "type" "neon_fp_mul_<Vetype>_scalar<q>")]
3197 )
3199 ;; vmulxq_laneq_f32, vmulxq_laneq_f64, vmulx_lane_f32
3201 (define_insn "*aarch64_mulx_elt<mode>"
3202 [(set (match_operand:VDQF 0 "register_operand" "=w")
3203 (unspec:VDQF
3204 [(match_operand:VDQF 1 "register_operand" "w")
3205 (vec_duplicate:VDQF
3206 (vec_select:<VEL>
3207 (match_operand:VDQF 2 "register_operand" "w")
3208 (parallel [(match_operand:SI 3 "immediate_operand" "i")])))]
3209 UNSPEC_FMULX))]
3210 "TARGET_SIMD"
3211 {
3212 operands[3] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[3])));
3213 return "fmulx\t%<v>0<Vmtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3214 }
3215 [(set_attr "type" "neon_fp_mul_<Vetype><q>")]
3216 )
3218 ;; vmulxq_lane
3220 (define_insn "*aarch64_mulx_elt_from_dup<mode>"
3221 [(set (match_operand:VHSDF 0 "register_operand" "=w")
3222 (unspec:VHSDF
3223 [(match_operand:VHSDF 1 "register_operand" "w")
3224 (vec_duplicate:VHSDF
3225 (match_operand:<VEL> 2 "register_operand" "<h_con>"))]
3226 UNSPEC_FMULX))]
3227 "TARGET_SIMD"
3228 "fmulx\t%0.<Vtype>, %1.<Vtype>, %2.<Vetype>[0]";
3229 [(set_attr "type" "neon<fp>_mul_<stype>_scalar<q>")]
3230 )
3232 ;; vmulxs_lane_f32, vmulxs_laneq_f32
3233 ;; vmulxd_lane_f64 == vmulx_lane_f64
3234 ;; vmulxd_laneq_f64 == vmulx_laneq_f64
3236 (define_insn "*aarch64_vgetfmulx<mode>"
3237 [(set (match_operand:<VEL> 0 "register_operand" "=w")
3238 (unspec:<VEL>
3239 [(match_operand:<VEL> 1 "register_operand" "w")
3240 (vec_select:<VEL>
3241 (match_operand:VDQF 2 "register_operand" "w")
3242 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))]
3243 UNSPEC_FMULX))]
3244 "TARGET_SIMD"
3245 {
3246 operands[3] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[3])));
3247 return "fmulx\t%<Vetype>0, %<Vetype>1, %2.<Vetype>[%3]";
3248 }
3249 [(set_attr "type" "fmul<Vetype>")]
3250 )
3251 ;; <su>q<addsub>
3253 (define_insn "aarch64_<su_optab><optab><mode>"
3254 [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
3255 (BINQOPS:VSDQ_I (match_operand:VSDQ_I 1 "register_operand" "w")
3256 (match_operand:VSDQ_I 2 "register_operand" "w")))]
3257 "TARGET_SIMD"
3258 "<su_optab><optab>\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
3259 [(set_attr "type" "neon_<optab><q>")]
3260 )
3262 ;; suqadd and usqadd
3264 (define_insn "aarch64_<sur>qadd<mode>"
3265 [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
3266 (unspec:VSDQ_I [(match_operand:VSDQ_I 1 "register_operand" "0")
3267 (match_operand:VSDQ_I 2 "register_operand" "w")]
3268 USSUQADD))]
3269 "TARGET_SIMD"
3270 "<sur>qadd\\t%<v>0<Vmtype>, %<v>2<Vmtype>"
3271 [(set_attr "type" "neon_qadd<q>")]
3272 )
3274 ;; sqmovun
3276 (define_insn "aarch64_sqmovun<mode>"
3277 [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
3278 (unspec:<VNARROWQ> [(match_operand:VSQN_HSDI 1 "register_operand" "w")]
3279 UNSPEC_SQXTUN))]
3280 "TARGET_SIMD"
3281 "sqxtun\\t%<vn2>0<Vmntype>, %<v>1<Vmtype>"
3282 [(set_attr "type" "neon_sat_shift_imm_narrow_q")]
3283 )
3285 ;; sqmovn and uqmovn
3287 (define_insn "aarch64_<sur>qmovn<mode>"
3288 [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
3289 (unspec:<VNARROWQ> [(match_operand:VSQN_HSDI 1 "register_operand" "w")]
3290 SUQMOVN))]
3291 "TARGET_SIMD"
3292 "<sur>qxtn\\t%<vn2>0<Vmntype>, %<v>1<Vmtype>"
3293 [(set_attr "type" "neon_sat_shift_imm_narrow_q")]
3294 )
3296 ;; <su>q<absneg>
3298 (define_insn "aarch64_s<optab><mode>"
3299 [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
3300 (UNQOPS:VSDQ_I
3301 (match_operand:VSDQ_I 1 "register_operand" "w")))]
3302 "TARGET_SIMD"
3303 "s<optab>\\t%<v>0<Vmtype>, %<v>1<Vmtype>"
3304 [(set_attr "type" "neon_<optab><q>")]
3305 )
3307 ;; sq<r>dmulh.
3309 (define_insn "aarch64_sq<r>dmulh<mode>"
3310 [(set (match_operand:VSDQ_HSI 0 "register_operand" "=w")
3311 (unspec:VSDQ_HSI
3312 [(match_operand:VSDQ_HSI 1 "register_operand" "w")
3313 (match_operand:VSDQ_HSI 2 "register_operand" "w")]
3314 VQDMULH))]
3315 "TARGET_SIMD"
3316 "sq<r>dmulh\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
3317 [(set_attr "type" "neon_sat_mul_<Vetype><q>")]
3318 )
3320 ;; sq<r>dmulh_lane
3322 (define_insn "aarch64_sq<r>dmulh_lane<mode>"
3323 [(set (match_operand:VDQHS 0 "register_operand" "=w")
3324 (unspec:VDQHS
3325 [(match_operand:VDQHS 1 "register_operand" "w")
3326 (vec_select:<VEL>
3327 (match_operand:<VCOND> 2 "register_operand" "<vwx>")
3328 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))]
3329 VQDMULH))]
3330 "TARGET_SIMD"
3331 "*
3332 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
3333 return \"sq<r>dmulh\\t%0.<Vtype>, %1.<Vtype>, %2.<Vetype>[%3]\";"
3334 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar<q>")]
3335 )
3337 (define_insn "aarch64_sq<r>dmulh_laneq<mode>"
3338 [(set (match_operand:VDQHS 0 "register_operand" "=w")
3339 (unspec:VDQHS
3340 [(match_operand:VDQHS 1 "register_operand" "w")
3341 (vec_select:<VEL>
3342 (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
3343 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))]
3344 VQDMULH))]
3345 "TARGET_SIMD"
3346 "*
3347 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[3])));
3348 return \"sq<r>dmulh\\t%0.<Vtype>, %1.<Vtype>, %2.<Vetype>[%3]\";"
3349 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar<q>")]
3350 )
3352 (define_insn "aarch64_sq<r>dmulh_lane<mode>"
3353 [(set (match_operand:SD_HSI 0 "register_operand" "=w")
3354 (unspec:SD_HSI
3355 [(match_operand:SD_HSI 1 "register_operand" "w")
3356 (vec_select:<VEL>
3357 (match_operand:<VCOND> 2 "register_operand" "<vwx>")
3358 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))]
3359 VQDMULH))]
3360 "TARGET_SIMD"
3361 "*
3362 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
3363 return \"sq<r>dmulh\\t%<v>0, %<v>1, %2.<v>[%3]\";"
3364 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar<q>")]
3365 )
3367 (define_insn "aarch64_sq<r>dmulh_laneq<mode>"
3368 [(set (match_operand:SD_HSI 0 "register_operand" "=w")
3369 (unspec:SD_HSI
3370 [(match_operand:SD_HSI 1 "register_operand" "w")
3371 (vec_select:<VEL>
3372 (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
3373 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))]
3374 VQDMULH))]
3375 "TARGET_SIMD"
3376 "*
3377 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[3])));
3378 return \"sq<r>dmulh\\t%<v>0, %<v>1, %2.<v>[%3]\";"
3379 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar<q>")]
3380 )
3382 ;; sqrdml[as]h.
3384 (define_insn "aarch64_sqrdml<SQRDMLH_AS:rdma_as>h<mode>"
3385 [(set (match_operand:VSDQ_HSI 0 "register_operand" "=w")
3386 (unspec:VSDQ_HSI
3387 [(match_operand:VSDQ_HSI 1 "register_operand" "0")
3388 (match_operand:VSDQ_HSI 2 "register_operand" "w")
3389 (match_operand:VSDQ_HSI 3 "register_operand" "w")]
3390 SQRDMLH_AS))]
3391 "TARGET_SIMD_RDMA"
3392 "sqrdml<SQRDMLH_AS:rdma_as>h\\t%<v>0<Vmtype>, %<v>2<Vmtype>, %<v>3<Vmtype>"
3393 [(set_attr "type" "neon_sat_mla_<Vetype>_long")]
3394 )
3396 ;; sqrdml[as]h_lane.
3398 (define_insn "aarch64_sqrdml<SQRDMLH_AS:rdma_as>h_lane<mode>"
3399 [(set (match_operand:VDQHS 0 "register_operand" "=w")
3400 (unspec:VDQHS
3401 [(match_operand:VDQHS 1 "register_operand" "0")
3402 (match_operand:VDQHS 2 "register_operand" "w")
3403 (vec_select:<VEL>
3404 (match_operand:<VCOND> 3 "register_operand" "<vwx>")
3405 (parallel [(match_operand:SI 4 "immediate_operand" "i")]))]
3406 SQRDMLH_AS))]
3407 "TARGET_SIMD_RDMA"
3408 {
3409 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[4])));
3410 return
3411 "sqrdml<SQRDMLH_AS:rdma_as>h\\t%0.<Vtype>, %2.<Vtype>, %3.<Vetype>[%4]";
3412 }
3413 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3414 )
3416 (define_insn "aarch64_sqrdml<SQRDMLH_AS:rdma_as>h_lane<mode>"
3417 [(set (match_operand:SD_HSI 0 "register_operand" "=w")
3418 (unspec:SD_HSI
3419 [(match_operand:SD_HSI 1 "register_operand" "0")
3420 (match_operand:SD_HSI 2 "register_operand" "w")
3421 (vec_select:<VEL>
3422 (match_operand:<VCOND> 3 "register_operand" "<vwx>")
3423 (parallel [(match_operand:SI 4 "immediate_operand" "i")]))]
3424 SQRDMLH_AS))]
3425 "TARGET_SIMD_RDMA"
3426 {
3427 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[4])));
3428 return
3429 "sqrdml<SQRDMLH_AS:rdma_as>h\\t%<v>0, %<v>2, %3.<Vetype>[%4]";
3430 }
3431 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3432 )
3434 ;; sqrdml[as]h_laneq.
3436 (define_insn "aarch64_sqrdml<SQRDMLH_AS:rdma_as>h_laneq<mode>"
3437 [(set (match_operand:VDQHS 0 "register_operand" "=w")
3438 (unspec:VDQHS
3439 [(match_operand:VDQHS 1 "register_operand" "0")
3440 (match_operand:VDQHS 2 "register_operand" "w")
3441 (vec_select:<VEL>
3442 (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
3443 (parallel [(match_operand:SI 4 "immediate_operand" "i")]))]
3444 SQRDMLH_AS))]
3445 "TARGET_SIMD_RDMA"
3446 {
3447 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[4])));
3448 return
3449 "sqrdml<SQRDMLH_AS:rdma_as>h\\t%0.<Vtype>, %2.<Vtype>, %3.<Vetype>[%4]";
3450 }
3451 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3452 )
3454 (define_insn "aarch64_sqrdml<SQRDMLH_AS:rdma_as>h_laneq<mode>"
3455 [(set (match_operand:SD_HSI 0 "register_operand" "=w")
3456 (unspec:SD_HSI
3457 [(match_operand:SD_HSI 1 "register_operand" "0")
3458 (match_operand:SD_HSI 2 "register_operand" "w")
3459 (vec_select:<VEL>
3460 (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
3461 (parallel [(match_operand:SI 4 "immediate_operand" "i")]))]
3462 SQRDMLH_AS))]
3463 "TARGET_SIMD_RDMA"
3464 {
3465 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[4])));
3466 return
3467 "sqrdml<SQRDMLH_AS:rdma_as>h\\t%<v>0, %<v>2, %3.<v>[%4]";
3468 }
3469 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3470 )
3472 ;; vqdml[sa]l
3474 (define_insn "aarch64_sqdml<SBINQOPS:as>l<mode>"
3475 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3476 (SBINQOPS:<VWIDE>
3477 (match_operand:<VWIDE> 1 "register_operand" "0")
3478 (ss_ashift:<VWIDE>
3479 (mult:<VWIDE>
3480 (sign_extend:<VWIDE>
3481 (match_operand:VSD_HSI 2 "register_operand" "w"))
3482 (sign_extend:<VWIDE>
3483 (match_operand:VSD_HSI 3 "register_operand" "w")))
3484 (const_int 1))))]
3485 "TARGET_SIMD"
3486 "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %<v>3<Vmtype>"
3487 [(set_attr "type" "neon_sat_mla_<Vetype>_long")]
3488 )
3490 ;; vqdml[sa]l_lane
3492 (define_insn "aarch64_sqdml<SBINQOPS:as>l_lane<mode>"
3493 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3494 (SBINQOPS:<VWIDE>
3495 (match_operand:<VWIDE> 1 "register_operand" "0")
3496 (ss_ashift:<VWIDE>
3497 (mult:<VWIDE>
3498 (sign_extend:<VWIDE>
3499 (match_operand:VD_HSI 2 "register_operand" "w"))
3500 (sign_extend:<VWIDE>
3501 (vec_duplicate:VD_HSI
3502 (vec_select:<VEL>
3503 (match_operand:<VCOND> 3 "register_operand" "<vwx>")
3504 (parallel [(match_operand:SI 4 "immediate_operand" "i")])))
3505 ))
3506 (const_int 1))))]
3507 "TARGET_SIMD"
3508 {
3509 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[4])));
3510 return
3511 "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
3512 }
3513 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3514 )
3516 (define_insn "aarch64_sqdml<SBINQOPS:as>l_laneq<mode>"
3517 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3518 (SBINQOPS:<VWIDE>
3519 (match_operand:<VWIDE> 1 "register_operand" "0")
3520 (ss_ashift:<VWIDE>
3521 (mult:<VWIDE>
3522 (sign_extend:<VWIDE>
3523 (match_operand:VD_HSI 2 "register_operand" "w"))
3524 (sign_extend:<VWIDE>
3525 (vec_duplicate:VD_HSI
3526 (vec_select:<VEL>
3527 (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
3528 (parallel [(match_operand:SI 4 "immediate_operand" "i")])))
3529 ))
3530 (const_int 1))))]
3531 "TARGET_SIMD"
3532 {
3533 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[4])));
3534 return
3535 "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
3536 }
3537 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3538 )
3540 (define_insn "aarch64_sqdml<SBINQOPS:as>l_lane<mode>"
3541 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3542 (SBINQOPS:<VWIDE>
3543 (match_operand:<VWIDE> 1 "register_operand" "0")
3544 (ss_ashift:<VWIDE>
3545 (mult:<VWIDE>
3546 (sign_extend:<VWIDE>
3547 (match_operand:SD_HSI 2 "register_operand" "w"))
3548 (sign_extend:<VWIDE>
3549 (vec_select:<VEL>
3550 (match_operand:<VCOND> 3 "register_operand" "<vwx>")
3551 (parallel [(match_operand:SI 4 "immediate_operand" "i")])))
3552 )
3553 (const_int 1))))]
3554 "TARGET_SIMD"
3555 {
3556 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[4])));
3557 return
3558 "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
3559 }
3560 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3561 )
3563 (define_insn "aarch64_sqdml<SBINQOPS:as>l_laneq<mode>"
3564 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3565 (SBINQOPS:<VWIDE>
3566 (match_operand:<VWIDE> 1 "register_operand" "0")
3567 (ss_ashift:<VWIDE>
3568 (mult:<VWIDE>
3569 (sign_extend:<VWIDE>
3570 (match_operand:SD_HSI 2 "register_operand" "w"))
3571 (sign_extend:<VWIDE>
3572 (vec_select:<VEL>
3573 (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
3574 (parallel [(match_operand:SI 4 "immediate_operand" "i")])))
3575 )
3576 (const_int 1))))]
3577 "TARGET_SIMD"
3578 {
3579 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[4])));
3580 return
3581 "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
3582 }
3583 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3584 )
3586 ;; vqdml[sa]l_n
3588 (define_insn "aarch64_sqdml<SBINQOPS:as>l_n<mode>"
3589 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3590 (SBINQOPS:<VWIDE>
3591 (match_operand:<VWIDE> 1 "register_operand" "0")
3592 (ss_ashift:<VWIDE>
3593 (mult:<VWIDE>
3594 (sign_extend:<VWIDE>
3595 (match_operand:VD_HSI 2 "register_operand" "w"))
3596 (sign_extend:<VWIDE>
3597 (vec_duplicate:VD_HSI
3598 (match_operand:<VEL> 3 "register_operand" "<vwx>"))))
3599 (const_int 1))))]
3600 "TARGET_SIMD"
3601 "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[0]"
3602 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3603 )
3605 ;; sqdml[as]l2
3607 (define_insn "aarch64_sqdml<SBINQOPS:as>l2<mode>_internal"
3608 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3609 (SBINQOPS:<VWIDE>
3610 (match_operand:<VWIDE> 1 "register_operand" "0")
3611 (ss_ashift:<VWIDE>
3612 (mult:<VWIDE>
3613 (sign_extend:<VWIDE>
3614 (vec_select:<VHALF>
3615 (match_operand:VQ_HSI 2 "register_operand" "w")
3616 (match_operand:VQ_HSI 4 "vect_par_cnst_hi_half" "")))
3617 (sign_extend:<VWIDE>
3618 (vec_select:<VHALF>
3619 (match_operand:VQ_HSI 3 "register_operand" "w")
3620 (match_dup 4))))
3621 (const_int 1))))]
3622 "TARGET_SIMD"
3623 "sqdml<SBINQOPS:as>l2\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %<v>3<Vmtype>"
3624 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3625 )
3627 (define_expand "aarch64_sqdmlal2<mode>"
3628 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3629 (match_operand:<VWIDE> 1 "register_operand" "w")
3630 (match_operand:VQ_HSI 2 "register_operand" "w")
3631 (match_operand:VQ_HSI 3 "register_operand" "w")]
3632 "TARGET_SIMD"
3633 {
3634 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3635 emit_insn (gen_aarch64_sqdmlal2<mode>_internal (operands[0], operands[1],
3636 operands[2], operands[3], p));
3637 DONE;
3638 })
3640 (define_expand "aarch64_sqdmlsl2<mode>"
3641 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3642 (match_operand:<VWIDE> 1 "register_operand" "w")
3643 (match_operand:VQ_HSI 2 "register_operand" "w")
3644 (match_operand:VQ_HSI 3 "register_operand" "w")]
3645 "TARGET_SIMD"
3646 {
3647 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3648 emit_insn (gen_aarch64_sqdmlsl2<mode>_internal (operands[0], operands[1],
3649 operands[2], operands[3], p));
3650 DONE;
3651 })
3653 ;; vqdml[sa]l2_lane
3655 (define_insn "aarch64_sqdml<SBINQOPS:as>l2_lane<mode>_internal"
3656 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3657 (SBINQOPS:<VWIDE>
3658 (match_operand:<VWIDE> 1 "register_operand" "0")
3659 (ss_ashift:<VWIDE>
3660 (mult:<VWIDE>
3661 (sign_extend:<VWIDE>
3662 (vec_select:<VHALF>
3663 (match_operand:VQ_HSI 2 "register_operand" "w")
3664 (match_operand:VQ_HSI 5 "vect_par_cnst_hi_half" "")))
3665 (sign_extend:<VWIDE>
3666 (vec_duplicate:<VHALF>
3667 (vec_select:<VEL>
3668 (match_operand:<VCOND> 3 "register_operand" "<vwx>")
3669 (parallel [(match_operand:SI 4 "immediate_operand" "i")])
3670 ))))
3671 (const_int 1))))]
3672 "TARGET_SIMD"
3673 {
3674 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[4])));
3675 return
3676 "sqdml<SBINQOPS:as>l2\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
3677 }
3678 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3679 )
3681 (define_insn "aarch64_sqdml<SBINQOPS:as>l2_laneq<mode>_internal"
3682 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3683 (SBINQOPS:<VWIDE>
3684 (match_operand:<VWIDE> 1 "register_operand" "0")
3685 (ss_ashift:<VWIDE>
3686 (mult:<VWIDE>
3687 (sign_extend:<VWIDE>
3688 (vec_select:<VHALF>
3689 (match_operand:VQ_HSI 2 "register_operand" "w")
3690 (match_operand:VQ_HSI 5 "vect_par_cnst_hi_half" "")))
3691 (sign_extend:<VWIDE>
3692 (vec_duplicate:<VHALF>
3693 (vec_select:<VEL>
3694 (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
3695 (parallel [(match_operand:SI 4 "immediate_operand" "i")])
3696 ))))
3697 (const_int 1))))]
3698 "TARGET_SIMD"
3699 {
3700 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[4])));
3701 return
3702 "sqdml<SBINQOPS:as>l2\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
3703 }
3704 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3705 )
3707 (define_expand "aarch64_sqdmlal2_lane<mode>"
3708 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3709 (match_operand:<VWIDE> 1 "register_operand" "w")
3710 (match_operand:VQ_HSI 2 "register_operand" "w")
3711 (match_operand:<VCOND> 3 "register_operand" "<vwx>")
3712 (match_operand:SI 4 "immediate_operand" "i")]
3713 "TARGET_SIMD"
3714 {
3715 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3716 emit_insn (gen_aarch64_sqdmlal2_lane<mode>_internal (operands[0], operands[1],
3717 operands[2], operands[3],
3718 operands[4], p));
3719 DONE;
3720 })
3722 (define_expand "aarch64_sqdmlal2_laneq<mode>"
3723 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3724 (match_operand:<VWIDE> 1 "register_operand" "w")
3725 (match_operand:VQ_HSI 2 "register_operand" "w")
3726 (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
3727 (match_operand:SI 4 "immediate_operand" "i")]
3728 "TARGET_SIMD"
3729 {
3730 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3731 emit_insn (gen_aarch64_sqdmlal2_laneq<mode>_internal (operands[0], operands[1],
3732 operands[2], operands[3],
3733 operands[4], p));
3734 DONE;
3735 })
3737 (define_expand "aarch64_sqdmlsl2_lane<mode>"
3738 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3739 (match_operand:<VWIDE> 1 "register_operand" "w")
3740 (match_operand:VQ_HSI 2 "register_operand" "w")
3741 (match_operand:<VCOND> 3 "register_operand" "<vwx>")
3742 (match_operand:SI 4 "immediate_operand" "i")]
3743 "TARGET_SIMD"
3744 {
3745 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3746 emit_insn (gen_aarch64_sqdmlsl2_lane<mode>_internal (operands[0], operands[1],
3747 operands[2], operands[3],
3748 operands[4], p));
3749 DONE;
3750 })
3752 (define_expand "aarch64_sqdmlsl2_laneq<mode>"
3753 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3754 (match_operand:<VWIDE> 1 "register_operand" "w")
3755 (match_operand:VQ_HSI 2 "register_operand" "w")
3756 (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
3757 (match_operand:SI 4 "immediate_operand" "i")]
3758 "TARGET_SIMD"
3759 {
3760 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3761 emit_insn (gen_aarch64_sqdmlsl2_laneq<mode>_internal (operands[0], operands[1],
3762 operands[2], operands[3],
3763 operands[4], p));
3764 DONE;
3765 })
3767 (define_insn "aarch64_sqdml<SBINQOPS:as>l2_n<mode>_internal"
3768 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3769 (SBINQOPS:<VWIDE>
3770 (match_operand:<VWIDE> 1 "register_operand" "0")
3771 (ss_ashift:<VWIDE>
3772 (mult:<VWIDE>
3773 (sign_extend:<VWIDE>
3774 (vec_select:<VHALF>
3775 (match_operand:VQ_HSI 2 "register_operand" "w")
3776 (match_operand:VQ_HSI 4 "vect_par_cnst_hi_half" "")))
3777 (sign_extend:<VWIDE>
3778 (vec_duplicate:<VHALF>
3779 (match_operand:<VEL> 3 "register_operand" "<vwx>"))))
3780 (const_int 1))))]
3781 "TARGET_SIMD"
3782 "sqdml<SBINQOPS:as>l2\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[0]"
3783 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3784 )
3786 (define_expand "aarch64_sqdmlal2_n<mode>"
3787 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3788 (match_operand:<VWIDE> 1 "register_operand" "w")
3789 (match_operand:VQ_HSI 2 "register_operand" "w")
3790 (match_operand:<VEL> 3 "register_operand" "w")]
3791 "TARGET_SIMD"
3792 {
3793 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3794 emit_insn (gen_aarch64_sqdmlal2_n<mode>_internal (operands[0], operands[1],
3795 operands[2], operands[3],
3796 p));
3797 DONE;
3798 })
3800 (define_expand "aarch64_sqdmlsl2_n<mode>"
3801 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3802 (match_operand:<VWIDE> 1 "register_operand" "w")
3803 (match_operand:VQ_HSI 2 "register_operand" "w")
3804 (match_operand:<VEL> 3 "register_operand" "w")]
3805 "TARGET_SIMD"
3806 {
3807 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3808 emit_insn (gen_aarch64_sqdmlsl2_n<mode>_internal (operands[0], operands[1],
3809 operands[2], operands[3],
3810 p));
3811 DONE;
3812 })
3814 ;; vqdmull
3816 (define_insn "aarch64_sqdmull<mode>"
3817 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3818 (ss_ashift:<VWIDE>
3819 (mult:<VWIDE>
3820 (sign_extend:<VWIDE>
3821 (match_operand:VSD_HSI 1 "register_operand" "w"))
3822 (sign_extend:<VWIDE>
3823 (match_operand:VSD_HSI 2 "register_operand" "w")))
3824 (const_int 1)))]
3825 "TARGET_SIMD"
3826 "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
3827 [(set_attr "type" "neon_sat_mul_<Vetype>_long")]
3828 )
3830 ;; vqdmull_lane
3832 (define_insn "aarch64_sqdmull_lane<mode>"
3833 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3834 (ss_ashift:<VWIDE>
3835 (mult:<VWIDE>
3836 (sign_extend:<VWIDE>
3837 (match_operand:VD_HSI 1 "register_operand" "w"))
3838 (sign_extend:<VWIDE>
3839 (vec_duplicate:VD_HSI
3840 (vec_select:<VEL>
3841 (match_operand:<VCOND> 2 "register_operand" "<vwx>")
3842 (parallel [(match_operand:SI 3 "immediate_operand" "i")])))
3843 ))
3844 (const_int 1)))]
3845 "TARGET_SIMD"
3846 {
3847 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
3848 return "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3849 }
3850 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3851 )
3853 (define_insn "aarch64_sqdmull_laneq<mode>"
3854 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3855 (ss_ashift:<VWIDE>
3856 (mult:<VWIDE>
3857 (sign_extend:<VWIDE>
3858 (match_operand:VD_HSI 1 "register_operand" "w"))
3859 (sign_extend:<VWIDE>
3860 (vec_duplicate:VD_HSI
3861 (vec_select:<VEL>
3862 (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
3863 (parallel [(match_operand:SI 3 "immediate_operand" "i")])))
3864 ))
3865 (const_int 1)))]
3866 "TARGET_SIMD"
3867 {
3868 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[3])));
3869 return "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3870 }
3871 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3872 )
3874 (define_insn "aarch64_sqdmull_lane<mode>"
3875 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3876 (ss_ashift:<VWIDE>
3877 (mult:<VWIDE>
3878 (sign_extend:<VWIDE>
3879 (match_operand:SD_HSI 1 "register_operand" "w"))
3880 (sign_extend:<VWIDE>
3881 (vec_select:<VEL>
3882 (match_operand:<VCOND> 2 "register_operand" "<vwx>")
3883 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))
3884 ))
3885 (const_int 1)))]
3886 "TARGET_SIMD"
3887 {
3888 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
3889 return "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3890 }
3891 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3892 )
3894 (define_insn "aarch64_sqdmull_laneq<mode>"
3895 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3896 (ss_ashift:<VWIDE>
3897 (mult:<VWIDE>
3898 (sign_extend:<VWIDE>
3899 (match_operand:SD_HSI 1 "register_operand" "w"))
3900 (sign_extend:<VWIDE>
3901 (vec_select:<VEL>
3902 (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
3903 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))
3904 ))
3905 (const_int 1)))]
3906 "TARGET_SIMD"
3907 {
3908 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[3])));
3909 return "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3910 }
3911 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3912 )
3914 ;; vqdmull_n
3916 (define_insn "aarch64_sqdmull_n<mode>"
3917 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3918 (ss_ashift:<VWIDE>
3919 (mult:<VWIDE>
3920 (sign_extend:<VWIDE>
3921 (match_operand:VD_HSI 1 "register_operand" "w"))
3922 (sign_extend:<VWIDE>
3923 (vec_duplicate:VD_HSI
3924 (match_operand:<VEL> 2 "register_operand" "<vwx>")))
3925 )
3926 (const_int 1)))]
3927 "TARGET_SIMD"
3928 "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[0]"
3929 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3930 )
3932 ;; vqdmull2
3936 (define_insn "aarch64_sqdmull2<mode>_internal"
3937 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3938 (ss_ashift:<VWIDE>
3939 (mult:<VWIDE>
3940 (sign_extend:<VWIDE>
3941 (vec_select:<VHALF>
3942 (match_operand:VQ_HSI 1 "register_operand" "w")
3943 (match_operand:VQ_HSI 3 "vect_par_cnst_hi_half" "")))
3944 (sign_extend:<VWIDE>
3945 (vec_select:<VHALF>
3946 (match_operand:VQ_HSI 2 "register_operand" "w")
3947 (match_dup 3)))
3948 )
3949 (const_int 1)))]
3950 "TARGET_SIMD"
3951 "sqdmull2\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
3952 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3953 )
3955 (define_expand "aarch64_sqdmull2<mode>"
3956 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3957 (match_operand:VQ_HSI 1 "register_operand" "w")
3958 (match_operand:VQ_HSI 2 "register_operand" "w")]
3959 "TARGET_SIMD"
3960 {
3961 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3962 emit_insn (gen_aarch64_sqdmull2<mode>_internal (operands[0], operands[1],
3963 operands[2], p));
3964 DONE;
3965 })
3967 ;; vqdmull2_lane
3969 (define_insn "aarch64_sqdmull2_lane<mode>_internal"
3970 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3971 (ss_ashift:<VWIDE>
3972 (mult:<VWIDE>
3973 (sign_extend:<VWIDE>
3974 (vec_select:<VHALF>
3975 (match_operand:VQ_HSI 1 "register_operand" "w")
3976 (match_operand:VQ_HSI 4 "vect_par_cnst_hi_half" "")))
3977 (sign_extend:<VWIDE>
3978 (vec_duplicate:<VHALF>
3979 (vec_select:<VEL>
3980 (match_operand:<VCOND> 2 "register_operand" "<vwx>")
3981 (parallel [(match_operand:SI 3 "immediate_operand" "i")])))
3982 ))
3983 (const_int 1)))]
3984 "TARGET_SIMD"
3985 {
3986 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
3987 return "sqdmull2\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3988 }
3989 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3990 )
3992 (define_insn "aarch64_sqdmull2_laneq<mode>_internal"
3993 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3994 (ss_ashift:<VWIDE>
3995 (mult:<VWIDE>
3996 (sign_extend:<VWIDE>
3997 (vec_select:<VHALF>
3998 (match_operand:VQ_HSI 1 "register_operand" "w")
3999 (match_operand:VQ_HSI 4 "vect_par_cnst_hi_half" "")))
4000 (sign_extend:<VWIDE>
4001 (vec_duplicate:<VHALF>
4002 (vec_select:<VEL>
4003 (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
4004 (parallel [(match_operand:SI 3 "immediate_operand" "i")])))
4005 ))
4006 (const_int 1)))]
4007 "TARGET_SIMD"
4008 {
4009 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[3])));
4010 return "sqdmull2\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
4011 }
4012 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
4013 )
4015 (define_expand "aarch64_sqdmull2_lane<mode>"
4016 [(match_operand:<VWIDE> 0 "register_operand" "=w")
4017 (match_operand:VQ_HSI 1 "register_operand" "w")
4018 (match_operand:<VCOND> 2 "register_operand" "<vwx>")
4019 (match_operand:SI 3 "immediate_operand" "i")]
4020 "TARGET_SIMD"
4021 {
4022 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
4023 emit_insn (gen_aarch64_sqdmull2_lane<mode>_internal (operands[0], operands[1],
4024 operands[2], operands[3],
4025 p));
4026 DONE;
4027 })
4029 (define_expand "aarch64_sqdmull2_laneq<mode>"
4030 [(match_operand:<VWIDE> 0 "register_operand" "=w")
4031 (match_operand:VQ_HSI 1 "register_operand" "w")
4032 (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
4033 (match_operand:SI 3 "immediate_operand" "i")]
4034 "TARGET_SIMD"
4035 {
4036 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
4037 emit_insn (gen_aarch64_sqdmull2_laneq<mode>_internal (operands[0], operands[1],
4038 operands[2], operands[3],
4039 p));
4040 DONE;
4041 })
4043 ;; vqdmull2_n
4045 (define_insn "aarch64_sqdmull2_n<mode>_internal"
4046 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
4047 (ss_ashift:<VWIDE>
4048 (mult:<VWIDE>
4049 (sign_extend:<VWIDE>
4050 (vec_select:<VHALF>
4051 (match_operand:VQ_HSI 1 "register_operand" "w")
4052 (match_operand:VQ_HSI 3 "vect_par_cnst_hi_half" "")))
4053 (sign_extend:<VWIDE>
4054 (vec_duplicate:<VHALF>
4055 (match_operand:<VEL> 2 "register_operand" "<vwx>")))
4056 )
4057 (const_int 1)))]
4058 "TARGET_SIMD"
4059 "sqdmull2\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[0]"
4060 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
4061 )
4063 (define_expand "aarch64_sqdmull2_n<mode>"
4064 [(match_operand:<VWIDE> 0 "register_operand" "=w")
4065 (match_operand:VQ_HSI 1 "register_operand" "w")
4066 (match_operand:<VEL> 2 "register_operand" "w")]
4067 "TARGET_SIMD"
4068 {
4069 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
4070 emit_insn (gen_aarch64_sqdmull2_n<mode>_internal (operands[0], operands[1],
4071 operands[2], p));
4072 DONE;
4073 })
4075 ;; vshl
4077 (define_insn "aarch64_<sur>shl<mode>"
4078 [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
4079 (unspec:VSDQ_I_DI
4080 [(match_operand:VSDQ_I_DI 1 "register_operand" "w")
4081 (match_operand:VSDQ_I_DI 2 "register_operand" "w")]
4082 VSHL))]
4083 "TARGET_SIMD"
4084 "<sur>shl\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>";
4085 [(set_attr "type" "neon_shift_reg<q>")]
4086 )
4089 ;; vqshl
4091 (define_insn "aarch64_<sur>q<r>shl<mode>"
4092 [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
4093 (unspec:VSDQ_I
4094 [(match_operand:VSDQ_I 1 "register_operand" "w")
4095 (match_operand:VSDQ_I 2 "register_operand" "w")]
4096 VQSHL))]
4097 "TARGET_SIMD"
4098 "<sur>q<r>shl\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>";
4099 [(set_attr "type" "neon_sat_shift_reg<q>")]
4100 )
4102 ;; vshll_n
4104 (define_insn "aarch64_<sur>shll_n<mode>"
4105 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
4106 (unspec:<VWIDE> [(match_operand:VD_BHSI 1 "register_operand" "w")
4107 (match_operand:SI 2
4108 "aarch64_simd_shift_imm_bitsize_<ve_mode>" "i")]
4109 VSHLL))]
4110 "TARGET_SIMD"
4111 {
4112 if (INTVAL (operands[2]) == GET_MODE_UNIT_BITSIZE (<MODE>mode))
4113 return "shll\\t%0.<Vwtype>, %1.<Vtype>, %2";
4114 else
4115 return "<sur>shll\\t%0.<Vwtype>, %1.<Vtype>, %2";
4116 }
4117 [(set_attr "type" "neon_shift_imm_long")]
4118 )
4120 ;; vshll_high_n
4122 (define_insn "aarch64_<sur>shll2_n<mode>"
4123 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
4124 (unspec:<VWIDE> [(match_operand:VQW 1 "register_operand" "w")
4125 (match_operand:SI 2 "immediate_operand" "i")]
4126 VSHLL))]
4127 "TARGET_SIMD"
4128 {
4129 if (INTVAL (operands[2]) == GET_MODE_UNIT_BITSIZE (<MODE>mode))
4130 return "shll2\\t%0.<Vwtype>, %1.<Vtype>, %2";
4131 else
4132 return "<sur>shll2\\t%0.<Vwtype>, %1.<Vtype>, %2";
4133 }
4134 [(set_attr "type" "neon_shift_imm_long")]
4135 )
4137 ;; vrshr_n
4139 (define_insn "aarch64_<sur>shr_n<mode>"
4140 [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
4141 (unspec:VSDQ_I_DI [(match_operand:VSDQ_I_DI 1 "register_operand" "w")
4142 (match_operand:SI 2
4143 "aarch64_simd_shift_imm_offset_<ve_mode>" "i")]
4144 VRSHR_N))]
4145 "TARGET_SIMD"
4146 "<sur>shr\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %2"
4147 [(set_attr "type" "neon_sat_shift_imm<q>")]
4148 )
4150 ;; v(r)sra_n
4152 (define_insn "aarch64_<sur>sra_n<mode>"
4153 [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
4154 (unspec:VSDQ_I_DI [(match_operand:VSDQ_I_DI 1 "register_operand" "0")
4155 (match_operand:VSDQ_I_DI 2 "register_operand" "w")
4156 (match_operand:SI 3
4157 "aarch64_simd_shift_imm_offset_<ve_mode>" "i")]
4158 VSRA))]
4159 "TARGET_SIMD"
4160 "<sur>sra\\t%<v>0<Vmtype>, %<v>2<Vmtype>, %3"
4161 [(set_attr "type" "neon_shift_acc<q>")]
4162 )
4164 ;; vs<lr>i_n
4166 (define_insn "aarch64_<sur>s<lr>i_n<mode>"
4167 [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
4168 (unspec:VSDQ_I_DI [(match_operand:VSDQ_I_DI 1 "register_operand" "0")
4169 (match_operand:VSDQ_I_DI 2 "register_operand" "w")
4170 (match_operand:SI 3
4171 "aarch64_simd_shift_imm_<offsetlr><ve_mode>" "i")]
4172 VSLRI))]
4173 "TARGET_SIMD"
4174 "s<lr>i\\t%<v>0<Vmtype>, %<v>2<Vmtype>, %3"
4175 [(set_attr "type" "neon_shift_imm<q>")]
4176 )
4178 ;; vqshl(u)
4180 (define_insn "aarch64_<sur>qshl<u>_n<mode>"
4181 [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
4182 (unspec:VSDQ_I [(match_operand:VSDQ_I 1 "register_operand" "w")
4183 (match_operand:SI 2
4184 "aarch64_simd_shift_imm_<ve_mode>" "i")]
4185 VQSHL_N))]
4186 "TARGET_SIMD"
4187 "<sur>qshl<u>\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %2"
4188 [(set_attr "type" "neon_sat_shift_imm<q>")]
4189 )
4192 ;; vq(r)shr(u)n_n
4194 (define_insn "aarch64_<sur>q<r>shr<u>n_n<mode>"
4195 [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
4196 (unspec:<VNARROWQ> [(match_operand:VSQN_HSDI 1 "register_operand" "w")
4197 (match_operand:SI 2
4198 "aarch64_simd_shift_imm_offset_<ve_mode>" "i")]
4199 VQSHRN_N))]
4200 "TARGET_SIMD"
4201 "<sur>q<r>shr<u>n\\t%<vn2>0<Vmntype>, %<v>1<Vmtype>, %2"
4202 [(set_attr "type" "neon_sat_shift_imm_narrow_q")]
4203 )
4206 ;; cm(eq|ge|gt|lt|le)
4207 ;; Note, we have constraints for Dz and Z as different expanders
4208 ;; have different ideas of what should be passed to this pattern.
4210 (define_insn "aarch64_cm<optab><mode>"
4211 [(set (match_operand:<V_INT_EQUIV> 0 "register_operand" "=w,w")
4212 (neg:<V_INT_EQUIV>
4213 (COMPARISONS:<V_INT_EQUIV>
4214 (match_operand:VDQ_I 1 "register_operand" "w,w")
4215 (match_operand:VDQ_I 2 "aarch64_simd_reg_or_zero" "w,ZDz")
4216 )))]
4217 "TARGET_SIMD"
4218 "@
4219 cm<n_optab>\t%<v>0<Vmtype>, %<v><cmp_1><Vmtype>, %<v><cmp_2><Vmtype>
4220 cm<optab>\t%<v>0<Vmtype>, %<v>1<Vmtype>, #0"
4221 [(set_attr "type" "neon_compare<q>, neon_compare_zero<q>")]
4222 )
4224 (define_insn_and_split "aarch64_cm<optab>di"
4225 [(set (match_operand:DI 0 "register_operand" "=w,w,r")
4226 (neg:DI
4227 (COMPARISONS:DI
4228 (match_operand:DI 1 "register_operand" "w,w,r")
4229 (match_operand:DI 2 "aarch64_simd_reg_or_zero" "w,ZDz,r")
4230 )))
4231 (clobber (reg:CC CC_REGNUM))]
4232 "TARGET_SIMD"
4233 "#"
4234 "reload_completed"
4235 [(set (match_operand:DI 0 "register_operand")
4236 (neg:DI
4237 (COMPARISONS:DI
4238 (match_operand:DI 1 "register_operand")
4239 (match_operand:DI 2 "aarch64_simd_reg_or_zero")
4240 )))]
4241 {
4242 /* If we are in the general purpose register file,
4243 we split to a sequence of comparison and store. */
4244 if (GP_REGNUM_P (REGNO (operands[0]))
4245 && GP_REGNUM_P (REGNO (operands[1])))
4246 {
4247 machine_mode mode = SELECT_CC_MODE (<CMP>, operands[1], operands[2]);
4248 rtx cc_reg = aarch64_gen_compare_reg (<CMP>, operands[1], operands[2]);
4249 rtx comparison = gen_rtx_<CMP> (mode, operands[1], operands[2]);
4250 emit_insn (gen_cstoredi_neg (operands[0], comparison, cc_reg));
4251 DONE;
4252 }
4253 /* Otherwise, we expand to a similar pattern which does not
4254 clobber CC_REGNUM. */
4255 }
4256 [(set_attr "type" "neon_compare, neon_compare_zero, multiple")]
4257 )
4259 (define_insn "*aarch64_cm<optab>di"
4260 [(set (match_operand:DI 0 "register_operand" "=w,w")
4261 (neg:DI
4262 (COMPARISONS:DI
4263 (match_operand:DI 1 "register_operand" "w,w")
4264 (match_operand:DI 2 "aarch64_simd_reg_or_zero" "w,ZDz")
4265 )))]
4266 "TARGET_SIMD && reload_completed"
4267 "@
4268 cm<n_optab>\t%d0, %d<cmp_1>, %d<cmp_2>
4269 cm<optab>\t%d0, %d1, #0"
4270 [(set_attr "type" "neon_compare, neon_compare_zero")]
4271 )
4273 ;; cm(hs|hi)
4275 (define_insn "aarch64_cm<optab><mode>"
4276 [(set (match_operand:<V_INT_EQUIV> 0 "register_operand" "=w")
4277 (neg:<V_INT_EQUIV>
4278 (UCOMPARISONS:<V_INT_EQUIV>
4279 (match_operand:VDQ_I 1 "register_operand" "w")
4280 (match_operand:VDQ_I 2 "register_operand" "w")
4281 )))]
4282 "TARGET_SIMD"
4283 "cm<n_optab>\t%<v>0<Vmtype>, %<v><cmp_1><Vmtype>, %<v><cmp_2><Vmtype>"
4284 [(set_attr "type" "neon_compare<q>")]
4285 )
4287 (define_insn_and_split "aarch64_cm<optab>di"
4288 [(set (match_operand:DI 0 "register_operand" "=w,r")
4289 (neg:DI
4290 (UCOMPARISONS:DI
4291 (match_operand:DI 1 "register_operand" "w,r")
4292 (match_operand:DI 2 "aarch64_simd_reg_or_zero" "w,r")
4293 )))
4294 (clobber (reg:CC CC_REGNUM))]
4295 "TARGET_SIMD"
4296 "#"
4297 "reload_completed"
4298 [(set (match_operand:DI 0 "register_operand")
4299 (neg:DI
4300 (UCOMPARISONS:DI
4301 (match_operand:DI 1 "register_operand")
4302 (match_operand:DI 2 "aarch64_simd_reg_or_zero")
4303 )))]
4304 {
4305 /* If we are in the general purpose register file,
4306 we split to a sequence of comparison and store. */
4307 if (GP_REGNUM_P (REGNO (operands[0]))
4308 && GP_REGNUM_P (REGNO (operands[1])))
4309 {
4310 machine_mode mode = CCmode;
4311 rtx cc_reg = aarch64_gen_compare_reg (<CMP>, operands[1], operands[2]);
4312 rtx comparison = gen_rtx_<CMP> (mode, operands[1], operands[2]);
4313 emit_insn (gen_cstoredi_neg (operands[0], comparison, cc_reg));
4314 DONE;
4315 }
4316 /* Otherwise, we expand to a similar pattern which does not
4317 clobber CC_REGNUM. */
4318 }
4319 [(set_attr "type" "neon_compare,multiple")]
4320 )
4322 (define_insn "*aarch64_cm<optab>di"
4323 [(set (match_operand:DI 0 "register_operand" "=w")
4324 (neg:DI
4325 (UCOMPARISONS:DI
4326 (match_operand:DI 1 "register_operand" "w")
4327 (match_operand:DI 2 "aarch64_simd_reg_or_zero" "w")
4328 )))]
4329 "TARGET_SIMD && reload_completed"
4330 "cm<n_optab>\t%d0, %d<cmp_1>, %d<cmp_2>"
4331 [(set_attr "type" "neon_compare")]
4332 )
4334 ;; cmtst
4336 ;; Although neg (ne (and x y) 0) is the natural way of expressing a cmtst,
4337 ;; we don't have any insns using ne, and aarch64_vcond outputs
4338 ;; not (neg (eq (and x y) 0))
4339 ;; which is rewritten by simplify_rtx as
4340 ;; plus (eq (and x y) 0) -1.
4342 (define_insn "aarch64_cmtst<mode>"
4343 [(set (match_operand:<V_INT_EQUIV> 0 "register_operand" "=w")
4344 (plus:<V_INT_EQUIV>
4345 (eq:<V_INT_EQUIV>
4346 (and:VDQ_I
4347 (match_operand:VDQ_I 1 "register_operand" "w")
4348 (match_operand:VDQ_I 2 "register_operand" "w"))
4349 (match_operand:VDQ_I 3 "aarch64_simd_imm_zero"))
4350 (match_operand:<V_INT_EQUIV> 4 "aarch64_simd_imm_minus_one")))
4351 ]
4352 "TARGET_SIMD"
4353 "cmtst\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
4354 [(set_attr "type" "neon_tst<q>")]
4355 )
4357 (define_insn_and_split "aarch64_cmtstdi"
4358 [(set (match_operand:DI 0 "register_operand" "=w,r")
4359 (neg:DI
4360 (ne:DI
4361 (and:DI
4362 (match_operand:DI 1 "register_operand" "w,r")
4363 (match_operand:DI 2 "register_operand" "w,r"))
4364 (const_int 0))))
4365 (clobber (reg:CC CC_REGNUM))]
4366 "TARGET_SIMD"
4367 "#"
4368 "reload_completed"
4369 [(set (match_operand:DI 0 "register_operand")
4370 (neg:DI
4371 (ne:DI
4372 (and:DI
4373 (match_operand:DI 1 "register_operand")
4374 (match_operand:DI 2 "register_operand"))
4375 (const_int 0))))]
4376 {
4377 /* If we are in the general purpose register file,
4378 we split to a sequence of comparison and store. */
4379 if (GP_REGNUM_P (REGNO (operands[0]))
4380 && GP_REGNUM_P (REGNO (operands[1])))
4381 {
4382 rtx and_tree = gen_rtx_AND (DImode, operands[1], operands[2]);
4383 machine_mode mode = SELECT_CC_MODE (NE, and_tree, const0_rtx);
4384 rtx cc_reg = aarch64_gen_compare_reg (NE, and_tree, const0_rtx);
4385 rtx comparison = gen_rtx_NE (mode, and_tree, const0_rtx);
4386 emit_insn (gen_cstoredi_neg (operands[0], comparison, cc_reg));
4387 DONE;
4388 }
4389 /* Otherwise, we expand to a similar pattern which does not
4390 clobber CC_REGNUM. */
4391 }
4392 [(set_attr "type" "neon_tst,multiple")]
4393 )
4395 (define_insn "*aarch64_cmtstdi"
4396 [(set (match_operand:DI 0 "register_operand" "=w")
4397 (neg:DI
4398 (ne:DI
4399 (and:DI
4400 (match_operand:DI 1 "register_operand" "w")
4401 (match_operand:DI 2 "register_operand" "w"))
4402 (const_int 0))))]
4403 "TARGET_SIMD"
4404 "cmtst\t%d0, %d1, %d2"
4405 [(set_attr "type" "neon_tst")]
4406 )
4408 ;; fcm(eq|ge|gt|le|lt)
4410 (define_insn "aarch64_cm<optab><mode>"
4411 [(set (match_operand:<V_INT_EQUIV> 0 "register_operand" "=w,w")
4412 (neg:<V_INT_EQUIV>
4413 (COMPARISONS:<V_INT_EQUIV>
4414 (match_operand:VHSDF_HSDF 1 "register_operand" "w,w")
4415 (match_operand:VHSDF_HSDF 2 "aarch64_simd_reg_or_zero" "w,YDz")
4416 )))]
4417 "TARGET_SIMD"
4418 "@
4419 fcm<n_optab>\t%<v>0<Vmtype>, %<v><cmp_1><Vmtype>, %<v><cmp_2><Vmtype>
4420 fcm<optab>\t%<v>0<Vmtype>, %<v>1<Vmtype>, 0"
4421 [(set_attr "type" "neon_fp_compare_<stype><q>")]
4422 )
4424 ;; fac(ge|gt)
4425 ;; Note we can also handle what would be fac(le|lt) by
4426 ;; generating fac(ge|gt).
4428 (define_insn "aarch64_fac<optab><mode>"
4429 [(set (match_operand:<V_INT_EQUIV> 0 "register_operand" "=w")
4430 (neg:<V_INT_EQUIV>
4431 (FAC_COMPARISONS:<V_INT_EQUIV>
4432 (abs:VHSDF_HSDF
4433 (match_operand:VHSDF_HSDF 1 "register_operand" "w"))
4434 (abs:VHSDF_HSDF
4435 (match_operand:VHSDF_HSDF 2 "register_operand" "w"))
4436 )))]
4437 "TARGET_SIMD"
4438 "fac<n_optab>\t%<v>0<Vmtype>, %<v><cmp_1><Vmtype>, %<v><cmp_2><Vmtype>"
4439 [(set_attr "type" "neon_fp_compare_<stype><q>")]
4440 )
4442 ;; addp
4444 (define_insn "aarch64_addp<mode>"
4445 [(set (match_operand:VD_BHSI 0 "register_operand" "=w")
4446 (unspec:VD_BHSI
4447 [(match_operand:VD_BHSI 1 "register_operand" "w")
4448 (match_operand:VD_BHSI 2 "register_operand" "w")]
4449 UNSPEC_ADDP))]
4450 "TARGET_SIMD"
4451 "addp\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
4452 [(set_attr "type" "neon_reduc_add<q>")]
4453 )
4455 (define_insn "aarch64_addpdi"
4456 [(set (match_operand:DI 0 "register_operand" "=w")
4457 (unspec:DI
4458 [(match_operand:V2DI 1 "register_operand" "w")]
4459 UNSPEC_ADDP))]
4460 "TARGET_SIMD"
4461 "addp\t%d0, %1.2d"
4462 [(set_attr "type" "neon_reduc_add")]
4463 )
4465 ;; sqrt
4467 (define_expand "sqrt<mode>2"
4468 [(set (match_operand:VHSDF 0 "register_operand" "=w")
4469 (sqrt:VHSDF (match_operand:VHSDF 1 "register_operand" "w")))]
4470 "TARGET_SIMD"
4471 {
4472 if (aarch64_emit_approx_sqrt (operands[0], operands[1], false))
4473 DONE;
4474 })
4476 (define_insn "*sqrt<mode>2"
4477 [(set (match_operand:VHSDF 0 "register_operand" "=w")
4478 (sqrt:VHSDF (match_operand:VHSDF 1 "register_operand" "w")))]
4479 "TARGET_SIMD"
4480 "fsqrt\\t%0.<Vtype>, %1.<Vtype>"
4481 [(set_attr "type" "neon_fp_sqrt_<stype><q>")]
4482 )
4484 ;; Patterns for vector struct loads and stores.
4486 (define_insn "aarch64_simd_ld2<mode>"
4487 [(set (match_operand:OI 0 "register_operand" "=w")
4488 (unspec:OI [(match_operand:OI 1 "aarch64_simd_struct_operand" "Utv")
4489 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4490 UNSPEC_LD2))]
4491 "TARGET_SIMD"
4492 "ld2\\t{%S0.<Vtype> - %T0.<Vtype>}, %1"
4493 [(set_attr "type" "neon_load2_2reg<q>")]
4494 )
4496 (define_insn "aarch64_simd_ld2r<mode>"
4497 [(set (match_operand:OI 0 "register_operand" "=w")
4498 (unspec:OI [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")
4499 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY) ]
4500 UNSPEC_LD2_DUP))]
4501 "TARGET_SIMD"
4502 "ld2r\\t{%S0.<Vtype> - %T0.<Vtype>}, %1"
4503 [(set_attr "type" "neon_load2_all_lanes<q>")]
4504 )
4506 (define_insn "aarch64_vec_load_lanesoi_lane<mode>"
4507 [(set (match_operand:OI 0 "register_operand" "=w")
4508 (unspec:OI [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")
4509 (match_operand:OI 2 "register_operand" "0")
4510 (match_operand:SI 3 "immediate_operand" "i")
4511 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY) ]
4512 UNSPEC_LD2_LANE))]
4513 "TARGET_SIMD"
4514 {
4515 operands[3] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[3])));
4516 return "ld2\\t{%S0.<Vetype> - %T0.<Vetype>}[%3], %1";
4517 }
4518 [(set_attr "type" "neon_load2_one_lane")]
4519 )
4521 (define_expand "vec_load_lanesoi<mode>"
4522 [(set (match_operand:OI 0 "register_operand" "=w")
4523 (unspec:OI [(match_operand:OI 1 "aarch64_simd_struct_operand" "Utv")
4524 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4525 UNSPEC_LD2))]
4526 "TARGET_SIMD"
4527 {
4528 if (BYTES_BIG_ENDIAN)
4529 {
4530 rtx tmp = gen_reg_rtx (OImode);
4531 rtx mask = aarch64_reverse_mask (<MODE>mode);
4532 emit_insn (gen_aarch64_simd_ld2<mode> (tmp, operands[1]));
4533 emit_insn (gen_aarch64_rev_reglistoi (operands[0], tmp, mask));
4534 }
4535 else
4536 emit_insn (gen_aarch64_simd_ld2<mode> (operands[0], operands[1]));
4537 DONE;
4538 })
4540 (define_insn "aarch64_simd_st2<mode>"
4541 [(set (match_operand:OI 0 "aarch64_simd_struct_operand" "=Utv")
4542 (unspec:OI [(match_operand:OI 1 "register_operand" "w")
4543 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4544 UNSPEC_ST2))]
4545 "TARGET_SIMD"
4546 "st2\\t{%S1.<Vtype> - %T1.<Vtype>}, %0"
4547 [(set_attr "type" "neon_store2_2reg<q>")]
4548 )
4550 ;; RTL uses GCC vector extension indices, so flip only for assembly.
4551 (define_insn "aarch64_vec_store_lanesoi_lane<mode>"
4552 [(set (match_operand:BLK 0 "aarch64_simd_struct_operand" "=Utv")
4553 (unspec:BLK [(match_operand:OI 1 "register_operand" "w")
4554 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)
4555 (match_operand:SI 2 "immediate_operand" "i")]
4556 UNSPEC_ST2_LANE))]
4557 "TARGET_SIMD"
4558 {
4559 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
4560 return "st2\\t{%S1.<Vetype> - %T1.<Vetype>}[%2], %0";
4561 }
4562 [(set_attr "type" "neon_store2_one_lane<q>")]
4563 )
4565 (define_expand "vec_store_lanesoi<mode>"
4566 [(set (match_operand:OI 0 "aarch64_simd_struct_operand" "=Utv")
4567 (unspec:OI [(match_operand:OI 1 "register_operand" "w")
4568 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4569 UNSPEC_ST2))]
4570 "TARGET_SIMD"
4571 {
4572 if (BYTES_BIG_ENDIAN)
4573 {
4574 rtx tmp = gen_reg_rtx (OImode);
4575 rtx mask = aarch64_reverse_mask (<MODE>mode);
4576 emit_insn (gen_aarch64_rev_reglistoi (tmp, operands[1], mask));
4577 emit_insn (gen_aarch64_simd_st2<mode> (operands[0], tmp));
4578 }
4579 else
4580 emit_insn (gen_aarch64_simd_st2<mode> (operands[0], operands[1]));
4581 DONE;
4582 })
4584 (define_insn "aarch64_simd_ld3<mode>"
4585 [(set (match_operand:CI 0 "register_operand" "=w")
4586 (unspec:CI [(match_operand:CI 1 "aarch64_simd_struct_operand" "Utv")
4587 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4588 UNSPEC_LD3))]
4589 "TARGET_SIMD"
4590 "ld3\\t{%S0.<Vtype> - %U0.<Vtype>}, %1"
4591 [(set_attr "type" "neon_load3_3reg<q>")]
4592 )
4594 (define_insn "aarch64_simd_ld3r<mode>"
4595 [(set (match_operand:CI 0 "register_operand" "=w")
4596 (unspec:CI [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")
4597 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY) ]
4598 UNSPEC_LD3_DUP))]
4599 "TARGET_SIMD"
4600 "ld3r\\t{%S0.<Vtype> - %U0.<Vtype>}, %1"
4601 [(set_attr "type" "neon_load3_all_lanes<q>")]
4602 )
4604 (define_insn "aarch64_vec_load_lanesci_lane<mode>"
4605 [(set (match_operand:CI 0 "register_operand" "=w")
4606 (unspec:CI [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")
4607 (match_operand:CI 2 "register_operand" "0")
4608 (match_operand:SI 3 "immediate_operand" "i")
4609 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4610 UNSPEC_LD3_LANE))]
4611 "TARGET_SIMD"
4612 {
4613 operands[3] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[3])));
4614 return "ld3\\t{%S0.<Vetype> - %U0.<Vetype>}[%3], %1";
4615 }
4616 [(set_attr "type" "neon_load3_one_lane")]
4617 )
4619 (define_expand "vec_load_lanesci<mode>"
4620 [(set (match_operand:CI 0 "register_operand" "=w")
4621 (unspec:CI [(match_operand:CI 1 "aarch64_simd_struct_operand" "Utv")
4622 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4623 UNSPEC_LD3))]
4624 "TARGET_SIMD"
4625 {
4626 if (BYTES_BIG_ENDIAN)
4627 {
4628 rtx tmp = gen_reg_rtx (CImode);
4629 rtx mask = aarch64_reverse_mask (<MODE>mode);
4630 emit_insn (gen_aarch64_simd_ld3<mode> (tmp, operands[1]));
4631 emit_insn (gen_aarch64_rev_reglistci (operands[0], tmp, mask));
4632 }
4633 else
4634 emit_insn (gen_aarch64_simd_ld3<mode> (operands[0], operands[1]));
4635 DONE;
4636 })
4638 (define_insn "aarch64_simd_st3<mode>"
4639 [(set (match_operand:CI 0 "aarch64_simd_struct_operand" "=Utv")
4640 (unspec:CI [(match_operand:CI 1 "register_operand" "w")
4641 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4642 UNSPEC_ST3))]
4643 "TARGET_SIMD"
4644 "st3\\t{%S1.<Vtype> - %U1.<Vtype>}, %0"
4645 [(set_attr "type" "neon_store3_3reg<q>")]
4646 )
4648 ;; RTL uses GCC vector extension indices, so flip only for assembly.
4649 (define_insn "aarch64_vec_store_lanesci_lane<mode>"
4650 [(set (match_operand:BLK 0 "aarch64_simd_struct_operand" "=Utv")
4651 (unspec:BLK [(match_operand:CI 1 "register_operand" "w")
4652 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)
4653 (match_operand:SI 2 "immediate_operand" "i")]
4654 UNSPEC_ST3_LANE))]
4655 "TARGET_SIMD"
4656 {
4657 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
4658 return "st3\\t{%S1.<Vetype> - %U1.<Vetype>}[%2], %0";
4659 }
4660 [(set_attr "type" "neon_store3_one_lane<q>")]
4661 )
4663 (define_expand "vec_store_lanesci<mode>"
4664 [(set (match_operand:CI 0 "aarch64_simd_struct_operand" "=Utv")
4665 (unspec:CI [(match_operand:CI 1 "register_operand" "w")
4666 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4667 UNSPEC_ST3))]
4668 "TARGET_SIMD"
4669 {
4670 if (BYTES_BIG_ENDIAN)
4671 {
4672 rtx tmp = gen_reg_rtx (CImode);
4673 rtx mask = aarch64_reverse_mask (<MODE>mode);
4674 emit_insn (gen_aarch64_rev_reglistci (tmp, operands[1], mask));
4675 emit_insn (gen_aarch64_simd_st3<mode> (operands[0], tmp));
4676 }
4677 else
4678 emit_insn (gen_aarch64_simd_st3<mode> (operands[0], operands[1]));
4679 DONE;
4680 })
4682 (define_insn "aarch64_simd_ld4<mode>"
4683 [(set (match_operand:XI 0 "register_operand" "=w")
4684 (unspec:XI [(match_operand:XI 1 "aarch64_simd_struct_operand" "Utv")
4685 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4686 UNSPEC_LD4))]
4687 "TARGET_SIMD"
4688 "ld4\\t{%S0.<Vtype> - %V0.<Vtype>}, %1"
4689 [(set_attr "type" "neon_load4_4reg<q>")]
4690 )
4692 (define_insn "aarch64_simd_ld4r<mode>"
4693 [(set (match_operand:XI 0 "register_operand" "=w")
4694 (unspec:XI [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")
4695 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY) ]
4696 UNSPEC_LD4_DUP))]
4697 "TARGET_SIMD"
4698 "ld4r\\t{%S0.<Vtype> - %V0.<Vtype>}, %1"
4699 [(set_attr "type" "neon_load4_all_lanes<q>")]
4700 )
4702 (define_insn "aarch64_vec_load_lanesxi_lane<mode>"
4703 [(set (match_operand:XI 0 "register_operand" "=w")
4704 (unspec:XI [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")
4705 (match_operand:XI 2 "register_operand" "0")
4706 (match_operand:SI 3 "immediate_operand" "i")
4707 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4708 UNSPEC_LD4_LANE))]
4709 "TARGET_SIMD"
4710 {
4711 operands[3] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[3])));
4712 return "ld4\\t{%S0.<Vetype> - %V0.<Vetype>}[%3], %1";
4713 }
4714 [(set_attr "type" "neon_load4_one_lane")]
4715 )
4717 (define_expand "vec_load_lanesxi<mode>"
4718 [(set (match_operand:XI 0 "register_operand" "=w")
4719 (unspec:XI [(match_operand:XI 1 "aarch64_simd_struct_operand" "Utv")
4720 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4721 UNSPEC_LD4))]
4722 "TARGET_SIMD"
4723 {
4724 if (BYTES_BIG_ENDIAN)
4725 {
4726 rtx tmp = gen_reg_rtx (XImode);
4727 rtx mask = aarch64_reverse_mask (<MODE>mode);
4728 emit_insn (gen_aarch64_simd_ld4<mode> (tmp, operands[1]));
4729 emit_insn (gen_aarch64_rev_reglistxi (operands[0], tmp, mask));
4730 }
4731 else
4732 emit_insn (gen_aarch64_simd_ld4<mode> (operands[0], operands[1]));
4733 DONE;
4734 })
4736 (define_insn "aarch64_simd_st4<mode>"
4737 [(set (match_operand:XI 0 "aarch64_simd_struct_operand" "=Utv")
4738 (unspec:XI [(match_operand:XI 1 "register_operand" "w")
4739 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4740 UNSPEC_ST4))]
4741 "TARGET_SIMD"
4742 "st4\\t{%S1.<Vtype> - %V1.<Vtype>}, %0"
4743 [(set_attr "type" "neon_store4_4reg<q>")]
4744 )
4746 ;; RTL uses GCC vector extension indices, so flip only for assembly.
4747 (define_insn "aarch64_vec_store_lanesxi_lane<mode>"
4748 [(set (match_operand:BLK 0 "aarch64_simd_struct_operand" "=Utv")
4749 (unspec:BLK [(match_operand:XI 1 "register_operand" "w")
4750 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)
4751 (match_operand:SI 2 "immediate_operand" "i")]
4752 UNSPEC_ST4_LANE))]
4753 "TARGET_SIMD"
4754 {
4755 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
4756 return "st4\\t{%S1.<Vetype> - %V1.<Vetype>}[%2], %0";
4757 }
4758 [(set_attr "type" "neon_store4_one_lane<q>")]
4759 )
4761 (define_expand "vec_store_lanesxi<mode>"
4762 [(set (match_operand:XI 0 "aarch64_simd_struct_operand" "=Utv")
4763 (unspec:XI [(match_operand:XI 1 "register_operand" "w")
4764 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4765 UNSPEC_ST4))]
4766 "TARGET_SIMD"
4767 {
4768 if (BYTES_BIG_ENDIAN)
4769 {
4770 rtx tmp = gen_reg_rtx (XImode);
4771 rtx mask = aarch64_reverse_mask (<MODE>mode);
4772 emit_insn (gen_aarch64_rev_reglistxi (tmp, operands[1], mask));
4773 emit_insn (gen_aarch64_simd_st4<mode> (operands[0], tmp));
4774 }
4775 else
4776 emit_insn (gen_aarch64_simd_st4<mode> (operands[0], operands[1]));
4777 DONE;
4778 })
4780 (define_insn_and_split "aarch64_rev_reglist<mode>"
4781 [(set (match_operand:VSTRUCT 0 "register_operand" "=&w")
4782 (unspec:VSTRUCT
4783 [(match_operand:VSTRUCT 1 "register_operand" "w")
4784 (match_operand:V16QI 2 "register_operand" "w")]
4785 UNSPEC_REV_REGLIST))]
4786 "TARGET_SIMD"
4787 "#"
4788 "&& reload_completed"
4789 [(const_int 0)]
4790 {
4791 int i;
4792 int nregs = GET_MODE_SIZE (<MODE>mode) / UNITS_PER_VREG;
4793 for (i = 0; i < nregs; i++)
4794 {
4795 rtx op0 = gen_rtx_REG (V16QImode, REGNO (operands[0]) + i);
4796 rtx op1 = gen_rtx_REG (V16QImode, REGNO (operands[1]) + i);
4797 emit_insn (gen_aarch64_tbl1v16qi (op0, op1, operands[2]));
4798 }
4799 DONE;
4800 }
4801 [(set_attr "type" "neon_tbl1_q")
4802 (set_attr "length" "<insn_count>")]
4803 )
4805 ;; Reload patterns for AdvSIMD register list operands.
4807 (define_expand "mov<mode>"
4808 [(set (match_operand:VSTRUCT 0 "nonimmediate_operand" "")
4809 (match_operand:VSTRUCT 1 "general_operand" ""))]
4810 "TARGET_SIMD"
4811 {
4812 if (can_create_pseudo_p ())
4813 {
4814 if (GET_CODE (operands[0]) != REG)
4815 operands[1] = force_reg (<MODE>mode, operands[1]);
4816 }
4817 })
4819 (define_insn "*aarch64_mov<mode>"
4820 [(set (match_operand:VSTRUCT 0 "aarch64_simd_nonimmediate_operand" "=w,Utv,w")
4821 (match_operand:VSTRUCT 1 "aarch64_simd_general_operand" " w,w,Utv"))]
4822 "TARGET_SIMD && !BYTES_BIG_ENDIAN
4823 && (register_operand (operands[0], <MODE>mode)
4824 || register_operand (operands[1], <MODE>mode))"
4825 "@
4826 #
4827 st1\\t{%S1.16b - %<Vendreg>1.16b}, %0
4828 ld1\\t{%S0.16b - %<Vendreg>0.16b}, %1"
4829 [(set_attr "type" "multiple,neon_store<nregs>_<nregs>reg_q,\
4830 neon_load<nregs>_<nregs>reg_q")
4831 (set_attr "length" "<insn_count>,4,4")]
4832 )
4834 (define_insn "aarch64_be_ld1<mode>"
4835 [(set (match_operand:VALLDI_F16 0 "register_operand" "=w")
4836 (unspec:VALLDI_F16 [(match_operand:VALLDI_F16 1
4837 "aarch64_simd_struct_operand" "Utv")]
4838 UNSPEC_LD1))]
4839 "TARGET_SIMD"
4840 "ld1\\t{%0<Vmtype>}, %1"
4841 [(set_attr "type" "neon_load1_1reg<q>")]
4842 )
4844 (define_insn "aarch64_be_st1<mode>"
4845 [(set (match_operand:VALLDI_F16 0 "aarch64_simd_struct_operand" "=Utv")
4846 (unspec:VALLDI_F16 [(match_operand:VALLDI_F16 1 "register_operand" "w")]
4847 UNSPEC_ST1))]
4848 "TARGET_SIMD"
4849 "st1\\t{%1<Vmtype>}, %0"
4850 [(set_attr "type" "neon_store1_1reg<q>")]
4851 )
4853 (define_insn "*aarch64_be_movoi"
4854 [(set (match_operand:OI 0 "nonimmediate_operand" "=w,m,w")
4855 (match_operand:OI 1 "general_operand" " w,w,m"))]
4856 "TARGET_SIMD && BYTES_BIG_ENDIAN
4857 && (register_operand (operands[0], OImode)
4858 || register_operand (operands[1], OImode))"
4859 "@
4860 #
4861 stp\\t%q1, %R1, %0
4862 ldp\\t%q0, %R0, %1"
4863 [(set_attr "type" "multiple,neon_stp_q,neon_ldp_q")
4864 (set_attr "length" "8,4,4")]
4865 )
4867 (define_insn "*aarch64_be_movci"
4868 [(set (match_operand:CI 0 "nonimmediate_operand" "=w,o,w")
4869 (match_operand:CI 1 "general_operand" " w,w,o"))]
4870 "TARGET_SIMD && BYTES_BIG_ENDIAN
4871 && (register_operand (operands[0], CImode)
4872 || register_operand (operands[1], CImode))"
4873 "#"
4874 [(set_attr "type" "multiple")
4875 (set_attr "length" "12,4,4")]
4876 )
4878 (define_insn "*aarch64_be_movxi"
4879 [(set (match_operand:XI 0 "nonimmediate_operand" "=w,o,w")
4880 (match_operand:XI 1 "general_operand" " w,w,o"))]
4881 "TARGET_SIMD && BYTES_BIG_ENDIAN
4882 && (register_operand (operands[0], XImode)
4883 || register_operand (operands[1], XImode))"
4884 "#"
4885 [(set_attr "type" "multiple")
4886 (set_attr "length" "16,4,4")]
4887 )
4889 (define_split
4890 [(set (match_operand:OI 0 "register_operand")
4891 (match_operand:OI 1 "register_operand"))]
4892 "TARGET_SIMD && reload_completed"
4893 [(const_int 0)]
4894 {
4895 aarch64_simd_emit_reg_reg_move (operands, TImode, 2);
4896 DONE;
4897 })
4899 (define_split
4900 [(set (match_operand:CI 0 "nonimmediate_operand")
4901 (match_operand:CI 1 "general_operand"))]
4902 "TARGET_SIMD && reload_completed"
4903 [(const_int 0)]
4904 {
4905 if (register_operand (operands[0], CImode)
4906 && register_operand (operands[1], CImode))
4907 {
4908 aarch64_simd_emit_reg_reg_move (operands, TImode, 3);
4909 DONE;
4910 }
4911 else if (BYTES_BIG_ENDIAN)
4912 {
4913 emit_move_insn (simplify_gen_subreg (OImode, operands[0], CImode, 0),
4914 simplify_gen_subreg (OImode, operands[1], CImode, 0));
4915 emit_move_insn (gen_lowpart (V16QImode,
4916 simplify_gen_subreg (TImode, operands[0],
4917 CImode, 32)),
4918 gen_lowpart (V16QImode,
4919 simplify_gen_subreg (TImode, operands[1],
4920 CImode, 32)));
4921 DONE;
4922 }
4923 else
4924 FAIL;
4925 })
4927 (define_split
4928 [(set (match_operand:XI 0 "nonimmediate_operand")
4929 (match_operand:XI 1 "general_operand"))]
4930 "TARGET_SIMD && reload_completed"
4931 [(const_int 0)]
4932 {
4933 if (register_operand (operands[0], XImode)
4934 && register_operand (operands[1], XImode))
4935 {
4936 aarch64_simd_emit_reg_reg_move (operands, TImode, 4);
4937 DONE;
4938 }
4939 else if (BYTES_BIG_ENDIAN)
4940 {
4941 emit_move_insn (simplify_gen_subreg (OImode, operands[0], XImode, 0),
4942 simplify_gen_subreg (OImode, operands[1], XImode, 0));
4943 emit_move_insn (simplify_gen_subreg (OImode, operands[0], XImode, 32),
4944 simplify_gen_subreg (OImode, operands[1], XImode, 32));
4945 DONE;
4946 }
4947 else
4948 FAIL;
4949 })
4951 (define_expand "aarch64_ld<VSTRUCT:nregs>r<VALLDIF:mode>"
4952 [(match_operand:VSTRUCT 0 "register_operand" "=w")
4953 (match_operand:DI 1 "register_operand" "w")
4954 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4955 "TARGET_SIMD"
4956 {
4957 rtx mem = gen_rtx_MEM (BLKmode, operands[1]);
4958 set_mem_size (mem, GET_MODE_SIZE (GET_MODE_INNER (<VALLDIF:MODE>mode))
4959 * <VSTRUCT:nregs>);
4961 emit_insn (gen_aarch64_simd_ld<VSTRUCT:nregs>r<VALLDIF:mode> (operands[0],
4962 mem));
4963 DONE;
4964 })
4966 (define_insn "aarch64_ld2<mode>_dreg"
4967 [(set (match_operand:OI 0 "register_operand" "=w")
4968 (unspec:OI [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")
4969 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4970 UNSPEC_LD2_DREG))]
4971 "TARGET_SIMD"
4972 "ld2\\t{%S0.<Vtype> - %T0.<Vtype>}, %1"
4973 [(set_attr "type" "neon_load2_2reg<q>")]
4974 )
4976 (define_insn "aarch64_ld2<mode>_dreg"
4977 [(set (match_operand:OI 0 "register_operand" "=w")
4978 (unspec:OI [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")
4979 (unspec:DX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4980 UNSPEC_LD2_DREG))]
4981 "TARGET_SIMD"
4982 "ld1\\t{%S0.1d - %T0.1d}, %1"
4983 [(set_attr "type" "neon_load1_2reg<q>")]
4984 )
4986 (define_insn "aarch64_ld3<mode>_dreg"
4987 [(set (match_operand:CI 0 "register_operand" "=w")
4988 (unspec:CI [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")
4989 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4990 UNSPEC_LD3_DREG))]
4991 "TARGET_SIMD"
4992 "ld3\\t{%S0.<Vtype> - %U0.<Vtype>}, %1"
4993 [(set_attr "type" "neon_load3_3reg<q>")]
4994 )
4996 (define_insn "aarch64_ld3<mode>_dreg"
4997 [(set (match_operand:CI 0 "register_operand" "=w")
4998 (unspec:CI [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")
4999 (unspec:DX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5000 UNSPEC_LD3_DREG))]
5001 "TARGET_SIMD"
5002 "ld1\\t{%S0.1d - %U0.1d}, %1"
5003 [(set_attr "type" "neon_load1_3reg<q>")]
5004 )
5006 (define_insn "aarch64_ld4<mode>_dreg"
5007 [(set (match_operand:XI 0 "register_operand" "=w")
5008 (unspec:XI [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")
5009 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5010 UNSPEC_LD4_DREG))]
5011 "TARGET_SIMD"
5012 "ld4\\t{%S0.<Vtype> - %V0.<Vtype>}, %1"
5013 [(set_attr "type" "neon_load4_4reg<q>")]
5014 )
5016 (define_insn "aarch64_ld4<mode>_dreg"
5017 [(set (match_operand:XI 0 "register_operand" "=w")
5018 (unspec:XI [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")
5019 (unspec:DX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5020 UNSPEC_LD4_DREG))]
5021 "TARGET_SIMD"
5022 "ld1\\t{%S0.1d - %V0.1d}, %1"
5023 [(set_attr "type" "neon_load1_4reg<q>")]
5024 )
5026 (define_expand "aarch64_ld<VSTRUCT:nregs><VDC:mode>"
5027 [(match_operand:VSTRUCT 0 "register_operand" "=w")
5028 (match_operand:DI 1 "register_operand" "r")
5029 (unspec:VDC [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5030 "TARGET_SIMD"
5031 {
5032 rtx mem = gen_rtx_MEM (BLKmode, operands[1]);
5033 set_mem_size (mem, <VSTRUCT:nregs> * 8);
5035 emit_insn (gen_aarch64_ld<VSTRUCT:nregs><VDC:mode>_dreg (operands[0], mem));
5036 DONE;
5037 })
5039 (define_expand "aarch64_ld1<VALL_F16:mode>"
5040 [(match_operand:VALL_F16 0 "register_operand")
5041 (match_operand:DI 1 "register_operand")]
5042 "TARGET_SIMD"
5043 {
5044 machine_mode mode = <VALL_F16:MODE>mode;
5045 rtx mem = gen_rtx_MEM (mode, operands[1]);
5047 if (BYTES_BIG_ENDIAN)
5048 emit_insn (gen_aarch64_be_ld1<VALL_F16:mode> (operands[0], mem));
5049 else
5050 emit_move_insn (operands[0], mem);
5051 DONE;
5052 })
5054 (define_expand "aarch64_ld<VSTRUCT:nregs><VQ:mode>"
5055 [(match_operand:VSTRUCT 0 "register_operand" "=w")
5056 (match_operand:DI 1 "register_operand" "r")
5057 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5058 "TARGET_SIMD"
5059 {
5060 machine_mode mode = <VSTRUCT:MODE>mode;
5061 rtx mem = gen_rtx_MEM (mode, operands[1]);
5063 emit_insn (gen_aarch64_simd_ld<VSTRUCT:nregs><VQ:mode> (operands[0], mem));
5064 DONE;
5065 })
5067 (define_expand "aarch64_ld<VSTRUCT:nregs>_lane<VALLDIF:mode>"
5068 [(match_operand:VSTRUCT 0 "register_operand" "=w")
5069 (match_operand:DI 1 "register_operand" "w")
5070 (match_operand:VSTRUCT 2 "register_operand" "0")
5071 (match_operand:SI 3 "immediate_operand" "i")
5072 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5073 "TARGET_SIMD"
5074 {
5075 rtx mem = gen_rtx_MEM (BLKmode, operands[1]);
5076 set_mem_size (mem, GET_MODE_SIZE (GET_MODE_INNER (<VALLDIF:MODE>mode))
5077 * <VSTRUCT:nregs>);
5079 aarch64_simd_lane_bounds (operands[3], 0,
5080 GET_MODE_NUNITS (<VALLDIF:MODE>mode),
5081 NULL);
5082 emit_insn (gen_aarch64_vec_load_lanes<VSTRUCT:mode>_lane<VALLDIF:mode> (
5083 operands[0], mem, operands[2], operands[3]));
5084 DONE;
5085 })
5087 ;; Expanders for builtins to extract vector registers from large
5088 ;; opaque integer modes.
5090 ;; D-register list.
5092 (define_expand "aarch64_get_dreg<VSTRUCT:mode><VDC:mode>"
5093 [(match_operand:VDC 0 "register_operand" "=w")
5094 (match_operand:VSTRUCT 1 "register_operand" "w")
5095 (match_operand:SI 2 "immediate_operand" "i")]
5096 "TARGET_SIMD"
5097 {
5098 int part = INTVAL (operands[2]);
5099 rtx temp = gen_reg_rtx (<VDC:VDBL>mode);
5100 int offset = part * 16;
5102 emit_move_insn (temp, gen_rtx_SUBREG (<VDC:VDBL>mode, operands[1], offset));
5103 emit_move_insn (operands[0], gen_lowpart (<VDC:MODE>mode, temp));
5104 DONE;
5105 })
5107 ;; Q-register list.
5109 (define_expand "aarch64_get_qreg<VSTRUCT:mode><VQ:mode>"
5110 [(match_operand:VQ 0 "register_operand" "=w")
5111 (match_operand:VSTRUCT 1 "register_operand" "w")
5112 (match_operand:SI 2 "immediate_operand" "i")]
5113 "TARGET_SIMD"
5114 {
5115 int part = INTVAL (operands[2]);
5116 int offset = part * 16;
5118 emit_move_insn (operands[0],
5119 gen_rtx_SUBREG (<VQ:MODE>mode, operands[1], offset));
5120 DONE;
5121 })
5123 ;; Permuted-store expanders for neon intrinsics.
5125 ;; Permute instructions
5127 ;; vec_perm support
5129 (define_expand "vec_perm_const<mode>"
5130 [(match_operand:VALL_F16 0 "register_operand")
5131 (match_operand:VALL_F16 1 "register_operand")
5132 (match_operand:VALL_F16 2 "register_operand")
5133 (match_operand:<V_INT_EQUIV> 3)]
5134 "TARGET_SIMD"
5135 {
5136 if (aarch64_expand_vec_perm_const (operands[0], operands[1],
5137 operands[2], operands[3]))
5138 DONE;
5139 else
5140 FAIL;
5141 })
5143 (define_expand "vec_perm<mode>"
5144 [(match_operand:VB 0 "register_operand")
5145 (match_operand:VB 1 "register_operand")
5146 (match_operand:VB 2 "register_operand")
5147 (match_operand:VB 3 "register_operand")]
5148 "TARGET_SIMD"
5149 {
5150 aarch64_expand_vec_perm (operands[0], operands[1],
5151 operands[2], operands[3]);
5152 DONE;
5153 })
5155 (define_insn "aarch64_tbl1<mode>"
5156 [(set (match_operand:VB 0 "register_operand" "=w")
5157 (unspec:VB [(match_operand:V16QI 1 "register_operand" "w")
5158 (match_operand:VB 2 "register_operand" "w")]
5159 UNSPEC_TBL))]
5160 "TARGET_SIMD"
5161 "tbl\\t%0.<Vtype>, {%1.16b}, %2.<Vtype>"
5162 [(set_attr "type" "neon_tbl1<q>")]
5163 )
5165 ;; Two source registers.
5167 (define_insn "aarch64_tbl2v16qi"
5168 [(set (match_operand:V16QI 0 "register_operand" "=w")
5169 (unspec:V16QI [(match_operand:OI 1 "register_operand" "w")
5170 (match_operand:V16QI 2 "register_operand" "w")]
5171 UNSPEC_TBL))]
5172 "TARGET_SIMD"
5173 "tbl\\t%0.16b, {%S1.16b - %T1.16b}, %2.16b"
5174 [(set_attr "type" "neon_tbl2_q")]
5175 )
5177 (define_insn "aarch64_tbl3<mode>"
5178 [(set (match_operand:VB 0 "register_operand" "=w")
5179 (unspec:VB [(match_operand:OI 1 "register_operand" "w")
5180 (match_operand:VB 2 "register_operand" "w")]
5181 UNSPEC_TBL))]
5182 "TARGET_SIMD"
5183 "tbl\\t%S0.<Vbtype>, {%S1.16b - %T1.16b}, %S2.<Vbtype>"
5184 [(set_attr "type" "neon_tbl3")]
5185 )
5187 (define_insn "aarch64_tbx4<mode>"
5188 [(set (match_operand:VB 0 "register_operand" "=w")
5189 (unspec:VB [(match_operand:VB 1 "register_operand" "0")
5190 (match_operand:OI 2 "register_operand" "w")
5191 (match_operand:VB 3 "register_operand" "w")]
5192 UNSPEC_TBX))]
5193 "TARGET_SIMD"
5194 "tbx\\t%S0.<Vbtype>, {%S2.16b - %T2.16b}, %S3.<Vbtype>"
5195 [(set_attr "type" "neon_tbl4")]
5196 )
5198 ;; Three source registers.
5200 (define_insn "aarch64_qtbl3<mode>"
5201 [(set (match_operand:VB 0 "register_operand" "=w")
5202 (unspec:VB [(match_operand:CI 1 "register_operand" "w")
5203 (match_operand:VB 2 "register_operand" "w")]
5204 UNSPEC_TBL))]
5205 "TARGET_SIMD"
5206 "tbl\\t%S0.<Vbtype>, {%S1.16b - %U1.16b}, %S2.<Vbtype>"
5207 [(set_attr "type" "neon_tbl3")]
5208 )
5210 (define_insn "aarch64_qtbx3<mode>"
5211 [(set (match_operand:VB 0 "register_operand" "=w")
5212 (unspec:VB [(match_operand:VB 1 "register_operand" "0")
5213 (match_operand:CI 2 "register_operand" "w")
5214 (match_operand:VB 3 "register_operand" "w")]
5215 UNSPEC_TBX))]
5216 "TARGET_SIMD"
5217 "tbx\\t%S0.<Vbtype>, {%S2.16b - %U2.16b}, %S3.<Vbtype>"
5218 [(set_attr "type" "neon_tbl3")]
5219 )
5221 ;; Four source registers.
5223 (define_insn "aarch64_qtbl4<mode>"
5224 [(set (match_operand:VB 0 "register_operand" "=w")
5225 (unspec:VB [(match_operand:XI 1 "register_operand" "w")
5226 (match_operand:VB 2 "register_operand" "w")]
5227 UNSPEC_TBL))]
5228 "TARGET_SIMD"
5229 "tbl\\t%S0.<Vbtype>, {%S1.16b - %V1.16b}, %S2.<Vbtype>"
5230 [(set_attr "type" "neon_tbl4")]
5231 )
5233 (define_insn "aarch64_qtbx4<mode>"
5234 [(set (match_operand:VB 0 "register_operand" "=w")
5235 (unspec:VB [(match_operand:VB 1 "register_operand" "0")
5236 (match_operand:XI 2 "register_operand" "w")
5237 (match_operand:VB 3 "register_operand" "w")]
5238 UNSPEC_TBX))]
5239 "TARGET_SIMD"
5240 "tbx\\t%S0.<Vbtype>, {%S2.16b - %V2.16b}, %S3.<Vbtype>"
5241 [(set_attr "type" "neon_tbl4")]
5242 )
5244 (define_insn_and_split "aarch64_combinev16qi"
5245 [(set (match_operand:OI 0 "register_operand" "=w")
5246 (unspec:OI [(match_operand:V16QI 1 "register_operand" "w")
5247 (match_operand:V16QI 2 "register_operand" "w")]
5248 UNSPEC_CONCAT))]
5249 "TARGET_SIMD"
5250 "#"
5251 "&& reload_completed"
5252 [(const_int 0)]
5253 {
5254 aarch64_split_combinev16qi (operands);
5255 DONE;
5256 }
5257 [(set_attr "type" "multiple")]
5258 )
5260 (define_insn "aarch64_<PERMUTE:perm_insn><PERMUTE:perm_hilo><mode>"
5261 [(set (match_operand:VALL_F16 0 "register_operand" "=w")
5262 (unspec:VALL_F16 [(match_operand:VALL_F16 1 "register_operand" "w")
5263 (match_operand:VALL_F16 2 "register_operand" "w")]
5264 PERMUTE))]
5265 "TARGET_SIMD"
5266 "<PERMUTE:perm_insn><PERMUTE:perm_hilo>\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
5267 [(set_attr "type" "neon_permute<q>")]
5268 )
5270 ;; Note immediate (third) operand is lane index not byte index.
5271 (define_insn "aarch64_ext<mode>"
5272 [(set (match_operand:VALL_F16 0 "register_operand" "=w")
5273 (unspec:VALL_F16 [(match_operand:VALL_F16 1 "register_operand" "w")
5274 (match_operand:VALL_F16 2 "register_operand" "w")
5275 (match_operand:SI 3 "immediate_operand" "i")]
5276 UNSPEC_EXT))]
5277 "TARGET_SIMD"
5278 {
5279 operands[3] = GEN_INT (INTVAL (operands[3])
5280 * GET_MODE_UNIT_SIZE (<MODE>mode));
5281 return "ext\\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>, #%3";
5282 }
5283 [(set_attr "type" "neon_ext<q>")]
5284 )
5286 (define_insn "aarch64_rev<REVERSE:rev_op><mode>"
5287 [(set (match_operand:VALL_F16 0 "register_operand" "=w")
5288 (unspec:VALL_F16 [(match_operand:VALL_F16 1 "register_operand" "w")]
5289 REVERSE))]
5290 "TARGET_SIMD"
5291 "rev<REVERSE:rev_op>\\t%0.<Vtype>, %1.<Vtype>"
5292 [(set_attr "type" "neon_rev<q>")]
5293 )
5295 (define_insn "aarch64_st2<mode>_dreg"
5296 [(set (match_operand:BLK 0 "aarch64_simd_struct_operand" "=Utv")
5297 (unspec:BLK [(match_operand:OI 1 "register_operand" "w")
5298 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5299 UNSPEC_ST2))]
5300 "TARGET_SIMD"
5301 "st2\\t{%S1.<Vtype> - %T1.<Vtype>}, %0"
5302 [(set_attr "type" "neon_store2_2reg")]
5303 )
5305 (define_insn "aarch64_st2<mode>_dreg"
5306 [(set (match_operand:BLK 0 "aarch64_simd_struct_operand" "=Utv")
5307 (unspec:BLK [(match_operand:OI 1 "register_operand" "w")
5308 (unspec:DX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5309 UNSPEC_ST2))]
5310 "TARGET_SIMD"
5311 "st1\\t{%S1.1d - %T1.1d}, %0"
5312 [(set_attr "type" "neon_store1_2reg")]
5313 )
5315 (define_insn "aarch64_st3<mode>_dreg"
5316 [(set (match_operand:BLK 0 "aarch64_simd_struct_operand" "=Utv")
5317 (unspec:BLK [(match_operand:CI 1 "register_operand" "w")
5318 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5319 UNSPEC_ST3))]
5320 "TARGET_SIMD"
5321 "st3\\t{%S1.<Vtype> - %U1.<Vtype>}, %0"
5322 [(set_attr "type" "neon_store3_3reg")]
5323 )
5325 (define_insn "aarch64_st3<mode>_dreg"
5326 [(set (match_operand:BLK 0 "aarch64_simd_struct_operand" "=Utv")
5327 (unspec:BLK [(match_operand:CI 1 "register_operand" "w")
5328 (unspec:DX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5329 UNSPEC_ST3))]
5330 "TARGET_SIMD"
5331 "st1\\t{%S1.1d - %U1.1d}, %0"
5332 [(set_attr "type" "neon_store1_3reg")]
5333 )
5335 (define_insn "aarch64_st4<mode>_dreg"
5336 [(set (match_operand:BLK 0 "aarch64_simd_struct_operand" "=Utv")
5337 (unspec:BLK [(match_operand:XI 1 "register_operand" "w")
5338 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5339 UNSPEC_ST4))]
5340 "TARGET_SIMD"
5341 "st4\\t{%S1.<Vtype> - %V1.<Vtype>}, %0"
5342 [(set_attr "type" "neon_store4_4reg")]
5343 )
5345 (define_insn "aarch64_st4<mode>_dreg"
5346 [(set (match_operand:BLK 0 "aarch64_simd_struct_operand" "=Utv")
5347 (unspec:BLK [(match_operand:XI 1 "register_operand" "w")
5348 (unspec:DX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5349 UNSPEC_ST4))]
5350 "TARGET_SIMD"
5351 "st1\\t{%S1.1d - %V1.1d}, %0"
5352 [(set_attr "type" "neon_store1_4reg")]
5353 )
5355 (define_expand "aarch64_st<VSTRUCT:nregs><VDC:mode>"
5356 [(match_operand:DI 0 "register_operand" "r")
5357 (match_operand:VSTRUCT 1 "register_operand" "w")
5358 (unspec:VDC [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5359 "TARGET_SIMD"
5360 {
5361 rtx mem = gen_rtx_MEM (BLKmode, operands[0]);
5362 set_mem_size (mem, <VSTRUCT:nregs> * 8);
5364 emit_insn (gen_aarch64_st<VSTRUCT:nregs><VDC:mode>_dreg (mem, operands[1]));
5365 DONE;
5366 })
5368 (define_expand "aarch64_st<VSTRUCT:nregs><VQ:mode>"
5369 [(match_operand:DI 0 "register_operand" "r")
5370 (match_operand:VSTRUCT 1 "register_operand" "w")
5371 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5372 "TARGET_SIMD"
5373 {
5374 machine_mode mode = <VSTRUCT:MODE>mode;
5375 rtx mem = gen_rtx_MEM (mode, operands[0]);
5377 emit_insn (gen_aarch64_simd_st<VSTRUCT:nregs><VQ:mode> (mem, operands[1]));
5378 DONE;
5379 })
5381 (define_expand "aarch64_st<VSTRUCT:nregs>_lane<VALLDIF:mode>"
5382 [(match_operand:DI 0 "register_operand" "r")
5383 (match_operand:VSTRUCT 1 "register_operand" "w")
5384 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)
5385 (match_operand:SI 2 "immediate_operand")]
5386 "TARGET_SIMD"
5387 {
5388 rtx mem = gen_rtx_MEM (BLKmode, operands[0]);
5389 set_mem_size (mem, GET_MODE_SIZE (GET_MODE_INNER (<VALLDIF:MODE>mode))
5390 * <VSTRUCT:nregs>);
5392 emit_insn (gen_aarch64_vec_store_lanes<VSTRUCT:mode>_lane<VALLDIF:mode> (
5393 mem, operands[1], operands[2]));
5394 DONE;
5395 })
5397 (define_expand "aarch64_st1<VALL_F16:mode>"
5398 [(match_operand:DI 0 "register_operand")
5399 (match_operand:VALL_F16 1 "register_operand")]
5400 "TARGET_SIMD"
5401 {
5402 machine_mode mode = <VALL_F16:MODE>mode;
5403 rtx mem = gen_rtx_MEM (mode, operands[0]);
5405 if (BYTES_BIG_ENDIAN)
5406 emit_insn (gen_aarch64_be_st1<VALL_F16:mode> (mem, operands[1]));
5407 else
5408 emit_move_insn (mem, operands[1]);
5409 DONE;
5410 })
5412 ;; Expander for builtins to insert vector registers into large
5413 ;; opaque integer modes.
5415 ;; Q-register list. We don't need a D-reg inserter as we zero
5416 ;; extend them in arm_neon.h and insert the resulting Q-regs.
5418 (define_expand "aarch64_set_qreg<VSTRUCT:mode><VQ:mode>"
5419 [(match_operand:VSTRUCT 0 "register_operand" "+w")
5420 (match_operand:VSTRUCT 1 "register_operand" "0")
5421 (match_operand:VQ 2 "register_operand" "w")
5422 (match_operand:SI 3 "immediate_operand" "i")]
5423 "TARGET_SIMD"
5424 {
5425 int part = INTVAL (operands[3]);
5426 int offset = part * 16;
5428 emit_move_insn (operands[0], operands[1]);
5429 emit_move_insn (gen_rtx_SUBREG (<VQ:MODE>mode, operands[0], offset),
5430 operands[2]);
5431 DONE;
5432 })
5434 ;; Standard pattern name vec_init<mode><Vel>.
5436 (define_expand "vec_init<mode><Vel>"
5437 [(match_operand:VALL_F16 0 "register_operand" "")
5438 (match_operand 1 "" "")]
5439 "TARGET_SIMD"
5440 {
5441 aarch64_expand_vector_init (operands[0], operands[1]);
5442 DONE;
5443 })
5445 (define_insn "*aarch64_simd_ld1r<mode>"
5446 [(set (match_operand:VALL_F16 0 "register_operand" "=w")
5447 (vec_duplicate:VALL_F16
5448 (match_operand:<VEL> 1 "aarch64_simd_struct_operand" "Utv")))]
5449 "TARGET_SIMD"
5450 "ld1r\\t{%0.<Vtype>}, %1"
5451 [(set_attr "type" "neon_load1_all_lanes")]
5452 )
5454 (define_insn "aarch64_frecpe<mode>"
5455 [(set (match_operand:VHSDF 0 "register_operand" "=w")
5456 (unspec:VHSDF [(match_operand:VHSDF 1 "register_operand" "w")]
5457 UNSPEC_FRECPE))]
5458 "TARGET_SIMD"
5459 "frecpe\\t%0.<Vtype>, %1.<Vtype>"
5460 [(set_attr "type" "neon_fp_recpe_<stype><q>")]
5461 )
5463 (define_insn "aarch64_frecp<FRECP:frecp_suffix><mode>"
5464 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5465 (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")]
5466 FRECP))]
5467 "TARGET_SIMD"
5468 "frecp<FRECP:frecp_suffix>\\t%<s>0, %<s>1"
5469 [(set_attr "type" "neon_fp_recp<FRECP:frecp_suffix>_<GPF_F16:stype>")]
5470 )
5472 (define_insn "aarch64_frecps<mode>"
5473 [(set (match_operand:VHSDF_HSDF 0 "register_operand" "=w")
5474 (unspec:VHSDF_HSDF
5475 [(match_operand:VHSDF_HSDF 1 "register_operand" "w")
5476 (match_operand:VHSDF_HSDF 2 "register_operand" "w")]
5477 UNSPEC_FRECPS))]
5478 "TARGET_SIMD"
5479 "frecps\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
5480 [(set_attr "type" "neon_fp_recps_<stype><q>")]
5481 )
5483 (define_insn "aarch64_urecpe<mode>"
5484 [(set (match_operand:VDQ_SI 0 "register_operand" "=w")
5485 (unspec:VDQ_SI [(match_operand:VDQ_SI 1 "register_operand" "w")]
5486 UNSPEC_URECPE))]
5487 "TARGET_SIMD"
5488 "urecpe\\t%0.<Vtype>, %1.<Vtype>"
5489 [(set_attr "type" "neon_fp_recpe_<Vetype><q>")])
5491 ;; Standard pattern name vec_extract<mode><Vel>.
5493 (define_expand "vec_extract<mode><Vel>"
5494 [(match_operand:<VEL> 0 "aarch64_simd_nonimmediate_operand" "")
5495 (match_operand:VALL_F16 1 "register_operand" "")
5496 (match_operand:SI 2 "immediate_operand" "")]
5497 "TARGET_SIMD"
5498 {
5499 emit_insn
5500 (gen_aarch64_get_lane<mode> (operands[0], operands[1], operands[2]));
5501 DONE;
5502 })
5504 ;; aes
5506 (define_insn "aarch64_crypto_aes<aes_op>v16qi"
5507 [(set (match_operand:V16QI 0 "register_operand" "=w")
5508 (unspec:V16QI [(match_operand:V16QI 1 "register_operand" "0")
5509 (match_operand:V16QI 2 "register_operand" "w")]
5510 CRYPTO_AES))]
5511 "TARGET_SIMD && TARGET_CRYPTO"
5512 "aes<aes_op>\\t%0.16b, %2.16b"
5513 [(set_attr "type" "crypto_aese")]
5514 )
5516 ;; When AES/AESMC fusion is enabled we want the register allocation to
5517 ;; look like:
5518 ;; AESE Vn, _
5519 ;; AESMC Vn, Vn
5520 ;; So prefer to tie operand 1 to operand 0 when fusing.
5522 (define_insn "aarch64_crypto_aes<aesmc_op>v16qi"
5523 [(set (match_operand:V16QI 0 "register_operand" "=w,w")
5524 (unspec:V16QI [(match_operand:V16QI 1 "register_operand" "0,w")]
5525 CRYPTO_AESMC))]
5526 "TARGET_SIMD && TARGET_CRYPTO"
5527 "aes<aesmc_op>\\t%0.16b, %1.16b"
5528 [(set_attr "type" "crypto_aesmc")
5529 (set_attr_alternative "enabled"
5530 [(if_then_else (match_test
5531 "aarch64_fusion_enabled_p (AARCH64_FUSE_AES_AESMC)")
5532 (const_string "yes" )
5533 (const_string "no"))
5534 (const_string "yes")])]
5535 )
5537 ;; sha1
5539 (define_insn "aarch64_crypto_sha1hsi"
5540 [(set (match_operand:SI 0 "register_operand" "=w")
5541 (unspec:SI [(match_operand:SI 1
5542 "register_operand" "w")]
5543 UNSPEC_SHA1H))]
5544 "TARGET_SIMD && TARGET_CRYPTO"
5545 "sha1h\\t%s0, %s1"
5546 [(set_attr "type" "crypto_sha1_fast")]
5547 )
5549 (define_insn "aarch64_crypto_sha1hv4si"
5550 [(set (match_operand:SI 0 "register_operand" "=w")
5551 (unspec:SI [(vec_select:SI (match_operand:V4SI 1 "register_operand" "w")
5552 (parallel [(const_int 0)]))]
5553 UNSPEC_SHA1H))]
5554 "TARGET_SIMD && TARGET_CRYPTO && !BYTES_BIG_ENDIAN"
5555 "sha1h\\t%s0, %s1"
5556 [(set_attr "type" "crypto_sha1_fast")]
5557 )
5559 (define_insn "aarch64_be_crypto_sha1hv4si"
5560 [(set (match_operand:SI 0 "register_operand" "=w")
5561 (unspec:SI [(vec_select:SI (match_operand:V4SI 1 "register_operand" "w")
5562 (parallel [(const_int 3)]))]
5563 UNSPEC_SHA1H))]
5564 "TARGET_SIMD && TARGET_CRYPTO && BYTES_BIG_ENDIAN"
5565 "sha1h\\t%s0, %s1"
5566 [(set_attr "type" "crypto_sha1_fast")]
5567 )
5569 (define_insn "aarch64_crypto_sha1su1v4si"
5570 [(set (match_operand:V4SI 0 "register_operand" "=w")
5571 (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
5572 (match_operand:V4SI 2 "register_operand" "w")]
5573 UNSPEC_SHA1SU1))]
5574 "TARGET_SIMD && TARGET_CRYPTO"
5575 "sha1su1\\t%0.4s, %2.4s"
5576 [(set_attr "type" "crypto_sha1_fast")]
5577 )
5579 (define_insn "aarch64_crypto_sha1<sha1_op>v4si"
5580 [(set (match_operand:V4SI 0 "register_operand" "=w")
5581 (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
5582 (match_operand:SI 2 "register_operand" "w")
5583 (match_operand:V4SI 3 "register_operand" "w")]
5584 CRYPTO_SHA1))]
5585 "TARGET_SIMD && TARGET_CRYPTO"
5586 "sha1<sha1_op>\\t%q0, %s2, %3.4s"
5587 [(set_attr "type" "crypto_sha1_slow")]
5588 )
5590 (define_insn "aarch64_crypto_sha1su0v4si"
5591 [(set (match_operand:V4SI 0 "register_operand" "=w")
5592 (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
5593 (match_operand:V4SI 2 "register_operand" "w")
5594 (match_operand:V4SI 3 "register_operand" "w")]
5595 UNSPEC_SHA1SU0))]
5596 "TARGET_SIMD && TARGET_CRYPTO"
5597 "sha1su0\\t%0.4s, %2.4s, %3.4s"
5598 [(set_attr "type" "crypto_sha1_xor")]
5599 )
5601 ;; sha256
5603 (define_insn "aarch64_crypto_sha256h<sha256_op>v4si"
5604 [(set (match_operand:V4SI 0 "register_operand" "=w")
5605 (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
5606 (match_operand:V4SI 2 "register_operand" "w")
5607 (match_operand:V4SI 3 "register_operand" "w")]
5608 CRYPTO_SHA256))]
5609 "TARGET_SIMD && TARGET_CRYPTO"
5610 "sha256h<sha256_op>\\t%q0, %q2, %3.4s"
5611 [(set_attr "type" "crypto_sha256_slow")]
5612 )
5614 (define_insn "aarch64_crypto_sha256su0v4si"
5615 [(set (match_operand:V4SI 0 "register_operand" "=w")
5616 (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
5617 (match_operand:V4SI 2 "register_operand" "w")]
5618 UNSPEC_SHA256SU0))]
5619 "TARGET_SIMD &&TARGET_CRYPTO"
5620 "sha256su0\\t%0.4s, %2.4s"
5621 [(set_attr "type" "crypto_sha256_fast")]
5622 )
5624 (define_insn "aarch64_crypto_sha256su1v4si"
5625 [(set (match_operand:V4SI 0 "register_operand" "=w")
5626 (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
5627 (match_operand:V4SI 2 "register_operand" "w")
5628 (match_operand:V4SI 3 "register_operand" "w")]
5629 UNSPEC_SHA256SU1))]
5630 "TARGET_SIMD &&TARGET_CRYPTO"
5631 "sha256su1\\t%0.4s, %2.4s, %3.4s"
5632 [(set_attr "type" "crypto_sha256_slow")]
5633 )
5635 ;; pmull
5637 (define_insn "aarch64_crypto_pmulldi"
5638 [(set (match_operand:TI 0 "register_operand" "=w")
5639 (unspec:TI [(match_operand:DI 1 "register_operand" "w")
5640 (match_operand:DI 2 "register_operand" "w")]
5641 UNSPEC_PMULL))]
5642 "TARGET_SIMD && TARGET_CRYPTO"
5643 "pmull\\t%0.1q, %1.1d, %2.1d"
5644 [(set_attr "type" "crypto_pmull")]
5645 )
5647 (define_insn "aarch64_crypto_pmullv2di"
5648 [(set (match_operand:TI 0 "register_operand" "=w")
5649 (unspec:TI [(match_operand:V2DI 1 "register_operand" "w")
5650 (match_operand:V2DI 2 "register_operand" "w")]
5651 UNSPEC_PMULL2))]
5652 "TARGET_SIMD && TARGET_CRYPTO"
5653 "pmull2\\t%0.1q, %1.2d, %2.2d"
5654 [(set_attr "type" "crypto_pmull")]
5655 )