| blob | ded8bff0973db17c9e09a801f10315fd59301bb3 |
1 ;; Machine description for AArch64 AdvSIMD architecture.
2 ;; Copyright (C) 2011-2016 Free Software Foundation, Inc.
3 ;; Contributed by ARM Ltd.
4 ;;
5 ;; This file is part of GCC.
6 ;;
7 ;; GCC is free software; you can redistribute it and/or modify it
8 ;; under the terms of the GNU General Public License as published by
9 ;; the Free Software Foundation; either version 3, or (at your option)
10 ;; any later version.
11 ;;
12 ;; GCC is distributed in the hope that it will be useful, but
13 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
14 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 ;; General Public License for more details.
16 ;;
17 ;; You should have received a copy of the GNU General Public License
18 ;; along with GCC; see the file COPYING3. If not see
19 ;; <http://www.gnu.org/licenses/>.
21 (define_expand "mov<mode>"
22 [(set (match_operand:VALL_F16 0 "nonimmediate_operand" "")
23 (match_operand:VALL_F16 1 "general_operand" ""))]
24 "TARGET_SIMD"
25 "
26 if (GET_CODE (operands[0]) == MEM)
27 operands[1] = force_reg (<MODE>mode, operands[1]);
28 "
29 )
31 (define_expand "movmisalign<mode>"
32 [(set (match_operand:VALL 0 "nonimmediate_operand" "")
33 (match_operand:VALL 1 "general_operand" ""))]
34 "TARGET_SIMD"
35 {
36 /* This pattern is not permitted to fail during expansion: if both arguments
37 are non-registers (e.g. memory := constant, which can be created by the
38 auto-vectorizer), force operand 1 into a register. */
39 if (!register_operand (operands[0], <MODE>mode)
40 && !register_operand (operands[1], <MODE>mode))
41 operands[1] = force_reg (<MODE>mode, operands[1]);
42 })
44 (define_insn "aarch64_simd_dup<mode>"
45 [(set (match_operand:VDQ_I 0 "register_operand" "=w, w")
46 (vec_duplicate:VDQ_I
47 (match_operand:<VEL> 1 "register_operand" "r, w")))]
48 "TARGET_SIMD"
49 "@
50 dup\\t%0.<Vtype>, %<vw>1
51 dup\\t%0.<Vtype>, %1.<Vetype>[0]"
52 [(set_attr "type" "neon_from_gp<q>, neon_dup<q>")]
53 )
55 (define_insn "aarch64_simd_dup<mode>"
56 [(set (match_operand:VDQF_F16 0 "register_operand" "=w")
57 (vec_duplicate:VDQF_F16
58 (match_operand:<VEL> 1 "register_operand" "w")))]
59 "TARGET_SIMD"
60 "dup\\t%0.<Vtype>, %1.<Vetype>[0]"
61 [(set_attr "type" "neon_dup<q>")]
62 )
64 (define_insn "aarch64_dup_lane<mode>"
65 [(set (match_operand:VALL_F16 0 "register_operand" "=w")
66 (vec_duplicate:VALL_F16
67 (vec_select:<VEL>
68 (match_operand:VALL_F16 1 "register_operand" "w")
69 (parallel [(match_operand:SI 2 "immediate_operand" "i")])
70 )))]
71 "TARGET_SIMD"
72 {
73 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
74 return "dup\\t%0.<Vtype>, %1.<Vetype>[%2]";
75 }
76 [(set_attr "type" "neon_dup<q>")]
77 )
79 (define_insn "aarch64_dup_lane_<vswap_width_name><mode>"
80 [(set (match_operand:VALL_F16 0 "register_operand" "=w")
81 (vec_duplicate:VALL_F16
82 (vec_select:<VEL>
83 (match_operand:<VSWAP_WIDTH> 1 "register_operand" "w")
84 (parallel [(match_operand:SI 2 "immediate_operand" "i")])
85 )))]
86 "TARGET_SIMD"
87 {
88 operands[2] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
89 INTVAL (operands[2])));
90 return "dup\\t%0.<Vtype>, %1.<Vetype>[%2]";
91 }
92 [(set_attr "type" "neon_dup<q>")]
93 )
95 (define_insn "*aarch64_simd_mov<mode>"
96 [(set (match_operand:VD 0 "nonimmediate_operand"
97 "=w, m, w, ?r, ?w, ?r, w")
98 (match_operand:VD 1 "general_operand"
99 "m, w, w, w, r, r, Dn"))]
100 "TARGET_SIMD
101 && (register_operand (operands[0], <MODE>mode)
102 || register_operand (operands[1], <MODE>mode))"
103 {
104 switch (which_alternative)
105 {
106 case 0: return "ldr\\t%d0, %1";
107 case 1: return "str\\t%d1, %0";
108 case 2: return "orr\t%0.<Vbtype>, %1.<Vbtype>, %1.<Vbtype>";
109 case 3: return "umov\t%0, %1.d[0]";
110 case 4: return "ins\t%0.d[0], %1";
111 case 5: return "mov\t%0, %1";
112 case 6:
113 return aarch64_output_simd_mov_immediate (operands[1],
114 <MODE>mode, 64);
115 default: gcc_unreachable ();
116 }
117 }
118 [(set_attr "type" "neon_load1_1reg<q>, neon_store1_1reg<q>,\
119 neon_logic<q>, neon_to_gp<q>, neon_from_gp<q>,\
120 mov_reg, neon_move<q>")]
121 )
123 (define_insn "*aarch64_simd_mov<mode>"
124 [(set (match_operand:VQ 0 "nonimmediate_operand"
125 "=w, m, w, ?r, ?w, ?r, w")
126 (match_operand:VQ 1 "general_operand"
127 "m, w, w, w, r, r, Dn"))]
128 "TARGET_SIMD
129 && (register_operand (operands[0], <MODE>mode)
130 || register_operand (operands[1], <MODE>mode))"
131 {
132 switch (which_alternative)
133 {
134 case 0:
135 return "ldr\\t%q0, %1";
136 case 1:
137 return "str\\t%q1, %0";
138 case 2:
139 return "orr\t%0.<Vbtype>, %1.<Vbtype>, %1.<Vbtype>";
140 case 3:
141 case 4:
142 case 5:
143 return "#";
144 case 6:
145 return aarch64_output_simd_mov_immediate (operands[1], <MODE>mode, 128);
146 default:
147 gcc_unreachable ();
148 }
149 }
150 [(set_attr "type" "neon_load1_1reg<q>, neon_store1_1reg<q>,\
151 neon_logic<q>, multiple, multiple, multiple,\
152 neon_move<q>")
153 (set_attr "length" "4,4,4,8,8,8,4")]
154 )
156 (define_insn "load_pair<mode>"
157 [(set (match_operand:VD 0 "register_operand" "=w")
158 (match_operand:VD 1 "aarch64_mem_pair_operand" "Ump"))
159 (set (match_operand:VD 2 "register_operand" "=w")
160 (match_operand:VD 3 "memory_operand" "m"))]
161 "TARGET_SIMD
162 && rtx_equal_p (XEXP (operands[3], 0),
163 plus_constant (Pmode,
164 XEXP (operands[1], 0),
165 GET_MODE_SIZE (<MODE>mode)))"
166 "ldp\\t%d0, %d2, %1"
167 [(set_attr "type" "neon_ldp")]
168 )
170 (define_insn "store_pair<mode>"
171 [(set (match_operand:VD 0 "aarch64_mem_pair_operand" "=Ump")
172 (match_operand:VD 1 "register_operand" "w"))
173 (set (match_operand:VD 2 "memory_operand" "=m")
174 (match_operand:VD 3 "register_operand" "w"))]
175 "TARGET_SIMD
176 && rtx_equal_p (XEXP (operands[2], 0),
177 plus_constant (Pmode,
178 XEXP (operands[0], 0),
179 GET_MODE_SIZE (<MODE>mode)))"
180 "stp\\t%d1, %d3, %0"
181 [(set_attr "type" "neon_stp")]
182 )
184 (define_split
185 [(set (match_operand:VQ 0 "register_operand" "")
186 (match_operand:VQ 1 "register_operand" ""))]
187 "TARGET_SIMD && reload_completed
188 && GP_REGNUM_P (REGNO (operands[0]))
189 && GP_REGNUM_P (REGNO (operands[1]))"
190 [(const_int 0)]
191 {
192 aarch64_simd_emit_reg_reg_move (operands, DImode, 2);
193 DONE;
194 })
196 (define_split
197 [(set (match_operand:VQ 0 "register_operand" "")
198 (match_operand:VQ 1 "register_operand" ""))]
199 "TARGET_SIMD && reload_completed
200 && ((FP_REGNUM_P (REGNO (operands[0])) && GP_REGNUM_P (REGNO (operands[1])))
201 || (GP_REGNUM_P (REGNO (operands[0])) && FP_REGNUM_P (REGNO (operands[1]))))"
202 [(const_int 0)]
203 {
204 aarch64_split_simd_move (operands[0], operands[1]);
205 DONE;
206 })
208 (define_expand "aarch64_split_simd_mov<mode>"
209 [(set (match_operand:VQ 0)
210 (match_operand:VQ 1))]
211 "TARGET_SIMD"
212 {
213 rtx dst = operands[0];
214 rtx src = operands[1];
216 if (GP_REGNUM_P (REGNO (src)))
217 {
218 rtx src_low_part = gen_lowpart (<VHALF>mode, src);
219 rtx src_high_part = gen_highpart (<VHALF>mode, src);
221 emit_insn
222 (gen_move_lo_quad_<mode> (dst, src_low_part));
223 emit_insn
224 (gen_move_hi_quad_<mode> (dst, src_high_part));
225 }
227 else
228 {
229 rtx dst_low_part = gen_lowpart (<VHALF>mode, dst);
230 rtx dst_high_part = gen_highpart (<VHALF>mode, dst);
231 rtx lo = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
232 rtx hi = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
234 emit_insn
235 (gen_aarch64_simd_mov_from_<mode>low (dst_low_part, src, lo));
236 emit_insn
237 (gen_aarch64_simd_mov_from_<mode>high (dst_high_part, src, hi));
238 }
239 DONE;
240 }
241 )
243 (define_insn "aarch64_simd_mov_from_<mode>low"
244 [(set (match_operand:<VHALF> 0 "register_operand" "=r")
245 (vec_select:<VHALF>
246 (match_operand:VQ 1 "register_operand" "w")
247 (match_operand:VQ 2 "vect_par_cnst_lo_half" "")))]
248 "TARGET_SIMD && reload_completed"
249 "umov\t%0, %1.d[0]"
250 [(set_attr "type" "neon_to_gp<q>")
251 (set_attr "length" "4")
252 ])
254 (define_insn "aarch64_simd_mov_from_<mode>high"
255 [(set (match_operand:<VHALF> 0 "register_operand" "=r")
256 (vec_select:<VHALF>
257 (match_operand:VQ 1 "register_operand" "w")
258 (match_operand:VQ 2 "vect_par_cnst_hi_half" "")))]
259 "TARGET_SIMD && reload_completed"
260 "umov\t%0, %1.d[1]"
261 [(set_attr "type" "neon_to_gp<q>")
262 (set_attr "length" "4")
263 ])
265 (define_insn "orn<mode>3"
266 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
267 (ior:VDQ_I (not:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w"))
268 (match_operand:VDQ_I 2 "register_operand" "w")))]
269 "TARGET_SIMD"
270 "orn\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>"
271 [(set_attr "type" "neon_logic<q>")]
272 )
274 (define_insn "bic<mode>3"
275 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
276 (and:VDQ_I (not:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w"))
277 (match_operand:VDQ_I 2 "register_operand" "w")))]
278 "TARGET_SIMD"
279 "bic\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>"
280 [(set_attr "type" "neon_logic<q>")]
281 )
283 (define_insn "add<mode>3"
284 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
285 (plus:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
286 (match_operand:VDQ_I 2 "register_operand" "w")))]
287 "TARGET_SIMD"
288 "add\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
289 [(set_attr "type" "neon_add<q>")]
290 )
292 (define_insn "sub<mode>3"
293 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
294 (minus:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
295 (match_operand:VDQ_I 2 "register_operand" "w")))]
296 "TARGET_SIMD"
297 "sub\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
298 [(set_attr "type" "neon_sub<q>")]
299 )
301 (define_insn "mul<mode>3"
302 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
303 (mult:VDQ_BHSI (match_operand:VDQ_BHSI 1 "register_operand" "w")
304 (match_operand:VDQ_BHSI 2 "register_operand" "w")))]
305 "TARGET_SIMD"
306 "mul\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
307 [(set_attr "type" "neon_mul_<Vetype><q>")]
308 )
310 (define_insn "bswap<mode>2"
311 [(set (match_operand:VDQHSD 0 "register_operand" "=w")
312 (bswap:VDQHSD (match_operand:VDQHSD 1 "register_operand" "w")))]
313 "TARGET_SIMD"
314 "rev<Vrevsuff>\\t%0.<Vbtype>, %1.<Vbtype>"
315 [(set_attr "type" "neon_rev<q>")]
316 )
318 (define_insn "aarch64_rbit<mode>"
319 [(set (match_operand:VB 0 "register_operand" "=w")
320 (unspec:VB [(match_operand:VB 1 "register_operand" "w")]
321 UNSPEC_RBIT))]
322 "TARGET_SIMD"
323 "rbit\\t%0.<Vbtype>, %1.<Vbtype>"
324 [(set_attr "type" "neon_rbit")]
325 )
327 (define_expand "ctz<mode>2"
328 [(set (match_operand:VS 0 "register_operand")
329 (ctz:VS (match_operand:VS 1 "register_operand")))]
330 "TARGET_SIMD"
331 {
332 emit_insn (gen_bswap<mode>2 (operands[0], operands[1]));
333 rtx op0_castsi2qi = simplify_gen_subreg(<VS:VSI2QI>mode, operands[0],
334 <MODE>mode, 0);
335 emit_insn (gen_aarch64_rbit<VS:vsi2qi> (op0_castsi2qi, op0_castsi2qi));
336 emit_insn (gen_clz<mode>2 (operands[0], operands[0]));
337 DONE;
338 }
339 )
341 (define_insn "*aarch64_mul3_elt<mode>"
342 [(set (match_operand:VMUL 0 "register_operand" "=w")
343 (mult:VMUL
344 (vec_duplicate:VMUL
345 (vec_select:<VEL>
346 (match_operand:VMUL 1 "register_operand" "<h_con>")
347 (parallel [(match_operand:SI 2 "immediate_operand")])))
348 (match_operand:VMUL 3 "register_operand" "w")))]
349 "TARGET_SIMD"
350 {
351 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
352 return "<f>mul\\t%0.<Vtype>, %3.<Vtype>, %1.<Vetype>[%2]";
353 }
354 [(set_attr "type" "neon<fp>_mul_<Vetype>_scalar<q>")]
355 )
357 (define_insn "*aarch64_mul3_elt_<vswap_width_name><mode>"
358 [(set (match_operand:VMUL_CHANGE_NLANES 0 "register_operand" "=w")
359 (mult:VMUL_CHANGE_NLANES
360 (vec_duplicate:VMUL_CHANGE_NLANES
361 (vec_select:<VEL>
362 (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
363 (parallel [(match_operand:SI 2 "immediate_operand")])))
364 (match_operand:VMUL_CHANGE_NLANES 3 "register_operand" "w")))]
365 "TARGET_SIMD"
366 {
367 operands[2] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
368 INTVAL (operands[2])));
369 return "<f>mul\\t%0.<Vtype>, %3.<Vtype>, %1.<Vetype>[%2]";
370 }
371 [(set_attr "type" "neon<fp>_mul_<Vetype>_scalar<q>")]
372 )
374 (define_insn "*aarch64_mul3_elt_to_128df"
375 [(set (match_operand:V2DF 0 "register_operand" "=w")
376 (mult:V2DF
377 (vec_duplicate:V2DF
378 (match_operand:DF 2 "register_operand" "w"))
379 (match_operand:V2DF 1 "register_operand" "w")))]
380 "TARGET_SIMD"
381 "fmul\\t%0.2d, %1.2d, %2.d[0]"
382 [(set_attr "type" "neon_fp_mul_d_scalar_q")]
383 )
385 (define_insn "aarch64_rsqrte_<mode>2"
386 [(set (match_operand:VALLF 0 "register_operand" "=w")
387 (unspec:VALLF [(match_operand:VALLF 1 "register_operand" "w")]
388 UNSPEC_RSQRTE))]
389 "TARGET_SIMD"
390 "frsqrte\\t%<v>0<Vmtype>, %<v>1<Vmtype>"
391 [(set_attr "type" "neon_fp_rsqrte_<Vetype><q>")])
393 (define_insn "aarch64_rsqrts_<mode>3"
394 [(set (match_operand:VALLF 0 "register_operand" "=w")
395 (unspec:VALLF [(match_operand:VALLF 1 "register_operand" "w")
396 (match_operand:VALLF 2 "register_operand" "w")]
397 UNSPEC_RSQRTS))]
398 "TARGET_SIMD"
399 "frsqrts\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
400 [(set_attr "type" "neon_fp_rsqrts_<Vetype><q>")])
402 (define_expand "rsqrt<mode>2"
403 [(set (match_operand:VALLF 0 "register_operand" "=w")
404 (unspec:VALLF [(match_operand:VALLF 1 "register_operand" "w")]
405 UNSPEC_RSQRT))]
406 "TARGET_SIMD"
407 {
408 aarch64_emit_approx_rsqrt (operands[0], operands[1]);
409 DONE;
410 })
412 (define_insn "*aarch64_mul3_elt_to_64v2df"
413 [(set (match_operand:DF 0 "register_operand" "=w")
414 (mult:DF
415 (vec_select:DF
416 (match_operand:V2DF 1 "register_operand" "w")
417 (parallel [(match_operand:SI 2 "immediate_operand")]))
418 (match_operand:DF 3 "register_operand" "w")))]
419 "TARGET_SIMD"
420 {
421 operands[2] = GEN_INT (ENDIAN_LANE_N (V2DFmode, INTVAL (operands[2])));
422 return "fmul\\t%0.2d, %3.2d, %1.d[%2]";
423 }
424 [(set_attr "type" "neon_fp_mul_d_scalar_q")]
425 )
427 (define_insn "neg<mode>2"
428 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
429 (neg:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")))]
430 "TARGET_SIMD"
431 "neg\t%0.<Vtype>, %1.<Vtype>"
432 [(set_attr "type" "neon_neg<q>")]
433 )
435 (define_insn "abs<mode>2"
436 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
437 (abs:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")))]
438 "TARGET_SIMD"
439 "abs\t%0.<Vtype>, %1.<Vtype>"
440 [(set_attr "type" "neon_abs<q>")]
441 )
443 ;; The intrinsic version of integer ABS must not be allowed to
444 ;; combine with any operation with an integerated ABS step, such
445 ;; as SABD.
446 (define_insn "aarch64_abs<mode>"
447 [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
448 (unspec:VSDQ_I_DI
449 [(match_operand:VSDQ_I_DI 1 "register_operand" "w")]
450 UNSPEC_ABS))]
451 "TARGET_SIMD"
452 "abs\t%<v>0<Vmtype>, %<v>1<Vmtype>"
453 [(set_attr "type" "neon_abs<q>")]
454 )
456 (define_insn "abd<mode>_3"
457 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
458 (abs:VDQ_BHSI (minus:VDQ_BHSI
459 (match_operand:VDQ_BHSI 1 "register_operand" "w")
460 (match_operand:VDQ_BHSI 2 "register_operand" "w"))))]
461 "TARGET_SIMD"
462 "sabd\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
463 [(set_attr "type" "neon_abd<q>")]
464 )
466 (define_insn "aba<mode>_3"
467 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
468 (plus:VDQ_BHSI (abs:VDQ_BHSI (minus:VDQ_BHSI
469 (match_operand:VDQ_BHSI 1 "register_operand" "w")
470 (match_operand:VDQ_BHSI 2 "register_operand" "w")))
471 (match_operand:VDQ_BHSI 3 "register_operand" "0")))]
472 "TARGET_SIMD"
473 "saba\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
474 [(set_attr "type" "neon_arith_acc<q>")]
475 )
477 (define_insn "fabd<mode>_3"
478 [(set (match_operand:VDQF 0 "register_operand" "=w")
479 (abs:VDQF (minus:VDQF
480 (match_operand:VDQF 1 "register_operand" "w")
481 (match_operand:VDQF 2 "register_operand" "w"))))]
482 "TARGET_SIMD"
483 "fabd\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
484 [(set_attr "type" "neon_fp_abd_<Vetype><q>")]
485 )
487 (define_insn "*fabd_scalar<mode>3"
488 [(set (match_operand:GPF 0 "register_operand" "=w")
489 (abs:GPF (minus:GPF
490 (match_operand:GPF 1 "register_operand" "w")
491 (match_operand:GPF 2 "register_operand" "w"))))]
492 "TARGET_SIMD"
493 "fabd\t%<s>0, %<s>1, %<s>2"
494 [(set_attr "type" "neon_fp_abd_<Vetype><q>")]
495 )
497 (define_insn "and<mode>3"
498 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
499 (and:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
500 (match_operand:VDQ_I 2 "register_operand" "w")))]
501 "TARGET_SIMD"
502 "and\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>"
503 [(set_attr "type" "neon_logic<q>")]
504 )
506 (define_insn "ior<mode>3"
507 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
508 (ior:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
509 (match_operand:VDQ_I 2 "register_operand" "w")))]
510 "TARGET_SIMD"
511 "orr\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>"
512 [(set_attr "type" "neon_logic<q>")]
513 )
515 (define_insn "xor<mode>3"
516 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
517 (xor:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
518 (match_operand:VDQ_I 2 "register_operand" "w")))]
519 "TARGET_SIMD"
520 "eor\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>"
521 [(set_attr "type" "neon_logic<q>")]
522 )
524 (define_insn "one_cmpl<mode>2"
525 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
526 (not:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")))]
527 "TARGET_SIMD"
528 "not\t%0.<Vbtype>, %1.<Vbtype>"
529 [(set_attr "type" "neon_logic<q>")]
530 )
532 (define_insn "aarch64_simd_vec_set<mode>"
533 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w,w,w")
534 (vec_merge:VDQ_BHSI
535 (vec_duplicate:VDQ_BHSI
536 (match_operand:<VEL> 1 "aarch64_simd_general_operand" "r,w,Utv"))
537 (match_operand:VDQ_BHSI 3 "register_operand" "0,0,0")
538 (match_operand:SI 2 "immediate_operand" "i,i,i")))]
539 "TARGET_SIMD"
540 {
541 int elt = ENDIAN_LANE_N (<MODE>mode, exact_log2 (INTVAL (operands[2])));
542 operands[2] = GEN_INT ((HOST_WIDE_INT) 1 << elt);
543 switch (which_alternative)
544 {
545 case 0:
546 return "ins\\t%0.<Vetype>[%p2], %w1";
547 case 1:
548 return "ins\\t%0.<Vetype>[%p2], %1.<Vetype>[0]";
549 case 2:
550 return "ld1\\t{%0.<Vetype>}[%p2], %1";
551 default:
552 gcc_unreachable ();
553 }
554 }
555 [(set_attr "type" "neon_from_gp<q>, neon_ins<q>, neon_load1_1reg<q>")]
556 )
558 (define_insn "aarch64_simd_lshr<mode>"
559 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
560 (lshiftrt:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
561 (match_operand:VDQ_I 2 "aarch64_simd_rshift_imm" "Dr")))]
562 "TARGET_SIMD"
563 "ushr\t%0.<Vtype>, %1.<Vtype>, %2"
564 [(set_attr "type" "neon_shift_imm<q>")]
565 )
567 (define_insn "aarch64_simd_ashr<mode>"
568 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
569 (ashiftrt:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
570 (match_operand:VDQ_I 2 "aarch64_simd_rshift_imm" "Dr")))]
571 "TARGET_SIMD"
572 "sshr\t%0.<Vtype>, %1.<Vtype>, %2"
573 [(set_attr "type" "neon_shift_imm<q>")]
574 )
576 (define_insn "aarch64_simd_imm_shl<mode>"
577 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
578 (ashift:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
579 (match_operand:VDQ_I 2 "aarch64_simd_lshift_imm" "Dl")))]
580 "TARGET_SIMD"
581 "shl\t%0.<Vtype>, %1.<Vtype>, %2"
582 [(set_attr "type" "neon_shift_imm<q>")]
583 )
585 (define_insn "aarch64_simd_reg_sshl<mode>"
586 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
587 (ashift:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
588 (match_operand:VDQ_I 2 "register_operand" "w")))]
589 "TARGET_SIMD"
590 "sshl\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
591 [(set_attr "type" "neon_shift_reg<q>")]
592 )
594 (define_insn "aarch64_simd_reg_shl<mode>_unsigned"
595 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
596 (unspec:VDQ_I [(match_operand:VDQ_I 1 "register_operand" "w")
597 (match_operand:VDQ_I 2 "register_operand" "w")]
598 UNSPEC_ASHIFT_UNSIGNED))]
599 "TARGET_SIMD"
600 "ushl\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
601 [(set_attr "type" "neon_shift_reg<q>")]
602 )
604 (define_insn "aarch64_simd_reg_shl<mode>_signed"
605 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
606 (unspec:VDQ_I [(match_operand:VDQ_I 1 "register_operand" "w")
607 (match_operand:VDQ_I 2 "register_operand" "w")]
608 UNSPEC_ASHIFT_SIGNED))]
609 "TARGET_SIMD"
610 "sshl\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
611 [(set_attr "type" "neon_shift_reg<q>")]
612 )
614 (define_expand "ashl<mode>3"
615 [(match_operand:VDQ_I 0 "register_operand" "")
616 (match_operand:VDQ_I 1 "register_operand" "")
617 (match_operand:SI 2 "general_operand" "")]
618 "TARGET_SIMD"
619 {
620 int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
621 int shift_amount;
623 if (CONST_INT_P (operands[2]))
624 {
625 shift_amount = INTVAL (operands[2]);
626 if (shift_amount >= 0 && shift_amount < bit_width)
627 {
628 rtx tmp = aarch64_simd_gen_const_vector_dup (<MODE>mode,
629 shift_amount);
630 emit_insn (gen_aarch64_simd_imm_shl<mode> (operands[0],
631 operands[1],
632 tmp));
633 DONE;
634 }
635 else
636 {
637 operands[2] = force_reg (SImode, operands[2]);
638 }
639 }
640 else if (MEM_P (operands[2]))
641 {
642 operands[2] = force_reg (SImode, operands[2]);
643 }
645 if (REG_P (operands[2]))
646 {
647 rtx tmp = gen_reg_rtx (<MODE>mode);
648 emit_insn (gen_aarch64_simd_dup<mode> (tmp,
649 convert_to_mode (<VEL>mode,
650 operands[2],
651 0)));
652 emit_insn (gen_aarch64_simd_reg_sshl<mode> (operands[0], operands[1],
653 tmp));
654 DONE;
655 }
656 else
657 FAIL;
658 }
659 )
661 (define_expand "lshr<mode>3"
662 [(match_operand:VDQ_I 0 "register_operand" "")
663 (match_operand:VDQ_I 1 "register_operand" "")
664 (match_operand:SI 2 "general_operand" "")]
665 "TARGET_SIMD"
666 {
667 int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
668 int shift_amount;
670 if (CONST_INT_P (operands[2]))
671 {
672 shift_amount = INTVAL (operands[2]);
673 if (shift_amount > 0 && shift_amount <= bit_width)
674 {
675 rtx tmp = aarch64_simd_gen_const_vector_dup (<MODE>mode,
676 shift_amount);
677 emit_insn (gen_aarch64_simd_lshr<mode> (operands[0],
678 operands[1],
679 tmp));
680 DONE;
681 }
682 else
683 operands[2] = force_reg (SImode, operands[2]);
684 }
685 else if (MEM_P (operands[2]))
686 {
687 operands[2] = force_reg (SImode, operands[2]);
688 }
690 if (REG_P (operands[2]))
691 {
692 rtx tmp = gen_reg_rtx (SImode);
693 rtx tmp1 = gen_reg_rtx (<MODE>mode);
694 emit_insn (gen_negsi2 (tmp, operands[2]));
695 emit_insn (gen_aarch64_simd_dup<mode> (tmp1,
696 convert_to_mode (<VEL>mode,
697 tmp, 0)));
698 emit_insn (gen_aarch64_simd_reg_shl<mode>_unsigned (operands[0],
699 operands[1],
700 tmp1));
701 DONE;
702 }
703 else
704 FAIL;
705 }
706 )
708 (define_expand "ashr<mode>3"
709 [(match_operand:VDQ_I 0 "register_operand" "")
710 (match_operand:VDQ_I 1 "register_operand" "")
711 (match_operand:SI 2 "general_operand" "")]
712 "TARGET_SIMD"
713 {
714 int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
715 int shift_amount;
717 if (CONST_INT_P (operands[2]))
718 {
719 shift_amount = INTVAL (operands[2]);
720 if (shift_amount > 0 && shift_amount <= bit_width)
721 {
722 rtx tmp = aarch64_simd_gen_const_vector_dup (<MODE>mode,
723 shift_amount);
724 emit_insn (gen_aarch64_simd_ashr<mode> (operands[0],
725 operands[1],
726 tmp));
727 DONE;
728 }
729 else
730 operands[2] = force_reg (SImode, operands[2]);
731 }
732 else if (MEM_P (operands[2]))
733 {
734 operands[2] = force_reg (SImode, operands[2]);
735 }
737 if (REG_P (operands[2]))
738 {
739 rtx tmp = gen_reg_rtx (SImode);
740 rtx tmp1 = gen_reg_rtx (<MODE>mode);
741 emit_insn (gen_negsi2 (tmp, operands[2]));
742 emit_insn (gen_aarch64_simd_dup<mode> (tmp1,
743 convert_to_mode (<VEL>mode,
744 tmp, 0)));
745 emit_insn (gen_aarch64_simd_reg_shl<mode>_signed (operands[0],
746 operands[1],
747 tmp1));
748 DONE;
749 }
750 else
751 FAIL;
752 }
753 )
755 (define_expand "vashl<mode>3"
756 [(match_operand:VDQ_I 0 "register_operand" "")
757 (match_operand:VDQ_I 1 "register_operand" "")
758 (match_operand:VDQ_I 2 "register_operand" "")]
759 "TARGET_SIMD"
760 {
761 emit_insn (gen_aarch64_simd_reg_sshl<mode> (operands[0], operands[1],
762 operands[2]));
763 DONE;
764 })
766 ;; Using mode VDQ_BHSI as there is no V2DImode neg!
767 ;; Negating individual lanes most certainly offsets the
768 ;; gain from vectorization.
769 (define_expand "vashr<mode>3"
770 [(match_operand:VDQ_BHSI 0 "register_operand" "")
771 (match_operand:VDQ_BHSI 1 "register_operand" "")
772 (match_operand:VDQ_BHSI 2 "register_operand" "")]
773 "TARGET_SIMD"
774 {
775 rtx neg = gen_reg_rtx (<MODE>mode);
776 emit (gen_neg<mode>2 (neg, operands[2]));
777 emit_insn (gen_aarch64_simd_reg_shl<mode>_signed (operands[0], operands[1],
778 neg));
779 DONE;
780 })
782 ;; DI vector shift
783 (define_expand "aarch64_ashr_simddi"
784 [(match_operand:DI 0 "register_operand" "=w")
785 (match_operand:DI 1 "register_operand" "w")
786 (match_operand:SI 2 "aarch64_shift_imm64_di" "")]
787 "TARGET_SIMD"
788 {
789 /* An arithmetic shift right by 64 fills the result with copies of the sign
790 bit, just like asr by 63 - however the standard pattern does not handle
791 a shift by 64. */
792 if (INTVAL (operands[2]) == 64)
793 operands[2] = GEN_INT (63);
794 emit_insn (gen_ashrdi3 (operands[0], operands[1], operands[2]));
795 DONE;
796 }
797 )
799 (define_expand "vlshr<mode>3"
800 [(match_operand:VDQ_BHSI 0 "register_operand" "")
801 (match_operand:VDQ_BHSI 1 "register_operand" "")
802 (match_operand:VDQ_BHSI 2 "register_operand" "")]
803 "TARGET_SIMD"
804 {
805 rtx neg = gen_reg_rtx (<MODE>mode);
806 emit (gen_neg<mode>2 (neg, operands[2]));
807 emit_insn (gen_aarch64_simd_reg_shl<mode>_unsigned (operands[0], operands[1],
808 neg));
809 DONE;
810 })
812 (define_expand "aarch64_lshr_simddi"
813 [(match_operand:DI 0 "register_operand" "=w")
814 (match_operand:DI 1 "register_operand" "w")
815 (match_operand:SI 2 "aarch64_shift_imm64_di" "")]
816 "TARGET_SIMD"
817 {
818 if (INTVAL (operands[2]) == 64)
819 emit_move_insn (operands[0], const0_rtx);
820 else
821 emit_insn (gen_lshrdi3 (operands[0], operands[1], operands[2]));
822 DONE;
823 }
824 )
826 (define_expand "vec_set<mode>"
827 [(match_operand:VDQ_BHSI 0 "register_operand")
828 (match_operand:<VEL> 1 "register_operand")
829 (match_operand:SI 2 "immediate_operand")]
830 "TARGET_SIMD"
831 {
832 HOST_WIDE_INT elem = (HOST_WIDE_INT) 1 << INTVAL (operands[2]);
833 emit_insn (gen_aarch64_simd_vec_set<mode> (operands[0], operands[1],
834 GEN_INT (elem), operands[0]));
835 DONE;
836 }
837 )
839 ;; For 64-bit modes we use ushl/r, as this does not require a SIMD zero.
840 (define_insn "vec_shr_<mode>"
841 [(set (match_operand:VD 0 "register_operand" "=w")
842 (unspec:VD [(match_operand:VD 1 "register_operand" "w")
843 (match_operand:SI 2 "immediate_operand" "i")]
844 UNSPEC_VEC_SHR))]
845 "TARGET_SIMD"
846 {
847 if (BYTES_BIG_ENDIAN)
848 return "shl %d0, %d1, %2";
849 else
850 return "ushr %d0, %d1, %2";
851 }
852 [(set_attr "type" "neon_shift_imm")]
853 )
855 (define_insn "aarch64_simd_vec_setv2di"
856 [(set (match_operand:V2DI 0 "register_operand" "=w,w")
857 (vec_merge:V2DI
858 (vec_duplicate:V2DI
859 (match_operand:DI 1 "register_operand" "r,w"))
860 (match_operand:V2DI 3 "register_operand" "0,0")
861 (match_operand:SI 2 "immediate_operand" "i,i")))]
862 "TARGET_SIMD"
863 {
864 int elt = ENDIAN_LANE_N (V2DImode, exact_log2 (INTVAL (operands[2])));
865 operands[2] = GEN_INT ((HOST_WIDE_INT) 1 << elt);
866 switch (which_alternative)
867 {
868 case 0:
869 return "ins\\t%0.d[%p2], %1";
870 case 1:
871 return "ins\\t%0.d[%p2], %1.d[0]";
872 default:
873 gcc_unreachable ();
874 }
875 }
876 [(set_attr "type" "neon_from_gp, neon_ins_q")]
877 )
879 (define_expand "vec_setv2di"
880 [(match_operand:V2DI 0 "register_operand")
881 (match_operand:DI 1 "register_operand")
882 (match_operand:SI 2 "immediate_operand")]
883 "TARGET_SIMD"
884 {
885 HOST_WIDE_INT elem = (HOST_WIDE_INT) 1 << INTVAL (operands[2]);
886 emit_insn (gen_aarch64_simd_vec_setv2di (operands[0], operands[1],
887 GEN_INT (elem), operands[0]));
888 DONE;
889 }
890 )
892 (define_insn "aarch64_simd_vec_set<mode>"
893 [(set (match_operand:VDQF_F16 0 "register_operand" "=w")
894 (vec_merge:VDQF_F16
895 (vec_duplicate:VDQF_F16
896 (match_operand:<VEL> 1 "register_operand" "w"))
897 (match_operand:VDQF_F16 3 "register_operand" "0")
898 (match_operand:SI 2 "immediate_operand" "i")))]
899 "TARGET_SIMD"
900 {
901 int elt = ENDIAN_LANE_N (<MODE>mode, exact_log2 (INTVAL (operands[2])));
903 operands[2] = GEN_INT ((HOST_WIDE_INT)1 << elt);
904 return "ins\t%0.<Vetype>[%p2], %1.<Vetype>[0]";
905 }
906 [(set_attr "type" "neon_ins<q>")]
907 )
909 (define_expand "vec_set<mode>"
910 [(match_operand:VDQF_F16 0 "register_operand" "+w")
911 (match_operand:<VEL> 1 "register_operand" "w")
912 (match_operand:SI 2 "immediate_operand" "")]
913 "TARGET_SIMD"
914 {
915 HOST_WIDE_INT elem = (HOST_WIDE_INT) 1 << INTVAL (operands[2]);
916 emit_insn (gen_aarch64_simd_vec_set<mode> (operands[0], operands[1],
917 GEN_INT (elem), operands[0]));
918 DONE;
919 }
920 )
923 (define_insn "aarch64_mla<mode>"
924 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
925 (plus:VDQ_BHSI (mult:VDQ_BHSI
926 (match_operand:VDQ_BHSI 2 "register_operand" "w")
927 (match_operand:VDQ_BHSI 3 "register_operand" "w"))
928 (match_operand:VDQ_BHSI 1 "register_operand" "0")))]
929 "TARGET_SIMD"
930 "mla\t%0.<Vtype>, %2.<Vtype>, %3.<Vtype>"
931 [(set_attr "type" "neon_mla_<Vetype><q>")]
932 )
934 (define_insn "*aarch64_mla_elt<mode>"
935 [(set (match_operand:VDQHS 0 "register_operand" "=w")
936 (plus:VDQHS
937 (mult:VDQHS
938 (vec_duplicate:VDQHS
939 (vec_select:<VEL>
940 (match_operand:VDQHS 1 "register_operand" "<h_con>")
941 (parallel [(match_operand:SI 2 "immediate_operand")])))
942 (match_operand:VDQHS 3 "register_operand" "w"))
943 (match_operand:VDQHS 4 "register_operand" "0")))]
944 "TARGET_SIMD"
945 {
946 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
947 return "mla\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
948 }
949 [(set_attr "type" "neon_mla_<Vetype>_scalar<q>")]
950 )
952 (define_insn "*aarch64_mla_elt_<vswap_width_name><mode>"
953 [(set (match_operand:VDQHS 0 "register_operand" "=w")
954 (plus:VDQHS
955 (mult:VDQHS
956 (vec_duplicate:VDQHS
957 (vec_select:<VEL>
958 (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
959 (parallel [(match_operand:SI 2 "immediate_operand")])))
960 (match_operand:VDQHS 3 "register_operand" "w"))
961 (match_operand:VDQHS 4 "register_operand" "0")))]
962 "TARGET_SIMD"
963 {
964 operands[2] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
965 INTVAL (operands[2])));
966 return "mla\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
967 }
968 [(set_attr "type" "neon_mla_<Vetype>_scalar<q>")]
969 )
971 (define_insn "aarch64_mls<mode>"
972 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
973 (minus:VDQ_BHSI (match_operand:VDQ_BHSI 1 "register_operand" "0")
974 (mult:VDQ_BHSI (match_operand:VDQ_BHSI 2 "register_operand" "w")
975 (match_operand:VDQ_BHSI 3 "register_operand" "w"))))]
976 "TARGET_SIMD"
977 "mls\t%0.<Vtype>, %2.<Vtype>, %3.<Vtype>"
978 [(set_attr "type" "neon_mla_<Vetype><q>")]
979 )
981 (define_insn "*aarch64_mls_elt<mode>"
982 [(set (match_operand:VDQHS 0 "register_operand" "=w")
983 (minus:VDQHS
984 (match_operand:VDQHS 4 "register_operand" "0")
985 (mult:VDQHS
986 (vec_duplicate:VDQHS
987 (vec_select:<VEL>
988 (match_operand:VDQHS 1 "register_operand" "<h_con>")
989 (parallel [(match_operand:SI 2 "immediate_operand")])))
990 (match_operand:VDQHS 3 "register_operand" "w"))))]
991 "TARGET_SIMD"
992 {
993 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
994 return "mls\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
995 }
996 [(set_attr "type" "neon_mla_<Vetype>_scalar<q>")]
997 )
999 (define_insn "*aarch64_mls_elt_<vswap_width_name><mode>"
1000 [(set (match_operand:VDQHS 0 "register_operand" "=w")
1001 (minus:VDQHS
1002 (match_operand:VDQHS 4 "register_operand" "0")
1003 (mult:VDQHS
1004 (vec_duplicate:VDQHS
1005 (vec_select:<VEL>
1006 (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
1007 (parallel [(match_operand:SI 2 "immediate_operand")])))
1008 (match_operand:VDQHS 3 "register_operand" "w"))))]
1009 "TARGET_SIMD"
1010 {
1011 operands[2] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
1012 INTVAL (operands[2])));
1013 return "mls\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
1014 }
1015 [(set_attr "type" "neon_mla_<Vetype>_scalar<q>")]
1016 )
1018 ;; Max/Min operations.
1019 (define_insn "<su><maxmin><mode>3"
1020 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
1021 (MAXMIN:VDQ_BHSI (match_operand:VDQ_BHSI 1 "register_operand" "w")
1022 (match_operand:VDQ_BHSI 2 "register_operand" "w")))]
1023 "TARGET_SIMD"
1024 "<su><maxmin>\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1025 [(set_attr "type" "neon_minmax<q>")]
1026 )
1028 (define_expand "<su><maxmin>v2di3"
1029 [(set (match_operand:V2DI 0 "register_operand" "")
1030 (MAXMIN:V2DI (match_operand:V2DI 1 "register_operand" "")
1031 (match_operand:V2DI 2 "register_operand" "")))]
1032 "TARGET_SIMD"
1033 {
1034 enum rtx_code cmp_operator;
1035 rtx cmp_fmt;
1037 switch (<CODE>)
1038 {
1039 case UMIN:
1040 cmp_operator = LTU;
1041 break;
1042 case SMIN:
1043 cmp_operator = LT;
1044 break;
1045 case UMAX:
1046 cmp_operator = GTU;
1047 break;
1048 case SMAX:
1049 cmp_operator = GT;
1050 break;
1051 default:
1052 gcc_unreachable ();
1053 }
1055 cmp_fmt = gen_rtx_fmt_ee (cmp_operator, V2DImode, operands[1], operands[2]);
1056 emit_insn (gen_aarch64_vcond_internalv2div2di (operands[0], operands[1],
1057 operands[2], cmp_fmt, operands[1], operands[2]));
1058 DONE;
1059 })
1061 ;; Pairwise Integer Max/Min operations.
1062 (define_insn "aarch64_<maxmin_uns>p<mode>"
1063 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
1064 (unspec:VDQ_BHSI [(match_operand:VDQ_BHSI 1 "register_operand" "w")
1065 (match_operand:VDQ_BHSI 2 "register_operand" "w")]
1066 MAXMINV))]
1067 "TARGET_SIMD"
1068 "<maxmin_uns_op>p\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1069 [(set_attr "type" "neon_minmax<q>")]
1070 )
1072 ;; Pairwise FP Max/Min operations.
1073 (define_insn "aarch64_<maxmin_uns>p<mode>"
1074 [(set (match_operand:VDQF 0 "register_operand" "=w")
1075 (unspec:VDQF [(match_operand:VDQF 1 "register_operand" "w")
1076 (match_operand:VDQF 2 "register_operand" "w")]
1077 FMAXMINV))]
1078 "TARGET_SIMD"
1079 "<maxmin_uns_op>p\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1080 [(set_attr "type" "neon_minmax<q>")]
1081 )
1083 ;; vec_concat gives a new vector with the low elements from operand 1, and
1084 ;; the high elements from operand 2. That is to say, given op1 = { a, b }
1085 ;; op2 = { c, d }, vec_concat (op1, op2) = { a, b, c, d }.
1086 ;; What that means, is that the RTL descriptions of the below patterns
1087 ;; need to change depending on endianness.
1089 ;; Move to the low architectural bits of the register.
1090 ;; On little-endian this is { operand, zeroes }
1091 ;; On big-endian this is { zeroes, operand }
1093 (define_insn "move_lo_quad_internal_<mode>"
1094 [(set (match_operand:VQ_NO2E 0 "register_operand" "=w,w,w")
1095 (vec_concat:VQ_NO2E
1096 (match_operand:<VHALF> 1 "register_operand" "w,r,r")
1097 (vec_duplicate:<VHALF> (const_int 0))))]
1098 "TARGET_SIMD && !BYTES_BIG_ENDIAN"
1099 "@
1100 dup\\t%d0, %1.d[0]
1101 fmov\\t%d0, %1
1102 dup\\t%d0, %1"
1103 [(set_attr "type" "neon_dup<q>,f_mcr,neon_dup<q>")
1104 (set_attr "simd" "yes,*,yes")
1105 (set_attr "fp" "*,yes,*")
1106 (set_attr "length" "4")]
1107 )
1109 (define_insn "move_lo_quad_internal_<mode>"
1110 [(set (match_operand:VQ_2E 0 "register_operand" "=w,w,w")
1111 (vec_concat:VQ_2E
1112 (match_operand:<VHALF> 1 "register_operand" "w,r,r")
1113 (const_int 0)))]
1114 "TARGET_SIMD && !BYTES_BIG_ENDIAN"
1115 "@
1116 dup\\t%d0, %1.d[0]
1117 fmov\\t%d0, %1
1118 dup\\t%d0, %1"
1119 [(set_attr "type" "neon_dup<q>,f_mcr,neon_dup<q>")
1120 (set_attr "simd" "yes,*,yes")
1121 (set_attr "fp" "*,yes,*")
1122 (set_attr "length" "4")]
1123 )
1125 (define_insn "move_lo_quad_internal_be_<mode>"
1126 [(set (match_operand:VQ_NO2E 0 "register_operand" "=w,w,w")
1127 (vec_concat:VQ_NO2E
1128 (vec_duplicate:<VHALF> (const_int 0))
1129 (match_operand:<VHALF> 1 "register_operand" "w,r,r")))]
1130 "TARGET_SIMD && BYTES_BIG_ENDIAN"
1131 "@
1132 dup\\t%d0, %1.d[0]
1133 fmov\\t%d0, %1
1134 dup\\t%d0, %1"
1135 [(set_attr "type" "neon_dup<q>,f_mcr,neon_dup<q>")
1136 (set_attr "simd" "yes,*,yes")
1137 (set_attr "fp" "*,yes,*")
1138 (set_attr "length" "4")]
1139 )
1141 (define_insn "move_lo_quad_internal_be_<mode>"
1142 [(set (match_operand:VQ_2E 0 "register_operand" "=w,w,w")
1143 (vec_concat:VQ_2E
1144 (const_int 0)
1145 (match_operand:<VHALF> 1 "register_operand" "w,r,r")))]
1146 "TARGET_SIMD && BYTES_BIG_ENDIAN"
1147 "@
1148 dup\\t%d0, %1.d[0]
1149 fmov\\t%d0, %1
1150 dup\\t%d0, %1"
1151 [(set_attr "type" "neon_dup<q>,f_mcr,neon_dup<q>")
1152 (set_attr "simd" "yes,*,yes")
1153 (set_attr "fp" "*,yes,*")
1154 (set_attr "length" "4")]
1155 )
1157 (define_expand "move_lo_quad_<mode>"
1158 [(match_operand:VQ 0 "register_operand")
1159 (match_operand:VQ 1 "register_operand")]
1160 "TARGET_SIMD"
1161 {
1162 if (BYTES_BIG_ENDIAN)
1163 emit_insn (gen_move_lo_quad_internal_be_<mode> (operands[0], operands[1]));
1164 else
1165 emit_insn (gen_move_lo_quad_internal_<mode> (operands[0], operands[1]));
1166 DONE;
1167 }
1168 )
1170 ;; Move operand1 to the high architectural bits of the register, keeping
1171 ;; the low architectural bits of operand2.
1172 ;; For little-endian this is { operand2, operand1 }
1173 ;; For big-endian this is { operand1, operand2 }
1175 (define_insn "aarch64_simd_move_hi_quad_<mode>"
1176 [(set (match_operand:VQ 0 "register_operand" "+w,w")
1177 (vec_concat:VQ
1178 (vec_select:<VHALF>
1179 (match_dup 0)
1180 (match_operand:VQ 2 "vect_par_cnst_lo_half" ""))
1181 (match_operand:<VHALF> 1 "register_operand" "w,r")))]
1182 "TARGET_SIMD && !BYTES_BIG_ENDIAN"
1183 "@
1184 ins\\t%0.d[1], %1.d[0]
1185 ins\\t%0.d[1], %1"
1186 [(set_attr "type" "neon_ins")]
1187 )
1189 (define_insn "aarch64_simd_move_hi_quad_be_<mode>"
1190 [(set (match_operand:VQ 0 "register_operand" "+w,w")
1191 (vec_concat:VQ
1192 (match_operand:<VHALF> 1 "register_operand" "w,r")
1193 (vec_select:<VHALF>
1194 (match_dup 0)
1195 (match_operand:VQ 2 "vect_par_cnst_lo_half" ""))))]
1196 "TARGET_SIMD && BYTES_BIG_ENDIAN"
1197 "@
1198 ins\\t%0.d[1], %1.d[0]
1199 ins\\t%0.d[1], %1"
1200 [(set_attr "type" "neon_ins")]
1201 )
1203 (define_expand "move_hi_quad_<mode>"
1204 [(match_operand:VQ 0 "register_operand" "")
1205 (match_operand:<VHALF> 1 "register_operand" "")]
1206 "TARGET_SIMD"
1207 {
1208 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
1209 if (BYTES_BIG_ENDIAN)
1210 emit_insn (gen_aarch64_simd_move_hi_quad_be_<mode> (operands[0],
1211 operands[1], p));
1212 else
1213 emit_insn (gen_aarch64_simd_move_hi_quad_<mode> (operands[0],
1214 operands[1], p));
1215 DONE;
1216 })
1218 ;; Narrowing operations.
1220 ;; For doubles.
1221 (define_insn "aarch64_simd_vec_pack_trunc_<mode>"
1222 [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
1223 (truncate:<VNARROWQ> (match_operand:VQN 1 "register_operand" "w")))]
1224 "TARGET_SIMD"
1225 "xtn\\t%0.<Vntype>, %1.<Vtype>"
1226 [(set_attr "type" "neon_shift_imm_narrow_q")]
1227 )
1229 (define_expand "vec_pack_trunc_<mode>"
1230 [(match_operand:<VNARROWD> 0 "register_operand" "")
1231 (match_operand:VDN 1 "register_operand" "")
1232 (match_operand:VDN 2 "register_operand" "")]
1233 "TARGET_SIMD"
1234 {
1235 rtx tempreg = gen_reg_rtx (<VDBL>mode);
1236 int lo = BYTES_BIG_ENDIAN ? 2 : 1;
1237 int hi = BYTES_BIG_ENDIAN ? 1 : 2;
1239 emit_insn (gen_move_lo_quad_<Vdbl> (tempreg, operands[lo]));
1240 emit_insn (gen_move_hi_quad_<Vdbl> (tempreg, operands[hi]));
1241 emit_insn (gen_aarch64_simd_vec_pack_trunc_<Vdbl> (operands[0], tempreg));
1242 DONE;
1243 })
1245 ;; For quads.
1247 (define_insn "vec_pack_trunc_<mode>"
1248 [(set (match_operand:<VNARROWQ2> 0 "register_operand" "=&w")
1249 (vec_concat:<VNARROWQ2>
1250 (truncate:<VNARROWQ> (match_operand:VQN 1 "register_operand" "w"))
1251 (truncate:<VNARROWQ> (match_operand:VQN 2 "register_operand" "w"))))]
1252 "TARGET_SIMD"
1253 {
1254 if (BYTES_BIG_ENDIAN)
1255 return "xtn\\t%0.<Vntype>, %2.<Vtype>\;xtn2\\t%0.<V2ntype>, %1.<Vtype>";
1256 else
1257 return "xtn\\t%0.<Vntype>, %1.<Vtype>\;xtn2\\t%0.<V2ntype>, %2.<Vtype>";
1258 }
1259 [(set_attr "type" "multiple")
1260 (set_attr "length" "8")]
1261 )
1263 ;; Widening operations.
1265 (define_insn "aarch64_simd_vec_unpack<su>_lo_<mode>"
1266 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1267 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1268 (match_operand:VQW 1 "register_operand" "w")
1269 (match_operand:VQW 2 "vect_par_cnst_lo_half" "")
1270 )))]
1271 "TARGET_SIMD"
1272 "<su>shll\t%0.<Vwtype>, %1.<Vhalftype>, 0"
1273 [(set_attr "type" "neon_shift_imm_long")]
1274 )
1276 (define_insn "aarch64_simd_vec_unpack<su>_hi_<mode>"
1277 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1278 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1279 (match_operand:VQW 1 "register_operand" "w")
1280 (match_operand:VQW 2 "vect_par_cnst_hi_half" "")
1281 )))]
1282 "TARGET_SIMD"
1283 "<su>shll2\t%0.<Vwtype>, %1.<Vtype>, 0"
1284 [(set_attr "type" "neon_shift_imm_long")]
1285 )
1287 (define_expand "vec_unpack<su>_hi_<mode>"
1288 [(match_operand:<VWIDE> 0 "register_operand" "")
1289 (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand"))]
1290 "TARGET_SIMD"
1291 {
1292 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
1293 emit_insn (gen_aarch64_simd_vec_unpack<su>_hi_<mode> (operands[0],
1294 operands[1], p));
1295 DONE;
1296 }
1297 )
1299 (define_expand "vec_unpack<su>_lo_<mode>"
1300 [(match_operand:<VWIDE> 0 "register_operand" "")
1301 (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand" ""))]
1302 "TARGET_SIMD"
1303 {
1304 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
1305 emit_insn (gen_aarch64_simd_vec_unpack<su>_lo_<mode> (operands[0],
1306 operands[1], p));
1307 DONE;
1308 }
1309 )
1311 ;; Widening arithmetic.
1313 (define_insn "*aarch64_<su>mlal_lo<mode>"
1314 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1315 (plus:<VWIDE>
1316 (mult:<VWIDE>
1317 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1318 (match_operand:VQW 2 "register_operand" "w")
1319 (match_operand:VQW 3 "vect_par_cnst_lo_half" "")))
1320 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1321 (match_operand:VQW 4 "register_operand" "w")
1322 (match_dup 3))))
1323 (match_operand:<VWIDE> 1 "register_operand" "0")))]
1324 "TARGET_SIMD"
1325 "<su>mlal\t%0.<Vwtype>, %2.<Vhalftype>, %4.<Vhalftype>"
1326 [(set_attr "type" "neon_mla_<Vetype>_long")]
1327 )
1329 (define_insn "*aarch64_<su>mlal_hi<mode>"
1330 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1331 (plus:<VWIDE>
1332 (mult:<VWIDE>
1333 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1334 (match_operand:VQW 2 "register_operand" "w")
1335 (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))
1336 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1337 (match_operand:VQW 4 "register_operand" "w")
1338 (match_dup 3))))
1339 (match_operand:<VWIDE> 1 "register_operand" "0")))]
1340 "TARGET_SIMD"
1341 "<su>mlal2\t%0.<Vwtype>, %2.<Vtype>, %4.<Vtype>"
1342 [(set_attr "type" "neon_mla_<Vetype>_long")]
1343 )
1345 (define_insn "*aarch64_<su>mlsl_lo<mode>"
1346 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1347 (minus:<VWIDE>
1348 (match_operand:<VWIDE> 1 "register_operand" "0")
1349 (mult:<VWIDE>
1350 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1351 (match_operand:VQW 2 "register_operand" "w")
1352 (match_operand:VQW 3 "vect_par_cnst_lo_half" "")))
1353 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1354 (match_operand:VQW 4 "register_operand" "w")
1355 (match_dup 3))))))]
1356 "TARGET_SIMD"
1357 "<su>mlsl\t%0.<Vwtype>, %2.<Vhalftype>, %4.<Vhalftype>"
1358 [(set_attr "type" "neon_mla_<Vetype>_long")]
1359 )
1361 (define_insn "*aarch64_<su>mlsl_hi<mode>"
1362 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1363 (minus:<VWIDE>
1364 (match_operand:<VWIDE> 1 "register_operand" "0")
1365 (mult:<VWIDE>
1366 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1367 (match_operand:VQW 2 "register_operand" "w")
1368 (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))
1369 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1370 (match_operand:VQW 4 "register_operand" "w")
1371 (match_dup 3))))))]
1372 "TARGET_SIMD"
1373 "<su>mlsl2\t%0.<Vwtype>, %2.<Vtype>, %4.<Vtype>"
1374 [(set_attr "type" "neon_mla_<Vetype>_long")]
1375 )
1377 (define_insn "*aarch64_<su>mlal<mode>"
1378 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1379 (plus:<VWIDE>
1380 (mult:<VWIDE>
1381 (ANY_EXTEND:<VWIDE>
1382 (match_operand:VD_BHSI 1 "register_operand" "w"))
1383 (ANY_EXTEND:<VWIDE>
1384 (match_operand:VD_BHSI 2 "register_operand" "w")))
1385 (match_operand:<VWIDE> 3 "register_operand" "0")))]
1386 "TARGET_SIMD"
1387 "<su>mlal\t%0.<Vwtype>, %1.<Vtype>, %2.<Vtype>"
1388 [(set_attr "type" "neon_mla_<Vetype>_long")]
1389 )
1391 (define_insn "*aarch64_<su>mlsl<mode>"
1392 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1393 (minus:<VWIDE>
1394 (match_operand:<VWIDE> 1 "register_operand" "0")
1395 (mult:<VWIDE>
1396 (ANY_EXTEND:<VWIDE>
1397 (match_operand:VD_BHSI 2 "register_operand" "w"))
1398 (ANY_EXTEND:<VWIDE>
1399 (match_operand:VD_BHSI 3 "register_operand" "w")))))]
1400 "TARGET_SIMD"
1401 "<su>mlsl\t%0.<Vwtype>, %2.<Vtype>, %3.<Vtype>"
1402 [(set_attr "type" "neon_mla_<Vetype>_long")]
1403 )
1405 (define_insn "aarch64_simd_vec_<su>mult_lo_<mode>"
1406 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1407 (mult:<VWIDE> (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1408 (match_operand:VQW 1 "register_operand" "w")
1409 (match_operand:VQW 3 "vect_par_cnst_lo_half" "")))
1410 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1411 (match_operand:VQW 2 "register_operand" "w")
1412 (match_dup 3)))))]
1413 "TARGET_SIMD"
1414 "<su>mull\\t%0.<Vwtype>, %1.<Vhalftype>, %2.<Vhalftype>"
1415 [(set_attr "type" "neon_mul_<Vetype>_long")]
1416 )
1418 (define_expand "vec_widen_<su>mult_lo_<mode>"
1419 [(match_operand:<VWIDE> 0 "register_operand" "")
1420 (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand" ""))
1421 (ANY_EXTEND:<VWIDE> (match_operand:VQW 2 "register_operand" ""))]
1422 "TARGET_SIMD"
1423 {
1424 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
1425 emit_insn (gen_aarch64_simd_vec_<su>mult_lo_<mode> (operands[0],
1426 operands[1],
1427 operands[2], p));
1428 DONE;
1429 }
1430 )
1432 (define_insn "aarch64_simd_vec_<su>mult_hi_<mode>"
1433 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1434 (mult:<VWIDE> (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1435 (match_operand:VQW 1 "register_operand" "w")
1436 (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))
1437 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1438 (match_operand:VQW 2 "register_operand" "w")
1439 (match_dup 3)))))]
1440 "TARGET_SIMD"
1441 "<su>mull2\\t%0.<Vwtype>, %1.<Vtype>, %2.<Vtype>"
1442 [(set_attr "type" "neon_mul_<Vetype>_long")]
1443 )
1445 (define_expand "vec_widen_<su>mult_hi_<mode>"
1446 [(match_operand:<VWIDE> 0 "register_operand" "")
1447 (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand" ""))
1448 (ANY_EXTEND:<VWIDE> (match_operand:VQW 2 "register_operand" ""))]
1449 "TARGET_SIMD"
1450 {
1451 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
1452 emit_insn (gen_aarch64_simd_vec_<su>mult_hi_<mode> (operands[0],
1453 operands[1],
1454 operands[2], p));
1455 DONE;
1457 }
1458 )
1460 ;; FP vector operations.
1461 ;; AArch64 AdvSIMD supports single-precision (32-bit) and
1462 ;; double-precision (64-bit) floating-point data types and arithmetic as
1463 ;; defined by the IEEE 754-2008 standard. This makes them vectorizable
1464 ;; without the need for -ffast-math or -funsafe-math-optimizations.
1465 ;;
1466 ;; Floating-point operations can raise an exception. Vectorizing such
1467 ;; operations are safe because of reasons explained below.
1468 ;;
1469 ;; ARMv8 permits an extension to enable trapped floating-point
1470 ;; exception handling, however this is an optional feature. In the
1471 ;; event of a floating-point exception being raised by vectorised
1472 ;; code then:
1473 ;; 1. If trapped floating-point exceptions are available, then a trap
1474 ;; will be taken when any lane raises an enabled exception. A trap
1475 ;; handler may determine which lane raised the exception.
1476 ;; 2. Alternatively a sticky exception flag is set in the
1477 ;; floating-point status register (FPSR). Software may explicitly
1478 ;; test the exception flags, in which case the tests will either
1479 ;; prevent vectorisation, allowing precise identification of the
1480 ;; failing operation, or if tested outside of vectorisable regions
1481 ;; then the specific operation and lane are not of interest.
1483 ;; FP arithmetic operations.
1485 (define_insn "add<mode>3"
1486 [(set (match_operand:VDQF 0 "register_operand" "=w")
1487 (plus:VDQF (match_operand:VDQF 1 "register_operand" "w")
1488 (match_operand:VDQF 2 "register_operand" "w")))]
1489 "TARGET_SIMD"
1490 "fadd\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1491 [(set_attr "type" "neon_fp_addsub_<Vetype><q>")]
1492 )
1494 (define_insn "sub<mode>3"
1495 [(set (match_operand:VDQF 0 "register_operand" "=w")
1496 (minus:VDQF (match_operand:VDQF 1 "register_operand" "w")
1497 (match_operand:VDQF 2 "register_operand" "w")))]
1498 "TARGET_SIMD"
1499 "fsub\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1500 [(set_attr "type" "neon_fp_addsub_<Vetype><q>")]
1501 )
1503 (define_insn "mul<mode>3"
1504 [(set (match_operand:VDQF 0 "register_operand" "=w")
1505 (mult:VDQF (match_operand:VDQF 1 "register_operand" "w")
1506 (match_operand:VDQF 2 "register_operand" "w")))]
1507 "TARGET_SIMD"
1508 "fmul\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1509 [(set_attr "type" "neon_fp_mul_<Vetype><q>")]
1510 )
1512 (define_insn "div<mode>3"
1513 [(set (match_operand:VDQF 0 "register_operand" "=w")
1514 (div:VDQF (match_operand:VDQF 1 "register_operand" "w")
1515 (match_operand:VDQF 2 "register_operand" "w")))]
1516 "TARGET_SIMD"
1517 "fdiv\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1518 [(set_attr "type" "neon_fp_div_<Vetype><q>")]
1519 )
1521 (define_insn "neg<mode>2"
1522 [(set (match_operand:VDQF 0 "register_operand" "=w")
1523 (neg:VDQF (match_operand:VDQF 1 "register_operand" "w")))]
1524 "TARGET_SIMD"
1525 "fneg\\t%0.<Vtype>, %1.<Vtype>"
1526 [(set_attr "type" "neon_fp_neg_<Vetype><q>")]
1527 )
1529 (define_insn "abs<mode>2"
1530 [(set (match_operand:VDQF 0 "register_operand" "=w")
1531 (abs:VDQF (match_operand:VDQF 1 "register_operand" "w")))]
1532 "TARGET_SIMD"
1533 "fabs\\t%0.<Vtype>, %1.<Vtype>"
1534 [(set_attr "type" "neon_fp_abs_<Vetype><q>")]
1535 )
1537 (define_insn "fma<mode>4"
1538 [(set (match_operand:VDQF 0 "register_operand" "=w")
1539 (fma:VDQF (match_operand:VDQF 1 "register_operand" "w")
1540 (match_operand:VDQF 2 "register_operand" "w")
1541 (match_operand:VDQF 3 "register_operand" "0")))]
1542 "TARGET_SIMD"
1543 "fmla\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1544 [(set_attr "type" "neon_fp_mla_<Vetype><q>")]
1545 )
1547 (define_insn "*aarch64_fma4_elt<mode>"
1548 [(set (match_operand:VDQF 0 "register_operand" "=w")
1549 (fma:VDQF
1550 (vec_duplicate:VDQF
1551 (vec_select:<VEL>
1552 (match_operand:VDQF 1 "register_operand" "<h_con>")
1553 (parallel [(match_operand:SI 2 "immediate_operand")])))
1554 (match_operand:VDQF 3 "register_operand" "w")
1555 (match_operand:VDQF 4 "register_operand" "0")))]
1556 "TARGET_SIMD"
1557 {
1558 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
1559 return "fmla\\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
1560 }
1561 [(set_attr "type" "neon_fp_mla_<Vetype>_scalar<q>")]
1562 )
1564 (define_insn "*aarch64_fma4_elt_<vswap_width_name><mode>"
1565 [(set (match_operand:VDQSF 0 "register_operand" "=w")
1566 (fma:VDQSF
1567 (vec_duplicate:VDQSF
1568 (vec_select:<VEL>
1569 (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
1570 (parallel [(match_operand:SI 2 "immediate_operand")])))
1571 (match_operand:VDQSF 3 "register_operand" "w")
1572 (match_operand:VDQSF 4 "register_operand" "0")))]
1573 "TARGET_SIMD"
1574 {
1575 operands[2] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
1576 INTVAL (operands[2])));
1577 return "fmla\\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
1578 }
1579 [(set_attr "type" "neon_fp_mla_<Vetype>_scalar<q>")]
1580 )
1582 (define_insn "*aarch64_fma4_elt_to_128df"
1583 [(set (match_operand:V2DF 0 "register_operand" "=w")
1584 (fma:V2DF
1585 (vec_duplicate:V2DF
1586 (match_operand:DF 1 "register_operand" "w"))
1587 (match_operand:V2DF 2 "register_operand" "w")
1588 (match_operand:V2DF 3 "register_operand" "0")))]
1589 "TARGET_SIMD"
1590 "fmla\\t%0.2d, %2.2d, %1.2d[0]"
1591 [(set_attr "type" "neon_fp_mla_d_scalar_q")]
1592 )
1594 (define_insn "*aarch64_fma4_elt_to_64v2df"
1595 [(set (match_operand:DF 0 "register_operand" "=w")
1596 (fma:DF
1597 (vec_select:DF
1598 (match_operand:V2DF 1 "register_operand" "w")
1599 (parallel [(match_operand:SI 2 "immediate_operand")]))
1600 (match_operand:DF 3 "register_operand" "w")
1601 (match_operand:DF 4 "register_operand" "0")))]
1602 "TARGET_SIMD"
1603 {
1604 operands[2] = GEN_INT (ENDIAN_LANE_N (V2DFmode, INTVAL (operands[2])));
1605 return "fmla\\t%0.2d, %3.2d, %1.2d[%2]";
1606 }
1607 [(set_attr "type" "neon_fp_mla_d_scalar_q")]
1608 )
1610 (define_insn "fnma<mode>4"
1611 [(set (match_operand:VDQF 0 "register_operand" "=w")
1612 (fma:VDQF
1613 (match_operand:VDQF 1 "register_operand" "w")
1614 (neg:VDQF
1615 (match_operand:VDQF 2 "register_operand" "w"))
1616 (match_operand:VDQF 3 "register_operand" "0")))]
1617 "TARGET_SIMD"
1618 "fmls\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1619 [(set_attr "type" "neon_fp_mla_<Vetype><q>")]
1620 )
1622 (define_insn "*aarch64_fnma4_elt<mode>"
1623 [(set (match_operand:VDQF 0 "register_operand" "=w")
1624 (fma:VDQF
1625 (neg:VDQF
1626 (match_operand:VDQF 3 "register_operand" "w"))
1627 (vec_duplicate:VDQF
1628 (vec_select:<VEL>
1629 (match_operand:VDQF 1 "register_operand" "<h_con>")
1630 (parallel [(match_operand:SI 2 "immediate_operand")])))
1631 (match_operand:VDQF 4 "register_operand" "0")))]
1632 "TARGET_SIMD"
1633 {
1634 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
1635 return "fmls\\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
1636 }
1637 [(set_attr "type" "neon_fp_mla_<Vetype>_scalar<q>")]
1638 )
1640 (define_insn "*aarch64_fnma4_elt_<vswap_width_name><mode>"
1641 [(set (match_operand:VDQSF 0 "register_operand" "=w")
1642 (fma:VDQSF
1643 (neg:VDQSF
1644 (match_operand:VDQSF 3 "register_operand" "w"))
1645 (vec_duplicate:VDQSF
1646 (vec_select:<VEL>
1647 (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
1648 (parallel [(match_operand:SI 2 "immediate_operand")])))
1649 (match_operand:VDQSF 4 "register_operand" "0")))]
1650 "TARGET_SIMD"
1651 {
1652 operands[2] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
1653 INTVAL (operands[2])));
1654 return "fmls\\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
1655 }
1656 [(set_attr "type" "neon_fp_mla_<Vetype>_scalar<q>")]
1657 )
1659 (define_insn "*aarch64_fnma4_elt_to_128df"
1660 [(set (match_operand:V2DF 0 "register_operand" "=w")
1661 (fma:V2DF
1662 (neg:V2DF
1663 (match_operand:V2DF 2 "register_operand" "w"))
1664 (vec_duplicate:V2DF
1665 (match_operand:DF 1 "register_operand" "w"))
1666 (match_operand:V2DF 3 "register_operand" "0")))]
1667 "TARGET_SIMD"
1668 "fmls\\t%0.2d, %2.2d, %1.2d[0]"
1669 [(set_attr "type" "neon_fp_mla_d_scalar_q")]
1670 )
1672 (define_insn "*aarch64_fnma4_elt_to_64v2df"
1673 [(set (match_operand:DF 0 "register_operand" "=w")
1674 (fma:DF
1675 (vec_select:DF
1676 (match_operand:V2DF 1 "register_operand" "w")
1677 (parallel [(match_operand:SI 2 "immediate_operand")]))
1678 (neg:DF
1679 (match_operand:DF 3 "register_operand" "w"))
1680 (match_operand:DF 4 "register_operand" "0")))]
1681 "TARGET_SIMD"
1682 {
1683 operands[2] = GEN_INT (ENDIAN_LANE_N (V2DFmode, INTVAL (operands[2])));
1684 return "fmls\\t%0.2d, %3.2d, %1.2d[%2]";
1685 }
1686 [(set_attr "type" "neon_fp_mla_d_scalar_q")]
1687 )
1689 ;; Vector versions of the floating-point frint patterns.
1690 ;; Expands to btrunc, ceil, floor, nearbyint, rint, round, frintn.
1691 (define_insn "<frint_pattern><mode>2"
1692 [(set (match_operand:VDQF 0 "register_operand" "=w")
1693 (unspec:VDQF [(match_operand:VDQF 1 "register_operand" "w")]
1694 FRINT))]
1695 "TARGET_SIMD"
1696 "frint<frint_suffix>\\t%0.<Vtype>, %1.<Vtype>"
1697 [(set_attr "type" "neon_fp_round_<Vetype><q>")]
1698 )
1700 ;; Vector versions of the fcvt standard patterns.
1701 ;; Expands to lbtrunc, lround, lceil, lfloor
1702 (define_insn "l<fcvt_pattern><su_optab><VDQF:mode><fcvt_target>2"
1703 [(set (match_operand:<FCVT_TARGET> 0 "register_operand" "=w")
1704 (FIXUORS:<FCVT_TARGET> (unspec:<FCVT_TARGET>
1705 [(match_operand:VDQF 1 "register_operand" "w")]
1706 FCVT)))]
1707 "TARGET_SIMD"
1708 "fcvt<frint_suffix><su>\\t%0.<Vtype>, %1.<Vtype>"
1709 [(set_attr "type" "neon_fp_to_int_<Vetype><q>")]
1710 )
1712 (define_insn "*aarch64_fcvt<su_optab><VDQF:mode><fcvt_target>2_mult"
1713 [(set (match_operand:<FCVT_TARGET> 0 "register_operand" "=w")
1714 (FIXUORS:<FCVT_TARGET> (unspec:<FCVT_TARGET>
1715 [(mult:VDQF
1716 (match_operand:VDQF 1 "register_operand" "w")
1717 (match_operand:VDQF 2 "aarch64_fp_vec_pow2" ""))]
1718 UNSPEC_FRINTZ)))]
1719 "TARGET_SIMD
1720 && IN_RANGE (aarch64_vec_fpconst_pow_of_2 (operands[2]), 1,
1721 GET_MODE_BITSIZE (GET_MODE_INNER (<VDQF:MODE>mode)))"
1722 {
1723 int fbits = aarch64_vec_fpconst_pow_of_2 (operands[2]);
1724 char buf[64];
1725 snprintf (buf, 64, "fcvtz<su>\\t%%0.<Vtype>, %%1.<Vtype>, #%d", fbits);
1726 output_asm_insn (buf, operands);
1727 return "";
1728 }
1729 [(set_attr "type" "neon_fp_to_int_<Vetype><q>")]
1730 )
1732 (define_expand "<optab><VDQF:mode><fcvt_target>2"
1733 [(set (match_operand:<FCVT_TARGET> 0 "register_operand")
1734 (FIXUORS:<FCVT_TARGET> (unspec:<FCVT_TARGET>
1735 [(match_operand:VDQF 1 "register_operand")]
1736 UNSPEC_FRINTZ)))]
1737 "TARGET_SIMD"
1738 {})
1740 (define_expand "<fix_trunc_optab><VDQF:mode><fcvt_target>2"
1741 [(set (match_operand:<FCVT_TARGET> 0 "register_operand")
1742 (FIXUORS:<FCVT_TARGET> (unspec:<FCVT_TARGET>
1743 [(match_operand:VDQF 1 "register_operand")]
1744 UNSPEC_FRINTZ)))]
1745 "TARGET_SIMD"
1746 {})
1748 (define_expand "ftrunc<VDQF:mode>2"
1749 [(set (match_operand:VDQF 0 "register_operand")
1750 (unspec:VDQF [(match_operand:VDQF 1 "register_operand")]
1751 UNSPEC_FRINTZ))]
1752 "TARGET_SIMD"
1753 {})
1755 (define_insn "<optab><fcvt_target><VDQF:mode>2"
1756 [(set (match_operand:VDQF 0 "register_operand" "=w")
1757 (FLOATUORS:VDQF
1758 (match_operand:<FCVT_TARGET> 1 "register_operand" "w")))]
1759 "TARGET_SIMD"
1760 "<su_optab>cvtf\\t%0.<Vtype>, %1.<Vtype>"
1761 [(set_attr "type" "neon_int_to_fp_<Vetype><q>")]
1762 )
1764 ;; Conversions between vectors of floats and doubles.
1765 ;; Contains a mix of patterns to match standard pattern names
1766 ;; and those for intrinsics.
1768 ;; Float widening operations.
1770 (define_insn "aarch64_simd_vec_unpacks_lo_<mode>"
1771 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1772 (float_extend:<VWIDE> (vec_select:<VHALF>
1773 (match_operand:VQ_HSF 1 "register_operand" "w")
1774 (match_operand:VQ_HSF 2 "vect_par_cnst_lo_half" "")
1775 )))]
1776 "TARGET_SIMD"
1777 "fcvtl\\t%0.<Vwtype>, %1.<Vhalftype>"
1778 [(set_attr "type" "neon_fp_cvt_widen_s")]
1779 )
1781 ;; ??? Note that the vectorizer usage of the vec_unpacks_[lo/hi] patterns
1782 ;; is inconsistent with vector ordering elsewhere in the compiler, in that
1783 ;; the meaning of HI and LO changes depending on the target endianness.
1784 ;; While elsewhere we map the higher numbered elements of a vector to
1785 ;; the lower architectural lanes of the vector, for these patterns we want
1786 ;; to always treat "hi" as referring to the higher architectural lanes.
1787 ;; Consequently, while the patterns below look inconsistent with our
1788 ;; other big-endian patterns their behavior is as required.
1790 (define_expand "vec_unpacks_lo_<mode>"
1791 [(match_operand:<VWIDE> 0 "register_operand" "")
1792 (match_operand:VQ_HSF 1 "register_operand" "")]
1793 "TARGET_SIMD"
1794 {
1795 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
1796 emit_insn (gen_aarch64_simd_vec_unpacks_lo_<mode> (operands[0],
1797 operands[1], p));
1798 DONE;
1799 }
1800 )
1802 (define_insn "aarch64_simd_vec_unpacks_hi_<mode>"
1803 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1804 (float_extend:<VWIDE> (vec_select:<VHALF>
1805 (match_operand:VQ_HSF 1 "register_operand" "w")
1806 (match_operand:VQ_HSF 2 "vect_par_cnst_hi_half" "")
1807 )))]
1808 "TARGET_SIMD"
1809 "fcvtl2\\t%0.<Vwtype>, %1.<Vtype>"
1810 [(set_attr "type" "neon_fp_cvt_widen_s")]
1811 )
1813 (define_expand "vec_unpacks_hi_<mode>"
1814 [(match_operand:<VWIDE> 0 "register_operand" "")
1815 (match_operand:VQ_HSF 1 "register_operand" "")]
1816 "TARGET_SIMD"
1817 {
1818 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
1819 emit_insn (gen_aarch64_simd_vec_unpacks_lo_<mode> (operands[0],
1820 operands[1], p));
1821 DONE;
1822 }
1823 )
1824 (define_insn "aarch64_float_extend_lo_<Vwide>"
1825 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1826 (float_extend:<VWIDE>
1827 (match_operand:VDF 1 "register_operand" "w")))]
1828 "TARGET_SIMD"
1829 "fcvtl\\t%0<Vmwtype>, %1<Vmtype>"
1830 [(set_attr "type" "neon_fp_cvt_widen_s")]
1831 )
1833 ;; Float narrowing operations.
1835 (define_insn "aarch64_float_truncate_lo_<mode>"
1836 [(set (match_operand:VDF 0 "register_operand" "=w")
1837 (float_truncate:VDF
1838 (match_operand:<VWIDE> 1 "register_operand" "w")))]
1839 "TARGET_SIMD"
1840 "fcvtn\\t%0.<Vtype>, %1<Vmwtype>"
1841 [(set_attr "type" "neon_fp_cvt_narrow_d_q")]
1842 )
1844 (define_insn "aarch64_float_truncate_hi_<Vdbl>_le"
1845 [(set (match_operand:<VDBL> 0 "register_operand" "=w")
1846 (vec_concat:<VDBL>
1847 (match_operand:VDF 1 "register_operand" "0")
1848 (float_truncate:VDF
1849 (match_operand:<VWIDE> 2 "register_operand" "w"))))]
1850 "TARGET_SIMD && !BYTES_BIG_ENDIAN"
1851 "fcvtn2\\t%0.<Vdtype>, %2<Vmwtype>"
1852 [(set_attr "type" "neon_fp_cvt_narrow_d_q")]
1853 )
1855 (define_insn "aarch64_float_truncate_hi_<Vdbl>_be"
1856 [(set (match_operand:<VDBL> 0 "register_operand" "=w")
1857 (vec_concat:<VDBL>
1858 (float_truncate:VDF
1859 (match_operand:<VWIDE> 2 "register_operand" "w"))
1860 (match_operand:VDF 1 "register_operand" "0")))]
1861 "TARGET_SIMD && BYTES_BIG_ENDIAN"
1862 "fcvtn2\\t%0.<Vdtype>, %2<Vmwtype>"
1863 [(set_attr "type" "neon_fp_cvt_narrow_d_q")]
1864 )
1866 (define_expand "aarch64_float_truncate_hi_<Vdbl>"
1867 [(match_operand:<VDBL> 0 "register_operand" "=w")
1868 (match_operand:VDF 1 "register_operand" "0")
1869 (match_operand:<VWIDE> 2 "register_operand" "w")]
1870 "TARGET_SIMD"
1871 {
1872 rtx (*gen) (rtx, rtx, rtx) = BYTES_BIG_ENDIAN
1873 ? gen_aarch64_float_truncate_hi_<Vdbl>_be
1874 : gen_aarch64_float_truncate_hi_<Vdbl>_le;
1875 emit_insn (gen (operands[0], operands[1], operands[2]));
1876 DONE;
1877 }
1878 )
1880 (define_expand "vec_pack_trunc_v2df"
1881 [(set (match_operand:V4SF 0 "register_operand")
1882 (vec_concat:V4SF
1883 (float_truncate:V2SF
1884 (match_operand:V2DF 1 "register_operand"))
1885 (float_truncate:V2SF
1886 (match_operand:V2DF 2 "register_operand"))
1887 ))]
1888 "TARGET_SIMD"
1889 {
1890 rtx tmp = gen_reg_rtx (V2SFmode);
1891 int lo = BYTES_BIG_ENDIAN ? 2 : 1;
1892 int hi = BYTES_BIG_ENDIAN ? 1 : 2;
1894 emit_insn (gen_aarch64_float_truncate_lo_v2sf (tmp, operands[lo]));
1895 emit_insn (gen_aarch64_float_truncate_hi_v4sf (operands[0],
1896 tmp, operands[hi]));
1897 DONE;
1898 }
1899 )
1901 (define_expand "vec_pack_trunc_df"
1902 [(set (match_operand:V2SF 0 "register_operand")
1903 (vec_concat:V2SF
1904 (float_truncate:SF
1905 (match_operand:DF 1 "register_operand"))
1906 (float_truncate:SF
1907 (match_operand:DF 2 "register_operand"))
1908 ))]
1909 "TARGET_SIMD"
1910 {
1911 rtx tmp = gen_reg_rtx (V2SFmode);
1912 int lo = BYTES_BIG_ENDIAN ? 2 : 1;
1913 int hi = BYTES_BIG_ENDIAN ? 1 : 2;
1915 emit_insn (gen_move_lo_quad_v2df (tmp, operands[lo]));
1916 emit_insn (gen_move_hi_quad_v2df (tmp, operands[hi]));
1917 emit_insn (gen_aarch64_float_truncate_lo_v2sf (operands[0], tmp));
1918 DONE;
1919 }
1920 )
1922 ;; FP Max/Min
1923 ;; Max/Min are introduced by idiom recognition by GCC's mid-end. An
1924 ;; expression like:
1925 ;; a = (b < c) ? b : c;
1926 ;; is idiom-matched as MIN_EXPR<b,c> only if -ffinite-math-only is enabled
1927 ;; either explicitly or indirectly via -ffast-math.
1928 ;;
1929 ;; MIN_EXPR and MAX_EXPR eventually map to 'smin' and 'smax' in RTL.
1930 ;; The 'smax' and 'smin' RTL standard pattern names do not specify which
1931 ;; operand will be returned when both operands are zero (i.e. they may not
1932 ;; honour signed zeroes), or when either operand is NaN. Therefore GCC
1933 ;; only introduces MIN_EXPR/MAX_EXPR in fast math mode or when not honouring
1934 ;; NaNs.
1936 (define_insn "<su><maxmin><mode>3"
1937 [(set (match_operand:VDQF 0 "register_operand" "=w")
1938 (FMAXMIN:VDQF (match_operand:VDQF 1 "register_operand" "w")
1939 (match_operand:VDQF 2 "register_operand" "w")))]
1940 "TARGET_SIMD"
1941 "f<maxmin>nm\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1942 [(set_attr "type" "neon_fp_minmax_<Vetype><q>")]
1943 )
1945 (define_insn "<maxmin_uns><mode>3"
1946 [(set (match_operand:VDQF 0 "register_operand" "=w")
1947 (unspec:VDQF [(match_operand:VDQF 1 "register_operand" "w")
1948 (match_operand:VDQF 2 "register_operand" "w")]
1949 FMAXMIN_UNS))]
1950 "TARGET_SIMD"
1951 "<maxmin_uns_op>\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1952 [(set_attr "type" "neon_fp_minmax_<Vetype><q>")]
1953 )
1955 ;; Auto-vectorized forms for the IEEE-754 fmax()/fmin() functions
1956 (define_insn "<fmaxmin><mode>3"
1957 [(set (match_operand:VDQF 0 "register_operand" "=w")
1958 (unspec:VDQF [(match_operand:VDQF 1 "register_operand" "w")
1959 (match_operand:VDQF 2 "register_operand" "w")]
1960 FMAXMIN))]
1961 "TARGET_SIMD"
1962 "<fmaxmin_op>\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1963 [(set_attr "type" "neon_fp_minmax_<Vetype><q>")]
1964 )
1966 ;; 'across lanes' add.
1968 (define_expand "reduc_plus_scal_<mode>"
1969 [(match_operand:<VEL> 0 "register_operand" "=w")
1970 (unspec:VDQ_I [(match_operand:VDQ_I 1 "register_operand" "w")]
1971 UNSPEC_ADDV)]
1972 "TARGET_SIMD"
1973 {
1974 rtx elt = GEN_INT (ENDIAN_LANE_N (<MODE>mode, 0));
1975 rtx scratch = gen_reg_rtx (<MODE>mode);
1976 emit_insn (gen_aarch64_reduc_plus_internal<mode> (scratch, operands[1]));
1977 emit_insn (gen_aarch64_get_lane<mode> (operands[0], scratch, elt));
1978 DONE;
1979 }
1980 )
1982 (define_expand "reduc_plus_scal_<mode>"
1983 [(match_operand:<VEL> 0 "register_operand" "=w")
1984 (match_operand:V2F 1 "register_operand" "w")]
1985 "TARGET_SIMD"
1986 {
1987 rtx elt = GEN_INT (ENDIAN_LANE_N (<MODE>mode, 0));
1988 rtx scratch = gen_reg_rtx (<MODE>mode);
1989 emit_insn (gen_aarch64_reduc_plus_internal<mode> (scratch, operands[1]));
1990 emit_insn (gen_aarch64_get_lane<mode> (operands[0], scratch, elt));
1991 DONE;
1992 }
1993 )
1995 (define_insn "aarch64_reduc_plus_internal<mode>"
1996 [(set (match_operand:VDQV 0 "register_operand" "=w")
1997 (unspec:VDQV [(match_operand:VDQV 1 "register_operand" "w")]
1998 UNSPEC_ADDV))]
1999 "TARGET_SIMD"
2000 "add<VDQV:vp>\\t%<Vetype>0, %1.<Vtype>"
2001 [(set_attr "type" "neon_reduc_add<q>")]
2002 )
2004 (define_insn "aarch64_reduc_plus_internalv2si"
2005 [(set (match_operand:V2SI 0 "register_operand" "=w")
2006 (unspec:V2SI [(match_operand:V2SI 1 "register_operand" "w")]
2007 UNSPEC_ADDV))]
2008 "TARGET_SIMD"
2009 "addp\\t%0.2s, %1.2s, %1.2s"
2010 [(set_attr "type" "neon_reduc_add")]
2011 )
2013 (define_insn "aarch64_reduc_plus_internal<mode>"
2014 [(set (match_operand:V2F 0 "register_operand" "=w")
2015 (unspec:V2F [(match_operand:V2F 1 "register_operand" "w")]
2016 UNSPEC_FADDV))]
2017 "TARGET_SIMD"
2018 "faddp\\t%<Vetype>0, %1.<Vtype>"
2019 [(set_attr "type" "neon_fp_reduc_add_<Vetype><q>")]
2020 )
2022 (define_insn "aarch64_addpv4sf"
2023 [(set (match_operand:V4SF 0 "register_operand" "=w")
2024 (unspec:V4SF [(match_operand:V4SF 1 "register_operand" "w")]
2025 UNSPEC_FADDV))]
2026 "TARGET_SIMD"
2027 "faddp\\t%0.4s, %1.4s, %1.4s"
2028 [(set_attr "type" "neon_fp_reduc_add_s_q")]
2029 )
2031 (define_expand "reduc_plus_scal_v4sf"
2032 [(set (match_operand:SF 0 "register_operand")
2033 (unspec:V4SF [(match_operand:V4SF 1 "register_operand")]
2034 UNSPEC_FADDV))]
2035 "TARGET_SIMD"
2036 {
2037 rtx elt = GEN_INT (ENDIAN_LANE_N (V4SFmode, 0));
2038 rtx scratch = gen_reg_rtx (V4SFmode);
2039 emit_insn (gen_aarch64_addpv4sf (scratch, operands[1]));
2040 emit_insn (gen_aarch64_addpv4sf (scratch, scratch));
2041 emit_insn (gen_aarch64_get_lanev4sf (operands[0], scratch, elt));
2042 DONE;
2043 })
2045 (define_insn "clrsb<mode>2"
2046 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
2047 (clrsb:VDQ_BHSI (match_operand:VDQ_BHSI 1 "register_operand" "w")))]
2048 "TARGET_SIMD"
2049 "cls\\t%0.<Vtype>, %1.<Vtype>"
2050 [(set_attr "type" "neon_cls<q>")]
2051 )
2053 (define_insn "clz<mode>2"
2054 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
2055 (clz:VDQ_BHSI (match_operand:VDQ_BHSI 1 "register_operand" "w")))]
2056 "TARGET_SIMD"
2057 "clz\\t%0.<Vtype>, %1.<Vtype>"
2058 [(set_attr "type" "neon_cls<q>")]
2059 )
2061 (define_insn "popcount<mode>2"
2062 [(set (match_operand:VB 0 "register_operand" "=w")
2063 (popcount:VB (match_operand:VB 1 "register_operand" "w")))]
2064 "TARGET_SIMD"
2065 "cnt\\t%0.<Vbtype>, %1.<Vbtype>"
2066 [(set_attr "type" "neon_cnt<q>")]
2067 )
2069 ;; 'across lanes' max and min ops.
2071 ;; Template for outputting a scalar, so we can create __builtins which can be
2072 ;; gimple_fold'd to the REDUC_(MAX|MIN)_EXPR tree code. (This is FP smax/smin).
2073 (define_expand "reduc_<maxmin_uns>_scal_<mode>"
2074 [(match_operand:<VEL> 0 "register_operand")
2075 (unspec:VDQF [(match_operand:VDQF 1 "register_operand")]
2076 FMAXMINV)]
2077 "TARGET_SIMD"
2078 {
2079 rtx elt = GEN_INT (ENDIAN_LANE_N (<MODE>mode, 0));
2080 rtx scratch = gen_reg_rtx (<MODE>mode);
2081 emit_insn (gen_aarch64_reduc_<maxmin_uns>_internal<mode> (scratch,
2082 operands[1]));
2083 emit_insn (gen_aarch64_get_lane<mode> (operands[0], scratch, elt));
2084 DONE;
2085 }
2086 )
2088 ;; Likewise for integer cases, signed and unsigned.
2089 (define_expand "reduc_<maxmin_uns>_scal_<mode>"
2090 [(match_operand:<VEL> 0 "register_operand")
2091 (unspec:VDQ_BHSI [(match_operand:VDQ_BHSI 1 "register_operand")]
2092 MAXMINV)]
2093 "TARGET_SIMD"
2094 {
2095 rtx elt = GEN_INT (ENDIAN_LANE_N (<MODE>mode, 0));
2096 rtx scratch = gen_reg_rtx (<MODE>mode);
2097 emit_insn (gen_aarch64_reduc_<maxmin_uns>_internal<mode> (scratch,
2098 operands[1]));
2099 emit_insn (gen_aarch64_get_lane<mode> (operands[0], scratch, elt));
2100 DONE;
2101 }
2102 )
2104 (define_insn "aarch64_reduc_<maxmin_uns>_internal<mode>"
2105 [(set (match_operand:VDQV_S 0 "register_operand" "=w")
2106 (unspec:VDQV_S [(match_operand:VDQV_S 1 "register_operand" "w")]
2107 MAXMINV))]
2108 "TARGET_SIMD"
2109 "<maxmin_uns_op>v\\t%<Vetype>0, %1.<Vtype>"
2110 [(set_attr "type" "neon_reduc_minmax<q>")]
2111 )
2113 (define_insn "aarch64_reduc_<maxmin_uns>_internalv2si"
2114 [(set (match_operand:V2SI 0 "register_operand" "=w")
2115 (unspec:V2SI [(match_operand:V2SI 1 "register_operand" "w")]
2116 MAXMINV))]
2117 "TARGET_SIMD"
2118 "<maxmin_uns_op>p\\t%0.2s, %1.2s, %1.2s"
2119 [(set_attr "type" "neon_reduc_minmax")]
2120 )
2122 (define_insn "aarch64_reduc_<maxmin_uns>_internal<mode>"
2123 [(set (match_operand:VDQF 0 "register_operand" "=w")
2124 (unspec:VDQF [(match_operand:VDQF 1 "register_operand" "w")]
2125 FMAXMINV))]
2126 "TARGET_SIMD"
2127 "<maxmin_uns_op><vp>\\t%<Vetype>0, %1.<Vtype>"
2128 [(set_attr "type" "neon_fp_reduc_minmax_<Vetype><q>")]
2129 )
2131 ;; aarch64_simd_bsl may compile to any of bsl/bif/bit depending on register
2132 ;; allocation.
2133 ;; Operand 1 is the mask, operands 2 and 3 are the bitfields from which
2134 ;; to select.
2135 ;;
2136 ;; Thus our BSL is of the form:
2137 ;; op0 = bsl (mask, op2, op3)
2138 ;; We can use any of:
2139 ;;
2140 ;; if (op0 = mask)
2141 ;; bsl mask, op1, op2
2142 ;; if (op0 = op1) (so 1-bits in mask choose bits from op2, else op0)
2143 ;; bit op0, op2, mask
2144 ;; if (op0 = op2) (so 0-bits in mask choose bits from op1, else op0)
2145 ;; bif op0, op1, mask
2146 ;;
2147 ;; This pattern is expanded to by the aarch64_simd_bsl<mode> expander.
2148 ;; Some forms of straight-line code may generate the equivalent form
2149 ;; in *aarch64_simd_bsl<mode>_alt.
2151 (define_insn "aarch64_simd_bsl<mode>_internal"
2152 [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w,w,w")
2153 (xor:VSDQ_I_DI
2154 (and:VSDQ_I_DI
2155 (xor:VSDQ_I_DI
2156 (match_operand:<V_cmp_result> 3 "register_operand" "w,0,w")
2157 (match_operand:VSDQ_I_DI 2 "register_operand" "w,w,0"))
2158 (match_operand:VSDQ_I_DI 1 "register_operand" "0,w,w"))
2159 (match_dup:<V_cmp_result> 3)
2160 ))]
2161 "TARGET_SIMD"
2162 "@
2163 bsl\\t%0.<Vbtype>, %2.<Vbtype>, %3.<Vbtype>
2164 bit\\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>
2165 bif\\t%0.<Vbtype>, %3.<Vbtype>, %1.<Vbtype>"
2166 [(set_attr "type" "neon_bsl<q>")]
2167 )
2169 ;; We need this form in addition to the above pattern to match the case
2170 ;; when combine tries merging three insns such that the second operand of
2171 ;; the outer XOR matches the second operand of the inner XOR rather than
2172 ;; the first. The two are equivalent but since recog doesn't try all
2173 ;; permutations of commutative operations, we have to have a separate pattern.
2175 (define_insn "*aarch64_simd_bsl<mode>_alt"
2176 [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w,w,w")
2177 (xor:VSDQ_I_DI
2178 (and:VSDQ_I_DI
2179 (xor:VSDQ_I_DI
2180 (match_operand:VSDQ_I_DI 3 "register_operand" "w,w,0")
2181 (match_operand:VSDQ_I_DI 2 "register_operand" "w,0,w"))
2182 (match_operand:VSDQ_I_DI 1 "register_operand" "0,w,w"))
2183 (match_dup:VSDQ_I_DI 2)))]
2184 "TARGET_SIMD"
2185 "@
2186 bsl\\t%0.<Vbtype>, %3.<Vbtype>, %2.<Vbtype>
2187 bit\\t%0.<Vbtype>, %3.<Vbtype>, %1.<Vbtype>
2188 bif\\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>"
2189 [(set_attr "type" "neon_bsl<q>")]
2190 )
2192 (define_expand "aarch64_simd_bsl<mode>"
2193 [(match_operand:VALLDIF 0 "register_operand")
2194 (match_operand:<V_cmp_result> 1 "register_operand")
2195 (match_operand:VALLDIF 2 "register_operand")
2196 (match_operand:VALLDIF 3 "register_operand")]
2197 "TARGET_SIMD"
2198 {
2199 /* We can't alias operands together if they have different modes. */
2200 rtx tmp = operands[0];
2201 if (FLOAT_MODE_P (<MODE>mode))
2202 {
2203 operands[2] = gen_lowpart (<V_cmp_result>mode, operands[2]);
2204 operands[3] = gen_lowpart (<V_cmp_result>mode, operands[3]);
2205 tmp = gen_reg_rtx (<V_cmp_result>mode);
2206 }
2207 operands[1] = gen_lowpart (<V_cmp_result>mode, operands[1]);
2208 emit_insn (gen_aarch64_simd_bsl<v_cmp_result>_internal (tmp,
2209 operands[1],
2210 operands[2],
2211 operands[3]));
2212 if (tmp != operands[0])
2213 emit_move_insn (operands[0], gen_lowpart (<MODE>mode, tmp));
2215 DONE;
2216 })
2218 (define_expand "aarch64_vcond_internal<mode><mode>"
2219 [(set (match_operand:VSDQ_I_DI 0 "register_operand")
2220 (if_then_else:VSDQ_I_DI
2221 (match_operator 3 "comparison_operator"
2222 [(match_operand:VSDQ_I_DI 4 "register_operand")
2223 (match_operand:VSDQ_I_DI 5 "nonmemory_operand")])
2224 (match_operand:VSDQ_I_DI 1 "nonmemory_operand")
2225 (match_operand:VSDQ_I_DI 2 "nonmemory_operand")))]
2226 "TARGET_SIMD"
2227 {
2228 rtx op1 = operands[1];
2229 rtx op2 = operands[2];
2230 rtx mask = gen_reg_rtx (<MODE>mode);
2231 enum rtx_code code = GET_CODE (operands[3]);
2233 /* Switching OP1 and OP2 is necessary for NE (to output a cmeq insn),
2234 and desirable for other comparisons if it results in FOO ? -1 : 0
2235 (this allows direct use of the comparison result without a bsl). */
2236 if (code == NE
2237 || (code != EQ
2238 && op1 == CONST0_RTX (<V_cmp_result>mode)
2239 && op2 == CONSTM1_RTX (<V_cmp_result>mode)))
2240 {
2241 op1 = operands[2];
2242 op2 = operands[1];
2243 switch (code)
2244 {
2245 case LE: code = GT; break;
2246 case LT: code = GE; break;
2247 case GE: code = LT; break;
2248 case GT: code = LE; break;
2249 /* No case EQ. */
2250 case NE: code = EQ; break;
2251 case LTU: code = GEU; break;
2252 case LEU: code = GTU; break;
2253 case GTU: code = LEU; break;
2254 case GEU: code = LTU; break;
2255 default: gcc_unreachable ();
2256 }
2257 }
2259 /* Make sure we can handle the last operand. */
2260 switch (code)
2261 {
2262 case NE:
2263 /* Normalized to EQ above. */
2264 gcc_unreachable ();
2266 case LE:
2267 case LT:
2268 case GE:
2269 case GT:
2270 case EQ:
2271 /* These instructions have a form taking an immediate zero. */
2272 if (operands[5] == CONST0_RTX (<MODE>mode))
2273 break;
2274 /* Fall through, as may need to load into register. */
2275 default:
2276 if (!REG_P (operands[5]))
2277 operands[5] = force_reg (<MODE>mode, operands[5]);
2278 break;
2279 }
2281 switch (code)
2282 {
2283 case LT:
2284 emit_insn (gen_aarch64_cmlt<mode> (mask, operands[4], operands[5]));
2285 break;
2287 case GE:
2288 emit_insn (gen_aarch64_cmge<mode> (mask, operands[4], operands[5]));
2289 break;
2291 case LE:
2292 emit_insn (gen_aarch64_cmle<mode> (mask, operands[4], operands[5]));
2293 break;
2295 case GT:
2296 emit_insn (gen_aarch64_cmgt<mode> (mask, operands[4], operands[5]));
2297 break;
2299 case LTU:
2300 emit_insn (gen_aarch64_cmgtu<mode> (mask, operands[5], operands[4]));
2301 break;
2303 case GEU:
2304 emit_insn (gen_aarch64_cmgeu<mode> (mask, operands[4], operands[5]));
2305 break;
2307 case LEU:
2308 emit_insn (gen_aarch64_cmgeu<mode> (mask, operands[5], operands[4]));
2309 break;
2311 case GTU:
2312 emit_insn (gen_aarch64_cmgtu<mode> (mask, operands[4], operands[5]));
2313 break;
2315 /* NE has been normalized to EQ above. */
2316 case EQ:
2317 emit_insn (gen_aarch64_cmeq<mode> (mask, operands[4], operands[5]));
2318 break;
2320 default:
2321 gcc_unreachable ();
2322 }
2324 /* If we have (a = (b CMP c) ? -1 : 0);
2325 Then we can simply move the generated mask. */
2327 if (op1 == CONSTM1_RTX (<V_cmp_result>mode)
2328 && op2 == CONST0_RTX (<V_cmp_result>mode))
2329 emit_move_insn (operands[0], mask);
2330 else
2331 {
2332 if (!REG_P (op1))
2333 op1 = force_reg (<MODE>mode, op1);
2334 if (!REG_P (op2))
2335 op2 = force_reg (<MODE>mode, op2);
2336 emit_insn (gen_aarch64_simd_bsl<mode> (operands[0], mask,
2337 op1, op2));
2338 }
2340 DONE;
2341 })
2343 (define_expand "aarch64_vcond_internal<VDQF_COND:mode><VDQF:mode>"
2344 [(set (match_operand:VDQF_COND 0 "register_operand")
2345 (if_then_else:VDQF
2346 (match_operator 3 "comparison_operator"
2347 [(match_operand:VDQF 4 "register_operand")
2348 (match_operand:VDQF 5 "nonmemory_operand")])
2349 (match_operand:VDQF_COND 1 "nonmemory_operand")
2350 (match_operand:VDQF_COND 2 "nonmemory_operand")))]
2351 "TARGET_SIMD"
2352 {
2353 int inverse = 0;
2354 int use_zero_form = 0;
2355 int swap_bsl_operands = 0;
2356 rtx op1 = operands[1];
2357 rtx op2 = operands[2];
2358 rtx mask = gen_reg_rtx (<VDQF_COND:V_cmp_result>mode);
2359 rtx tmp = gen_reg_rtx (<VDQF_COND:V_cmp_result>mode);
2361 rtx (*base_comparison) (rtx, rtx, rtx);
2362 rtx (*complimentary_comparison) (rtx, rtx, rtx);
2364 switch (GET_CODE (operands[3]))
2365 {
2366 case GE:
2367 case GT:
2368 case LE:
2369 case LT:
2370 case EQ:
2371 if (operands[5] == CONST0_RTX (<MODE>mode))
2372 {
2373 use_zero_form = 1;
2374 break;
2375 }
2376 /* Fall through. */
2377 default:
2378 if (!REG_P (operands[5]))
2379 operands[5] = force_reg (<VDQF:MODE>mode, operands[5]);
2380 }
2382 switch (GET_CODE (operands[3]))
2383 {
2384 case LT:
2385 case UNLT:
2386 inverse = 1;
2387 /* Fall through. */
2388 case GE:
2389 case UNGE:
2390 case ORDERED:
2391 case UNORDERED:
2392 base_comparison = gen_aarch64_cmge<VDQF:mode>;
2393 complimentary_comparison = gen_aarch64_cmgt<VDQF:mode>;
2394 break;
2395 case LE:
2396 case UNLE:
2397 inverse = 1;
2398 /* Fall through. */
2399 case GT:
2400 case UNGT:
2401 base_comparison = gen_aarch64_cmgt<VDQF:mode>;
2402 complimentary_comparison = gen_aarch64_cmge<VDQF:mode>;
2403 break;
2404 case EQ:
2405 case NE:
2406 case UNEQ:
2407 base_comparison = gen_aarch64_cmeq<VDQF:mode>;
2408 complimentary_comparison = gen_aarch64_cmeq<VDQF:mode>;
2409 break;
2410 default:
2411 gcc_unreachable ();
2412 }
2414 switch (GET_CODE (operands[3]))
2415 {
2416 case LT:
2417 case LE:
2418 case GT:
2419 case GE:
2420 case EQ:
2421 /* The easy case. Here we emit one of FCMGE, FCMGT or FCMEQ.
2422 As a LT b <=> b GE a && a LE b <=> b GT a. Our transformations are:
2423 a GE b -> a GE b
2424 a GT b -> a GT b
2425 a LE b -> b GE a
2426 a LT b -> b GT a
2427 a EQ b -> a EQ b
2428 Note that there also exist direct comparison against 0 forms,
2429 so catch those as a special case. */
2430 if (use_zero_form)
2431 {
2432 inverse = 0;
2433 switch (GET_CODE (operands[3]))
2434 {
2435 case LT:
2436 base_comparison = gen_aarch64_cmlt<VDQF:mode>;
2437 break;
2438 case LE:
2439 base_comparison = gen_aarch64_cmle<VDQF:mode>;
2440 break;
2441 default:
2442 /* Do nothing, other zero form cases already have the correct
2443 base_comparison. */
2444 break;
2445 }
2446 }
2448 if (!inverse)
2449 emit_insn (base_comparison (mask, operands[4], operands[5]));
2450 else
2451 emit_insn (complimentary_comparison (mask, operands[5], operands[4]));
2452 break;
2453 case UNLT:
2454 case UNLE:
2455 case UNGT:
2456 case UNGE:
2457 case NE:
2458 /* FCM returns false for lanes which are unordered, so if we use
2459 the inverse of the comparison we actually want to emit, then
2460 swap the operands to BSL, we will end up with the correct result.
2461 Note that a NE NaN and NaN NE b are true for all a, b.
2463 Our transformations are:
2464 a GE b -> !(b GT a)
2465 a GT b -> !(b GE a)
2466 a LE b -> !(a GT b)
2467 a LT b -> !(a GE b)
2468 a NE b -> !(a EQ b) */
2470 if (inverse)
2471 emit_insn (base_comparison (mask, operands[4], operands[5]));
2472 else
2473 emit_insn (complimentary_comparison (mask, operands[5], operands[4]));
2475 swap_bsl_operands = 1;
2476 break;
2477 case UNEQ:
2478 /* We check (a > b || b > a). combining these comparisons give us
2479 true iff !(a != b && a ORDERED b), swapping the operands to BSL
2480 will then give us (a == b || a UNORDERED b) as intended. */
2482 emit_insn (gen_aarch64_cmgt<VDQF:mode> (mask, operands[4], operands[5]));
2483 emit_insn (gen_aarch64_cmgt<VDQF:mode> (tmp, operands[5], operands[4]));
2484 emit_insn (gen_ior<VDQF_COND:v_cmp_result>3 (mask, mask, tmp));
2485 swap_bsl_operands = 1;
2486 break;
2487 case UNORDERED:
2488 /* Operands are ORDERED iff (a > b || b >= a).
2489 Swapping the operands to BSL will give the UNORDERED case. */
2490 swap_bsl_operands = 1;
2491 /* Fall through. */
2492 case ORDERED:
2493 emit_insn (gen_aarch64_cmgt<VDQF:mode> (tmp, operands[4], operands[5]));
2494 emit_insn (gen_aarch64_cmge<VDQF:mode> (mask, operands[5], operands[4]));
2495 emit_insn (gen_ior<VDQF_COND:v_cmp_result>3 (mask, mask, tmp));
2496 break;
2497 default:
2498 gcc_unreachable ();
2499 }
2501 if (swap_bsl_operands)
2502 {
2503 op1 = operands[2];
2504 op2 = operands[1];
2505 }
2507 /* If we have (a = (b CMP c) ? -1 : 0);
2508 Then we can simply move the generated mask. */
2510 if (op1 == CONSTM1_RTX (<VDQF_COND:V_cmp_result>mode)
2511 && op2 == CONST0_RTX (<VDQF_COND:V_cmp_result>mode))
2512 emit_move_insn (operands[0], mask);
2513 else
2514 {
2515 if (!REG_P (op1))
2516 op1 = force_reg (<VDQF_COND:MODE>mode, op1);
2517 if (!REG_P (op2))
2518 op2 = force_reg (<VDQF_COND:MODE>mode, op2);
2519 emit_insn (gen_aarch64_simd_bsl<VDQF_COND:mode> (operands[0], mask,
2520 op1, op2));
2521 }
2523 DONE;
2524 })
2526 (define_expand "vcond<mode><mode>"
2527 [(set (match_operand:VALLDI 0 "register_operand")
2528 (if_then_else:VALLDI
2529 (match_operator 3 "comparison_operator"
2530 [(match_operand:VALLDI 4 "register_operand")
2531 (match_operand:VALLDI 5 "nonmemory_operand")])
2532 (match_operand:VALLDI 1 "nonmemory_operand")
2533 (match_operand:VALLDI 2 "nonmemory_operand")))]
2534 "TARGET_SIMD"
2535 {
2536 emit_insn (gen_aarch64_vcond_internal<mode><mode> (operands[0], operands[1],
2537 operands[2], operands[3],
2538 operands[4], operands[5]));
2539 DONE;
2540 })
2542 (define_expand "vcond<v_cmp_result><mode>"
2543 [(set (match_operand:<V_cmp_result> 0 "register_operand")
2544 (if_then_else:<V_cmp_result>
2545 (match_operator 3 "comparison_operator"
2546 [(match_operand:VDQF 4 "register_operand")
2547 (match_operand:VDQF 5 "nonmemory_operand")])
2548 (match_operand:<V_cmp_result> 1 "nonmemory_operand")
2549 (match_operand:<V_cmp_result> 2 "nonmemory_operand")))]
2550 "TARGET_SIMD"
2551 {
2552 emit_insn (gen_aarch64_vcond_internal<v_cmp_result><mode> (
2553 operands[0], operands[1],
2554 operands[2], operands[3],
2555 operands[4], operands[5]));
2556 DONE;
2557 })
2559 (define_expand "vcondu<mode><mode>"
2560 [(set (match_operand:VSDQ_I_DI 0 "register_operand")
2561 (if_then_else:VSDQ_I_DI
2562 (match_operator 3 "comparison_operator"
2563 [(match_operand:VSDQ_I_DI 4 "register_operand")
2564 (match_operand:VSDQ_I_DI 5 "nonmemory_operand")])
2565 (match_operand:VSDQ_I_DI 1 "nonmemory_operand")
2566 (match_operand:VSDQ_I_DI 2 "nonmemory_operand")))]
2567 "TARGET_SIMD"
2568 {
2569 emit_insn (gen_aarch64_vcond_internal<mode><mode> (operands[0], operands[1],
2570 operands[2], operands[3],
2571 operands[4], operands[5]));
2572 DONE;
2573 })
2575 ;; Patterns for AArch64 SIMD Intrinsics.
2577 ;; Lane extraction with sign extension to general purpose register.
2578 (define_insn "*aarch64_get_lane_extend<GPI:mode><VDQQH:mode>"
2579 [(set (match_operand:GPI 0 "register_operand" "=r")
2580 (sign_extend:GPI
2581 (vec_select:<VEL>
2582 (match_operand:VDQQH 1 "register_operand" "w")
2583 (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2584 "TARGET_SIMD"
2585 {
2586 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
2587 return "smov\\t%<GPI:w>0, %1.<VDQQH:Vetype>[%2]";
2588 }
2589 [(set_attr "type" "neon_to_gp<q>")]
2590 )
2592 (define_insn "*aarch64_get_lane_zero_extendsi<mode>"
2593 [(set (match_operand:SI 0 "register_operand" "=r")
2594 (zero_extend:SI
2595 (vec_select:<VEL>
2596 (match_operand:VDQQH 1 "register_operand" "w")
2597 (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2598 "TARGET_SIMD"
2599 {
2600 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
2601 return "umov\\t%w0, %1.<Vetype>[%2]";
2602 }
2603 [(set_attr "type" "neon_to_gp<q>")]
2604 )
2606 ;; Lane extraction of a value, neither sign nor zero extension
2607 ;; is guaranteed so upper bits should be considered undefined.
2608 ;; RTL uses GCC vector extension indices throughout so flip only for assembly.
2609 (define_insn "aarch64_get_lane<mode>"
2610 [(set (match_operand:<VEL> 0 "aarch64_simd_nonimmediate_operand" "=r, w, Utv")
2611 (vec_select:<VEL>
2612 (match_operand:VALL_F16 1 "register_operand" "w, w, w")
2613 (parallel [(match_operand:SI 2 "immediate_operand" "i, i, i")])))]
2614 "TARGET_SIMD"
2615 {
2616 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
2617 switch (which_alternative)
2618 {
2619 case 0:
2620 return "umov\\t%<vwcore>0, %1.<Vetype>[%2]";
2621 case 1:
2622 return "dup\\t%<Vetype>0, %1.<Vetype>[%2]";
2623 case 2:
2624 return "st1\\t{%1.<Vetype>}[%2], %0";
2625 default:
2626 gcc_unreachable ();
2627 }
2628 }
2629 [(set_attr "type" "neon_to_gp<q>, neon_dup<q>, neon_store1_one_lane<q>")]
2630 )
2632 ;; In this insn, operand 1 should be low, and operand 2 the high part of the
2633 ;; dest vector.
2635 (define_insn "*aarch64_combinez<mode>"
2636 [(set (match_operand:<VDBL> 0 "register_operand" "=w,w,w")
2637 (vec_concat:<VDBL>
2638 (match_operand:VD_BHSI 1 "general_operand" "w,r,m")
2639 (match_operand:VD_BHSI 2 "aarch64_simd_imm_zero" "Dz,Dz,Dz")))]
2640 "TARGET_SIMD && !BYTES_BIG_ENDIAN"
2641 "@
2642 mov\\t%0.8b, %1.8b
2643 fmov\t%d0, %1
2644 ldr\\t%d0, %1"
2645 [(set_attr "type" "neon_move<q>, neon_from_gp, neon_load1_1reg")
2646 (set_attr "simd" "yes,*,yes")
2647 (set_attr "fp" "*,yes,*")]
2648 )
2650 (define_insn "*aarch64_combinez_be<mode>"
2651 [(set (match_operand:<VDBL> 0 "register_operand" "=w,w,w")
2652 (vec_concat:<VDBL>
2653 (match_operand:VD_BHSI 2 "aarch64_simd_imm_zero" "Dz,Dz,Dz")
2654 (match_operand:VD_BHSI 1 "general_operand" "w,r,m")))]
2655 "TARGET_SIMD && BYTES_BIG_ENDIAN"
2656 "@
2657 mov\\t%0.8b, %1.8b
2658 fmov\t%d0, %1
2659 ldr\\t%d0, %1"
2660 [(set_attr "type" "neon_move<q>, neon_from_gp, neon_load1_1reg")
2661 (set_attr "simd" "yes,*,yes")
2662 (set_attr "fp" "*,yes,*")]
2663 )
2665 (define_expand "aarch64_combine<mode>"
2666 [(match_operand:<VDBL> 0 "register_operand")
2667 (match_operand:VDC 1 "register_operand")
2668 (match_operand:VDC 2 "register_operand")]
2669 "TARGET_SIMD"
2670 {
2671 rtx op1, op2;
2672 if (BYTES_BIG_ENDIAN)
2673 {
2674 op1 = operands[2];
2675 op2 = operands[1];
2676 }
2677 else
2678 {
2679 op1 = operands[1];
2680 op2 = operands[2];
2681 }
2682 emit_insn (gen_aarch64_combine_internal<mode> (operands[0], op1, op2));
2683 DONE;
2684 }
2685 )
2687 (define_insn_and_split "aarch64_combine_internal<mode>"
2688 [(set (match_operand:<VDBL> 0 "register_operand" "=&w")
2689 (vec_concat:<VDBL> (match_operand:VDC 1 "register_operand" "w")
2690 (match_operand:VDC 2 "register_operand" "w")))]
2691 "TARGET_SIMD"
2692 "#"
2693 "&& reload_completed"
2694 [(const_int 0)]
2695 {
2696 if (BYTES_BIG_ENDIAN)
2697 aarch64_split_simd_combine (operands[0], operands[2], operands[1]);
2698 else
2699 aarch64_split_simd_combine (operands[0], operands[1], operands[2]);
2700 DONE;
2701 }
2702 [(set_attr "type" "multiple")]
2703 )
2705 (define_expand "aarch64_simd_combine<mode>"
2706 [(match_operand:<VDBL> 0 "register_operand")
2707 (match_operand:VDC 1 "register_operand")
2708 (match_operand:VDC 2 "register_operand")]
2709 "TARGET_SIMD"
2710 {
2711 emit_insn (gen_move_lo_quad_<Vdbl> (operands[0], operands[1]));
2712 emit_insn (gen_move_hi_quad_<Vdbl> (operands[0], operands[2]));
2713 DONE;
2714 }
2715 [(set_attr "type" "multiple")]
2716 )
2718 ;; <su><addsub>l<q>.
2720 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>l<mode>_hi_internal"
2721 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2722 (ADDSUB:<VWIDE> (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
2723 (match_operand:VQW 1 "register_operand" "w")
2724 (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))
2725 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
2726 (match_operand:VQW 2 "register_operand" "w")
2727 (match_dup 3)))))]
2728 "TARGET_SIMD"
2729 "<ANY_EXTEND:su><ADDSUB:optab>l2\t%0.<Vwtype>, %1.<Vtype>, %2.<Vtype>"
2730 [(set_attr "type" "neon_<ADDSUB:optab>_long")]
2731 )
2733 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>l<mode>_lo_internal"
2734 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2735 (ADDSUB:<VWIDE> (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
2736 (match_operand:VQW 1 "register_operand" "w")
2737 (match_operand:VQW 3 "vect_par_cnst_lo_half" "")))
2738 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
2739 (match_operand:VQW 2 "register_operand" "w")
2740 (match_dup 3)))))]
2741 "TARGET_SIMD"
2742 "<ANY_EXTEND:su><ADDSUB:optab>l\t%0.<Vwtype>, %1.<Vhalftype>, %2.<Vhalftype>"
2743 [(set_attr "type" "neon_<ADDSUB:optab>_long")]
2744 )
2747 (define_expand "aarch64_saddl2<mode>"
2748 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2749 (match_operand:VQW 1 "register_operand" "w")
2750 (match_operand:VQW 2 "register_operand" "w")]
2751 "TARGET_SIMD"
2752 {
2753 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2754 emit_insn (gen_aarch64_saddl<mode>_hi_internal (operands[0], operands[1],
2755 operands[2], p));
2756 DONE;
2757 })
2759 (define_expand "aarch64_uaddl2<mode>"
2760 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2761 (match_operand:VQW 1 "register_operand" "w")
2762 (match_operand:VQW 2 "register_operand" "w")]
2763 "TARGET_SIMD"
2764 {
2765 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2766 emit_insn (gen_aarch64_uaddl<mode>_hi_internal (operands[0], operands[1],
2767 operands[2], p));
2768 DONE;
2769 })
2771 (define_expand "aarch64_ssubl2<mode>"
2772 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2773 (match_operand:VQW 1 "register_operand" "w")
2774 (match_operand:VQW 2 "register_operand" "w")]
2775 "TARGET_SIMD"
2776 {
2777 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2778 emit_insn (gen_aarch64_ssubl<mode>_hi_internal (operands[0], operands[1],
2779 operands[2], p));
2780 DONE;
2781 })
2783 (define_expand "aarch64_usubl2<mode>"
2784 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2785 (match_operand:VQW 1 "register_operand" "w")
2786 (match_operand:VQW 2 "register_operand" "w")]
2787 "TARGET_SIMD"
2788 {
2789 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2790 emit_insn (gen_aarch64_usubl<mode>_hi_internal (operands[0], operands[1],
2791 operands[2], p));
2792 DONE;
2793 })
2795 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>l<mode>"
2796 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2797 (ADDSUB:<VWIDE> (ANY_EXTEND:<VWIDE>
2798 (match_operand:VD_BHSI 1 "register_operand" "w"))
2799 (ANY_EXTEND:<VWIDE>
2800 (match_operand:VD_BHSI 2 "register_operand" "w"))))]
2801 "TARGET_SIMD"
2802 "<ANY_EXTEND:su><ADDSUB:optab>l\t%0.<Vwtype>, %1.<Vtype>, %2.<Vtype>"
2803 [(set_attr "type" "neon_<ADDSUB:optab>_long")]
2804 )
2806 ;; <su><addsub>w<q>.
2808 (define_expand "widen_ssum<mode>3"
2809 [(set (match_operand:<VDBLW> 0 "register_operand" "")
2810 (plus:<VDBLW> (sign_extend:<VDBLW>
2811 (match_operand:VQW 1 "register_operand" ""))
2812 (match_operand:<VDBLW> 2 "register_operand" "")))]
2813 "TARGET_SIMD"
2814 {
2815 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
2816 rtx temp = gen_reg_rtx (GET_MODE (operands[0]));
2818 emit_insn (gen_aarch64_saddw<mode>_internal (temp, operands[2],
2819 operands[1], p));
2820 emit_insn (gen_aarch64_saddw2<mode> (operands[0], temp, operands[1]));
2821 DONE;
2822 }
2823 )
2825 (define_expand "widen_ssum<mode>3"
2826 [(set (match_operand:<VWIDE> 0 "register_operand" "")
2827 (plus:<VWIDE> (sign_extend:<VWIDE>
2828 (match_operand:VD_BHSI 1 "register_operand" ""))
2829 (match_operand:<VWIDE> 2 "register_operand" "")))]
2830 "TARGET_SIMD"
2831 {
2832 emit_insn (gen_aarch64_saddw<mode> (operands[0], operands[2], operands[1]));
2833 DONE;
2834 })
2836 (define_expand "widen_usum<mode>3"
2837 [(set (match_operand:<VDBLW> 0 "register_operand" "")
2838 (plus:<VDBLW> (zero_extend:<VDBLW>
2839 (match_operand:VQW 1 "register_operand" ""))
2840 (match_operand:<VDBLW> 2 "register_operand" "")))]
2841 "TARGET_SIMD"
2842 {
2843 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
2844 rtx temp = gen_reg_rtx (GET_MODE (operands[0]));
2846 emit_insn (gen_aarch64_uaddw<mode>_internal (temp, operands[2],
2847 operands[1], p));
2848 emit_insn (gen_aarch64_uaddw2<mode> (operands[0], temp, operands[1]));
2849 DONE;
2850 }
2851 )
2853 (define_expand "widen_usum<mode>3"
2854 [(set (match_operand:<VWIDE> 0 "register_operand" "")
2855 (plus:<VWIDE> (zero_extend:<VWIDE>
2856 (match_operand:VD_BHSI 1 "register_operand" ""))
2857 (match_operand:<VWIDE> 2 "register_operand" "")))]
2858 "TARGET_SIMD"
2859 {
2860 emit_insn (gen_aarch64_uaddw<mode> (operands[0], operands[2], operands[1]));
2861 DONE;
2862 })
2864 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>w<mode>"
2865 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2866 (ADDSUB:<VWIDE> (match_operand:<VWIDE> 1 "register_operand" "w")
2867 (ANY_EXTEND:<VWIDE>
2868 (match_operand:VD_BHSI 2 "register_operand" "w"))))]
2869 "TARGET_SIMD"
2870 "<ANY_EXTEND:su><ADDSUB:optab>w\\t%0.<Vwtype>, %1.<Vwtype>, %2.<Vtype>"
2871 [(set_attr "type" "neon_<ADDSUB:optab>_widen")]
2872 )
2874 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>w<mode>_internal"
2875 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2876 (ADDSUB:<VWIDE> (match_operand:<VWIDE> 1 "register_operand" "w")
2877 (ANY_EXTEND:<VWIDE>
2878 (vec_select:<VHALF>
2879 (match_operand:VQW 2 "register_operand" "w")
2880 (match_operand:VQW 3 "vect_par_cnst_lo_half" "")))))]
2881 "TARGET_SIMD"
2882 "<ANY_EXTEND:su><ADDSUB:optab>w\\t%0.<Vwtype>, %1.<Vwtype>, %2.<Vhalftype>"
2883 [(set_attr "type" "neon_<ADDSUB:optab>_widen")]
2884 )
2886 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>w2<mode>_internal"
2887 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2888 (ADDSUB:<VWIDE> (match_operand:<VWIDE> 1 "register_operand" "w")
2889 (ANY_EXTEND:<VWIDE>
2890 (vec_select:<VHALF>
2891 (match_operand:VQW 2 "register_operand" "w")
2892 (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))))]
2893 "TARGET_SIMD"
2894 "<ANY_EXTEND:su><ADDSUB:optab>w2\\t%0.<Vwtype>, %1.<Vwtype>, %2.<Vtype>"
2895 [(set_attr "type" "neon_<ADDSUB:optab>_widen")]
2896 )
2898 (define_expand "aarch64_saddw2<mode>"
2899 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2900 (match_operand:<VWIDE> 1 "register_operand" "w")
2901 (match_operand:VQW 2 "register_operand" "w")]
2902 "TARGET_SIMD"
2903 {
2904 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2905 emit_insn (gen_aarch64_saddw2<mode>_internal (operands[0], operands[1],
2906 operands[2], p));
2907 DONE;
2908 })
2910 (define_expand "aarch64_uaddw2<mode>"
2911 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2912 (match_operand:<VWIDE> 1 "register_operand" "w")
2913 (match_operand:VQW 2 "register_operand" "w")]
2914 "TARGET_SIMD"
2915 {
2916 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2917 emit_insn (gen_aarch64_uaddw2<mode>_internal (operands[0], operands[1],
2918 operands[2], p));
2919 DONE;
2920 })
2923 (define_expand "aarch64_ssubw2<mode>"
2924 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2925 (match_operand:<VWIDE> 1 "register_operand" "w")
2926 (match_operand:VQW 2 "register_operand" "w")]
2927 "TARGET_SIMD"
2928 {
2929 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2930 emit_insn (gen_aarch64_ssubw2<mode>_internal (operands[0], operands[1],
2931 operands[2], p));
2932 DONE;
2933 })
2935 (define_expand "aarch64_usubw2<mode>"
2936 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2937 (match_operand:<VWIDE> 1 "register_operand" "w")
2938 (match_operand:VQW 2 "register_operand" "w")]
2939 "TARGET_SIMD"
2940 {
2941 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2942 emit_insn (gen_aarch64_usubw2<mode>_internal (operands[0], operands[1],
2943 operands[2], p));
2944 DONE;
2945 })
2947 ;; <su><r>h<addsub>.
2949 (define_insn "aarch64_<sur>h<addsub><mode>"
2950 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
2951 (unspec:VDQ_BHSI [(match_operand:VDQ_BHSI 1 "register_operand" "w")
2952 (match_operand:VDQ_BHSI 2 "register_operand" "w")]
2953 HADDSUB))]
2954 "TARGET_SIMD"
2955 "<sur>h<addsub>\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
2956 [(set_attr "type" "neon_<addsub>_halve<q>")]
2957 )
2959 ;; <r><addsub>hn<q>.
2961 (define_insn "aarch64_<sur><addsub>hn<mode>"
2962 [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
2963 (unspec:<VNARROWQ> [(match_operand:VQN 1 "register_operand" "w")
2964 (match_operand:VQN 2 "register_operand" "w")]
2965 ADDSUBHN))]
2966 "TARGET_SIMD"
2967 "<sur><addsub>hn\\t%0.<Vntype>, %1.<Vtype>, %2.<Vtype>"
2968 [(set_attr "type" "neon_<addsub>_halve_narrow_q")]
2969 )
2971 (define_insn "aarch64_<sur><addsub>hn2<mode>"
2972 [(set (match_operand:<VNARROWQ2> 0 "register_operand" "=w")
2973 (unspec:<VNARROWQ2> [(match_operand:<VNARROWQ> 1 "register_operand" "0")
2974 (match_operand:VQN 2 "register_operand" "w")
2975 (match_operand:VQN 3 "register_operand" "w")]
2976 ADDSUBHN2))]
2977 "TARGET_SIMD"
2978 "<sur><addsub>hn2\\t%0.<V2ntype>, %2.<Vtype>, %3.<Vtype>"
2979 [(set_attr "type" "neon_<addsub>_halve_narrow_q")]
2980 )
2982 ;; pmul.
2984 (define_insn "aarch64_pmul<mode>"
2985 [(set (match_operand:VB 0 "register_operand" "=w")
2986 (unspec:VB [(match_operand:VB 1 "register_operand" "w")
2987 (match_operand:VB 2 "register_operand" "w")]
2988 UNSPEC_PMUL))]
2989 "TARGET_SIMD"
2990 "pmul\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
2991 [(set_attr "type" "neon_mul_<Vetype><q>")]
2992 )
2994 ;; fmulx.
2996 (define_insn "aarch64_fmulx<mode>"
2997 [(set (match_operand:VALLF 0 "register_operand" "=w")
2998 (unspec:VALLF [(match_operand:VALLF 1 "register_operand" "w")
2999 (match_operand:VALLF 2 "register_operand" "w")]
3000 UNSPEC_FMULX))]
3001 "TARGET_SIMD"
3002 "fmulx\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
3003 [(set_attr "type" "neon_fp_mul_<Vetype>")]
3004 )
3006 ;; vmulxq_lane_f32, and vmulx_laneq_f32
3008 (define_insn "*aarch64_mulx_elt_<vswap_width_name><mode>"
3009 [(set (match_operand:VDQSF 0 "register_operand" "=w")
3010 (unspec:VDQSF
3011 [(match_operand:VDQSF 1 "register_operand" "w")
3012 (vec_duplicate:VDQSF
3013 (vec_select:<VEL>
3014 (match_operand:<VSWAP_WIDTH> 2 "register_operand" "w")
3015 (parallel [(match_operand:SI 3 "immediate_operand" "i")])))]
3016 UNSPEC_FMULX))]
3017 "TARGET_SIMD"
3018 {
3019 operands[3] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
3020 INTVAL (operands[3])));
3021 return "fmulx\t%<v>0<Vmtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3022 }
3023 [(set_attr "type" "neon_fp_mul_<Vetype>_scalar<q>")]
3024 )
3026 ;; vmulxq_laneq_f32, vmulxq_laneq_f64, vmulx_lane_f32
3028 (define_insn "*aarch64_mulx_elt<mode>"
3029 [(set (match_operand:VDQF 0 "register_operand" "=w")
3030 (unspec:VDQF
3031 [(match_operand:VDQF 1 "register_operand" "w")
3032 (vec_duplicate:VDQF
3033 (vec_select:<VEL>
3034 (match_operand:VDQF 2 "register_operand" "w")
3035 (parallel [(match_operand:SI 3 "immediate_operand" "i")])))]
3036 UNSPEC_FMULX))]
3037 "TARGET_SIMD"
3038 {
3039 operands[3] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[3])));
3040 return "fmulx\t%<v>0<Vmtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3041 }
3042 [(set_attr "type" "neon_fp_mul_<Vetype><q>")]
3043 )
3045 ;; vmulxq_lane_f64
3047 (define_insn "*aarch64_mulx_elt_to_64v2df"
3048 [(set (match_operand:V2DF 0 "register_operand" "=w")
3049 (unspec:V2DF
3050 [(match_operand:V2DF 1 "register_operand" "w")
3051 (vec_duplicate:V2DF
3052 (match_operand:DF 2 "register_operand" "w"))]
3053 UNSPEC_FMULX))]
3054 "TARGET_SIMD"
3055 {
3056 return "fmulx\t%0.2d, %1.2d, %2.d[0]";
3057 }
3058 [(set_attr "type" "neon_fp_mul_d_scalar_q")]
3059 )
3061 ;; vmulxs_lane_f32, vmulxs_laneq_f32
3062 ;; vmulxd_lane_f64 == vmulx_lane_f64
3063 ;; vmulxd_laneq_f64 == vmulx_laneq_f64
3065 (define_insn "*aarch64_vgetfmulx<mode>"
3066 [(set (match_operand:<VEL> 0 "register_operand" "=w")
3067 (unspec:<VEL>
3068 [(match_operand:<VEL> 1 "register_operand" "w")
3069 (vec_select:<VEL>
3070 (match_operand:VDQF_DF 2 "register_operand" "w")
3071 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))]
3072 UNSPEC_FMULX))]
3073 "TARGET_SIMD"
3074 {
3075 operands[3] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[3])));
3076 return "fmulx\t%<Vetype>0, %<Vetype>1, %2.<Vetype>[%3]";
3077 }
3078 [(set_attr "type" "fmul<Vetype>")]
3079 )
3080 ;; <su>q<addsub>
3082 (define_insn "aarch64_<su_optab><optab><mode>"
3083 [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
3084 (BINQOPS:VSDQ_I (match_operand:VSDQ_I 1 "register_operand" "w")
3085 (match_operand:VSDQ_I 2 "register_operand" "w")))]
3086 "TARGET_SIMD"
3087 "<su_optab><optab>\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
3088 [(set_attr "type" "neon_<optab><q>")]
3089 )
3091 ;; suqadd and usqadd
3093 (define_insn "aarch64_<sur>qadd<mode>"
3094 [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
3095 (unspec:VSDQ_I [(match_operand:VSDQ_I 1 "register_operand" "0")
3096 (match_operand:VSDQ_I 2 "register_operand" "w")]
3097 USSUQADD))]
3098 "TARGET_SIMD"
3099 "<sur>qadd\\t%<v>0<Vmtype>, %<v>2<Vmtype>"
3100 [(set_attr "type" "neon_qadd<q>")]
3101 )
3103 ;; sqmovun
3105 (define_insn "aarch64_sqmovun<mode>"
3106 [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
3107 (unspec:<VNARROWQ> [(match_operand:VSQN_HSDI 1 "register_operand" "w")]
3108 UNSPEC_SQXTUN))]
3109 "TARGET_SIMD"
3110 "sqxtun\\t%<vn2>0<Vmntype>, %<v>1<Vmtype>"
3111 [(set_attr "type" "neon_sat_shift_imm_narrow_q")]
3112 )
3114 ;; sqmovn and uqmovn
3116 (define_insn "aarch64_<sur>qmovn<mode>"
3117 [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
3118 (unspec:<VNARROWQ> [(match_operand:VSQN_HSDI 1 "register_operand" "w")]
3119 SUQMOVN))]
3120 "TARGET_SIMD"
3121 "<sur>qxtn\\t%<vn2>0<Vmntype>, %<v>1<Vmtype>"
3122 [(set_attr "type" "neon_sat_shift_imm_narrow_q")]
3123 )
3125 ;; <su>q<absneg>
3127 (define_insn "aarch64_s<optab><mode>"
3128 [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
3129 (UNQOPS:VSDQ_I
3130 (match_operand:VSDQ_I 1 "register_operand" "w")))]
3131 "TARGET_SIMD"
3132 "s<optab>\\t%<v>0<Vmtype>, %<v>1<Vmtype>"
3133 [(set_attr "type" "neon_<optab><q>")]
3134 )
3136 ;; sq<r>dmulh.
3138 (define_insn "aarch64_sq<r>dmulh<mode>"
3139 [(set (match_operand:VSDQ_HSI 0 "register_operand" "=w")
3140 (unspec:VSDQ_HSI
3141 [(match_operand:VSDQ_HSI 1 "register_operand" "w")
3142 (match_operand:VSDQ_HSI 2 "register_operand" "w")]
3143 VQDMULH))]
3144 "TARGET_SIMD"
3145 "sq<r>dmulh\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
3146 [(set_attr "type" "neon_sat_mul_<Vetype><q>")]
3147 )
3149 ;; sq<r>dmulh_lane
3151 (define_insn "aarch64_sq<r>dmulh_lane<mode>"
3152 [(set (match_operand:VDQHS 0 "register_operand" "=w")
3153 (unspec:VDQHS
3154 [(match_operand:VDQHS 1 "register_operand" "w")
3155 (vec_select:<VEL>
3156 (match_operand:<VCOND> 2 "register_operand" "<vwx>")
3157 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))]
3158 VQDMULH))]
3159 "TARGET_SIMD"
3160 "*
3161 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
3162 return \"sq<r>dmulh\\t%0.<Vtype>, %1.<Vtype>, %2.<Vetype>[%3]\";"
3163 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar<q>")]
3164 )
3166 (define_insn "aarch64_sq<r>dmulh_laneq<mode>"
3167 [(set (match_operand:VDQHS 0 "register_operand" "=w")
3168 (unspec:VDQHS
3169 [(match_operand:VDQHS 1 "register_operand" "w")
3170 (vec_select:<VEL>
3171 (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
3172 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))]
3173 VQDMULH))]
3174 "TARGET_SIMD"
3175 "*
3176 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[3])));
3177 return \"sq<r>dmulh\\t%0.<Vtype>, %1.<Vtype>, %2.<Vetype>[%3]\";"
3178 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar<q>")]
3179 )
3181 (define_insn "aarch64_sq<r>dmulh_lane<mode>"
3182 [(set (match_operand:SD_HSI 0 "register_operand" "=w")
3183 (unspec:SD_HSI
3184 [(match_operand:SD_HSI 1 "register_operand" "w")
3185 (vec_select:<VEL>
3186 (match_operand:<VCOND> 2 "register_operand" "<vwx>")
3187 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))]
3188 VQDMULH))]
3189 "TARGET_SIMD"
3190 "*
3191 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
3192 return \"sq<r>dmulh\\t%<v>0, %<v>1, %2.<v>[%3]\";"
3193 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar<q>")]
3194 )
3196 (define_insn "aarch64_sq<r>dmulh_laneq<mode>"
3197 [(set (match_operand:SD_HSI 0 "register_operand" "=w")
3198 (unspec:SD_HSI
3199 [(match_operand:SD_HSI 1 "register_operand" "w")
3200 (vec_select:<VEL>
3201 (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
3202 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))]
3203 VQDMULH))]
3204 "TARGET_SIMD"
3205 "*
3206 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[3])));
3207 return \"sq<r>dmulh\\t%<v>0, %<v>1, %2.<v>[%3]\";"
3208 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar<q>")]
3209 )
3211 ;; sqrdml[as]h.
3213 (define_insn "aarch64_sqrdml<SQRDMLH_AS:rdma_as>h<mode>"
3214 [(set (match_operand:VSDQ_HSI 0 "register_operand" "=w")
3215 (unspec:VSDQ_HSI
3216 [(match_operand:VSDQ_HSI 1 "register_operand" "0")
3217 (match_operand:VSDQ_HSI 2 "register_operand" "w")
3218 (match_operand:VSDQ_HSI 3 "register_operand" "w")]
3219 SQRDMLH_AS))]
3220 "TARGET_SIMD_RDMA"
3221 "sqrdml<SQRDMLH_AS:rdma_as>h\\t%<v>0<Vmtype>, %<v>2<Vmtype>, %<v>3<Vmtype>"
3222 [(set_attr "type" "neon_sat_mla_<Vetype>_long")]
3223 )
3225 ;; sqrdml[as]h_lane.
3227 (define_insn "aarch64_sqrdml<SQRDMLH_AS:rdma_as>h_lane<mode>"
3228 [(set (match_operand:VDQHS 0 "register_operand" "=w")
3229 (unspec:VDQHS
3230 [(match_operand:VDQHS 1 "register_operand" "0")
3231 (match_operand:VDQHS 2 "register_operand" "w")
3232 (vec_select:<VEL>
3233 (match_operand:<VCOND> 3 "register_operand" "<vwx>")
3234 (parallel [(match_operand:SI 4 "immediate_operand" "i")]))]
3235 SQRDMLH_AS))]
3236 "TARGET_SIMD_RDMA"
3237 {
3238 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[4])));
3239 return
3240 "sqrdml<SQRDMLH_AS:rdma_as>h\\t%0.<Vtype>, %2.<Vtype>, %3.<Vetype>[%4]";
3241 }
3242 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3243 )
3245 (define_insn "aarch64_sqrdml<SQRDMLH_AS:rdma_as>h_lane<mode>"
3246 [(set (match_operand:SD_HSI 0 "register_operand" "=w")
3247 (unspec:SD_HSI
3248 [(match_operand:SD_HSI 1 "register_operand" "0")
3249 (match_operand:SD_HSI 2 "register_operand" "w")
3250 (vec_select:<VEL>
3251 (match_operand:<VCOND> 3 "register_operand" "<vwx>")
3252 (parallel [(match_operand:SI 4 "immediate_operand" "i")]))]
3253 SQRDMLH_AS))]
3254 "TARGET_SIMD_RDMA"
3255 {
3256 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[4])));
3257 return
3258 "sqrdml<SQRDMLH_AS:rdma_as>h\\t%<v>0, %<v>2, %3.<Vetype>[%4]";
3259 }
3260 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3261 )
3263 ;; sqrdml[as]h_laneq.
3265 (define_insn "aarch64_sqrdml<SQRDMLH_AS:rdma_as>h_laneq<mode>"
3266 [(set (match_operand:VDQHS 0 "register_operand" "=w")
3267 (unspec:VDQHS
3268 [(match_operand:VDQHS 1 "register_operand" "0")
3269 (match_operand:VDQHS 2 "register_operand" "w")
3270 (vec_select:<VEL>
3271 (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
3272 (parallel [(match_operand:SI 4 "immediate_operand" "i")]))]
3273 SQRDMLH_AS))]
3274 "TARGET_SIMD_RDMA"
3275 {
3276 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[4])));
3277 return
3278 "sqrdml<SQRDMLH_AS:rdma_as>h\\t%0.<Vtype>, %2.<Vtype>, %3.<Vetype>[%4]";
3279 }
3280 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3281 )
3283 (define_insn "aarch64_sqrdml<SQRDMLH_AS:rdma_as>h_laneq<mode>"
3284 [(set (match_operand:SD_HSI 0 "register_operand" "=w")
3285 (unspec:SD_HSI
3286 [(match_operand:SD_HSI 1 "register_operand" "0")
3287 (match_operand:SD_HSI 2 "register_operand" "w")
3288 (vec_select:<VEL>
3289 (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
3290 (parallel [(match_operand:SI 4 "immediate_operand" "i")]))]
3291 SQRDMLH_AS))]
3292 "TARGET_SIMD_RDMA"
3293 {
3294 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[4])));
3295 return
3296 "sqrdml<SQRDMLH_AS:rdma_as>h\\t%<v>0, %<v>2, %3.<v>[%4]";
3297 }
3298 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3299 )
3301 ;; vqdml[sa]l
3303 (define_insn "aarch64_sqdml<SBINQOPS:as>l<mode>"
3304 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3305 (SBINQOPS:<VWIDE>
3306 (match_operand:<VWIDE> 1 "register_operand" "0")
3307 (ss_ashift:<VWIDE>
3308 (mult:<VWIDE>
3309 (sign_extend:<VWIDE>
3310 (match_operand:VSD_HSI 2 "register_operand" "w"))
3311 (sign_extend:<VWIDE>
3312 (match_operand:VSD_HSI 3 "register_operand" "w")))
3313 (const_int 1))))]
3314 "TARGET_SIMD"
3315 "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %<v>3<Vmtype>"
3316 [(set_attr "type" "neon_sat_mla_<Vetype>_long")]
3317 )
3319 ;; vqdml[sa]l_lane
3321 (define_insn "aarch64_sqdml<SBINQOPS:as>l_lane<mode>"
3322 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3323 (SBINQOPS:<VWIDE>
3324 (match_operand:<VWIDE> 1 "register_operand" "0")
3325 (ss_ashift:<VWIDE>
3326 (mult:<VWIDE>
3327 (sign_extend:<VWIDE>
3328 (match_operand:VD_HSI 2 "register_operand" "w"))
3329 (sign_extend:<VWIDE>
3330 (vec_duplicate:VD_HSI
3331 (vec_select:<VEL>
3332 (match_operand:<VCOND> 3 "register_operand" "<vwx>")
3333 (parallel [(match_operand:SI 4 "immediate_operand" "i")])))
3334 ))
3335 (const_int 1))))]
3336 "TARGET_SIMD"
3337 {
3338 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[4])));
3339 return
3340 "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
3341 }
3342 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3343 )
3345 (define_insn "aarch64_sqdml<SBINQOPS:as>l_laneq<mode>"
3346 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3347 (SBINQOPS:<VWIDE>
3348 (match_operand:<VWIDE> 1 "register_operand" "0")
3349 (ss_ashift:<VWIDE>
3350 (mult:<VWIDE>
3351 (sign_extend:<VWIDE>
3352 (match_operand:VD_HSI 2 "register_operand" "w"))
3353 (sign_extend:<VWIDE>
3354 (vec_duplicate:VD_HSI
3355 (vec_select:<VEL>
3356 (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
3357 (parallel [(match_operand:SI 4 "immediate_operand" "i")])))
3358 ))
3359 (const_int 1))))]
3360 "TARGET_SIMD"
3361 {
3362 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[4])));
3363 return
3364 "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
3365 }
3366 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3367 )
3369 (define_insn "aarch64_sqdml<SBINQOPS:as>l_lane<mode>"
3370 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3371 (SBINQOPS:<VWIDE>
3372 (match_operand:<VWIDE> 1 "register_operand" "0")
3373 (ss_ashift:<VWIDE>
3374 (mult:<VWIDE>
3375 (sign_extend:<VWIDE>
3376 (match_operand:SD_HSI 2 "register_operand" "w"))
3377 (sign_extend:<VWIDE>
3378 (vec_select:<VEL>
3379 (match_operand:<VCOND> 3 "register_operand" "<vwx>")
3380 (parallel [(match_operand:SI 4 "immediate_operand" "i")])))
3381 )
3382 (const_int 1))))]
3383 "TARGET_SIMD"
3384 {
3385 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[4])));
3386 return
3387 "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
3388 }
3389 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3390 )
3392 (define_insn "aarch64_sqdml<SBINQOPS:as>l_laneq<mode>"
3393 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3394 (SBINQOPS:<VWIDE>
3395 (match_operand:<VWIDE> 1 "register_operand" "0")
3396 (ss_ashift:<VWIDE>
3397 (mult:<VWIDE>
3398 (sign_extend:<VWIDE>
3399 (match_operand:SD_HSI 2 "register_operand" "w"))
3400 (sign_extend:<VWIDE>
3401 (vec_select:<VEL>
3402 (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
3403 (parallel [(match_operand:SI 4 "immediate_operand" "i")])))
3404 )
3405 (const_int 1))))]
3406 "TARGET_SIMD"
3407 {
3408 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[4])));
3409 return
3410 "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
3411 }
3412 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3413 )
3415 ;; vqdml[sa]l_n
3417 (define_insn "aarch64_sqdml<SBINQOPS:as>l_n<mode>"
3418 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3419 (SBINQOPS:<VWIDE>
3420 (match_operand:<VWIDE> 1 "register_operand" "0")
3421 (ss_ashift:<VWIDE>
3422 (mult:<VWIDE>
3423 (sign_extend:<VWIDE>
3424 (match_operand:VD_HSI 2 "register_operand" "w"))
3425 (sign_extend:<VWIDE>
3426 (vec_duplicate:VD_HSI
3427 (match_operand:<VEL> 3 "register_operand" "<vwx>"))))
3428 (const_int 1))))]
3429 "TARGET_SIMD"
3430 "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[0]"
3431 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3432 )
3434 ;; sqdml[as]l2
3436 (define_insn "aarch64_sqdml<SBINQOPS:as>l2<mode>_internal"
3437 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3438 (SBINQOPS:<VWIDE>
3439 (match_operand:<VWIDE> 1 "register_operand" "0")
3440 (ss_ashift:<VWIDE>
3441 (mult:<VWIDE>
3442 (sign_extend:<VWIDE>
3443 (vec_select:<VHALF>
3444 (match_operand:VQ_HSI 2 "register_operand" "w")
3445 (match_operand:VQ_HSI 4 "vect_par_cnst_hi_half" "")))
3446 (sign_extend:<VWIDE>
3447 (vec_select:<VHALF>
3448 (match_operand:VQ_HSI 3 "register_operand" "w")
3449 (match_dup 4))))
3450 (const_int 1))))]
3451 "TARGET_SIMD"
3452 "sqdml<SBINQOPS:as>l2\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %<v>3<Vmtype>"
3453 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3454 )
3456 (define_expand "aarch64_sqdmlal2<mode>"
3457 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3458 (match_operand:<VWIDE> 1 "register_operand" "w")
3459 (match_operand:VQ_HSI 2 "register_operand" "w")
3460 (match_operand:VQ_HSI 3 "register_operand" "w")]
3461 "TARGET_SIMD"
3462 {
3463 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3464 emit_insn (gen_aarch64_sqdmlal2<mode>_internal (operands[0], operands[1],
3465 operands[2], operands[3], p));
3466 DONE;
3467 })
3469 (define_expand "aarch64_sqdmlsl2<mode>"
3470 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3471 (match_operand:<VWIDE> 1 "register_operand" "w")
3472 (match_operand:VQ_HSI 2 "register_operand" "w")
3473 (match_operand:VQ_HSI 3 "register_operand" "w")]
3474 "TARGET_SIMD"
3475 {
3476 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3477 emit_insn (gen_aarch64_sqdmlsl2<mode>_internal (operands[0], operands[1],
3478 operands[2], operands[3], p));
3479 DONE;
3480 })
3482 ;; vqdml[sa]l2_lane
3484 (define_insn "aarch64_sqdml<SBINQOPS:as>l2_lane<mode>_internal"
3485 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3486 (SBINQOPS:<VWIDE>
3487 (match_operand:<VWIDE> 1 "register_operand" "0")
3488 (ss_ashift:<VWIDE>
3489 (mult:<VWIDE>
3490 (sign_extend:<VWIDE>
3491 (vec_select:<VHALF>
3492 (match_operand:VQ_HSI 2 "register_operand" "w")
3493 (match_operand:VQ_HSI 5 "vect_par_cnst_hi_half" "")))
3494 (sign_extend:<VWIDE>
3495 (vec_duplicate:<VHALF>
3496 (vec_select:<VEL>
3497 (match_operand:<VCOND> 3 "register_operand" "<vwx>")
3498 (parallel [(match_operand:SI 4 "immediate_operand" "i")])
3499 ))))
3500 (const_int 1))))]
3501 "TARGET_SIMD"
3502 {
3503 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[4])));
3504 return
3505 "sqdml<SBINQOPS:as>l2\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
3506 }
3507 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3508 )
3510 (define_insn "aarch64_sqdml<SBINQOPS:as>l2_laneq<mode>_internal"
3511 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3512 (SBINQOPS:<VWIDE>
3513 (match_operand:<VWIDE> 1 "register_operand" "0")
3514 (ss_ashift:<VWIDE>
3515 (mult:<VWIDE>
3516 (sign_extend:<VWIDE>
3517 (vec_select:<VHALF>
3518 (match_operand:VQ_HSI 2 "register_operand" "w")
3519 (match_operand:VQ_HSI 5 "vect_par_cnst_hi_half" "")))
3520 (sign_extend:<VWIDE>
3521 (vec_duplicate:<VHALF>
3522 (vec_select:<VEL>
3523 (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
3524 (parallel [(match_operand:SI 4 "immediate_operand" "i")])
3525 ))))
3526 (const_int 1))))]
3527 "TARGET_SIMD"
3528 {
3529 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[4])));
3530 return
3531 "sqdml<SBINQOPS:as>l2\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
3532 }
3533 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3534 )
3536 (define_expand "aarch64_sqdmlal2_lane<mode>"
3537 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3538 (match_operand:<VWIDE> 1 "register_operand" "w")
3539 (match_operand:VQ_HSI 2 "register_operand" "w")
3540 (match_operand:<VCOND> 3 "register_operand" "<vwx>")
3541 (match_operand:SI 4 "immediate_operand" "i")]
3542 "TARGET_SIMD"
3543 {
3544 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3545 emit_insn (gen_aarch64_sqdmlal2_lane<mode>_internal (operands[0], operands[1],
3546 operands[2], operands[3],
3547 operands[4], p));
3548 DONE;
3549 })
3551 (define_expand "aarch64_sqdmlal2_laneq<mode>"
3552 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3553 (match_operand:<VWIDE> 1 "register_operand" "w")
3554 (match_operand:VQ_HSI 2 "register_operand" "w")
3555 (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
3556 (match_operand:SI 4 "immediate_operand" "i")]
3557 "TARGET_SIMD"
3558 {
3559 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3560 emit_insn (gen_aarch64_sqdmlal2_laneq<mode>_internal (operands[0], operands[1],
3561 operands[2], operands[3],
3562 operands[4], p));
3563 DONE;
3564 })
3566 (define_expand "aarch64_sqdmlsl2_lane<mode>"
3567 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3568 (match_operand:<VWIDE> 1 "register_operand" "w")
3569 (match_operand:VQ_HSI 2 "register_operand" "w")
3570 (match_operand:<VCOND> 3 "register_operand" "<vwx>")
3571 (match_operand:SI 4 "immediate_operand" "i")]
3572 "TARGET_SIMD"
3573 {
3574 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3575 emit_insn (gen_aarch64_sqdmlsl2_lane<mode>_internal (operands[0], operands[1],
3576 operands[2], operands[3],
3577 operands[4], p));
3578 DONE;
3579 })
3581 (define_expand "aarch64_sqdmlsl2_laneq<mode>"
3582 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3583 (match_operand:<VWIDE> 1 "register_operand" "w")
3584 (match_operand:VQ_HSI 2 "register_operand" "w")
3585 (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
3586 (match_operand:SI 4 "immediate_operand" "i")]
3587 "TARGET_SIMD"
3588 {
3589 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3590 emit_insn (gen_aarch64_sqdmlsl2_laneq<mode>_internal (operands[0], operands[1],
3591 operands[2], operands[3],
3592 operands[4], p));
3593 DONE;
3594 })
3596 (define_insn "aarch64_sqdml<SBINQOPS:as>l2_n<mode>_internal"
3597 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3598 (SBINQOPS:<VWIDE>
3599 (match_operand:<VWIDE> 1 "register_operand" "0")
3600 (ss_ashift:<VWIDE>
3601 (mult:<VWIDE>
3602 (sign_extend:<VWIDE>
3603 (vec_select:<VHALF>
3604 (match_operand:VQ_HSI 2 "register_operand" "w")
3605 (match_operand:VQ_HSI 4 "vect_par_cnst_hi_half" "")))
3606 (sign_extend:<VWIDE>
3607 (vec_duplicate:<VHALF>
3608 (match_operand:<VEL> 3 "register_operand" "<vwx>"))))
3609 (const_int 1))))]
3610 "TARGET_SIMD"
3611 "sqdml<SBINQOPS:as>l2\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[0]"
3612 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3613 )
3615 (define_expand "aarch64_sqdmlal2_n<mode>"
3616 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3617 (match_operand:<VWIDE> 1 "register_operand" "w")
3618 (match_operand:VQ_HSI 2 "register_operand" "w")
3619 (match_operand:<VEL> 3 "register_operand" "w")]
3620 "TARGET_SIMD"
3621 {
3622 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3623 emit_insn (gen_aarch64_sqdmlal2_n<mode>_internal (operands[0], operands[1],
3624 operands[2], operands[3],
3625 p));
3626 DONE;
3627 })
3629 (define_expand "aarch64_sqdmlsl2_n<mode>"
3630 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3631 (match_operand:<VWIDE> 1 "register_operand" "w")
3632 (match_operand:VQ_HSI 2 "register_operand" "w")
3633 (match_operand:<VEL> 3 "register_operand" "w")]
3634 "TARGET_SIMD"
3635 {
3636 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3637 emit_insn (gen_aarch64_sqdmlsl2_n<mode>_internal (operands[0], operands[1],
3638 operands[2], operands[3],
3639 p));
3640 DONE;
3641 })
3643 ;; vqdmull
3645 (define_insn "aarch64_sqdmull<mode>"
3646 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3647 (ss_ashift:<VWIDE>
3648 (mult:<VWIDE>
3649 (sign_extend:<VWIDE>
3650 (match_operand:VSD_HSI 1 "register_operand" "w"))
3651 (sign_extend:<VWIDE>
3652 (match_operand:VSD_HSI 2 "register_operand" "w")))
3653 (const_int 1)))]
3654 "TARGET_SIMD"
3655 "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
3656 [(set_attr "type" "neon_sat_mul_<Vetype>_long")]
3657 )
3659 ;; vqdmull_lane
3661 (define_insn "aarch64_sqdmull_lane<mode>"
3662 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3663 (ss_ashift:<VWIDE>
3664 (mult:<VWIDE>
3665 (sign_extend:<VWIDE>
3666 (match_operand:VD_HSI 1 "register_operand" "w"))
3667 (sign_extend:<VWIDE>
3668 (vec_duplicate:VD_HSI
3669 (vec_select:<VEL>
3670 (match_operand:<VCOND> 2 "register_operand" "<vwx>")
3671 (parallel [(match_operand:SI 3 "immediate_operand" "i")])))
3672 ))
3673 (const_int 1)))]
3674 "TARGET_SIMD"
3675 {
3676 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
3677 return "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3678 }
3679 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3680 )
3682 (define_insn "aarch64_sqdmull_laneq<mode>"
3683 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3684 (ss_ashift:<VWIDE>
3685 (mult:<VWIDE>
3686 (sign_extend:<VWIDE>
3687 (match_operand:VD_HSI 1 "register_operand" "w"))
3688 (sign_extend:<VWIDE>
3689 (vec_duplicate:VD_HSI
3690 (vec_select:<VEL>
3691 (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
3692 (parallel [(match_operand:SI 3 "immediate_operand" "i")])))
3693 ))
3694 (const_int 1)))]
3695 "TARGET_SIMD"
3696 {
3697 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[3])));
3698 return "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3699 }
3700 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3701 )
3703 (define_insn "aarch64_sqdmull_lane<mode>"
3704 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3705 (ss_ashift:<VWIDE>
3706 (mult:<VWIDE>
3707 (sign_extend:<VWIDE>
3708 (match_operand:SD_HSI 1 "register_operand" "w"))
3709 (sign_extend:<VWIDE>
3710 (vec_select:<VEL>
3711 (match_operand:<VCOND> 2 "register_operand" "<vwx>")
3712 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))
3713 ))
3714 (const_int 1)))]
3715 "TARGET_SIMD"
3716 {
3717 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
3718 return "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3719 }
3720 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3721 )
3723 (define_insn "aarch64_sqdmull_laneq<mode>"
3724 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3725 (ss_ashift:<VWIDE>
3726 (mult:<VWIDE>
3727 (sign_extend:<VWIDE>
3728 (match_operand:SD_HSI 1 "register_operand" "w"))
3729 (sign_extend:<VWIDE>
3730 (vec_select:<VEL>
3731 (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
3732 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))
3733 ))
3734 (const_int 1)))]
3735 "TARGET_SIMD"
3736 {
3737 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[3])));
3738 return "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3739 }
3740 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3741 )
3743 ;; vqdmull_n
3745 (define_insn "aarch64_sqdmull_n<mode>"
3746 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3747 (ss_ashift:<VWIDE>
3748 (mult:<VWIDE>
3749 (sign_extend:<VWIDE>
3750 (match_operand:VD_HSI 1 "register_operand" "w"))
3751 (sign_extend:<VWIDE>
3752 (vec_duplicate:VD_HSI
3753 (match_operand:<VEL> 2 "register_operand" "<vwx>")))
3754 )
3755 (const_int 1)))]
3756 "TARGET_SIMD"
3757 "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[0]"
3758 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3759 )
3761 ;; vqdmull2
3765 (define_insn "aarch64_sqdmull2<mode>_internal"
3766 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3767 (ss_ashift:<VWIDE>
3768 (mult:<VWIDE>
3769 (sign_extend:<VWIDE>
3770 (vec_select:<VHALF>
3771 (match_operand:VQ_HSI 1 "register_operand" "w")
3772 (match_operand:VQ_HSI 3 "vect_par_cnst_hi_half" "")))
3773 (sign_extend:<VWIDE>
3774 (vec_select:<VHALF>
3775 (match_operand:VQ_HSI 2 "register_operand" "w")
3776 (match_dup 3)))
3777 )
3778 (const_int 1)))]
3779 "TARGET_SIMD"
3780 "sqdmull2\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
3781 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3782 )
3784 (define_expand "aarch64_sqdmull2<mode>"
3785 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3786 (match_operand:VQ_HSI 1 "register_operand" "w")
3787 (match_operand:VQ_HSI 2 "register_operand" "w")]
3788 "TARGET_SIMD"
3789 {
3790 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3791 emit_insn (gen_aarch64_sqdmull2<mode>_internal (operands[0], operands[1],
3792 operands[2], p));
3793 DONE;
3794 })
3796 ;; vqdmull2_lane
3798 (define_insn "aarch64_sqdmull2_lane<mode>_internal"
3799 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3800 (ss_ashift:<VWIDE>
3801 (mult:<VWIDE>
3802 (sign_extend:<VWIDE>
3803 (vec_select:<VHALF>
3804 (match_operand:VQ_HSI 1 "register_operand" "w")
3805 (match_operand:VQ_HSI 4 "vect_par_cnst_hi_half" "")))
3806 (sign_extend:<VWIDE>
3807 (vec_duplicate:<VHALF>
3808 (vec_select:<VEL>
3809 (match_operand:<VCOND> 2 "register_operand" "<vwx>")
3810 (parallel [(match_operand:SI 3 "immediate_operand" "i")])))
3811 ))
3812 (const_int 1)))]
3813 "TARGET_SIMD"
3814 {
3815 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
3816 return "sqdmull2\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3817 }
3818 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3819 )
3821 (define_insn "aarch64_sqdmull2_laneq<mode>_internal"
3822 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3823 (ss_ashift:<VWIDE>
3824 (mult:<VWIDE>
3825 (sign_extend:<VWIDE>
3826 (vec_select:<VHALF>
3827 (match_operand:VQ_HSI 1 "register_operand" "w")
3828 (match_operand:VQ_HSI 4 "vect_par_cnst_hi_half" "")))
3829 (sign_extend:<VWIDE>
3830 (vec_duplicate:<VHALF>
3831 (vec_select:<VEL>
3832 (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
3833 (parallel [(match_operand:SI 3 "immediate_operand" "i")])))
3834 ))
3835 (const_int 1)))]
3836 "TARGET_SIMD"
3837 {
3838 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[3])));
3839 return "sqdmull2\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3840 }
3841 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3842 )
3844 (define_expand "aarch64_sqdmull2_lane<mode>"
3845 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3846 (match_operand:VQ_HSI 1 "register_operand" "w")
3847 (match_operand:<VCOND> 2 "register_operand" "<vwx>")
3848 (match_operand:SI 3 "immediate_operand" "i")]
3849 "TARGET_SIMD"
3850 {
3851 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3852 emit_insn (gen_aarch64_sqdmull2_lane<mode>_internal (operands[0], operands[1],
3853 operands[2], operands[3],
3854 p));
3855 DONE;
3856 })
3858 (define_expand "aarch64_sqdmull2_laneq<mode>"
3859 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3860 (match_operand:VQ_HSI 1 "register_operand" "w")
3861 (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
3862 (match_operand:SI 3 "immediate_operand" "i")]
3863 "TARGET_SIMD"
3864 {
3865 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3866 emit_insn (gen_aarch64_sqdmull2_laneq<mode>_internal (operands[0], operands[1],
3867 operands[2], operands[3],
3868 p));
3869 DONE;
3870 })
3872 ;; vqdmull2_n
3874 (define_insn "aarch64_sqdmull2_n<mode>_internal"
3875 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3876 (ss_ashift:<VWIDE>
3877 (mult:<VWIDE>
3878 (sign_extend:<VWIDE>
3879 (vec_select:<VHALF>
3880 (match_operand:VQ_HSI 1 "register_operand" "w")
3881 (match_operand:VQ_HSI 3 "vect_par_cnst_hi_half" "")))
3882 (sign_extend:<VWIDE>
3883 (vec_duplicate:<VHALF>
3884 (match_operand:<VEL> 2 "register_operand" "<vwx>")))
3885 )
3886 (const_int 1)))]
3887 "TARGET_SIMD"
3888 "sqdmull2\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[0]"
3889 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3890 )
3892 (define_expand "aarch64_sqdmull2_n<mode>"
3893 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3894 (match_operand:VQ_HSI 1 "register_operand" "w")
3895 (match_operand:<VEL> 2 "register_operand" "w")]
3896 "TARGET_SIMD"
3897 {
3898 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3899 emit_insn (gen_aarch64_sqdmull2_n<mode>_internal (operands[0], operands[1],
3900 operands[2], p));
3901 DONE;
3902 })
3904 ;; vshl
3906 (define_insn "aarch64_<sur>shl<mode>"
3907 [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
3908 (unspec:VSDQ_I_DI
3909 [(match_operand:VSDQ_I_DI 1 "register_operand" "w")
3910 (match_operand:VSDQ_I_DI 2 "register_operand" "w")]
3911 VSHL))]
3912 "TARGET_SIMD"
3913 "<sur>shl\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>";
3914 [(set_attr "type" "neon_shift_reg<q>")]
3915 )
3918 ;; vqshl
3920 (define_insn "aarch64_<sur>q<r>shl<mode>"
3921 [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
3922 (unspec:VSDQ_I
3923 [(match_operand:VSDQ_I 1 "register_operand" "w")
3924 (match_operand:VSDQ_I 2 "register_operand" "w")]
3925 VQSHL))]
3926 "TARGET_SIMD"
3927 "<sur>q<r>shl\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>";
3928 [(set_attr "type" "neon_sat_shift_reg<q>")]
3929 )
3931 ;; vshll_n
3933 (define_insn "aarch64_<sur>shll_n<mode>"
3934 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3935 (unspec:<VWIDE> [(match_operand:VD_BHSI 1 "register_operand" "w")
3936 (match_operand:SI 2
3937 "aarch64_simd_shift_imm_bitsize_<ve_mode>" "i")]
3938 VSHLL))]
3939 "TARGET_SIMD"
3940 "*
3941 int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
3942 if (INTVAL (operands[2]) == bit_width)
3943 {
3944 return \"shll\\t%0.<Vwtype>, %1.<Vtype>, %2\";
3945 }
3946 else {
3947 return \"<sur>shll\\t%0.<Vwtype>, %1.<Vtype>, %2\";
3948 }"
3949 [(set_attr "type" "neon_shift_imm_long")]
3950 )
3952 ;; vshll_high_n
3954 (define_insn "aarch64_<sur>shll2_n<mode>"
3955 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3956 (unspec:<VWIDE> [(match_operand:VQW 1 "register_operand" "w")
3957 (match_operand:SI 2 "immediate_operand" "i")]
3958 VSHLL))]
3959 "TARGET_SIMD"
3960 "*
3961 int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
3962 if (INTVAL (operands[2]) == bit_width)
3963 {
3964 return \"shll2\\t%0.<Vwtype>, %1.<Vtype>, %2\";
3965 }
3966 else {
3967 return \"<sur>shll2\\t%0.<Vwtype>, %1.<Vtype>, %2\";
3968 }"
3969 [(set_attr "type" "neon_shift_imm_long")]
3970 )
3972 ;; vrshr_n
3974 (define_insn "aarch64_<sur>shr_n<mode>"
3975 [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
3976 (unspec:VSDQ_I_DI [(match_operand:VSDQ_I_DI 1 "register_operand" "w")
3977 (match_operand:SI 2
3978 "aarch64_simd_shift_imm_offset_<ve_mode>" "i")]
3979 VRSHR_N))]
3980 "TARGET_SIMD"
3981 "<sur>shr\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %2"
3982 [(set_attr "type" "neon_sat_shift_imm<q>")]
3983 )
3985 ;; v(r)sra_n
3987 (define_insn "aarch64_<sur>sra_n<mode>"
3988 [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
3989 (unspec:VSDQ_I_DI [(match_operand:VSDQ_I_DI 1 "register_operand" "0")
3990 (match_operand:VSDQ_I_DI 2 "register_operand" "w")
3991 (match_operand:SI 3
3992 "aarch64_simd_shift_imm_offset_<ve_mode>" "i")]
3993 VSRA))]
3994 "TARGET_SIMD"
3995 "<sur>sra\\t%<v>0<Vmtype>, %<v>2<Vmtype>, %3"
3996 [(set_attr "type" "neon_shift_acc<q>")]
3997 )
3999 ;; vs<lr>i_n
4001 (define_insn "aarch64_<sur>s<lr>i_n<mode>"
4002 [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
4003 (unspec:VSDQ_I_DI [(match_operand:VSDQ_I_DI 1 "register_operand" "0")
4004 (match_operand:VSDQ_I_DI 2 "register_operand" "w")
4005 (match_operand:SI 3
4006 "aarch64_simd_shift_imm_<offsetlr><ve_mode>" "i")]
4007 VSLRI))]
4008 "TARGET_SIMD"
4009 "s<lr>i\\t%<v>0<Vmtype>, %<v>2<Vmtype>, %3"
4010 [(set_attr "type" "neon_shift_imm<q>")]
4011 )
4013 ;; vqshl(u)
4015 (define_insn "aarch64_<sur>qshl<u>_n<mode>"
4016 [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
4017 (unspec:VSDQ_I [(match_operand:VSDQ_I 1 "register_operand" "w")
4018 (match_operand:SI 2
4019 "aarch64_simd_shift_imm_<ve_mode>" "i")]
4020 VQSHL_N))]
4021 "TARGET_SIMD"
4022 "<sur>qshl<u>\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %2"
4023 [(set_attr "type" "neon_sat_shift_imm<q>")]
4024 )
4027 ;; vq(r)shr(u)n_n
4029 (define_insn "aarch64_<sur>q<r>shr<u>n_n<mode>"
4030 [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
4031 (unspec:<VNARROWQ> [(match_operand:VSQN_HSDI 1 "register_operand" "w")
4032 (match_operand:SI 2
4033 "aarch64_simd_shift_imm_offset_<ve_mode>" "i")]
4034 VQSHRN_N))]
4035 "TARGET_SIMD"
4036 "<sur>q<r>shr<u>n\\t%<vn2>0<Vmntype>, %<v>1<Vmtype>, %2"
4037 [(set_attr "type" "neon_sat_shift_imm_narrow_q")]
4038 )
4041 ;; cm(eq|ge|gt|lt|le)
4042 ;; Note, we have constraints for Dz and Z as different expanders
4043 ;; have different ideas of what should be passed to this pattern.
4045 (define_insn "aarch64_cm<optab><mode>"
4046 [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w,w")
4047 (neg:<V_cmp_result>
4048 (COMPARISONS:<V_cmp_result>
4049 (match_operand:VDQ_I 1 "register_operand" "w,w")
4050 (match_operand:VDQ_I 2 "aarch64_simd_reg_or_zero" "w,ZDz")
4051 )))]
4052 "TARGET_SIMD"
4053 "@
4054 cm<n_optab>\t%<v>0<Vmtype>, %<v><cmp_1><Vmtype>, %<v><cmp_2><Vmtype>
4055 cm<optab>\t%<v>0<Vmtype>, %<v>1<Vmtype>, #0"
4056 [(set_attr "type" "neon_compare<q>, neon_compare_zero<q>")]
4057 )
4059 (define_insn_and_split "aarch64_cm<optab>di"
4060 [(set (match_operand:DI 0 "register_operand" "=w,w,r")
4061 (neg:DI
4062 (COMPARISONS:DI
4063 (match_operand:DI 1 "register_operand" "w,w,r")
4064 (match_operand:DI 2 "aarch64_simd_reg_or_zero" "w,ZDz,r")
4065 )))
4066 (clobber (reg:CC CC_REGNUM))]
4067 "TARGET_SIMD"
4068 "#"
4069 "reload_completed"
4070 [(set (match_operand:DI 0 "register_operand")
4071 (neg:DI
4072 (COMPARISONS:DI
4073 (match_operand:DI 1 "register_operand")
4074 (match_operand:DI 2 "aarch64_simd_reg_or_zero")
4075 )))]
4076 {
4077 /* If we are in the general purpose register file,
4078 we split to a sequence of comparison and store. */
4079 if (GP_REGNUM_P (REGNO (operands[0]))
4080 && GP_REGNUM_P (REGNO (operands[1])))
4081 {
4082 machine_mode mode = SELECT_CC_MODE (<CMP>, operands[1], operands[2]);
4083 rtx cc_reg = aarch64_gen_compare_reg (<CMP>, operands[1], operands[2]);
4084 rtx comparison = gen_rtx_<CMP> (mode, operands[1], operands[2]);
4085 emit_insn (gen_cstoredi_neg (operands[0], comparison, cc_reg));
4086 DONE;
4087 }
4088 /* Otherwise, we expand to a similar pattern which does not
4089 clobber CC_REGNUM. */
4090 }
4091 [(set_attr "type" "neon_compare, neon_compare_zero, multiple")]
4092 )
4094 (define_insn "*aarch64_cm<optab>di"
4095 [(set (match_operand:DI 0 "register_operand" "=w,w")
4096 (neg:DI
4097 (COMPARISONS:DI
4098 (match_operand:DI 1 "register_operand" "w,w")
4099 (match_operand:DI 2 "aarch64_simd_reg_or_zero" "w,ZDz")
4100 )))]
4101 "TARGET_SIMD && reload_completed"
4102 "@
4103 cm<n_optab>\t%d0, %d<cmp_1>, %d<cmp_2>
4104 cm<optab>\t%d0, %d1, #0"
4105 [(set_attr "type" "neon_compare, neon_compare_zero")]
4106 )
4108 ;; cm(hs|hi)
4110 (define_insn "aarch64_cm<optab><mode>"
4111 [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w")
4112 (neg:<V_cmp_result>
4113 (UCOMPARISONS:<V_cmp_result>
4114 (match_operand:VDQ_I 1 "register_operand" "w")
4115 (match_operand:VDQ_I 2 "register_operand" "w")
4116 )))]
4117 "TARGET_SIMD"
4118 "cm<n_optab>\t%<v>0<Vmtype>, %<v><cmp_1><Vmtype>, %<v><cmp_2><Vmtype>"
4119 [(set_attr "type" "neon_compare<q>")]
4120 )
4122 (define_insn_and_split "aarch64_cm<optab>di"
4123 [(set (match_operand:DI 0 "register_operand" "=w,r")
4124 (neg:DI
4125 (UCOMPARISONS:DI
4126 (match_operand:DI 1 "register_operand" "w,r")
4127 (match_operand:DI 2 "aarch64_simd_reg_or_zero" "w,r")
4128 )))
4129 (clobber (reg:CC CC_REGNUM))]
4130 "TARGET_SIMD"
4131 "#"
4132 "reload_completed"
4133 [(set (match_operand:DI 0 "register_operand")
4134 (neg:DI
4135 (UCOMPARISONS:DI
4136 (match_operand:DI 1 "register_operand")
4137 (match_operand:DI 2 "aarch64_simd_reg_or_zero")
4138 )))]
4139 {
4140 /* If we are in the general purpose register file,
4141 we split to a sequence of comparison and store. */
4142 if (GP_REGNUM_P (REGNO (operands[0]))
4143 && GP_REGNUM_P (REGNO (operands[1])))
4144 {
4145 machine_mode mode = CCmode;
4146 rtx cc_reg = aarch64_gen_compare_reg (<CMP>, operands[1], operands[2]);
4147 rtx comparison = gen_rtx_<CMP> (mode, operands[1], operands[2]);
4148 emit_insn (gen_cstoredi_neg (operands[0], comparison, cc_reg));
4149 DONE;
4150 }
4151 /* Otherwise, we expand to a similar pattern which does not
4152 clobber CC_REGNUM. */
4153 }
4154 [(set_attr "type" "neon_compare,multiple")]
4155 )
4157 (define_insn "*aarch64_cm<optab>di"
4158 [(set (match_operand:DI 0 "register_operand" "=w")
4159 (neg:DI
4160 (UCOMPARISONS:DI
4161 (match_operand:DI 1 "register_operand" "w")
4162 (match_operand:DI 2 "aarch64_simd_reg_or_zero" "w")
4163 )))]
4164 "TARGET_SIMD && reload_completed"
4165 "cm<n_optab>\t%d0, %d<cmp_1>, %d<cmp_2>"
4166 [(set_attr "type" "neon_compare")]
4167 )
4169 ;; cmtst
4171 ;; Although neg (ne (and x y) 0) is the natural way of expressing a cmtst,
4172 ;; we don't have any insns using ne, and aarch64_vcond_internal outputs
4173 ;; not (neg (eq (and x y) 0))
4174 ;; which is rewritten by simplify_rtx as
4175 ;; plus (eq (and x y) 0) -1.
4177 (define_insn "aarch64_cmtst<mode>"
4178 [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w")
4179 (plus:<V_cmp_result>
4180 (eq:<V_cmp_result>
4181 (and:VDQ_I
4182 (match_operand:VDQ_I 1 "register_operand" "w")
4183 (match_operand:VDQ_I 2 "register_operand" "w"))
4184 (match_operand:VDQ_I 3 "aarch64_simd_imm_zero"))
4185 (match_operand:<V_cmp_result> 4 "aarch64_simd_imm_minus_one")))
4186 ]
4187 "TARGET_SIMD"
4188 "cmtst\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
4189 [(set_attr "type" "neon_tst<q>")]
4190 )
4192 (define_insn_and_split "aarch64_cmtstdi"
4193 [(set (match_operand:DI 0 "register_operand" "=w,r")
4194 (neg:DI
4195 (ne:DI
4196 (and:DI
4197 (match_operand:DI 1 "register_operand" "w,r")
4198 (match_operand:DI 2 "register_operand" "w,r"))
4199 (const_int 0))))
4200 (clobber (reg:CC CC_REGNUM))]
4201 "TARGET_SIMD"
4202 "#"
4203 "reload_completed"
4204 [(set (match_operand:DI 0 "register_operand")
4205 (neg:DI
4206 (ne:DI
4207 (and:DI
4208 (match_operand:DI 1 "register_operand")
4209 (match_operand:DI 2 "register_operand"))
4210 (const_int 0))))]
4211 {
4212 /* If we are in the general purpose register file,
4213 we split to a sequence of comparison and store. */
4214 if (GP_REGNUM_P (REGNO (operands[0]))
4215 && GP_REGNUM_P (REGNO (operands[1])))
4216 {
4217 rtx and_tree = gen_rtx_AND (DImode, operands[1], operands[2]);
4218 machine_mode mode = SELECT_CC_MODE (NE, and_tree, const0_rtx);
4219 rtx cc_reg = aarch64_gen_compare_reg (NE, and_tree, const0_rtx);
4220 rtx comparison = gen_rtx_NE (mode, and_tree, const0_rtx);
4221 emit_insn (gen_cstoredi_neg (operands[0], comparison, cc_reg));
4222 DONE;
4223 }
4224 /* Otherwise, we expand to a similar pattern which does not
4225 clobber CC_REGNUM. */
4226 }
4227 [(set_attr "type" "neon_tst,multiple")]
4228 )
4230 (define_insn "*aarch64_cmtstdi"
4231 [(set (match_operand:DI 0 "register_operand" "=w")
4232 (neg:DI
4233 (ne:DI
4234 (and:DI
4235 (match_operand:DI 1 "register_operand" "w")
4236 (match_operand:DI 2 "register_operand" "w"))
4237 (const_int 0))))]
4238 "TARGET_SIMD"
4239 "cmtst\t%d0, %d1, %d2"
4240 [(set_attr "type" "neon_tst")]
4241 )
4243 ;; fcm(eq|ge|gt|le|lt)
4245 (define_insn "aarch64_cm<optab><mode>"
4246 [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w,w")
4247 (neg:<V_cmp_result>
4248 (COMPARISONS:<V_cmp_result>
4249 (match_operand:VALLF 1 "register_operand" "w,w")
4250 (match_operand:VALLF 2 "aarch64_simd_reg_or_zero" "w,YDz")
4251 )))]
4252 "TARGET_SIMD"
4253 "@
4254 fcm<n_optab>\t%<v>0<Vmtype>, %<v><cmp_1><Vmtype>, %<v><cmp_2><Vmtype>
4255 fcm<optab>\t%<v>0<Vmtype>, %<v>1<Vmtype>, 0"
4256 [(set_attr "type" "neon_fp_compare_<Vetype><q>")]
4257 )
4259 ;; fac(ge|gt)
4260 ;; Note we can also handle what would be fac(le|lt) by
4261 ;; generating fac(ge|gt).
4263 (define_insn "*aarch64_fac<optab><mode>"
4264 [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w")
4265 (neg:<V_cmp_result>
4266 (FAC_COMPARISONS:<V_cmp_result>
4267 (abs:VALLF (match_operand:VALLF 1 "register_operand" "w"))
4268 (abs:VALLF (match_operand:VALLF 2 "register_operand" "w"))
4269 )))]
4270 "TARGET_SIMD"
4271 "fac<n_optab>\t%<v>0<Vmtype>, %<v><cmp_1><Vmtype>, %<v><cmp_2><Vmtype>"
4272 [(set_attr "type" "neon_fp_compare_<Vetype><q>")]
4273 )
4275 ;; addp
4277 (define_insn "aarch64_addp<mode>"
4278 [(set (match_operand:VD_BHSI 0 "register_operand" "=w")
4279 (unspec:VD_BHSI
4280 [(match_operand:VD_BHSI 1 "register_operand" "w")
4281 (match_operand:VD_BHSI 2 "register_operand" "w")]
4282 UNSPEC_ADDP))]
4283 "TARGET_SIMD"
4284 "addp\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
4285 [(set_attr "type" "neon_reduc_add<q>")]
4286 )
4288 (define_insn "aarch64_addpdi"
4289 [(set (match_operand:DI 0 "register_operand" "=w")
4290 (unspec:DI
4291 [(match_operand:V2DI 1 "register_operand" "w")]
4292 UNSPEC_ADDP))]
4293 "TARGET_SIMD"
4294 "addp\t%d0, %1.2d"
4295 [(set_attr "type" "neon_reduc_add")]
4296 )
4298 ;; sqrt
4300 (define_insn "sqrt<mode>2"
4301 [(set (match_operand:VDQF 0 "register_operand" "=w")
4302 (sqrt:VDQF (match_operand:VDQF 1 "register_operand" "w")))]
4303 "TARGET_SIMD"
4304 "fsqrt\\t%0.<Vtype>, %1.<Vtype>"
4305 [(set_attr "type" "neon_fp_sqrt_<Vetype><q>")]
4306 )
4308 ;; Patterns for vector struct loads and stores.
4310 (define_insn "aarch64_simd_ld2<mode>"
4311 [(set (match_operand:OI 0 "register_operand" "=w")
4312 (unspec:OI [(match_operand:OI 1 "aarch64_simd_struct_operand" "Utv")
4313 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4314 UNSPEC_LD2))]
4315 "TARGET_SIMD"
4316 "ld2\\t{%S0.<Vtype> - %T0.<Vtype>}, %1"
4317 [(set_attr "type" "neon_load2_2reg<q>")]
4318 )
4320 (define_insn "aarch64_simd_ld2r<mode>"
4321 [(set (match_operand:OI 0 "register_operand" "=w")
4322 (unspec:OI [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")
4323 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY) ]
4324 UNSPEC_LD2_DUP))]
4325 "TARGET_SIMD"
4326 "ld2r\\t{%S0.<Vtype> - %T0.<Vtype>}, %1"
4327 [(set_attr "type" "neon_load2_all_lanes<q>")]
4328 )
4330 (define_insn "aarch64_vec_load_lanesoi_lane<mode>"
4331 [(set (match_operand:OI 0 "register_operand" "=w")
4332 (unspec:OI [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")
4333 (match_operand:OI 2 "register_operand" "0")
4334 (match_operand:SI 3 "immediate_operand" "i")
4335 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY) ]
4336 UNSPEC_LD2_LANE))]
4337 "TARGET_SIMD"
4338 {
4339 operands[3] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[3])));
4340 return "ld2\\t{%S0.<Vetype> - %T0.<Vetype>}[%3], %1";
4341 }
4342 [(set_attr "type" "neon_load2_one_lane")]
4343 )
4345 (define_expand "vec_load_lanesoi<mode>"
4346 [(set (match_operand:OI 0 "register_operand" "=w")
4347 (unspec:OI [(match_operand:OI 1 "aarch64_simd_struct_operand" "Utv")
4348 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4349 UNSPEC_LD2))]
4350 "TARGET_SIMD"
4351 {
4352 if (BYTES_BIG_ENDIAN)
4353 {
4354 rtx tmp = gen_reg_rtx (OImode);
4355 rtx mask = aarch64_reverse_mask (<MODE>mode);
4356 emit_insn (gen_aarch64_simd_ld2<mode> (tmp, operands[1]));
4357 emit_insn (gen_aarch64_rev_reglistoi (operands[0], tmp, mask));
4358 }
4359 else
4360 emit_insn (gen_aarch64_simd_ld2<mode> (operands[0], operands[1]));
4361 DONE;
4362 })
4364 (define_insn "aarch64_simd_st2<mode>"
4365 [(set (match_operand:OI 0 "aarch64_simd_struct_operand" "=Utv")
4366 (unspec:OI [(match_operand:OI 1 "register_operand" "w")
4367 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4368 UNSPEC_ST2))]
4369 "TARGET_SIMD"
4370 "st2\\t{%S1.<Vtype> - %T1.<Vtype>}, %0"
4371 [(set_attr "type" "neon_store2_2reg<q>")]
4372 )
4374 ;; RTL uses GCC vector extension indices, so flip only for assembly.
4375 (define_insn "aarch64_vec_store_lanesoi_lane<mode>"
4376 [(set (match_operand:BLK 0 "aarch64_simd_struct_operand" "=Utv")
4377 (unspec:BLK [(match_operand:OI 1 "register_operand" "w")
4378 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)
4379 (match_operand:SI 2 "immediate_operand" "i")]
4380 UNSPEC_ST2_LANE))]
4381 "TARGET_SIMD"
4382 {
4383 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
4384 return "st2\\t{%S1.<Vetype> - %T1.<Vetype>}[%2], %0";
4385 }
4386 [(set_attr "type" "neon_store2_one_lane<q>")]
4387 )
4389 (define_expand "vec_store_lanesoi<mode>"
4390 [(set (match_operand:OI 0 "aarch64_simd_struct_operand" "=Utv")
4391 (unspec:OI [(match_operand:OI 1 "register_operand" "w")
4392 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4393 UNSPEC_ST2))]
4394 "TARGET_SIMD"
4395 {
4396 if (BYTES_BIG_ENDIAN)
4397 {
4398 rtx tmp = gen_reg_rtx (OImode);
4399 rtx mask = aarch64_reverse_mask (<MODE>mode);
4400 emit_insn (gen_aarch64_rev_reglistoi (tmp, operands[1], mask));
4401 emit_insn (gen_aarch64_simd_st2<mode> (operands[0], tmp));
4402 }
4403 else
4404 emit_insn (gen_aarch64_simd_st2<mode> (operands[0], operands[1]));
4405 DONE;
4406 })
4408 (define_insn "aarch64_simd_ld3<mode>"
4409 [(set (match_operand:CI 0 "register_operand" "=w")
4410 (unspec:CI [(match_operand:CI 1 "aarch64_simd_struct_operand" "Utv")
4411 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4412 UNSPEC_LD3))]
4413 "TARGET_SIMD"
4414 "ld3\\t{%S0.<Vtype> - %U0.<Vtype>}, %1"
4415 [(set_attr "type" "neon_load3_3reg<q>")]
4416 )
4418 (define_insn "aarch64_simd_ld3r<mode>"
4419 [(set (match_operand:CI 0 "register_operand" "=w")
4420 (unspec:CI [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")
4421 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY) ]
4422 UNSPEC_LD3_DUP))]
4423 "TARGET_SIMD"
4424 "ld3r\\t{%S0.<Vtype> - %U0.<Vtype>}, %1"
4425 [(set_attr "type" "neon_load3_all_lanes<q>")]
4426 )
4428 (define_insn "aarch64_vec_load_lanesci_lane<mode>"
4429 [(set (match_operand:CI 0 "register_operand" "=w")
4430 (unspec:CI [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")
4431 (match_operand:CI 2 "register_operand" "0")
4432 (match_operand:SI 3 "immediate_operand" "i")
4433 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4434 UNSPEC_LD3_LANE))]
4435 "TARGET_SIMD"
4436 {
4437 operands[3] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[3])));
4438 return "ld3\\t{%S0.<Vetype> - %U0.<Vetype>}[%3], %1";
4439 }
4440 [(set_attr "type" "neon_load3_one_lane")]
4441 )
4443 (define_expand "vec_load_lanesci<mode>"
4444 [(set (match_operand:CI 0 "register_operand" "=w")
4445 (unspec:CI [(match_operand:CI 1 "aarch64_simd_struct_operand" "Utv")
4446 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4447 UNSPEC_LD3))]
4448 "TARGET_SIMD"
4449 {
4450 if (BYTES_BIG_ENDIAN)
4451 {
4452 rtx tmp = gen_reg_rtx (CImode);
4453 rtx mask = aarch64_reverse_mask (<MODE>mode);
4454 emit_insn (gen_aarch64_simd_ld3<mode> (tmp, operands[1]));
4455 emit_insn (gen_aarch64_rev_reglistci (operands[0], tmp, mask));
4456 }
4457 else
4458 emit_insn (gen_aarch64_simd_ld3<mode> (operands[0], operands[1]));
4459 DONE;
4460 })
4462 (define_insn "aarch64_simd_st3<mode>"
4463 [(set (match_operand:CI 0 "aarch64_simd_struct_operand" "=Utv")
4464 (unspec:CI [(match_operand:CI 1 "register_operand" "w")
4465 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4466 UNSPEC_ST3))]
4467 "TARGET_SIMD"
4468 "st3\\t{%S1.<Vtype> - %U1.<Vtype>}, %0"
4469 [(set_attr "type" "neon_store3_3reg<q>")]
4470 )
4472 ;; RTL uses GCC vector extension indices, so flip only for assembly.
4473 (define_insn "aarch64_vec_store_lanesci_lane<mode>"
4474 [(set (match_operand:BLK 0 "aarch64_simd_struct_operand" "=Utv")
4475 (unspec:BLK [(match_operand:CI 1 "register_operand" "w")
4476 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)
4477 (match_operand:SI 2 "immediate_operand" "i")]
4478 UNSPEC_ST3_LANE))]
4479 "TARGET_SIMD"
4480 {
4481 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
4482 return "st3\\t{%S1.<Vetype> - %U1.<Vetype>}[%2], %0";
4483 }
4484 [(set_attr "type" "neon_store3_one_lane<q>")]
4485 )
4487 (define_expand "vec_store_lanesci<mode>"
4488 [(set (match_operand:CI 0 "aarch64_simd_struct_operand" "=Utv")
4489 (unspec:CI [(match_operand:CI 1 "register_operand" "w")
4490 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4491 UNSPEC_ST3))]
4492 "TARGET_SIMD"
4493 {
4494 if (BYTES_BIG_ENDIAN)
4495 {
4496 rtx tmp = gen_reg_rtx (CImode);
4497 rtx mask = aarch64_reverse_mask (<MODE>mode);
4498 emit_insn (gen_aarch64_rev_reglistci (tmp, operands[1], mask));
4499 emit_insn (gen_aarch64_simd_st3<mode> (operands[0], tmp));
4500 }
4501 else
4502 emit_insn (gen_aarch64_simd_st3<mode> (operands[0], operands[1]));
4503 DONE;
4504 })
4506 (define_insn "aarch64_simd_ld4<mode>"
4507 [(set (match_operand:XI 0 "register_operand" "=w")
4508 (unspec:XI [(match_operand:XI 1 "aarch64_simd_struct_operand" "Utv")
4509 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4510 UNSPEC_LD4))]
4511 "TARGET_SIMD"
4512 "ld4\\t{%S0.<Vtype> - %V0.<Vtype>}, %1"
4513 [(set_attr "type" "neon_load4_4reg<q>")]
4514 )
4516 (define_insn "aarch64_simd_ld4r<mode>"
4517 [(set (match_operand:XI 0 "register_operand" "=w")
4518 (unspec:XI [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")
4519 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY) ]
4520 UNSPEC_LD4_DUP))]
4521 "TARGET_SIMD"
4522 "ld4r\\t{%S0.<Vtype> - %V0.<Vtype>}, %1"
4523 [(set_attr "type" "neon_load4_all_lanes<q>")]
4524 )
4526 (define_insn "aarch64_vec_load_lanesxi_lane<mode>"
4527 [(set (match_operand:XI 0 "register_operand" "=w")
4528 (unspec:XI [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")
4529 (match_operand:XI 2 "register_operand" "0")
4530 (match_operand:SI 3 "immediate_operand" "i")
4531 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4532 UNSPEC_LD4_LANE))]
4533 "TARGET_SIMD"
4534 {
4535 operands[3] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[3])));
4536 return "ld4\\t{%S0.<Vetype> - %V0.<Vetype>}[%3], %1";
4537 }
4538 [(set_attr "type" "neon_load4_one_lane")]
4539 )
4541 (define_expand "vec_load_lanesxi<mode>"
4542 [(set (match_operand:XI 0 "register_operand" "=w")
4543 (unspec:XI [(match_operand:XI 1 "aarch64_simd_struct_operand" "Utv")
4544 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4545 UNSPEC_LD4))]
4546 "TARGET_SIMD"
4547 {
4548 if (BYTES_BIG_ENDIAN)
4549 {
4550 rtx tmp = gen_reg_rtx (XImode);
4551 rtx mask = aarch64_reverse_mask (<MODE>mode);
4552 emit_insn (gen_aarch64_simd_ld4<mode> (tmp, operands[1]));
4553 emit_insn (gen_aarch64_rev_reglistxi (operands[0], tmp, mask));
4554 }
4555 else
4556 emit_insn (gen_aarch64_simd_ld4<mode> (operands[0], operands[1]));
4557 DONE;
4558 })
4560 (define_insn "aarch64_simd_st4<mode>"
4561 [(set (match_operand:XI 0 "aarch64_simd_struct_operand" "=Utv")
4562 (unspec:XI [(match_operand:XI 1 "register_operand" "w")
4563 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4564 UNSPEC_ST4))]
4565 "TARGET_SIMD"
4566 "st4\\t{%S1.<Vtype> - %V1.<Vtype>}, %0"
4567 [(set_attr "type" "neon_store4_4reg<q>")]
4568 )
4570 ;; RTL uses GCC vector extension indices, so flip only for assembly.
4571 (define_insn "aarch64_vec_store_lanesxi_lane<mode>"
4572 [(set (match_operand:BLK 0 "aarch64_simd_struct_operand" "=Utv")
4573 (unspec:BLK [(match_operand:XI 1 "register_operand" "w")
4574 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)
4575 (match_operand:SI 2 "immediate_operand" "i")]
4576 UNSPEC_ST4_LANE))]
4577 "TARGET_SIMD"
4578 {
4579 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
4580 return "st4\\t{%S1.<Vetype> - %V1.<Vetype>}[%2], %0";
4581 }
4582 [(set_attr "type" "neon_store4_one_lane<q>")]
4583 )
4585 (define_expand "vec_store_lanesxi<mode>"
4586 [(set (match_operand:XI 0 "aarch64_simd_struct_operand" "=Utv")
4587 (unspec:XI [(match_operand:XI 1 "register_operand" "w")
4588 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4589 UNSPEC_ST4))]
4590 "TARGET_SIMD"
4591 {
4592 if (BYTES_BIG_ENDIAN)
4593 {
4594 rtx tmp = gen_reg_rtx (XImode);
4595 rtx mask = aarch64_reverse_mask (<MODE>mode);
4596 emit_insn (gen_aarch64_rev_reglistxi (tmp, operands[1], mask));
4597 emit_insn (gen_aarch64_simd_st4<mode> (operands[0], tmp));
4598 }
4599 else
4600 emit_insn (gen_aarch64_simd_st4<mode> (operands[0], operands[1]));
4601 DONE;
4602 })
4604 (define_insn_and_split "aarch64_rev_reglist<mode>"
4605 [(set (match_operand:VSTRUCT 0 "register_operand" "=&w")
4606 (unspec:VSTRUCT
4607 [(match_operand:VSTRUCT 1 "register_operand" "w")
4608 (match_operand:V16QI 2 "register_operand" "w")]
4609 UNSPEC_REV_REGLIST))]
4610 "TARGET_SIMD"
4611 "#"
4612 "&& reload_completed"
4613 [(const_int 0)]
4614 {
4615 int i;
4616 int nregs = GET_MODE_SIZE (<MODE>mode) / UNITS_PER_VREG;
4617 for (i = 0; i < nregs; i++)
4618 {
4619 rtx op0 = gen_rtx_REG (V16QImode, REGNO (operands[0]) + i);
4620 rtx op1 = gen_rtx_REG (V16QImode, REGNO (operands[1]) + i);
4621 emit_insn (gen_aarch64_tbl1v16qi (op0, op1, operands[2]));
4622 }
4623 DONE;
4624 }
4625 [(set_attr "type" "neon_tbl1_q")
4626 (set_attr "length" "<insn_count>")]
4627 )
4629 ;; Reload patterns for AdvSIMD register list operands.
4631 (define_expand "mov<mode>"
4632 [(set (match_operand:VSTRUCT 0 "nonimmediate_operand" "")
4633 (match_operand:VSTRUCT 1 "general_operand" ""))]
4634 "TARGET_SIMD"
4635 {
4636 if (can_create_pseudo_p ())
4637 {
4638 if (GET_CODE (operands[0]) != REG)
4639 operands[1] = force_reg (<MODE>mode, operands[1]);
4640 }
4641 })
4643 (define_insn "*aarch64_mov<mode>"
4644 [(set (match_operand:VSTRUCT 0 "aarch64_simd_nonimmediate_operand" "=w,Utv,w")
4645 (match_operand:VSTRUCT 1 "aarch64_simd_general_operand" " w,w,Utv"))]
4646 "TARGET_SIMD && !BYTES_BIG_ENDIAN
4647 && (register_operand (operands[0], <MODE>mode)
4648 || register_operand (operands[1], <MODE>mode))"
4649 "@
4650 #
4651 st1\\t{%S1.16b - %<Vendreg>1.16b}, %0
4652 ld1\\t{%S0.16b - %<Vendreg>0.16b}, %1"
4653 [(set_attr "type" "multiple,neon_store<nregs>_<nregs>reg_q,\
4654 neon_load<nregs>_<nregs>reg_q")
4655 (set (attr "length") (symbol_ref "aarch64_simd_attr_length_move (insn)"))]
4656 )
4658 (define_insn "aarch64_be_ld1<mode>"
4659 [(set (match_operand:VALLDI_F16 0 "register_operand" "=w")
4660 (unspec:VALLDI_F16 [(match_operand:VALLDI_F16 1
4661 "aarch64_simd_struct_operand" "Utv")]
4662 UNSPEC_LD1))]
4663 "TARGET_SIMD"
4664 "ld1\\t{%0<Vmtype>}, %1"
4665 [(set_attr "type" "neon_load1_1reg<q>")]
4666 )
4668 (define_insn "aarch64_be_st1<mode>"
4669 [(set (match_operand:VALLDI_F16 0 "aarch64_simd_struct_operand" "=Utv")
4670 (unspec:VALLDI_F16 [(match_operand:VALLDI_F16 1 "register_operand" "w")]
4671 UNSPEC_ST1))]
4672 "TARGET_SIMD"
4673 "st1\\t{%1<Vmtype>}, %0"
4674 [(set_attr "type" "neon_store1_1reg<q>")]
4675 )
4677 (define_insn "*aarch64_be_movoi"
4678 [(set (match_operand:OI 0 "nonimmediate_operand" "=w,m,w")
4679 (match_operand:OI 1 "general_operand" " w,w,m"))]
4680 "TARGET_SIMD && BYTES_BIG_ENDIAN
4681 && (register_operand (operands[0], OImode)
4682 || register_operand (operands[1], OImode))"
4683 "@
4684 #
4685 stp\\t%q1, %R1, %0
4686 ldp\\t%q0, %R0, %1"
4687 [(set_attr "type" "multiple,neon_stp_q,neon_ldp_q")
4688 (set (attr "length") (symbol_ref "aarch64_simd_attr_length_move (insn)"))]
4689 )
4691 (define_insn "*aarch64_be_movci"
4692 [(set (match_operand:CI 0 "nonimmediate_operand" "=w,o,w")
4693 (match_operand:CI 1 "general_operand" " w,w,o"))]
4694 "TARGET_SIMD && BYTES_BIG_ENDIAN
4695 && (register_operand (operands[0], CImode)
4696 || register_operand (operands[1], CImode))"
4697 "#"
4698 [(set_attr "type" "multiple")
4699 (set (attr "length") (symbol_ref "aarch64_simd_attr_length_move (insn)"))]
4700 )
4702 (define_insn "*aarch64_be_movxi"
4703 [(set (match_operand:XI 0 "nonimmediate_operand" "=w,o,w")
4704 (match_operand:XI 1 "general_operand" " w,w,o"))]
4705 "TARGET_SIMD && BYTES_BIG_ENDIAN
4706 && (register_operand (operands[0], XImode)
4707 || register_operand (operands[1], XImode))"
4708 "#"
4709 [(set_attr "type" "multiple")
4710 (set (attr "length") (symbol_ref "aarch64_simd_attr_length_move (insn)"))]
4711 )
4713 (define_split
4714 [(set (match_operand:OI 0 "register_operand")
4715 (match_operand:OI 1 "register_operand"))]
4716 "TARGET_SIMD && reload_completed"
4717 [(const_int 0)]
4718 {
4719 aarch64_simd_emit_reg_reg_move (operands, TImode, 2);
4720 DONE;
4721 })
4723 (define_split
4724 [(set (match_operand:CI 0 "nonimmediate_operand")
4725 (match_operand:CI 1 "general_operand"))]
4726 "TARGET_SIMD && reload_completed"
4727 [(const_int 0)]
4728 {
4729 if (register_operand (operands[0], CImode)
4730 && register_operand (operands[1], CImode))
4731 {
4732 aarch64_simd_emit_reg_reg_move (operands, TImode, 3);
4733 DONE;
4734 }
4735 else if (BYTES_BIG_ENDIAN)
4736 {
4737 emit_move_insn (simplify_gen_subreg (OImode, operands[0], CImode, 0),
4738 simplify_gen_subreg (OImode, operands[1], CImode, 0));
4739 emit_move_insn (gen_lowpart (V16QImode,
4740 simplify_gen_subreg (TImode, operands[0],
4741 CImode, 32)),
4742 gen_lowpart (V16QImode,
4743 simplify_gen_subreg (TImode, operands[1],
4744 CImode, 32)));
4745 DONE;
4746 }
4747 else
4748 FAIL;
4749 })
4751 (define_split
4752 [(set (match_operand:XI 0 "nonimmediate_operand")
4753 (match_operand:XI 1 "general_operand"))]
4754 "TARGET_SIMD && reload_completed"
4755 [(const_int 0)]
4756 {
4757 if (register_operand (operands[0], XImode)
4758 && register_operand (operands[1], XImode))
4759 {
4760 aarch64_simd_emit_reg_reg_move (operands, TImode, 4);
4761 DONE;
4762 }
4763 else if (BYTES_BIG_ENDIAN)
4764 {
4765 emit_move_insn (simplify_gen_subreg (OImode, operands[0], XImode, 0),
4766 simplify_gen_subreg (OImode, operands[1], XImode, 0));
4767 emit_move_insn (simplify_gen_subreg (OImode, operands[0], XImode, 32),
4768 simplify_gen_subreg (OImode, operands[1], XImode, 32));
4769 DONE;
4770 }
4771 else
4772 FAIL;
4773 })
4775 (define_expand "aarch64_ld<VSTRUCT:nregs>r<VALLDIF:mode>"
4776 [(match_operand:VSTRUCT 0 "register_operand" "=w")
4777 (match_operand:DI 1 "register_operand" "w")
4778 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4779 "TARGET_SIMD"
4780 {
4781 rtx mem = gen_rtx_MEM (BLKmode, operands[1]);
4782 set_mem_size (mem, GET_MODE_SIZE (GET_MODE_INNER (<VALLDIF:MODE>mode))
4783 * <VSTRUCT:nregs>);
4785 emit_insn (gen_aarch64_simd_ld<VSTRUCT:nregs>r<VALLDIF:mode> (operands[0],
4786 mem));
4787 DONE;
4788 })
4790 (define_insn "aarch64_ld2<mode>_dreg"
4791 [(set (match_operand:OI 0 "register_operand" "=w")
4792 (subreg:OI
4793 (vec_concat:<VRL2>
4794 (vec_concat:<VDBL>
4795 (unspec:VD
4796 [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")]
4797 UNSPEC_LD2)
4798 (vec_duplicate:VD (const_int 0)))
4799 (vec_concat:<VDBL>
4800 (unspec:VD [(match_dup 1)]
4801 UNSPEC_LD2)
4802 (vec_duplicate:VD (const_int 0)))) 0))]
4803 "TARGET_SIMD"
4804 "ld2\\t{%S0.<Vtype> - %T0.<Vtype>}, %1"
4805 [(set_attr "type" "neon_load2_2reg<q>")]
4806 )
4808 (define_insn "aarch64_ld2<mode>_dreg"
4809 [(set (match_operand:OI 0 "register_operand" "=w")
4810 (subreg:OI
4811 (vec_concat:<VRL2>
4812 (vec_concat:<VDBL>
4813 (unspec:DX
4814 [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")]
4815 UNSPEC_LD2)
4816 (const_int 0))
4817 (vec_concat:<VDBL>
4818 (unspec:DX [(match_dup 1)]
4819 UNSPEC_LD2)
4820 (const_int 0))) 0))]
4821 "TARGET_SIMD"
4822 "ld1\\t{%S0.1d - %T0.1d}, %1"
4823 [(set_attr "type" "neon_load1_2reg<q>")]
4824 )
4826 (define_insn "aarch64_ld3<mode>_dreg"
4827 [(set (match_operand:CI 0 "register_operand" "=w")
4828 (subreg:CI
4829 (vec_concat:<VRL3>
4830 (vec_concat:<VRL2>
4831 (vec_concat:<VDBL>
4832 (unspec:VD
4833 [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")]
4834 UNSPEC_LD3)
4835 (vec_duplicate:VD (const_int 0)))
4836 (vec_concat:<VDBL>
4837 (unspec:VD [(match_dup 1)]
4838 UNSPEC_LD3)
4839 (vec_duplicate:VD (const_int 0))))
4840 (vec_concat:<VDBL>
4841 (unspec:VD [(match_dup 1)]
4842 UNSPEC_LD3)
4843 (vec_duplicate:VD (const_int 0)))) 0))]
4844 "TARGET_SIMD"
4845 "ld3\\t{%S0.<Vtype> - %U0.<Vtype>}, %1"
4846 [(set_attr "type" "neon_load3_3reg<q>")]
4847 )
4849 (define_insn "aarch64_ld3<mode>_dreg"
4850 [(set (match_operand:CI 0 "register_operand" "=w")
4851 (subreg:CI
4852 (vec_concat:<VRL3>
4853 (vec_concat:<VRL2>
4854 (vec_concat:<VDBL>
4855 (unspec:DX
4856 [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")]
4857 UNSPEC_LD3)
4858 (const_int 0))
4859 (vec_concat:<VDBL>
4860 (unspec:DX [(match_dup 1)]
4861 UNSPEC_LD3)
4862 (const_int 0)))
4863 (vec_concat:<VDBL>
4864 (unspec:DX [(match_dup 1)]
4865 UNSPEC_LD3)
4866 (const_int 0))) 0))]
4867 "TARGET_SIMD"
4868 "ld1\\t{%S0.1d - %U0.1d}, %1"
4869 [(set_attr "type" "neon_load1_3reg<q>")]
4870 )
4872 (define_insn "aarch64_ld4<mode>_dreg"
4873 [(set (match_operand:XI 0 "register_operand" "=w")
4874 (subreg:XI
4875 (vec_concat:<VRL4>
4876 (vec_concat:<VRL2>
4877 (vec_concat:<VDBL>
4878 (unspec:VD
4879 [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")]
4880 UNSPEC_LD4)
4881 (vec_duplicate:VD (const_int 0)))
4882 (vec_concat:<VDBL>
4883 (unspec:VD [(match_dup 1)]
4884 UNSPEC_LD4)
4885 (vec_duplicate:VD (const_int 0))))
4886 (vec_concat:<VRL2>
4887 (vec_concat:<VDBL>
4888 (unspec:VD [(match_dup 1)]
4889 UNSPEC_LD4)
4890 (vec_duplicate:VD (const_int 0)))
4891 (vec_concat:<VDBL>
4892 (unspec:VD [(match_dup 1)]
4893 UNSPEC_LD4)
4894 (vec_duplicate:VD (const_int 0))))) 0))]
4895 "TARGET_SIMD"
4896 "ld4\\t{%S0.<Vtype> - %V0.<Vtype>}, %1"
4897 [(set_attr "type" "neon_load4_4reg<q>")]
4898 )
4900 (define_insn "aarch64_ld4<mode>_dreg"
4901 [(set (match_operand:XI 0 "register_operand" "=w")
4902 (subreg:XI
4903 (vec_concat:<VRL4>
4904 (vec_concat:<VRL2>
4905 (vec_concat:<VDBL>
4906 (unspec:DX
4907 [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")]
4908 UNSPEC_LD4)
4909 (const_int 0))
4910 (vec_concat:<VDBL>
4911 (unspec:DX [(match_dup 1)]
4912 UNSPEC_LD4)
4913 (const_int 0)))
4914 (vec_concat:<VRL2>
4915 (vec_concat:<VDBL>
4916 (unspec:DX [(match_dup 1)]
4917 UNSPEC_LD4)
4918 (const_int 0))
4919 (vec_concat:<VDBL>
4920 (unspec:DX [(match_dup 1)]
4921 UNSPEC_LD4)
4922 (const_int 0)))) 0))]
4923 "TARGET_SIMD"
4924 "ld1\\t{%S0.1d - %V0.1d}, %1"
4925 [(set_attr "type" "neon_load1_4reg<q>")]
4926 )
4928 (define_expand "aarch64_ld<VSTRUCT:nregs><VDC:mode>"
4929 [(match_operand:VSTRUCT 0 "register_operand" "=w")
4930 (match_operand:DI 1 "register_operand" "r")
4931 (unspec:VDC [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4932 "TARGET_SIMD"
4933 {
4934 rtx mem = gen_rtx_MEM (BLKmode, operands[1]);
4935 set_mem_size (mem, <VSTRUCT:nregs> * 8);
4937 emit_insn (gen_aarch64_ld<VSTRUCT:nregs><VDC:mode>_dreg (operands[0], mem));
4938 DONE;
4939 })
4941 (define_expand "aarch64_ld1<VALL_F16:mode>"
4942 [(match_operand:VALL_F16 0 "register_operand")
4943 (match_operand:DI 1 "register_operand")]
4944 "TARGET_SIMD"
4945 {
4946 machine_mode mode = <VALL_F16:MODE>mode;
4947 rtx mem = gen_rtx_MEM (mode, operands[1]);
4949 if (BYTES_BIG_ENDIAN)
4950 emit_insn (gen_aarch64_be_ld1<VALL_F16:mode> (operands[0], mem));
4951 else
4952 emit_move_insn (operands[0], mem);
4953 DONE;
4954 })
4956 (define_expand "aarch64_ld<VSTRUCT:nregs><VQ:mode>"
4957 [(match_operand:VSTRUCT 0 "register_operand" "=w")
4958 (match_operand:DI 1 "register_operand" "r")
4959 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4960 "TARGET_SIMD"
4961 {
4962 machine_mode mode = <VSTRUCT:MODE>mode;
4963 rtx mem = gen_rtx_MEM (mode, operands[1]);
4965 emit_insn (gen_aarch64_simd_ld<VSTRUCT:nregs><VQ:mode> (operands[0], mem));
4966 DONE;
4967 })
4969 (define_expand "aarch64_ld<VSTRUCT:nregs>_lane<VALLDIF:mode>"
4970 [(match_operand:VSTRUCT 0 "register_operand" "=w")
4971 (match_operand:DI 1 "register_operand" "w")
4972 (match_operand:VSTRUCT 2 "register_operand" "0")
4973 (match_operand:SI 3 "immediate_operand" "i")
4974 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4975 "TARGET_SIMD"
4976 {
4977 rtx mem = gen_rtx_MEM (BLKmode, operands[1]);
4978 set_mem_size (mem, GET_MODE_SIZE (GET_MODE_INNER (<VALLDIF:MODE>mode))
4979 * <VSTRUCT:nregs>);
4981 aarch64_simd_lane_bounds (operands[3], 0,
4982 GET_MODE_NUNITS (<VALLDIF:MODE>mode),
4983 NULL);
4984 emit_insn (gen_aarch64_vec_load_lanes<VSTRUCT:mode>_lane<VALLDIF:mode> (
4985 operands[0], mem, operands[2], operands[3]));
4986 DONE;
4987 })
4989 ;; Expanders for builtins to extract vector registers from large
4990 ;; opaque integer modes.
4992 ;; D-register list.
4994 (define_expand "aarch64_get_dreg<VSTRUCT:mode><VDC:mode>"
4995 [(match_operand:VDC 0 "register_operand" "=w")
4996 (match_operand:VSTRUCT 1 "register_operand" "w")
4997 (match_operand:SI 2 "immediate_operand" "i")]
4998 "TARGET_SIMD"
4999 {
5000 int part = INTVAL (operands[2]);
5001 rtx temp = gen_reg_rtx (<VDC:VDBL>mode);
5002 int offset = part * 16;
5004 emit_move_insn (temp, gen_rtx_SUBREG (<VDC:VDBL>mode, operands[1], offset));
5005 emit_move_insn (operands[0], gen_lowpart (<VDC:MODE>mode, temp));
5006 DONE;
5007 })
5009 ;; Q-register list.
5011 (define_expand "aarch64_get_qreg<VSTRUCT:mode><VQ:mode>"
5012 [(match_operand:VQ 0 "register_operand" "=w")
5013 (match_operand:VSTRUCT 1 "register_operand" "w")
5014 (match_operand:SI 2 "immediate_operand" "i")]
5015 "TARGET_SIMD"
5016 {
5017 int part = INTVAL (operands[2]);
5018 int offset = part * 16;
5020 emit_move_insn (operands[0],
5021 gen_rtx_SUBREG (<VQ:MODE>mode, operands[1], offset));
5022 DONE;
5023 })
5025 ;; Permuted-store expanders for neon intrinsics.
5027 ;; Permute instructions
5029 ;; vec_perm support
5031 (define_expand "vec_perm_const<mode>"
5032 [(match_operand:VALL_F16 0 "register_operand")
5033 (match_operand:VALL_F16 1 "register_operand")
5034 (match_operand:VALL_F16 2 "register_operand")
5035 (match_operand:<V_cmp_result> 3)]
5036 "TARGET_SIMD"
5037 {
5038 if (aarch64_expand_vec_perm_const (operands[0], operands[1],
5039 operands[2], operands[3]))
5040 DONE;
5041 else
5042 FAIL;
5043 })
5045 (define_expand "vec_perm<mode>"
5046 [(match_operand:VB 0 "register_operand")
5047 (match_operand:VB 1 "register_operand")
5048 (match_operand:VB 2 "register_operand")
5049 (match_operand:VB 3 "register_operand")]
5050 "TARGET_SIMD"
5051 {
5052 aarch64_expand_vec_perm (operands[0], operands[1],
5053 operands[2], operands[3]);
5054 DONE;
5055 })
5057 (define_insn "aarch64_tbl1<mode>"
5058 [(set (match_operand:VB 0 "register_operand" "=w")
5059 (unspec:VB [(match_operand:V16QI 1 "register_operand" "w")
5060 (match_operand:VB 2 "register_operand" "w")]
5061 UNSPEC_TBL))]
5062 "TARGET_SIMD"
5063 "tbl\\t%0.<Vtype>, {%1.16b}, %2.<Vtype>"
5064 [(set_attr "type" "neon_tbl1<q>")]
5065 )
5067 ;; Two source registers.
5069 (define_insn "aarch64_tbl2v16qi"
5070 [(set (match_operand:V16QI 0 "register_operand" "=w")
5071 (unspec:V16QI [(match_operand:OI 1 "register_operand" "w")
5072 (match_operand:V16QI 2 "register_operand" "w")]
5073 UNSPEC_TBL))]
5074 "TARGET_SIMD"
5075 "tbl\\t%0.16b, {%S1.16b - %T1.16b}, %2.16b"
5076 [(set_attr "type" "neon_tbl2_q")]
5077 )
5079 (define_insn "aarch64_tbl3<mode>"
5080 [(set (match_operand:VB 0 "register_operand" "=w")
5081 (unspec:VB [(match_operand:OI 1 "register_operand" "w")
5082 (match_operand:VB 2 "register_operand" "w")]
5083 UNSPEC_TBL))]
5084 "TARGET_SIMD"
5085 "tbl\\t%S0.<Vbtype>, {%S1.16b - %T1.16b}, %S2.<Vbtype>"
5086 [(set_attr "type" "neon_tbl3")]
5087 )
5089 (define_insn "aarch64_tbx4<mode>"
5090 [(set (match_operand:VB 0 "register_operand" "=w")
5091 (unspec:VB [(match_operand:VB 1 "register_operand" "0")
5092 (match_operand:OI 2 "register_operand" "w")
5093 (match_operand:VB 3 "register_operand" "w")]
5094 UNSPEC_TBX))]
5095 "TARGET_SIMD"
5096 "tbx\\t%S0.<Vbtype>, {%S2.16b - %T2.16b}, %S3.<Vbtype>"
5097 [(set_attr "type" "neon_tbl4")]
5098 )
5100 ;; Three source registers.
5102 (define_insn "aarch64_qtbl3<mode>"
5103 [(set (match_operand:VB 0 "register_operand" "=w")
5104 (unspec:VB [(match_operand:CI 1 "register_operand" "w")
5105 (match_operand:VB 2 "register_operand" "w")]
5106 UNSPEC_TBL))]
5107 "TARGET_SIMD"
5108 "tbl\\t%S0.<Vbtype>, {%S1.16b - %U1.16b}, %S2.<Vbtype>"
5109 [(set_attr "type" "neon_tbl3")]
5110 )
5112 (define_insn "aarch64_qtbx3<mode>"
5113 [(set (match_operand:VB 0 "register_operand" "=w")
5114 (unspec:VB [(match_operand:VB 1 "register_operand" "0")
5115 (match_operand:CI 2 "register_operand" "w")
5116 (match_operand:VB 3 "register_operand" "w")]
5117 UNSPEC_TBX))]
5118 "TARGET_SIMD"
5119 "tbx\\t%S0.<Vbtype>, {%S2.16b - %U2.16b}, %S3.<Vbtype>"
5120 [(set_attr "type" "neon_tbl3")]
5121 )
5123 ;; Four source registers.
5125 (define_insn "aarch64_qtbl4<mode>"
5126 [(set (match_operand:VB 0 "register_operand" "=w")
5127 (unspec:VB [(match_operand:XI 1 "register_operand" "w")
5128 (match_operand:VB 2 "register_operand" "w")]
5129 UNSPEC_TBL))]
5130 "TARGET_SIMD"
5131 "tbl\\t%S0.<Vbtype>, {%S1.16b - %V1.16b}, %S2.<Vbtype>"
5132 [(set_attr "type" "neon_tbl4")]
5133 )
5135 (define_insn "aarch64_qtbx4<mode>"
5136 [(set (match_operand:VB 0 "register_operand" "=w")
5137 (unspec:VB [(match_operand:VB 1 "register_operand" "0")
5138 (match_operand:XI 2 "register_operand" "w")
5139 (match_operand:VB 3 "register_operand" "w")]
5140 UNSPEC_TBX))]
5141 "TARGET_SIMD"
5142 "tbx\\t%S0.<Vbtype>, {%S2.16b - %V2.16b}, %S3.<Vbtype>"
5143 [(set_attr "type" "neon_tbl4")]
5144 )
5146 (define_insn_and_split "aarch64_combinev16qi"
5147 [(set (match_operand:OI 0 "register_operand" "=w")
5148 (unspec:OI [(match_operand:V16QI 1 "register_operand" "w")
5149 (match_operand:V16QI 2 "register_operand" "w")]
5150 UNSPEC_CONCAT))]
5151 "TARGET_SIMD"
5152 "#"
5153 "&& reload_completed"
5154 [(const_int 0)]
5155 {
5156 aarch64_split_combinev16qi (operands);
5157 DONE;
5158 }
5159 [(set_attr "type" "multiple")]
5160 )
5162 (define_insn "aarch64_<PERMUTE:perm_insn><PERMUTE:perm_hilo><mode>"
5163 [(set (match_operand:VALL 0 "register_operand" "=w")
5164 (unspec:VALL [(match_operand:VALL 1 "register_operand" "w")
5165 (match_operand:VALL 2 "register_operand" "w")]
5166 PERMUTE))]
5167 "TARGET_SIMD"
5168 "<PERMUTE:perm_insn><PERMUTE:perm_hilo>\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
5169 [(set_attr "type" "neon_permute<q>")]
5170 )
5172 ;; Note immediate (third) operand is lane index not byte index.
5173 (define_insn "aarch64_ext<mode>"
5174 [(set (match_operand:VALL 0 "register_operand" "=w")
5175 (unspec:VALL [(match_operand:VALL 1 "register_operand" "w")
5176 (match_operand:VALL 2 "register_operand" "w")
5177 (match_operand:SI 3 "immediate_operand" "i")]
5178 UNSPEC_EXT))]
5179 "TARGET_SIMD"
5180 {
5181 operands[3] = GEN_INT (INTVAL (operands[3])
5182 * GET_MODE_UNIT_SIZE (<MODE>mode));
5183 return "ext\\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>, #%3";
5184 }
5185 [(set_attr "type" "neon_ext<q>")]
5186 )
5188 (define_insn "aarch64_rev<REVERSE:rev_op><mode>"
5189 [(set (match_operand:VALL 0 "register_operand" "=w")
5190 (unspec:VALL [(match_operand:VALL 1 "register_operand" "w")]
5191 REVERSE))]
5192 "TARGET_SIMD"
5193 "rev<REVERSE:rev_op>\\t%0.<Vtype>, %1.<Vtype>"
5194 [(set_attr "type" "neon_rev<q>")]
5195 )
5197 (define_insn "aarch64_st2<mode>_dreg"
5198 [(set (match_operand:BLK 0 "aarch64_simd_struct_operand" "=Utv")
5199 (unspec:BLK [(match_operand:OI 1 "register_operand" "w")
5200 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5201 UNSPEC_ST2))]
5202 "TARGET_SIMD"
5203 "st2\\t{%S1.<Vtype> - %T1.<Vtype>}, %0"
5204 [(set_attr "type" "neon_store2_2reg")]
5205 )
5207 (define_insn "aarch64_st2<mode>_dreg"
5208 [(set (match_operand:BLK 0 "aarch64_simd_struct_operand" "=Utv")
5209 (unspec:BLK [(match_operand:OI 1 "register_operand" "w")
5210 (unspec:DX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5211 UNSPEC_ST2))]
5212 "TARGET_SIMD"
5213 "st1\\t{%S1.1d - %T1.1d}, %0"
5214 [(set_attr "type" "neon_store1_2reg")]
5215 )
5217 (define_insn "aarch64_st3<mode>_dreg"
5218 [(set (match_operand:BLK 0 "aarch64_simd_struct_operand" "=Utv")
5219 (unspec:BLK [(match_operand:CI 1 "register_operand" "w")
5220 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5221 UNSPEC_ST3))]
5222 "TARGET_SIMD"
5223 "st3\\t{%S1.<Vtype> - %U1.<Vtype>}, %0"
5224 [(set_attr "type" "neon_store3_3reg")]
5225 )
5227 (define_insn "aarch64_st3<mode>_dreg"
5228 [(set (match_operand:BLK 0 "aarch64_simd_struct_operand" "=Utv")
5229 (unspec:BLK [(match_operand:CI 1 "register_operand" "w")
5230 (unspec:DX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5231 UNSPEC_ST3))]
5232 "TARGET_SIMD"
5233 "st1\\t{%S1.1d - %U1.1d}, %0"
5234 [(set_attr "type" "neon_store1_3reg")]
5235 )
5237 (define_insn "aarch64_st4<mode>_dreg"
5238 [(set (match_operand:BLK 0 "aarch64_simd_struct_operand" "=Utv")
5239 (unspec:BLK [(match_operand:XI 1 "register_operand" "w")
5240 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5241 UNSPEC_ST4))]
5242 "TARGET_SIMD"
5243 "st4\\t{%S1.<Vtype> - %V1.<Vtype>}, %0"
5244 [(set_attr "type" "neon_store4_4reg")]
5245 )
5247 (define_insn "aarch64_st4<mode>_dreg"
5248 [(set (match_operand:BLK 0 "aarch64_simd_struct_operand" "=Utv")
5249 (unspec:BLK [(match_operand:XI 1 "register_operand" "w")
5250 (unspec:DX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5251 UNSPEC_ST4))]
5252 "TARGET_SIMD"
5253 "st1\\t{%S1.1d - %V1.1d}, %0"
5254 [(set_attr "type" "neon_store1_4reg")]
5255 )
5257 (define_expand "aarch64_st<VSTRUCT:nregs><VDC:mode>"
5258 [(match_operand:DI 0 "register_operand" "r")
5259 (match_operand:VSTRUCT 1 "register_operand" "w")
5260 (unspec:VDC [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5261 "TARGET_SIMD"
5262 {
5263 rtx mem = gen_rtx_MEM (BLKmode, operands[0]);
5264 set_mem_size (mem, <VSTRUCT:nregs> * 8);
5266 emit_insn (gen_aarch64_st<VSTRUCT:nregs><VDC:mode>_dreg (mem, operands[1]));
5267 DONE;
5268 })
5270 (define_expand "aarch64_st<VSTRUCT:nregs><VQ:mode>"
5271 [(match_operand:DI 0 "register_operand" "r")
5272 (match_operand:VSTRUCT 1 "register_operand" "w")
5273 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5274 "TARGET_SIMD"
5275 {
5276 machine_mode mode = <VSTRUCT:MODE>mode;
5277 rtx mem = gen_rtx_MEM (mode, operands[0]);
5279 emit_insn (gen_aarch64_simd_st<VSTRUCT:nregs><VQ:mode> (mem, operands[1]));
5280 DONE;
5281 })
5283 (define_expand "aarch64_st<VSTRUCT:nregs>_lane<VALLDIF:mode>"
5284 [(match_operand:DI 0 "register_operand" "r")
5285 (match_operand:VSTRUCT 1 "register_operand" "w")
5286 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)
5287 (match_operand:SI 2 "immediate_operand")]
5288 "TARGET_SIMD"
5289 {
5290 rtx mem = gen_rtx_MEM (BLKmode, operands[0]);
5291 set_mem_size (mem, GET_MODE_SIZE (GET_MODE_INNER (<VALLDIF:MODE>mode))
5292 * <VSTRUCT:nregs>);
5294 emit_insn (gen_aarch64_vec_store_lanes<VSTRUCT:mode>_lane<VALLDIF:mode> (
5295 mem, operands[1], operands[2]));
5296 DONE;
5297 })
5299 (define_expand "aarch64_st1<VALL_F16:mode>"
5300 [(match_operand:DI 0 "register_operand")
5301 (match_operand:VALL_F16 1 "register_operand")]
5302 "TARGET_SIMD"
5303 {
5304 machine_mode mode = <VALL_F16:MODE>mode;
5305 rtx mem = gen_rtx_MEM (mode, operands[0]);
5307 if (BYTES_BIG_ENDIAN)
5308 emit_insn (gen_aarch64_be_st1<VALL_F16:mode> (mem, operands[1]));
5309 else
5310 emit_move_insn (mem, operands[1]);
5311 DONE;
5312 })
5314 ;; Expander for builtins to insert vector registers into large
5315 ;; opaque integer modes.
5317 ;; Q-register list. We don't need a D-reg inserter as we zero
5318 ;; extend them in arm_neon.h and insert the resulting Q-regs.
5320 (define_expand "aarch64_set_qreg<VSTRUCT:mode><VQ:mode>"
5321 [(match_operand:VSTRUCT 0 "register_operand" "+w")
5322 (match_operand:VSTRUCT 1 "register_operand" "0")
5323 (match_operand:VQ 2 "register_operand" "w")
5324 (match_operand:SI 3 "immediate_operand" "i")]
5325 "TARGET_SIMD"
5326 {
5327 int part = INTVAL (operands[3]);
5328 int offset = part * 16;
5330 emit_move_insn (operands[0], operands[1]);
5331 emit_move_insn (gen_rtx_SUBREG (<VQ:MODE>mode, operands[0], offset),
5332 operands[2]);
5333 DONE;
5334 })
5336 ;; Standard pattern name vec_init<mode>.
5338 (define_expand "vec_init<mode>"
5339 [(match_operand:VALL_F16 0 "register_operand" "")
5340 (match_operand 1 "" "")]
5341 "TARGET_SIMD"
5342 {
5343 aarch64_expand_vector_init (operands[0], operands[1]);
5344 DONE;
5345 })
5347 (define_insn "*aarch64_simd_ld1r<mode>"
5348 [(set (match_operand:VALL_F16 0 "register_operand" "=w")
5349 (vec_duplicate:VALL_F16
5350 (match_operand:<VEL> 1 "aarch64_simd_struct_operand" "Utv")))]
5351 "TARGET_SIMD"
5352 "ld1r\\t{%0.<Vtype>}, %1"
5353 [(set_attr "type" "neon_load1_all_lanes")]
5354 )
5356 (define_insn "aarch64_frecpe<mode>"
5357 [(set (match_operand:VDQF 0 "register_operand" "=w")
5358 (unspec:VDQF [(match_operand:VDQF 1 "register_operand" "w")]
5359 UNSPEC_FRECPE))]
5360 "TARGET_SIMD"
5361 "frecpe\\t%0.<Vtype>, %1.<Vtype>"
5362 [(set_attr "type" "neon_fp_recpe_<Vetype><q>")]
5363 )
5365 (define_insn "aarch64_frecp<FRECP:frecp_suffix><mode>"
5366 [(set (match_operand:GPF 0 "register_operand" "=w")
5367 (unspec:GPF [(match_operand:GPF 1 "register_operand" "w")]
5368 FRECP))]
5369 "TARGET_SIMD"
5370 "frecp<FRECP:frecp_suffix>\\t%<s>0, %<s>1"
5371 [(set_attr "type" "neon_fp_recp<FRECP:frecp_suffix>_<GPF:Vetype><GPF:q>")]
5372 )
5374 (define_insn "aarch64_frecps<mode>"
5375 [(set (match_operand:VALLF 0 "register_operand" "=w")
5376 (unspec:VALLF [(match_operand:VALLF 1 "register_operand" "w")
5377 (match_operand:VALLF 2 "register_operand" "w")]
5378 UNSPEC_FRECPS))]
5379 "TARGET_SIMD"
5380 "frecps\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
5381 [(set_attr "type" "neon_fp_recps_<Vetype><q>")]
5382 )
5384 (define_insn "aarch64_urecpe<mode>"
5385 [(set (match_operand:VDQ_SI 0 "register_operand" "=w")
5386 (unspec:VDQ_SI [(match_operand:VDQ_SI 1 "register_operand" "w")]
5387 UNSPEC_URECPE))]
5388 "TARGET_SIMD"
5389 "urecpe\\t%0.<Vtype>, %1.<Vtype>"
5390 [(set_attr "type" "neon_fp_recpe_<Vetype><q>")])
5392 ;; Standard pattern name vec_extract<mode>.
5394 (define_expand "vec_extract<mode>"
5395 [(match_operand:<VEL> 0 "aarch64_simd_nonimmediate_operand" "")
5396 (match_operand:VALL_F16 1 "register_operand" "")
5397 (match_operand:SI 2 "immediate_operand" "")]
5398 "TARGET_SIMD"
5399 {
5400 emit_insn
5401 (gen_aarch64_get_lane<mode> (operands[0], operands[1], operands[2]));
5402 DONE;
5403 })
5405 ;; aes
5407 (define_insn "aarch64_crypto_aes<aes_op>v16qi"
5408 [(set (match_operand:V16QI 0 "register_operand" "=w")
5409 (unspec:V16QI [(match_operand:V16QI 1 "register_operand" "0")
5410 (match_operand:V16QI 2 "register_operand" "w")]
5411 CRYPTO_AES))]
5412 "TARGET_SIMD && TARGET_CRYPTO"
5413 "aes<aes_op>\\t%0.16b, %2.16b"
5414 [(set_attr "type" "crypto_aese")]
5415 )
5417 (define_insn "aarch64_crypto_aes<aesmc_op>v16qi"
5418 [(set (match_operand:V16QI 0 "register_operand" "=w")
5419 (unspec:V16QI [(match_operand:V16QI 1 "register_operand" "w")]
5420 CRYPTO_AESMC))]
5421 "TARGET_SIMD && TARGET_CRYPTO"
5422 "aes<aesmc_op>\\t%0.16b, %1.16b"
5423 [(set_attr "type" "crypto_aesmc")]
5424 )
5426 ;; sha1
5428 (define_insn "aarch64_crypto_sha1hsi"
5429 [(set (match_operand:SI 0 "register_operand" "=w")
5430 (unspec:SI [(match_operand:SI 1
5431 "register_operand" "w")]
5432 UNSPEC_SHA1H))]
5433 "TARGET_SIMD && TARGET_CRYPTO"
5434 "sha1h\\t%s0, %s1"
5435 [(set_attr "type" "crypto_sha1_fast")]
5436 )
5438 (define_insn "aarch64_crypto_sha1su1v4si"
5439 [(set (match_operand:V4SI 0 "register_operand" "=w")
5440 (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
5441 (match_operand:V4SI 2 "register_operand" "w")]
5442 UNSPEC_SHA1SU1))]
5443 "TARGET_SIMD && TARGET_CRYPTO"
5444 "sha1su1\\t%0.4s, %2.4s"
5445 [(set_attr "type" "crypto_sha1_fast")]
5446 )
5448 (define_insn "aarch64_crypto_sha1<sha1_op>v4si"
5449 [(set (match_operand:V4SI 0 "register_operand" "=w")
5450 (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
5451 (match_operand:SI 2 "register_operand" "w")
5452 (match_operand:V4SI 3 "register_operand" "w")]
5453 CRYPTO_SHA1))]
5454 "TARGET_SIMD && TARGET_CRYPTO"
5455 "sha1<sha1_op>\\t%q0, %s2, %3.4s"
5456 [(set_attr "type" "crypto_sha1_slow")]
5457 )
5459 (define_insn "aarch64_crypto_sha1su0v4si"
5460 [(set (match_operand:V4SI 0 "register_operand" "=w")
5461 (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
5462 (match_operand:V4SI 2 "register_operand" "w")
5463 (match_operand:V4SI 3 "register_operand" "w")]
5464 UNSPEC_SHA1SU0))]
5465 "TARGET_SIMD && TARGET_CRYPTO"
5466 "sha1su0\\t%0.4s, %2.4s, %3.4s"
5467 [(set_attr "type" "crypto_sha1_xor")]
5468 )
5470 ;; sha256
5472 (define_insn "aarch64_crypto_sha256h<sha256_op>v4si"
5473 [(set (match_operand:V4SI 0 "register_operand" "=w")
5474 (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
5475 (match_operand:V4SI 2 "register_operand" "w")
5476 (match_operand:V4SI 3 "register_operand" "w")]
5477 CRYPTO_SHA256))]
5478 "TARGET_SIMD && TARGET_CRYPTO"
5479 "sha256h<sha256_op>\\t%q0, %q2, %3.4s"
5480 [(set_attr "type" "crypto_sha256_slow")]
5481 )
5483 (define_insn "aarch64_crypto_sha256su0v4si"
5484 [(set (match_operand:V4SI 0 "register_operand" "=w")
5485 (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
5486 (match_operand:V4SI 2 "register_operand" "w")]
5487 UNSPEC_SHA256SU0))]
5488 "TARGET_SIMD &&TARGET_CRYPTO"
5489 "sha256su0\\t%0.4s, %2.4s"
5490 [(set_attr "type" "crypto_sha256_fast")]
5491 )
5493 (define_insn "aarch64_crypto_sha256su1v4si"
5494 [(set (match_operand:V4SI 0 "register_operand" "=w")
5495 (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
5496 (match_operand:V4SI 2 "register_operand" "w")
5497 (match_operand:V4SI 3 "register_operand" "w")]
5498 UNSPEC_SHA256SU1))]
5499 "TARGET_SIMD &&TARGET_CRYPTO"
5500 "sha256su1\\t%0.4s, %2.4s, %3.4s"
5501 [(set_attr "type" "crypto_sha256_slow")]
5502 )
5504 ;; pmull
5506 (define_insn "aarch64_crypto_pmulldi"
5507 [(set (match_operand:TI 0 "register_operand" "=w")
5508 (unspec:TI [(match_operand:DI 1 "register_operand" "w")
5509 (match_operand:DI 2 "register_operand" "w")]
5510 UNSPEC_PMULL))]
5511 "TARGET_SIMD && TARGET_CRYPTO"
5512 "pmull\\t%0.1q, %1.1d, %2.1d"
5513 [(set_attr "type" "neon_mul_d_long")]
5514 )
5516 (define_insn "aarch64_crypto_pmullv2di"
5517 [(set (match_operand:TI 0 "register_operand" "=w")
5518 (unspec:TI [(match_operand:V2DI 1 "register_operand" "w")
5519 (match_operand:V2DI 2 "register_operand" "w")]
5520 UNSPEC_PMULL2))]
5521 "TARGET_SIMD && TARGET_CRYPTO"
5522 "pmull2\\t%0.1q, %1.2d, %2.2d"
5523 [(set_attr "type" "neon_mul_d_long")]
5524 )