| blob | 030a1013caa8a965bcd1615c9686d0be715be921 |
1 ;; Machine description for AArch64 AdvSIMD architecture.
2 ;; Copyright (C) 2011-2015 Free Software Foundation, Inc.
3 ;; Contributed by ARM Ltd.
4 ;;
5 ;; This file is part of GCC.
6 ;;
7 ;; GCC is free software; you can redistribute it and/or modify it
8 ;; under the terms of the GNU General Public License as published by
9 ;; the Free Software Foundation; either version 3, or (at your option)
10 ;; any later version.
11 ;;
12 ;; GCC is distributed in the hope that it will be useful, but
13 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
14 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 ;; General Public License for more details.
16 ;;
17 ;; You should have received a copy of the GNU General Public License
18 ;; along with GCC; see the file COPYING3. If not see
19 ;; <http://www.gnu.org/licenses/>.
21 (define_expand "mov<mode>"
22 [(set (match_operand:VALL_F16 0 "nonimmediate_operand" "")
23 (match_operand:VALL_F16 1 "general_operand" ""))]
24 "TARGET_SIMD"
25 "
26 if (GET_CODE (operands[0]) == MEM)
27 operands[1] = force_reg (<MODE>mode, operands[1]);
28 "
29 )
31 (define_expand "movmisalign<mode>"
32 [(set (match_operand:VALL 0 "nonimmediate_operand" "")
33 (match_operand:VALL 1 "general_operand" ""))]
34 "TARGET_SIMD"
35 {
36 /* This pattern is not permitted to fail during expansion: if both arguments
37 are non-registers (e.g. memory := constant, which can be created by the
38 auto-vectorizer), force operand 1 into a register. */
39 if (!register_operand (operands[0], <MODE>mode)
40 && !register_operand (operands[1], <MODE>mode))
41 operands[1] = force_reg (<MODE>mode, operands[1]);
42 })
44 (define_insn "aarch64_simd_dup<mode>"
45 [(set (match_operand:VDQ_I 0 "register_operand" "=w, w")
46 (vec_duplicate:VDQ_I
47 (match_operand:<VEL> 1 "register_operand" "r, w")))]
48 "TARGET_SIMD"
49 "@
50 dup\\t%0.<Vtype>, %<vw>1
51 dup\\t%0.<Vtype>, %1.<Vetype>[0]"
52 [(set_attr "type" "neon_from_gp<q>, neon_dup<q>")]
53 )
55 (define_insn "aarch64_simd_dup<mode>"
56 [(set (match_operand:VDQF_F16 0 "register_operand" "=w")
57 (vec_duplicate:VDQF_F16
58 (match_operand:<VEL> 1 "register_operand" "w")))]
59 "TARGET_SIMD"
60 "dup\\t%0.<Vtype>, %1.<Vetype>[0]"
61 [(set_attr "type" "neon_dup<q>")]
62 )
64 (define_insn "aarch64_dup_lane<mode>"
65 [(set (match_operand:VALL_F16 0 "register_operand" "=w")
66 (vec_duplicate:VALL_F16
67 (vec_select:<VEL>
68 (match_operand:VALL_F16 1 "register_operand" "w")
69 (parallel [(match_operand:SI 2 "immediate_operand" "i")])
70 )))]
71 "TARGET_SIMD"
72 {
73 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
74 return "dup\\t%0.<Vtype>, %1.<Vetype>[%2]";
75 }
76 [(set_attr "type" "neon_dup<q>")]
77 )
79 (define_insn "aarch64_dup_lane_<vswap_width_name><mode>"
80 [(set (match_operand:VALL_F16 0 "register_operand" "=w")
81 (vec_duplicate:VALL_F16
82 (vec_select:<VEL>
83 (match_operand:<VSWAP_WIDTH> 1 "register_operand" "w")
84 (parallel [(match_operand:SI 2 "immediate_operand" "i")])
85 )))]
86 "TARGET_SIMD"
87 {
88 operands[2] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
89 INTVAL (operands[2])));
90 return "dup\\t%0.<Vtype>, %1.<Vetype>[%2]";
91 }
92 [(set_attr "type" "neon_dup<q>")]
93 )
95 (define_insn "*aarch64_simd_mov<mode>"
96 [(set (match_operand:VD 0 "nonimmediate_operand"
97 "=w, m, w, ?r, ?w, ?r, w")
98 (match_operand:VD 1 "general_operand"
99 "m, w, w, w, r, r, Dn"))]
100 "TARGET_SIMD
101 && (register_operand (operands[0], <MODE>mode)
102 || register_operand (operands[1], <MODE>mode))"
103 {
104 switch (which_alternative)
105 {
106 case 0: return "ldr\\t%d0, %1";
107 case 1: return "str\\t%d1, %0";
108 case 2: return "orr\t%0.<Vbtype>, %1.<Vbtype>, %1.<Vbtype>";
109 case 3: return "umov\t%0, %1.d[0]";
110 case 4: return "ins\t%0.d[0], %1";
111 case 5: return "mov\t%0, %1";
112 case 6:
113 return aarch64_output_simd_mov_immediate (operands[1],
114 <MODE>mode, 64);
115 default: gcc_unreachable ();
116 }
117 }
118 [(set_attr "type" "neon_load1_1reg<q>, neon_store1_1reg<q>,\
119 neon_logic<q>, neon_to_gp<q>, neon_from_gp<q>,\
120 mov_reg, neon_move<q>")]
121 )
123 (define_insn "*aarch64_simd_mov<mode>"
124 [(set (match_operand:VQ 0 "nonimmediate_operand"
125 "=w, m, w, ?r, ?w, ?r, w")
126 (match_operand:VQ 1 "general_operand"
127 "m, w, w, w, r, r, Dn"))]
128 "TARGET_SIMD
129 && (register_operand (operands[0], <MODE>mode)
130 || register_operand (operands[1], <MODE>mode))"
131 {
132 switch (which_alternative)
133 {
134 case 0:
135 return "ldr\\t%q0, %1";
136 case 1:
137 return "str\\t%q1, %0";
138 case 2:
139 return "orr\t%0.<Vbtype>, %1.<Vbtype>, %1.<Vbtype>";
140 case 3:
141 case 4:
142 case 5:
143 return "#";
144 case 6:
145 return aarch64_output_simd_mov_immediate (operands[1], <MODE>mode, 128);
146 default:
147 gcc_unreachable ();
148 }
149 }
150 [(set_attr "type" "neon_load1_1reg<q>, neon_store1_1reg<q>,\
151 neon_logic<q>, multiple, multiple, multiple,\
152 neon_move<q>")
153 (set_attr "length" "4,4,4,8,8,8,4")]
154 )
156 (define_insn "load_pair<mode>"
157 [(set (match_operand:VD 0 "register_operand" "=w")
158 (match_operand:VD 1 "aarch64_mem_pair_operand" "Ump"))
159 (set (match_operand:VD 2 "register_operand" "=w")
160 (match_operand:VD 3 "memory_operand" "m"))]
161 "TARGET_SIMD
162 && rtx_equal_p (XEXP (operands[3], 0),
163 plus_constant (Pmode,
164 XEXP (operands[1], 0),
165 GET_MODE_SIZE (<MODE>mode)))"
166 "ldp\\t%d0, %d2, %1"
167 [(set_attr "type" "neon_ldp")]
168 )
170 (define_insn "store_pair<mode>"
171 [(set (match_operand:VD 0 "aarch64_mem_pair_operand" "=Ump")
172 (match_operand:VD 1 "register_operand" "w"))
173 (set (match_operand:VD 2 "memory_operand" "=m")
174 (match_operand:VD 3 "register_operand" "w"))]
175 "TARGET_SIMD
176 && rtx_equal_p (XEXP (operands[2], 0),
177 plus_constant (Pmode,
178 XEXP (operands[0], 0),
179 GET_MODE_SIZE (<MODE>mode)))"
180 "stp\\t%d1, %d3, %0"
181 [(set_attr "type" "neon_stp")]
182 )
184 (define_split
185 [(set (match_operand:VQ 0 "register_operand" "")
186 (match_operand:VQ 1 "register_operand" ""))]
187 "TARGET_SIMD && reload_completed
188 && GP_REGNUM_P (REGNO (operands[0]))
189 && GP_REGNUM_P (REGNO (operands[1]))"
190 [(const_int 0)]
191 {
192 aarch64_simd_emit_reg_reg_move (operands, DImode, 2);
193 DONE;
194 })
196 (define_split
197 [(set (match_operand:VQ 0 "register_operand" "")
198 (match_operand:VQ 1 "register_operand" ""))]
199 "TARGET_SIMD && reload_completed
200 && ((FP_REGNUM_P (REGNO (operands[0])) && GP_REGNUM_P (REGNO (operands[1])))
201 || (GP_REGNUM_P (REGNO (operands[0])) && FP_REGNUM_P (REGNO (operands[1]))))"
202 [(const_int 0)]
203 {
204 aarch64_split_simd_move (operands[0], operands[1]);
205 DONE;
206 })
208 (define_expand "aarch64_split_simd_mov<mode>"
209 [(set (match_operand:VQ 0)
210 (match_operand:VQ 1))]
211 "TARGET_SIMD"
212 {
213 rtx dst = operands[0];
214 rtx src = operands[1];
216 if (GP_REGNUM_P (REGNO (src)))
217 {
218 rtx src_low_part = gen_lowpart (<VHALF>mode, src);
219 rtx src_high_part = gen_highpart (<VHALF>mode, src);
221 emit_insn
222 (gen_move_lo_quad_<mode> (dst, src_low_part));
223 emit_insn
224 (gen_move_hi_quad_<mode> (dst, src_high_part));
225 }
227 else
228 {
229 rtx dst_low_part = gen_lowpart (<VHALF>mode, dst);
230 rtx dst_high_part = gen_highpart (<VHALF>mode, dst);
231 rtx lo = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
232 rtx hi = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
234 emit_insn
235 (gen_aarch64_simd_mov_from_<mode>low (dst_low_part, src, lo));
236 emit_insn
237 (gen_aarch64_simd_mov_from_<mode>high (dst_high_part, src, hi));
238 }
239 DONE;
240 }
241 )
243 (define_insn "aarch64_simd_mov_from_<mode>low"
244 [(set (match_operand:<VHALF> 0 "register_operand" "=r")
245 (vec_select:<VHALF>
246 (match_operand:VQ 1 "register_operand" "w")
247 (match_operand:VQ 2 "vect_par_cnst_lo_half" "")))]
248 "TARGET_SIMD && reload_completed"
249 "umov\t%0, %1.d[0]"
250 [(set_attr "type" "neon_to_gp<q>")
251 (set_attr "length" "4")
252 ])
254 (define_insn "aarch64_simd_mov_from_<mode>high"
255 [(set (match_operand:<VHALF> 0 "register_operand" "=r")
256 (vec_select:<VHALF>
257 (match_operand:VQ 1 "register_operand" "w")
258 (match_operand:VQ 2 "vect_par_cnst_hi_half" "")))]
259 "TARGET_SIMD && reload_completed"
260 "umov\t%0, %1.d[1]"
261 [(set_attr "type" "neon_to_gp<q>")
262 (set_attr "length" "4")
263 ])
265 (define_insn "orn<mode>3"
266 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
267 (ior:VDQ_I (not:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w"))
268 (match_operand:VDQ_I 2 "register_operand" "w")))]
269 "TARGET_SIMD"
270 "orn\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>"
271 [(set_attr "type" "neon_logic<q>")]
272 )
274 (define_insn "bic<mode>3"
275 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
276 (and:VDQ_I (not:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w"))
277 (match_operand:VDQ_I 2 "register_operand" "w")))]
278 "TARGET_SIMD"
279 "bic\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>"
280 [(set_attr "type" "neon_logic<q>")]
281 )
283 (define_insn "add<mode>3"
284 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
285 (plus:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
286 (match_operand:VDQ_I 2 "register_operand" "w")))]
287 "TARGET_SIMD"
288 "add\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
289 [(set_attr "type" "neon_add<q>")]
290 )
292 (define_insn "sub<mode>3"
293 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
294 (minus:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
295 (match_operand:VDQ_I 2 "register_operand" "w")))]
296 "TARGET_SIMD"
297 "sub\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
298 [(set_attr "type" "neon_sub<q>")]
299 )
301 (define_insn "mul<mode>3"
302 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
303 (mult:VDQ_BHSI (match_operand:VDQ_BHSI 1 "register_operand" "w")
304 (match_operand:VDQ_BHSI 2 "register_operand" "w")))]
305 "TARGET_SIMD"
306 "mul\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
307 [(set_attr "type" "neon_mul_<Vetype><q>")]
308 )
310 (define_insn "bswap<mode>2"
311 [(set (match_operand:VDQHSD 0 "register_operand" "=w")
312 (bswap:VDQHSD (match_operand:VDQHSD 1 "register_operand" "w")))]
313 "TARGET_SIMD"
314 "rev<Vrevsuff>\\t%0.<Vbtype>, %1.<Vbtype>"
315 [(set_attr "type" "neon_rev<q>")]
316 )
318 (define_insn "aarch64_rbit<mode>"
319 [(set (match_operand:VB 0 "register_operand" "=w")
320 (unspec:VB [(match_operand:VB 1 "register_operand" "w")]
321 UNSPEC_RBIT))]
322 "TARGET_SIMD"
323 "rbit\\t%0.<Vbtype>, %1.<Vbtype>"
324 [(set_attr "type" "neon_rbit")]
325 )
327 (define_expand "ctz<mode>2"
328 [(set (match_operand:VS 0 "register_operand")
329 (ctz:VS (match_operand:VS 1 "register_operand")))]
330 "TARGET_SIMD"
331 {
332 emit_insn (gen_bswap<mode>2 (operands[0], operands[1]));
333 rtx op0_castsi2qi = simplify_gen_subreg(<VS:VSI2QI>mode, operands[0],
334 <MODE>mode, 0);
335 emit_insn (gen_aarch64_rbit<VS:vsi2qi> (op0_castsi2qi, op0_castsi2qi));
336 emit_insn (gen_clz<mode>2 (operands[0], operands[0]));
337 DONE;
338 }
339 )
341 (define_insn "*aarch64_mul3_elt<mode>"
342 [(set (match_operand:VMUL 0 "register_operand" "=w")
343 (mult:VMUL
344 (vec_duplicate:VMUL
345 (vec_select:<VEL>
346 (match_operand:VMUL 1 "register_operand" "<h_con>")
347 (parallel [(match_operand:SI 2 "immediate_operand")])))
348 (match_operand:VMUL 3 "register_operand" "w")))]
349 "TARGET_SIMD"
350 {
351 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
352 return "<f>mul\\t%0.<Vtype>, %3.<Vtype>, %1.<Vetype>[%2]";
353 }
354 [(set_attr "type" "neon<fp>_mul_<Vetype>_scalar<q>")]
355 )
357 (define_insn "*aarch64_mul3_elt_<vswap_width_name><mode>"
358 [(set (match_operand:VMUL_CHANGE_NLANES 0 "register_operand" "=w")
359 (mult:VMUL_CHANGE_NLANES
360 (vec_duplicate:VMUL_CHANGE_NLANES
361 (vec_select:<VEL>
362 (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
363 (parallel [(match_operand:SI 2 "immediate_operand")])))
364 (match_operand:VMUL_CHANGE_NLANES 3 "register_operand" "w")))]
365 "TARGET_SIMD"
366 {
367 operands[2] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
368 INTVAL (operands[2])));
369 return "<f>mul\\t%0.<Vtype>, %3.<Vtype>, %1.<Vetype>[%2]";
370 }
371 [(set_attr "type" "neon<fp>_mul_<Vetype>_scalar<q>")]
372 )
374 (define_insn "*aarch64_mul3_elt_to_128df"
375 [(set (match_operand:V2DF 0 "register_operand" "=w")
376 (mult:V2DF
377 (vec_duplicate:V2DF
378 (match_operand:DF 2 "register_operand" "w"))
379 (match_operand:V2DF 1 "register_operand" "w")))]
380 "TARGET_SIMD"
381 "fmul\\t%0.2d, %1.2d, %2.d[0]"
382 [(set_attr "type" "neon_fp_mul_d_scalar_q")]
383 )
385 (define_insn "aarch64_rsqrte_<mode>2"
386 [(set (match_operand:VALLF 0 "register_operand" "=w")
387 (unspec:VALLF [(match_operand:VALLF 1 "register_operand" "w")]
388 UNSPEC_RSQRTE))]
389 "TARGET_SIMD"
390 "frsqrte\\t%<v>0<Vmtype>, %<v>1<Vmtype>"
391 [(set_attr "type" "neon_fp_rsqrte_<Vetype><q>")])
393 (define_insn "aarch64_rsqrts_<mode>3"
394 [(set (match_operand:VALLF 0 "register_operand" "=w")
395 (unspec:VALLF [(match_operand:VALLF 1 "register_operand" "w")
396 (match_operand:VALLF 2 "register_operand" "w")]
397 UNSPEC_RSQRTS))]
398 "TARGET_SIMD"
399 "frsqrts\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
400 [(set_attr "type" "neon_fp_rsqrts_<Vetype><q>")])
402 (define_expand "rsqrt<mode>2"
403 [(set (match_operand:VALLF 0 "register_operand" "=w")
404 (unspec:VALLF [(match_operand:VALLF 1 "register_operand" "w")]
405 UNSPEC_RSQRT))]
406 "TARGET_SIMD"
407 {
408 aarch64_emit_swrsqrt (operands[0], operands[1]);
409 DONE;
410 })
412 (define_insn "*aarch64_mul3_elt_to_64v2df"
413 [(set (match_operand:DF 0 "register_operand" "=w")
414 (mult:DF
415 (vec_select:DF
416 (match_operand:V2DF 1 "register_operand" "w")
417 (parallel [(match_operand:SI 2 "immediate_operand")]))
418 (match_operand:DF 3 "register_operand" "w")))]
419 "TARGET_SIMD"
420 {
421 operands[2] = GEN_INT (ENDIAN_LANE_N (V2DFmode, INTVAL (operands[2])));
422 return "fmul\\t%0.2d, %3.2d, %1.d[%2]";
423 }
424 [(set_attr "type" "neon_fp_mul_d_scalar_q")]
425 )
427 (define_insn "neg<mode>2"
428 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
429 (neg:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")))]
430 "TARGET_SIMD"
431 "neg\t%0.<Vtype>, %1.<Vtype>"
432 [(set_attr "type" "neon_neg<q>")]
433 )
435 (define_insn "abs<mode>2"
436 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
437 (abs:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")))]
438 "TARGET_SIMD"
439 "abs\t%0.<Vtype>, %1.<Vtype>"
440 [(set_attr "type" "neon_abs<q>")]
441 )
443 ;; The intrinsic version of integer ABS must not be allowed to
444 ;; combine with any operation with an integerated ABS step, such
445 ;; as SABD.
446 (define_insn "aarch64_abs<mode>"
447 [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
448 (unspec:VSDQ_I_DI
449 [(match_operand:VSDQ_I_DI 1 "register_operand" "w")]
450 UNSPEC_ABS))]
451 "TARGET_SIMD"
452 "abs\t%<v>0<Vmtype>, %<v>1<Vmtype>"
453 [(set_attr "type" "neon_abs<q>")]
454 )
456 (define_insn "abd<mode>_3"
457 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
458 (abs:VDQ_BHSI (minus:VDQ_BHSI
459 (match_operand:VDQ_BHSI 1 "register_operand" "w")
460 (match_operand:VDQ_BHSI 2 "register_operand" "w"))))]
461 "TARGET_SIMD"
462 "sabd\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
463 [(set_attr "type" "neon_abd<q>")]
464 )
466 (define_insn "aba<mode>_3"
467 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
468 (plus:VDQ_BHSI (abs:VDQ_BHSI (minus:VDQ_BHSI
469 (match_operand:VDQ_BHSI 1 "register_operand" "w")
470 (match_operand:VDQ_BHSI 2 "register_operand" "w")))
471 (match_operand:VDQ_BHSI 3 "register_operand" "0")))]
472 "TARGET_SIMD"
473 "saba\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
474 [(set_attr "type" "neon_arith_acc<q>")]
475 )
477 (define_insn "fabd<mode>_3"
478 [(set (match_operand:VDQF 0 "register_operand" "=w")
479 (abs:VDQF (minus:VDQF
480 (match_operand:VDQF 1 "register_operand" "w")
481 (match_operand:VDQF 2 "register_operand" "w"))))]
482 "TARGET_SIMD"
483 "fabd\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
484 [(set_attr "type" "neon_fp_abd_<Vetype><q>")]
485 )
487 (define_insn "*fabd_scalar<mode>3"
488 [(set (match_operand:GPF 0 "register_operand" "=w")
489 (abs:GPF (minus:GPF
490 (match_operand:GPF 1 "register_operand" "w")
491 (match_operand:GPF 2 "register_operand" "w"))))]
492 "TARGET_SIMD"
493 "fabd\t%<s>0, %<s>1, %<s>2"
494 [(set_attr "type" "neon_fp_abd_<Vetype><q>")]
495 )
497 (define_insn "and<mode>3"
498 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
499 (and:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
500 (match_operand:VDQ_I 2 "register_operand" "w")))]
501 "TARGET_SIMD"
502 "and\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>"
503 [(set_attr "type" "neon_logic<q>")]
504 )
506 (define_insn "ior<mode>3"
507 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
508 (ior:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
509 (match_operand:VDQ_I 2 "register_operand" "w")))]
510 "TARGET_SIMD"
511 "orr\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>"
512 [(set_attr "type" "neon_logic<q>")]
513 )
515 (define_insn "xor<mode>3"
516 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
517 (xor:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
518 (match_operand:VDQ_I 2 "register_operand" "w")))]
519 "TARGET_SIMD"
520 "eor\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>"
521 [(set_attr "type" "neon_logic<q>")]
522 )
524 (define_insn "one_cmpl<mode>2"
525 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
526 (not:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")))]
527 "TARGET_SIMD"
528 "not\t%0.<Vbtype>, %1.<Vbtype>"
529 [(set_attr "type" "neon_logic<q>")]
530 )
532 (define_insn "aarch64_simd_vec_set<mode>"
533 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w,w,w")
534 (vec_merge:VDQ_BHSI
535 (vec_duplicate:VDQ_BHSI
536 (match_operand:<VEL> 1 "aarch64_simd_general_operand" "r,w,Utv"))
537 (match_operand:VDQ_BHSI 3 "register_operand" "0,0,0")
538 (match_operand:SI 2 "immediate_operand" "i,i,i")))]
539 "TARGET_SIMD"
540 {
541 int elt = ENDIAN_LANE_N (<MODE>mode, exact_log2 (INTVAL (operands[2])));
542 operands[2] = GEN_INT ((HOST_WIDE_INT) 1 << elt);
543 switch (which_alternative)
544 {
545 case 0:
546 return "ins\\t%0.<Vetype>[%p2], %w1";
547 case 1:
548 return "ins\\t%0.<Vetype>[%p2], %1.<Vetype>[0]";
549 case 2:
550 return "ld1\\t{%0.<Vetype>}[%p2], %1";
551 default:
552 gcc_unreachable ();
553 }
554 }
555 [(set_attr "type" "neon_from_gp<q>, neon_ins<q>, neon_load1_1reg<q>")]
556 )
558 (define_insn "aarch64_simd_lshr<mode>"
559 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
560 (lshiftrt:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
561 (match_operand:VDQ_I 2 "aarch64_simd_rshift_imm" "Dr")))]
562 "TARGET_SIMD"
563 "ushr\t%0.<Vtype>, %1.<Vtype>, %2"
564 [(set_attr "type" "neon_shift_imm<q>")]
565 )
567 (define_insn "aarch64_simd_ashr<mode>"
568 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
569 (ashiftrt:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
570 (match_operand:VDQ_I 2 "aarch64_simd_rshift_imm" "Dr")))]
571 "TARGET_SIMD"
572 "sshr\t%0.<Vtype>, %1.<Vtype>, %2"
573 [(set_attr "type" "neon_shift_imm<q>")]
574 )
576 (define_insn "aarch64_simd_imm_shl<mode>"
577 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
578 (ashift:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
579 (match_operand:VDQ_I 2 "aarch64_simd_lshift_imm" "Dl")))]
580 "TARGET_SIMD"
581 "shl\t%0.<Vtype>, %1.<Vtype>, %2"
582 [(set_attr "type" "neon_shift_imm<q>")]
583 )
585 (define_insn "aarch64_simd_reg_sshl<mode>"
586 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
587 (ashift:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
588 (match_operand:VDQ_I 2 "register_operand" "w")))]
589 "TARGET_SIMD"
590 "sshl\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
591 [(set_attr "type" "neon_shift_reg<q>")]
592 )
594 (define_insn "aarch64_simd_reg_shl<mode>_unsigned"
595 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
596 (unspec:VDQ_I [(match_operand:VDQ_I 1 "register_operand" "w")
597 (match_operand:VDQ_I 2 "register_operand" "w")]
598 UNSPEC_ASHIFT_UNSIGNED))]
599 "TARGET_SIMD"
600 "ushl\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
601 [(set_attr "type" "neon_shift_reg<q>")]
602 )
604 (define_insn "aarch64_simd_reg_shl<mode>_signed"
605 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
606 (unspec:VDQ_I [(match_operand:VDQ_I 1 "register_operand" "w")
607 (match_operand:VDQ_I 2 "register_operand" "w")]
608 UNSPEC_ASHIFT_SIGNED))]
609 "TARGET_SIMD"
610 "sshl\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
611 [(set_attr "type" "neon_shift_reg<q>")]
612 )
614 (define_expand "ashl<mode>3"
615 [(match_operand:VDQ_I 0 "register_operand" "")
616 (match_operand:VDQ_I 1 "register_operand" "")
617 (match_operand:SI 2 "general_operand" "")]
618 "TARGET_SIMD"
619 {
620 int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
621 int shift_amount;
623 if (CONST_INT_P (operands[2]))
624 {
625 shift_amount = INTVAL (operands[2]);
626 if (shift_amount >= 0 && shift_amount < bit_width)
627 {
628 rtx tmp = aarch64_simd_gen_const_vector_dup (<MODE>mode,
629 shift_amount);
630 emit_insn (gen_aarch64_simd_imm_shl<mode> (operands[0],
631 operands[1],
632 tmp));
633 DONE;
634 }
635 else
636 {
637 operands[2] = force_reg (SImode, operands[2]);
638 }
639 }
640 else if (MEM_P (operands[2]))
641 {
642 operands[2] = force_reg (SImode, operands[2]);
643 }
645 if (REG_P (operands[2]))
646 {
647 rtx tmp = gen_reg_rtx (<MODE>mode);
648 emit_insn (gen_aarch64_simd_dup<mode> (tmp,
649 convert_to_mode (<VEL>mode,
650 operands[2],
651 0)));
652 emit_insn (gen_aarch64_simd_reg_sshl<mode> (operands[0], operands[1],
653 tmp));
654 DONE;
655 }
656 else
657 FAIL;
658 }
659 )
661 (define_expand "lshr<mode>3"
662 [(match_operand:VDQ_I 0 "register_operand" "")
663 (match_operand:VDQ_I 1 "register_operand" "")
664 (match_operand:SI 2 "general_operand" "")]
665 "TARGET_SIMD"
666 {
667 int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
668 int shift_amount;
670 if (CONST_INT_P (operands[2]))
671 {
672 shift_amount = INTVAL (operands[2]);
673 if (shift_amount > 0 && shift_amount <= bit_width)
674 {
675 rtx tmp = aarch64_simd_gen_const_vector_dup (<MODE>mode,
676 shift_amount);
677 emit_insn (gen_aarch64_simd_lshr<mode> (operands[0],
678 operands[1],
679 tmp));
680 DONE;
681 }
682 else
683 operands[2] = force_reg (SImode, operands[2]);
684 }
685 else if (MEM_P (operands[2]))
686 {
687 operands[2] = force_reg (SImode, operands[2]);
688 }
690 if (REG_P (operands[2]))
691 {
692 rtx tmp = gen_reg_rtx (SImode);
693 rtx tmp1 = gen_reg_rtx (<MODE>mode);
694 emit_insn (gen_negsi2 (tmp, operands[2]));
695 emit_insn (gen_aarch64_simd_dup<mode> (tmp1,
696 convert_to_mode (<VEL>mode,
697 tmp, 0)));
698 emit_insn (gen_aarch64_simd_reg_shl<mode>_unsigned (operands[0],
699 operands[1],
700 tmp1));
701 DONE;
702 }
703 else
704 FAIL;
705 }
706 )
708 (define_expand "ashr<mode>3"
709 [(match_operand:VDQ_I 0 "register_operand" "")
710 (match_operand:VDQ_I 1 "register_operand" "")
711 (match_operand:SI 2 "general_operand" "")]
712 "TARGET_SIMD"
713 {
714 int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
715 int shift_amount;
717 if (CONST_INT_P (operands[2]))
718 {
719 shift_amount = INTVAL (operands[2]);
720 if (shift_amount > 0 && shift_amount <= bit_width)
721 {
722 rtx tmp = aarch64_simd_gen_const_vector_dup (<MODE>mode,
723 shift_amount);
724 emit_insn (gen_aarch64_simd_ashr<mode> (operands[0],
725 operands[1],
726 tmp));
727 DONE;
728 }
729 else
730 operands[2] = force_reg (SImode, operands[2]);
731 }
732 else if (MEM_P (operands[2]))
733 {
734 operands[2] = force_reg (SImode, operands[2]);
735 }
737 if (REG_P (operands[2]))
738 {
739 rtx tmp = gen_reg_rtx (SImode);
740 rtx tmp1 = gen_reg_rtx (<MODE>mode);
741 emit_insn (gen_negsi2 (tmp, operands[2]));
742 emit_insn (gen_aarch64_simd_dup<mode> (tmp1,
743 convert_to_mode (<VEL>mode,
744 tmp, 0)));
745 emit_insn (gen_aarch64_simd_reg_shl<mode>_signed (operands[0],
746 operands[1],
747 tmp1));
748 DONE;
749 }
750 else
751 FAIL;
752 }
753 )
755 (define_expand "vashl<mode>3"
756 [(match_operand:VDQ_I 0 "register_operand" "")
757 (match_operand:VDQ_I 1 "register_operand" "")
758 (match_operand:VDQ_I 2 "register_operand" "")]
759 "TARGET_SIMD"
760 {
761 emit_insn (gen_aarch64_simd_reg_sshl<mode> (operands[0], operands[1],
762 operands[2]));
763 DONE;
764 })
766 ;; Using mode VDQ_BHSI as there is no V2DImode neg!
767 ;; Negating individual lanes most certainly offsets the
768 ;; gain from vectorization.
769 (define_expand "vashr<mode>3"
770 [(match_operand:VDQ_BHSI 0 "register_operand" "")
771 (match_operand:VDQ_BHSI 1 "register_operand" "")
772 (match_operand:VDQ_BHSI 2 "register_operand" "")]
773 "TARGET_SIMD"
774 {
775 rtx neg = gen_reg_rtx (<MODE>mode);
776 emit (gen_neg<mode>2 (neg, operands[2]));
777 emit_insn (gen_aarch64_simd_reg_shl<mode>_signed (operands[0], operands[1],
778 neg));
779 DONE;
780 })
782 ;; DI vector shift
783 (define_expand "aarch64_ashr_simddi"
784 [(match_operand:DI 0 "register_operand" "=w")
785 (match_operand:DI 1 "register_operand" "w")
786 (match_operand:SI 2 "aarch64_shift_imm64_di" "")]
787 "TARGET_SIMD"
788 {
789 /* An arithmetic shift right by 64 fills the result with copies of the sign
790 bit, just like asr by 63 - however the standard pattern does not handle
791 a shift by 64. */
792 if (INTVAL (operands[2]) == 64)
793 operands[2] = GEN_INT (63);
794 emit_insn (gen_ashrdi3 (operands[0], operands[1], operands[2]));
795 DONE;
796 }
797 )
799 (define_expand "vlshr<mode>3"
800 [(match_operand:VDQ_BHSI 0 "register_operand" "")
801 (match_operand:VDQ_BHSI 1 "register_operand" "")
802 (match_operand:VDQ_BHSI 2 "register_operand" "")]
803 "TARGET_SIMD"
804 {
805 rtx neg = gen_reg_rtx (<MODE>mode);
806 emit (gen_neg<mode>2 (neg, operands[2]));
807 emit_insn (gen_aarch64_simd_reg_shl<mode>_unsigned (operands[0], operands[1],
808 neg));
809 DONE;
810 })
812 (define_expand "aarch64_lshr_simddi"
813 [(match_operand:DI 0 "register_operand" "=w")
814 (match_operand:DI 1 "register_operand" "w")
815 (match_operand:SI 2 "aarch64_shift_imm64_di" "")]
816 "TARGET_SIMD"
817 {
818 if (INTVAL (operands[2]) == 64)
819 emit_move_insn (operands[0], const0_rtx);
820 else
821 emit_insn (gen_lshrdi3 (operands[0], operands[1], operands[2]));
822 DONE;
823 }
824 )
826 (define_expand "vec_set<mode>"
827 [(match_operand:VDQ_BHSI 0 "register_operand")
828 (match_operand:<VEL> 1 "register_operand")
829 (match_operand:SI 2 "immediate_operand")]
830 "TARGET_SIMD"
831 {
832 HOST_WIDE_INT elem = (HOST_WIDE_INT) 1 << INTVAL (operands[2]);
833 emit_insn (gen_aarch64_simd_vec_set<mode> (operands[0], operands[1],
834 GEN_INT (elem), operands[0]));
835 DONE;
836 }
837 )
839 ;; For 64-bit modes we use ushl/r, as this does not require a SIMD zero.
840 (define_insn "vec_shr_<mode>"
841 [(set (match_operand:VD 0 "register_operand" "=w")
842 (unspec:VD [(match_operand:VD 1 "register_operand" "w")
843 (match_operand:SI 2 "immediate_operand" "i")]
844 UNSPEC_VEC_SHR))]
845 "TARGET_SIMD"
846 {
847 if (BYTES_BIG_ENDIAN)
848 return "shl %d0, %d1, %2";
849 else
850 return "ushr %d0, %d1, %2";
851 }
852 [(set_attr "type" "neon_shift_imm")]
853 )
855 (define_insn "aarch64_simd_vec_setv2di"
856 [(set (match_operand:V2DI 0 "register_operand" "=w,w")
857 (vec_merge:V2DI
858 (vec_duplicate:V2DI
859 (match_operand:DI 1 "register_operand" "r,w"))
860 (match_operand:V2DI 3 "register_operand" "0,0")
861 (match_operand:SI 2 "immediate_operand" "i,i")))]
862 "TARGET_SIMD"
863 {
864 int elt = ENDIAN_LANE_N (V2DImode, exact_log2 (INTVAL (operands[2])));
865 operands[2] = GEN_INT ((HOST_WIDE_INT) 1 << elt);
866 switch (which_alternative)
867 {
868 case 0:
869 return "ins\\t%0.d[%p2], %1";
870 case 1:
871 return "ins\\t%0.d[%p2], %1.d[0]";
872 default:
873 gcc_unreachable ();
874 }
875 }
876 [(set_attr "type" "neon_from_gp, neon_ins_q")]
877 )
879 (define_expand "vec_setv2di"
880 [(match_operand:V2DI 0 "register_operand")
881 (match_operand:DI 1 "register_operand")
882 (match_operand:SI 2 "immediate_operand")]
883 "TARGET_SIMD"
884 {
885 HOST_WIDE_INT elem = (HOST_WIDE_INT) 1 << INTVAL (operands[2]);
886 emit_insn (gen_aarch64_simd_vec_setv2di (operands[0], operands[1],
887 GEN_INT (elem), operands[0]));
888 DONE;
889 }
890 )
892 (define_insn "aarch64_simd_vec_set<mode>"
893 [(set (match_operand:VDQF_F16 0 "register_operand" "=w")
894 (vec_merge:VDQF_F16
895 (vec_duplicate:VDQF_F16
896 (match_operand:<VEL> 1 "register_operand" "w"))
897 (match_operand:VDQF_F16 3 "register_operand" "0")
898 (match_operand:SI 2 "immediate_operand" "i")))]
899 "TARGET_SIMD"
900 {
901 int elt = ENDIAN_LANE_N (<MODE>mode, exact_log2 (INTVAL (operands[2])));
903 operands[2] = GEN_INT ((HOST_WIDE_INT)1 << elt);
904 return "ins\t%0.<Vetype>[%p2], %1.<Vetype>[0]";
905 }
906 [(set_attr "type" "neon_ins<q>")]
907 )
909 (define_expand "vec_set<mode>"
910 [(match_operand:VDQF_F16 0 "register_operand" "+w")
911 (match_operand:<VEL> 1 "register_operand" "w")
912 (match_operand:SI 2 "immediate_operand" "")]
913 "TARGET_SIMD"
914 {
915 HOST_WIDE_INT elem = (HOST_WIDE_INT) 1 << INTVAL (operands[2]);
916 emit_insn (gen_aarch64_simd_vec_set<mode> (operands[0], operands[1],
917 GEN_INT (elem), operands[0]));
918 DONE;
919 }
920 )
923 (define_insn "aarch64_mla<mode>"
924 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
925 (plus:VDQ_BHSI (mult:VDQ_BHSI
926 (match_operand:VDQ_BHSI 2 "register_operand" "w")
927 (match_operand:VDQ_BHSI 3 "register_operand" "w"))
928 (match_operand:VDQ_BHSI 1 "register_operand" "0")))]
929 "TARGET_SIMD"
930 "mla\t%0.<Vtype>, %2.<Vtype>, %3.<Vtype>"
931 [(set_attr "type" "neon_mla_<Vetype><q>")]
932 )
934 (define_insn "*aarch64_mla_elt<mode>"
935 [(set (match_operand:VDQHS 0 "register_operand" "=w")
936 (plus:VDQHS
937 (mult:VDQHS
938 (vec_duplicate:VDQHS
939 (vec_select:<VEL>
940 (match_operand:VDQHS 1 "register_operand" "<h_con>")
941 (parallel [(match_operand:SI 2 "immediate_operand")])))
942 (match_operand:VDQHS 3 "register_operand" "w"))
943 (match_operand:VDQHS 4 "register_operand" "0")))]
944 "TARGET_SIMD"
945 {
946 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
947 return "mla\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
948 }
949 [(set_attr "type" "neon_mla_<Vetype>_scalar<q>")]
950 )
952 (define_insn "*aarch64_mla_elt_<vswap_width_name><mode>"
953 [(set (match_operand:VDQHS 0 "register_operand" "=w")
954 (plus:VDQHS
955 (mult:VDQHS
956 (vec_duplicate:VDQHS
957 (vec_select:<VEL>
958 (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
959 (parallel [(match_operand:SI 2 "immediate_operand")])))
960 (match_operand:VDQHS 3 "register_operand" "w"))
961 (match_operand:VDQHS 4 "register_operand" "0")))]
962 "TARGET_SIMD"
963 {
964 operands[2] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
965 INTVAL (operands[2])));
966 return "mla\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
967 }
968 [(set_attr "type" "neon_mla_<Vetype>_scalar<q>")]
969 )
971 (define_insn "aarch64_mls<mode>"
972 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
973 (minus:VDQ_BHSI (match_operand:VDQ_BHSI 1 "register_operand" "0")
974 (mult:VDQ_BHSI (match_operand:VDQ_BHSI 2 "register_operand" "w")
975 (match_operand:VDQ_BHSI 3 "register_operand" "w"))))]
976 "TARGET_SIMD"
977 "mls\t%0.<Vtype>, %2.<Vtype>, %3.<Vtype>"
978 [(set_attr "type" "neon_mla_<Vetype><q>")]
979 )
981 (define_insn "*aarch64_mls_elt<mode>"
982 [(set (match_operand:VDQHS 0 "register_operand" "=w")
983 (minus:VDQHS
984 (match_operand:VDQHS 4 "register_operand" "0")
985 (mult:VDQHS
986 (vec_duplicate:VDQHS
987 (vec_select:<VEL>
988 (match_operand:VDQHS 1 "register_operand" "<h_con>")
989 (parallel [(match_operand:SI 2 "immediate_operand")])))
990 (match_operand:VDQHS 3 "register_operand" "w"))))]
991 "TARGET_SIMD"
992 {
993 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
994 return "mls\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
995 }
996 [(set_attr "type" "neon_mla_<Vetype>_scalar<q>")]
997 )
999 (define_insn "*aarch64_mls_elt_<vswap_width_name><mode>"
1000 [(set (match_operand:VDQHS 0 "register_operand" "=w")
1001 (minus:VDQHS
1002 (match_operand:VDQHS 4 "register_operand" "0")
1003 (mult:VDQHS
1004 (vec_duplicate:VDQHS
1005 (vec_select:<VEL>
1006 (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
1007 (parallel [(match_operand:SI 2 "immediate_operand")])))
1008 (match_operand:VDQHS 3 "register_operand" "w"))))]
1009 "TARGET_SIMD"
1010 {
1011 operands[2] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
1012 INTVAL (operands[2])));
1013 return "mls\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
1014 }
1015 [(set_attr "type" "neon_mla_<Vetype>_scalar<q>")]
1016 )
1018 ;; Max/Min operations.
1019 (define_insn "<su><maxmin><mode>3"
1020 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
1021 (MAXMIN:VDQ_BHSI (match_operand:VDQ_BHSI 1 "register_operand" "w")
1022 (match_operand:VDQ_BHSI 2 "register_operand" "w")))]
1023 "TARGET_SIMD"
1024 "<su><maxmin>\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1025 [(set_attr "type" "neon_minmax<q>")]
1026 )
1028 (define_expand "<su><maxmin>v2di3"
1029 [(set (match_operand:V2DI 0 "register_operand" "")
1030 (MAXMIN:V2DI (match_operand:V2DI 1 "register_operand" "")
1031 (match_operand:V2DI 2 "register_operand" "")))]
1032 "TARGET_SIMD"
1033 {
1034 enum rtx_code cmp_operator;
1035 rtx cmp_fmt;
1037 switch (<CODE>)
1038 {
1039 case UMIN:
1040 cmp_operator = LTU;
1041 break;
1042 case SMIN:
1043 cmp_operator = LT;
1044 break;
1045 case UMAX:
1046 cmp_operator = GTU;
1047 break;
1048 case SMAX:
1049 cmp_operator = GT;
1050 break;
1051 default:
1052 gcc_unreachable ();
1053 }
1055 cmp_fmt = gen_rtx_fmt_ee (cmp_operator, V2DImode, operands[1], operands[2]);
1056 emit_insn (gen_aarch64_vcond_internalv2div2di (operands[0], operands[1],
1057 operands[2], cmp_fmt, operands[1], operands[2]));
1058 DONE;
1059 })
1061 ;; Pairwise Integer Max/Min operations.
1062 (define_insn "aarch64_<maxmin_uns>p<mode>"
1063 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
1064 (unspec:VDQ_BHSI [(match_operand:VDQ_BHSI 1 "register_operand" "w")
1065 (match_operand:VDQ_BHSI 2 "register_operand" "w")]
1066 MAXMINV))]
1067 "TARGET_SIMD"
1068 "<maxmin_uns_op>p\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1069 [(set_attr "type" "neon_minmax<q>")]
1070 )
1072 ;; Pairwise FP Max/Min operations.
1073 (define_insn "aarch64_<maxmin_uns>p<mode>"
1074 [(set (match_operand:VDQF 0 "register_operand" "=w")
1075 (unspec:VDQF [(match_operand:VDQF 1 "register_operand" "w")
1076 (match_operand:VDQF 2 "register_operand" "w")]
1077 FMAXMINV))]
1078 "TARGET_SIMD"
1079 "<maxmin_uns_op>p\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1080 [(set_attr "type" "neon_minmax<q>")]
1081 )
1083 ;; vec_concat gives a new vector with the low elements from operand 1, and
1084 ;; the high elements from operand 2. That is to say, given op1 = { a, b }
1085 ;; op2 = { c, d }, vec_concat (op1, op2) = { a, b, c, d }.
1086 ;; What that means, is that the RTL descriptions of the below patterns
1087 ;; need to change depending on endianness.
1089 ;; Move to the low architectural bits of the register.
1090 ;; On little-endian this is { operand, zeroes }
1091 ;; On big-endian this is { zeroes, operand }
1093 (define_insn "move_lo_quad_internal_<mode>"
1094 [(set (match_operand:VQ_NO2E 0 "register_operand" "=w,w,w")
1095 (vec_concat:VQ_NO2E
1096 (match_operand:<VHALF> 1 "register_operand" "w,r,r")
1097 (vec_duplicate:<VHALF> (const_int 0))))]
1098 "TARGET_SIMD && !BYTES_BIG_ENDIAN"
1099 "@
1100 dup\\t%d0, %1.d[0]
1101 fmov\\t%d0, %1
1102 dup\\t%d0, %1"
1103 [(set_attr "type" "neon_dup<q>,f_mcr,neon_dup<q>")
1104 (set_attr "simd" "yes,*,yes")
1105 (set_attr "fp" "*,yes,*")
1106 (set_attr "length" "4")]
1107 )
1109 (define_insn "move_lo_quad_internal_<mode>"
1110 [(set (match_operand:VQ_2E 0 "register_operand" "=w,w,w")
1111 (vec_concat:VQ_2E
1112 (match_operand:<VHALF> 1 "register_operand" "w,r,r")
1113 (const_int 0)))]
1114 "TARGET_SIMD && !BYTES_BIG_ENDIAN"
1115 "@
1116 dup\\t%d0, %1.d[0]
1117 fmov\\t%d0, %1
1118 dup\\t%d0, %1"
1119 [(set_attr "type" "neon_dup<q>,f_mcr,neon_dup<q>")
1120 (set_attr "simd" "yes,*,yes")
1121 (set_attr "fp" "*,yes,*")
1122 (set_attr "length" "4")]
1123 )
1125 (define_insn "move_lo_quad_internal_be_<mode>"
1126 [(set (match_operand:VQ_NO2E 0 "register_operand" "=w,w,w")
1127 (vec_concat:VQ_NO2E
1128 (vec_duplicate:<VHALF> (const_int 0))
1129 (match_operand:<VHALF> 1 "register_operand" "w,r,r")))]
1130 "TARGET_SIMD && BYTES_BIG_ENDIAN"
1131 "@
1132 dup\\t%d0, %1.d[0]
1133 fmov\\t%d0, %1
1134 dup\\t%d0, %1"
1135 [(set_attr "type" "neon_dup<q>,f_mcr,neon_dup<q>")
1136 (set_attr "simd" "yes,*,yes")
1137 (set_attr "fp" "*,yes,*")
1138 (set_attr "length" "4")]
1139 )
1141 (define_insn "move_lo_quad_internal_be_<mode>"
1142 [(set (match_operand:VQ_2E 0 "register_operand" "=w,w,w")
1143 (vec_concat:VQ_2E
1144 (const_int 0)
1145 (match_operand:<VHALF> 1 "register_operand" "w,r,r")))]
1146 "TARGET_SIMD && BYTES_BIG_ENDIAN"
1147 "@
1148 dup\\t%d0, %1.d[0]
1149 fmov\\t%d0, %1
1150 dup\\t%d0, %1"
1151 [(set_attr "type" "neon_dup<q>,f_mcr,neon_dup<q>")
1152 (set_attr "simd" "yes,*,yes")
1153 (set_attr "fp" "*,yes,*")
1154 (set_attr "length" "4")]
1155 )
1157 (define_expand "move_lo_quad_<mode>"
1158 [(match_operand:VQ 0 "register_operand")
1159 (match_operand:VQ 1 "register_operand")]
1160 "TARGET_SIMD"
1161 {
1162 if (BYTES_BIG_ENDIAN)
1163 emit_insn (gen_move_lo_quad_internal_be_<mode> (operands[0], operands[1]));
1164 else
1165 emit_insn (gen_move_lo_quad_internal_<mode> (operands[0], operands[1]));
1166 DONE;
1167 }
1168 )
1170 ;; Move operand1 to the high architectural bits of the register, keeping
1171 ;; the low architectural bits of operand2.
1172 ;; For little-endian this is { operand2, operand1 }
1173 ;; For big-endian this is { operand1, operand2 }
1175 (define_insn "aarch64_simd_move_hi_quad_<mode>"
1176 [(set (match_operand:VQ 0 "register_operand" "+w,w")
1177 (vec_concat:VQ
1178 (vec_select:<VHALF>
1179 (match_dup 0)
1180 (match_operand:VQ 2 "vect_par_cnst_lo_half" ""))
1181 (match_operand:<VHALF> 1 "register_operand" "w,r")))]
1182 "TARGET_SIMD && !BYTES_BIG_ENDIAN"
1183 "@
1184 ins\\t%0.d[1], %1.d[0]
1185 ins\\t%0.d[1], %1"
1186 [(set_attr "type" "neon_ins")]
1187 )
1189 (define_insn "aarch64_simd_move_hi_quad_be_<mode>"
1190 [(set (match_operand:VQ 0 "register_operand" "+w,w")
1191 (vec_concat:VQ
1192 (match_operand:<VHALF> 1 "register_operand" "w,r")
1193 (vec_select:<VHALF>
1194 (match_dup 0)
1195 (match_operand:VQ 2 "vect_par_cnst_lo_half" ""))))]
1196 "TARGET_SIMD && BYTES_BIG_ENDIAN"
1197 "@
1198 ins\\t%0.d[1], %1.d[0]
1199 ins\\t%0.d[1], %1"
1200 [(set_attr "type" "neon_ins")]
1201 )
1203 (define_expand "move_hi_quad_<mode>"
1204 [(match_operand:VQ 0 "register_operand" "")
1205 (match_operand:<VHALF> 1 "register_operand" "")]
1206 "TARGET_SIMD"
1207 {
1208 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
1209 if (BYTES_BIG_ENDIAN)
1210 emit_insn (gen_aarch64_simd_move_hi_quad_be_<mode> (operands[0],
1211 operands[1], p));
1212 else
1213 emit_insn (gen_aarch64_simd_move_hi_quad_<mode> (operands[0],
1214 operands[1], p));
1215 DONE;
1216 })
1218 ;; Narrowing operations.
1220 ;; For doubles.
1221 (define_insn "aarch64_simd_vec_pack_trunc_<mode>"
1222 [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
1223 (truncate:<VNARROWQ> (match_operand:VQN 1 "register_operand" "w")))]
1224 "TARGET_SIMD"
1225 "xtn\\t%0.<Vntype>, %1.<Vtype>"
1226 [(set_attr "type" "neon_shift_imm_narrow_q")]
1227 )
1229 (define_expand "vec_pack_trunc_<mode>"
1230 [(match_operand:<VNARROWD> 0 "register_operand" "")
1231 (match_operand:VDN 1 "register_operand" "")
1232 (match_operand:VDN 2 "register_operand" "")]
1233 "TARGET_SIMD"
1234 {
1235 rtx tempreg = gen_reg_rtx (<VDBL>mode);
1236 int lo = BYTES_BIG_ENDIAN ? 2 : 1;
1237 int hi = BYTES_BIG_ENDIAN ? 1 : 2;
1239 emit_insn (gen_move_lo_quad_<Vdbl> (tempreg, operands[lo]));
1240 emit_insn (gen_move_hi_quad_<Vdbl> (tempreg, operands[hi]));
1241 emit_insn (gen_aarch64_simd_vec_pack_trunc_<Vdbl> (operands[0], tempreg));
1242 DONE;
1243 })
1245 ;; For quads.
1247 (define_insn "vec_pack_trunc_<mode>"
1248 [(set (match_operand:<VNARROWQ2> 0 "register_operand" "=&w")
1249 (vec_concat:<VNARROWQ2>
1250 (truncate:<VNARROWQ> (match_operand:VQN 1 "register_operand" "w"))
1251 (truncate:<VNARROWQ> (match_operand:VQN 2 "register_operand" "w"))))]
1252 "TARGET_SIMD"
1253 {
1254 if (BYTES_BIG_ENDIAN)
1255 return "xtn\\t%0.<Vntype>, %2.<Vtype>\;xtn2\\t%0.<V2ntype>, %1.<Vtype>";
1256 else
1257 return "xtn\\t%0.<Vntype>, %1.<Vtype>\;xtn2\\t%0.<V2ntype>, %2.<Vtype>";
1258 }
1259 [(set_attr "type" "multiple")
1260 (set_attr "length" "8")]
1261 )
1263 ;; Widening operations.
1265 (define_insn "aarch64_simd_vec_unpack<su>_lo_<mode>"
1266 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1267 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1268 (match_operand:VQW 1 "register_operand" "w")
1269 (match_operand:VQW 2 "vect_par_cnst_lo_half" "")
1270 )))]
1271 "TARGET_SIMD"
1272 "<su>shll\t%0.<Vwtype>, %1.<Vhalftype>, 0"
1273 [(set_attr "type" "neon_shift_imm_long")]
1274 )
1276 (define_insn "aarch64_simd_vec_unpack<su>_hi_<mode>"
1277 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1278 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1279 (match_operand:VQW 1 "register_operand" "w")
1280 (match_operand:VQW 2 "vect_par_cnst_hi_half" "")
1281 )))]
1282 "TARGET_SIMD"
1283 "<su>shll2\t%0.<Vwtype>, %1.<Vtype>, 0"
1284 [(set_attr "type" "neon_shift_imm_long")]
1285 )
1287 (define_expand "vec_unpack<su>_hi_<mode>"
1288 [(match_operand:<VWIDE> 0 "register_operand" "")
1289 (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand"))]
1290 "TARGET_SIMD"
1291 {
1292 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
1293 emit_insn (gen_aarch64_simd_vec_unpack<su>_hi_<mode> (operands[0],
1294 operands[1], p));
1295 DONE;
1296 }
1297 )
1299 (define_expand "vec_unpack<su>_lo_<mode>"
1300 [(match_operand:<VWIDE> 0 "register_operand" "")
1301 (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand" ""))]
1302 "TARGET_SIMD"
1303 {
1304 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
1305 emit_insn (gen_aarch64_simd_vec_unpack<su>_lo_<mode> (operands[0],
1306 operands[1], p));
1307 DONE;
1308 }
1309 )
1311 ;; Widening arithmetic.
1313 (define_insn "*aarch64_<su>mlal_lo<mode>"
1314 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1315 (plus:<VWIDE>
1316 (mult:<VWIDE>
1317 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1318 (match_operand:VQW 2 "register_operand" "w")
1319 (match_operand:VQW 3 "vect_par_cnst_lo_half" "")))
1320 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1321 (match_operand:VQW 4 "register_operand" "w")
1322 (match_dup 3))))
1323 (match_operand:<VWIDE> 1 "register_operand" "0")))]
1324 "TARGET_SIMD"
1325 "<su>mlal\t%0.<Vwtype>, %2.<Vhalftype>, %4.<Vhalftype>"
1326 [(set_attr "type" "neon_mla_<Vetype>_long")]
1327 )
1329 (define_insn "*aarch64_<su>mlal_hi<mode>"
1330 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1331 (plus:<VWIDE>
1332 (mult:<VWIDE>
1333 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1334 (match_operand:VQW 2 "register_operand" "w")
1335 (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))
1336 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1337 (match_operand:VQW 4 "register_operand" "w")
1338 (match_dup 3))))
1339 (match_operand:<VWIDE> 1 "register_operand" "0")))]
1340 "TARGET_SIMD"
1341 "<su>mlal2\t%0.<Vwtype>, %2.<Vtype>, %4.<Vtype>"
1342 [(set_attr "type" "neon_mla_<Vetype>_long")]
1343 )
1345 (define_insn "*aarch64_<su>mlsl_lo<mode>"
1346 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1347 (minus:<VWIDE>
1348 (match_operand:<VWIDE> 1 "register_operand" "0")
1349 (mult:<VWIDE>
1350 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1351 (match_operand:VQW 2 "register_operand" "w")
1352 (match_operand:VQW 3 "vect_par_cnst_lo_half" "")))
1353 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1354 (match_operand:VQW 4 "register_operand" "w")
1355 (match_dup 3))))))]
1356 "TARGET_SIMD"
1357 "<su>mlsl\t%0.<Vwtype>, %2.<Vhalftype>, %4.<Vhalftype>"
1358 [(set_attr "type" "neon_mla_<Vetype>_long")]
1359 )
1361 (define_insn "*aarch64_<su>mlsl_hi<mode>"
1362 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1363 (minus:<VWIDE>
1364 (match_operand:<VWIDE> 1 "register_operand" "0")
1365 (mult:<VWIDE>
1366 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1367 (match_operand:VQW 2 "register_operand" "w")
1368 (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))
1369 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1370 (match_operand:VQW 4 "register_operand" "w")
1371 (match_dup 3))))))]
1372 "TARGET_SIMD"
1373 "<su>mlsl2\t%0.<Vwtype>, %2.<Vtype>, %4.<Vtype>"
1374 [(set_attr "type" "neon_mla_<Vetype>_long")]
1375 )
1377 (define_insn "*aarch64_<su>mlal<mode>"
1378 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1379 (plus:<VWIDE>
1380 (mult:<VWIDE>
1381 (ANY_EXTEND:<VWIDE>
1382 (match_operand:VD_BHSI 1 "register_operand" "w"))
1383 (ANY_EXTEND:<VWIDE>
1384 (match_operand:VD_BHSI 2 "register_operand" "w")))
1385 (match_operand:<VWIDE> 3 "register_operand" "0")))]
1386 "TARGET_SIMD"
1387 "<su>mlal\t%0.<Vwtype>, %1.<Vtype>, %2.<Vtype>"
1388 [(set_attr "type" "neon_mla_<Vetype>_long")]
1389 )
1391 (define_insn "*aarch64_<su>mlsl<mode>"
1392 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1393 (minus:<VWIDE>
1394 (match_operand:<VWIDE> 1 "register_operand" "0")
1395 (mult:<VWIDE>
1396 (ANY_EXTEND:<VWIDE>
1397 (match_operand:VD_BHSI 2 "register_operand" "w"))
1398 (ANY_EXTEND:<VWIDE>
1399 (match_operand:VD_BHSI 3 "register_operand" "w")))))]
1400 "TARGET_SIMD"
1401 "<su>mlsl\t%0.<Vwtype>, %2.<Vtype>, %3.<Vtype>"
1402 [(set_attr "type" "neon_mla_<Vetype>_long")]
1403 )
1405 (define_insn "aarch64_simd_vec_<su>mult_lo_<mode>"
1406 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1407 (mult:<VWIDE> (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1408 (match_operand:VQW 1 "register_operand" "w")
1409 (match_operand:VQW 3 "vect_par_cnst_lo_half" "")))
1410 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1411 (match_operand:VQW 2 "register_operand" "w")
1412 (match_dup 3)))))]
1413 "TARGET_SIMD"
1414 "<su>mull\\t%0.<Vwtype>, %1.<Vhalftype>, %2.<Vhalftype>"
1415 [(set_attr "type" "neon_mul_<Vetype>_long")]
1416 )
1418 (define_expand "vec_widen_<su>mult_lo_<mode>"
1419 [(match_operand:<VWIDE> 0 "register_operand" "")
1420 (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand" ""))
1421 (ANY_EXTEND:<VWIDE> (match_operand:VQW 2 "register_operand" ""))]
1422 "TARGET_SIMD"
1423 {
1424 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
1425 emit_insn (gen_aarch64_simd_vec_<su>mult_lo_<mode> (operands[0],
1426 operands[1],
1427 operands[2], p));
1428 DONE;
1429 }
1430 )
1432 (define_insn "aarch64_simd_vec_<su>mult_hi_<mode>"
1433 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1434 (mult:<VWIDE> (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1435 (match_operand:VQW 1 "register_operand" "w")
1436 (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))
1437 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1438 (match_operand:VQW 2 "register_operand" "w")
1439 (match_dup 3)))))]
1440 "TARGET_SIMD"
1441 "<su>mull2\\t%0.<Vwtype>, %1.<Vtype>, %2.<Vtype>"
1442 [(set_attr "type" "neon_mul_<Vetype>_long")]
1443 )
1445 (define_expand "vec_widen_<su>mult_hi_<mode>"
1446 [(match_operand:<VWIDE> 0 "register_operand" "")
1447 (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand" ""))
1448 (ANY_EXTEND:<VWIDE> (match_operand:VQW 2 "register_operand" ""))]
1449 "TARGET_SIMD"
1450 {
1451 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
1452 emit_insn (gen_aarch64_simd_vec_<su>mult_hi_<mode> (operands[0],
1453 operands[1],
1454 operands[2], p));
1455 DONE;
1457 }
1458 )
1460 ;; FP vector operations.
1461 ;; AArch64 AdvSIMD supports single-precision (32-bit) and
1462 ;; double-precision (64-bit) floating-point data types and arithmetic as
1463 ;; defined by the IEEE 754-2008 standard. This makes them vectorizable
1464 ;; without the need for -ffast-math or -funsafe-math-optimizations.
1465 ;;
1466 ;; Floating-point operations can raise an exception. Vectorizing such
1467 ;; operations are safe because of reasons explained below.
1468 ;;
1469 ;; ARMv8 permits an extension to enable trapped floating-point
1470 ;; exception handling, however this is an optional feature. In the
1471 ;; event of a floating-point exception being raised by vectorised
1472 ;; code then:
1473 ;; 1. If trapped floating-point exceptions are available, then a trap
1474 ;; will be taken when any lane raises an enabled exception. A trap
1475 ;; handler may determine which lane raised the exception.
1476 ;; 2. Alternatively a sticky exception flag is set in the
1477 ;; floating-point status register (FPSR). Software may explicitly
1478 ;; test the exception flags, in which case the tests will either
1479 ;; prevent vectorisation, allowing precise identification of the
1480 ;; failing operation, or if tested outside of vectorisable regions
1481 ;; then the specific operation and lane are not of interest.
1483 ;; FP arithmetic operations.
1485 (define_insn "add<mode>3"
1486 [(set (match_operand:VDQF 0 "register_operand" "=w")
1487 (plus:VDQF (match_operand:VDQF 1 "register_operand" "w")
1488 (match_operand:VDQF 2 "register_operand" "w")))]
1489 "TARGET_SIMD"
1490 "fadd\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1491 [(set_attr "type" "neon_fp_addsub_<Vetype><q>")]
1492 )
1494 (define_insn "sub<mode>3"
1495 [(set (match_operand:VDQF 0 "register_operand" "=w")
1496 (minus:VDQF (match_operand:VDQF 1 "register_operand" "w")
1497 (match_operand:VDQF 2 "register_operand" "w")))]
1498 "TARGET_SIMD"
1499 "fsub\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1500 [(set_attr "type" "neon_fp_addsub_<Vetype><q>")]
1501 )
1503 (define_insn "mul<mode>3"
1504 [(set (match_operand:VDQF 0 "register_operand" "=w")
1505 (mult:VDQF (match_operand:VDQF 1 "register_operand" "w")
1506 (match_operand:VDQF 2 "register_operand" "w")))]
1507 "TARGET_SIMD"
1508 "fmul\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1509 [(set_attr "type" "neon_fp_mul_<Vetype><q>")]
1510 )
1512 (define_insn "div<mode>3"
1513 [(set (match_operand:VDQF 0 "register_operand" "=w")
1514 (div:VDQF (match_operand:VDQF 1 "register_operand" "w")
1515 (match_operand:VDQF 2 "register_operand" "w")))]
1516 "TARGET_SIMD"
1517 "fdiv\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1518 [(set_attr "type" "neon_fp_div_<Vetype><q>")]
1519 )
1521 (define_insn "neg<mode>2"
1522 [(set (match_operand:VDQF 0 "register_operand" "=w")
1523 (neg:VDQF (match_operand:VDQF 1 "register_operand" "w")))]
1524 "TARGET_SIMD"
1525 "fneg\\t%0.<Vtype>, %1.<Vtype>"
1526 [(set_attr "type" "neon_fp_neg_<Vetype><q>")]
1527 )
1529 (define_insn "abs<mode>2"
1530 [(set (match_operand:VDQF 0 "register_operand" "=w")
1531 (abs:VDQF (match_operand:VDQF 1 "register_operand" "w")))]
1532 "TARGET_SIMD"
1533 "fabs\\t%0.<Vtype>, %1.<Vtype>"
1534 [(set_attr "type" "neon_fp_abs_<Vetype><q>")]
1535 )
1537 (define_insn "fma<mode>4"
1538 [(set (match_operand:VDQF 0 "register_operand" "=w")
1539 (fma:VDQF (match_operand:VDQF 1 "register_operand" "w")
1540 (match_operand:VDQF 2 "register_operand" "w")
1541 (match_operand:VDQF 3 "register_operand" "0")))]
1542 "TARGET_SIMD"
1543 "fmla\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1544 [(set_attr "type" "neon_fp_mla_<Vetype><q>")]
1545 )
1547 (define_insn "*aarch64_fma4_elt<mode>"
1548 [(set (match_operand:VDQF 0 "register_operand" "=w")
1549 (fma:VDQF
1550 (vec_duplicate:VDQF
1551 (vec_select:<VEL>
1552 (match_operand:VDQF 1 "register_operand" "<h_con>")
1553 (parallel [(match_operand:SI 2 "immediate_operand")])))
1554 (match_operand:VDQF 3 "register_operand" "w")
1555 (match_operand:VDQF 4 "register_operand" "0")))]
1556 "TARGET_SIMD"
1557 {
1558 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
1559 return "fmla\\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
1560 }
1561 [(set_attr "type" "neon_fp_mla_<Vetype>_scalar<q>")]
1562 )
1564 (define_insn "*aarch64_fma4_elt_<vswap_width_name><mode>"
1565 [(set (match_operand:VDQSF 0 "register_operand" "=w")
1566 (fma:VDQSF
1567 (vec_duplicate:VDQSF
1568 (vec_select:<VEL>
1569 (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
1570 (parallel [(match_operand:SI 2 "immediate_operand")])))
1571 (match_operand:VDQSF 3 "register_operand" "w")
1572 (match_operand:VDQSF 4 "register_operand" "0")))]
1573 "TARGET_SIMD"
1574 {
1575 operands[2] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
1576 INTVAL (operands[2])));
1577 return "fmla\\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
1578 }
1579 [(set_attr "type" "neon_fp_mla_<Vetype>_scalar<q>")]
1580 )
1582 (define_insn "*aarch64_fma4_elt_to_128df"
1583 [(set (match_operand:V2DF 0 "register_operand" "=w")
1584 (fma:V2DF
1585 (vec_duplicate:V2DF
1586 (match_operand:DF 1 "register_operand" "w"))
1587 (match_operand:V2DF 2 "register_operand" "w")
1588 (match_operand:V2DF 3 "register_operand" "0")))]
1589 "TARGET_SIMD"
1590 "fmla\\t%0.2d, %2.2d, %1.2d[0]"
1591 [(set_attr "type" "neon_fp_mla_d_scalar_q")]
1592 )
1594 (define_insn "*aarch64_fma4_elt_to_64v2df"
1595 [(set (match_operand:DF 0 "register_operand" "=w")
1596 (fma:DF
1597 (vec_select:DF
1598 (match_operand:V2DF 1 "register_operand" "w")
1599 (parallel [(match_operand:SI 2 "immediate_operand")]))
1600 (match_operand:DF 3 "register_operand" "w")
1601 (match_operand:DF 4 "register_operand" "0")))]
1602 "TARGET_SIMD"
1603 {
1604 operands[2] = GEN_INT (ENDIAN_LANE_N (V2DFmode, INTVAL (operands[2])));
1605 return "fmla\\t%0.2d, %3.2d, %1.2d[%2]";
1606 }
1607 [(set_attr "type" "neon_fp_mla_d_scalar_q")]
1608 )
1610 (define_insn "fnma<mode>4"
1611 [(set (match_operand:VDQF 0 "register_operand" "=w")
1612 (fma:VDQF
1613 (match_operand:VDQF 1 "register_operand" "w")
1614 (neg:VDQF
1615 (match_operand:VDQF 2 "register_operand" "w"))
1616 (match_operand:VDQF 3 "register_operand" "0")))]
1617 "TARGET_SIMD"
1618 "fmls\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1619 [(set_attr "type" "neon_fp_mla_<Vetype><q>")]
1620 )
1622 (define_insn "*aarch64_fnma4_elt<mode>"
1623 [(set (match_operand:VDQF 0 "register_operand" "=w")
1624 (fma:VDQF
1625 (neg:VDQF
1626 (match_operand:VDQF 3 "register_operand" "w"))
1627 (vec_duplicate:VDQF
1628 (vec_select:<VEL>
1629 (match_operand:VDQF 1 "register_operand" "<h_con>")
1630 (parallel [(match_operand:SI 2 "immediate_operand")])))
1631 (match_operand:VDQF 4 "register_operand" "0")))]
1632 "TARGET_SIMD"
1633 {
1634 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
1635 return "fmls\\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
1636 }
1637 [(set_attr "type" "neon_fp_mla_<Vetype>_scalar<q>")]
1638 )
1640 (define_insn "*aarch64_fnma4_elt_<vswap_width_name><mode>"
1641 [(set (match_operand:VDQSF 0 "register_operand" "=w")
1642 (fma:VDQSF
1643 (neg:VDQSF
1644 (match_operand:VDQSF 3 "register_operand" "w"))
1645 (vec_duplicate:VDQSF
1646 (vec_select:<VEL>
1647 (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
1648 (parallel [(match_operand:SI 2 "immediate_operand")])))
1649 (match_operand:VDQSF 4 "register_operand" "0")))]
1650 "TARGET_SIMD"
1651 {
1652 operands[2] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
1653 INTVAL (operands[2])));
1654 return "fmls\\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
1655 }
1656 [(set_attr "type" "neon_fp_mla_<Vetype>_scalar<q>")]
1657 )
1659 (define_insn "*aarch64_fnma4_elt_to_128df"
1660 [(set (match_operand:V2DF 0 "register_operand" "=w")
1661 (fma:V2DF
1662 (neg:V2DF
1663 (match_operand:V2DF 2 "register_operand" "w"))
1664 (vec_duplicate:V2DF
1665 (match_operand:DF 1 "register_operand" "w"))
1666 (match_operand:V2DF 3 "register_operand" "0")))]
1667 "TARGET_SIMD"
1668 "fmls\\t%0.2d, %2.2d, %1.2d[0]"
1669 [(set_attr "type" "neon_fp_mla_d_scalar_q")]
1670 )
1672 (define_insn "*aarch64_fnma4_elt_to_64v2df"
1673 [(set (match_operand:DF 0 "register_operand" "=w")
1674 (fma:DF
1675 (vec_select:DF
1676 (match_operand:V2DF 1 "register_operand" "w")
1677 (parallel [(match_operand:SI 2 "immediate_operand")]))
1678 (neg:DF
1679 (match_operand:DF 3 "register_operand" "w"))
1680 (match_operand:DF 4 "register_operand" "0")))]
1681 "TARGET_SIMD"
1682 {
1683 operands[2] = GEN_INT (ENDIAN_LANE_N (V2DFmode, INTVAL (operands[2])));
1684 return "fmls\\t%0.2d, %3.2d, %1.2d[%2]";
1685 }
1686 [(set_attr "type" "neon_fp_mla_d_scalar_q")]
1687 )
1689 ;; Vector versions of the floating-point frint patterns.
1690 ;; Expands to btrunc, ceil, floor, nearbyint, rint, round, frintn.
1691 (define_insn "<frint_pattern><mode>2"
1692 [(set (match_operand:VDQF 0 "register_operand" "=w")
1693 (unspec:VDQF [(match_operand:VDQF 1 "register_operand" "w")]
1694 FRINT))]
1695 "TARGET_SIMD"
1696 "frint<frint_suffix>\\t%0.<Vtype>, %1.<Vtype>"
1697 [(set_attr "type" "neon_fp_round_<Vetype><q>")]
1698 )
1700 ;; Vector versions of the fcvt standard patterns.
1701 ;; Expands to lbtrunc, lround, lceil, lfloor
1702 (define_insn "l<fcvt_pattern><su_optab><VDQF:mode><fcvt_target>2"
1703 [(set (match_operand:<FCVT_TARGET> 0 "register_operand" "=w")
1704 (FIXUORS:<FCVT_TARGET> (unspec:<FCVT_TARGET>
1705 [(match_operand:VDQF 1 "register_operand" "w")]
1706 FCVT)))]
1707 "TARGET_SIMD"
1708 "fcvt<frint_suffix><su>\\t%0.<Vtype>, %1.<Vtype>"
1709 [(set_attr "type" "neon_fp_to_int_<Vetype><q>")]
1710 )
1712 (define_insn "*aarch64_fcvt<su_optab><VDQF:mode><fcvt_target>2_mult"
1713 [(set (match_operand:<FCVT_TARGET> 0 "register_operand" "=w")
1714 (FIXUORS:<FCVT_TARGET> (unspec:<FCVT_TARGET>
1715 [(mult:VDQF
1716 (match_operand:VDQF 1 "register_operand" "w")
1717 (match_operand:VDQF 2 "aarch64_fp_vec_pow2" ""))]
1718 UNSPEC_FRINTZ)))]
1719 "TARGET_SIMD
1720 && IN_RANGE (aarch64_vec_fpconst_pow_of_2 (operands[2]), 1,
1721 GET_MODE_BITSIZE (GET_MODE_INNER (<VDQF:MODE>mode)))"
1722 {
1723 int fbits = aarch64_vec_fpconst_pow_of_2 (operands[2]);
1724 char buf[64];
1725 snprintf (buf, 64, "fcvtz<su>\\t%%0.<Vtype>, %%1.<Vtype>, #%d", fbits);
1726 output_asm_insn (buf, operands);
1727 return "";
1728 }
1729 [(set_attr "type" "neon_fp_to_int_<Vetype><q>")]
1730 )
1732 (define_expand "<optab><VDQF:mode><fcvt_target>2"
1733 [(set (match_operand:<FCVT_TARGET> 0 "register_operand")
1734 (FIXUORS:<FCVT_TARGET> (unspec:<FCVT_TARGET>
1735 [(match_operand:VDQF 1 "register_operand")]
1736 UNSPEC_FRINTZ)))]
1737 "TARGET_SIMD"
1738 {})
1740 (define_expand "<fix_trunc_optab><VDQF:mode><fcvt_target>2"
1741 [(set (match_operand:<FCVT_TARGET> 0 "register_operand")
1742 (FIXUORS:<FCVT_TARGET> (unspec:<FCVT_TARGET>
1743 [(match_operand:VDQF 1 "register_operand")]
1744 UNSPEC_FRINTZ)))]
1745 "TARGET_SIMD"
1746 {})
1748 (define_expand "ftrunc<VDQF:mode>2"
1749 [(set (match_operand:VDQF 0 "register_operand")
1750 (unspec:VDQF [(match_operand:VDQF 1 "register_operand")]
1751 UNSPEC_FRINTZ))]
1752 "TARGET_SIMD"
1753 {})
1755 (define_insn "<optab><fcvt_target><VDQF:mode>2"
1756 [(set (match_operand:VDQF 0 "register_operand" "=w")
1757 (FLOATUORS:VDQF
1758 (match_operand:<FCVT_TARGET> 1 "register_operand" "w")))]
1759 "TARGET_SIMD"
1760 "<su_optab>cvtf\\t%0.<Vtype>, %1.<Vtype>"
1761 [(set_attr "type" "neon_int_to_fp_<Vetype><q>")]
1762 )
1764 ;; Conversions between vectors of floats and doubles.
1765 ;; Contains a mix of patterns to match standard pattern names
1766 ;; and those for intrinsics.
1768 ;; Float widening operations.
1770 (define_insn "aarch64_simd_vec_unpacks_lo_<mode>"
1771 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1772 (float_extend:<VWIDE> (vec_select:<VHALF>
1773 (match_operand:VQ_HSF 1 "register_operand" "w")
1774 (match_operand:VQ_HSF 2 "vect_par_cnst_lo_half" "")
1775 )))]
1776 "TARGET_SIMD"
1777 "fcvtl\\t%0.<Vwtype>, %1.<Vhalftype>"
1778 [(set_attr "type" "neon_fp_cvt_widen_s")]
1779 )
1781 ;; ??? Note that the vectorizer usage of the vec_unpacks_[lo/hi] patterns
1782 ;; is inconsistent with vector ordering elsewhere in the compiler, in that
1783 ;; the meaning of HI and LO changes depending on the target endianness.
1784 ;; While elsewhere we map the higher numbered elements of a vector to
1785 ;; the lower architectural lanes of the vector, for these patterns we want
1786 ;; to always treat "hi" as referring to the higher architectural lanes.
1787 ;; Consequently, while the patterns below look inconsistent with our
1788 ;; other big-endian patterns their behaviour is as required.
1790 (define_expand "vec_unpacks_lo_<mode>"
1791 [(match_operand:<VWIDE> 0 "register_operand" "")
1792 (match_operand:VQ_HSF 1 "register_operand" "")]
1793 "TARGET_SIMD"
1794 {
1795 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
1796 emit_insn (gen_aarch64_simd_vec_unpacks_lo_<mode> (operands[0],
1797 operands[1], p));
1798 DONE;
1799 }
1800 )
1802 (define_insn "aarch64_simd_vec_unpacks_hi_<mode>"
1803 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1804 (float_extend:<VWIDE> (vec_select:<VHALF>
1805 (match_operand:VQ_HSF 1 "register_operand" "w")
1806 (match_operand:VQ_HSF 2 "vect_par_cnst_hi_half" "")
1807 )))]
1808 "TARGET_SIMD"
1809 "fcvtl2\\t%0.<Vwtype>, %1.<Vtype>"
1810 [(set_attr "type" "neon_fp_cvt_widen_s")]
1811 )
1813 (define_expand "vec_unpacks_hi_<mode>"
1814 [(match_operand:<VWIDE> 0 "register_operand" "")
1815 (match_operand:VQ_HSF 1 "register_operand" "")]
1816 "TARGET_SIMD"
1817 {
1818 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
1819 emit_insn (gen_aarch64_simd_vec_unpacks_lo_<mode> (operands[0],
1820 operands[1], p));
1821 DONE;
1822 }
1823 )
1824 (define_insn "aarch64_float_extend_lo_<Vwide>"
1825 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1826 (float_extend:<VWIDE>
1827 (match_operand:VDF 1 "register_operand" "w")))]
1828 "TARGET_SIMD"
1829 "fcvtl\\t%0<Vmwtype>, %1<Vmtype>"
1830 [(set_attr "type" "neon_fp_cvt_widen_s")]
1831 )
1833 ;; Float narrowing operations.
1835 (define_insn "aarch64_float_truncate_lo_<mode>"
1836 [(set (match_operand:VDF 0 "register_operand" "=w")
1837 (float_truncate:VDF
1838 (match_operand:<VWIDE> 1 "register_operand" "w")))]
1839 "TARGET_SIMD"
1840 "fcvtn\\t%0.<Vtype>, %1<Vmwtype>"
1841 [(set_attr "type" "neon_fp_cvt_narrow_d_q")]
1842 )
1844 (define_insn "aarch64_float_truncate_hi_<Vdbl>_le"
1845 [(set (match_operand:<VDBL> 0 "register_operand" "=w")
1846 (vec_concat:<VDBL>
1847 (match_operand:VDF 1 "register_operand" "0")
1848 (float_truncate:VDF
1849 (match_operand:<VWIDE> 2 "register_operand" "w"))))]
1850 "TARGET_SIMD && !BYTES_BIG_ENDIAN"
1851 "fcvtn2\\t%0.<Vdtype>, %2<Vmwtype>"
1852 [(set_attr "type" "neon_fp_cvt_narrow_d_q")]
1853 )
1855 (define_insn "aarch64_float_truncate_hi_<Vdbl>_be"
1856 [(set (match_operand:<VDBL> 0 "register_operand" "=w")
1857 (vec_concat:<VDBL>
1858 (float_truncate:VDF
1859 (match_operand:<VWIDE> 2 "register_operand" "w"))
1860 (match_operand:VDF 1 "register_operand" "0")))]
1861 "TARGET_SIMD && BYTES_BIG_ENDIAN"
1862 "fcvtn2\\t%0.<Vdtype>, %2<Vmwtype>"
1863 [(set_attr "type" "neon_fp_cvt_narrow_d_q")]
1864 )
1866 (define_expand "aarch64_float_truncate_hi_<Vdbl>"
1867 [(match_operand:<VDBL> 0 "register_operand" "=w")
1868 (match_operand:VDF 1 "register_operand" "0")
1869 (match_operand:<VWIDE> 2 "register_operand" "w")]
1870 "TARGET_SIMD"
1871 {
1872 rtx (*gen) (rtx, rtx, rtx) = BYTES_BIG_ENDIAN
1873 ? gen_aarch64_float_truncate_hi_<Vdbl>_be
1874 : gen_aarch64_float_truncate_hi_<Vdbl>_le;
1875 emit_insn (gen (operands[0], operands[1], operands[2]));
1876 DONE;
1877 }
1878 )
1880 (define_expand "vec_pack_trunc_v2df"
1881 [(set (match_operand:V4SF 0 "register_operand")
1882 (vec_concat:V4SF
1883 (float_truncate:V2SF
1884 (match_operand:V2DF 1 "register_operand"))
1885 (float_truncate:V2SF
1886 (match_operand:V2DF 2 "register_operand"))
1887 ))]
1888 "TARGET_SIMD"
1889 {
1890 rtx tmp = gen_reg_rtx (V2SFmode);
1891 int lo = BYTES_BIG_ENDIAN ? 2 : 1;
1892 int hi = BYTES_BIG_ENDIAN ? 1 : 2;
1894 emit_insn (gen_aarch64_float_truncate_lo_v2sf (tmp, operands[lo]));
1895 emit_insn (gen_aarch64_float_truncate_hi_v4sf (operands[0],
1896 tmp, operands[hi]));
1897 DONE;
1898 }
1899 )
1901 (define_expand "vec_pack_trunc_df"
1902 [(set (match_operand:V2SF 0 "register_operand")
1903 (vec_concat:V2SF
1904 (float_truncate:SF
1905 (match_operand:DF 1 "register_operand"))
1906 (float_truncate:SF
1907 (match_operand:DF 2 "register_operand"))
1908 ))]
1909 "TARGET_SIMD"
1910 {
1911 rtx tmp = gen_reg_rtx (V2SFmode);
1912 int lo = BYTES_BIG_ENDIAN ? 2 : 1;
1913 int hi = BYTES_BIG_ENDIAN ? 1 : 2;
1915 emit_insn (gen_move_lo_quad_v2df (tmp, operands[lo]));
1916 emit_insn (gen_move_hi_quad_v2df (tmp, operands[hi]));
1917 emit_insn (gen_aarch64_float_truncate_lo_v2sf (operands[0], tmp));
1918 DONE;
1919 }
1920 )
1922 (define_insn "aarch64_vmls<mode>"
1923 [(set (match_operand:VDQF 0 "register_operand" "=w")
1924 (minus:VDQF (match_operand:VDQF 1 "register_operand" "0")
1925 (mult:VDQF (match_operand:VDQF 2 "register_operand" "w")
1926 (match_operand:VDQF 3 "register_operand" "w"))))]
1927 "TARGET_SIMD"
1928 "fmls\\t%0.<Vtype>, %2.<Vtype>, %3.<Vtype>"
1929 [(set_attr "type" "neon_fp_mla_<Vetype>_scalar<q>")]
1930 )
1932 ;; FP Max/Min
1933 ;; Max/Min are introduced by idiom recognition by GCC's mid-end. An
1934 ;; expression like:
1935 ;; a = (b < c) ? b : c;
1936 ;; is idiom-matched as MIN_EXPR<b,c> only if -ffinite-math-only is enabled
1937 ;; either explicitly or indirectly via -ffast-math.
1938 ;;
1939 ;; MIN_EXPR and MAX_EXPR eventually map to 'smin' and 'smax' in RTL.
1940 ;; The 'smax' and 'smin' RTL standard pattern names do not specify which
1941 ;; operand will be returned when both operands are zero (i.e. they may not
1942 ;; honour signed zeroes), or when either operand is NaN. Therefore GCC
1943 ;; only introduces MIN_EXPR/MAX_EXPR in fast math mode or when not honouring
1944 ;; NaNs.
1946 (define_insn "<su><maxmin><mode>3"
1947 [(set (match_operand:VDQF 0 "register_operand" "=w")
1948 (FMAXMIN:VDQF (match_operand:VDQF 1 "register_operand" "w")
1949 (match_operand:VDQF 2 "register_operand" "w")))]
1950 "TARGET_SIMD"
1951 "f<maxmin>nm\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1952 [(set_attr "type" "neon_fp_minmax_<Vetype><q>")]
1953 )
1955 (define_insn "<maxmin_uns><mode>3"
1956 [(set (match_operand:VDQF 0 "register_operand" "=w")
1957 (unspec:VDQF [(match_operand:VDQF 1 "register_operand" "w")
1958 (match_operand:VDQF 2 "register_operand" "w")]
1959 FMAXMIN_UNS))]
1960 "TARGET_SIMD"
1961 "<maxmin_uns_op>\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1962 [(set_attr "type" "neon_fp_minmax_<Vetype><q>")]
1963 )
1965 ;; Auto-vectorized forms for the IEEE-754 fmax()/fmin() functions
1966 (define_insn "<fmaxmin><mode>3"
1967 [(set (match_operand:VDQF 0 "register_operand" "=w")
1968 (unspec:VDQF [(match_operand:VDQF 1 "register_operand" "w")
1969 (match_operand:VDQF 2 "register_operand" "w")]
1970 FMAXMIN))]
1971 "TARGET_SIMD"
1972 "<fmaxmin_op>\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1973 [(set_attr "type" "neon_fp_minmax_<Vetype><q>")]
1974 )
1976 ;; 'across lanes' add.
1978 (define_expand "reduc_plus_scal_<mode>"
1979 [(match_operand:<VEL> 0 "register_operand" "=w")
1980 (unspec:VDQ_I [(match_operand:VDQ_I 1 "register_operand" "w")]
1981 UNSPEC_ADDV)]
1982 "TARGET_SIMD"
1983 {
1984 rtx elt = GEN_INT (ENDIAN_LANE_N (<MODE>mode, 0));
1985 rtx scratch = gen_reg_rtx (<MODE>mode);
1986 emit_insn (gen_aarch64_reduc_plus_internal<mode> (scratch, operands[1]));
1987 emit_insn (gen_aarch64_get_lane<mode> (operands[0], scratch, elt));
1988 DONE;
1989 }
1990 )
1992 (define_expand "reduc_plus_scal_<mode>"
1993 [(match_operand:<VEL> 0 "register_operand" "=w")
1994 (match_operand:V2F 1 "register_operand" "w")]
1995 "TARGET_SIMD"
1996 {
1997 rtx elt = GEN_INT (ENDIAN_LANE_N (<MODE>mode, 0));
1998 rtx scratch = gen_reg_rtx (<MODE>mode);
1999 emit_insn (gen_aarch64_reduc_plus_internal<mode> (scratch, operands[1]));
2000 emit_insn (gen_aarch64_get_lane<mode> (operands[0], scratch, elt));
2001 DONE;
2002 }
2003 )
2005 (define_insn "aarch64_reduc_plus_internal<mode>"
2006 [(set (match_operand:VDQV 0 "register_operand" "=w")
2007 (unspec:VDQV [(match_operand:VDQV 1 "register_operand" "w")]
2008 UNSPEC_ADDV))]
2009 "TARGET_SIMD"
2010 "add<VDQV:vp>\\t%<Vetype>0, %1.<Vtype>"
2011 [(set_attr "type" "neon_reduc_add<q>")]
2012 )
2014 (define_insn "aarch64_reduc_plus_internalv2si"
2015 [(set (match_operand:V2SI 0 "register_operand" "=w")
2016 (unspec:V2SI [(match_operand:V2SI 1 "register_operand" "w")]
2017 UNSPEC_ADDV))]
2018 "TARGET_SIMD"
2019 "addp\\t%0.2s, %1.2s, %1.2s"
2020 [(set_attr "type" "neon_reduc_add")]
2021 )
2023 (define_insn "aarch64_reduc_plus_internal<mode>"
2024 [(set (match_operand:V2F 0 "register_operand" "=w")
2025 (unspec:V2F [(match_operand:V2F 1 "register_operand" "w")]
2026 UNSPEC_FADDV))]
2027 "TARGET_SIMD"
2028 "faddp\\t%<Vetype>0, %1.<Vtype>"
2029 [(set_attr "type" "neon_fp_reduc_add_<Vetype><q>")]
2030 )
2032 (define_insn "aarch64_addpv4sf"
2033 [(set (match_operand:V4SF 0 "register_operand" "=w")
2034 (unspec:V4SF [(match_operand:V4SF 1 "register_operand" "w")]
2035 UNSPEC_FADDV))]
2036 "TARGET_SIMD"
2037 "faddp\\t%0.4s, %1.4s, %1.4s"
2038 [(set_attr "type" "neon_fp_reduc_add_s_q")]
2039 )
2041 (define_expand "reduc_plus_scal_v4sf"
2042 [(set (match_operand:SF 0 "register_operand")
2043 (unspec:V4SF [(match_operand:V4SF 1 "register_operand")]
2044 UNSPEC_FADDV))]
2045 "TARGET_SIMD"
2046 {
2047 rtx elt = GEN_INT (ENDIAN_LANE_N (V4SFmode, 0));
2048 rtx scratch = gen_reg_rtx (V4SFmode);
2049 emit_insn (gen_aarch64_addpv4sf (scratch, operands[1]));
2050 emit_insn (gen_aarch64_addpv4sf (scratch, scratch));
2051 emit_insn (gen_aarch64_get_lanev4sf (operands[0], scratch, elt));
2052 DONE;
2053 })
2055 (define_insn "clrsb<mode>2"
2056 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
2057 (clrsb:VDQ_BHSI (match_operand:VDQ_BHSI 1 "register_operand" "w")))]
2058 "TARGET_SIMD"
2059 "cls\\t%0.<Vtype>, %1.<Vtype>"
2060 [(set_attr "type" "neon_cls<q>")]
2061 )
2063 (define_insn "clz<mode>2"
2064 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
2065 (clz:VDQ_BHSI (match_operand:VDQ_BHSI 1 "register_operand" "w")))]
2066 "TARGET_SIMD"
2067 "clz\\t%0.<Vtype>, %1.<Vtype>"
2068 [(set_attr "type" "neon_cls<q>")]
2069 )
2071 (define_insn "popcount<mode>2"
2072 [(set (match_operand:VB 0 "register_operand" "=w")
2073 (popcount:VB (match_operand:VB 1 "register_operand" "w")))]
2074 "TARGET_SIMD"
2075 "cnt\\t%0.<Vbtype>, %1.<Vbtype>"
2076 [(set_attr "type" "neon_cnt<q>")]
2077 )
2079 ;; 'across lanes' max and min ops.
2081 ;; Template for outputting a scalar, so we can create __builtins which can be
2082 ;; gimple_fold'd to the REDUC_(MAX|MIN)_EXPR tree code. (This is FP smax/smin).
2083 (define_expand "reduc_<maxmin_uns>_scal_<mode>"
2084 [(match_operand:<VEL> 0 "register_operand")
2085 (unspec:VDQF [(match_operand:VDQF 1 "register_operand")]
2086 FMAXMINV)]
2087 "TARGET_SIMD"
2088 {
2089 rtx elt = GEN_INT (ENDIAN_LANE_N (<MODE>mode, 0));
2090 rtx scratch = gen_reg_rtx (<MODE>mode);
2091 emit_insn (gen_aarch64_reduc_<maxmin_uns>_internal<mode> (scratch,
2092 operands[1]));
2093 emit_insn (gen_aarch64_get_lane<mode> (operands[0], scratch, elt));
2094 DONE;
2095 }
2096 )
2098 ;; Likewise for integer cases, signed and unsigned.
2099 (define_expand "reduc_<maxmin_uns>_scal_<mode>"
2100 [(match_operand:<VEL> 0 "register_operand")
2101 (unspec:VDQ_BHSI [(match_operand:VDQ_BHSI 1 "register_operand")]
2102 MAXMINV)]
2103 "TARGET_SIMD"
2104 {
2105 rtx elt = GEN_INT (ENDIAN_LANE_N (<MODE>mode, 0));
2106 rtx scratch = gen_reg_rtx (<MODE>mode);
2107 emit_insn (gen_aarch64_reduc_<maxmin_uns>_internal<mode> (scratch,
2108 operands[1]));
2109 emit_insn (gen_aarch64_get_lane<mode> (operands[0], scratch, elt));
2110 DONE;
2111 }
2112 )
2114 (define_insn "aarch64_reduc_<maxmin_uns>_internal<mode>"
2115 [(set (match_operand:VDQV_S 0 "register_operand" "=w")
2116 (unspec:VDQV_S [(match_operand:VDQV_S 1 "register_operand" "w")]
2117 MAXMINV))]
2118 "TARGET_SIMD"
2119 "<maxmin_uns_op>v\\t%<Vetype>0, %1.<Vtype>"
2120 [(set_attr "type" "neon_reduc_minmax<q>")]
2121 )
2123 (define_insn "aarch64_reduc_<maxmin_uns>_internalv2si"
2124 [(set (match_operand:V2SI 0 "register_operand" "=w")
2125 (unspec:V2SI [(match_operand:V2SI 1 "register_operand" "w")]
2126 MAXMINV))]
2127 "TARGET_SIMD"
2128 "<maxmin_uns_op>p\\t%0.2s, %1.2s, %1.2s"
2129 [(set_attr "type" "neon_reduc_minmax")]
2130 )
2132 (define_insn "aarch64_reduc_<maxmin_uns>_internal<mode>"
2133 [(set (match_operand:VDQF 0 "register_operand" "=w")
2134 (unspec:VDQF [(match_operand:VDQF 1 "register_operand" "w")]
2135 FMAXMINV))]
2136 "TARGET_SIMD"
2137 "<maxmin_uns_op><vp>\\t%<Vetype>0, %1.<Vtype>"
2138 [(set_attr "type" "neon_fp_reduc_minmax_<Vetype><q>")]
2139 )
2141 ;; aarch64_simd_bsl may compile to any of bsl/bif/bit depending on register
2142 ;; allocation.
2143 ;; Operand 1 is the mask, operands 2 and 3 are the bitfields from which
2144 ;; to select.
2145 ;;
2146 ;; Thus our BSL is of the form:
2147 ;; op0 = bsl (mask, op2, op3)
2148 ;; We can use any of:
2149 ;;
2150 ;; if (op0 = mask)
2151 ;; bsl mask, op1, op2
2152 ;; if (op0 = op1) (so 1-bits in mask choose bits from op2, else op0)
2153 ;; bit op0, op2, mask
2154 ;; if (op0 = op2) (so 0-bits in mask choose bits from op1, else op0)
2155 ;; bif op0, op1, mask
2157 (define_insn "aarch64_simd_bsl<mode>_internal"
2158 [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w,w,w")
2159 (xor:VSDQ_I_DI
2160 (and:VSDQ_I_DI
2161 (xor:VSDQ_I_DI
2162 (match_operand:<V_cmp_result> 3 "register_operand" "w,0,w")
2163 (match_operand:VSDQ_I_DI 2 "register_operand" "w,w,0"))
2164 (match_operand:VSDQ_I_DI 1 "register_operand" "0,w,w"))
2165 (match_dup:<V_cmp_result> 3)
2166 ))]
2167 "TARGET_SIMD"
2168 "@
2169 bsl\\t%0.<Vbtype>, %2.<Vbtype>, %3.<Vbtype>
2170 bit\\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>
2171 bif\\t%0.<Vbtype>, %3.<Vbtype>, %1.<Vbtype>"
2172 [(set_attr "type" "neon_bsl<q>")]
2173 )
2175 (define_expand "aarch64_simd_bsl<mode>"
2176 [(match_operand:VALLDIF 0 "register_operand")
2177 (match_operand:<V_cmp_result> 1 "register_operand")
2178 (match_operand:VALLDIF 2 "register_operand")
2179 (match_operand:VALLDIF 3 "register_operand")]
2180 "TARGET_SIMD"
2181 {
2182 /* We can't alias operands together if they have different modes. */
2183 rtx tmp = operands[0];
2184 if (FLOAT_MODE_P (<MODE>mode))
2185 {
2186 operands[2] = gen_lowpart (<V_cmp_result>mode, operands[2]);
2187 operands[3] = gen_lowpart (<V_cmp_result>mode, operands[3]);
2188 tmp = gen_reg_rtx (<V_cmp_result>mode);
2189 }
2190 operands[1] = gen_lowpart (<V_cmp_result>mode, operands[1]);
2191 emit_insn (gen_aarch64_simd_bsl<v_cmp_result>_internal (tmp,
2192 operands[1],
2193 operands[2],
2194 operands[3]));
2195 if (tmp != operands[0])
2196 emit_move_insn (operands[0], gen_lowpart (<MODE>mode, tmp));
2198 DONE;
2199 })
2201 (define_expand "aarch64_vcond_internal<mode><mode>"
2202 [(set (match_operand:VSDQ_I_DI 0 "register_operand")
2203 (if_then_else:VSDQ_I_DI
2204 (match_operator 3 "comparison_operator"
2205 [(match_operand:VSDQ_I_DI 4 "register_operand")
2206 (match_operand:VSDQ_I_DI 5 "nonmemory_operand")])
2207 (match_operand:VSDQ_I_DI 1 "nonmemory_operand")
2208 (match_operand:VSDQ_I_DI 2 "nonmemory_operand")))]
2209 "TARGET_SIMD"
2210 {
2211 rtx op1 = operands[1];
2212 rtx op2 = operands[2];
2213 rtx mask = gen_reg_rtx (<MODE>mode);
2214 enum rtx_code code = GET_CODE (operands[3]);
2216 /* Switching OP1 and OP2 is necessary for NE (to output a cmeq insn),
2217 and desirable for other comparisons if it results in FOO ? -1 : 0
2218 (this allows direct use of the comparison result without a bsl). */
2219 if (code == NE
2220 || (code != EQ
2221 && op1 == CONST0_RTX (<V_cmp_result>mode)
2222 && op2 == CONSTM1_RTX (<V_cmp_result>mode)))
2223 {
2224 op1 = operands[2];
2225 op2 = operands[1];
2226 switch (code)
2227 {
2228 case LE: code = GT; break;
2229 case LT: code = GE; break;
2230 case GE: code = LT; break;
2231 case GT: code = LE; break;
2232 /* No case EQ. */
2233 case NE: code = EQ; break;
2234 case LTU: code = GEU; break;
2235 case LEU: code = GTU; break;
2236 case GTU: code = LEU; break;
2237 case GEU: code = LTU; break;
2238 default: gcc_unreachable ();
2239 }
2240 }
2242 /* Make sure we can handle the last operand. */
2243 switch (code)
2244 {
2245 case NE:
2246 /* Normalized to EQ above. */
2247 gcc_unreachable ();
2249 case LE:
2250 case LT:
2251 case GE:
2252 case GT:
2253 case EQ:
2254 /* These instructions have a form taking an immediate zero. */
2255 if (operands[5] == CONST0_RTX (<MODE>mode))
2256 break;
2257 /* Fall through, as may need to load into register. */
2258 default:
2259 if (!REG_P (operands[5]))
2260 operands[5] = force_reg (<MODE>mode, operands[5]);
2261 break;
2262 }
2264 switch (code)
2265 {
2266 case LT:
2267 emit_insn (gen_aarch64_cmlt<mode> (mask, operands[4], operands[5]));
2268 break;
2270 case GE:
2271 emit_insn (gen_aarch64_cmge<mode> (mask, operands[4], operands[5]));
2272 break;
2274 case LE:
2275 emit_insn (gen_aarch64_cmle<mode> (mask, operands[4], operands[5]));
2276 break;
2278 case GT:
2279 emit_insn (gen_aarch64_cmgt<mode> (mask, operands[4], operands[5]));
2280 break;
2282 case LTU:
2283 emit_insn (gen_aarch64_cmgtu<mode> (mask, operands[5], operands[4]));
2284 break;
2286 case GEU:
2287 emit_insn (gen_aarch64_cmgeu<mode> (mask, operands[4], operands[5]));
2288 break;
2290 case LEU:
2291 emit_insn (gen_aarch64_cmgeu<mode> (mask, operands[5], operands[4]));
2292 break;
2294 case GTU:
2295 emit_insn (gen_aarch64_cmgtu<mode> (mask, operands[4], operands[5]));
2296 break;
2298 /* NE has been normalized to EQ above. */
2299 case EQ:
2300 emit_insn (gen_aarch64_cmeq<mode> (mask, operands[4], operands[5]));
2301 break;
2303 default:
2304 gcc_unreachable ();
2305 }
2307 /* If we have (a = (b CMP c) ? -1 : 0);
2308 Then we can simply move the generated mask. */
2310 if (op1 == CONSTM1_RTX (<V_cmp_result>mode)
2311 && op2 == CONST0_RTX (<V_cmp_result>mode))
2312 emit_move_insn (operands[0], mask);
2313 else
2314 {
2315 if (!REG_P (op1))
2316 op1 = force_reg (<MODE>mode, op1);
2317 if (!REG_P (op2))
2318 op2 = force_reg (<MODE>mode, op2);
2319 emit_insn (gen_aarch64_simd_bsl<mode> (operands[0], mask,
2320 op1, op2));
2321 }
2323 DONE;
2324 })
2326 (define_expand "aarch64_vcond_internal<VDQF_COND:mode><VDQF:mode>"
2327 [(set (match_operand:VDQF_COND 0 "register_operand")
2328 (if_then_else:VDQF
2329 (match_operator 3 "comparison_operator"
2330 [(match_operand:VDQF 4 "register_operand")
2331 (match_operand:VDQF 5 "nonmemory_operand")])
2332 (match_operand:VDQF_COND 1 "nonmemory_operand")
2333 (match_operand:VDQF_COND 2 "nonmemory_operand")))]
2334 "TARGET_SIMD"
2335 {
2336 int inverse = 0;
2337 int use_zero_form = 0;
2338 int swap_bsl_operands = 0;
2339 rtx op1 = operands[1];
2340 rtx op2 = operands[2];
2341 rtx mask = gen_reg_rtx (<VDQF_COND:V_cmp_result>mode);
2342 rtx tmp = gen_reg_rtx (<VDQF_COND:V_cmp_result>mode);
2344 rtx (*base_comparison) (rtx, rtx, rtx);
2345 rtx (*complimentary_comparison) (rtx, rtx, rtx);
2347 switch (GET_CODE (operands[3]))
2348 {
2349 case GE:
2350 case GT:
2351 case LE:
2352 case LT:
2353 case EQ:
2354 if (operands[5] == CONST0_RTX (<MODE>mode))
2355 {
2356 use_zero_form = 1;
2357 break;
2358 }
2359 /* Fall through. */
2360 default:
2361 if (!REG_P (operands[5]))
2362 operands[5] = force_reg (<VDQF:MODE>mode, operands[5]);
2363 }
2365 switch (GET_CODE (operands[3]))
2366 {
2367 case LT:
2368 case UNLT:
2369 inverse = 1;
2370 /* Fall through. */
2371 case GE:
2372 case UNGE:
2373 case ORDERED:
2374 case UNORDERED:
2375 base_comparison = gen_aarch64_cmge<VDQF:mode>;
2376 complimentary_comparison = gen_aarch64_cmgt<VDQF:mode>;
2377 break;
2378 case LE:
2379 case UNLE:
2380 inverse = 1;
2381 /* Fall through. */
2382 case GT:
2383 case UNGT:
2384 base_comparison = gen_aarch64_cmgt<VDQF:mode>;
2385 complimentary_comparison = gen_aarch64_cmge<VDQF:mode>;
2386 break;
2387 case EQ:
2388 case NE:
2389 case UNEQ:
2390 base_comparison = gen_aarch64_cmeq<VDQF:mode>;
2391 complimentary_comparison = gen_aarch64_cmeq<VDQF:mode>;
2392 break;
2393 default:
2394 gcc_unreachable ();
2395 }
2397 switch (GET_CODE (operands[3]))
2398 {
2399 case LT:
2400 case LE:
2401 case GT:
2402 case GE:
2403 case EQ:
2404 /* The easy case. Here we emit one of FCMGE, FCMGT or FCMEQ.
2405 As a LT b <=> b GE a && a LE b <=> b GT a. Our transformations are:
2406 a GE b -> a GE b
2407 a GT b -> a GT b
2408 a LE b -> b GE a
2409 a LT b -> b GT a
2410 a EQ b -> a EQ b
2411 Note that there also exist direct comparison against 0 forms,
2412 so catch those as a special case. */
2413 if (use_zero_form)
2414 {
2415 inverse = 0;
2416 switch (GET_CODE (operands[3]))
2417 {
2418 case LT:
2419 base_comparison = gen_aarch64_cmlt<VDQF:mode>;
2420 break;
2421 case LE:
2422 base_comparison = gen_aarch64_cmle<VDQF:mode>;
2423 break;
2424 default:
2425 /* Do nothing, other zero form cases already have the correct
2426 base_comparison. */
2427 break;
2428 }
2429 }
2431 if (!inverse)
2432 emit_insn (base_comparison (mask, operands[4], operands[5]));
2433 else
2434 emit_insn (complimentary_comparison (mask, operands[5], operands[4]));
2435 break;
2436 case UNLT:
2437 case UNLE:
2438 case UNGT:
2439 case UNGE:
2440 case NE:
2441 /* FCM returns false for lanes which are unordered, so if we use
2442 the inverse of the comparison we actually want to emit, then
2443 swap the operands to BSL, we will end up with the correct result.
2444 Note that a NE NaN and NaN NE b are true for all a, b.
2446 Our transformations are:
2447 a GE b -> !(b GT a)
2448 a GT b -> !(b GE a)
2449 a LE b -> !(a GT b)
2450 a LT b -> !(a GE b)
2451 a NE b -> !(a EQ b) */
2453 if (inverse)
2454 emit_insn (base_comparison (mask, operands[4], operands[5]));
2455 else
2456 emit_insn (complimentary_comparison (mask, operands[5], operands[4]));
2458 swap_bsl_operands = 1;
2459 break;
2460 case UNEQ:
2461 /* We check (a > b || b > a). combining these comparisons give us
2462 true iff !(a != b && a ORDERED b), swapping the operands to BSL
2463 will then give us (a == b || a UNORDERED b) as intended. */
2465 emit_insn (gen_aarch64_cmgt<VDQF:mode> (mask, operands[4], operands[5]));
2466 emit_insn (gen_aarch64_cmgt<VDQF:mode> (tmp, operands[5], operands[4]));
2467 emit_insn (gen_ior<VDQF_COND:v_cmp_result>3 (mask, mask, tmp));
2468 swap_bsl_operands = 1;
2469 break;
2470 case UNORDERED:
2471 /* Operands are ORDERED iff (a > b || b >= a).
2472 Swapping the operands to BSL will give the UNORDERED case. */
2473 swap_bsl_operands = 1;
2474 /* Fall through. */
2475 case ORDERED:
2476 emit_insn (gen_aarch64_cmgt<VDQF:mode> (tmp, operands[4], operands[5]));
2477 emit_insn (gen_aarch64_cmge<VDQF:mode> (mask, operands[5], operands[4]));
2478 emit_insn (gen_ior<VDQF_COND:v_cmp_result>3 (mask, mask, tmp));
2479 break;
2480 default:
2481 gcc_unreachable ();
2482 }
2484 if (swap_bsl_operands)
2485 {
2486 op1 = operands[2];
2487 op2 = operands[1];
2488 }
2490 /* If we have (a = (b CMP c) ? -1 : 0);
2491 Then we can simply move the generated mask. */
2493 if (op1 == CONSTM1_RTX (<VDQF_COND:V_cmp_result>mode)
2494 && op2 == CONST0_RTX (<VDQF_COND:V_cmp_result>mode))
2495 emit_move_insn (operands[0], mask);
2496 else
2497 {
2498 if (!REG_P (op1))
2499 op1 = force_reg (<VDQF_COND:MODE>mode, op1);
2500 if (!REG_P (op2))
2501 op2 = force_reg (<VDQF_COND:MODE>mode, op2);
2502 emit_insn (gen_aarch64_simd_bsl<VDQF_COND:mode> (operands[0], mask,
2503 op1, op2));
2504 }
2506 DONE;
2507 })
2509 (define_expand "vcond<mode><mode>"
2510 [(set (match_operand:VALLDI 0 "register_operand")
2511 (if_then_else:VALLDI
2512 (match_operator 3 "comparison_operator"
2513 [(match_operand:VALLDI 4 "register_operand")
2514 (match_operand:VALLDI 5 "nonmemory_operand")])
2515 (match_operand:VALLDI 1 "nonmemory_operand")
2516 (match_operand:VALLDI 2 "nonmemory_operand")))]
2517 "TARGET_SIMD"
2518 {
2519 emit_insn (gen_aarch64_vcond_internal<mode><mode> (operands[0], operands[1],
2520 operands[2], operands[3],
2521 operands[4], operands[5]));
2522 DONE;
2523 })
2525 (define_expand "vcond<v_cmp_result><mode>"
2526 [(set (match_operand:<V_cmp_result> 0 "register_operand")
2527 (if_then_else:<V_cmp_result>
2528 (match_operator 3 "comparison_operator"
2529 [(match_operand:VDQF 4 "register_operand")
2530 (match_operand:VDQF 5 "nonmemory_operand")])
2531 (match_operand:<V_cmp_result> 1 "nonmemory_operand")
2532 (match_operand:<V_cmp_result> 2 "nonmemory_operand")))]
2533 "TARGET_SIMD"
2534 {
2535 emit_insn (gen_aarch64_vcond_internal<v_cmp_result><mode> (
2536 operands[0], operands[1],
2537 operands[2], operands[3],
2538 operands[4], operands[5]));
2539 DONE;
2540 })
2542 (define_expand "vcondu<mode><mode>"
2543 [(set (match_operand:VSDQ_I_DI 0 "register_operand")
2544 (if_then_else:VSDQ_I_DI
2545 (match_operator 3 "comparison_operator"
2546 [(match_operand:VSDQ_I_DI 4 "register_operand")
2547 (match_operand:VSDQ_I_DI 5 "nonmemory_operand")])
2548 (match_operand:VSDQ_I_DI 1 "nonmemory_operand")
2549 (match_operand:VSDQ_I_DI 2 "nonmemory_operand")))]
2550 "TARGET_SIMD"
2551 {
2552 emit_insn (gen_aarch64_vcond_internal<mode><mode> (operands[0], operands[1],
2553 operands[2], operands[3],
2554 operands[4], operands[5]));
2555 DONE;
2556 })
2558 ;; Patterns for AArch64 SIMD Intrinsics.
2560 ;; Lane extraction with sign extension to general purpose register.
2561 (define_insn "*aarch64_get_lane_extend<GPI:mode><VDQQH:mode>"
2562 [(set (match_operand:GPI 0 "register_operand" "=r")
2563 (sign_extend:GPI
2564 (vec_select:<VEL>
2565 (match_operand:VDQQH 1 "register_operand" "w")
2566 (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2567 "TARGET_SIMD"
2568 {
2569 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
2570 return "smov\\t%<GPI:w>0, %1.<VDQQH:Vetype>[%2]";
2571 }
2572 [(set_attr "type" "neon_to_gp<q>")]
2573 )
2575 (define_insn "*aarch64_get_lane_zero_extendsi<mode>"
2576 [(set (match_operand:SI 0 "register_operand" "=r")
2577 (zero_extend:SI
2578 (vec_select:<VEL>
2579 (match_operand:VDQQH 1 "register_operand" "w")
2580 (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2581 "TARGET_SIMD"
2582 {
2583 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
2584 return "umov\\t%w0, %1.<Vetype>[%2]";
2585 }
2586 [(set_attr "type" "neon_to_gp<q>")]
2587 )
2589 ;; Lane extraction of a value, neither sign nor zero extension
2590 ;; is guaranteed so upper bits should be considered undefined.
2591 ;; RTL uses GCC vector extension indices throughout so flip only for assembly.
2592 (define_insn "aarch64_get_lane<mode>"
2593 [(set (match_operand:<VEL> 0 "aarch64_simd_nonimmediate_operand" "=r, w, Utv")
2594 (vec_select:<VEL>
2595 (match_operand:VALL_F16 1 "register_operand" "w, w, w")
2596 (parallel [(match_operand:SI 2 "immediate_operand" "i, i, i")])))]
2597 "TARGET_SIMD"
2598 {
2599 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
2600 switch (which_alternative)
2601 {
2602 case 0:
2603 return "umov\\t%<vwcore>0, %1.<Vetype>[%2]";
2604 case 1:
2605 return "dup\\t%<Vetype>0, %1.<Vetype>[%2]";
2606 case 2:
2607 return "st1\\t{%1.<Vetype>}[%2], %0";
2608 default:
2609 gcc_unreachable ();
2610 }
2611 }
2612 [(set_attr "type" "neon_to_gp<q>, neon_dup<q>, neon_store1_one_lane<q>")]
2613 )
2615 ;; In this insn, operand 1 should be low, and operand 2 the high part of the
2616 ;; dest vector.
2618 (define_insn "*aarch64_combinez<mode>"
2619 [(set (match_operand:<VDBL> 0 "register_operand" "=w,w,w")
2620 (vec_concat:<VDBL>
2621 (match_operand:VD_BHSI 1 "general_operand" "w,r,m")
2622 (match_operand:VD_BHSI 2 "aarch64_simd_imm_zero" "Dz,Dz,Dz")))]
2623 "TARGET_SIMD && !BYTES_BIG_ENDIAN"
2624 "@
2625 mov\\t%0.8b, %1.8b
2626 fmov\t%d0, %1
2627 ldr\\t%d0, %1"
2628 [(set_attr "type" "neon_move<q>, neon_from_gp, neon_load1_1reg")
2629 (set_attr "simd" "yes,*,yes")
2630 (set_attr "fp" "*,yes,*")]
2631 )
2633 (define_insn "*aarch64_combinez_be<mode>"
2634 [(set (match_operand:<VDBL> 0 "register_operand" "=w,w,w")
2635 (vec_concat:<VDBL>
2636 (match_operand:VD_BHSI 2 "aarch64_simd_imm_zero" "Dz,Dz,Dz")
2637 (match_operand:VD_BHSI 1 "general_operand" "w,r,m")))]
2638 "TARGET_SIMD && BYTES_BIG_ENDIAN"
2639 "@
2640 mov\\t%0.8b, %1.8b
2641 fmov\t%d0, %1
2642 ldr\\t%d0, %1"
2643 [(set_attr "type" "neon_move<q>, neon_from_gp, neon_load1_1reg")
2644 (set_attr "simd" "yes,*,yes")
2645 (set_attr "fp" "*,yes,*")]
2646 )
2648 (define_expand "aarch64_combine<mode>"
2649 [(match_operand:<VDBL> 0 "register_operand")
2650 (match_operand:VDC 1 "register_operand")
2651 (match_operand:VDC 2 "register_operand")]
2652 "TARGET_SIMD"
2653 {
2654 rtx op1, op2;
2655 if (BYTES_BIG_ENDIAN)
2656 {
2657 op1 = operands[2];
2658 op2 = operands[1];
2659 }
2660 else
2661 {
2662 op1 = operands[1];
2663 op2 = operands[2];
2664 }
2665 emit_insn (gen_aarch64_combine_internal<mode> (operands[0], op1, op2));
2666 DONE;
2667 }
2668 )
2670 (define_insn_and_split "aarch64_combine_internal<mode>"
2671 [(set (match_operand:<VDBL> 0 "register_operand" "=&w")
2672 (vec_concat:<VDBL> (match_operand:VDC 1 "register_operand" "w")
2673 (match_operand:VDC 2 "register_operand" "w")))]
2674 "TARGET_SIMD"
2675 "#"
2676 "&& reload_completed"
2677 [(const_int 0)]
2678 {
2679 if (BYTES_BIG_ENDIAN)
2680 aarch64_split_simd_combine (operands[0], operands[2], operands[1]);
2681 else
2682 aarch64_split_simd_combine (operands[0], operands[1], operands[2]);
2683 DONE;
2684 }
2685 [(set_attr "type" "multiple")]
2686 )
2688 (define_expand "aarch64_simd_combine<mode>"
2689 [(match_operand:<VDBL> 0 "register_operand")
2690 (match_operand:VDC 1 "register_operand")
2691 (match_operand:VDC 2 "register_operand")]
2692 "TARGET_SIMD"
2693 {
2694 emit_insn (gen_move_lo_quad_<Vdbl> (operands[0], operands[1]));
2695 emit_insn (gen_move_hi_quad_<Vdbl> (operands[0], operands[2]));
2696 DONE;
2697 }
2698 [(set_attr "type" "multiple")]
2699 )
2701 ;; <su><addsub>l<q>.
2703 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>l<mode>_hi_internal"
2704 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2705 (ADDSUB:<VWIDE> (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
2706 (match_operand:VQW 1 "register_operand" "w")
2707 (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))
2708 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
2709 (match_operand:VQW 2 "register_operand" "w")
2710 (match_dup 3)))))]
2711 "TARGET_SIMD"
2712 "<ANY_EXTEND:su><ADDSUB:optab>l2\t%0.<Vwtype>, %1.<Vtype>, %2.<Vtype>"
2713 [(set_attr "type" "neon_<ADDSUB:optab>_long")]
2714 )
2716 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>l<mode>_lo_internal"
2717 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2718 (ADDSUB:<VWIDE> (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
2719 (match_operand:VQW 1 "register_operand" "w")
2720 (match_operand:VQW 3 "vect_par_cnst_lo_half" "")))
2721 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
2722 (match_operand:VQW 2 "register_operand" "w")
2723 (match_dup 3)))))]
2724 "TARGET_SIMD"
2725 "<ANY_EXTEND:su><ADDSUB:optab>l\t%0.<Vwtype>, %1.<Vhalftype>, %2.<Vhalftype>"
2726 [(set_attr "type" "neon_<ADDSUB:optab>_long")]
2727 )
2730 (define_expand "aarch64_saddl2<mode>"
2731 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2732 (match_operand:VQW 1 "register_operand" "w")
2733 (match_operand:VQW 2 "register_operand" "w")]
2734 "TARGET_SIMD"
2735 {
2736 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2737 emit_insn (gen_aarch64_saddl<mode>_hi_internal (operands[0], operands[1],
2738 operands[2], p));
2739 DONE;
2740 })
2742 (define_expand "aarch64_uaddl2<mode>"
2743 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2744 (match_operand:VQW 1 "register_operand" "w")
2745 (match_operand:VQW 2 "register_operand" "w")]
2746 "TARGET_SIMD"
2747 {
2748 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2749 emit_insn (gen_aarch64_uaddl<mode>_hi_internal (operands[0], operands[1],
2750 operands[2], p));
2751 DONE;
2752 })
2754 (define_expand "aarch64_ssubl2<mode>"
2755 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2756 (match_operand:VQW 1 "register_operand" "w")
2757 (match_operand:VQW 2 "register_operand" "w")]
2758 "TARGET_SIMD"
2759 {
2760 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2761 emit_insn (gen_aarch64_ssubl<mode>_hi_internal (operands[0], operands[1],
2762 operands[2], p));
2763 DONE;
2764 })
2766 (define_expand "aarch64_usubl2<mode>"
2767 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2768 (match_operand:VQW 1 "register_operand" "w")
2769 (match_operand:VQW 2 "register_operand" "w")]
2770 "TARGET_SIMD"
2771 {
2772 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2773 emit_insn (gen_aarch64_usubl<mode>_hi_internal (operands[0], operands[1],
2774 operands[2], p));
2775 DONE;
2776 })
2778 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>l<mode>"
2779 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2780 (ADDSUB:<VWIDE> (ANY_EXTEND:<VWIDE>
2781 (match_operand:VD_BHSI 1 "register_operand" "w"))
2782 (ANY_EXTEND:<VWIDE>
2783 (match_operand:VD_BHSI 2 "register_operand" "w"))))]
2784 "TARGET_SIMD"
2785 "<ANY_EXTEND:su><ADDSUB:optab>l\t%0.<Vwtype>, %1.<Vtype>, %2.<Vtype>"
2786 [(set_attr "type" "neon_<ADDSUB:optab>_long")]
2787 )
2789 ;; <su><addsub>w<q>.
2791 (define_expand "widen_ssum<mode>3"
2792 [(set (match_operand:<VDBLW> 0 "register_operand" "")
2793 (plus:<VDBLW> (sign_extend:<VDBLW>
2794 (match_operand:VQW 1 "register_operand" ""))
2795 (match_operand:<VDBLW> 2 "register_operand" "")))]
2796 "TARGET_SIMD"
2797 {
2798 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
2799 rtx temp = gen_reg_rtx (GET_MODE (operands[0]));
2801 emit_insn (gen_aarch64_saddw<mode>_internal (temp, operands[2],
2802 operands[1], p));
2803 emit_insn (gen_aarch64_saddw2<mode> (operands[0], temp, operands[1]));
2804 DONE;
2805 }
2806 )
2808 (define_expand "widen_ssum<mode>3"
2809 [(set (match_operand:<VWIDE> 0 "register_operand" "")
2810 (plus:<VWIDE> (sign_extend:<VWIDE>
2811 (match_operand:VD_BHSI 1 "register_operand" ""))
2812 (match_operand:<VWIDE> 2 "register_operand" "")))]
2813 "TARGET_SIMD"
2814 {
2815 emit_insn (gen_aarch64_saddw<mode> (operands[0], operands[2], operands[1]));
2816 DONE;
2817 })
2819 (define_expand "widen_usum<mode>3"
2820 [(set (match_operand:<VDBLW> 0 "register_operand" "")
2821 (plus:<VDBLW> (zero_extend:<VDBLW>
2822 (match_operand:VQW 1 "register_operand" ""))
2823 (match_operand:<VDBLW> 2 "register_operand" "")))]
2824 "TARGET_SIMD"
2825 {
2826 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
2827 rtx temp = gen_reg_rtx (GET_MODE (operands[0]));
2829 emit_insn (gen_aarch64_uaddw<mode>_internal (temp, operands[2],
2830 operands[1], p));
2831 emit_insn (gen_aarch64_uaddw2<mode> (operands[0], temp, operands[1]));
2832 DONE;
2833 }
2834 )
2836 (define_expand "widen_usum<mode>3"
2837 [(set (match_operand:<VWIDE> 0 "register_operand" "")
2838 (plus:<VWIDE> (zero_extend:<VWIDE>
2839 (match_operand:VD_BHSI 1 "register_operand" ""))
2840 (match_operand:<VWIDE> 2 "register_operand" "")))]
2841 "TARGET_SIMD"
2842 {
2843 emit_insn (gen_aarch64_uaddw<mode> (operands[0], operands[2], operands[1]));
2844 DONE;
2845 })
2847 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>w<mode>"
2848 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2849 (ADDSUB:<VWIDE> (match_operand:<VWIDE> 1 "register_operand" "w")
2850 (ANY_EXTEND:<VWIDE>
2851 (match_operand:VD_BHSI 2 "register_operand" "w"))))]
2852 "TARGET_SIMD"
2853 "<ANY_EXTEND:su><ADDSUB:optab>w\\t%0.<Vwtype>, %1.<Vwtype>, %2.<Vtype>"
2854 [(set_attr "type" "neon_<ADDSUB:optab>_widen")]
2855 )
2857 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>w<mode>_internal"
2858 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2859 (ADDSUB:<VWIDE> (match_operand:<VWIDE> 1 "register_operand" "w")
2860 (ANY_EXTEND:<VWIDE>
2861 (vec_select:<VHALF>
2862 (match_operand:VQW 2 "register_operand" "w")
2863 (match_operand:VQW 3 "vect_par_cnst_lo_half" "")))))]
2864 "TARGET_SIMD"
2865 "<ANY_EXTEND:su><ADDSUB:optab>w\\t%0.<Vwtype>, %1.<Vwtype>, %2.<Vhalftype>"
2866 [(set_attr "type" "neon_<ADDSUB:optab>_widen")]
2867 )
2869 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>w2<mode>_internal"
2870 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2871 (ADDSUB:<VWIDE> (match_operand:<VWIDE> 1 "register_operand" "w")
2872 (ANY_EXTEND:<VWIDE>
2873 (vec_select:<VHALF>
2874 (match_operand:VQW 2 "register_operand" "w")
2875 (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))))]
2876 "TARGET_SIMD"
2877 "<ANY_EXTEND:su><ADDSUB:optab>w2\\t%0.<Vwtype>, %1.<Vwtype>, %2.<Vtype>"
2878 [(set_attr "type" "neon_<ADDSUB:optab>_widen")]
2879 )
2881 (define_expand "aarch64_saddw2<mode>"
2882 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2883 (match_operand:<VWIDE> 1 "register_operand" "w")
2884 (match_operand:VQW 2 "register_operand" "w")]
2885 "TARGET_SIMD"
2886 {
2887 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2888 emit_insn (gen_aarch64_saddw2<mode>_internal (operands[0], operands[1],
2889 operands[2], p));
2890 DONE;
2891 })
2893 (define_expand "aarch64_uaddw2<mode>"
2894 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2895 (match_operand:<VWIDE> 1 "register_operand" "w")
2896 (match_operand:VQW 2 "register_operand" "w")]
2897 "TARGET_SIMD"
2898 {
2899 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2900 emit_insn (gen_aarch64_uaddw2<mode>_internal (operands[0], operands[1],
2901 operands[2], p));
2902 DONE;
2903 })
2906 (define_expand "aarch64_ssubw2<mode>"
2907 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2908 (match_operand:<VWIDE> 1 "register_operand" "w")
2909 (match_operand:VQW 2 "register_operand" "w")]
2910 "TARGET_SIMD"
2911 {
2912 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2913 emit_insn (gen_aarch64_ssubw2<mode>_internal (operands[0], operands[1],
2914 operands[2], p));
2915 DONE;
2916 })
2918 (define_expand "aarch64_usubw2<mode>"
2919 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2920 (match_operand:<VWIDE> 1 "register_operand" "w")
2921 (match_operand:VQW 2 "register_operand" "w")]
2922 "TARGET_SIMD"
2923 {
2924 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2925 emit_insn (gen_aarch64_usubw2<mode>_internal (operands[0], operands[1],
2926 operands[2], p));
2927 DONE;
2928 })
2930 ;; <su><r>h<addsub>.
2932 (define_insn "aarch64_<sur>h<addsub><mode>"
2933 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
2934 (unspec:VDQ_BHSI [(match_operand:VDQ_BHSI 1 "register_operand" "w")
2935 (match_operand:VDQ_BHSI 2 "register_operand" "w")]
2936 HADDSUB))]
2937 "TARGET_SIMD"
2938 "<sur>h<addsub>\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
2939 [(set_attr "type" "neon_<addsub>_halve<q>")]
2940 )
2942 ;; <r><addsub>hn<q>.
2944 (define_insn "aarch64_<sur><addsub>hn<mode>"
2945 [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
2946 (unspec:<VNARROWQ> [(match_operand:VQN 1 "register_operand" "w")
2947 (match_operand:VQN 2 "register_operand" "w")]
2948 ADDSUBHN))]
2949 "TARGET_SIMD"
2950 "<sur><addsub>hn\\t%0.<Vntype>, %1.<Vtype>, %2.<Vtype>"
2951 [(set_attr "type" "neon_<addsub>_halve_narrow_q")]
2952 )
2954 (define_insn "aarch64_<sur><addsub>hn2<mode>"
2955 [(set (match_operand:<VNARROWQ2> 0 "register_operand" "=w")
2956 (unspec:<VNARROWQ2> [(match_operand:<VNARROWQ> 1 "register_operand" "0")
2957 (match_operand:VQN 2 "register_operand" "w")
2958 (match_operand:VQN 3 "register_operand" "w")]
2959 ADDSUBHN2))]
2960 "TARGET_SIMD"
2961 "<sur><addsub>hn2\\t%0.<V2ntype>, %2.<Vtype>, %3.<Vtype>"
2962 [(set_attr "type" "neon_<addsub>_halve_narrow_q")]
2963 )
2965 ;; pmul.
2967 (define_insn "aarch64_pmul<mode>"
2968 [(set (match_operand:VB 0 "register_operand" "=w")
2969 (unspec:VB [(match_operand:VB 1 "register_operand" "w")
2970 (match_operand:VB 2 "register_operand" "w")]
2971 UNSPEC_PMUL))]
2972 "TARGET_SIMD"
2973 "pmul\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
2974 [(set_attr "type" "neon_mul_<Vetype><q>")]
2975 )
2977 ;; fmulx.
2979 (define_insn "aarch64_fmulx<mode>"
2980 [(set (match_operand:VALLF 0 "register_operand" "=w")
2981 (unspec:VALLF [(match_operand:VALLF 1 "register_operand" "w")
2982 (match_operand:VALLF 2 "register_operand" "w")]
2983 UNSPEC_FMULX))]
2984 "TARGET_SIMD"
2985 "fmulx\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
2986 [(set_attr "type" "neon_fp_mul_<Vetype>")]
2987 )
2989 ;; vmulxq_lane_f32, and vmulx_laneq_f32
2991 (define_insn "*aarch64_mulx_elt_<vswap_width_name><mode>"
2992 [(set (match_operand:VDQSF 0 "register_operand" "=w")
2993 (unspec:VDQSF
2994 [(match_operand:VDQSF 1 "register_operand" "w")
2995 (vec_duplicate:VDQSF
2996 (vec_select:<VEL>
2997 (match_operand:<VSWAP_WIDTH> 2 "register_operand" "w")
2998 (parallel [(match_operand:SI 3 "immediate_operand" "i")])))]
2999 UNSPEC_FMULX))]
3000 "TARGET_SIMD"
3001 {
3002 operands[3] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
3003 INTVAL (operands[3])));
3004 return "fmulx\t%<v>0<Vmtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3005 }
3006 [(set_attr "type" "neon_fp_mul_<Vetype>_scalar<q>")]
3007 )
3009 ;; vmulxq_laneq_f32, vmulxq_laneq_f64, vmulx_lane_f32
3011 (define_insn "*aarch64_mulx_elt<mode>"
3012 [(set (match_operand:VDQF 0 "register_operand" "=w")
3013 (unspec:VDQF
3014 [(match_operand:VDQF 1 "register_operand" "w")
3015 (vec_duplicate:VDQF
3016 (vec_select:<VEL>
3017 (match_operand:VDQF 2 "register_operand" "w")
3018 (parallel [(match_operand:SI 3 "immediate_operand" "i")])))]
3019 UNSPEC_FMULX))]
3020 "TARGET_SIMD"
3021 {
3022 operands[3] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[3])));
3023 return "fmulx\t%<v>0<Vmtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3024 }
3025 [(set_attr "type" "neon_fp_mul_<Vetype><q>")]
3026 )
3028 ;; vmulxq_lane_f64
3030 (define_insn "*aarch64_mulx_elt_to_64v2df"
3031 [(set (match_operand:V2DF 0 "register_operand" "=w")
3032 (unspec:V2DF
3033 [(match_operand:V2DF 1 "register_operand" "w")
3034 (vec_duplicate:V2DF
3035 (match_operand:DF 2 "register_operand" "w"))]
3036 UNSPEC_FMULX))]
3037 "TARGET_SIMD"
3038 {
3039 return "fmulx\t%0.2d, %1.2d, %2.d[0]";
3040 }
3041 [(set_attr "type" "neon_fp_mul_d_scalar_q")]
3042 )
3044 ;; vmulxs_lane_f32, vmulxs_laneq_f32
3045 ;; vmulxd_lane_f64 == vmulx_lane_f64
3046 ;; vmulxd_laneq_f64 == vmulx_laneq_f64
3048 (define_insn "*aarch64_vgetfmulx<mode>"
3049 [(set (match_operand:<VEL> 0 "register_operand" "=w")
3050 (unspec:<VEL>
3051 [(match_operand:<VEL> 1 "register_operand" "w")
3052 (vec_select:<VEL>
3053 (match_operand:VDQF_DF 2 "register_operand" "w")
3054 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))]
3055 UNSPEC_FMULX))]
3056 "TARGET_SIMD"
3057 {
3058 operands[3] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[3])));
3059 return "fmulx\t%<Vetype>0, %<Vetype>1, %2.<Vetype>[%3]";
3060 }
3061 [(set_attr "type" "fmul<Vetype>")]
3062 )
3063 ;; <su>q<addsub>
3065 (define_insn "aarch64_<su_optab><optab><mode>"
3066 [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
3067 (BINQOPS:VSDQ_I (match_operand:VSDQ_I 1 "register_operand" "w")
3068 (match_operand:VSDQ_I 2 "register_operand" "w")))]
3069 "TARGET_SIMD"
3070 "<su_optab><optab>\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
3071 [(set_attr "type" "neon_<optab><q>")]
3072 )
3074 ;; suqadd and usqadd
3076 (define_insn "aarch64_<sur>qadd<mode>"
3077 [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
3078 (unspec:VSDQ_I [(match_operand:VSDQ_I 1 "register_operand" "0")
3079 (match_operand:VSDQ_I 2 "register_operand" "w")]
3080 USSUQADD))]
3081 "TARGET_SIMD"
3082 "<sur>qadd\\t%<v>0<Vmtype>, %<v>2<Vmtype>"
3083 [(set_attr "type" "neon_qadd<q>")]
3084 )
3086 ;; sqmovun
3088 (define_insn "aarch64_sqmovun<mode>"
3089 [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
3090 (unspec:<VNARROWQ> [(match_operand:VSQN_HSDI 1 "register_operand" "w")]
3091 UNSPEC_SQXTUN))]
3092 "TARGET_SIMD"
3093 "sqxtun\\t%<vn2>0<Vmntype>, %<v>1<Vmtype>"
3094 [(set_attr "type" "neon_sat_shift_imm_narrow_q")]
3095 )
3097 ;; sqmovn and uqmovn
3099 (define_insn "aarch64_<sur>qmovn<mode>"
3100 [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
3101 (unspec:<VNARROWQ> [(match_operand:VSQN_HSDI 1 "register_operand" "w")]
3102 SUQMOVN))]
3103 "TARGET_SIMD"
3104 "<sur>qxtn\\t%<vn2>0<Vmntype>, %<v>1<Vmtype>"
3105 [(set_attr "type" "neon_sat_shift_imm_narrow_q")]
3106 )
3108 ;; <su>q<absneg>
3110 (define_insn "aarch64_s<optab><mode>"
3111 [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
3112 (UNQOPS:VSDQ_I
3113 (match_operand:VSDQ_I 1 "register_operand" "w")))]
3114 "TARGET_SIMD"
3115 "s<optab>\\t%<v>0<Vmtype>, %<v>1<Vmtype>"
3116 [(set_attr "type" "neon_<optab><q>")]
3117 )
3119 ;; sq<r>dmulh.
3121 (define_insn "aarch64_sq<r>dmulh<mode>"
3122 [(set (match_operand:VSDQ_HSI 0 "register_operand" "=w")
3123 (unspec:VSDQ_HSI
3124 [(match_operand:VSDQ_HSI 1 "register_operand" "w")
3125 (match_operand:VSDQ_HSI 2 "register_operand" "w")]
3126 VQDMULH))]
3127 "TARGET_SIMD"
3128 "sq<r>dmulh\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
3129 [(set_attr "type" "neon_sat_mul_<Vetype><q>")]
3130 )
3132 ;; sq<r>dmulh_lane
3134 (define_insn "aarch64_sq<r>dmulh_lane<mode>"
3135 [(set (match_operand:VDQHS 0 "register_operand" "=w")
3136 (unspec:VDQHS
3137 [(match_operand:VDQHS 1 "register_operand" "w")
3138 (vec_select:<VEL>
3139 (match_operand:<VCOND> 2 "register_operand" "<vwx>")
3140 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))]
3141 VQDMULH))]
3142 "TARGET_SIMD"
3143 "*
3144 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
3145 return \"sq<r>dmulh\\t%0.<Vtype>, %1.<Vtype>, %2.<Vetype>[%3]\";"
3146 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar<q>")]
3147 )
3149 (define_insn "aarch64_sq<r>dmulh_laneq<mode>"
3150 [(set (match_operand:VDQHS 0 "register_operand" "=w")
3151 (unspec:VDQHS
3152 [(match_operand:VDQHS 1 "register_operand" "w")
3153 (vec_select:<VEL>
3154 (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
3155 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))]
3156 VQDMULH))]
3157 "TARGET_SIMD"
3158 "*
3159 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[3])));
3160 return \"sq<r>dmulh\\t%0.<Vtype>, %1.<Vtype>, %2.<Vetype>[%3]\";"
3161 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar<q>")]
3162 )
3164 (define_insn "aarch64_sq<r>dmulh_lane<mode>"
3165 [(set (match_operand:SD_HSI 0 "register_operand" "=w")
3166 (unspec:SD_HSI
3167 [(match_operand:SD_HSI 1 "register_operand" "w")
3168 (vec_select:<VEL>
3169 (match_operand:<VCOND> 2 "register_operand" "<vwx>")
3170 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))]
3171 VQDMULH))]
3172 "TARGET_SIMD"
3173 "*
3174 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
3175 return \"sq<r>dmulh\\t%<v>0, %<v>1, %2.<v>[%3]\";"
3176 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar<q>")]
3177 )
3179 (define_insn "aarch64_sq<r>dmulh_laneq<mode>"
3180 [(set (match_operand:SD_HSI 0 "register_operand" "=w")
3181 (unspec:SD_HSI
3182 [(match_operand:SD_HSI 1 "register_operand" "w")
3183 (vec_select:<VEL>
3184 (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
3185 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))]
3186 VQDMULH))]
3187 "TARGET_SIMD"
3188 "*
3189 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[3])));
3190 return \"sq<r>dmulh\\t%<v>0, %<v>1, %2.<v>[%3]\";"
3191 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar<q>")]
3192 )
3194 ;; sqrdml[as]h.
3196 (define_insn "aarch64_sqrdml<SQRDMLH_AS:rdma_as>h<mode>"
3197 [(set (match_operand:VSDQ_HSI 0 "register_operand" "=w")
3198 (unspec:VSDQ_HSI
3199 [(match_operand:VSDQ_HSI 1 "register_operand" "0")
3200 (match_operand:VSDQ_HSI 2 "register_operand" "w")
3201 (match_operand:VSDQ_HSI 3 "register_operand" "w")]
3202 SQRDMLH_AS))]
3203 "TARGET_SIMD_RDMA"
3204 "sqrdml<SQRDMLH_AS:rdma_as>h\\t%<v>0<Vmtype>, %<v>2<Vmtype>, %<v>3<Vmtype>"
3205 [(set_attr "type" "neon_sat_mla_<Vetype>_long")]
3206 )
3208 ;; sqrdml[as]h_lane.
3210 (define_insn "aarch64_sqrdml<SQRDMLH_AS:rdma_as>h_lane<mode>"
3211 [(set (match_operand:VDQHS 0 "register_operand" "=w")
3212 (unspec:VDQHS
3213 [(match_operand:VDQHS 1 "register_operand" "0")
3214 (match_operand:VDQHS 2 "register_operand" "w")
3215 (vec_select:<VEL>
3216 (match_operand:<VCOND> 3 "register_operand" "w")
3217 (parallel [(match_operand:SI 4 "immediate_operand" "i")]))]
3218 SQRDMLH_AS))]
3219 "TARGET_SIMD_RDMA"
3220 {
3221 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[4])));
3222 return
3223 "sqrdml<SQRDMLH_AS:rdma_as>h\\t%0.<Vtype>, %2.<Vtype>, %3.<Vetype>[%4]";
3224 }
3225 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3226 )
3228 (define_insn "aarch64_sqrdml<SQRDMLH_AS:rdma_as>h_lane<mode>"
3229 [(set (match_operand:SD_HSI 0 "register_operand" "=w")
3230 (unspec:SD_HSI
3231 [(match_operand:SD_HSI 1 "register_operand" "0")
3232 (match_operand:SD_HSI 2 "register_operand" "w")
3233 (vec_select:<VEL>
3234 (match_operand:<VCOND> 3 "register_operand" "w")
3235 (parallel [(match_operand:SI 4 "immediate_operand" "i")]))]
3236 SQRDMLH_AS))]
3237 "TARGET_SIMD_RDMA"
3238 {
3239 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[4])));
3240 return
3241 "sqrdml<SQRDMLH_AS:rdma_as>h\\t%<v>0, %<v>2, %3.<Vetype>[%4]";
3242 }
3243 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3244 )
3246 ;; sqrdml[as]h_laneq.
3248 (define_insn "aarch64_sqrdml<SQRDMLH_AS:rdma_as>h_laneq<mode>"
3249 [(set (match_operand:VDQHS 0 "register_operand" "=w")
3250 (unspec:VDQHS
3251 [(match_operand:VDQHS 1 "register_operand" "0")
3252 (match_operand:VDQHS 2 "register_operand" "w")
3253 (vec_select:<VEL>
3254 (match_operand:<VCONQ> 3 "register_operand" "w")
3255 (parallel [(match_operand:SI 4 "immediate_operand" "i")]))]
3256 SQRDMLH_AS))]
3257 "TARGET_SIMD_RDMA"
3258 {
3259 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[4])));
3260 return
3261 "sqrdml<SQRDMLH_AS:rdma_as>h\\t%0.<Vtype>, %2.<Vtype>, %3.<Vetype>[%4]";
3262 }
3263 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3264 )
3266 (define_insn "aarch64_sqrdml<SQRDMLH_AS:rdma_as>h_laneq<mode>"
3267 [(set (match_operand:SD_HSI 0 "register_operand" "=w")
3268 (unspec:SD_HSI
3269 [(match_operand:SD_HSI 1 "register_operand" "0")
3270 (match_operand:SD_HSI 2 "register_operand" "w")
3271 (vec_select:<VEL>
3272 (match_operand:<VCONQ> 3 "register_operand" "w")
3273 (parallel [(match_operand:SI 4 "immediate_operand" "i")]))]
3274 SQRDMLH_AS))]
3275 "TARGET_SIMD_RDMA"
3276 {
3277 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[4])));
3278 return
3279 "sqrdml<SQRDMLH_AS:rdma_as>h\\t%<v>0, %<v>2, %3.<v>[%4]";
3280 }
3281 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3282 )
3284 ;; vqdml[sa]l
3286 (define_insn "aarch64_sqdml<SBINQOPS:as>l<mode>"
3287 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3288 (SBINQOPS:<VWIDE>
3289 (match_operand:<VWIDE> 1 "register_operand" "0")
3290 (ss_ashift:<VWIDE>
3291 (mult:<VWIDE>
3292 (sign_extend:<VWIDE>
3293 (match_operand:VSD_HSI 2 "register_operand" "w"))
3294 (sign_extend:<VWIDE>
3295 (match_operand:VSD_HSI 3 "register_operand" "w")))
3296 (const_int 1))))]
3297 "TARGET_SIMD"
3298 "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %<v>3<Vmtype>"
3299 [(set_attr "type" "neon_sat_mla_<Vetype>_long")]
3300 )
3302 ;; vqdml[sa]l_lane
3304 (define_insn "aarch64_sqdml<SBINQOPS:as>l_lane<mode>"
3305 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3306 (SBINQOPS:<VWIDE>
3307 (match_operand:<VWIDE> 1 "register_operand" "0")
3308 (ss_ashift:<VWIDE>
3309 (mult:<VWIDE>
3310 (sign_extend:<VWIDE>
3311 (match_operand:VD_HSI 2 "register_operand" "w"))
3312 (sign_extend:<VWIDE>
3313 (vec_duplicate:VD_HSI
3314 (vec_select:<VEL>
3315 (match_operand:<VCOND> 3 "register_operand" "<vwx>")
3316 (parallel [(match_operand:SI 4 "immediate_operand" "i")])))
3317 ))
3318 (const_int 1))))]
3319 "TARGET_SIMD"
3320 {
3321 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[4])));
3322 return
3323 "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
3324 }
3325 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3326 )
3328 (define_insn "aarch64_sqdml<SBINQOPS:as>l_laneq<mode>"
3329 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3330 (SBINQOPS:<VWIDE>
3331 (match_operand:<VWIDE> 1 "register_operand" "0")
3332 (ss_ashift:<VWIDE>
3333 (mult:<VWIDE>
3334 (sign_extend:<VWIDE>
3335 (match_operand:VD_HSI 2 "register_operand" "w"))
3336 (sign_extend:<VWIDE>
3337 (vec_duplicate:VD_HSI
3338 (vec_select:<VEL>
3339 (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
3340 (parallel [(match_operand:SI 4 "immediate_operand" "i")])))
3341 ))
3342 (const_int 1))))]
3343 "TARGET_SIMD"
3344 {
3345 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[4])));
3346 return
3347 "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
3348 }
3349 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3350 )
3352 (define_insn "aarch64_sqdml<SBINQOPS:as>l_lane<mode>"
3353 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3354 (SBINQOPS:<VWIDE>
3355 (match_operand:<VWIDE> 1 "register_operand" "0")
3356 (ss_ashift:<VWIDE>
3357 (mult:<VWIDE>
3358 (sign_extend:<VWIDE>
3359 (match_operand:SD_HSI 2 "register_operand" "w"))
3360 (sign_extend:<VWIDE>
3361 (vec_select:<VEL>
3362 (match_operand:<VCOND> 3 "register_operand" "<vwx>")
3363 (parallel [(match_operand:SI 4 "immediate_operand" "i")])))
3364 )
3365 (const_int 1))))]
3366 "TARGET_SIMD"
3367 {
3368 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[4])));
3369 return
3370 "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
3371 }
3372 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3373 )
3375 (define_insn "aarch64_sqdml<SBINQOPS:as>l_laneq<mode>"
3376 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3377 (SBINQOPS:<VWIDE>
3378 (match_operand:<VWIDE> 1 "register_operand" "0")
3379 (ss_ashift:<VWIDE>
3380 (mult:<VWIDE>
3381 (sign_extend:<VWIDE>
3382 (match_operand:SD_HSI 2 "register_operand" "w"))
3383 (sign_extend:<VWIDE>
3384 (vec_select:<VEL>
3385 (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
3386 (parallel [(match_operand:SI 4 "immediate_operand" "i")])))
3387 )
3388 (const_int 1))))]
3389 "TARGET_SIMD"
3390 {
3391 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[4])));
3392 return
3393 "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
3394 }
3395 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3396 )
3398 ;; vqdml[sa]l_n
3400 (define_insn "aarch64_sqdml<SBINQOPS:as>l_n<mode>"
3401 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3402 (SBINQOPS:<VWIDE>
3403 (match_operand:<VWIDE> 1 "register_operand" "0")
3404 (ss_ashift:<VWIDE>
3405 (mult:<VWIDE>
3406 (sign_extend:<VWIDE>
3407 (match_operand:VD_HSI 2 "register_operand" "w"))
3408 (sign_extend:<VWIDE>
3409 (vec_duplicate:VD_HSI
3410 (match_operand:<VEL> 3 "register_operand" "<vwx>"))))
3411 (const_int 1))))]
3412 "TARGET_SIMD"
3413 "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[0]"
3414 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3415 )
3417 ;; sqdml[as]l2
3419 (define_insn "aarch64_sqdml<SBINQOPS:as>l2<mode>_internal"
3420 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3421 (SBINQOPS:<VWIDE>
3422 (match_operand:<VWIDE> 1 "register_operand" "0")
3423 (ss_ashift:<VWIDE>
3424 (mult:<VWIDE>
3425 (sign_extend:<VWIDE>
3426 (vec_select:<VHALF>
3427 (match_operand:VQ_HSI 2 "register_operand" "w")
3428 (match_operand:VQ_HSI 4 "vect_par_cnst_hi_half" "")))
3429 (sign_extend:<VWIDE>
3430 (vec_select:<VHALF>
3431 (match_operand:VQ_HSI 3 "register_operand" "w")
3432 (match_dup 4))))
3433 (const_int 1))))]
3434 "TARGET_SIMD"
3435 "sqdml<SBINQOPS:as>l2\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %<v>3<Vmtype>"
3436 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3437 )
3439 (define_expand "aarch64_sqdmlal2<mode>"
3440 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3441 (match_operand:<VWIDE> 1 "register_operand" "w")
3442 (match_operand:VQ_HSI 2 "register_operand" "w")
3443 (match_operand:VQ_HSI 3 "register_operand" "w")]
3444 "TARGET_SIMD"
3445 {
3446 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3447 emit_insn (gen_aarch64_sqdmlal2<mode>_internal (operands[0], operands[1],
3448 operands[2], operands[3], p));
3449 DONE;
3450 })
3452 (define_expand "aarch64_sqdmlsl2<mode>"
3453 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3454 (match_operand:<VWIDE> 1 "register_operand" "w")
3455 (match_operand:VQ_HSI 2 "register_operand" "w")
3456 (match_operand:VQ_HSI 3 "register_operand" "w")]
3457 "TARGET_SIMD"
3458 {
3459 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3460 emit_insn (gen_aarch64_sqdmlsl2<mode>_internal (operands[0], operands[1],
3461 operands[2], operands[3], p));
3462 DONE;
3463 })
3465 ;; vqdml[sa]l2_lane
3467 (define_insn "aarch64_sqdml<SBINQOPS:as>l2_lane<mode>_internal"
3468 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3469 (SBINQOPS:<VWIDE>
3470 (match_operand:<VWIDE> 1 "register_operand" "0")
3471 (ss_ashift:<VWIDE>
3472 (mult:<VWIDE>
3473 (sign_extend:<VWIDE>
3474 (vec_select:<VHALF>
3475 (match_operand:VQ_HSI 2 "register_operand" "w")
3476 (match_operand:VQ_HSI 5 "vect_par_cnst_hi_half" "")))
3477 (sign_extend:<VWIDE>
3478 (vec_duplicate:<VHALF>
3479 (vec_select:<VEL>
3480 (match_operand:<VCOND> 3 "register_operand" "<vwx>")
3481 (parallel [(match_operand:SI 4 "immediate_operand" "i")])
3482 ))))
3483 (const_int 1))))]
3484 "TARGET_SIMD"
3485 {
3486 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[4])));
3487 return
3488 "sqdml<SBINQOPS:as>l2\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
3489 }
3490 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3491 )
3493 (define_insn "aarch64_sqdml<SBINQOPS:as>l2_laneq<mode>_internal"
3494 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3495 (SBINQOPS:<VWIDE>
3496 (match_operand:<VWIDE> 1 "register_operand" "0")
3497 (ss_ashift:<VWIDE>
3498 (mult:<VWIDE>
3499 (sign_extend:<VWIDE>
3500 (vec_select:<VHALF>
3501 (match_operand:VQ_HSI 2 "register_operand" "w")
3502 (match_operand:VQ_HSI 5 "vect_par_cnst_hi_half" "")))
3503 (sign_extend:<VWIDE>
3504 (vec_duplicate:<VHALF>
3505 (vec_select:<VEL>
3506 (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
3507 (parallel [(match_operand:SI 4 "immediate_operand" "i")])
3508 ))))
3509 (const_int 1))))]
3510 "TARGET_SIMD"
3511 {
3512 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[4])));
3513 return
3514 "sqdml<SBINQOPS:as>l2\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
3515 }
3516 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3517 )
3519 (define_expand "aarch64_sqdmlal2_lane<mode>"
3520 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3521 (match_operand:<VWIDE> 1 "register_operand" "w")
3522 (match_operand:VQ_HSI 2 "register_operand" "w")
3523 (match_operand:<VCOND> 3 "register_operand" "<vwx>")
3524 (match_operand:SI 4 "immediate_operand" "i")]
3525 "TARGET_SIMD"
3526 {
3527 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3528 emit_insn (gen_aarch64_sqdmlal2_lane<mode>_internal (operands[0], operands[1],
3529 operands[2], operands[3],
3530 operands[4], p));
3531 DONE;
3532 })
3534 (define_expand "aarch64_sqdmlal2_laneq<mode>"
3535 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3536 (match_operand:<VWIDE> 1 "register_operand" "w")
3537 (match_operand:VQ_HSI 2 "register_operand" "w")
3538 (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
3539 (match_operand:SI 4 "immediate_operand" "i")]
3540 "TARGET_SIMD"
3541 {
3542 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3543 emit_insn (gen_aarch64_sqdmlal2_laneq<mode>_internal (operands[0], operands[1],
3544 operands[2], operands[3],
3545 operands[4], p));
3546 DONE;
3547 })
3549 (define_expand "aarch64_sqdmlsl2_lane<mode>"
3550 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3551 (match_operand:<VWIDE> 1 "register_operand" "w")
3552 (match_operand:VQ_HSI 2 "register_operand" "w")
3553 (match_operand:<VCOND> 3 "register_operand" "<vwx>")
3554 (match_operand:SI 4 "immediate_operand" "i")]
3555 "TARGET_SIMD"
3556 {
3557 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3558 emit_insn (gen_aarch64_sqdmlsl2_lane<mode>_internal (operands[0], operands[1],
3559 operands[2], operands[3],
3560 operands[4], p));
3561 DONE;
3562 })
3564 (define_expand "aarch64_sqdmlsl2_laneq<mode>"
3565 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3566 (match_operand:<VWIDE> 1 "register_operand" "w")
3567 (match_operand:VQ_HSI 2 "register_operand" "w")
3568 (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
3569 (match_operand:SI 4 "immediate_operand" "i")]
3570 "TARGET_SIMD"
3571 {
3572 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3573 emit_insn (gen_aarch64_sqdmlsl2_laneq<mode>_internal (operands[0], operands[1],
3574 operands[2], operands[3],
3575 operands[4], p));
3576 DONE;
3577 })
3579 (define_insn "aarch64_sqdml<SBINQOPS:as>l2_n<mode>_internal"
3580 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3581 (SBINQOPS:<VWIDE>
3582 (match_operand:<VWIDE> 1 "register_operand" "0")
3583 (ss_ashift:<VWIDE>
3584 (mult:<VWIDE>
3585 (sign_extend:<VWIDE>
3586 (vec_select:<VHALF>
3587 (match_operand:VQ_HSI 2 "register_operand" "w")
3588 (match_operand:VQ_HSI 4 "vect_par_cnst_hi_half" "")))
3589 (sign_extend:<VWIDE>
3590 (vec_duplicate:<VHALF>
3591 (match_operand:<VEL> 3 "register_operand" "<vwx>"))))
3592 (const_int 1))))]
3593 "TARGET_SIMD"
3594 "sqdml<SBINQOPS:as>l2\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[0]"
3595 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3596 )
3598 (define_expand "aarch64_sqdmlal2_n<mode>"
3599 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3600 (match_operand:<VWIDE> 1 "register_operand" "w")
3601 (match_operand:VQ_HSI 2 "register_operand" "w")
3602 (match_operand:<VEL> 3 "register_operand" "w")]
3603 "TARGET_SIMD"
3604 {
3605 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3606 emit_insn (gen_aarch64_sqdmlal2_n<mode>_internal (operands[0], operands[1],
3607 operands[2], operands[3],
3608 p));
3609 DONE;
3610 })
3612 (define_expand "aarch64_sqdmlsl2_n<mode>"
3613 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3614 (match_operand:<VWIDE> 1 "register_operand" "w")
3615 (match_operand:VQ_HSI 2 "register_operand" "w")
3616 (match_operand:<VEL> 3 "register_operand" "w")]
3617 "TARGET_SIMD"
3618 {
3619 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3620 emit_insn (gen_aarch64_sqdmlsl2_n<mode>_internal (operands[0], operands[1],
3621 operands[2], operands[3],
3622 p));
3623 DONE;
3624 })
3626 ;; vqdmull
3628 (define_insn "aarch64_sqdmull<mode>"
3629 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3630 (ss_ashift:<VWIDE>
3631 (mult:<VWIDE>
3632 (sign_extend:<VWIDE>
3633 (match_operand:VSD_HSI 1 "register_operand" "w"))
3634 (sign_extend:<VWIDE>
3635 (match_operand:VSD_HSI 2 "register_operand" "w")))
3636 (const_int 1)))]
3637 "TARGET_SIMD"
3638 "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
3639 [(set_attr "type" "neon_sat_mul_<Vetype>_long")]
3640 )
3642 ;; vqdmull_lane
3644 (define_insn "aarch64_sqdmull_lane<mode>"
3645 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3646 (ss_ashift:<VWIDE>
3647 (mult:<VWIDE>
3648 (sign_extend:<VWIDE>
3649 (match_operand:VD_HSI 1 "register_operand" "w"))
3650 (sign_extend:<VWIDE>
3651 (vec_duplicate:VD_HSI
3652 (vec_select:<VEL>
3653 (match_operand:<VCOND> 2 "register_operand" "<vwx>")
3654 (parallel [(match_operand:SI 3 "immediate_operand" "i")])))
3655 ))
3656 (const_int 1)))]
3657 "TARGET_SIMD"
3658 {
3659 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
3660 return "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3661 }
3662 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3663 )
3665 (define_insn "aarch64_sqdmull_laneq<mode>"
3666 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3667 (ss_ashift:<VWIDE>
3668 (mult:<VWIDE>
3669 (sign_extend:<VWIDE>
3670 (match_operand:VD_HSI 1 "register_operand" "w"))
3671 (sign_extend:<VWIDE>
3672 (vec_duplicate:VD_HSI
3673 (vec_select:<VEL>
3674 (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
3675 (parallel [(match_operand:SI 3 "immediate_operand" "i")])))
3676 ))
3677 (const_int 1)))]
3678 "TARGET_SIMD"
3679 {
3680 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[3])));
3681 return "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3682 }
3683 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3684 )
3686 (define_insn "aarch64_sqdmull_lane<mode>"
3687 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3688 (ss_ashift:<VWIDE>
3689 (mult:<VWIDE>
3690 (sign_extend:<VWIDE>
3691 (match_operand:SD_HSI 1 "register_operand" "w"))
3692 (sign_extend:<VWIDE>
3693 (vec_select:<VEL>
3694 (match_operand:<VCOND> 2 "register_operand" "<vwx>")
3695 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))
3696 ))
3697 (const_int 1)))]
3698 "TARGET_SIMD"
3699 {
3700 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
3701 return "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3702 }
3703 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3704 )
3706 (define_insn "aarch64_sqdmull_laneq<mode>"
3707 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3708 (ss_ashift:<VWIDE>
3709 (mult:<VWIDE>
3710 (sign_extend:<VWIDE>
3711 (match_operand:SD_HSI 1 "register_operand" "w"))
3712 (sign_extend:<VWIDE>
3713 (vec_select:<VEL>
3714 (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
3715 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))
3716 ))
3717 (const_int 1)))]
3718 "TARGET_SIMD"
3719 {
3720 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[3])));
3721 return "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3722 }
3723 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3724 )
3726 ;; vqdmull_n
3728 (define_insn "aarch64_sqdmull_n<mode>"
3729 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3730 (ss_ashift:<VWIDE>
3731 (mult:<VWIDE>
3732 (sign_extend:<VWIDE>
3733 (match_operand:VD_HSI 1 "register_operand" "w"))
3734 (sign_extend:<VWIDE>
3735 (vec_duplicate:VD_HSI
3736 (match_operand:<VEL> 2 "register_operand" "<vwx>")))
3737 )
3738 (const_int 1)))]
3739 "TARGET_SIMD"
3740 "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[0]"
3741 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3742 )
3744 ;; vqdmull2
3748 (define_insn "aarch64_sqdmull2<mode>_internal"
3749 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3750 (ss_ashift:<VWIDE>
3751 (mult:<VWIDE>
3752 (sign_extend:<VWIDE>
3753 (vec_select:<VHALF>
3754 (match_operand:VQ_HSI 1 "register_operand" "w")
3755 (match_operand:VQ_HSI 3 "vect_par_cnst_hi_half" "")))
3756 (sign_extend:<VWIDE>
3757 (vec_select:<VHALF>
3758 (match_operand:VQ_HSI 2 "register_operand" "w")
3759 (match_dup 3)))
3760 )
3761 (const_int 1)))]
3762 "TARGET_SIMD"
3763 "sqdmull2\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
3764 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3765 )
3767 (define_expand "aarch64_sqdmull2<mode>"
3768 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3769 (match_operand:VQ_HSI 1 "register_operand" "w")
3770 (match_operand:VQ_HSI 2 "register_operand" "w")]
3771 "TARGET_SIMD"
3772 {
3773 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3774 emit_insn (gen_aarch64_sqdmull2<mode>_internal (operands[0], operands[1],
3775 operands[2], p));
3776 DONE;
3777 })
3779 ;; vqdmull2_lane
3781 (define_insn "aarch64_sqdmull2_lane<mode>_internal"
3782 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3783 (ss_ashift:<VWIDE>
3784 (mult:<VWIDE>
3785 (sign_extend:<VWIDE>
3786 (vec_select:<VHALF>
3787 (match_operand:VQ_HSI 1 "register_operand" "w")
3788 (match_operand:VQ_HSI 4 "vect_par_cnst_hi_half" "")))
3789 (sign_extend:<VWIDE>
3790 (vec_duplicate:<VHALF>
3791 (vec_select:<VEL>
3792 (match_operand:<VCOND> 2 "register_operand" "<vwx>")
3793 (parallel [(match_operand:SI 3 "immediate_operand" "i")])))
3794 ))
3795 (const_int 1)))]
3796 "TARGET_SIMD"
3797 {
3798 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
3799 return "sqdmull2\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3800 }
3801 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3802 )
3804 (define_insn "aarch64_sqdmull2_laneq<mode>_internal"
3805 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3806 (ss_ashift:<VWIDE>
3807 (mult:<VWIDE>
3808 (sign_extend:<VWIDE>
3809 (vec_select:<VHALF>
3810 (match_operand:VQ_HSI 1 "register_operand" "w")
3811 (match_operand:VQ_HSI 4 "vect_par_cnst_hi_half" "")))
3812 (sign_extend:<VWIDE>
3813 (vec_duplicate:<VHALF>
3814 (vec_select:<VEL>
3815 (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
3816 (parallel [(match_operand:SI 3 "immediate_operand" "i")])))
3817 ))
3818 (const_int 1)))]
3819 "TARGET_SIMD"
3820 {
3821 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[3])));
3822 return "sqdmull2\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3823 }
3824 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3825 )
3827 (define_expand "aarch64_sqdmull2_lane<mode>"
3828 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3829 (match_operand:VQ_HSI 1 "register_operand" "w")
3830 (match_operand:<VCOND> 2 "register_operand" "<vwx>")
3831 (match_operand:SI 3 "immediate_operand" "i")]
3832 "TARGET_SIMD"
3833 {
3834 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3835 emit_insn (gen_aarch64_sqdmull2_lane<mode>_internal (operands[0], operands[1],
3836 operands[2], operands[3],
3837 p));
3838 DONE;
3839 })
3841 (define_expand "aarch64_sqdmull2_laneq<mode>"
3842 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3843 (match_operand:VQ_HSI 1 "register_operand" "w")
3844 (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
3845 (match_operand:SI 3 "immediate_operand" "i")]
3846 "TARGET_SIMD"
3847 {
3848 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3849 emit_insn (gen_aarch64_sqdmull2_laneq<mode>_internal (operands[0], operands[1],
3850 operands[2], operands[3],
3851 p));
3852 DONE;
3853 })
3855 ;; vqdmull2_n
3857 (define_insn "aarch64_sqdmull2_n<mode>_internal"
3858 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3859 (ss_ashift:<VWIDE>
3860 (mult:<VWIDE>
3861 (sign_extend:<VWIDE>
3862 (vec_select:<VHALF>
3863 (match_operand:VQ_HSI 1 "register_operand" "w")
3864 (match_operand:VQ_HSI 3 "vect_par_cnst_hi_half" "")))
3865 (sign_extend:<VWIDE>
3866 (vec_duplicate:<VHALF>
3867 (match_operand:<VEL> 2 "register_operand" "<vwx>")))
3868 )
3869 (const_int 1)))]
3870 "TARGET_SIMD"
3871 "sqdmull2\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[0]"
3872 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3873 )
3875 (define_expand "aarch64_sqdmull2_n<mode>"
3876 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3877 (match_operand:VQ_HSI 1 "register_operand" "w")
3878 (match_operand:<VEL> 2 "register_operand" "w")]
3879 "TARGET_SIMD"
3880 {
3881 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3882 emit_insn (gen_aarch64_sqdmull2_n<mode>_internal (operands[0], operands[1],
3883 operands[2], p));
3884 DONE;
3885 })
3887 ;; vshl
3889 (define_insn "aarch64_<sur>shl<mode>"
3890 [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
3891 (unspec:VSDQ_I_DI
3892 [(match_operand:VSDQ_I_DI 1 "register_operand" "w")
3893 (match_operand:VSDQ_I_DI 2 "register_operand" "w")]
3894 VSHL))]
3895 "TARGET_SIMD"
3896 "<sur>shl\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>";
3897 [(set_attr "type" "neon_shift_reg<q>")]
3898 )
3901 ;; vqshl
3903 (define_insn "aarch64_<sur>q<r>shl<mode>"
3904 [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
3905 (unspec:VSDQ_I
3906 [(match_operand:VSDQ_I 1 "register_operand" "w")
3907 (match_operand:VSDQ_I 2 "register_operand" "w")]
3908 VQSHL))]
3909 "TARGET_SIMD"
3910 "<sur>q<r>shl\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>";
3911 [(set_attr "type" "neon_sat_shift_reg<q>")]
3912 )
3914 ;; vshll_n
3916 (define_insn "aarch64_<sur>shll_n<mode>"
3917 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3918 (unspec:<VWIDE> [(match_operand:VD_BHSI 1 "register_operand" "w")
3919 (match_operand:SI 2
3920 "aarch64_simd_shift_imm_bitsize_<ve_mode>" "i")]
3921 VSHLL))]
3922 "TARGET_SIMD"
3923 "*
3924 int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
3925 if (INTVAL (operands[2]) == bit_width)
3926 {
3927 return \"shll\\t%0.<Vwtype>, %1.<Vtype>, %2\";
3928 }
3929 else {
3930 return \"<sur>shll\\t%0.<Vwtype>, %1.<Vtype>, %2\";
3931 }"
3932 [(set_attr "type" "neon_shift_imm_long")]
3933 )
3935 ;; vshll_high_n
3937 (define_insn "aarch64_<sur>shll2_n<mode>"
3938 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3939 (unspec:<VWIDE> [(match_operand:VQW 1 "register_operand" "w")
3940 (match_operand:SI 2 "immediate_operand" "i")]
3941 VSHLL))]
3942 "TARGET_SIMD"
3943 "*
3944 int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
3945 if (INTVAL (operands[2]) == bit_width)
3946 {
3947 return \"shll2\\t%0.<Vwtype>, %1.<Vtype>, %2\";
3948 }
3949 else {
3950 return \"<sur>shll2\\t%0.<Vwtype>, %1.<Vtype>, %2\";
3951 }"
3952 [(set_attr "type" "neon_shift_imm_long")]
3953 )
3955 ;; vrshr_n
3957 (define_insn "aarch64_<sur>shr_n<mode>"
3958 [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
3959 (unspec:VSDQ_I_DI [(match_operand:VSDQ_I_DI 1 "register_operand" "w")
3960 (match_operand:SI 2
3961 "aarch64_simd_shift_imm_offset_<ve_mode>" "i")]
3962 VRSHR_N))]
3963 "TARGET_SIMD"
3964 "<sur>shr\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %2"
3965 [(set_attr "type" "neon_sat_shift_imm<q>")]
3966 )
3968 ;; v(r)sra_n
3970 (define_insn "aarch64_<sur>sra_n<mode>"
3971 [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
3972 (unspec:VSDQ_I_DI [(match_operand:VSDQ_I_DI 1 "register_operand" "0")
3973 (match_operand:VSDQ_I_DI 2 "register_operand" "w")
3974 (match_operand:SI 3
3975 "aarch64_simd_shift_imm_offset_<ve_mode>" "i")]
3976 VSRA))]
3977 "TARGET_SIMD"
3978 "<sur>sra\\t%<v>0<Vmtype>, %<v>2<Vmtype>, %3"
3979 [(set_attr "type" "neon_shift_acc<q>")]
3980 )
3982 ;; vs<lr>i_n
3984 (define_insn "aarch64_<sur>s<lr>i_n<mode>"
3985 [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
3986 (unspec:VSDQ_I_DI [(match_operand:VSDQ_I_DI 1 "register_operand" "0")
3987 (match_operand:VSDQ_I_DI 2 "register_operand" "w")
3988 (match_operand:SI 3
3989 "aarch64_simd_shift_imm_<offsetlr><ve_mode>" "i")]
3990 VSLRI))]
3991 "TARGET_SIMD"
3992 "s<lr>i\\t%<v>0<Vmtype>, %<v>2<Vmtype>, %3"
3993 [(set_attr "type" "neon_shift_imm<q>")]
3994 )
3996 ;; vqshl(u)
3998 (define_insn "aarch64_<sur>qshl<u>_n<mode>"
3999 [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
4000 (unspec:VSDQ_I [(match_operand:VSDQ_I 1 "register_operand" "w")
4001 (match_operand:SI 2
4002 "aarch64_simd_shift_imm_<ve_mode>" "i")]
4003 VQSHL_N))]
4004 "TARGET_SIMD"
4005 "<sur>qshl<u>\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %2"
4006 [(set_attr "type" "neon_sat_shift_imm<q>")]
4007 )
4010 ;; vq(r)shr(u)n_n
4012 (define_insn "aarch64_<sur>q<r>shr<u>n_n<mode>"
4013 [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
4014 (unspec:<VNARROWQ> [(match_operand:VSQN_HSDI 1 "register_operand" "w")
4015 (match_operand:SI 2
4016 "aarch64_simd_shift_imm_offset_<ve_mode>" "i")]
4017 VQSHRN_N))]
4018 "TARGET_SIMD"
4019 "<sur>q<r>shr<u>n\\t%<vn2>0<Vmntype>, %<v>1<Vmtype>, %2"
4020 [(set_attr "type" "neon_sat_shift_imm_narrow_q")]
4021 )
4024 ;; cm(eq|ge|gt|lt|le)
4025 ;; Note, we have constraints for Dz and Z as different expanders
4026 ;; have different ideas of what should be passed to this pattern.
4028 (define_insn "aarch64_cm<optab><mode>"
4029 [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w,w")
4030 (neg:<V_cmp_result>
4031 (COMPARISONS:<V_cmp_result>
4032 (match_operand:VDQ_I 1 "register_operand" "w,w")
4033 (match_operand:VDQ_I 2 "aarch64_simd_reg_or_zero" "w,ZDz")
4034 )))]
4035 "TARGET_SIMD"
4036 "@
4037 cm<n_optab>\t%<v>0<Vmtype>, %<v><cmp_1><Vmtype>, %<v><cmp_2><Vmtype>
4038 cm<optab>\t%<v>0<Vmtype>, %<v>1<Vmtype>, #0"
4039 [(set_attr "type" "neon_compare<q>, neon_compare_zero<q>")]
4040 )
4042 (define_insn_and_split "aarch64_cm<optab>di"
4043 [(set (match_operand:DI 0 "register_operand" "=w,w,r")
4044 (neg:DI
4045 (COMPARISONS:DI
4046 (match_operand:DI 1 "register_operand" "w,w,r")
4047 (match_operand:DI 2 "aarch64_simd_reg_or_zero" "w,ZDz,r")
4048 )))
4049 (clobber (reg:CC CC_REGNUM))]
4050 "TARGET_SIMD"
4051 "#"
4052 "reload_completed"
4053 [(set (match_operand:DI 0 "register_operand")
4054 (neg:DI
4055 (COMPARISONS:DI
4056 (match_operand:DI 1 "register_operand")
4057 (match_operand:DI 2 "aarch64_simd_reg_or_zero")
4058 )))]
4059 {
4060 /* If we are in the general purpose register file,
4061 we split to a sequence of comparison and store. */
4062 if (GP_REGNUM_P (REGNO (operands[0]))
4063 && GP_REGNUM_P (REGNO (operands[1])))
4064 {
4065 machine_mode mode = SELECT_CC_MODE (<CMP>, operands[1], operands[2]);
4066 rtx cc_reg = aarch64_gen_compare_reg (<CMP>, operands[1], operands[2]);
4067 rtx comparison = gen_rtx_<CMP> (mode, operands[1], operands[2]);
4068 emit_insn (gen_cstoredi_neg (operands[0], comparison, cc_reg));
4069 DONE;
4070 }
4071 /* Otherwise, we expand to a similar pattern which does not
4072 clobber CC_REGNUM. */
4073 }
4074 [(set_attr "type" "neon_compare, neon_compare_zero, multiple")]
4075 )
4077 (define_insn "*aarch64_cm<optab>di"
4078 [(set (match_operand:DI 0 "register_operand" "=w,w")
4079 (neg:DI
4080 (COMPARISONS:DI
4081 (match_operand:DI 1 "register_operand" "w,w")
4082 (match_operand:DI 2 "aarch64_simd_reg_or_zero" "w,ZDz")
4083 )))]
4084 "TARGET_SIMD && reload_completed"
4085 "@
4086 cm<n_optab>\t%d0, %d<cmp_1>, %d<cmp_2>
4087 cm<optab>\t%d0, %d1, #0"
4088 [(set_attr "type" "neon_compare, neon_compare_zero")]
4089 )
4091 ;; cm(hs|hi)
4093 (define_insn "aarch64_cm<optab><mode>"
4094 [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w")
4095 (neg:<V_cmp_result>
4096 (UCOMPARISONS:<V_cmp_result>
4097 (match_operand:VDQ_I 1 "register_operand" "w")
4098 (match_operand:VDQ_I 2 "register_operand" "w")
4099 )))]
4100 "TARGET_SIMD"
4101 "cm<n_optab>\t%<v>0<Vmtype>, %<v><cmp_1><Vmtype>, %<v><cmp_2><Vmtype>"
4102 [(set_attr "type" "neon_compare<q>")]
4103 )
4105 (define_insn_and_split "aarch64_cm<optab>di"
4106 [(set (match_operand:DI 0 "register_operand" "=w,r")
4107 (neg:DI
4108 (UCOMPARISONS:DI
4109 (match_operand:DI 1 "register_operand" "w,r")
4110 (match_operand:DI 2 "aarch64_simd_reg_or_zero" "w,r")
4111 )))
4112 (clobber (reg:CC CC_REGNUM))]
4113 "TARGET_SIMD"
4114 "#"
4115 "reload_completed"
4116 [(set (match_operand:DI 0 "register_operand")
4117 (neg:DI
4118 (UCOMPARISONS:DI
4119 (match_operand:DI 1 "register_operand")
4120 (match_operand:DI 2 "aarch64_simd_reg_or_zero")
4121 )))]
4122 {
4123 /* If we are in the general purpose register file,
4124 we split to a sequence of comparison and store. */
4125 if (GP_REGNUM_P (REGNO (operands[0]))
4126 && GP_REGNUM_P (REGNO (operands[1])))
4127 {
4128 machine_mode mode = CCmode;
4129 rtx cc_reg = aarch64_gen_compare_reg (<CMP>, operands[1], operands[2]);
4130 rtx comparison = gen_rtx_<CMP> (mode, operands[1], operands[2]);
4131 emit_insn (gen_cstoredi_neg (operands[0], comparison, cc_reg));
4132 DONE;
4133 }
4134 /* Otherwise, we expand to a similar pattern which does not
4135 clobber CC_REGNUM. */
4136 }
4137 [(set_attr "type" "neon_compare,multiple")]
4138 )
4140 (define_insn "*aarch64_cm<optab>di"
4141 [(set (match_operand:DI 0 "register_operand" "=w")
4142 (neg:DI
4143 (UCOMPARISONS:DI
4144 (match_operand:DI 1 "register_operand" "w")
4145 (match_operand:DI 2 "aarch64_simd_reg_or_zero" "w")
4146 )))]
4147 "TARGET_SIMD && reload_completed"
4148 "cm<n_optab>\t%d0, %d<cmp_1>, %d<cmp_2>"
4149 [(set_attr "type" "neon_compare")]
4150 )
4152 ;; cmtst
4154 ;; Although neg (ne (and x y) 0) is the natural way of expressing a cmtst,
4155 ;; we don't have any insns using ne, and aarch64_vcond_internal outputs
4156 ;; not (neg (eq (and x y) 0))
4157 ;; which is rewritten by simplify_rtx as
4158 ;; plus (eq (and x y) 0) -1.
4160 (define_insn "aarch64_cmtst<mode>"
4161 [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w")
4162 (plus:<V_cmp_result>
4163 (eq:<V_cmp_result>
4164 (and:VDQ_I
4165 (match_operand:VDQ_I 1 "register_operand" "w")
4166 (match_operand:VDQ_I 2 "register_operand" "w"))
4167 (match_operand:VDQ_I 3 "aarch64_simd_imm_zero"))
4168 (match_operand:<V_cmp_result> 4 "aarch64_simd_imm_minus_one")))
4169 ]
4170 "TARGET_SIMD"
4171 "cmtst\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
4172 [(set_attr "type" "neon_tst<q>")]
4173 )
4175 (define_insn_and_split "aarch64_cmtstdi"
4176 [(set (match_operand:DI 0 "register_operand" "=w,r")
4177 (neg:DI
4178 (ne:DI
4179 (and:DI
4180 (match_operand:DI 1 "register_operand" "w,r")
4181 (match_operand:DI 2 "register_operand" "w,r"))
4182 (const_int 0))))
4183 (clobber (reg:CC CC_REGNUM))]
4184 "TARGET_SIMD"
4185 "#"
4186 "reload_completed"
4187 [(set (match_operand:DI 0 "register_operand")
4188 (neg:DI
4189 (ne:DI
4190 (and:DI
4191 (match_operand:DI 1 "register_operand")
4192 (match_operand:DI 2 "register_operand"))
4193 (const_int 0))))]
4194 {
4195 /* If we are in the general purpose register file,
4196 we split to a sequence of comparison and store. */
4197 if (GP_REGNUM_P (REGNO (operands[0]))
4198 && GP_REGNUM_P (REGNO (operands[1])))
4199 {
4200 rtx and_tree = gen_rtx_AND (DImode, operands[1], operands[2]);
4201 machine_mode mode = SELECT_CC_MODE (NE, and_tree, const0_rtx);
4202 rtx cc_reg = aarch64_gen_compare_reg (NE, and_tree, const0_rtx);
4203 rtx comparison = gen_rtx_NE (mode, and_tree, const0_rtx);
4204 emit_insn (gen_cstoredi_neg (operands[0], comparison, cc_reg));
4205 DONE;
4206 }
4207 /* Otherwise, we expand to a similar pattern which does not
4208 clobber CC_REGNUM. */
4209 }
4210 [(set_attr "type" "neon_tst,multiple")]
4211 )
4213 (define_insn "*aarch64_cmtstdi"
4214 [(set (match_operand:DI 0 "register_operand" "=w")
4215 (neg:DI
4216 (ne:DI
4217 (and:DI
4218 (match_operand:DI 1 "register_operand" "w")
4219 (match_operand:DI 2 "register_operand" "w"))
4220 (const_int 0))))]
4221 "TARGET_SIMD"
4222 "cmtst\t%d0, %d1, %d2"
4223 [(set_attr "type" "neon_tst")]
4224 )
4226 ;; fcm(eq|ge|gt|le|lt)
4228 (define_insn "aarch64_cm<optab><mode>"
4229 [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w,w")
4230 (neg:<V_cmp_result>
4231 (COMPARISONS:<V_cmp_result>
4232 (match_operand:VALLF 1 "register_operand" "w,w")
4233 (match_operand:VALLF 2 "aarch64_simd_reg_or_zero" "w,YDz")
4234 )))]
4235 "TARGET_SIMD"
4236 "@
4237 fcm<n_optab>\t%<v>0<Vmtype>, %<v><cmp_1><Vmtype>, %<v><cmp_2><Vmtype>
4238 fcm<optab>\t%<v>0<Vmtype>, %<v>1<Vmtype>, 0"
4239 [(set_attr "type" "neon_fp_compare_<Vetype><q>")]
4240 )
4242 ;; fac(ge|gt)
4243 ;; Note we can also handle what would be fac(le|lt) by
4244 ;; generating fac(ge|gt).
4246 (define_insn "*aarch64_fac<optab><mode>"
4247 [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w")
4248 (neg:<V_cmp_result>
4249 (FAC_COMPARISONS:<V_cmp_result>
4250 (abs:VALLF (match_operand:VALLF 1 "register_operand" "w"))
4251 (abs:VALLF (match_operand:VALLF 2 "register_operand" "w"))
4252 )))]
4253 "TARGET_SIMD"
4254 "fac<n_optab>\t%<v>0<Vmtype>, %<v><cmp_1><Vmtype>, %<v><cmp_2><Vmtype>"
4255 [(set_attr "type" "neon_fp_compare_<Vetype><q>")]
4256 )
4258 ;; addp
4260 (define_insn "aarch64_addp<mode>"
4261 [(set (match_operand:VD_BHSI 0 "register_operand" "=w")
4262 (unspec:VD_BHSI
4263 [(match_operand:VD_BHSI 1 "register_operand" "w")
4264 (match_operand:VD_BHSI 2 "register_operand" "w")]
4265 UNSPEC_ADDP))]
4266 "TARGET_SIMD"
4267 "addp\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
4268 [(set_attr "type" "neon_reduc_add<q>")]
4269 )
4271 (define_insn "aarch64_addpdi"
4272 [(set (match_operand:DI 0 "register_operand" "=w")
4273 (unspec:DI
4274 [(match_operand:V2DI 1 "register_operand" "w")]
4275 UNSPEC_ADDP))]
4276 "TARGET_SIMD"
4277 "addp\t%d0, %1.2d"
4278 [(set_attr "type" "neon_reduc_add")]
4279 )
4281 ;; sqrt
4283 (define_insn "sqrt<mode>2"
4284 [(set (match_operand:VDQF 0 "register_operand" "=w")
4285 (sqrt:VDQF (match_operand:VDQF 1 "register_operand" "w")))]
4286 "TARGET_SIMD"
4287 "fsqrt\\t%0.<Vtype>, %1.<Vtype>"
4288 [(set_attr "type" "neon_fp_sqrt_<Vetype><q>")]
4289 )
4291 ;; Patterns for vector struct loads and stores.
4293 (define_insn "aarch64_simd_ld2<mode>"
4294 [(set (match_operand:OI 0 "register_operand" "=w")
4295 (unspec:OI [(match_operand:OI 1 "aarch64_simd_struct_operand" "Utv")
4296 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4297 UNSPEC_LD2))]
4298 "TARGET_SIMD"
4299 "ld2\\t{%S0.<Vtype> - %T0.<Vtype>}, %1"
4300 [(set_attr "type" "neon_load2_2reg<q>")]
4301 )
4303 (define_insn "aarch64_simd_ld2r<mode>"
4304 [(set (match_operand:OI 0 "register_operand" "=w")
4305 (unspec:OI [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")
4306 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY) ]
4307 UNSPEC_LD2_DUP))]
4308 "TARGET_SIMD"
4309 "ld2r\\t{%S0.<Vtype> - %T0.<Vtype>}, %1"
4310 [(set_attr "type" "neon_load2_all_lanes<q>")]
4311 )
4313 (define_insn "aarch64_vec_load_lanesoi_lane<mode>"
4314 [(set (match_operand:OI 0 "register_operand" "=w")
4315 (unspec:OI [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")
4316 (match_operand:OI 2 "register_operand" "0")
4317 (match_operand:SI 3 "immediate_operand" "i")
4318 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY) ]
4319 UNSPEC_LD2_LANE))]
4320 "TARGET_SIMD"
4321 {
4322 operands[3] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[3])));
4323 return "ld2\\t{%S0.<Vetype> - %T0.<Vetype>}[%3], %1";
4324 }
4325 [(set_attr "type" "neon_load2_one_lane")]
4326 )
4328 (define_expand "vec_load_lanesoi<mode>"
4329 [(set (match_operand:OI 0 "register_operand" "=w")
4330 (unspec:OI [(match_operand:OI 1 "aarch64_simd_struct_operand" "Utv")
4331 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4332 UNSPEC_LD2))]
4333 "TARGET_SIMD"
4334 {
4335 if (BYTES_BIG_ENDIAN)
4336 {
4337 rtx tmp = gen_reg_rtx (OImode);
4338 rtx mask = aarch64_reverse_mask (<MODE>mode);
4339 emit_insn (gen_aarch64_simd_ld2<mode> (tmp, operands[1]));
4340 emit_insn (gen_aarch64_rev_reglistoi (operands[0], tmp, mask));
4341 }
4342 else
4343 emit_insn (gen_aarch64_simd_ld2<mode> (operands[0], operands[1]));
4344 DONE;
4345 })
4347 (define_insn "aarch64_simd_st2<mode>"
4348 [(set (match_operand:OI 0 "aarch64_simd_struct_operand" "=Utv")
4349 (unspec:OI [(match_operand:OI 1 "register_operand" "w")
4350 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4351 UNSPEC_ST2))]
4352 "TARGET_SIMD"
4353 "st2\\t{%S1.<Vtype> - %T1.<Vtype>}, %0"
4354 [(set_attr "type" "neon_store2_2reg<q>")]
4355 )
4357 ;; RTL uses GCC vector extension indices, so flip only for assembly.
4358 (define_insn "aarch64_vec_store_lanesoi_lane<mode>"
4359 [(set (match_operand:BLK 0 "aarch64_simd_struct_operand" "=Utv")
4360 (unspec:BLK [(match_operand:OI 1 "register_operand" "w")
4361 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)
4362 (match_operand:SI 2 "immediate_operand" "i")]
4363 UNSPEC_ST2_LANE))]
4364 "TARGET_SIMD"
4365 {
4366 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
4367 return "st2\\t{%S1.<Vetype> - %T1.<Vetype>}[%2], %0";
4368 }
4369 [(set_attr "type" "neon_store2_one_lane<q>")]
4370 )
4372 (define_expand "vec_store_lanesoi<mode>"
4373 [(set (match_operand:OI 0 "aarch64_simd_struct_operand" "=Utv")
4374 (unspec:OI [(match_operand:OI 1 "register_operand" "w")
4375 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4376 UNSPEC_ST2))]
4377 "TARGET_SIMD"
4378 {
4379 if (BYTES_BIG_ENDIAN)
4380 {
4381 rtx tmp = gen_reg_rtx (OImode);
4382 rtx mask = aarch64_reverse_mask (<MODE>mode);
4383 emit_insn (gen_aarch64_rev_reglistoi (tmp, operands[1], mask));
4384 emit_insn (gen_aarch64_simd_st2<mode> (operands[0], tmp));
4385 }
4386 else
4387 emit_insn (gen_aarch64_simd_st2<mode> (operands[0], operands[1]));
4388 DONE;
4389 })
4391 (define_insn "aarch64_simd_ld3<mode>"
4392 [(set (match_operand:CI 0 "register_operand" "=w")
4393 (unspec:CI [(match_operand:CI 1 "aarch64_simd_struct_operand" "Utv")
4394 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4395 UNSPEC_LD3))]
4396 "TARGET_SIMD"
4397 "ld3\\t{%S0.<Vtype> - %U0.<Vtype>}, %1"
4398 [(set_attr "type" "neon_load3_3reg<q>")]
4399 )
4401 (define_insn "aarch64_simd_ld3r<mode>"
4402 [(set (match_operand:CI 0 "register_operand" "=w")
4403 (unspec:CI [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")
4404 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY) ]
4405 UNSPEC_LD3_DUP))]
4406 "TARGET_SIMD"
4407 "ld3r\\t{%S0.<Vtype> - %U0.<Vtype>}, %1"
4408 [(set_attr "type" "neon_load3_all_lanes<q>")]
4409 )
4411 (define_insn "aarch64_vec_load_lanesci_lane<mode>"
4412 [(set (match_operand:CI 0 "register_operand" "=w")
4413 (unspec:CI [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")
4414 (match_operand:CI 2 "register_operand" "0")
4415 (match_operand:SI 3 "immediate_operand" "i")
4416 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4417 UNSPEC_LD3_LANE))]
4418 "TARGET_SIMD"
4419 {
4420 operands[3] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[3])));
4421 return "ld3\\t{%S0.<Vetype> - %U0.<Vetype>}[%3], %1";
4422 }
4423 [(set_attr "type" "neon_load3_one_lane")]
4424 )
4426 (define_expand "vec_load_lanesci<mode>"
4427 [(set (match_operand:CI 0 "register_operand" "=w")
4428 (unspec:CI [(match_operand:CI 1 "aarch64_simd_struct_operand" "Utv")
4429 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4430 UNSPEC_LD3))]
4431 "TARGET_SIMD"
4432 {
4433 if (BYTES_BIG_ENDIAN)
4434 {
4435 rtx tmp = gen_reg_rtx (CImode);
4436 rtx mask = aarch64_reverse_mask (<MODE>mode);
4437 emit_insn (gen_aarch64_simd_ld3<mode> (tmp, operands[1]));
4438 emit_insn (gen_aarch64_rev_reglistci (operands[0], tmp, mask));
4439 }
4440 else
4441 emit_insn (gen_aarch64_simd_ld3<mode> (operands[0], operands[1]));
4442 DONE;
4443 })
4445 (define_insn "aarch64_simd_st3<mode>"
4446 [(set (match_operand:CI 0 "aarch64_simd_struct_operand" "=Utv")
4447 (unspec:CI [(match_operand:CI 1 "register_operand" "w")
4448 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4449 UNSPEC_ST3))]
4450 "TARGET_SIMD"
4451 "st3\\t{%S1.<Vtype> - %U1.<Vtype>}, %0"
4452 [(set_attr "type" "neon_store3_3reg<q>")]
4453 )
4455 ;; RTL uses GCC vector extension indices, so flip only for assembly.
4456 (define_insn "aarch64_vec_store_lanesci_lane<mode>"
4457 [(set (match_operand:BLK 0 "aarch64_simd_struct_operand" "=Utv")
4458 (unspec:BLK [(match_operand:CI 1 "register_operand" "w")
4459 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)
4460 (match_operand:SI 2 "immediate_operand" "i")]
4461 UNSPEC_ST3_LANE))]
4462 "TARGET_SIMD"
4463 {
4464 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
4465 return "st3\\t{%S1.<Vetype> - %U1.<Vetype>}[%2], %0";
4466 }
4467 [(set_attr "type" "neon_store3_one_lane<q>")]
4468 )
4470 (define_expand "vec_store_lanesci<mode>"
4471 [(set (match_operand:CI 0 "aarch64_simd_struct_operand" "=Utv")
4472 (unspec:CI [(match_operand:CI 1 "register_operand" "w")
4473 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4474 UNSPEC_ST3))]
4475 "TARGET_SIMD"
4476 {
4477 if (BYTES_BIG_ENDIAN)
4478 {
4479 rtx tmp = gen_reg_rtx (CImode);
4480 rtx mask = aarch64_reverse_mask (<MODE>mode);
4481 emit_insn (gen_aarch64_rev_reglistci (tmp, operands[1], mask));
4482 emit_insn (gen_aarch64_simd_st3<mode> (operands[0], tmp));
4483 }
4484 else
4485 emit_insn (gen_aarch64_simd_st3<mode> (operands[0], operands[1]));
4486 DONE;
4487 })
4489 (define_insn "aarch64_simd_ld4<mode>"
4490 [(set (match_operand:XI 0 "register_operand" "=w")
4491 (unspec:XI [(match_operand:XI 1 "aarch64_simd_struct_operand" "Utv")
4492 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4493 UNSPEC_LD4))]
4494 "TARGET_SIMD"
4495 "ld4\\t{%S0.<Vtype> - %V0.<Vtype>}, %1"
4496 [(set_attr "type" "neon_load4_4reg<q>")]
4497 )
4499 (define_insn "aarch64_simd_ld4r<mode>"
4500 [(set (match_operand:XI 0 "register_operand" "=w")
4501 (unspec:XI [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")
4502 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY) ]
4503 UNSPEC_LD4_DUP))]
4504 "TARGET_SIMD"
4505 "ld4r\\t{%S0.<Vtype> - %V0.<Vtype>}, %1"
4506 [(set_attr "type" "neon_load4_all_lanes<q>")]
4507 )
4509 (define_insn "aarch64_vec_load_lanesxi_lane<mode>"
4510 [(set (match_operand:XI 0 "register_operand" "=w")
4511 (unspec:XI [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")
4512 (match_operand:XI 2 "register_operand" "0")
4513 (match_operand:SI 3 "immediate_operand" "i")
4514 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4515 UNSPEC_LD4_LANE))]
4516 "TARGET_SIMD"
4517 {
4518 operands[3] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[3])));
4519 return "ld4\\t{%S0.<Vetype> - %V0.<Vetype>}[%3], %1";
4520 }
4521 [(set_attr "type" "neon_load4_one_lane")]
4522 )
4524 (define_expand "vec_load_lanesxi<mode>"
4525 [(set (match_operand:XI 0 "register_operand" "=w")
4526 (unspec:XI [(match_operand:XI 1 "aarch64_simd_struct_operand" "Utv")
4527 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4528 UNSPEC_LD4))]
4529 "TARGET_SIMD"
4530 {
4531 if (BYTES_BIG_ENDIAN)
4532 {
4533 rtx tmp = gen_reg_rtx (XImode);
4534 rtx mask = aarch64_reverse_mask (<MODE>mode);
4535 emit_insn (gen_aarch64_simd_ld4<mode> (tmp, operands[1]));
4536 emit_insn (gen_aarch64_rev_reglistxi (operands[0], tmp, mask));
4537 }
4538 else
4539 emit_insn (gen_aarch64_simd_ld4<mode> (operands[0], operands[1]));
4540 DONE;
4541 })
4543 (define_insn "aarch64_simd_st4<mode>"
4544 [(set (match_operand:XI 0 "aarch64_simd_struct_operand" "=Utv")
4545 (unspec:XI [(match_operand:XI 1 "register_operand" "w")
4546 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4547 UNSPEC_ST4))]
4548 "TARGET_SIMD"
4549 "st4\\t{%S1.<Vtype> - %V1.<Vtype>}, %0"
4550 [(set_attr "type" "neon_store4_4reg<q>")]
4551 )
4553 ;; RTL uses GCC vector extension indices, so flip only for assembly.
4554 (define_insn "aarch64_vec_store_lanesxi_lane<mode>"
4555 [(set (match_operand:BLK 0 "aarch64_simd_struct_operand" "=Utv")
4556 (unspec:BLK [(match_operand:XI 1 "register_operand" "w")
4557 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)
4558 (match_operand:SI 2 "immediate_operand" "i")]
4559 UNSPEC_ST4_LANE))]
4560 "TARGET_SIMD"
4561 {
4562 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
4563 return "st4\\t{%S1.<Vetype> - %V1.<Vetype>}[%2], %0";
4564 }
4565 [(set_attr "type" "neon_store4_one_lane<q>")]
4566 )
4568 (define_expand "vec_store_lanesxi<mode>"
4569 [(set (match_operand:XI 0 "aarch64_simd_struct_operand" "=Utv")
4570 (unspec:XI [(match_operand:XI 1 "register_operand" "w")
4571 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4572 UNSPEC_ST4))]
4573 "TARGET_SIMD"
4574 {
4575 if (BYTES_BIG_ENDIAN)
4576 {
4577 rtx tmp = gen_reg_rtx (XImode);
4578 rtx mask = aarch64_reverse_mask (<MODE>mode);
4579 emit_insn (gen_aarch64_rev_reglistxi (tmp, operands[1], mask));
4580 emit_insn (gen_aarch64_simd_st4<mode> (operands[0], tmp));
4581 }
4582 else
4583 emit_insn (gen_aarch64_simd_st4<mode> (operands[0], operands[1]));
4584 DONE;
4585 })
4587 (define_insn_and_split "aarch64_rev_reglist<mode>"
4588 [(set (match_operand:VSTRUCT 0 "register_operand" "=&w")
4589 (unspec:VSTRUCT
4590 [(match_operand:VSTRUCT 1 "register_operand" "w")
4591 (match_operand:V16QI 2 "register_operand" "w")]
4592 UNSPEC_REV_REGLIST))]
4593 "TARGET_SIMD"
4594 "#"
4595 "&& reload_completed"
4596 [(const_int 0)]
4597 {
4598 int i;
4599 int nregs = GET_MODE_SIZE (<MODE>mode) / UNITS_PER_VREG;
4600 for (i = 0; i < nregs; i++)
4601 {
4602 rtx op0 = gen_rtx_REG (V16QImode, REGNO (operands[0]) + i);
4603 rtx op1 = gen_rtx_REG (V16QImode, REGNO (operands[1]) + i);
4604 emit_insn (gen_aarch64_tbl1v16qi (op0, op1, operands[2]));
4605 }
4606 DONE;
4607 }
4608 [(set_attr "type" "neon_tbl1_q")
4609 (set_attr "length" "<insn_count>")]
4610 )
4612 ;; Reload patterns for AdvSIMD register list operands.
4614 (define_expand "mov<mode>"
4615 [(set (match_operand:VSTRUCT 0 "nonimmediate_operand" "")
4616 (match_operand:VSTRUCT 1 "general_operand" ""))]
4617 "TARGET_SIMD"
4618 {
4619 if (can_create_pseudo_p ())
4620 {
4621 if (GET_CODE (operands[0]) != REG)
4622 operands[1] = force_reg (<MODE>mode, operands[1]);
4623 }
4624 })
4626 (define_insn "*aarch64_mov<mode>"
4627 [(set (match_operand:VSTRUCT 0 "aarch64_simd_nonimmediate_operand" "=w,Utv,w")
4628 (match_operand:VSTRUCT 1 "aarch64_simd_general_operand" " w,w,Utv"))]
4629 "TARGET_SIMD && !BYTES_BIG_ENDIAN
4630 && (register_operand (operands[0], <MODE>mode)
4631 || register_operand (operands[1], <MODE>mode))"
4632 "@
4633 #
4634 st1\\t{%S1.16b - %<Vendreg>1.16b}, %0
4635 ld1\\t{%S0.16b - %<Vendreg>0.16b}, %1"
4636 [(set_attr "type" "multiple,neon_store<nregs>_<nregs>reg_q,\
4637 neon_load<nregs>_<nregs>reg_q")
4638 (set (attr "length") (symbol_ref "aarch64_simd_attr_length_move (insn)"))]
4639 )
4641 (define_insn "aarch64_be_ld1<mode>"
4642 [(set (match_operand:VALLDI_F16 0 "register_operand" "=w")
4643 (unspec:VALLDI_F16 [(match_operand:VALLDI_F16 1
4644 "aarch64_simd_struct_operand" "Utv")]
4645 UNSPEC_LD1))]
4646 "TARGET_SIMD"
4647 "ld1\\t{%0<Vmtype>}, %1"
4648 [(set_attr "type" "neon_load1_1reg<q>")]
4649 )
4651 (define_insn "aarch64_be_st1<mode>"
4652 [(set (match_operand:VALLDI_F16 0 "aarch64_simd_struct_operand" "=Utv")
4653 (unspec:VALLDI_F16 [(match_operand:VALLDI_F16 1 "register_operand" "w")]
4654 UNSPEC_ST1))]
4655 "TARGET_SIMD"
4656 "st1\\t{%1<Vmtype>}, %0"
4657 [(set_attr "type" "neon_store1_1reg<q>")]
4658 )
4660 (define_insn "*aarch64_be_movoi"
4661 [(set (match_operand:OI 0 "nonimmediate_operand" "=w,m,w")
4662 (match_operand:OI 1 "general_operand" " w,w,m"))]
4663 "TARGET_SIMD && BYTES_BIG_ENDIAN
4664 && (register_operand (operands[0], OImode)
4665 || register_operand (operands[1], OImode))"
4666 "@
4667 #
4668 stp\\t%q1, %R1, %0
4669 ldp\\t%q0, %R0, %1"
4670 [(set_attr "type" "multiple,neon_stp_q,neon_ldp_q")
4671 (set (attr "length") (symbol_ref "aarch64_simd_attr_length_move (insn)"))]
4672 )
4674 (define_insn "*aarch64_be_movci"
4675 [(set (match_operand:CI 0 "nonimmediate_operand" "=w,o,w")
4676 (match_operand:CI 1 "general_operand" " w,w,o"))]
4677 "TARGET_SIMD && BYTES_BIG_ENDIAN
4678 && (register_operand (operands[0], CImode)
4679 || register_operand (operands[1], CImode))"
4680 "#"
4681 [(set_attr "type" "multiple")
4682 (set (attr "length") (symbol_ref "aarch64_simd_attr_length_move (insn)"))]
4683 )
4685 (define_insn "*aarch64_be_movxi"
4686 [(set (match_operand:XI 0 "nonimmediate_operand" "=w,o,w")
4687 (match_operand:XI 1 "general_operand" " w,w,o"))]
4688 "TARGET_SIMD && BYTES_BIG_ENDIAN
4689 && (register_operand (operands[0], XImode)
4690 || register_operand (operands[1], XImode))"
4691 "#"
4692 [(set_attr "type" "multiple")
4693 (set (attr "length") (symbol_ref "aarch64_simd_attr_length_move (insn)"))]
4694 )
4696 (define_split
4697 [(set (match_operand:OI 0 "register_operand")
4698 (match_operand:OI 1 "register_operand"))]
4699 "TARGET_SIMD && reload_completed"
4700 [(const_int 0)]
4701 {
4702 aarch64_simd_emit_reg_reg_move (operands, TImode, 2);
4703 DONE;
4704 })
4706 (define_split
4707 [(set (match_operand:CI 0 "nonimmediate_operand")
4708 (match_operand:CI 1 "general_operand"))]
4709 "TARGET_SIMD && reload_completed"
4710 [(const_int 0)]
4711 {
4712 if (register_operand (operands[0], CImode)
4713 && register_operand (operands[1], CImode))
4714 {
4715 aarch64_simd_emit_reg_reg_move (operands, TImode, 3);
4716 DONE;
4717 }
4718 else if (BYTES_BIG_ENDIAN)
4719 {
4720 emit_move_insn (simplify_gen_subreg (OImode, operands[0], CImode, 0),
4721 simplify_gen_subreg (OImode, operands[1], CImode, 0));
4722 emit_move_insn (gen_lowpart (V16QImode,
4723 simplify_gen_subreg (TImode, operands[0],
4724 CImode, 32)),
4725 gen_lowpart (V16QImode,
4726 simplify_gen_subreg (TImode, operands[1],
4727 CImode, 32)));
4728 DONE;
4729 }
4730 else
4731 FAIL;
4732 })
4734 (define_split
4735 [(set (match_operand:XI 0 "nonimmediate_operand")
4736 (match_operand:XI 1 "general_operand"))]
4737 "TARGET_SIMD && reload_completed"
4738 [(const_int 0)]
4739 {
4740 if (register_operand (operands[0], XImode)
4741 && register_operand (operands[1], XImode))
4742 {
4743 aarch64_simd_emit_reg_reg_move (operands, TImode, 4);
4744 DONE;
4745 }
4746 else if (BYTES_BIG_ENDIAN)
4747 {
4748 emit_move_insn (simplify_gen_subreg (OImode, operands[0], XImode, 0),
4749 simplify_gen_subreg (OImode, operands[1], XImode, 0));
4750 emit_move_insn (simplify_gen_subreg (OImode, operands[0], XImode, 32),
4751 simplify_gen_subreg (OImode, operands[1], XImode, 32));
4752 DONE;
4753 }
4754 else
4755 FAIL;
4756 })
4758 (define_expand "aarch64_ld<VSTRUCT:nregs>r<VALLDIF:mode>"
4759 [(match_operand:VSTRUCT 0 "register_operand" "=w")
4760 (match_operand:DI 1 "register_operand" "w")
4761 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4762 "TARGET_SIMD"
4763 {
4764 rtx mem = gen_rtx_MEM (BLKmode, operands[1]);
4765 set_mem_size (mem, GET_MODE_SIZE (GET_MODE_INNER (<VALLDIF:MODE>mode))
4766 * <VSTRUCT:nregs>);
4768 emit_insn (gen_aarch64_simd_ld<VSTRUCT:nregs>r<VALLDIF:mode> (operands[0],
4769 mem));
4770 DONE;
4771 })
4773 (define_insn "aarch64_ld2<mode>_dreg"
4774 [(set (match_operand:OI 0 "register_operand" "=w")
4775 (subreg:OI
4776 (vec_concat:<VRL2>
4777 (vec_concat:<VDBL>
4778 (unspec:VD
4779 [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")]
4780 UNSPEC_LD2)
4781 (vec_duplicate:VD (const_int 0)))
4782 (vec_concat:<VDBL>
4783 (unspec:VD [(match_dup 1)]
4784 UNSPEC_LD2)
4785 (vec_duplicate:VD (const_int 0)))) 0))]
4786 "TARGET_SIMD"
4787 "ld2\\t{%S0.<Vtype> - %T0.<Vtype>}, %1"
4788 [(set_attr "type" "neon_load2_2reg<q>")]
4789 )
4791 (define_insn "aarch64_ld2<mode>_dreg"
4792 [(set (match_operand:OI 0 "register_operand" "=w")
4793 (subreg:OI
4794 (vec_concat:<VRL2>
4795 (vec_concat:<VDBL>
4796 (unspec:DX
4797 [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")]
4798 UNSPEC_LD2)
4799 (const_int 0))
4800 (vec_concat:<VDBL>
4801 (unspec:DX [(match_dup 1)]
4802 UNSPEC_LD2)
4803 (const_int 0))) 0))]
4804 "TARGET_SIMD"
4805 "ld1\\t{%S0.1d - %T0.1d}, %1"
4806 [(set_attr "type" "neon_load1_2reg<q>")]
4807 )
4809 (define_insn "aarch64_ld3<mode>_dreg"
4810 [(set (match_operand:CI 0 "register_operand" "=w")
4811 (subreg:CI
4812 (vec_concat:<VRL3>
4813 (vec_concat:<VRL2>
4814 (vec_concat:<VDBL>
4815 (unspec:VD
4816 [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")]
4817 UNSPEC_LD3)
4818 (vec_duplicate:VD (const_int 0)))
4819 (vec_concat:<VDBL>
4820 (unspec:VD [(match_dup 1)]
4821 UNSPEC_LD3)
4822 (vec_duplicate:VD (const_int 0))))
4823 (vec_concat:<VDBL>
4824 (unspec:VD [(match_dup 1)]
4825 UNSPEC_LD3)
4826 (vec_duplicate:VD (const_int 0)))) 0))]
4827 "TARGET_SIMD"
4828 "ld3\\t{%S0.<Vtype> - %U0.<Vtype>}, %1"
4829 [(set_attr "type" "neon_load3_3reg<q>")]
4830 )
4832 (define_insn "aarch64_ld3<mode>_dreg"
4833 [(set (match_operand:CI 0 "register_operand" "=w")
4834 (subreg:CI
4835 (vec_concat:<VRL3>
4836 (vec_concat:<VRL2>
4837 (vec_concat:<VDBL>
4838 (unspec:DX
4839 [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")]
4840 UNSPEC_LD3)
4841 (const_int 0))
4842 (vec_concat:<VDBL>
4843 (unspec:DX [(match_dup 1)]
4844 UNSPEC_LD3)
4845 (const_int 0)))
4846 (vec_concat:<VDBL>
4847 (unspec:DX [(match_dup 1)]
4848 UNSPEC_LD3)
4849 (const_int 0))) 0))]
4850 "TARGET_SIMD"
4851 "ld1\\t{%S0.1d - %U0.1d}, %1"
4852 [(set_attr "type" "neon_load1_3reg<q>")]
4853 )
4855 (define_insn "aarch64_ld4<mode>_dreg"
4856 [(set (match_operand:XI 0 "register_operand" "=w")
4857 (subreg:XI
4858 (vec_concat:<VRL4>
4859 (vec_concat:<VRL2>
4860 (vec_concat:<VDBL>
4861 (unspec:VD
4862 [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")]
4863 UNSPEC_LD4)
4864 (vec_duplicate:VD (const_int 0)))
4865 (vec_concat:<VDBL>
4866 (unspec:VD [(match_dup 1)]
4867 UNSPEC_LD4)
4868 (vec_duplicate:VD (const_int 0))))
4869 (vec_concat:<VRL2>
4870 (vec_concat:<VDBL>
4871 (unspec:VD [(match_dup 1)]
4872 UNSPEC_LD4)
4873 (vec_duplicate:VD (const_int 0)))
4874 (vec_concat:<VDBL>
4875 (unspec:VD [(match_dup 1)]
4876 UNSPEC_LD4)
4877 (vec_duplicate:VD (const_int 0))))) 0))]
4878 "TARGET_SIMD"
4879 "ld4\\t{%S0.<Vtype> - %V0.<Vtype>}, %1"
4880 [(set_attr "type" "neon_load4_4reg<q>")]
4881 )
4883 (define_insn "aarch64_ld4<mode>_dreg"
4884 [(set (match_operand:XI 0 "register_operand" "=w")
4885 (subreg:XI
4886 (vec_concat:<VRL4>
4887 (vec_concat:<VRL2>
4888 (vec_concat:<VDBL>
4889 (unspec:DX
4890 [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")]
4891 UNSPEC_LD4)
4892 (const_int 0))
4893 (vec_concat:<VDBL>
4894 (unspec:DX [(match_dup 1)]
4895 UNSPEC_LD4)
4896 (const_int 0)))
4897 (vec_concat:<VRL2>
4898 (vec_concat:<VDBL>
4899 (unspec:DX [(match_dup 1)]
4900 UNSPEC_LD4)
4901 (const_int 0))
4902 (vec_concat:<VDBL>
4903 (unspec:DX [(match_dup 1)]
4904 UNSPEC_LD4)
4905 (const_int 0)))) 0))]
4906 "TARGET_SIMD"
4907 "ld1\\t{%S0.1d - %V0.1d}, %1"
4908 [(set_attr "type" "neon_load1_4reg<q>")]
4909 )
4911 (define_expand "aarch64_ld<VSTRUCT:nregs><VDC:mode>"
4912 [(match_operand:VSTRUCT 0 "register_operand" "=w")
4913 (match_operand:DI 1 "register_operand" "r")
4914 (unspec:VDC [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4915 "TARGET_SIMD"
4916 {
4917 rtx mem = gen_rtx_MEM (BLKmode, operands[1]);
4918 set_mem_size (mem, <VSTRUCT:nregs> * 8);
4920 emit_insn (gen_aarch64_ld<VSTRUCT:nregs><VDC:mode>_dreg (operands[0], mem));
4921 DONE;
4922 })
4924 (define_expand "aarch64_ld1<VALL_F16:mode>"
4925 [(match_operand:VALL_F16 0 "register_operand")
4926 (match_operand:DI 1 "register_operand")]
4927 "TARGET_SIMD"
4928 {
4929 machine_mode mode = <VALL_F16:MODE>mode;
4930 rtx mem = gen_rtx_MEM (mode, operands[1]);
4932 if (BYTES_BIG_ENDIAN)
4933 emit_insn (gen_aarch64_be_ld1<VALL_F16:mode> (operands[0], mem));
4934 else
4935 emit_move_insn (operands[0], mem);
4936 DONE;
4937 })
4939 (define_expand "aarch64_ld<VSTRUCT:nregs><VQ:mode>"
4940 [(match_operand:VSTRUCT 0 "register_operand" "=w")
4941 (match_operand:DI 1 "register_operand" "r")
4942 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4943 "TARGET_SIMD"
4944 {
4945 machine_mode mode = <VSTRUCT:MODE>mode;
4946 rtx mem = gen_rtx_MEM (mode, operands[1]);
4948 emit_insn (gen_aarch64_simd_ld<VSTRUCT:nregs><VQ:mode> (operands[0], mem));
4949 DONE;
4950 })
4952 (define_expand "aarch64_ld<VSTRUCT:nregs>_lane<VALLDIF:mode>"
4953 [(match_operand:VSTRUCT 0 "register_operand" "=w")
4954 (match_operand:DI 1 "register_operand" "w")
4955 (match_operand:VSTRUCT 2 "register_operand" "0")
4956 (match_operand:SI 3 "immediate_operand" "i")
4957 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4958 "TARGET_SIMD"
4959 {
4960 rtx mem = gen_rtx_MEM (BLKmode, operands[1]);
4961 set_mem_size (mem, GET_MODE_SIZE (GET_MODE_INNER (<VALLDIF:MODE>mode))
4962 * <VSTRUCT:nregs>);
4964 aarch64_simd_lane_bounds (operands[3], 0,
4965 GET_MODE_NUNITS (<VALLDIF:MODE>mode),
4966 NULL);
4967 emit_insn (gen_aarch64_vec_load_lanes<VSTRUCT:mode>_lane<VALLDIF:mode> (
4968 operands[0], mem, operands[2], operands[3]));
4969 DONE;
4970 })
4972 ;; Expanders for builtins to extract vector registers from large
4973 ;; opaque integer modes.
4975 ;; D-register list.
4977 (define_expand "aarch64_get_dreg<VSTRUCT:mode><VDC:mode>"
4978 [(match_operand:VDC 0 "register_operand" "=w")
4979 (match_operand:VSTRUCT 1 "register_operand" "w")
4980 (match_operand:SI 2 "immediate_operand" "i")]
4981 "TARGET_SIMD"
4982 {
4983 int part = INTVAL (operands[2]);
4984 rtx temp = gen_reg_rtx (<VDC:VDBL>mode);
4985 int offset = part * 16;
4987 emit_move_insn (temp, gen_rtx_SUBREG (<VDC:VDBL>mode, operands[1], offset));
4988 emit_move_insn (operands[0], gen_lowpart (<VDC:MODE>mode, temp));
4989 DONE;
4990 })
4992 ;; Q-register list.
4994 (define_expand "aarch64_get_qreg<VSTRUCT:mode><VQ:mode>"
4995 [(match_operand:VQ 0 "register_operand" "=w")
4996 (match_operand:VSTRUCT 1 "register_operand" "w")
4997 (match_operand:SI 2 "immediate_operand" "i")]
4998 "TARGET_SIMD"
4999 {
5000 int part = INTVAL (operands[2]);
5001 int offset = part * 16;
5003 emit_move_insn (operands[0],
5004 gen_rtx_SUBREG (<VQ:MODE>mode, operands[1], offset));
5005 DONE;
5006 })
5008 ;; Permuted-store expanders for neon intrinsics.
5010 ;; Permute instructions
5012 ;; vec_perm support
5014 (define_expand "vec_perm_const<mode>"
5015 [(match_operand:VALL_F16 0 "register_operand")
5016 (match_operand:VALL_F16 1 "register_operand")
5017 (match_operand:VALL_F16 2 "register_operand")
5018 (match_operand:<V_cmp_result> 3)]
5019 "TARGET_SIMD"
5020 {
5021 if (aarch64_expand_vec_perm_const (operands[0], operands[1],
5022 operands[2], operands[3]))
5023 DONE;
5024 else
5025 FAIL;
5026 })
5028 (define_expand "vec_perm<mode>"
5029 [(match_operand:VB 0 "register_operand")
5030 (match_operand:VB 1 "register_operand")
5031 (match_operand:VB 2 "register_operand")
5032 (match_operand:VB 3 "register_operand")]
5033 "TARGET_SIMD"
5034 {
5035 aarch64_expand_vec_perm (operands[0], operands[1],
5036 operands[2], operands[3]);
5037 DONE;
5038 })
5040 (define_insn "aarch64_tbl1<mode>"
5041 [(set (match_operand:VB 0 "register_operand" "=w")
5042 (unspec:VB [(match_operand:V16QI 1 "register_operand" "w")
5043 (match_operand:VB 2 "register_operand" "w")]
5044 UNSPEC_TBL))]
5045 "TARGET_SIMD"
5046 "tbl\\t%0.<Vtype>, {%1.16b}, %2.<Vtype>"
5047 [(set_attr "type" "neon_tbl1<q>")]
5048 )
5050 ;; Two source registers.
5052 (define_insn "aarch64_tbl2v16qi"
5053 [(set (match_operand:V16QI 0 "register_operand" "=w")
5054 (unspec:V16QI [(match_operand:OI 1 "register_operand" "w")
5055 (match_operand:V16QI 2 "register_operand" "w")]
5056 UNSPEC_TBL))]
5057 "TARGET_SIMD"
5058 "tbl\\t%0.16b, {%S1.16b - %T1.16b}, %2.16b"
5059 [(set_attr "type" "neon_tbl2_q")]
5060 )
5062 (define_insn "aarch64_tbl3<mode>"
5063 [(set (match_operand:VB 0 "register_operand" "=w")
5064 (unspec:VB [(match_operand:OI 1 "register_operand" "w")
5065 (match_operand:VB 2 "register_operand" "w")]
5066 UNSPEC_TBL))]
5067 "TARGET_SIMD"
5068 "tbl\\t%S0.<Vbtype>, {%S1.16b - %T1.16b}, %S2.<Vbtype>"
5069 [(set_attr "type" "neon_tbl3")]
5070 )
5072 (define_insn "aarch64_tbx4<mode>"
5073 [(set (match_operand:VB 0 "register_operand" "=w")
5074 (unspec:VB [(match_operand:VB 1 "register_operand" "0")
5075 (match_operand:OI 2 "register_operand" "w")
5076 (match_operand:VB 3 "register_operand" "w")]
5077 UNSPEC_TBX))]
5078 "TARGET_SIMD"
5079 "tbx\\t%S0.<Vbtype>, {%S2.16b - %T2.16b}, %S3.<Vbtype>"
5080 [(set_attr "type" "neon_tbl4")]
5081 )
5083 ;; Three source registers.
5085 (define_insn "aarch64_qtbl3<mode>"
5086 [(set (match_operand:VB 0 "register_operand" "=w")
5087 (unspec:VB [(match_operand:CI 1 "register_operand" "w")
5088 (match_operand:VB 2 "register_operand" "w")]
5089 UNSPEC_TBL))]
5090 "TARGET_SIMD"
5091 "tbl\\t%S0.<Vbtype>, {%S1.16b - %U1.16b}, %S2.<Vbtype>"
5092 [(set_attr "type" "neon_tbl3")]
5093 )
5095 (define_insn "aarch64_qtbx3<mode>"
5096 [(set (match_operand:VB 0 "register_operand" "=w")
5097 (unspec:VB [(match_operand:VB 1 "register_operand" "0")
5098 (match_operand:CI 2 "register_operand" "w")
5099 (match_operand:VB 3 "register_operand" "w")]
5100 UNSPEC_TBX))]
5101 "TARGET_SIMD"
5102 "tbx\\t%S0.<Vbtype>, {%S2.16b - %U2.16b}, %S3.<Vbtype>"
5103 [(set_attr "type" "neon_tbl3")]
5104 )
5106 ;; Four source registers.
5108 (define_insn "aarch64_qtbl4<mode>"
5109 [(set (match_operand:VB 0 "register_operand" "=w")
5110 (unspec:VB [(match_operand:XI 1 "register_operand" "w")
5111 (match_operand:VB 2 "register_operand" "w")]
5112 UNSPEC_TBL))]
5113 "TARGET_SIMD"
5114 "tbl\\t%S0.<Vbtype>, {%S1.16b - %V1.16b}, %S2.<Vbtype>"
5115 [(set_attr "type" "neon_tbl4")]
5116 )
5118 (define_insn "aarch64_qtbx4<mode>"
5119 [(set (match_operand:VB 0 "register_operand" "=w")
5120 (unspec:VB [(match_operand:VB 1 "register_operand" "0")
5121 (match_operand:XI 2 "register_operand" "w")
5122 (match_operand:VB 3 "register_operand" "w")]
5123 UNSPEC_TBX))]
5124 "TARGET_SIMD"
5125 "tbx\\t%S0.<Vbtype>, {%S2.16b - %V2.16b}, %S3.<Vbtype>"
5126 [(set_attr "type" "neon_tbl4")]
5127 )
5129 (define_insn_and_split "aarch64_combinev16qi"
5130 [(set (match_operand:OI 0 "register_operand" "=w")
5131 (unspec:OI [(match_operand:V16QI 1 "register_operand" "w")
5132 (match_operand:V16QI 2 "register_operand" "w")]
5133 UNSPEC_CONCAT))]
5134 "TARGET_SIMD"
5135 "#"
5136 "&& reload_completed"
5137 [(const_int 0)]
5138 {
5139 aarch64_split_combinev16qi (operands);
5140 DONE;
5141 }
5142 [(set_attr "type" "multiple")]
5143 )
5145 (define_insn "aarch64_<PERMUTE:perm_insn><PERMUTE:perm_hilo><mode>"
5146 [(set (match_operand:VALL 0 "register_operand" "=w")
5147 (unspec:VALL [(match_operand:VALL 1 "register_operand" "w")
5148 (match_operand:VALL 2 "register_operand" "w")]
5149 PERMUTE))]
5150 "TARGET_SIMD"
5151 "<PERMUTE:perm_insn><PERMUTE:perm_hilo>\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
5152 [(set_attr "type" "neon_permute<q>")]
5153 )
5155 ;; Note immediate (third) operand is lane index not byte index.
5156 (define_insn "aarch64_ext<mode>"
5157 [(set (match_operand:VALL 0 "register_operand" "=w")
5158 (unspec:VALL [(match_operand:VALL 1 "register_operand" "w")
5159 (match_operand:VALL 2 "register_operand" "w")
5160 (match_operand:SI 3 "immediate_operand" "i")]
5161 UNSPEC_EXT))]
5162 "TARGET_SIMD"
5163 {
5164 operands[3] = GEN_INT (INTVAL (operands[3])
5165 * GET_MODE_UNIT_SIZE (<MODE>mode));
5166 return "ext\\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>, #%3";
5167 }
5168 [(set_attr "type" "neon_ext<q>")]
5169 )
5171 (define_insn "aarch64_rev<REVERSE:rev_op><mode>"
5172 [(set (match_operand:VALL 0 "register_operand" "=w")
5173 (unspec:VALL [(match_operand:VALL 1 "register_operand" "w")]
5174 REVERSE))]
5175 "TARGET_SIMD"
5176 "rev<REVERSE:rev_op>\\t%0.<Vtype>, %1.<Vtype>"
5177 [(set_attr "type" "neon_rev<q>")]
5178 )
5180 (define_insn "aarch64_st2<mode>_dreg"
5181 [(set (match_operand:BLK 0 "aarch64_simd_struct_operand" "=Utv")
5182 (unspec:BLK [(match_operand:OI 1 "register_operand" "w")
5183 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5184 UNSPEC_ST2))]
5185 "TARGET_SIMD"
5186 "st2\\t{%S1.<Vtype> - %T1.<Vtype>}, %0"
5187 [(set_attr "type" "neon_store2_2reg")]
5188 )
5190 (define_insn "aarch64_st2<mode>_dreg"
5191 [(set (match_operand:BLK 0 "aarch64_simd_struct_operand" "=Utv")
5192 (unspec:BLK [(match_operand:OI 1 "register_operand" "w")
5193 (unspec:DX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5194 UNSPEC_ST2))]
5195 "TARGET_SIMD"
5196 "st1\\t{%S1.1d - %T1.1d}, %0"
5197 [(set_attr "type" "neon_store1_2reg")]
5198 )
5200 (define_insn "aarch64_st3<mode>_dreg"
5201 [(set (match_operand:BLK 0 "aarch64_simd_struct_operand" "=Utv")
5202 (unspec:BLK [(match_operand:CI 1 "register_operand" "w")
5203 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5204 UNSPEC_ST3))]
5205 "TARGET_SIMD"
5206 "st3\\t{%S1.<Vtype> - %U1.<Vtype>}, %0"
5207 [(set_attr "type" "neon_store3_3reg")]
5208 )
5210 (define_insn "aarch64_st3<mode>_dreg"
5211 [(set (match_operand:BLK 0 "aarch64_simd_struct_operand" "=Utv")
5212 (unspec:BLK [(match_operand:CI 1 "register_operand" "w")
5213 (unspec:DX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5214 UNSPEC_ST3))]
5215 "TARGET_SIMD"
5216 "st1\\t{%S1.1d - %U1.1d}, %0"
5217 [(set_attr "type" "neon_store1_3reg")]
5218 )
5220 (define_insn "aarch64_st4<mode>_dreg"
5221 [(set (match_operand:BLK 0 "aarch64_simd_struct_operand" "=Utv")
5222 (unspec:BLK [(match_operand:XI 1 "register_operand" "w")
5223 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5224 UNSPEC_ST4))]
5225 "TARGET_SIMD"
5226 "st4\\t{%S1.<Vtype> - %V1.<Vtype>}, %0"
5227 [(set_attr "type" "neon_store4_4reg")]
5228 )
5230 (define_insn "aarch64_st4<mode>_dreg"
5231 [(set (match_operand:BLK 0 "aarch64_simd_struct_operand" "=Utv")
5232 (unspec:BLK [(match_operand:XI 1 "register_operand" "w")
5233 (unspec:DX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5234 UNSPEC_ST4))]
5235 "TARGET_SIMD"
5236 "st1\\t{%S1.1d - %V1.1d}, %0"
5237 [(set_attr "type" "neon_store1_4reg")]
5238 )
5240 (define_expand "aarch64_st<VSTRUCT:nregs><VDC:mode>"
5241 [(match_operand:DI 0 "register_operand" "r")
5242 (match_operand:VSTRUCT 1 "register_operand" "w")
5243 (unspec:VDC [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5244 "TARGET_SIMD"
5245 {
5246 rtx mem = gen_rtx_MEM (BLKmode, operands[0]);
5247 set_mem_size (mem, <VSTRUCT:nregs> * 8);
5249 emit_insn (gen_aarch64_st<VSTRUCT:nregs><VDC:mode>_dreg (mem, operands[1]));
5250 DONE;
5251 })
5253 (define_expand "aarch64_st<VSTRUCT:nregs><VQ:mode>"
5254 [(match_operand:DI 0 "register_operand" "r")
5255 (match_operand:VSTRUCT 1 "register_operand" "w")
5256 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5257 "TARGET_SIMD"
5258 {
5259 machine_mode mode = <VSTRUCT:MODE>mode;
5260 rtx mem = gen_rtx_MEM (mode, operands[0]);
5262 emit_insn (gen_aarch64_simd_st<VSTRUCT:nregs><VQ:mode> (mem, operands[1]));
5263 DONE;
5264 })
5266 (define_expand "aarch64_st<VSTRUCT:nregs>_lane<VALLDIF:mode>"
5267 [(match_operand:DI 0 "register_operand" "r")
5268 (match_operand:VSTRUCT 1 "register_operand" "w")
5269 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)
5270 (match_operand:SI 2 "immediate_operand")]
5271 "TARGET_SIMD"
5272 {
5273 rtx mem = gen_rtx_MEM (BLKmode, operands[0]);
5274 set_mem_size (mem, GET_MODE_SIZE (GET_MODE_INNER (<VALLDIF:MODE>mode))
5275 * <VSTRUCT:nregs>);
5277 emit_insn (gen_aarch64_vec_store_lanes<VSTRUCT:mode>_lane<VALLDIF:mode> (
5278 mem, operands[1], operands[2]));
5279 DONE;
5280 })
5282 (define_expand "aarch64_st1<VALL_F16:mode>"
5283 [(match_operand:DI 0 "register_operand")
5284 (match_operand:VALL_F16 1 "register_operand")]
5285 "TARGET_SIMD"
5286 {
5287 machine_mode mode = <VALL_F16:MODE>mode;
5288 rtx mem = gen_rtx_MEM (mode, operands[0]);
5290 if (BYTES_BIG_ENDIAN)
5291 emit_insn (gen_aarch64_be_st1<VALL_F16:mode> (mem, operands[1]));
5292 else
5293 emit_move_insn (mem, operands[1]);
5294 DONE;
5295 })
5297 ;; Expander for builtins to insert vector registers into large
5298 ;; opaque integer modes.
5300 ;; Q-register list. We don't need a D-reg inserter as we zero
5301 ;; extend them in arm_neon.h and insert the resulting Q-regs.
5303 (define_expand "aarch64_set_qreg<VSTRUCT:mode><VQ:mode>"
5304 [(match_operand:VSTRUCT 0 "register_operand" "+w")
5305 (match_operand:VSTRUCT 1 "register_operand" "0")
5306 (match_operand:VQ 2 "register_operand" "w")
5307 (match_operand:SI 3 "immediate_operand" "i")]
5308 "TARGET_SIMD"
5309 {
5310 int part = INTVAL (operands[3]);
5311 int offset = part * 16;
5313 emit_move_insn (operands[0], operands[1]);
5314 emit_move_insn (gen_rtx_SUBREG (<VQ:MODE>mode, operands[0], offset),
5315 operands[2]);
5316 DONE;
5317 })
5319 ;; Standard pattern name vec_init<mode>.
5321 (define_expand "vec_init<mode>"
5322 [(match_operand:VALL_F16 0 "register_operand" "")
5323 (match_operand 1 "" "")]
5324 "TARGET_SIMD"
5325 {
5326 aarch64_expand_vector_init (operands[0], operands[1]);
5327 DONE;
5328 })
5330 (define_insn "*aarch64_simd_ld1r<mode>"
5331 [(set (match_operand:VALL_F16 0 "register_operand" "=w")
5332 (vec_duplicate:VALL_F16
5333 (match_operand:<VEL> 1 "aarch64_simd_struct_operand" "Utv")))]
5334 "TARGET_SIMD"
5335 "ld1r\\t{%0.<Vtype>}, %1"
5336 [(set_attr "type" "neon_load1_all_lanes")]
5337 )
5339 (define_insn "aarch64_frecpe<mode>"
5340 [(set (match_operand:VDQF 0 "register_operand" "=w")
5341 (unspec:VDQF [(match_operand:VDQF 1 "register_operand" "w")]
5342 UNSPEC_FRECPE))]
5343 "TARGET_SIMD"
5344 "frecpe\\t%0.<Vtype>, %1.<Vtype>"
5345 [(set_attr "type" "neon_fp_recpe_<Vetype><q>")]
5346 )
5348 (define_insn "aarch64_frecp<FRECP:frecp_suffix><mode>"
5349 [(set (match_operand:GPF 0 "register_operand" "=w")
5350 (unspec:GPF [(match_operand:GPF 1 "register_operand" "w")]
5351 FRECP))]
5352 "TARGET_SIMD"
5353 "frecp<FRECP:frecp_suffix>\\t%<s>0, %<s>1"
5354 [(set_attr "type" "neon_fp_recp<FRECP:frecp_suffix>_<GPF:Vetype><GPF:q>")]
5355 )
5357 (define_insn "aarch64_frecps<mode>"
5358 [(set (match_operand:VALLF 0 "register_operand" "=w")
5359 (unspec:VALLF [(match_operand:VALLF 1 "register_operand" "w")
5360 (match_operand:VALLF 2 "register_operand" "w")]
5361 UNSPEC_FRECPS))]
5362 "TARGET_SIMD"
5363 "frecps\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
5364 [(set_attr "type" "neon_fp_recps_<Vetype><q>")]
5365 )
5367 (define_insn "aarch64_urecpe<mode>"
5368 [(set (match_operand:VDQ_SI 0 "register_operand" "=w")
5369 (unspec:VDQ_SI [(match_operand:VDQ_SI 1 "register_operand" "w")]
5370 UNSPEC_URECPE))]
5371 "TARGET_SIMD"
5372 "urecpe\\t%0.<Vtype>, %1.<Vtype>"
5373 [(set_attr "type" "neon_fp_recpe_<Vetype><q>")])
5375 ;; Standard pattern name vec_extract<mode>.
5377 (define_expand "vec_extract<mode>"
5378 [(match_operand:<VEL> 0 "aarch64_simd_nonimmediate_operand" "")
5379 (match_operand:VALL_F16 1 "register_operand" "")
5380 (match_operand:SI 2 "immediate_operand" "")]
5381 "TARGET_SIMD"
5382 {
5383 emit_insn
5384 (gen_aarch64_get_lane<mode> (operands[0], operands[1], operands[2]));
5385 DONE;
5386 })
5388 ;; aes
5390 (define_insn "aarch64_crypto_aes<aes_op>v16qi"
5391 [(set (match_operand:V16QI 0 "register_operand" "=w")
5392 (unspec:V16QI [(match_operand:V16QI 1 "register_operand" "0")
5393 (match_operand:V16QI 2 "register_operand" "w")]
5394 CRYPTO_AES))]
5395 "TARGET_SIMD && TARGET_CRYPTO"
5396 "aes<aes_op>\\t%0.16b, %2.16b"
5397 [(set_attr "type" "crypto_aese")]
5398 )
5400 (define_insn "aarch64_crypto_aes<aesmc_op>v16qi"
5401 [(set (match_operand:V16QI 0 "register_operand" "=w")
5402 (unspec:V16QI [(match_operand:V16QI 1 "register_operand" "w")]
5403 CRYPTO_AESMC))]
5404 "TARGET_SIMD && TARGET_CRYPTO"
5405 "aes<aesmc_op>\\t%0.16b, %1.16b"
5406 [(set_attr "type" "crypto_aesmc")]
5407 )
5409 ;; sha1
5411 (define_insn "aarch64_crypto_sha1hsi"
5412 [(set (match_operand:SI 0 "register_operand" "=w")
5413 (unspec:SI [(match_operand:SI 1
5414 "register_operand" "w")]
5415 UNSPEC_SHA1H))]
5416 "TARGET_SIMD && TARGET_CRYPTO"
5417 "sha1h\\t%s0, %s1"
5418 [(set_attr "type" "crypto_sha1_fast")]
5419 )
5421 (define_insn "aarch64_crypto_sha1su1v4si"
5422 [(set (match_operand:V4SI 0 "register_operand" "=w")
5423 (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
5424 (match_operand:V4SI 2 "register_operand" "w")]
5425 UNSPEC_SHA1SU1))]
5426 "TARGET_SIMD && TARGET_CRYPTO"
5427 "sha1su1\\t%0.4s, %2.4s"
5428 [(set_attr "type" "crypto_sha1_fast")]
5429 )
5431 (define_insn "aarch64_crypto_sha1<sha1_op>v4si"
5432 [(set (match_operand:V4SI 0 "register_operand" "=w")
5433 (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
5434 (match_operand:SI 2 "register_operand" "w")
5435 (match_operand:V4SI 3 "register_operand" "w")]
5436 CRYPTO_SHA1))]
5437 "TARGET_SIMD && TARGET_CRYPTO"
5438 "sha1<sha1_op>\\t%q0, %s2, %3.4s"
5439 [(set_attr "type" "crypto_sha1_slow")]
5440 )
5442 (define_insn "aarch64_crypto_sha1su0v4si"
5443 [(set (match_operand:V4SI 0 "register_operand" "=w")
5444 (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
5445 (match_operand:V4SI 2 "register_operand" "w")
5446 (match_operand:V4SI 3 "register_operand" "w")]
5447 UNSPEC_SHA1SU0))]
5448 "TARGET_SIMD && TARGET_CRYPTO"
5449 "sha1su0\\t%0.4s, %2.4s, %3.4s"
5450 [(set_attr "type" "crypto_sha1_xor")]
5451 )
5453 ;; sha256
5455 (define_insn "aarch64_crypto_sha256h<sha256_op>v4si"
5456 [(set (match_operand:V4SI 0 "register_operand" "=w")
5457 (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
5458 (match_operand:V4SI 2 "register_operand" "w")
5459 (match_operand:V4SI 3 "register_operand" "w")]
5460 CRYPTO_SHA256))]
5461 "TARGET_SIMD && TARGET_CRYPTO"
5462 "sha256h<sha256_op>\\t%q0, %q2, %3.4s"
5463 [(set_attr "type" "crypto_sha256_slow")]
5464 )
5466 (define_insn "aarch64_crypto_sha256su0v4si"
5467 [(set (match_operand:V4SI 0 "register_operand" "=w")
5468 (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
5469 (match_operand:V4SI 2 "register_operand" "w")]
5470 UNSPEC_SHA256SU0))]
5471 "TARGET_SIMD &&TARGET_CRYPTO"
5472 "sha256su0\\t%0.4s, %2.4s"
5473 [(set_attr "type" "crypto_sha256_fast")]
5474 )
5476 (define_insn "aarch64_crypto_sha256su1v4si"
5477 [(set (match_operand:V4SI 0 "register_operand" "=w")
5478 (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
5479 (match_operand:V4SI 2 "register_operand" "w")
5480 (match_operand:V4SI 3 "register_operand" "w")]
5481 UNSPEC_SHA256SU1))]
5482 "TARGET_SIMD &&TARGET_CRYPTO"
5483 "sha256su1\\t%0.4s, %2.4s, %3.4s"
5484 [(set_attr "type" "crypto_sha256_slow")]
5485 )
5487 ;; pmull
5489 (define_insn "aarch64_crypto_pmulldi"
5490 [(set (match_operand:TI 0 "register_operand" "=w")
5491 (unspec:TI [(match_operand:DI 1 "register_operand" "w")
5492 (match_operand:DI 2 "register_operand" "w")]
5493 UNSPEC_PMULL))]
5494 "TARGET_SIMD && TARGET_CRYPTO"
5495 "pmull\\t%0.1q, %1.1d, %2.1d"
5496 [(set_attr "type" "neon_mul_d_long")]
5497 )
5499 (define_insn "aarch64_crypto_pmullv2di"
5500 [(set (match_operand:TI 0 "register_operand" "=w")
5501 (unspec:TI [(match_operand:V2DI 1 "register_operand" "w")
5502 (match_operand:V2DI 2 "register_operand" "w")]
5503 UNSPEC_PMULL2))]
5504 "TARGET_SIMD && TARGET_CRYPTO"
5505 "pmull2\\t%0.1q, %1.2d, %2.2d"
5506 [(set_attr "type" "neon_mul_d_long")]
5507 )