| blob | 9a6da35f6f9154c46464ca47d9c4efa9a69a6b8d |
1 ;; Machine description for AArch64 AdvSIMD architecture.
2 ;; Copyright (C) 2011-2017 Free Software Foundation, Inc.
3 ;; Contributed by ARM Ltd.
4 ;;
5 ;; This file is part of GCC.
6 ;;
7 ;; GCC is free software; you can redistribute it and/or modify it
8 ;; under the terms of the GNU General Public License as published by
9 ;; the Free Software Foundation; either version 3, or (at your option)
10 ;; any later version.
11 ;;
12 ;; GCC is distributed in the hope that it will be useful, but
13 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
14 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 ;; General Public License for more details.
16 ;;
17 ;; You should have received a copy of the GNU General Public License
18 ;; along with GCC; see the file COPYING3. If not see
19 ;; <http://www.gnu.org/licenses/>.
21 (define_expand "mov<mode>"
22 [(set (match_operand:VALL_F16 0 "nonimmediate_operand" "")
23 (match_operand:VALL_F16 1 "general_operand" ""))]
24 "TARGET_SIMD"
25 "
26 /* Force the operand into a register if it is not an
27 immediate whose use can be replaced with xzr.
28 If the mode is 16 bytes wide, then we will be doing
29 a stp in DI mode, so we check the validity of that.
30 If the mode is 8 bytes wide, then we will do doing a
31 normal str, so the check need not apply. */
32 if (GET_CODE (operands[0]) == MEM
33 && !(aarch64_simd_imm_zero (operands[1], <MODE>mode)
34 && ((GET_MODE_SIZE (<MODE>mode) == 16
35 && aarch64_mem_pair_operand (operands[0], DImode))
36 || GET_MODE_SIZE (<MODE>mode) == 8)))
37 operands[1] = force_reg (<MODE>mode, operands[1]);
38 "
39 )
41 (define_expand "movmisalign<mode>"
42 [(set (match_operand:VALL 0 "nonimmediate_operand" "")
43 (match_operand:VALL 1 "general_operand" ""))]
44 "TARGET_SIMD"
45 {
46 /* This pattern is not permitted to fail during expansion: if both arguments
47 are non-registers (e.g. memory := constant, which can be created by the
48 auto-vectorizer), force operand 1 into a register. */
49 if (!register_operand (operands[0], <MODE>mode)
50 && !register_operand (operands[1], <MODE>mode))
51 operands[1] = force_reg (<MODE>mode, operands[1]);
52 })
54 (define_insn "aarch64_simd_dup<mode>"
55 [(set (match_operand:VDQ_I 0 "register_operand" "=w, w")
56 (vec_duplicate:VDQ_I
57 (match_operand:<VEL> 1 "register_operand" "w,?r")))]
58 "TARGET_SIMD"
59 "@
60 dup\\t%0.<Vtype>, %1.<Vetype>[0]
61 dup\\t%0.<Vtype>, %<vw>1"
62 [(set_attr "type" "neon_dup<q>, neon_from_gp<q>")]
63 )
65 (define_insn "aarch64_simd_dup<mode>"
66 [(set (match_operand:VDQF_F16 0 "register_operand" "=w")
67 (vec_duplicate:VDQF_F16
68 (match_operand:<VEL> 1 "register_operand" "w")))]
69 "TARGET_SIMD"
70 "dup\\t%0.<Vtype>, %1.<Vetype>[0]"
71 [(set_attr "type" "neon_dup<q>")]
72 )
74 (define_insn "aarch64_dup_lane<mode>"
75 [(set (match_operand:VALL_F16 0 "register_operand" "=w")
76 (vec_duplicate:VALL_F16
77 (vec_select:<VEL>
78 (match_operand:VALL_F16 1 "register_operand" "w")
79 (parallel [(match_operand:SI 2 "immediate_operand" "i")])
80 )))]
81 "TARGET_SIMD"
82 {
83 operands[2] = aarch64_endian_lane_rtx (<MODE>mode, INTVAL (operands[2]));
84 return "dup\\t%0.<Vtype>, %1.<Vetype>[%2]";
85 }
86 [(set_attr "type" "neon_dup<q>")]
87 )
89 (define_insn "aarch64_dup_lane_<vswap_width_name><mode>"
90 [(set (match_operand:VALL_F16_NO_V2Q 0 "register_operand" "=w")
91 (vec_duplicate:VALL_F16_NO_V2Q
92 (vec_select:<VEL>
93 (match_operand:<VSWAP_WIDTH> 1 "register_operand" "w")
94 (parallel [(match_operand:SI 2 "immediate_operand" "i")])
95 )))]
96 "TARGET_SIMD"
97 {
98 operands[2] = aarch64_endian_lane_rtx (<VSWAP_WIDTH>mode, INTVAL (operands[2]));
99 return "dup\\t%0.<Vtype>, %1.<Vetype>[%2]";
100 }
101 [(set_attr "type" "neon_dup<q>")]
102 )
104 (define_insn "*aarch64_simd_mov<VD:mode>"
105 [(set (match_operand:VD 0 "nonimmediate_operand"
106 "=w, m, m, w, ?r, ?w, ?r, w")
107 (match_operand:VD 1 "general_operand"
108 "m, Dz, w, w, w, r, r, Dn"))]
109 "TARGET_SIMD
110 && (register_operand (operands[0], <MODE>mode)
111 || aarch64_simd_reg_or_zero (operands[1], <MODE>mode))"
112 {
113 switch (which_alternative)
114 {
115 case 0: return "ldr\t%d0, %1";
116 case 1: return "str\txzr, %0";
117 case 2: return "str\t%d1, %0";
118 case 3: return "mov\t%0.<Vbtype>, %1.<Vbtype>";
119 case 4: return "umov\t%0, %1.d[0]";
120 case 5: return "fmov\t%d0, %1";
121 case 6: return "mov\t%0, %1";
122 case 7:
123 return aarch64_output_simd_mov_immediate (operands[1],
124 <MODE>mode, 64);
125 default: gcc_unreachable ();
126 }
127 }
128 [(set_attr "type" "neon_load1_1reg<q>, store_8, neon_store1_1reg<q>,\
129 neon_logic<q>, neon_to_gp<q>, f_mcr,\
130 mov_reg, neon_move<q>")]
131 )
133 (define_insn "*aarch64_simd_mov<VQ:mode>"
134 [(set (match_operand:VQ 0 "nonimmediate_operand"
135 "=w, Umq, m, w, ?r, ?w, ?r, w")
136 (match_operand:VQ 1 "general_operand"
137 "m, Dz, w, w, w, r, r, Dn"))]
138 "TARGET_SIMD
139 && (register_operand (operands[0], <MODE>mode)
140 || aarch64_simd_reg_or_zero (operands[1], <MODE>mode))"
141 {
142 switch (which_alternative)
143 {
144 case 0:
145 return "ldr\t%q0, %1";
146 case 1:
147 return "stp\txzr, xzr, %0";
148 case 2:
149 return "str\t%q1, %0";
150 case 3:
151 return "mov\t%0.<Vbtype>, %1.<Vbtype>";
152 case 4:
153 case 5:
154 case 6:
155 return "#";
156 case 7:
157 return aarch64_output_simd_mov_immediate (operands[1], <MODE>mode, 128);
158 default:
159 gcc_unreachable ();
160 }
161 }
162 [(set_attr "type" "neon_load1_1reg<q>, store_16, neon_store1_1reg<q>,\
163 neon_logic<q>, multiple, multiple,\
164 multiple, neon_move<q>")
165 (set_attr "length" "4,4,4,4,8,8,8,4")]
166 )
168 ;; When storing lane zero we can use the normal STR and its more permissive
169 ;; addressing modes.
171 (define_insn "aarch64_store_lane0<mode>"
172 [(set (match_operand:<VEL> 0 "memory_operand" "=m")
173 (vec_select:<VEL> (match_operand:VALL_F16 1 "register_operand" "w")
174 (parallel [(match_operand 2 "const_int_operand" "n")])))]
175 "TARGET_SIMD
176 && ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])) == 0"
177 "str\\t%<Vetype>1, %0"
178 [(set_attr "type" "neon_store1_1reg<q>")]
179 )
181 (define_insn "load_pair<mode>"
182 [(set (match_operand:VD 0 "register_operand" "=w")
183 (match_operand:VD 1 "aarch64_mem_pair_operand" "Ump"))
184 (set (match_operand:VD 2 "register_operand" "=w")
185 (match_operand:VD 3 "memory_operand" "m"))]
186 "TARGET_SIMD
187 && rtx_equal_p (XEXP (operands[3], 0),
188 plus_constant (Pmode,
189 XEXP (operands[1], 0),
190 GET_MODE_SIZE (<MODE>mode)))"
191 "ldp\\t%d0, %d2, %1"
192 [(set_attr "type" "neon_ldp")]
193 )
195 (define_insn "store_pair<mode>"
196 [(set (match_operand:VD 0 "aarch64_mem_pair_operand" "=Ump")
197 (match_operand:VD 1 "register_operand" "w"))
198 (set (match_operand:VD 2 "memory_operand" "=m")
199 (match_operand:VD 3 "register_operand" "w"))]
200 "TARGET_SIMD
201 && rtx_equal_p (XEXP (operands[2], 0),
202 plus_constant (Pmode,
203 XEXP (operands[0], 0),
204 GET_MODE_SIZE (<MODE>mode)))"
205 "stp\\t%d1, %d3, %0"
206 [(set_attr "type" "neon_stp")]
207 )
209 (define_split
210 [(set (match_operand:VQ 0 "register_operand" "")
211 (match_operand:VQ 1 "register_operand" ""))]
212 "TARGET_SIMD && reload_completed
213 && GP_REGNUM_P (REGNO (operands[0]))
214 && GP_REGNUM_P (REGNO (operands[1]))"
215 [(const_int 0)]
216 {
217 aarch64_simd_emit_reg_reg_move (operands, DImode, 2);
218 DONE;
219 })
221 (define_split
222 [(set (match_operand:VQ 0 "register_operand" "")
223 (match_operand:VQ 1 "register_operand" ""))]
224 "TARGET_SIMD && reload_completed
225 && ((FP_REGNUM_P (REGNO (operands[0])) && GP_REGNUM_P (REGNO (operands[1])))
226 || (GP_REGNUM_P (REGNO (operands[0])) && FP_REGNUM_P (REGNO (operands[1]))))"
227 [(const_int 0)]
228 {
229 aarch64_split_simd_move (operands[0], operands[1]);
230 DONE;
231 })
233 (define_expand "aarch64_split_simd_mov<mode>"
234 [(set (match_operand:VQ 0)
235 (match_operand:VQ 1))]
236 "TARGET_SIMD"
237 {
238 rtx dst = operands[0];
239 rtx src = operands[1];
241 if (GP_REGNUM_P (REGNO (src)))
242 {
243 rtx src_low_part = gen_lowpart (<VHALF>mode, src);
244 rtx src_high_part = gen_highpart (<VHALF>mode, src);
246 emit_insn
247 (gen_move_lo_quad_<mode> (dst, src_low_part));
248 emit_insn
249 (gen_move_hi_quad_<mode> (dst, src_high_part));
250 }
252 else
253 {
254 rtx dst_low_part = gen_lowpart (<VHALF>mode, dst);
255 rtx dst_high_part = gen_highpart (<VHALF>mode, dst);
256 rtx lo = aarch64_simd_vect_par_cnst_half (<MODE>mode, <nunits>, false);
257 rtx hi = aarch64_simd_vect_par_cnst_half (<MODE>mode, <nunits>, true);
259 emit_insn
260 (gen_aarch64_simd_mov_from_<mode>low (dst_low_part, src, lo));
261 emit_insn
262 (gen_aarch64_simd_mov_from_<mode>high (dst_high_part, src, hi));
263 }
264 DONE;
265 }
266 )
268 (define_insn "aarch64_simd_mov_from_<mode>low"
269 [(set (match_operand:<VHALF> 0 "register_operand" "=r")
270 (vec_select:<VHALF>
271 (match_operand:VQ 1 "register_operand" "w")
272 (match_operand:VQ 2 "vect_par_cnst_lo_half" "")))]
273 "TARGET_SIMD && reload_completed"
274 "umov\t%0, %1.d[0]"
275 [(set_attr "type" "neon_to_gp<q>")
276 (set_attr "length" "4")
277 ])
279 (define_insn "aarch64_simd_mov_from_<mode>high"
280 [(set (match_operand:<VHALF> 0 "register_operand" "=r")
281 (vec_select:<VHALF>
282 (match_operand:VQ 1 "register_operand" "w")
283 (match_operand:VQ 2 "vect_par_cnst_hi_half" "")))]
284 "TARGET_SIMD && reload_completed"
285 "umov\t%0, %1.d[1]"
286 [(set_attr "type" "neon_to_gp<q>")
287 (set_attr "length" "4")
288 ])
290 (define_insn "orn<mode>3"
291 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
292 (ior:VDQ_I (not:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w"))
293 (match_operand:VDQ_I 2 "register_operand" "w")))]
294 "TARGET_SIMD"
295 "orn\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>"
296 [(set_attr "type" "neon_logic<q>")]
297 )
299 (define_insn "bic<mode>3"
300 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
301 (and:VDQ_I (not:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w"))
302 (match_operand:VDQ_I 2 "register_operand" "w")))]
303 "TARGET_SIMD"
304 "bic\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>"
305 [(set_attr "type" "neon_logic<q>")]
306 )
308 (define_insn "add<mode>3"
309 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
310 (plus:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
311 (match_operand:VDQ_I 2 "register_operand" "w")))]
312 "TARGET_SIMD"
313 "add\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
314 [(set_attr "type" "neon_add<q>")]
315 )
317 (define_insn "sub<mode>3"
318 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
319 (minus:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
320 (match_operand:VDQ_I 2 "register_operand" "w")))]
321 "TARGET_SIMD"
322 "sub\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
323 [(set_attr "type" "neon_sub<q>")]
324 )
326 (define_insn "mul<mode>3"
327 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
328 (mult:VDQ_BHSI (match_operand:VDQ_BHSI 1 "register_operand" "w")
329 (match_operand:VDQ_BHSI 2 "register_operand" "w")))]
330 "TARGET_SIMD"
331 "mul\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
332 [(set_attr "type" "neon_mul_<Vetype><q>")]
333 )
335 (define_insn "bswap<mode>2"
336 [(set (match_operand:VDQHSD 0 "register_operand" "=w")
337 (bswap:VDQHSD (match_operand:VDQHSD 1 "register_operand" "w")))]
338 "TARGET_SIMD"
339 "rev<Vrevsuff>\\t%0.<Vbtype>, %1.<Vbtype>"
340 [(set_attr "type" "neon_rev<q>")]
341 )
343 (define_insn "aarch64_rbit<mode>"
344 [(set (match_operand:VB 0 "register_operand" "=w")
345 (unspec:VB [(match_operand:VB 1 "register_operand" "w")]
346 UNSPEC_RBIT))]
347 "TARGET_SIMD"
348 "rbit\\t%0.<Vbtype>, %1.<Vbtype>"
349 [(set_attr "type" "neon_rbit")]
350 )
352 (define_expand "ctz<mode>2"
353 [(set (match_operand:VS 0 "register_operand")
354 (ctz:VS (match_operand:VS 1 "register_operand")))]
355 "TARGET_SIMD"
356 {
357 emit_insn (gen_bswap<mode>2 (operands[0], operands[1]));
358 rtx op0_castsi2qi = simplify_gen_subreg(<VS:VSI2QI>mode, operands[0],
359 <MODE>mode, 0);
360 emit_insn (gen_aarch64_rbit<VS:vsi2qi> (op0_castsi2qi, op0_castsi2qi));
361 emit_insn (gen_clz<mode>2 (operands[0], operands[0]));
362 DONE;
363 }
364 )
366 (define_expand "xorsign<mode>3"
367 [(match_operand:VHSDF 0 "register_operand")
368 (match_operand:VHSDF 1 "register_operand")
369 (match_operand:VHSDF 2 "register_operand")]
370 "TARGET_SIMD"
371 {
373 machine_mode imode = <V_INT_EQUIV>mode;
374 rtx v_bitmask = gen_reg_rtx (imode);
375 rtx op1x = gen_reg_rtx (imode);
376 rtx op2x = gen_reg_rtx (imode);
378 rtx arg1 = lowpart_subreg (imode, operands[1], <MODE>mode);
379 rtx arg2 = lowpart_subreg (imode, operands[2], <MODE>mode);
381 int bits = GET_MODE_UNIT_BITSIZE (<MODE>mode) - 1;
383 emit_move_insn (v_bitmask,
384 aarch64_simd_gen_const_vector_dup (<V_INT_EQUIV>mode,
385 HOST_WIDE_INT_M1U << bits));
387 emit_insn (gen_and<v_int_equiv>3 (op2x, v_bitmask, arg2));
388 emit_insn (gen_xor<v_int_equiv>3 (op1x, arg1, op2x));
389 emit_move_insn (operands[0],
390 lowpart_subreg (<MODE>mode, op1x, imode));
391 DONE;
392 }
393 )
395 ;; These instructions map to the __builtins for the Dot Product operations.
396 (define_insn "aarch64_<sur>dot<vsi2qi>"
397 [(set (match_operand:VS 0 "register_operand" "=w")
398 (plus:VS (match_operand:VS 1 "register_operand" "0")
399 (unspec:VS [(match_operand:<VSI2QI> 2 "register_operand" "w")
400 (match_operand:<VSI2QI> 3 "register_operand" "w")]
401 DOTPROD)))]
402 "TARGET_DOTPROD"
403 "<sur>dot\\t%0.<Vtype>, %2.<Vdottype>, %3.<Vdottype>"
404 [(set_attr "type" "neon_dot")]
405 )
407 ;; These expands map to the Dot Product optab the vectorizer checks for.
408 ;; The auto-vectorizer expects a dot product builtin that also does an
409 ;; accumulation into the provided register.
410 ;; Given the following pattern
411 ;;
412 ;; for (i=0; i<len; i++) {
413 ;; c = a[i] * b[i];
414 ;; r += c;
415 ;; }
416 ;; return result;
417 ;;
418 ;; This can be auto-vectorized to
419 ;; r = a[0]*b[0] + a[1]*b[1] + a[2]*b[2] + a[3]*b[3];
420 ;;
421 ;; given enough iterations. However the vectorizer can keep unrolling the loop
422 ;; r += a[4]*b[4] + a[5]*b[5] + a[6]*b[6] + a[7]*b[7];
423 ;; r += a[8]*b[8] + a[9]*b[9] + a[10]*b[10] + a[11]*b[11];
424 ;; ...
425 ;;
426 ;; and so the vectorizer provides r, in which the result has to be accumulated.
427 (define_expand "<sur>dot_prod<vsi2qi>"
428 [(set (match_operand:VS 0 "register_operand")
429 (plus:VS (unspec:VS [(match_operand:<VSI2QI> 1 "register_operand")
430 (match_operand:<VSI2QI> 2 "register_operand")]
431 DOTPROD)
432 (match_operand:VS 3 "register_operand")))]
433 "TARGET_DOTPROD"
434 {
435 emit_insn (
436 gen_aarch64_<sur>dot<vsi2qi> (operands[3], operands[3], operands[1],
437 operands[2]));
438 emit_insn (gen_rtx_SET (operands[0], operands[3]));
439 DONE;
440 })
442 ;; These instructions map to the __builtins for the Dot Product
443 ;; indexed operations.
444 (define_insn "aarch64_<sur>dot_lane<vsi2qi>"
445 [(set (match_operand:VS 0 "register_operand" "=w")
446 (plus:VS (match_operand:VS 1 "register_operand" "0")
447 (unspec:VS [(match_operand:<VSI2QI> 2 "register_operand" "w")
448 (match_operand:V8QI 3 "register_operand" "<h_con>")
449 (match_operand:SI 4 "immediate_operand" "i")]
450 DOTPROD)))]
451 "TARGET_DOTPROD"
452 {
453 operands[4]
454 = GEN_INT (ENDIAN_LANE_N (V8QImode, INTVAL (operands[4])));
455 return "<sur>dot\\t%0.<Vtype>, %2.<Vdottype>, %3.4b[%4]";
456 }
457 [(set_attr "type" "neon_dot")]
458 )
460 (define_insn "aarch64_<sur>dot_laneq<vsi2qi>"
461 [(set (match_operand:VS 0 "register_operand" "=w")
462 (plus:VS (match_operand:VS 1 "register_operand" "0")
463 (unspec:VS [(match_operand:<VSI2QI> 2 "register_operand" "w")
464 (match_operand:V16QI 3 "register_operand" "<h_con>")
465 (match_operand:SI 4 "immediate_operand" "i")]
466 DOTPROD)))]
467 "TARGET_DOTPROD"
468 {
469 operands[4]
470 = GEN_INT (ENDIAN_LANE_N (V16QImode, INTVAL (operands[4])));
471 return "<sur>dot\\t%0.<Vtype>, %2.<Vdottype>, %3.4b[%4]";
472 }
473 [(set_attr "type" "neon_dot")]
474 )
476 (define_expand "copysign<mode>3"
477 [(match_operand:VHSDF 0 "register_operand")
478 (match_operand:VHSDF 1 "register_operand")
479 (match_operand:VHSDF 2 "register_operand")]
480 "TARGET_FLOAT && TARGET_SIMD"
481 {
482 rtx v_bitmask = gen_reg_rtx (<V_INT_EQUIV>mode);
483 int bits = GET_MODE_UNIT_BITSIZE (<MODE>mode) - 1;
485 emit_move_insn (v_bitmask,
486 aarch64_simd_gen_const_vector_dup (<V_INT_EQUIV>mode,
487 HOST_WIDE_INT_M1U << bits));
488 emit_insn (gen_aarch64_simd_bsl<mode> (operands[0], v_bitmask,
489 operands[2], operands[1]));
490 DONE;
491 }
492 )
494 (define_insn "*aarch64_mul3_elt<mode>"
495 [(set (match_operand:VMUL 0 "register_operand" "=w")
496 (mult:VMUL
497 (vec_duplicate:VMUL
498 (vec_select:<VEL>
499 (match_operand:VMUL 1 "register_operand" "<h_con>")
500 (parallel [(match_operand:SI 2 "immediate_operand")])))
501 (match_operand:VMUL 3 "register_operand" "w")))]
502 "TARGET_SIMD"
503 {
504 operands[2] = aarch64_endian_lane_rtx (<MODE>mode, INTVAL (operands[2]));
505 return "<f>mul\\t%0.<Vtype>, %3.<Vtype>, %1.<Vetype>[%2]";
506 }
507 [(set_attr "type" "neon<fp>_mul_<stype>_scalar<q>")]
508 )
510 (define_insn "*aarch64_mul3_elt_<vswap_width_name><mode>"
511 [(set (match_operand:VMUL_CHANGE_NLANES 0 "register_operand" "=w")
512 (mult:VMUL_CHANGE_NLANES
513 (vec_duplicate:VMUL_CHANGE_NLANES
514 (vec_select:<VEL>
515 (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
516 (parallel [(match_operand:SI 2 "immediate_operand")])))
517 (match_operand:VMUL_CHANGE_NLANES 3 "register_operand" "w")))]
518 "TARGET_SIMD"
519 {
520 operands[2] = aarch64_endian_lane_rtx (<VSWAP_WIDTH>mode, INTVAL (operands[2]));
521 return "<f>mul\\t%0.<Vtype>, %3.<Vtype>, %1.<Vetype>[%2]";
522 }
523 [(set_attr "type" "neon<fp>_mul_<Vetype>_scalar<q>")]
524 )
526 (define_insn "*aarch64_mul3_elt_from_dup<mode>"
527 [(set (match_operand:VMUL 0 "register_operand" "=w")
528 (mult:VMUL
529 (vec_duplicate:VMUL
530 (match_operand:<VEL> 1 "register_operand" "<h_con>"))
531 (match_operand:VMUL 2 "register_operand" "w")))]
532 "TARGET_SIMD"
533 "<f>mul\t%0.<Vtype>, %2.<Vtype>, %1.<Vetype>[0]";
534 [(set_attr "type" "neon<fp>_mul_<stype>_scalar<q>")]
535 )
537 (define_insn "aarch64_rsqrte<mode>"
538 [(set (match_operand:VHSDF_HSDF 0 "register_operand" "=w")
539 (unspec:VHSDF_HSDF [(match_operand:VHSDF_HSDF 1 "register_operand" "w")]
540 UNSPEC_RSQRTE))]
541 "TARGET_SIMD"
542 "frsqrte\\t%<v>0<Vmtype>, %<v>1<Vmtype>"
543 [(set_attr "type" "neon_fp_rsqrte_<stype><q>")])
545 (define_insn "aarch64_rsqrts<mode>"
546 [(set (match_operand:VHSDF_HSDF 0 "register_operand" "=w")
547 (unspec:VHSDF_HSDF [(match_operand:VHSDF_HSDF 1 "register_operand" "w")
548 (match_operand:VHSDF_HSDF 2 "register_operand" "w")]
549 UNSPEC_RSQRTS))]
550 "TARGET_SIMD"
551 "frsqrts\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
552 [(set_attr "type" "neon_fp_rsqrts_<stype><q>")])
554 (define_expand "rsqrt<mode>2"
555 [(set (match_operand:VALLF 0 "register_operand" "=w")
556 (unspec:VALLF [(match_operand:VALLF 1 "register_operand" "w")]
557 UNSPEC_RSQRT))]
558 "TARGET_SIMD"
559 {
560 aarch64_emit_approx_sqrt (operands[0], operands[1], true);
561 DONE;
562 })
564 (define_insn "*aarch64_mul3_elt_to_64v2df"
565 [(set (match_operand:DF 0 "register_operand" "=w")
566 (mult:DF
567 (vec_select:DF
568 (match_operand:V2DF 1 "register_operand" "w")
569 (parallel [(match_operand:SI 2 "immediate_operand")]))
570 (match_operand:DF 3 "register_operand" "w")))]
571 "TARGET_SIMD"
572 {
573 operands[2] = aarch64_endian_lane_rtx (V2DFmode, INTVAL (operands[2]));
574 return "fmul\\t%0.2d, %3.2d, %1.d[%2]";
575 }
576 [(set_attr "type" "neon_fp_mul_d_scalar_q")]
577 )
579 (define_insn "neg<mode>2"
580 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
581 (neg:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")))]
582 "TARGET_SIMD"
583 "neg\t%0.<Vtype>, %1.<Vtype>"
584 [(set_attr "type" "neon_neg<q>")]
585 )
587 (define_insn "abs<mode>2"
588 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
589 (abs:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")))]
590 "TARGET_SIMD"
591 "abs\t%0.<Vtype>, %1.<Vtype>"
592 [(set_attr "type" "neon_abs<q>")]
593 )
595 ;; The intrinsic version of integer ABS must not be allowed to
596 ;; combine with any operation with an integerated ABS step, such
597 ;; as SABD.
598 (define_insn "aarch64_abs<mode>"
599 [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
600 (unspec:VSDQ_I_DI
601 [(match_operand:VSDQ_I_DI 1 "register_operand" "w")]
602 UNSPEC_ABS))]
603 "TARGET_SIMD"
604 "abs\t%<v>0<Vmtype>, %<v>1<Vmtype>"
605 [(set_attr "type" "neon_abs<q>")]
606 )
608 (define_insn "abd<mode>_3"
609 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
610 (abs:VDQ_BHSI (minus:VDQ_BHSI
611 (match_operand:VDQ_BHSI 1 "register_operand" "w")
612 (match_operand:VDQ_BHSI 2 "register_operand" "w"))))]
613 "TARGET_SIMD"
614 "sabd\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
615 [(set_attr "type" "neon_abd<q>")]
616 )
618 (define_insn "aba<mode>_3"
619 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
620 (plus:VDQ_BHSI (abs:VDQ_BHSI (minus:VDQ_BHSI
621 (match_operand:VDQ_BHSI 1 "register_operand" "w")
622 (match_operand:VDQ_BHSI 2 "register_operand" "w")))
623 (match_operand:VDQ_BHSI 3 "register_operand" "0")))]
624 "TARGET_SIMD"
625 "saba\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
626 [(set_attr "type" "neon_arith_acc<q>")]
627 )
629 (define_insn "fabd<mode>3"
630 [(set (match_operand:VHSDF_HSDF 0 "register_operand" "=w")
631 (abs:VHSDF_HSDF
632 (minus:VHSDF_HSDF
633 (match_operand:VHSDF_HSDF 1 "register_operand" "w")
634 (match_operand:VHSDF_HSDF 2 "register_operand" "w"))))]
635 "TARGET_SIMD"
636 "fabd\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
637 [(set_attr "type" "neon_fp_abd_<stype><q>")]
638 )
640 ;; For AND (vector, register) and BIC (vector, immediate)
641 (define_insn "and<mode>3"
642 [(set (match_operand:VDQ_I 0 "register_operand" "=w,w")
643 (and:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w,0")
644 (match_operand:VDQ_I 2 "aarch64_reg_or_bic_imm" "w,Db")))]
645 "TARGET_SIMD"
646 {
647 switch (which_alternative)
648 {
649 case 0:
650 return "and\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>";
651 case 1:
652 return aarch64_output_simd_mov_immediate (operands[2],
653 <MODE>mode, GET_MODE_BITSIZE (<MODE>mode), AARCH64_CHECK_BIC);
654 default:
655 gcc_unreachable ();
656 }
657 }
658 [(set_attr "type" "neon_logic<q>")]
659 )
661 ;; For ORR (vector, register) and ORR (vector, immediate)
662 (define_insn "ior<mode>3"
663 [(set (match_operand:VDQ_I 0 "register_operand" "=w,w")
664 (ior:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w,0")
665 (match_operand:VDQ_I 2 "aarch64_reg_or_orr_imm" "w,Do")))]
666 "TARGET_SIMD"
667 {
668 switch (which_alternative)
669 {
670 case 0:
671 return "orr\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>";
672 case 1:
673 return aarch64_output_simd_mov_immediate (operands[2],
674 <MODE>mode, GET_MODE_BITSIZE (<MODE>mode), AARCH64_CHECK_ORR);
675 default:
676 gcc_unreachable ();
677 }
678 }
679 [(set_attr "type" "neon_logic<q>")]
680 )
682 (define_insn "xor<mode>3"
683 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
684 (xor:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
685 (match_operand:VDQ_I 2 "register_operand" "w")))]
686 "TARGET_SIMD"
687 "eor\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>"
688 [(set_attr "type" "neon_logic<q>")]
689 )
691 (define_insn "one_cmpl<mode>2"
692 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
693 (not:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")))]
694 "TARGET_SIMD"
695 "not\t%0.<Vbtype>, %1.<Vbtype>"
696 [(set_attr "type" "neon_logic<q>")]
697 )
699 (define_insn "aarch64_simd_vec_set<mode>"
700 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w,w,w")
701 (vec_merge:VDQ_BHSI
702 (vec_duplicate:VDQ_BHSI
703 (match_operand:<VEL> 1 "aarch64_simd_general_operand" "r,w,Utv"))
704 (match_operand:VDQ_BHSI 3 "register_operand" "0,0,0")
705 (match_operand:SI 2 "immediate_operand" "i,i,i")))]
706 "TARGET_SIMD"
707 {
708 int elt = ENDIAN_LANE_N (<nunits>, exact_log2 (INTVAL (operands[2])));
709 operands[2] = GEN_INT ((HOST_WIDE_INT) 1 << elt);
710 switch (which_alternative)
711 {
712 case 0:
713 return "ins\\t%0.<Vetype>[%p2], %w1";
714 case 1:
715 return "ins\\t%0.<Vetype>[%p2], %1.<Vetype>[0]";
716 case 2:
717 return "ld1\\t{%0.<Vetype>}[%p2], %1";
718 default:
719 gcc_unreachable ();
720 }
721 }
722 [(set_attr "type" "neon_from_gp<q>, neon_ins<q>, neon_load1_one_lane<q>")]
723 )
725 (define_insn "*aarch64_simd_vec_copy_lane<mode>"
726 [(set (match_operand:VALL_F16 0 "register_operand" "=w")
727 (vec_merge:VALL_F16
728 (vec_duplicate:VALL_F16
729 (vec_select:<VEL>
730 (match_operand:VALL_F16 3 "register_operand" "w")
731 (parallel
732 [(match_operand:SI 4 "immediate_operand" "i")])))
733 (match_operand:VALL_F16 1 "register_operand" "0")
734 (match_operand:SI 2 "immediate_operand" "i")))]
735 "TARGET_SIMD"
736 {
737 int elt = ENDIAN_LANE_N (<MODE>mode, exact_log2 (INTVAL (operands[2])));
738 operands[2] = GEN_INT (HOST_WIDE_INT_1 << elt);
739 operands[4] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[4])));
741 return "ins\t%0.<Vetype>[%p2], %3.<Vetype>[%4]";
742 }
743 [(set_attr "type" "neon_ins<q>")]
744 )
746 (define_insn "*aarch64_simd_vec_copy_lane_<vswap_width_name><mode>"
747 [(set (match_operand:VALL_F16_NO_V2Q 0 "register_operand" "=w")
748 (vec_merge:VALL_F16_NO_V2Q
749 (vec_duplicate:VALL_F16_NO_V2Q
750 (vec_select:<VEL>
751 (match_operand:<VSWAP_WIDTH> 3 "register_operand" "w")
752 (parallel
753 [(match_operand:SI 4 "immediate_operand" "i")])))
754 (match_operand:VALL_F16_NO_V2Q 1 "register_operand" "0")
755 (match_operand:SI 2 "immediate_operand" "i")))]
756 "TARGET_SIMD"
757 {
758 int elt = ENDIAN_LANE_N (<MODE>mode, exact_log2 (INTVAL (operands[2])));
759 operands[2] = GEN_INT (HOST_WIDE_INT_1 << elt);
760 operands[4] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
761 INTVAL (operands[4])));
763 return "ins\t%0.<Vetype>[%p2], %3.<Vetype>[%4]";
764 }
765 [(set_attr "type" "neon_ins<q>")]
766 )
768 (define_insn "aarch64_simd_lshr<mode>"
769 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
770 (lshiftrt:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
771 (match_operand:VDQ_I 2 "aarch64_simd_rshift_imm" "Dr")))]
772 "TARGET_SIMD"
773 "ushr\t%0.<Vtype>, %1.<Vtype>, %2"
774 [(set_attr "type" "neon_shift_imm<q>")]
775 )
777 (define_insn "aarch64_simd_ashr<mode>"
778 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
779 (ashiftrt:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
780 (match_operand:VDQ_I 2 "aarch64_simd_rshift_imm" "Dr")))]
781 "TARGET_SIMD"
782 "sshr\t%0.<Vtype>, %1.<Vtype>, %2"
783 [(set_attr "type" "neon_shift_imm<q>")]
784 )
786 (define_insn "aarch64_simd_imm_shl<mode>"
787 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
788 (ashift:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
789 (match_operand:VDQ_I 2 "aarch64_simd_lshift_imm" "Dl")))]
790 "TARGET_SIMD"
791 "shl\t%0.<Vtype>, %1.<Vtype>, %2"
792 [(set_attr "type" "neon_shift_imm<q>")]
793 )
795 (define_insn "aarch64_simd_reg_sshl<mode>"
796 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
797 (ashift:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
798 (match_operand:VDQ_I 2 "register_operand" "w")))]
799 "TARGET_SIMD"
800 "sshl\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
801 [(set_attr "type" "neon_shift_reg<q>")]
802 )
804 (define_insn "aarch64_simd_reg_shl<mode>_unsigned"
805 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
806 (unspec:VDQ_I [(match_operand:VDQ_I 1 "register_operand" "w")
807 (match_operand:VDQ_I 2 "register_operand" "w")]
808 UNSPEC_ASHIFT_UNSIGNED))]
809 "TARGET_SIMD"
810 "ushl\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
811 [(set_attr "type" "neon_shift_reg<q>")]
812 )
814 (define_insn "aarch64_simd_reg_shl<mode>_signed"
815 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
816 (unspec:VDQ_I [(match_operand:VDQ_I 1 "register_operand" "w")
817 (match_operand:VDQ_I 2 "register_operand" "w")]
818 UNSPEC_ASHIFT_SIGNED))]
819 "TARGET_SIMD"
820 "sshl\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
821 [(set_attr "type" "neon_shift_reg<q>")]
822 )
824 (define_expand "ashl<mode>3"
825 [(match_operand:VDQ_I 0 "register_operand" "")
826 (match_operand:VDQ_I 1 "register_operand" "")
827 (match_operand:SI 2 "general_operand" "")]
828 "TARGET_SIMD"
829 {
830 int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
831 int shift_amount;
833 if (CONST_INT_P (operands[2]))
834 {
835 shift_amount = INTVAL (operands[2]);
836 if (shift_amount >= 0 && shift_amount < bit_width)
837 {
838 rtx tmp = aarch64_simd_gen_const_vector_dup (<MODE>mode,
839 shift_amount);
840 emit_insn (gen_aarch64_simd_imm_shl<mode> (operands[0],
841 operands[1],
842 tmp));
843 DONE;
844 }
845 else
846 {
847 operands[2] = force_reg (SImode, operands[2]);
848 }
849 }
850 else if (MEM_P (operands[2]))
851 {
852 operands[2] = force_reg (SImode, operands[2]);
853 }
855 if (REG_P (operands[2]))
856 {
857 rtx tmp = gen_reg_rtx (<MODE>mode);
858 emit_insn (gen_aarch64_simd_dup<mode> (tmp,
859 convert_to_mode (<VEL>mode,
860 operands[2],
861 0)));
862 emit_insn (gen_aarch64_simd_reg_sshl<mode> (operands[0], operands[1],
863 tmp));
864 DONE;
865 }
866 else
867 FAIL;
868 }
869 )
871 (define_expand "lshr<mode>3"
872 [(match_operand:VDQ_I 0 "register_operand" "")
873 (match_operand:VDQ_I 1 "register_operand" "")
874 (match_operand:SI 2 "general_operand" "")]
875 "TARGET_SIMD"
876 {
877 int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
878 int shift_amount;
880 if (CONST_INT_P (operands[2]))
881 {
882 shift_amount = INTVAL (operands[2]);
883 if (shift_amount > 0 && shift_amount <= bit_width)
884 {
885 rtx tmp = aarch64_simd_gen_const_vector_dup (<MODE>mode,
886 shift_amount);
887 emit_insn (gen_aarch64_simd_lshr<mode> (operands[0],
888 operands[1],
889 tmp));
890 DONE;
891 }
892 else
893 operands[2] = force_reg (SImode, operands[2]);
894 }
895 else if (MEM_P (operands[2]))
896 {
897 operands[2] = force_reg (SImode, operands[2]);
898 }
900 if (REG_P (operands[2]))
901 {
902 rtx tmp = gen_reg_rtx (SImode);
903 rtx tmp1 = gen_reg_rtx (<MODE>mode);
904 emit_insn (gen_negsi2 (tmp, operands[2]));
905 emit_insn (gen_aarch64_simd_dup<mode> (tmp1,
906 convert_to_mode (<VEL>mode,
907 tmp, 0)));
908 emit_insn (gen_aarch64_simd_reg_shl<mode>_unsigned (operands[0],
909 operands[1],
910 tmp1));
911 DONE;
912 }
913 else
914 FAIL;
915 }
916 )
918 (define_expand "ashr<mode>3"
919 [(match_operand:VDQ_I 0 "register_operand" "")
920 (match_operand:VDQ_I 1 "register_operand" "")
921 (match_operand:SI 2 "general_operand" "")]
922 "TARGET_SIMD"
923 {
924 int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
925 int shift_amount;
927 if (CONST_INT_P (operands[2]))
928 {
929 shift_amount = INTVAL (operands[2]);
930 if (shift_amount > 0 && shift_amount <= bit_width)
931 {
932 rtx tmp = aarch64_simd_gen_const_vector_dup (<MODE>mode,
933 shift_amount);
934 emit_insn (gen_aarch64_simd_ashr<mode> (operands[0],
935 operands[1],
936 tmp));
937 DONE;
938 }
939 else
940 operands[2] = force_reg (SImode, operands[2]);
941 }
942 else if (MEM_P (operands[2]))
943 {
944 operands[2] = force_reg (SImode, operands[2]);
945 }
947 if (REG_P (operands[2]))
948 {
949 rtx tmp = gen_reg_rtx (SImode);
950 rtx tmp1 = gen_reg_rtx (<MODE>mode);
951 emit_insn (gen_negsi2 (tmp, operands[2]));
952 emit_insn (gen_aarch64_simd_dup<mode> (tmp1,
953 convert_to_mode (<VEL>mode,
954 tmp, 0)));
955 emit_insn (gen_aarch64_simd_reg_shl<mode>_signed (operands[0],
956 operands[1],
957 tmp1));
958 DONE;
959 }
960 else
961 FAIL;
962 }
963 )
965 (define_expand "vashl<mode>3"
966 [(match_operand:VDQ_I 0 "register_operand" "")
967 (match_operand:VDQ_I 1 "register_operand" "")
968 (match_operand:VDQ_I 2 "register_operand" "")]
969 "TARGET_SIMD"
970 {
971 emit_insn (gen_aarch64_simd_reg_sshl<mode> (operands[0], operands[1],
972 operands[2]));
973 DONE;
974 })
976 ;; Using mode VDQ_BHSI as there is no V2DImode neg!
977 ;; Negating individual lanes most certainly offsets the
978 ;; gain from vectorization.
979 (define_expand "vashr<mode>3"
980 [(match_operand:VDQ_BHSI 0 "register_operand" "")
981 (match_operand:VDQ_BHSI 1 "register_operand" "")
982 (match_operand:VDQ_BHSI 2 "register_operand" "")]
983 "TARGET_SIMD"
984 {
985 rtx neg = gen_reg_rtx (<MODE>mode);
986 emit (gen_neg<mode>2 (neg, operands[2]));
987 emit_insn (gen_aarch64_simd_reg_shl<mode>_signed (operands[0], operands[1],
988 neg));
989 DONE;
990 })
992 ;; DI vector shift
993 (define_expand "aarch64_ashr_simddi"
994 [(match_operand:DI 0 "register_operand" "=w")
995 (match_operand:DI 1 "register_operand" "w")
996 (match_operand:SI 2 "aarch64_shift_imm64_di" "")]
997 "TARGET_SIMD"
998 {
999 /* An arithmetic shift right by 64 fills the result with copies of the sign
1000 bit, just like asr by 63 - however the standard pattern does not handle
1001 a shift by 64. */
1002 if (INTVAL (operands[2]) == 64)
1003 operands[2] = GEN_INT (63);
1004 emit_insn (gen_ashrdi3 (operands[0], operands[1], operands[2]));
1005 DONE;
1006 }
1007 )
1009 (define_expand "vlshr<mode>3"
1010 [(match_operand:VDQ_BHSI 0 "register_operand" "")
1011 (match_operand:VDQ_BHSI 1 "register_operand" "")
1012 (match_operand:VDQ_BHSI 2 "register_operand" "")]
1013 "TARGET_SIMD"
1014 {
1015 rtx neg = gen_reg_rtx (<MODE>mode);
1016 emit (gen_neg<mode>2 (neg, operands[2]));
1017 emit_insn (gen_aarch64_simd_reg_shl<mode>_unsigned (operands[0], operands[1],
1018 neg));
1019 DONE;
1020 })
1022 (define_expand "aarch64_lshr_simddi"
1023 [(match_operand:DI 0 "register_operand" "=w")
1024 (match_operand:DI 1 "register_operand" "w")
1025 (match_operand:SI 2 "aarch64_shift_imm64_di" "")]
1026 "TARGET_SIMD"
1027 {
1028 if (INTVAL (operands[2]) == 64)
1029 emit_move_insn (operands[0], const0_rtx);
1030 else
1031 emit_insn (gen_lshrdi3 (operands[0], operands[1], operands[2]));
1032 DONE;
1033 }
1034 )
1036 (define_expand "vec_set<mode>"
1037 [(match_operand:VDQ_BHSI 0 "register_operand")
1038 (match_operand:<VEL> 1 "register_operand")
1039 (match_operand:SI 2 "immediate_operand")]
1040 "TARGET_SIMD"
1041 {
1042 HOST_WIDE_INT elem = (HOST_WIDE_INT) 1 << INTVAL (operands[2]);
1043 emit_insn (gen_aarch64_simd_vec_set<mode> (operands[0], operands[1],
1044 GEN_INT (elem), operands[0]));
1045 DONE;
1046 }
1047 )
1049 ;; For 64-bit modes we use ushl/r, as this does not require a SIMD zero.
1050 (define_insn "vec_shr_<mode>"
1051 [(set (match_operand:VD 0 "register_operand" "=w")
1052 (unspec:VD [(match_operand:VD 1 "register_operand" "w")
1053 (match_operand:SI 2 "immediate_operand" "i")]
1054 UNSPEC_VEC_SHR))]
1055 "TARGET_SIMD"
1056 {
1057 if (BYTES_BIG_ENDIAN)
1058 return "shl %d0, %d1, %2";
1059 else
1060 return "ushr %d0, %d1, %2";
1061 }
1062 [(set_attr "type" "neon_shift_imm")]
1063 )
1065 (define_insn "aarch64_simd_vec_setv2di"
1066 [(set (match_operand:V2DI 0 "register_operand" "=w,w")
1067 (vec_merge:V2DI
1068 (vec_duplicate:V2DI
1069 (match_operand:DI 1 "register_operand" "r,w"))
1070 (match_operand:V2DI 3 "register_operand" "0,0")
1071 (match_operand:SI 2 "immediate_operand" "i,i")))]
1072 "TARGET_SIMD"
1073 {
1074 int elt = ENDIAN_LANE_N (2, exact_log2 (INTVAL (operands[2])));
1075 operands[2] = GEN_INT ((HOST_WIDE_INT) 1 << elt);
1076 switch (which_alternative)
1077 {
1078 case 0:
1079 return "ins\\t%0.d[%p2], %1";
1080 case 1:
1081 return "ins\\t%0.d[%p2], %1.d[0]";
1082 default:
1083 gcc_unreachable ();
1084 }
1085 }
1086 [(set_attr "type" "neon_from_gp, neon_ins_q")]
1087 )
1089 (define_expand "vec_setv2di"
1090 [(match_operand:V2DI 0 "register_operand")
1091 (match_operand:DI 1 "register_operand")
1092 (match_operand:SI 2 "immediate_operand")]
1093 "TARGET_SIMD"
1094 {
1095 HOST_WIDE_INT elem = (HOST_WIDE_INT) 1 << INTVAL (operands[2]);
1096 emit_insn (gen_aarch64_simd_vec_setv2di (operands[0], operands[1],
1097 GEN_INT (elem), operands[0]));
1098 DONE;
1099 }
1100 )
1102 (define_insn "aarch64_simd_vec_set<mode>"
1103 [(set (match_operand:VDQF_F16 0 "register_operand" "=w")
1104 (vec_merge:VDQF_F16
1105 (vec_duplicate:VDQF_F16
1106 (match_operand:<VEL> 1 "register_operand" "w"))
1107 (match_operand:VDQF_F16 3 "register_operand" "0")
1108 (match_operand:SI 2 "immediate_operand" "i")))]
1109 "TARGET_SIMD"
1110 {
1111 int elt = ENDIAN_LANE_N (<nunits>, exact_log2 (INTVAL (operands[2])));
1113 operands[2] = GEN_INT ((HOST_WIDE_INT)1 << elt);
1114 return "ins\t%0.<Vetype>[%p2], %1.<Vetype>[0]";
1115 }
1116 [(set_attr "type" "neon_ins<q>")]
1117 )
1119 (define_expand "vec_set<mode>"
1120 [(match_operand:VDQF_F16 0 "register_operand" "+w")
1121 (match_operand:<VEL> 1 "register_operand" "w")
1122 (match_operand:SI 2 "immediate_operand" "")]
1123 "TARGET_SIMD"
1124 {
1125 HOST_WIDE_INT elem = (HOST_WIDE_INT) 1 << INTVAL (operands[2]);
1126 emit_insn (gen_aarch64_simd_vec_set<mode> (operands[0], operands[1],
1127 GEN_INT (elem), operands[0]));
1128 DONE;
1129 }
1130 )
1133 (define_insn "aarch64_mla<mode>"
1134 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
1135 (plus:VDQ_BHSI (mult:VDQ_BHSI
1136 (match_operand:VDQ_BHSI 2 "register_operand" "w")
1137 (match_operand:VDQ_BHSI 3 "register_operand" "w"))
1138 (match_operand:VDQ_BHSI 1 "register_operand" "0")))]
1139 "TARGET_SIMD"
1140 "mla\t%0.<Vtype>, %2.<Vtype>, %3.<Vtype>"
1141 [(set_attr "type" "neon_mla_<Vetype><q>")]
1142 )
1144 (define_insn "*aarch64_mla_elt<mode>"
1145 [(set (match_operand:VDQHS 0 "register_operand" "=w")
1146 (plus:VDQHS
1147 (mult:VDQHS
1148 (vec_duplicate:VDQHS
1149 (vec_select:<VEL>
1150 (match_operand:VDQHS 1 "register_operand" "<h_con>")
1151 (parallel [(match_operand:SI 2 "immediate_operand")])))
1152 (match_operand:VDQHS 3 "register_operand" "w"))
1153 (match_operand:VDQHS 4 "register_operand" "0")))]
1154 "TARGET_SIMD"
1155 {
1156 operands[2] = aarch64_endian_lane_rtx (<MODE>mode, INTVAL (operands[2]));
1157 return "mla\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
1158 }
1159 [(set_attr "type" "neon_mla_<Vetype>_scalar<q>")]
1160 )
1162 (define_insn "*aarch64_mla_elt_<vswap_width_name><mode>"
1163 [(set (match_operand:VDQHS 0 "register_operand" "=w")
1164 (plus:VDQHS
1165 (mult:VDQHS
1166 (vec_duplicate:VDQHS
1167 (vec_select:<VEL>
1168 (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
1169 (parallel [(match_operand:SI 2 "immediate_operand")])))
1170 (match_operand:VDQHS 3 "register_operand" "w"))
1171 (match_operand:VDQHS 4 "register_operand" "0")))]
1172 "TARGET_SIMD"
1173 {
1174 operands[2] = aarch64_endian_lane_rtx (<VSWAP_WIDTH>mode, INTVAL (operands[2]));
1175 return "mla\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
1176 }
1177 [(set_attr "type" "neon_mla_<Vetype>_scalar<q>")]
1178 )
1180 (define_insn "*aarch64_mla_elt_merge<mode>"
1181 [(set (match_operand:VDQHS 0 "register_operand" "=w")
1182 (plus:VDQHS
1183 (mult:VDQHS (vec_duplicate:VDQHS
1184 (match_operand:<VEL> 1 "register_operand" "<h_con>"))
1185 (match_operand:VDQHS 2 "register_operand" "w"))
1186 (match_operand:VDQHS 3 "register_operand" "0")))]
1187 "TARGET_SIMD"
1188 "mla\t%0.<Vtype>, %2.<Vtype>, %1.<Vetype>[0]"
1189 [(set_attr "type" "neon_mla_<Vetype>_scalar<q>")]
1190 )
1192 (define_insn "aarch64_mls<mode>"
1193 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
1194 (minus:VDQ_BHSI (match_operand:VDQ_BHSI 1 "register_operand" "0")
1195 (mult:VDQ_BHSI (match_operand:VDQ_BHSI 2 "register_operand" "w")
1196 (match_operand:VDQ_BHSI 3 "register_operand" "w"))))]
1197 "TARGET_SIMD"
1198 "mls\t%0.<Vtype>, %2.<Vtype>, %3.<Vtype>"
1199 [(set_attr "type" "neon_mla_<Vetype><q>")]
1200 )
1202 (define_insn "*aarch64_mls_elt<mode>"
1203 [(set (match_operand:VDQHS 0 "register_operand" "=w")
1204 (minus:VDQHS
1205 (match_operand:VDQHS 4 "register_operand" "0")
1206 (mult:VDQHS
1207 (vec_duplicate:VDQHS
1208 (vec_select:<VEL>
1209 (match_operand:VDQHS 1 "register_operand" "<h_con>")
1210 (parallel [(match_operand:SI 2 "immediate_operand")])))
1211 (match_operand:VDQHS 3 "register_operand" "w"))))]
1212 "TARGET_SIMD"
1213 {
1214 operands[2] = aarch64_endian_lane_rtx (<MODE>mode, INTVAL (operands[2]));
1215 return "mls\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
1216 }
1217 [(set_attr "type" "neon_mla_<Vetype>_scalar<q>")]
1218 )
1220 (define_insn "*aarch64_mls_elt_<vswap_width_name><mode>"
1221 [(set (match_operand:VDQHS 0 "register_operand" "=w")
1222 (minus:VDQHS
1223 (match_operand:VDQHS 4 "register_operand" "0")
1224 (mult:VDQHS
1225 (vec_duplicate:VDQHS
1226 (vec_select:<VEL>
1227 (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
1228 (parallel [(match_operand:SI 2 "immediate_operand")])))
1229 (match_operand:VDQHS 3 "register_operand" "w"))))]
1230 "TARGET_SIMD"
1231 {
1232 operands[2] = aarch64_endian_lane_rtx (<VSWAP_WIDTH>mode, INTVAL (operands[2]));
1233 return "mls\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
1234 }
1235 [(set_attr "type" "neon_mla_<Vetype>_scalar<q>")]
1236 )
1238 (define_insn "*aarch64_mls_elt_merge<mode>"
1239 [(set (match_operand:VDQHS 0 "register_operand" "=w")
1240 (minus:VDQHS
1241 (match_operand:VDQHS 1 "register_operand" "0")
1242 (mult:VDQHS (vec_duplicate:VDQHS
1243 (match_operand:<VEL> 2 "register_operand" "<h_con>"))
1244 (match_operand:VDQHS 3 "register_operand" "w"))))]
1245 "TARGET_SIMD"
1246 "mls\t%0.<Vtype>, %3.<Vtype>, %2.<Vetype>[0]"
1247 [(set_attr "type" "neon_mla_<Vetype>_scalar<q>")]
1248 )
1250 ;; Max/Min operations.
1251 (define_insn "<su><maxmin><mode>3"
1252 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
1253 (MAXMIN:VDQ_BHSI (match_operand:VDQ_BHSI 1 "register_operand" "w")
1254 (match_operand:VDQ_BHSI 2 "register_operand" "w")))]
1255 "TARGET_SIMD"
1256 "<su><maxmin>\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1257 [(set_attr "type" "neon_minmax<q>")]
1258 )
1260 (define_expand "<su><maxmin>v2di3"
1261 [(set (match_operand:V2DI 0 "register_operand" "")
1262 (MAXMIN:V2DI (match_operand:V2DI 1 "register_operand" "")
1263 (match_operand:V2DI 2 "register_operand" "")))]
1264 "TARGET_SIMD"
1265 {
1266 enum rtx_code cmp_operator;
1267 rtx cmp_fmt;
1269 switch (<CODE>)
1270 {
1271 case UMIN:
1272 cmp_operator = LTU;
1273 break;
1274 case SMIN:
1275 cmp_operator = LT;
1276 break;
1277 case UMAX:
1278 cmp_operator = GTU;
1279 break;
1280 case SMAX:
1281 cmp_operator = GT;
1282 break;
1283 default:
1284 gcc_unreachable ();
1285 }
1287 cmp_fmt = gen_rtx_fmt_ee (cmp_operator, V2DImode, operands[1], operands[2]);
1288 emit_insn (gen_vcondv2div2di (operands[0], operands[1],
1289 operands[2], cmp_fmt, operands[1], operands[2]));
1290 DONE;
1291 })
1293 ;; Pairwise Integer Max/Min operations.
1294 (define_insn "aarch64_<maxmin_uns>p<mode>"
1295 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
1296 (unspec:VDQ_BHSI [(match_operand:VDQ_BHSI 1 "register_operand" "w")
1297 (match_operand:VDQ_BHSI 2 "register_operand" "w")]
1298 MAXMINV))]
1299 "TARGET_SIMD"
1300 "<maxmin_uns_op>p\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1301 [(set_attr "type" "neon_minmax<q>")]
1302 )
1304 ;; Pairwise FP Max/Min operations.
1305 (define_insn "aarch64_<maxmin_uns>p<mode>"
1306 [(set (match_operand:VHSDF 0 "register_operand" "=w")
1307 (unspec:VHSDF [(match_operand:VHSDF 1 "register_operand" "w")
1308 (match_operand:VHSDF 2 "register_operand" "w")]
1309 FMAXMINV))]
1310 "TARGET_SIMD"
1311 "<maxmin_uns_op>p\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1312 [(set_attr "type" "neon_minmax<q>")]
1313 )
1315 ;; vec_concat gives a new vector with the low elements from operand 1, and
1316 ;; the high elements from operand 2. That is to say, given op1 = { a, b }
1317 ;; op2 = { c, d }, vec_concat (op1, op2) = { a, b, c, d }.
1318 ;; What that means, is that the RTL descriptions of the below patterns
1319 ;; need to change depending on endianness.
1321 ;; Move to the low architectural bits of the register.
1322 ;; On little-endian this is { operand, zeroes }
1323 ;; On big-endian this is { zeroes, operand }
1325 (define_insn "move_lo_quad_internal_<mode>"
1326 [(set (match_operand:VQ_NO2E 0 "register_operand" "=w,w,w")
1327 (vec_concat:VQ_NO2E
1328 (match_operand:<VHALF> 1 "register_operand" "w,r,r")
1329 (vec_duplicate:<VHALF> (const_int 0))))]
1330 "TARGET_SIMD && !BYTES_BIG_ENDIAN"
1331 "@
1332 dup\\t%d0, %1.d[0]
1333 fmov\\t%d0, %1
1334 dup\\t%d0, %1"
1335 [(set_attr "type" "neon_dup<q>,f_mcr,neon_dup<q>")
1336 (set_attr "simd" "yes,*,yes")
1337 (set_attr "fp" "*,yes,*")
1338 (set_attr "length" "4")]
1339 )
1341 (define_insn "move_lo_quad_internal_<mode>"
1342 [(set (match_operand:VQ_2E 0 "register_operand" "=w,w,w")
1343 (vec_concat:VQ_2E
1344 (match_operand:<VHALF> 1 "register_operand" "w,r,r")
1345 (const_int 0)))]
1346 "TARGET_SIMD && !BYTES_BIG_ENDIAN"
1347 "@
1348 dup\\t%d0, %1.d[0]
1349 fmov\\t%d0, %1
1350 dup\\t%d0, %1"
1351 [(set_attr "type" "neon_dup<q>,f_mcr,neon_dup<q>")
1352 (set_attr "simd" "yes,*,yes")
1353 (set_attr "fp" "*,yes,*")
1354 (set_attr "length" "4")]
1355 )
1357 (define_insn "move_lo_quad_internal_be_<mode>"
1358 [(set (match_operand:VQ_NO2E 0 "register_operand" "=w,w,w")
1359 (vec_concat:VQ_NO2E
1360 (vec_duplicate:<VHALF> (const_int 0))
1361 (match_operand:<VHALF> 1 "register_operand" "w,r,r")))]
1362 "TARGET_SIMD && BYTES_BIG_ENDIAN"
1363 "@
1364 dup\\t%d0, %1.d[0]
1365 fmov\\t%d0, %1
1366 dup\\t%d0, %1"
1367 [(set_attr "type" "neon_dup<q>,f_mcr,neon_dup<q>")
1368 (set_attr "simd" "yes,*,yes")
1369 (set_attr "fp" "*,yes,*")
1370 (set_attr "length" "4")]
1371 )
1373 (define_insn "move_lo_quad_internal_be_<mode>"
1374 [(set (match_operand:VQ_2E 0 "register_operand" "=w,w,w")
1375 (vec_concat:VQ_2E
1376 (const_int 0)
1377 (match_operand:<VHALF> 1 "register_operand" "w,r,r")))]
1378 "TARGET_SIMD && BYTES_BIG_ENDIAN"
1379 "@
1380 dup\\t%d0, %1.d[0]
1381 fmov\\t%d0, %1
1382 dup\\t%d0, %1"
1383 [(set_attr "type" "neon_dup<q>,f_mcr,neon_dup<q>")
1384 (set_attr "simd" "yes,*,yes")
1385 (set_attr "fp" "*,yes,*")
1386 (set_attr "length" "4")]
1387 )
1389 (define_expand "move_lo_quad_<mode>"
1390 [(match_operand:VQ 0 "register_operand")
1391 (match_operand:VQ 1 "register_operand")]
1392 "TARGET_SIMD"
1393 {
1394 if (BYTES_BIG_ENDIAN)
1395 emit_insn (gen_move_lo_quad_internal_be_<mode> (operands[0], operands[1]));
1396 else
1397 emit_insn (gen_move_lo_quad_internal_<mode> (operands[0], operands[1]));
1398 DONE;
1399 }
1400 )
1402 ;; Move operand1 to the high architectural bits of the register, keeping
1403 ;; the low architectural bits of operand2.
1404 ;; For little-endian this is { operand2, operand1 }
1405 ;; For big-endian this is { operand1, operand2 }
1407 (define_insn "aarch64_simd_move_hi_quad_<mode>"
1408 [(set (match_operand:VQ 0 "register_operand" "+w,w")
1409 (vec_concat:VQ
1410 (vec_select:<VHALF>
1411 (match_dup 0)
1412 (match_operand:VQ 2 "vect_par_cnst_lo_half" ""))
1413 (match_operand:<VHALF> 1 "register_operand" "w,r")))]
1414 "TARGET_SIMD && !BYTES_BIG_ENDIAN"
1415 "@
1416 ins\\t%0.d[1], %1.d[0]
1417 ins\\t%0.d[1], %1"
1418 [(set_attr "type" "neon_ins")]
1419 )
1421 (define_insn "aarch64_simd_move_hi_quad_be_<mode>"
1422 [(set (match_operand:VQ 0 "register_operand" "+w,w")
1423 (vec_concat:VQ
1424 (match_operand:<VHALF> 1 "register_operand" "w,r")
1425 (vec_select:<VHALF>
1426 (match_dup 0)
1427 (match_operand:VQ 2 "vect_par_cnst_lo_half" ""))))]
1428 "TARGET_SIMD && BYTES_BIG_ENDIAN"
1429 "@
1430 ins\\t%0.d[1], %1.d[0]
1431 ins\\t%0.d[1], %1"
1432 [(set_attr "type" "neon_ins")]
1433 )
1435 (define_expand "move_hi_quad_<mode>"
1436 [(match_operand:VQ 0 "register_operand" "")
1437 (match_operand:<VHALF> 1 "register_operand" "")]
1438 "TARGET_SIMD"
1439 {
1440 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, <nunits>, false);
1441 if (BYTES_BIG_ENDIAN)
1442 emit_insn (gen_aarch64_simd_move_hi_quad_be_<mode> (operands[0],
1443 operands[1], p));
1444 else
1445 emit_insn (gen_aarch64_simd_move_hi_quad_<mode> (operands[0],
1446 operands[1], p));
1447 DONE;
1448 })
1450 ;; Narrowing operations.
1452 ;; For doubles.
1453 (define_insn "aarch64_simd_vec_pack_trunc_<mode>"
1454 [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
1455 (truncate:<VNARROWQ> (match_operand:VQN 1 "register_operand" "w")))]
1456 "TARGET_SIMD"
1457 "xtn\\t%0.<Vntype>, %1.<Vtype>"
1458 [(set_attr "type" "neon_shift_imm_narrow_q")]
1459 )
1461 (define_expand "vec_pack_trunc_<mode>"
1462 [(match_operand:<VNARROWD> 0 "register_operand" "")
1463 (match_operand:VDN 1 "register_operand" "")
1464 (match_operand:VDN 2 "register_operand" "")]
1465 "TARGET_SIMD"
1466 {
1467 rtx tempreg = gen_reg_rtx (<VDBL>mode);
1468 int lo = BYTES_BIG_ENDIAN ? 2 : 1;
1469 int hi = BYTES_BIG_ENDIAN ? 1 : 2;
1471 emit_insn (gen_move_lo_quad_<Vdbl> (tempreg, operands[lo]));
1472 emit_insn (gen_move_hi_quad_<Vdbl> (tempreg, operands[hi]));
1473 emit_insn (gen_aarch64_simd_vec_pack_trunc_<Vdbl> (operands[0], tempreg));
1474 DONE;
1475 })
1477 ;; For quads.
1479 (define_insn "vec_pack_trunc_<mode>"
1480 [(set (match_operand:<VNARROWQ2> 0 "register_operand" "=&w")
1481 (vec_concat:<VNARROWQ2>
1482 (truncate:<VNARROWQ> (match_operand:VQN 1 "register_operand" "w"))
1483 (truncate:<VNARROWQ> (match_operand:VQN 2 "register_operand" "w"))))]
1484 "TARGET_SIMD"
1485 {
1486 if (BYTES_BIG_ENDIAN)
1487 return "xtn\\t%0.<Vntype>, %2.<Vtype>\;xtn2\\t%0.<V2ntype>, %1.<Vtype>";
1488 else
1489 return "xtn\\t%0.<Vntype>, %1.<Vtype>\;xtn2\\t%0.<V2ntype>, %2.<Vtype>";
1490 }
1491 [(set_attr "type" "multiple")
1492 (set_attr "length" "8")]
1493 )
1495 ;; Widening operations.
1497 (define_insn "aarch64_simd_vec_unpack<su>_lo_<mode>"
1498 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1499 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1500 (match_operand:VQW 1 "register_operand" "w")
1501 (match_operand:VQW 2 "vect_par_cnst_lo_half" "")
1502 )))]
1503 "TARGET_SIMD"
1504 "<su>shll\t%0.<Vwtype>, %1.<Vhalftype>, 0"
1505 [(set_attr "type" "neon_shift_imm_long")]
1506 )
1508 (define_insn "aarch64_simd_vec_unpack<su>_hi_<mode>"
1509 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1510 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1511 (match_operand:VQW 1 "register_operand" "w")
1512 (match_operand:VQW 2 "vect_par_cnst_hi_half" "")
1513 )))]
1514 "TARGET_SIMD"
1515 "<su>shll2\t%0.<Vwtype>, %1.<Vtype>, 0"
1516 [(set_attr "type" "neon_shift_imm_long")]
1517 )
1519 (define_expand "vec_unpack<su>_hi_<mode>"
1520 [(match_operand:<VWIDE> 0 "register_operand" "")
1521 (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand"))]
1522 "TARGET_SIMD"
1523 {
1524 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, <nunits>, true);
1525 emit_insn (gen_aarch64_simd_vec_unpack<su>_hi_<mode> (operands[0],
1526 operands[1], p));
1527 DONE;
1528 }
1529 )
1531 (define_expand "vec_unpack<su>_lo_<mode>"
1532 [(match_operand:<VWIDE> 0 "register_operand" "")
1533 (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand" ""))]
1534 "TARGET_SIMD"
1535 {
1536 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, <nunits>, false);
1537 emit_insn (gen_aarch64_simd_vec_unpack<su>_lo_<mode> (operands[0],
1538 operands[1], p));
1539 DONE;
1540 }
1541 )
1543 ;; Widening arithmetic.
1545 (define_insn "*aarch64_<su>mlal_lo<mode>"
1546 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1547 (plus:<VWIDE>
1548 (mult:<VWIDE>
1549 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1550 (match_operand:VQW 2 "register_operand" "w")
1551 (match_operand:VQW 3 "vect_par_cnst_lo_half" "")))
1552 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1553 (match_operand:VQW 4 "register_operand" "w")
1554 (match_dup 3))))
1555 (match_operand:<VWIDE> 1 "register_operand" "0")))]
1556 "TARGET_SIMD"
1557 "<su>mlal\t%0.<Vwtype>, %2.<Vhalftype>, %4.<Vhalftype>"
1558 [(set_attr "type" "neon_mla_<Vetype>_long")]
1559 )
1561 (define_insn "*aarch64_<su>mlal_hi<mode>"
1562 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1563 (plus:<VWIDE>
1564 (mult:<VWIDE>
1565 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1566 (match_operand:VQW 2 "register_operand" "w")
1567 (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))
1568 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1569 (match_operand:VQW 4 "register_operand" "w")
1570 (match_dup 3))))
1571 (match_operand:<VWIDE> 1 "register_operand" "0")))]
1572 "TARGET_SIMD"
1573 "<su>mlal2\t%0.<Vwtype>, %2.<Vtype>, %4.<Vtype>"
1574 [(set_attr "type" "neon_mla_<Vetype>_long")]
1575 )
1577 (define_insn "*aarch64_<su>mlsl_lo<mode>"
1578 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1579 (minus:<VWIDE>
1580 (match_operand:<VWIDE> 1 "register_operand" "0")
1581 (mult:<VWIDE>
1582 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1583 (match_operand:VQW 2 "register_operand" "w")
1584 (match_operand:VQW 3 "vect_par_cnst_lo_half" "")))
1585 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1586 (match_operand:VQW 4 "register_operand" "w")
1587 (match_dup 3))))))]
1588 "TARGET_SIMD"
1589 "<su>mlsl\t%0.<Vwtype>, %2.<Vhalftype>, %4.<Vhalftype>"
1590 [(set_attr "type" "neon_mla_<Vetype>_long")]
1591 )
1593 (define_insn "*aarch64_<su>mlsl_hi<mode>"
1594 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1595 (minus:<VWIDE>
1596 (match_operand:<VWIDE> 1 "register_operand" "0")
1597 (mult:<VWIDE>
1598 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1599 (match_operand:VQW 2 "register_operand" "w")
1600 (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))
1601 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1602 (match_operand:VQW 4 "register_operand" "w")
1603 (match_dup 3))))))]
1604 "TARGET_SIMD"
1605 "<su>mlsl2\t%0.<Vwtype>, %2.<Vtype>, %4.<Vtype>"
1606 [(set_attr "type" "neon_mla_<Vetype>_long")]
1607 )
1609 (define_insn "*aarch64_<su>mlal<mode>"
1610 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1611 (plus:<VWIDE>
1612 (mult:<VWIDE>
1613 (ANY_EXTEND:<VWIDE>
1614 (match_operand:VD_BHSI 1 "register_operand" "w"))
1615 (ANY_EXTEND:<VWIDE>
1616 (match_operand:VD_BHSI 2 "register_operand" "w")))
1617 (match_operand:<VWIDE> 3 "register_operand" "0")))]
1618 "TARGET_SIMD"
1619 "<su>mlal\t%0.<Vwtype>, %1.<Vtype>, %2.<Vtype>"
1620 [(set_attr "type" "neon_mla_<Vetype>_long")]
1621 )
1623 (define_insn "*aarch64_<su>mlsl<mode>"
1624 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1625 (minus:<VWIDE>
1626 (match_operand:<VWIDE> 1 "register_operand" "0")
1627 (mult:<VWIDE>
1628 (ANY_EXTEND:<VWIDE>
1629 (match_operand:VD_BHSI 2 "register_operand" "w"))
1630 (ANY_EXTEND:<VWIDE>
1631 (match_operand:VD_BHSI 3 "register_operand" "w")))))]
1632 "TARGET_SIMD"
1633 "<su>mlsl\t%0.<Vwtype>, %2.<Vtype>, %3.<Vtype>"
1634 [(set_attr "type" "neon_mla_<Vetype>_long")]
1635 )
1637 (define_insn "aarch64_simd_vec_<su>mult_lo_<mode>"
1638 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1639 (mult:<VWIDE> (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1640 (match_operand:VQW 1 "register_operand" "w")
1641 (match_operand:VQW 3 "vect_par_cnst_lo_half" "")))
1642 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1643 (match_operand:VQW 2 "register_operand" "w")
1644 (match_dup 3)))))]
1645 "TARGET_SIMD"
1646 "<su>mull\\t%0.<Vwtype>, %1.<Vhalftype>, %2.<Vhalftype>"
1647 [(set_attr "type" "neon_mul_<Vetype>_long")]
1648 )
1650 (define_expand "vec_widen_<su>mult_lo_<mode>"
1651 [(match_operand:<VWIDE> 0 "register_operand" "")
1652 (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand" ""))
1653 (ANY_EXTEND:<VWIDE> (match_operand:VQW 2 "register_operand" ""))]
1654 "TARGET_SIMD"
1655 {
1656 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, <nunits>, false);
1657 emit_insn (gen_aarch64_simd_vec_<su>mult_lo_<mode> (operands[0],
1658 operands[1],
1659 operands[2], p));
1660 DONE;
1661 }
1662 )
1664 (define_insn "aarch64_simd_vec_<su>mult_hi_<mode>"
1665 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1666 (mult:<VWIDE> (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1667 (match_operand:VQW 1 "register_operand" "w")
1668 (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))
1669 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1670 (match_operand:VQW 2 "register_operand" "w")
1671 (match_dup 3)))))]
1672 "TARGET_SIMD"
1673 "<su>mull2\\t%0.<Vwtype>, %1.<Vtype>, %2.<Vtype>"
1674 [(set_attr "type" "neon_mul_<Vetype>_long")]
1675 )
1677 (define_expand "vec_widen_<su>mult_hi_<mode>"
1678 [(match_operand:<VWIDE> 0 "register_operand" "")
1679 (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand" ""))
1680 (ANY_EXTEND:<VWIDE> (match_operand:VQW 2 "register_operand" ""))]
1681 "TARGET_SIMD"
1682 {
1683 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, <nunits>, true);
1684 emit_insn (gen_aarch64_simd_vec_<su>mult_hi_<mode> (operands[0],
1685 operands[1],
1686 operands[2], p));
1687 DONE;
1689 }
1690 )
1692 ;; FP vector operations.
1693 ;; AArch64 AdvSIMD supports single-precision (32-bit) and
1694 ;; double-precision (64-bit) floating-point data types and arithmetic as
1695 ;; defined by the IEEE 754-2008 standard. This makes them vectorizable
1696 ;; without the need for -ffast-math or -funsafe-math-optimizations.
1697 ;;
1698 ;; Floating-point operations can raise an exception. Vectorizing such
1699 ;; operations are safe because of reasons explained below.
1700 ;;
1701 ;; ARMv8 permits an extension to enable trapped floating-point
1702 ;; exception handling, however this is an optional feature. In the
1703 ;; event of a floating-point exception being raised by vectorised
1704 ;; code then:
1705 ;; 1. If trapped floating-point exceptions are available, then a trap
1706 ;; will be taken when any lane raises an enabled exception. A trap
1707 ;; handler may determine which lane raised the exception.
1708 ;; 2. Alternatively a sticky exception flag is set in the
1709 ;; floating-point status register (FPSR). Software may explicitly
1710 ;; test the exception flags, in which case the tests will either
1711 ;; prevent vectorisation, allowing precise identification of the
1712 ;; failing operation, or if tested outside of vectorisable regions
1713 ;; then the specific operation and lane are not of interest.
1715 ;; FP arithmetic operations.
1717 (define_insn "add<mode>3"
1718 [(set (match_operand:VHSDF 0 "register_operand" "=w")
1719 (plus:VHSDF (match_operand:VHSDF 1 "register_operand" "w")
1720 (match_operand:VHSDF 2 "register_operand" "w")))]
1721 "TARGET_SIMD"
1722 "fadd\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1723 [(set_attr "type" "neon_fp_addsub_<stype><q>")]
1724 )
1726 (define_insn "sub<mode>3"
1727 [(set (match_operand:VHSDF 0 "register_operand" "=w")
1728 (minus:VHSDF (match_operand:VHSDF 1 "register_operand" "w")
1729 (match_operand:VHSDF 2 "register_operand" "w")))]
1730 "TARGET_SIMD"
1731 "fsub\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1732 [(set_attr "type" "neon_fp_addsub_<stype><q>")]
1733 )
1735 (define_insn "mul<mode>3"
1736 [(set (match_operand:VHSDF 0 "register_operand" "=w")
1737 (mult:VHSDF (match_operand:VHSDF 1 "register_operand" "w")
1738 (match_operand:VHSDF 2 "register_operand" "w")))]
1739 "TARGET_SIMD"
1740 "fmul\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1741 [(set_attr "type" "neon_fp_mul_<stype><q>")]
1742 )
1744 (define_expand "div<mode>3"
1745 [(set (match_operand:VHSDF 0 "register_operand" "=w")
1746 (div:VHSDF (match_operand:VHSDF 1 "register_operand" "w")
1747 (match_operand:VHSDF 2 "register_operand" "w")))]
1748 "TARGET_SIMD"
1749 {
1750 if (aarch64_emit_approx_div (operands[0], operands[1], operands[2]))
1751 DONE;
1753 operands[1] = force_reg (<MODE>mode, operands[1]);
1754 })
1756 (define_insn "*div<mode>3"
1757 [(set (match_operand:VHSDF 0 "register_operand" "=w")
1758 (div:VHSDF (match_operand:VHSDF 1 "register_operand" "w")
1759 (match_operand:VHSDF 2 "register_operand" "w")))]
1760 "TARGET_SIMD"
1761 "fdiv\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1762 [(set_attr "type" "neon_fp_div_<stype><q>")]
1763 )
1765 (define_insn "neg<mode>2"
1766 [(set (match_operand:VHSDF 0 "register_operand" "=w")
1767 (neg:VHSDF (match_operand:VHSDF 1 "register_operand" "w")))]
1768 "TARGET_SIMD"
1769 "fneg\\t%0.<Vtype>, %1.<Vtype>"
1770 [(set_attr "type" "neon_fp_neg_<stype><q>")]
1771 )
1773 (define_insn "abs<mode>2"
1774 [(set (match_operand:VHSDF 0 "register_operand" "=w")
1775 (abs:VHSDF (match_operand:VHSDF 1 "register_operand" "w")))]
1776 "TARGET_SIMD"
1777 "fabs\\t%0.<Vtype>, %1.<Vtype>"
1778 [(set_attr "type" "neon_fp_abs_<stype><q>")]
1779 )
1781 (define_insn "fma<mode>4"
1782 [(set (match_operand:VHSDF 0 "register_operand" "=w")
1783 (fma:VHSDF (match_operand:VHSDF 1 "register_operand" "w")
1784 (match_operand:VHSDF 2 "register_operand" "w")
1785 (match_operand:VHSDF 3 "register_operand" "0")))]
1786 "TARGET_SIMD"
1787 "fmla\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1788 [(set_attr "type" "neon_fp_mla_<stype><q>")]
1789 )
1791 (define_insn "*aarch64_fma4_elt<mode>"
1792 [(set (match_operand:VDQF 0 "register_operand" "=w")
1793 (fma:VDQF
1794 (vec_duplicate:VDQF
1795 (vec_select:<VEL>
1796 (match_operand:VDQF 1 "register_operand" "<h_con>")
1797 (parallel [(match_operand:SI 2 "immediate_operand")])))
1798 (match_operand:VDQF 3 "register_operand" "w")
1799 (match_operand:VDQF 4 "register_operand" "0")))]
1800 "TARGET_SIMD"
1801 {
1802 operands[2] = aarch64_endian_lane_rtx (<MODE>mode, INTVAL (operands[2]));
1803 return "fmla\\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
1804 }
1805 [(set_attr "type" "neon_fp_mla_<Vetype>_scalar<q>")]
1806 )
1808 (define_insn "*aarch64_fma4_elt_<vswap_width_name><mode>"
1809 [(set (match_operand:VDQSF 0 "register_operand" "=w")
1810 (fma:VDQSF
1811 (vec_duplicate:VDQSF
1812 (vec_select:<VEL>
1813 (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
1814 (parallel [(match_operand:SI 2 "immediate_operand")])))
1815 (match_operand:VDQSF 3 "register_operand" "w")
1816 (match_operand:VDQSF 4 "register_operand" "0")))]
1817 "TARGET_SIMD"
1818 {
1819 operands[2] = aarch64_endian_lane_rtx (<VSWAP_WIDTH>mode, INTVAL (operands[2]));
1820 return "fmla\\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
1821 }
1822 [(set_attr "type" "neon_fp_mla_<Vetype>_scalar<q>")]
1823 )
1825 (define_insn "*aarch64_fma4_elt_from_dup<mode>"
1826 [(set (match_operand:VMUL 0 "register_operand" "=w")
1827 (fma:VMUL
1828 (vec_duplicate:VMUL
1829 (match_operand:<VEL> 1 "register_operand" "<h_con>"))
1830 (match_operand:VMUL 2 "register_operand" "w")
1831 (match_operand:VMUL 3 "register_operand" "0")))]
1832 "TARGET_SIMD"
1833 "fmla\t%0.<Vtype>, %2.<Vtype>, %1.<Vetype>[0]"
1834 [(set_attr "type" "neon<fp>_mla_<stype>_scalar<q>")]
1835 )
1837 (define_insn "*aarch64_fma4_elt_to_64v2df"
1838 [(set (match_operand:DF 0 "register_operand" "=w")
1839 (fma:DF
1840 (vec_select:DF
1841 (match_operand:V2DF 1 "register_operand" "w")
1842 (parallel [(match_operand:SI 2 "immediate_operand")]))
1843 (match_operand:DF 3 "register_operand" "w")
1844 (match_operand:DF 4 "register_operand" "0")))]
1845 "TARGET_SIMD"
1846 {
1847 operands[2] = aarch64_endian_lane_rtx (V2DFmode, INTVAL (operands[2]));
1848 return "fmla\\t%0.2d, %3.2d, %1.2d[%2]";
1849 }
1850 [(set_attr "type" "neon_fp_mla_d_scalar_q")]
1851 )
1853 (define_insn "fnma<mode>4"
1854 [(set (match_operand:VHSDF 0 "register_operand" "=w")
1855 (fma:VHSDF
1856 (match_operand:VHSDF 1 "register_operand" "w")
1857 (neg:VHSDF
1858 (match_operand:VHSDF 2 "register_operand" "w"))
1859 (match_operand:VHSDF 3 "register_operand" "0")))]
1860 "TARGET_SIMD"
1861 "fmls\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1862 [(set_attr "type" "neon_fp_mla_<stype><q>")]
1863 )
1865 (define_insn "*aarch64_fnma4_elt<mode>"
1866 [(set (match_operand:VDQF 0 "register_operand" "=w")
1867 (fma:VDQF
1868 (neg:VDQF
1869 (match_operand:VDQF 3 "register_operand" "w"))
1870 (vec_duplicate:VDQF
1871 (vec_select:<VEL>
1872 (match_operand:VDQF 1 "register_operand" "<h_con>")
1873 (parallel [(match_operand:SI 2 "immediate_operand")])))
1874 (match_operand:VDQF 4 "register_operand" "0")))]
1875 "TARGET_SIMD"
1876 {
1877 operands[2] = aarch64_endian_lane_rtx (<MODE>mode, INTVAL (operands[2]));
1878 return "fmls\\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
1879 }
1880 [(set_attr "type" "neon_fp_mla_<Vetype>_scalar<q>")]
1881 )
1883 (define_insn "*aarch64_fnma4_elt_<vswap_width_name><mode>"
1884 [(set (match_operand:VDQSF 0 "register_operand" "=w")
1885 (fma:VDQSF
1886 (neg:VDQSF
1887 (match_operand:VDQSF 3 "register_operand" "w"))
1888 (vec_duplicate:VDQSF
1889 (vec_select:<VEL>
1890 (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
1891 (parallel [(match_operand:SI 2 "immediate_operand")])))
1892 (match_operand:VDQSF 4 "register_operand" "0")))]
1893 "TARGET_SIMD"
1894 {
1895 operands[2] = aarch64_endian_lane_rtx (<VSWAP_WIDTH>mode, INTVAL (operands[2]));
1896 return "fmls\\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
1897 }
1898 [(set_attr "type" "neon_fp_mla_<Vetype>_scalar<q>")]
1899 )
1901 (define_insn "*aarch64_fnma4_elt_from_dup<mode>"
1902 [(set (match_operand:VMUL 0 "register_operand" "=w")
1903 (fma:VMUL
1904 (neg:VMUL
1905 (match_operand:VMUL 2 "register_operand" "w"))
1906 (vec_duplicate:VMUL
1907 (match_operand:<VEL> 1 "register_operand" "<h_con>"))
1908 (match_operand:VMUL 3 "register_operand" "0")))]
1909 "TARGET_SIMD"
1910 "fmls\t%0.<Vtype>, %2.<Vtype>, %1.<Vetype>[0]"
1911 [(set_attr "type" "neon<fp>_mla_<stype>_scalar<q>")]
1912 )
1914 (define_insn "*aarch64_fnma4_elt_to_64v2df"
1915 [(set (match_operand:DF 0 "register_operand" "=w")
1916 (fma:DF
1917 (vec_select:DF
1918 (match_operand:V2DF 1 "register_operand" "w")
1919 (parallel [(match_operand:SI 2 "immediate_operand")]))
1920 (neg:DF
1921 (match_operand:DF 3 "register_operand" "w"))
1922 (match_operand:DF 4 "register_operand" "0")))]
1923 "TARGET_SIMD"
1924 {
1925 operands[2] = aarch64_endian_lane_rtx (V2DFmode, INTVAL (operands[2]));
1926 return "fmls\\t%0.2d, %3.2d, %1.2d[%2]";
1927 }
1928 [(set_attr "type" "neon_fp_mla_d_scalar_q")]
1929 )
1931 ;; Vector versions of the floating-point frint patterns.
1932 ;; Expands to btrunc, ceil, floor, nearbyint, rint, round, frintn.
1933 (define_insn "<frint_pattern><mode>2"
1934 [(set (match_operand:VHSDF 0 "register_operand" "=w")
1935 (unspec:VHSDF [(match_operand:VHSDF 1 "register_operand" "w")]
1936 FRINT))]
1937 "TARGET_SIMD"
1938 "frint<frint_suffix>\\t%0.<Vtype>, %1.<Vtype>"
1939 [(set_attr "type" "neon_fp_round_<stype><q>")]
1940 )
1942 ;; Vector versions of the fcvt standard patterns.
1943 ;; Expands to lbtrunc, lround, lceil, lfloor
1944 (define_insn "l<fcvt_pattern><su_optab><VHSDF:mode><fcvt_target>2"
1945 [(set (match_operand:<FCVT_TARGET> 0 "register_operand" "=w")
1946 (FIXUORS:<FCVT_TARGET> (unspec:<FCVT_TARGET>
1947 [(match_operand:VHSDF 1 "register_operand" "w")]
1948 FCVT)))]
1949 "TARGET_SIMD"
1950 "fcvt<frint_suffix><su>\\t%0.<Vtype>, %1.<Vtype>"
1951 [(set_attr "type" "neon_fp_to_int_<stype><q>")]
1952 )
1954 ;; HF Scalar variants of related SIMD instructions.
1955 (define_insn "l<fcvt_pattern><su_optab>hfhi2"
1956 [(set (match_operand:HI 0 "register_operand" "=w")
1957 (FIXUORS:HI (unspec:HF [(match_operand:HF 1 "register_operand" "w")]
1958 FCVT)))]
1959 "TARGET_SIMD_F16INST"
1960 "fcvt<frint_suffix><su>\t%h0, %h1"
1961 [(set_attr "type" "neon_fp_to_int_s")]
1962 )
1964 (define_insn "<optab>_trunchfhi2"
1965 [(set (match_operand:HI 0 "register_operand" "=w")
1966 (FIXUORS:HI (match_operand:HF 1 "register_operand" "w")))]
1967 "TARGET_SIMD_F16INST"
1968 "fcvtz<su>\t%h0, %h1"
1969 [(set_attr "type" "neon_fp_to_int_s")]
1970 )
1972 (define_insn "<optab>hihf2"
1973 [(set (match_operand:HF 0 "register_operand" "=w")
1974 (FLOATUORS:HF (match_operand:HI 1 "register_operand" "w")))]
1975 "TARGET_SIMD_F16INST"
1976 "<su_optab>cvtf\t%h0, %h1"
1977 [(set_attr "type" "neon_int_to_fp_s")]
1978 )
1980 (define_insn "*aarch64_fcvt<su_optab><VDQF:mode><fcvt_target>2_mult"
1981 [(set (match_operand:<FCVT_TARGET> 0 "register_operand" "=w")
1982 (FIXUORS:<FCVT_TARGET> (unspec:<FCVT_TARGET>
1983 [(mult:VDQF
1984 (match_operand:VDQF 1 "register_operand" "w")
1985 (match_operand:VDQF 2 "aarch64_fp_vec_pow2" ""))]
1986 UNSPEC_FRINTZ)))]
1987 "TARGET_SIMD
1988 && IN_RANGE (aarch64_vec_fpconst_pow_of_2 (operands[2]), 1,
1989 GET_MODE_BITSIZE (GET_MODE_INNER (<VDQF:MODE>mode)))"
1990 {
1991 int fbits = aarch64_vec_fpconst_pow_of_2 (operands[2]);
1992 char buf[64];
1993 snprintf (buf, 64, "fcvtz<su>\\t%%0.<Vtype>, %%1.<Vtype>, #%d", fbits);
1994 output_asm_insn (buf, operands);
1995 return "";
1996 }
1997 [(set_attr "type" "neon_fp_to_int_<Vetype><q>")]
1998 )
2000 (define_expand "<optab><VHSDF:mode><fcvt_target>2"
2001 [(set (match_operand:<FCVT_TARGET> 0 "register_operand")
2002 (FIXUORS:<FCVT_TARGET> (unspec:<FCVT_TARGET>
2003 [(match_operand:VHSDF 1 "register_operand")]
2004 UNSPEC_FRINTZ)))]
2005 "TARGET_SIMD"
2006 {})
2008 (define_expand "<fix_trunc_optab><VHSDF:mode><fcvt_target>2"
2009 [(set (match_operand:<FCVT_TARGET> 0 "register_operand")
2010 (FIXUORS:<FCVT_TARGET> (unspec:<FCVT_TARGET>
2011 [(match_operand:VHSDF 1 "register_operand")]
2012 UNSPEC_FRINTZ)))]
2013 "TARGET_SIMD"
2014 {})
2016 (define_expand "ftrunc<VHSDF:mode>2"
2017 [(set (match_operand:VHSDF 0 "register_operand")
2018 (unspec:VHSDF [(match_operand:VHSDF 1 "register_operand")]
2019 UNSPEC_FRINTZ))]
2020 "TARGET_SIMD"
2021 {})
2023 (define_insn "<optab><fcvt_target><VHSDF:mode>2"
2024 [(set (match_operand:VHSDF 0 "register_operand" "=w")
2025 (FLOATUORS:VHSDF
2026 (match_operand:<FCVT_TARGET> 1 "register_operand" "w")))]
2027 "TARGET_SIMD"
2028 "<su_optab>cvtf\\t%0.<Vtype>, %1.<Vtype>"
2029 [(set_attr "type" "neon_int_to_fp_<stype><q>")]
2030 )
2032 ;; Conversions between vectors of floats and doubles.
2033 ;; Contains a mix of patterns to match standard pattern names
2034 ;; and those for intrinsics.
2036 ;; Float widening operations.
2038 (define_insn "aarch64_simd_vec_unpacks_lo_<mode>"
2039 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2040 (float_extend:<VWIDE> (vec_select:<VHALF>
2041 (match_operand:VQ_HSF 1 "register_operand" "w")
2042 (match_operand:VQ_HSF 2 "vect_par_cnst_lo_half" "")
2043 )))]
2044 "TARGET_SIMD"
2045 "fcvtl\\t%0.<Vwtype>, %1.<Vhalftype>"
2046 [(set_attr "type" "neon_fp_cvt_widen_s")]
2047 )
2049 ;; Convert between fixed-point and floating-point (vector modes)
2051 (define_insn "<FCVT_F2FIXED:fcvt_fixed_insn><VHSDF:mode>3"
2052 [(set (match_operand:<VHSDF:FCVT_TARGET> 0 "register_operand" "=w")
2053 (unspec:<VHSDF:FCVT_TARGET>
2054 [(match_operand:VHSDF 1 "register_operand" "w")
2055 (match_operand:SI 2 "immediate_operand" "i")]
2056 FCVT_F2FIXED))]
2057 "TARGET_SIMD"
2058 "<FCVT_F2FIXED:fcvt_fixed_insn>\t%<v>0<Vmtype>, %<v>1<Vmtype>, #%2"
2059 [(set_attr "type" "neon_fp_to_int_<VHSDF:stype><q>")]
2060 )
2062 (define_insn "<FCVT_FIXED2F:fcvt_fixed_insn><VDQ_HSDI:mode>3"
2063 [(set (match_operand:<VDQ_HSDI:FCVT_TARGET> 0 "register_operand" "=w")
2064 (unspec:<VDQ_HSDI:FCVT_TARGET>
2065 [(match_operand:VDQ_HSDI 1 "register_operand" "w")
2066 (match_operand:SI 2 "immediate_operand" "i")]
2067 FCVT_FIXED2F))]
2068 "TARGET_SIMD"
2069 "<FCVT_FIXED2F:fcvt_fixed_insn>\t%<v>0<Vmtype>, %<v>1<Vmtype>, #%2"
2070 [(set_attr "type" "neon_int_to_fp_<VDQ_HSDI:stype><q>")]
2071 )
2073 ;; ??? Note that the vectorizer usage of the vec_unpacks_[lo/hi] patterns
2074 ;; is inconsistent with vector ordering elsewhere in the compiler, in that
2075 ;; the meaning of HI and LO changes depending on the target endianness.
2076 ;; While elsewhere we map the higher numbered elements of a vector to
2077 ;; the lower architectural lanes of the vector, for these patterns we want
2078 ;; to always treat "hi" as referring to the higher architectural lanes.
2079 ;; Consequently, while the patterns below look inconsistent with our
2080 ;; other big-endian patterns their behavior is as required.
2082 (define_expand "vec_unpacks_lo_<mode>"
2083 [(match_operand:<VWIDE> 0 "register_operand" "")
2084 (match_operand:VQ_HSF 1 "register_operand" "")]
2085 "TARGET_SIMD"
2086 {
2087 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, <nunits>, false);
2088 emit_insn (gen_aarch64_simd_vec_unpacks_lo_<mode> (operands[0],
2089 operands[1], p));
2090 DONE;
2091 }
2092 )
2094 (define_insn "aarch64_simd_vec_unpacks_hi_<mode>"
2095 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2096 (float_extend:<VWIDE> (vec_select:<VHALF>
2097 (match_operand:VQ_HSF 1 "register_operand" "w")
2098 (match_operand:VQ_HSF 2 "vect_par_cnst_hi_half" "")
2099 )))]
2100 "TARGET_SIMD"
2101 "fcvtl2\\t%0.<Vwtype>, %1.<Vtype>"
2102 [(set_attr "type" "neon_fp_cvt_widen_s")]
2103 )
2105 (define_expand "vec_unpacks_hi_<mode>"
2106 [(match_operand:<VWIDE> 0 "register_operand" "")
2107 (match_operand:VQ_HSF 1 "register_operand" "")]
2108 "TARGET_SIMD"
2109 {
2110 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, <nunits>, true);
2111 emit_insn (gen_aarch64_simd_vec_unpacks_lo_<mode> (operands[0],
2112 operands[1], p));
2113 DONE;
2114 }
2115 )
2116 (define_insn "aarch64_float_extend_lo_<Vwide>"
2117 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2118 (float_extend:<VWIDE>
2119 (match_operand:VDF 1 "register_operand" "w")))]
2120 "TARGET_SIMD"
2121 "fcvtl\\t%0<Vmwtype>, %1<Vmtype>"
2122 [(set_attr "type" "neon_fp_cvt_widen_s")]
2123 )
2125 ;; Float narrowing operations.
2127 (define_insn "aarch64_float_truncate_lo_<mode>"
2128 [(set (match_operand:VDF 0 "register_operand" "=w")
2129 (float_truncate:VDF
2130 (match_operand:<VWIDE> 1 "register_operand" "w")))]
2131 "TARGET_SIMD"
2132 "fcvtn\\t%0.<Vtype>, %1<Vmwtype>"
2133 [(set_attr "type" "neon_fp_cvt_narrow_d_q")]
2134 )
2136 (define_insn "aarch64_float_truncate_hi_<Vdbl>_le"
2137 [(set (match_operand:<VDBL> 0 "register_operand" "=w")
2138 (vec_concat:<VDBL>
2139 (match_operand:VDF 1 "register_operand" "0")
2140 (float_truncate:VDF
2141 (match_operand:<VWIDE> 2 "register_operand" "w"))))]
2142 "TARGET_SIMD && !BYTES_BIG_ENDIAN"
2143 "fcvtn2\\t%0.<Vdtype>, %2<Vmwtype>"
2144 [(set_attr "type" "neon_fp_cvt_narrow_d_q")]
2145 )
2147 (define_insn "aarch64_float_truncate_hi_<Vdbl>_be"
2148 [(set (match_operand:<VDBL> 0 "register_operand" "=w")
2149 (vec_concat:<VDBL>
2150 (float_truncate:VDF
2151 (match_operand:<VWIDE> 2 "register_operand" "w"))
2152 (match_operand:VDF 1 "register_operand" "0")))]
2153 "TARGET_SIMD && BYTES_BIG_ENDIAN"
2154 "fcvtn2\\t%0.<Vdtype>, %2<Vmwtype>"
2155 [(set_attr "type" "neon_fp_cvt_narrow_d_q")]
2156 )
2158 (define_expand "aarch64_float_truncate_hi_<Vdbl>"
2159 [(match_operand:<VDBL> 0 "register_operand" "=w")
2160 (match_operand:VDF 1 "register_operand" "0")
2161 (match_operand:<VWIDE> 2 "register_operand" "w")]
2162 "TARGET_SIMD"
2163 {
2164 rtx (*gen) (rtx, rtx, rtx) = BYTES_BIG_ENDIAN
2165 ? gen_aarch64_float_truncate_hi_<Vdbl>_be
2166 : gen_aarch64_float_truncate_hi_<Vdbl>_le;
2167 emit_insn (gen (operands[0], operands[1], operands[2]));
2168 DONE;
2169 }
2170 )
2172 (define_expand "vec_pack_trunc_v2df"
2173 [(set (match_operand:V4SF 0 "register_operand")
2174 (vec_concat:V4SF
2175 (float_truncate:V2SF
2176 (match_operand:V2DF 1 "register_operand"))
2177 (float_truncate:V2SF
2178 (match_operand:V2DF 2 "register_operand"))
2179 ))]
2180 "TARGET_SIMD"
2181 {
2182 rtx tmp = gen_reg_rtx (V2SFmode);
2183 int lo = BYTES_BIG_ENDIAN ? 2 : 1;
2184 int hi = BYTES_BIG_ENDIAN ? 1 : 2;
2186 emit_insn (gen_aarch64_float_truncate_lo_v2sf (tmp, operands[lo]));
2187 emit_insn (gen_aarch64_float_truncate_hi_v4sf (operands[0],
2188 tmp, operands[hi]));
2189 DONE;
2190 }
2191 )
2193 (define_expand "vec_pack_trunc_df"
2194 [(set (match_operand:V2SF 0 "register_operand")
2195 (vec_concat:V2SF
2196 (float_truncate:SF
2197 (match_operand:DF 1 "register_operand"))
2198 (float_truncate:SF
2199 (match_operand:DF 2 "register_operand"))
2200 ))]
2201 "TARGET_SIMD"
2202 {
2203 rtx tmp = gen_reg_rtx (V2SFmode);
2204 int lo = BYTES_BIG_ENDIAN ? 2 : 1;
2205 int hi = BYTES_BIG_ENDIAN ? 1 : 2;
2207 emit_insn (gen_move_lo_quad_v2df (tmp, operands[lo]));
2208 emit_insn (gen_move_hi_quad_v2df (tmp, operands[hi]));
2209 emit_insn (gen_aarch64_float_truncate_lo_v2sf (operands[0], tmp));
2210 DONE;
2211 }
2212 )
2214 ;; FP Max/Min
2215 ;; Max/Min are introduced by idiom recognition by GCC's mid-end. An
2216 ;; expression like:
2217 ;; a = (b < c) ? b : c;
2218 ;; is idiom-matched as MIN_EXPR<b,c> only if -ffinite-math-only is enabled
2219 ;; either explicitly or indirectly via -ffast-math.
2220 ;;
2221 ;; MIN_EXPR and MAX_EXPR eventually map to 'smin' and 'smax' in RTL.
2222 ;; The 'smax' and 'smin' RTL standard pattern names do not specify which
2223 ;; operand will be returned when both operands are zero (i.e. they may not
2224 ;; honour signed zeroes), or when either operand is NaN. Therefore GCC
2225 ;; only introduces MIN_EXPR/MAX_EXPR in fast math mode or when not honouring
2226 ;; NaNs.
2228 (define_insn "<su><maxmin><mode>3"
2229 [(set (match_operand:VHSDF 0 "register_operand" "=w")
2230 (FMAXMIN:VHSDF (match_operand:VHSDF 1 "register_operand" "w")
2231 (match_operand:VHSDF 2 "register_operand" "w")))]
2232 "TARGET_SIMD"
2233 "f<maxmin>nm\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
2234 [(set_attr "type" "neon_fp_minmax_<stype><q>")]
2235 )
2237 ;; Vector forms for fmax, fmin, fmaxnm, fminnm.
2238 ;; fmaxnm and fminnm are used for the fmax<mode>3 standard pattern names,
2239 ;; which implement the IEEE fmax ()/fmin () functions.
2240 (define_insn "<maxmin_uns><mode>3"
2241 [(set (match_operand:VHSDF 0 "register_operand" "=w")
2242 (unspec:VHSDF [(match_operand:VHSDF 1 "register_operand" "w")
2243 (match_operand:VHSDF 2 "register_operand" "w")]
2244 FMAXMIN_UNS))]
2245 "TARGET_SIMD"
2246 "<maxmin_uns_op>\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
2247 [(set_attr "type" "neon_fp_minmax_<stype><q>")]
2248 )
2250 ;; 'across lanes' add.
2252 (define_expand "reduc_plus_scal_<mode>"
2253 [(match_operand:<VEL> 0 "register_operand" "=w")
2254 (unspec:VDQ_I [(match_operand:VDQ_I 1 "register_operand" "w")]
2255 UNSPEC_ADDV)]
2256 "TARGET_SIMD"
2257 {
2258 rtx elt = aarch64_endian_lane_rtx (<MODE>mode, 0);
2259 rtx scratch = gen_reg_rtx (<MODE>mode);
2260 emit_insn (gen_aarch64_reduc_plus_internal<mode> (scratch, operands[1]));
2261 emit_insn (gen_aarch64_get_lane<mode> (operands[0], scratch, elt));
2262 DONE;
2263 }
2264 )
2266 (define_insn "aarch64_faddp<mode>"
2267 [(set (match_operand:VHSDF 0 "register_operand" "=w")
2268 (unspec:VHSDF [(match_operand:VHSDF 1 "register_operand" "w")
2269 (match_operand:VHSDF 2 "register_operand" "w")]
2270 UNSPEC_FADDV))]
2271 "TARGET_SIMD"
2272 "faddp\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
2273 [(set_attr "type" "neon_fp_reduc_add_<stype><q>")]
2274 )
2276 (define_insn "aarch64_reduc_plus_internal<mode>"
2277 [(set (match_operand:VDQV 0 "register_operand" "=w")
2278 (unspec:VDQV [(match_operand:VDQV 1 "register_operand" "w")]
2279 UNSPEC_ADDV))]
2280 "TARGET_SIMD"
2281 "add<VDQV:vp>\\t%<Vetype>0, %1.<Vtype>"
2282 [(set_attr "type" "neon_reduc_add<q>")]
2283 )
2285 (define_insn "aarch64_reduc_plus_internalv2si"
2286 [(set (match_operand:V2SI 0 "register_operand" "=w")
2287 (unspec:V2SI [(match_operand:V2SI 1 "register_operand" "w")]
2288 UNSPEC_ADDV))]
2289 "TARGET_SIMD"
2290 "addp\\t%0.2s, %1.2s, %1.2s"
2291 [(set_attr "type" "neon_reduc_add")]
2292 )
2294 (define_insn "reduc_plus_scal_<mode>"
2295 [(set (match_operand:<VEL> 0 "register_operand" "=w")
2296 (unspec:<VEL> [(match_operand:V2F 1 "register_operand" "w")]
2297 UNSPEC_FADDV))]
2298 "TARGET_SIMD"
2299 "faddp\\t%<Vetype>0, %1.<Vtype>"
2300 [(set_attr "type" "neon_fp_reduc_add_<Vetype><q>")]
2301 )
2303 (define_expand "reduc_plus_scal_v4sf"
2304 [(set (match_operand:SF 0 "register_operand")
2305 (unspec:V4SF [(match_operand:V4SF 1 "register_operand")]
2306 UNSPEC_FADDV))]
2307 "TARGET_SIMD"
2308 {
2309 rtx elt = aarch64_endian_lane_rtx (V4SFmode, 0);
2310 rtx scratch = gen_reg_rtx (V4SFmode);
2311 emit_insn (gen_aarch64_faddpv4sf (scratch, operands[1], operands[1]));
2312 emit_insn (gen_aarch64_faddpv4sf (scratch, scratch, scratch));
2313 emit_insn (gen_aarch64_get_lanev4sf (operands[0], scratch, elt));
2314 DONE;
2315 })
2317 (define_insn "clrsb<mode>2"
2318 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
2319 (clrsb:VDQ_BHSI (match_operand:VDQ_BHSI 1 "register_operand" "w")))]
2320 "TARGET_SIMD"
2321 "cls\\t%0.<Vtype>, %1.<Vtype>"
2322 [(set_attr "type" "neon_cls<q>")]
2323 )
2325 (define_insn "clz<mode>2"
2326 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
2327 (clz:VDQ_BHSI (match_operand:VDQ_BHSI 1 "register_operand" "w")))]
2328 "TARGET_SIMD"
2329 "clz\\t%0.<Vtype>, %1.<Vtype>"
2330 [(set_attr "type" "neon_cls<q>")]
2331 )
2333 (define_insn "popcount<mode>2"
2334 [(set (match_operand:VB 0 "register_operand" "=w")
2335 (popcount:VB (match_operand:VB 1 "register_operand" "w")))]
2336 "TARGET_SIMD"
2337 "cnt\\t%0.<Vbtype>, %1.<Vbtype>"
2338 [(set_attr "type" "neon_cnt<q>")]
2339 )
2341 ;; 'across lanes' max and min ops.
2343 ;; Template for outputting a scalar, so we can create __builtins which can be
2344 ;; gimple_fold'd to the REDUC_(MAX|MIN)_EXPR tree code. (This is FP smax/smin).
2345 (define_expand "reduc_<maxmin_uns>_scal_<mode>"
2346 [(match_operand:<VEL> 0 "register_operand")
2347 (unspec:VHSDF [(match_operand:VHSDF 1 "register_operand")]
2348 FMAXMINV)]
2349 "TARGET_SIMD"
2350 {
2351 rtx elt = aarch64_endian_lane_rtx (<MODE>mode, 0);
2352 rtx scratch = gen_reg_rtx (<MODE>mode);
2353 emit_insn (gen_aarch64_reduc_<maxmin_uns>_internal<mode> (scratch,
2354 operands[1]));
2355 emit_insn (gen_aarch64_get_lane<mode> (operands[0], scratch, elt));
2356 DONE;
2357 }
2358 )
2360 ;; Likewise for integer cases, signed and unsigned.
2361 (define_expand "reduc_<maxmin_uns>_scal_<mode>"
2362 [(match_operand:<VEL> 0 "register_operand")
2363 (unspec:VDQ_BHSI [(match_operand:VDQ_BHSI 1 "register_operand")]
2364 MAXMINV)]
2365 "TARGET_SIMD"
2366 {
2367 rtx elt = aarch64_endian_lane_rtx (<MODE>mode, 0);
2368 rtx scratch = gen_reg_rtx (<MODE>mode);
2369 emit_insn (gen_aarch64_reduc_<maxmin_uns>_internal<mode> (scratch,
2370 operands[1]));
2371 emit_insn (gen_aarch64_get_lane<mode> (operands[0], scratch, elt));
2372 DONE;
2373 }
2374 )
2376 (define_insn "aarch64_reduc_<maxmin_uns>_internal<mode>"
2377 [(set (match_operand:VDQV_S 0 "register_operand" "=w")
2378 (unspec:VDQV_S [(match_operand:VDQV_S 1 "register_operand" "w")]
2379 MAXMINV))]
2380 "TARGET_SIMD"
2381 "<maxmin_uns_op>v\\t%<Vetype>0, %1.<Vtype>"
2382 [(set_attr "type" "neon_reduc_minmax<q>")]
2383 )
2385 (define_insn "aarch64_reduc_<maxmin_uns>_internalv2si"
2386 [(set (match_operand:V2SI 0 "register_operand" "=w")
2387 (unspec:V2SI [(match_operand:V2SI 1 "register_operand" "w")]
2388 MAXMINV))]
2389 "TARGET_SIMD"
2390 "<maxmin_uns_op>p\\t%0.2s, %1.2s, %1.2s"
2391 [(set_attr "type" "neon_reduc_minmax")]
2392 )
2394 (define_insn "aarch64_reduc_<maxmin_uns>_internal<mode>"
2395 [(set (match_operand:VHSDF 0 "register_operand" "=w")
2396 (unspec:VHSDF [(match_operand:VHSDF 1 "register_operand" "w")]
2397 FMAXMINV))]
2398 "TARGET_SIMD"
2399 "<maxmin_uns_op><vp>\\t%<Vetype>0, %1.<Vtype>"
2400 [(set_attr "type" "neon_fp_reduc_minmax_<stype><q>")]
2401 )
2403 ;; aarch64_simd_bsl may compile to any of bsl/bif/bit depending on register
2404 ;; allocation.
2405 ;; Operand 1 is the mask, operands 2 and 3 are the bitfields from which
2406 ;; to select.
2407 ;;
2408 ;; Thus our BSL is of the form:
2409 ;; op0 = bsl (mask, op2, op3)
2410 ;; We can use any of:
2411 ;;
2412 ;; if (op0 = mask)
2413 ;; bsl mask, op1, op2
2414 ;; if (op0 = op1) (so 1-bits in mask choose bits from op2, else op0)
2415 ;; bit op0, op2, mask
2416 ;; if (op0 = op2) (so 0-bits in mask choose bits from op1, else op0)
2417 ;; bif op0, op1, mask
2418 ;;
2419 ;; This pattern is expanded to by the aarch64_simd_bsl<mode> expander.
2420 ;; Some forms of straight-line code may generate the equivalent form
2421 ;; in *aarch64_simd_bsl<mode>_alt.
2423 (define_insn "aarch64_simd_bsl<mode>_internal"
2424 [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w,w,w")
2425 (xor:VSDQ_I_DI
2426 (and:VSDQ_I_DI
2427 (xor:VSDQ_I_DI
2428 (match_operand:<V_INT_EQUIV> 3 "register_operand" "w,0,w")
2429 (match_operand:VSDQ_I_DI 2 "register_operand" "w,w,0"))
2430 (match_operand:VSDQ_I_DI 1 "register_operand" "0,w,w"))
2431 (match_dup:<V_INT_EQUIV> 3)
2432 ))]
2433 "TARGET_SIMD"
2434 "@
2435 bsl\\t%0.<Vbtype>, %2.<Vbtype>, %3.<Vbtype>
2436 bit\\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>
2437 bif\\t%0.<Vbtype>, %3.<Vbtype>, %1.<Vbtype>"
2438 [(set_attr "type" "neon_bsl<q>")]
2439 )
2441 ;; We need this form in addition to the above pattern to match the case
2442 ;; when combine tries merging three insns such that the second operand of
2443 ;; the outer XOR matches the second operand of the inner XOR rather than
2444 ;; the first. The two are equivalent but since recog doesn't try all
2445 ;; permutations of commutative operations, we have to have a separate pattern.
2447 (define_insn "*aarch64_simd_bsl<mode>_alt"
2448 [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w,w,w")
2449 (xor:VSDQ_I_DI
2450 (and:VSDQ_I_DI
2451 (xor:VSDQ_I_DI
2452 (match_operand:VSDQ_I_DI 3 "register_operand" "w,w,0")
2453 (match_operand:VSDQ_I_DI 2 "register_operand" "w,0,w"))
2454 (match_operand:VSDQ_I_DI 1 "register_operand" "0,w,w"))
2455 (match_dup:VSDQ_I_DI 2)))]
2456 "TARGET_SIMD"
2457 "@
2458 bsl\\t%0.<Vbtype>, %3.<Vbtype>, %2.<Vbtype>
2459 bit\\t%0.<Vbtype>, %3.<Vbtype>, %1.<Vbtype>
2460 bif\\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>"
2461 [(set_attr "type" "neon_bsl<q>")]
2462 )
2464 (define_expand "aarch64_simd_bsl<mode>"
2465 [(match_operand:VALLDIF 0 "register_operand")
2466 (match_operand:<V_INT_EQUIV> 1 "register_operand")
2467 (match_operand:VALLDIF 2 "register_operand")
2468 (match_operand:VALLDIF 3 "register_operand")]
2469 "TARGET_SIMD"
2470 {
2471 /* We can't alias operands together if they have different modes. */
2472 rtx tmp = operands[0];
2473 if (FLOAT_MODE_P (<MODE>mode))
2474 {
2475 operands[2] = gen_lowpart (<V_INT_EQUIV>mode, operands[2]);
2476 operands[3] = gen_lowpart (<V_INT_EQUIV>mode, operands[3]);
2477 tmp = gen_reg_rtx (<V_INT_EQUIV>mode);
2478 }
2479 operands[1] = gen_lowpart (<V_INT_EQUIV>mode, operands[1]);
2480 emit_insn (gen_aarch64_simd_bsl<v_int_equiv>_internal (tmp,
2481 operands[1],
2482 operands[2],
2483 operands[3]));
2484 if (tmp != operands[0])
2485 emit_move_insn (operands[0], gen_lowpart (<MODE>mode, tmp));
2487 DONE;
2488 })
2490 (define_expand "vcond_mask_<mode><v_int_equiv>"
2491 [(match_operand:VALLDI 0 "register_operand")
2492 (match_operand:VALLDI 1 "nonmemory_operand")
2493 (match_operand:VALLDI 2 "nonmemory_operand")
2494 (match_operand:<V_INT_EQUIV> 3 "register_operand")]
2495 "TARGET_SIMD"
2496 {
2497 /* If we have (a = (P) ? -1 : 0);
2498 Then we can simply move the generated mask (result must be int). */
2499 if (operands[1] == CONSTM1_RTX (<MODE>mode)
2500 && operands[2] == CONST0_RTX (<MODE>mode))
2501 emit_move_insn (operands[0], operands[3]);
2502 /* Similarly, (a = (P) ? 0 : -1) is just inverting the generated mask. */
2503 else if (operands[1] == CONST0_RTX (<MODE>mode)
2504 && operands[2] == CONSTM1_RTX (<MODE>mode))
2505 emit_insn (gen_one_cmpl<v_int_equiv>2 (operands[0], operands[3]));
2506 else
2507 {
2508 if (!REG_P (operands[1]))
2509 operands[1] = force_reg (<MODE>mode, operands[1]);
2510 if (!REG_P (operands[2]))
2511 operands[2] = force_reg (<MODE>mode, operands[2]);
2512 emit_insn (gen_aarch64_simd_bsl<mode> (operands[0], operands[3],
2513 operands[1], operands[2]));
2514 }
2516 DONE;
2517 })
2519 ;; Patterns comparing two vectors to produce a mask.
2521 (define_expand "vec_cmp<mode><mode>"
2522 [(set (match_operand:VSDQ_I_DI 0 "register_operand")
2523 (match_operator 1 "comparison_operator"
2524 [(match_operand:VSDQ_I_DI 2 "register_operand")
2525 (match_operand:VSDQ_I_DI 3 "nonmemory_operand")]))]
2526 "TARGET_SIMD"
2527 {
2528 rtx mask = operands[0];
2529 enum rtx_code code = GET_CODE (operands[1]);
2531 switch (code)
2532 {
2533 case NE:
2534 case LE:
2535 case LT:
2536 case GE:
2537 case GT:
2538 case EQ:
2539 if (operands[3] == CONST0_RTX (<MODE>mode))
2540 break;
2542 /* Fall through. */
2543 default:
2544 if (!REG_P (operands[3]))
2545 operands[3] = force_reg (<MODE>mode, operands[3]);
2547 break;
2548 }
2550 switch (code)
2551 {
2552 case LT:
2553 emit_insn (gen_aarch64_cmlt<mode> (mask, operands[2], operands[3]));
2554 break;
2556 case GE:
2557 emit_insn (gen_aarch64_cmge<mode> (mask, operands[2], operands[3]));
2558 break;
2560 case LE:
2561 emit_insn (gen_aarch64_cmle<mode> (mask, operands[2], operands[3]));
2562 break;
2564 case GT:
2565 emit_insn (gen_aarch64_cmgt<mode> (mask, operands[2], operands[3]));
2566 break;
2568 case LTU:
2569 emit_insn (gen_aarch64_cmgtu<mode> (mask, operands[3], operands[2]));
2570 break;
2572 case GEU:
2573 emit_insn (gen_aarch64_cmgeu<mode> (mask, operands[2], operands[3]));
2574 break;
2576 case LEU:
2577 emit_insn (gen_aarch64_cmgeu<mode> (mask, operands[3], operands[2]));
2578 break;
2580 case GTU:
2581 emit_insn (gen_aarch64_cmgtu<mode> (mask, operands[2], operands[3]));
2582 break;
2584 case NE:
2585 /* Handle NE as !EQ. */
2586 emit_insn (gen_aarch64_cmeq<mode> (mask, operands[2], operands[3]));
2587 emit_insn (gen_one_cmpl<v_int_equiv>2 (mask, mask));
2588 break;
2590 case EQ:
2591 emit_insn (gen_aarch64_cmeq<mode> (mask, operands[2], operands[3]));
2592 break;
2594 default:
2595 gcc_unreachable ();
2596 }
2598 DONE;
2599 })
2601 (define_expand "vec_cmp<mode><v_int_equiv>"
2602 [(set (match_operand:<V_INT_EQUIV> 0 "register_operand")
2603 (match_operator 1 "comparison_operator"
2604 [(match_operand:VDQF 2 "register_operand")
2605 (match_operand:VDQF 3 "nonmemory_operand")]))]
2606 "TARGET_SIMD"
2607 {
2608 int use_zero_form = 0;
2609 enum rtx_code code = GET_CODE (operands[1]);
2610 rtx tmp = gen_reg_rtx (<V_INT_EQUIV>mode);
2612 rtx (*comparison) (rtx, rtx, rtx) = NULL;
2614 switch (code)
2615 {
2616 case LE:
2617 case LT:
2618 case GE:
2619 case GT:
2620 case EQ:
2621 if (operands[3] == CONST0_RTX (<MODE>mode))
2622 {
2623 use_zero_form = 1;
2624 break;
2625 }
2626 /* Fall through. */
2627 default:
2628 if (!REG_P (operands[3]))
2629 operands[3] = force_reg (<MODE>mode, operands[3]);
2631 break;
2632 }
2634 switch (code)
2635 {
2636 case LT:
2637 if (use_zero_form)
2638 {
2639 comparison = gen_aarch64_cmlt<mode>;
2640 break;
2641 }
2642 /* Fall through. */
2643 case UNGE:
2644 std::swap (operands[2], operands[3]);
2645 /* Fall through. */
2646 case UNLE:
2647 case GT:
2648 comparison = gen_aarch64_cmgt<mode>;
2649 break;
2650 case LE:
2651 if (use_zero_form)
2652 {
2653 comparison = gen_aarch64_cmle<mode>;
2654 break;
2655 }
2656 /* Fall through. */
2657 case UNGT:
2658 std::swap (operands[2], operands[3]);
2659 /* Fall through. */
2660 case UNLT:
2661 case GE:
2662 comparison = gen_aarch64_cmge<mode>;
2663 break;
2664 case NE:
2665 case EQ:
2666 comparison = gen_aarch64_cmeq<mode>;
2667 break;
2668 case UNEQ:
2669 case ORDERED:
2670 case UNORDERED:
2671 break;
2672 default:
2673 gcc_unreachable ();
2674 }
2676 switch (code)
2677 {
2678 case UNGE:
2679 case UNGT:
2680 case UNLE:
2681 case UNLT:
2682 case NE:
2683 /* FCM returns false for lanes which are unordered, so if we use
2684 the inverse of the comparison we actually want to emit, then
2685 invert the result, we will end up with the correct result.
2686 Note that a NE NaN and NaN NE b are true for all a, b.
2688 Our transformations are:
2689 a UNGE b -> !(b GT a)
2690 a UNGT b -> !(b GE a)
2691 a UNLE b -> !(a GT b)
2692 a UNLT b -> !(a GE b)
2693 a NE b -> !(a EQ b) */
2694 gcc_assert (comparison != NULL);
2695 emit_insn (comparison (operands[0], operands[2], operands[3]));
2696 emit_insn (gen_one_cmpl<v_int_equiv>2 (operands[0], operands[0]));
2697 break;
2699 case LT:
2700 case LE:
2701 case GT:
2702 case GE:
2703 case EQ:
2704 /* The easy case. Here we emit one of FCMGE, FCMGT or FCMEQ.
2705 As a LT b <=> b GE a && a LE b <=> b GT a. Our transformations are:
2706 a GE b -> a GE b
2707 a GT b -> a GT b
2708 a LE b -> b GE a
2709 a LT b -> b GT a
2710 a EQ b -> a EQ b */
2711 gcc_assert (comparison != NULL);
2712 emit_insn (comparison (operands[0], operands[2], operands[3]));
2713 break;
2715 case UNEQ:
2716 /* We first check (a > b || b > a) which is !UNEQ, inverting
2717 this result will then give us (a == b || a UNORDERED b). */
2718 emit_insn (gen_aarch64_cmgt<mode> (operands[0],
2719 operands[2], operands[3]));
2720 emit_insn (gen_aarch64_cmgt<mode> (tmp, operands[3], operands[2]));
2721 emit_insn (gen_ior<v_int_equiv>3 (operands[0], operands[0], tmp));
2722 emit_insn (gen_one_cmpl<v_int_equiv>2 (operands[0], operands[0]));
2723 break;
2725 case UNORDERED:
2726 /* Operands are ORDERED iff (a > b || b >= a), so we can compute
2727 UNORDERED as !ORDERED. */
2728 emit_insn (gen_aarch64_cmgt<mode> (tmp, operands[2], operands[3]));
2729 emit_insn (gen_aarch64_cmge<mode> (operands[0],
2730 operands[3], operands[2]));
2731 emit_insn (gen_ior<v_int_equiv>3 (operands[0], operands[0], tmp));
2732 emit_insn (gen_one_cmpl<v_int_equiv>2 (operands[0], operands[0]));
2733 break;
2735 case ORDERED:
2736 emit_insn (gen_aarch64_cmgt<mode> (tmp, operands[2], operands[3]));
2737 emit_insn (gen_aarch64_cmge<mode> (operands[0],
2738 operands[3], operands[2]));
2739 emit_insn (gen_ior<v_int_equiv>3 (operands[0], operands[0], tmp));
2740 break;
2742 default:
2743 gcc_unreachable ();
2744 }
2746 DONE;
2747 })
2749 (define_expand "vec_cmpu<mode><mode>"
2750 [(set (match_operand:VSDQ_I_DI 0 "register_operand")
2751 (match_operator 1 "comparison_operator"
2752 [(match_operand:VSDQ_I_DI 2 "register_operand")
2753 (match_operand:VSDQ_I_DI 3 "nonmemory_operand")]))]
2754 "TARGET_SIMD"
2755 {
2756 emit_insn (gen_vec_cmp<mode><mode> (operands[0], operands[1],
2757 operands[2], operands[3]));
2758 DONE;
2759 })
2761 (define_expand "vcond<mode><mode>"
2762 [(set (match_operand:VALLDI 0 "register_operand")
2763 (if_then_else:VALLDI
2764 (match_operator 3 "comparison_operator"
2765 [(match_operand:VALLDI 4 "register_operand")
2766 (match_operand:VALLDI 5 "nonmemory_operand")])
2767 (match_operand:VALLDI 1 "nonmemory_operand")
2768 (match_operand:VALLDI 2 "nonmemory_operand")))]
2769 "TARGET_SIMD"
2770 {
2771 rtx mask = gen_reg_rtx (<V_INT_EQUIV>mode);
2772 enum rtx_code code = GET_CODE (operands[3]);
2774 /* NE is handled as !EQ in vec_cmp patterns, we can explicitly invert
2775 it as well as switch operands 1/2 in order to avoid the additional
2776 NOT instruction. */
2777 if (code == NE)
2778 {
2779 operands[3] = gen_rtx_fmt_ee (EQ, GET_MODE (operands[3]),
2780 operands[4], operands[5]);
2781 std::swap (operands[1], operands[2]);
2782 }
2783 emit_insn (gen_vec_cmp<mode><v_int_equiv> (mask, operands[3],
2784 operands[4], operands[5]));
2785 emit_insn (gen_vcond_mask_<mode><v_int_equiv> (operands[0], operands[1],
2786 operands[2], mask));
2788 DONE;
2789 })
2791 (define_expand "vcond<v_cmp_mixed><mode>"
2792 [(set (match_operand:<V_cmp_mixed> 0 "register_operand")
2793 (if_then_else:<V_cmp_mixed>
2794 (match_operator 3 "comparison_operator"
2795 [(match_operand:VDQF_COND 4 "register_operand")
2796 (match_operand:VDQF_COND 5 "nonmemory_operand")])
2797 (match_operand:<V_cmp_mixed> 1 "nonmemory_operand")
2798 (match_operand:<V_cmp_mixed> 2 "nonmemory_operand")))]
2799 "TARGET_SIMD"
2800 {
2801 rtx mask = gen_reg_rtx (<V_INT_EQUIV>mode);
2802 enum rtx_code code = GET_CODE (operands[3]);
2804 /* NE is handled as !EQ in vec_cmp patterns, we can explicitly invert
2805 it as well as switch operands 1/2 in order to avoid the additional
2806 NOT instruction. */
2807 if (code == NE)
2808 {
2809 operands[3] = gen_rtx_fmt_ee (EQ, GET_MODE (operands[3]),
2810 operands[4], operands[5]);
2811 std::swap (operands[1], operands[2]);
2812 }
2813 emit_insn (gen_vec_cmp<mode><v_int_equiv> (mask, operands[3],
2814 operands[4], operands[5]));
2815 emit_insn (gen_vcond_mask_<v_cmp_mixed><v_int_equiv> (
2816 operands[0], operands[1],
2817 operands[2], mask));
2819 DONE;
2820 })
2822 (define_expand "vcondu<mode><mode>"
2823 [(set (match_operand:VSDQ_I_DI 0 "register_operand")
2824 (if_then_else:VSDQ_I_DI
2825 (match_operator 3 "comparison_operator"
2826 [(match_operand:VSDQ_I_DI 4 "register_operand")
2827 (match_operand:VSDQ_I_DI 5 "nonmemory_operand")])
2828 (match_operand:VSDQ_I_DI 1 "nonmemory_operand")
2829 (match_operand:VSDQ_I_DI 2 "nonmemory_operand")))]
2830 "TARGET_SIMD"
2831 {
2832 rtx mask = gen_reg_rtx (<MODE>mode);
2833 enum rtx_code code = GET_CODE (operands[3]);
2835 /* NE is handled as !EQ in vec_cmp patterns, we can explicitly invert
2836 it as well as switch operands 1/2 in order to avoid the additional
2837 NOT instruction. */
2838 if (code == NE)
2839 {
2840 operands[3] = gen_rtx_fmt_ee (EQ, GET_MODE (operands[3]),
2841 operands[4], operands[5]);
2842 std::swap (operands[1], operands[2]);
2843 }
2844 emit_insn (gen_vec_cmp<mode><mode> (mask, operands[3],
2845 operands[4], operands[5]));
2846 emit_insn (gen_vcond_mask_<mode><v_int_equiv> (operands[0], operands[1],
2847 operands[2], mask));
2848 DONE;
2849 })
2851 (define_expand "vcondu<mode><v_cmp_mixed>"
2852 [(set (match_operand:VDQF 0 "register_operand")
2853 (if_then_else:VDQF
2854 (match_operator 3 "comparison_operator"
2855 [(match_operand:<V_cmp_mixed> 4 "register_operand")
2856 (match_operand:<V_cmp_mixed> 5 "nonmemory_operand")])
2857 (match_operand:VDQF 1 "nonmemory_operand")
2858 (match_operand:VDQF 2 "nonmemory_operand")))]
2859 "TARGET_SIMD"
2860 {
2861 rtx mask = gen_reg_rtx (<V_INT_EQUIV>mode);
2862 enum rtx_code code = GET_CODE (operands[3]);
2864 /* NE is handled as !EQ in vec_cmp patterns, we can explicitly invert
2865 it as well as switch operands 1/2 in order to avoid the additional
2866 NOT instruction. */
2867 if (code == NE)
2868 {
2869 operands[3] = gen_rtx_fmt_ee (EQ, GET_MODE (operands[3]),
2870 operands[4], operands[5]);
2871 std::swap (operands[1], operands[2]);
2872 }
2873 emit_insn (gen_vec_cmp<v_cmp_mixed><v_cmp_mixed> (
2874 mask, operands[3],
2875 operands[4], operands[5]));
2876 emit_insn (gen_vcond_mask_<mode><v_int_equiv> (operands[0], operands[1],
2877 operands[2], mask));
2878 DONE;
2879 })
2881 ;; Patterns for AArch64 SIMD Intrinsics.
2883 ;; Lane extraction with sign extension to general purpose register.
2884 (define_insn "*aarch64_get_lane_extend<GPI:mode><VDQQH:mode>"
2885 [(set (match_operand:GPI 0 "register_operand" "=r")
2886 (sign_extend:GPI
2887 (vec_select:<VEL>
2888 (match_operand:VDQQH 1 "register_operand" "w")
2889 (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2890 "TARGET_SIMD"
2891 {
2892 operands[2] = aarch64_endian_lane_rtx (<MODE>mode, INTVAL (operands[2]));
2893 return "smov\\t%<GPI:w>0, %1.<VDQQH:Vetype>[%2]";
2894 }
2895 [(set_attr "type" "neon_to_gp<q>")]
2896 )
2898 (define_insn "*aarch64_get_lane_zero_extendsi<mode>"
2899 [(set (match_operand:SI 0 "register_operand" "=r")
2900 (zero_extend:SI
2901 (vec_select:<VEL>
2902 (match_operand:VDQQH 1 "register_operand" "w")
2903 (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2904 "TARGET_SIMD"
2905 {
2906 operands[2] = aarch64_endian_lane_rtx (<MODE>mode, INTVAL (operands[2]));
2907 return "umov\\t%w0, %1.<Vetype>[%2]";
2908 }
2909 [(set_attr "type" "neon_to_gp<q>")]
2910 )
2912 ;; Lane extraction of a value, neither sign nor zero extension
2913 ;; is guaranteed so upper bits should be considered undefined.
2914 ;; RTL uses GCC vector extension indices throughout so flip only for assembly.
2915 (define_insn "aarch64_get_lane<mode>"
2916 [(set (match_operand:<VEL> 0 "aarch64_simd_nonimmediate_operand" "=r, w, Utv")
2917 (vec_select:<VEL>
2918 (match_operand:VALL_F16 1 "register_operand" "w, w, w")
2919 (parallel [(match_operand:SI 2 "immediate_operand" "i, i, i")])))]
2920 "TARGET_SIMD"
2921 {
2922 operands[2] = aarch64_endian_lane_rtx (<MODE>mode, INTVAL (operands[2]));
2923 switch (which_alternative)
2924 {
2925 case 0:
2926 return "umov\\t%<vwcore>0, %1.<Vetype>[%2]";
2927 case 1:
2928 return "dup\\t%<Vetype>0, %1.<Vetype>[%2]";
2929 case 2:
2930 return "st1\\t{%1.<Vetype>}[%2], %0";
2931 default:
2932 gcc_unreachable ();
2933 }
2934 }
2935 [(set_attr "type" "neon_to_gp<q>, neon_dup<q>, neon_store1_one_lane<q>")]
2936 )
2938 (define_insn "load_pair_lanes<mode>"
2939 [(set (match_operand:<VDBL> 0 "register_operand" "=w")
2940 (vec_concat:<VDBL>
2941 (match_operand:VDC 1 "memory_operand" "Utq")
2942 (match_operand:VDC 2 "memory_operand" "m")))]
2943 "TARGET_SIMD && !STRICT_ALIGNMENT
2944 && rtx_equal_p (XEXP (operands[2], 0),
2945 plus_constant (Pmode,
2946 XEXP (operands[1], 0),
2947 GET_MODE_SIZE (<MODE>mode)))"
2948 "ldr\\t%q0, %1"
2949 [(set_attr "type" "neon_load1_1reg_q")]
2950 )
2952 (define_insn "store_pair_lanes<mode>"
2953 [(set (match_operand:<VDBL> 0 "aarch64_mem_pair_lanes_operand" "=Uml, Uml")
2954 (vec_concat:<VDBL>
2955 (match_operand:VDC 1 "register_operand" "w, r")
2956 (match_operand:VDC 2 "register_operand" "w, r")))]
2957 "TARGET_SIMD"
2958 "@
2959 stp\\t%d1, %d2, %0
2960 stp\\t%x1, %x2, %0"
2961 [(set_attr "type" "neon_stp, store_16")]
2962 )
2964 ;; In this insn, operand 1 should be low, and operand 2 the high part of the
2965 ;; dest vector.
2967 (define_insn "*aarch64_combinez<mode>"
2968 [(set (match_operand:<VDBL> 0 "register_operand" "=w,w,w")
2969 (vec_concat:<VDBL>
2970 (match_operand:VDC 1 "general_operand" "w,?r,m")
2971 (match_operand:VDC 2 "aarch64_simd_or_scalar_imm_zero")))]
2972 "TARGET_SIMD && !BYTES_BIG_ENDIAN"
2973 "@
2974 mov\\t%0.8b, %1.8b
2975 fmov\t%d0, %1
2976 ldr\\t%d0, %1"
2977 [(set_attr "type" "neon_move<q>, neon_from_gp, neon_load1_1reg")
2978 (set_attr "simd" "yes,*,yes")
2979 (set_attr "fp" "*,yes,*")]
2980 )
2982 (define_insn "*aarch64_combinez_be<mode>"
2983 [(set (match_operand:<VDBL> 0 "register_operand" "=w,w,w")
2984 (vec_concat:<VDBL>
2985 (match_operand:VDC 2 "aarch64_simd_or_scalar_imm_zero")
2986 (match_operand:VDC 1 "general_operand" "w,?r,m")))]
2987 "TARGET_SIMD && BYTES_BIG_ENDIAN"
2988 "@
2989 mov\\t%0.8b, %1.8b
2990 fmov\t%d0, %1
2991 ldr\\t%d0, %1"
2992 [(set_attr "type" "neon_move<q>, neon_from_gp, neon_load1_1reg")
2993 (set_attr "simd" "yes,*,yes")
2994 (set_attr "fp" "*,yes,*")]
2995 )
2997 (define_expand "aarch64_combine<mode>"
2998 [(match_operand:<VDBL> 0 "register_operand")
2999 (match_operand:VDC 1 "register_operand")
3000 (match_operand:VDC 2 "register_operand")]
3001 "TARGET_SIMD"
3002 {
3003 aarch64_split_simd_combine (operands[0], operands[1], operands[2]);
3005 DONE;
3006 }
3007 )
3009 (define_expand "aarch64_simd_combine<mode>"
3010 [(match_operand:<VDBL> 0 "register_operand")
3011 (match_operand:VDC 1 "register_operand")
3012 (match_operand:VDC 2 "register_operand")]
3013 "TARGET_SIMD"
3014 {
3015 emit_insn (gen_move_lo_quad_<Vdbl> (operands[0], operands[1]));
3016 emit_insn (gen_move_hi_quad_<Vdbl> (operands[0], operands[2]));
3017 DONE;
3018 }
3019 [(set_attr "type" "multiple")]
3020 )
3022 ;; <su><addsub>l<q>.
3024 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>l<mode>_hi_internal"
3025 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3026 (ADDSUB:<VWIDE> (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
3027 (match_operand:VQW 1 "register_operand" "w")
3028 (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))
3029 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
3030 (match_operand:VQW 2 "register_operand" "w")
3031 (match_dup 3)))))]
3032 "TARGET_SIMD"
3033 "<ANY_EXTEND:su><ADDSUB:optab>l2\t%0.<Vwtype>, %1.<Vtype>, %2.<Vtype>"
3034 [(set_attr "type" "neon_<ADDSUB:optab>_long")]
3035 )
3037 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>l<mode>_lo_internal"
3038 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3039 (ADDSUB:<VWIDE> (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
3040 (match_operand:VQW 1 "register_operand" "w")
3041 (match_operand:VQW 3 "vect_par_cnst_lo_half" "")))
3042 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
3043 (match_operand:VQW 2 "register_operand" "w")
3044 (match_dup 3)))))]
3045 "TARGET_SIMD"
3046 "<ANY_EXTEND:su><ADDSUB:optab>l\t%0.<Vwtype>, %1.<Vhalftype>, %2.<Vhalftype>"
3047 [(set_attr "type" "neon_<ADDSUB:optab>_long")]
3048 )
3051 (define_expand "aarch64_saddl2<mode>"
3052 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3053 (match_operand:VQW 1 "register_operand" "w")
3054 (match_operand:VQW 2 "register_operand" "w")]
3055 "TARGET_SIMD"
3056 {
3057 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, <nunits>, true);
3058 emit_insn (gen_aarch64_saddl<mode>_hi_internal (operands[0], operands[1],
3059 operands[2], p));
3060 DONE;
3061 })
3063 (define_expand "aarch64_uaddl2<mode>"
3064 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3065 (match_operand:VQW 1 "register_operand" "w")
3066 (match_operand:VQW 2 "register_operand" "w")]
3067 "TARGET_SIMD"
3068 {
3069 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, <nunits>, true);
3070 emit_insn (gen_aarch64_uaddl<mode>_hi_internal (operands[0], operands[1],
3071 operands[2], p));
3072 DONE;
3073 })
3075 (define_expand "aarch64_ssubl2<mode>"
3076 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3077 (match_operand:VQW 1 "register_operand" "w")
3078 (match_operand:VQW 2 "register_operand" "w")]
3079 "TARGET_SIMD"
3080 {
3081 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, <nunits>, true);
3082 emit_insn (gen_aarch64_ssubl<mode>_hi_internal (operands[0], operands[1],
3083 operands[2], p));
3084 DONE;
3085 })
3087 (define_expand "aarch64_usubl2<mode>"
3088 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3089 (match_operand:VQW 1 "register_operand" "w")
3090 (match_operand:VQW 2 "register_operand" "w")]
3091 "TARGET_SIMD"
3092 {
3093 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, <nunits>, true);
3094 emit_insn (gen_aarch64_usubl<mode>_hi_internal (operands[0], operands[1],
3095 operands[2], p));
3096 DONE;
3097 })
3099 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>l<mode>"
3100 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3101 (ADDSUB:<VWIDE> (ANY_EXTEND:<VWIDE>
3102 (match_operand:VD_BHSI 1 "register_operand" "w"))
3103 (ANY_EXTEND:<VWIDE>
3104 (match_operand:VD_BHSI 2 "register_operand" "w"))))]
3105 "TARGET_SIMD"
3106 "<ANY_EXTEND:su><ADDSUB:optab>l\t%0.<Vwtype>, %1.<Vtype>, %2.<Vtype>"
3107 [(set_attr "type" "neon_<ADDSUB:optab>_long")]
3108 )
3110 ;; <su><addsub>w<q>.
3112 (define_expand "widen_ssum<mode>3"
3113 [(set (match_operand:<VDBLW> 0 "register_operand" "")
3114 (plus:<VDBLW> (sign_extend:<VDBLW>
3115 (match_operand:VQW 1 "register_operand" ""))
3116 (match_operand:<VDBLW> 2 "register_operand" "")))]
3117 "TARGET_SIMD"
3118 {
3119 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, <nunits>, false);
3120 rtx temp = gen_reg_rtx (GET_MODE (operands[0]));
3122 emit_insn (gen_aarch64_saddw<mode>_internal (temp, operands[2],
3123 operands[1], p));
3124 emit_insn (gen_aarch64_saddw2<mode> (operands[0], temp, operands[1]));
3125 DONE;
3126 }
3127 )
3129 (define_expand "widen_ssum<mode>3"
3130 [(set (match_operand:<VWIDE> 0 "register_operand" "")
3131 (plus:<VWIDE> (sign_extend:<VWIDE>
3132 (match_operand:VD_BHSI 1 "register_operand" ""))
3133 (match_operand:<VWIDE> 2 "register_operand" "")))]
3134 "TARGET_SIMD"
3135 {
3136 emit_insn (gen_aarch64_saddw<mode> (operands[0], operands[2], operands[1]));
3137 DONE;
3138 })
3140 (define_expand "widen_usum<mode>3"
3141 [(set (match_operand:<VDBLW> 0 "register_operand" "")
3142 (plus:<VDBLW> (zero_extend:<VDBLW>
3143 (match_operand:VQW 1 "register_operand" ""))
3144 (match_operand:<VDBLW> 2 "register_operand" "")))]
3145 "TARGET_SIMD"
3146 {
3147 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, <nunits>, false);
3148 rtx temp = gen_reg_rtx (GET_MODE (operands[0]));
3150 emit_insn (gen_aarch64_uaddw<mode>_internal (temp, operands[2],
3151 operands[1], p));
3152 emit_insn (gen_aarch64_uaddw2<mode> (operands[0], temp, operands[1]));
3153 DONE;
3154 }
3155 )
3157 (define_expand "widen_usum<mode>3"
3158 [(set (match_operand:<VWIDE> 0 "register_operand" "")
3159 (plus:<VWIDE> (zero_extend:<VWIDE>
3160 (match_operand:VD_BHSI 1 "register_operand" ""))
3161 (match_operand:<VWIDE> 2 "register_operand" "")))]
3162 "TARGET_SIMD"
3163 {
3164 emit_insn (gen_aarch64_uaddw<mode> (operands[0], operands[2], operands[1]));
3165 DONE;
3166 })
3168 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>w<mode>"
3169 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3170 (ADDSUB:<VWIDE> (match_operand:<VWIDE> 1 "register_operand" "w")
3171 (ANY_EXTEND:<VWIDE>
3172 (match_operand:VD_BHSI 2 "register_operand" "w"))))]
3173 "TARGET_SIMD"
3174 "<ANY_EXTEND:su><ADDSUB:optab>w\\t%0.<Vwtype>, %1.<Vwtype>, %2.<Vtype>"
3175 [(set_attr "type" "neon_<ADDSUB:optab>_widen")]
3176 )
3178 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>w<mode>_internal"
3179 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3180 (ADDSUB:<VWIDE> (match_operand:<VWIDE> 1 "register_operand" "w")
3181 (ANY_EXTEND:<VWIDE>
3182 (vec_select:<VHALF>
3183 (match_operand:VQW 2 "register_operand" "w")
3184 (match_operand:VQW 3 "vect_par_cnst_lo_half" "")))))]
3185 "TARGET_SIMD"
3186 "<ANY_EXTEND:su><ADDSUB:optab>w\\t%0.<Vwtype>, %1.<Vwtype>, %2.<Vhalftype>"
3187 [(set_attr "type" "neon_<ADDSUB:optab>_widen")]
3188 )
3190 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>w2<mode>_internal"
3191 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3192 (ADDSUB:<VWIDE> (match_operand:<VWIDE> 1 "register_operand" "w")
3193 (ANY_EXTEND:<VWIDE>
3194 (vec_select:<VHALF>
3195 (match_operand:VQW 2 "register_operand" "w")
3196 (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))))]
3197 "TARGET_SIMD"
3198 "<ANY_EXTEND:su><ADDSUB:optab>w2\\t%0.<Vwtype>, %1.<Vwtype>, %2.<Vtype>"
3199 [(set_attr "type" "neon_<ADDSUB:optab>_widen")]
3200 )
3202 (define_expand "aarch64_saddw2<mode>"
3203 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3204 (match_operand:<VWIDE> 1 "register_operand" "w")
3205 (match_operand:VQW 2 "register_operand" "w")]
3206 "TARGET_SIMD"
3207 {
3208 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, <nunits>, true);
3209 emit_insn (gen_aarch64_saddw2<mode>_internal (operands[0], operands[1],
3210 operands[2], p));
3211 DONE;
3212 })
3214 (define_expand "aarch64_uaddw2<mode>"
3215 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3216 (match_operand:<VWIDE> 1 "register_operand" "w")
3217 (match_operand:VQW 2 "register_operand" "w")]
3218 "TARGET_SIMD"
3219 {
3220 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, <nunits>, true);
3221 emit_insn (gen_aarch64_uaddw2<mode>_internal (operands[0], operands[1],
3222 operands[2], p));
3223 DONE;
3224 })
3227 (define_expand "aarch64_ssubw2<mode>"
3228 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3229 (match_operand:<VWIDE> 1 "register_operand" "w")
3230 (match_operand:VQW 2 "register_operand" "w")]
3231 "TARGET_SIMD"
3232 {
3233 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, <nunits>, true);
3234 emit_insn (gen_aarch64_ssubw2<mode>_internal (operands[0], operands[1],
3235 operands[2], p));
3236 DONE;
3237 })
3239 (define_expand "aarch64_usubw2<mode>"
3240 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3241 (match_operand:<VWIDE> 1 "register_operand" "w")
3242 (match_operand:VQW 2 "register_operand" "w")]
3243 "TARGET_SIMD"
3244 {
3245 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, <nunits>, true);
3246 emit_insn (gen_aarch64_usubw2<mode>_internal (operands[0], operands[1],
3247 operands[2], p));
3248 DONE;
3249 })
3251 ;; <su><r>h<addsub>.
3253 (define_insn "aarch64_<sur>h<addsub><mode>"
3254 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
3255 (unspec:VDQ_BHSI [(match_operand:VDQ_BHSI 1 "register_operand" "w")
3256 (match_operand:VDQ_BHSI 2 "register_operand" "w")]
3257 HADDSUB))]
3258 "TARGET_SIMD"
3259 "<sur>h<addsub>\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
3260 [(set_attr "type" "neon_<addsub>_halve<q>")]
3261 )
3263 ;; <r><addsub>hn<q>.
3265 (define_insn "aarch64_<sur><addsub>hn<mode>"
3266 [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
3267 (unspec:<VNARROWQ> [(match_operand:VQN 1 "register_operand" "w")
3268 (match_operand:VQN 2 "register_operand" "w")]
3269 ADDSUBHN))]
3270 "TARGET_SIMD"
3271 "<sur><addsub>hn\\t%0.<Vntype>, %1.<Vtype>, %2.<Vtype>"
3272 [(set_attr "type" "neon_<addsub>_halve_narrow_q")]
3273 )
3275 (define_insn "aarch64_<sur><addsub>hn2<mode>"
3276 [(set (match_operand:<VNARROWQ2> 0 "register_operand" "=w")
3277 (unspec:<VNARROWQ2> [(match_operand:<VNARROWQ> 1 "register_operand" "0")
3278 (match_operand:VQN 2 "register_operand" "w")
3279 (match_operand:VQN 3 "register_operand" "w")]
3280 ADDSUBHN2))]
3281 "TARGET_SIMD"
3282 "<sur><addsub>hn2\\t%0.<V2ntype>, %2.<Vtype>, %3.<Vtype>"
3283 [(set_attr "type" "neon_<addsub>_halve_narrow_q")]
3284 )
3286 ;; pmul.
3288 (define_insn "aarch64_pmul<mode>"
3289 [(set (match_operand:VB 0 "register_operand" "=w")
3290 (unspec:VB [(match_operand:VB 1 "register_operand" "w")
3291 (match_operand:VB 2 "register_operand" "w")]
3292 UNSPEC_PMUL))]
3293 "TARGET_SIMD"
3294 "pmul\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
3295 [(set_attr "type" "neon_mul_<Vetype><q>")]
3296 )
3298 ;; fmulx.
3300 (define_insn "aarch64_fmulx<mode>"
3301 [(set (match_operand:VHSDF_HSDF 0 "register_operand" "=w")
3302 (unspec:VHSDF_HSDF
3303 [(match_operand:VHSDF_HSDF 1 "register_operand" "w")
3304 (match_operand:VHSDF_HSDF 2 "register_operand" "w")]
3305 UNSPEC_FMULX))]
3306 "TARGET_SIMD"
3307 "fmulx\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
3308 [(set_attr "type" "neon_fp_mul_<stype>")]
3309 )
3311 ;; vmulxq_lane_f32, and vmulx_laneq_f32
3313 (define_insn "*aarch64_mulx_elt_<vswap_width_name><mode>"
3314 [(set (match_operand:VDQSF 0 "register_operand" "=w")
3315 (unspec:VDQSF
3316 [(match_operand:VDQSF 1 "register_operand" "w")
3317 (vec_duplicate:VDQSF
3318 (vec_select:<VEL>
3319 (match_operand:<VSWAP_WIDTH> 2 "register_operand" "w")
3320 (parallel [(match_operand:SI 3 "immediate_operand" "i")])))]
3321 UNSPEC_FMULX))]
3322 "TARGET_SIMD"
3323 {
3324 operands[3] = aarch64_endian_lane_rtx (<VSWAP_WIDTH>mode, INTVAL (operands[3]));
3325 return "fmulx\t%<v>0<Vmtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3326 }
3327 [(set_attr "type" "neon_fp_mul_<Vetype>_scalar<q>")]
3328 )
3330 ;; vmulxq_laneq_f32, vmulxq_laneq_f64, vmulx_lane_f32
3332 (define_insn "*aarch64_mulx_elt<mode>"
3333 [(set (match_operand:VDQF 0 "register_operand" "=w")
3334 (unspec:VDQF
3335 [(match_operand:VDQF 1 "register_operand" "w")
3336 (vec_duplicate:VDQF
3337 (vec_select:<VEL>
3338 (match_operand:VDQF 2 "register_operand" "w")
3339 (parallel [(match_operand:SI 3 "immediate_operand" "i")])))]
3340 UNSPEC_FMULX))]
3341 "TARGET_SIMD"
3342 {
3343 operands[3] = aarch64_endian_lane_rtx (<MODE>mode, INTVAL (operands[3]));
3344 return "fmulx\t%<v>0<Vmtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3345 }
3346 [(set_attr "type" "neon_fp_mul_<Vetype><q>")]
3347 )
3349 ;; vmulxq_lane
3351 (define_insn "*aarch64_mulx_elt_from_dup<mode>"
3352 [(set (match_operand:VHSDF 0 "register_operand" "=w")
3353 (unspec:VHSDF
3354 [(match_operand:VHSDF 1 "register_operand" "w")
3355 (vec_duplicate:VHSDF
3356 (match_operand:<VEL> 2 "register_operand" "<h_con>"))]
3357 UNSPEC_FMULX))]
3358 "TARGET_SIMD"
3359 "fmulx\t%0.<Vtype>, %1.<Vtype>, %2.<Vetype>[0]";
3360 [(set_attr "type" "neon<fp>_mul_<stype>_scalar<q>")]
3361 )
3363 ;; vmulxs_lane_f32, vmulxs_laneq_f32
3364 ;; vmulxd_lane_f64 == vmulx_lane_f64
3365 ;; vmulxd_laneq_f64 == vmulx_laneq_f64
3367 (define_insn "*aarch64_vgetfmulx<mode>"
3368 [(set (match_operand:<VEL> 0 "register_operand" "=w")
3369 (unspec:<VEL>
3370 [(match_operand:<VEL> 1 "register_operand" "w")
3371 (vec_select:<VEL>
3372 (match_operand:VDQF 2 "register_operand" "w")
3373 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))]
3374 UNSPEC_FMULX))]
3375 "TARGET_SIMD"
3376 {
3377 operands[3] = aarch64_endian_lane_rtx (<MODE>mode, INTVAL (operands[3]));
3378 return "fmulx\t%<Vetype>0, %<Vetype>1, %2.<Vetype>[%3]";
3379 }
3380 [(set_attr "type" "fmul<Vetype>")]
3381 )
3382 ;; <su>q<addsub>
3384 (define_insn "aarch64_<su_optab><optab><mode>"
3385 [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
3386 (BINQOPS:VSDQ_I (match_operand:VSDQ_I 1 "register_operand" "w")
3387 (match_operand:VSDQ_I 2 "register_operand" "w")))]
3388 "TARGET_SIMD"
3389 "<su_optab><optab>\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
3390 [(set_attr "type" "neon_<optab><q>")]
3391 )
3393 ;; suqadd and usqadd
3395 (define_insn "aarch64_<sur>qadd<mode>"
3396 [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
3397 (unspec:VSDQ_I [(match_operand:VSDQ_I 1 "register_operand" "0")
3398 (match_operand:VSDQ_I 2 "register_operand" "w")]
3399 USSUQADD))]
3400 "TARGET_SIMD"
3401 "<sur>qadd\\t%<v>0<Vmtype>, %<v>2<Vmtype>"
3402 [(set_attr "type" "neon_qadd<q>")]
3403 )
3405 ;; sqmovun
3407 (define_insn "aarch64_sqmovun<mode>"
3408 [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
3409 (unspec:<VNARROWQ> [(match_operand:VSQN_HSDI 1 "register_operand" "w")]
3410 UNSPEC_SQXTUN))]
3411 "TARGET_SIMD"
3412 "sqxtun\\t%<vn2>0<Vmntype>, %<v>1<Vmtype>"
3413 [(set_attr "type" "neon_sat_shift_imm_narrow_q")]
3414 )
3416 ;; sqmovn and uqmovn
3418 (define_insn "aarch64_<sur>qmovn<mode>"
3419 [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
3420 (unspec:<VNARROWQ> [(match_operand:VSQN_HSDI 1 "register_operand" "w")]
3421 SUQMOVN))]
3422 "TARGET_SIMD"
3423 "<sur>qxtn\\t%<vn2>0<Vmntype>, %<v>1<Vmtype>"
3424 [(set_attr "type" "neon_sat_shift_imm_narrow_q")]
3425 )
3427 ;; <su>q<absneg>
3429 (define_insn "aarch64_s<optab><mode>"
3430 [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
3431 (UNQOPS:VSDQ_I
3432 (match_operand:VSDQ_I 1 "register_operand" "w")))]
3433 "TARGET_SIMD"
3434 "s<optab>\\t%<v>0<Vmtype>, %<v>1<Vmtype>"
3435 [(set_attr "type" "neon_<optab><q>")]
3436 )
3438 ;; sq<r>dmulh.
3440 (define_insn "aarch64_sq<r>dmulh<mode>"
3441 [(set (match_operand:VSDQ_HSI 0 "register_operand" "=w")
3442 (unspec:VSDQ_HSI
3443 [(match_operand:VSDQ_HSI 1 "register_operand" "w")
3444 (match_operand:VSDQ_HSI 2 "register_operand" "w")]
3445 VQDMULH))]
3446 "TARGET_SIMD"
3447 "sq<r>dmulh\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
3448 [(set_attr "type" "neon_sat_mul_<Vetype><q>")]
3449 )
3451 ;; sq<r>dmulh_lane
3453 (define_insn "aarch64_sq<r>dmulh_lane<mode>"
3454 [(set (match_operand:VDQHS 0 "register_operand" "=w")
3455 (unspec:VDQHS
3456 [(match_operand:VDQHS 1 "register_operand" "w")
3457 (vec_select:<VEL>
3458 (match_operand:<VCOND> 2 "register_operand" "<vwx>")
3459 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))]
3460 VQDMULH))]
3461 "TARGET_SIMD"
3462 "*
3463 operands[3] = aarch64_endian_lane_rtx (<VCOND>mode, INTVAL (operands[3]));
3464 return \"sq<r>dmulh\\t%0.<Vtype>, %1.<Vtype>, %2.<Vetype>[%3]\";"
3465 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar<q>")]
3466 )
3468 (define_insn "aarch64_sq<r>dmulh_laneq<mode>"
3469 [(set (match_operand:VDQHS 0 "register_operand" "=w")
3470 (unspec:VDQHS
3471 [(match_operand:VDQHS 1 "register_operand" "w")
3472 (vec_select:<VEL>
3473 (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
3474 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))]
3475 VQDMULH))]
3476 "TARGET_SIMD"
3477 "*
3478 operands[3] = aarch64_endian_lane_rtx (<VCONQ>mode, INTVAL (operands[3]));
3479 return \"sq<r>dmulh\\t%0.<Vtype>, %1.<Vtype>, %2.<Vetype>[%3]\";"
3480 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar<q>")]
3481 )
3483 (define_insn "aarch64_sq<r>dmulh_lane<mode>"
3484 [(set (match_operand:SD_HSI 0 "register_operand" "=w")
3485 (unspec:SD_HSI
3486 [(match_operand:SD_HSI 1 "register_operand" "w")
3487 (vec_select:<VEL>
3488 (match_operand:<VCOND> 2 "register_operand" "<vwx>")
3489 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))]
3490 VQDMULH))]
3491 "TARGET_SIMD"
3492 "*
3493 operands[3] = aarch64_endian_lane_rtx (<VCOND>mode, INTVAL (operands[3]));
3494 return \"sq<r>dmulh\\t%<v>0, %<v>1, %2.<v>[%3]\";"
3495 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar<q>")]
3496 )
3498 (define_insn "aarch64_sq<r>dmulh_laneq<mode>"
3499 [(set (match_operand:SD_HSI 0 "register_operand" "=w")
3500 (unspec:SD_HSI
3501 [(match_operand:SD_HSI 1 "register_operand" "w")
3502 (vec_select:<VEL>
3503 (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
3504 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))]
3505 VQDMULH))]
3506 "TARGET_SIMD"
3507 "*
3508 operands[3] = aarch64_endian_lane_rtx (<VCONQ>mode, INTVAL (operands[3]));
3509 return \"sq<r>dmulh\\t%<v>0, %<v>1, %2.<v>[%3]\";"
3510 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar<q>")]
3511 )
3513 ;; sqrdml[as]h.
3515 (define_insn "aarch64_sqrdml<SQRDMLH_AS:rdma_as>h<mode>"
3516 [(set (match_operand:VSDQ_HSI 0 "register_operand" "=w")
3517 (unspec:VSDQ_HSI
3518 [(match_operand:VSDQ_HSI 1 "register_operand" "0")
3519 (match_operand:VSDQ_HSI 2 "register_operand" "w")
3520 (match_operand:VSDQ_HSI 3 "register_operand" "w")]
3521 SQRDMLH_AS))]
3522 "TARGET_SIMD_RDMA"
3523 "sqrdml<SQRDMLH_AS:rdma_as>h\\t%<v>0<Vmtype>, %<v>2<Vmtype>, %<v>3<Vmtype>"
3524 [(set_attr "type" "neon_sat_mla_<Vetype>_long")]
3525 )
3527 ;; sqrdml[as]h_lane.
3529 (define_insn "aarch64_sqrdml<SQRDMLH_AS:rdma_as>h_lane<mode>"
3530 [(set (match_operand:VDQHS 0 "register_operand" "=w")
3531 (unspec:VDQHS
3532 [(match_operand:VDQHS 1 "register_operand" "0")
3533 (match_operand:VDQHS 2 "register_operand" "w")
3534 (vec_select:<VEL>
3535 (match_operand:<VCOND> 3 "register_operand" "<vwx>")
3536 (parallel [(match_operand:SI 4 "immediate_operand" "i")]))]
3537 SQRDMLH_AS))]
3538 "TARGET_SIMD_RDMA"
3539 {
3540 operands[4] = aarch64_endian_lane_rtx (<VCOND>mode, INTVAL (operands[4]));
3541 return
3542 "sqrdml<SQRDMLH_AS:rdma_as>h\\t%0.<Vtype>, %2.<Vtype>, %3.<Vetype>[%4]";
3543 }
3544 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3545 )
3547 (define_insn "aarch64_sqrdml<SQRDMLH_AS:rdma_as>h_lane<mode>"
3548 [(set (match_operand:SD_HSI 0 "register_operand" "=w")
3549 (unspec:SD_HSI
3550 [(match_operand:SD_HSI 1 "register_operand" "0")
3551 (match_operand:SD_HSI 2 "register_operand" "w")
3552 (vec_select:<VEL>
3553 (match_operand:<VCOND> 3 "register_operand" "<vwx>")
3554 (parallel [(match_operand:SI 4 "immediate_operand" "i")]))]
3555 SQRDMLH_AS))]
3556 "TARGET_SIMD_RDMA"
3557 {
3558 operands[4] = aarch64_endian_lane_rtx (<VCOND>mode, INTVAL (operands[4]));
3559 return
3560 "sqrdml<SQRDMLH_AS:rdma_as>h\\t%<v>0, %<v>2, %3.<Vetype>[%4]";
3561 }
3562 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3563 )
3565 ;; sqrdml[as]h_laneq.
3567 (define_insn "aarch64_sqrdml<SQRDMLH_AS:rdma_as>h_laneq<mode>"
3568 [(set (match_operand:VDQHS 0 "register_operand" "=w")
3569 (unspec:VDQHS
3570 [(match_operand:VDQHS 1 "register_operand" "0")
3571 (match_operand:VDQHS 2 "register_operand" "w")
3572 (vec_select:<VEL>
3573 (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
3574 (parallel [(match_operand:SI 4 "immediate_operand" "i")]))]
3575 SQRDMLH_AS))]
3576 "TARGET_SIMD_RDMA"
3577 {
3578 operands[4] = aarch64_endian_lane_rtx (<VCONQ>mode, INTVAL (operands[4]));
3579 return
3580 "sqrdml<SQRDMLH_AS:rdma_as>h\\t%0.<Vtype>, %2.<Vtype>, %3.<Vetype>[%4]";
3581 }
3582 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3583 )
3585 (define_insn "aarch64_sqrdml<SQRDMLH_AS:rdma_as>h_laneq<mode>"
3586 [(set (match_operand:SD_HSI 0 "register_operand" "=w")
3587 (unspec:SD_HSI
3588 [(match_operand:SD_HSI 1 "register_operand" "0")
3589 (match_operand:SD_HSI 2 "register_operand" "w")
3590 (vec_select:<VEL>
3591 (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
3592 (parallel [(match_operand:SI 4 "immediate_operand" "i")]))]
3593 SQRDMLH_AS))]
3594 "TARGET_SIMD_RDMA"
3595 {
3596 operands[4] = aarch64_endian_lane_rtx (<VCONQ>mode, INTVAL (operands[4]));
3597 return
3598 "sqrdml<SQRDMLH_AS:rdma_as>h\\t%<v>0, %<v>2, %3.<v>[%4]";
3599 }
3600 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3601 )
3603 ;; vqdml[sa]l
3605 (define_insn "aarch64_sqdml<SBINQOPS:as>l<mode>"
3606 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3607 (SBINQOPS:<VWIDE>
3608 (match_operand:<VWIDE> 1 "register_operand" "0")
3609 (ss_ashift:<VWIDE>
3610 (mult:<VWIDE>
3611 (sign_extend:<VWIDE>
3612 (match_operand:VSD_HSI 2 "register_operand" "w"))
3613 (sign_extend:<VWIDE>
3614 (match_operand:VSD_HSI 3 "register_operand" "w")))
3615 (const_int 1))))]
3616 "TARGET_SIMD"
3617 "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %<v>3<Vmtype>"
3618 [(set_attr "type" "neon_sat_mla_<Vetype>_long")]
3619 )
3621 ;; vqdml[sa]l_lane
3623 (define_insn "aarch64_sqdml<SBINQOPS:as>l_lane<mode>"
3624 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3625 (SBINQOPS:<VWIDE>
3626 (match_operand:<VWIDE> 1 "register_operand" "0")
3627 (ss_ashift:<VWIDE>
3628 (mult:<VWIDE>
3629 (sign_extend:<VWIDE>
3630 (match_operand:VD_HSI 2 "register_operand" "w"))
3631 (sign_extend:<VWIDE>
3632 (vec_duplicate:VD_HSI
3633 (vec_select:<VEL>
3634 (match_operand:<VCOND> 3 "register_operand" "<vwx>")
3635 (parallel [(match_operand:SI 4 "immediate_operand" "i")])))
3636 ))
3637 (const_int 1))))]
3638 "TARGET_SIMD"
3639 {
3640 operands[4] = aarch64_endian_lane_rtx (<VCOND>mode, INTVAL (operands[4]));
3641 return
3642 "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
3643 }
3644 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3645 )
3647 (define_insn "aarch64_sqdml<SBINQOPS:as>l_laneq<mode>"
3648 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3649 (SBINQOPS:<VWIDE>
3650 (match_operand:<VWIDE> 1 "register_operand" "0")
3651 (ss_ashift:<VWIDE>
3652 (mult:<VWIDE>
3653 (sign_extend:<VWIDE>
3654 (match_operand:VD_HSI 2 "register_operand" "w"))
3655 (sign_extend:<VWIDE>
3656 (vec_duplicate:VD_HSI
3657 (vec_select:<VEL>
3658 (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
3659 (parallel [(match_operand:SI 4 "immediate_operand" "i")])))
3660 ))
3661 (const_int 1))))]
3662 "TARGET_SIMD"
3663 {
3664 operands[4] = aarch64_endian_lane_rtx (<VCONQ>mode, INTVAL (operands[4]));
3665 return
3666 "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
3667 }
3668 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3669 )
3671 (define_insn "aarch64_sqdml<SBINQOPS:as>l_lane<mode>"
3672 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3673 (SBINQOPS:<VWIDE>
3674 (match_operand:<VWIDE> 1 "register_operand" "0")
3675 (ss_ashift:<VWIDE>
3676 (mult:<VWIDE>
3677 (sign_extend:<VWIDE>
3678 (match_operand:SD_HSI 2 "register_operand" "w"))
3679 (sign_extend:<VWIDE>
3680 (vec_select:<VEL>
3681 (match_operand:<VCOND> 3 "register_operand" "<vwx>")
3682 (parallel [(match_operand:SI 4 "immediate_operand" "i")])))
3683 )
3684 (const_int 1))))]
3685 "TARGET_SIMD"
3686 {
3687 operands[4] = aarch64_endian_lane_rtx (<VCOND>mode, INTVAL (operands[4]));
3688 return
3689 "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
3690 }
3691 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3692 )
3694 (define_insn "aarch64_sqdml<SBINQOPS:as>l_laneq<mode>"
3695 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3696 (SBINQOPS:<VWIDE>
3697 (match_operand:<VWIDE> 1 "register_operand" "0")
3698 (ss_ashift:<VWIDE>
3699 (mult:<VWIDE>
3700 (sign_extend:<VWIDE>
3701 (match_operand:SD_HSI 2 "register_operand" "w"))
3702 (sign_extend:<VWIDE>
3703 (vec_select:<VEL>
3704 (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
3705 (parallel [(match_operand:SI 4 "immediate_operand" "i")])))
3706 )
3707 (const_int 1))))]
3708 "TARGET_SIMD"
3709 {
3710 operands[4] = aarch64_endian_lane_rtx (<VCONQ>mode, INTVAL (operands[4]));
3711 return
3712 "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
3713 }
3714 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3715 )
3717 ;; vqdml[sa]l_n
3719 (define_insn "aarch64_sqdml<SBINQOPS:as>l_n<mode>"
3720 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3721 (SBINQOPS:<VWIDE>
3722 (match_operand:<VWIDE> 1 "register_operand" "0")
3723 (ss_ashift:<VWIDE>
3724 (mult:<VWIDE>
3725 (sign_extend:<VWIDE>
3726 (match_operand:VD_HSI 2 "register_operand" "w"))
3727 (sign_extend:<VWIDE>
3728 (vec_duplicate:VD_HSI
3729 (match_operand:<VEL> 3 "register_operand" "<vwx>"))))
3730 (const_int 1))))]
3731 "TARGET_SIMD"
3732 "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[0]"
3733 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3734 )
3736 ;; sqdml[as]l2
3738 (define_insn "aarch64_sqdml<SBINQOPS:as>l2<mode>_internal"
3739 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3740 (SBINQOPS:<VWIDE>
3741 (match_operand:<VWIDE> 1 "register_operand" "0")
3742 (ss_ashift:<VWIDE>
3743 (mult:<VWIDE>
3744 (sign_extend:<VWIDE>
3745 (vec_select:<VHALF>
3746 (match_operand:VQ_HSI 2 "register_operand" "w")
3747 (match_operand:VQ_HSI 4 "vect_par_cnst_hi_half" "")))
3748 (sign_extend:<VWIDE>
3749 (vec_select:<VHALF>
3750 (match_operand:VQ_HSI 3 "register_operand" "w")
3751 (match_dup 4))))
3752 (const_int 1))))]
3753 "TARGET_SIMD"
3754 "sqdml<SBINQOPS:as>l2\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %<v>3<Vmtype>"
3755 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3756 )
3758 (define_expand "aarch64_sqdmlal2<mode>"
3759 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3760 (match_operand:<VWIDE> 1 "register_operand" "w")
3761 (match_operand:VQ_HSI 2 "register_operand" "w")
3762 (match_operand:VQ_HSI 3 "register_operand" "w")]
3763 "TARGET_SIMD"
3764 {
3765 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, <nunits>, true);
3766 emit_insn (gen_aarch64_sqdmlal2<mode>_internal (operands[0], operands[1],
3767 operands[2], operands[3], p));
3768 DONE;
3769 })
3771 (define_expand "aarch64_sqdmlsl2<mode>"
3772 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3773 (match_operand:<VWIDE> 1 "register_operand" "w")
3774 (match_operand:VQ_HSI 2 "register_operand" "w")
3775 (match_operand:VQ_HSI 3 "register_operand" "w")]
3776 "TARGET_SIMD"
3777 {
3778 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, <nunits>, true);
3779 emit_insn (gen_aarch64_sqdmlsl2<mode>_internal (operands[0], operands[1],
3780 operands[2], operands[3], p));
3781 DONE;
3782 })
3784 ;; vqdml[sa]l2_lane
3786 (define_insn "aarch64_sqdml<SBINQOPS:as>l2_lane<mode>_internal"
3787 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3788 (SBINQOPS:<VWIDE>
3789 (match_operand:<VWIDE> 1 "register_operand" "0")
3790 (ss_ashift:<VWIDE>
3791 (mult:<VWIDE>
3792 (sign_extend:<VWIDE>
3793 (vec_select:<VHALF>
3794 (match_operand:VQ_HSI 2 "register_operand" "w")
3795 (match_operand:VQ_HSI 5 "vect_par_cnst_hi_half" "")))
3796 (sign_extend:<VWIDE>
3797 (vec_duplicate:<VHALF>
3798 (vec_select:<VEL>
3799 (match_operand:<VCOND> 3 "register_operand" "<vwx>")
3800 (parallel [(match_operand:SI 4 "immediate_operand" "i")])
3801 ))))
3802 (const_int 1))))]
3803 "TARGET_SIMD"
3804 {
3805 operands[4] = aarch64_endian_lane_rtx (<VCOND>mode, INTVAL (operands[4]));
3806 return
3807 "sqdml<SBINQOPS:as>l2\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
3808 }
3809 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3810 )
3812 (define_insn "aarch64_sqdml<SBINQOPS:as>l2_laneq<mode>_internal"
3813 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3814 (SBINQOPS:<VWIDE>
3815 (match_operand:<VWIDE> 1 "register_operand" "0")
3816 (ss_ashift:<VWIDE>
3817 (mult:<VWIDE>
3818 (sign_extend:<VWIDE>
3819 (vec_select:<VHALF>
3820 (match_operand:VQ_HSI 2 "register_operand" "w")
3821 (match_operand:VQ_HSI 5 "vect_par_cnst_hi_half" "")))
3822 (sign_extend:<VWIDE>
3823 (vec_duplicate:<VHALF>
3824 (vec_select:<VEL>
3825 (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
3826 (parallel [(match_operand:SI 4 "immediate_operand" "i")])
3827 ))))
3828 (const_int 1))))]
3829 "TARGET_SIMD"
3830 {
3831 operands[4] = aarch64_endian_lane_rtx (<VCONQ>mode, INTVAL (operands[4]));
3832 return
3833 "sqdml<SBINQOPS:as>l2\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
3834 }
3835 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3836 )
3838 (define_expand "aarch64_sqdmlal2_lane<mode>"
3839 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3840 (match_operand:<VWIDE> 1 "register_operand" "w")
3841 (match_operand:VQ_HSI 2 "register_operand" "w")
3842 (match_operand:<VCOND> 3 "register_operand" "<vwx>")
3843 (match_operand:SI 4 "immediate_operand" "i")]
3844 "TARGET_SIMD"
3845 {
3846 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, <nunits>, true);
3847 emit_insn (gen_aarch64_sqdmlal2_lane<mode>_internal (operands[0], operands[1],
3848 operands[2], operands[3],
3849 operands[4], p));
3850 DONE;
3851 })
3853 (define_expand "aarch64_sqdmlal2_laneq<mode>"
3854 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3855 (match_operand:<VWIDE> 1 "register_operand" "w")
3856 (match_operand:VQ_HSI 2 "register_operand" "w")
3857 (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
3858 (match_operand:SI 4 "immediate_operand" "i")]
3859 "TARGET_SIMD"
3860 {
3861 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, <nunits>, true);
3862 emit_insn (gen_aarch64_sqdmlal2_laneq<mode>_internal (operands[0], operands[1],
3863 operands[2], operands[3],
3864 operands[4], p));
3865 DONE;
3866 })
3868 (define_expand "aarch64_sqdmlsl2_lane<mode>"
3869 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3870 (match_operand:<VWIDE> 1 "register_operand" "w")
3871 (match_operand:VQ_HSI 2 "register_operand" "w")
3872 (match_operand:<VCOND> 3 "register_operand" "<vwx>")
3873 (match_operand:SI 4 "immediate_operand" "i")]
3874 "TARGET_SIMD"
3875 {
3876 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, <nunits>, true);
3877 emit_insn (gen_aarch64_sqdmlsl2_lane<mode>_internal (operands[0], operands[1],
3878 operands[2], operands[3],
3879 operands[4], p));
3880 DONE;
3881 })
3883 (define_expand "aarch64_sqdmlsl2_laneq<mode>"
3884 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3885 (match_operand:<VWIDE> 1 "register_operand" "w")
3886 (match_operand:VQ_HSI 2 "register_operand" "w")
3887 (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
3888 (match_operand:SI 4 "immediate_operand" "i")]
3889 "TARGET_SIMD"
3890 {
3891 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, <nunits>, true);
3892 emit_insn (gen_aarch64_sqdmlsl2_laneq<mode>_internal (operands[0], operands[1],
3893 operands[2], operands[3],
3894 operands[4], p));
3895 DONE;
3896 })
3898 (define_insn "aarch64_sqdml<SBINQOPS:as>l2_n<mode>_internal"
3899 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3900 (SBINQOPS:<VWIDE>
3901 (match_operand:<VWIDE> 1 "register_operand" "0")
3902 (ss_ashift:<VWIDE>
3903 (mult:<VWIDE>
3904 (sign_extend:<VWIDE>
3905 (vec_select:<VHALF>
3906 (match_operand:VQ_HSI 2 "register_operand" "w")
3907 (match_operand:VQ_HSI 4 "vect_par_cnst_hi_half" "")))
3908 (sign_extend:<VWIDE>
3909 (vec_duplicate:<VHALF>
3910 (match_operand:<VEL> 3 "register_operand" "<vwx>"))))
3911 (const_int 1))))]
3912 "TARGET_SIMD"
3913 "sqdml<SBINQOPS:as>l2\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[0]"
3914 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3915 )
3917 (define_expand "aarch64_sqdmlal2_n<mode>"
3918 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3919 (match_operand:<VWIDE> 1 "register_operand" "w")
3920 (match_operand:VQ_HSI 2 "register_operand" "w")
3921 (match_operand:<VEL> 3 "register_operand" "w")]
3922 "TARGET_SIMD"
3923 {
3924 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, <nunits>, true);
3925 emit_insn (gen_aarch64_sqdmlal2_n<mode>_internal (operands[0], operands[1],
3926 operands[2], operands[3],
3927 p));
3928 DONE;
3929 })
3931 (define_expand "aarch64_sqdmlsl2_n<mode>"
3932 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3933 (match_operand:<VWIDE> 1 "register_operand" "w")
3934 (match_operand:VQ_HSI 2 "register_operand" "w")
3935 (match_operand:<VEL> 3 "register_operand" "w")]
3936 "TARGET_SIMD"
3937 {
3938 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, <nunits>, true);
3939 emit_insn (gen_aarch64_sqdmlsl2_n<mode>_internal (operands[0], operands[1],
3940 operands[2], operands[3],
3941 p));
3942 DONE;
3943 })
3945 ;; vqdmull
3947 (define_insn "aarch64_sqdmull<mode>"
3948 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3949 (ss_ashift:<VWIDE>
3950 (mult:<VWIDE>
3951 (sign_extend:<VWIDE>
3952 (match_operand:VSD_HSI 1 "register_operand" "w"))
3953 (sign_extend:<VWIDE>
3954 (match_operand:VSD_HSI 2 "register_operand" "w")))
3955 (const_int 1)))]
3956 "TARGET_SIMD"
3957 "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
3958 [(set_attr "type" "neon_sat_mul_<Vetype>_long")]
3959 )
3961 ;; vqdmull_lane
3963 (define_insn "aarch64_sqdmull_lane<mode>"
3964 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3965 (ss_ashift:<VWIDE>
3966 (mult:<VWIDE>
3967 (sign_extend:<VWIDE>
3968 (match_operand:VD_HSI 1 "register_operand" "w"))
3969 (sign_extend:<VWIDE>
3970 (vec_duplicate:VD_HSI
3971 (vec_select:<VEL>
3972 (match_operand:<VCOND> 2 "register_operand" "<vwx>")
3973 (parallel [(match_operand:SI 3 "immediate_operand" "i")])))
3974 ))
3975 (const_int 1)))]
3976 "TARGET_SIMD"
3977 {
3978 operands[3] = aarch64_endian_lane_rtx (<VCOND>mode, INTVAL (operands[3]));
3979 return "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3980 }
3981 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3982 )
3984 (define_insn "aarch64_sqdmull_laneq<mode>"
3985 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3986 (ss_ashift:<VWIDE>
3987 (mult:<VWIDE>
3988 (sign_extend:<VWIDE>
3989 (match_operand:VD_HSI 1 "register_operand" "w"))
3990 (sign_extend:<VWIDE>
3991 (vec_duplicate:VD_HSI
3992 (vec_select:<VEL>
3993 (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
3994 (parallel [(match_operand:SI 3 "immediate_operand" "i")])))
3995 ))
3996 (const_int 1)))]
3997 "TARGET_SIMD"
3998 {
3999 operands[3] = aarch64_endian_lane_rtx (<VCONQ>mode, INTVAL (operands[3]));
4000 return "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
4001 }
4002 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
4003 )
4005 (define_insn "aarch64_sqdmull_lane<mode>"
4006 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
4007 (ss_ashift:<VWIDE>
4008 (mult:<VWIDE>
4009 (sign_extend:<VWIDE>
4010 (match_operand:SD_HSI 1 "register_operand" "w"))
4011 (sign_extend:<VWIDE>
4012 (vec_select:<VEL>
4013 (match_operand:<VCOND> 2 "register_operand" "<vwx>")
4014 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))
4015 ))
4016 (const_int 1)))]
4017 "TARGET_SIMD"
4018 {
4019 operands[3] = aarch64_endian_lane_rtx (<VCOND>mode, INTVAL (operands[3]));
4020 return "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
4021 }
4022 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
4023 )
4025 (define_insn "aarch64_sqdmull_laneq<mode>"
4026 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
4027 (ss_ashift:<VWIDE>
4028 (mult:<VWIDE>
4029 (sign_extend:<VWIDE>
4030 (match_operand:SD_HSI 1 "register_operand" "w"))
4031 (sign_extend:<VWIDE>
4032 (vec_select:<VEL>
4033 (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
4034 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))
4035 ))
4036 (const_int 1)))]
4037 "TARGET_SIMD"
4038 {
4039 operands[3] = aarch64_endian_lane_rtx (<VCONQ>mode, INTVAL (operands[3]));
4040 return "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
4041 }
4042 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
4043 )
4045 ;; vqdmull_n
4047 (define_insn "aarch64_sqdmull_n<mode>"
4048 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
4049 (ss_ashift:<VWIDE>
4050 (mult:<VWIDE>
4051 (sign_extend:<VWIDE>
4052 (match_operand:VD_HSI 1 "register_operand" "w"))
4053 (sign_extend:<VWIDE>
4054 (vec_duplicate:VD_HSI
4055 (match_operand:<VEL> 2 "register_operand" "<vwx>")))
4056 )
4057 (const_int 1)))]
4058 "TARGET_SIMD"
4059 "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[0]"
4060 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
4061 )
4063 ;; vqdmull2
4067 (define_insn "aarch64_sqdmull2<mode>_internal"
4068 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
4069 (ss_ashift:<VWIDE>
4070 (mult:<VWIDE>
4071 (sign_extend:<VWIDE>
4072 (vec_select:<VHALF>
4073 (match_operand:VQ_HSI 1 "register_operand" "w")
4074 (match_operand:VQ_HSI 3 "vect_par_cnst_hi_half" "")))
4075 (sign_extend:<VWIDE>
4076 (vec_select:<VHALF>
4077 (match_operand:VQ_HSI 2 "register_operand" "w")
4078 (match_dup 3)))
4079 )
4080 (const_int 1)))]
4081 "TARGET_SIMD"
4082 "sqdmull2\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
4083 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
4084 )
4086 (define_expand "aarch64_sqdmull2<mode>"
4087 [(match_operand:<VWIDE> 0 "register_operand" "=w")
4088 (match_operand:VQ_HSI 1 "register_operand" "w")
4089 (match_operand:VQ_HSI 2 "register_operand" "w")]
4090 "TARGET_SIMD"
4091 {
4092 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, <nunits>, true);
4093 emit_insn (gen_aarch64_sqdmull2<mode>_internal (operands[0], operands[1],
4094 operands[2], p));
4095 DONE;
4096 })
4098 ;; vqdmull2_lane
4100 (define_insn "aarch64_sqdmull2_lane<mode>_internal"
4101 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
4102 (ss_ashift:<VWIDE>
4103 (mult:<VWIDE>
4104 (sign_extend:<VWIDE>
4105 (vec_select:<VHALF>
4106 (match_operand:VQ_HSI 1 "register_operand" "w")
4107 (match_operand:VQ_HSI 4 "vect_par_cnst_hi_half" "")))
4108 (sign_extend:<VWIDE>
4109 (vec_duplicate:<VHALF>
4110 (vec_select:<VEL>
4111 (match_operand:<VCOND> 2 "register_operand" "<vwx>")
4112 (parallel [(match_operand:SI 3 "immediate_operand" "i")])))
4113 ))
4114 (const_int 1)))]
4115 "TARGET_SIMD"
4116 {
4117 operands[3] = aarch64_endian_lane_rtx (<VCOND>mode, INTVAL (operands[3]));
4118 return "sqdmull2\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
4119 }
4120 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
4121 )
4123 (define_insn "aarch64_sqdmull2_laneq<mode>_internal"
4124 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
4125 (ss_ashift:<VWIDE>
4126 (mult:<VWIDE>
4127 (sign_extend:<VWIDE>
4128 (vec_select:<VHALF>
4129 (match_operand:VQ_HSI 1 "register_operand" "w")
4130 (match_operand:VQ_HSI 4 "vect_par_cnst_hi_half" "")))
4131 (sign_extend:<VWIDE>
4132 (vec_duplicate:<VHALF>
4133 (vec_select:<VEL>
4134 (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
4135 (parallel [(match_operand:SI 3 "immediate_operand" "i")])))
4136 ))
4137 (const_int 1)))]
4138 "TARGET_SIMD"
4139 {
4140 operands[3] = aarch64_endian_lane_rtx (<VCONQ>mode, INTVAL (operands[3]));
4141 return "sqdmull2\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
4142 }
4143 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
4144 )
4146 (define_expand "aarch64_sqdmull2_lane<mode>"
4147 [(match_operand:<VWIDE> 0 "register_operand" "=w")
4148 (match_operand:VQ_HSI 1 "register_operand" "w")
4149 (match_operand:<VCOND> 2 "register_operand" "<vwx>")
4150 (match_operand:SI 3 "immediate_operand" "i")]
4151 "TARGET_SIMD"
4152 {
4153 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, <nunits>, true);
4154 emit_insn (gen_aarch64_sqdmull2_lane<mode>_internal (operands[0], operands[1],
4155 operands[2], operands[3],
4156 p));
4157 DONE;
4158 })
4160 (define_expand "aarch64_sqdmull2_laneq<mode>"
4161 [(match_operand:<VWIDE> 0 "register_operand" "=w")
4162 (match_operand:VQ_HSI 1 "register_operand" "w")
4163 (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
4164 (match_operand:SI 3 "immediate_operand" "i")]
4165 "TARGET_SIMD"
4166 {
4167 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, <nunits>, true);
4168 emit_insn (gen_aarch64_sqdmull2_laneq<mode>_internal (operands[0], operands[1],
4169 operands[2], operands[3],
4170 p));
4171 DONE;
4172 })
4174 ;; vqdmull2_n
4176 (define_insn "aarch64_sqdmull2_n<mode>_internal"
4177 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
4178 (ss_ashift:<VWIDE>
4179 (mult:<VWIDE>
4180 (sign_extend:<VWIDE>
4181 (vec_select:<VHALF>
4182 (match_operand:VQ_HSI 1 "register_operand" "w")
4183 (match_operand:VQ_HSI 3 "vect_par_cnst_hi_half" "")))
4184 (sign_extend:<VWIDE>
4185 (vec_duplicate:<VHALF>
4186 (match_operand:<VEL> 2 "register_operand" "<vwx>")))
4187 )
4188 (const_int 1)))]
4189 "TARGET_SIMD"
4190 "sqdmull2\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[0]"
4191 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
4192 )
4194 (define_expand "aarch64_sqdmull2_n<mode>"
4195 [(match_operand:<VWIDE> 0 "register_operand" "=w")
4196 (match_operand:VQ_HSI 1 "register_operand" "w")
4197 (match_operand:<VEL> 2 "register_operand" "w")]
4198 "TARGET_SIMD"
4199 {
4200 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, <nunits>, true);
4201 emit_insn (gen_aarch64_sqdmull2_n<mode>_internal (operands[0], operands[1],
4202 operands[2], p));
4203 DONE;
4204 })
4206 ;; vshl
4208 (define_insn "aarch64_<sur>shl<mode>"
4209 [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
4210 (unspec:VSDQ_I_DI
4211 [(match_operand:VSDQ_I_DI 1 "register_operand" "w")
4212 (match_operand:VSDQ_I_DI 2 "register_operand" "w")]
4213 VSHL))]
4214 "TARGET_SIMD"
4215 "<sur>shl\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>";
4216 [(set_attr "type" "neon_shift_reg<q>")]
4217 )
4220 ;; vqshl
4222 (define_insn "aarch64_<sur>q<r>shl<mode>"
4223 [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
4224 (unspec:VSDQ_I
4225 [(match_operand:VSDQ_I 1 "register_operand" "w")
4226 (match_operand:VSDQ_I 2 "register_operand" "w")]
4227 VQSHL))]
4228 "TARGET_SIMD"
4229 "<sur>q<r>shl\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>";
4230 [(set_attr "type" "neon_sat_shift_reg<q>")]
4231 )
4233 ;; vshll_n
4235 (define_insn "aarch64_<sur>shll_n<mode>"
4236 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
4237 (unspec:<VWIDE> [(match_operand:VD_BHSI 1 "register_operand" "w")
4238 (match_operand:SI 2
4239 "aarch64_simd_shift_imm_bitsize_<ve_mode>" "i")]
4240 VSHLL))]
4241 "TARGET_SIMD"
4242 {
4243 if (INTVAL (operands[2]) == GET_MODE_UNIT_BITSIZE (<MODE>mode))
4244 return "shll\\t%0.<Vwtype>, %1.<Vtype>, %2";
4245 else
4246 return "<sur>shll\\t%0.<Vwtype>, %1.<Vtype>, %2";
4247 }
4248 [(set_attr "type" "neon_shift_imm_long")]
4249 )
4251 ;; vshll_high_n
4253 (define_insn "aarch64_<sur>shll2_n<mode>"
4254 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
4255 (unspec:<VWIDE> [(match_operand:VQW 1 "register_operand" "w")
4256 (match_operand:SI 2 "immediate_operand" "i")]
4257 VSHLL))]
4258 "TARGET_SIMD"
4259 {
4260 if (INTVAL (operands[2]) == GET_MODE_UNIT_BITSIZE (<MODE>mode))
4261 return "shll2\\t%0.<Vwtype>, %1.<Vtype>, %2";
4262 else
4263 return "<sur>shll2\\t%0.<Vwtype>, %1.<Vtype>, %2";
4264 }
4265 [(set_attr "type" "neon_shift_imm_long")]
4266 )
4268 ;; vrshr_n
4270 (define_insn "aarch64_<sur>shr_n<mode>"
4271 [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
4272 (unspec:VSDQ_I_DI [(match_operand:VSDQ_I_DI 1 "register_operand" "w")
4273 (match_operand:SI 2
4274 "aarch64_simd_shift_imm_offset_<ve_mode>" "i")]
4275 VRSHR_N))]
4276 "TARGET_SIMD"
4277 "<sur>shr\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %2"
4278 [(set_attr "type" "neon_sat_shift_imm<q>")]
4279 )
4281 ;; v(r)sra_n
4283 (define_insn "aarch64_<sur>sra_n<mode>"
4284 [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
4285 (unspec:VSDQ_I_DI [(match_operand:VSDQ_I_DI 1 "register_operand" "0")
4286 (match_operand:VSDQ_I_DI 2 "register_operand" "w")
4287 (match_operand:SI 3
4288 "aarch64_simd_shift_imm_offset_<ve_mode>" "i")]
4289 VSRA))]
4290 "TARGET_SIMD"
4291 "<sur>sra\\t%<v>0<Vmtype>, %<v>2<Vmtype>, %3"
4292 [(set_attr "type" "neon_shift_acc<q>")]
4293 )
4295 ;; vs<lr>i_n
4297 (define_insn "aarch64_<sur>s<lr>i_n<mode>"
4298 [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
4299 (unspec:VSDQ_I_DI [(match_operand:VSDQ_I_DI 1 "register_operand" "0")
4300 (match_operand:VSDQ_I_DI 2 "register_operand" "w")
4301 (match_operand:SI 3
4302 "aarch64_simd_shift_imm_<offsetlr><ve_mode>" "i")]
4303 VSLRI))]
4304 "TARGET_SIMD"
4305 "s<lr>i\\t%<v>0<Vmtype>, %<v>2<Vmtype>, %3"
4306 [(set_attr "type" "neon_shift_imm<q>")]
4307 )
4309 ;; vqshl(u)
4311 (define_insn "aarch64_<sur>qshl<u>_n<mode>"
4312 [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
4313 (unspec:VSDQ_I [(match_operand:VSDQ_I 1 "register_operand" "w")
4314 (match_operand:SI 2
4315 "aarch64_simd_shift_imm_<ve_mode>" "i")]
4316 VQSHL_N))]
4317 "TARGET_SIMD"
4318 "<sur>qshl<u>\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %2"
4319 [(set_attr "type" "neon_sat_shift_imm<q>")]
4320 )
4323 ;; vq(r)shr(u)n_n
4325 (define_insn "aarch64_<sur>q<r>shr<u>n_n<mode>"
4326 [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
4327 (unspec:<VNARROWQ> [(match_operand:VSQN_HSDI 1 "register_operand" "w")
4328 (match_operand:SI 2
4329 "aarch64_simd_shift_imm_offset_<ve_mode>" "i")]
4330 VQSHRN_N))]
4331 "TARGET_SIMD"
4332 "<sur>q<r>shr<u>n\\t%<vn2>0<Vmntype>, %<v>1<Vmtype>, %2"
4333 [(set_attr "type" "neon_sat_shift_imm_narrow_q")]
4334 )
4337 ;; cm(eq|ge|gt|lt|le)
4338 ;; Note, we have constraints for Dz and Z as different expanders
4339 ;; have different ideas of what should be passed to this pattern.
4341 (define_insn "aarch64_cm<optab><mode>"
4342 [(set (match_operand:<V_INT_EQUIV> 0 "register_operand" "=w,w")
4343 (neg:<V_INT_EQUIV>
4344 (COMPARISONS:<V_INT_EQUIV>
4345 (match_operand:VDQ_I 1 "register_operand" "w,w")
4346 (match_operand:VDQ_I 2 "aarch64_simd_reg_or_zero" "w,ZDz")
4347 )))]
4348 "TARGET_SIMD"
4349 "@
4350 cm<n_optab>\t%<v>0<Vmtype>, %<v><cmp_1><Vmtype>, %<v><cmp_2><Vmtype>
4351 cm<optab>\t%<v>0<Vmtype>, %<v>1<Vmtype>, #0"
4352 [(set_attr "type" "neon_compare<q>, neon_compare_zero<q>")]
4353 )
4355 (define_insn_and_split "aarch64_cm<optab>di"
4356 [(set (match_operand:DI 0 "register_operand" "=w,w,r")
4357 (neg:DI
4358 (COMPARISONS:DI
4359 (match_operand:DI 1 "register_operand" "w,w,r")
4360 (match_operand:DI 2 "aarch64_simd_reg_or_zero" "w,ZDz,r")
4361 )))
4362 (clobber (reg:CC CC_REGNUM))]
4363 "TARGET_SIMD"
4364 "#"
4365 "reload_completed"
4366 [(set (match_operand:DI 0 "register_operand")
4367 (neg:DI
4368 (COMPARISONS:DI
4369 (match_operand:DI 1 "register_operand")
4370 (match_operand:DI 2 "aarch64_simd_reg_or_zero")
4371 )))]
4372 {
4373 /* If we are in the general purpose register file,
4374 we split to a sequence of comparison and store. */
4375 if (GP_REGNUM_P (REGNO (operands[0]))
4376 && GP_REGNUM_P (REGNO (operands[1])))
4377 {
4378 machine_mode mode = SELECT_CC_MODE (<CMP>, operands[1], operands[2]);
4379 rtx cc_reg = aarch64_gen_compare_reg (<CMP>, operands[1], operands[2]);
4380 rtx comparison = gen_rtx_<CMP> (mode, operands[1], operands[2]);
4381 emit_insn (gen_cstoredi_neg (operands[0], comparison, cc_reg));
4382 DONE;
4383 }
4384 /* Otherwise, we expand to a similar pattern which does not
4385 clobber CC_REGNUM. */
4386 }
4387 [(set_attr "type" "neon_compare, neon_compare_zero, multiple")]
4388 )
4390 (define_insn "*aarch64_cm<optab>di"
4391 [(set (match_operand:DI 0 "register_operand" "=w,w")
4392 (neg:DI
4393 (COMPARISONS:DI
4394 (match_operand:DI 1 "register_operand" "w,w")
4395 (match_operand:DI 2 "aarch64_simd_reg_or_zero" "w,ZDz")
4396 )))]
4397 "TARGET_SIMD && reload_completed"
4398 "@
4399 cm<n_optab>\t%d0, %d<cmp_1>, %d<cmp_2>
4400 cm<optab>\t%d0, %d1, #0"
4401 [(set_attr "type" "neon_compare, neon_compare_zero")]
4402 )
4404 ;; cm(hs|hi)
4406 (define_insn "aarch64_cm<optab><mode>"
4407 [(set (match_operand:<V_INT_EQUIV> 0 "register_operand" "=w")
4408 (neg:<V_INT_EQUIV>
4409 (UCOMPARISONS:<V_INT_EQUIV>
4410 (match_operand:VDQ_I 1 "register_operand" "w")
4411 (match_operand:VDQ_I 2 "register_operand" "w")
4412 )))]
4413 "TARGET_SIMD"
4414 "cm<n_optab>\t%<v>0<Vmtype>, %<v><cmp_1><Vmtype>, %<v><cmp_2><Vmtype>"
4415 [(set_attr "type" "neon_compare<q>")]
4416 )
4418 (define_insn_and_split "aarch64_cm<optab>di"
4419 [(set (match_operand:DI 0 "register_operand" "=w,r")
4420 (neg:DI
4421 (UCOMPARISONS:DI
4422 (match_operand:DI 1 "register_operand" "w,r")
4423 (match_operand:DI 2 "aarch64_simd_reg_or_zero" "w,r")
4424 )))
4425 (clobber (reg:CC CC_REGNUM))]
4426 "TARGET_SIMD"
4427 "#"
4428 "reload_completed"
4429 [(set (match_operand:DI 0 "register_operand")
4430 (neg:DI
4431 (UCOMPARISONS:DI
4432 (match_operand:DI 1 "register_operand")
4433 (match_operand:DI 2 "aarch64_simd_reg_or_zero")
4434 )))]
4435 {
4436 /* If we are in the general purpose register file,
4437 we split to a sequence of comparison and store. */
4438 if (GP_REGNUM_P (REGNO (operands[0]))
4439 && GP_REGNUM_P (REGNO (operands[1])))
4440 {
4441 machine_mode mode = CCmode;
4442 rtx cc_reg = aarch64_gen_compare_reg (<CMP>, operands[1], operands[2]);
4443 rtx comparison = gen_rtx_<CMP> (mode, operands[1], operands[2]);
4444 emit_insn (gen_cstoredi_neg (operands[0], comparison, cc_reg));
4445 DONE;
4446 }
4447 /* Otherwise, we expand to a similar pattern which does not
4448 clobber CC_REGNUM. */
4449 }
4450 [(set_attr "type" "neon_compare,multiple")]
4451 )
4453 (define_insn "*aarch64_cm<optab>di"
4454 [(set (match_operand:DI 0 "register_operand" "=w")
4455 (neg:DI
4456 (UCOMPARISONS:DI
4457 (match_operand:DI 1 "register_operand" "w")
4458 (match_operand:DI 2 "aarch64_simd_reg_or_zero" "w")
4459 )))]
4460 "TARGET_SIMD && reload_completed"
4461 "cm<n_optab>\t%d0, %d<cmp_1>, %d<cmp_2>"
4462 [(set_attr "type" "neon_compare")]
4463 )
4465 ;; cmtst
4467 ;; Although neg (ne (and x y) 0) is the natural way of expressing a cmtst,
4468 ;; we don't have any insns using ne, and aarch64_vcond outputs
4469 ;; not (neg (eq (and x y) 0))
4470 ;; which is rewritten by simplify_rtx as
4471 ;; plus (eq (and x y) 0) -1.
4473 (define_insn "aarch64_cmtst<mode>"
4474 [(set (match_operand:<V_INT_EQUIV> 0 "register_operand" "=w")
4475 (plus:<V_INT_EQUIV>
4476 (eq:<V_INT_EQUIV>
4477 (and:VDQ_I
4478 (match_operand:VDQ_I 1 "register_operand" "w")
4479 (match_operand:VDQ_I 2 "register_operand" "w"))
4480 (match_operand:VDQ_I 3 "aarch64_simd_imm_zero"))
4481 (match_operand:<V_INT_EQUIV> 4 "aarch64_simd_imm_minus_one")))
4482 ]
4483 "TARGET_SIMD"
4484 "cmtst\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
4485 [(set_attr "type" "neon_tst<q>")]
4486 )
4488 (define_insn_and_split "aarch64_cmtstdi"
4489 [(set (match_operand:DI 0 "register_operand" "=w,r")
4490 (neg:DI
4491 (ne:DI
4492 (and:DI
4493 (match_operand:DI 1 "register_operand" "w,r")
4494 (match_operand:DI 2 "register_operand" "w,r"))
4495 (const_int 0))))
4496 (clobber (reg:CC CC_REGNUM))]
4497 "TARGET_SIMD"
4498 "#"
4499 "reload_completed"
4500 [(set (match_operand:DI 0 "register_operand")
4501 (neg:DI
4502 (ne:DI
4503 (and:DI
4504 (match_operand:DI 1 "register_operand")
4505 (match_operand:DI 2 "register_operand"))
4506 (const_int 0))))]
4507 {
4508 /* If we are in the general purpose register file,
4509 we split to a sequence of comparison and store. */
4510 if (GP_REGNUM_P (REGNO (operands[0]))
4511 && GP_REGNUM_P (REGNO (operands[1])))
4512 {
4513 rtx and_tree = gen_rtx_AND (DImode, operands[1], operands[2]);
4514 machine_mode mode = SELECT_CC_MODE (NE, and_tree, const0_rtx);
4515 rtx cc_reg = aarch64_gen_compare_reg (NE, and_tree, const0_rtx);
4516 rtx comparison = gen_rtx_NE (mode, and_tree, const0_rtx);
4517 emit_insn (gen_cstoredi_neg (operands[0], comparison, cc_reg));
4518 DONE;
4519 }
4520 /* Otherwise, we expand to a similar pattern which does not
4521 clobber CC_REGNUM. */
4522 }
4523 [(set_attr "type" "neon_tst,multiple")]
4524 )
4526 (define_insn "*aarch64_cmtstdi"
4527 [(set (match_operand:DI 0 "register_operand" "=w")
4528 (neg:DI
4529 (ne:DI
4530 (and:DI
4531 (match_operand:DI 1 "register_operand" "w")
4532 (match_operand:DI 2 "register_operand" "w"))
4533 (const_int 0))))]
4534 "TARGET_SIMD"
4535 "cmtst\t%d0, %d1, %d2"
4536 [(set_attr "type" "neon_tst")]
4537 )
4539 ;; fcm(eq|ge|gt|le|lt)
4541 (define_insn "aarch64_cm<optab><mode>"
4542 [(set (match_operand:<V_INT_EQUIV> 0 "register_operand" "=w,w")
4543 (neg:<V_INT_EQUIV>
4544 (COMPARISONS:<V_INT_EQUIV>
4545 (match_operand:VHSDF_HSDF 1 "register_operand" "w,w")
4546 (match_operand:VHSDF_HSDF 2 "aarch64_simd_reg_or_zero" "w,YDz")
4547 )))]
4548 "TARGET_SIMD"
4549 "@
4550 fcm<n_optab>\t%<v>0<Vmtype>, %<v><cmp_1><Vmtype>, %<v><cmp_2><Vmtype>
4551 fcm<optab>\t%<v>0<Vmtype>, %<v>1<Vmtype>, 0"
4552 [(set_attr "type" "neon_fp_compare_<stype><q>")]
4553 )
4555 ;; fac(ge|gt)
4556 ;; Note we can also handle what would be fac(le|lt) by
4557 ;; generating fac(ge|gt).
4559 (define_insn "aarch64_fac<optab><mode>"
4560 [(set (match_operand:<V_INT_EQUIV> 0 "register_operand" "=w")
4561 (neg:<V_INT_EQUIV>
4562 (FAC_COMPARISONS:<V_INT_EQUIV>
4563 (abs:VHSDF_HSDF
4564 (match_operand:VHSDF_HSDF 1 "register_operand" "w"))
4565 (abs:VHSDF_HSDF
4566 (match_operand:VHSDF_HSDF 2 "register_operand" "w"))
4567 )))]
4568 "TARGET_SIMD"
4569 "fac<n_optab>\t%<v>0<Vmtype>, %<v><cmp_1><Vmtype>, %<v><cmp_2><Vmtype>"
4570 [(set_attr "type" "neon_fp_compare_<stype><q>")]
4571 )
4573 ;; addp
4575 (define_insn "aarch64_addp<mode>"
4576 [(set (match_operand:VD_BHSI 0 "register_operand" "=w")
4577 (unspec:VD_BHSI
4578 [(match_operand:VD_BHSI 1 "register_operand" "w")
4579 (match_operand:VD_BHSI 2 "register_operand" "w")]
4580 UNSPEC_ADDP))]
4581 "TARGET_SIMD"
4582 "addp\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
4583 [(set_attr "type" "neon_reduc_add<q>")]
4584 )
4586 (define_insn "aarch64_addpdi"
4587 [(set (match_operand:DI 0 "register_operand" "=w")
4588 (unspec:DI
4589 [(match_operand:V2DI 1 "register_operand" "w")]
4590 UNSPEC_ADDP))]
4591 "TARGET_SIMD"
4592 "addp\t%d0, %1.2d"
4593 [(set_attr "type" "neon_reduc_add")]
4594 )
4596 ;; sqrt
4598 (define_expand "sqrt<mode>2"
4599 [(set (match_operand:VHSDF 0 "register_operand" "=w")
4600 (sqrt:VHSDF (match_operand:VHSDF 1 "register_operand" "w")))]
4601 "TARGET_SIMD"
4602 {
4603 if (aarch64_emit_approx_sqrt (operands[0], operands[1], false))
4604 DONE;
4605 })
4607 (define_insn "*sqrt<mode>2"
4608 [(set (match_operand:VHSDF 0 "register_operand" "=w")
4609 (sqrt:VHSDF (match_operand:VHSDF 1 "register_operand" "w")))]
4610 "TARGET_SIMD"
4611 "fsqrt\\t%0.<Vtype>, %1.<Vtype>"
4612 [(set_attr "type" "neon_fp_sqrt_<stype><q>")]
4613 )
4615 ;; Patterns for vector struct loads and stores.
4617 (define_insn "aarch64_simd_ld2<mode>"
4618 [(set (match_operand:OI 0 "register_operand" "=w")
4619 (unspec:OI [(match_operand:OI 1 "aarch64_simd_struct_operand" "Utv")
4620 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4621 UNSPEC_LD2))]
4622 "TARGET_SIMD"
4623 "ld2\\t{%S0.<Vtype> - %T0.<Vtype>}, %1"
4624 [(set_attr "type" "neon_load2_2reg<q>")]
4625 )
4627 (define_insn "aarch64_simd_ld2r<mode>"
4628 [(set (match_operand:OI 0 "register_operand" "=w")
4629 (unspec:OI [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")
4630 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY) ]
4631 UNSPEC_LD2_DUP))]
4632 "TARGET_SIMD"
4633 "ld2r\\t{%S0.<Vtype> - %T0.<Vtype>}, %1"
4634 [(set_attr "type" "neon_load2_all_lanes<q>")]
4635 )
4637 (define_insn "aarch64_vec_load_lanesoi_lane<mode>"
4638 [(set (match_operand:OI 0 "register_operand" "=w")
4639 (unspec:OI [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")
4640 (match_operand:OI 2 "register_operand" "0")
4641 (match_operand:SI 3 "immediate_operand" "i")
4642 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY) ]
4643 UNSPEC_LD2_LANE))]
4644 "TARGET_SIMD"
4645 {
4646 operands[3] = aarch64_endian_lane_rtx (<MODE>mode, INTVAL (operands[3]));
4647 return "ld2\\t{%S0.<Vetype> - %T0.<Vetype>}[%3], %1";
4648 }
4649 [(set_attr "type" "neon_load2_one_lane")]
4650 )
4652 (define_expand "vec_load_lanesoi<mode>"
4653 [(set (match_operand:OI 0 "register_operand" "=w")
4654 (unspec:OI [(match_operand:OI 1 "aarch64_simd_struct_operand" "Utv")
4655 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4656 UNSPEC_LD2))]
4657 "TARGET_SIMD"
4658 {
4659 if (BYTES_BIG_ENDIAN)
4660 {
4661 rtx tmp = gen_reg_rtx (OImode);
4662 rtx mask = aarch64_reverse_mask (<MODE>mode, <nunits>);
4663 emit_insn (gen_aarch64_simd_ld2<mode> (tmp, operands[1]));
4664 emit_insn (gen_aarch64_rev_reglistoi (operands[0], tmp, mask));
4665 }
4666 else
4667 emit_insn (gen_aarch64_simd_ld2<mode> (operands[0], operands[1]));
4668 DONE;
4669 })
4671 (define_insn "aarch64_simd_st2<mode>"
4672 [(set (match_operand:OI 0 "aarch64_simd_struct_operand" "=Utv")
4673 (unspec:OI [(match_operand:OI 1 "register_operand" "w")
4674 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4675 UNSPEC_ST2))]
4676 "TARGET_SIMD"
4677 "st2\\t{%S1.<Vtype> - %T1.<Vtype>}, %0"
4678 [(set_attr "type" "neon_store2_2reg<q>")]
4679 )
4681 ;; RTL uses GCC vector extension indices, so flip only for assembly.
4682 (define_insn "aarch64_vec_store_lanesoi_lane<mode>"
4683 [(set (match_operand:BLK 0 "aarch64_simd_struct_operand" "=Utv")
4684 (unspec:BLK [(match_operand:OI 1 "register_operand" "w")
4685 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)
4686 (match_operand:SI 2 "immediate_operand" "i")]
4687 UNSPEC_ST2_LANE))]
4688 "TARGET_SIMD"
4689 {
4690 operands[2] = aarch64_endian_lane_rtx (<MODE>mode, INTVAL (operands[2]));
4691 return "st2\\t{%S1.<Vetype> - %T1.<Vetype>}[%2], %0";
4692 }
4693 [(set_attr "type" "neon_store2_one_lane<q>")]
4694 )
4696 (define_expand "vec_store_lanesoi<mode>"
4697 [(set (match_operand:OI 0 "aarch64_simd_struct_operand" "=Utv")
4698 (unspec:OI [(match_operand:OI 1 "register_operand" "w")
4699 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4700 UNSPEC_ST2))]
4701 "TARGET_SIMD"
4702 {
4703 if (BYTES_BIG_ENDIAN)
4704 {
4705 rtx tmp = gen_reg_rtx (OImode);
4706 rtx mask = aarch64_reverse_mask (<MODE>mode, <nunits>);
4707 emit_insn (gen_aarch64_rev_reglistoi (tmp, operands[1], mask));
4708 emit_insn (gen_aarch64_simd_st2<mode> (operands[0], tmp));
4709 }
4710 else
4711 emit_insn (gen_aarch64_simd_st2<mode> (operands[0], operands[1]));
4712 DONE;
4713 })
4715 (define_insn "aarch64_simd_ld3<mode>"
4716 [(set (match_operand:CI 0 "register_operand" "=w")
4717 (unspec:CI [(match_operand:CI 1 "aarch64_simd_struct_operand" "Utv")
4718 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4719 UNSPEC_LD3))]
4720 "TARGET_SIMD"
4721 "ld3\\t{%S0.<Vtype> - %U0.<Vtype>}, %1"
4722 [(set_attr "type" "neon_load3_3reg<q>")]
4723 )
4725 (define_insn "aarch64_simd_ld3r<mode>"
4726 [(set (match_operand:CI 0 "register_operand" "=w")
4727 (unspec:CI [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")
4728 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY) ]
4729 UNSPEC_LD3_DUP))]
4730 "TARGET_SIMD"
4731 "ld3r\\t{%S0.<Vtype> - %U0.<Vtype>}, %1"
4732 [(set_attr "type" "neon_load3_all_lanes<q>")]
4733 )
4735 (define_insn "aarch64_vec_load_lanesci_lane<mode>"
4736 [(set (match_operand:CI 0 "register_operand" "=w")
4737 (unspec:CI [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")
4738 (match_operand:CI 2 "register_operand" "0")
4739 (match_operand:SI 3 "immediate_operand" "i")
4740 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4741 UNSPEC_LD3_LANE))]
4742 "TARGET_SIMD"
4743 {
4744 operands[3] = aarch64_endian_lane_rtx (<MODE>mode, INTVAL (operands[3]));
4745 return "ld3\\t{%S0.<Vetype> - %U0.<Vetype>}[%3], %1";
4746 }
4747 [(set_attr "type" "neon_load3_one_lane")]
4748 )
4750 (define_expand "vec_load_lanesci<mode>"
4751 [(set (match_operand:CI 0 "register_operand" "=w")
4752 (unspec:CI [(match_operand:CI 1 "aarch64_simd_struct_operand" "Utv")
4753 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4754 UNSPEC_LD3))]
4755 "TARGET_SIMD"
4756 {
4757 if (BYTES_BIG_ENDIAN)
4758 {
4759 rtx tmp = gen_reg_rtx (CImode);
4760 rtx mask = aarch64_reverse_mask (<MODE>mode, <nunits>);
4761 emit_insn (gen_aarch64_simd_ld3<mode> (tmp, operands[1]));
4762 emit_insn (gen_aarch64_rev_reglistci (operands[0], tmp, mask));
4763 }
4764 else
4765 emit_insn (gen_aarch64_simd_ld3<mode> (operands[0], operands[1]));
4766 DONE;
4767 })
4769 (define_insn "aarch64_simd_st3<mode>"
4770 [(set (match_operand:CI 0 "aarch64_simd_struct_operand" "=Utv")
4771 (unspec:CI [(match_operand:CI 1 "register_operand" "w")
4772 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4773 UNSPEC_ST3))]
4774 "TARGET_SIMD"
4775 "st3\\t{%S1.<Vtype> - %U1.<Vtype>}, %0"
4776 [(set_attr "type" "neon_store3_3reg<q>")]
4777 )
4779 ;; RTL uses GCC vector extension indices, so flip only for assembly.
4780 (define_insn "aarch64_vec_store_lanesci_lane<mode>"
4781 [(set (match_operand:BLK 0 "aarch64_simd_struct_operand" "=Utv")
4782 (unspec:BLK [(match_operand:CI 1 "register_operand" "w")
4783 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)
4784 (match_operand:SI 2 "immediate_operand" "i")]
4785 UNSPEC_ST3_LANE))]
4786 "TARGET_SIMD"
4787 {
4788 operands[2] = aarch64_endian_lane_rtx (<MODE>mode, INTVAL (operands[2]));
4789 return "st3\\t{%S1.<Vetype> - %U1.<Vetype>}[%2], %0";
4790 }
4791 [(set_attr "type" "neon_store3_one_lane<q>")]
4792 )
4794 (define_expand "vec_store_lanesci<mode>"
4795 [(set (match_operand:CI 0 "aarch64_simd_struct_operand" "=Utv")
4796 (unspec:CI [(match_operand:CI 1 "register_operand" "w")
4797 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4798 UNSPEC_ST3))]
4799 "TARGET_SIMD"
4800 {
4801 if (BYTES_BIG_ENDIAN)
4802 {
4803 rtx tmp = gen_reg_rtx (CImode);
4804 rtx mask = aarch64_reverse_mask (<MODE>mode, <nunits>);
4805 emit_insn (gen_aarch64_rev_reglistci (tmp, operands[1], mask));
4806 emit_insn (gen_aarch64_simd_st3<mode> (operands[0], tmp));
4807 }
4808 else
4809 emit_insn (gen_aarch64_simd_st3<mode> (operands[0], operands[1]));
4810 DONE;
4811 })
4813 (define_insn "aarch64_simd_ld4<mode>"
4814 [(set (match_operand:XI 0 "register_operand" "=w")
4815 (unspec:XI [(match_operand:XI 1 "aarch64_simd_struct_operand" "Utv")
4816 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4817 UNSPEC_LD4))]
4818 "TARGET_SIMD"
4819 "ld4\\t{%S0.<Vtype> - %V0.<Vtype>}, %1"
4820 [(set_attr "type" "neon_load4_4reg<q>")]
4821 )
4823 (define_insn "aarch64_simd_ld4r<mode>"
4824 [(set (match_operand:XI 0 "register_operand" "=w")
4825 (unspec:XI [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")
4826 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY) ]
4827 UNSPEC_LD4_DUP))]
4828 "TARGET_SIMD"
4829 "ld4r\\t{%S0.<Vtype> - %V0.<Vtype>}, %1"
4830 [(set_attr "type" "neon_load4_all_lanes<q>")]
4831 )
4833 (define_insn "aarch64_vec_load_lanesxi_lane<mode>"
4834 [(set (match_operand:XI 0 "register_operand" "=w")
4835 (unspec:XI [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")
4836 (match_operand:XI 2 "register_operand" "0")
4837 (match_operand:SI 3 "immediate_operand" "i")
4838 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4839 UNSPEC_LD4_LANE))]
4840 "TARGET_SIMD"
4841 {
4842 operands[3] = aarch64_endian_lane_rtx (<MODE>mode, INTVAL (operands[3]));
4843 return "ld4\\t{%S0.<Vetype> - %V0.<Vetype>}[%3], %1";
4844 }
4845 [(set_attr "type" "neon_load4_one_lane")]
4846 )
4848 (define_expand "vec_load_lanesxi<mode>"
4849 [(set (match_operand:XI 0 "register_operand" "=w")
4850 (unspec:XI [(match_operand:XI 1 "aarch64_simd_struct_operand" "Utv")
4851 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4852 UNSPEC_LD4))]
4853 "TARGET_SIMD"
4854 {
4855 if (BYTES_BIG_ENDIAN)
4856 {
4857 rtx tmp = gen_reg_rtx (XImode);
4858 rtx mask = aarch64_reverse_mask (<MODE>mode, <nunits>);
4859 emit_insn (gen_aarch64_simd_ld4<mode> (tmp, operands[1]));
4860 emit_insn (gen_aarch64_rev_reglistxi (operands[0], tmp, mask));
4861 }
4862 else
4863 emit_insn (gen_aarch64_simd_ld4<mode> (operands[0], operands[1]));
4864 DONE;
4865 })
4867 (define_insn "aarch64_simd_st4<mode>"
4868 [(set (match_operand:XI 0 "aarch64_simd_struct_operand" "=Utv")
4869 (unspec:XI [(match_operand:XI 1 "register_operand" "w")
4870 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4871 UNSPEC_ST4))]
4872 "TARGET_SIMD"
4873 "st4\\t{%S1.<Vtype> - %V1.<Vtype>}, %0"
4874 [(set_attr "type" "neon_store4_4reg<q>")]
4875 )
4877 ;; RTL uses GCC vector extension indices, so flip only for assembly.
4878 (define_insn "aarch64_vec_store_lanesxi_lane<mode>"
4879 [(set (match_operand:BLK 0 "aarch64_simd_struct_operand" "=Utv")
4880 (unspec:BLK [(match_operand:XI 1 "register_operand" "w")
4881 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)
4882 (match_operand:SI 2 "immediate_operand" "i")]
4883 UNSPEC_ST4_LANE))]
4884 "TARGET_SIMD"
4885 {
4886 operands[2] = aarch64_endian_lane_rtx (<MODE>mode, INTVAL (operands[2]));
4887 return "st4\\t{%S1.<Vetype> - %V1.<Vetype>}[%2], %0";
4888 }
4889 [(set_attr "type" "neon_store4_one_lane<q>")]
4890 )
4892 (define_expand "vec_store_lanesxi<mode>"
4893 [(set (match_operand:XI 0 "aarch64_simd_struct_operand" "=Utv")
4894 (unspec:XI [(match_operand:XI 1 "register_operand" "w")
4895 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4896 UNSPEC_ST4))]
4897 "TARGET_SIMD"
4898 {
4899 if (BYTES_BIG_ENDIAN)
4900 {
4901 rtx tmp = gen_reg_rtx (XImode);
4902 rtx mask = aarch64_reverse_mask (<MODE>mode, <nunits>);
4903 emit_insn (gen_aarch64_rev_reglistxi (tmp, operands[1], mask));
4904 emit_insn (gen_aarch64_simd_st4<mode> (operands[0], tmp));
4905 }
4906 else
4907 emit_insn (gen_aarch64_simd_st4<mode> (operands[0], operands[1]));
4908 DONE;
4909 })
4911 (define_insn_and_split "aarch64_rev_reglist<mode>"
4912 [(set (match_operand:VSTRUCT 0 "register_operand" "=&w")
4913 (unspec:VSTRUCT
4914 [(match_operand:VSTRUCT 1 "register_operand" "w")
4915 (match_operand:V16QI 2 "register_operand" "w")]
4916 UNSPEC_REV_REGLIST))]
4917 "TARGET_SIMD"
4918 "#"
4919 "&& reload_completed"
4920 [(const_int 0)]
4921 {
4922 int i;
4923 int nregs = GET_MODE_SIZE (<MODE>mode) / UNITS_PER_VREG;
4924 for (i = 0; i < nregs; i++)
4925 {
4926 rtx op0 = gen_rtx_REG (V16QImode, REGNO (operands[0]) + i);
4927 rtx op1 = gen_rtx_REG (V16QImode, REGNO (operands[1]) + i);
4928 emit_insn (gen_aarch64_tbl1v16qi (op0, op1, operands[2]));
4929 }
4930 DONE;
4931 }
4932 [(set_attr "type" "neon_tbl1_q")
4933 (set_attr "length" "<insn_count>")]
4934 )
4936 ;; Reload patterns for AdvSIMD register list operands.
4938 (define_expand "mov<mode>"
4939 [(set (match_operand:VSTRUCT 0 "nonimmediate_operand" "")
4940 (match_operand:VSTRUCT 1 "general_operand" ""))]
4941 "TARGET_SIMD"
4942 {
4943 if (can_create_pseudo_p ())
4944 {
4945 if (GET_CODE (operands[0]) != REG)
4946 operands[1] = force_reg (<MODE>mode, operands[1]);
4947 }
4948 })
4950 (define_insn "*aarch64_mov<mode>"
4951 [(set (match_operand:VSTRUCT 0 "aarch64_simd_nonimmediate_operand" "=w,Utv,w")
4952 (match_operand:VSTRUCT 1 "aarch64_simd_general_operand" " w,w,Utv"))]
4953 "TARGET_SIMD && !BYTES_BIG_ENDIAN
4954 && (register_operand (operands[0], <MODE>mode)
4955 || register_operand (operands[1], <MODE>mode))"
4956 "@
4957 #
4958 st1\\t{%S1.16b - %<Vendreg>1.16b}, %0
4959 ld1\\t{%S0.16b - %<Vendreg>0.16b}, %1"
4960 [(set_attr "type" "multiple,neon_store<nregs>_<nregs>reg_q,\
4961 neon_load<nregs>_<nregs>reg_q")
4962 (set_attr "length" "<insn_count>,4,4")]
4963 )
4965 (define_insn "aarch64_be_ld1<mode>"
4966 [(set (match_operand:VALLDI_F16 0 "register_operand" "=w")
4967 (unspec:VALLDI_F16 [(match_operand:VALLDI_F16 1
4968 "aarch64_simd_struct_operand" "Utv")]
4969 UNSPEC_LD1))]
4970 "TARGET_SIMD"
4971 "ld1\\t{%0<Vmtype>}, %1"
4972 [(set_attr "type" "neon_load1_1reg<q>")]
4973 )
4975 (define_insn "aarch64_be_st1<mode>"
4976 [(set (match_operand:VALLDI_F16 0 "aarch64_simd_struct_operand" "=Utv")
4977 (unspec:VALLDI_F16 [(match_operand:VALLDI_F16 1 "register_operand" "w")]
4978 UNSPEC_ST1))]
4979 "TARGET_SIMD"
4980 "st1\\t{%1<Vmtype>}, %0"
4981 [(set_attr "type" "neon_store1_1reg<q>")]
4982 )
4984 (define_insn "*aarch64_be_movoi"
4985 [(set (match_operand:OI 0 "nonimmediate_operand" "=w,m,w")
4986 (match_operand:OI 1 "general_operand" " w,w,m"))]
4987 "TARGET_SIMD && BYTES_BIG_ENDIAN
4988 && (register_operand (operands[0], OImode)
4989 || register_operand (operands[1], OImode))"
4990 "@
4991 #
4992 stp\\t%q1, %R1, %0
4993 ldp\\t%q0, %R0, %1"
4994 [(set_attr "type" "multiple,neon_stp_q,neon_ldp_q")
4995 (set_attr "length" "8,4,4")]
4996 )
4998 (define_insn "*aarch64_be_movci"
4999 [(set (match_operand:CI 0 "nonimmediate_operand" "=w,o,w")
5000 (match_operand:CI 1 "general_operand" " w,w,o"))]
5001 "TARGET_SIMD && BYTES_BIG_ENDIAN
5002 && (register_operand (operands[0], CImode)
5003 || register_operand (operands[1], CImode))"
5004 "#"
5005 [(set_attr "type" "multiple")
5006 (set_attr "length" "12,4,4")]
5007 )
5009 (define_insn "*aarch64_be_movxi"
5010 [(set (match_operand:XI 0 "nonimmediate_operand" "=w,o,w")
5011 (match_operand:XI 1 "general_operand" " w,w,o"))]
5012 "TARGET_SIMD && BYTES_BIG_ENDIAN
5013 && (register_operand (operands[0], XImode)
5014 || register_operand (operands[1], XImode))"
5015 "#"
5016 [(set_attr "type" "multiple")
5017 (set_attr "length" "16,4,4")]
5018 )
5020 (define_split
5021 [(set (match_operand:OI 0 "register_operand")
5022 (match_operand:OI 1 "register_operand"))]
5023 "TARGET_SIMD && reload_completed"
5024 [(const_int 0)]
5025 {
5026 aarch64_simd_emit_reg_reg_move (operands, TImode, 2);
5027 DONE;
5028 })
5030 (define_split
5031 [(set (match_operand:CI 0 "nonimmediate_operand")
5032 (match_operand:CI 1 "general_operand"))]
5033 "TARGET_SIMD && reload_completed"
5034 [(const_int 0)]
5035 {
5036 if (register_operand (operands[0], CImode)
5037 && register_operand (operands[1], CImode))
5038 {
5039 aarch64_simd_emit_reg_reg_move (operands, TImode, 3);
5040 DONE;
5041 }
5042 else if (BYTES_BIG_ENDIAN)
5043 {
5044 emit_move_insn (simplify_gen_subreg (OImode, operands[0], CImode, 0),
5045 simplify_gen_subreg (OImode, operands[1], CImode, 0));
5046 emit_move_insn (gen_lowpart (V16QImode,
5047 simplify_gen_subreg (TImode, operands[0],
5048 CImode, 32)),
5049 gen_lowpart (V16QImode,
5050 simplify_gen_subreg (TImode, operands[1],
5051 CImode, 32)));
5052 DONE;
5053 }
5054 else
5055 FAIL;
5056 })
5058 (define_split
5059 [(set (match_operand:XI 0 "nonimmediate_operand")
5060 (match_operand:XI 1 "general_operand"))]
5061 "TARGET_SIMD && reload_completed"
5062 [(const_int 0)]
5063 {
5064 if (register_operand (operands[0], XImode)
5065 && register_operand (operands[1], XImode))
5066 {
5067 aarch64_simd_emit_reg_reg_move (operands, TImode, 4);
5068 DONE;
5069 }
5070 else if (BYTES_BIG_ENDIAN)
5071 {
5072 emit_move_insn (simplify_gen_subreg (OImode, operands[0], XImode, 0),
5073 simplify_gen_subreg (OImode, operands[1], XImode, 0));
5074 emit_move_insn (simplify_gen_subreg (OImode, operands[0], XImode, 32),
5075 simplify_gen_subreg (OImode, operands[1], XImode, 32));
5076 DONE;
5077 }
5078 else
5079 FAIL;
5080 })
5082 (define_expand "aarch64_ld<VSTRUCT:nregs>r<VALLDIF:mode>"
5083 [(match_operand:VSTRUCT 0 "register_operand" "=w")
5084 (match_operand:DI 1 "register_operand" "w")
5085 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5086 "TARGET_SIMD"
5087 {
5088 rtx mem = gen_rtx_MEM (BLKmode, operands[1]);
5089 set_mem_size (mem, GET_MODE_SIZE (GET_MODE_INNER (<VALLDIF:MODE>mode))
5090 * <VSTRUCT:nregs>);
5092 emit_insn (gen_aarch64_simd_ld<VSTRUCT:nregs>r<VALLDIF:mode> (operands[0],
5093 mem));
5094 DONE;
5095 })
5097 (define_insn "aarch64_ld2<mode>_dreg"
5098 [(set (match_operand:OI 0 "register_operand" "=w")
5099 (unspec:OI [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")
5100 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5101 UNSPEC_LD2_DREG))]
5102 "TARGET_SIMD"
5103 "ld2\\t{%S0.<Vtype> - %T0.<Vtype>}, %1"
5104 [(set_attr "type" "neon_load2_2reg<q>")]
5105 )
5107 (define_insn "aarch64_ld2<mode>_dreg"
5108 [(set (match_operand:OI 0 "register_operand" "=w")
5109 (unspec:OI [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")
5110 (unspec:DX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5111 UNSPEC_LD2_DREG))]
5112 "TARGET_SIMD"
5113 "ld1\\t{%S0.1d - %T0.1d}, %1"
5114 [(set_attr "type" "neon_load1_2reg<q>")]
5115 )
5117 (define_insn "aarch64_ld3<mode>_dreg"
5118 [(set (match_operand:CI 0 "register_operand" "=w")
5119 (unspec:CI [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")
5120 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5121 UNSPEC_LD3_DREG))]
5122 "TARGET_SIMD"
5123 "ld3\\t{%S0.<Vtype> - %U0.<Vtype>}, %1"
5124 [(set_attr "type" "neon_load3_3reg<q>")]
5125 )
5127 (define_insn "aarch64_ld3<mode>_dreg"
5128 [(set (match_operand:CI 0 "register_operand" "=w")
5129 (unspec:CI [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")
5130 (unspec:DX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5131 UNSPEC_LD3_DREG))]
5132 "TARGET_SIMD"
5133 "ld1\\t{%S0.1d - %U0.1d}, %1"
5134 [(set_attr "type" "neon_load1_3reg<q>")]
5135 )
5137 (define_insn "aarch64_ld4<mode>_dreg"
5138 [(set (match_operand:XI 0 "register_operand" "=w")
5139 (unspec:XI [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")
5140 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5141 UNSPEC_LD4_DREG))]
5142 "TARGET_SIMD"
5143 "ld4\\t{%S0.<Vtype> - %V0.<Vtype>}, %1"
5144 [(set_attr "type" "neon_load4_4reg<q>")]
5145 )
5147 (define_insn "aarch64_ld4<mode>_dreg"
5148 [(set (match_operand:XI 0 "register_operand" "=w")
5149 (unspec:XI [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")
5150 (unspec:DX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5151 UNSPEC_LD4_DREG))]
5152 "TARGET_SIMD"
5153 "ld1\\t{%S0.1d - %V0.1d}, %1"
5154 [(set_attr "type" "neon_load1_4reg<q>")]
5155 )
5157 (define_expand "aarch64_ld<VSTRUCT:nregs><VDC:mode>"
5158 [(match_operand:VSTRUCT 0 "register_operand" "=w")
5159 (match_operand:DI 1 "register_operand" "r")
5160 (unspec:VDC [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5161 "TARGET_SIMD"
5162 {
5163 rtx mem = gen_rtx_MEM (BLKmode, operands[1]);
5164 set_mem_size (mem, <VSTRUCT:nregs> * 8);
5166 emit_insn (gen_aarch64_ld<VSTRUCT:nregs><VDC:mode>_dreg (operands[0], mem));
5167 DONE;
5168 })
5170 (define_expand "aarch64_ld1<VALL_F16:mode>"
5171 [(match_operand:VALL_F16 0 "register_operand")
5172 (match_operand:DI 1 "register_operand")]
5173 "TARGET_SIMD"
5174 {
5175 machine_mode mode = <VALL_F16:MODE>mode;
5176 rtx mem = gen_rtx_MEM (mode, operands[1]);
5178 if (BYTES_BIG_ENDIAN)
5179 emit_insn (gen_aarch64_be_ld1<VALL_F16:mode> (operands[0], mem));
5180 else
5181 emit_move_insn (operands[0], mem);
5182 DONE;
5183 })
5185 (define_expand "aarch64_ld<VSTRUCT:nregs><VQ:mode>"
5186 [(match_operand:VSTRUCT 0 "register_operand" "=w")
5187 (match_operand:DI 1 "register_operand" "r")
5188 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5189 "TARGET_SIMD"
5190 {
5191 machine_mode mode = <VSTRUCT:MODE>mode;
5192 rtx mem = gen_rtx_MEM (mode, operands[1]);
5194 emit_insn (gen_aarch64_simd_ld<VSTRUCT:nregs><VQ:mode> (operands[0], mem));
5195 DONE;
5196 })
5198 (define_expand "aarch64_ld<VSTRUCT:nregs>_lane<VALLDIF:mode>"
5199 [(match_operand:VSTRUCT 0 "register_operand" "=w")
5200 (match_operand:DI 1 "register_operand" "w")
5201 (match_operand:VSTRUCT 2 "register_operand" "0")
5202 (match_operand:SI 3 "immediate_operand" "i")
5203 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5204 "TARGET_SIMD"
5205 {
5206 rtx mem = gen_rtx_MEM (BLKmode, operands[1]);
5207 set_mem_size (mem, GET_MODE_SIZE (GET_MODE_INNER (<VALLDIF:MODE>mode))
5208 * <VSTRUCT:nregs>);
5210 aarch64_simd_lane_bounds (operands[3], 0,
5211 GET_MODE_NUNITS (<VALLDIF:MODE>mode),
5212 NULL);
5213 emit_insn (gen_aarch64_vec_load_lanes<VSTRUCT:mode>_lane<VALLDIF:mode> (
5214 operands[0], mem, operands[2], operands[3]));
5215 DONE;
5216 })
5218 ;; Expanders for builtins to extract vector registers from large
5219 ;; opaque integer modes.
5221 ;; D-register list.
5223 (define_expand "aarch64_get_dreg<VSTRUCT:mode><VDC:mode>"
5224 [(match_operand:VDC 0 "register_operand" "=w")
5225 (match_operand:VSTRUCT 1 "register_operand" "w")
5226 (match_operand:SI 2 "immediate_operand" "i")]
5227 "TARGET_SIMD"
5228 {
5229 int part = INTVAL (operands[2]);
5230 rtx temp = gen_reg_rtx (<VDC:VDBL>mode);
5231 int offset = part * 16;
5233 emit_move_insn (temp, gen_rtx_SUBREG (<VDC:VDBL>mode, operands[1], offset));
5234 emit_move_insn (operands[0], gen_lowpart (<VDC:MODE>mode, temp));
5235 DONE;
5236 })
5238 ;; Q-register list.
5240 (define_expand "aarch64_get_qreg<VSTRUCT:mode><VQ:mode>"
5241 [(match_operand:VQ 0 "register_operand" "=w")
5242 (match_operand:VSTRUCT 1 "register_operand" "w")
5243 (match_operand:SI 2 "immediate_operand" "i")]
5244 "TARGET_SIMD"
5245 {
5246 int part = INTVAL (operands[2]);
5247 int offset = part * 16;
5249 emit_move_insn (operands[0],
5250 gen_rtx_SUBREG (<VQ:MODE>mode, operands[1], offset));
5251 DONE;
5252 })
5254 ;; Permuted-store expanders for neon intrinsics.
5256 ;; Permute instructions
5258 ;; vec_perm support
5260 (define_expand "vec_perm_const<mode>"
5261 [(match_operand:VALL_F16 0 "register_operand")
5262 (match_operand:VALL_F16 1 "register_operand")
5263 (match_operand:VALL_F16 2 "register_operand")
5264 (match_operand:<V_INT_EQUIV> 3)]
5265 "TARGET_SIMD"
5266 {
5267 if (aarch64_expand_vec_perm_const (operands[0], operands[1],
5268 operands[2], operands[3], <nunits>))
5269 DONE;
5270 else
5271 FAIL;
5272 })
5274 (define_expand "vec_perm<mode>"
5275 [(match_operand:VB 0 "register_operand")
5276 (match_operand:VB 1 "register_operand")
5277 (match_operand:VB 2 "register_operand")
5278 (match_operand:VB 3 "register_operand")]
5279 "TARGET_SIMD"
5280 {
5281 aarch64_expand_vec_perm (operands[0], operands[1],
5282 operands[2], operands[3], <nunits>);
5283 DONE;
5284 })
5286 (define_insn "aarch64_tbl1<mode>"
5287 [(set (match_operand:VB 0 "register_operand" "=w")
5288 (unspec:VB [(match_operand:V16QI 1 "register_operand" "w")
5289 (match_operand:VB 2 "register_operand" "w")]
5290 UNSPEC_TBL))]
5291 "TARGET_SIMD"
5292 "tbl\\t%0.<Vtype>, {%1.16b}, %2.<Vtype>"
5293 [(set_attr "type" "neon_tbl1<q>")]
5294 )
5296 ;; Two source registers.
5298 (define_insn "aarch64_tbl2v16qi"
5299 [(set (match_operand:V16QI 0 "register_operand" "=w")
5300 (unspec:V16QI [(match_operand:OI 1 "register_operand" "w")
5301 (match_operand:V16QI 2 "register_operand" "w")]
5302 UNSPEC_TBL))]
5303 "TARGET_SIMD"
5304 "tbl\\t%0.16b, {%S1.16b - %T1.16b}, %2.16b"
5305 [(set_attr "type" "neon_tbl2_q")]
5306 )
5308 (define_insn "aarch64_tbl3<mode>"
5309 [(set (match_operand:VB 0 "register_operand" "=w")
5310 (unspec:VB [(match_operand:OI 1 "register_operand" "w")
5311 (match_operand:VB 2 "register_operand" "w")]
5312 UNSPEC_TBL))]
5313 "TARGET_SIMD"
5314 "tbl\\t%S0.<Vbtype>, {%S1.16b - %T1.16b}, %S2.<Vbtype>"
5315 [(set_attr "type" "neon_tbl3")]
5316 )
5318 (define_insn "aarch64_tbx4<mode>"
5319 [(set (match_operand:VB 0 "register_operand" "=w")
5320 (unspec:VB [(match_operand:VB 1 "register_operand" "0")
5321 (match_operand:OI 2 "register_operand" "w")
5322 (match_operand:VB 3 "register_operand" "w")]
5323 UNSPEC_TBX))]
5324 "TARGET_SIMD"
5325 "tbx\\t%S0.<Vbtype>, {%S2.16b - %T2.16b}, %S3.<Vbtype>"
5326 [(set_attr "type" "neon_tbl4")]
5327 )
5329 ;; Three source registers.
5331 (define_insn "aarch64_qtbl3<mode>"
5332 [(set (match_operand:VB 0 "register_operand" "=w")
5333 (unspec:VB [(match_operand:CI 1 "register_operand" "w")
5334 (match_operand:VB 2 "register_operand" "w")]
5335 UNSPEC_TBL))]
5336 "TARGET_SIMD"
5337 "tbl\\t%S0.<Vbtype>, {%S1.16b - %U1.16b}, %S2.<Vbtype>"
5338 [(set_attr "type" "neon_tbl3")]
5339 )
5341 (define_insn "aarch64_qtbx3<mode>"
5342 [(set (match_operand:VB 0 "register_operand" "=w")
5343 (unspec:VB [(match_operand:VB 1 "register_operand" "0")
5344 (match_operand:CI 2 "register_operand" "w")
5345 (match_operand:VB 3 "register_operand" "w")]
5346 UNSPEC_TBX))]
5347 "TARGET_SIMD"
5348 "tbx\\t%S0.<Vbtype>, {%S2.16b - %U2.16b}, %S3.<Vbtype>"
5349 [(set_attr "type" "neon_tbl3")]
5350 )
5352 ;; Four source registers.
5354 (define_insn "aarch64_qtbl4<mode>"
5355 [(set (match_operand:VB 0 "register_operand" "=w")
5356 (unspec:VB [(match_operand:XI 1 "register_operand" "w")
5357 (match_operand:VB 2 "register_operand" "w")]
5358 UNSPEC_TBL))]
5359 "TARGET_SIMD"
5360 "tbl\\t%S0.<Vbtype>, {%S1.16b - %V1.16b}, %S2.<Vbtype>"
5361 [(set_attr "type" "neon_tbl4")]
5362 )
5364 (define_insn "aarch64_qtbx4<mode>"
5365 [(set (match_operand:VB 0 "register_operand" "=w")
5366 (unspec:VB [(match_operand:VB 1 "register_operand" "0")
5367 (match_operand:XI 2 "register_operand" "w")
5368 (match_operand:VB 3 "register_operand" "w")]
5369 UNSPEC_TBX))]
5370 "TARGET_SIMD"
5371 "tbx\\t%S0.<Vbtype>, {%S2.16b - %V2.16b}, %S3.<Vbtype>"
5372 [(set_attr "type" "neon_tbl4")]
5373 )
5375 (define_insn_and_split "aarch64_combinev16qi"
5376 [(set (match_operand:OI 0 "register_operand" "=w")
5377 (unspec:OI [(match_operand:V16QI 1 "register_operand" "w")
5378 (match_operand:V16QI 2 "register_operand" "w")]
5379 UNSPEC_CONCAT))]
5380 "TARGET_SIMD"
5381 "#"
5382 "&& reload_completed"
5383 [(const_int 0)]
5384 {
5385 aarch64_split_combinev16qi (operands);
5386 DONE;
5387 }
5388 [(set_attr "type" "multiple")]
5389 )
5391 ;; This instruction's pattern is generated directly by
5392 ;; aarch64_expand_vec_perm_const, so any changes to the pattern would
5393 ;; need corresponding changes there.
5394 (define_insn "aarch64_<PERMUTE:perm_insn><PERMUTE:perm_hilo><mode>"
5395 [(set (match_operand:VALL_F16 0 "register_operand" "=w")
5396 (unspec:VALL_F16 [(match_operand:VALL_F16 1 "register_operand" "w")
5397 (match_operand:VALL_F16 2 "register_operand" "w")]
5398 PERMUTE))]
5399 "TARGET_SIMD"
5400 "<PERMUTE:perm_insn><PERMUTE:perm_hilo>\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
5401 [(set_attr "type" "neon_permute<q>")]
5402 )
5404 ;; This instruction's pattern is generated directly by
5405 ;; aarch64_expand_vec_perm_const, so any changes to the pattern would
5406 ;; need corresponding changes there. Note that the immediate (third)
5407 ;; operand is a lane index not a byte index.
5408 (define_insn "aarch64_ext<mode>"
5409 [(set (match_operand:VALL_F16 0 "register_operand" "=w")
5410 (unspec:VALL_F16 [(match_operand:VALL_F16 1 "register_operand" "w")
5411 (match_operand:VALL_F16 2 "register_operand" "w")
5412 (match_operand:SI 3 "immediate_operand" "i")]
5413 UNSPEC_EXT))]
5414 "TARGET_SIMD"
5415 {
5416 operands[3] = GEN_INT (INTVAL (operands[3])
5417 * GET_MODE_UNIT_SIZE (<MODE>mode));
5418 return "ext\\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>, #%3";
5419 }
5420 [(set_attr "type" "neon_ext<q>")]
5421 )
5423 ;; This instruction's pattern is generated directly by
5424 ;; aarch64_expand_vec_perm_const, so any changes to the pattern would
5425 ;; need corresponding changes there.
5426 (define_insn "aarch64_rev<REVERSE:rev_op><mode>"
5427 [(set (match_operand:VALL_F16 0 "register_operand" "=w")
5428 (unspec:VALL_F16 [(match_operand:VALL_F16 1 "register_operand" "w")]
5429 REVERSE))]
5430 "TARGET_SIMD"
5431 "rev<REVERSE:rev_op>\\t%0.<Vtype>, %1.<Vtype>"
5432 [(set_attr "type" "neon_rev<q>")]
5433 )
5435 (define_insn "aarch64_st2<mode>_dreg"
5436 [(set (match_operand:BLK 0 "aarch64_simd_struct_operand" "=Utv")
5437 (unspec:BLK [(match_operand:OI 1 "register_operand" "w")
5438 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5439 UNSPEC_ST2))]
5440 "TARGET_SIMD"
5441 "st2\\t{%S1.<Vtype> - %T1.<Vtype>}, %0"
5442 [(set_attr "type" "neon_store2_2reg")]
5443 )
5445 (define_insn "aarch64_st2<mode>_dreg"
5446 [(set (match_operand:BLK 0 "aarch64_simd_struct_operand" "=Utv")
5447 (unspec:BLK [(match_operand:OI 1 "register_operand" "w")
5448 (unspec:DX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5449 UNSPEC_ST2))]
5450 "TARGET_SIMD"
5451 "st1\\t{%S1.1d - %T1.1d}, %0"
5452 [(set_attr "type" "neon_store1_2reg")]
5453 )
5455 (define_insn "aarch64_st3<mode>_dreg"
5456 [(set (match_operand:BLK 0 "aarch64_simd_struct_operand" "=Utv")
5457 (unspec:BLK [(match_operand:CI 1 "register_operand" "w")
5458 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5459 UNSPEC_ST3))]
5460 "TARGET_SIMD"
5461 "st3\\t{%S1.<Vtype> - %U1.<Vtype>}, %0"
5462 [(set_attr "type" "neon_store3_3reg")]
5463 )
5465 (define_insn "aarch64_st3<mode>_dreg"
5466 [(set (match_operand:BLK 0 "aarch64_simd_struct_operand" "=Utv")
5467 (unspec:BLK [(match_operand:CI 1 "register_operand" "w")
5468 (unspec:DX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5469 UNSPEC_ST3))]
5470 "TARGET_SIMD"
5471 "st1\\t{%S1.1d - %U1.1d}, %0"
5472 [(set_attr "type" "neon_store1_3reg")]
5473 )
5475 (define_insn "aarch64_st4<mode>_dreg"
5476 [(set (match_operand:BLK 0 "aarch64_simd_struct_operand" "=Utv")
5477 (unspec:BLK [(match_operand:XI 1 "register_operand" "w")
5478 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5479 UNSPEC_ST4))]
5480 "TARGET_SIMD"
5481 "st4\\t{%S1.<Vtype> - %V1.<Vtype>}, %0"
5482 [(set_attr "type" "neon_store4_4reg")]
5483 )
5485 (define_insn "aarch64_st4<mode>_dreg"
5486 [(set (match_operand:BLK 0 "aarch64_simd_struct_operand" "=Utv")
5487 (unspec:BLK [(match_operand:XI 1 "register_operand" "w")
5488 (unspec:DX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5489 UNSPEC_ST4))]
5490 "TARGET_SIMD"
5491 "st1\\t{%S1.1d - %V1.1d}, %0"
5492 [(set_attr "type" "neon_store1_4reg")]
5493 )
5495 (define_expand "aarch64_st<VSTRUCT:nregs><VDC:mode>"
5496 [(match_operand:DI 0 "register_operand" "r")
5497 (match_operand:VSTRUCT 1 "register_operand" "w")
5498 (unspec:VDC [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5499 "TARGET_SIMD"
5500 {
5501 rtx mem = gen_rtx_MEM (BLKmode, operands[0]);
5502 set_mem_size (mem, <VSTRUCT:nregs> * 8);
5504 emit_insn (gen_aarch64_st<VSTRUCT:nregs><VDC:mode>_dreg (mem, operands[1]));
5505 DONE;
5506 })
5508 (define_expand "aarch64_st<VSTRUCT:nregs><VQ:mode>"
5509 [(match_operand:DI 0 "register_operand" "r")
5510 (match_operand:VSTRUCT 1 "register_operand" "w")
5511 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5512 "TARGET_SIMD"
5513 {
5514 machine_mode mode = <VSTRUCT:MODE>mode;
5515 rtx mem = gen_rtx_MEM (mode, operands[0]);
5517 emit_insn (gen_aarch64_simd_st<VSTRUCT:nregs><VQ:mode> (mem, operands[1]));
5518 DONE;
5519 })
5521 (define_expand "aarch64_st<VSTRUCT:nregs>_lane<VALLDIF:mode>"
5522 [(match_operand:DI 0 "register_operand" "r")
5523 (match_operand:VSTRUCT 1 "register_operand" "w")
5524 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)
5525 (match_operand:SI 2 "immediate_operand")]
5526 "TARGET_SIMD"
5527 {
5528 rtx mem = gen_rtx_MEM (BLKmode, operands[0]);
5529 set_mem_size (mem, GET_MODE_SIZE (GET_MODE_INNER (<VALLDIF:MODE>mode))
5530 * <VSTRUCT:nregs>);
5532 emit_insn (gen_aarch64_vec_store_lanes<VSTRUCT:mode>_lane<VALLDIF:mode> (
5533 mem, operands[1], operands[2]));
5534 DONE;
5535 })
5537 (define_expand "aarch64_st1<VALL_F16:mode>"
5538 [(match_operand:DI 0 "register_operand")
5539 (match_operand:VALL_F16 1 "register_operand")]
5540 "TARGET_SIMD"
5541 {
5542 machine_mode mode = <VALL_F16:MODE>mode;
5543 rtx mem = gen_rtx_MEM (mode, operands[0]);
5545 if (BYTES_BIG_ENDIAN)
5546 emit_insn (gen_aarch64_be_st1<VALL_F16:mode> (mem, operands[1]));
5547 else
5548 emit_move_insn (mem, operands[1]);
5549 DONE;
5550 })
5552 ;; Expander for builtins to insert vector registers into large
5553 ;; opaque integer modes.
5555 ;; Q-register list. We don't need a D-reg inserter as we zero
5556 ;; extend them in arm_neon.h and insert the resulting Q-regs.
5558 (define_expand "aarch64_set_qreg<VSTRUCT:mode><VQ:mode>"
5559 [(match_operand:VSTRUCT 0 "register_operand" "+w")
5560 (match_operand:VSTRUCT 1 "register_operand" "0")
5561 (match_operand:VQ 2 "register_operand" "w")
5562 (match_operand:SI 3 "immediate_operand" "i")]
5563 "TARGET_SIMD"
5564 {
5565 int part = INTVAL (operands[3]);
5566 int offset = part * 16;
5568 emit_move_insn (operands[0], operands[1]);
5569 emit_move_insn (gen_rtx_SUBREG (<VQ:MODE>mode, operands[0], offset),
5570 operands[2]);
5571 DONE;
5572 })
5574 ;; Standard pattern name vec_init<mode><Vel>.
5576 (define_expand "vec_init<mode><Vel>"
5577 [(match_operand:VALL_F16 0 "register_operand" "")
5578 (match_operand 1 "" "")]
5579 "TARGET_SIMD"
5580 {
5581 aarch64_expand_vector_init (operands[0], operands[1]);
5582 DONE;
5583 })
5585 (define_insn "*aarch64_simd_ld1r<mode>"
5586 [(set (match_operand:VALL_F16 0 "register_operand" "=w")
5587 (vec_duplicate:VALL_F16
5588 (match_operand:<VEL> 1 "aarch64_simd_struct_operand" "Utv")))]
5589 "TARGET_SIMD"
5590 "ld1r\\t{%0.<Vtype>}, %1"
5591 [(set_attr "type" "neon_load1_all_lanes")]
5592 )
5594 (define_insn "aarch64_frecpe<mode>"
5595 [(set (match_operand:VHSDF 0 "register_operand" "=w")
5596 (unspec:VHSDF [(match_operand:VHSDF 1 "register_operand" "w")]
5597 UNSPEC_FRECPE))]
5598 "TARGET_SIMD"
5599 "frecpe\\t%0.<Vtype>, %1.<Vtype>"
5600 [(set_attr "type" "neon_fp_recpe_<stype><q>")]
5601 )
5603 (define_insn "aarch64_frecp<FRECP:frecp_suffix><mode>"
5604 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5605 (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")]
5606 FRECP))]
5607 "TARGET_SIMD"
5608 "frecp<FRECP:frecp_suffix>\\t%<s>0, %<s>1"
5609 [(set_attr "type" "neon_fp_recp<FRECP:frecp_suffix>_<GPF_F16:stype>")]
5610 )
5612 (define_insn "aarch64_frecps<mode>"
5613 [(set (match_operand:VHSDF_HSDF 0 "register_operand" "=w")
5614 (unspec:VHSDF_HSDF
5615 [(match_operand:VHSDF_HSDF 1 "register_operand" "w")
5616 (match_operand:VHSDF_HSDF 2 "register_operand" "w")]
5617 UNSPEC_FRECPS))]
5618 "TARGET_SIMD"
5619 "frecps\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
5620 [(set_attr "type" "neon_fp_recps_<stype><q>")]
5621 )
5623 (define_insn "aarch64_urecpe<mode>"
5624 [(set (match_operand:VDQ_SI 0 "register_operand" "=w")
5625 (unspec:VDQ_SI [(match_operand:VDQ_SI 1 "register_operand" "w")]
5626 UNSPEC_URECPE))]
5627 "TARGET_SIMD"
5628 "urecpe\\t%0.<Vtype>, %1.<Vtype>"
5629 [(set_attr "type" "neon_fp_recpe_<Vetype><q>")])
5631 ;; Standard pattern name vec_extract<mode><Vel>.
5633 (define_expand "vec_extract<mode><Vel>"
5634 [(match_operand:<VEL> 0 "aarch64_simd_nonimmediate_operand" "")
5635 (match_operand:VALL_F16 1 "register_operand" "")
5636 (match_operand:SI 2 "immediate_operand" "")]
5637 "TARGET_SIMD"
5638 {
5639 emit_insn
5640 (gen_aarch64_get_lane<mode> (operands[0], operands[1], operands[2]));
5641 DONE;
5642 })
5644 ;; aes
5646 (define_insn "aarch64_crypto_aes<aes_op>v16qi"
5647 [(set (match_operand:V16QI 0 "register_operand" "=w")
5648 (unspec:V16QI [(match_operand:V16QI 1 "register_operand" "0")
5649 (match_operand:V16QI 2 "register_operand" "w")]
5650 CRYPTO_AES))]
5651 "TARGET_SIMD && TARGET_CRYPTO"
5652 "aes<aes_op>\\t%0.16b, %2.16b"
5653 [(set_attr "type" "crypto_aese")]
5654 )
5656 ;; When AES/AESMC fusion is enabled we want the register allocation to
5657 ;; look like:
5658 ;; AESE Vn, _
5659 ;; AESMC Vn, Vn
5660 ;; So prefer to tie operand 1 to operand 0 when fusing.
5662 (define_insn "aarch64_crypto_aes<aesmc_op>v16qi"
5663 [(set (match_operand:V16QI 0 "register_operand" "=w,w")
5664 (unspec:V16QI [(match_operand:V16QI 1 "register_operand" "0,w")]
5665 CRYPTO_AESMC))]
5666 "TARGET_SIMD && TARGET_CRYPTO"
5667 "aes<aesmc_op>\\t%0.16b, %1.16b"
5668 [(set_attr "type" "crypto_aesmc")
5669 (set_attr_alternative "enabled"
5670 [(if_then_else (match_test
5671 "aarch64_fusion_enabled_p (AARCH64_FUSE_AES_AESMC)")
5672 (const_string "yes" )
5673 (const_string "no"))
5674 (const_string "yes")])]
5675 )
5677 ;; sha1
5679 (define_insn "aarch64_crypto_sha1hsi"
5680 [(set (match_operand:SI 0 "register_operand" "=w")
5681 (unspec:SI [(match_operand:SI 1
5682 "register_operand" "w")]
5683 UNSPEC_SHA1H))]
5684 "TARGET_SIMD && TARGET_CRYPTO"
5685 "sha1h\\t%s0, %s1"
5686 [(set_attr "type" "crypto_sha1_fast")]
5687 )
5689 (define_insn "aarch64_crypto_sha1hv4si"
5690 [(set (match_operand:SI 0 "register_operand" "=w")
5691 (unspec:SI [(vec_select:SI (match_operand:V4SI 1 "register_operand" "w")
5692 (parallel [(const_int 0)]))]
5693 UNSPEC_SHA1H))]
5694 "TARGET_SIMD && TARGET_CRYPTO && !BYTES_BIG_ENDIAN"
5695 "sha1h\\t%s0, %s1"
5696 [(set_attr "type" "crypto_sha1_fast")]
5697 )
5699 (define_insn "aarch64_be_crypto_sha1hv4si"
5700 [(set (match_operand:SI 0 "register_operand" "=w")
5701 (unspec:SI [(vec_select:SI (match_operand:V4SI 1 "register_operand" "w")
5702 (parallel [(const_int 3)]))]
5703 UNSPEC_SHA1H))]
5704 "TARGET_SIMD && TARGET_CRYPTO && BYTES_BIG_ENDIAN"
5705 "sha1h\\t%s0, %s1"
5706 [(set_attr "type" "crypto_sha1_fast")]
5707 )
5709 (define_insn "aarch64_crypto_sha1su1v4si"
5710 [(set (match_operand:V4SI 0 "register_operand" "=w")
5711 (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
5712 (match_operand:V4SI 2 "register_operand" "w")]
5713 UNSPEC_SHA1SU1))]
5714 "TARGET_SIMD && TARGET_CRYPTO"
5715 "sha1su1\\t%0.4s, %2.4s"
5716 [(set_attr "type" "crypto_sha1_fast")]
5717 )
5719 (define_insn "aarch64_crypto_sha1<sha1_op>v4si"
5720 [(set (match_operand:V4SI 0 "register_operand" "=w")
5721 (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
5722 (match_operand:SI 2 "register_operand" "w")
5723 (match_operand:V4SI 3 "register_operand" "w")]
5724 CRYPTO_SHA1))]
5725 "TARGET_SIMD && TARGET_CRYPTO"
5726 "sha1<sha1_op>\\t%q0, %s2, %3.4s"
5727 [(set_attr "type" "crypto_sha1_slow")]
5728 )
5730 (define_insn "aarch64_crypto_sha1su0v4si"
5731 [(set (match_operand:V4SI 0 "register_operand" "=w")
5732 (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
5733 (match_operand:V4SI 2 "register_operand" "w")
5734 (match_operand:V4SI 3 "register_operand" "w")]
5735 UNSPEC_SHA1SU0))]
5736 "TARGET_SIMD && TARGET_CRYPTO"
5737 "sha1su0\\t%0.4s, %2.4s, %3.4s"
5738 [(set_attr "type" "crypto_sha1_xor")]
5739 )
5741 ;; sha256
5743 (define_insn "aarch64_crypto_sha256h<sha256_op>v4si"
5744 [(set (match_operand:V4SI 0 "register_operand" "=w")
5745 (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
5746 (match_operand:V4SI 2 "register_operand" "w")
5747 (match_operand:V4SI 3 "register_operand" "w")]
5748 CRYPTO_SHA256))]
5749 "TARGET_SIMD && TARGET_CRYPTO"
5750 "sha256h<sha256_op>\\t%q0, %q2, %3.4s"
5751 [(set_attr "type" "crypto_sha256_slow")]
5752 )
5754 (define_insn "aarch64_crypto_sha256su0v4si"
5755 [(set (match_operand:V4SI 0 "register_operand" "=w")
5756 (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
5757 (match_operand:V4SI 2 "register_operand" "w")]
5758 UNSPEC_SHA256SU0))]
5759 "TARGET_SIMD &&TARGET_CRYPTO"
5760 "sha256su0\\t%0.4s, %2.4s"
5761 [(set_attr "type" "crypto_sha256_fast")]
5762 )
5764 (define_insn "aarch64_crypto_sha256su1v4si"
5765 [(set (match_operand:V4SI 0 "register_operand" "=w")
5766 (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
5767 (match_operand:V4SI 2 "register_operand" "w")
5768 (match_operand:V4SI 3 "register_operand" "w")]
5769 UNSPEC_SHA256SU1))]
5770 "TARGET_SIMD &&TARGET_CRYPTO"
5771 "sha256su1\\t%0.4s, %2.4s, %3.4s"
5772 [(set_attr "type" "crypto_sha256_slow")]
5773 )
5775 ;; pmull
5777 (define_insn "aarch64_crypto_pmulldi"
5778 [(set (match_operand:TI 0 "register_operand" "=w")
5779 (unspec:TI [(match_operand:DI 1 "register_operand" "w")
5780 (match_operand:DI 2 "register_operand" "w")]
5781 UNSPEC_PMULL))]
5782 "TARGET_SIMD && TARGET_CRYPTO"
5783 "pmull\\t%0.1q, %1.1d, %2.1d"
5784 [(set_attr "type" "crypto_pmull")]
5785 )
5787 (define_insn "aarch64_crypto_pmullv2di"
5788 [(set (match_operand:TI 0 "register_operand" "=w")
5789 (unspec:TI [(match_operand:V2DI 1 "register_operand" "w")
5790 (match_operand:V2DI 2 "register_operand" "w")]
5791 UNSPEC_PMULL2))]
5792 "TARGET_SIMD && TARGET_CRYPTO"
5793 "pmull2\\t%0.1q, %1.2d, %2.2d"
5794 [(set_attr "type" "crypto_pmull")]
5795 )