| blob | c94946563e620ada88df1cfa0f7724143a147df7 |
1 ;; Machine description for AArch64 AdvSIMD architecture.
2 ;; Copyright (C) 2011-2017 Free Software Foundation, Inc.
3 ;; Contributed by ARM Ltd.
4 ;;
5 ;; This file is part of GCC.
6 ;;
7 ;; GCC is free software; you can redistribute it and/or modify it
8 ;; under the terms of the GNU General Public License as published by
9 ;; the Free Software Foundation; either version 3, or (at your option)
10 ;; any later version.
11 ;;
12 ;; GCC is distributed in the hope that it will be useful, but
13 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
14 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 ;; General Public License for more details.
16 ;;
17 ;; You should have received a copy of the GNU General Public License
18 ;; along with GCC; see the file COPYING3. If not see
19 ;; <http://www.gnu.org/licenses/>.
21 (define_expand "mov<mode>"
22 [(set (match_operand:VALL_F16 0 "nonimmediate_operand" "")
23 (match_operand:VALL_F16 1 "general_operand" ""))]
24 "TARGET_SIMD"
25 "
26 if (GET_CODE (operands[0]) == MEM)
27 operands[1] = force_reg (<MODE>mode, operands[1]);
28 "
29 )
31 (define_expand "movmisalign<mode>"
32 [(set (match_operand:VALL 0 "nonimmediate_operand" "")
33 (match_operand:VALL 1 "general_operand" ""))]
34 "TARGET_SIMD"
35 {
36 /* This pattern is not permitted to fail during expansion: if both arguments
37 are non-registers (e.g. memory := constant, which can be created by the
38 auto-vectorizer), force operand 1 into a register. */
39 if (!register_operand (operands[0], <MODE>mode)
40 && !register_operand (operands[1], <MODE>mode))
41 operands[1] = force_reg (<MODE>mode, operands[1]);
42 })
44 (define_insn "aarch64_simd_dup<mode>"
45 [(set (match_operand:VDQ_I 0 "register_operand" "=w, w")
46 (vec_duplicate:VDQ_I
47 (match_operand:<VEL> 1 "register_operand" "w,?r")))]
48 "TARGET_SIMD"
49 "@
50 dup\\t%0.<Vtype>, %1.<Vetype>[0]
51 dup\\t%0.<Vtype>, %<vw>1"
52 [(set_attr "type" "neon_dup<q>, neon_from_gp<q>")]
53 )
55 (define_insn "aarch64_simd_dup<mode>"
56 [(set (match_operand:VDQF_F16 0 "register_operand" "=w")
57 (vec_duplicate:VDQF_F16
58 (match_operand:<VEL> 1 "register_operand" "w")))]
59 "TARGET_SIMD"
60 "dup\\t%0.<Vtype>, %1.<Vetype>[0]"
61 [(set_attr "type" "neon_dup<q>")]
62 )
64 (define_insn "aarch64_dup_lane<mode>"
65 [(set (match_operand:VALL_F16 0 "register_operand" "=w")
66 (vec_duplicate:VALL_F16
67 (vec_select:<VEL>
68 (match_operand:VALL_F16 1 "register_operand" "w")
69 (parallel [(match_operand:SI 2 "immediate_operand" "i")])
70 )))]
71 "TARGET_SIMD"
72 {
73 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
74 return "dup\\t%0.<Vtype>, %1.<Vetype>[%2]";
75 }
76 [(set_attr "type" "neon_dup<q>")]
77 )
79 (define_insn "aarch64_dup_lane_<vswap_width_name><mode>"
80 [(set (match_operand:VALL_F16_NO_V2Q 0 "register_operand" "=w")
81 (vec_duplicate:VALL_F16_NO_V2Q
82 (vec_select:<VEL>
83 (match_operand:<VSWAP_WIDTH> 1 "register_operand" "w")
84 (parallel [(match_operand:SI 2 "immediate_operand" "i")])
85 )))]
86 "TARGET_SIMD"
87 {
88 operands[2] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
89 INTVAL (operands[2])));
90 return "dup\\t%0.<Vtype>, %1.<Vetype>[%2]";
91 }
92 [(set_attr "type" "neon_dup<q>")]
93 )
95 (define_insn "*aarch64_simd_mov<mode>"
96 [(set (match_operand:VD 0 "nonimmediate_operand"
97 "=w, m, w, ?r, ?w, ?r, w")
98 (match_operand:VD 1 "general_operand"
99 "m, w, w, w, r, r, Dn"))]
100 "TARGET_SIMD
101 && (register_operand (operands[0], <MODE>mode)
102 || register_operand (operands[1], <MODE>mode))"
103 {
104 switch (which_alternative)
105 {
106 case 0: return "ldr\\t%d0, %1";
107 case 1: return "str\\t%d1, %0";
108 case 2: return "orr\t%0.<Vbtype>, %1.<Vbtype>, %1.<Vbtype>";
109 case 3: return "umov\t%0, %1.d[0]";
110 case 4: return "fmov\t%d0, %1";
111 case 5: return "mov\t%0, %1";
112 case 6:
113 return aarch64_output_simd_mov_immediate (operands[1],
114 <MODE>mode, 64);
115 default: gcc_unreachable ();
116 }
117 }
118 [(set_attr "type" "neon_load1_1reg<q>, neon_store1_1reg<q>,\
119 neon_logic<q>, neon_to_gp<q>, f_mcr,\
120 mov_reg, neon_move<q>")]
121 )
123 (define_insn "*aarch64_simd_mov<mode>"
124 [(set (match_operand:VQ 0 "nonimmediate_operand"
125 "=w, m, w, ?r, ?w, ?r, w")
126 (match_operand:VQ 1 "general_operand"
127 "m, w, w, w, r, r, Dn"))]
128 "TARGET_SIMD
129 && (register_operand (operands[0], <MODE>mode)
130 || register_operand (operands[1], <MODE>mode))"
131 {
132 switch (which_alternative)
133 {
134 case 0:
135 return "ldr\\t%q0, %1";
136 case 1:
137 return "str\\t%q1, %0";
138 case 2:
139 return "orr\t%0.<Vbtype>, %1.<Vbtype>, %1.<Vbtype>";
140 case 3:
141 case 4:
142 case 5:
143 return "#";
144 case 6:
145 return aarch64_output_simd_mov_immediate (operands[1], <MODE>mode, 128);
146 default:
147 gcc_unreachable ();
148 }
149 }
150 [(set_attr "type" "neon_load1_1reg<q>, neon_store1_1reg<q>,\
151 neon_logic<q>, multiple, multiple, multiple,\
152 neon_move<q>")
153 (set_attr "length" "4,4,4,8,8,8,4")]
154 )
156 ;; When storing lane zero we can use the normal STR and its more permissive
157 ;; addressing modes.
159 (define_insn "aarch64_store_lane0<mode>"
160 [(set (match_operand:<VEL> 0 "memory_operand" "=m")
161 (vec_select:<VEL> (match_operand:VALL_F16 1 "register_operand" "w")
162 (parallel [(match_operand 2 "const_int_operand" "n")])))]
163 "TARGET_SIMD
164 && ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])) == 0"
165 "str\\t%<Vetype>1, %0"
166 [(set_attr "type" "neon_store1_1reg<q>")]
167 )
169 (define_insn "load_pair<mode>"
170 [(set (match_operand:VD 0 "register_operand" "=w")
171 (match_operand:VD 1 "aarch64_mem_pair_operand" "Ump"))
172 (set (match_operand:VD 2 "register_operand" "=w")
173 (match_operand:VD 3 "memory_operand" "m"))]
174 "TARGET_SIMD
175 && rtx_equal_p (XEXP (operands[3], 0),
176 plus_constant (Pmode,
177 XEXP (operands[1], 0),
178 GET_MODE_SIZE (<MODE>mode)))"
179 "ldp\\t%d0, %d2, %1"
180 [(set_attr "type" "neon_ldp")]
181 )
183 (define_insn "store_pair<mode>"
184 [(set (match_operand:VD 0 "aarch64_mem_pair_operand" "=Ump")
185 (match_operand:VD 1 "register_operand" "w"))
186 (set (match_operand:VD 2 "memory_operand" "=m")
187 (match_operand:VD 3 "register_operand" "w"))]
188 "TARGET_SIMD
189 && rtx_equal_p (XEXP (operands[2], 0),
190 plus_constant (Pmode,
191 XEXP (operands[0], 0),
192 GET_MODE_SIZE (<MODE>mode)))"
193 "stp\\t%d1, %d3, %0"
194 [(set_attr "type" "neon_stp")]
195 )
197 (define_split
198 [(set (match_operand:VQ 0 "register_operand" "")
199 (match_operand:VQ 1 "register_operand" ""))]
200 "TARGET_SIMD && reload_completed
201 && GP_REGNUM_P (REGNO (operands[0]))
202 && GP_REGNUM_P (REGNO (operands[1]))"
203 [(const_int 0)]
204 {
205 aarch64_simd_emit_reg_reg_move (operands, DImode, 2);
206 DONE;
207 })
209 (define_split
210 [(set (match_operand:VQ 0 "register_operand" "")
211 (match_operand:VQ 1 "register_operand" ""))]
212 "TARGET_SIMD && reload_completed
213 && ((FP_REGNUM_P (REGNO (operands[0])) && GP_REGNUM_P (REGNO (operands[1])))
214 || (GP_REGNUM_P (REGNO (operands[0])) && FP_REGNUM_P (REGNO (operands[1]))))"
215 [(const_int 0)]
216 {
217 aarch64_split_simd_move (operands[0], operands[1]);
218 DONE;
219 })
221 (define_expand "aarch64_split_simd_mov<mode>"
222 [(set (match_operand:VQ 0)
223 (match_operand:VQ 1))]
224 "TARGET_SIMD"
225 {
226 rtx dst = operands[0];
227 rtx src = operands[1];
229 if (GP_REGNUM_P (REGNO (src)))
230 {
231 rtx src_low_part = gen_lowpart (<VHALF>mode, src);
232 rtx src_high_part = gen_highpart (<VHALF>mode, src);
234 emit_insn
235 (gen_move_lo_quad_<mode> (dst, src_low_part));
236 emit_insn
237 (gen_move_hi_quad_<mode> (dst, src_high_part));
238 }
240 else
241 {
242 rtx dst_low_part = gen_lowpart (<VHALF>mode, dst);
243 rtx dst_high_part = gen_highpart (<VHALF>mode, dst);
244 rtx lo = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
245 rtx hi = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
247 emit_insn
248 (gen_aarch64_simd_mov_from_<mode>low (dst_low_part, src, lo));
249 emit_insn
250 (gen_aarch64_simd_mov_from_<mode>high (dst_high_part, src, hi));
251 }
252 DONE;
253 }
254 )
256 (define_insn "aarch64_simd_mov_from_<mode>low"
257 [(set (match_operand:<VHALF> 0 "register_operand" "=r")
258 (vec_select:<VHALF>
259 (match_operand:VQ 1 "register_operand" "w")
260 (match_operand:VQ 2 "vect_par_cnst_lo_half" "")))]
261 "TARGET_SIMD && reload_completed"
262 "umov\t%0, %1.d[0]"
263 [(set_attr "type" "neon_to_gp<q>")
264 (set_attr "length" "4")
265 ])
267 (define_insn "aarch64_simd_mov_from_<mode>high"
268 [(set (match_operand:<VHALF> 0 "register_operand" "=r")
269 (vec_select:<VHALF>
270 (match_operand:VQ 1 "register_operand" "w")
271 (match_operand:VQ 2 "vect_par_cnst_hi_half" "")))]
272 "TARGET_SIMD && reload_completed"
273 "umov\t%0, %1.d[1]"
274 [(set_attr "type" "neon_to_gp<q>")
275 (set_attr "length" "4")
276 ])
278 (define_insn "orn<mode>3"
279 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
280 (ior:VDQ_I (not:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w"))
281 (match_operand:VDQ_I 2 "register_operand" "w")))]
282 "TARGET_SIMD"
283 "orn\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>"
284 [(set_attr "type" "neon_logic<q>")]
285 )
287 (define_insn "bic<mode>3"
288 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
289 (and:VDQ_I (not:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w"))
290 (match_operand:VDQ_I 2 "register_operand" "w")))]
291 "TARGET_SIMD"
292 "bic\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>"
293 [(set_attr "type" "neon_logic<q>")]
294 )
296 (define_insn "add<mode>3"
297 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
298 (plus:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
299 (match_operand:VDQ_I 2 "register_operand" "w")))]
300 "TARGET_SIMD"
301 "add\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
302 [(set_attr "type" "neon_add<q>")]
303 )
305 (define_insn "sub<mode>3"
306 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
307 (minus:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
308 (match_operand:VDQ_I 2 "register_operand" "w")))]
309 "TARGET_SIMD"
310 "sub\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
311 [(set_attr "type" "neon_sub<q>")]
312 )
314 (define_insn "mul<mode>3"
315 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
316 (mult:VDQ_BHSI (match_operand:VDQ_BHSI 1 "register_operand" "w")
317 (match_operand:VDQ_BHSI 2 "register_operand" "w")))]
318 "TARGET_SIMD"
319 "mul\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
320 [(set_attr "type" "neon_mul_<Vetype><q>")]
321 )
323 (define_insn "bswap<mode>2"
324 [(set (match_operand:VDQHSD 0 "register_operand" "=w")
325 (bswap:VDQHSD (match_operand:VDQHSD 1 "register_operand" "w")))]
326 "TARGET_SIMD"
327 "rev<Vrevsuff>\\t%0.<Vbtype>, %1.<Vbtype>"
328 [(set_attr "type" "neon_rev<q>")]
329 )
331 (define_insn "aarch64_rbit<mode>"
332 [(set (match_operand:VB 0 "register_operand" "=w")
333 (unspec:VB [(match_operand:VB 1 "register_operand" "w")]
334 UNSPEC_RBIT))]
335 "TARGET_SIMD"
336 "rbit\\t%0.<Vbtype>, %1.<Vbtype>"
337 [(set_attr "type" "neon_rbit")]
338 )
340 (define_expand "ctz<mode>2"
341 [(set (match_operand:VS 0 "register_operand")
342 (ctz:VS (match_operand:VS 1 "register_operand")))]
343 "TARGET_SIMD"
344 {
345 emit_insn (gen_bswap<mode>2 (operands[0], operands[1]));
346 rtx op0_castsi2qi = simplify_gen_subreg(<VS:VSI2QI>mode, operands[0],
347 <MODE>mode, 0);
348 emit_insn (gen_aarch64_rbit<VS:vsi2qi> (op0_castsi2qi, op0_castsi2qi));
349 emit_insn (gen_clz<mode>2 (operands[0], operands[0]));
350 DONE;
351 }
352 )
354 (define_expand "copysign<mode>3"
355 [(match_operand:VHSDF 0 "register_operand")
356 (match_operand:VHSDF 1 "register_operand")
357 (match_operand:VHSDF 2 "register_operand")]
358 "TARGET_FLOAT && TARGET_SIMD"
359 {
360 rtx v_bitmask = gen_reg_rtx (<V_cmp_result>mode);
361 int bits = GET_MODE_UNIT_BITSIZE (<MODE>mode) - 1;
363 emit_move_insn (v_bitmask,
364 aarch64_simd_gen_const_vector_dup (<V_cmp_result>mode,
365 HOST_WIDE_INT_M1U << bits));
366 emit_insn (gen_aarch64_simd_bsl<mode> (operands[0], v_bitmask,
367 operands[2], operands[1]));
368 DONE;
369 }
370 )
372 (define_insn "*aarch64_mul3_elt<mode>"
373 [(set (match_operand:VMUL 0 "register_operand" "=w")
374 (mult:VMUL
375 (vec_duplicate:VMUL
376 (vec_select:<VEL>
377 (match_operand:VMUL 1 "register_operand" "<h_con>")
378 (parallel [(match_operand:SI 2 "immediate_operand")])))
379 (match_operand:VMUL 3 "register_operand" "w")))]
380 "TARGET_SIMD"
381 {
382 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
383 return "<f>mul\\t%0.<Vtype>, %3.<Vtype>, %1.<Vetype>[%2]";
384 }
385 [(set_attr "type" "neon<fp>_mul_<stype>_scalar<q>")]
386 )
388 (define_insn "*aarch64_mul3_elt_<vswap_width_name><mode>"
389 [(set (match_operand:VMUL_CHANGE_NLANES 0 "register_operand" "=w")
390 (mult:VMUL_CHANGE_NLANES
391 (vec_duplicate:VMUL_CHANGE_NLANES
392 (vec_select:<VEL>
393 (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
394 (parallel [(match_operand:SI 2 "immediate_operand")])))
395 (match_operand:VMUL_CHANGE_NLANES 3 "register_operand" "w")))]
396 "TARGET_SIMD"
397 {
398 operands[2] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
399 INTVAL (operands[2])));
400 return "<f>mul\\t%0.<Vtype>, %3.<Vtype>, %1.<Vetype>[%2]";
401 }
402 [(set_attr "type" "neon<fp>_mul_<Vetype>_scalar<q>")]
403 )
405 (define_insn "*aarch64_mul3_elt_from_dup<mode>"
406 [(set (match_operand:VMUL 0 "register_operand" "=w")
407 (mult:VMUL
408 (vec_duplicate:VMUL
409 (match_operand:<VEL> 1 "register_operand" "<h_con>"))
410 (match_operand:VMUL 2 "register_operand" "w")))]
411 "TARGET_SIMD"
412 "<f>mul\t%0.<Vtype>, %2.<Vtype>, %1.<Vetype>[0]";
413 [(set_attr "type" "neon<fp>_mul_<stype>_scalar<q>")]
414 )
416 (define_insn "aarch64_rsqrte<mode>"
417 [(set (match_operand:VHSDF_HSDF 0 "register_operand" "=w")
418 (unspec:VHSDF_HSDF [(match_operand:VHSDF_HSDF 1 "register_operand" "w")]
419 UNSPEC_RSQRTE))]
420 "TARGET_SIMD"
421 "frsqrte\\t%<v>0<Vmtype>, %<v>1<Vmtype>"
422 [(set_attr "type" "neon_fp_rsqrte_<stype><q>")])
424 (define_insn "aarch64_rsqrts<mode>"
425 [(set (match_operand:VHSDF_HSDF 0 "register_operand" "=w")
426 (unspec:VHSDF_HSDF [(match_operand:VHSDF_HSDF 1 "register_operand" "w")
427 (match_operand:VHSDF_HSDF 2 "register_operand" "w")]
428 UNSPEC_RSQRTS))]
429 "TARGET_SIMD"
430 "frsqrts\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
431 [(set_attr "type" "neon_fp_rsqrts_<stype><q>")])
433 (define_expand "rsqrt<mode>2"
434 [(set (match_operand:VALLF 0 "register_operand" "=w")
435 (unspec:VALLF [(match_operand:VALLF 1 "register_operand" "w")]
436 UNSPEC_RSQRT))]
437 "TARGET_SIMD"
438 {
439 aarch64_emit_approx_sqrt (operands[0], operands[1], true);
440 DONE;
441 })
443 (define_insn "*aarch64_mul3_elt_to_64v2df"
444 [(set (match_operand:DF 0 "register_operand" "=w")
445 (mult:DF
446 (vec_select:DF
447 (match_operand:V2DF 1 "register_operand" "w")
448 (parallel [(match_operand:SI 2 "immediate_operand")]))
449 (match_operand:DF 3 "register_operand" "w")))]
450 "TARGET_SIMD"
451 {
452 operands[2] = GEN_INT (ENDIAN_LANE_N (V2DFmode, INTVAL (operands[2])));
453 return "fmul\\t%0.2d, %3.2d, %1.d[%2]";
454 }
455 [(set_attr "type" "neon_fp_mul_d_scalar_q")]
456 )
458 (define_insn "neg<mode>2"
459 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
460 (neg:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")))]
461 "TARGET_SIMD"
462 "neg\t%0.<Vtype>, %1.<Vtype>"
463 [(set_attr "type" "neon_neg<q>")]
464 )
466 (define_insn "abs<mode>2"
467 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
468 (abs:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")))]
469 "TARGET_SIMD"
470 "abs\t%0.<Vtype>, %1.<Vtype>"
471 [(set_attr "type" "neon_abs<q>")]
472 )
474 ;; The intrinsic version of integer ABS must not be allowed to
475 ;; combine with any operation with an integerated ABS step, such
476 ;; as SABD.
477 (define_insn "aarch64_abs<mode>"
478 [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
479 (unspec:VSDQ_I_DI
480 [(match_operand:VSDQ_I_DI 1 "register_operand" "w")]
481 UNSPEC_ABS))]
482 "TARGET_SIMD"
483 "abs\t%<v>0<Vmtype>, %<v>1<Vmtype>"
484 [(set_attr "type" "neon_abs<q>")]
485 )
487 (define_insn "abd<mode>_3"
488 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
489 (abs:VDQ_BHSI (minus:VDQ_BHSI
490 (match_operand:VDQ_BHSI 1 "register_operand" "w")
491 (match_operand:VDQ_BHSI 2 "register_operand" "w"))))]
492 "TARGET_SIMD"
493 "sabd\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
494 [(set_attr "type" "neon_abd<q>")]
495 )
497 (define_insn "aba<mode>_3"
498 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
499 (plus:VDQ_BHSI (abs:VDQ_BHSI (minus:VDQ_BHSI
500 (match_operand:VDQ_BHSI 1 "register_operand" "w")
501 (match_operand:VDQ_BHSI 2 "register_operand" "w")))
502 (match_operand:VDQ_BHSI 3 "register_operand" "0")))]
503 "TARGET_SIMD"
504 "saba\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
505 [(set_attr "type" "neon_arith_acc<q>")]
506 )
508 (define_insn "fabd<mode>3"
509 [(set (match_operand:VHSDF_HSDF 0 "register_operand" "=w")
510 (abs:VHSDF_HSDF
511 (minus:VHSDF_HSDF
512 (match_operand:VHSDF_HSDF 1 "register_operand" "w")
513 (match_operand:VHSDF_HSDF 2 "register_operand" "w"))))]
514 "TARGET_SIMD"
515 "fabd\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
516 [(set_attr "type" "neon_fp_abd_<stype><q>")]
517 )
519 (define_insn "and<mode>3"
520 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
521 (and:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
522 (match_operand:VDQ_I 2 "register_operand" "w")))]
523 "TARGET_SIMD"
524 "and\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>"
525 [(set_attr "type" "neon_logic<q>")]
526 )
528 (define_insn "ior<mode>3"
529 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
530 (ior:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
531 (match_operand:VDQ_I 2 "register_operand" "w")))]
532 "TARGET_SIMD"
533 "orr\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>"
534 [(set_attr "type" "neon_logic<q>")]
535 )
537 (define_insn "xor<mode>3"
538 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
539 (xor:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
540 (match_operand:VDQ_I 2 "register_operand" "w")))]
541 "TARGET_SIMD"
542 "eor\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>"
543 [(set_attr "type" "neon_logic<q>")]
544 )
546 (define_insn "one_cmpl<mode>2"
547 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
548 (not:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")))]
549 "TARGET_SIMD"
550 "not\t%0.<Vbtype>, %1.<Vbtype>"
551 [(set_attr "type" "neon_logic<q>")]
552 )
554 (define_insn "aarch64_simd_vec_set<mode>"
555 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w,w,w")
556 (vec_merge:VDQ_BHSI
557 (vec_duplicate:VDQ_BHSI
558 (match_operand:<VEL> 1 "aarch64_simd_general_operand" "r,w,Utv"))
559 (match_operand:VDQ_BHSI 3 "register_operand" "0,0,0")
560 (match_operand:SI 2 "immediate_operand" "i,i,i")))]
561 "TARGET_SIMD"
562 {
563 int elt = ENDIAN_LANE_N (<MODE>mode, exact_log2 (INTVAL (operands[2])));
564 operands[2] = GEN_INT ((HOST_WIDE_INT) 1 << elt);
565 switch (which_alternative)
566 {
567 case 0:
568 return "ins\\t%0.<Vetype>[%p2], %w1";
569 case 1:
570 return "ins\\t%0.<Vetype>[%p2], %1.<Vetype>[0]";
571 case 2:
572 return "ld1\\t{%0.<Vetype>}[%p2], %1";
573 default:
574 gcc_unreachable ();
575 }
576 }
577 [(set_attr "type" "neon_from_gp<q>, neon_ins<q>, neon_load1_one_lane<q>")]
578 )
580 (define_insn "*aarch64_simd_vec_copy_lane<mode>"
581 [(set (match_operand:VALL_F16 0 "register_operand" "=w")
582 (vec_merge:VALL_F16
583 (vec_duplicate:VALL_F16
584 (vec_select:<VEL>
585 (match_operand:VALL_F16 3 "register_operand" "w")
586 (parallel
587 [(match_operand:SI 4 "immediate_operand" "i")])))
588 (match_operand:VALL_F16 1 "register_operand" "0")
589 (match_operand:SI 2 "immediate_operand" "i")))]
590 "TARGET_SIMD"
591 {
592 int elt = ENDIAN_LANE_N (<MODE>mode, exact_log2 (INTVAL (operands[2])));
593 operands[2] = GEN_INT (HOST_WIDE_INT_1 << elt);
594 operands[4] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[4])));
596 return "ins\t%0.<Vetype>[%p2], %3.<Vetype>[%4]";
597 }
598 [(set_attr "type" "neon_ins<q>")]
599 )
601 (define_insn "*aarch64_simd_vec_copy_lane_<vswap_width_name><mode>"
602 [(set (match_operand:VALL_F16_NO_V2Q 0 "register_operand" "=w")
603 (vec_merge:VALL_F16_NO_V2Q
604 (vec_duplicate:VALL_F16_NO_V2Q
605 (vec_select:<VEL>
606 (match_operand:<VSWAP_WIDTH> 3 "register_operand" "w")
607 (parallel
608 [(match_operand:SI 4 "immediate_operand" "i")])))
609 (match_operand:VALL_F16_NO_V2Q 1 "register_operand" "0")
610 (match_operand:SI 2 "immediate_operand" "i")))]
611 "TARGET_SIMD"
612 {
613 int elt = ENDIAN_LANE_N (<MODE>mode, exact_log2 (INTVAL (operands[2])));
614 operands[2] = GEN_INT (HOST_WIDE_INT_1 << elt);
615 operands[4] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
616 INTVAL (operands[4])));
618 return "ins\t%0.<Vetype>[%p2], %3.<Vetype>[%4]";
619 }
620 [(set_attr "type" "neon_ins<q>")]
621 )
623 (define_insn "aarch64_simd_lshr<mode>"
624 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
625 (lshiftrt:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
626 (match_operand:VDQ_I 2 "aarch64_simd_rshift_imm" "Dr")))]
627 "TARGET_SIMD"
628 "ushr\t%0.<Vtype>, %1.<Vtype>, %2"
629 [(set_attr "type" "neon_shift_imm<q>")]
630 )
632 (define_insn "aarch64_simd_ashr<mode>"
633 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
634 (ashiftrt:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
635 (match_operand:VDQ_I 2 "aarch64_simd_rshift_imm" "Dr")))]
636 "TARGET_SIMD"
637 "sshr\t%0.<Vtype>, %1.<Vtype>, %2"
638 [(set_attr "type" "neon_shift_imm<q>")]
639 )
641 (define_insn "aarch64_simd_imm_shl<mode>"
642 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
643 (ashift:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
644 (match_operand:VDQ_I 2 "aarch64_simd_lshift_imm" "Dl")))]
645 "TARGET_SIMD"
646 "shl\t%0.<Vtype>, %1.<Vtype>, %2"
647 [(set_attr "type" "neon_shift_imm<q>")]
648 )
650 (define_insn "aarch64_simd_reg_sshl<mode>"
651 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
652 (ashift:VDQ_I (match_operand:VDQ_I 1 "register_operand" "w")
653 (match_operand:VDQ_I 2 "register_operand" "w")))]
654 "TARGET_SIMD"
655 "sshl\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
656 [(set_attr "type" "neon_shift_reg<q>")]
657 )
659 (define_insn "aarch64_simd_reg_shl<mode>_unsigned"
660 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
661 (unspec:VDQ_I [(match_operand:VDQ_I 1 "register_operand" "w")
662 (match_operand:VDQ_I 2 "register_operand" "w")]
663 UNSPEC_ASHIFT_UNSIGNED))]
664 "TARGET_SIMD"
665 "ushl\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
666 [(set_attr "type" "neon_shift_reg<q>")]
667 )
669 (define_insn "aarch64_simd_reg_shl<mode>_signed"
670 [(set (match_operand:VDQ_I 0 "register_operand" "=w")
671 (unspec:VDQ_I [(match_operand:VDQ_I 1 "register_operand" "w")
672 (match_operand:VDQ_I 2 "register_operand" "w")]
673 UNSPEC_ASHIFT_SIGNED))]
674 "TARGET_SIMD"
675 "sshl\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
676 [(set_attr "type" "neon_shift_reg<q>")]
677 )
679 (define_expand "ashl<mode>3"
680 [(match_operand:VDQ_I 0 "register_operand" "")
681 (match_operand:VDQ_I 1 "register_operand" "")
682 (match_operand:SI 2 "general_operand" "")]
683 "TARGET_SIMD"
684 {
685 int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
686 int shift_amount;
688 if (CONST_INT_P (operands[2]))
689 {
690 shift_amount = INTVAL (operands[2]);
691 if (shift_amount >= 0 && shift_amount < bit_width)
692 {
693 rtx tmp = aarch64_simd_gen_const_vector_dup (<MODE>mode,
694 shift_amount);
695 emit_insn (gen_aarch64_simd_imm_shl<mode> (operands[0],
696 operands[1],
697 tmp));
698 DONE;
699 }
700 else
701 {
702 operands[2] = force_reg (SImode, operands[2]);
703 }
704 }
705 else if (MEM_P (operands[2]))
706 {
707 operands[2] = force_reg (SImode, operands[2]);
708 }
710 if (REG_P (operands[2]))
711 {
712 rtx tmp = gen_reg_rtx (<MODE>mode);
713 emit_insn (gen_aarch64_simd_dup<mode> (tmp,
714 convert_to_mode (<VEL>mode,
715 operands[2],
716 0)));
717 emit_insn (gen_aarch64_simd_reg_sshl<mode> (operands[0], operands[1],
718 tmp));
719 DONE;
720 }
721 else
722 FAIL;
723 }
724 )
726 (define_expand "lshr<mode>3"
727 [(match_operand:VDQ_I 0 "register_operand" "")
728 (match_operand:VDQ_I 1 "register_operand" "")
729 (match_operand:SI 2 "general_operand" "")]
730 "TARGET_SIMD"
731 {
732 int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
733 int shift_amount;
735 if (CONST_INT_P (operands[2]))
736 {
737 shift_amount = INTVAL (operands[2]);
738 if (shift_amount > 0 && shift_amount <= bit_width)
739 {
740 rtx tmp = aarch64_simd_gen_const_vector_dup (<MODE>mode,
741 shift_amount);
742 emit_insn (gen_aarch64_simd_lshr<mode> (operands[0],
743 operands[1],
744 tmp));
745 DONE;
746 }
747 else
748 operands[2] = force_reg (SImode, operands[2]);
749 }
750 else if (MEM_P (operands[2]))
751 {
752 operands[2] = force_reg (SImode, operands[2]);
753 }
755 if (REG_P (operands[2]))
756 {
757 rtx tmp = gen_reg_rtx (SImode);
758 rtx tmp1 = gen_reg_rtx (<MODE>mode);
759 emit_insn (gen_negsi2 (tmp, operands[2]));
760 emit_insn (gen_aarch64_simd_dup<mode> (tmp1,
761 convert_to_mode (<VEL>mode,
762 tmp, 0)));
763 emit_insn (gen_aarch64_simd_reg_shl<mode>_unsigned (operands[0],
764 operands[1],
765 tmp1));
766 DONE;
767 }
768 else
769 FAIL;
770 }
771 )
773 (define_expand "ashr<mode>3"
774 [(match_operand:VDQ_I 0 "register_operand" "")
775 (match_operand:VDQ_I 1 "register_operand" "")
776 (match_operand:SI 2 "general_operand" "")]
777 "TARGET_SIMD"
778 {
779 int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
780 int shift_amount;
782 if (CONST_INT_P (operands[2]))
783 {
784 shift_amount = INTVAL (operands[2]);
785 if (shift_amount > 0 && shift_amount <= bit_width)
786 {
787 rtx tmp = aarch64_simd_gen_const_vector_dup (<MODE>mode,
788 shift_amount);
789 emit_insn (gen_aarch64_simd_ashr<mode> (operands[0],
790 operands[1],
791 tmp));
792 DONE;
793 }
794 else
795 operands[2] = force_reg (SImode, operands[2]);
796 }
797 else if (MEM_P (operands[2]))
798 {
799 operands[2] = force_reg (SImode, operands[2]);
800 }
802 if (REG_P (operands[2]))
803 {
804 rtx tmp = gen_reg_rtx (SImode);
805 rtx tmp1 = gen_reg_rtx (<MODE>mode);
806 emit_insn (gen_negsi2 (tmp, operands[2]));
807 emit_insn (gen_aarch64_simd_dup<mode> (tmp1,
808 convert_to_mode (<VEL>mode,
809 tmp, 0)));
810 emit_insn (gen_aarch64_simd_reg_shl<mode>_signed (operands[0],
811 operands[1],
812 tmp1));
813 DONE;
814 }
815 else
816 FAIL;
817 }
818 )
820 (define_expand "vashl<mode>3"
821 [(match_operand:VDQ_I 0 "register_operand" "")
822 (match_operand:VDQ_I 1 "register_operand" "")
823 (match_operand:VDQ_I 2 "register_operand" "")]
824 "TARGET_SIMD"
825 {
826 emit_insn (gen_aarch64_simd_reg_sshl<mode> (operands[0], operands[1],
827 operands[2]));
828 DONE;
829 })
831 ;; Using mode VDQ_BHSI as there is no V2DImode neg!
832 ;; Negating individual lanes most certainly offsets the
833 ;; gain from vectorization.
834 (define_expand "vashr<mode>3"
835 [(match_operand:VDQ_BHSI 0 "register_operand" "")
836 (match_operand:VDQ_BHSI 1 "register_operand" "")
837 (match_operand:VDQ_BHSI 2 "register_operand" "")]
838 "TARGET_SIMD"
839 {
840 rtx neg = gen_reg_rtx (<MODE>mode);
841 emit (gen_neg<mode>2 (neg, operands[2]));
842 emit_insn (gen_aarch64_simd_reg_shl<mode>_signed (operands[0], operands[1],
843 neg));
844 DONE;
845 })
847 ;; DI vector shift
848 (define_expand "aarch64_ashr_simddi"
849 [(match_operand:DI 0 "register_operand" "=w")
850 (match_operand:DI 1 "register_operand" "w")
851 (match_operand:SI 2 "aarch64_shift_imm64_di" "")]
852 "TARGET_SIMD"
853 {
854 /* An arithmetic shift right by 64 fills the result with copies of the sign
855 bit, just like asr by 63 - however the standard pattern does not handle
856 a shift by 64. */
857 if (INTVAL (operands[2]) == 64)
858 operands[2] = GEN_INT (63);
859 emit_insn (gen_ashrdi3 (operands[0], operands[1], operands[2]));
860 DONE;
861 }
862 )
864 (define_expand "vlshr<mode>3"
865 [(match_operand:VDQ_BHSI 0 "register_operand" "")
866 (match_operand:VDQ_BHSI 1 "register_operand" "")
867 (match_operand:VDQ_BHSI 2 "register_operand" "")]
868 "TARGET_SIMD"
869 {
870 rtx neg = gen_reg_rtx (<MODE>mode);
871 emit (gen_neg<mode>2 (neg, operands[2]));
872 emit_insn (gen_aarch64_simd_reg_shl<mode>_unsigned (operands[0], operands[1],
873 neg));
874 DONE;
875 })
877 (define_expand "aarch64_lshr_simddi"
878 [(match_operand:DI 0 "register_operand" "=w")
879 (match_operand:DI 1 "register_operand" "w")
880 (match_operand:SI 2 "aarch64_shift_imm64_di" "")]
881 "TARGET_SIMD"
882 {
883 if (INTVAL (operands[2]) == 64)
884 emit_move_insn (operands[0], const0_rtx);
885 else
886 emit_insn (gen_lshrdi3 (operands[0], operands[1], operands[2]));
887 DONE;
888 }
889 )
891 (define_expand "vec_set<mode>"
892 [(match_operand:VDQ_BHSI 0 "register_operand")
893 (match_operand:<VEL> 1 "register_operand")
894 (match_operand:SI 2 "immediate_operand")]
895 "TARGET_SIMD"
896 {
897 HOST_WIDE_INT elem = (HOST_WIDE_INT) 1 << INTVAL (operands[2]);
898 emit_insn (gen_aarch64_simd_vec_set<mode> (operands[0], operands[1],
899 GEN_INT (elem), operands[0]));
900 DONE;
901 }
902 )
904 ;; For 64-bit modes we use ushl/r, as this does not require a SIMD zero.
905 (define_insn "vec_shr_<mode>"
906 [(set (match_operand:VD 0 "register_operand" "=w")
907 (unspec:VD [(match_operand:VD 1 "register_operand" "w")
908 (match_operand:SI 2 "immediate_operand" "i")]
909 UNSPEC_VEC_SHR))]
910 "TARGET_SIMD"
911 {
912 if (BYTES_BIG_ENDIAN)
913 return "shl %d0, %d1, %2";
914 else
915 return "ushr %d0, %d1, %2";
916 }
917 [(set_attr "type" "neon_shift_imm")]
918 )
920 (define_insn "aarch64_simd_vec_setv2di"
921 [(set (match_operand:V2DI 0 "register_operand" "=w,w")
922 (vec_merge:V2DI
923 (vec_duplicate:V2DI
924 (match_operand:DI 1 "register_operand" "r,w"))
925 (match_operand:V2DI 3 "register_operand" "0,0")
926 (match_operand:SI 2 "immediate_operand" "i,i")))]
927 "TARGET_SIMD"
928 {
929 int elt = ENDIAN_LANE_N (V2DImode, exact_log2 (INTVAL (operands[2])));
930 operands[2] = GEN_INT ((HOST_WIDE_INT) 1 << elt);
931 switch (which_alternative)
932 {
933 case 0:
934 return "ins\\t%0.d[%p2], %1";
935 case 1:
936 return "ins\\t%0.d[%p2], %1.d[0]";
937 default:
938 gcc_unreachable ();
939 }
940 }
941 [(set_attr "type" "neon_from_gp, neon_ins_q")]
942 )
944 (define_expand "vec_setv2di"
945 [(match_operand:V2DI 0 "register_operand")
946 (match_operand:DI 1 "register_operand")
947 (match_operand:SI 2 "immediate_operand")]
948 "TARGET_SIMD"
949 {
950 HOST_WIDE_INT elem = (HOST_WIDE_INT) 1 << INTVAL (operands[2]);
951 emit_insn (gen_aarch64_simd_vec_setv2di (operands[0], operands[1],
952 GEN_INT (elem), operands[0]));
953 DONE;
954 }
955 )
957 (define_insn "aarch64_simd_vec_set<mode>"
958 [(set (match_operand:VDQF_F16 0 "register_operand" "=w")
959 (vec_merge:VDQF_F16
960 (vec_duplicate:VDQF_F16
961 (match_operand:<VEL> 1 "register_operand" "w"))
962 (match_operand:VDQF_F16 3 "register_operand" "0")
963 (match_operand:SI 2 "immediate_operand" "i")))]
964 "TARGET_SIMD"
965 {
966 int elt = ENDIAN_LANE_N (<MODE>mode, exact_log2 (INTVAL (operands[2])));
968 operands[2] = GEN_INT ((HOST_WIDE_INT)1 << elt);
969 return "ins\t%0.<Vetype>[%p2], %1.<Vetype>[0]";
970 }
971 [(set_attr "type" "neon_ins<q>")]
972 )
974 (define_expand "vec_set<mode>"
975 [(match_operand:VDQF_F16 0 "register_operand" "+w")
976 (match_operand:<VEL> 1 "register_operand" "w")
977 (match_operand:SI 2 "immediate_operand" "")]
978 "TARGET_SIMD"
979 {
980 HOST_WIDE_INT elem = (HOST_WIDE_INT) 1 << INTVAL (operands[2]);
981 emit_insn (gen_aarch64_simd_vec_set<mode> (operands[0], operands[1],
982 GEN_INT (elem), operands[0]));
983 DONE;
984 }
985 )
988 (define_insn "aarch64_mla<mode>"
989 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
990 (plus:VDQ_BHSI (mult:VDQ_BHSI
991 (match_operand:VDQ_BHSI 2 "register_operand" "w")
992 (match_operand:VDQ_BHSI 3 "register_operand" "w"))
993 (match_operand:VDQ_BHSI 1 "register_operand" "0")))]
994 "TARGET_SIMD"
995 "mla\t%0.<Vtype>, %2.<Vtype>, %3.<Vtype>"
996 [(set_attr "type" "neon_mla_<Vetype><q>")]
997 )
999 (define_insn "*aarch64_mla_elt<mode>"
1000 [(set (match_operand:VDQHS 0 "register_operand" "=w")
1001 (plus:VDQHS
1002 (mult:VDQHS
1003 (vec_duplicate:VDQHS
1004 (vec_select:<VEL>
1005 (match_operand:VDQHS 1 "register_operand" "<h_con>")
1006 (parallel [(match_operand:SI 2 "immediate_operand")])))
1007 (match_operand:VDQHS 3 "register_operand" "w"))
1008 (match_operand:VDQHS 4 "register_operand" "0")))]
1009 "TARGET_SIMD"
1010 {
1011 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
1012 return "mla\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
1013 }
1014 [(set_attr "type" "neon_mla_<Vetype>_scalar<q>")]
1015 )
1017 (define_insn "*aarch64_mla_elt_<vswap_width_name><mode>"
1018 [(set (match_operand:VDQHS 0 "register_operand" "=w")
1019 (plus:VDQHS
1020 (mult:VDQHS
1021 (vec_duplicate:VDQHS
1022 (vec_select:<VEL>
1023 (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
1024 (parallel [(match_operand:SI 2 "immediate_operand")])))
1025 (match_operand:VDQHS 3 "register_operand" "w"))
1026 (match_operand:VDQHS 4 "register_operand" "0")))]
1027 "TARGET_SIMD"
1028 {
1029 operands[2] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
1030 INTVAL (operands[2])));
1031 return "mla\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
1032 }
1033 [(set_attr "type" "neon_mla_<Vetype>_scalar<q>")]
1034 )
1036 (define_insn "aarch64_mls<mode>"
1037 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
1038 (minus:VDQ_BHSI (match_operand:VDQ_BHSI 1 "register_operand" "0")
1039 (mult:VDQ_BHSI (match_operand:VDQ_BHSI 2 "register_operand" "w")
1040 (match_operand:VDQ_BHSI 3 "register_operand" "w"))))]
1041 "TARGET_SIMD"
1042 "mls\t%0.<Vtype>, %2.<Vtype>, %3.<Vtype>"
1043 [(set_attr "type" "neon_mla_<Vetype><q>")]
1044 )
1046 (define_insn "*aarch64_mls_elt<mode>"
1047 [(set (match_operand:VDQHS 0 "register_operand" "=w")
1048 (minus:VDQHS
1049 (match_operand:VDQHS 4 "register_operand" "0")
1050 (mult:VDQHS
1051 (vec_duplicate:VDQHS
1052 (vec_select:<VEL>
1053 (match_operand:VDQHS 1 "register_operand" "<h_con>")
1054 (parallel [(match_operand:SI 2 "immediate_operand")])))
1055 (match_operand:VDQHS 3 "register_operand" "w"))))]
1056 "TARGET_SIMD"
1057 {
1058 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
1059 return "mls\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
1060 }
1061 [(set_attr "type" "neon_mla_<Vetype>_scalar<q>")]
1062 )
1064 (define_insn "*aarch64_mls_elt_<vswap_width_name><mode>"
1065 [(set (match_operand:VDQHS 0 "register_operand" "=w")
1066 (minus:VDQHS
1067 (match_operand:VDQHS 4 "register_operand" "0")
1068 (mult:VDQHS
1069 (vec_duplicate:VDQHS
1070 (vec_select:<VEL>
1071 (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
1072 (parallel [(match_operand:SI 2 "immediate_operand")])))
1073 (match_operand:VDQHS 3 "register_operand" "w"))))]
1074 "TARGET_SIMD"
1075 {
1076 operands[2] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
1077 INTVAL (operands[2])));
1078 return "mls\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
1079 }
1080 [(set_attr "type" "neon_mla_<Vetype>_scalar<q>")]
1081 )
1083 ;; Max/Min operations.
1084 (define_insn "<su><maxmin><mode>3"
1085 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
1086 (MAXMIN:VDQ_BHSI (match_operand:VDQ_BHSI 1 "register_operand" "w")
1087 (match_operand:VDQ_BHSI 2 "register_operand" "w")))]
1088 "TARGET_SIMD"
1089 "<su><maxmin>\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1090 [(set_attr "type" "neon_minmax<q>")]
1091 )
1093 (define_expand "<su><maxmin>v2di3"
1094 [(set (match_operand:V2DI 0 "register_operand" "")
1095 (MAXMIN:V2DI (match_operand:V2DI 1 "register_operand" "")
1096 (match_operand:V2DI 2 "register_operand" "")))]
1097 "TARGET_SIMD"
1098 {
1099 enum rtx_code cmp_operator;
1100 rtx cmp_fmt;
1102 switch (<CODE>)
1103 {
1104 case UMIN:
1105 cmp_operator = LTU;
1106 break;
1107 case SMIN:
1108 cmp_operator = LT;
1109 break;
1110 case UMAX:
1111 cmp_operator = GTU;
1112 break;
1113 case SMAX:
1114 cmp_operator = GT;
1115 break;
1116 default:
1117 gcc_unreachable ();
1118 }
1120 cmp_fmt = gen_rtx_fmt_ee (cmp_operator, V2DImode, operands[1], operands[2]);
1121 emit_insn (gen_vcondv2div2di (operands[0], operands[1],
1122 operands[2], cmp_fmt, operands[1], operands[2]));
1123 DONE;
1124 })
1126 ;; Pairwise Integer Max/Min operations.
1127 (define_insn "aarch64_<maxmin_uns>p<mode>"
1128 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
1129 (unspec:VDQ_BHSI [(match_operand:VDQ_BHSI 1 "register_operand" "w")
1130 (match_operand:VDQ_BHSI 2 "register_operand" "w")]
1131 MAXMINV))]
1132 "TARGET_SIMD"
1133 "<maxmin_uns_op>p\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1134 [(set_attr "type" "neon_minmax<q>")]
1135 )
1137 ;; Pairwise FP Max/Min operations.
1138 (define_insn "aarch64_<maxmin_uns>p<mode>"
1139 [(set (match_operand:VHSDF 0 "register_operand" "=w")
1140 (unspec:VHSDF [(match_operand:VHSDF 1 "register_operand" "w")
1141 (match_operand:VHSDF 2 "register_operand" "w")]
1142 FMAXMINV))]
1143 "TARGET_SIMD"
1144 "<maxmin_uns_op>p\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1145 [(set_attr "type" "neon_minmax<q>")]
1146 )
1148 ;; vec_concat gives a new vector with the low elements from operand 1, and
1149 ;; the high elements from operand 2. That is to say, given op1 = { a, b }
1150 ;; op2 = { c, d }, vec_concat (op1, op2) = { a, b, c, d }.
1151 ;; What that means, is that the RTL descriptions of the below patterns
1152 ;; need to change depending on endianness.
1154 ;; Move to the low architectural bits of the register.
1155 ;; On little-endian this is { operand, zeroes }
1156 ;; On big-endian this is { zeroes, operand }
1158 (define_insn "move_lo_quad_internal_<mode>"
1159 [(set (match_operand:VQ_NO2E 0 "register_operand" "=w,w,w")
1160 (vec_concat:VQ_NO2E
1161 (match_operand:<VHALF> 1 "register_operand" "w,r,r")
1162 (vec_duplicate:<VHALF> (const_int 0))))]
1163 "TARGET_SIMD && !BYTES_BIG_ENDIAN"
1164 "@
1165 dup\\t%d0, %1.d[0]
1166 fmov\\t%d0, %1
1167 dup\\t%d0, %1"
1168 [(set_attr "type" "neon_dup<q>,f_mcr,neon_dup<q>")
1169 (set_attr "simd" "yes,*,yes")
1170 (set_attr "fp" "*,yes,*")
1171 (set_attr "length" "4")]
1172 )
1174 (define_insn "move_lo_quad_internal_<mode>"
1175 [(set (match_operand:VQ_2E 0 "register_operand" "=w,w,w")
1176 (vec_concat:VQ_2E
1177 (match_operand:<VHALF> 1 "register_operand" "w,r,r")
1178 (const_int 0)))]
1179 "TARGET_SIMD && !BYTES_BIG_ENDIAN"
1180 "@
1181 dup\\t%d0, %1.d[0]
1182 fmov\\t%d0, %1
1183 dup\\t%d0, %1"
1184 [(set_attr "type" "neon_dup<q>,f_mcr,neon_dup<q>")
1185 (set_attr "simd" "yes,*,yes")
1186 (set_attr "fp" "*,yes,*")
1187 (set_attr "length" "4")]
1188 )
1190 (define_insn "move_lo_quad_internal_be_<mode>"
1191 [(set (match_operand:VQ_NO2E 0 "register_operand" "=w,w,w")
1192 (vec_concat:VQ_NO2E
1193 (vec_duplicate:<VHALF> (const_int 0))
1194 (match_operand:<VHALF> 1 "register_operand" "w,r,r")))]
1195 "TARGET_SIMD && BYTES_BIG_ENDIAN"
1196 "@
1197 dup\\t%d0, %1.d[0]
1198 fmov\\t%d0, %1
1199 dup\\t%d0, %1"
1200 [(set_attr "type" "neon_dup<q>,f_mcr,neon_dup<q>")
1201 (set_attr "simd" "yes,*,yes")
1202 (set_attr "fp" "*,yes,*")
1203 (set_attr "length" "4")]
1204 )
1206 (define_insn "move_lo_quad_internal_be_<mode>"
1207 [(set (match_operand:VQ_2E 0 "register_operand" "=w,w,w")
1208 (vec_concat:VQ_2E
1209 (const_int 0)
1210 (match_operand:<VHALF> 1 "register_operand" "w,r,r")))]
1211 "TARGET_SIMD && BYTES_BIG_ENDIAN"
1212 "@
1213 dup\\t%d0, %1.d[0]
1214 fmov\\t%d0, %1
1215 dup\\t%d0, %1"
1216 [(set_attr "type" "neon_dup<q>,f_mcr,neon_dup<q>")
1217 (set_attr "simd" "yes,*,yes")
1218 (set_attr "fp" "*,yes,*")
1219 (set_attr "length" "4")]
1220 )
1222 (define_expand "move_lo_quad_<mode>"
1223 [(match_operand:VQ 0 "register_operand")
1224 (match_operand:VQ 1 "register_operand")]
1225 "TARGET_SIMD"
1226 {
1227 if (BYTES_BIG_ENDIAN)
1228 emit_insn (gen_move_lo_quad_internal_be_<mode> (operands[0], operands[1]));
1229 else
1230 emit_insn (gen_move_lo_quad_internal_<mode> (operands[0], operands[1]));
1231 DONE;
1232 }
1233 )
1235 ;; Move operand1 to the high architectural bits of the register, keeping
1236 ;; the low architectural bits of operand2.
1237 ;; For little-endian this is { operand2, operand1 }
1238 ;; For big-endian this is { operand1, operand2 }
1240 (define_insn "aarch64_simd_move_hi_quad_<mode>"
1241 [(set (match_operand:VQ 0 "register_operand" "+w,w")
1242 (vec_concat:VQ
1243 (vec_select:<VHALF>
1244 (match_dup 0)
1245 (match_operand:VQ 2 "vect_par_cnst_lo_half" ""))
1246 (match_operand:<VHALF> 1 "register_operand" "w,r")))]
1247 "TARGET_SIMD && !BYTES_BIG_ENDIAN"
1248 "@
1249 ins\\t%0.d[1], %1.d[0]
1250 ins\\t%0.d[1], %1"
1251 [(set_attr "type" "neon_ins")]
1252 )
1254 (define_insn "aarch64_simd_move_hi_quad_be_<mode>"
1255 [(set (match_operand:VQ 0 "register_operand" "+w,w")
1256 (vec_concat:VQ
1257 (match_operand:<VHALF> 1 "register_operand" "w,r")
1258 (vec_select:<VHALF>
1259 (match_dup 0)
1260 (match_operand:VQ 2 "vect_par_cnst_lo_half" ""))))]
1261 "TARGET_SIMD && BYTES_BIG_ENDIAN"
1262 "@
1263 ins\\t%0.d[1], %1.d[0]
1264 ins\\t%0.d[1], %1"
1265 [(set_attr "type" "neon_ins")]
1266 )
1268 (define_expand "move_hi_quad_<mode>"
1269 [(match_operand:VQ 0 "register_operand" "")
1270 (match_operand:<VHALF> 1 "register_operand" "")]
1271 "TARGET_SIMD"
1272 {
1273 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
1274 if (BYTES_BIG_ENDIAN)
1275 emit_insn (gen_aarch64_simd_move_hi_quad_be_<mode> (operands[0],
1276 operands[1], p));
1277 else
1278 emit_insn (gen_aarch64_simd_move_hi_quad_<mode> (operands[0],
1279 operands[1], p));
1280 DONE;
1281 })
1283 ;; Narrowing operations.
1285 ;; For doubles.
1286 (define_insn "aarch64_simd_vec_pack_trunc_<mode>"
1287 [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
1288 (truncate:<VNARROWQ> (match_operand:VQN 1 "register_operand" "w")))]
1289 "TARGET_SIMD"
1290 "xtn\\t%0.<Vntype>, %1.<Vtype>"
1291 [(set_attr "type" "neon_shift_imm_narrow_q")]
1292 )
1294 (define_expand "vec_pack_trunc_<mode>"
1295 [(match_operand:<VNARROWD> 0 "register_operand" "")
1296 (match_operand:VDN 1 "register_operand" "")
1297 (match_operand:VDN 2 "register_operand" "")]
1298 "TARGET_SIMD"
1299 {
1300 rtx tempreg = gen_reg_rtx (<VDBL>mode);
1301 int lo = BYTES_BIG_ENDIAN ? 2 : 1;
1302 int hi = BYTES_BIG_ENDIAN ? 1 : 2;
1304 emit_insn (gen_move_lo_quad_<Vdbl> (tempreg, operands[lo]));
1305 emit_insn (gen_move_hi_quad_<Vdbl> (tempreg, operands[hi]));
1306 emit_insn (gen_aarch64_simd_vec_pack_trunc_<Vdbl> (operands[0], tempreg));
1307 DONE;
1308 })
1310 ;; For quads.
1312 (define_insn "vec_pack_trunc_<mode>"
1313 [(set (match_operand:<VNARROWQ2> 0 "register_operand" "=&w")
1314 (vec_concat:<VNARROWQ2>
1315 (truncate:<VNARROWQ> (match_operand:VQN 1 "register_operand" "w"))
1316 (truncate:<VNARROWQ> (match_operand:VQN 2 "register_operand" "w"))))]
1317 "TARGET_SIMD"
1318 {
1319 if (BYTES_BIG_ENDIAN)
1320 return "xtn\\t%0.<Vntype>, %2.<Vtype>\;xtn2\\t%0.<V2ntype>, %1.<Vtype>";
1321 else
1322 return "xtn\\t%0.<Vntype>, %1.<Vtype>\;xtn2\\t%0.<V2ntype>, %2.<Vtype>";
1323 }
1324 [(set_attr "type" "multiple")
1325 (set_attr "length" "8")]
1326 )
1328 ;; Widening operations.
1330 (define_insn "aarch64_simd_vec_unpack<su>_lo_<mode>"
1331 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1332 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1333 (match_operand:VQW 1 "register_operand" "w")
1334 (match_operand:VQW 2 "vect_par_cnst_lo_half" "")
1335 )))]
1336 "TARGET_SIMD"
1337 "<su>shll\t%0.<Vwtype>, %1.<Vhalftype>, 0"
1338 [(set_attr "type" "neon_shift_imm_long")]
1339 )
1341 (define_insn "aarch64_simd_vec_unpack<su>_hi_<mode>"
1342 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1343 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1344 (match_operand:VQW 1 "register_operand" "w")
1345 (match_operand:VQW 2 "vect_par_cnst_hi_half" "")
1346 )))]
1347 "TARGET_SIMD"
1348 "<su>shll2\t%0.<Vwtype>, %1.<Vtype>, 0"
1349 [(set_attr "type" "neon_shift_imm_long")]
1350 )
1352 (define_expand "vec_unpack<su>_hi_<mode>"
1353 [(match_operand:<VWIDE> 0 "register_operand" "")
1354 (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand"))]
1355 "TARGET_SIMD"
1356 {
1357 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
1358 emit_insn (gen_aarch64_simd_vec_unpack<su>_hi_<mode> (operands[0],
1359 operands[1], p));
1360 DONE;
1361 }
1362 )
1364 (define_expand "vec_unpack<su>_lo_<mode>"
1365 [(match_operand:<VWIDE> 0 "register_operand" "")
1366 (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand" ""))]
1367 "TARGET_SIMD"
1368 {
1369 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
1370 emit_insn (gen_aarch64_simd_vec_unpack<su>_lo_<mode> (operands[0],
1371 operands[1], p));
1372 DONE;
1373 }
1374 )
1376 ;; Widening arithmetic.
1378 (define_insn "*aarch64_<su>mlal_lo<mode>"
1379 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1380 (plus:<VWIDE>
1381 (mult:<VWIDE>
1382 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1383 (match_operand:VQW 2 "register_operand" "w")
1384 (match_operand:VQW 3 "vect_par_cnst_lo_half" "")))
1385 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1386 (match_operand:VQW 4 "register_operand" "w")
1387 (match_dup 3))))
1388 (match_operand:<VWIDE> 1 "register_operand" "0")))]
1389 "TARGET_SIMD"
1390 "<su>mlal\t%0.<Vwtype>, %2.<Vhalftype>, %4.<Vhalftype>"
1391 [(set_attr "type" "neon_mla_<Vetype>_long")]
1392 )
1394 (define_insn "*aarch64_<su>mlal_hi<mode>"
1395 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1396 (plus:<VWIDE>
1397 (mult:<VWIDE>
1398 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1399 (match_operand:VQW 2 "register_operand" "w")
1400 (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))
1401 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1402 (match_operand:VQW 4 "register_operand" "w")
1403 (match_dup 3))))
1404 (match_operand:<VWIDE> 1 "register_operand" "0")))]
1405 "TARGET_SIMD"
1406 "<su>mlal2\t%0.<Vwtype>, %2.<Vtype>, %4.<Vtype>"
1407 [(set_attr "type" "neon_mla_<Vetype>_long")]
1408 )
1410 (define_insn "*aarch64_<su>mlsl_lo<mode>"
1411 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1412 (minus:<VWIDE>
1413 (match_operand:<VWIDE> 1 "register_operand" "0")
1414 (mult:<VWIDE>
1415 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1416 (match_operand:VQW 2 "register_operand" "w")
1417 (match_operand:VQW 3 "vect_par_cnst_lo_half" "")))
1418 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1419 (match_operand:VQW 4 "register_operand" "w")
1420 (match_dup 3))))))]
1421 "TARGET_SIMD"
1422 "<su>mlsl\t%0.<Vwtype>, %2.<Vhalftype>, %4.<Vhalftype>"
1423 [(set_attr "type" "neon_mla_<Vetype>_long")]
1424 )
1426 (define_insn "*aarch64_<su>mlsl_hi<mode>"
1427 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1428 (minus:<VWIDE>
1429 (match_operand:<VWIDE> 1 "register_operand" "0")
1430 (mult:<VWIDE>
1431 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1432 (match_operand:VQW 2 "register_operand" "w")
1433 (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))
1434 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1435 (match_operand:VQW 4 "register_operand" "w")
1436 (match_dup 3))))))]
1437 "TARGET_SIMD"
1438 "<su>mlsl2\t%0.<Vwtype>, %2.<Vtype>, %4.<Vtype>"
1439 [(set_attr "type" "neon_mla_<Vetype>_long")]
1440 )
1442 (define_insn "*aarch64_<su>mlal<mode>"
1443 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1444 (plus:<VWIDE>
1445 (mult:<VWIDE>
1446 (ANY_EXTEND:<VWIDE>
1447 (match_operand:VD_BHSI 1 "register_operand" "w"))
1448 (ANY_EXTEND:<VWIDE>
1449 (match_operand:VD_BHSI 2 "register_operand" "w")))
1450 (match_operand:<VWIDE> 3 "register_operand" "0")))]
1451 "TARGET_SIMD"
1452 "<su>mlal\t%0.<Vwtype>, %1.<Vtype>, %2.<Vtype>"
1453 [(set_attr "type" "neon_mla_<Vetype>_long")]
1454 )
1456 (define_insn "*aarch64_<su>mlsl<mode>"
1457 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1458 (minus:<VWIDE>
1459 (match_operand:<VWIDE> 1 "register_operand" "0")
1460 (mult:<VWIDE>
1461 (ANY_EXTEND:<VWIDE>
1462 (match_operand:VD_BHSI 2 "register_operand" "w"))
1463 (ANY_EXTEND:<VWIDE>
1464 (match_operand:VD_BHSI 3 "register_operand" "w")))))]
1465 "TARGET_SIMD"
1466 "<su>mlsl\t%0.<Vwtype>, %2.<Vtype>, %3.<Vtype>"
1467 [(set_attr "type" "neon_mla_<Vetype>_long")]
1468 )
1470 (define_insn "aarch64_simd_vec_<su>mult_lo_<mode>"
1471 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1472 (mult:<VWIDE> (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1473 (match_operand:VQW 1 "register_operand" "w")
1474 (match_operand:VQW 3 "vect_par_cnst_lo_half" "")))
1475 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1476 (match_operand:VQW 2 "register_operand" "w")
1477 (match_dup 3)))))]
1478 "TARGET_SIMD"
1479 "<su>mull\\t%0.<Vwtype>, %1.<Vhalftype>, %2.<Vhalftype>"
1480 [(set_attr "type" "neon_mul_<Vetype>_long")]
1481 )
1483 (define_expand "vec_widen_<su>mult_lo_<mode>"
1484 [(match_operand:<VWIDE> 0 "register_operand" "")
1485 (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand" ""))
1486 (ANY_EXTEND:<VWIDE> (match_operand:VQW 2 "register_operand" ""))]
1487 "TARGET_SIMD"
1488 {
1489 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
1490 emit_insn (gen_aarch64_simd_vec_<su>mult_lo_<mode> (operands[0],
1491 operands[1],
1492 operands[2], p));
1493 DONE;
1494 }
1495 )
1497 (define_insn "aarch64_simd_vec_<su>mult_hi_<mode>"
1498 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1499 (mult:<VWIDE> (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1500 (match_operand:VQW 1 "register_operand" "w")
1501 (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))
1502 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1503 (match_operand:VQW 2 "register_operand" "w")
1504 (match_dup 3)))))]
1505 "TARGET_SIMD"
1506 "<su>mull2\\t%0.<Vwtype>, %1.<Vtype>, %2.<Vtype>"
1507 [(set_attr "type" "neon_mul_<Vetype>_long")]
1508 )
1510 (define_expand "vec_widen_<su>mult_hi_<mode>"
1511 [(match_operand:<VWIDE> 0 "register_operand" "")
1512 (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand" ""))
1513 (ANY_EXTEND:<VWIDE> (match_operand:VQW 2 "register_operand" ""))]
1514 "TARGET_SIMD"
1515 {
1516 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
1517 emit_insn (gen_aarch64_simd_vec_<su>mult_hi_<mode> (operands[0],
1518 operands[1],
1519 operands[2], p));
1520 DONE;
1522 }
1523 )
1525 ;; FP vector operations.
1526 ;; AArch64 AdvSIMD supports single-precision (32-bit) and
1527 ;; double-precision (64-bit) floating-point data types and arithmetic as
1528 ;; defined by the IEEE 754-2008 standard. This makes them vectorizable
1529 ;; without the need for -ffast-math or -funsafe-math-optimizations.
1530 ;;
1531 ;; Floating-point operations can raise an exception. Vectorizing such
1532 ;; operations are safe because of reasons explained below.
1533 ;;
1534 ;; ARMv8 permits an extension to enable trapped floating-point
1535 ;; exception handling, however this is an optional feature. In the
1536 ;; event of a floating-point exception being raised by vectorised
1537 ;; code then:
1538 ;; 1. If trapped floating-point exceptions are available, then a trap
1539 ;; will be taken when any lane raises an enabled exception. A trap
1540 ;; handler may determine which lane raised the exception.
1541 ;; 2. Alternatively a sticky exception flag is set in the
1542 ;; floating-point status register (FPSR). Software may explicitly
1543 ;; test the exception flags, in which case the tests will either
1544 ;; prevent vectorisation, allowing precise identification of the
1545 ;; failing operation, or if tested outside of vectorisable regions
1546 ;; then the specific operation and lane are not of interest.
1548 ;; FP arithmetic operations.
1550 (define_insn "add<mode>3"
1551 [(set (match_operand:VHSDF 0 "register_operand" "=w")
1552 (plus:VHSDF (match_operand:VHSDF 1 "register_operand" "w")
1553 (match_operand:VHSDF 2 "register_operand" "w")))]
1554 "TARGET_SIMD"
1555 "fadd\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1556 [(set_attr "type" "neon_fp_addsub_<stype><q>")]
1557 )
1559 (define_insn "sub<mode>3"
1560 [(set (match_operand:VHSDF 0 "register_operand" "=w")
1561 (minus:VHSDF (match_operand:VHSDF 1 "register_operand" "w")
1562 (match_operand:VHSDF 2 "register_operand" "w")))]
1563 "TARGET_SIMD"
1564 "fsub\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1565 [(set_attr "type" "neon_fp_addsub_<stype><q>")]
1566 )
1568 (define_insn "mul<mode>3"
1569 [(set (match_operand:VHSDF 0 "register_operand" "=w")
1570 (mult:VHSDF (match_operand:VHSDF 1 "register_operand" "w")
1571 (match_operand:VHSDF 2 "register_operand" "w")))]
1572 "TARGET_SIMD"
1573 "fmul\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1574 [(set_attr "type" "neon_fp_mul_<stype><q>")]
1575 )
1577 (define_expand "div<mode>3"
1578 [(set (match_operand:VHSDF 0 "register_operand" "=w")
1579 (div:VHSDF (match_operand:VHSDF 1 "register_operand" "w")
1580 (match_operand:VHSDF 2 "register_operand" "w")))]
1581 "TARGET_SIMD"
1582 {
1583 if (aarch64_emit_approx_div (operands[0], operands[1], operands[2]))
1584 DONE;
1586 operands[1] = force_reg (<MODE>mode, operands[1]);
1587 })
1589 (define_insn "*div<mode>3"
1590 [(set (match_operand:VHSDF 0 "register_operand" "=w")
1591 (div:VHSDF (match_operand:VHSDF 1 "register_operand" "w")
1592 (match_operand:VHSDF 2 "register_operand" "w")))]
1593 "TARGET_SIMD"
1594 "fdiv\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1595 [(set_attr "type" "neon_fp_div_<stype><q>")]
1596 )
1598 (define_insn "neg<mode>2"
1599 [(set (match_operand:VHSDF 0 "register_operand" "=w")
1600 (neg:VHSDF (match_operand:VHSDF 1 "register_operand" "w")))]
1601 "TARGET_SIMD"
1602 "fneg\\t%0.<Vtype>, %1.<Vtype>"
1603 [(set_attr "type" "neon_fp_neg_<stype><q>")]
1604 )
1606 (define_insn "abs<mode>2"
1607 [(set (match_operand:VHSDF 0 "register_operand" "=w")
1608 (abs:VHSDF (match_operand:VHSDF 1 "register_operand" "w")))]
1609 "TARGET_SIMD"
1610 "fabs\\t%0.<Vtype>, %1.<Vtype>"
1611 [(set_attr "type" "neon_fp_abs_<stype><q>")]
1612 )
1614 (define_insn "fma<mode>4"
1615 [(set (match_operand:VHSDF 0 "register_operand" "=w")
1616 (fma:VHSDF (match_operand:VHSDF 1 "register_operand" "w")
1617 (match_operand:VHSDF 2 "register_operand" "w")
1618 (match_operand:VHSDF 3 "register_operand" "0")))]
1619 "TARGET_SIMD"
1620 "fmla\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1621 [(set_attr "type" "neon_fp_mla_<stype><q>")]
1622 )
1624 (define_insn "*aarch64_fma4_elt<mode>"
1625 [(set (match_operand:VDQF 0 "register_operand" "=w")
1626 (fma:VDQF
1627 (vec_duplicate:VDQF
1628 (vec_select:<VEL>
1629 (match_operand:VDQF 1 "register_operand" "<h_con>")
1630 (parallel [(match_operand:SI 2 "immediate_operand")])))
1631 (match_operand:VDQF 3 "register_operand" "w")
1632 (match_operand:VDQF 4 "register_operand" "0")))]
1633 "TARGET_SIMD"
1634 {
1635 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
1636 return "fmla\\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
1637 }
1638 [(set_attr "type" "neon_fp_mla_<Vetype>_scalar<q>")]
1639 )
1641 (define_insn "*aarch64_fma4_elt_<vswap_width_name><mode>"
1642 [(set (match_operand:VDQSF 0 "register_operand" "=w")
1643 (fma:VDQSF
1644 (vec_duplicate:VDQSF
1645 (vec_select:<VEL>
1646 (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
1647 (parallel [(match_operand:SI 2 "immediate_operand")])))
1648 (match_operand:VDQSF 3 "register_operand" "w")
1649 (match_operand:VDQSF 4 "register_operand" "0")))]
1650 "TARGET_SIMD"
1651 {
1652 operands[2] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
1653 INTVAL (operands[2])));
1654 return "fmla\\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
1655 }
1656 [(set_attr "type" "neon_fp_mla_<Vetype>_scalar<q>")]
1657 )
1659 (define_insn "*aarch64_fma4_elt_from_dup<mode>"
1660 [(set (match_operand:VMUL 0 "register_operand" "=w")
1661 (fma:VMUL
1662 (vec_duplicate:VMUL
1663 (match_operand:<VEL> 1 "register_operand" "<h_con>"))
1664 (match_operand:VMUL 2 "register_operand" "w")
1665 (match_operand:VMUL 3 "register_operand" "0")))]
1666 "TARGET_SIMD"
1667 "fmla\t%0.<Vtype>, %2.<Vtype>, %1.<Vetype>[0]"
1668 [(set_attr "type" "neon<fp>_mla_<stype>_scalar<q>")]
1669 )
1671 (define_insn "*aarch64_fma4_elt_to_64v2df"
1672 [(set (match_operand:DF 0 "register_operand" "=w")
1673 (fma:DF
1674 (vec_select:DF
1675 (match_operand:V2DF 1 "register_operand" "w")
1676 (parallel [(match_operand:SI 2 "immediate_operand")]))
1677 (match_operand:DF 3 "register_operand" "w")
1678 (match_operand:DF 4 "register_operand" "0")))]
1679 "TARGET_SIMD"
1680 {
1681 operands[2] = GEN_INT (ENDIAN_LANE_N (V2DFmode, INTVAL (operands[2])));
1682 return "fmla\\t%0.2d, %3.2d, %1.2d[%2]";
1683 }
1684 [(set_attr "type" "neon_fp_mla_d_scalar_q")]
1685 )
1687 (define_insn "fnma<mode>4"
1688 [(set (match_operand:VHSDF 0 "register_operand" "=w")
1689 (fma:VHSDF
1690 (match_operand:VHSDF 1 "register_operand" "w")
1691 (neg:VHSDF
1692 (match_operand:VHSDF 2 "register_operand" "w"))
1693 (match_operand:VHSDF 3 "register_operand" "0")))]
1694 "TARGET_SIMD"
1695 "fmls\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1696 [(set_attr "type" "neon_fp_mla_<stype><q>")]
1697 )
1699 (define_insn "*aarch64_fnma4_elt<mode>"
1700 [(set (match_operand:VDQF 0 "register_operand" "=w")
1701 (fma:VDQF
1702 (neg:VDQF
1703 (match_operand:VDQF 3 "register_operand" "w"))
1704 (vec_duplicate:VDQF
1705 (vec_select:<VEL>
1706 (match_operand:VDQF 1 "register_operand" "<h_con>")
1707 (parallel [(match_operand:SI 2 "immediate_operand")])))
1708 (match_operand:VDQF 4 "register_operand" "0")))]
1709 "TARGET_SIMD"
1710 {
1711 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
1712 return "fmls\\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
1713 }
1714 [(set_attr "type" "neon_fp_mla_<Vetype>_scalar<q>")]
1715 )
1717 (define_insn "*aarch64_fnma4_elt_<vswap_width_name><mode>"
1718 [(set (match_operand:VDQSF 0 "register_operand" "=w")
1719 (fma:VDQSF
1720 (neg:VDQSF
1721 (match_operand:VDQSF 3 "register_operand" "w"))
1722 (vec_duplicate:VDQSF
1723 (vec_select:<VEL>
1724 (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
1725 (parallel [(match_operand:SI 2 "immediate_operand")])))
1726 (match_operand:VDQSF 4 "register_operand" "0")))]
1727 "TARGET_SIMD"
1728 {
1729 operands[2] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
1730 INTVAL (operands[2])));
1731 return "fmls\\t%0.<Vtype>, %3.<Vtype>, %1.<Vtype>[%2]";
1732 }
1733 [(set_attr "type" "neon_fp_mla_<Vetype>_scalar<q>")]
1734 )
1736 (define_insn "*aarch64_fnma4_elt_from_dup<mode>"
1737 [(set (match_operand:VMUL 0 "register_operand" "=w")
1738 (fma:VMUL
1739 (neg:VMUL
1740 (match_operand:VMUL 2 "register_operand" "w"))
1741 (vec_duplicate:VMUL
1742 (match_operand:<VEL> 1 "register_operand" "<h_con>"))
1743 (match_operand:VMUL 3 "register_operand" "0")))]
1744 "TARGET_SIMD"
1745 "fmls\t%0.<Vtype>, %2.<Vtype>, %1.<Vetype>[0]"
1746 [(set_attr "type" "neon<fp>_mla_<stype>_scalar<q>")]
1747 )
1749 (define_insn "*aarch64_fnma4_elt_to_64v2df"
1750 [(set (match_operand:DF 0 "register_operand" "=w")
1751 (fma:DF
1752 (vec_select:DF
1753 (match_operand:V2DF 1 "register_operand" "w")
1754 (parallel [(match_operand:SI 2 "immediate_operand")]))
1755 (neg:DF
1756 (match_operand:DF 3 "register_operand" "w"))
1757 (match_operand:DF 4 "register_operand" "0")))]
1758 "TARGET_SIMD"
1759 {
1760 operands[2] = GEN_INT (ENDIAN_LANE_N (V2DFmode, INTVAL (operands[2])));
1761 return "fmls\\t%0.2d, %3.2d, %1.2d[%2]";
1762 }
1763 [(set_attr "type" "neon_fp_mla_d_scalar_q")]
1764 )
1766 ;; Vector versions of the floating-point frint patterns.
1767 ;; Expands to btrunc, ceil, floor, nearbyint, rint, round, frintn.
1768 (define_insn "<frint_pattern><mode>2"
1769 [(set (match_operand:VHSDF 0 "register_operand" "=w")
1770 (unspec:VHSDF [(match_operand:VHSDF 1 "register_operand" "w")]
1771 FRINT))]
1772 "TARGET_SIMD"
1773 "frint<frint_suffix>\\t%0.<Vtype>, %1.<Vtype>"
1774 [(set_attr "type" "neon_fp_round_<stype><q>")]
1775 )
1777 ;; Vector versions of the fcvt standard patterns.
1778 ;; Expands to lbtrunc, lround, lceil, lfloor
1779 (define_insn "l<fcvt_pattern><su_optab><VHSDF:mode><fcvt_target>2"
1780 [(set (match_operand:<FCVT_TARGET> 0 "register_operand" "=w")
1781 (FIXUORS:<FCVT_TARGET> (unspec:<FCVT_TARGET>
1782 [(match_operand:VHSDF 1 "register_operand" "w")]
1783 FCVT)))]
1784 "TARGET_SIMD"
1785 "fcvt<frint_suffix><su>\\t%0.<Vtype>, %1.<Vtype>"
1786 [(set_attr "type" "neon_fp_to_int_<stype><q>")]
1787 )
1789 ;; HF Scalar variants of related SIMD instructions.
1790 (define_insn "l<fcvt_pattern><su_optab>hfhi2"
1791 [(set (match_operand:HI 0 "register_operand" "=w")
1792 (FIXUORS:HI (unspec:HF [(match_operand:HF 1 "register_operand" "w")]
1793 FCVT)))]
1794 "TARGET_SIMD_F16INST"
1795 "fcvt<frint_suffix><su>\t%h0, %h1"
1796 [(set_attr "type" "neon_fp_to_int_s")]
1797 )
1799 (define_insn "<optab>_trunchfhi2"
1800 [(set (match_operand:HI 0 "register_operand" "=w")
1801 (FIXUORS:HI (match_operand:HF 1 "register_operand" "w")))]
1802 "TARGET_SIMD_F16INST"
1803 "fcvtz<su>\t%h0, %h1"
1804 [(set_attr "type" "neon_fp_to_int_s")]
1805 )
1807 (define_insn "<optab>hihf2"
1808 [(set (match_operand:HF 0 "register_operand" "=w")
1809 (FLOATUORS:HF (match_operand:HI 1 "register_operand" "w")))]
1810 "TARGET_SIMD_F16INST"
1811 "<su_optab>cvtf\t%h0, %h1"
1812 [(set_attr "type" "neon_int_to_fp_s")]
1813 )
1815 (define_insn "*aarch64_fcvt<su_optab><VDQF:mode><fcvt_target>2_mult"
1816 [(set (match_operand:<FCVT_TARGET> 0 "register_operand" "=w")
1817 (FIXUORS:<FCVT_TARGET> (unspec:<FCVT_TARGET>
1818 [(mult:VDQF
1819 (match_operand:VDQF 1 "register_operand" "w")
1820 (match_operand:VDQF 2 "aarch64_fp_vec_pow2" ""))]
1821 UNSPEC_FRINTZ)))]
1822 "TARGET_SIMD
1823 && IN_RANGE (aarch64_vec_fpconst_pow_of_2 (operands[2]), 1,
1824 GET_MODE_BITSIZE (GET_MODE_INNER (<VDQF:MODE>mode)))"
1825 {
1826 int fbits = aarch64_vec_fpconst_pow_of_2 (operands[2]);
1827 char buf[64];
1828 snprintf (buf, 64, "fcvtz<su>\\t%%0.<Vtype>, %%1.<Vtype>, #%d", fbits);
1829 output_asm_insn (buf, operands);
1830 return "";
1831 }
1832 [(set_attr "type" "neon_fp_to_int_<Vetype><q>")]
1833 )
1835 (define_expand "<optab><VHSDF:mode><fcvt_target>2"
1836 [(set (match_operand:<FCVT_TARGET> 0 "register_operand")
1837 (FIXUORS:<FCVT_TARGET> (unspec:<FCVT_TARGET>
1838 [(match_operand:VHSDF 1 "register_operand")]
1839 UNSPEC_FRINTZ)))]
1840 "TARGET_SIMD"
1841 {})
1843 (define_expand "<fix_trunc_optab><VHSDF:mode><fcvt_target>2"
1844 [(set (match_operand:<FCVT_TARGET> 0 "register_operand")
1845 (FIXUORS:<FCVT_TARGET> (unspec:<FCVT_TARGET>
1846 [(match_operand:VHSDF 1 "register_operand")]
1847 UNSPEC_FRINTZ)))]
1848 "TARGET_SIMD"
1849 {})
1851 (define_expand "ftrunc<VHSDF:mode>2"
1852 [(set (match_operand:VHSDF 0 "register_operand")
1853 (unspec:VHSDF [(match_operand:VHSDF 1 "register_operand")]
1854 UNSPEC_FRINTZ))]
1855 "TARGET_SIMD"
1856 {})
1858 (define_insn "<optab><fcvt_target><VHSDF:mode>2"
1859 [(set (match_operand:VHSDF 0 "register_operand" "=w")
1860 (FLOATUORS:VHSDF
1861 (match_operand:<FCVT_TARGET> 1 "register_operand" "w")))]
1862 "TARGET_SIMD"
1863 "<su_optab>cvtf\\t%0.<Vtype>, %1.<Vtype>"
1864 [(set_attr "type" "neon_int_to_fp_<stype><q>")]
1865 )
1867 ;; Conversions between vectors of floats and doubles.
1868 ;; Contains a mix of patterns to match standard pattern names
1869 ;; and those for intrinsics.
1871 ;; Float widening operations.
1873 (define_insn "aarch64_simd_vec_unpacks_lo_<mode>"
1874 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1875 (float_extend:<VWIDE> (vec_select:<VHALF>
1876 (match_operand:VQ_HSF 1 "register_operand" "w")
1877 (match_operand:VQ_HSF 2 "vect_par_cnst_lo_half" "")
1878 )))]
1879 "TARGET_SIMD"
1880 "fcvtl\\t%0.<Vwtype>, %1.<Vhalftype>"
1881 [(set_attr "type" "neon_fp_cvt_widen_s")]
1882 )
1884 ;; Convert between fixed-point and floating-point (vector modes)
1886 (define_insn "<FCVT_F2FIXED:fcvt_fixed_insn><VHSDF:mode>3"
1887 [(set (match_operand:<VHSDF:FCVT_TARGET> 0 "register_operand" "=w")
1888 (unspec:<VHSDF:FCVT_TARGET>
1889 [(match_operand:VHSDF 1 "register_operand" "w")
1890 (match_operand:SI 2 "immediate_operand" "i")]
1891 FCVT_F2FIXED))]
1892 "TARGET_SIMD"
1893 "<FCVT_F2FIXED:fcvt_fixed_insn>\t%<v>0<Vmtype>, %<v>1<Vmtype>, #%2"
1894 [(set_attr "type" "neon_fp_to_int_<VHSDF:stype><q>")]
1895 )
1897 (define_insn "<FCVT_FIXED2F:fcvt_fixed_insn><VDQ_HSDI:mode>3"
1898 [(set (match_operand:<VDQ_HSDI:FCVT_TARGET> 0 "register_operand" "=w")
1899 (unspec:<VDQ_HSDI:FCVT_TARGET>
1900 [(match_operand:VDQ_HSDI 1 "register_operand" "w")
1901 (match_operand:SI 2 "immediate_operand" "i")]
1902 FCVT_FIXED2F))]
1903 "TARGET_SIMD"
1904 "<FCVT_FIXED2F:fcvt_fixed_insn>\t%<v>0<Vmtype>, %<v>1<Vmtype>, #%2"
1905 [(set_attr "type" "neon_int_to_fp_<VDQ_HSDI:stype><q>")]
1906 )
1908 ;; ??? Note that the vectorizer usage of the vec_unpacks_[lo/hi] patterns
1909 ;; is inconsistent with vector ordering elsewhere in the compiler, in that
1910 ;; the meaning of HI and LO changes depending on the target endianness.
1911 ;; While elsewhere we map the higher numbered elements of a vector to
1912 ;; the lower architectural lanes of the vector, for these patterns we want
1913 ;; to always treat "hi" as referring to the higher architectural lanes.
1914 ;; Consequently, while the patterns below look inconsistent with our
1915 ;; other big-endian patterns their behavior is as required.
1917 (define_expand "vec_unpacks_lo_<mode>"
1918 [(match_operand:<VWIDE> 0 "register_operand" "")
1919 (match_operand:VQ_HSF 1 "register_operand" "")]
1920 "TARGET_SIMD"
1921 {
1922 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
1923 emit_insn (gen_aarch64_simd_vec_unpacks_lo_<mode> (operands[0],
1924 operands[1], p));
1925 DONE;
1926 }
1927 )
1929 (define_insn "aarch64_simd_vec_unpacks_hi_<mode>"
1930 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1931 (float_extend:<VWIDE> (vec_select:<VHALF>
1932 (match_operand:VQ_HSF 1 "register_operand" "w")
1933 (match_operand:VQ_HSF 2 "vect_par_cnst_hi_half" "")
1934 )))]
1935 "TARGET_SIMD"
1936 "fcvtl2\\t%0.<Vwtype>, %1.<Vtype>"
1937 [(set_attr "type" "neon_fp_cvt_widen_s")]
1938 )
1940 (define_expand "vec_unpacks_hi_<mode>"
1941 [(match_operand:<VWIDE> 0 "register_operand" "")
1942 (match_operand:VQ_HSF 1 "register_operand" "")]
1943 "TARGET_SIMD"
1944 {
1945 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
1946 emit_insn (gen_aarch64_simd_vec_unpacks_lo_<mode> (operands[0],
1947 operands[1], p));
1948 DONE;
1949 }
1950 )
1951 (define_insn "aarch64_float_extend_lo_<Vwide>"
1952 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1953 (float_extend:<VWIDE>
1954 (match_operand:VDF 1 "register_operand" "w")))]
1955 "TARGET_SIMD"
1956 "fcvtl\\t%0<Vmwtype>, %1<Vmtype>"
1957 [(set_attr "type" "neon_fp_cvt_widen_s")]
1958 )
1960 ;; Float narrowing operations.
1962 (define_insn "aarch64_float_truncate_lo_<mode>"
1963 [(set (match_operand:VDF 0 "register_operand" "=w")
1964 (float_truncate:VDF
1965 (match_operand:<VWIDE> 1 "register_operand" "w")))]
1966 "TARGET_SIMD"
1967 "fcvtn\\t%0.<Vtype>, %1<Vmwtype>"
1968 [(set_attr "type" "neon_fp_cvt_narrow_d_q")]
1969 )
1971 (define_insn "aarch64_float_truncate_hi_<Vdbl>_le"
1972 [(set (match_operand:<VDBL> 0 "register_operand" "=w")
1973 (vec_concat:<VDBL>
1974 (match_operand:VDF 1 "register_operand" "0")
1975 (float_truncate:VDF
1976 (match_operand:<VWIDE> 2 "register_operand" "w"))))]
1977 "TARGET_SIMD && !BYTES_BIG_ENDIAN"
1978 "fcvtn2\\t%0.<Vdtype>, %2<Vmwtype>"
1979 [(set_attr "type" "neon_fp_cvt_narrow_d_q")]
1980 )
1982 (define_insn "aarch64_float_truncate_hi_<Vdbl>_be"
1983 [(set (match_operand:<VDBL> 0 "register_operand" "=w")
1984 (vec_concat:<VDBL>
1985 (float_truncate:VDF
1986 (match_operand:<VWIDE> 2 "register_operand" "w"))
1987 (match_operand:VDF 1 "register_operand" "0")))]
1988 "TARGET_SIMD && BYTES_BIG_ENDIAN"
1989 "fcvtn2\\t%0.<Vdtype>, %2<Vmwtype>"
1990 [(set_attr "type" "neon_fp_cvt_narrow_d_q")]
1991 )
1993 (define_expand "aarch64_float_truncate_hi_<Vdbl>"
1994 [(match_operand:<VDBL> 0 "register_operand" "=w")
1995 (match_operand:VDF 1 "register_operand" "0")
1996 (match_operand:<VWIDE> 2 "register_operand" "w")]
1997 "TARGET_SIMD"
1998 {
1999 rtx (*gen) (rtx, rtx, rtx) = BYTES_BIG_ENDIAN
2000 ? gen_aarch64_float_truncate_hi_<Vdbl>_be
2001 : gen_aarch64_float_truncate_hi_<Vdbl>_le;
2002 emit_insn (gen (operands[0], operands[1], operands[2]));
2003 DONE;
2004 }
2005 )
2007 (define_expand "vec_pack_trunc_v2df"
2008 [(set (match_operand:V4SF 0 "register_operand")
2009 (vec_concat:V4SF
2010 (float_truncate:V2SF
2011 (match_operand:V2DF 1 "register_operand"))
2012 (float_truncate:V2SF
2013 (match_operand:V2DF 2 "register_operand"))
2014 ))]
2015 "TARGET_SIMD"
2016 {
2017 rtx tmp = gen_reg_rtx (V2SFmode);
2018 int lo = BYTES_BIG_ENDIAN ? 2 : 1;
2019 int hi = BYTES_BIG_ENDIAN ? 1 : 2;
2021 emit_insn (gen_aarch64_float_truncate_lo_v2sf (tmp, operands[lo]));
2022 emit_insn (gen_aarch64_float_truncate_hi_v4sf (operands[0],
2023 tmp, operands[hi]));
2024 DONE;
2025 }
2026 )
2028 (define_expand "vec_pack_trunc_df"
2029 [(set (match_operand:V2SF 0 "register_operand")
2030 (vec_concat:V2SF
2031 (float_truncate:SF
2032 (match_operand:DF 1 "register_operand"))
2033 (float_truncate:SF
2034 (match_operand:DF 2 "register_operand"))
2035 ))]
2036 "TARGET_SIMD"
2037 {
2038 rtx tmp = gen_reg_rtx (V2SFmode);
2039 int lo = BYTES_BIG_ENDIAN ? 2 : 1;
2040 int hi = BYTES_BIG_ENDIAN ? 1 : 2;
2042 emit_insn (gen_move_lo_quad_v2df (tmp, operands[lo]));
2043 emit_insn (gen_move_hi_quad_v2df (tmp, operands[hi]));
2044 emit_insn (gen_aarch64_float_truncate_lo_v2sf (operands[0], tmp));
2045 DONE;
2046 }
2047 )
2049 ;; FP Max/Min
2050 ;; Max/Min are introduced by idiom recognition by GCC's mid-end. An
2051 ;; expression like:
2052 ;; a = (b < c) ? b : c;
2053 ;; is idiom-matched as MIN_EXPR<b,c> only if -ffinite-math-only is enabled
2054 ;; either explicitly or indirectly via -ffast-math.
2055 ;;
2056 ;; MIN_EXPR and MAX_EXPR eventually map to 'smin' and 'smax' in RTL.
2057 ;; The 'smax' and 'smin' RTL standard pattern names do not specify which
2058 ;; operand will be returned when both operands are zero (i.e. they may not
2059 ;; honour signed zeroes), or when either operand is NaN. Therefore GCC
2060 ;; only introduces MIN_EXPR/MAX_EXPR in fast math mode or when not honouring
2061 ;; NaNs.
2063 (define_insn "<su><maxmin><mode>3"
2064 [(set (match_operand:VHSDF 0 "register_operand" "=w")
2065 (FMAXMIN:VHSDF (match_operand:VHSDF 1 "register_operand" "w")
2066 (match_operand:VHSDF 2 "register_operand" "w")))]
2067 "TARGET_SIMD"
2068 "f<maxmin>nm\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
2069 [(set_attr "type" "neon_fp_minmax_<stype><q>")]
2070 )
2072 ;; Vector forms for fmax, fmin, fmaxnm, fminnm.
2073 ;; fmaxnm and fminnm are used for the fmax<mode>3 standard pattern names,
2074 ;; which implement the IEEE fmax ()/fmin () functions.
2075 (define_insn "<maxmin_uns><mode>3"
2076 [(set (match_operand:VHSDF 0 "register_operand" "=w")
2077 (unspec:VHSDF [(match_operand:VHSDF 1 "register_operand" "w")
2078 (match_operand:VHSDF 2 "register_operand" "w")]
2079 FMAXMIN_UNS))]
2080 "TARGET_SIMD"
2081 "<maxmin_uns_op>\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
2082 [(set_attr "type" "neon_fp_minmax_<stype><q>")]
2083 )
2085 ;; 'across lanes' add.
2087 (define_expand "reduc_plus_scal_<mode>"
2088 [(match_operand:<VEL> 0 "register_operand" "=w")
2089 (unspec:VDQ_I [(match_operand:VDQ_I 1 "register_operand" "w")]
2090 UNSPEC_ADDV)]
2091 "TARGET_SIMD"
2092 {
2093 rtx elt = GEN_INT (ENDIAN_LANE_N (<MODE>mode, 0));
2094 rtx scratch = gen_reg_rtx (<MODE>mode);
2095 emit_insn (gen_aarch64_reduc_plus_internal<mode> (scratch, operands[1]));
2096 emit_insn (gen_aarch64_get_lane<mode> (operands[0], scratch, elt));
2097 DONE;
2098 }
2099 )
2101 (define_insn "aarch64_faddp<mode>"
2102 [(set (match_operand:VHSDF 0 "register_operand" "=w")
2103 (unspec:VHSDF [(match_operand:VHSDF 1 "register_operand" "w")
2104 (match_operand:VHSDF 2 "register_operand" "w")]
2105 UNSPEC_FADDV))]
2106 "TARGET_SIMD"
2107 "faddp\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
2108 [(set_attr "type" "neon_fp_reduc_add_<stype><q>")]
2109 )
2111 (define_insn "aarch64_reduc_plus_internal<mode>"
2112 [(set (match_operand:VDQV 0 "register_operand" "=w")
2113 (unspec:VDQV [(match_operand:VDQV 1 "register_operand" "w")]
2114 UNSPEC_ADDV))]
2115 "TARGET_SIMD"
2116 "add<VDQV:vp>\\t%<Vetype>0, %1.<Vtype>"
2117 [(set_attr "type" "neon_reduc_add<q>")]
2118 )
2120 (define_insn "aarch64_reduc_plus_internalv2si"
2121 [(set (match_operand:V2SI 0 "register_operand" "=w")
2122 (unspec:V2SI [(match_operand:V2SI 1 "register_operand" "w")]
2123 UNSPEC_ADDV))]
2124 "TARGET_SIMD"
2125 "addp\\t%0.2s, %1.2s, %1.2s"
2126 [(set_attr "type" "neon_reduc_add")]
2127 )
2129 (define_insn "reduc_plus_scal_<mode>"
2130 [(set (match_operand:<VEL> 0 "register_operand" "=w")
2131 (unspec:<VEL> [(match_operand:V2F 1 "register_operand" "w")]
2132 UNSPEC_FADDV))]
2133 "TARGET_SIMD"
2134 "faddp\\t%<Vetype>0, %1.<Vtype>"
2135 [(set_attr "type" "neon_fp_reduc_add_<Vetype><q>")]
2136 )
2138 (define_expand "reduc_plus_scal_v4sf"
2139 [(set (match_operand:SF 0 "register_operand")
2140 (unspec:V4SF [(match_operand:V4SF 1 "register_operand")]
2141 UNSPEC_FADDV))]
2142 "TARGET_SIMD"
2143 {
2144 rtx elt = GEN_INT (ENDIAN_LANE_N (V4SFmode, 0));
2145 rtx scratch = gen_reg_rtx (V4SFmode);
2146 emit_insn (gen_aarch64_faddpv4sf (scratch, operands[1], operands[1]));
2147 emit_insn (gen_aarch64_faddpv4sf (scratch, scratch, scratch));
2148 emit_insn (gen_aarch64_get_lanev4sf (operands[0], scratch, elt));
2149 DONE;
2150 })
2152 (define_insn "clrsb<mode>2"
2153 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
2154 (clrsb:VDQ_BHSI (match_operand:VDQ_BHSI 1 "register_operand" "w")))]
2155 "TARGET_SIMD"
2156 "cls\\t%0.<Vtype>, %1.<Vtype>"
2157 [(set_attr "type" "neon_cls<q>")]
2158 )
2160 (define_insn "clz<mode>2"
2161 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
2162 (clz:VDQ_BHSI (match_operand:VDQ_BHSI 1 "register_operand" "w")))]
2163 "TARGET_SIMD"
2164 "clz\\t%0.<Vtype>, %1.<Vtype>"
2165 [(set_attr "type" "neon_cls<q>")]
2166 )
2168 (define_insn "popcount<mode>2"
2169 [(set (match_operand:VB 0 "register_operand" "=w")
2170 (popcount:VB (match_operand:VB 1 "register_operand" "w")))]
2171 "TARGET_SIMD"
2172 "cnt\\t%0.<Vbtype>, %1.<Vbtype>"
2173 [(set_attr "type" "neon_cnt<q>")]
2174 )
2176 ;; 'across lanes' max and min ops.
2178 ;; Template for outputting a scalar, so we can create __builtins which can be
2179 ;; gimple_fold'd to the REDUC_(MAX|MIN)_EXPR tree code. (This is FP smax/smin).
2180 (define_expand "reduc_<maxmin_uns>_scal_<mode>"
2181 [(match_operand:<VEL> 0 "register_operand")
2182 (unspec:VHSDF [(match_operand:VHSDF 1 "register_operand")]
2183 FMAXMINV)]
2184 "TARGET_SIMD"
2185 {
2186 rtx elt = GEN_INT (ENDIAN_LANE_N (<MODE>mode, 0));
2187 rtx scratch = gen_reg_rtx (<MODE>mode);
2188 emit_insn (gen_aarch64_reduc_<maxmin_uns>_internal<mode> (scratch,
2189 operands[1]));
2190 emit_insn (gen_aarch64_get_lane<mode> (operands[0], scratch, elt));
2191 DONE;
2192 }
2193 )
2195 ;; Likewise for integer cases, signed and unsigned.
2196 (define_expand "reduc_<maxmin_uns>_scal_<mode>"
2197 [(match_operand:<VEL> 0 "register_operand")
2198 (unspec:VDQ_BHSI [(match_operand:VDQ_BHSI 1 "register_operand")]
2199 MAXMINV)]
2200 "TARGET_SIMD"
2201 {
2202 rtx elt = GEN_INT (ENDIAN_LANE_N (<MODE>mode, 0));
2203 rtx scratch = gen_reg_rtx (<MODE>mode);
2204 emit_insn (gen_aarch64_reduc_<maxmin_uns>_internal<mode> (scratch,
2205 operands[1]));
2206 emit_insn (gen_aarch64_get_lane<mode> (operands[0], scratch, elt));
2207 DONE;
2208 }
2209 )
2211 (define_insn "aarch64_reduc_<maxmin_uns>_internal<mode>"
2212 [(set (match_operand:VDQV_S 0 "register_operand" "=w")
2213 (unspec:VDQV_S [(match_operand:VDQV_S 1 "register_operand" "w")]
2214 MAXMINV))]
2215 "TARGET_SIMD"
2216 "<maxmin_uns_op>v\\t%<Vetype>0, %1.<Vtype>"
2217 [(set_attr "type" "neon_reduc_minmax<q>")]
2218 )
2220 (define_insn "aarch64_reduc_<maxmin_uns>_internalv2si"
2221 [(set (match_operand:V2SI 0 "register_operand" "=w")
2222 (unspec:V2SI [(match_operand:V2SI 1 "register_operand" "w")]
2223 MAXMINV))]
2224 "TARGET_SIMD"
2225 "<maxmin_uns_op>p\\t%0.2s, %1.2s, %1.2s"
2226 [(set_attr "type" "neon_reduc_minmax")]
2227 )
2229 (define_insn "aarch64_reduc_<maxmin_uns>_internal<mode>"
2230 [(set (match_operand:VHSDF 0 "register_operand" "=w")
2231 (unspec:VHSDF [(match_operand:VHSDF 1 "register_operand" "w")]
2232 FMAXMINV))]
2233 "TARGET_SIMD"
2234 "<maxmin_uns_op><vp>\\t%<Vetype>0, %1.<Vtype>"
2235 [(set_attr "type" "neon_fp_reduc_minmax_<stype><q>")]
2236 )
2238 ;; aarch64_simd_bsl may compile to any of bsl/bif/bit depending on register
2239 ;; allocation.
2240 ;; Operand 1 is the mask, operands 2 and 3 are the bitfields from which
2241 ;; to select.
2242 ;;
2243 ;; Thus our BSL is of the form:
2244 ;; op0 = bsl (mask, op2, op3)
2245 ;; We can use any of:
2246 ;;
2247 ;; if (op0 = mask)
2248 ;; bsl mask, op1, op2
2249 ;; if (op0 = op1) (so 1-bits in mask choose bits from op2, else op0)
2250 ;; bit op0, op2, mask
2251 ;; if (op0 = op2) (so 0-bits in mask choose bits from op1, else op0)
2252 ;; bif op0, op1, mask
2253 ;;
2254 ;; This pattern is expanded to by the aarch64_simd_bsl<mode> expander.
2255 ;; Some forms of straight-line code may generate the equivalent form
2256 ;; in *aarch64_simd_bsl<mode>_alt.
2258 (define_insn "aarch64_simd_bsl<mode>_internal"
2259 [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w,w,w")
2260 (xor:VSDQ_I_DI
2261 (and:VSDQ_I_DI
2262 (xor:VSDQ_I_DI
2263 (match_operand:<V_cmp_result> 3 "register_operand" "w,0,w")
2264 (match_operand:VSDQ_I_DI 2 "register_operand" "w,w,0"))
2265 (match_operand:VSDQ_I_DI 1 "register_operand" "0,w,w"))
2266 (match_dup:<V_cmp_result> 3)
2267 ))]
2268 "TARGET_SIMD"
2269 "@
2270 bsl\\t%0.<Vbtype>, %2.<Vbtype>, %3.<Vbtype>
2271 bit\\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>
2272 bif\\t%0.<Vbtype>, %3.<Vbtype>, %1.<Vbtype>"
2273 [(set_attr "type" "neon_bsl<q>")]
2274 )
2276 ;; We need this form in addition to the above pattern to match the case
2277 ;; when combine tries merging three insns such that the second operand of
2278 ;; the outer XOR matches the second operand of the inner XOR rather than
2279 ;; the first. The two are equivalent but since recog doesn't try all
2280 ;; permutations of commutative operations, we have to have a separate pattern.
2282 (define_insn "*aarch64_simd_bsl<mode>_alt"
2283 [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w,w,w")
2284 (xor:VSDQ_I_DI
2285 (and:VSDQ_I_DI
2286 (xor:VSDQ_I_DI
2287 (match_operand:VSDQ_I_DI 3 "register_operand" "w,w,0")
2288 (match_operand:VSDQ_I_DI 2 "register_operand" "w,0,w"))
2289 (match_operand:VSDQ_I_DI 1 "register_operand" "0,w,w"))
2290 (match_dup:VSDQ_I_DI 2)))]
2291 "TARGET_SIMD"
2292 "@
2293 bsl\\t%0.<Vbtype>, %3.<Vbtype>, %2.<Vbtype>
2294 bit\\t%0.<Vbtype>, %3.<Vbtype>, %1.<Vbtype>
2295 bif\\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>"
2296 [(set_attr "type" "neon_bsl<q>")]
2297 )
2299 (define_expand "aarch64_simd_bsl<mode>"
2300 [(match_operand:VALLDIF 0 "register_operand")
2301 (match_operand:<V_cmp_result> 1 "register_operand")
2302 (match_operand:VALLDIF 2 "register_operand")
2303 (match_operand:VALLDIF 3 "register_operand")]
2304 "TARGET_SIMD"
2305 {
2306 /* We can't alias operands together if they have different modes. */
2307 rtx tmp = operands[0];
2308 if (FLOAT_MODE_P (<MODE>mode))
2309 {
2310 operands[2] = gen_lowpart (<V_cmp_result>mode, operands[2]);
2311 operands[3] = gen_lowpart (<V_cmp_result>mode, operands[3]);
2312 tmp = gen_reg_rtx (<V_cmp_result>mode);
2313 }
2314 operands[1] = gen_lowpart (<V_cmp_result>mode, operands[1]);
2315 emit_insn (gen_aarch64_simd_bsl<v_cmp_result>_internal (tmp,
2316 operands[1],
2317 operands[2],
2318 operands[3]));
2319 if (tmp != operands[0])
2320 emit_move_insn (operands[0], gen_lowpart (<MODE>mode, tmp));
2322 DONE;
2323 })
2325 (define_expand "vcond_mask_<mode><v_cmp_result>"
2326 [(match_operand:VALLDI 0 "register_operand")
2327 (match_operand:VALLDI 1 "nonmemory_operand")
2328 (match_operand:VALLDI 2 "nonmemory_operand")
2329 (match_operand:<V_cmp_result> 3 "register_operand")]
2330 "TARGET_SIMD"
2331 {
2332 /* If we have (a = (P) ? -1 : 0);
2333 Then we can simply move the generated mask (result must be int). */
2334 if (operands[1] == CONSTM1_RTX (<MODE>mode)
2335 && operands[2] == CONST0_RTX (<MODE>mode))
2336 emit_move_insn (operands[0], operands[3]);
2337 /* Similarly, (a = (P) ? 0 : -1) is just inverting the generated mask. */
2338 else if (operands[1] == CONST0_RTX (<MODE>mode)
2339 && operands[2] == CONSTM1_RTX (<MODE>mode))
2340 emit_insn (gen_one_cmpl<v_cmp_result>2 (operands[0], operands[3]));
2341 else
2342 {
2343 if (!REG_P (operands[1]))
2344 operands[1] = force_reg (<MODE>mode, operands[1]);
2345 if (!REG_P (operands[2]))
2346 operands[2] = force_reg (<MODE>mode, operands[2]);
2347 emit_insn (gen_aarch64_simd_bsl<mode> (operands[0], operands[3],
2348 operands[1], operands[2]));
2349 }
2351 DONE;
2352 })
2354 ;; Patterns comparing two vectors to produce a mask.
2356 (define_expand "vec_cmp<mode><mode>"
2357 [(set (match_operand:VSDQ_I_DI 0 "register_operand")
2358 (match_operator 1 "comparison_operator"
2359 [(match_operand:VSDQ_I_DI 2 "register_operand")
2360 (match_operand:VSDQ_I_DI 3 "nonmemory_operand")]))]
2361 "TARGET_SIMD"
2362 {
2363 rtx mask = operands[0];
2364 enum rtx_code code = GET_CODE (operands[1]);
2366 switch (code)
2367 {
2368 case NE:
2369 case LE:
2370 case LT:
2371 case GE:
2372 case GT:
2373 case EQ:
2374 if (operands[3] == CONST0_RTX (<MODE>mode))
2375 break;
2377 /* Fall through. */
2378 default:
2379 if (!REG_P (operands[3]))
2380 operands[3] = force_reg (<MODE>mode, operands[3]);
2382 break;
2383 }
2385 switch (code)
2386 {
2387 case LT:
2388 emit_insn (gen_aarch64_cmlt<mode> (mask, operands[2], operands[3]));
2389 break;
2391 case GE:
2392 emit_insn (gen_aarch64_cmge<mode> (mask, operands[2], operands[3]));
2393 break;
2395 case LE:
2396 emit_insn (gen_aarch64_cmle<mode> (mask, operands[2], operands[3]));
2397 break;
2399 case GT:
2400 emit_insn (gen_aarch64_cmgt<mode> (mask, operands[2], operands[3]));
2401 break;
2403 case LTU:
2404 emit_insn (gen_aarch64_cmgtu<mode> (mask, operands[3], operands[2]));
2405 break;
2407 case GEU:
2408 emit_insn (gen_aarch64_cmgeu<mode> (mask, operands[2], operands[3]));
2409 break;
2411 case LEU:
2412 emit_insn (gen_aarch64_cmgeu<mode> (mask, operands[3], operands[2]));
2413 break;
2415 case GTU:
2416 emit_insn (gen_aarch64_cmgtu<mode> (mask, operands[2], operands[3]));
2417 break;
2419 case NE:
2420 /* Handle NE as !EQ. */
2421 emit_insn (gen_aarch64_cmeq<mode> (mask, operands[2], operands[3]));
2422 emit_insn (gen_one_cmpl<v_cmp_result>2 (mask, mask));
2423 break;
2425 case EQ:
2426 emit_insn (gen_aarch64_cmeq<mode> (mask, operands[2], operands[3]));
2427 break;
2429 default:
2430 gcc_unreachable ();
2431 }
2433 DONE;
2434 })
2436 (define_expand "vec_cmp<mode><v_cmp_result>"
2437 [(set (match_operand:<V_cmp_result> 0 "register_operand")
2438 (match_operator 1 "comparison_operator"
2439 [(match_operand:VDQF 2 "register_operand")
2440 (match_operand:VDQF 3 "nonmemory_operand")]))]
2441 "TARGET_SIMD"
2442 {
2443 int use_zero_form = 0;
2444 enum rtx_code code = GET_CODE (operands[1]);
2445 rtx tmp = gen_reg_rtx (<V_cmp_result>mode);
2447 rtx (*comparison) (rtx, rtx, rtx) = NULL;
2449 switch (code)
2450 {
2451 case LE:
2452 case LT:
2453 case GE:
2454 case GT:
2455 case EQ:
2456 if (operands[3] == CONST0_RTX (<MODE>mode))
2457 {
2458 use_zero_form = 1;
2459 break;
2460 }
2461 /* Fall through. */
2462 default:
2463 if (!REG_P (operands[3]))
2464 operands[3] = force_reg (<MODE>mode, operands[3]);
2466 break;
2467 }
2469 switch (code)
2470 {
2471 case LT:
2472 if (use_zero_form)
2473 {
2474 comparison = gen_aarch64_cmlt<mode>;
2475 break;
2476 }
2477 /* Fall through. */
2478 case UNGE:
2479 std::swap (operands[2], operands[3]);
2480 /* Fall through. */
2481 case UNLE:
2482 case GT:
2483 comparison = gen_aarch64_cmgt<mode>;
2484 break;
2485 case LE:
2486 if (use_zero_form)
2487 {
2488 comparison = gen_aarch64_cmle<mode>;
2489 break;
2490 }
2491 /* Fall through. */
2492 case UNGT:
2493 std::swap (operands[2], operands[3]);
2494 /* Fall through. */
2495 case UNLT:
2496 case GE:
2497 comparison = gen_aarch64_cmge<mode>;
2498 break;
2499 case NE:
2500 case EQ:
2501 comparison = gen_aarch64_cmeq<mode>;
2502 break;
2503 case UNEQ:
2504 case ORDERED:
2505 case UNORDERED:
2506 break;
2507 default:
2508 gcc_unreachable ();
2509 }
2511 switch (code)
2512 {
2513 case UNGE:
2514 case UNGT:
2515 case UNLE:
2516 case UNLT:
2517 case NE:
2518 /* FCM returns false for lanes which are unordered, so if we use
2519 the inverse of the comparison we actually want to emit, then
2520 invert the result, we will end up with the correct result.
2521 Note that a NE NaN and NaN NE b are true for all a, b.
2523 Our transformations are:
2524 a UNGE b -> !(b GT a)
2525 a UNGT b -> !(b GE a)
2526 a UNLE b -> !(a GT b)
2527 a UNLT b -> !(a GE b)
2528 a NE b -> !(a EQ b) */
2529 gcc_assert (comparison != NULL);
2530 emit_insn (comparison (operands[0], operands[2], operands[3]));
2531 emit_insn (gen_one_cmpl<v_cmp_result>2 (operands[0], operands[0]));
2532 break;
2534 case LT:
2535 case LE:
2536 case GT:
2537 case GE:
2538 case EQ:
2539 /* The easy case. Here we emit one of FCMGE, FCMGT or FCMEQ.
2540 As a LT b <=> b GE a && a LE b <=> b GT a. Our transformations are:
2541 a GE b -> a GE b
2542 a GT b -> a GT b
2543 a LE b -> b GE a
2544 a LT b -> b GT a
2545 a EQ b -> a EQ b */
2546 gcc_assert (comparison != NULL);
2547 emit_insn (comparison (operands[0], operands[2], operands[3]));
2548 break;
2550 case UNEQ:
2551 /* We first check (a > b || b > a) which is !UNEQ, inverting
2552 this result will then give us (a == b || a UNORDERED b). */
2553 emit_insn (gen_aarch64_cmgt<mode> (operands[0],
2554 operands[2], operands[3]));
2555 emit_insn (gen_aarch64_cmgt<mode> (tmp, operands[3], operands[2]));
2556 emit_insn (gen_ior<v_cmp_result>3 (operands[0], operands[0], tmp));
2557 emit_insn (gen_one_cmpl<v_cmp_result>2 (operands[0], operands[0]));
2558 break;
2560 case UNORDERED:
2561 /* Operands are ORDERED iff (a > b || b >= a), so we can compute
2562 UNORDERED as !ORDERED. */
2563 emit_insn (gen_aarch64_cmgt<mode> (tmp, operands[2], operands[3]));
2564 emit_insn (gen_aarch64_cmge<mode> (operands[0],
2565 operands[3], operands[2]));
2566 emit_insn (gen_ior<v_cmp_result>3 (operands[0], operands[0], tmp));
2567 emit_insn (gen_one_cmpl<v_cmp_result>2 (operands[0], operands[0]));
2568 break;
2570 case ORDERED:
2571 emit_insn (gen_aarch64_cmgt<mode> (tmp, operands[2], operands[3]));
2572 emit_insn (gen_aarch64_cmge<mode> (operands[0],
2573 operands[3], operands[2]));
2574 emit_insn (gen_ior<v_cmp_result>3 (operands[0], operands[0], tmp));
2575 break;
2577 default:
2578 gcc_unreachable ();
2579 }
2581 DONE;
2582 })
2584 (define_expand "vec_cmpu<mode><mode>"
2585 [(set (match_operand:VSDQ_I_DI 0 "register_operand")
2586 (match_operator 1 "comparison_operator"
2587 [(match_operand:VSDQ_I_DI 2 "register_operand")
2588 (match_operand:VSDQ_I_DI 3 "nonmemory_operand")]))]
2589 "TARGET_SIMD"
2590 {
2591 emit_insn (gen_vec_cmp<mode><mode> (operands[0], operands[1],
2592 operands[2], operands[3]));
2593 DONE;
2594 })
2596 (define_expand "vcond<mode><mode>"
2597 [(set (match_operand:VALLDI 0 "register_operand")
2598 (if_then_else:VALLDI
2599 (match_operator 3 "comparison_operator"
2600 [(match_operand:VALLDI 4 "register_operand")
2601 (match_operand:VALLDI 5 "nonmemory_operand")])
2602 (match_operand:VALLDI 1 "nonmemory_operand")
2603 (match_operand:VALLDI 2 "nonmemory_operand")))]
2604 "TARGET_SIMD"
2605 {
2606 rtx mask = gen_reg_rtx (<V_cmp_result>mode);
2607 enum rtx_code code = GET_CODE (operands[3]);
2609 /* NE is handled as !EQ in vec_cmp patterns, we can explicitly invert
2610 it as well as switch operands 1/2 in order to avoid the additional
2611 NOT instruction. */
2612 if (code == NE)
2613 {
2614 operands[3] = gen_rtx_fmt_ee (EQ, GET_MODE (operands[3]),
2615 operands[4], operands[5]);
2616 std::swap (operands[1], operands[2]);
2617 }
2618 emit_insn (gen_vec_cmp<mode><v_cmp_result> (mask, operands[3],
2619 operands[4], operands[5]));
2620 emit_insn (gen_vcond_mask_<mode><v_cmp_result> (operands[0], operands[1],
2621 operands[2], mask));
2623 DONE;
2624 })
2626 (define_expand "vcond<v_cmp_mixed><mode>"
2627 [(set (match_operand:<V_cmp_mixed> 0 "register_operand")
2628 (if_then_else:<V_cmp_mixed>
2629 (match_operator 3 "comparison_operator"
2630 [(match_operand:VDQF_COND 4 "register_operand")
2631 (match_operand:VDQF_COND 5 "nonmemory_operand")])
2632 (match_operand:<V_cmp_mixed> 1 "nonmemory_operand")
2633 (match_operand:<V_cmp_mixed> 2 "nonmemory_operand")))]
2634 "TARGET_SIMD"
2635 {
2636 rtx mask = gen_reg_rtx (<V_cmp_result>mode);
2637 enum rtx_code code = GET_CODE (operands[3]);
2639 /* NE is handled as !EQ in vec_cmp patterns, we can explicitly invert
2640 it as well as switch operands 1/2 in order to avoid the additional
2641 NOT instruction. */
2642 if (code == NE)
2643 {
2644 operands[3] = gen_rtx_fmt_ee (EQ, GET_MODE (operands[3]),
2645 operands[4], operands[5]);
2646 std::swap (operands[1], operands[2]);
2647 }
2648 emit_insn (gen_vec_cmp<mode><v_cmp_result> (mask, operands[3],
2649 operands[4], operands[5]));
2650 emit_insn (gen_vcond_mask_<v_cmp_mixed><v_cmp_result> (
2651 operands[0], operands[1],
2652 operands[2], mask));
2654 DONE;
2655 })
2657 (define_expand "vcondu<mode><mode>"
2658 [(set (match_operand:VSDQ_I_DI 0 "register_operand")
2659 (if_then_else:VSDQ_I_DI
2660 (match_operator 3 "comparison_operator"
2661 [(match_operand:VSDQ_I_DI 4 "register_operand")
2662 (match_operand:VSDQ_I_DI 5 "nonmemory_operand")])
2663 (match_operand:VSDQ_I_DI 1 "nonmemory_operand")
2664 (match_operand:VSDQ_I_DI 2 "nonmemory_operand")))]
2665 "TARGET_SIMD"
2666 {
2667 rtx mask = gen_reg_rtx (<MODE>mode);
2668 enum rtx_code code = GET_CODE (operands[3]);
2670 /* NE is handled as !EQ in vec_cmp patterns, we can explicitly invert
2671 it as well as switch operands 1/2 in order to avoid the additional
2672 NOT instruction. */
2673 if (code == NE)
2674 {
2675 operands[3] = gen_rtx_fmt_ee (EQ, GET_MODE (operands[3]),
2676 operands[4], operands[5]);
2677 std::swap (operands[1], operands[2]);
2678 }
2679 emit_insn (gen_vec_cmp<mode><mode> (mask, operands[3],
2680 operands[4], operands[5]));
2681 emit_insn (gen_vcond_mask_<mode><v_cmp_result> (operands[0], operands[1],
2682 operands[2], mask));
2683 DONE;
2684 })
2686 (define_expand "vcondu<mode><v_cmp_mixed>"
2687 [(set (match_operand:VDQF 0 "register_operand")
2688 (if_then_else:VDQF
2689 (match_operator 3 "comparison_operator"
2690 [(match_operand:<V_cmp_mixed> 4 "register_operand")
2691 (match_operand:<V_cmp_mixed> 5 "nonmemory_operand")])
2692 (match_operand:VDQF 1 "nonmemory_operand")
2693 (match_operand:VDQF 2 "nonmemory_operand")))]
2694 "TARGET_SIMD"
2695 {
2696 rtx mask = gen_reg_rtx (<V_cmp_result>mode);
2697 enum rtx_code code = GET_CODE (operands[3]);
2699 /* NE is handled as !EQ in vec_cmp patterns, we can explicitly invert
2700 it as well as switch operands 1/2 in order to avoid the additional
2701 NOT instruction. */
2702 if (code == NE)
2703 {
2704 operands[3] = gen_rtx_fmt_ee (EQ, GET_MODE (operands[3]),
2705 operands[4], operands[5]);
2706 std::swap (operands[1], operands[2]);
2707 }
2708 emit_insn (gen_vec_cmp<v_cmp_mixed><v_cmp_mixed> (
2709 mask, operands[3],
2710 operands[4], operands[5]));
2711 emit_insn (gen_vcond_mask_<mode><v_cmp_result> (operands[0], operands[1],
2712 operands[2], mask));
2713 DONE;
2714 })
2716 ;; Patterns for AArch64 SIMD Intrinsics.
2718 ;; Lane extraction with sign extension to general purpose register.
2719 (define_insn "*aarch64_get_lane_extend<GPI:mode><VDQQH:mode>"
2720 [(set (match_operand:GPI 0 "register_operand" "=r")
2721 (sign_extend:GPI
2722 (vec_select:<VEL>
2723 (match_operand:VDQQH 1 "register_operand" "w")
2724 (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2725 "TARGET_SIMD"
2726 {
2727 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
2728 return "smov\\t%<GPI:w>0, %1.<VDQQH:Vetype>[%2]";
2729 }
2730 [(set_attr "type" "neon_to_gp<q>")]
2731 )
2733 (define_insn "*aarch64_get_lane_zero_extendsi<mode>"
2734 [(set (match_operand:SI 0 "register_operand" "=r")
2735 (zero_extend:SI
2736 (vec_select:<VEL>
2737 (match_operand:VDQQH 1 "register_operand" "w")
2738 (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2739 "TARGET_SIMD"
2740 {
2741 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
2742 return "umov\\t%w0, %1.<Vetype>[%2]";
2743 }
2744 [(set_attr "type" "neon_to_gp<q>")]
2745 )
2747 ;; Lane extraction of a value, neither sign nor zero extension
2748 ;; is guaranteed so upper bits should be considered undefined.
2749 ;; RTL uses GCC vector extension indices throughout so flip only for assembly.
2750 (define_insn "aarch64_get_lane<mode>"
2751 [(set (match_operand:<VEL> 0 "aarch64_simd_nonimmediate_operand" "=r, w, Utv")
2752 (vec_select:<VEL>
2753 (match_operand:VALL_F16 1 "register_operand" "w, w, w")
2754 (parallel [(match_operand:SI 2 "immediate_operand" "i, i, i")])))]
2755 "TARGET_SIMD"
2756 {
2757 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
2758 switch (which_alternative)
2759 {
2760 case 0:
2761 return "umov\\t%<vwcore>0, %1.<Vetype>[%2]";
2762 case 1:
2763 return "dup\\t%<Vetype>0, %1.<Vetype>[%2]";
2764 case 2:
2765 return "st1\\t{%1.<Vetype>}[%2], %0";
2766 default:
2767 gcc_unreachable ();
2768 }
2769 }
2770 [(set_attr "type" "neon_to_gp<q>, neon_dup<q>, neon_store1_one_lane<q>")]
2771 )
2773 ;; In this insn, operand 1 should be low, and operand 2 the high part of the
2774 ;; dest vector.
2776 (define_insn "*aarch64_combinez<mode>"
2777 [(set (match_operand:<VDBL> 0 "register_operand" "=w,w,w")
2778 (vec_concat:<VDBL>
2779 (match_operand:VD_BHSI 1 "general_operand" "w,?r,m")
2780 (match_operand:VD_BHSI 2 "aarch64_simd_imm_zero" "Dz,Dz,Dz")))]
2781 "TARGET_SIMD && !BYTES_BIG_ENDIAN"
2782 "@
2783 mov\\t%0.8b, %1.8b
2784 fmov\t%d0, %1
2785 ldr\\t%d0, %1"
2786 [(set_attr "type" "neon_move<q>, neon_from_gp, neon_load1_1reg")
2787 (set_attr "simd" "yes,*,yes")
2788 (set_attr "fp" "*,yes,*")]
2789 )
2791 (define_insn "*aarch64_combinez_be<mode>"
2792 [(set (match_operand:<VDBL> 0 "register_operand" "=w,w,w")
2793 (vec_concat:<VDBL>
2794 (match_operand:VD_BHSI 2 "aarch64_simd_imm_zero" "Dz,Dz,Dz")
2795 (match_operand:VD_BHSI 1 "general_operand" "w,?r,m")))]
2796 "TARGET_SIMD && BYTES_BIG_ENDIAN"
2797 "@
2798 mov\\t%0.8b, %1.8b
2799 fmov\t%d0, %1
2800 ldr\\t%d0, %1"
2801 [(set_attr "type" "neon_move<q>, neon_from_gp, neon_load1_1reg")
2802 (set_attr "simd" "yes,*,yes")
2803 (set_attr "fp" "*,yes,*")]
2804 )
2806 (define_expand "aarch64_combine<mode>"
2807 [(match_operand:<VDBL> 0 "register_operand")
2808 (match_operand:VDC 1 "register_operand")
2809 (match_operand:VDC 2 "register_operand")]
2810 "TARGET_SIMD"
2811 {
2812 rtx op1, op2;
2813 if (BYTES_BIG_ENDIAN)
2814 {
2815 op1 = operands[2];
2816 op2 = operands[1];
2817 }
2818 else
2819 {
2820 op1 = operands[1];
2821 op2 = operands[2];
2822 }
2823 emit_insn (gen_aarch64_combine_internal<mode> (operands[0], op1, op2));
2824 DONE;
2825 }
2826 )
2828 (define_insn_and_split "aarch64_combine_internal<mode>"
2829 [(set (match_operand:<VDBL> 0 "register_operand" "=&w")
2830 (vec_concat:<VDBL> (match_operand:VDC 1 "register_operand" "w")
2831 (match_operand:VDC 2 "register_operand" "w")))]
2832 "TARGET_SIMD"
2833 "#"
2834 "&& reload_completed"
2835 [(const_int 0)]
2836 {
2837 if (BYTES_BIG_ENDIAN)
2838 aarch64_split_simd_combine (operands[0], operands[2], operands[1]);
2839 else
2840 aarch64_split_simd_combine (operands[0], operands[1], operands[2]);
2841 DONE;
2842 }
2843 [(set_attr "type" "multiple")]
2844 )
2846 (define_expand "aarch64_simd_combine<mode>"
2847 [(match_operand:<VDBL> 0 "register_operand")
2848 (match_operand:VDC 1 "register_operand")
2849 (match_operand:VDC 2 "register_operand")]
2850 "TARGET_SIMD"
2851 {
2852 emit_insn (gen_move_lo_quad_<Vdbl> (operands[0], operands[1]));
2853 emit_insn (gen_move_hi_quad_<Vdbl> (operands[0], operands[2]));
2854 DONE;
2855 }
2856 [(set_attr "type" "multiple")]
2857 )
2859 ;; <su><addsub>l<q>.
2861 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>l<mode>_hi_internal"
2862 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2863 (ADDSUB:<VWIDE> (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
2864 (match_operand:VQW 1 "register_operand" "w")
2865 (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))
2866 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
2867 (match_operand:VQW 2 "register_operand" "w")
2868 (match_dup 3)))))]
2869 "TARGET_SIMD"
2870 "<ANY_EXTEND:su><ADDSUB:optab>l2\t%0.<Vwtype>, %1.<Vtype>, %2.<Vtype>"
2871 [(set_attr "type" "neon_<ADDSUB:optab>_long")]
2872 )
2874 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>l<mode>_lo_internal"
2875 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2876 (ADDSUB:<VWIDE> (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
2877 (match_operand:VQW 1 "register_operand" "w")
2878 (match_operand:VQW 3 "vect_par_cnst_lo_half" "")))
2879 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
2880 (match_operand:VQW 2 "register_operand" "w")
2881 (match_dup 3)))))]
2882 "TARGET_SIMD"
2883 "<ANY_EXTEND:su><ADDSUB:optab>l\t%0.<Vwtype>, %1.<Vhalftype>, %2.<Vhalftype>"
2884 [(set_attr "type" "neon_<ADDSUB:optab>_long")]
2885 )
2888 (define_expand "aarch64_saddl2<mode>"
2889 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2890 (match_operand:VQW 1 "register_operand" "w")
2891 (match_operand:VQW 2 "register_operand" "w")]
2892 "TARGET_SIMD"
2893 {
2894 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2895 emit_insn (gen_aarch64_saddl<mode>_hi_internal (operands[0], operands[1],
2896 operands[2], p));
2897 DONE;
2898 })
2900 (define_expand "aarch64_uaddl2<mode>"
2901 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2902 (match_operand:VQW 1 "register_operand" "w")
2903 (match_operand:VQW 2 "register_operand" "w")]
2904 "TARGET_SIMD"
2905 {
2906 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2907 emit_insn (gen_aarch64_uaddl<mode>_hi_internal (operands[0], operands[1],
2908 operands[2], p));
2909 DONE;
2910 })
2912 (define_expand "aarch64_ssubl2<mode>"
2913 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2914 (match_operand:VQW 1 "register_operand" "w")
2915 (match_operand:VQW 2 "register_operand" "w")]
2916 "TARGET_SIMD"
2917 {
2918 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2919 emit_insn (gen_aarch64_ssubl<mode>_hi_internal (operands[0], operands[1],
2920 operands[2], p));
2921 DONE;
2922 })
2924 (define_expand "aarch64_usubl2<mode>"
2925 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2926 (match_operand:VQW 1 "register_operand" "w")
2927 (match_operand:VQW 2 "register_operand" "w")]
2928 "TARGET_SIMD"
2929 {
2930 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2931 emit_insn (gen_aarch64_usubl<mode>_hi_internal (operands[0], operands[1],
2932 operands[2], p));
2933 DONE;
2934 })
2936 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>l<mode>"
2937 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2938 (ADDSUB:<VWIDE> (ANY_EXTEND:<VWIDE>
2939 (match_operand:VD_BHSI 1 "register_operand" "w"))
2940 (ANY_EXTEND:<VWIDE>
2941 (match_operand:VD_BHSI 2 "register_operand" "w"))))]
2942 "TARGET_SIMD"
2943 "<ANY_EXTEND:su><ADDSUB:optab>l\t%0.<Vwtype>, %1.<Vtype>, %2.<Vtype>"
2944 [(set_attr "type" "neon_<ADDSUB:optab>_long")]
2945 )
2947 ;; <su><addsub>w<q>.
2949 (define_expand "widen_ssum<mode>3"
2950 [(set (match_operand:<VDBLW> 0 "register_operand" "")
2951 (plus:<VDBLW> (sign_extend:<VDBLW>
2952 (match_operand:VQW 1 "register_operand" ""))
2953 (match_operand:<VDBLW> 2 "register_operand" "")))]
2954 "TARGET_SIMD"
2955 {
2956 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
2957 rtx temp = gen_reg_rtx (GET_MODE (operands[0]));
2959 emit_insn (gen_aarch64_saddw<mode>_internal (temp, operands[2],
2960 operands[1], p));
2961 emit_insn (gen_aarch64_saddw2<mode> (operands[0], temp, operands[1]));
2962 DONE;
2963 }
2964 )
2966 (define_expand "widen_ssum<mode>3"
2967 [(set (match_operand:<VWIDE> 0 "register_operand" "")
2968 (plus:<VWIDE> (sign_extend:<VWIDE>
2969 (match_operand:VD_BHSI 1 "register_operand" ""))
2970 (match_operand:<VWIDE> 2 "register_operand" "")))]
2971 "TARGET_SIMD"
2972 {
2973 emit_insn (gen_aarch64_saddw<mode> (operands[0], operands[2], operands[1]));
2974 DONE;
2975 })
2977 (define_expand "widen_usum<mode>3"
2978 [(set (match_operand:<VDBLW> 0 "register_operand" "")
2979 (plus:<VDBLW> (zero_extend:<VDBLW>
2980 (match_operand:VQW 1 "register_operand" ""))
2981 (match_operand:<VDBLW> 2 "register_operand" "")))]
2982 "TARGET_SIMD"
2983 {
2984 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
2985 rtx temp = gen_reg_rtx (GET_MODE (operands[0]));
2987 emit_insn (gen_aarch64_uaddw<mode>_internal (temp, operands[2],
2988 operands[1], p));
2989 emit_insn (gen_aarch64_uaddw2<mode> (operands[0], temp, operands[1]));
2990 DONE;
2991 }
2992 )
2994 (define_expand "widen_usum<mode>3"
2995 [(set (match_operand:<VWIDE> 0 "register_operand" "")
2996 (plus:<VWIDE> (zero_extend:<VWIDE>
2997 (match_operand:VD_BHSI 1 "register_operand" ""))
2998 (match_operand:<VWIDE> 2 "register_operand" "")))]
2999 "TARGET_SIMD"
3000 {
3001 emit_insn (gen_aarch64_uaddw<mode> (operands[0], operands[2], operands[1]));
3002 DONE;
3003 })
3005 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>w<mode>"
3006 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3007 (ADDSUB:<VWIDE> (match_operand:<VWIDE> 1 "register_operand" "w")
3008 (ANY_EXTEND:<VWIDE>
3009 (match_operand:VD_BHSI 2 "register_operand" "w"))))]
3010 "TARGET_SIMD"
3011 "<ANY_EXTEND:su><ADDSUB:optab>w\\t%0.<Vwtype>, %1.<Vwtype>, %2.<Vtype>"
3012 [(set_attr "type" "neon_<ADDSUB:optab>_widen")]
3013 )
3015 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>w<mode>_internal"
3016 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3017 (ADDSUB:<VWIDE> (match_operand:<VWIDE> 1 "register_operand" "w")
3018 (ANY_EXTEND:<VWIDE>
3019 (vec_select:<VHALF>
3020 (match_operand:VQW 2 "register_operand" "w")
3021 (match_operand:VQW 3 "vect_par_cnst_lo_half" "")))))]
3022 "TARGET_SIMD"
3023 "<ANY_EXTEND:su><ADDSUB:optab>w\\t%0.<Vwtype>, %1.<Vwtype>, %2.<Vhalftype>"
3024 [(set_attr "type" "neon_<ADDSUB:optab>_widen")]
3025 )
3027 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>w2<mode>_internal"
3028 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3029 (ADDSUB:<VWIDE> (match_operand:<VWIDE> 1 "register_operand" "w")
3030 (ANY_EXTEND:<VWIDE>
3031 (vec_select:<VHALF>
3032 (match_operand:VQW 2 "register_operand" "w")
3033 (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))))]
3034 "TARGET_SIMD"
3035 "<ANY_EXTEND:su><ADDSUB:optab>w2\\t%0.<Vwtype>, %1.<Vwtype>, %2.<Vtype>"
3036 [(set_attr "type" "neon_<ADDSUB:optab>_widen")]
3037 )
3039 (define_expand "aarch64_saddw2<mode>"
3040 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3041 (match_operand:<VWIDE> 1 "register_operand" "w")
3042 (match_operand:VQW 2 "register_operand" "w")]
3043 "TARGET_SIMD"
3044 {
3045 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3046 emit_insn (gen_aarch64_saddw2<mode>_internal (operands[0], operands[1],
3047 operands[2], p));
3048 DONE;
3049 })
3051 (define_expand "aarch64_uaddw2<mode>"
3052 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3053 (match_operand:<VWIDE> 1 "register_operand" "w")
3054 (match_operand:VQW 2 "register_operand" "w")]
3055 "TARGET_SIMD"
3056 {
3057 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3058 emit_insn (gen_aarch64_uaddw2<mode>_internal (operands[0], operands[1],
3059 operands[2], p));
3060 DONE;
3061 })
3064 (define_expand "aarch64_ssubw2<mode>"
3065 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3066 (match_operand:<VWIDE> 1 "register_operand" "w")
3067 (match_operand:VQW 2 "register_operand" "w")]
3068 "TARGET_SIMD"
3069 {
3070 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3071 emit_insn (gen_aarch64_ssubw2<mode>_internal (operands[0], operands[1],
3072 operands[2], p));
3073 DONE;
3074 })
3076 (define_expand "aarch64_usubw2<mode>"
3077 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3078 (match_operand:<VWIDE> 1 "register_operand" "w")
3079 (match_operand:VQW 2 "register_operand" "w")]
3080 "TARGET_SIMD"
3081 {
3082 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3083 emit_insn (gen_aarch64_usubw2<mode>_internal (operands[0], operands[1],
3084 operands[2], p));
3085 DONE;
3086 })
3088 ;; <su><r>h<addsub>.
3090 (define_insn "aarch64_<sur>h<addsub><mode>"
3091 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
3092 (unspec:VDQ_BHSI [(match_operand:VDQ_BHSI 1 "register_operand" "w")
3093 (match_operand:VDQ_BHSI 2 "register_operand" "w")]
3094 HADDSUB))]
3095 "TARGET_SIMD"
3096 "<sur>h<addsub>\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
3097 [(set_attr "type" "neon_<addsub>_halve<q>")]
3098 )
3100 ;; <r><addsub>hn<q>.
3102 (define_insn "aarch64_<sur><addsub>hn<mode>"
3103 [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
3104 (unspec:<VNARROWQ> [(match_operand:VQN 1 "register_operand" "w")
3105 (match_operand:VQN 2 "register_operand" "w")]
3106 ADDSUBHN))]
3107 "TARGET_SIMD"
3108 "<sur><addsub>hn\\t%0.<Vntype>, %1.<Vtype>, %2.<Vtype>"
3109 [(set_attr "type" "neon_<addsub>_halve_narrow_q")]
3110 )
3112 (define_insn "aarch64_<sur><addsub>hn2<mode>"
3113 [(set (match_operand:<VNARROWQ2> 0 "register_operand" "=w")
3114 (unspec:<VNARROWQ2> [(match_operand:<VNARROWQ> 1 "register_operand" "0")
3115 (match_operand:VQN 2 "register_operand" "w")
3116 (match_operand:VQN 3 "register_operand" "w")]
3117 ADDSUBHN2))]
3118 "TARGET_SIMD"
3119 "<sur><addsub>hn2\\t%0.<V2ntype>, %2.<Vtype>, %3.<Vtype>"
3120 [(set_attr "type" "neon_<addsub>_halve_narrow_q")]
3121 )
3123 ;; pmul.
3125 (define_insn "aarch64_pmul<mode>"
3126 [(set (match_operand:VB 0 "register_operand" "=w")
3127 (unspec:VB [(match_operand:VB 1 "register_operand" "w")
3128 (match_operand:VB 2 "register_operand" "w")]
3129 UNSPEC_PMUL))]
3130 "TARGET_SIMD"
3131 "pmul\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
3132 [(set_attr "type" "neon_mul_<Vetype><q>")]
3133 )
3135 ;; fmulx.
3137 (define_insn "aarch64_fmulx<mode>"
3138 [(set (match_operand:VHSDF_HSDF 0 "register_operand" "=w")
3139 (unspec:VHSDF_HSDF
3140 [(match_operand:VHSDF_HSDF 1 "register_operand" "w")
3141 (match_operand:VHSDF_HSDF 2 "register_operand" "w")]
3142 UNSPEC_FMULX))]
3143 "TARGET_SIMD"
3144 "fmulx\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
3145 [(set_attr "type" "neon_fp_mul_<stype>")]
3146 )
3148 ;; vmulxq_lane_f32, and vmulx_laneq_f32
3150 (define_insn "*aarch64_mulx_elt_<vswap_width_name><mode>"
3151 [(set (match_operand:VDQSF 0 "register_operand" "=w")
3152 (unspec:VDQSF
3153 [(match_operand:VDQSF 1 "register_operand" "w")
3154 (vec_duplicate:VDQSF
3155 (vec_select:<VEL>
3156 (match_operand:<VSWAP_WIDTH> 2 "register_operand" "w")
3157 (parallel [(match_operand:SI 3 "immediate_operand" "i")])))]
3158 UNSPEC_FMULX))]
3159 "TARGET_SIMD"
3160 {
3161 operands[3] = GEN_INT (ENDIAN_LANE_N (<VSWAP_WIDTH>mode,
3162 INTVAL (operands[3])));
3163 return "fmulx\t%<v>0<Vmtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3164 }
3165 [(set_attr "type" "neon_fp_mul_<Vetype>_scalar<q>")]
3166 )
3168 ;; vmulxq_laneq_f32, vmulxq_laneq_f64, vmulx_lane_f32
3170 (define_insn "*aarch64_mulx_elt<mode>"
3171 [(set (match_operand:VDQF 0 "register_operand" "=w")
3172 (unspec:VDQF
3173 [(match_operand:VDQF 1 "register_operand" "w")
3174 (vec_duplicate:VDQF
3175 (vec_select:<VEL>
3176 (match_operand:VDQF 2 "register_operand" "w")
3177 (parallel [(match_operand:SI 3 "immediate_operand" "i")])))]
3178 UNSPEC_FMULX))]
3179 "TARGET_SIMD"
3180 {
3181 operands[3] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[3])));
3182 return "fmulx\t%<v>0<Vmtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3183 }
3184 [(set_attr "type" "neon_fp_mul_<Vetype><q>")]
3185 )
3187 ;; vmulxq_lane
3189 (define_insn "*aarch64_mulx_elt_from_dup<mode>"
3190 [(set (match_operand:VHSDF 0 "register_operand" "=w")
3191 (unspec:VHSDF
3192 [(match_operand:VHSDF 1 "register_operand" "w")
3193 (vec_duplicate:VHSDF
3194 (match_operand:<VEL> 2 "register_operand" "<h_con>"))]
3195 UNSPEC_FMULX))]
3196 "TARGET_SIMD"
3197 "fmulx\t%0.<Vtype>, %1.<Vtype>, %2.<Vetype>[0]";
3198 [(set_attr "type" "neon<fp>_mul_<stype>_scalar<q>")]
3199 )
3201 ;; vmulxs_lane_f32, vmulxs_laneq_f32
3202 ;; vmulxd_lane_f64 == vmulx_lane_f64
3203 ;; vmulxd_laneq_f64 == vmulx_laneq_f64
3205 (define_insn "*aarch64_vgetfmulx<mode>"
3206 [(set (match_operand:<VEL> 0 "register_operand" "=w")
3207 (unspec:<VEL>
3208 [(match_operand:<VEL> 1 "register_operand" "w")
3209 (vec_select:<VEL>
3210 (match_operand:VDQF 2 "register_operand" "w")
3211 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))]
3212 UNSPEC_FMULX))]
3213 "TARGET_SIMD"
3214 {
3215 operands[3] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[3])));
3216 return "fmulx\t%<Vetype>0, %<Vetype>1, %2.<Vetype>[%3]";
3217 }
3218 [(set_attr "type" "fmul<Vetype>")]
3219 )
3220 ;; <su>q<addsub>
3222 (define_insn "aarch64_<su_optab><optab><mode>"
3223 [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
3224 (BINQOPS:VSDQ_I (match_operand:VSDQ_I 1 "register_operand" "w")
3225 (match_operand:VSDQ_I 2 "register_operand" "w")))]
3226 "TARGET_SIMD"
3227 "<su_optab><optab>\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
3228 [(set_attr "type" "neon_<optab><q>")]
3229 )
3231 ;; suqadd and usqadd
3233 (define_insn "aarch64_<sur>qadd<mode>"
3234 [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
3235 (unspec:VSDQ_I [(match_operand:VSDQ_I 1 "register_operand" "0")
3236 (match_operand:VSDQ_I 2 "register_operand" "w")]
3237 USSUQADD))]
3238 "TARGET_SIMD"
3239 "<sur>qadd\\t%<v>0<Vmtype>, %<v>2<Vmtype>"
3240 [(set_attr "type" "neon_qadd<q>")]
3241 )
3243 ;; sqmovun
3245 (define_insn "aarch64_sqmovun<mode>"
3246 [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
3247 (unspec:<VNARROWQ> [(match_operand:VSQN_HSDI 1 "register_operand" "w")]
3248 UNSPEC_SQXTUN))]
3249 "TARGET_SIMD"
3250 "sqxtun\\t%<vn2>0<Vmntype>, %<v>1<Vmtype>"
3251 [(set_attr "type" "neon_sat_shift_imm_narrow_q")]
3252 )
3254 ;; sqmovn and uqmovn
3256 (define_insn "aarch64_<sur>qmovn<mode>"
3257 [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
3258 (unspec:<VNARROWQ> [(match_operand:VSQN_HSDI 1 "register_operand" "w")]
3259 SUQMOVN))]
3260 "TARGET_SIMD"
3261 "<sur>qxtn\\t%<vn2>0<Vmntype>, %<v>1<Vmtype>"
3262 [(set_attr "type" "neon_sat_shift_imm_narrow_q")]
3263 )
3265 ;; <su>q<absneg>
3267 (define_insn "aarch64_s<optab><mode>"
3268 [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
3269 (UNQOPS:VSDQ_I
3270 (match_operand:VSDQ_I 1 "register_operand" "w")))]
3271 "TARGET_SIMD"
3272 "s<optab>\\t%<v>0<Vmtype>, %<v>1<Vmtype>"
3273 [(set_attr "type" "neon_<optab><q>")]
3274 )
3276 ;; sq<r>dmulh.
3278 (define_insn "aarch64_sq<r>dmulh<mode>"
3279 [(set (match_operand:VSDQ_HSI 0 "register_operand" "=w")
3280 (unspec:VSDQ_HSI
3281 [(match_operand:VSDQ_HSI 1 "register_operand" "w")
3282 (match_operand:VSDQ_HSI 2 "register_operand" "w")]
3283 VQDMULH))]
3284 "TARGET_SIMD"
3285 "sq<r>dmulh\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
3286 [(set_attr "type" "neon_sat_mul_<Vetype><q>")]
3287 )
3289 ;; sq<r>dmulh_lane
3291 (define_insn "aarch64_sq<r>dmulh_lane<mode>"
3292 [(set (match_operand:VDQHS 0 "register_operand" "=w")
3293 (unspec:VDQHS
3294 [(match_operand:VDQHS 1 "register_operand" "w")
3295 (vec_select:<VEL>
3296 (match_operand:<VCOND> 2 "register_operand" "<vwx>")
3297 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))]
3298 VQDMULH))]
3299 "TARGET_SIMD"
3300 "*
3301 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
3302 return \"sq<r>dmulh\\t%0.<Vtype>, %1.<Vtype>, %2.<Vetype>[%3]\";"
3303 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar<q>")]
3304 )
3306 (define_insn "aarch64_sq<r>dmulh_laneq<mode>"
3307 [(set (match_operand:VDQHS 0 "register_operand" "=w")
3308 (unspec:VDQHS
3309 [(match_operand:VDQHS 1 "register_operand" "w")
3310 (vec_select:<VEL>
3311 (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
3312 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))]
3313 VQDMULH))]
3314 "TARGET_SIMD"
3315 "*
3316 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[3])));
3317 return \"sq<r>dmulh\\t%0.<Vtype>, %1.<Vtype>, %2.<Vetype>[%3]\";"
3318 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar<q>")]
3319 )
3321 (define_insn "aarch64_sq<r>dmulh_lane<mode>"
3322 [(set (match_operand:SD_HSI 0 "register_operand" "=w")
3323 (unspec:SD_HSI
3324 [(match_operand:SD_HSI 1 "register_operand" "w")
3325 (vec_select:<VEL>
3326 (match_operand:<VCOND> 2 "register_operand" "<vwx>")
3327 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))]
3328 VQDMULH))]
3329 "TARGET_SIMD"
3330 "*
3331 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
3332 return \"sq<r>dmulh\\t%<v>0, %<v>1, %2.<v>[%3]\";"
3333 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar<q>")]
3334 )
3336 (define_insn "aarch64_sq<r>dmulh_laneq<mode>"
3337 [(set (match_operand:SD_HSI 0 "register_operand" "=w")
3338 (unspec:SD_HSI
3339 [(match_operand:SD_HSI 1 "register_operand" "w")
3340 (vec_select:<VEL>
3341 (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
3342 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))]
3343 VQDMULH))]
3344 "TARGET_SIMD"
3345 "*
3346 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[3])));
3347 return \"sq<r>dmulh\\t%<v>0, %<v>1, %2.<v>[%3]\";"
3348 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar<q>")]
3349 )
3351 ;; sqrdml[as]h.
3353 (define_insn "aarch64_sqrdml<SQRDMLH_AS:rdma_as>h<mode>"
3354 [(set (match_operand:VSDQ_HSI 0 "register_operand" "=w")
3355 (unspec:VSDQ_HSI
3356 [(match_operand:VSDQ_HSI 1 "register_operand" "0")
3357 (match_operand:VSDQ_HSI 2 "register_operand" "w")
3358 (match_operand:VSDQ_HSI 3 "register_operand" "w")]
3359 SQRDMLH_AS))]
3360 "TARGET_SIMD_RDMA"
3361 "sqrdml<SQRDMLH_AS:rdma_as>h\\t%<v>0<Vmtype>, %<v>2<Vmtype>, %<v>3<Vmtype>"
3362 [(set_attr "type" "neon_sat_mla_<Vetype>_long")]
3363 )
3365 ;; sqrdml[as]h_lane.
3367 (define_insn "aarch64_sqrdml<SQRDMLH_AS:rdma_as>h_lane<mode>"
3368 [(set (match_operand:VDQHS 0 "register_operand" "=w")
3369 (unspec:VDQHS
3370 [(match_operand:VDQHS 1 "register_operand" "0")
3371 (match_operand:VDQHS 2 "register_operand" "w")
3372 (vec_select:<VEL>
3373 (match_operand:<VCOND> 3 "register_operand" "<vwx>")
3374 (parallel [(match_operand:SI 4 "immediate_operand" "i")]))]
3375 SQRDMLH_AS))]
3376 "TARGET_SIMD_RDMA"
3377 {
3378 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[4])));
3379 return
3380 "sqrdml<SQRDMLH_AS:rdma_as>h\\t%0.<Vtype>, %2.<Vtype>, %3.<Vetype>[%4]";
3381 }
3382 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3383 )
3385 (define_insn "aarch64_sqrdml<SQRDMLH_AS:rdma_as>h_lane<mode>"
3386 [(set (match_operand:SD_HSI 0 "register_operand" "=w")
3387 (unspec:SD_HSI
3388 [(match_operand:SD_HSI 1 "register_operand" "0")
3389 (match_operand:SD_HSI 2 "register_operand" "w")
3390 (vec_select:<VEL>
3391 (match_operand:<VCOND> 3 "register_operand" "<vwx>")
3392 (parallel [(match_operand:SI 4 "immediate_operand" "i")]))]
3393 SQRDMLH_AS))]
3394 "TARGET_SIMD_RDMA"
3395 {
3396 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[4])));
3397 return
3398 "sqrdml<SQRDMLH_AS:rdma_as>h\\t%<v>0, %<v>2, %3.<Vetype>[%4]";
3399 }
3400 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3401 )
3403 ;; sqrdml[as]h_laneq.
3405 (define_insn "aarch64_sqrdml<SQRDMLH_AS:rdma_as>h_laneq<mode>"
3406 [(set (match_operand:VDQHS 0 "register_operand" "=w")
3407 (unspec:VDQHS
3408 [(match_operand:VDQHS 1 "register_operand" "0")
3409 (match_operand:VDQHS 2 "register_operand" "w")
3410 (vec_select:<VEL>
3411 (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
3412 (parallel [(match_operand:SI 4 "immediate_operand" "i")]))]
3413 SQRDMLH_AS))]
3414 "TARGET_SIMD_RDMA"
3415 {
3416 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[4])));
3417 return
3418 "sqrdml<SQRDMLH_AS:rdma_as>h\\t%0.<Vtype>, %2.<Vtype>, %3.<Vetype>[%4]";
3419 }
3420 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3421 )
3423 (define_insn "aarch64_sqrdml<SQRDMLH_AS:rdma_as>h_laneq<mode>"
3424 [(set (match_operand:SD_HSI 0 "register_operand" "=w")
3425 (unspec:SD_HSI
3426 [(match_operand:SD_HSI 1 "register_operand" "0")
3427 (match_operand:SD_HSI 2 "register_operand" "w")
3428 (vec_select:<VEL>
3429 (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
3430 (parallel [(match_operand:SI 4 "immediate_operand" "i")]))]
3431 SQRDMLH_AS))]
3432 "TARGET_SIMD_RDMA"
3433 {
3434 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[4])));
3435 return
3436 "sqrdml<SQRDMLH_AS:rdma_as>h\\t%<v>0, %<v>2, %3.<v>[%4]";
3437 }
3438 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3439 )
3441 ;; vqdml[sa]l
3443 (define_insn "aarch64_sqdml<SBINQOPS:as>l<mode>"
3444 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3445 (SBINQOPS:<VWIDE>
3446 (match_operand:<VWIDE> 1 "register_operand" "0")
3447 (ss_ashift:<VWIDE>
3448 (mult:<VWIDE>
3449 (sign_extend:<VWIDE>
3450 (match_operand:VSD_HSI 2 "register_operand" "w"))
3451 (sign_extend:<VWIDE>
3452 (match_operand:VSD_HSI 3 "register_operand" "w")))
3453 (const_int 1))))]
3454 "TARGET_SIMD"
3455 "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %<v>3<Vmtype>"
3456 [(set_attr "type" "neon_sat_mla_<Vetype>_long")]
3457 )
3459 ;; vqdml[sa]l_lane
3461 (define_insn "aarch64_sqdml<SBINQOPS:as>l_lane<mode>"
3462 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3463 (SBINQOPS:<VWIDE>
3464 (match_operand:<VWIDE> 1 "register_operand" "0")
3465 (ss_ashift:<VWIDE>
3466 (mult:<VWIDE>
3467 (sign_extend:<VWIDE>
3468 (match_operand:VD_HSI 2 "register_operand" "w"))
3469 (sign_extend:<VWIDE>
3470 (vec_duplicate:VD_HSI
3471 (vec_select:<VEL>
3472 (match_operand:<VCOND> 3 "register_operand" "<vwx>")
3473 (parallel [(match_operand:SI 4 "immediate_operand" "i")])))
3474 ))
3475 (const_int 1))))]
3476 "TARGET_SIMD"
3477 {
3478 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[4])));
3479 return
3480 "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
3481 }
3482 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3483 )
3485 (define_insn "aarch64_sqdml<SBINQOPS:as>l_laneq<mode>"
3486 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3487 (SBINQOPS:<VWIDE>
3488 (match_operand:<VWIDE> 1 "register_operand" "0")
3489 (ss_ashift:<VWIDE>
3490 (mult:<VWIDE>
3491 (sign_extend:<VWIDE>
3492 (match_operand:VD_HSI 2 "register_operand" "w"))
3493 (sign_extend:<VWIDE>
3494 (vec_duplicate:VD_HSI
3495 (vec_select:<VEL>
3496 (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
3497 (parallel [(match_operand:SI 4 "immediate_operand" "i")])))
3498 ))
3499 (const_int 1))))]
3500 "TARGET_SIMD"
3501 {
3502 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[4])));
3503 return
3504 "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
3505 }
3506 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3507 )
3509 (define_insn "aarch64_sqdml<SBINQOPS:as>l_lane<mode>"
3510 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3511 (SBINQOPS:<VWIDE>
3512 (match_operand:<VWIDE> 1 "register_operand" "0")
3513 (ss_ashift:<VWIDE>
3514 (mult:<VWIDE>
3515 (sign_extend:<VWIDE>
3516 (match_operand:SD_HSI 2 "register_operand" "w"))
3517 (sign_extend:<VWIDE>
3518 (vec_select:<VEL>
3519 (match_operand:<VCOND> 3 "register_operand" "<vwx>")
3520 (parallel [(match_operand:SI 4 "immediate_operand" "i")])))
3521 )
3522 (const_int 1))))]
3523 "TARGET_SIMD"
3524 {
3525 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[4])));
3526 return
3527 "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
3528 }
3529 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3530 )
3532 (define_insn "aarch64_sqdml<SBINQOPS:as>l_laneq<mode>"
3533 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3534 (SBINQOPS:<VWIDE>
3535 (match_operand:<VWIDE> 1 "register_operand" "0")
3536 (ss_ashift:<VWIDE>
3537 (mult:<VWIDE>
3538 (sign_extend:<VWIDE>
3539 (match_operand:SD_HSI 2 "register_operand" "w"))
3540 (sign_extend:<VWIDE>
3541 (vec_select:<VEL>
3542 (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
3543 (parallel [(match_operand:SI 4 "immediate_operand" "i")])))
3544 )
3545 (const_int 1))))]
3546 "TARGET_SIMD"
3547 {
3548 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[4])));
3549 return
3550 "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
3551 }
3552 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3553 )
3555 ;; vqdml[sa]l_n
3557 (define_insn "aarch64_sqdml<SBINQOPS:as>l_n<mode>"
3558 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3559 (SBINQOPS:<VWIDE>
3560 (match_operand:<VWIDE> 1 "register_operand" "0")
3561 (ss_ashift:<VWIDE>
3562 (mult:<VWIDE>
3563 (sign_extend:<VWIDE>
3564 (match_operand:VD_HSI 2 "register_operand" "w"))
3565 (sign_extend:<VWIDE>
3566 (vec_duplicate:VD_HSI
3567 (match_operand:<VEL> 3 "register_operand" "<vwx>"))))
3568 (const_int 1))))]
3569 "TARGET_SIMD"
3570 "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[0]"
3571 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3572 )
3574 ;; sqdml[as]l2
3576 (define_insn "aarch64_sqdml<SBINQOPS:as>l2<mode>_internal"
3577 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3578 (SBINQOPS:<VWIDE>
3579 (match_operand:<VWIDE> 1 "register_operand" "0")
3580 (ss_ashift:<VWIDE>
3581 (mult:<VWIDE>
3582 (sign_extend:<VWIDE>
3583 (vec_select:<VHALF>
3584 (match_operand:VQ_HSI 2 "register_operand" "w")
3585 (match_operand:VQ_HSI 4 "vect_par_cnst_hi_half" "")))
3586 (sign_extend:<VWIDE>
3587 (vec_select:<VHALF>
3588 (match_operand:VQ_HSI 3 "register_operand" "w")
3589 (match_dup 4))))
3590 (const_int 1))))]
3591 "TARGET_SIMD"
3592 "sqdml<SBINQOPS:as>l2\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %<v>3<Vmtype>"
3593 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3594 )
3596 (define_expand "aarch64_sqdmlal2<mode>"
3597 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3598 (match_operand:<VWIDE> 1 "register_operand" "w")
3599 (match_operand:VQ_HSI 2 "register_operand" "w")
3600 (match_operand:VQ_HSI 3 "register_operand" "w")]
3601 "TARGET_SIMD"
3602 {
3603 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3604 emit_insn (gen_aarch64_sqdmlal2<mode>_internal (operands[0], operands[1],
3605 operands[2], operands[3], p));
3606 DONE;
3607 })
3609 (define_expand "aarch64_sqdmlsl2<mode>"
3610 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3611 (match_operand:<VWIDE> 1 "register_operand" "w")
3612 (match_operand:VQ_HSI 2 "register_operand" "w")
3613 (match_operand:VQ_HSI 3 "register_operand" "w")]
3614 "TARGET_SIMD"
3615 {
3616 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3617 emit_insn (gen_aarch64_sqdmlsl2<mode>_internal (operands[0], operands[1],
3618 operands[2], operands[3], p));
3619 DONE;
3620 })
3622 ;; vqdml[sa]l2_lane
3624 (define_insn "aarch64_sqdml<SBINQOPS:as>l2_lane<mode>_internal"
3625 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3626 (SBINQOPS:<VWIDE>
3627 (match_operand:<VWIDE> 1 "register_operand" "0")
3628 (ss_ashift:<VWIDE>
3629 (mult:<VWIDE>
3630 (sign_extend:<VWIDE>
3631 (vec_select:<VHALF>
3632 (match_operand:VQ_HSI 2 "register_operand" "w")
3633 (match_operand:VQ_HSI 5 "vect_par_cnst_hi_half" "")))
3634 (sign_extend:<VWIDE>
3635 (vec_duplicate:<VHALF>
3636 (vec_select:<VEL>
3637 (match_operand:<VCOND> 3 "register_operand" "<vwx>")
3638 (parallel [(match_operand:SI 4 "immediate_operand" "i")])
3639 ))))
3640 (const_int 1))))]
3641 "TARGET_SIMD"
3642 {
3643 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[4])));
3644 return
3645 "sqdml<SBINQOPS:as>l2\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
3646 }
3647 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3648 )
3650 (define_insn "aarch64_sqdml<SBINQOPS:as>l2_laneq<mode>_internal"
3651 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3652 (SBINQOPS:<VWIDE>
3653 (match_operand:<VWIDE> 1 "register_operand" "0")
3654 (ss_ashift:<VWIDE>
3655 (mult:<VWIDE>
3656 (sign_extend:<VWIDE>
3657 (vec_select:<VHALF>
3658 (match_operand:VQ_HSI 2 "register_operand" "w")
3659 (match_operand:VQ_HSI 5 "vect_par_cnst_hi_half" "")))
3660 (sign_extend:<VWIDE>
3661 (vec_duplicate:<VHALF>
3662 (vec_select:<VEL>
3663 (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
3664 (parallel [(match_operand:SI 4 "immediate_operand" "i")])
3665 ))))
3666 (const_int 1))))]
3667 "TARGET_SIMD"
3668 {
3669 operands[4] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[4])));
3670 return
3671 "sqdml<SBINQOPS:as>l2\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]";
3672 }
3673 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3674 )
3676 (define_expand "aarch64_sqdmlal2_lane<mode>"
3677 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3678 (match_operand:<VWIDE> 1 "register_operand" "w")
3679 (match_operand:VQ_HSI 2 "register_operand" "w")
3680 (match_operand:<VCOND> 3 "register_operand" "<vwx>")
3681 (match_operand:SI 4 "immediate_operand" "i")]
3682 "TARGET_SIMD"
3683 {
3684 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3685 emit_insn (gen_aarch64_sqdmlal2_lane<mode>_internal (operands[0], operands[1],
3686 operands[2], operands[3],
3687 operands[4], p));
3688 DONE;
3689 })
3691 (define_expand "aarch64_sqdmlal2_laneq<mode>"
3692 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3693 (match_operand:<VWIDE> 1 "register_operand" "w")
3694 (match_operand:VQ_HSI 2 "register_operand" "w")
3695 (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
3696 (match_operand:SI 4 "immediate_operand" "i")]
3697 "TARGET_SIMD"
3698 {
3699 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3700 emit_insn (gen_aarch64_sqdmlal2_laneq<mode>_internal (operands[0], operands[1],
3701 operands[2], operands[3],
3702 operands[4], p));
3703 DONE;
3704 })
3706 (define_expand "aarch64_sqdmlsl2_lane<mode>"
3707 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3708 (match_operand:<VWIDE> 1 "register_operand" "w")
3709 (match_operand:VQ_HSI 2 "register_operand" "w")
3710 (match_operand:<VCOND> 3 "register_operand" "<vwx>")
3711 (match_operand:SI 4 "immediate_operand" "i")]
3712 "TARGET_SIMD"
3713 {
3714 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3715 emit_insn (gen_aarch64_sqdmlsl2_lane<mode>_internal (operands[0], operands[1],
3716 operands[2], operands[3],
3717 operands[4], p));
3718 DONE;
3719 })
3721 (define_expand "aarch64_sqdmlsl2_laneq<mode>"
3722 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3723 (match_operand:<VWIDE> 1 "register_operand" "w")
3724 (match_operand:VQ_HSI 2 "register_operand" "w")
3725 (match_operand:<VCONQ> 3 "register_operand" "<vwx>")
3726 (match_operand:SI 4 "immediate_operand" "i")]
3727 "TARGET_SIMD"
3728 {
3729 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3730 emit_insn (gen_aarch64_sqdmlsl2_laneq<mode>_internal (operands[0], operands[1],
3731 operands[2], operands[3],
3732 operands[4], p));
3733 DONE;
3734 })
3736 (define_insn "aarch64_sqdml<SBINQOPS:as>l2_n<mode>_internal"
3737 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3738 (SBINQOPS:<VWIDE>
3739 (match_operand:<VWIDE> 1 "register_operand" "0")
3740 (ss_ashift:<VWIDE>
3741 (mult:<VWIDE>
3742 (sign_extend:<VWIDE>
3743 (vec_select:<VHALF>
3744 (match_operand:VQ_HSI 2 "register_operand" "w")
3745 (match_operand:VQ_HSI 4 "vect_par_cnst_hi_half" "")))
3746 (sign_extend:<VWIDE>
3747 (vec_duplicate:<VHALF>
3748 (match_operand:<VEL> 3 "register_operand" "<vwx>"))))
3749 (const_int 1))))]
3750 "TARGET_SIMD"
3751 "sqdml<SBINQOPS:as>l2\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[0]"
3752 [(set_attr "type" "neon_sat_mla_<Vetype>_scalar_long")]
3753 )
3755 (define_expand "aarch64_sqdmlal2_n<mode>"
3756 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3757 (match_operand:<VWIDE> 1 "register_operand" "w")
3758 (match_operand:VQ_HSI 2 "register_operand" "w")
3759 (match_operand:<VEL> 3 "register_operand" "w")]
3760 "TARGET_SIMD"
3761 {
3762 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3763 emit_insn (gen_aarch64_sqdmlal2_n<mode>_internal (operands[0], operands[1],
3764 operands[2], operands[3],
3765 p));
3766 DONE;
3767 })
3769 (define_expand "aarch64_sqdmlsl2_n<mode>"
3770 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3771 (match_operand:<VWIDE> 1 "register_operand" "w")
3772 (match_operand:VQ_HSI 2 "register_operand" "w")
3773 (match_operand:<VEL> 3 "register_operand" "w")]
3774 "TARGET_SIMD"
3775 {
3776 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3777 emit_insn (gen_aarch64_sqdmlsl2_n<mode>_internal (operands[0], operands[1],
3778 operands[2], operands[3],
3779 p));
3780 DONE;
3781 })
3783 ;; vqdmull
3785 (define_insn "aarch64_sqdmull<mode>"
3786 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3787 (ss_ashift:<VWIDE>
3788 (mult:<VWIDE>
3789 (sign_extend:<VWIDE>
3790 (match_operand:VSD_HSI 1 "register_operand" "w"))
3791 (sign_extend:<VWIDE>
3792 (match_operand:VSD_HSI 2 "register_operand" "w")))
3793 (const_int 1)))]
3794 "TARGET_SIMD"
3795 "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
3796 [(set_attr "type" "neon_sat_mul_<Vetype>_long")]
3797 )
3799 ;; vqdmull_lane
3801 (define_insn "aarch64_sqdmull_lane<mode>"
3802 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3803 (ss_ashift:<VWIDE>
3804 (mult:<VWIDE>
3805 (sign_extend:<VWIDE>
3806 (match_operand:VD_HSI 1 "register_operand" "w"))
3807 (sign_extend:<VWIDE>
3808 (vec_duplicate:VD_HSI
3809 (vec_select:<VEL>
3810 (match_operand:<VCOND> 2 "register_operand" "<vwx>")
3811 (parallel [(match_operand:SI 3 "immediate_operand" "i")])))
3812 ))
3813 (const_int 1)))]
3814 "TARGET_SIMD"
3815 {
3816 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
3817 return "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3818 }
3819 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3820 )
3822 (define_insn "aarch64_sqdmull_laneq<mode>"
3823 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3824 (ss_ashift:<VWIDE>
3825 (mult:<VWIDE>
3826 (sign_extend:<VWIDE>
3827 (match_operand:VD_HSI 1 "register_operand" "w"))
3828 (sign_extend:<VWIDE>
3829 (vec_duplicate:VD_HSI
3830 (vec_select:<VEL>
3831 (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
3832 (parallel [(match_operand:SI 3 "immediate_operand" "i")])))
3833 ))
3834 (const_int 1)))]
3835 "TARGET_SIMD"
3836 {
3837 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[3])));
3838 return "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3839 }
3840 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3841 )
3843 (define_insn "aarch64_sqdmull_lane<mode>"
3844 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3845 (ss_ashift:<VWIDE>
3846 (mult:<VWIDE>
3847 (sign_extend:<VWIDE>
3848 (match_operand:SD_HSI 1 "register_operand" "w"))
3849 (sign_extend:<VWIDE>
3850 (vec_select:<VEL>
3851 (match_operand:<VCOND> 2 "register_operand" "<vwx>")
3852 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))
3853 ))
3854 (const_int 1)))]
3855 "TARGET_SIMD"
3856 {
3857 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
3858 return "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3859 }
3860 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3861 )
3863 (define_insn "aarch64_sqdmull_laneq<mode>"
3864 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3865 (ss_ashift:<VWIDE>
3866 (mult:<VWIDE>
3867 (sign_extend:<VWIDE>
3868 (match_operand:SD_HSI 1 "register_operand" "w"))
3869 (sign_extend:<VWIDE>
3870 (vec_select:<VEL>
3871 (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
3872 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))
3873 ))
3874 (const_int 1)))]
3875 "TARGET_SIMD"
3876 {
3877 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[3])));
3878 return "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3879 }
3880 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3881 )
3883 ;; vqdmull_n
3885 (define_insn "aarch64_sqdmull_n<mode>"
3886 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3887 (ss_ashift:<VWIDE>
3888 (mult:<VWIDE>
3889 (sign_extend:<VWIDE>
3890 (match_operand:VD_HSI 1 "register_operand" "w"))
3891 (sign_extend:<VWIDE>
3892 (vec_duplicate:VD_HSI
3893 (match_operand:<VEL> 2 "register_operand" "<vwx>")))
3894 )
3895 (const_int 1)))]
3896 "TARGET_SIMD"
3897 "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[0]"
3898 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3899 )
3901 ;; vqdmull2
3905 (define_insn "aarch64_sqdmull2<mode>_internal"
3906 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3907 (ss_ashift:<VWIDE>
3908 (mult:<VWIDE>
3909 (sign_extend:<VWIDE>
3910 (vec_select:<VHALF>
3911 (match_operand:VQ_HSI 1 "register_operand" "w")
3912 (match_operand:VQ_HSI 3 "vect_par_cnst_hi_half" "")))
3913 (sign_extend:<VWIDE>
3914 (vec_select:<VHALF>
3915 (match_operand:VQ_HSI 2 "register_operand" "w")
3916 (match_dup 3)))
3917 )
3918 (const_int 1)))]
3919 "TARGET_SIMD"
3920 "sqdmull2\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
3921 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3922 )
3924 (define_expand "aarch64_sqdmull2<mode>"
3925 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3926 (match_operand:VQ_HSI 1 "register_operand" "w")
3927 (match_operand:VQ_HSI 2 "register_operand" "w")]
3928 "TARGET_SIMD"
3929 {
3930 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3931 emit_insn (gen_aarch64_sqdmull2<mode>_internal (operands[0], operands[1],
3932 operands[2], p));
3933 DONE;
3934 })
3936 ;; vqdmull2_lane
3938 (define_insn "aarch64_sqdmull2_lane<mode>_internal"
3939 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3940 (ss_ashift:<VWIDE>
3941 (mult:<VWIDE>
3942 (sign_extend:<VWIDE>
3943 (vec_select:<VHALF>
3944 (match_operand:VQ_HSI 1 "register_operand" "w")
3945 (match_operand:VQ_HSI 4 "vect_par_cnst_hi_half" "")))
3946 (sign_extend:<VWIDE>
3947 (vec_duplicate:<VHALF>
3948 (vec_select:<VEL>
3949 (match_operand:<VCOND> 2 "register_operand" "<vwx>")
3950 (parallel [(match_operand:SI 3 "immediate_operand" "i")])))
3951 ))
3952 (const_int 1)))]
3953 "TARGET_SIMD"
3954 {
3955 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCOND>mode, INTVAL (operands[3])));
3956 return "sqdmull2\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3957 }
3958 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3959 )
3961 (define_insn "aarch64_sqdmull2_laneq<mode>_internal"
3962 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3963 (ss_ashift:<VWIDE>
3964 (mult:<VWIDE>
3965 (sign_extend:<VWIDE>
3966 (vec_select:<VHALF>
3967 (match_operand:VQ_HSI 1 "register_operand" "w")
3968 (match_operand:VQ_HSI 4 "vect_par_cnst_hi_half" "")))
3969 (sign_extend:<VWIDE>
3970 (vec_duplicate:<VHALF>
3971 (vec_select:<VEL>
3972 (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
3973 (parallel [(match_operand:SI 3 "immediate_operand" "i")])))
3974 ))
3975 (const_int 1)))]
3976 "TARGET_SIMD"
3977 {
3978 operands[3] = GEN_INT (ENDIAN_LANE_N (<VCONQ>mode, INTVAL (operands[3])));
3979 return "sqdmull2\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]";
3980 }
3981 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
3982 )
3984 (define_expand "aarch64_sqdmull2_lane<mode>"
3985 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3986 (match_operand:VQ_HSI 1 "register_operand" "w")
3987 (match_operand:<VCOND> 2 "register_operand" "<vwx>")
3988 (match_operand:SI 3 "immediate_operand" "i")]
3989 "TARGET_SIMD"
3990 {
3991 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3992 emit_insn (gen_aarch64_sqdmull2_lane<mode>_internal (operands[0], operands[1],
3993 operands[2], operands[3],
3994 p));
3995 DONE;
3996 })
3998 (define_expand "aarch64_sqdmull2_laneq<mode>"
3999 [(match_operand:<VWIDE> 0 "register_operand" "=w")
4000 (match_operand:VQ_HSI 1 "register_operand" "w")
4001 (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
4002 (match_operand:SI 3 "immediate_operand" "i")]
4003 "TARGET_SIMD"
4004 {
4005 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
4006 emit_insn (gen_aarch64_sqdmull2_laneq<mode>_internal (operands[0], operands[1],
4007 operands[2], operands[3],
4008 p));
4009 DONE;
4010 })
4012 ;; vqdmull2_n
4014 (define_insn "aarch64_sqdmull2_n<mode>_internal"
4015 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
4016 (ss_ashift:<VWIDE>
4017 (mult:<VWIDE>
4018 (sign_extend:<VWIDE>
4019 (vec_select:<VHALF>
4020 (match_operand:VQ_HSI 1 "register_operand" "w")
4021 (match_operand:VQ_HSI 3 "vect_par_cnst_hi_half" "")))
4022 (sign_extend:<VWIDE>
4023 (vec_duplicate:<VHALF>
4024 (match_operand:<VEL> 2 "register_operand" "<vwx>")))
4025 )
4026 (const_int 1)))]
4027 "TARGET_SIMD"
4028 "sqdmull2\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[0]"
4029 [(set_attr "type" "neon_sat_mul_<Vetype>_scalar_long")]
4030 )
4032 (define_expand "aarch64_sqdmull2_n<mode>"
4033 [(match_operand:<VWIDE> 0 "register_operand" "=w")
4034 (match_operand:VQ_HSI 1 "register_operand" "w")
4035 (match_operand:<VEL> 2 "register_operand" "w")]
4036 "TARGET_SIMD"
4037 {
4038 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
4039 emit_insn (gen_aarch64_sqdmull2_n<mode>_internal (operands[0], operands[1],
4040 operands[2], p));
4041 DONE;
4042 })
4044 ;; vshl
4046 (define_insn "aarch64_<sur>shl<mode>"
4047 [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
4048 (unspec:VSDQ_I_DI
4049 [(match_operand:VSDQ_I_DI 1 "register_operand" "w")
4050 (match_operand:VSDQ_I_DI 2 "register_operand" "w")]
4051 VSHL))]
4052 "TARGET_SIMD"
4053 "<sur>shl\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>";
4054 [(set_attr "type" "neon_shift_reg<q>")]
4055 )
4058 ;; vqshl
4060 (define_insn "aarch64_<sur>q<r>shl<mode>"
4061 [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
4062 (unspec:VSDQ_I
4063 [(match_operand:VSDQ_I 1 "register_operand" "w")
4064 (match_operand:VSDQ_I 2 "register_operand" "w")]
4065 VQSHL))]
4066 "TARGET_SIMD"
4067 "<sur>q<r>shl\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>";
4068 [(set_attr "type" "neon_sat_shift_reg<q>")]
4069 )
4071 ;; vshll_n
4073 (define_insn "aarch64_<sur>shll_n<mode>"
4074 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
4075 (unspec:<VWIDE> [(match_operand:VD_BHSI 1 "register_operand" "w")
4076 (match_operand:SI 2
4077 "aarch64_simd_shift_imm_bitsize_<ve_mode>" "i")]
4078 VSHLL))]
4079 "TARGET_SIMD"
4080 {
4081 if (INTVAL (operands[2]) == GET_MODE_UNIT_BITSIZE (<MODE>mode))
4082 return "shll\\t%0.<Vwtype>, %1.<Vtype>, %2";
4083 else
4084 return "<sur>shll\\t%0.<Vwtype>, %1.<Vtype>, %2";
4085 }
4086 [(set_attr "type" "neon_shift_imm_long")]
4087 )
4089 ;; vshll_high_n
4091 (define_insn "aarch64_<sur>shll2_n<mode>"
4092 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
4093 (unspec:<VWIDE> [(match_operand:VQW 1 "register_operand" "w")
4094 (match_operand:SI 2 "immediate_operand" "i")]
4095 VSHLL))]
4096 "TARGET_SIMD"
4097 {
4098 if (INTVAL (operands[2]) == GET_MODE_UNIT_BITSIZE (<MODE>mode))
4099 return "shll2\\t%0.<Vwtype>, %1.<Vtype>, %2";
4100 else
4101 return "<sur>shll2\\t%0.<Vwtype>, %1.<Vtype>, %2";
4102 }
4103 [(set_attr "type" "neon_shift_imm_long")]
4104 )
4106 ;; vrshr_n
4108 (define_insn "aarch64_<sur>shr_n<mode>"
4109 [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
4110 (unspec:VSDQ_I_DI [(match_operand:VSDQ_I_DI 1 "register_operand" "w")
4111 (match_operand:SI 2
4112 "aarch64_simd_shift_imm_offset_<ve_mode>" "i")]
4113 VRSHR_N))]
4114 "TARGET_SIMD"
4115 "<sur>shr\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %2"
4116 [(set_attr "type" "neon_sat_shift_imm<q>")]
4117 )
4119 ;; v(r)sra_n
4121 (define_insn "aarch64_<sur>sra_n<mode>"
4122 [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
4123 (unspec:VSDQ_I_DI [(match_operand:VSDQ_I_DI 1 "register_operand" "0")
4124 (match_operand:VSDQ_I_DI 2 "register_operand" "w")
4125 (match_operand:SI 3
4126 "aarch64_simd_shift_imm_offset_<ve_mode>" "i")]
4127 VSRA))]
4128 "TARGET_SIMD"
4129 "<sur>sra\\t%<v>0<Vmtype>, %<v>2<Vmtype>, %3"
4130 [(set_attr "type" "neon_shift_acc<q>")]
4131 )
4133 ;; vs<lr>i_n
4135 (define_insn "aarch64_<sur>s<lr>i_n<mode>"
4136 [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
4137 (unspec:VSDQ_I_DI [(match_operand:VSDQ_I_DI 1 "register_operand" "0")
4138 (match_operand:VSDQ_I_DI 2 "register_operand" "w")
4139 (match_operand:SI 3
4140 "aarch64_simd_shift_imm_<offsetlr><ve_mode>" "i")]
4141 VSLRI))]
4142 "TARGET_SIMD"
4143 "s<lr>i\\t%<v>0<Vmtype>, %<v>2<Vmtype>, %3"
4144 [(set_attr "type" "neon_shift_imm<q>")]
4145 )
4147 ;; vqshl(u)
4149 (define_insn "aarch64_<sur>qshl<u>_n<mode>"
4150 [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
4151 (unspec:VSDQ_I [(match_operand:VSDQ_I 1 "register_operand" "w")
4152 (match_operand:SI 2
4153 "aarch64_simd_shift_imm_<ve_mode>" "i")]
4154 VQSHL_N))]
4155 "TARGET_SIMD"
4156 "<sur>qshl<u>\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %2"
4157 [(set_attr "type" "neon_sat_shift_imm<q>")]
4158 )
4161 ;; vq(r)shr(u)n_n
4163 (define_insn "aarch64_<sur>q<r>shr<u>n_n<mode>"
4164 [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
4165 (unspec:<VNARROWQ> [(match_operand:VSQN_HSDI 1 "register_operand" "w")
4166 (match_operand:SI 2
4167 "aarch64_simd_shift_imm_offset_<ve_mode>" "i")]
4168 VQSHRN_N))]
4169 "TARGET_SIMD"
4170 "<sur>q<r>shr<u>n\\t%<vn2>0<Vmntype>, %<v>1<Vmtype>, %2"
4171 [(set_attr "type" "neon_sat_shift_imm_narrow_q")]
4172 )
4175 ;; cm(eq|ge|gt|lt|le)
4176 ;; Note, we have constraints for Dz and Z as different expanders
4177 ;; have different ideas of what should be passed to this pattern.
4179 (define_insn "aarch64_cm<optab><mode>"
4180 [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w,w")
4181 (neg:<V_cmp_result>
4182 (COMPARISONS:<V_cmp_result>
4183 (match_operand:VDQ_I 1 "register_operand" "w,w")
4184 (match_operand:VDQ_I 2 "aarch64_simd_reg_or_zero" "w,ZDz")
4185 )))]
4186 "TARGET_SIMD"
4187 "@
4188 cm<n_optab>\t%<v>0<Vmtype>, %<v><cmp_1><Vmtype>, %<v><cmp_2><Vmtype>
4189 cm<optab>\t%<v>0<Vmtype>, %<v>1<Vmtype>, #0"
4190 [(set_attr "type" "neon_compare<q>, neon_compare_zero<q>")]
4191 )
4193 (define_insn_and_split "aarch64_cm<optab>di"
4194 [(set (match_operand:DI 0 "register_operand" "=w,w,r")
4195 (neg:DI
4196 (COMPARISONS:DI
4197 (match_operand:DI 1 "register_operand" "w,w,r")
4198 (match_operand:DI 2 "aarch64_simd_reg_or_zero" "w,ZDz,r")
4199 )))
4200 (clobber (reg:CC CC_REGNUM))]
4201 "TARGET_SIMD"
4202 "#"
4203 "reload_completed"
4204 [(set (match_operand:DI 0 "register_operand")
4205 (neg:DI
4206 (COMPARISONS:DI
4207 (match_operand:DI 1 "register_operand")
4208 (match_operand:DI 2 "aarch64_simd_reg_or_zero")
4209 )))]
4210 {
4211 /* If we are in the general purpose register file,
4212 we split to a sequence of comparison and store. */
4213 if (GP_REGNUM_P (REGNO (operands[0]))
4214 && GP_REGNUM_P (REGNO (operands[1])))
4215 {
4216 machine_mode mode = SELECT_CC_MODE (<CMP>, operands[1], operands[2]);
4217 rtx cc_reg = aarch64_gen_compare_reg (<CMP>, operands[1], operands[2]);
4218 rtx comparison = gen_rtx_<CMP> (mode, operands[1], operands[2]);
4219 emit_insn (gen_cstoredi_neg (operands[0], comparison, cc_reg));
4220 DONE;
4221 }
4222 /* Otherwise, we expand to a similar pattern which does not
4223 clobber CC_REGNUM. */
4224 }
4225 [(set_attr "type" "neon_compare, neon_compare_zero, multiple")]
4226 )
4228 (define_insn "*aarch64_cm<optab>di"
4229 [(set (match_operand:DI 0 "register_operand" "=w,w")
4230 (neg:DI
4231 (COMPARISONS:DI
4232 (match_operand:DI 1 "register_operand" "w,w")
4233 (match_operand:DI 2 "aarch64_simd_reg_or_zero" "w,ZDz")
4234 )))]
4235 "TARGET_SIMD && reload_completed"
4236 "@
4237 cm<n_optab>\t%d0, %d<cmp_1>, %d<cmp_2>
4238 cm<optab>\t%d0, %d1, #0"
4239 [(set_attr "type" "neon_compare, neon_compare_zero")]
4240 )
4242 ;; cm(hs|hi)
4244 (define_insn "aarch64_cm<optab><mode>"
4245 [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w")
4246 (neg:<V_cmp_result>
4247 (UCOMPARISONS:<V_cmp_result>
4248 (match_operand:VDQ_I 1 "register_operand" "w")
4249 (match_operand:VDQ_I 2 "register_operand" "w")
4250 )))]
4251 "TARGET_SIMD"
4252 "cm<n_optab>\t%<v>0<Vmtype>, %<v><cmp_1><Vmtype>, %<v><cmp_2><Vmtype>"
4253 [(set_attr "type" "neon_compare<q>")]
4254 )
4256 (define_insn_and_split "aarch64_cm<optab>di"
4257 [(set (match_operand:DI 0 "register_operand" "=w,r")
4258 (neg:DI
4259 (UCOMPARISONS:DI
4260 (match_operand:DI 1 "register_operand" "w,r")
4261 (match_operand:DI 2 "aarch64_simd_reg_or_zero" "w,r")
4262 )))
4263 (clobber (reg:CC CC_REGNUM))]
4264 "TARGET_SIMD"
4265 "#"
4266 "reload_completed"
4267 [(set (match_operand:DI 0 "register_operand")
4268 (neg:DI
4269 (UCOMPARISONS:DI
4270 (match_operand:DI 1 "register_operand")
4271 (match_operand:DI 2 "aarch64_simd_reg_or_zero")
4272 )))]
4273 {
4274 /* If we are in the general purpose register file,
4275 we split to a sequence of comparison and store. */
4276 if (GP_REGNUM_P (REGNO (operands[0]))
4277 && GP_REGNUM_P (REGNO (operands[1])))
4278 {
4279 machine_mode mode = CCmode;
4280 rtx cc_reg = aarch64_gen_compare_reg (<CMP>, operands[1], operands[2]);
4281 rtx comparison = gen_rtx_<CMP> (mode, operands[1], operands[2]);
4282 emit_insn (gen_cstoredi_neg (operands[0], comparison, cc_reg));
4283 DONE;
4284 }
4285 /* Otherwise, we expand to a similar pattern which does not
4286 clobber CC_REGNUM. */
4287 }
4288 [(set_attr "type" "neon_compare,multiple")]
4289 )
4291 (define_insn "*aarch64_cm<optab>di"
4292 [(set (match_operand:DI 0 "register_operand" "=w")
4293 (neg:DI
4294 (UCOMPARISONS:DI
4295 (match_operand:DI 1 "register_operand" "w")
4296 (match_operand:DI 2 "aarch64_simd_reg_or_zero" "w")
4297 )))]
4298 "TARGET_SIMD && reload_completed"
4299 "cm<n_optab>\t%d0, %d<cmp_1>, %d<cmp_2>"
4300 [(set_attr "type" "neon_compare")]
4301 )
4303 ;; cmtst
4305 ;; Although neg (ne (and x y) 0) is the natural way of expressing a cmtst,
4306 ;; we don't have any insns using ne, and aarch64_vcond outputs
4307 ;; not (neg (eq (and x y) 0))
4308 ;; which is rewritten by simplify_rtx as
4309 ;; plus (eq (and x y) 0) -1.
4311 (define_insn "aarch64_cmtst<mode>"
4312 [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w")
4313 (plus:<V_cmp_result>
4314 (eq:<V_cmp_result>
4315 (and:VDQ_I
4316 (match_operand:VDQ_I 1 "register_operand" "w")
4317 (match_operand:VDQ_I 2 "register_operand" "w"))
4318 (match_operand:VDQ_I 3 "aarch64_simd_imm_zero"))
4319 (match_operand:<V_cmp_result> 4 "aarch64_simd_imm_minus_one")))
4320 ]
4321 "TARGET_SIMD"
4322 "cmtst\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
4323 [(set_attr "type" "neon_tst<q>")]
4324 )
4326 (define_insn_and_split "aarch64_cmtstdi"
4327 [(set (match_operand:DI 0 "register_operand" "=w,r")
4328 (neg:DI
4329 (ne:DI
4330 (and:DI
4331 (match_operand:DI 1 "register_operand" "w,r")
4332 (match_operand:DI 2 "register_operand" "w,r"))
4333 (const_int 0))))
4334 (clobber (reg:CC CC_REGNUM))]
4335 "TARGET_SIMD"
4336 "#"
4337 "reload_completed"
4338 [(set (match_operand:DI 0 "register_operand")
4339 (neg:DI
4340 (ne:DI
4341 (and:DI
4342 (match_operand:DI 1 "register_operand")
4343 (match_operand:DI 2 "register_operand"))
4344 (const_int 0))))]
4345 {
4346 /* If we are in the general purpose register file,
4347 we split to a sequence of comparison and store. */
4348 if (GP_REGNUM_P (REGNO (operands[0]))
4349 && GP_REGNUM_P (REGNO (operands[1])))
4350 {
4351 rtx and_tree = gen_rtx_AND (DImode, operands[1], operands[2]);
4352 machine_mode mode = SELECT_CC_MODE (NE, and_tree, const0_rtx);
4353 rtx cc_reg = aarch64_gen_compare_reg (NE, and_tree, const0_rtx);
4354 rtx comparison = gen_rtx_NE (mode, and_tree, const0_rtx);
4355 emit_insn (gen_cstoredi_neg (operands[0], comparison, cc_reg));
4356 DONE;
4357 }
4358 /* Otherwise, we expand to a similar pattern which does not
4359 clobber CC_REGNUM. */
4360 }
4361 [(set_attr "type" "neon_tst,multiple")]
4362 )
4364 (define_insn "*aarch64_cmtstdi"
4365 [(set (match_operand:DI 0 "register_operand" "=w")
4366 (neg:DI
4367 (ne:DI
4368 (and:DI
4369 (match_operand:DI 1 "register_operand" "w")
4370 (match_operand:DI 2 "register_operand" "w"))
4371 (const_int 0))))]
4372 "TARGET_SIMD"
4373 "cmtst\t%d0, %d1, %d2"
4374 [(set_attr "type" "neon_tst")]
4375 )
4377 ;; fcm(eq|ge|gt|le|lt)
4379 (define_insn "aarch64_cm<optab><mode>"
4380 [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w,w")
4381 (neg:<V_cmp_result>
4382 (COMPARISONS:<V_cmp_result>
4383 (match_operand:VHSDF_HSDF 1 "register_operand" "w,w")
4384 (match_operand:VHSDF_HSDF 2 "aarch64_simd_reg_or_zero" "w,YDz")
4385 )))]
4386 "TARGET_SIMD"
4387 "@
4388 fcm<n_optab>\t%<v>0<Vmtype>, %<v><cmp_1><Vmtype>, %<v><cmp_2><Vmtype>
4389 fcm<optab>\t%<v>0<Vmtype>, %<v>1<Vmtype>, 0"
4390 [(set_attr "type" "neon_fp_compare_<stype><q>")]
4391 )
4393 ;; fac(ge|gt)
4394 ;; Note we can also handle what would be fac(le|lt) by
4395 ;; generating fac(ge|gt).
4397 (define_insn "aarch64_fac<optab><mode>"
4398 [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w")
4399 (neg:<V_cmp_result>
4400 (FAC_COMPARISONS:<V_cmp_result>
4401 (abs:VHSDF_HSDF
4402 (match_operand:VHSDF_HSDF 1 "register_operand" "w"))
4403 (abs:VHSDF_HSDF
4404 (match_operand:VHSDF_HSDF 2 "register_operand" "w"))
4405 )))]
4406 "TARGET_SIMD"
4407 "fac<n_optab>\t%<v>0<Vmtype>, %<v><cmp_1><Vmtype>, %<v><cmp_2><Vmtype>"
4408 [(set_attr "type" "neon_fp_compare_<stype><q>")]
4409 )
4411 ;; addp
4413 (define_insn "aarch64_addp<mode>"
4414 [(set (match_operand:VD_BHSI 0 "register_operand" "=w")
4415 (unspec:VD_BHSI
4416 [(match_operand:VD_BHSI 1 "register_operand" "w")
4417 (match_operand:VD_BHSI 2 "register_operand" "w")]
4418 UNSPEC_ADDP))]
4419 "TARGET_SIMD"
4420 "addp\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
4421 [(set_attr "type" "neon_reduc_add<q>")]
4422 )
4424 (define_insn "aarch64_addpdi"
4425 [(set (match_operand:DI 0 "register_operand" "=w")
4426 (unspec:DI
4427 [(match_operand:V2DI 1 "register_operand" "w")]
4428 UNSPEC_ADDP))]
4429 "TARGET_SIMD"
4430 "addp\t%d0, %1.2d"
4431 [(set_attr "type" "neon_reduc_add")]
4432 )
4434 ;; sqrt
4436 (define_expand "sqrt<mode>2"
4437 [(set (match_operand:VHSDF 0 "register_operand" "=w")
4438 (sqrt:VHSDF (match_operand:VHSDF 1 "register_operand" "w")))]
4439 "TARGET_SIMD"
4440 {
4441 if (aarch64_emit_approx_sqrt (operands[0], operands[1], false))
4442 DONE;
4443 })
4445 (define_insn "*sqrt<mode>2"
4446 [(set (match_operand:VHSDF 0 "register_operand" "=w")
4447 (sqrt:VHSDF (match_operand:VHSDF 1 "register_operand" "w")))]
4448 "TARGET_SIMD"
4449 "fsqrt\\t%0.<Vtype>, %1.<Vtype>"
4450 [(set_attr "type" "neon_fp_sqrt_<stype><q>")]
4451 )
4453 ;; Patterns for vector struct loads and stores.
4455 (define_insn "aarch64_simd_ld2<mode>"
4456 [(set (match_operand:OI 0 "register_operand" "=w")
4457 (unspec:OI [(match_operand:OI 1 "aarch64_simd_struct_operand" "Utv")
4458 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4459 UNSPEC_LD2))]
4460 "TARGET_SIMD"
4461 "ld2\\t{%S0.<Vtype> - %T0.<Vtype>}, %1"
4462 [(set_attr "type" "neon_load2_2reg<q>")]
4463 )
4465 (define_insn "aarch64_simd_ld2r<mode>"
4466 [(set (match_operand:OI 0 "register_operand" "=w")
4467 (unspec:OI [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")
4468 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY) ]
4469 UNSPEC_LD2_DUP))]
4470 "TARGET_SIMD"
4471 "ld2r\\t{%S0.<Vtype> - %T0.<Vtype>}, %1"
4472 [(set_attr "type" "neon_load2_all_lanes<q>")]
4473 )
4475 (define_insn "aarch64_vec_load_lanesoi_lane<mode>"
4476 [(set (match_operand:OI 0 "register_operand" "=w")
4477 (unspec:OI [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")
4478 (match_operand:OI 2 "register_operand" "0")
4479 (match_operand:SI 3 "immediate_operand" "i")
4480 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY) ]
4481 UNSPEC_LD2_LANE))]
4482 "TARGET_SIMD"
4483 {
4484 operands[3] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[3])));
4485 return "ld2\\t{%S0.<Vetype> - %T0.<Vetype>}[%3], %1";
4486 }
4487 [(set_attr "type" "neon_load2_one_lane")]
4488 )
4490 (define_expand "vec_load_lanesoi<mode>"
4491 [(set (match_operand:OI 0 "register_operand" "=w")
4492 (unspec:OI [(match_operand:OI 1 "aarch64_simd_struct_operand" "Utv")
4493 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4494 UNSPEC_LD2))]
4495 "TARGET_SIMD"
4496 {
4497 if (BYTES_BIG_ENDIAN)
4498 {
4499 rtx tmp = gen_reg_rtx (OImode);
4500 rtx mask = aarch64_reverse_mask (<MODE>mode);
4501 emit_insn (gen_aarch64_simd_ld2<mode> (tmp, operands[1]));
4502 emit_insn (gen_aarch64_rev_reglistoi (operands[0], tmp, mask));
4503 }
4504 else
4505 emit_insn (gen_aarch64_simd_ld2<mode> (operands[0], operands[1]));
4506 DONE;
4507 })
4509 (define_insn "aarch64_simd_st2<mode>"
4510 [(set (match_operand:OI 0 "aarch64_simd_struct_operand" "=Utv")
4511 (unspec:OI [(match_operand:OI 1 "register_operand" "w")
4512 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4513 UNSPEC_ST2))]
4514 "TARGET_SIMD"
4515 "st2\\t{%S1.<Vtype> - %T1.<Vtype>}, %0"
4516 [(set_attr "type" "neon_store2_2reg<q>")]
4517 )
4519 ;; RTL uses GCC vector extension indices, so flip only for assembly.
4520 (define_insn "aarch64_vec_store_lanesoi_lane<mode>"
4521 [(set (match_operand:BLK 0 "aarch64_simd_struct_operand" "=Utv")
4522 (unspec:BLK [(match_operand:OI 1 "register_operand" "w")
4523 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)
4524 (match_operand:SI 2 "immediate_operand" "i")]
4525 UNSPEC_ST2_LANE))]
4526 "TARGET_SIMD"
4527 {
4528 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
4529 return "st2\\t{%S1.<Vetype> - %T1.<Vetype>}[%2], %0";
4530 }
4531 [(set_attr "type" "neon_store2_one_lane<q>")]
4532 )
4534 (define_expand "vec_store_lanesoi<mode>"
4535 [(set (match_operand:OI 0 "aarch64_simd_struct_operand" "=Utv")
4536 (unspec:OI [(match_operand:OI 1 "register_operand" "w")
4537 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4538 UNSPEC_ST2))]
4539 "TARGET_SIMD"
4540 {
4541 if (BYTES_BIG_ENDIAN)
4542 {
4543 rtx tmp = gen_reg_rtx (OImode);
4544 rtx mask = aarch64_reverse_mask (<MODE>mode);
4545 emit_insn (gen_aarch64_rev_reglistoi (tmp, operands[1], mask));
4546 emit_insn (gen_aarch64_simd_st2<mode> (operands[0], tmp));
4547 }
4548 else
4549 emit_insn (gen_aarch64_simd_st2<mode> (operands[0], operands[1]));
4550 DONE;
4551 })
4553 (define_insn "aarch64_simd_ld3<mode>"
4554 [(set (match_operand:CI 0 "register_operand" "=w")
4555 (unspec:CI [(match_operand:CI 1 "aarch64_simd_struct_operand" "Utv")
4556 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4557 UNSPEC_LD3))]
4558 "TARGET_SIMD"
4559 "ld3\\t{%S0.<Vtype> - %U0.<Vtype>}, %1"
4560 [(set_attr "type" "neon_load3_3reg<q>")]
4561 )
4563 (define_insn "aarch64_simd_ld3r<mode>"
4564 [(set (match_operand:CI 0 "register_operand" "=w")
4565 (unspec:CI [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")
4566 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY) ]
4567 UNSPEC_LD3_DUP))]
4568 "TARGET_SIMD"
4569 "ld3r\\t{%S0.<Vtype> - %U0.<Vtype>}, %1"
4570 [(set_attr "type" "neon_load3_all_lanes<q>")]
4571 )
4573 (define_insn "aarch64_vec_load_lanesci_lane<mode>"
4574 [(set (match_operand:CI 0 "register_operand" "=w")
4575 (unspec:CI [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")
4576 (match_operand:CI 2 "register_operand" "0")
4577 (match_operand:SI 3 "immediate_operand" "i")
4578 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4579 UNSPEC_LD3_LANE))]
4580 "TARGET_SIMD"
4581 {
4582 operands[3] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[3])));
4583 return "ld3\\t{%S0.<Vetype> - %U0.<Vetype>}[%3], %1";
4584 }
4585 [(set_attr "type" "neon_load3_one_lane")]
4586 )
4588 (define_expand "vec_load_lanesci<mode>"
4589 [(set (match_operand:CI 0 "register_operand" "=w")
4590 (unspec:CI [(match_operand:CI 1 "aarch64_simd_struct_operand" "Utv")
4591 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4592 UNSPEC_LD3))]
4593 "TARGET_SIMD"
4594 {
4595 if (BYTES_BIG_ENDIAN)
4596 {
4597 rtx tmp = gen_reg_rtx (CImode);
4598 rtx mask = aarch64_reverse_mask (<MODE>mode);
4599 emit_insn (gen_aarch64_simd_ld3<mode> (tmp, operands[1]));
4600 emit_insn (gen_aarch64_rev_reglistci (operands[0], tmp, mask));
4601 }
4602 else
4603 emit_insn (gen_aarch64_simd_ld3<mode> (operands[0], operands[1]));
4604 DONE;
4605 })
4607 (define_insn "aarch64_simd_st3<mode>"
4608 [(set (match_operand:CI 0 "aarch64_simd_struct_operand" "=Utv")
4609 (unspec:CI [(match_operand:CI 1 "register_operand" "w")
4610 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4611 UNSPEC_ST3))]
4612 "TARGET_SIMD"
4613 "st3\\t{%S1.<Vtype> - %U1.<Vtype>}, %0"
4614 [(set_attr "type" "neon_store3_3reg<q>")]
4615 )
4617 ;; RTL uses GCC vector extension indices, so flip only for assembly.
4618 (define_insn "aarch64_vec_store_lanesci_lane<mode>"
4619 [(set (match_operand:BLK 0 "aarch64_simd_struct_operand" "=Utv")
4620 (unspec:BLK [(match_operand:CI 1 "register_operand" "w")
4621 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)
4622 (match_operand:SI 2 "immediate_operand" "i")]
4623 UNSPEC_ST3_LANE))]
4624 "TARGET_SIMD"
4625 {
4626 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
4627 return "st3\\t{%S1.<Vetype> - %U1.<Vetype>}[%2], %0";
4628 }
4629 [(set_attr "type" "neon_store3_one_lane<q>")]
4630 )
4632 (define_expand "vec_store_lanesci<mode>"
4633 [(set (match_operand:CI 0 "aarch64_simd_struct_operand" "=Utv")
4634 (unspec:CI [(match_operand:CI 1 "register_operand" "w")
4635 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4636 UNSPEC_ST3))]
4637 "TARGET_SIMD"
4638 {
4639 if (BYTES_BIG_ENDIAN)
4640 {
4641 rtx tmp = gen_reg_rtx (CImode);
4642 rtx mask = aarch64_reverse_mask (<MODE>mode);
4643 emit_insn (gen_aarch64_rev_reglistci (tmp, operands[1], mask));
4644 emit_insn (gen_aarch64_simd_st3<mode> (operands[0], tmp));
4645 }
4646 else
4647 emit_insn (gen_aarch64_simd_st3<mode> (operands[0], operands[1]));
4648 DONE;
4649 })
4651 (define_insn "aarch64_simd_ld4<mode>"
4652 [(set (match_operand:XI 0 "register_operand" "=w")
4653 (unspec:XI [(match_operand:XI 1 "aarch64_simd_struct_operand" "Utv")
4654 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4655 UNSPEC_LD4))]
4656 "TARGET_SIMD"
4657 "ld4\\t{%S0.<Vtype> - %V0.<Vtype>}, %1"
4658 [(set_attr "type" "neon_load4_4reg<q>")]
4659 )
4661 (define_insn "aarch64_simd_ld4r<mode>"
4662 [(set (match_operand:XI 0 "register_operand" "=w")
4663 (unspec:XI [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")
4664 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY) ]
4665 UNSPEC_LD4_DUP))]
4666 "TARGET_SIMD"
4667 "ld4r\\t{%S0.<Vtype> - %V0.<Vtype>}, %1"
4668 [(set_attr "type" "neon_load4_all_lanes<q>")]
4669 )
4671 (define_insn "aarch64_vec_load_lanesxi_lane<mode>"
4672 [(set (match_operand:XI 0 "register_operand" "=w")
4673 (unspec:XI [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")
4674 (match_operand:XI 2 "register_operand" "0")
4675 (match_operand:SI 3 "immediate_operand" "i")
4676 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4677 UNSPEC_LD4_LANE))]
4678 "TARGET_SIMD"
4679 {
4680 operands[3] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[3])));
4681 return "ld4\\t{%S0.<Vetype> - %V0.<Vetype>}[%3], %1";
4682 }
4683 [(set_attr "type" "neon_load4_one_lane")]
4684 )
4686 (define_expand "vec_load_lanesxi<mode>"
4687 [(set (match_operand:XI 0 "register_operand" "=w")
4688 (unspec:XI [(match_operand:XI 1 "aarch64_simd_struct_operand" "Utv")
4689 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4690 UNSPEC_LD4))]
4691 "TARGET_SIMD"
4692 {
4693 if (BYTES_BIG_ENDIAN)
4694 {
4695 rtx tmp = gen_reg_rtx (XImode);
4696 rtx mask = aarch64_reverse_mask (<MODE>mode);
4697 emit_insn (gen_aarch64_simd_ld4<mode> (tmp, operands[1]));
4698 emit_insn (gen_aarch64_rev_reglistxi (operands[0], tmp, mask));
4699 }
4700 else
4701 emit_insn (gen_aarch64_simd_ld4<mode> (operands[0], operands[1]));
4702 DONE;
4703 })
4705 (define_insn "aarch64_simd_st4<mode>"
4706 [(set (match_operand:XI 0 "aarch64_simd_struct_operand" "=Utv")
4707 (unspec:XI [(match_operand:XI 1 "register_operand" "w")
4708 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4709 UNSPEC_ST4))]
4710 "TARGET_SIMD"
4711 "st4\\t{%S1.<Vtype> - %V1.<Vtype>}, %0"
4712 [(set_attr "type" "neon_store4_4reg<q>")]
4713 )
4715 ;; RTL uses GCC vector extension indices, so flip only for assembly.
4716 (define_insn "aarch64_vec_store_lanesxi_lane<mode>"
4717 [(set (match_operand:BLK 0 "aarch64_simd_struct_operand" "=Utv")
4718 (unspec:BLK [(match_operand:XI 1 "register_operand" "w")
4719 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)
4720 (match_operand:SI 2 "immediate_operand" "i")]
4721 UNSPEC_ST4_LANE))]
4722 "TARGET_SIMD"
4723 {
4724 operands[2] = GEN_INT (ENDIAN_LANE_N (<MODE>mode, INTVAL (operands[2])));
4725 return "st4\\t{%S1.<Vetype> - %V1.<Vetype>}[%2], %0";
4726 }
4727 [(set_attr "type" "neon_store4_one_lane<q>")]
4728 )
4730 (define_expand "vec_store_lanesxi<mode>"
4731 [(set (match_operand:XI 0 "aarch64_simd_struct_operand" "=Utv")
4732 (unspec:XI [(match_operand:XI 1 "register_operand" "w")
4733 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4734 UNSPEC_ST4))]
4735 "TARGET_SIMD"
4736 {
4737 if (BYTES_BIG_ENDIAN)
4738 {
4739 rtx tmp = gen_reg_rtx (XImode);
4740 rtx mask = aarch64_reverse_mask (<MODE>mode);
4741 emit_insn (gen_aarch64_rev_reglistxi (tmp, operands[1], mask));
4742 emit_insn (gen_aarch64_simd_st4<mode> (operands[0], tmp));
4743 }
4744 else
4745 emit_insn (gen_aarch64_simd_st4<mode> (operands[0], operands[1]));
4746 DONE;
4747 })
4749 (define_insn_and_split "aarch64_rev_reglist<mode>"
4750 [(set (match_operand:VSTRUCT 0 "register_operand" "=&w")
4751 (unspec:VSTRUCT
4752 [(match_operand:VSTRUCT 1 "register_operand" "w")
4753 (match_operand:V16QI 2 "register_operand" "w")]
4754 UNSPEC_REV_REGLIST))]
4755 "TARGET_SIMD"
4756 "#"
4757 "&& reload_completed"
4758 [(const_int 0)]
4759 {
4760 int i;
4761 int nregs = GET_MODE_SIZE (<MODE>mode) / UNITS_PER_VREG;
4762 for (i = 0; i < nregs; i++)
4763 {
4764 rtx op0 = gen_rtx_REG (V16QImode, REGNO (operands[0]) + i);
4765 rtx op1 = gen_rtx_REG (V16QImode, REGNO (operands[1]) + i);
4766 emit_insn (gen_aarch64_tbl1v16qi (op0, op1, operands[2]));
4767 }
4768 DONE;
4769 }
4770 [(set_attr "type" "neon_tbl1_q")
4771 (set_attr "length" "<insn_count>")]
4772 )
4774 ;; Reload patterns for AdvSIMD register list operands.
4776 (define_expand "mov<mode>"
4777 [(set (match_operand:VSTRUCT 0 "nonimmediate_operand" "")
4778 (match_operand:VSTRUCT 1 "general_operand" ""))]
4779 "TARGET_SIMD"
4780 {
4781 if (can_create_pseudo_p ())
4782 {
4783 if (GET_CODE (operands[0]) != REG)
4784 operands[1] = force_reg (<MODE>mode, operands[1]);
4785 }
4786 })
4788 (define_insn "*aarch64_mov<mode>"
4789 [(set (match_operand:VSTRUCT 0 "aarch64_simd_nonimmediate_operand" "=w,Utv,w")
4790 (match_operand:VSTRUCT 1 "aarch64_simd_general_operand" " w,w,Utv"))]
4791 "TARGET_SIMD && !BYTES_BIG_ENDIAN
4792 && (register_operand (operands[0], <MODE>mode)
4793 || register_operand (operands[1], <MODE>mode))"
4794 "@
4795 #
4796 st1\\t{%S1.16b - %<Vendreg>1.16b}, %0
4797 ld1\\t{%S0.16b - %<Vendreg>0.16b}, %1"
4798 [(set_attr "type" "multiple,neon_store<nregs>_<nregs>reg_q,\
4799 neon_load<nregs>_<nregs>reg_q")
4800 (set_attr "length" "<insn_count>,4,4")]
4801 )
4803 (define_insn "aarch64_be_ld1<mode>"
4804 [(set (match_operand:VALLDI_F16 0 "register_operand" "=w")
4805 (unspec:VALLDI_F16 [(match_operand:VALLDI_F16 1
4806 "aarch64_simd_struct_operand" "Utv")]
4807 UNSPEC_LD1))]
4808 "TARGET_SIMD"
4809 "ld1\\t{%0<Vmtype>}, %1"
4810 [(set_attr "type" "neon_load1_1reg<q>")]
4811 )
4813 (define_insn "aarch64_be_st1<mode>"
4814 [(set (match_operand:VALLDI_F16 0 "aarch64_simd_struct_operand" "=Utv")
4815 (unspec:VALLDI_F16 [(match_operand:VALLDI_F16 1 "register_operand" "w")]
4816 UNSPEC_ST1))]
4817 "TARGET_SIMD"
4818 "st1\\t{%1<Vmtype>}, %0"
4819 [(set_attr "type" "neon_store1_1reg<q>")]
4820 )
4822 (define_insn "*aarch64_be_movoi"
4823 [(set (match_operand:OI 0 "nonimmediate_operand" "=w,m,w")
4824 (match_operand:OI 1 "general_operand" " w,w,m"))]
4825 "TARGET_SIMD && BYTES_BIG_ENDIAN
4826 && (register_operand (operands[0], OImode)
4827 || register_operand (operands[1], OImode))"
4828 "@
4829 #
4830 stp\\t%q1, %R1, %0
4831 ldp\\t%q0, %R0, %1"
4832 [(set_attr "type" "multiple,neon_stp_q,neon_ldp_q")
4833 (set_attr "length" "8,4,4")]
4834 )
4836 (define_insn "*aarch64_be_movci"
4837 [(set (match_operand:CI 0 "nonimmediate_operand" "=w,o,w")
4838 (match_operand:CI 1 "general_operand" " w,w,o"))]
4839 "TARGET_SIMD && BYTES_BIG_ENDIAN
4840 && (register_operand (operands[0], CImode)
4841 || register_operand (operands[1], CImode))"
4842 "#"
4843 [(set_attr "type" "multiple")
4844 (set_attr "length" "12,4,4")]
4845 )
4847 (define_insn "*aarch64_be_movxi"
4848 [(set (match_operand:XI 0 "nonimmediate_operand" "=w,o,w")
4849 (match_operand:XI 1 "general_operand" " w,w,o"))]
4850 "TARGET_SIMD && BYTES_BIG_ENDIAN
4851 && (register_operand (operands[0], XImode)
4852 || register_operand (operands[1], XImode))"
4853 "#"
4854 [(set_attr "type" "multiple")
4855 (set_attr "length" "16,4,4")]
4856 )
4858 (define_split
4859 [(set (match_operand:OI 0 "register_operand")
4860 (match_operand:OI 1 "register_operand"))]
4861 "TARGET_SIMD && reload_completed"
4862 [(const_int 0)]
4863 {
4864 aarch64_simd_emit_reg_reg_move (operands, TImode, 2);
4865 DONE;
4866 })
4868 (define_split
4869 [(set (match_operand:CI 0 "nonimmediate_operand")
4870 (match_operand:CI 1 "general_operand"))]
4871 "TARGET_SIMD && reload_completed"
4872 [(const_int 0)]
4873 {
4874 if (register_operand (operands[0], CImode)
4875 && register_operand (operands[1], CImode))
4876 {
4877 aarch64_simd_emit_reg_reg_move (operands, TImode, 3);
4878 DONE;
4879 }
4880 else if (BYTES_BIG_ENDIAN)
4881 {
4882 emit_move_insn (simplify_gen_subreg (OImode, operands[0], CImode, 0),
4883 simplify_gen_subreg (OImode, operands[1], CImode, 0));
4884 emit_move_insn (gen_lowpart (V16QImode,
4885 simplify_gen_subreg (TImode, operands[0],
4886 CImode, 32)),
4887 gen_lowpart (V16QImode,
4888 simplify_gen_subreg (TImode, operands[1],
4889 CImode, 32)));
4890 DONE;
4891 }
4892 else
4893 FAIL;
4894 })
4896 (define_split
4897 [(set (match_operand:XI 0 "nonimmediate_operand")
4898 (match_operand:XI 1 "general_operand"))]
4899 "TARGET_SIMD && reload_completed"
4900 [(const_int 0)]
4901 {
4902 if (register_operand (operands[0], XImode)
4903 && register_operand (operands[1], XImode))
4904 {
4905 aarch64_simd_emit_reg_reg_move (operands, TImode, 4);
4906 DONE;
4907 }
4908 else if (BYTES_BIG_ENDIAN)
4909 {
4910 emit_move_insn (simplify_gen_subreg (OImode, operands[0], XImode, 0),
4911 simplify_gen_subreg (OImode, operands[1], XImode, 0));
4912 emit_move_insn (simplify_gen_subreg (OImode, operands[0], XImode, 32),
4913 simplify_gen_subreg (OImode, operands[1], XImode, 32));
4914 DONE;
4915 }
4916 else
4917 FAIL;
4918 })
4920 (define_expand "aarch64_ld<VSTRUCT:nregs>r<VALLDIF:mode>"
4921 [(match_operand:VSTRUCT 0 "register_operand" "=w")
4922 (match_operand:DI 1 "register_operand" "w")
4923 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4924 "TARGET_SIMD"
4925 {
4926 rtx mem = gen_rtx_MEM (BLKmode, operands[1]);
4927 set_mem_size (mem, GET_MODE_SIZE (GET_MODE_INNER (<VALLDIF:MODE>mode))
4928 * <VSTRUCT:nregs>);
4930 emit_insn (gen_aarch64_simd_ld<VSTRUCT:nregs>r<VALLDIF:mode> (operands[0],
4931 mem));
4932 DONE;
4933 })
4935 (define_insn "aarch64_ld2<mode>_dreg_le"
4936 [(set (match_operand:OI 0 "register_operand" "=w")
4937 (subreg:OI
4938 (vec_concat:<VRL2>
4939 (vec_concat:<VDBL>
4940 (unspec:VD
4941 [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")]
4942 UNSPEC_LD2)
4943 (vec_duplicate:VD (const_int 0)))
4944 (vec_concat:<VDBL>
4945 (unspec:VD [(match_dup 1)]
4946 UNSPEC_LD2)
4947 (vec_duplicate:VD (const_int 0)))) 0))]
4948 "TARGET_SIMD && !BYTES_BIG_ENDIAN"
4949 "ld2\\t{%S0.<Vtype> - %T0.<Vtype>}, %1"
4950 [(set_attr "type" "neon_load2_2reg<q>")]
4951 )
4953 (define_insn "aarch64_ld2<mode>_dreg_be"
4954 [(set (match_operand:OI 0 "register_operand" "=w")
4955 (subreg:OI
4956 (vec_concat:<VRL2>
4957 (vec_concat:<VDBL>
4958 (vec_duplicate:VD (const_int 0))
4959 (unspec:VD
4960 [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")]
4961 UNSPEC_LD2))
4962 (vec_concat:<VDBL>
4963 (vec_duplicate:VD (const_int 0))
4964 (unspec:VD [(match_dup 1)]
4965 UNSPEC_LD2))) 0))]
4966 "TARGET_SIMD && BYTES_BIG_ENDIAN"
4967 "ld2\\t{%S0.<Vtype> - %T0.<Vtype>}, %1"
4968 [(set_attr "type" "neon_load2_2reg<q>")]
4969 )
4971 (define_insn "aarch64_ld2<mode>_dreg_le"
4972 [(set (match_operand:OI 0 "register_operand" "=w")
4973 (subreg:OI
4974 (vec_concat:<VRL2>
4975 (vec_concat:<VDBL>
4976 (unspec:DX
4977 [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")]
4978 UNSPEC_LD2)
4979 (const_int 0))
4980 (vec_concat:<VDBL>
4981 (unspec:DX [(match_dup 1)]
4982 UNSPEC_LD2)
4983 (const_int 0))) 0))]
4984 "TARGET_SIMD && !BYTES_BIG_ENDIAN"
4985 "ld1\\t{%S0.1d - %T0.1d}, %1"
4986 [(set_attr "type" "neon_load1_2reg<q>")]
4987 )
4989 (define_insn "aarch64_ld2<mode>_dreg_be"
4990 [(set (match_operand:OI 0 "register_operand" "=w")
4991 (subreg:OI
4992 (vec_concat:<VRL2>
4993 (vec_concat:<VDBL>
4994 (const_int 0)
4995 (unspec:DX
4996 [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")]
4997 UNSPEC_LD2))
4998 (vec_concat:<VDBL>
4999 (const_int 0)
5000 (unspec:DX [(match_dup 1)]
5001 UNSPEC_LD2))) 0))]
5002 "TARGET_SIMD && BYTES_BIG_ENDIAN"
5003 "ld1\\t{%S0.1d - %T0.1d}, %1"
5004 [(set_attr "type" "neon_load1_2reg<q>")]
5005 )
5007 (define_insn "aarch64_ld3<mode>_dreg_le"
5008 [(set (match_operand:CI 0 "register_operand" "=w")
5009 (subreg:CI
5010 (vec_concat:<VRL3>
5011 (vec_concat:<VRL2>
5012 (vec_concat:<VDBL>
5013 (unspec:VD
5014 [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")]
5015 UNSPEC_LD3)
5016 (vec_duplicate:VD (const_int 0)))
5017 (vec_concat:<VDBL>
5018 (unspec:VD [(match_dup 1)]
5019 UNSPEC_LD3)
5020 (vec_duplicate:VD (const_int 0))))
5021 (vec_concat:<VDBL>
5022 (unspec:VD [(match_dup 1)]
5023 UNSPEC_LD3)
5024 (vec_duplicate:VD (const_int 0)))) 0))]
5025 "TARGET_SIMD && !BYTES_BIG_ENDIAN"
5026 "ld3\\t{%S0.<Vtype> - %U0.<Vtype>}, %1"
5027 [(set_attr "type" "neon_load3_3reg<q>")]
5028 )
5030 (define_insn "aarch64_ld3<mode>_dreg_be"
5031 [(set (match_operand:CI 0 "register_operand" "=w")
5032 (subreg:CI
5033 (vec_concat:<VRL3>
5034 (vec_concat:<VRL2>
5035 (vec_concat:<VDBL>
5036 (vec_duplicate:VD (const_int 0))
5037 (unspec:VD
5038 [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")]
5039 UNSPEC_LD3))
5040 (vec_concat:<VDBL>
5041 (vec_duplicate:VD (const_int 0))
5042 (unspec:VD [(match_dup 1)]
5043 UNSPEC_LD3)))
5044 (vec_concat:<VDBL>
5045 (vec_duplicate:VD (const_int 0))
5046 (unspec:VD [(match_dup 1)]
5047 UNSPEC_LD3))) 0))]
5048 "TARGET_SIMD && BYTES_BIG_ENDIAN"
5049 "ld3\\t{%S0.<Vtype> - %U0.<Vtype>}, %1"
5050 [(set_attr "type" "neon_load3_3reg<q>")]
5051 )
5053 (define_insn "aarch64_ld3<mode>_dreg_le"
5054 [(set (match_operand:CI 0 "register_operand" "=w")
5055 (subreg:CI
5056 (vec_concat:<VRL3>
5057 (vec_concat:<VRL2>
5058 (vec_concat:<VDBL>
5059 (unspec:DX
5060 [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")]
5061 UNSPEC_LD3)
5062 (const_int 0))
5063 (vec_concat:<VDBL>
5064 (unspec:DX [(match_dup 1)]
5065 UNSPEC_LD3)
5066 (const_int 0)))
5067 (vec_concat:<VDBL>
5068 (unspec:DX [(match_dup 1)]
5069 UNSPEC_LD3)
5070 (const_int 0))) 0))]
5071 "TARGET_SIMD && !BYTES_BIG_ENDIAN"
5072 "ld1\\t{%S0.1d - %U0.1d}, %1"
5073 [(set_attr "type" "neon_load1_3reg<q>")]
5074 )
5076 (define_insn "aarch64_ld3<mode>_dreg_be"
5077 [(set (match_operand:CI 0 "register_operand" "=w")
5078 (subreg:CI
5079 (vec_concat:<VRL3>
5080 (vec_concat:<VRL2>
5081 (vec_concat:<VDBL>
5082 (const_int 0)
5083 (unspec:DX
5084 [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")]
5085 UNSPEC_LD3))
5086 (vec_concat:<VDBL>
5087 (const_int 0)
5088 (unspec:DX [(match_dup 1)]
5089 UNSPEC_LD3)))
5090 (vec_concat:<VDBL>
5091 (const_int 0)
5092 (unspec:DX [(match_dup 1)]
5093 UNSPEC_LD3))) 0))]
5094 "TARGET_SIMD && BYTES_BIG_ENDIAN"
5095 "ld1\\t{%S0.1d - %U0.1d}, %1"
5096 [(set_attr "type" "neon_load1_3reg<q>")]
5097 )
5099 (define_insn "aarch64_ld4<mode>_dreg_le"
5100 [(set (match_operand:XI 0 "register_operand" "=w")
5101 (subreg:XI
5102 (vec_concat:<VRL4>
5103 (vec_concat:<VRL2>
5104 (vec_concat:<VDBL>
5105 (unspec:VD
5106 [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")]
5107 UNSPEC_LD4)
5108 (vec_duplicate:VD (const_int 0)))
5109 (vec_concat:<VDBL>
5110 (unspec:VD [(match_dup 1)]
5111 UNSPEC_LD4)
5112 (vec_duplicate:VD (const_int 0))))
5113 (vec_concat:<VRL2>
5114 (vec_concat:<VDBL>
5115 (unspec:VD [(match_dup 1)]
5116 UNSPEC_LD4)
5117 (vec_duplicate:VD (const_int 0)))
5118 (vec_concat:<VDBL>
5119 (unspec:VD [(match_dup 1)]
5120 UNSPEC_LD4)
5121 (vec_duplicate:VD (const_int 0))))) 0))]
5122 "TARGET_SIMD && !BYTES_BIG_ENDIAN"
5123 "ld4\\t{%S0.<Vtype> - %V0.<Vtype>}, %1"
5124 [(set_attr "type" "neon_load4_4reg<q>")]
5125 )
5127 (define_insn "aarch64_ld4<mode>_dreg_be"
5128 [(set (match_operand:XI 0 "register_operand" "=w")
5129 (subreg:XI
5130 (vec_concat:<VRL4>
5131 (vec_concat:<VRL2>
5132 (vec_concat:<VDBL>
5133 (vec_duplicate:VD (const_int 0))
5134 (unspec:VD
5135 [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")]
5136 UNSPEC_LD4))
5137 (vec_concat:<VDBL>
5138 (vec_duplicate:VD (const_int 0))
5139 (unspec:VD [(match_dup 1)]
5140 UNSPEC_LD4)))
5141 (vec_concat:<VRL2>
5142 (vec_concat:<VDBL>
5143 (vec_duplicate:VD (const_int 0))
5144 (unspec:VD [(match_dup 1)]
5145 UNSPEC_LD4))
5146 (vec_concat:<VDBL>
5147 (vec_duplicate:VD (const_int 0))
5148 (unspec:VD [(match_dup 1)]
5149 UNSPEC_LD4)))) 0))]
5150 "TARGET_SIMD && BYTES_BIG_ENDIAN"
5151 "ld4\\t{%S0.<Vtype> - %V0.<Vtype>}, %1"
5152 [(set_attr "type" "neon_load4_4reg<q>")]
5153 )
5155 (define_insn "aarch64_ld4<mode>_dreg_le"
5156 [(set (match_operand:XI 0 "register_operand" "=w")
5157 (subreg:XI
5158 (vec_concat:<VRL4>
5159 (vec_concat:<VRL2>
5160 (vec_concat:<VDBL>
5161 (unspec:DX
5162 [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")]
5163 UNSPEC_LD4)
5164 (const_int 0))
5165 (vec_concat:<VDBL>
5166 (unspec:DX [(match_dup 1)]
5167 UNSPEC_LD4)
5168 (const_int 0)))
5169 (vec_concat:<VRL2>
5170 (vec_concat:<VDBL>
5171 (unspec:DX [(match_dup 1)]
5172 UNSPEC_LD4)
5173 (const_int 0))
5174 (vec_concat:<VDBL>
5175 (unspec:DX [(match_dup 1)]
5176 UNSPEC_LD4)
5177 (const_int 0)))) 0))]
5178 "TARGET_SIMD && !BYTES_BIG_ENDIAN"
5179 "ld1\\t{%S0.1d - %V0.1d}, %1"
5180 [(set_attr "type" "neon_load1_4reg<q>")]
5181 )
5183 (define_insn "aarch64_ld4<mode>_dreg_be"
5184 [(set (match_operand:XI 0 "register_operand" "=w")
5185 (subreg:XI
5186 (vec_concat:<VRL4>
5187 (vec_concat:<VRL2>
5188 (vec_concat:<VDBL>
5189 (const_int 0)
5190 (unspec:DX
5191 [(match_operand:BLK 1 "aarch64_simd_struct_operand" "Utv")]
5192 UNSPEC_LD4))
5193 (vec_concat:<VDBL>
5194 (const_int 0)
5195 (unspec:DX [(match_dup 1)]
5196 UNSPEC_LD4)))
5197 (vec_concat:<VRL2>
5198 (vec_concat:<VDBL>
5199 (const_int 0)
5200 (unspec:DX [(match_dup 1)]
5201 UNSPEC_LD4))
5202 (vec_concat:<VDBL>
5203 (const_int 0)
5204 (unspec:DX [(match_dup 1)]
5205 UNSPEC_LD4)))) 0))]
5206 "TARGET_SIMD && BYTES_BIG_ENDIAN"
5207 "ld1\\t{%S0.1d - %V0.1d}, %1"
5208 [(set_attr "type" "neon_load1_4reg<q>")]
5209 )
5211 (define_expand "aarch64_ld<VSTRUCT:nregs><VDC:mode>"
5212 [(match_operand:VSTRUCT 0 "register_operand" "=w")
5213 (match_operand:DI 1 "register_operand" "r")
5214 (unspec:VDC [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5215 "TARGET_SIMD"
5216 {
5217 rtx mem = gen_rtx_MEM (BLKmode, operands[1]);
5218 set_mem_size (mem, <VSTRUCT:nregs> * 8);
5220 if (BYTES_BIG_ENDIAN)
5221 emit_insn (gen_aarch64_ld<VSTRUCT:nregs><VDC:mode>_dreg_be (operands[0],
5222 mem));
5223 else
5224 emit_insn (gen_aarch64_ld<VSTRUCT:nregs><VDC:mode>_dreg_le (operands[0],
5225 mem));
5226 DONE;
5227 })
5229 (define_expand "aarch64_ld1<VALL_F16:mode>"
5230 [(match_operand:VALL_F16 0 "register_operand")
5231 (match_operand:DI 1 "register_operand")]
5232 "TARGET_SIMD"
5233 {
5234 machine_mode mode = <VALL_F16:MODE>mode;
5235 rtx mem = gen_rtx_MEM (mode, operands[1]);
5237 if (BYTES_BIG_ENDIAN)
5238 emit_insn (gen_aarch64_be_ld1<VALL_F16:mode> (operands[0], mem));
5239 else
5240 emit_move_insn (operands[0], mem);
5241 DONE;
5242 })
5244 (define_expand "aarch64_ld<VSTRUCT:nregs><VQ:mode>"
5245 [(match_operand:VSTRUCT 0 "register_operand" "=w")
5246 (match_operand:DI 1 "register_operand" "r")
5247 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5248 "TARGET_SIMD"
5249 {
5250 machine_mode mode = <VSTRUCT:MODE>mode;
5251 rtx mem = gen_rtx_MEM (mode, operands[1]);
5253 emit_insn (gen_aarch64_simd_ld<VSTRUCT:nregs><VQ:mode> (operands[0], mem));
5254 DONE;
5255 })
5257 (define_expand "aarch64_ld<VSTRUCT:nregs>_lane<VALLDIF:mode>"
5258 [(match_operand:VSTRUCT 0 "register_operand" "=w")
5259 (match_operand:DI 1 "register_operand" "w")
5260 (match_operand:VSTRUCT 2 "register_operand" "0")
5261 (match_operand:SI 3 "immediate_operand" "i")
5262 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5263 "TARGET_SIMD"
5264 {
5265 rtx mem = gen_rtx_MEM (BLKmode, operands[1]);
5266 set_mem_size (mem, GET_MODE_SIZE (GET_MODE_INNER (<VALLDIF:MODE>mode))
5267 * <VSTRUCT:nregs>);
5269 aarch64_simd_lane_bounds (operands[3], 0,
5270 GET_MODE_NUNITS (<VALLDIF:MODE>mode),
5271 NULL);
5272 emit_insn (gen_aarch64_vec_load_lanes<VSTRUCT:mode>_lane<VALLDIF:mode> (
5273 operands[0], mem, operands[2], operands[3]));
5274 DONE;
5275 })
5277 ;; Expanders for builtins to extract vector registers from large
5278 ;; opaque integer modes.
5280 ;; D-register list.
5282 (define_expand "aarch64_get_dreg<VSTRUCT:mode><VDC:mode>"
5283 [(match_operand:VDC 0 "register_operand" "=w")
5284 (match_operand:VSTRUCT 1 "register_operand" "w")
5285 (match_operand:SI 2 "immediate_operand" "i")]
5286 "TARGET_SIMD"
5287 {
5288 int part = INTVAL (operands[2]);
5289 rtx temp = gen_reg_rtx (<VDC:VDBL>mode);
5290 int offset = part * 16;
5292 emit_move_insn (temp, gen_rtx_SUBREG (<VDC:VDBL>mode, operands[1], offset));
5293 emit_move_insn (operands[0], gen_lowpart (<VDC:MODE>mode, temp));
5294 DONE;
5295 })
5297 ;; Q-register list.
5299 (define_expand "aarch64_get_qreg<VSTRUCT:mode><VQ:mode>"
5300 [(match_operand:VQ 0 "register_operand" "=w")
5301 (match_operand:VSTRUCT 1 "register_operand" "w")
5302 (match_operand:SI 2 "immediate_operand" "i")]
5303 "TARGET_SIMD"
5304 {
5305 int part = INTVAL (operands[2]);
5306 int offset = part * 16;
5308 emit_move_insn (operands[0],
5309 gen_rtx_SUBREG (<VQ:MODE>mode, operands[1], offset));
5310 DONE;
5311 })
5313 ;; Permuted-store expanders for neon intrinsics.
5315 ;; Permute instructions
5317 ;; vec_perm support
5319 (define_expand "vec_perm_const<mode>"
5320 [(match_operand:VALL_F16 0 "register_operand")
5321 (match_operand:VALL_F16 1 "register_operand")
5322 (match_operand:VALL_F16 2 "register_operand")
5323 (match_operand:<V_cmp_result> 3)]
5324 "TARGET_SIMD"
5325 {
5326 if (aarch64_expand_vec_perm_const (operands[0], operands[1],
5327 operands[2], operands[3]))
5328 DONE;
5329 else
5330 FAIL;
5331 })
5333 (define_expand "vec_perm<mode>"
5334 [(match_operand:VB 0 "register_operand")
5335 (match_operand:VB 1 "register_operand")
5336 (match_operand:VB 2 "register_operand")
5337 (match_operand:VB 3 "register_operand")]
5338 "TARGET_SIMD"
5339 {
5340 aarch64_expand_vec_perm (operands[0], operands[1],
5341 operands[2], operands[3]);
5342 DONE;
5343 })
5345 (define_insn "aarch64_tbl1<mode>"
5346 [(set (match_operand:VB 0 "register_operand" "=w")
5347 (unspec:VB [(match_operand:V16QI 1 "register_operand" "w")
5348 (match_operand:VB 2 "register_operand" "w")]
5349 UNSPEC_TBL))]
5350 "TARGET_SIMD"
5351 "tbl\\t%0.<Vtype>, {%1.16b}, %2.<Vtype>"
5352 [(set_attr "type" "neon_tbl1<q>")]
5353 )
5355 ;; Two source registers.
5357 (define_insn "aarch64_tbl2v16qi"
5358 [(set (match_operand:V16QI 0 "register_operand" "=w")
5359 (unspec:V16QI [(match_operand:OI 1 "register_operand" "w")
5360 (match_operand:V16QI 2 "register_operand" "w")]
5361 UNSPEC_TBL))]
5362 "TARGET_SIMD"
5363 "tbl\\t%0.16b, {%S1.16b - %T1.16b}, %2.16b"
5364 [(set_attr "type" "neon_tbl2_q")]
5365 )
5367 (define_insn "aarch64_tbl3<mode>"
5368 [(set (match_operand:VB 0 "register_operand" "=w")
5369 (unspec:VB [(match_operand:OI 1 "register_operand" "w")
5370 (match_operand:VB 2 "register_operand" "w")]
5371 UNSPEC_TBL))]
5372 "TARGET_SIMD"
5373 "tbl\\t%S0.<Vbtype>, {%S1.16b - %T1.16b}, %S2.<Vbtype>"
5374 [(set_attr "type" "neon_tbl3")]
5375 )
5377 (define_insn "aarch64_tbx4<mode>"
5378 [(set (match_operand:VB 0 "register_operand" "=w")
5379 (unspec:VB [(match_operand:VB 1 "register_operand" "0")
5380 (match_operand:OI 2 "register_operand" "w")
5381 (match_operand:VB 3 "register_operand" "w")]
5382 UNSPEC_TBX))]
5383 "TARGET_SIMD"
5384 "tbx\\t%S0.<Vbtype>, {%S2.16b - %T2.16b}, %S3.<Vbtype>"
5385 [(set_attr "type" "neon_tbl4")]
5386 )
5388 ;; Three source registers.
5390 (define_insn "aarch64_qtbl3<mode>"
5391 [(set (match_operand:VB 0 "register_operand" "=w")
5392 (unspec:VB [(match_operand:CI 1 "register_operand" "w")
5393 (match_operand:VB 2 "register_operand" "w")]
5394 UNSPEC_TBL))]
5395 "TARGET_SIMD"
5396 "tbl\\t%S0.<Vbtype>, {%S1.16b - %U1.16b}, %S2.<Vbtype>"
5397 [(set_attr "type" "neon_tbl3")]
5398 )
5400 (define_insn "aarch64_qtbx3<mode>"
5401 [(set (match_operand:VB 0 "register_operand" "=w")
5402 (unspec:VB [(match_operand:VB 1 "register_operand" "0")
5403 (match_operand:CI 2 "register_operand" "w")
5404 (match_operand:VB 3 "register_operand" "w")]
5405 UNSPEC_TBX))]
5406 "TARGET_SIMD"
5407 "tbx\\t%S0.<Vbtype>, {%S2.16b - %U2.16b}, %S3.<Vbtype>"
5408 [(set_attr "type" "neon_tbl3")]
5409 )
5411 ;; Four source registers.
5413 (define_insn "aarch64_qtbl4<mode>"
5414 [(set (match_operand:VB 0 "register_operand" "=w")
5415 (unspec:VB [(match_operand:XI 1 "register_operand" "w")
5416 (match_operand:VB 2 "register_operand" "w")]
5417 UNSPEC_TBL))]
5418 "TARGET_SIMD"
5419 "tbl\\t%S0.<Vbtype>, {%S1.16b - %V1.16b}, %S2.<Vbtype>"
5420 [(set_attr "type" "neon_tbl4")]
5421 )
5423 (define_insn "aarch64_qtbx4<mode>"
5424 [(set (match_operand:VB 0 "register_operand" "=w")
5425 (unspec:VB [(match_operand:VB 1 "register_operand" "0")
5426 (match_operand:XI 2 "register_operand" "w")
5427 (match_operand:VB 3 "register_operand" "w")]
5428 UNSPEC_TBX))]
5429 "TARGET_SIMD"
5430 "tbx\\t%S0.<Vbtype>, {%S2.16b - %V2.16b}, %S3.<Vbtype>"
5431 [(set_attr "type" "neon_tbl4")]
5432 )
5434 (define_insn_and_split "aarch64_combinev16qi"
5435 [(set (match_operand:OI 0 "register_operand" "=w")
5436 (unspec:OI [(match_operand:V16QI 1 "register_operand" "w")
5437 (match_operand:V16QI 2 "register_operand" "w")]
5438 UNSPEC_CONCAT))]
5439 "TARGET_SIMD"
5440 "#"
5441 "&& reload_completed"
5442 [(const_int 0)]
5443 {
5444 aarch64_split_combinev16qi (operands);
5445 DONE;
5446 }
5447 [(set_attr "type" "multiple")]
5448 )
5450 (define_insn "aarch64_<PERMUTE:perm_insn><PERMUTE:perm_hilo><mode>"
5451 [(set (match_operand:VALL_F16 0 "register_operand" "=w")
5452 (unspec:VALL_F16 [(match_operand:VALL_F16 1 "register_operand" "w")
5453 (match_operand:VALL_F16 2 "register_operand" "w")]
5454 PERMUTE))]
5455 "TARGET_SIMD"
5456 "<PERMUTE:perm_insn><PERMUTE:perm_hilo>\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
5457 [(set_attr "type" "neon_permute<q>")]
5458 )
5460 ;; Note immediate (third) operand is lane index not byte index.
5461 (define_insn "aarch64_ext<mode>"
5462 [(set (match_operand:VALL_F16 0 "register_operand" "=w")
5463 (unspec:VALL_F16 [(match_operand:VALL_F16 1 "register_operand" "w")
5464 (match_operand:VALL_F16 2 "register_operand" "w")
5465 (match_operand:SI 3 "immediate_operand" "i")]
5466 UNSPEC_EXT))]
5467 "TARGET_SIMD"
5468 {
5469 operands[3] = GEN_INT (INTVAL (operands[3])
5470 * GET_MODE_UNIT_SIZE (<MODE>mode));
5471 return "ext\\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>, #%3";
5472 }
5473 [(set_attr "type" "neon_ext<q>")]
5474 )
5476 (define_insn "aarch64_rev<REVERSE:rev_op><mode>"
5477 [(set (match_operand:VALL_F16 0 "register_operand" "=w")
5478 (unspec:VALL_F16 [(match_operand:VALL_F16 1 "register_operand" "w")]
5479 REVERSE))]
5480 "TARGET_SIMD"
5481 "rev<REVERSE:rev_op>\\t%0.<Vtype>, %1.<Vtype>"
5482 [(set_attr "type" "neon_rev<q>")]
5483 )
5485 (define_insn "aarch64_st2<mode>_dreg"
5486 [(set (match_operand:BLK 0 "aarch64_simd_struct_operand" "=Utv")
5487 (unspec:BLK [(match_operand:OI 1 "register_operand" "w")
5488 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5489 UNSPEC_ST2))]
5490 "TARGET_SIMD"
5491 "st2\\t{%S1.<Vtype> - %T1.<Vtype>}, %0"
5492 [(set_attr "type" "neon_store2_2reg")]
5493 )
5495 (define_insn "aarch64_st2<mode>_dreg"
5496 [(set (match_operand:BLK 0 "aarch64_simd_struct_operand" "=Utv")
5497 (unspec:BLK [(match_operand:OI 1 "register_operand" "w")
5498 (unspec:DX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5499 UNSPEC_ST2))]
5500 "TARGET_SIMD"
5501 "st1\\t{%S1.1d - %T1.1d}, %0"
5502 [(set_attr "type" "neon_store1_2reg")]
5503 )
5505 (define_insn "aarch64_st3<mode>_dreg"
5506 [(set (match_operand:BLK 0 "aarch64_simd_struct_operand" "=Utv")
5507 (unspec:BLK [(match_operand:CI 1 "register_operand" "w")
5508 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5509 UNSPEC_ST3))]
5510 "TARGET_SIMD"
5511 "st3\\t{%S1.<Vtype> - %U1.<Vtype>}, %0"
5512 [(set_attr "type" "neon_store3_3reg")]
5513 )
5515 (define_insn "aarch64_st3<mode>_dreg"
5516 [(set (match_operand:BLK 0 "aarch64_simd_struct_operand" "=Utv")
5517 (unspec:BLK [(match_operand:CI 1 "register_operand" "w")
5518 (unspec:DX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5519 UNSPEC_ST3))]
5520 "TARGET_SIMD"
5521 "st1\\t{%S1.1d - %U1.1d}, %0"
5522 [(set_attr "type" "neon_store1_3reg")]
5523 )
5525 (define_insn "aarch64_st4<mode>_dreg"
5526 [(set (match_operand:BLK 0 "aarch64_simd_struct_operand" "=Utv")
5527 (unspec:BLK [(match_operand:XI 1 "register_operand" "w")
5528 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5529 UNSPEC_ST4))]
5530 "TARGET_SIMD"
5531 "st4\\t{%S1.<Vtype> - %V1.<Vtype>}, %0"
5532 [(set_attr "type" "neon_store4_4reg")]
5533 )
5535 (define_insn "aarch64_st4<mode>_dreg"
5536 [(set (match_operand:BLK 0 "aarch64_simd_struct_operand" "=Utv")
5537 (unspec:BLK [(match_operand:XI 1 "register_operand" "w")
5538 (unspec:DX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5539 UNSPEC_ST4))]
5540 "TARGET_SIMD"
5541 "st1\\t{%S1.1d - %V1.1d}, %0"
5542 [(set_attr "type" "neon_store1_4reg")]
5543 )
5545 (define_expand "aarch64_st<VSTRUCT:nregs><VDC:mode>"
5546 [(match_operand:DI 0 "register_operand" "r")
5547 (match_operand:VSTRUCT 1 "register_operand" "w")
5548 (unspec:VDC [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5549 "TARGET_SIMD"
5550 {
5551 rtx mem = gen_rtx_MEM (BLKmode, operands[0]);
5552 set_mem_size (mem, <VSTRUCT:nregs> * 8);
5554 emit_insn (gen_aarch64_st<VSTRUCT:nregs><VDC:mode>_dreg (mem, operands[1]));
5555 DONE;
5556 })
5558 (define_expand "aarch64_st<VSTRUCT:nregs><VQ:mode>"
5559 [(match_operand:DI 0 "register_operand" "r")
5560 (match_operand:VSTRUCT 1 "register_operand" "w")
5561 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5562 "TARGET_SIMD"
5563 {
5564 machine_mode mode = <VSTRUCT:MODE>mode;
5565 rtx mem = gen_rtx_MEM (mode, operands[0]);
5567 emit_insn (gen_aarch64_simd_st<VSTRUCT:nregs><VQ:mode> (mem, operands[1]));
5568 DONE;
5569 })
5571 (define_expand "aarch64_st<VSTRUCT:nregs>_lane<VALLDIF:mode>"
5572 [(match_operand:DI 0 "register_operand" "r")
5573 (match_operand:VSTRUCT 1 "register_operand" "w")
5574 (unspec:VALLDIF [(const_int 0)] UNSPEC_VSTRUCTDUMMY)
5575 (match_operand:SI 2 "immediate_operand")]
5576 "TARGET_SIMD"
5577 {
5578 rtx mem = gen_rtx_MEM (BLKmode, operands[0]);
5579 set_mem_size (mem, GET_MODE_SIZE (GET_MODE_INNER (<VALLDIF:MODE>mode))
5580 * <VSTRUCT:nregs>);
5582 emit_insn (gen_aarch64_vec_store_lanes<VSTRUCT:mode>_lane<VALLDIF:mode> (
5583 mem, operands[1], operands[2]));
5584 DONE;
5585 })
5587 (define_expand "aarch64_st1<VALL_F16:mode>"
5588 [(match_operand:DI 0 "register_operand")
5589 (match_operand:VALL_F16 1 "register_operand")]
5590 "TARGET_SIMD"
5591 {
5592 machine_mode mode = <VALL_F16:MODE>mode;
5593 rtx mem = gen_rtx_MEM (mode, operands[0]);
5595 if (BYTES_BIG_ENDIAN)
5596 emit_insn (gen_aarch64_be_st1<VALL_F16:mode> (mem, operands[1]));
5597 else
5598 emit_move_insn (mem, operands[1]);
5599 DONE;
5600 })
5602 ;; Expander for builtins to insert vector registers into large
5603 ;; opaque integer modes.
5605 ;; Q-register list. We don't need a D-reg inserter as we zero
5606 ;; extend them in arm_neon.h and insert the resulting Q-regs.
5608 (define_expand "aarch64_set_qreg<VSTRUCT:mode><VQ:mode>"
5609 [(match_operand:VSTRUCT 0 "register_operand" "+w")
5610 (match_operand:VSTRUCT 1 "register_operand" "0")
5611 (match_operand:VQ 2 "register_operand" "w")
5612 (match_operand:SI 3 "immediate_operand" "i")]
5613 "TARGET_SIMD"
5614 {
5615 int part = INTVAL (operands[3]);
5616 int offset = part * 16;
5618 emit_move_insn (operands[0], operands[1]);
5619 emit_move_insn (gen_rtx_SUBREG (<VQ:MODE>mode, operands[0], offset),
5620 operands[2]);
5621 DONE;
5622 })
5624 ;; Standard pattern name vec_init<mode>.
5626 (define_expand "vec_init<mode>"
5627 [(match_operand:VALL_F16 0 "register_operand" "")
5628 (match_operand 1 "" "")]
5629 "TARGET_SIMD"
5630 {
5631 aarch64_expand_vector_init (operands[0], operands[1]);
5632 DONE;
5633 })
5635 (define_insn "*aarch64_simd_ld1r<mode>"
5636 [(set (match_operand:VALL_F16 0 "register_operand" "=w")
5637 (vec_duplicate:VALL_F16
5638 (match_operand:<VEL> 1 "aarch64_simd_struct_operand" "Utv")))]
5639 "TARGET_SIMD"
5640 "ld1r\\t{%0.<Vtype>}, %1"
5641 [(set_attr "type" "neon_load1_all_lanes")]
5642 )
5644 (define_insn "aarch64_frecpe<mode>"
5645 [(set (match_operand:VHSDF 0 "register_operand" "=w")
5646 (unspec:VHSDF [(match_operand:VHSDF 1 "register_operand" "w")]
5647 UNSPEC_FRECPE))]
5648 "TARGET_SIMD"
5649 "frecpe\\t%0.<Vtype>, %1.<Vtype>"
5650 [(set_attr "type" "neon_fp_recpe_<stype><q>")]
5651 )
5653 (define_insn "aarch64_frecp<FRECP:frecp_suffix><mode>"
5654 [(set (match_operand:GPF_F16 0 "register_operand" "=w")
5655 (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")]
5656 FRECP))]
5657 "TARGET_SIMD"
5658 "frecp<FRECP:frecp_suffix>\\t%<s>0, %<s>1"
5659 [(set_attr "type" "neon_fp_recp<FRECP:frecp_suffix>_<GPF_F16:stype>")]
5660 )
5662 (define_insn "aarch64_frecps<mode>"
5663 [(set (match_operand:VHSDF_HSDF 0 "register_operand" "=w")
5664 (unspec:VHSDF_HSDF
5665 [(match_operand:VHSDF_HSDF 1 "register_operand" "w")
5666 (match_operand:VHSDF_HSDF 2 "register_operand" "w")]
5667 UNSPEC_FRECPS))]
5668 "TARGET_SIMD"
5669 "frecps\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
5670 [(set_attr "type" "neon_fp_recps_<stype><q>")]
5671 )
5673 (define_insn "aarch64_urecpe<mode>"
5674 [(set (match_operand:VDQ_SI 0 "register_operand" "=w")
5675 (unspec:VDQ_SI [(match_operand:VDQ_SI 1 "register_operand" "w")]
5676 UNSPEC_URECPE))]
5677 "TARGET_SIMD"
5678 "urecpe\\t%0.<Vtype>, %1.<Vtype>"
5679 [(set_attr "type" "neon_fp_recpe_<Vetype><q>")])
5681 ;; Standard pattern name vec_extract<mode>.
5683 (define_expand "vec_extract<mode>"
5684 [(match_operand:<VEL> 0 "aarch64_simd_nonimmediate_operand" "")
5685 (match_operand:VALL_F16 1 "register_operand" "")
5686 (match_operand:SI 2 "immediate_operand" "")]
5687 "TARGET_SIMD"
5688 {
5689 emit_insn
5690 (gen_aarch64_get_lane<mode> (operands[0], operands[1], operands[2]));
5691 DONE;
5692 })
5694 ;; aes
5696 (define_insn "aarch64_crypto_aes<aes_op>v16qi"
5697 [(set (match_operand:V16QI 0 "register_operand" "=w")
5698 (unspec:V16QI [(match_operand:V16QI 1 "register_operand" "0")
5699 (match_operand:V16QI 2 "register_operand" "w")]
5700 CRYPTO_AES))]
5701 "TARGET_SIMD && TARGET_CRYPTO"
5702 "aes<aes_op>\\t%0.16b, %2.16b"
5703 [(set_attr "type" "crypto_aese")]
5704 )
5706 ;; When AES/AESMC fusion is enabled we want the register allocation to
5707 ;; look like:
5708 ;; AESE Vn, _
5709 ;; AESMC Vn, Vn
5710 ;; So prefer to tie operand 1 to operand 0 when fusing.
5712 (define_insn "aarch64_crypto_aes<aesmc_op>v16qi"
5713 [(set (match_operand:V16QI 0 "register_operand" "=w,w")
5714 (unspec:V16QI [(match_operand:V16QI 1 "register_operand" "0,w")]
5715 CRYPTO_AESMC))]
5716 "TARGET_SIMD && TARGET_CRYPTO"
5717 "aes<aesmc_op>\\t%0.16b, %1.16b"
5718 [(set_attr "type" "crypto_aesmc")
5719 (set_attr_alternative "enabled"
5720 [(if_then_else (match_test
5721 "aarch64_fusion_enabled_p (AARCH64_FUSE_AES_AESMC)")
5722 (const_string "yes" )
5723 (const_string "no"))
5724 (const_string "yes")])]
5725 )
5727 ;; sha1
5729 (define_insn "aarch64_crypto_sha1hsi"
5730 [(set (match_operand:SI 0 "register_operand" "=w")
5731 (unspec:SI [(match_operand:SI 1
5732 "register_operand" "w")]
5733 UNSPEC_SHA1H))]
5734 "TARGET_SIMD && TARGET_CRYPTO"
5735 "sha1h\\t%s0, %s1"
5736 [(set_attr "type" "crypto_sha1_fast")]
5737 )
5739 (define_insn "aarch64_crypto_sha1hv4si"
5740 [(set (match_operand:SI 0 "register_operand" "=w")
5741 (unspec:SI [(vec_select:SI (match_operand:V4SI 1 "register_operand" "w")
5742 (parallel [(const_int 0)]))]
5743 UNSPEC_SHA1H))]
5744 "TARGET_SIMD && TARGET_CRYPTO && !BYTES_BIG_ENDIAN"
5745 "sha1h\\t%s0, %s1"
5746 [(set_attr "type" "crypto_sha1_fast")]
5747 )
5749 (define_insn "aarch64_be_crypto_sha1hv4si"
5750 [(set (match_operand:SI 0 "register_operand" "=w")
5751 (unspec:SI [(vec_select:SI (match_operand:V4SI 1 "register_operand" "w")
5752 (parallel [(const_int 3)]))]
5753 UNSPEC_SHA1H))]
5754 "TARGET_SIMD && TARGET_CRYPTO && BYTES_BIG_ENDIAN"
5755 "sha1h\\t%s0, %s1"
5756 [(set_attr "type" "crypto_sha1_fast")]
5757 )
5759 (define_insn "aarch64_crypto_sha1su1v4si"
5760 [(set (match_operand:V4SI 0 "register_operand" "=w")
5761 (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
5762 (match_operand:V4SI 2 "register_operand" "w")]
5763 UNSPEC_SHA1SU1))]
5764 "TARGET_SIMD && TARGET_CRYPTO"
5765 "sha1su1\\t%0.4s, %2.4s"
5766 [(set_attr "type" "crypto_sha1_fast")]
5767 )
5769 (define_insn "aarch64_crypto_sha1<sha1_op>v4si"
5770 [(set (match_operand:V4SI 0 "register_operand" "=w")
5771 (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
5772 (match_operand:SI 2 "register_operand" "w")
5773 (match_operand:V4SI 3 "register_operand" "w")]
5774 CRYPTO_SHA1))]
5775 "TARGET_SIMD && TARGET_CRYPTO"
5776 "sha1<sha1_op>\\t%q0, %s2, %3.4s"
5777 [(set_attr "type" "crypto_sha1_slow")]
5778 )
5780 (define_insn "aarch64_crypto_sha1su0v4si"
5781 [(set (match_operand:V4SI 0 "register_operand" "=w")
5782 (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
5783 (match_operand:V4SI 2 "register_operand" "w")
5784 (match_operand:V4SI 3 "register_operand" "w")]
5785 UNSPEC_SHA1SU0))]
5786 "TARGET_SIMD && TARGET_CRYPTO"
5787 "sha1su0\\t%0.4s, %2.4s, %3.4s"
5788 [(set_attr "type" "crypto_sha1_xor")]
5789 )
5791 ;; sha256
5793 (define_insn "aarch64_crypto_sha256h<sha256_op>v4si"
5794 [(set (match_operand:V4SI 0 "register_operand" "=w")
5795 (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
5796 (match_operand:V4SI 2 "register_operand" "w")
5797 (match_operand:V4SI 3 "register_operand" "w")]
5798 CRYPTO_SHA256))]
5799 "TARGET_SIMD && TARGET_CRYPTO"
5800 "sha256h<sha256_op>\\t%q0, %q2, %3.4s"
5801 [(set_attr "type" "crypto_sha256_slow")]
5802 )
5804 (define_insn "aarch64_crypto_sha256su0v4si"
5805 [(set (match_operand:V4SI 0 "register_operand" "=w")
5806 (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
5807 (match_operand:V4SI 2 "register_operand" "w")]
5808 UNSPEC_SHA256SU0))]
5809 "TARGET_SIMD &&TARGET_CRYPTO"
5810 "sha256su0\\t%0.4s, %2.4s"
5811 [(set_attr "type" "crypto_sha256_fast")]
5812 )
5814 (define_insn "aarch64_crypto_sha256su1v4si"
5815 [(set (match_operand:V4SI 0 "register_operand" "=w")
5816 (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
5817 (match_operand:V4SI 2 "register_operand" "w")
5818 (match_operand:V4SI 3 "register_operand" "w")]
5819 UNSPEC_SHA256SU1))]
5820 "TARGET_SIMD &&TARGET_CRYPTO"
5821 "sha256su1\\t%0.4s, %2.4s, %3.4s"
5822 [(set_attr "type" "crypto_sha256_slow")]
5823 )
5825 ;; pmull
5827 (define_insn "aarch64_crypto_pmulldi"
5828 [(set (match_operand:TI 0 "register_operand" "=w")
5829 (unspec:TI [(match_operand:DI 1 "register_operand" "w")
5830 (match_operand:DI 2 "register_operand" "w")]
5831 UNSPEC_PMULL))]
5832 "TARGET_SIMD && TARGET_CRYPTO"
5833 "pmull\\t%0.1q, %1.1d, %2.1d"
5834 [(set_attr "type" "crypto_pmull")]
5835 )
5837 (define_insn "aarch64_crypto_pmullv2di"
5838 [(set (match_operand:TI 0 "register_operand" "=w")
5839 (unspec:TI [(match_operand:V2DI 1 "register_operand" "w")
5840 (match_operand:V2DI 2 "register_operand" "w")]
5841 UNSPEC_PMULL2))]
5842 "TARGET_SIMD && TARGET_CRYPTO"
5843 "pmull2\\t%0.1q, %1.2d, %2.2d"
5844 [(set_attr "type" "crypto_pmull")]
5845 )