| blob | 04d5794ffcae73a8b33844f3147e4315747deb69 |
1 ;; Machine description for AArch64 AdvSIMD architecture.
2 ;; Copyright (C) 2011-2013 Free Software Foundation, Inc.
3 ;; Contributed by ARM Ltd.
4 ;;
5 ;; This file is part of GCC.
6 ;;
7 ;; GCC is free software; you can redistribute it and/or modify it
8 ;; under the terms of the GNU General Public License as published by
9 ;; the Free Software Foundation; either version 3, or (at your option)
10 ;; any later version.
11 ;;
12 ;; GCC is distributed in the hope that it will be useful, but
13 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
14 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 ;; General Public License for more details.
16 ;;
17 ;; You should have received a copy of the GNU General Public License
18 ;; along with GCC; see the file COPYING3. If not see
19 ;; <http://www.gnu.org/licenses/>.
22 ; Main data types used by the insntructions
24 (define_attr "simd_mode" "unknown,none,V8QI,V16QI,V4HI,V8HI,V2SI,V4SI,V2DI,V2SF,V4SF,V2DF,OI,CI,XI,DI,DF,SI,SF,HI,QI"
25 (const_string "unknown"))
28 ; Classification of AdvSIMD instructions for scheduling purposes.
29 ; Do not set this attribute and the "v8type" attribute together in
30 ; any instruction pattern.
32 ; simd_abd integer absolute difference and accumulate.
33 ; simd_abdl integer absolute difference and accumulate (long).
34 ; simd_adal integer add and accumulate (long).
35 ; simd_add integer addition/subtraction.
36 ; simd_addl integer addition/subtraction (long).
37 ; simd_addlv across lanes integer sum (long).
38 ; simd_addn integer addition/subtraction (narrow).
39 ; simd_addn2 integer addition/subtraction (narrow, high).
40 ; simd_addv across lanes integer sum.
41 ; simd_cls count leading sign/zero bits.
42 ; simd_cmp compare / create mask.
43 ; simd_cnt population count.
44 ; simd_dup duplicate element.
45 ; simd_dupgp duplicate general purpose register.
46 ; simd_ext bitwise extract from pair.
47 ; simd_fabd floating point absolute difference.
48 ; simd_fadd floating point add/sub.
49 ; simd_fcmp floating point compare.
50 ; simd_fcvti floating point convert to integer.
51 ; simd_fcvtl floating-point convert upsize.
52 ; simd_fcvtn floating-point convert downsize (narrow).
53 ; simd_fcvtn2 floating-point convert downsize (narrow, high).
54 ; simd_fdiv floating point division.
55 ; simd_fminmax floating point min/max.
56 ; simd_fminmaxv across lanes floating point min/max.
57 ; simd_fmla floating point multiply-add.
58 ; simd_fmla_elt floating point multiply-add (by element).
59 ; simd_fmul floating point multiply.
60 ; simd_fmul_elt floating point multiply (by element).
61 ; simd_fnegabs floating point neg/abs.
62 ; simd_frecpe floating point reciprocal estimate.
63 ; simd_frecps floating point reciprocal step.
64 ; simd_frecpx floating point reciprocal exponent.
65 ; simd_frint floating point round to integer.
66 ; simd_fsqrt floating point square root.
67 ; simd_icvtf integer convert to floating point.
68 ; simd_ins insert element.
69 ; simd_insgp insert general purpose register.
70 ; simd_load1 load multiple structures to one register (LD1).
71 ; simd_load1r load single structure to all lanes of one register (LD1R).
72 ; simd_load1s load single structure to one lane of one register (LD1 [index]).
73 ; simd_load2 load multiple structures to two registers (LD1, LD2).
74 ; simd_load2r load single structure to all lanes of two registers (LD1R, LD2R).
75 ; simd_load2s load single structure to one lane of two registers (LD2 [index]).
76 ; simd_load3 load multiple structures to three registers (LD1, LD3).
77 ; simd_load3r load single structure to all lanes of three registers (LD3R).
78 ; simd_load3s load single structure to one lane of three registers (LD3 [index]).
79 ; simd_load4 load multiple structures to four registers (LD1, LD2, LD4).
80 ; simd_load4r load single structure to all lanes of four registers (LD4R).
81 ; simd_load4s load single structure to one lane of four registers (LD4 [index]).
82 ; simd_logic logical operation.
83 ; simd_logic_imm logcial operation (immediate).
84 ; simd_minmax integer min/max.
85 ; simd_minmaxv across lanes integer min/max,
86 ; simd_mla integer multiply-accumulate.
87 ; simd_mla_elt integer multiply-accumulate (by element).
88 ; simd_mlal integer multiply-accumulate (long).
89 ; simd_mlal_elt integer multiply-accumulate (by element, long).
90 ; simd_move move register.
91 ; simd_move_imm move immediate.
92 ; simd_movgp move element to general purpose register.
93 ; simd_mul integer multiply.
94 ; simd_mul_elt integer multiply (by element).
95 ; simd_mull integer multiply (long).
96 ; simd_mull_elt integer multiply (by element, long).
97 ; simd_negabs integer negate/absolute.
98 ; simd_rbit bitwise reverse.
99 ; simd_rcpe integer reciprocal estimate.
100 ; simd_rcps integer reciprocal square root.
101 ; simd_rev element reverse.
102 ; simd_sat_add integer saturating addition/subtraction.
103 ; simd_sat_mlal integer saturating multiply-accumulate (long).
104 ; simd_sat_mlal_elt integer saturating multiply-accumulate (by element, long).
105 ; simd_sat_mul integer saturating multiply.
106 ; simd_sat_mul_elt integer saturating multiply (by element).
107 ; simd_sat_mull integer saturating multiply (long).
108 ; simd_sat_mull_elt integer saturating multiply (by element, long).
109 ; simd_sat_negabs integer saturating negate/absolute.
110 ; simd_sat_shift integer saturating shift.
111 ; simd_sat_shift_imm integer saturating shift (immediate).
112 ; simd_sat_shiftn_imm integer saturating shift (narrow, immediate).
113 ; simd_sat_shiftn2_imm integer saturating shift (narrow, high, immediate).
114 ; simd_shift shift register/vector.
115 ; simd_shift_acc shift accumulate.
116 ; simd_shift_imm shift immediate.
117 ; simd_shift_imm_acc shift immediate and accumualte.
118 ; simd_shiftl shift register/vector (long).
119 ; simd_shiftl_imm shift register/vector (long, immediate).
120 ; simd_shiftn_imm shift register/vector (narrow, immediate).
121 ; simd_shiftn2_imm shift register/vector (narrow, high, immediate).
122 ; simd_store1 store multiple structures from one register (ST1).
123 ; simd_store1s store single structure from one lane of one register (ST1 [index]).
124 ; simd_store2 store multiple structures from two registers (ST1, ST2).
125 ; simd_store2s store single structure from one lane of two registers (ST2 [index]).
126 ; simd_store3 store multiple structures from three registers (ST1, ST3).
127 ; simd_store3s store single structure from one lane of three register (ST3 [index]).
128 ; simd_store4 store multiple structures from four registers (ST1, ST2, ST4).
129 ; simd_store4s store single structure from one lane for four registers (ST4 [index]).
130 ; simd_tbl table lookup.
131 ; simd_trn transpose.
132 ; simd_uzp unzip.
133 ; simd_zip zip.
135 (define_attr "simd_type"
136 "simd_abd,\
137 simd_abdl,\
138 simd_adal,\
139 simd_add,\
140 simd_addl,\
141 simd_addlv,\
142 simd_addn,\
143 simd_addn2,\
144 simd_addv,\
145 simd_cls,\
146 simd_cmp,\
147 simd_cnt,\
148 simd_dup,\
149 simd_dupgp,\
150 simd_ext,\
151 simd_fabd,\
152 simd_fadd,\
153 simd_fcmp,\
154 simd_fcvti,\
155 simd_fcvtl,\
156 simd_fcvtn,\
157 simd_fcvtn2,\
158 simd_fdiv,\
159 simd_fminmax,\
160 simd_fminmaxv,\
161 simd_fmla,\
162 simd_fmla_elt,\
163 simd_fmul,\
164 simd_fmul_elt,\
165 simd_fnegabs,\
166 simd_frecpe,\
167 simd_frecps,\
168 simd_frecpx,\
169 simd_frint,\
170 simd_fsqrt,\
171 simd_icvtf,\
172 simd_ins,\
173 simd_insgp,\
174 simd_load1,\
175 simd_load1r,\
176 simd_load1s,\
177 simd_load2,\
178 simd_load2r,\
179 simd_load2s,\
180 simd_load3,\
181 simd_load3r,\
182 simd_load3s,\
183 simd_load4,\
184 simd_load4r,\
185 simd_load4s,\
186 simd_logic,\
187 simd_logic_imm,\
188 simd_minmax,\
189 simd_minmaxv,\
190 simd_mla,\
191 simd_mla_elt,\
192 simd_mlal,\
193 simd_mlal_elt,\
194 simd_movgp,\
195 simd_move,\
196 simd_move_imm,\
197 simd_mul,\
198 simd_mul_elt,\
199 simd_mull,\
200 simd_mull_elt,\
201 simd_negabs,\
202 simd_rbit,\
203 simd_rcpe,\
204 simd_rcps,\
205 simd_rev,\
206 simd_sat_add,\
207 simd_sat_mlal,\
208 simd_sat_mlal_elt,\
209 simd_sat_mul,\
210 simd_sat_mul_elt,\
211 simd_sat_mull,\
212 simd_sat_mull_elt,\
213 simd_sat_negabs,\
214 simd_sat_shift,\
215 simd_sat_shift_imm,\
216 simd_sat_shiftn_imm,\
217 simd_sat_shiftn2_imm,\
218 simd_shift,\
219 simd_shift_acc,\
220 simd_shift_imm,\
221 simd_shift_imm_acc,\
222 simd_shiftl,\
223 simd_shiftl_imm,\
224 simd_shiftn_imm,\
225 simd_shiftn2_imm,\
226 simd_store1,\
227 simd_store1s,\
228 simd_store2,\
229 simd_store2s,\
230 simd_store3,\
231 simd_store3s,\
232 simd_store4,\
233 simd_store4s,\
234 simd_tbl,\
235 simd_trn,\
236 simd_uzp,\
237 simd_zip,\
238 none"
239 (const_string "none"))
242 ; The "neon_type" attribute is used by the AArch32 backend. Below is a mapping
243 ; from "simd_type" to "neon_type".
245 (define_attr "neon_type"
246 "neon_int_1,neon_int_2,neon_int_3,neon_int_4,neon_int_5,neon_vqneg_vqabs,
247 neon_vmov,neon_vaba,neon_vsma,neon_vaba_qqq,
248 neon_mul_ddd_8_16_qdd_16_8_long_32_16_long,neon_mul_qqq_8_16_32_ddd_32,
249 neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar,
250 neon_mla_ddd_8_16_qdd_16_8_long_32_16_long,neon_mla_qqq_8_16,
251 neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long,
252 neon_mla_qqq_32_qqd_32_scalar,neon_mul_ddd_16_scalar_32_16_long_scalar,
253 neon_mul_qqd_32_scalar,neon_mla_ddd_16_scalar_qdd_32_16_long_scalar,
254 neon_shift_1,neon_shift_2,neon_shift_3,neon_vshl_ddd,
255 neon_vqshl_vrshl_vqrshl_qqq,neon_vsra_vrsra,neon_fp_vadd_ddd_vabs_dd,
256 neon_fp_vadd_qqq_vabs_qq,neon_fp_vsum,neon_fp_vmul_ddd,neon_fp_vmul_qqd,
257 neon_fp_vmla_ddd,neon_fp_vmla_qqq,neon_fp_vmla_ddd_scalar,
258 neon_fp_vmla_qqq_scalar,neon_fp_vrecps_vrsqrts_ddd,
259 neon_fp_vrecps_vrsqrts_qqq,neon_bp_simple,neon_bp_2cycle,neon_bp_3cycle,
260 neon_ldr,neon_str,neon_vld1_1_2_regs,neon_vld1_3_4_regs,
261 neon_vld2_2_regs_vld1_vld2_all_lanes,neon_vld2_4_regs,neon_vld3_vld4,
262 neon_vst1_1_2_regs_vst2_2_regs,neon_vst1_3_4_regs,
263 neon_vst2_4_regs_vst3_vst4,neon_vst3_vst4,neon_vld1_vld2_lane,
264 neon_vld3_vld4_lane,neon_vst1_vst2_lane,neon_vst3_vst4_lane,
265 neon_vld3_vld4_all_lanes,neon_mcr,neon_mcr_2_mcrr,neon_mrc,neon_mrrc,
266 neon_ldm_2,neon_stm_2,none,unknown"
267 (cond [
268 (eq_attr "simd_type" "simd_dup") (const_string "neon_bp_simple")
269 (eq_attr "simd_type" "simd_movgp") (const_string "neon_bp_simple")
270 (eq_attr "simd_type" "simd_add,simd_logic,simd_logic_imm") (const_string "neon_int_1")
271 (eq_attr "simd_type" "simd_negabs,simd_addlv") (const_string "neon_int_3")
272 (eq_attr "simd_type" "simd_addn,simd_addn2,simd_addl,simd_sat_add,simd_sat_negabs") (const_string "neon_int_4")
273 (eq_attr "simd_type" "simd_move") (const_string "neon_vmov")
274 (eq_attr "simd_type" "simd_ins") (const_string "neon_mcr")
275 (and (eq_attr "simd_type" "simd_mul,simd_sat_mul") (eq_attr "simd_mode" "V8QI,V4HI")) (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long")
276 (and (eq_attr "simd_type" "simd_mul,simd_sat_mul") (eq_attr "simd_mode" "V2SI,V8QI,V16QI,V2SI")) (const_string "neon_mul_qqq_8_16_32_ddd_32")
277 (and (eq_attr "simd_type" "simd_mull,simd_sat_mull") (eq_attr "simd_mode" "V8QI,V16QI,V4HI,V8HI")) (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long")
278 (and (eq_attr "simd_type" "simd_mull,simd_sat_mull") (eq_attr "simd_mode" "V2SI,V4SI,V2DI")) (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")
279 (and (eq_attr "simd_type" "simd_mla,simd_sat_mlal") (eq_attr "simd_mode" "V8QI,V4HI")) (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
280 (and (eq_attr "simd_type" "simd_mla,simd_sat_mlal") (eq_attr "simd_mode" "V2SI")) (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")
281 (and (eq_attr "simd_type" "simd_mla,simd_sat_mlal") (eq_attr "simd_mode" "V16QI,V8HI")) (const_string "neon_mla_qqq_8_16")
282 (and (eq_attr "simd_type" "simd_mla,simd_sat_mlal") (eq_attr "simd_mode" "V4SI")) (const_string "neon_mla_qqq_32_qqd_32_scalar")
283 (and (eq_attr "simd_type" "simd_mlal") (eq_attr "simd_mode" "V8QI,V16QI,V4HI,V8HI")) (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
284 (and (eq_attr "simd_type" "simd_mlal") (eq_attr "simd_mode" "V2SI,V4SI,V2DI")) (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")
285 (and (eq_attr "simd_type" "simd_fmla") (eq_attr "simd_mode" "V2SF")) (const_string "neon_fp_vmla_ddd")
286 (and (eq_attr "simd_type" "simd_fmla") (eq_attr "simd_mode" "V4SF,V2DF")) (const_string "neon_fp_vmla_qqq")
287 (and (eq_attr "simd_type" "simd_fmla_elt") (eq_attr "simd_mode" "V2SF")) (const_string "neon_fp_vmla_ddd_scalar")
288 (and (eq_attr "simd_type" "simd_fmla_elt") (eq_attr "simd_mode" "V4SF,V2DF")) (const_string "neon_fp_vmla_qqq_scalar")
289 (and (eq_attr "simd_type" "simd_fmul,simd_fmul_elt,simd_fdiv,simd_fsqrt") (eq_attr "simd_mode" "V2SF")) (const_string "neon_fp_vmul_ddd")
290 (and (eq_attr "simd_type" "simd_fmul,simd_fmul_elt,simd_fdiv,simd_fsqrt") (eq_attr "simd_mode" "V4SF,V2DF")) (const_string "neon_fp_vmul_qqd")
291 (and (eq_attr "simd_type" "simd_fadd") (eq_attr "simd_mode" "V2SF")) (const_string "neon_fp_vadd_ddd_vabs_dd")
292 (and (eq_attr "simd_type" "simd_fadd") (eq_attr "simd_mode" "V4SF,V2DF")) (const_string "neon_fp_vadd_qqq_vabs_qq")
293 (and (eq_attr "simd_type" "simd_fnegabs,simd_fminmax,simd_fminmaxv") (eq_attr "simd_mode" "V2SF")) (const_string "neon_fp_vadd_ddd_vabs_dd")
294 (and (eq_attr "simd_type" "simd_fnegabs,simd_fminmax,simd_fminmaxv") (eq_attr "simd_mode" "V4SF,V2DF")) (const_string "neon_fp_vadd_qqq_vabs_qq")
295 (and (eq_attr "simd_type" "simd_shift,simd_shift_acc") (eq_attr "simd_mode" "V8QI,V4HI,V2SI")) (const_string "neon_vshl_ddd")
296 (and (eq_attr "simd_type" "simd_shift,simd_shift_acc") (eq_attr "simd_mode" "V16QI,V8HI,V4SI,V2DI")) (const_string "neon_shift_3")
297 (eq_attr "simd_type" "simd_minmax,simd_minmaxv") (const_string "neon_int_5")
298 (eq_attr "simd_type" "simd_shiftn_imm,simd_shiftn2_imm,simd_shiftl_imm,") (const_string "neon_shift_1")
299 (eq_attr "simd_type" "simd_load1,simd_load2") (const_string "neon_vld1_1_2_regs")
300 (eq_attr "simd_type" "simd_load3,simd_load3") (const_string "neon_vld1_3_4_regs")
301 (eq_attr "simd_type" "simd_load1r,simd_load2r,simd_load3r,simd_load4r") (const_string "neon_vld2_2_regs_vld1_vld2_all_lanes")
302 (eq_attr "simd_type" "simd_load1s,simd_load2s") (const_string "neon_vld1_vld2_lane")
303 (eq_attr "simd_type" "simd_load3s,simd_load4s") (const_string "neon_vld3_vld4_lane")
304 (eq_attr "simd_type" "simd_store1,simd_store2") (const_string "neon_vst1_1_2_regs_vst2_2_regs")
305 (eq_attr "simd_type" "simd_store3,simd_store4") (const_string "neon_vst1_3_4_regs")
306 (eq_attr "simd_type" "simd_store1s,simd_store2s") (const_string "neon_vst1_vst2_lane")
307 (eq_attr "simd_type" "simd_store3s,simd_store4s") (const_string "neon_vst3_vst4_lane")
308 (and (eq_attr "simd_type" "simd_frecpe,simd_frecps") (eq_attr "simd_mode" "V2SF")) (const_string "neon_fp_vrecps_vrsqrts_ddd")
309 (and (eq_attr "simd_type" "simd_frecpe,simd_frecps") (eq_attr "simd_mode" "V4SF,V2DF")) (const_string "neon_fp_vrecps_vrsqrts_qqq")
310 (eq_attr "simd_type" "none") (const_string "none")
311 ]
312 (const_string "unknown")))
315 (define_expand "mov<mode>"
316 [(set (match_operand:VALL 0 "aarch64_simd_nonimmediate_operand" "")
317 (match_operand:VALL 1 "aarch64_simd_general_operand" ""))]
318 "TARGET_SIMD"
319 "
320 if (GET_CODE (operands[0]) == MEM)
321 operands[1] = force_reg (<MODE>mode, operands[1]);
322 "
323 )
325 (define_expand "movmisalign<mode>"
326 [(set (match_operand:VALL 0 "aarch64_simd_nonimmediate_operand" "")
327 (match_operand:VALL 1 "aarch64_simd_general_operand" ""))]
328 "TARGET_SIMD"
329 {
330 /* This pattern is not permitted to fail during expansion: if both arguments
331 are non-registers (e.g. memory := constant, which can be created by the
332 auto-vectorizer), force operand 1 into a register. */
333 if (!register_operand (operands[0], <MODE>mode)
334 && !register_operand (operands[1], <MODE>mode))
335 operands[1] = force_reg (<MODE>mode, operands[1]);
336 })
338 (define_insn "aarch64_simd_dup<mode>"
339 [(set (match_operand:VDQ 0 "register_operand" "=w, w")
340 (vec_duplicate:VDQ (match_operand:<VEL> 1 "register_operand" "r, w")))]
341 "TARGET_SIMD"
342 "@
343 dup\\t%0.<Vtype>, %<vw>1
344 dup\\t%0.<Vtype>, %1.<Vetype>[0]"
345 [(set_attr "simd_type" "simd_dupgp, simd_dup")
346 (set_attr "simd_mode" "<MODE>")]
347 )
349 (define_insn "aarch64_simd_dup<mode>"
350 [(set (match_operand:VDQF 0 "register_operand" "=w")
351 (vec_duplicate:VDQF (match_operand:<VEL> 1 "register_operand" "w")))]
352 "TARGET_SIMD"
353 "dup\\t%0.<Vtype>, %1.<Vetype>[0]"
354 [(set_attr "simd_type" "simd_dup")
355 (set_attr "simd_mode" "<MODE>")]
356 )
358 (define_insn "aarch64_dup_lane<mode>"
359 [(set (match_operand:VALL 0 "register_operand" "=w")
360 (vec_duplicate:VALL
361 (vec_select:<VEL>
362 (match_operand:VALL 1 "register_operand" "w")
363 (parallel [(match_operand:SI 2 "immediate_operand" "i")])
364 )))]
365 "TARGET_SIMD"
366 "dup\\t%0.<Vtype>, %1.<Vetype>[%2]"
367 [(set_attr "simd_type" "simd_dup")
368 (set_attr "simd_mode" "<MODE>")]
369 )
371 (define_insn "aarch64_dup_lane_<vswap_width_name><mode>"
372 [(set (match_operand:VALL 0 "register_operand" "=w")
373 (vec_duplicate:VALL
374 (vec_select:<VEL>
375 (match_operand:<VSWAP_WIDTH> 1 "register_operand" "w")
376 (parallel [(match_operand:SI 2 "immediate_operand" "i")])
377 )))]
378 "TARGET_SIMD"
379 "dup\\t%0.<Vtype>, %1.<Vetype>[%2]"
380 [(set_attr "simd_type" "simd_dup")
381 (set_attr "simd_mode" "<MODE>")]
382 )
384 (define_insn "*aarch64_simd_mov<mode>"
385 [(set (match_operand:VD 0 "aarch64_simd_nonimmediate_operand"
386 "=w, Utv, w, ?r, ?w, ?r, w")
387 (match_operand:VD 1 "aarch64_simd_general_operand"
388 "Utv, w, w, w, r, r, Dn"))]
389 "TARGET_SIMD
390 && (register_operand (operands[0], <MODE>mode)
391 || register_operand (operands[1], <MODE>mode))"
392 {
393 switch (which_alternative)
394 {
395 case 0: return "ld1\t{%0.<Vtype>}, %1";
396 case 1: return "st1\t{%1.<Vtype>}, %0";
397 case 2: return "orr\t%0.<Vbtype>, %1.<Vbtype>, %1.<Vbtype>";
398 case 3: return "umov\t%0, %1.d[0]";
399 case 4: return "ins\t%0.d[0], %1";
400 case 5: return "mov\t%0, %1";
401 case 6:
402 return aarch64_output_simd_mov_immediate (operands[1],
403 <MODE>mode, 64);
404 default: gcc_unreachable ();
405 }
406 }
407 [(set_attr "simd_type" "simd_load1,simd_store1,simd_move,simd_movgp,simd_insgp,simd_move,simd_move_imm")
408 (set_attr "simd_mode" "<MODE>")]
409 )
411 (define_insn "*aarch64_simd_mov<mode>"
412 [(set (match_operand:VQ 0 "aarch64_simd_nonimmediate_operand"
413 "=w, Utv, w, ?r, ?w, ?r, w")
414 (match_operand:VQ 1 "aarch64_simd_general_operand"
415 "Utv, w, w, w, r, r, Dn"))]
416 "TARGET_SIMD
417 && (register_operand (operands[0], <MODE>mode)
418 || register_operand (operands[1], <MODE>mode))"
419 {
420 switch (which_alternative)
421 {
422 case 0:
423 return "ld1\t{%0.<Vtype>}, %1";
424 case 1:
425 return "st1\t{%1.<Vtype>}, %0";
426 case 2:
427 return "orr\t%0.<Vbtype>, %1.<Vbtype>, %1.<Vbtype>";
428 case 3:
429 case 4:
430 case 5:
431 return "#";
432 case 6:
433 return aarch64_output_simd_mov_immediate (operands[1], <MODE>mode, 128);
434 default:
435 gcc_unreachable ();
436 }
437 }
438 [(set_attr "simd_type" "simd_load1,simd_store1,simd_move,simd_movgp,simd_insgp,simd_move,simd_move_imm")
439 (set_attr "simd_mode" "<MODE>")
440 (set_attr "length" "4,4,4,8,8,8,4")]
441 )
443 (define_split
444 [(set (match_operand:VQ 0 "register_operand" "")
445 (match_operand:VQ 1 "register_operand" ""))]
446 "TARGET_SIMD && reload_completed
447 && GP_REGNUM_P (REGNO (operands[0]))
448 && GP_REGNUM_P (REGNO (operands[1]))"
449 [(set (match_dup 0) (match_dup 1))
450 (set (match_dup 2) (match_dup 3))]
451 {
452 int rdest = REGNO (operands[0]);
453 int rsrc = REGNO (operands[1]);
454 rtx dest[2], src[2];
456 dest[0] = gen_rtx_REG (DImode, rdest);
457 src[0] = gen_rtx_REG (DImode, rsrc);
458 dest[1] = gen_rtx_REG (DImode, rdest + 1);
459 src[1] = gen_rtx_REG (DImode, rsrc + 1);
461 aarch64_simd_disambiguate_copy (operands, dest, src, 2);
462 })
464 (define_split
465 [(set (match_operand:VQ 0 "register_operand" "")
466 (match_operand:VQ 1 "register_operand" ""))]
467 "TARGET_SIMD && reload_completed
468 && ((FP_REGNUM_P (REGNO (operands[0])) && GP_REGNUM_P (REGNO (operands[1])))
469 || (GP_REGNUM_P (REGNO (operands[0])) && FP_REGNUM_P (REGNO (operands[1]))))"
470 [(const_int 0)]
471 {
472 aarch64_split_simd_move (operands[0], operands[1]);
473 DONE;
474 })
476 (define_expand "aarch64_split_simd_mov<mode>"
477 [(set (match_operand:VQ 0)
478 (match_operand:VQ 1))]
479 "TARGET_SIMD"
480 {
481 rtx dst = operands[0];
482 rtx src = operands[1];
484 if (GP_REGNUM_P (REGNO (src)))
485 {
486 rtx src_low_part = gen_lowpart (<VHALF>mode, src);
487 rtx src_high_part = gen_highpart (<VHALF>mode, src);
489 emit_insn
490 (gen_move_lo_quad_<mode> (dst, src_low_part));
491 emit_insn
492 (gen_move_hi_quad_<mode> (dst, src_high_part));
493 }
495 else
496 {
497 rtx dst_low_part = gen_lowpart (<VHALF>mode, dst);
498 rtx dst_high_part = gen_highpart (<VHALF>mode, dst);
499 rtx lo = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
500 rtx hi = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
502 emit_insn
503 (gen_aarch64_simd_mov_from_<mode>low (dst_low_part, src, lo));
504 emit_insn
505 (gen_aarch64_simd_mov_from_<mode>high (dst_high_part, src, hi));
506 }
507 DONE;
508 }
509 )
511 (define_insn "aarch64_simd_mov_from_<mode>low"
512 [(set (match_operand:<VHALF> 0 "register_operand" "=r")
513 (vec_select:<VHALF>
514 (match_operand:VQ 1 "register_operand" "w")
515 (match_operand:VQ 2 "vect_par_cnst_lo_half" "")))]
516 "TARGET_SIMD && reload_completed"
517 "umov\t%0, %1.d[0]"
518 [(set_attr "simd_type" "simd_movgp")
519 (set_attr "simd_mode" "<MODE>")
520 (set_attr "length" "4")
521 ])
523 (define_insn "aarch64_simd_mov_from_<mode>high"
524 [(set (match_operand:<VHALF> 0 "register_operand" "=r")
525 (vec_select:<VHALF>
526 (match_operand:VQ 1 "register_operand" "w")
527 (match_operand:VQ 2 "vect_par_cnst_hi_half" "")))]
528 "TARGET_SIMD && reload_completed"
529 "umov\t%0, %1.d[1]"
530 [(set_attr "simd_type" "simd_movgp")
531 (set_attr "simd_mode" "<MODE>")
532 (set_attr "length" "4")
533 ])
535 (define_insn "orn<mode>3"
536 [(set (match_operand:VDQ 0 "register_operand" "=w")
537 (ior:VDQ (not:VDQ (match_operand:VDQ 1 "register_operand" "w"))
538 (match_operand:VDQ 2 "register_operand" "w")))]
539 "TARGET_SIMD"
540 "orn\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>"
541 [(set_attr "simd_type" "simd_logic")
542 (set_attr "simd_mode" "<MODE>")]
543 )
545 (define_insn "bic<mode>3"
546 [(set (match_operand:VDQ 0 "register_operand" "=w")
547 (and:VDQ (not:VDQ (match_operand:VDQ 1 "register_operand" "w"))
548 (match_operand:VDQ 2 "register_operand" "w")))]
549 "TARGET_SIMD"
550 "bic\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>"
551 [(set_attr "simd_type" "simd_logic")
552 (set_attr "simd_mode" "<MODE>")]
553 )
555 (define_insn "add<mode>3"
556 [(set (match_operand:VDQ 0 "register_operand" "=w")
557 (plus:VDQ (match_operand:VDQ 1 "register_operand" "w")
558 (match_operand:VDQ 2 "register_operand" "w")))]
559 "TARGET_SIMD"
560 "add\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
561 [(set_attr "simd_type" "simd_add")
562 (set_attr "simd_mode" "<MODE>")]
563 )
565 (define_insn "sub<mode>3"
566 [(set (match_operand:VDQ 0 "register_operand" "=w")
567 (minus:VDQ (match_operand:VDQ 1 "register_operand" "w")
568 (match_operand:VDQ 2 "register_operand" "w")))]
569 "TARGET_SIMD"
570 "sub\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
571 [(set_attr "simd_type" "simd_add")
572 (set_attr "simd_mode" "<MODE>")]
573 )
575 (define_insn "mul<mode>3"
576 [(set (match_operand:VDQM 0 "register_operand" "=w")
577 (mult:VDQM (match_operand:VDQM 1 "register_operand" "w")
578 (match_operand:VDQM 2 "register_operand" "w")))]
579 "TARGET_SIMD"
580 "mul\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
581 [(set_attr "simd_type" "simd_mul")
582 (set_attr "simd_mode" "<MODE>")]
583 )
585 (define_insn "*aarch64_mul3_elt<mode>"
586 [(set (match_operand:VMUL 0 "register_operand" "=w")
587 (mult:VMUL
588 (vec_duplicate:VMUL
589 (vec_select:<VEL>
590 (match_operand:VMUL 1 "register_operand" "<h_con>")
591 (parallel [(match_operand:SI 2 "immediate_operand")])))
592 (match_operand:VMUL 3 "register_operand" "w")))]
593 "TARGET_SIMD"
594 "<f>mul\\t%0.<Vtype>, %3.<Vtype>, %1.<Vetype>[%2]"
595 [(set_attr "simd_type" "simd_<f>mul_elt")
596 (set_attr "simd_mode" "<MODE>")]
597 )
599 (define_insn "*aarch64_mul3_elt_<vswap_width_name><mode>"
600 [(set (match_operand:VMUL_CHANGE_NLANES 0 "register_operand" "=w")
601 (mult:VMUL_CHANGE_NLANES
602 (vec_duplicate:VMUL_CHANGE_NLANES
603 (vec_select:<VEL>
604 (match_operand:<VSWAP_WIDTH> 1 "register_operand" "<h_con>")
605 (parallel [(match_operand:SI 2 "immediate_operand")])))
606 (match_operand:VMUL_CHANGE_NLANES 3 "register_operand" "w")))]
607 "TARGET_SIMD"
608 "<f>mul\\t%0.<Vtype>, %3.<Vtype>, %1.<Vetype>[%2]"
609 [(set_attr "simd_type" "simd_<f>mul_elt")
610 (set_attr "simd_mode" "<MODE>")]
611 )
613 (define_insn "*aarch64_mul3_elt_to_128df"
614 [(set (match_operand:V2DF 0 "register_operand" "=w")
615 (mult:V2DF
616 (vec_duplicate:V2DF
617 (match_operand:DF 2 "register_operand" "w"))
618 (match_operand:V2DF 1 "register_operand" "w")))]
619 "TARGET_SIMD"
620 "fmul\\t%0.2d, %1.2d, %2.d[0]"
621 [(set_attr "simd_type" "simd_fmul_elt")
622 (set_attr "simd_mode" "V2DF")]
623 )
625 (define_insn "*aarch64_mul3_elt_to_64v2df"
626 [(set (match_operand:DF 0 "register_operand" "=w")
627 (mult:DF
628 (vec_select:DF
629 (match_operand:V2DF 1 "register_operand" "w")
630 (parallel [(match_operand:SI 2 "immediate_operand")]))
631 (match_operand:DF 3 "register_operand" "w")))]
632 "TARGET_SIMD"
633 "fmul\\t%0.2d, %3.2d, %1.d[%2]"
634 [(set_attr "simd_type" "simd_fmul_elt")
635 (set_attr "simd_mode" "V2DF")]
636 )
638 (define_insn "neg<mode>2"
639 [(set (match_operand:VDQ 0 "register_operand" "=w")
640 (neg:VDQ (match_operand:VDQ 1 "register_operand" "w")))]
641 "TARGET_SIMD"
642 "neg\t%0.<Vtype>, %1.<Vtype>"
643 [(set_attr "simd_type" "simd_negabs")
644 (set_attr "simd_mode" "<MODE>")]
645 )
647 (define_insn "abs<mode>2"
648 [(set (match_operand:VDQ 0 "register_operand" "=w")
649 (abs:VDQ (match_operand:VDQ 1 "register_operand" "w")))]
650 "TARGET_SIMD"
651 "abs\t%0.<Vtype>, %1.<Vtype>"
652 [(set_attr "simd_type" "simd_negabs")
653 (set_attr "simd_mode" "<MODE>")]
654 )
656 (define_insn "abd<mode>_3"
657 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
658 (abs:VDQ_BHSI (minus:VDQ_BHSI
659 (match_operand:VDQ_BHSI 1 "register_operand" "w")
660 (match_operand:VDQ_BHSI 2 "register_operand" "w"))))]
661 "TARGET_SIMD"
662 "sabd\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
663 [(set_attr "simd_type" "simd_abd")
664 (set_attr "simd_mode" "<MODE>")]
665 )
667 (define_insn "aba<mode>_3"
668 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
669 (plus:VDQ_BHSI (abs:VDQ_BHSI (minus:VDQ_BHSI
670 (match_operand:VDQ_BHSI 1 "register_operand" "w")
671 (match_operand:VDQ_BHSI 2 "register_operand" "w")))
672 (match_operand:VDQ_BHSI 3 "register_operand" "0")))]
673 "TARGET_SIMD"
674 "saba\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
675 [(set_attr "simd_type" "simd_abd")
676 (set_attr "simd_mode" "<MODE>")]
677 )
679 (define_insn "fabd<mode>_3"
680 [(set (match_operand:VDQF 0 "register_operand" "=w")
681 (abs:VDQF (minus:VDQF
682 (match_operand:VDQF 1 "register_operand" "w")
683 (match_operand:VDQF 2 "register_operand" "w"))))]
684 "TARGET_SIMD"
685 "fabd\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
686 [(set_attr "simd_type" "simd_fabd")
687 (set_attr "simd_mode" "<MODE>")]
688 )
690 (define_insn "*fabd_scalar<mode>3"
691 [(set (match_operand:GPF 0 "register_operand" "=w")
692 (abs:GPF (minus:GPF
693 (match_operand:GPF 1 "register_operand" "w")
694 (match_operand:GPF 2 "register_operand" "w"))))]
695 "TARGET_SIMD"
696 "fabd\t%<s>0, %<s>1, %<s>2"
697 [(set_attr "simd_type" "simd_fabd")
698 (set_attr "mode" "<MODE>")]
699 )
701 (define_insn "and<mode>3"
702 [(set (match_operand:VDQ 0 "register_operand" "=w")
703 (and:VDQ (match_operand:VDQ 1 "register_operand" "w")
704 (match_operand:VDQ 2 "register_operand" "w")))]
705 "TARGET_SIMD"
706 "and\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>"
707 [(set_attr "simd_type" "simd_logic")
708 (set_attr "simd_mode" "<MODE>")]
709 )
711 (define_insn "ior<mode>3"
712 [(set (match_operand:VDQ 0 "register_operand" "=w")
713 (ior:VDQ (match_operand:VDQ 1 "register_operand" "w")
714 (match_operand:VDQ 2 "register_operand" "w")))]
715 "TARGET_SIMD"
716 "orr\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>"
717 [(set_attr "simd_type" "simd_logic")
718 (set_attr "simd_mode" "<MODE>")]
719 )
721 (define_insn "xor<mode>3"
722 [(set (match_operand:VDQ 0 "register_operand" "=w")
723 (xor:VDQ (match_operand:VDQ 1 "register_operand" "w")
724 (match_operand:VDQ 2 "register_operand" "w")))]
725 "TARGET_SIMD"
726 "eor\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>"
727 [(set_attr "simd_type" "simd_logic")
728 (set_attr "simd_mode" "<MODE>")]
729 )
731 (define_insn "one_cmpl<mode>2"
732 [(set (match_operand:VDQ 0 "register_operand" "=w")
733 (not:VDQ (match_operand:VDQ 1 "register_operand" "w")))]
734 "TARGET_SIMD"
735 "not\t%0.<Vbtype>, %1.<Vbtype>"
736 [(set_attr "simd_type" "simd_logic")
737 (set_attr "simd_mode" "<MODE>")]
738 )
740 (define_insn "aarch64_simd_vec_set<mode>"
741 [(set (match_operand:VQ_S 0 "register_operand" "=w")
742 (vec_merge:VQ_S
743 (vec_duplicate:VQ_S
744 (match_operand:<VEL> 1 "register_operand" "r"))
745 (match_operand:VQ_S 3 "register_operand" "0")
746 (match_operand:SI 2 "immediate_operand" "i")))]
747 "TARGET_SIMD"
748 "ins\t%0.<Vetype>[%p2], %w1";
749 [(set_attr "simd_type" "simd_insgp")
750 (set_attr "simd_mode" "<MODE>")]
751 )
753 (define_insn "aarch64_simd_lshr<mode>"
754 [(set (match_operand:VDQ 0 "register_operand" "=w")
755 (lshiftrt:VDQ (match_operand:VDQ 1 "register_operand" "w")
756 (match_operand:VDQ 2 "aarch64_simd_rshift_imm" "Dr")))]
757 "TARGET_SIMD"
758 "ushr\t%0.<Vtype>, %1.<Vtype>, %2"
759 [(set_attr "simd_type" "simd_shift_imm")
760 (set_attr "simd_mode" "<MODE>")]
761 )
763 (define_insn "aarch64_simd_ashr<mode>"
764 [(set (match_operand:VDQ 0 "register_operand" "=w")
765 (ashiftrt:VDQ (match_operand:VDQ 1 "register_operand" "w")
766 (match_operand:VDQ 2 "aarch64_simd_rshift_imm" "Dr")))]
767 "TARGET_SIMD"
768 "sshr\t%0.<Vtype>, %1.<Vtype>, %2"
769 [(set_attr "simd_type" "simd_shift_imm")
770 (set_attr "simd_mode" "<MODE>")]
771 )
773 (define_insn "aarch64_simd_imm_shl<mode>"
774 [(set (match_operand:VDQ 0 "register_operand" "=w")
775 (ashift:VDQ (match_operand:VDQ 1 "register_operand" "w")
776 (match_operand:VDQ 2 "aarch64_simd_lshift_imm" "Dl")))]
777 "TARGET_SIMD"
778 "shl\t%0.<Vtype>, %1.<Vtype>, %2"
779 [(set_attr "simd_type" "simd_shift_imm")
780 (set_attr "simd_mode" "<MODE>")]
781 )
783 (define_insn "aarch64_simd_reg_sshl<mode>"
784 [(set (match_operand:VDQ 0 "register_operand" "=w")
785 (ashift:VDQ (match_operand:VDQ 1 "register_operand" "w")
786 (match_operand:VDQ 2 "register_operand" "w")))]
787 "TARGET_SIMD"
788 "sshl\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
789 [(set_attr "simd_type" "simd_shift")
790 (set_attr "simd_mode" "<MODE>")]
791 )
793 (define_insn "aarch64_simd_reg_shl<mode>_unsigned"
794 [(set (match_operand:VDQ 0 "register_operand" "=w")
795 (unspec:VDQ [(match_operand:VDQ 1 "register_operand" "w")
796 (match_operand:VDQ 2 "register_operand" "w")]
797 UNSPEC_ASHIFT_UNSIGNED))]
798 "TARGET_SIMD"
799 "ushl\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
800 [(set_attr "simd_type" "simd_shift")
801 (set_attr "simd_mode" "<MODE>")]
802 )
804 (define_insn "aarch64_simd_reg_shl<mode>_signed"
805 [(set (match_operand:VDQ 0 "register_operand" "=w")
806 (unspec:VDQ [(match_operand:VDQ 1 "register_operand" "w")
807 (match_operand:VDQ 2 "register_operand" "w")]
808 UNSPEC_ASHIFT_SIGNED))]
809 "TARGET_SIMD"
810 "sshl\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
811 [(set_attr "simd_type" "simd_shift")
812 (set_attr "simd_mode" "<MODE>")]
813 )
815 (define_expand "ashl<mode>3"
816 [(match_operand:VDQ 0 "register_operand" "")
817 (match_operand:VDQ 1 "register_operand" "")
818 (match_operand:SI 2 "general_operand" "")]
819 "TARGET_SIMD"
820 {
821 int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
822 int shift_amount;
824 if (CONST_INT_P (operands[2]))
825 {
826 shift_amount = INTVAL (operands[2]);
827 if (shift_amount >= 0 && shift_amount < bit_width)
828 {
829 rtx tmp = aarch64_simd_gen_const_vector_dup (<MODE>mode,
830 shift_amount);
831 emit_insn (gen_aarch64_simd_imm_shl<mode> (operands[0],
832 operands[1],
833 tmp));
834 DONE;
835 }
836 else
837 {
838 operands[2] = force_reg (SImode, operands[2]);
839 }
840 }
841 else if (MEM_P (operands[2]))
842 {
843 operands[2] = force_reg (SImode, operands[2]);
844 }
846 if (REG_P (operands[2]))
847 {
848 rtx tmp = gen_reg_rtx (<MODE>mode);
849 emit_insn (gen_aarch64_simd_dup<mode> (tmp,
850 convert_to_mode (<VEL>mode,
851 operands[2],
852 0)));
853 emit_insn (gen_aarch64_simd_reg_sshl<mode> (operands[0], operands[1],
854 tmp));
855 DONE;
856 }
857 else
858 FAIL;
859 }
860 )
862 (define_expand "lshr<mode>3"
863 [(match_operand:VDQ 0 "register_operand" "")
864 (match_operand:VDQ 1 "register_operand" "")
865 (match_operand:SI 2 "general_operand" "")]
866 "TARGET_SIMD"
867 {
868 int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
869 int shift_amount;
871 if (CONST_INT_P (operands[2]))
872 {
873 shift_amount = INTVAL (operands[2]);
874 if (shift_amount > 0 && shift_amount <= bit_width)
875 {
876 rtx tmp = aarch64_simd_gen_const_vector_dup (<MODE>mode,
877 shift_amount);
878 emit_insn (gen_aarch64_simd_lshr<mode> (operands[0],
879 operands[1],
880 tmp));
881 DONE;
882 }
883 else
884 operands[2] = force_reg (SImode, operands[2]);
885 }
886 else if (MEM_P (operands[2]))
887 {
888 operands[2] = force_reg (SImode, operands[2]);
889 }
891 if (REG_P (operands[2]))
892 {
893 rtx tmp = gen_reg_rtx (SImode);
894 rtx tmp1 = gen_reg_rtx (<MODE>mode);
895 emit_insn (gen_negsi2 (tmp, operands[2]));
896 emit_insn (gen_aarch64_simd_dup<mode> (tmp1,
897 convert_to_mode (<VEL>mode,
898 tmp, 0)));
899 emit_insn (gen_aarch64_simd_reg_shl<mode>_unsigned (operands[0],
900 operands[1],
901 tmp1));
902 DONE;
903 }
904 else
905 FAIL;
906 }
907 )
909 (define_expand "ashr<mode>3"
910 [(match_operand:VDQ 0 "register_operand" "")
911 (match_operand:VDQ 1 "register_operand" "")
912 (match_operand:SI 2 "general_operand" "")]
913 "TARGET_SIMD"
914 {
915 int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
916 int shift_amount;
918 if (CONST_INT_P (operands[2]))
919 {
920 shift_amount = INTVAL (operands[2]);
921 if (shift_amount > 0 && shift_amount <= bit_width)
922 {
923 rtx tmp = aarch64_simd_gen_const_vector_dup (<MODE>mode,
924 shift_amount);
925 emit_insn (gen_aarch64_simd_ashr<mode> (operands[0],
926 operands[1],
927 tmp));
928 DONE;
929 }
930 else
931 operands[2] = force_reg (SImode, operands[2]);
932 }
933 else if (MEM_P (operands[2]))
934 {
935 operands[2] = force_reg (SImode, operands[2]);
936 }
938 if (REG_P (operands[2]))
939 {
940 rtx tmp = gen_reg_rtx (SImode);
941 rtx tmp1 = gen_reg_rtx (<MODE>mode);
942 emit_insn (gen_negsi2 (tmp, operands[2]));
943 emit_insn (gen_aarch64_simd_dup<mode> (tmp1,
944 convert_to_mode (<VEL>mode,
945 tmp, 0)));
946 emit_insn (gen_aarch64_simd_reg_shl<mode>_signed (operands[0],
947 operands[1],
948 tmp1));
949 DONE;
950 }
951 else
952 FAIL;
953 }
954 )
956 (define_expand "vashl<mode>3"
957 [(match_operand:VDQ 0 "register_operand" "")
958 (match_operand:VDQ 1 "register_operand" "")
959 (match_operand:VDQ 2 "register_operand" "")]
960 "TARGET_SIMD"
961 {
962 emit_insn (gen_aarch64_simd_reg_sshl<mode> (operands[0], operands[1],
963 operands[2]));
964 DONE;
965 })
967 ;; Using mode VQ_S as there is no V2DImode neg!
968 ;; Negating individual lanes most certainly offsets the
969 ;; gain from vectorization.
970 (define_expand "vashr<mode>3"
971 [(match_operand:VQ_S 0 "register_operand" "")
972 (match_operand:VQ_S 1 "register_operand" "")
973 (match_operand:VQ_S 2 "register_operand" "")]
974 "TARGET_SIMD"
975 {
976 rtx neg = gen_reg_rtx (<MODE>mode);
977 emit (gen_neg<mode>2 (neg, operands[2]));
978 emit_insn (gen_aarch64_simd_reg_shl<mode>_signed (operands[0], operands[1],
979 neg));
980 DONE;
981 })
983 (define_expand "vlshr<mode>3"
984 [(match_operand:VQ_S 0 "register_operand" "")
985 (match_operand:VQ_S 1 "register_operand" "")
986 (match_operand:VQ_S 2 "register_operand" "")]
987 "TARGET_SIMD"
988 {
989 rtx neg = gen_reg_rtx (<MODE>mode);
990 emit (gen_neg<mode>2 (neg, operands[2]));
991 emit_insn (gen_aarch64_simd_reg_shl<mode>_unsigned (operands[0], operands[1],
992 neg));
993 DONE;
994 })
996 (define_expand "vec_set<mode>"
997 [(match_operand:VQ_S 0 "register_operand" "+w")
998 (match_operand:<VEL> 1 "register_operand" "r")
999 (match_operand:SI 2 "immediate_operand" "")]
1000 "TARGET_SIMD"
1001 {
1002 HOST_WIDE_INT elem = (HOST_WIDE_INT) 1 << INTVAL (operands[2]);
1003 emit_insn (gen_aarch64_simd_vec_set<mode> (operands[0], operands[1],
1004 GEN_INT (elem), operands[0]));
1005 DONE;
1006 }
1007 )
1009 (define_insn "aarch64_simd_vec_setv2di"
1010 [(set (match_operand:V2DI 0 "register_operand" "=w")
1011 (vec_merge:V2DI
1012 (vec_duplicate:V2DI
1013 (match_operand:DI 1 "register_operand" "r"))
1014 (match_operand:V2DI 3 "register_operand" "0")
1015 (match_operand:SI 2 "immediate_operand" "i")))]
1016 "TARGET_SIMD"
1017 "ins\t%0.d[%p2], %1";
1018 [(set_attr "simd_type" "simd_insgp")
1019 (set_attr "simd_mode" "V2DI")]
1020 )
1022 (define_expand "vec_setv2di"
1023 [(match_operand:V2DI 0 "register_operand" "+w")
1024 (match_operand:DI 1 "register_operand" "r")
1025 (match_operand:SI 2 "immediate_operand" "")]
1026 "TARGET_SIMD"
1027 {
1028 HOST_WIDE_INT elem = (HOST_WIDE_INT) 1 << INTVAL (operands[2]);
1029 emit_insn (gen_aarch64_simd_vec_setv2di (operands[0], operands[1],
1030 GEN_INT (elem), operands[0]));
1031 DONE;
1032 }
1033 )
1035 (define_insn "aarch64_simd_vec_set<mode>"
1036 [(set (match_operand:VDQF 0 "register_operand" "=w")
1037 (vec_merge:VDQF
1038 (vec_duplicate:VDQF
1039 (match_operand:<VEL> 1 "register_operand" "w"))
1040 (match_operand:VDQF 3 "register_operand" "0")
1041 (match_operand:SI 2 "immediate_operand" "i")))]
1042 "TARGET_SIMD"
1043 "ins\t%0.<Vetype>[%p2], %1.<Vetype>[0]";
1044 [(set_attr "simd_type" "simd_ins")
1045 (set_attr "simd_mode" "<MODE>")]
1046 )
1048 (define_expand "vec_set<mode>"
1049 [(match_operand:VDQF 0 "register_operand" "+w")
1050 (match_operand:<VEL> 1 "register_operand" "w")
1051 (match_operand:SI 2 "immediate_operand" "")]
1052 "TARGET_SIMD"
1053 {
1054 HOST_WIDE_INT elem = (HOST_WIDE_INT) 1 << INTVAL (operands[2]);
1055 emit_insn (gen_aarch64_simd_vec_set<mode> (operands[0], operands[1],
1056 GEN_INT (elem), operands[0]));
1057 DONE;
1058 }
1059 )
1062 (define_insn "aarch64_mla<mode>"
1063 [(set (match_operand:VQ_S 0 "register_operand" "=w")
1064 (plus:VQ_S (mult:VQ_S (match_operand:VQ_S 2 "register_operand" "w")
1065 (match_operand:VQ_S 3 "register_operand" "w"))
1066 (match_operand:VQ_S 1 "register_operand" "0")))]
1067 "TARGET_SIMD"
1068 "mla\t%0.<Vtype>, %2.<Vtype>, %3.<Vtype>"
1069 [(set_attr "simd_type" "simd_mla")
1070 (set_attr "simd_mode" "<MODE>")]
1071 )
1073 (define_insn "aarch64_mls<mode>"
1074 [(set (match_operand:VQ_S 0 "register_operand" "=w")
1075 (minus:VQ_S (match_operand:VQ_S 1 "register_operand" "0")
1076 (mult:VQ_S (match_operand:VQ_S 2 "register_operand" "w")
1077 (match_operand:VQ_S 3 "register_operand" "w"))))]
1078 "TARGET_SIMD"
1079 "mls\t%0.<Vtype>, %2.<Vtype>, %3.<Vtype>"
1080 [(set_attr "simd_type" "simd_mla")
1081 (set_attr "simd_mode" "<MODE>")]
1082 )
1084 ;; Max/Min operations.
1085 (define_insn "<su><maxmin><mode>3"
1086 [(set (match_operand:VQ_S 0 "register_operand" "=w")
1087 (MAXMIN:VQ_S (match_operand:VQ_S 1 "register_operand" "w")
1088 (match_operand:VQ_S 2 "register_operand" "w")))]
1089 "TARGET_SIMD"
1090 "<su><maxmin>\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1091 [(set_attr "simd_type" "simd_minmax")
1092 (set_attr "simd_mode" "<MODE>")]
1093 )
1095 ;; Move into low-half clearing high half to 0.
1097 (define_insn "move_lo_quad_<mode>"
1098 [(set (match_operand:VQ 0 "register_operand" "=w,w,w")
1099 (vec_concat:VQ
1100 (match_operand:<VHALF> 1 "register_operand" "w,r,r")
1101 (vec_duplicate:<VHALF> (const_int 0))))]
1102 "TARGET_SIMD"
1103 "@
1104 dup\\t%d0, %1.d[0]
1105 fmov\\t%d0, %1
1106 dup\\t%d0, %1"
1107 [(set_attr "v8type" "*,fmov,*")
1108 (set_attr "type" "*,fmov,*")
1109 (set_attr "simd_type" "simd_dup,*,simd_dup")
1110 (set_attr "simd_mode" "<MODE>")
1111 (set_attr "simd" "yes,*,yes")
1112 (set_attr "fp" "*,yes,*")
1113 (set_attr "length" "4")]
1114 )
1116 ;; Move into high-half.
1118 (define_insn "aarch64_simd_move_hi_quad_<mode>"
1119 [(set (match_operand:VQ 0 "register_operand" "+w,w")
1120 (vec_concat:VQ
1121 (vec_select:<VHALF>
1122 (match_dup 0)
1123 (match_operand:VQ 2 "vect_par_cnst_lo_half" ""))
1124 (match_operand:<VHALF> 1 "register_operand" "w,r")))]
1125 "TARGET_SIMD"
1126 "@
1127 ins\\t%0.d[1], %1.d[0]
1128 ins\\t%0.d[1], %1"
1129 [(set_attr "simd_type" "simd_ins,simd_ins")
1130 (set_attr "simd_mode" "<MODE>")
1131 (set_attr "length" "4")]
1132 )
1134 (define_expand "move_hi_quad_<mode>"
1135 [(match_operand:VQ 0 "register_operand" "")
1136 (match_operand:<VHALF> 1 "register_operand" "")]
1137 "TARGET_SIMD"
1138 {
1139 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
1140 emit_insn (gen_aarch64_simd_move_hi_quad_<mode> (operands[0],
1141 operands[1], p));
1142 DONE;
1143 })
1145 ;; Narrowing operations.
1147 ;; For doubles.
1148 (define_insn "aarch64_simd_vec_pack_trunc_<mode>"
1149 [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
1150 (truncate:<VNARROWQ> (match_operand:VQN 1 "register_operand" "w")))]
1151 "TARGET_SIMD"
1152 "xtn\\t%0.<Vntype>, %1.<Vtype>"
1153 [(set_attr "simd_type" "simd_shiftn_imm")
1154 (set_attr "simd_mode" "<MODE>")]
1155 )
1157 (define_expand "vec_pack_trunc_<mode>"
1158 [(match_operand:<VNARROWD> 0 "register_operand" "")
1159 (match_operand:VDN 1 "register_operand" "")
1160 (match_operand:VDN 2 "register_operand" "")]
1161 "TARGET_SIMD"
1162 {
1163 rtx tempreg = gen_reg_rtx (<VDBL>mode);
1165 emit_insn (gen_move_lo_quad_<Vdbl> (tempreg, operands[1]));
1166 emit_insn (gen_move_hi_quad_<Vdbl> (tempreg, operands[2]));
1167 emit_insn (gen_aarch64_simd_vec_pack_trunc_<Vdbl> (operands[0], tempreg));
1168 DONE;
1169 })
1171 ;; For quads.
1173 (define_insn "vec_pack_trunc_<mode>"
1174 [(set (match_operand:<VNARROWQ2> 0 "register_operand" "+&w")
1175 (vec_concat:<VNARROWQ2>
1176 (truncate:<VNARROWQ> (match_operand:VQN 1 "register_operand" "w"))
1177 (truncate:<VNARROWQ> (match_operand:VQN 2 "register_operand" "w"))))]
1178 "TARGET_SIMD"
1179 "xtn\\t%0.<Vntype>, %1.<Vtype>\;xtn2\\t%0.<V2ntype>, %2.<Vtype>"
1180 [(set_attr "simd_type" "simd_shiftn2_imm")
1181 (set_attr "simd_mode" "<MODE>")
1182 (set_attr "length" "8")]
1183 )
1185 ;; Widening operations.
1187 (define_insn "aarch64_simd_vec_unpack<su>_lo_<mode>"
1188 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1189 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1190 (match_operand:VQW 1 "register_operand" "w")
1191 (match_operand:VQW 2 "vect_par_cnst_lo_half" "")
1192 )))]
1193 "TARGET_SIMD"
1194 "<su>shll %0.<Vwtype>, %1.<Vhalftype>, 0"
1195 [(set_attr "simd_type" "simd_shiftl_imm")
1196 (set_attr "simd_mode" "<MODE>")]
1197 )
1199 (define_insn "aarch64_simd_vec_unpack<su>_hi_<mode>"
1200 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1201 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1202 (match_operand:VQW 1 "register_operand" "w")
1203 (match_operand:VQW 2 "vect_par_cnst_hi_half" "")
1204 )))]
1205 "TARGET_SIMD"
1206 "<su>shll2 %0.<Vwtype>, %1.<Vtype>, 0"
1207 [(set_attr "simd_type" "simd_shiftl_imm")
1208 (set_attr "simd_mode" "<MODE>")]
1209 )
1211 (define_expand "vec_unpack<su>_hi_<mode>"
1212 [(match_operand:<VWIDE> 0 "register_operand" "")
1213 (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand"))]
1214 "TARGET_SIMD"
1215 {
1216 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
1217 emit_insn (gen_aarch64_simd_vec_unpack<su>_hi_<mode> (operands[0],
1218 operands[1], p));
1219 DONE;
1220 }
1221 )
1223 (define_expand "vec_unpack<su>_lo_<mode>"
1224 [(match_operand:<VWIDE> 0 "register_operand" "")
1225 (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand" ""))]
1226 "TARGET_SIMD"
1227 {
1228 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
1229 emit_insn (gen_aarch64_simd_vec_unpack<su>_lo_<mode> (operands[0],
1230 operands[1], p));
1231 DONE;
1232 }
1233 )
1235 ;; Widening arithmetic.
1237 (define_insn "*aarch64_<su>mlal_lo<mode>"
1238 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1239 (plus:<VWIDE>
1240 (mult:<VWIDE>
1241 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1242 (match_operand:VQW 2 "register_operand" "w")
1243 (match_operand:VQW 3 "vect_par_cnst_lo_half" "")))
1244 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1245 (match_operand:VQW 4 "register_operand" "w")
1246 (match_dup 3))))
1247 (match_operand:<VWIDE> 1 "register_operand" "0")))]
1248 "TARGET_SIMD"
1249 "<su>mlal\t%0.<Vwtype>, %2.<Vhalftype>, %4.<Vhalftype>"
1250 [(set_attr "simd_type" "simd_mlal")
1251 (set_attr "simd_mode" "<MODE>")]
1252 )
1254 (define_insn "*aarch64_<su>mlal_hi<mode>"
1255 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1256 (plus:<VWIDE>
1257 (mult:<VWIDE>
1258 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1259 (match_operand:VQW 2 "register_operand" "w")
1260 (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))
1261 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1262 (match_operand:VQW 4 "register_operand" "w")
1263 (match_dup 3))))
1264 (match_operand:<VWIDE> 1 "register_operand" "0")))]
1265 "TARGET_SIMD"
1266 "<su>mlal2\t%0.<Vwtype>, %2.<Vtype>, %4.<Vtype>"
1267 [(set_attr "simd_type" "simd_mlal")
1268 (set_attr "simd_mode" "<MODE>")]
1269 )
1271 (define_insn "*aarch64_<su>mlsl_lo<mode>"
1272 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1273 (minus:<VWIDE>
1274 (match_operand:<VWIDE> 1 "register_operand" "0")
1275 (mult:<VWIDE>
1276 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1277 (match_operand:VQW 2 "register_operand" "w")
1278 (match_operand:VQW 3 "vect_par_cnst_lo_half" "")))
1279 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1280 (match_operand:VQW 4 "register_operand" "w")
1281 (match_dup 3))))))]
1282 "TARGET_SIMD"
1283 "<su>mlsl\t%0.<Vwtype>, %2.<Vhalftype>, %4.<Vhalftype>"
1284 [(set_attr "simd_type" "simd_mlal")
1285 (set_attr "simd_mode" "<MODE>")]
1286 )
1288 (define_insn "*aarch64_<su>mlsl_hi<mode>"
1289 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1290 (minus:<VWIDE>
1291 (match_operand:<VWIDE> 1 "register_operand" "0")
1292 (mult:<VWIDE>
1293 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1294 (match_operand:VQW 2 "register_operand" "w")
1295 (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))
1296 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1297 (match_operand:VQW 4 "register_operand" "w")
1298 (match_dup 3))))))]
1299 "TARGET_SIMD"
1300 "<su>mlsl2\t%0.<Vwtype>, %2.<Vtype>, %4.<Vtype>"
1301 [(set_attr "simd_type" "simd_mlal")
1302 (set_attr "simd_mode" "<MODE>")]
1303 )
1305 (define_insn "*aarch64_<su>mlal<mode>"
1306 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1307 (plus:<VWIDE>
1308 (mult:<VWIDE>
1309 (ANY_EXTEND:<VWIDE>
1310 (match_operand:VDW 1 "register_operand" "w"))
1311 (ANY_EXTEND:<VWIDE>
1312 (match_operand:VDW 2 "register_operand" "w")))
1313 (match_operand:<VWIDE> 3 "register_operand" "0")))]
1314 "TARGET_SIMD"
1315 "<su>mlal\t%0.<Vwtype>, %1.<Vtype>, %2.<Vtype>"
1316 [(set_attr "simd_type" "simd_mlal")
1317 (set_attr "simd_mode" "<MODE>")]
1318 )
1320 (define_insn "*aarch64_<su>mlsl<mode>"
1321 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1322 (minus:<VWIDE>
1323 (match_operand:<VWIDE> 1 "register_operand" "0")
1324 (mult:<VWIDE>
1325 (ANY_EXTEND:<VWIDE>
1326 (match_operand:VDW 2 "register_operand" "w"))
1327 (ANY_EXTEND:<VWIDE>
1328 (match_operand:VDW 3 "register_operand" "w")))))]
1329 "TARGET_SIMD"
1330 "<su>mlsl\t%0.<Vwtype>, %2.<Vtype>, %3.<Vtype>"
1331 [(set_attr "simd_type" "simd_mlal")
1332 (set_attr "simd_mode" "<MODE>")]
1333 )
1335 (define_insn "aarch64_simd_vec_<su>mult_lo_<mode>"
1336 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1337 (mult:<VWIDE> (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1338 (match_operand:VQW 1 "register_operand" "w")
1339 (match_operand:VQW 3 "vect_par_cnst_lo_half" "")))
1340 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1341 (match_operand:VQW 2 "register_operand" "w")
1342 (match_dup 3)))))]
1343 "TARGET_SIMD"
1344 "<su>mull\\t%0.<Vwtype>, %1.<Vhalftype>, %2.<Vhalftype>"
1345 [(set_attr "simd_type" "simd_mull")
1346 (set_attr "simd_mode" "<MODE>")]
1347 )
1349 (define_expand "vec_widen_<su>mult_lo_<mode>"
1350 [(match_operand:<VWIDE> 0 "register_operand" "")
1351 (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand" ""))
1352 (ANY_EXTEND:<VWIDE> (match_operand:VQW 2 "register_operand" ""))]
1353 "TARGET_SIMD"
1354 {
1355 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, false);
1356 emit_insn (gen_aarch64_simd_vec_<su>mult_lo_<mode> (operands[0],
1357 operands[1],
1358 operands[2], p));
1359 DONE;
1360 }
1361 )
1363 (define_insn "aarch64_simd_vec_<su>mult_hi_<mode>"
1364 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
1365 (mult:<VWIDE> (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1366 (match_operand:VQW 1 "register_operand" "w")
1367 (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))
1368 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
1369 (match_operand:VQW 2 "register_operand" "w")
1370 (match_dup 3)))))]
1371 "TARGET_SIMD"
1372 "<su>mull2\\t%0.<Vwtype>, %1.<Vtype>, %2.<Vtype>"
1373 [(set_attr "simd_type" "simd_mull")
1374 (set_attr "simd_mode" "<MODE>")]
1375 )
1377 (define_expand "vec_widen_<su>mult_hi_<mode>"
1378 [(match_operand:<VWIDE> 0 "register_operand" "")
1379 (ANY_EXTEND:<VWIDE> (match_operand:VQW 1 "register_operand" ""))
1380 (ANY_EXTEND:<VWIDE> (match_operand:VQW 2 "register_operand" ""))]
1381 "TARGET_SIMD"
1382 {
1383 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
1384 emit_insn (gen_aarch64_simd_vec_<su>mult_hi_<mode> (operands[0],
1385 operands[1],
1386 operands[2], p));
1387 DONE;
1389 }
1390 )
1392 ;; FP vector operations.
1393 ;; AArch64 AdvSIMD supports single-precision (32-bit) and
1394 ;; double-precision (64-bit) floating-point data types and arithmetic as
1395 ;; defined by the IEEE 754-2008 standard. This makes them vectorizable
1396 ;; without the need for -ffast-math or -funsafe-math-optimizations.
1397 ;;
1398 ;; Floating-point operations can raise an exception. Vectorizing such
1399 ;; operations are safe because of reasons explained below.
1400 ;;
1401 ;; ARMv8 permits an extension to enable trapped floating-point
1402 ;; exception handling, however this is an optional feature. In the
1403 ;; event of a floating-point exception being raised by vectorised
1404 ;; code then:
1405 ;; 1. If trapped floating-point exceptions are available, then a trap
1406 ;; will be taken when any lane raises an enabled exception. A trap
1407 ;; handler may determine which lane raised the exception.
1408 ;; 2. Alternatively a sticky exception flag is set in the
1409 ;; floating-point status register (FPSR). Software may explicitly
1410 ;; test the exception flags, in which case the tests will either
1411 ;; prevent vectorisation, allowing precise identification of the
1412 ;; failing operation, or if tested outside of vectorisable regions
1413 ;; then the specific operation and lane are not of interest.
1415 ;; FP arithmetic operations.
1417 (define_insn "add<mode>3"
1418 [(set (match_operand:VDQF 0 "register_operand" "=w")
1419 (plus:VDQF (match_operand:VDQF 1 "register_operand" "w")
1420 (match_operand:VDQF 2 "register_operand" "w")))]
1421 "TARGET_SIMD"
1422 "fadd\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1423 [(set_attr "simd_type" "simd_fadd")
1424 (set_attr "simd_mode" "<MODE>")]
1425 )
1427 (define_insn "sub<mode>3"
1428 [(set (match_operand:VDQF 0 "register_operand" "=w")
1429 (minus:VDQF (match_operand:VDQF 1 "register_operand" "w")
1430 (match_operand:VDQF 2 "register_operand" "w")))]
1431 "TARGET_SIMD"
1432 "fsub\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1433 [(set_attr "simd_type" "simd_fadd")
1434 (set_attr "simd_mode" "<MODE>")]
1435 )
1437 (define_insn "mul<mode>3"
1438 [(set (match_operand:VDQF 0 "register_operand" "=w")
1439 (mult:VDQF (match_operand:VDQF 1 "register_operand" "w")
1440 (match_operand:VDQF 2 "register_operand" "w")))]
1441 "TARGET_SIMD"
1442 "fmul\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1443 [(set_attr "simd_type" "simd_fmul")
1444 (set_attr "simd_mode" "<MODE>")]
1445 )
1447 (define_insn "div<mode>3"
1448 [(set (match_operand:VDQF 0 "register_operand" "=w")
1449 (div:VDQF (match_operand:VDQF 1 "register_operand" "w")
1450 (match_operand:VDQF 2 "register_operand" "w")))]
1451 "TARGET_SIMD"
1452 "fdiv\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1453 [(set_attr "simd_type" "simd_fdiv")
1454 (set_attr "simd_mode" "<MODE>")]
1455 )
1457 (define_insn "neg<mode>2"
1458 [(set (match_operand:VDQF 0 "register_operand" "=w")
1459 (neg:VDQF (match_operand:VDQF 1 "register_operand" "w")))]
1460 "TARGET_SIMD"
1461 "fneg\\t%0.<Vtype>, %1.<Vtype>"
1462 [(set_attr "simd_type" "simd_fnegabs")
1463 (set_attr "simd_mode" "<MODE>")]
1464 )
1466 (define_insn "abs<mode>2"
1467 [(set (match_operand:VDQF 0 "register_operand" "=w")
1468 (abs:VDQF (match_operand:VDQF 1 "register_operand" "w")))]
1469 "TARGET_SIMD"
1470 "fabs\\t%0.<Vtype>, %1.<Vtype>"
1471 [(set_attr "simd_type" "simd_fnegabs")
1472 (set_attr "simd_mode" "<MODE>")]
1473 )
1475 (define_insn "fma<mode>4"
1476 [(set (match_operand:VDQF 0 "register_operand" "=w")
1477 (fma:VDQF (match_operand:VDQF 1 "register_operand" "w")
1478 (match_operand:VDQF 2 "register_operand" "w")
1479 (match_operand:VDQF 3 "register_operand" "0")))]
1480 "TARGET_SIMD"
1481 "fmla\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1482 [(set_attr "simd_type" "simd_fmla")
1483 (set_attr "simd_mode" "<MODE>")]
1484 )
1486 ;; Vector versions of the floating-point frint patterns.
1487 ;; Expands to btrunc, ceil, floor, nearbyint, rint, round.
1488 (define_insn "<frint_pattern><mode>2"
1489 [(set (match_operand:VDQF 0 "register_operand" "=w")
1490 (unspec:VDQF [(match_operand:VDQF 1 "register_operand" "w")]
1491 FRINT))]
1492 "TARGET_SIMD"
1493 "frint<frint_suffix>\\t%0.<Vtype>, %1.<Vtype>"
1494 [(set_attr "simd_type" "simd_frint")
1495 (set_attr "simd_mode" "<MODE>")]
1496 )
1498 ;; Vector versions of the fcvt standard patterns.
1499 ;; Expands to lbtrunc, lround, lceil, lfloor
1500 (define_insn "l<fcvt_pattern><su_optab><VDQF:mode><fcvt_target>2"
1501 [(set (match_operand:<FCVT_TARGET> 0 "register_operand" "=w")
1502 (FIXUORS:<FCVT_TARGET> (unspec:<FCVT_TARGET>
1503 [(match_operand:VDQF 1 "register_operand" "w")]
1504 FCVT)))]
1505 "TARGET_SIMD"
1506 "fcvt<frint_suffix><su>\\t%0.<Vtype>, %1.<Vtype>"
1507 [(set_attr "simd_type" "simd_fcvti")
1508 (set_attr "simd_mode" "<MODE>")]
1509 )
1511 (define_expand "<optab><VDQF:mode><fcvt_target>2"
1512 [(set (match_operand:<FCVT_TARGET> 0 "register_operand")
1513 (FIXUORS:<FCVT_TARGET> (unspec:<FCVT_TARGET>
1514 [(match_operand:VDQF 1 "register_operand")]
1515 UNSPEC_FRINTZ)))]
1516 "TARGET_SIMD"
1517 {})
1519 (define_expand "<fix_trunc_optab><VDQF:mode><fcvt_target>2"
1520 [(set (match_operand:<FCVT_TARGET> 0 "register_operand")
1521 (FIXUORS:<FCVT_TARGET> (unspec:<FCVT_TARGET>
1522 [(match_operand:VDQF 1 "register_operand")]
1523 UNSPEC_FRINTZ)))]
1524 "TARGET_SIMD"
1525 {})
1527 (define_expand "ftrunc<VDQF:mode>2"
1528 [(set (match_operand:VDQF 0 "register_operand")
1529 (unspec:VDQF [(match_operand:VDQF 1 "register_operand")]
1530 UNSPEC_FRINTZ))]
1531 "TARGET_SIMD"
1532 {})
1534 (define_insn "<optab><fcvt_target><VDQF:mode>2"
1535 [(set (match_operand:VDQF 0 "register_operand" "=w")
1536 (FLOATUORS:VDQF
1537 (match_operand:<FCVT_TARGET> 1 "register_operand" "w")))]
1538 "TARGET_SIMD"
1539 "<su_optab>cvtf\\t%0.<Vtype>, %1.<Vtype>"
1540 [(set_attr "simd_type" "simd_icvtf")
1541 (set_attr "simd_mode" "<MODE>")]
1542 )
1544 ;; Conversions between vectors of floats and doubles.
1545 ;; Contains a mix of patterns to match standard pattern names
1546 ;; and those for intrinsics.
1548 ;; Float widening operations.
1550 (define_insn "vec_unpacks_lo_v4sf"
1551 [(set (match_operand:V2DF 0 "register_operand" "=w")
1552 (float_extend:V2DF
1553 (vec_select:V2SF
1554 (match_operand:V4SF 1 "register_operand" "w")
1555 (parallel [(const_int 0) (const_int 1)])
1556 )))]
1557 "TARGET_SIMD"
1558 "fcvtl\\t%0.2d, %1.2s"
1559 [(set_attr "simd_type" "simd_fcvtl")
1560 (set_attr "simd_mode" "V2DF")]
1561 )
1563 (define_insn "aarch64_float_extend_lo_v2df"
1564 [(set (match_operand:V2DF 0 "register_operand" "=w")
1565 (float_extend:V2DF
1566 (match_operand:V2SF 1 "register_operand" "w")))]
1567 "TARGET_SIMD"
1568 "fcvtl\\t%0.2d, %1.2s"
1569 [(set_attr "simd_type" "simd_fcvtl")
1570 (set_attr "simd_mode" "V2DF")]
1571 )
1573 (define_insn "vec_unpacks_hi_v4sf"
1574 [(set (match_operand:V2DF 0 "register_operand" "=w")
1575 (float_extend:V2DF
1576 (vec_select:V2SF
1577 (match_operand:V4SF 1 "register_operand" "w")
1578 (parallel [(const_int 2) (const_int 3)])
1579 )))]
1580 "TARGET_SIMD"
1581 "fcvtl2\\t%0.2d, %1.4s"
1582 [(set_attr "simd_type" "simd_fcvtl")
1583 (set_attr "simd_mode" "V2DF")]
1584 )
1586 ;; Float narrowing operations.
1588 (define_insn "aarch64_float_truncate_lo_v2sf"
1589 [(set (match_operand:V2SF 0 "register_operand" "=w")
1590 (float_truncate:V2SF
1591 (match_operand:V2DF 1 "register_operand" "w")))]
1592 "TARGET_SIMD"
1593 "fcvtn\\t%0.2s, %1.2d"
1594 [(set_attr "simd_type" "simd_fcvtl")
1595 (set_attr "simd_mode" "V2SF")]
1596 )
1598 (define_insn "aarch64_float_truncate_hi_v4sf"
1599 [(set (match_operand:V4SF 0 "register_operand" "=w")
1600 (vec_concat:V4SF
1601 (match_operand:V2SF 1 "register_operand" "0")
1602 (float_truncate:V2SF
1603 (match_operand:V2DF 2 "register_operand" "w"))))]
1604 "TARGET_SIMD"
1605 "fcvtn2\\t%0.4s, %2.2d"
1606 [(set_attr "simd_type" "simd_fcvtl")
1607 (set_attr "simd_mode" "V4SF")]
1608 )
1610 (define_expand "vec_pack_trunc_v2df"
1611 [(set (match_operand:V4SF 0 "register_operand")
1612 (vec_concat:V4SF
1613 (float_truncate:V2SF
1614 (match_operand:V2DF 1 "register_operand"))
1615 (float_truncate:V2SF
1616 (match_operand:V2DF 2 "register_operand"))
1617 ))]
1618 "TARGET_SIMD"
1619 {
1620 rtx tmp = gen_reg_rtx (V2SFmode);
1621 emit_insn (gen_aarch64_float_truncate_lo_v2sf (tmp, operands[1]));
1622 emit_insn (gen_aarch64_float_truncate_hi_v4sf (operands[0],
1623 tmp, operands[2]));
1624 DONE;
1625 }
1626 )
1628 (define_expand "vec_pack_trunc_df"
1629 [(set (match_operand:V2SF 0 "register_operand")
1630 (vec_concat:V2SF
1631 (float_truncate:SF
1632 (match_operand:DF 1 "register_operand"))
1633 (float_truncate:SF
1634 (match_operand:DF 2 "register_operand"))
1635 ))]
1636 "TARGET_SIMD"
1637 {
1638 rtx tmp = gen_reg_rtx (V2SFmode);
1639 emit_insn (gen_move_lo_quad_v2df (tmp, operands[1]));
1640 emit_insn (gen_move_hi_quad_v2df (tmp, operands[2]));
1641 emit_insn (gen_aarch64_float_truncate_lo_v2sf (operands[0], tmp));
1642 DONE;
1643 }
1644 )
1646 (define_insn "aarch64_vmls<mode>"
1647 [(set (match_operand:VDQF 0 "register_operand" "=w")
1648 (minus:VDQF (match_operand:VDQF 1 "register_operand" "0")
1649 (mult:VDQF (match_operand:VDQF 2 "register_operand" "w")
1650 (match_operand:VDQF 3 "register_operand" "w"))))]
1651 "TARGET_SIMD"
1652 "fmls\\t%0.<Vtype>, %2.<Vtype>, %3.<Vtype>"
1653 [(set_attr "simd_type" "simd_fmla")
1654 (set_attr "simd_mode" "<MODE>")]
1655 )
1657 ;; FP Max/Min
1658 ;; Max/Min are introduced by idiom recognition by GCC's mid-end. An
1659 ;; expression like:
1660 ;; a = (b < c) ? b : c;
1661 ;; is idiom-matched as MIN_EXPR<b,c> only if -ffinite-math-only is enabled
1662 ;; either explicitly or indirectly via -ffast-math.
1663 ;;
1664 ;; MIN_EXPR and MAX_EXPR eventually map to 'smin' and 'smax' in RTL.
1665 ;; The 'smax' and 'smin' RTL standard pattern names do not specify which
1666 ;; operand will be returned when both operands are zero (i.e. they may not
1667 ;; honour signed zeroes), or when either operand is NaN. Therefore GCC
1668 ;; only introduces MIN_EXPR/MAX_EXPR in fast math mode or when not honouring
1669 ;; NaNs.
1671 (define_insn "<su><maxmin><mode>3"
1672 [(set (match_operand:VDQF 0 "register_operand" "=w")
1673 (FMAXMIN:VDQF (match_operand:VDQF 1 "register_operand" "w")
1674 (match_operand:VDQF 2 "register_operand" "w")))]
1675 "TARGET_SIMD"
1676 "f<maxmin>nm\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1677 [(set_attr "simd_type" "simd_fminmax")
1678 (set_attr "simd_mode" "<MODE>")]
1679 )
1681 (define_insn "<maxmin_uns><mode>3"
1682 [(set (match_operand:VDQF 0 "register_operand" "=w")
1683 (unspec:VDQF [(match_operand:VDQF 1 "register_operand" "w")
1684 (match_operand:VDQF 2 "register_operand" "w")]
1685 FMAXMIN_UNS))]
1686 "TARGET_SIMD"
1687 "<maxmin_uns_op>\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
1688 [(set_attr "simd_type" "simd_fminmax")
1689 (set_attr "simd_mode" "<MODE>")]
1690 )
1692 ;; 'across lanes' add.
1694 (define_insn "reduc_<sur>plus_<mode>"
1695 [(set (match_operand:VDQV 0 "register_operand" "=w")
1696 (unspec:VDQV [(match_operand:VDQV 1 "register_operand" "w")]
1697 SUADDV))]
1698 "TARGET_SIMD"
1699 "addv\\t%<Vetype>0, %1.<Vtype>"
1700 [(set_attr "simd_type" "simd_addv")
1701 (set_attr "simd_mode" "<MODE>")]
1702 )
1704 (define_insn "reduc_<sur>plus_v2di"
1705 [(set (match_operand:V2DI 0 "register_operand" "=w")
1706 (unspec:V2DI [(match_operand:V2DI 1 "register_operand" "w")]
1707 SUADDV))]
1708 "TARGET_SIMD"
1709 "addp\\t%d0, %1.2d"
1710 [(set_attr "simd_type" "simd_addv")
1711 (set_attr "simd_mode" "V2DI")]
1712 )
1714 (define_insn "reduc_<sur>plus_v2si"
1715 [(set (match_operand:V2SI 0 "register_operand" "=w")
1716 (unspec:V2SI [(match_operand:V2SI 1 "register_operand" "w")]
1717 SUADDV))]
1718 "TARGET_SIMD"
1719 "addp\\t%0.2s, %1.2s, %1.2s"
1720 [(set_attr "simd_type" "simd_addv")
1721 (set_attr "simd_mode" "V2SI")]
1722 )
1724 (define_insn "reduc_<sur>plus_<mode>"
1725 [(set (match_operand:V2F 0 "register_operand" "=w")
1726 (unspec:V2F [(match_operand:V2F 1 "register_operand" "w")]
1727 SUADDV))]
1728 "TARGET_SIMD"
1729 "faddp\\t%<Vetype>0, %1.<Vtype>"
1730 [(set_attr "simd_type" "simd_fadd")
1731 (set_attr "simd_mode" "<MODE>")]
1732 )
1734 (define_insn "aarch64_addpv4sf"
1735 [(set (match_operand:V4SF 0 "register_operand" "=w")
1736 (unspec:V4SF [(match_operand:V4SF 1 "register_operand" "w")]
1737 UNSPEC_FADDV))]
1738 "TARGET_SIMD"
1739 "faddp\\t%0.4s, %1.4s, %1.4s"
1740 [(set_attr "simd_type" "simd_fadd")
1741 (set_attr "simd_mode" "V4SF")]
1742 )
1744 (define_expand "reduc_<sur>plus_v4sf"
1745 [(set (match_operand:V4SF 0 "register_operand")
1746 (unspec:V4SF [(match_operand:V4SF 1 "register_operand")]
1747 SUADDV))]
1748 "TARGET_SIMD"
1749 {
1750 rtx tmp = gen_reg_rtx (V4SFmode);
1751 emit_insn (gen_aarch64_addpv4sf (tmp, operands[1]));
1752 emit_insn (gen_aarch64_addpv4sf (operands[0], tmp));
1753 DONE;
1754 })
1756 (define_insn "clz<mode>2"
1757 [(set (match_operand:VDQ_BHSI 0 "register_operand" "=w")
1758 (clz:VDQ_BHSI (match_operand:VDQ_BHSI 1 "register_operand" "w")))]
1759 "TARGET_SIMD"
1760 "clz\\t%0.<Vtype>, %1.<Vtype>"
1761 [(set_attr "simd_type" "simd_cls")
1762 (set_attr "simd_mode" "<MODE>")]
1763 )
1765 ;; 'across lanes' max and min ops.
1767 (define_insn "reduc_<maxmin_uns>_<mode>"
1768 [(set (match_operand:VDQV 0 "register_operand" "=w")
1769 (unspec:VDQV [(match_operand:VDQV 1 "register_operand" "w")]
1770 MAXMINV))]
1771 "TARGET_SIMD"
1772 "<maxmin_uns_op>v\\t%<Vetype>0, %1.<Vtype>"
1773 [(set_attr "simd_type" "simd_minmaxv")
1774 (set_attr "simd_mode" "<MODE>")]
1775 )
1777 (define_insn "reduc_<maxmin_uns>_v2di"
1778 [(set (match_operand:V2DI 0 "register_operand" "=w")
1779 (unspec:V2DI [(match_operand:V2DI 1 "register_operand" "w")]
1780 MAXMINV))]
1781 "TARGET_SIMD"
1782 "<maxmin_uns_op>p\\t%d0, %1.2d"
1783 [(set_attr "simd_type" "simd_minmaxv")
1784 (set_attr "simd_mode" "V2DI")]
1785 )
1787 (define_insn "reduc_<maxmin_uns>_v2si"
1788 [(set (match_operand:V2SI 0 "register_operand" "=w")
1789 (unspec:V2SI [(match_operand:V2SI 1 "register_operand" "w")]
1790 MAXMINV))]
1791 "TARGET_SIMD"
1792 "<maxmin_uns_op>p\\t%0.2s, %1.2s, %1.2s"
1793 [(set_attr "simd_type" "simd_minmaxv")
1794 (set_attr "simd_mode" "V2SI")]
1795 )
1797 (define_insn "reduc_<maxmin_uns>_<mode>"
1798 [(set (match_operand:V2F 0 "register_operand" "=w")
1799 (unspec:V2F [(match_operand:V2F 1 "register_operand" "w")]
1800 FMAXMINV))]
1801 "TARGET_SIMD"
1802 "<maxmin_uns_op>p\\t%<Vetype>0, %1.<Vtype>"
1803 [(set_attr "simd_type" "simd_fminmaxv")
1804 (set_attr "simd_mode" "<MODE>")]
1805 )
1807 (define_insn "reduc_<maxmin_uns>_v4sf"
1808 [(set (match_operand:V4SF 0 "register_operand" "=w")
1809 (unspec:V4SF [(match_operand:V4SF 1 "register_operand" "w")]
1810 FMAXMINV))]
1811 "TARGET_SIMD"
1812 "<maxmin_uns_op>v\\t%s0, %1.4s"
1813 [(set_attr "simd_type" "simd_fminmaxv")
1814 (set_attr "simd_mode" "V4SF")]
1815 )
1817 ;; aarch64_simd_bsl may compile to any of bsl/bif/bit depending on register
1818 ;; allocation.
1819 ;; Operand 1 is the mask, operands 2 and 3 are the bitfields from which
1820 ;; to select.
1821 ;;
1822 ;; Thus our BSL is of the form:
1823 ;; op0 = bsl (mask, op2, op3)
1824 ;; We can use any of:
1825 ;;
1826 ;; if (op0 = mask)
1827 ;; bsl mask, op1, op2
1828 ;; if (op0 = op1) (so 1-bits in mask choose bits from op2, else op0)
1829 ;; bit op0, op2, mask
1830 ;; if (op0 = op2) (so 0-bits in mask choose bits from op1, else op0)
1831 ;; bif op0, op1, mask
1833 (define_insn "aarch64_simd_bsl<mode>_internal"
1834 [(set (match_operand:VALL 0 "register_operand" "=w,w,w")
1835 (ior:VALL
1836 (and:VALL
1837 (match_operand:<V_cmp_result> 1 "register_operand" " 0,w,w")
1838 (match_operand:VALL 2 "register_operand" " w,w,0"))
1839 (and:VALL
1840 (not:<V_cmp_result>
1841 (match_dup:<V_cmp_result> 1))
1842 (match_operand:VALL 3 "register_operand" " w,0,w"))
1843 ))]
1844 "TARGET_SIMD"
1845 "@
1846 bsl\\t%0.<Vbtype>, %2.<Vbtype>, %3.<Vbtype>
1847 bit\\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>
1848 bif\\t%0.<Vbtype>, %3.<Vbtype>, %1.<Vbtype>"
1849 )
1851 (define_expand "aarch64_simd_bsl<mode>"
1852 [(match_operand:VALL 0 "register_operand")
1853 (match_operand:<V_cmp_result> 1 "register_operand")
1854 (match_operand:VALL 2 "register_operand")
1855 (match_operand:VALL 3 "register_operand")]
1856 "TARGET_SIMD"
1857 {
1858 /* We can't alias operands together if they have different modes. */
1859 operands[1] = gen_lowpart (<V_cmp_result>mode, operands[1]);
1860 emit_insn (gen_aarch64_simd_bsl<mode>_internal (operands[0], operands[1],
1861 operands[2], operands[3]));
1862 DONE;
1863 })
1865 (define_expand "aarch64_vcond_internal<mode><mode>"
1866 [(set (match_operand:VDQ 0 "register_operand")
1867 (if_then_else:VDQ
1868 (match_operator 3 "comparison_operator"
1869 [(match_operand:VDQ 4 "register_operand")
1870 (match_operand:VDQ 5 "nonmemory_operand")])
1871 (match_operand:VDQ 1 "nonmemory_operand")
1872 (match_operand:VDQ 2 "nonmemory_operand")))]
1873 "TARGET_SIMD"
1874 {
1875 int inverse = 0, has_zero_imm_form = 0;
1876 rtx op1 = operands[1];
1877 rtx op2 = operands[2];
1878 rtx mask = gen_reg_rtx (<MODE>mode);
1880 switch (GET_CODE (operands[3]))
1881 {
1882 case LE:
1883 case LT:
1884 case NE:
1885 inverse = 1;
1886 /* Fall through. */
1887 case GE:
1888 case GT:
1889 case EQ:
1890 has_zero_imm_form = 1;
1891 break;
1892 case LEU:
1893 case LTU:
1894 inverse = 1;
1895 break;
1896 default:
1897 break;
1898 }
1900 if (!REG_P (operands[5])
1901 && (operands[5] != CONST0_RTX (<MODE>mode) || !has_zero_imm_form))
1902 operands[5] = force_reg (<MODE>mode, operands[5]);
1904 switch (GET_CODE (operands[3]))
1905 {
1906 case LT:
1907 case GE:
1908 emit_insn (gen_aarch64_cmge<mode> (mask, operands[4], operands[5]));
1909 break;
1911 case LE:
1912 case GT:
1913 emit_insn (gen_aarch64_cmgt<mode> (mask, operands[4], operands[5]));
1914 break;
1916 case LTU:
1917 case GEU:
1918 emit_insn (gen_aarch64_cmgeu<mode> (mask, operands[4], operands[5]));
1919 break;
1921 case LEU:
1922 case GTU:
1923 emit_insn (gen_aarch64_cmgtu<mode> (mask, operands[4], operands[5]));
1924 break;
1926 case NE:
1927 case EQ:
1928 emit_insn (gen_aarch64_cmeq<mode> (mask, operands[4], operands[5]));
1929 break;
1931 default:
1932 gcc_unreachable ();
1933 }
1935 if (inverse)
1936 {
1937 op1 = operands[2];
1938 op2 = operands[1];
1939 }
1941 /* If we have (a = (b CMP c) ? -1 : 0);
1942 Then we can simply move the generated mask. */
1944 if (op1 == CONSTM1_RTX (<V_cmp_result>mode)
1945 && op2 == CONST0_RTX (<V_cmp_result>mode))
1946 emit_move_insn (operands[0], mask);
1947 else
1948 {
1949 if (!REG_P (op1))
1950 op1 = force_reg (<MODE>mode, op1);
1951 if (!REG_P (op2))
1952 op2 = force_reg (<MODE>mode, op2);
1953 emit_insn (gen_aarch64_simd_bsl<mode> (operands[0], mask,
1954 op1, op2));
1955 }
1957 DONE;
1958 })
1960 (define_expand "aarch64_vcond_internal<VDQF_COND:mode><VDQF:mode>"
1961 [(set (match_operand:VDQF_COND 0 "register_operand")
1962 (if_then_else:VDQF
1963 (match_operator 3 "comparison_operator"
1964 [(match_operand:VDQF 4 "register_operand")
1965 (match_operand:VDQF 5 "nonmemory_operand")])
1966 (match_operand:VDQF_COND 1 "nonmemory_operand")
1967 (match_operand:VDQF_COND 2 "nonmemory_operand")))]
1968 "TARGET_SIMD"
1969 {
1970 int inverse = 0;
1971 int use_zero_form = 0;
1972 int swap_bsl_operands = 0;
1973 rtx op1 = operands[1];
1974 rtx op2 = operands[2];
1975 rtx mask = gen_reg_rtx (<VDQF_COND:V_cmp_result>mode);
1976 rtx tmp = gen_reg_rtx (<VDQF_COND:V_cmp_result>mode);
1978 rtx (*base_comparison) (rtx, rtx, rtx);
1979 rtx (*complimentary_comparison) (rtx, rtx, rtx);
1981 switch (GET_CODE (operands[3]))
1982 {
1983 case GE:
1984 case GT:
1985 case LE:
1986 case LT:
1987 case EQ:
1988 if (operands[5] == CONST0_RTX (<MODE>mode))
1989 {
1990 use_zero_form = 1;
1991 break;
1992 }
1993 /* Fall through. */
1994 default:
1995 if (!REG_P (operands[5]))
1996 operands[5] = force_reg (<VDQF:MODE>mode, operands[5]);
1997 }
1999 switch (GET_CODE (operands[3]))
2000 {
2001 case LT:
2002 case UNLT:
2003 inverse = 1;
2004 /* Fall through. */
2005 case GE:
2006 case UNGE:
2007 case ORDERED:
2008 case UNORDERED:
2009 base_comparison = gen_aarch64_cmge<VDQF:mode>;
2010 complimentary_comparison = gen_aarch64_cmgt<VDQF:mode>;
2011 break;
2012 case LE:
2013 case UNLE:
2014 inverse = 1;
2015 /* Fall through. */
2016 case GT:
2017 case UNGT:
2018 base_comparison = gen_aarch64_cmgt<VDQF:mode>;
2019 complimentary_comparison = gen_aarch64_cmge<VDQF:mode>;
2020 break;
2021 case EQ:
2022 case NE:
2023 case UNEQ:
2024 base_comparison = gen_aarch64_cmeq<VDQF:mode>;
2025 complimentary_comparison = gen_aarch64_cmeq<VDQF:mode>;
2026 break;
2027 default:
2028 gcc_unreachable ();
2029 }
2031 switch (GET_CODE (operands[3]))
2032 {
2033 case LT:
2034 case LE:
2035 case GT:
2036 case GE:
2037 case EQ:
2038 /* The easy case. Here we emit one of FCMGE, FCMGT or FCMEQ.
2039 As a LT b <=> b GE a && a LE b <=> b GT a. Our transformations are:
2040 a GE b -> a GE b
2041 a GT b -> a GT b
2042 a LE b -> b GE a
2043 a LT b -> b GT a
2044 a EQ b -> a EQ b
2045 Note that there also exist direct comparison against 0 forms,
2046 so catch those as a special case. */
2047 if (use_zero_form)
2048 {
2049 inverse = 0;
2050 switch (GET_CODE (operands[3]))
2051 {
2052 case LT:
2053 base_comparison = gen_aarch64_cmlt<VDQF:mode>;
2054 break;
2055 case LE:
2056 base_comparison = gen_aarch64_cmle<VDQF:mode>;
2057 break;
2058 default:
2059 /* Do nothing, other zero form cases already have the correct
2060 base_comparison. */
2061 break;
2062 }
2063 }
2065 if (!inverse)
2066 emit_insn (base_comparison (mask, operands[4], operands[5]));
2067 else
2068 emit_insn (complimentary_comparison (mask, operands[5], operands[4]));
2069 break;
2070 case UNLT:
2071 case UNLE:
2072 case UNGT:
2073 case UNGE:
2074 case NE:
2075 /* FCM returns false for lanes which are unordered, so if we use
2076 the inverse of the comparison we actually want to emit, then
2077 swap the operands to BSL, we will end up with the correct result.
2078 Note that a NE NaN and NaN NE b are true for all a, b.
2080 Our transformations are:
2081 a GE b -> !(b GT a)
2082 a GT b -> !(b GE a)
2083 a LE b -> !(a GT b)
2084 a LT b -> !(a GE b)
2085 a NE b -> !(a EQ b) */
2087 if (inverse)
2088 emit_insn (base_comparison (mask, operands[4], operands[5]));
2089 else
2090 emit_insn (complimentary_comparison (mask, operands[5], operands[4]));
2092 swap_bsl_operands = 1;
2093 break;
2094 case UNEQ:
2095 /* We check (a > b || b > a). combining these comparisons give us
2096 true iff !(a != b && a ORDERED b), swapping the operands to BSL
2097 will then give us (a == b || a UNORDERED b) as intended. */
2099 emit_insn (gen_aarch64_cmgt<VDQF:mode> (mask, operands[4], operands[5]));
2100 emit_insn (gen_aarch64_cmgt<VDQF:mode> (tmp, operands[5], operands[4]));
2101 emit_insn (gen_ior<VDQF_COND:v_cmp_result>3 (mask, mask, tmp));
2102 swap_bsl_operands = 1;
2103 break;
2104 case UNORDERED:
2105 /* Operands are ORDERED iff (a > b || b >= a).
2106 Swapping the operands to BSL will give the UNORDERED case. */
2107 swap_bsl_operands = 1;
2108 /* Fall through. */
2109 case ORDERED:
2110 emit_insn (gen_aarch64_cmgt<VDQF:mode> (tmp, operands[4], operands[5]));
2111 emit_insn (gen_aarch64_cmge<VDQF:mode> (mask, operands[5], operands[4]));
2112 emit_insn (gen_ior<VDQF_COND:v_cmp_result>3 (mask, mask, tmp));
2113 break;
2114 default:
2115 gcc_unreachable ();
2116 }
2118 if (swap_bsl_operands)
2119 {
2120 op1 = operands[2];
2121 op2 = operands[1];
2122 }
2124 /* If we have (a = (b CMP c) ? -1 : 0);
2125 Then we can simply move the generated mask. */
2127 if (op1 == CONSTM1_RTX (<VDQF_COND:V_cmp_result>mode)
2128 && op2 == CONST0_RTX (<VDQF_COND:V_cmp_result>mode))
2129 emit_move_insn (operands[0], mask);
2130 else
2131 {
2132 if (!REG_P (op1))
2133 op1 = force_reg (<VDQF_COND:MODE>mode, op1);
2134 if (!REG_P (op2))
2135 op2 = force_reg (<VDQF_COND:MODE>mode, op2);
2136 emit_insn (gen_aarch64_simd_bsl<VDQF_COND:mode> (operands[0], mask,
2137 op1, op2));
2138 }
2140 DONE;
2141 })
2143 (define_expand "vcond<mode><mode>"
2144 [(set (match_operand:VALL 0 "register_operand")
2145 (if_then_else:VALL
2146 (match_operator 3 "comparison_operator"
2147 [(match_operand:VALL 4 "register_operand")
2148 (match_operand:VALL 5 "nonmemory_operand")])
2149 (match_operand:VALL 1 "nonmemory_operand")
2150 (match_operand:VALL 2 "nonmemory_operand")))]
2151 "TARGET_SIMD"
2152 {
2153 emit_insn (gen_aarch64_vcond_internal<mode><mode> (operands[0], operands[1],
2154 operands[2], operands[3],
2155 operands[4], operands[5]));
2156 DONE;
2157 })
2159 (define_expand "vcond<v_cmp_result><mode>"
2160 [(set (match_operand:<V_cmp_result> 0 "register_operand")
2161 (if_then_else:<V_cmp_result>
2162 (match_operator 3 "comparison_operator"
2163 [(match_operand:VDQF 4 "register_operand")
2164 (match_operand:VDQF 5 "nonmemory_operand")])
2165 (match_operand:<V_cmp_result> 1 "nonmemory_operand")
2166 (match_operand:<V_cmp_result> 2 "nonmemory_operand")))]
2167 "TARGET_SIMD"
2168 {
2169 emit_insn (gen_aarch64_vcond_internal<v_cmp_result><mode> (
2170 operands[0], operands[1],
2171 operands[2], operands[3],
2172 operands[4], operands[5]));
2173 DONE;
2174 })
2176 (define_expand "vcondu<mode><mode>"
2177 [(set (match_operand:VDQ 0 "register_operand")
2178 (if_then_else:VDQ
2179 (match_operator 3 "comparison_operator"
2180 [(match_operand:VDQ 4 "register_operand")
2181 (match_operand:VDQ 5 "nonmemory_operand")])
2182 (match_operand:VDQ 1 "nonmemory_operand")
2183 (match_operand:VDQ 2 "nonmemory_operand")))]
2184 "TARGET_SIMD"
2185 {
2186 emit_insn (gen_aarch64_vcond_internal<mode><mode> (operands[0], operands[1],
2187 operands[2], operands[3],
2188 operands[4], operands[5]));
2189 DONE;
2190 })
2192 ;; Patterns for AArch64 SIMD Intrinsics.
2194 (define_expand "aarch64_create<mode>"
2195 [(match_operand:VD_RE 0 "register_operand" "")
2196 (match_operand:DI 1 "general_operand" "")]
2197 "TARGET_SIMD"
2198 {
2199 rtx src = gen_lowpart (<MODE>mode, operands[1]);
2200 emit_move_insn (operands[0], src);
2201 DONE;
2202 })
2204 ;; Lane extraction with sign extension to general purpose register.
2205 (define_insn "*aarch64_get_lane_extend<GPI:mode><VDQQH:mode>"
2206 [(set (match_operand:GPI 0 "register_operand" "=r")
2207 (sign_extend:GPI
2208 (vec_select:<VEL>
2209 (match_operand:VDQQH 1 "register_operand" "w")
2210 (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2211 "TARGET_SIMD"
2212 "smov\\t%<GPI:w>0, %1.<VDQQH:Vetype>[%2]"
2213 [(set_attr "simd_type" "simd_movgp")
2214 (set_attr "simd_mode" "<VDQQH:MODE>")]
2215 )
2217 (define_insn "*aarch64_get_lane_zero_extendsi<mode>"
2218 [(set (match_operand:SI 0 "register_operand" "=r")
2219 (zero_extend:SI
2220 (vec_select:<VEL>
2221 (match_operand:VDQQH 1 "register_operand" "w")
2222 (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2223 "TARGET_SIMD"
2224 "umov\\t%w0, %1.<Vetype>[%2]"
2225 [(set_attr "simd_type" "simd_movgp")
2226 (set_attr "simd_mode" "<MODE>")]
2227 )
2229 ;; Lane extraction of a value, neither sign nor zero extension
2230 ;; is guaranteed so upper bits should be considered undefined.
2231 (define_insn "aarch64_get_lane<mode>"
2232 [(set (match_operand:<VEL> 0 "register_operand" "=r, w")
2233 (vec_select:<VEL>
2234 (match_operand:VALL 1 "register_operand" "w, w")
2235 (parallel [(match_operand:SI 2 "immediate_operand" "i, i")])))]
2236 "TARGET_SIMD"
2237 "@
2238 umov\\t%<vwcore>0, %1.<Vetype>[%2]
2239 dup\\t%<Vetype>0, %1.<Vetype>[%2]"
2240 [(set_attr "simd_type" "simd_movgp, simd_dup")
2241 (set_attr "simd_mode" "<MODE>")]
2242 )
2244 (define_expand "aarch64_get_lanedi"
2245 [(match_operand:DI 0 "register_operand")
2246 (match_operand:DI 1 "register_operand")
2247 (match_operand:SI 2 "immediate_operand")]
2248 "TARGET_SIMD"
2249 {
2250 aarch64_simd_lane_bounds (operands[2], 0, 1);
2251 emit_move_insn (operands[0], operands[1]);
2252 DONE;
2253 })
2255 (define_expand "aarch64_reinterpretv8qi<mode>"
2256 [(match_operand:V8QI 0 "register_operand" "")
2257 (match_operand:VDC 1 "register_operand" "")]
2258 "TARGET_SIMD"
2259 {
2260 aarch64_simd_reinterpret (operands[0], operands[1]);
2261 DONE;
2262 })
2264 (define_expand "aarch64_reinterpretv4hi<mode>"
2265 [(match_operand:V4HI 0 "register_operand" "")
2266 (match_operand:VDC 1 "register_operand" "")]
2267 "TARGET_SIMD"
2268 {
2269 aarch64_simd_reinterpret (operands[0], operands[1]);
2270 DONE;
2271 })
2273 (define_expand "aarch64_reinterpretv2si<mode>"
2274 [(match_operand:V2SI 0 "register_operand" "")
2275 (match_operand:VDC 1 "register_operand" "")]
2276 "TARGET_SIMD"
2277 {
2278 aarch64_simd_reinterpret (operands[0], operands[1]);
2279 DONE;
2280 })
2282 (define_expand "aarch64_reinterpretv2sf<mode>"
2283 [(match_operand:V2SF 0 "register_operand" "")
2284 (match_operand:VDC 1 "register_operand" "")]
2285 "TARGET_SIMD"
2286 {
2287 aarch64_simd_reinterpret (operands[0], operands[1]);
2288 DONE;
2289 })
2291 (define_expand "aarch64_reinterpretdi<mode>"
2292 [(match_operand:DI 0 "register_operand" "")
2293 (match_operand:VD_RE 1 "register_operand" "")]
2294 "TARGET_SIMD"
2295 {
2296 aarch64_simd_reinterpret (operands[0], operands[1]);
2297 DONE;
2298 })
2300 (define_expand "aarch64_reinterpretv16qi<mode>"
2301 [(match_operand:V16QI 0 "register_operand" "")
2302 (match_operand:VQ 1 "register_operand" "")]
2303 "TARGET_SIMD"
2304 {
2305 aarch64_simd_reinterpret (operands[0], operands[1]);
2306 DONE;
2307 })
2309 (define_expand "aarch64_reinterpretv8hi<mode>"
2310 [(match_operand:V8HI 0 "register_operand" "")
2311 (match_operand:VQ 1 "register_operand" "")]
2312 "TARGET_SIMD"
2313 {
2314 aarch64_simd_reinterpret (operands[0], operands[1]);
2315 DONE;
2316 })
2318 (define_expand "aarch64_reinterpretv4si<mode>"
2319 [(match_operand:V4SI 0 "register_operand" "")
2320 (match_operand:VQ 1 "register_operand" "")]
2321 "TARGET_SIMD"
2322 {
2323 aarch64_simd_reinterpret (operands[0], operands[1]);
2324 DONE;
2325 })
2327 (define_expand "aarch64_reinterpretv4sf<mode>"
2328 [(match_operand:V4SF 0 "register_operand" "")
2329 (match_operand:VQ 1 "register_operand" "")]
2330 "TARGET_SIMD"
2331 {
2332 aarch64_simd_reinterpret (operands[0], operands[1]);
2333 DONE;
2334 })
2336 (define_expand "aarch64_reinterpretv2di<mode>"
2337 [(match_operand:V2DI 0 "register_operand" "")
2338 (match_operand:VQ 1 "register_operand" "")]
2339 "TARGET_SIMD"
2340 {
2341 aarch64_simd_reinterpret (operands[0], operands[1]);
2342 DONE;
2343 })
2345 (define_expand "aarch64_reinterpretv2df<mode>"
2346 [(match_operand:V2DF 0 "register_operand" "")
2347 (match_operand:VQ 1 "register_operand" "")]
2348 "TARGET_SIMD"
2349 {
2350 aarch64_simd_reinterpret (operands[0], operands[1]);
2351 DONE;
2352 })
2354 ;; In this insn, operand 1 should be low, and operand 2 the high part of the
2355 ;; dest vector.
2357 (define_insn "*aarch64_combinez<mode>"
2358 [(set (match_operand:<VDBL> 0 "register_operand" "=&w")
2359 (vec_concat:<VDBL>
2360 (match_operand:VDIC 1 "register_operand" "w")
2361 (match_operand:VDIC 2 "aarch64_simd_imm_zero" "Dz")))]
2362 "TARGET_SIMD"
2363 "mov\\t%0.8b, %1.8b"
2364 [(set_attr "simd_type" "simd_move")
2365 (set_attr "simd_mode" "<MODE>")]
2366 )
2368 (define_insn_and_split "aarch64_combine<mode>"
2369 [(set (match_operand:<VDBL> 0 "register_operand" "=&w")
2370 (vec_concat:<VDBL> (match_operand:VDC 1 "register_operand" "w")
2371 (match_operand:VDC 2 "register_operand" "w")))]
2372 "TARGET_SIMD"
2373 "#"
2374 "&& reload_completed"
2375 [(const_int 0)]
2376 {
2377 aarch64_split_simd_combine (operands[0], operands[1], operands[2]);
2378 DONE;
2379 })
2381 (define_expand "aarch64_simd_combine<mode>"
2382 [(set (match_operand:<VDBL> 0 "register_operand" "=&w")
2383 (vec_concat:<VDBL> (match_operand:VDC 1 "register_operand" "w")
2384 (match_operand:VDC 2 "register_operand" "w")))]
2385 "TARGET_SIMD"
2386 {
2387 emit_insn (gen_move_lo_quad_<Vdbl> (operands[0], operands[1]));
2388 emit_insn (gen_move_hi_quad_<Vdbl> (operands[0], operands[2]));
2389 DONE;
2390 })
2392 ;; <su><addsub>l<q>.
2394 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>l2<mode>_internal"
2395 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2396 (ADDSUB:<VWIDE> (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
2397 (match_operand:VQW 1 "register_operand" "w")
2398 (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))
2399 (ANY_EXTEND:<VWIDE> (vec_select:<VHALF>
2400 (match_operand:VQW 2 "register_operand" "w")
2401 (match_dup 3)))))]
2402 "TARGET_SIMD"
2403 "<ANY_EXTEND:su><ADDSUB:optab>l2 %0.<Vwtype>, %1.<Vtype>, %2.<Vtype>"
2404 [(set_attr "simd_type" "simd_addl")
2405 (set_attr "simd_mode" "<MODE>")]
2406 )
2408 (define_expand "aarch64_saddl2<mode>"
2409 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2410 (match_operand:VQW 1 "register_operand" "w")
2411 (match_operand:VQW 2 "register_operand" "w")]
2412 "TARGET_SIMD"
2413 {
2414 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2415 emit_insn (gen_aarch64_saddl2<mode>_internal (operands[0], operands[1],
2416 operands[2], p));
2417 DONE;
2418 })
2420 (define_expand "aarch64_uaddl2<mode>"
2421 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2422 (match_operand:VQW 1 "register_operand" "w")
2423 (match_operand:VQW 2 "register_operand" "w")]
2424 "TARGET_SIMD"
2425 {
2426 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2427 emit_insn (gen_aarch64_uaddl2<mode>_internal (operands[0], operands[1],
2428 operands[2], p));
2429 DONE;
2430 })
2432 (define_expand "aarch64_ssubl2<mode>"
2433 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2434 (match_operand:VQW 1 "register_operand" "w")
2435 (match_operand:VQW 2 "register_operand" "w")]
2436 "TARGET_SIMD"
2437 {
2438 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2439 emit_insn (gen_aarch64_ssubl2<mode>_internal (operands[0], operands[1],
2440 operands[2], p));
2441 DONE;
2442 })
2444 (define_expand "aarch64_usubl2<mode>"
2445 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2446 (match_operand:VQW 1 "register_operand" "w")
2447 (match_operand:VQW 2 "register_operand" "w")]
2448 "TARGET_SIMD"
2449 {
2450 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2451 emit_insn (gen_aarch64_usubl2<mode>_internal (operands[0], operands[1],
2452 operands[2], p));
2453 DONE;
2454 })
2456 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>l<mode>"
2457 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2458 (ADDSUB:<VWIDE> (ANY_EXTEND:<VWIDE>
2459 (match_operand:VDW 1 "register_operand" "w"))
2460 (ANY_EXTEND:<VWIDE>
2461 (match_operand:VDW 2 "register_operand" "w"))))]
2462 "TARGET_SIMD"
2463 "<ANY_EXTEND:su><ADDSUB:optab>l %0.<Vwtype>, %1.<Vtype>, %2.<Vtype>"
2464 [(set_attr "simd_type" "simd_addl")
2465 (set_attr "simd_mode" "<MODE>")]
2466 )
2468 ;; <su><addsub>w<q>.
2470 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>w<mode>"
2471 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2472 (ADDSUB:<VWIDE> (match_operand:<VWIDE> 1 "register_operand" "w")
2473 (ANY_EXTEND:<VWIDE>
2474 (match_operand:VDW 2 "register_operand" "w"))))]
2475 "TARGET_SIMD"
2476 "<ANY_EXTEND:su><ADDSUB:optab>w\\t%0.<Vwtype>, %1.<Vwtype>, %2.<Vtype>"
2477 [(set_attr "simd_type" "simd_addl")
2478 (set_attr "simd_mode" "<MODE>")]
2479 )
2481 (define_insn "aarch64_<ANY_EXTEND:su><ADDSUB:optab>w2<mode>_internal"
2482 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2483 (ADDSUB:<VWIDE> (match_operand:<VWIDE> 1 "register_operand" "w")
2484 (ANY_EXTEND:<VWIDE>
2485 (vec_select:<VHALF>
2486 (match_operand:VQW 2 "register_operand" "w")
2487 (match_operand:VQW 3 "vect_par_cnst_hi_half" "")))))]
2488 "TARGET_SIMD"
2489 "<ANY_EXTEND:su><ADDSUB:optab>w2\\t%0.<Vwtype>, %1.<Vwtype>, %2.<Vtype>"
2490 [(set_attr "simd_type" "simd_addl")
2491 (set_attr "simd_mode" "<MODE>")]
2492 )
2494 (define_expand "aarch64_saddw2<mode>"
2495 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2496 (match_operand:<VWIDE> 1 "register_operand" "w")
2497 (match_operand:VQW 2 "register_operand" "w")]
2498 "TARGET_SIMD"
2499 {
2500 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2501 emit_insn (gen_aarch64_saddw2<mode>_internal (operands[0], operands[1],
2502 operands[2], p));
2503 DONE;
2504 })
2506 (define_expand "aarch64_uaddw2<mode>"
2507 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2508 (match_operand:<VWIDE> 1 "register_operand" "w")
2509 (match_operand:VQW 2 "register_operand" "w")]
2510 "TARGET_SIMD"
2511 {
2512 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2513 emit_insn (gen_aarch64_uaddw2<mode>_internal (operands[0], operands[1],
2514 operands[2], p));
2515 DONE;
2516 })
2519 (define_expand "aarch64_ssubw2<mode>"
2520 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2521 (match_operand:<VWIDE> 1 "register_operand" "w")
2522 (match_operand:VQW 2 "register_operand" "w")]
2523 "TARGET_SIMD"
2524 {
2525 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2526 emit_insn (gen_aarch64_ssubw2<mode>_internal (operands[0], operands[1],
2527 operands[2], p));
2528 DONE;
2529 })
2531 (define_expand "aarch64_usubw2<mode>"
2532 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2533 (match_operand:<VWIDE> 1 "register_operand" "w")
2534 (match_operand:VQW 2 "register_operand" "w")]
2535 "TARGET_SIMD"
2536 {
2537 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2538 emit_insn (gen_aarch64_usubw2<mode>_internal (operands[0], operands[1],
2539 operands[2], p));
2540 DONE;
2541 })
2543 ;; <su><r>h<addsub>.
2545 (define_insn "aarch64_<sur>h<addsub><mode>"
2546 [(set (match_operand:VQ_S 0 "register_operand" "=w")
2547 (unspec:VQ_S [(match_operand:VQ_S 1 "register_operand" "w")
2548 (match_operand:VQ_S 2 "register_operand" "w")]
2549 HADDSUB))]
2550 "TARGET_SIMD"
2551 "<sur>h<addsub>\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
2552 [(set_attr "simd_type" "simd_add")
2553 (set_attr "simd_mode" "<MODE>")]
2554 )
2556 ;; <r><addsub>hn<q>.
2558 (define_insn "aarch64_<sur><addsub>hn<mode>"
2559 [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
2560 (unspec:<VNARROWQ> [(match_operand:VQN 1 "register_operand" "w")
2561 (match_operand:VQN 2 "register_operand" "w")]
2562 ADDSUBHN))]
2563 "TARGET_SIMD"
2564 "<sur><addsub>hn\\t%0.<Vntype>, %1.<Vtype>, %2.<Vtype>"
2565 [(set_attr "simd_type" "simd_addn")
2566 (set_attr "simd_mode" "<MODE>")]
2567 )
2569 (define_insn "aarch64_<sur><addsub>hn2<mode>"
2570 [(set (match_operand:<VNARROWQ2> 0 "register_operand" "=w")
2571 (unspec:<VNARROWQ2> [(match_operand:<VNARROWQ> 1 "register_operand" "0")
2572 (match_operand:VQN 2 "register_operand" "w")
2573 (match_operand:VQN 3 "register_operand" "w")]
2574 ADDSUBHN2))]
2575 "TARGET_SIMD"
2576 "<sur><addsub>hn2\\t%0.<V2ntype>, %2.<Vtype>, %3.<Vtype>"
2577 [(set_attr "simd_type" "simd_addn2")
2578 (set_attr "simd_mode" "<MODE>")]
2579 )
2581 ;; pmul.
2583 (define_insn "aarch64_pmul<mode>"
2584 [(set (match_operand:VB 0 "register_operand" "=w")
2585 (unspec:VB [(match_operand:VB 1 "register_operand" "w")
2586 (match_operand:VB 2 "register_operand" "w")]
2587 UNSPEC_PMUL))]
2588 "TARGET_SIMD"
2589 "pmul\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
2590 [(set_attr "simd_type" "simd_mul")
2591 (set_attr "simd_mode" "<MODE>")]
2592 )
2594 ;; <su>q<addsub>
2596 (define_insn "aarch64_<su_optab><optab><mode>"
2597 [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
2598 (BINQOPS:VSDQ_I (match_operand:VSDQ_I 1 "register_operand" "w")
2599 (match_operand:VSDQ_I 2 "register_operand" "w")))]
2600 "TARGET_SIMD"
2601 "<su_optab><optab>\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
2602 [(set_attr "simd_type" "simd_add")
2603 (set_attr "simd_mode" "<MODE>")]
2604 )
2606 ;; suqadd and usqadd
2608 (define_insn "aarch64_<sur>qadd<mode>"
2609 [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
2610 (unspec:VSDQ_I [(match_operand:VSDQ_I 1 "register_operand" "0")
2611 (match_operand:VSDQ_I 2 "register_operand" "w")]
2612 USSUQADD))]
2613 "TARGET_SIMD"
2614 "<sur>qadd\\t%<v>0<Vmtype>, %<v>2<Vmtype>"
2615 [(set_attr "simd_type" "simd_sat_add")
2616 (set_attr "simd_mode" "<MODE>")]
2617 )
2619 ;; sqmovun
2621 (define_insn "aarch64_sqmovun<mode>"
2622 [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
2623 (unspec:<VNARROWQ> [(match_operand:VSQN_HSDI 1 "register_operand" "w")]
2624 UNSPEC_SQXTUN))]
2625 "TARGET_SIMD"
2626 "sqxtun\\t%<vn2>0<Vmntype>, %<v>1<Vmtype>"
2627 [(set_attr "simd_type" "simd_sat_shiftn_imm")
2628 (set_attr "simd_mode" "<MODE>")]
2629 )
2631 ;; sqmovn and uqmovn
2633 (define_insn "aarch64_<sur>qmovn<mode>"
2634 [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
2635 (unspec:<VNARROWQ> [(match_operand:VSQN_HSDI 1 "register_operand" "w")]
2636 SUQMOVN))]
2637 "TARGET_SIMD"
2638 "<sur>qxtn\\t%<vn2>0<Vmntype>, %<v>1<Vmtype>"
2639 [(set_attr "simd_type" "simd_sat_shiftn_imm")
2640 (set_attr "simd_mode" "<MODE>")]
2641 )
2643 ;; <su>q<absneg>
2645 (define_insn "aarch64_s<optab><mode>"
2646 [(set (match_operand:VSDQ_I_BHSI 0 "register_operand" "=w")
2647 (UNQOPS:VSDQ_I_BHSI
2648 (match_operand:VSDQ_I_BHSI 1 "register_operand" "w")))]
2649 "TARGET_SIMD"
2650 "s<optab>\\t%<v>0<Vmtype>, %<v>1<Vmtype>"
2651 [(set_attr "simd_type" "simd_sat_negabs")
2652 (set_attr "simd_mode" "<MODE>")]
2653 )
2655 ;; sq<r>dmulh.
2657 (define_insn "aarch64_sq<r>dmulh<mode>"
2658 [(set (match_operand:VSDQ_HSI 0 "register_operand" "=w")
2659 (unspec:VSDQ_HSI
2660 [(match_operand:VSDQ_HSI 1 "register_operand" "w")
2661 (match_operand:VSDQ_HSI 2 "register_operand" "w")]
2662 VQDMULH))]
2663 "TARGET_SIMD"
2664 "sq<r>dmulh\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
2665 [(set_attr "simd_type" "simd_sat_mul")
2666 (set_attr "simd_mode" "<MODE>")]
2667 )
2669 ;; sq<r>dmulh_lane
2671 (define_insn "aarch64_sq<r>dmulh_lane<mode>"
2672 [(set (match_operand:VDQHS 0 "register_operand" "=w")
2673 (unspec:VDQHS
2674 [(match_operand:VDQHS 1 "register_operand" "w")
2675 (vec_select:<VEL>
2676 (match_operand:<VCOND> 2 "register_operand" "<vwx>")
2677 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))]
2678 VQDMULH))]
2679 "TARGET_SIMD"
2680 "*
2681 aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCOND>mode));
2682 return \"sq<r>dmulh\\t%0.<Vtype>, %1.<Vtype>, %2.<Vetype>[%3]\";"
2683 [(set_attr "simd_type" "simd_sat_mul")
2684 (set_attr "simd_mode" "<MODE>")]
2685 )
2687 (define_insn "aarch64_sq<r>dmulh_laneq<mode>"
2688 [(set (match_operand:VDQHS 0 "register_operand" "=w")
2689 (unspec:VDQHS
2690 [(match_operand:VDQHS 1 "register_operand" "w")
2691 (vec_select:<VEL>
2692 (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
2693 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))]
2694 VQDMULH))]
2695 "TARGET_SIMD"
2696 "*
2697 aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCONQ>mode));
2698 return \"sq<r>dmulh\\t%0.<Vtype>, %1.<Vtype>, %2.<Vetype>[%3]\";"
2699 [(set_attr "simd_type" "simd_sat_mul")
2700 (set_attr "simd_mode" "<MODE>")]
2701 )
2703 (define_insn "aarch64_sq<r>dmulh_lane<mode>"
2704 [(set (match_operand:SD_HSI 0 "register_operand" "=w")
2705 (unspec:SD_HSI
2706 [(match_operand:SD_HSI 1 "register_operand" "w")
2707 (vec_select:<VEL>
2708 (match_operand:<VCONQ> 2 "register_operand" "<vwx>")
2709 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))]
2710 VQDMULH))]
2711 "TARGET_SIMD"
2712 "*
2713 aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCONQ>mode));
2714 return \"sq<r>dmulh\\t%<v>0, %<v>1, %2.<v>[%3]\";"
2715 [(set_attr "simd_type" "simd_sat_mul")
2716 (set_attr "simd_mode" "<MODE>")]
2717 )
2719 ;; vqdml[sa]l
2721 (define_insn "aarch64_sqdml<SBINQOPS:as>l<mode>"
2722 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2723 (SBINQOPS:<VWIDE>
2724 (match_operand:<VWIDE> 1 "register_operand" "0")
2725 (ss_ashift:<VWIDE>
2726 (mult:<VWIDE>
2727 (sign_extend:<VWIDE>
2728 (match_operand:VSD_HSI 2 "register_operand" "w"))
2729 (sign_extend:<VWIDE>
2730 (match_operand:VSD_HSI 3 "register_operand" "w")))
2731 (const_int 1))))]
2732 "TARGET_SIMD"
2733 "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %<v>3<Vmtype>"
2734 [(set_attr "simd_type" "simd_sat_mlal")
2735 (set_attr "simd_mode" "<MODE>")]
2736 )
2738 ;; vqdml[sa]l_lane
2740 (define_insn "aarch64_sqdml<SBINQOPS:as>l_lane<mode>_internal"
2741 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2742 (SBINQOPS:<VWIDE>
2743 (match_operand:<VWIDE> 1 "register_operand" "0")
2744 (ss_ashift:<VWIDE>
2745 (mult:<VWIDE>
2746 (sign_extend:<VWIDE>
2747 (match_operand:VD_HSI 2 "register_operand" "w"))
2748 (sign_extend:<VWIDE>
2749 (vec_duplicate:VD_HSI
2750 (vec_select:<VEL>
2751 (match_operand:<VCON> 3 "register_operand" "<vwx>")
2752 (parallel [(match_operand:SI 4 "immediate_operand" "i")])))
2753 ))
2754 (const_int 1))))]
2755 "TARGET_SIMD"
2756 "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]"
2757 [(set_attr "simd_type" "simd_sat_mlal")
2758 (set_attr "simd_mode" "<MODE>")]
2759 )
2761 (define_insn "aarch64_sqdml<SBINQOPS:as>l_lane<mode>_internal"
2762 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2763 (SBINQOPS:<VWIDE>
2764 (match_operand:<VWIDE> 1 "register_operand" "0")
2765 (ss_ashift:<VWIDE>
2766 (mult:<VWIDE>
2767 (sign_extend:<VWIDE>
2768 (match_operand:SD_HSI 2 "register_operand" "w"))
2769 (sign_extend:<VWIDE>
2770 (vec_select:<VEL>
2771 (match_operand:<VCON> 3 "register_operand" "<vwx>")
2772 (parallel [(match_operand:SI 4 "immediate_operand" "i")])))
2773 )
2774 (const_int 1))))]
2775 "TARGET_SIMD"
2776 "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]"
2777 [(set_attr "simd_type" "simd_sat_mlal")
2778 (set_attr "simd_mode" "<MODE>")]
2779 )
2781 (define_expand "aarch64_sqdmlal_lane<mode>"
2782 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2783 (match_operand:<VWIDE> 1 "register_operand" "0")
2784 (match_operand:VSD_HSI 2 "register_operand" "w")
2785 (match_operand:<VCON> 3 "register_operand" "<vwx>")
2786 (match_operand:SI 4 "immediate_operand" "i")]
2787 "TARGET_SIMD"
2788 {
2789 aarch64_simd_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<VCON>mode) / 2);
2790 emit_insn (gen_aarch64_sqdmlal_lane<mode>_internal (operands[0], operands[1],
2791 operands[2], operands[3],
2792 operands[4]));
2793 DONE;
2794 })
2796 (define_expand "aarch64_sqdmlal_laneq<mode>"
2797 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2798 (match_operand:<VWIDE> 1 "register_operand" "0")
2799 (match_operand:VSD_HSI 2 "register_operand" "w")
2800 (match_operand:<VCON> 3 "register_operand" "<vwx>")
2801 (match_operand:SI 4 "immediate_operand" "i")]
2802 "TARGET_SIMD"
2803 {
2804 aarch64_simd_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<VCON>mode));
2805 emit_insn (gen_aarch64_sqdmlal_lane<mode>_internal (operands[0], operands[1],
2806 operands[2], operands[3],
2807 operands[4]));
2808 DONE;
2809 })
2811 (define_expand "aarch64_sqdmlsl_lane<mode>"
2812 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2813 (match_operand:<VWIDE> 1 "register_operand" "0")
2814 (match_operand:VSD_HSI 2 "register_operand" "w")
2815 (match_operand:<VCON> 3 "register_operand" "<vwx>")
2816 (match_operand:SI 4 "immediate_operand" "i")]
2817 "TARGET_SIMD"
2818 {
2819 aarch64_simd_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<VCON>mode) / 2);
2820 emit_insn (gen_aarch64_sqdmlsl_lane<mode>_internal (operands[0], operands[1],
2821 operands[2], operands[3],
2822 operands[4]));
2823 DONE;
2824 })
2826 (define_expand "aarch64_sqdmlsl_laneq<mode>"
2827 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2828 (match_operand:<VWIDE> 1 "register_operand" "0")
2829 (match_operand:VSD_HSI 2 "register_operand" "w")
2830 (match_operand:<VCON> 3 "register_operand" "<vwx>")
2831 (match_operand:SI 4 "immediate_operand" "i")]
2832 "TARGET_SIMD"
2833 {
2834 aarch64_simd_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<VCON>mode));
2835 emit_insn (gen_aarch64_sqdmlsl_lane<mode>_internal (operands[0], operands[1],
2836 operands[2], operands[3],
2837 operands[4]));
2838 DONE;
2839 })
2841 ;; vqdml[sa]l_n
2843 (define_insn "aarch64_sqdml<SBINQOPS:as>l_n<mode>"
2844 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2845 (SBINQOPS:<VWIDE>
2846 (match_operand:<VWIDE> 1 "register_operand" "0")
2847 (ss_ashift:<VWIDE>
2848 (mult:<VWIDE>
2849 (sign_extend:<VWIDE>
2850 (match_operand:VD_HSI 2 "register_operand" "w"))
2851 (sign_extend:<VWIDE>
2852 (vec_duplicate:VD_HSI
2853 (match_operand:<VEL> 3 "register_operand" "<vwx>"))))
2854 (const_int 1))))]
2855 "TARGET_SIMD"
2856 "sqdml<SBINQOPS:as>l\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[0]"
2857 [(set_attr "simd_type" "simd_sat_mlal")
2858 (set_attr "simd_mode" "<MODE>")]
2859 )
2861 ;; sqdml[as]l2
2863 (define_insn "aarch64_sqdml<SBINQOPS:as>l2<mode>_internal"
2864 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2865 (SBINQOPS:<VWIDE>
2866 (match_operand:<VWIDE> 1 "register_operand" "0")
2867 (ss_ashift:<VWIDE>
2868 (mult:<VWIDE>
2869 (sign_extend:<VWIDE>
2870 (vec_select:<VHALF>
2871 (match_operand:VQ_HSI 2 "register_operand" "w")
2872 (match_operand:VQ_HSI 4 "vect_par_cnst_hi_half" "")))
2873 (sign_extend:<VWIDE>
2874 (vec_select:<VHALF>
2875 (match_operand:VQ_HSI 3 "register_operand" "w")
2876 (match_dup 4))))
2877 (const_int 1))))]
2878 "TARGET_SIMD"
2879 "sqdml<SBINQOPS:as>l2\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %<v>3<Vmtype>"
2880 [(set_attr "simd_type" "simd_sat_mlal")
2881 (set_attr "simd_mode" "<MODE>")]
2882 )
2884 (define_expand "aarch64_sqdmlal2<mode>"
2885 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2886 (match_operand:<VWIDE> 1 "register_operand" "w")
2887 (match_operand:VQ_HSI 2 "register_operand" "w")
2888 (match_operand:VQ_HSI 3 "register_operand" "w")]
2889 "TARGET_SIMD"
2890 {
2891 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2892 emit_insn (gen_aarch64_sqdmlal2<mode>_internal (operands[0], operands[1],
2893 operands[2], operands[3], p));
2894 DONE;
2895 })
2897 (define_expand "aarch64_sqdmlsl2<mode>"
2898 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2899 (match_operand:<VWIDE> 1 "register_operand" "w")
2900 (match_operand:VQ_HSI 2 "register_operand" "w")
2901 (match_operand:VQ_HSI 3 "register_operand" "w")]
2902 "TARGET_SIMD"
2903 {
2904 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2905 emit_insn (gen_aarch64_sqdmlsl2<mode>_internal (operands[0], operands[1],
2906 operands[2], operands[3], p));
2907 DONE;
2908 })
2910 ;; vqdml[sa]l2_lane
2912 (define_insn "aarch64_sqdml<SBINQOPS:as>l2_lane<mode>_internal"
2913 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
2914 (SBINQOPS:<VWIDE>
2915 (match_operand:<VWIDE> 1 "register_operand" "0")
2916 (ss_ashift:<VWIDE>
2917 (mult:<VWIDE>
2918 (sign_extend:<VWIDE>
2919 (vec_select:<VHALF>
2920 (match_operand:VQ_HSI 2 "register_operand" "w")
2921 (match_operand:VQ_HSI 5 "vect_par_cnst_hi_half" "")))
2922 (sign_extend:<VWIDE>
2923 (vec_duplicate:<VHALF>
2924 (vec_select:<VEL>
2925 (match_operand:<VCON> 3 "register_operand" "<vwx>")
2926 (parallel [(match_operand:SI 4 "immediate_operand" "i")])
2927 ))))
2928 (const_int 1))))]
2929 "TARGET_SIMD"
2930 "sqdml<SBINQOPS:as>l2\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[%4]"
2931 [(set_attr "simd_type" "simd_sat_mlal")
2932 (set_attr "simd_mode" "<MODE>")]
2933 )
2935 (define_expand "aarch64_sqdmlal2_lane<mode>"
2936 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2937 (match_operand:<VWIDE> 1 "register_operand" "w")
2938 (match_operand:VQ_HSI 2 "register_operand" "w")
2939 (match_operand:<VCON> 3 "register_operand" "<vwx>")
2940 (match_operand:SI 4 "immediate_operand" "i")]
2941 "TARGET_SIMD"
2942 {
2943 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2944 aarch64_simd_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode) / 2);
2945 emit_insn (gen_aarch64_sqdmlal2_lane<mode>_internal (operands[0], operands[1],
2946 operands[2], operands[3],
2947 operands[4], p));
2948 DONE;
2949 })
2951 (define_expand "aarch64_sqdmlal2_laneq<mode>"
2952 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2953 (match_operand:<VWIDE> 1 "register_operand" "w")
2954 (match_operand:VQ_HSI 2 "register_operand" "w")
2955 (match_operand:<VCON> 3 "register_operand" "<vwx>")
2956 (match_operand:SI 4 "immediate_operand" "i")]
2957 "TARGET_SIMD"
2958 {
2959 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2960 aarch64_simd_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
2961 emit_insn (gen_aarch64_sqdmlal2_lane<mode>_internal (operands[0], operands[1],
2962 operands[2], operands[3],
2963 operands[4], p));
2964 DONE;
2965 })
2967 (define_expand "aarch64_sqdmlsl2_lane<mode>"
2968 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2969 (match_operand:<VWIDE> 1 "register_operand" "w")
2970 (match_operand:VQ_HSI 2 "register_operand" "w")
2971 (match_operand:<VCON> 3 "register_operand" "<vwx>")
2972 (match_operand:SI 4 "immediate_operand" "i")]
2973 "TARGET_SIMD"
2974 {
2975 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2976 aarch64_simd_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode) / 2);
2977 emit_insn (gen_aarch64_sqdmlsl2_lane<mode>_internal (operands[0], operands[1],
2978 operands[2], operands[3],
2979 operands[4], p));
2980 DONE;
2981 })
2983 (define_expand "aarch64_sqdmlsl2_laneq<mode>"
2984 [(match_operand:<VWIDE> 0 "register_operand" "=w")
2985 (match_operand:<VWIDE> 1 "register_operand" "w")
2986 (match_operand:VQ_HSI 2 "register_operand" "w")
2987 (match_operand:<VCON> 3 "register_operand" "<vwx>")
2988 (match_operand:SI 4 "immediate_operand" "i")]
2989 "TARGET_SIMD"
2990 {
2991 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
2992 aarch64_simd_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
2993 emit_insn (gen_aarch64_sqdmlsl2_lane<mode>_internal (operands[0], operands[1],
2994 operands[2], operands[3],
2995 operands[4], p));
2996 DONE;
2997 })
2999 (define_insn "aarch64_sqdml<SBINQOPS:as>l2_n<mode>_internal"
3000 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3001 (SBINQOPS:<VWIDE>
3002 (match_operand:<VWIDE> 1 "register_operand" "0")
3003 (ss_ashift:<VWIDE>
3004 (mult:<VWIDE>
3005 (sign_extend:<VWIDE>
3006 (vec_select:<VHALF>
3007 (match_operand:VQ_HSI 2 "register_operand" "w")
3008 (match_operand:VQ_HSI 4 "vect_par_cnst_hi_half" "")))
3009 (sign_extend:<VWIDE>
3010 (vec_duplicate:<VHALF>
3011 (match_operand:<VEL> 3 "register_operand" "<vwx>"))))
3012 (const_int 1))))]
3013 "TARGET_SIMD"
3014 "sqdml<SBINQOPS:as>l2\\t%<vw2>0<Vmwtype>, %<v>2<Vmtype>, %3.<Vetype>[0]"
3015 [(set_attr "simd_type" "simd_sat_mlal")
3016 (set_attr "simd_mode" "<MODE>")]
3017 )
3019 (define_expand "aarch64_sqdmlal2_n<mode>"
3020 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3021 (match_operand:<VWIDE> 1 "register_operand" "w")
3022 (match_operand:VQ_HSI 2 "register_operand" "w")
3023 (match_operand:<VEL> 3 "register_operand" "w")]
3024 "TARGET_SIMD"
3025 {
3026 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3027 emit_insn (gen_aarch64_sqdmlal2_n<mode>_internal (operands[0], operands[1],
3028 operands[2], operands[3],
3029 p));
3030 DONE;
3031 })
3033 (define_expand "aarch64_sqdmlsl2_n<mode>"
3034 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3035 (match_operand:<VWIDE> 1 "register_operand" "w")
3036 (match_operand:VQ_HSI 2 "register_operand" "w")
3037 (match_operand:<VEL> 3 "register_operand" "w")]
3038 "TARGET_SIMD"
3039 {
3040 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3041 emit_insn (gen_aarch64_sqdmlsl2_n<mode>_internal (operands[0], operands[1],
3042 operands[2], operands[3],
3043 p));
3044 DONE;
3045 })
3047 ;; vqdmull
3049 (define_insn "aarch64_sqdmull<mode>"
3050 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3051 (ss_ashift:<VWIDE>
3052 (mult:<VWIDE>
3053 (sign_extend:<VWIDE>
3054 (match_operand:VSD_HSI 1 "register_operand" "w"))
3055 (sign_extend:<VWIDE>
3056 (match_operand:VSD_HSI 2 "register_operand" "w")))
3057 (const_int 1)))]
3058 "TARGET_SIMD"
3059 "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
3060 [(set_attr "simd_type" "simd_sat_mul")
3061 (set_attr "simd_mode" "<MODE>")]
3062 )
3064 ;; vqdmull_lane
3066 (define_insn "aarch64_sqdmull_lane<mode>_internal"
3067 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3068 (ss_ashift:<VWIDE>
3069 (mult:<VWIDE>
3070 (sign_extend:<VWIDE>
3071 (match_operand:VD_HSI 1 "register_operand" "w"))
3072 (sign_extend:<VWIDE>
3073 (vec_duplicate:VD_HSI
3074 (vec_select:<VEL>
3075 (match_operand:<VCON> 2 "register_operand" "<vwx>")
3076 (parallel [(match_operand:SI 3 "immediate_operand" "i")])))
3077 ))
3078 (const_int 1)))]
3079 "TARGET_SIMD"
3080 "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]"
3081 [(set_attr "simd_type" "simd_sat_mul")
3082 (set_attr "simd_mode" "<MODE>")]
3083 )
3085 (define_insn "aarch64_sqdmull_lane<mode>_internal"
3086 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3087 (ss_ashift:<VWIDE>
3088 (mult:<VWIDE>
3089 (sign_extend:<VWIDE>
3090 (match_operand:SD_HSI 1 "register_operand" "w"))
3091 (sign_extend:<VWIDE>
3092 (vec_select:<VEL>
3093 (match_operand:<VCON> 2 "register_operand" "<vwx>")
3094 (parallel [(match_operand:SI 3 "immediate_operand" "i")]))
3095 ))
3096 (const_int 1)))]
3097 "TARGET_SIMD"
3098 "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]"
3099 [(set_attr "simd_type" "simd_sat_mul")
3100 (set_attr "simd_mode" "<MODE>")]
3101 )
3103 (define_expand "aarch64_sqdmull_lane<mode>"
3104 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3105 (match_operand:VSD_HSI 1 "register_operand" "w")
3106 (match_operand:<VCON> 2 "register_operand" "<vwx>")
3107 (match_operand:SI 3 "immediate_operand" "i")]
3108 "TARGET_SIMD"
3109 {
3110 aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCON>mode) / 2);
3111 emit_insn (gen_aarch64_sqdmull_lane<mode>_internal (operands[0], operands[1],
3112 operands[2], operands[3]));
3113 DONE;
3114 })
3116 (define_expand "aarch64_sqdmull_laneq<mode>"
3117 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3118 (match_operand:VD_HSI 1 "register_operand" "w")
3119 (match_operand:<VCON> 2 "register_operand" "<vwx>")
3120 (match_operand:SI 3 "immediate_operand" "i")]
3121 "TARGET_SIMD"
3122 {
3123 aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<VCON>mode));
3124 emit_insn (gen_aarch64_sqdmull_lane<mode>_internal
3125 (operands[0], operands[1], operands[2], operands[3]));
3126 DONE;
3127 })
3129 ;; vqdmull_n
3131 (define_insn "aarch64_sqdmull_n<mode>"
3132 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3133 (ss_ashift:<VWIDE>
3134 (mult:<VWIDE>
3135 (sign_extend:<VWIDE>
3136 (match_operand:VD_HSI 1 "register_operand" "w"))
3137 (sign_extend:<VWIDE>
3138 (vec_duplicate:VD_HSI
3139 (match_operand:<VEL> 2 "register_operand" "<vwx>")))
3140 )
3141 (const_int 1)))]
3142 "TARGET_SIMD"
3143 "sqdmull\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[0]"
3144 [(set_attr "simd_type" "simd_sat_mul")
3145 (set_attr "simd_mode" "<MODE>")]
3146 )
3148 ;; vqdmull2
3152 (define_insn "aarch64_sqdmull2<mode>_internal"
3153 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3154 (ss_ashift:<VWIDE>
3155 (mult:<VWIDE>
3156 (sign_extend:<VWIDE>
3157 (vec_select:<VHALF>
3158 (match_operand:VQ_HSI 1 "register_operand" "w")
3159 (match_operand:VQ_HSI 3 "vect_par_cnst_hi_half" "")))
3160 (sign_extend:<VWIDE>
3161 (vec_select:<VHALF>
3162 (match_operand:VQ_HSI 2 "register_operand" "w")
3163 (match_dup 3)))
3164 )
3165 (const_int 1)))]
3166 "TARGET_SIMD"
3167 "sqdmull2\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
3168 [(set_attr "simd_type" "simd_sat_mul")
3169 (set_attr "simd_mode" "<MODE>")]
3170 )
3172 (define_expand "aarch64_sqdmull2<mode>"
3173 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3174 (match_operand:VQ_HSI 1 "register_operand" "w")
3175 (match_operand:<VCON> 2 "register_operand" "w")]
3176 "TARGET_SIMD"
3177 {
3178 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3179 emit_insn (gen_aarch64_sqdmull2<mode>_internal (operands[0], operands[1],
3180 operands[2], p));
3181 DONE;
3182 })
3184 ;; vqdmull2_lane
3186 (define_insn "aarch64_sqdmull2_lane<mode>_internal"
3187 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3188 (ss_ashift:<VWIDE>
3189 (mult:<VWIDE>
3190 (sign_extend:<VWIDE>
3191 (vec_select:<VHALF>
3192 (match_operand:VQ_HSI 1 "register_operand" "w")
3193 (match_operand:VQ_HSI 4 "vect_par_cnst_hi_half" "")))
3194 (sign_extend:<VWIDE>
3195 (vec_duplicate:<VHALF>
3196 (vec_select:<VEL>
3197 (match_operand:<VCON> 2 "register_operand" "<vwx>")
3198 (parallel [(match_operand:SI 3 "immediate_operand" "i")])))
3199 ))
3200 (const_int 1)))]
3201 "TARGET_SIMD"
3202 "sqdmull2\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[%3]"
3203 [(set_attr "simd_type" "simd_sat_mul")
3204 (set_attr "simd_mode" "<MODE>")]
3205 )
3207 (define_expand "aarch64_sqdmull2_lane<mode>"
3208 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3209 (match_operand:VQ_HSI 1 "register_operand" "w")
3210 (match_operand:<VCON> 2 "register_operand" "<vwx>")
3211 (match_operand:SI 3 "immediate_operand" "i")]
3212 "TARGET_SIMD"
3213 {
3214 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3215 aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode) / 2);
3216 emit_insn (gen_aarch64_sqdmull2_lane<mode>_internal (operands[0], operands[1],
3217 operands[2], operands[3],
3218 p));
3219 DONE;
3220 })
3222 (define_expand "aarch64_sqdmull2_laneq<mode>"
3223 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3224 (match_operand:VQ_HSI 1 "register_operand" "w")
3225 (match_operand:<VCON> 2 "register_operand" "<vwx>")
3226 (match_operand:SI 3 "immediate_operand" "i")]
3227 "TARGET_SIMD"
3228 {
3229 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3230 aarch64_simd_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3231 emit_insn (gen_aarch64_sqdmull2_lane<mode>_internal (operands[0], operands[1],
3232 operands[2], operands[3],
3233 p));
3234 DONE;
3235 })
3237 ;; vqdmull2_n
3239 (define_insn "aarch64_sqdmull2_n<mode>_internal"
3240 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3241 (ss_ashift:<VWIDE>
3242 (mult:<VWIDE>
3243 (sign_extend:<VWIDE>
3244 (vec_select:<VHALF>
3245 (match_operand:VQ_HSI 1 "register_operand" "w")
3246 (match_operand:VQ_HSI 3 "vect_par_cnst_hi_half" "")))
3247 (sign_extend:<VWIDE>
3248 (vec_duplicate:<VHALF>
3249 (match_operand:<VEL> 2 "register_operand" "<vwx>")))
3250 )
3251 (const_int 1)))]
3252 "TARGET_SIMD"
3253 "sqdmull2\\t%<vw2>0<Vmwtype>, %<v>1<Vmtype>, %2.<Vetype>[0]"
3254 [(set_attr "simd_type" "simd_sat_mul")
3255 (set_attr "simd_mode" "<MODE>")]
3256 )
3258 (define_expand "aarch64_sqdmull2_n<mode>"
3259 [(match_operand:<VWIDE> 0 "register_operand" "=w")
3260 (match_operand:VQ_HSI 1 "register_operand" "w")
3261 (match_operand:<VEL> 2 "register_operand" "w")]
3262 "TARGET_SIMD"
3263 {
3264 rtx p = aarch64_simd_vect_par_cnst_half (<MODE>mode, true);
3265 emit_insn (gen_aarch64_sqdmull2_n<mode>_internal (operands[0], operands[1],
3266 operands[2], p));
3267 DONE;
3268 })
3270 ;; vshl
3272 (define_insn "aarch64_<sur>shl<mode>"
3273 [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
3274 (unspec:VSDQ_I_DI
3275 [(match_operand:VSDQ_I_DI 1 "register_operand" "w")
3276 (match_operand:VSDQ_I_DI 2 "register_operand" "w")]
3277 VSHL))]
3278 "TARGET_SIMD"
3279 "<sur>shl\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>";
3280 [(set_attr "simd_type" "simd_shift")
3281 (set_attr "simd_mode" "<MODE>")]
3282 )
3285 ;; vqshl
3287 (define_insn "aarch64_<sur>q<r>shl<mode>"
3288 [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
3289 (unspec:VSDQ_I
3290 [(match_operand:VSDQ_I 1 "register_operand" "w")
3291 (match_operand:VSDQ_I 2 "register_operand" "w")]
3292 VQSHL))]
3293 "TARGET_SIMD"
3294 "<sur>q<r>shl\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>";
3295 [(set_attr "simd_type" "simd_sat_shift")
3296 (set_attr "simd_mode" "<MODE>")]
3297 )
3299 ;; vshll_n
3301 (define_insn "aarch64_<sur>shll_n<mode>"
3302 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3303 (unspec:<VWIDE> [(match_operand:VDW 1 "register_operand" "w")
3304 (match_operand:SI 2 "immediate_operand" "i")]
3305 VSHLL))]
3306 "TARGET_SIMD"
3307 "*
3308 int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
3309 aarch64_simd_const_bounds (operands[2], 0, bit_width + 1);
3310 if (INTVAL (operands[2]) == bit_width)
3311 {
3312 return \"shll\\t%0.<Vwtype>, %1.<Vtype>, %2\";
3313 }
3314 else {
3315 return \"<sur>shll\\t%0.<Vwtype>, %1.<Vtype>, %2\";
3316 }"
3317 [(set_attr "simd_type" "simd_shift_imm")
3318 (set_attr "simd_mode" "<MODE>")]
3319 )
3321 ;; vshll_high_n
3323 (define_insn "aarch64_<sur>shll2_n<mode>"
3324 [(set (match_operand:<VWIDE> 0 "register_operand" "=w")
3325 (unspec:<VWIDE> [(match_operand:VQW 1 "register_operand" "w")
3326 (match_operand:SI 2 "immediate_operand" "i")]
3327 VSHLL))]
3328 "TARGET_SIMD"
3329 "*
3330 int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
3331 aarch64_simd_const_bounds (operands[2], 0, bit_width + 1);
3332 if (INTVAL (operands[2]) == bit_width)
3333 {
3334 return \"shll2\\t%0.<Vwtype>, %1.<Vtype>, %2\";
3335 }
3336 else {
3337 return \"<sur>shll2\\t%0.<Vwtype>, %1.<Vtype>, %2\";
3338 }"
3339 [(set_attr "simd_type" "simd_shift_imm")
3340 (set_attr "simd_mode" "<MODE>")]
3341 )
3343 ;; vrshr_n
3345 (define_insn "aarch64_<sur>shr_n<mode>"
3346 [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
3347 (unspec:VSDQ_I_DI [(match_operand:VSDQ_I_DI 1 "register_operand" "w")
3348 (match_operand:SI 2 "immediate_operand" "i")]
3349 VRSHR_N))]
3350 "TARGET_SIMD"
3351 "*
3352 int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
3353 aarch64_simd_const_bounds (operands[2], 1, bit_width + 1);
3354 return \"<sur>shr\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %2\";"
3355 [(set_attr "simd_type" "simd_shift_imm")
3356 (set_attr "simd_mode" "<MODE>")]
3357 )
3359 ;; v(r)sra_n
3361 (define_insn "aarch64_<sur>sra_n<mode>"
3362 [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
3363 (unspec:VSDQ_I_DI [(match_operand:VSDQ_I_DI 1 "register_operand" "0")
3364 (match_operand:VSDQ_I_DI 2 "register_operand" "w")
3365 (match_operand:SI 3 "immediate_operand" "i")]
3366 VSRA))]
3367 "TARGET_SIMD"
3368 "*
3369 int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
3370 aarch64_simd_const_bounds (operands[3], 1, bit_width + 1);
3371 return \"<sur>sra\\t%<v>0<Vmtype>, %<v>2<Vmtype>, %3\";"
3372 [(set_attr "simd_type" "simd_shift_imm_acc")
3373 (set_attr "simd_mode" "<MODE>")]
3374 )
3376 ;; vs<lr>i_n
3378 (define_insn "aarch64_<sur>s<lr>i_n<mode>"
3379 [(set (match_operand:VSDQ_I_DI 0 "register_operand" "=w")
3380 (unspec:VSDQ_I_DI [(match_operand:VSDQ_I_DI 1 "register_operand" "0")
3381 (match_operand:VSDQ_I_DI 2 "register_operand" "w")
3382 (match_operand:SI 3 "immediate_operand" "i")]
3383 VSLRI))]
3384 "TARGET_SIMD"
3385 "*
3386 int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
3387 aarch64_simd_const_bounds (operands[3], 1 - <VSLRI:offsetlr>,
3388 bit_width - <VSLRI:offsetlr> + 1);
3389 return \"s<lr>i\\t%<v>0<Vmtype>, %<v>2<Vmtype>, %3\";"
3390 [(set_attr "simd_type" "simd_shift_imm")
3391 (set_attr "simd_mode" "<MODE>")]
3392 )
3394 ;; vqshl(u)
3396 (define_insn "aarch64_<sur>qshl<u>_n<mode>"
3397 [(set (match_operand:VSDQ_I 0 "register_operand" "=w")
3398 (unspec:VSDQ_I [(match_operand:VSDQ_I 1 "register_operand" "w")
3399 (match_operand:SI 2 "immediate_operand" "i")]
3400 VQSHL_N))]
3401 "TARGET_SIMD"
3402 "*
3403 int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
3404 aarch64_simd_const_bounds (operands[2], 0, bit_width);
3405 return \"<sur>qshl<u>\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %2\";"
3406 [(set_attr "simd_type" "simd_sat_shift_imm")
3407 (set_attr "simd_mode" "<MODE>")]
3408 )
3411 ;; vq(r)shr(u)n_n
3413 (define_insn "aarch64_<sur>q<r>shr<u>n_n<mode>"
3414 [(set (match_operand:<VNARROWQ> 0 "register_operand" "=w")
3415 (unspec:<VNARROWQ> [(match_operand:VSQN_HSDI 1 "register_operand" "w")
3416 (match_operand:SI 2 "immediate_operand" "i")]
3417 VQSHRN_N))]
3418 "TARGET_SIMD"
3419 "*
3420 int bit_width = GET_MODE_UNIT_SIZE (<MODE>mode) * BITS_PER_UNIT;
3421 aarch64_simd_const_bounds (operands[2], 1, bit_width + 1);
3422 return \"<sur>q<r>shr<u>n\\t%<vn2>0<Vmntype>, %<v>1<Vmtype>, %2\";"
3423 [(set_attr "simd_type" "simd_sat_shiftn_imm")
3424 (set_attr "simd_mode" "<MODE>")]
3425 )
3428 ;; cm(eq|ge|gt|lt|le)
3429 ;; Note, we have constraints for Dz and Z as different expanders
3430 ;; have different ideas of what should be passed to this pattern.
3432 (define_insn "aarch64_cm<optab><mode>"
3433 [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w,w")
3434 (neg:<V_cmp_result>
3435 (COMPARISONS:<V_cmp_result>
3436 (match_operand:VDQ 1 "register_operand" "w,w")
3437 (match_operand:VDQ 2 "aarch64_simd_reg_or_zero" "w,ZDz")
3438 )))]
3439 "TARGET_SIMD"
3440 "@
3441 cm<n_optab>\t%<v>0<Vmtype>, %<v><cmp_1><Vmtype>, %<v><cmp_2><Vmtype>
3442 cm<optab>\t%<v>0<Vmtype>, %<v>1<Vmtype>, #0"
3443 [(set_attr "simd_type" "simd_cmp")
3444 (set_attr "simd_mode" "<MODE>")]
3445 )
3447 (define_insn_and_split "aarch64_cm<optab>di"
3448 [(set (match_operand:DI 0 "register_operand" "=w,w,r")
3449 (neg:DI
3450 (COMPARISONS:DI
3451 (match_operand:DI 1 "register_operand" "w,w,r")
3452 (match_operand:DI 2 "aarch64_simd_reg_or_zero" "w,ZDz,r")
3453 )))
3454 (clobber (reg:CC CC_REGNUM))]
3455 "TARGET_SIMD"
3456 "@
3457 cm<n_optab>\t%d0, %d<cmp_1>, %d<cmp_2>
3458 cm<optab>\t%d0, %d1, #0
3459 #"
3460 "reload_completed
3461 /* We need to prevent the split from
3462 happening in the 'w' constraint cases. */
3463 && GP_REGNUM_P (REGNO (operands[0]))
3464 && GP_REGNUM_P (REGNO (operands[1]))"
3465 [(const_int 0)]
3466 {
3467 enum machine_mode mode = SELECT_CC_MODE (<CMP>, operands[1], operands[2]);
3468 rtx cc_reg = aarch64_gen_compare_reg (<CMP>, operands[1], operands[2]);
3469 rtx comparison = gen_rtx_<CMP> (mode, operands[1], operands[2]);
3470 emit_insn (gen_cstoredi_neg (operands[0], comparison, cc_reg));
3471 DONE;
3472 }
3473 [(set_attr "simd_type" "simd_cmp")
3474 (set_attr "simd_mode" "DI")]
3475 )
3477 ;; cm(hs|hi)
3479 (define_insn "aarch64_cm<optab><mode>"
3480 [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w")
3481 (neg:<V_cmp_result>
3482 (UCOMPARISONS:<V_cmp_result>
3483 (match_operand:VDQ 1 "register_operand" "w")
3484 (match_operand:VDQ 2 "register_operand" "w")
3485 )))]
3486 "TARGET_SIMD"
3487 "cm<n_optab>\t%<v>0<Vmtype>, %<v><cmp_1><Vmtype>, %<v><cmp_2><Vmtype>"
3488 [(set_attr "simd_type" "simd_cmp")
3489 (set_attr "simd_mode" "<MODE>")]
3490 )
3492 (define_insn_and_split "aarch64_cm<optab>di"
3493 [(set (match_operand:DI 0 "register_operand" "=w,r")
3494 (neg:DI
3495 (UCOMPARISONS:DI
3496 (match_operand:DI 1 "register_operand" "w,r")
3497 (match_operand:DI 2 "aarch64_simd_reg_or_zero" "w,r")
3498 )))
3499 (clobber (reg:CC CC_REGNUM))]
3500 "TARGET_SIMD"
3501 "@
3502 cm<n_optab>\t%d0, %d<cmp_1>, %d<cmp_2>
3503 #"
3504 "reload_completed
3505 /* We need to prevent the split from
3506 happening in the 'w' constraint cases. */
3507 && GP_REGNUM_P (REGNO (operands[0]))
3508 && GP_REGNUM_P (REGNO (operands[1]))"
3509 [(const_int 0)]
3510 {
3511 enum machine_mode mode = CCmode;
3512 rtx cc_reg = aarch64_gen_compare_reg (<CMP>, operands[1], operands[2]);
3513 rtx comparison = gen_rtx_<CMP> (mode, operands[1], operands[2]);
3514 emit_insn (gen_cstoredi_neg (operands[0], comparison, cc_reg));
3515 DONE;
3516 }
3517 [(set_attr "simd_type" "simd_cmp")
3518 (set_attr "simd_mode" "DI")]
3519 )
3521 ;; cmtst
3523 (define_insn "aarch64_cmtst<mode>"
3524 [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w")
3525 (neg:<V_cmp_result>
3526 (ne:<V_cmp_result>
3527 (and:VDQ
3528 (match_operand:VDQ 1 "register_operand" "w")
3529 (match_operand:VDQ 2 "register_operand" "w"))
3530 (vec_duplicate:<V_cmp_result> (const_int 0)))))]
3531 "TARGET_SIMD"
3532 "cmtst\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
3533 [(set_attr "simd_type" "simd_cmp")
3534 (set_attr "simd_mode" "<MODE>")]
3535 )
3537 (define_insn_and_split "aarch64_cmtstdi"
3538 [(set (match_operand:DI 0 "register_operand" "=w,r")
3539 (neg:DI
3540 (ne:DI
3541 (and:DI
3542 (match_operand:DI 1 "register_operand" "w,r")
3543 (match_operand:DI 2 "register_operand" "w,r"))
3544 (const_int 0))))
3545 (clobber (reg:CC CC_REGNUM))]
3546 "TARGET_SIMD"
3547 "@
3548 cmtst\t%d0, %d1, %d2
3549 #"
3550 "reload_completed
3551 /* We need to prevent the split from
3552 happening in the 'w' constraint cases. */
3553 && GP_REGNUM_P (REGNO (operands[0]))
3554 && GP_REGNUM_P (REGNO (operands[1]))"
3555 [(const_int 0)]
3556 {
3557 rtx and_tree = gen_rtx_AND (DImode, operands[1], operands[2]);
3558 enum machine_mode mode = SELECT_CC_MODE (NE, and_tree, const0_rtx);
3559 rtx cc_reg = aarch64_gen_compare_reg (NE, and_tree, const0_rtx);
3560 rtx comparison = gen_rtx_NE (mode, and_tree, const0_rtx);
3561 emit_insn (gen_cstoredi_neg (operands[0], comparison, cc_reg));
3562 DONE;
3563 }
3564 [(set_attr "simd_type" "simd_cmp")
3565 (set_attr "simd_mode" "DI")]
3566 )
3568 ;; fcm(eq|ge|gt|le|lt)
3570 (define_insn "aarch64_cm<optab><mode>"
3571 [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w,w")
3572 (neg:<V_cmp_result>
3573 (COMPARISONS:<V_cmp_result>
3574 (match_operand:VALLF 1 "register_operand" "w,w")
3575 (match_operand:VALLF 2 "aarch64_simd_reg_or_zero" "w,YDz")
3576 )))]
3577 "TARGET_SIMD"
3578 "@
3579 fcm<n_optab>\t%<v>0<Vmtype>, %<v><cmp_1><Vmtype>, %<v><cmp_2><Vmtype>
3580 fcm<optab>\t%<v>0<Vmtype>, %<v>1<Vmtype>, 0"
3581 [(set_attr "simd_type" "simd_fcmp")
3582 (set_attr "simd_mode" "<MODE>")]
3583 )
3585 ;; fac(ge|gt)
3586 ;; Note we can also handle what would be fac(le|lt) by
3587 ;; generating fac(ge|gt).
3589 (define_insn "*aarch64_fac<optab><mode>"
3590 [(set (match_operand:<V_cmp_result> 0 "register_operand" "=w")
3591 (neg:<V_cmp_result>
3592 (FAC_COMPARISONS:<V_cmp_result>
3593 (abs:VALLF (match_operand:VALLF 1 "register_operand" "w"))
3594 (abs:VALLF (match_operand:VALLF 2 "register_operand" "w"))
3595 )))]
3596 "TARGET_SIMD"
3597 "fac<n_optab>\t%<v>0<Vmtype>, %<v><cmp_1><Vmtype>, %<v><cmp_2><Vmtype>"
3598 [(set_attr "simd_type" "simd_fcmp")
3599 (set_attr "simd_mode" "<MODE>")]
3600 )
3602 ;; addp
3604 (define_insn "aarch64_addp<mode>"
3605 [(set (match_operand:VD_BHSI 0 "register_operand" "=w")
3606 (unspec:VD_BHSI
3607 [(match_operand:VD_BHSI 1 "register_operand" "w")
3608 (match_operand:VD_BHSI 2 "register_operand" "w")]
3609 UNSPEC_ADDP))]
3610 "TARGET_SIMD"
3611 "addp\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
3612 [(set_attr "simd_type" "simd_add")
3613 (set_attr "simd_mode" "<MODE>")]
3614 )
3616 (define_insn "aarch64_addpdi"
3617 [(set (match_operand:DI 0 "register_operand" "=w")
3618 (unspec:DI
3619 [(match_operand:V2DI 1 "register_operand" "w")]
3620 UNSPEC_ADDP))]
3621 "TARGET_SIMD"
3622 "addp\t%d0, %1.2d"
3623 [(set_attr "simd_type" "simd_add")
3624 (set_attr "simd_mode" "DI")]
3625 )
3627 ;; sqrt
3629 (define_insn "sqrt<mode>2"
3630 [(set (match_operand:VDQF 0 "register_operand" "=w")
3631 (sqrt:VDQF (match_operand:VDQF 1 "register_operand" "w")))]
3632 "TARGET_SIMD"
3633 "fsqrt\\t%0.<Vtype>, %1.<Vtype>"
3634 [(set_attr "simd_type" "simd_fsqrt")
3635 (set_attr "simd_mode" "<MODE>")]
3636 )
3638 ;; Patterns for vector struct loads and stores.
3640 (define_insn "vec_load_lanesoi<mode>"
3641 [(set (match_operand:OI 0 "register_operand" "=w")
3642 (unspec:OI [(match_operand:OI 1 "aarch64_simd_struct_operand" "Utv")
3643 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3644 UNSPEC_LD2))]
3645 "TARGET_SIMD"
3646 "ld2\\t{%S0.<Vtype> - %T0.<Vtype>}, %1"
3647 [(set_attr "simd_type" "simd_load2")
3648 (set_attr "simd_mode" "<MODE>")])
3650 (define_insn "vec_store_lanesoi<mode>"
3651 [(set (match_operand:OI 0 "aarch64_simd_struct_operand" "=Utv")
3652 (unspec:OI [(match_operand:OI 1 "register_operand" "w")
3653 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3654 UNSPEC_ST2))]
3655 "TARGET_SIMD"
3656 "st2\\t{%S1.<Vtype> - %T1.<Vtype>}, %0"
3657 [(set_attr "simd_type" "simd_store2")
3658 (set_attr "simd_mode" "<MODE>")])
3660 (define_insn "vec_load_lanesci<mode>"
3661 [(set (match_operand:CI 0 "register_operand" "=w")
3662 (unspec:CI [(match_operand:CI 1 "aarch64_simd_struct_operand" "Utv")
3663 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3664 UNSPEC_LD3))]
3665 "TARGET_SIMD"
3666 "ld3\\t{%S0.<Vtype> - %U0.<Vtype>}, %1"
3667 [(set_attr "simd_type" "simd_load3")
3668 (set_attr "simd_mode" "<MODE>")])
3670 (define_insn "vec_store_lanesci<mode>"
3671 [(set (match_operand:CI 0 "aarch64_simd_struct_operand" "=Utv")
3672 (unspec:CI [(match_operand:CI 1 "register_operand" "w")
3673 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3674 UNSPEC_ST3))]
3675 "TARGET_SIMD"
3676 "st3\\t{%S1.<Vtype> - %U1.<Vtype>}, %0"
3677 [(set_attr "simd_type" "simd_store3")
3678 (set_attr "simd_mode" "<MODE>")])
3680 (define_insn "vec_load_lanesxi<mode>"
3681 [(set (match_operand:XI 0 "register_operand" "=w")
3682 (unspec:XI [(match_operand:XI 1 "aarch64_simd_struct_operand" "Utv")
3683 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3684 UNSPEC_LD4))]
3685 "TARGET_SIMD"
3686 "ld4\\t{%S0.<Vtype> - %V0.<Vtype>}, %1"
3687 [(set_attr "simd_type" "simd_load4")
3688 (set_attr "simd_mode" "<MODE>")])
3690 (define_insn "vec_store_lanesxi<mode>"
3691 [(set (match_operand:XI 0 "aarch64_simd_struct_operand" "=Utv")
3692 (unspec:XI [(match_operand:XI 1 "register_operand" "w")
3693 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3694 UNSPEC_ST4))]
3695 "TARGET_SIMD"
3696 "st4\\t{%S1.<Vtype> - %V1.<Vtype>}, %0"
3697 [(set_attr "simd_type" "simd_store4")
3698 (set_attr "simd_mode" "<MODE>")])
3700 ;; Reload patterns for AdvSIMD register list operands.
3702 (define_expand "mov<mode>"
3703 [(set (match_operand:VSTRUCT 0 "aarch64_simd_nonimmediate_operand" "")
3704 (match_operand:VSTRUCT 1 "aarch64_simd_general_operand" ""))]
3705 "TARGET_SIMD"
3706 {
3707 if (can_create_pseudo_p ())
3708 {
3709 if (GET_CODE (operands[0]) != REG)
3710 operands[1] = force_reg (<MODE>mode, operands[1]);
3711 }
3712 })
3714 (define_insn "*aarch64_mov<mode>"
3715 [(set (match_operand:VSTRUCT 0 "aarch64_simd_nonimmediate_operand" "=w,Utv,w")
3716 (match_operand:VSTRUCT 1 "aarch64_simd_general_operand" " w,w,Utv"))]
3717 "TARGET_SIMD
3718 && (register_operand (operands[0], <MODE>mode)
3719 || register_operand (operands[1], <MODE>mode))"
3721 {
3722 switch (which_alternative)
3723 {
3724 case 0: return "#";
3725 case 1: return "st1\\t{%S1.16b - %<Vendreg>1.16b}, %0";
3726 case 2: return "ld1\\t{%S0.16b - %<Vendreg>0.16b}, %1";
3727 default: gcc_unreachable ();
3728 }
3729 }
3730 [(set_attr "simd_type" "simd_move,simd_store<nregs>,simd_load<nregs>")
3731 (set (attr "length") (symbol_ref "aarch64_simd_attr_length_move (insn)"))
3732 (set_attr "simd_mode" "<MODE>")])
3734 (define_split
3735 [(set (match_operand:OI 0 "register_operand" "")
3736 (match_operand:OI 1 "register_operand" ""))]
3737 "TARGET_SIMD && reload_completed"
3738 [(set (match_dup 0) (match_dup 1))
3739 (set (match_dup 2) (match_dup 3))]
3740 {
3741 int rdest = REGNO (operands[0]);
3742 int rsrc = REGNO (operands[1]);
3743 rtx dest[2], src[2];
3745 dest[0] = gen_rtx_REG (TFmode, rdest);
3746 src[0] = gen_rtx_REG (TFmode, rsrc);
3747 dest[1] = gen_rtx_REG (TFmode, rdest + 1);
3748 src[1] = gen_rtx_REG (TFmode, rsrc + 1);
3750 aarch64_simd_disambiguate_copy (operands, dest, src, 2);
3751 })
3753 (define_split
3754 [(set (match_operand:CI 0 "register_operand" "")
3755 (match_operand:CI 1 "register_operand" ""))]
3756 "TARGET_SIMD && reload_completed"
3757 [(set (match_dup 0) (match_dup 1))
3758 (set (match_dup 2) (match_dup 3))
3759 (set (match_dup 4) (match_dup 5))]
3760 {
3761 int rdest = REGNO (operands[0]);
3762 int rsrc = REGNO (operands[1]);
3763 rtx dest[3], src[3];
3765 dest[0] = gen_rtx_REG (TFmode, rdest);
3766 src[0] = gen_rtx_REG (TFmode, rsrc);
3767 dest[1] = gen_rtx_REG (TFmode, rdest + 1);
3768 src[1] = gen_rtx_REG (TFmode, rsrc + 1);
3769 dest[2] = gen_rtx_REG (TFmode, rdest + 2);
3770 src[2] = gen_rtx_REG (TFmode, rsrc + 2);
3772 aarch64_simd_disambiguate_copy (operands, dest, src, 3);
3773 })
3775 (define_split
3776 [(set (match_operand:XI 0 "register_operand" "")
3777 (match_operand:XI 1 "register_operand" ""))]
3778 "TARGET_SIMD && reload_completed"
3779 [(set (match_dup 0) (match_dup 1))
3780 (set (match_dup 2) (match_dup 3))
3781 (set (match_dup 4) (match_dup 5))
3782 (set (match_dup 6) (match_dup 7))]
3783 {
3784 int rdest = REGNO (operands[0]);
3785 int rsrc = REGNO (operands[1]);
3786 rtx dest[4], src[4];
3788 dest[0] = gen_rtx_REG (TFmode, rdest);
3789 src[0] = gen_rtx_REG (TFmode, rsrc);
3790 dest[1] = gen_rtx_REG (TFmode, rdest + 1);
3791 src[1] = gen_rtx_REG (TFmode, rsrc + 1);
3792 dest[2] = gen_rtx_REG (TFmode, rdest + 2);
3793 src[2] = gen_rtx_REG (TFmode, rsrc + 2);
3794 dest[3] = gen_rtx_REG (TFmode, rdest + 3);
3795 src[3] = gen_rtx_REG (TFmode, rsrc + 3);
3797 aarch64_simd_disambiguate_copy (operands, dest, src, 4);
3798 })
3800 (define_insn "aarch64_ld2<mode>_dreg"
3801 [(set (match_operand:OI 0 "register_operand" "=w")
3802 (subreg:OI
3803 (vec_concat:<VRL2>
3804 (vec_concat:<VDBL>
3805 (unspec:VD [(match_operand:TI 1 "aarch64_simd_struct_operand" "Utv")]
3806 UNSPEC_LD2)
3807 (vec_duplicate:VD (const_int 0)))
3808 (vec_concat:<VDBL>
3809 (unspec:VD [(match_dup 1)]
3810 UNSPEC_LD2)
3811 (vec_duplicate:VD (const_int 0)))) 0))]
3812 "TARGET_SIMD"
3813 "ld2\\t{%S0.<Vtype> - %T0.<Vtype>}, %1"
3814 [(set_attr "simd_type" "simd_load2")
3815 (set_attr "simd_mode" "<MODE>")])
3817 (define_insn "aarch64_ld2<mode>_dreg"
3818 [(set (match_operand:OI 0 "register_operand" "=w")
3819 (subreg:OI
3820 (vec_concat:<VRL2>
3821 (vec_concat:<VDBL>
3822 (unspec:DX [(match_operand:TI 1 "aarch64_simd_struct_operand" "Utv")]
3823 UNSPEC_LD2)
3824 (const_int 0))
3825 (vec_concat:<VDBL>
3826 (unspec:DX [(match_dup 1)]
3827 UNSPEC_LD2)
3828 (const_int 0))) 0))]
3829 "TARGET_SIMD"
3830 "ld1\\t{%S0.1d - %T0.1d}, %1"
3831 [(set_attr "simd_type" "simd_load2")
3832 (set_attr "simd_mode" "<MODE>")])
3834 (define_insn "aarch64_ld3<mode>_dreg"
3835 [(set (match_operand:CI 0 "register_operand" "=w")
3836 (subreg:CI
3837 (vec_concat:<VRL3>
3838 (vec_concat:<VRL2>
3839 (vec_concat:<VDBL>
3840 (unspec:VD [(match_operand:EI 1 "aarch64_simd_struct_operand" "Utv")]
3841 UNSPEC_LD3)
3842 (vec_duplicate:VD (const_int 0)))
3843 (vec_concat:<VDBL>
3844 (unspec:VD [(match_dup 1)]
3845 UNSPEC_LD3)
3846 (vec_duplicate:VD (const_int 0))))
3847 (vec_concat:<VDBL>
3848 (unspec:VD [(match_dup 1)]
3849 UNSPEC_LD3)
3850 (vec_duplicate:VD (const_int 0)))) 0))]
3851 "TARGET_SIMD"
3852 "ld3\\t{%S0.<Vtype> - %U0.<Vtype>}, %1"
3853 [(set_attr "simd_type" "simd_load3")
3854 (set_attr "simd_mode" "<MODE>")])
3856 (define_insn "aarch64_ld3<mode>_dreg"
3857 [(set (match_operand:CI 0 "register_operand" "=w")
3858 (subreg:CI
3859 (vec_concat:<VRL3>
3860 (vec_concat:<VRL2>
3861 (vec_concat:<VDBL>
3862 (unspec:DX [(match_operand:EI 1 "aarch64_simd_struct_operand" "Utv")]
3863 UNSPEC_LD3)
3864 (const_int 0))
3865 (vec_concat:<VDBL>
3866 (unspec:DX [(match_dup 1)]
3867 UNSPEC_LD3)
3868 (const_int 0)))
3869 (vec_concat:<VDBL>
3870 (unspec:DX [(match_dup 1)]
3871 UNSPEC_LD3)
3872 (const_int 0))) 0))]
3873 "TARGET_SIMD"
3874 "ld1\\t{%S0.1d - %U0.1d}, %1"
3875 [(set_attr "simd_type" "simd_load3")
3876 (set_attr "simd_mode" "<MODE>")])
3878 (define_insn "aarch64_ld4<mode>_dreg"
3879 [(set (match_operand:XI 0 "register_operand" "=w")
3880 (subreg:XI
3881 (vec_concat:<VRL4>
3882 (vec_concat:<VRL2>
3883 (vec_concat:<VDBL>
3884 (unspec:VD [(match_operand:OI 1 "aarch64_simd_struct_operand" "Utv")]
3885 UNSPEC_LD4)
3886 (vec_duplicate:VD (const_int 0)))
3887 (vec_concat:<VDBL>
3888 (unspec:VD [(match_dup 1)]
3889 UNSPEC_LD4)
3890 (vec_duplicate:VD (const_int 0))))
3891 (vec_concat:<VRL2>
3892 (vec_concat:<VDBL>
3893 (unspec:VD [(match_dup 1)]
3894 UNSPEC_LD4)
3895 (vec_duplicate:VD (const_int 0)))
3896 (vec_concat:<VDBL>
3897 (unspec:VD [(match_dup 1)]
3898 UNSPEC_LD4)
3899 (vec_duplicate:VD (const_int 0))))) 0))]
3900 "TARGET_SIMD"
3901 "ld4\\t{%S0.<Vtype> - %V0.<Vtype>}, %1"
3902 [(set_attr "simd_type" "simd_load4")
3903 (set_attr "simd_mode" "<MODE>")])
3905 (define_insn "aarch64_ld4<mode>_dreg"
3906 [(set (match_operand:XI 0 "register_operand" "=w")
3907 (subreg:XI
3908 (vec_concat:<VRL4>
3909 (vec_concat:<VRL2>
3910 (vec_concat:<VDBL>
3911 (unspec:DX [(match_operand:OI 1 "aarch64_simd_struct_operand" "Utv")]
3912 UNSPEC_LD4)
3913 (const_int 0))
3914 (vec_concat:<VDBL>
3915 (unspec:DX [(match_dup 1)]
3916 UNSPEC_LD4)
3917 (const_int 0)))
3918 (vec_concat:<VRL2>
3919 (vec_concat:<VDBL>
3920 (unspec:DX [(match_dup 1)]
3921 UNSPEC_LD4)
3922 (const_int 0))
3923 (vec_concat:<VDBL>
3924 (unspec:DX [(match_dup 1)]
3925 UNSPEC_LD4)
3926 (const_int 0)))) 0))]
3927 "TARGET_SIMD"
3928 "ld1\\t{%S0.1d - %V0.1d}, %1"
3929 [(set_attr "simd_type" "simd_load4")
3930 (set_attr "simd_mode" "<MODE>")])
3932 (define_expand "aarch64_ld<VSTRUCT:nregs><VDC:mode>"
3933 [(match_operand:VSTRUCT 0 "register_operand" "=w")
3934 (match_operand:DI 1 "register_operand" "r")
3935 (unspec:VDC [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3936 "TARGET_SIMD"
3937 {
3938 enum machine_mode mode = <VSTRUCT:VSTRUCT_DREG>mode;
3939 rtx mem = gen_rtx_MEM (mode, operands[1]);
3941 emit_insn (gen_aarch64_ld<VSTRUCT:nregs><VDC:mode>_dreg (operands[0], mem));
3942 DONE;
3943 })
3945 (define_expand "aarch64_ld1<VALL:mode>"
3946 [(match_operand:VALL 0 "register_operand")
3947 (match_operand:DI 1 "register_operand")]
3948 "TARGET_SIMD"
3949 {
3950 enum machine_mode mode = <VALL:MODE>mode;
3951 rtx mem = gen_rtx_MEM (mode, operands[1]);
3952 emit_move_insn (operands[0], mem);
3953 DONE;
3954 })
3956 (define_expand "aarch64_ld<VSTRUCT:nregs><VQ:mode>"
3957 [(match_operand:VSTRUCT 0 "register_operand" "=w")
3958 (match_operand:DI 1 "register_operand" "r")
3959 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
3960 "TARGET_SIMD"
3961 {
3962 enum machine_mode mode = <VSTRUCT:MODE>mode;
3963 rtx mem = gen_rtx_MEM (mode, operands[1]);
3965 emit_insn (gen_vec_load_lanes<VSTRUCT:mode><VQ:mode> (operands[0], mem));
3966 DONE;
3967 })
3969 ;; Expanders for builtins to extract vector registers from large
3970 ;; opaque integer modes.
3972 ;; D-register list.
3974 (define_expand "aarch64_get_dreg<VSTRUCT:mode><VDC:mode>"
3975 [(match_operand:VDC 0 "register_operand" "=w")
3976 (match_operand:VSTRUCT 1 "register_operand" "w")
3977 (match_operand:SI 2 "immediate_operand" "i")]
3978 "TARGET_SIMD"
3979 {
3980 int part = INTVAL (operands[2]);
3981 rtx temp = gen_reg_rtx (<VDC:VDBL>mode);
3982 int offset = part * 16;
3984 emit_move_insn (temp, gen_rtx_SUBREG (<VDC:VDBL>mode, operands[1], offset));
3985 emit_move_insn (operands[0], gen_lowpart (<VDC:MODE>mode, temp));
3986 DONE;
3987 })
3989 ;; Q-register list.
3991 (define_expand "aarch64_get_qreg<VSTRUCT:mode><VQ:mode>"
3992 [(match_operand:VQ 0 "register_operand" "=w")
3993 (match_operand:VSTRUCT 1 "register_operand" "w")
3994 (match_operand:SI 2 "immediate_operand" "i")]
3995 "TARGET_SIMD"
3996 {
3997 int part = INTVAL (operands[2]);
3998 int offset = part * 16;
4000 emit_move_insn (operands[0],
4001 gen_rtx_SUBREG (<VQ:MODE>mode, operands[1], offset));
4002 DONE;
4003 })
4005 ;; Permuted-store expanders for neon intrinsics.
4007 ;; Permute instructions
4009 ;; vec_perm support
4011 (define_expand "vec_perm_const<mode>"
4012 [(match_operand:VALL 0 "register_operand")
4013 (match_operand:VALL 1 "register_operand")
4014 (match_operand:VALL 2 "register_operand")
4015 (match_operand:<V_cmp_result> 3)]
4016 "TARGET_SIMD"
4017 {
4018 if (aarch64_expand_vec_perm_const (operands[0], operands[1],
4019 operands[2], operands[3]))
4020 DONE;
4021 else
4022 FAIL;
4023 })
4025 (define_expand "vec_perm<mode>"
4026 [(match_operand:VB 0 "register_operand")
4027 (match_operand:VB 1 "register_operand")
4028 (match_operand:VB 2 "register_operand")
4029 (match_operand:VB 3 "register_operand")]
4030 "TARGET_SIMD"
4031 {
4032 aarch64_expand_vec_perm (operands[0], operands[1],
4033 operands[2], operands[3]);
4034 DONE;
4035 })
4037 (define_insn "aarch64_tbl1<mode>"
4038 [(set (match_operand:VB 0 "register_operand" "=w")
4039 (unspec:VB [(match_operand:V16QI 1 "register_operand" "w")
4040 (match_operand:VB 2 "register_operand" "w")]
4041 UNSPEC_TBL))]
4042 "TARGET_SIMD"
4043 "tbl\\t%0.<Vtype>, {%1.16b}, %2.<Vtype>"
4044 [(set_attr "simd_type" "simd_tbl")
4045 (set_attr "simd_mode" "<MODE>")]
4046 )
4048 ;; Two source registers.
4050 (define_insn "aarch64_tbl2v16qi"
4051 [(set (match_operand:V16QI 0 "register_operand" "=w")
4052 (unspec:V16QI [(match_operand:OI 1 "register_operand" "w")
4053 (match_operand:V16QI 2 "register_operand" "w")]
4054 UNSPEC_TBL))]
4055 "TARGET_SIMD"
4056 "tbl\\t%0.16b, {%S1.16b - %T1.16b}, %2.16b"
4057 [(set_attr "simd_type" "simd_tbl")
4058 (set_attr "simd_mode" "V16QI")]
4059 )
4061 (define_insn_and_split "aarch64_combinev16qi"
4062 [(set (match_operand:OI 0 "register_operand" "=w")
4063 (unspec:OI [(match_operand:V16QI 1 "register_operand" "w")
4064 (match_operand:V16QI 2 "register_operand" "w")]
4065 UNSPEC_CONCAT))]
4066 "TARGET_SIMD"
4067 "#"
4068 "&& reload_completed"
4069 [(const_int 0)]
4070 {
4071 aarch64_split_combinev16qi (operands);
4072 DONE;
4073 })
4075 (define_insn "aarch64_<PERMUTE:perm_insn><PERMUTE:perm_hilo><mode>"
4076 [(set (match_operand:VALL 0 "register_operand" "=w")
4077 (unspec:VALL [(match_operand:VALL 1 "register_operand" "w")
4078 (match_operand:VALL 2 "register_operand" "w")]
4079 PERMUTE))]
4080 "TARGET_SIMD"
4081 "<PERMUTE:perm_insn><PERMUTE:perm_hilo>\\t%0.<Vtype>, %1.<Vtype>, %2.<Vtype>"
4082 [(set_attr "simd_type" "simd_<PERMUTE:perm_insn>")
4083 (set_attr "simd_mode" "<MODE>")]
4084 )
4086 (define_insn "aarch64_st2<mode>_dreg"
4087 [(set (match_operand:TI 0 "aarch64_simd_struct_operand" "=Utv")
4088 (unspec:TI [(match_operand:OI 1 "register_operand" "w")
4089 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4090 UNSPEC_ST2))]
4091 "TARGET_SIMD"
4092 "st2\\t{%S1.<Vtype> - %T1.<Vtype>}, %0"
4093 [(set_attr "simd_type" "simd_store2")
4094 (set_attr "simd_mode" "<MODE>")])
4096 (define_insn "aarch64_st2<mode>_dreg"
4097 [(set (match_operand:TI 0 "aarch64_simd_struct_operand" "=Utv")
4098 (unspec:TI [(match_operand:OI 1 "register_operand" "w")
4099 (unspec:DX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4100 UNSPEC_ST2))]
4101 "TARGET_SIMD"
4102 "st1\\t{%S1.1d - %T1.1d}, %0"
4103 [(set_attr "simd_type" "simd_store2")
4104 (set_attr "simd_mode" "<MODE>")])
4106 (define_insn "aarch64_st3<mode>_dreg"
4107 [(set (match_operand:EI 0 "aarch64_simd_struct_operand" "=Utv")
4108 (unspec:EI [(match_operand:CI 1 "register_operand" "w")
4109 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4110 UNSPEC_ST3))]
4111 "TARGET_SIMD"
4112 "st3\\t{%S1.<Vtype> - %U1.<Vtype>}, %0"
4113 [(set_attr "simd_type" "simd_store3")
4114 (set_attr "simd_mode" "<MODE>")])
4116 (define_insn "aarch64_st3<mode>_dreg"
4117 [(set (match_operand:EI 0 "aarch64_simd_struct_operand" "=Utv")
4118 (unspec:EI [(match_operand:CI 1 "register_operand" "w")
4119 (unspec:DX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4120 UNSPEC_ST3))]
4121 "TARGET_SIMD"
4122 "st1\\t{%S1.1d - %U1.1d}, %0"
4123 [(set_attr "simd_type" "simd_store3")
4124 (set_attr "simd_mode" "<MODE>")])
4126 (define_insn "aarch64_st4<mode>_dreg"
4127 [(set (match_operand:OI 0 "aarch64_simd_struct_operand" "=Utv")
4128 (unspec:OI [(match_operand:XI 1 "register_operand" "w")
4129 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4130 UNSPEC_ST4))]
4131 "TARGET_SIMD"
4132 "st4\\t{%S1.<Vtype> - %V1.<Vtype>}, %0"
4133 [(set_attr "simd_type" "simd_store4")
4134 (set_attr "simd_mode" "<MODE>")])
4136 (define_insn "aarch64_st4<mode>_dreg"
4137 [(set (match_operand:OI 0 "aarch64_simd_struct_operand" "=Utv")
4138 (unspec:OI [(match_operand:XI 1 "register_operand" "w")
4139 (unspec:DX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4140 UNSPEC_ST4))]
4141 "TARGET_SIMD"
4142 "st1\\t{%S1.1d - %V1.1d}, %0"
4143 [(set_attr "simd_type" "simd_store4")
4144 (set_attr "simd_mode" "<MODE>")])
4146 (define_expand "aarch64_st<VSTRUCT:nregs><VDC:mode>"
4147 [(match_operand:DI 0 "register_operand" "r")
4148 (match_operand:VSTRUCT 1 "register_operand" "w")
4149 (unspec:VDC [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4150 "TARGET_SIMD"
4151 {
4152 enum machine_mode mode = <VSTRUCT:VSTRUCT_DREG>mode;
4153 rtx mem = gen_rtx_MEM (mode, operands[0]);
4155 emit_insn (gen_aarch64_st<VSTRUCT:nregs><VDC:mode>_dreg (mem, operands[1]));
4156 DONE;
4157 })
4159 (define_expand "aarch64_st<VSTRUCT:nregs><VQ:mode>"
4160 [(match_operand:DI 0 "register_operand" "r")
4161 (match_operand:VSTRUCT 1 "register_operand" "w")
4162 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4163 "TARGET_SIMD"
4164 {
4165 enum machine_mode mode = <VSTRUCT:MODE>mode;
4166 rtx mem = gen_rtx_MEM (mode, operands[0]);
4168 emit_insn (gen_vec_store_lanes<VSTRUCT:mode><VQ:mode> (mem, operands[1]));
4169 DONE;
4170 })
4172 (define_expand "aarch64_st1<VALL:mode>"
4173 [(match_operand:DI 0 "register_operand")
4174 (match_operand:VALL 1 "register_operand")]
4175 "TARGET_SIMD"
4176 {
4177 enum machine_mode mode = <VALL:MODE>mode;
4178 rtx mem = gen_rtx_MEM (mode, operands[0]);
4179 emit_move_insn (mem, operands[1]);
4180 DONE;
4181 })
4183 ;; Expander for builtins to insert vector registers into large
4184 ;; opaque integer modes.
4186 ;; Q-register list. We don't need a D-reg inserter as we zero
4187 ;; extend them in arm_neon.h and insert the resulting Q-regs.
4189 (define_expand "aarch64_set_qreg<VSTRUCT:mode><VQ:mode>"
4190 [(match_operand:VSTRUCT 0 "register_operand" "+w")
4191 (match_operand:VSTRUCT 1 "register_operand" "0")
4192 (match_operand:VQ 2 "register_operand" "w")
4193 (match_operand:SI 3 "immediate_operand" "i")]
4194 "TARGET_SIMD"
4195 {
4196 int part = INTVAL (operands[3]);
4197 int offset = part * 16;
4199 emit_move_insn (operands[0], operands[1]);
4200 emit_move_insn (gen_rtx_SUBREG (<VQ:MODE>mode, operands[0], offset),
4201 operands[2]);
4202 DONE;
4203 })
4205 ;; Standard pattern name vec_init<mode>.
4207 (define_expand "vec_init<mode>"
4208 [(match_operand:VALL 0 "register_operand" "")
4209 (match_operand 1 "" "")]
4210 "TARGET_SIMD"
4211 {
4212 aarch64_expand_vector_init (operands[0], operands[1]);
4213 DONE;
4214 })
4216 (define_insn "*aarch64_simd_ld1r<mode>"
4217 [(set (match_operand:VALLDI 0 "register_operand" "=w")
4218 (vec_duplicate:VALLDI
4219 (match_operand:<VEL> 1 "aarch64_simd_struct_operand" "Utv")))]
4220 "TARGET_SIMD"
4221 "ld1r\\t{%0.<Vtype>}, %1"
4222 [(set_attr "simd_type" "simd_load1r")
4223 (set_attr "simd_mode" "<MODE>")])
4225 (define_insn "aarch64_frecpe<mode>"
4226 [(set (match_operand:VDQF 0 "register_operand" "=w")
4227 (unspec:VDQF [(match_operand:VDQF 1 "register_operand" "w")]
4228 UNSPEC_FRECPE))]
4229 "TARGET_SIMD"
4230 "frecpe\\t%0.<Vtype>, %1.<Vtype>"
4231 [(set_attr "simd_type" "simd_frecpe")
4232 (set_attr "simd_mode" "<MODE>")]
4233 )
4235 (define_insn "aarch64_frecp<FRECP:frecp_suffix><mode>"
4236 [(set (match_operand:GPF 0 "register_operand" "=w")
4237 (unspec:GPF [(match_operand:GPF 1 "register_operand" "w")]
4238 FRECP))]
4239 "TARGET_SIMD"
4240 "frecp<FRECP:frecp_suffix>\\t%<s>0, %<s>1"
4241 [(set_attr "simd_type" "simd_frecp<FRECP:frecp_suffix>")
4242 (set_attr "mode" "<MODE>")]
4243 )
4245 (define_insn "aarch64_frecps<mode>"
4246 [(set (match_operand:VALLF 0 "register_operand" "=w")
4247 (unspec:VALLF [(match_operand:VALLF 1 "register_operand" "w")
4248 (match_operand:VALLF 2 "register_operand" "w")]
4249 UNSPEC_FRECPS))]
4250 "TARGET_SIMD"
4251 "frecps\\t%<v>0<Vmtype>, %<v>1<Vmtype>, %<v>2<Vmtype>"
4252 [(set_attr "simd_type" "simd_frecps")
4253 (set_attr "simd_mode" "<MODE>")]
4254 )