1 ;; GCC machine description for SSE instructions
2 ;; Copyright (C) 2005-2015 Free Software Foundation, Inc.
4 ;; This file is part of GCC.
6 ;; GCC is free software; you can redistribute it and/or modify
7 ;; it under the terms of the GNU General Public License as published by
8 ;; the Free Software Foundation; either version 3, or (at your option)
11 ;; GCC is distributed in the hope that it will be useful,
12 ;; but WITHOUT ANY WARRANTY; without even the implied warranty of
13 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 ;; GNU General Public License for more details.
16 ;; You should have received a copy of the GNU General Public License
17 ;; along with GCC; see the file COPYING3. If not see
18 ;; <http://www.gnu.org/licenses/>.
20 (define_c_enum "unspec" [
55 UNSPEC_XOP_UNSIGNED_CMP
66 UNSPEC_AESKEYGENASSIST
87 ;; For AVX512F support
91 UNSPEC_UNSIGNED_FIX_NOTRUNC
106 UNSPEC_COMPRESS_STORE
111 ;; For embed. rounding feature
112 UNSPEC_EMBEDDED_ROUNDING
114 ;; For AVX512PF support
115 UNSPEC_GATHER_PREFETCH
116 UNSPEC_SCATTER_PREFETCH
118 ;; For AVX512ER support
132 ;; For AVX512BW support
140 ;; For AVX512DQ support
145 ;; For AVX512IFMA support
149 ;; For AVX512VBMI support
153 (define_c_enum "unspecv" [
163 ;; All vector modes including V?TImode, used in move patterns.
164 (define_mode_iterator VMOVE
165 [(V64QI "TARGET_AVX512F") (V32QI "TARGET_AVX") V16QI
166 (V32HI "TARGET_AVX512F") (V16HI "TARGET_AVX") V8HI
167 (V16SI "TARGET_AVX512F") (V8SI "TARGET_AVX") V4SI
168 (V8DI "TARGET_AVX512F") (V4DI "TARGET_AVX") V2DI
169 (V4TI "TARGET_AVX512BW") (V2TI "TARGET_AVX") V1TI
170 (V16SF "TARGET_AVX512F") (V8SF "TARGET_AVX") V4SF
171 (V8DF "TARGET_AVX512F") (V4DF "TARGET_AVX") V2DF])
173 ;; All AVX-512{F,VL} vector modes. Supposed TARGET_AVX512F baseline.
174 (define_mode_iterator V48_AVX512VL
175 [V16SI (V8SI "TARGET_AVX512VL") (V4SI "TARGET_AVX512VL")
176 V8DI (V4DI "TARGET_AVX512VL") (V2DI "TARGET_AVX512VL")
177 V16SF (V8SF "TARGET_AVX512VL") (V4SF "TARGET_AVX512VL")
178 V8DF (V4DF "TARGET_AVX512VL") (V2DF "TARGET_AVX512VL")])
180 ;; 1,2 byte AVX-512{BW,VL} vector modes. Supposed TARGET_AVX512BW baseline.
181 (define_mode_iterator VI12_AVX512VL
182 [V64QI (V16QI "TARGET_AVX512VL") (V32QI "TARGET_AVX512VL")
183 V32HI (V16HI "TARGET_AVX512VL") (V8HI "TARGET_AVX512VL")])
185 (define_mode_iterator VI1_AVX512VL
186 [V64QI (V16QI "TARGET_AVX512VL") (V32QI "TARGET_AVX512VL")])
189 (define_mode_iterator V
190 [(V32QI "TARGET_AVX") V16QI
191 (V16HI "TARGET_AVX") V8HI
192 (V16SI "TARGET_AVX512F") (V8SI "TARGET_AVX") V4SI
193 (V8DI "TARGET_AVX512F") (V4DI "TARGET_AVX") V2DI
194 (V16SF "TARGET_AVX512F") (V8SF "TARGET_AVX") V4SF
195 (V8DF "TARGET_AVX512F") (V4DF "TARGET_AVX") (V2DF "TARGET_SSE2")])
197 ;; All 128bit vector modes
198 (define_mode_iterator V_128
199 [V16QI V8HI V4SI V2DI V4SF (V2DF "TARGET_SSE2")])
201 ;; All 256bit vector modes
202 (define_mode_iterator V_256
203 [V32QI V16HI V8SI V4DI V8SF V4DF])
205 ;; All 512bit vector modes
206 (define_mode_iterator V_512 [V64QI V32HI V16SI V8DI V16SF V8DF])
208 ;; All 256bit and 512bit vector modes
209 (define_mode_iterator V_256_512
210 [V32QI V16HI V8SI V4DI V8SF V4DF
211 (V64QI "TARGET_AVX512F") (V32HI "TARGET_AVX512F") (V16SI "TARGET_AVX512F")
212 (V8DI "TARGET_AVX512F") (V16SF "TARGET_AVX512F") (V8DF "TARGET_AVX512F")])
214 ;; All vector float modes
215 (define_mode_iterator VF
216 [(V16SF "TARGET_AVX512F") (V8SF "TARGET_AVX") V4SF
217 (V8DF "TARGET_AVX512F") (V4DF "TARGET_AVX") (V2DF "TARGET_SSE2")])
219 ;; 128- and 256-bit float vector modes
220 (define_mode_iterator VF_128_256
221 [(V8SF "TARGET_AVX") V4SF
222 (V4DF "TARGET_AVX") (V2DF "TARGET_SSE2")])
224 ;; All SFmode vector float modes
225 (define_mode_iterator VF1
226 [(V16SF "TARGET_AVX512F") (V8SF "TARGET_AVX") V4SF])
228 ;; 128- and 256-bit SF vector modes
229 (define_mode_iterator VF1_128_256
230 [(V8SF "TARGET_AVX") V4SF])
232 (define_mode_iterator VF1_128_256VL
233 [V8SF (V4SF "TARGET_AVX512VL")])
235 ;; All DFmode vector float modes
236 (define_mode_iterator VF2
237 [(V8DF "TARGET_AVX512F") (V4DF "TARGET_AVX") V2DF])
239 ;; 128- and 256-bit DF vector modes
240 (define_mode_iterator VF2_128_256
241 [(V4DF "TARGET_AVX") V2DF])
243 (define_mode_iterator VF2_512_256
244 [(V8DF "TARGET_AVX512F") V4DF])
246 (define_mode_iterator VF2_512_256VL
247 [V8DF (V4DF "TARGET_AVX512VL")])
249 ;; All 128bit vector float modes
250 (define_mode_iterator VF_128
251 [V4SF (V2DF "TARGET_SSE2")])
253 ;; All 256bit vector float modes
254 (define_mode_iterator VF_256
257 ;; All 512bit vector float modes
258 (define_mode_iterator VF_512
261 (define_mode_iterator VI48_AVX512VL
262 [V16SI (V8SI "TARGET_AVX512VL") (V4SI "TARGET_AVX512VL")
263 V8DI (V4DI "TARGET_AVX512VL") (V2DI "TARGET_AVX512VL")])
265 (define_mode_iterator VF_AVX512VL
266 [V16SF (V8SF "TARGET_AVX512VL") (V4SF "TARGET_AVX512VL")
267 V8DF (V4DF "TARGET_AVX512VL") (V2DF "TARGET_AVX512VL")])
269 (define_mode_iterator VF2_AVX512VL
270 [V8DF (V4DF "TARGET_AVX512VL") (V2DF "TARGET_AVX512VL")])
272 (define_mode_iterator VF1_AVX512VL
273 [V16SF (V8SF "TARGET_AVX512VL") (V4SF "TARGET_AVX512VL")])
275 ;; All vector integer modes
276 (define_mode_iterator VI
277 [(V16SI "TARGET_AVX512F") (V8DI "TARGET_AVX512F")
278 (V64QI "TARGET_AVX512BW") (V32QI "TARGET_AVX") V16QI
279 (V32HI "TARGET_AVX512BW") (V16HI "TARGET_AVX") V8HI
280 (V8SI "TARGET_AVX") V4SI
281 (V4DI "TARGET_AVX") V2DI])
283 (define_mode_iterator VI_AVX2
284 [(V64QI "TARGET_AVX512BW") (V32QI "TARGET_AVX2") V16QI
285 (V32HI "TARGET_AVX512BW") (V16HI "TARGET_AVX2") V8HI
286 (V16SI "TARGET_AVX512F") (V8SI "TARGET_AVX2") V4SI
287 (V8DI "TARGET_AVX512F") (V4DI "TARGET_AVX2") V2DI])
289 ;; All QImode vector integer modes
290 (define_mode_iterator VI1
291 [(V32QI "TARGET_AVX") V16QI])
293 (define_mode_iterator VI_ULOADSTORE_BW_AVX512VL
295 V32HI (V8HI "TARGET_AVX512VL") (V16HI "TARGET_AVX512VL")])
297 (define_mode_iterator VI_ULOADSTORE_F_AVX512VL
298 [V16SI (V8SI "TARGET_AVX512VL") (V4SI "TARGET_AVX512VL")
299 V8DI (V4DI "TARGET_AVX512VL") (V2DI "TARGET_AVX512VL")])
301 ;; All DImode vector integer modes
302 (define_mode_iterator VI8
303 [(V8DI "TARGET_AVX512F") (V4DI "TARGET_AVX") V2DI])
305 (define_mode_iterator VI8_AVX512VL
306 [V8DI (V4DI "TARGET_AVX512VL") (V2DI "TARGET_AVX512VL")])
308 (define_mode_iterator VI8_256_512
309 [V8DI (V4DI "TARGET_AVX512VL")])
311 (define_mode_iterator VI1_AVX2
312 [(V32QI "TARGET_AVX2") V16QI])
314 (define_mode_iterator VI1_AVX512
315 [(V64QI "TARGET_AVX512BW") (V32QI "TARGET_AVX2") V16QI])
317 (define_mode_iterator VI2_AVX2
318 [(V32HI "TARGET_AVX512BW") (V16HI "TARGET_AVX2") V8HI])
320 (define_mode_iterator VI2_AVX512F
321 [(V32HI "TARGET_AVX512F") (V16HI "TARGET_AVX2") V8HI])
323 (define_mode_iterator VI4_AVX
324 [(V8SI "TARGET_AVX") V4SI])
326 (define_mode_iterator VI4_AVX2
327 [(V8SI "TARGET_AVX2") V4SI])
329 (define_mode_iterator VI4_AVX512F
330 [(V16SI "TARGET_AVX512F") (V8SI "TARGET_AVX2") V4SI])
332 (define_mode_iterator VI4_AVX512VL
333 [V16SI (V8SI "TARGET_AVX512VL") (V4SI "TARGET_AVX512VL")])
335 (define_mode_iterator VI48_AVX512F_AVX512VL
336 [V4SI V8SI (V16SI "TARGET_AVX512F")
337 (V2DI "TARGET_AVX512VL") (V4DI "TARGET_AVX512VL") (V8DI "TARGET_AVX512F")])
339 (define_mode_iterator VI2_AVX512VL
340 [(V8HI "TARGET_AVX512VL") (V16HI "TARGET_AVX512VL") V32HI])
342 (define_mode_iterator VI8_AVX2_AVX512BW
343 [(V8DI "TARGET_AVX512BW") (V4DI "TARGET_AVX2") V2DI])
345 (define_mode_iterator VI8_AVX2
346 [(V4DI "TARGET_AVX2") V2DI])
348 (define_mode_iterator VI8_AVX2_AVX512F
349 [(V8DI "TARGET_AVX512F") (V4DI "TARGET_AVX2") V2DI])
351 (define_mode_iterator VI4_128_8_256
355 (define_mode_iterator V8FI
359 (define_mode_iterator V16FI
362 ;; ??? We should probably use TImode instead.
363 (define_mode_iterator VIMAX_AVX2
364 [(V4TI "TARGET_AVX512BW") (V2TI "TARGET_AVX2") V1TI])
366 ;; ??? This should probably be dropped in favor of VIMAX_AVX2.
367 (define_mode_iterator SSESCALARMODE
368 [(V4TI "TARGET_AVX512BW") (V2TI "TARGET_AVX2") TI])
370 (define_mode_iterator VI12_AVX2
371 [(V64QI "TARGET_AVX512BW") (V32QI "TARGET_AVX2") V16QI
372 (V32HI "TARGET_AVX512BW") (V16HI "TARGET_AVX2") V8HI])
374 (define_mode_iterator VI24_AVX2
375 [(V16HI "TARGET_AVX2") V8HI
376 (V8SI "TARGET_AVX2") V4SI])
378 (define_mode_iterator VI124_AVX512F
379 [(V32QI "TARGET_AVX2") V16QI
380 (V32HI "TARGET_AVX512F") (V16HI "TARGET_AVX2") V8HI
381 (V16SI "TARGET_AVX512F") (V8SI "TARGET_AVX2") V4SI])
383 (define_mode_iterator VI124_AVX2
384 [(V32QI "TARGET_AVX2") V16QI
385 (V16HI "TARGET_AVX2") V8HI
386 (V8SI "TARGET_AVX2") V4SI])
388 (define_mode_iterator VI2_AVX2_AVX512BW
389 [(V32HI "TARGET_AVX512BW") (V16HI "TARGET_AVX2") V8HI])
391 (define_mode_iterator VI48_AVX2
392 [(V8SI "TARGET_AVX2") V4SI
393 (V4DI "TARGET_AVX2") V2DI])
395 (define_mode_iterator VI248_AVX2_8_AVX512F
396 [(V16HI "TARGET_AVX2") V8HI
397 (V8SI "TARGET_AVX2") V4SI
398 (V8DI "TARGET_AVX512F") (V4DI "TARGET_AVX2") V2DI])
400 (define_mode_iterator VI248_AVX512BW_AVX512VL
401 [(V32HI "TARGET_AVX512BW")
402 (V4DI "TARGET_AVX512VL") V16SI V8DI])
404 ;; Suppose TARGET_AVX512VL as baseline
405 (define_mode_iterator VI24_AVX512BW_1
406 [(V16HI "TARGET_AVX512BW") (V8HI "TARGET_AVX512BW")
409 (define_mode_iterator VI48_AVX512F
410 [(V16SI "TARGET_AVX512F") V8SI V4SI
411 (V8DI "TARGET_AVX512F") V4DI V2DI])
413 (define_mode_iterator V48_AVX2
416 (V4SI "TARGET_AVX2") (V2DI "TARGET_AVX2")
417 (V8SI "TARGET_AVX2") (V4DI "TARGET_AVX2")])
419 (define_mode_attr avx512
420 [(V16QI "avx512vl") (V32QI "avx512vl") (V64QI "avx512bw")
421 (V8HI "avx512vl") (V16HI "avx512vl") (V32HI "avx512bw")
422 (V4SI "avx512vl") (V8SI "avx512vl") (V16SI "avx512f")
423 (V2DI "avx512vl") (V4DI "avx512vl") (V8DI "avx512f")
424 (V4SF "avx512vl") (V8SF "avx512vl") (V16SF "avx512f")
425 (V2DF "avx512vl") (V4DF "avx512vl") (V8DF "avx512f")])
427 (define_mode_attr sse2_avx_avx512f
428 [(V16QI "sse2") (V32QI "avx") (V64QI "avx512f")
429 (V8HI "avx512vl") (V16HI "avx512vl") (V32HI "avx512bw")
430 (V4SI "sse2") (V8SI "avx") (V16SI "avx512f")
431 (V2DI "avx512vl") (V4DI "avx512vl") (V8DI "avx512f")
432 (V16SF "avx512f") (V8SF "avx") (V4SF "avx")
433 (V8DF "avx512f") (V4DF "avx") (V2DF "avx")])
435 (define_mode_attr sse2_avx2
436 [(V16QI "sse2") (V32QI "avx2") (V64QI "avx512bw")
437 (V8HI "sse2") (V16HI "avx2") (V32HI "avx512bw")
438 (V4SI "sse2") (V8SI "avx2") (V16SI "avx512f")
439 (V2DI "sse2") (V4DI "avx2") (V8DI "avx512f")
440 (V1TI "sse2") (V2TI "avx2") (V4TI "avx512bw")])
442 (define_mode_attr ssse3_avx2
443 [(V16QI "ssse3") (V32QI "avx2") (V64QI "avx512bw")
444 (V4HI "ssse3") (V8HI "ssse3") (V16HI "avx2") (V32HI "avx512bw")
445 (V4SI "ssse3") (V8SI "avx2")
446 (V2DI "ssse3") (V4DI "avx2")
447 (TI "ssse3") (V2TI "avx2") (V4TI "avx512bw")])
449 (define_mode_attr sse4_1_avx2
450 [(V16QI "sse4_1") (V32QI "avx2") (V64QI "avx512bw")
451 (V8HI "sse4_1") (V16HI "avx2") (V32HI "avx512bw")
452 (V4SI "sse4_1") (V8SI "avx2") (V16SI "avx512f")
453 (V2DI "sse4_1") (V4DI "avx2") (V8DI "avx512dq")])
455 (define_mode_attr avx_avx2
456 [(V4SF "avx") (V2DF "avx")
457 (V8SF "avx") (V4DF "avx")
458 (V4SI "avx2") (V2DI "avx2")
459 (V8SI "avx2") (V4DI "avx2")])
461 (define_mode_attr vec_avx2
462 [(V16QI "vec") (V32QI "avx2")
463 (V8HI "vec") (V16HI "avx2")
464 (V4SI "vec") (V8SI "avx2")
465 (V2DI "vec") (V4DI "avx2")])
467 (define_mode_attr avx2_avx512
468 [(V4SI "avx2") (V8SI "avx2") (V16SI "avx512f")
469 (V2DI "avx2") (V4DI "avx2") (V8DI "avx512f")
470 (V4SF "avx2") (V8SF "avx2") (V16SF "avx512f")
471 (V2DF "avx2") (V4DF "avx2") (V8DF "avx512f")
472 (V8HI "avx512vl") (V16HI "avx512vl") (V32HI "avx512bw")])
474 (define_mode_attr shuffletype
475 [(V16SF "f") (V16SI "i") (V8DF "f") (V8DI "i")
476 (V8SF "f") (V8SI "i") (V4DF "f") (V4DI "i")
477 (V4SF "f") (V4SI "i") (V2DF "f") (V2DI "i")
478 (V32QI "i") (V16HI "u") (V16QI "i") (V8HI "i")
479 (V64QI "i") (V1TI "i") (V2TI "i")])
481 (define_mode_attr ssequartermode
482 [(V16SF "V4SF") (V8DF "V2DF") (V16SI "V4SI") (V8DI "V2DI")])
484 (define_mode_attr ssedoublemodelower
485 [(V16QI "v16hi") (V32QI "v32hi") (V64QI "v64hi")
486 (V8HI "v8si") (V16HI "v16si") (V32HI "v32si")
487 (V4SI "v4di") (V8SI "v8di") (V16SI "v16di")])
489 (define_mode_attr ssedoublemode
490 [(V16SF "V32SF") (V16SI "V32SI") (V8DI "V16DI") (V8DF "V16DF")
491 (V8SF "V16SF") (V8SI "V16SI") (V4DI "V8DI") (V4DF "V8DF")
492 (V16HI "V16SI") (V8HI "V8SI") (V4HI "V4SI") (V4SI "V4DI")
493 (V32HI "V32SI") (V32QI "V32HI") (V16QI "V16HI") (V64QI "V64HI")])
495 (define_mode_attr ssebytemode
496 [(V8DI "V64QI") (V4DI "V32QI") (V2DI "V16QI")])
498 ;; All 128bit vector integer modes
499 (define_mode_iterator VI_128 [V16QI V8HI V4SI V2DI])
501 ;; All 256bit vector integer modes
502 (define_mode_iterator VI_256 [V32QI V16HI V8SI V4DI])
504 ;; All 512bit vector integer modes
505 (define_mode_iterator VI_512 [V64QI V32HI V16SI V8DI])
507 ;; Various 128bit vector integer mode combinations
508 (define_mode_iterator VI12_128 [V16QI V8HI])
509 (define_mode_iterator VI14_128 [V16QI V4SI])
510 (define_mode_iterator VI124_128 [V16QI V8HI V4SI])
511 (define_mode_iterator VI24_128 [V8HI V4SI])
512 (define_mode_iterator VI248_128 [V8HI V4SI V2DI])
513 (define_mode_iterator VI48_128 [V4SI V2DI])
515 ;; Various 256bit and 512 vector integer mode combinations
516 (define_mode_iterator VI124_256 [V32QI V16HI V8SI])
517 (define_mode_iterator VI124_256_AVX512F_AVX512BW
519 (V64QI "TARGET_AVX512BW")
520 (V32HI "TARGET_AVX512BW")
521 (V16SI "TARGET_AVX512F")])
522 (define_mode_iterator VI48_256 [V8SI V4DI])
523 (define_mode_iterator VI48_512 [V16SI V8DI])
524 (define_mode_iterator VI4_256_8_512 [V8SI V8DI])
525 (define_mode_iterator VI_AVX512BW
526 [V16SI V8DI (V32HI "TARGET_AVX512BW") (V64QI "TARGET_AVX512BW")])
528 ;; Int-float size matches
529 (define_mode_iterator VI4F_128 [V4SI V4SF])
530 (define_mode_iterator VI8F_128 [V2DI V2DF])
531 (define_mode_iterator VI4F_256 [V8SI V8SF])
532 (define_mode_iterator VI8F_256 [V4DI V4DF])
533 (define_mode_iterator VI8F_256_512
534 [V4DI V4DF (V8DI "TARGET_AVX512F") (V8DF "TARGET_AVX512F")])
535 (define_mode_iterator VI48F_256_512
537 (V16SI "TARGET_AVX512F") (V16SF "TARGET_AVX512F")
538 (V8DI "TARGET_AVX512F") (V8DF "TARGET_AVX512F")
539 (V4DI "TARGET_AVX512VL") (V4DF "TARGET_AVX512VL")])
540 (define_mode_iterator VF48_I1248
541 [V16SI V16SF V8DI V8DF V32HI V64QI])
542 (define_mode_iterator VI48F
543 [V16SI V16SF V8DI V8DF
544 (V8SI "TARGET_AVX512VL") (V8SF "TARGET_AVX512VL")
545 (V4DI "TARGET_AVX512VL") (V4DF "TARGET_AVX512VL")
546 (V4SI "TARGET_AVX512VL") (V4SF "TARGET_AVX512VL")
547 (V2DI "TARGET_AVX512VL") (V2DF "TARGET_AVX512VL")])
548 (define_mode_iterator VI48F_256 [V8SI V8SF V4DI V4DF])
550 ;; Mapping from float mode to required SSE level
551 (define_mode_attr sse
552 [(SF "sse") (DF "sse2")
553 (V4SF "sse") (V2DF "sse2")
554 (V16SF "avx512f") (V8SF "avx")
555 (V8DF "avx512f") (V4DF "avx")])
557 (define_mode_attr sse2
558 [(V16QI "sse2") (V32QI "avx") (V64QI "avx512f")
559 (V2DI "sse2") (V4DI "avx") (V8DI "avx512f")])
561 (define_mode_attr sse3
562 [(V16QI "sse3") (V32QI "avx")])
564 (define_mode_attr sse4_1
565 [(V4SF "sse4_1") (V2DF "sse4_1")
566 (V8SF "avx") (V4DF "avx")
569 (define_mode_attr avxsizesuffix
570 [(V64QI "512") (V32HI "512") (V16SI "512") (V8DI "512")
571 (V32QI "256") (V16HI "256") (V8SI "256") (V4DI "256")
572 (V16QI "") (V8HI "") (V4SI "") (V2DI "")
573 (V16SF "512") (V8DF "512")
574 (V8SF "256") (V4DF "256")
575 (V4SF "") (V2DF "")])
577 ;; SSE instruction mode
578 (define_mode_attr sseinsnmode
579 [(V64QI "XI") (V32HI "XI") (V16SI "XI") (V8DI "XI") (V4TI "XI")
580 (V32QI "OI") (V16HI "OI") (V8SI "OI") (V4DI "OI") (V2TI "OI")
581 (V16QI "TI") (V8HI "TI") (V4SI "TI") (V2DI "TI") (V1TI "TI")
582 (V16SF "V16SF") (V8DF "V8DF")
583 (V8SF "V8SF") (V4DF "V4DF")
584 (V4SF "V4SF") (V2DF "V2DF")
587 ;; Mapping of vector modes to corresponding mask size
588 (define_mode_attr avx512fmaskmode
589 [(V64QI "DI") (V32QI "SI") (V16QI "HI")
590 (V32HI "SI") (V16HI "HI") (V8HI "QI") (V4HI "QI")
591 (V16SI "HI") (V8SI "QI") (V4SI "QI")
592 (V8DI "QI") (V4DI "QI") (V2DI "QI")
593 (V16SF "HI") (V8SF "QI") (V4SF "QI")
594 (V8DF "QI") (V4DF "QI") (V2DF "QI")])
596 ;; Mapping of vector float modes to an integer mode of the same size
597 (define_mode_attr sseintvecmode
598 [(V16SF "V16SI") (V8DF "V8DI")
599 (V8SF "V8SI") (V4DF "V4DI")
600 (V4SF "V4SI") (V2DF "V2DI")
601 (V16SI "V16SI") (V8DI "V8DI")
602 (V8SI "V8SI") (V4DI "V4DI")
603 (V4SI "V4SI") (V2DI "V2DI")
604 (V16HI "V16HI") (V8HI "V8HI")
605 (V32HI "V32HI") (V64QI "V64QI")
606 (V32QI "V32QI") (V16QI "V16QI")])
608 (define_mode_attr sseintvecmode2
609 [(V8DF "XI") (V4DF "OI") (V2DF "TI")
610 (V8SF "OI") (V4SF "TI")])
612 (define_mode_attr sseintvecmodelower
613 [(V16SF "v16si") (V8DF "v8di")
614 (V8SF "v8si") (V4DF "v4di")
615 (V4SF "v4si") (V2DF "v2di")
616 (V8SI "v8si") (V4DI "v4di")
617 (V4SI "v4si") (V2DI "v2di")
618 (V16HI "v16hi") (V8HI "v8hi")
619 (V32QI "v32qi") (V16QI "v16qi")])
621 ;; Mapping of vector modes to a vector mode of double size
622 (define_mode_attr ssedoublevecmode
623 [(V32QI "V64QI") (V16HI "V32HI") (V8SI "V16SI") (V4DI "V8DI")
624 (V16QI "V32QI") (V8HI "V16HI") (V4SI "V8SI") (V2DI "V4DI")
625 (V8SF "V16SF") (V4DF "V8DF")
626 (V4SF "V8SF") (V2DF "V4DF")])
628 ;; Mapping of vector modes to a vector mode of half size
629 (define_mode_attr ssehalfvecmode
630 [(V64QI "V32QI") (V32HI "V16HI") (V16SI "V8SI") (V8DI "V4DI")
631 (V32QI "V16QI") (V16HI "V8HI") (V8SI "V4SI") (V4DI "V2DI")
632 (V16QI "V8QI") (V8HI "V4HI") (V4SI "V2SI")
633 (V16SF "V8SF") (V8DF "V4DF")
634 (V8SF "V4SF") (V4DF "V2DF")
637 ;; Mapping of vector modes ti packed single mode of the same size
638 (define_mode_attr ssePSmode
639 [(V16SI "V16SF") (V8DF "V16SF")
640 (V16SF "V16SF") (V8DI "V16SF")
641 (V64QI "V16SF") (V32QI "V8SF") (V16QI "V4SF")
642 (V32HI "V16SF") (V16HI "V8SF") (V8HI "V4SF")
643 (V8SI "V8SF") (V4SI "V4SF")
644 (V4DI "V8SF") (V2DI "V4SF")
645 (V4TI "V16SF") (V2TI "V8SF") (V1TI "V4SF")
646 (V8SF "V8SF") (V4SF "V4SF")
647 (V4DF "V8SF") (V2DF "V4SF")])
649 (define_mode_attr ssePSmode2
650 [(V8DI "V8SF") (V4DI "V4SF")])
652 ;; Mapping of vector modes back to the scalar modes
653 (define_mode_attr ssescalarmode
654 [(V64QI "QI") (V32QI "QI") (V16QI "QI")
655 (V32HI "HI") (V16HI "HI") (V8HI "HI")
656 (V16SI "SI") (V8SI "SI") (V4SI "SI")
657 (V8DI "DI") (V4DI "DI") (V2DI "DI")
658 (V16SF "SF") (V8SF "SF") (V4SF "SF")
659 (V8DF "DF") (V4DF "DF") (V2DF "DF")])
661 ;; Mapping of vector modes to the 128bit modes
662 (define_mode_attr ssexmmmode
663 [(V64QI "V16QI") (V32QI "V16QI") (V16QI "V16QI")
664 (V32HI "V8HI") (V16HI "V8HI") (V8HI "V8HI")
665 (V16SI "V4SI") (V8SI "V4SI") (V4SI "V4SI")
666 (V8DI "V2DI") (V4DI "V2DI") (V2DI "V2DI")
667 (V16SF "V4SF") (V8SF "V4SF") (V4SF "V4SF")
668 (V8DF "V2DF") (V4DF "V2DF") (V2DF "V2DF")])
670 ;; Pointer size override for scalar modes (Intel asm dialect)
671 (define_mode_attr iptr
672 [(V32QI "b") (V16HI "w") (V8SI "k") (V4DI "q")
673 (V16QI "b") (V8HI "w") (V4SI "k") (V2DI "q")
674 (V8SF "k") (V4DF "q")
675 (V4SF "k") (V2DF "q")
678 ;; Number of scalar elements in each vector type
679 (define_mode_attr ssescalarnum
680 [(V64QI "64") (V16SI "16") (V8DI "8")
681 (V32QI "32") (V16HI "16") (V8SI "8") (V4DI "4")
682 (V16QI "16") (V8HI "8") (V4SI "4") (V2DI "2")
683 (V16SF "16") (V8DF "8")
684 (V8SF "8") (V4DF "4")
685 (V4SF "4") (V2DF "2")])
687 ;; Mask of scalar elements in each vector type
688 (define_mode_attr ssescalarnummask
689 [(V32QI "31") (V16HI "15") (V8SI "7") (V4DI "3")
690 (V16QI "15") (V8HI "7") (V4SI "3") (V2DI "1")
691 (V8SF "7") (V4DF "3")
692 (V4SF "3") (V2DF "1")])
694 (define_mode_attr ssescalarsize
695 [(V8DI "64") (V4DI "64") (V2DI "64")
696 (V64QI "8") (V32QI "8") (V16QI "8")
697 (V32HI "16") (V16HI "16") (V8HI "16")
698 (V16SI "32") (V8SI "32") (V4SI "32")
699 (V16SF "32") (V8DF "64")])
701 ;; SSE prefix for integer vector modes
702 (define_mode_attr sseintprefix
703 [(V2DI "p") (V2DF "")
708 (V16SI "p") (V16SF "")
709 (V16QI "p") (V8HI "p")
710 (V32QI "p") (V16HI "p")
711 (V64QI "p") (V32HI "p")])
713 ;; SSE scalar suffix for vector modes
714 (define_mode_attr ssescalarmodesuffix
716 (V8SF "ss") (V4DF "sd")
717 (V4SF "ss") (V2DF "sd")
718 (V8SI "ss") (V4DI "sd")
721 ;; Pack/unpack vector modes
722 (define_mode_attr sseunpackmode
723 [(V16QI "V8HI") (V8HI "V4SI") (V4SI "V2DI")
724 (V32QI "V16HI") (V16HI "V8SI") (V8SI "V4DI")
725 (V32HI "V16SI") (V64QI "V32HI") (V16SI "V8DI")])
727 (define_mode_attr ssepackmode
728 [(V8HI "V16QI") (V4SI "V8HI") (V2DI "V4SI")
729 (V16HI "V32QI") (V8SI "V16HI") (V4DI "V8SI")
730 (V32HI "V64QI") (V16SI "V32HI") (V8DI "V16SI")])
732 ;; Mapping of the max integer size for xop rotate immediate constraint
733 (define_mode_attr sserotatemax
734 [(V16QI "7") (V8HI "15") (V4SI "31") (V2DI "63")])
736 ;; Mapping of mode to cast intrinsic name
737 (define_mode_attr castmode
738 [(V8SI "si") (V8SF "ps") (V4DF "pd")
739 (V16SI "si") (V16SF "ps") (V8DF "pd")])
741 ;; Instruction suffix for sign and zero extensions.
742 (define_code_attr extsuffix [(sign_extend "sx") (zero_extend "zx")])
744 ;; i128 for integer vectors and TARGET_AVX2, f128 otherwise.
745 ;; i64x4 or f64x4 for 512bit modes.
746 (define_mode_attr i128
747 [(V16SF "f64x4") (V8SF "f128") (V8DF "f64x4") (V4DF "f128")
748 (V64QI "i64x4") (V32QI "%~128") (V32HI "i64x4") (V16HI "%~128")
749 (V16SI "i64x4") (V8SI "%~128") (V8DI "i64x4") (V4DI "%~128")])
752 (define_mode_iterator AVX256MODE2P [V8SI V8SF V4DF])
753 (define_mode_iterator AVX512MODE2P [V16SI V16SF V8DF])
755 ;; Mapping for dbpsabbw modes
756 (define_mode_attr dbpsadbwmode
757 [(V32HI "V64QI") (V16HI "V32QI") (V8HI "V16QI")])
759 ;; Mapping suffixes for broadcast
760 (define_mode_attr bcstscalarsuff
761 [(V64QI "b") (V32QI "b") (V16QI "b")
762 (V32HI "w") (V16HI "w") (V8HI "w")
763 (V16SI "d") (V8SI "d") (V4SI "d")
764 (V8DI "q") (V4DI "q") (V2DI "q")
765 (V16SF "ss") (V8SF "ss") (V4SF "ss")
766 (V8DF "sd") (V4DF "sd") (V2DF "sd")])
768 ;; Tie mode of assembler operand to mode iterator
769 (define_mode_attr concat_tg_mode
770 [(V32QI "t") (V16HI "t") (V8SI "t") (V4DI "t") (V8SF "t") (V4DF "t")
771 (V64QI "g") (V32HI "g") (V16SI "g") (V8DI "g") (V16SF "g") (V8DF "g")])
774 ;; Include define_subst patterns for instructions with mask
777 ;; Patterns whose name begins with "sse{,2,3}_" are invoked by intrinsics.
779 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
783 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
785 ;; All of these patterns are enabled for SSE1 as well as SSE2.
786 ;; This is essential for maintaining stable calling conventions.
788 (define_expand "mov<mode>"
789 [(set (match_operand:VMOVE 0 "nonimmediate_operand")
790 (match_operand:VMOVE 1 "nonimmediate_operand"))]
793 ix86_expand_vector_move (<MODE>mode, operands);
797 (define_insn "*mov<mode>_internal"
798 [(set (match_operand:VMOVE 0 "nonimmediate_operand" "=v,v ,m")
799 (match_operand:VMOVE 1 "nonimmediate_or_sse_const_operand" "C ,vm,v"))]
801 && (register_operand (operands[0], <MODE>mode)
802 || register_operand (operands[1], <MODE>mode))"
804 int mode = get_attr_mode (insn);
805 switch (which_alternative)
808 return standard_sse_constant_opcode (insn, operands[1]);
811 /* There is no evex-encoded vmov* for sizes smaller than 64-bytes
812 in avx512f, so we need to use workarounds, to access sse registers
813 16-31, which are evex-only. In avx512vl we don't need workarounds. */
814 if (TARGET_AVX512F && <MODE_SIZE> < 64 && !TARGET_AVX512VL
815 && ((REG_P (operands[0]) && EXT_REX_SSE_REGNO_P (REGNO (operands[0])))
816 || (REG_P (operands[1]) && EXT_REX_SSE_REGNO_P (REGNO (operands[1])))))
818 if (memory_operand (operands[0], <MODE>mode))
820 if (<MODE_SIZE> == 32)
821 return "vextract<shuffletype>64x4\t{$0x0, %g1, %0|%0, %g1, 0x0}";
822 else if (<MODE_SIZE> == 16)
823 return "vextract<shuffletype>32x4\t{$0x0, %g1, %0|%0, %g1, 0x0}";
827 else if (memory_operand (operands[1], <MODE>mode))
829 if (<MODE_SIZE> == 32)
830 return "vbroadcast<shuffletype>64x4\t{%1, %g0|%g0, %1}";
831 else if (<MODE_SIZE> == 16)
832 return "vbroadcast<shuffletype>32x4\t{%1, %g0|%g0, %1}";
837 /* Reg -> reg move is always aligned. Just use wider move. */
842 return "vmovaps\t{%g1, %g0|%g0, %g1}";
845 return "vmovapd\t{%g1, %g0|%g0, %g1}";
848 return "vmovdqa64\t{%g1, %g0|%g0, %g1}";
859 && (misaligned_operand (operands[0], <MODE>mode)
860 || misaligned_operand (operands[1], <MODE>mode)))
861 return "vmovups\t{%1, %0|%0, %1}";
863 return "%vmovaps\t{%1, %0|%0, %1}";
869 && (misaligned_operand (operands[0], <MODE>mode)
870 || misaligned_operand (operands[1], <MODE>mode)))
871 return "vmovupd\t{%1, %0|%0, %1}";
873 return "%vmovapd\t{%1, %0|%0, %1}";
878 && (misaligned_operand (operands[0], <MODE>mode)
879 || misaligned_operand (operands[1], <MODE>mode)))
880 return TARGET_AVX512VL ? "vmovdqu64\t{%1, %0|%0, %1}"
881 : "vmovdqu\t{%1, %0|%0, %1}";
883 return TARGET_AVX512VL ? "vmovdqa64\t{%1, %0|%0, %1}"
884 : "%vmovdqa\t{%1, %0|%0, %1}";
886 if (misaligned_operand (operands[0], <MODE>mode)
887 || misaligned_operand (operands[1], <MODE>mode))
888 return "vmovdqu64\t{%1, %0|%0, %1}";
890 return "vmovdqa64\t{%1, %0|%0, %1}";
899 [(set_attr "type" "sselog1,ssemov,ssemov")
900 (set_attr "prefix" "maybe_vex")
902 (cond [(and (match_test "<MODE_SIZE> == 16")
903 (ior (match_test "TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL")
904 (and (eq_attr "alternative" "2")
905 (match_test "TARGET_SSE_TYPELESS_STORES"))))
906 (const_string "<ssePSmode>")
907 (match_test "TARGET_AVX")
908 (const_string "<sseinsnmode>")
909 (ior (not (match_test "TARGET_SSE2"))
910 (match_test "optimize_function_for_size_p (cfun)"))
911 (const_string "V4SF")
912 (and (eq_attr "alternative" "0")
913 (match_test "TARGET_SSE_LOAD0_BY_PXOR"))
916 (const_string "<sseinsnmode>")))])
918 (define_insn "<avx512>_load<mode>_mask"
919 [(set (match_operand:V48_AVX512VL 0 "register_operand" "=v,v")
920 (vec_merge:V48_AVX512VL
921 (match_operand:V48_AVX512VL 1 "nonimmediate_operand" "v,m")
922 (match_operand:V48_AVX512VL 2 "vector_move_operand" "0C,0C")
923 (match_operand:<avx512fmaskmode> 3 "register_operand" "Yk,Yk")))]
926 static char buf [64];
929 const char *sse_suffix;
931 if (FLOAT_MODE_P (GET_MODE_INNER (<MODE>mode)))
934 sse_suffix = "<ssemodesuffix>";
939 sse_suffix = "<ssescalarsize>";
942 if (misaligned_operand (operands[1], <MODE>mode))
947 snprintf (buf, sizeof (buf), "%s%s%s\t{%%1, %%0%%{%%3%%}%%N2|%%0%%{%%3%%}%%N2, %%1}",
948 insn_op, align, sse_suffix);
951 [(set_attr "type" "ssemov")
952 (set_attr "prefix" "evex")
953 (set_attr "memory" "none,load")
954 (set_attr "mode" "<sseinsnmode>")])
956 (define_insn "<avx512>_load<mode>_mask"
957 [(set (match_operand:VI12_AVX512VL 0 "register_operand" "=v,v")
958 (vec_merge:VI12_AVX512VL
959 (match_operand:VI12_AVX512VL 1 "nonimmediate_operand" "v,m")
960 (match_operand:VI12_AVX512VL 2 "vector_move_operand" "0C,0C")
961 (match_operand:<avx512fmaskmode> 3 "register_operand" "Yk,Yk")))]
963 "vmovdqu<ssescalarsize>\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"
964 [(set_attr "type" "ssemov")
965 (set_attr "prefix" "evex")
966 (set_attr "memory" "none,load")
967 (set_attr "mode" "<sseinsnmode>")])
969 (define_insn "<avx512>_blendm<mode>"
970 [(set (match_operand:V48_AVX512VL 0 "register_operand" "=v")
971 (vec_merge:V48_AVX512VL
972 (match_operand:V48_AVX512VL 2 "nonimmediate_operand" "vm")
973 (match_operand:V48_AVX512VL 1 "register_operand" "v")
974 (match_operand:<avx512fmaskmode> 3 "register_operand" "Yk")))]
976 "vblendm<ssemodesuffix>\t{%2, %1, %0%{%3%}|%0%{%3%}, %1, %2}"
977 [(set_attr "type" "ssemov")
978 (set_attr "prefix" "evex")
979 (set_attr "mode" "<sseinsnmode>")])
981 (define_insn "<avx512>_blendm<mode>"
982 [(set (match_operand:VI12_AVX512VL 0 "register_operand" "=v")
983 (vec_merge:VI12_AVX512VL
984 (match_operand:VI12_AVX512VL 2 "nonimmediate_operand" "vm")
985 (match_operand:VI12_AVX512VL 1 "register_operand" "v")
986 (match_operand:<avx512fmaskmode> 3 "register_operand" "Yk")))]
988 "vpblendm<ssemodesuffix>\t{%2, %1, %0%{%3%}|%0%{%3%}, %1, %2}"
989 [(set_attr "type" "ssemov")
990 (set_attr "prefix" "evex")
991 (set_attr "mode" "<sseinsnmode>")])
993 (define_insn "<avx512>_store<mode>_mask"
994 [(set (match_operand:V48_AVX512VL 0 "memory_operand" "=m")
995 (vec_merge:V48_AVX512VL
996 (match_operand:V48_AVX512VL 1 "register_operand" "v")
998 (match_operand:<avx512fmaskmode> 2 "register_operand" "Yk")))]
1001 static char buf [64];
1003 const char *insn_op;
1004 const char *sse_suffix;
1006 if (FLOAT_MODE_P (GET_MODE_INNER (<MODE>mode)))
1009 sse_suffix = "<ssemodesuffix>";
1014 sse_suffix = "<ssescalarsize>";
1017 if (misaligned_operand (operands[1], <MODE>mode))
1022 snprintf (buf, sizeof (buf), "%s%s%s\t{%%1, %%0%%{%%2%%}|%%0%%{%%2%%}, %%1}",
1023 insn_op, align, sse_suffix);
1026 [(set_attr "type" "ssemov")
1027 (set_attr "prefix" "evex")
1028 (set_attr "memory" "store")
1029 (set_attr "mode" "<sseinsnmode>")])
1031 (define_insn "<avx512>_store<mode>_mask"
1032 [(set (match_operand:VI12_AVX512VL 0 "memory_operand" "=m")
1033 (vec_merge:VI12_AVX512VL
1034 (match_operand:VI12_AVX512VL 1 "register_operand" "v")
1036 (match_operand:<avx512fmaskmode> 2 "register_operand" "Yk")))]
1038 "vmovdqu<ssescalarsize>\t{%1, %0%{%2%}|%0%{%2%}, %1}"
1039 [(set_attr "type" "ssemov")
1040 (set_attr "prefix" "evex")
1041 (set_attr "memory" "store")
1042 (set_attr "mode" "<sseinsnmode>")])
1044 (define_insn "sse2_movq128"
1045 [(set (match_operand:V2DI 0 "register_operand" "=x")
1048 (match_operand:V2DI 1 "nonimmediate_operand" "xm")
1049 (parallel [(const_int 0)]))
1052 "%vmovq\t{%1, %0|%0, %q1}"
1053 [(set_attr "type" "ssemov")
1054 (set_attr "prefix" "maybe_vex")
1055 (set_attr "mode" "TI")])
1057 ;; Move a DI from a 32-bit register pair (e.g. %edx:%eax) to an xmm.
1058 ;; We'd rather avoid this entirely; if the 32-bit reg pair was loaded
1059 ;; from memory, we'd prefer to load the memory directly into the %xmm
1060 ;; register. To facilitate this happy circumstance, this pattern won't
1061 ;; split until after register allocation. If the 64-bit value didn't
1062 ;; come from memory, this is the best we can do. This is much better
1063 ;; than storing %edx:%eax into a stack temporary and loading an %xmm
1066 (define_insn_and_split "movdi_to_sse"
1068 [(set (match_operand:V4SI 0 "register_operand" "=?x,x")
1069 (subreg:V4SI (match_operand:DI 1 "nonimmediate_operand" "r,m") 0))
1070 (clobber (match_scratch:V4SI 2 "=&x,X"))])]
1071 "!TARGET_64BIT && TARGET_SSE2 && TARGET_INTER_UNIT_MOVES_TO_VEC"
1073 "&& reload_completed"
1076 if (register_operand (operands[1], DImode))
1078 /* The DImode arrived in a pair of integral registers (e.g. %edx:%eax).
1079 Assemble the 64-bit DImode value in an xmm register. */
1080 emit_insn (gen_sse2_loadld (operands[0], CONST0_RTX (V4SImode),
1081 gen_rtx_SUBREG (SImode, operands[1], 0)));
1082 emit_insn (gen_sse2_loadld (operands[2], CONST0_RTX (V4SImode),
1083 gen_rtx_SUBREG (SImode, operands[1], 4)));
1084 emit_insn (gen_vec_interleave_lowv4si (operands[0], operands[0],
1087 else if (memory_operand (operands[1], DImode))
1089 rtx tmp = gen_reg_rtx (V2DImode);
1090 emit_insn (gen_vec_concatv2di (tmp, operands[1], const0_rtx));
1091 emit_move_insn (operands[0], gen_lowpart (V4SImode, tmp));
1098 [(set (match_operand:V4SF 0 "register_operand")
1099 (match_operand:V4SF 1 "zero_extended_scalar_load_operand"))]
1100 "TARGET_SSE && reload_completed"
1103 (vec_duplicate:V4SF (match_dup 1))
1107 operands[1] = simplify_gen_subreg (SFmode, operands[1], V4SFmode, 0);
1108 operands[2] = CONST0_RTX (V4SFmode);
1112 [(set (match_operand:V2DF 0 "register_operand")
1113 (match_operand:V2DF 1 "zero_extended_scalar_load_operand"))]
1114 "TARGET_SSE2 && reload_completed"
1115 [(set (match_dup 0) (vec_concat:V2DF (match_dup 1) (match_dup 2)))]
1117 operands[1] = simplify_gen_subreg (DFmode, operands[1], V2DFmode, 0);
1118 operands[2] = CONST0_RTX (DFmode);
1121 (define_expand "movmisalign<mode>"
1122 [(set (match_operand:VMOVE 0 "nonimmediate_operand")
1123 (match_operand:VMOVE 1 "nonimmediate_operand"))]
1126 ix86_expand_vector_move_misalign (<MODE>mode, operands);
1130 (define_expand "<sse>_loadu<ssemodesuffix><avxsizesuffix><mask_name>"
1131 [(set (match_operand:VF 0 "register_operand")
1132 (unspec:VF [(match_operand:VF 1 "nonimmediate_operand")]
1134 "TARGET_SSE && <mask_mode512bit_condition>"
1136 /* For AVX, normal *mov<mode>_internal pattern will handle unaligned loads
1137 just fine if misaligned_operand is true, and without the UNSPEC it can
1138 be combined with arithmetic instructions. If misaligned_operand is
1139 false, still emit UNSPEC_LOADU insn to honor user's request for
1142 && misaligned_operand (operands[1], <MODE>mode))
1144 rtx src = operands[1];
1146 src = gen_rtx_VEC_MERGE (<MODE>mode, operands[1],
1147 operands[2 * <mask_applied>],
1148 operands[3 * <mask_applied>]);
1149 emit_insn (gen_rtx_SET (VOIDmode, operands[0], src));
1154 (define_insn "*<sse>_loadu<ssemodesuffix><avxsizesuffix><mask_name>"
1155 [(set (match_operand:VF 0 "register_operand" "=v")
1157 [(match_operand:VF 1 "nonimmediate_operand" "vm")]
1159 "TARGET_SSE && <mask_mode512bit_condition>"
1161 switch (get_attr_mode (insn))
1166 return "%vmovups\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}";
1168 return "%vmovu<ssemodesuffix>\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}";
1171 [(set_attr "type" "ssemov")
1172 (set_attr "movu" "1")
1173 (set_attr "ssememalign" "8")
1174 (set_attr "prefix" "maybe_vex")
1176 (cond [(and (match_test "<MODE_SIZE> == 16")
1177 (match_test "TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL"))
1178 (const_string "<ssePSmode>")
1179 (match_test "TARGET_AVX")
1180 (const_string "<MODE>")
1181 (match_test "optimize_function_for_size_p (cfun)")
1182 (const_string "V4SF")
1184 (const_string "<MODE>")))])
1186 (define_insn "<sse>_storeu<ssemodesuffix><avxsizesuffix>"
1187 [(set (match_operand:VF 0 "memory_operand" "=m")
1189 [(match_operand:VF 1 "register_operand" "v")]
1193 switch (get_attr_mode (insn))
1198 return "%vmovups\t{%1, %0|%0, %1}";
1200 return "%vmovu<ssemodesuffix>\t{%1, %0|%0, %1}";
1203 [(set_attr "type" "ssemov")
1204 (set_attr "movu" "1")
1205 (set_attr "ssememalign" "8")
1206 (set_attr "prefix" "maybe_vex")
1208 (cond [(and (match_test "<MODE_SIZE> == 16")
1209 (ior (match_test "TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL")
1210 (match_test "TARGET_SSE_TYPELESS_STORES")))
1211 (const_string "<ssePSmode>")
1212 (match_test "TARGET_AVX")
1213 (const_string "<MODE>")
1214 (match_test "optimize_function_for_size_p (cfun)")
1215 (const_string "V4SF")
1217 (const_string "<MODE>")))])
1219 (define_insn "<avx512>_storeu<ssemodesuffix><avxsizesuffix>_mask"
1220 [(set (match_operand:VF_AVX512VL 0 "memory_operand" "=m")
1221 (vec_merge:VF_AVX512VL
1223 [(match_operand:VF_AVX512VL 1 "register_operand" "v")]
1226 (match_operand:<avx512fmaskmode> 2 "register_operand" "Yk")))]
1229 switch (get_attr_mode (insn))
1234 return "vmovups\t{%1, %0%{%2%}|%0%{%2%}, %1}";
1236 return "vmovu<ssemodesuffix>\t{%1, %0%{%2%}|%0%{%2%}, %1}";
1239 [(set_attr "type" "ssemov")
1240 (set_attr "movu" "1")
1241 (set_attr "memory" "store")
1242 (set_attr "prefix" "evex")
1243 (set_attr "mode" "<sseinsnmode>")])
1245 /* For AVX, normal *mov<mode>_internal pattern will handle unaligned loads
1246 just fine if misaligned_operand is true, and without the UNSPEC it can
1247 be combined with arithmetic instructions. If misaligned_operand is
1248 false, still emit UNSPEC_LOADU insn to honor user's request for
1250 (define_expand "<sse2_avx_avx512f>_loaddqu<mode><mask_name>"
1251 [(set (match_operand:VI1 0 "register_operand")
1253 [(match_operand:VI1 1 "nonimmediate_operand")]
1255 "TARGET_SSE2 && <mask_avx512vl_condition> && <mask_avx512bw_condition>"
1258 && misaligned_operand (operands[1], <MODE>mode))
1260 rtx src = operands[1];
1262 src = gen_rtx_VEC_MERGE (<MODE>mode, operands[1],
1263 operands[2 * <mask_applied>],
1264 operands[3 * <mask_applied>]);
1265 emit_insn (gen_rtx_SET (VOIDmode, operands[0], src));
1270 (define_expand "<sse2_avx_avx512f>_loaddqu<mode><mask_name>"
1271 [(set (match_operand:VI_ULOADSTORE_BW_AVX512VL 0 "register_operand")
1272 (unspec:VI_ULOADSTORE_BW_AVX512VL
1273 [(match_operand:VI_ULOADSTORE_BW_AVX512VL 1 "nonimmediate_operand")]
1277 if (misaligned_operand (operands[1], <MODE>mode))
1279 rtx src = operands[1];
1281 src = gen_rtx_VEC_MERGE (<MODE>mode, operands[1],
1282 operands[2 * <mask_applied>],
1283 operands[3 * <mask_applied>]);
1284 emit_insn (gen_rtx_SET (VOIDmode, operands[0], src));
1289 (define_expand "<sse2_avx_avx512f>_loaddqu<mode><mask_name>"
1290 [(set (match_operand:VI_ULOADSTORE_F_AVX512VL 0 "register_operand")
1291 (unspec:VI_ULOADSTORE_F_AVX512VL
1292 [(match_operand:VI_ULOADSTORE_F_AVX512VL 1 "nonimmediate_operand")]
1296 if (misaligned_operand (operands[1], <MODE>mode))
1298 rtx src = operands[1];
1300 src = gen_rtx_VEC_MERGE (<MODE>mode, operands[1],
1301 operands[2 * <mask_applied>],
1302 operands[3 * <mask_applied>]);
1303 emit_insn (gen_rtx_SET (VOIDmode, operands[0], src));
1308 (define_insn "*<sse2_avx_avx512f>_loaddqu<mode><mask_name>"
1309 [(set (match_operand:VI1 0 "register_operand" "=v")
1311 [(match_operand:VI1 1 "nonimmediate_operand" "vm")]
1313 "TARGET_SSE2 && <mask_avx512vl_condition> && <mask_avx512bw_condition>"
1315 switch (get_attr_mode (insn))
1319 return "%vmovups\t{%1, %0|%0, %1}";
1321 if (!(TARGET_AVX512VL && TARGET_AVX512BW))
1322 return "%vmovdqu\t{%1, %0|%0, %1}";
1324 return "vmovdqu<ssescalarsize>\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}";
1327 [(set_attr "type" "ssemov")
1328 (set_attr "movu" "1")
1329 (set_attr "ssememalign" "8")
1330 (set (attr "prefix_data16")
1332 (match_test "TARGET_AVX")
1334 (const_string "1")))
1335 (set_attr "prefix" "maybe_vex")
1337 (cond [(and (match_test "<MODE_SIZE> == 16")
1338 (match_test "TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL"))
1339 (const_string "<ssePSmode>")
1340 (match_test "TARGET_AVX")
1341 (const_string "<sseinsnmode>")
1342 (match_test "optimize_function_for_size_p (cfun)")
1343 (const_string "V4SF")
1345 (const_string "<sseinsnmode>")))])
1347 (define_insn "*<sse2_avx_avx512f>_loaddqu<mode><mask_name>"
1348 [(set (match_operand:VI_ULOADSTORE_BW_AVX512VL 0 "register_operand" "=v")
1349 (unspec:VI_ULOADSTORE_BW_AVX512VL
1350 [(match_operand:VI_ULOADSTORE_BW_AVX512VL 1 "nonimmediate_operand" "vm")]
1353 "vmovdqu<ssescalarsize>\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}";
1354 [(set_attr "type" "ssemov")
1355 (set_attr "movu" "1")
1356 (set_attr "ssememalign" "8")
1357 (set_attr "prefix" "maybe_evex")])
1359 (define_insn "*<sse2_avx_avx512f>_loaddqu<mode><mask_name>"
1360 [(set (match_operand:VI_ULOADSTORE_F_AVX512VL 0 "register_operand" "=v")
1361 (unspec:VI_ULOADSTORE_F_AVX512VL
1362 [(match_operand:VI_ULOADSTORE_F_AVX512VL 1 "nonimmediate_operand" "vm")]
1365 "vmovdqu<ssescalarsize>\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}";
1366 [(set_attr "type" "ssemov")
1367 (set_attr "movu" "1")
1368 (set_attr "ssememalign" "8")
1369 (set_attr "prefix" "maybe_evex")])
1371 (define_insn "<sse2_avx_avx512f>_storedqu<mode>"
1372 [(set (match_operand:VI1 0 "memory_operand" "=m")
1374 [(match_operand:VI1 1 "register_operand" "v")]
1378 switch (get_attr_mode (insn))
1383 return "%vmovups\t{%1, %0|%0, %1}";
1389 if (!(TARGET_AVX512VL && TARGET_AVX512BW))
1390 return "%vmovdqu\t{%1, %0|%0, %1}";
1392 return "vmovdqu<ssescalarsize>\t{%1, %0|%0, %1}";
1396 [(set_attr "type" "ssemov")
1397 (set_attr "movu" "1")
1398 (set_attr "ssememalign" "8")
1399 (set (attr "prefix_data16")
1401 (match_test "TARGET_AVX")
1403 (const_string "1")))
1404 (set_attr "prefix" "maybe_vex")
1406 (cond [(and (match_test "<MODE_SIZE> == 16")
1407 (ior (match_test "TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL")
1408 (match_test "TARGET_SSE_TYPELESS_STORES")))
1409 (const_string "<ssePSmode>")
1410 (match_test "TARGET_AVX")
1411 (const_string "<sseinsnmode>")
1412 (match_test "optimize_function_for_size_p (cfun)")
1413 (const_string "V4SF")
1415 (const_string "<sseinsnmode>")))])
1417 (define_insn "<sse2_avx_avx512f>_storedqu<mode>"
1418 [(set (match_operand:VI_ULOADSTORE_BW_AVX512VL 0 "memory_operand" "=m")
1419 (unspec:VI_ULOADSTORE_BW_AVX512VL
1420 [(match_operand:VI_ULOADSTORE_BW_AVX512VL 1 "register_operand" "v")]
1423 "vmovdqu<ssescalarsize>\t{%1, %0|%0, %1}"
1424 [(set_attr "type" "ssemov")
1425 (set_attr "movu" "1")
1426 (set_attr "ssememalign" "8")
1427 (set_attr "prefix" "maybe_evex")])
1429 (define_insn "<sse2_avx_avx512f>_storedqu<mode>"
1430 [(set (match_operand:VI_ULOADSTORE_F_AVX512VL 0 "memory_operand" "=m")
1431 (unspec:VI_ULOADSTORE_F_AVX512VL
1432 [(match_operand:VI_ULOADSTORE_F_AVX512VL 1 "register_operand" "v")]
1435 "vmovdqu<ssescalarsize>\t{%1, %0|%0, %1}"
1436 [(set_attr "type" "ssemov")
1437 (set_attr "movu" "1")
1438 (set_attr "ssememalign" "8")
1439 (set_attr "prefix" "maybe_vex")])
1441 (define_insn "<avx512>_storedqu<mode>_mask"
1442 [(set (match_operand:VI48_AVX512VL 0 "memory_operand" "=m")
1443 (vec_merge:VI48_AVX512VL
1444 (unspec:VI48_AVX512VL
1445 [(match_operand:VI48_AVX512VL 1 "register_operand" "v")]
1448 (match_operand:<avx512fmaskmode> 2 "register_operand" "Yk")))]
1450 "vmovdqu<ssescalarsize>\t{%1, %0%{%2%}|%0%{%2%}, %1}"
1451 [(set_attr "type" "ssemov")
1452 (set_attr "movu" "1")
1453 (set_attr "memory" "store")
1454 (set_attr "prefix" "evex")
1455 (set_attr "mode" "<sseinsnmode>")])
1457 (define_insn "<avx512>_storedqu<mode>_mask"
1458 [(set (match_operand:VI12_AVX512VL 0 "memory_operand" "=m")
1459 (vec_merge:VI12_AVX512VL
1460 (unspec:VI12_AVX512VL
1461 [(match_operand:VI12_AVX512VL 1 "register_operand" "v")]
1464 (match_operand:<avx512fmaskmode> 2 "register_operand" "Yk")))]
1466 "vmovdqu<ssescalarsize>\t{%1, %0%{%2%}|%0%{%2%}, %1}"
1467 [(set_attr "type" "ssemov")
1468 (set_attr "movu" "1")
1469 (set_attr "memory" "store")
1470 (set_attr "prefix" "evex")
1471 (set_attr "mode" "<sseinsnmode>")])
1473 (define_insn "<sse3>_lddqu<avxsizesuffix>"
1474 [(set (match_operand:VI1 0 "register_operand" "=x")
1475 (unspec:VI1 [(match_operand:VI1 1 "memory_operand" "m")]
1478 "%vlddqu\t{%1, %0|%0, %1}"
1479 [(set_attr "type" "ssemov")
1480 (set_attr "movu" "1")
1481 (set_attr "ssememalign" "8")
1482 (set (attr "prefix_data16")
1484 (match_test "TARGET_AVX")
1486 (const_string "0")))
1487 (set (attr "prefix_rep")
1489 (match_test "TARGET_AVX")
1491 (const_string "1")))
1492 (set_attr "prefix" "maybe_vex")
1493 (set_attr "mode" "<sseinsnmode>")])
1495 (define_insn "sse2_movnti<mode>"
1496 [(set (match_operand:SWI48 0 "memory_operand" "=m")
1497 (unspec:SWI48 [(match_operand:SWI48 1 "register_operand" "r")]
1500 "movnti\t{%1, %0|%0, %1}"
1501 [(set_attr "type" "ssemov")
1502 (set_attr "prefix_data16" "0")
1503 (set_attr "mode" "<MODE>")])
1505 (define_insn "<sse>_movnt<mode>"
1506 [(set (match_operand:VF 0 "memory_operand" "=m")
1508 [(match_operand:VF 1 "register_operand" "v")]
1511 "%vmovnt<ssemodesuffix>\t{%1, %0|%0, %1}"
1512 [(set_attr "type" "ssemov")
1513 (set_attr "prefix" "maybe_vex")
1514 (set_attr "mode" "<MODE>")])
1516 (define_insn "<sse2>_movnt<mode>"
1517 [(set (match_operand:VI8 0 "memory_operand" "=m")
1518 (unspec:VI8 [(match_operand:VI8 1 "register_operand" "v")]
1521 "%vmovntdq\t{%1, %0|%0, %1}"
1522 [(set_attr "type" "ssecvt")
1523 (set (attr "prefix_data16")
1525 (match_test "TARGET_AVX")
1527 (const_string "1")))
1528 (set_attr "prefix" "maybe_vex")
1529 (set_attr "mode" "<sseinsnmode>")])
1531 ; Expand patterns for non-temporal stores. At the moment, only those
1532 ; that directly map to insns are defined; it would be possible to
1533 ; define patterns for other modes that would expand to several insns.
1535 ;; Modes handled by storent patterns.
1536 (define_mode_iterator STORENT_MODE
1537 [(DI "TARGET_SSE2 && TARGET_64BIT") (SI "TARGET_SSE2")
1538 (SF "TARGET_SSE4A") (DF "TARGET_SSE4A")
1539 (V8DI "TARGET_AVX512F") (V4DI "TARGET_AVX") (V2DI "TARGET_SSE2")
1540 (V16SF "TARGET_AVX512F") (V8SF "TARGET_AVX") V4SF
1541 (V8DF "TARGET_AVX512F") (V4DF "TARGET_AVX") (V2DF "TARGET_SSE2")])
1543 (define_expand "storent<mode>"
1544 [(set (match_operand:STORENT_MODE 0 "memory_operand")
1545 (unspec:STORENT_MODE
1546 [(match_operand:STORENT_MODE 1 "register_operand")]
1550 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
1552 ;; Parallel floating point arithmetic
1554 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
1556 (define_expand "<code><mode>2"
1557 [(set (match_operand:VF 0 "register_operand")
1559 (match_operand:VF 1 "register_operand")))]
1561 "ix86_expand_fp_absneg_operator (<CODE>, <MODE>mode, operands); DONE;")
1563 (define_insn_and_split "*absneg<mode>2"
1564 [(set (match_operand:VF 0 "register_operand" "=x,x,v,v")
1565 (match_operator:VF 3 "absneg_operator"
1566 [(match_operand:VF 1 "nonimmediate_operand" "0, xm, v, m")]))
1567 (use (match_operand:VF 2 "nonimmediate_operand" "xm, 0, vm,v"))]
1570 "&& reload_completed"
1573 enum rtx_code absneg_op;
1579 if (MEM_P (operands[1]))
1580 op1 = operands[2], op2 = operands[1];
1582 op1 = operands[1], op2 = operands[2];
1587 if (rtx_equal_p (operands[0], operands[1]))
1593 absneg_op = GET_CODE (operands[3]) == NEG ? XOR : AND;
1594 t = gen_rtx_fmt_ee (absneg_op, <MODE>mode, op1, op2);
1595 t = gen_rtx_SET (VOIDmode, operands[0], t);
1599 [(set_attr "isa" "noavx,noavx,avx,avx")])
1601 (define_expand "<plusminus_insn><mode>3<mask_name><round_name>"
1602 [(set (match_operand:VF 0 "register_operand")
1604 (match_operand:VF 1 "<round_nimm_predicate>")
1605 (match_operand:VF 2 "<round_nimm_predicate>")))]
1606 "TARGET_SSE && <mask_mode512bit_condition> && <round_mode512bit_condition>"
1607 "ix86_fixup_binary_operands_no_copy (<CODE>, <MODE>mode, operands);")
1609 (define_insn "*<plusminus_insn><mode>3<mask_name><round_name>"
1610 [(set (match_operand:VF 0 "register_operand" "=x,v")
1612 (match_operand:VF 1 "<round_nimm_predicate>" "<comm>0,v")
1613 (match_operand:VF 2 "<round_nimm_predicate>" "xm,<round_constraint>")))]
1614 "TARGET_SSE && ix86_binary_operator_ok (<CODE>, <MODE>mode, operands) && <mask_mode512bit_condition> && <round_mode512bit_condition>"
1616 <plusminus_mnemonic><ssemodesuffix>\t{%2, %0|%0, %2}
1617 v<plusminus_mnemonic><ssemodesuffix>\t{<round_mask_op3>%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2<round_mask_op3>}"
1618 [(set_attr "isa" "noavx,avx")
1619 (set_attr "type" "sseadd")
1620 (set_attr "prefix" "<mask_prefix3>")
1621 (set_attr "mode" "<MODE>")])
1623 (define_insn "<sse>_vm<plusminus_insn><mode>3<round_name>"
1624 [(set (match_operand:VF_128 0 "register_operand" "=x,v")
1627 (match_operand:VF_128 1 "register_operand" "0,v")
1628 (match_operand:VF_128 2 "nonimmediate_operand" "xm,<round_constraint>"))
1633 <plusminus_mnemonic><ssescalarmodesuffix>\t{%2, %0|%0, %<iptr>2}
1634 v<plusminus_mnemonic><ssescalarmodesuffix>\t{<round_op3>%2, %1, %0|%0, %1, %<iptr>2<round_op3>}"
1635 [(set_attr "isa" "noavx,avx")
1636 (set_attr "type" "sseadd")
1637 (set_attr "prefix" "<round_prefix>")
1638 (set_attr "mode" "<ssescalarmode>")])
1640 (define_expand "mul<mode>3<mask_name><round_name>"
1641 [(set (match_operand:VF 0 "register_operand")
1643 (match_operand:VF 1 "<round_nimm_predicate>")
1644 (match_operand:VF 2 "<round_nimm_predicate>")))]
1645 "TARGET_SSE && <mask_mode512bit_condition> && <round_mode512bit_condition>"
1646 "ix86_fixup_binary_operands_no_copy (MULT, <MODE>mode, operands);")
1648 (define_insn "*mul<mode>3<mask_name><round_name>"
1649 [(set (match_operand:VF 0 "register_operand" "=x,v")
1651 (match_operand:VF 1 "<round_nimm_predicate>" "%0,v")
1652 (match_operand:VF 2 "<round_nimm_predicate>" "xm,<round_constraint>")))]
1653 "TARGET_SSE && ix86_binary_operator_ok (MULT, <MODE>mode, operands) && <mask_mode512bit_condition> && <round_mode512bit_condition>"
1655 mul<ssemodesuffix>\t{%2, %0|%0, %2}
1656 vmul<ssemodesuffix>\t{<round_mask_op3>%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2<round_mask_op3>}"
1657 [(set_attr "isa" "noavx,avx")
1658 (set_attr "type" "ssemul")
1659 (set_attr "prefix" "<mask_prefix3>")
1660 (set_attr "btver2_decode" "direct,double")
1661 (set_attr "mode" "<MODE>")])
1663 (define_insn "<sse>_vm<multdiv_mnemonic><mode>3<round_name>"
1664 [(set (match_operand:VF_128 0 "register_operand" "=x,v")
1667 (match_operand:VF_128 1 "register_operand" "0,v")
1668 (match_operand:VF_128 2 "nonimmediate_operand" "xm,<round_constraint>"))
1673 <multdiv_mnemonic><ssescalarmodesuffix>\t{%2, %0|%0, %<iptr>2}
1674 v<multdiv_mnemonic><ssescalarmodesuffix>\t{<round_op3>%2, %1, %0|%0, %1, %<iptr>2<round_op3>}"
1675 [(set_attr "isa" "noavx,avx")
1676 (set_attr "type" "sse<multdiv_mnemonic>")
1677 (set_attr "prefix" "<round_prefix>")
1678 (set_attr "btver2_decode" "direct,double")
1679 (set_attr "mode" "<ssescalarmode>")])
1681 (define_expand "div<mode>3"
1682 [(set (match_operand:VF2 0 "register_operand")
1683 (div:VF2 (match_operand:VF2 1 "register_operand")
1684 (match_operand:VF2 2 "nonimmediate_operand")))]
1686 "ix86_fixup_binary_operands_no_copy (DIV, <MODE>mode, operands);")
1688 (define_expand "div<mode>3"
1689 [(set (match_operand:VF1 0 "register_operand")
1690 (div:VF1 (match_operand:VF1 1 "register_operand")
1691 (match_operand:VF1 2 "nonimmediate_operand")))]
1694 ix86_fixup_binary_operands_no_copy (DIV, <MODE>mode, operands);
1697 && TARGET_RECIP_VEC_DIV
1698 && !optimize_insn_for_size_p ()
1699 && flag_finite_math_only && !flag_trapping_math
1700 && flag_unsafe_math_optimizations)
1702 ix86_emit_swdivsf (operands[0], operands[1], operands[2], <MODE>mode);
1707 (define_insn "<sse>_div<mode>3<mask_name><round_name>"
1708 [(set (match_operand:VF 0 "register_operand" "=x,v")
1710 (match_operand:VF 1 "register_operand" "0,v")
1711 (match_operand:VF 2 "<round_nimm_predicate>" "xm,<round_constraint>")))]
1712 "TARGET_SSE && <mask_mode512bit_condition> && <round_mode512bit_condition>"
1714 div<ssemodesuffix>\t{%2, %0|%0, %2}
1715 vdiv<ssemodesuffix>\t{<round_mask_op3>%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2<round_mask_op3>}"
1716 [(set_attr "isa" "noavx,avx")
1717 (set_attr "type" "ssediv")
1718 (set_attr "prefix" "<mask_prefix3>")
1719 (set_attr "mode" "<MODE>")])
1721 (define_insn "<sse>_rcp<mode>2"
1722 [(set (match_operand:VF1_128_256 0 "register_operand" "=x")
1724 [(match_operand:VF1_128_256 1 "nonimmediate_operand" "xm")] UNSPEC_RCP))]
1726 "%vrcpps\t{%1, %0|%0, %1}"
1727 [(set_attr "type" "sse")
1728 (set_attr "atom_sse_attr" "rcp")
1729 (set_attr "btver2_sse_attr" "rcp")
1730 (set_attr "prefix" "maybe_vex")
1731 (set_attr "mode" "<MODE>")])
1733 (define_insn "sse_vmrcpv4sf2"
1734 [(set (match_operand:V4SF 0 "register_operand" "=x,x")
1736 (unspec:V4SF [(match_operand:V4SF 1 "nonimmediate_operand" "xm,xm")]
1738 (match_operand:V4SF 2 "register_operand" "0,x")
1742 rcpss\t{%1, %0|%0, %k1}
1743 vrcpss\t{%1, %2, %0|%0, %2, %k1}"
1744 [(set_attr "isa" "noavx,avx")
1745 (set_attr "type" "sse")
1746 (set_attr "ssememalign" "32")
1747 (set_attr "atom_sse_attr" "rcp")
1748 (set_attr "btver2_sse_attr" "rcp")
1749 (set_attr "prefix" "orig,vex")
1750 (set_attr "mode" "SF")])
1752 (define_insn "<mask_codefor>rcp14<mode><mask_name>"
1753 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v")
1755 [(match_operand:VF_AVX512VL 1 "nonimmediate_operand" "vm")]
1758 "vrcp14<ssemodesuffix>\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
1759 [(set_attr "type" "sse")
1760 (set_attr "prefix" "evex")
1761 (set_attr "mode" "<MODE>")])
1763 (define_insn "srcp14<mode>"
1764 [(set (match_operand:VF_128 0 "register_operand" "=v")
1767 [(match_operand:VF_128 1 "nonimmediate_operand" "vm")]
1769 (match_operand:VF_128 2 "register_operand" "v")
1772 "vrcp14<ssescalarmodesuffix>\t{%1, %2, %0|%0, %2, %1}"
1773 [(set_attr "type" "sse")
1774 (set_attr "prefix" "evex")
1775 (set_attr "mode" "<MODE>")])
1777 (define_expand "sqrt<mode>2"
1778 [(set (match_operand:VF2 0 "register_operand")
1779 (sqrt:VF2 (match_operand:VF2 1 "nonimmediate_operand")))]
1782 (define_expand "sqrt<mode>2"
1783 [(set (match_operand:VF1 0 "register_operand")
1784 (sqrt:VF1 (match_operand:VF1 1 "nonimmediate_operand")))]
1788 && TARGET_RECIP_VEC_SQRT
1789 && !optimize_insn_for_size_p ()
1790 && flag_finite_math_only && !flag_trapping_math
1791 && flag_unsafe_math_optimizations)
1793 ix86_emit_swsqrtsf (operands[0], operands[1], <MODE>mode, false);
1798 (define_insn "<sse>_sqrt<mode>2<mask_name><round_name>"
1799 [(set (match_operand:VF 0 "register_operand" "=v")
1800 (sqrt:VF (match_operand:VF 1 "<round_nimm_predicate>" "<round_constraint>")))]
1801 "TARGET_SSE && <mask_mode512bit_condition> && <round_mode512bit_condition>"
1802 "%vsqrt<ssemodesuffix>\t{<round_mask_op2>%1, %0<mask_operand2>|%0<mask_operand2>, %1<round_mask_op2>}"
1803 [(set_attr "type" "sse")
1804 (set_attr "atom_sse_attr" "sqrt")
1805 (set_attr "btver2_sse_attr" "sqrt")
1806 (set_attr "prefix" "maybe_vex")
1807 (set_attr "mode" "<MODE>")])
1809 (define_insn "<sse>_vmsqrt<mode>2<round_name>"
1810 [(set (match_operand:VF_128 0 "register_operand" "=x,v")
1813 (match_operand:VF_128 1 "nonimmediate_operand" "xm,<round_constraint>"))
1814 (match_operand:VF_128 2 "register_operand" "0,v")
1818 sqrt<ssescalarmodesuffix>\t{%1, %0|%0, %<iptr>1}
1819 vsqrt<ssescalarmodesuffix>\t{<round_op3>%1, %2, %0|%0, %2, %<iptr>1<round_op3>}"
1820 [(set_attr "isa" "noavx,avx")
1821 (set_attr "type" "sse")
1822 (set_attr "atom_sse_attr" "sqrt")
1823 (set_attr "prefix" "<round_prefix>")
1824 (set_attr "btver2_sse_attr" "sqrt")
1825 (set_attr "mode" "<ssescalarmode>")])
1827 (define_expand "rsqrt<mode>2"
1828 [(set (match_operand:VF1_128_256 0 "register_operand")
1830 [(match_operand:VF1_128_256 1 "nonimmediate_operand")] UNSPEC_RSQRT))]
1833 ix86_emit_swsqrtsf (operands[0], operands[1], <MODE>mode, true);
1837 (define_insn "<sse>_rsqrt<mode>2"
1838 [(set (match_operand:VF1_128_256 0 "register_operand" "=x")
1840 [(match_operand:VF1_128_256 1 "nonimmediate_operand" "xm")] UNSPEC_RSQRT))]
1842 "%vrsqrtps\t{%1, %0|%0, %1}"
1843 [(set_attr "type" "sse")
1844 (set_attr "prefix" "maybe_vex")
1845 (set_attr "mode" "<MODE>")])
1847 (define_insn "<mask_codefor>rsqrt14<mode><mask_name>"
1848 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v")
1850 [(match_operand:VF_AVX512VL 1 "nonimmediate_operand" "vm")]
1853 "vrsqrt14<ssemodesuffix>\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
1854 [(set_attr "type" "sse")
1855 (set_attr "prefix" "evex")
1856 (set_attr "mode" "<MODE>")])
1858 (define_insn "rsqrt14<mode>"
1859 [(set (match_operand:VF_128 0 "register_operand" "=v")
1862 [(match_operand:VF_128 1 "nonimmediate_operand" "vm")]
1864 (match_operand:VF_128 2 "register_operand" "v")
1867 "vrsqrt14<ssescalarmodesuffix>\t{%1, %2, %0|%0, %2, %1}"
1868 [(set_attr "type" "sse")
1869 (set_attr "prefix" "evex")
1870 (set_attr "mode" "<MODE>")])
1872 (define_insn "sse_vmrsqrtv4sf2"
1873 [(set (match_operand:V4SF 0 "register_operand" "=x,x")
1875 (unspec:V4SF [(match_operand:V4SF 1 "nonimmediate_operand" "xm,xm")]
1877 (match_operand:V4SF 2 "register_operand" "0,x")
1881 rsqrtss\t{%1, %0|%0, %k1}
1882 vrsqrtss\t{%1, %2, %0|%0, %2, %k1}"
1883 [(set_attr "isa" "noavx,avx")
1884 (set_attr "type" "sse")
1885 (set_attr "ssememalign" "32")
1886 (set_attr "prefix" "orig,vex")
1887 (set_attr "mode" "SF")])
1889 ;; ??? For !flag_finite_math_only, the representation with SMIN/SMAX
1890 ;; isn't really correct, as those rtl operators aren't defined when
1891 ;; applied to NaNs. Hopefully the optimizers won't get too smart on us.
1893 (define_expand "<code><mode>3<mask_name><round_saeonly_name>"
1894 [(set (match_operand:VF 0 "register_operand")
1896 (match_operand:VF 1 "<round_saeonly_nimm_predicate>")
1897 (match_operand:VF 2 "<round_saeonly_nimm_predicate>")))]
1898 "TARGET_SSE && <mask_mode512bit_condition> && <round_saeonly_mode512bit_condition>"
1900 if (!flag_finite_math_only)
1901 operands[1] = force_reg (<MODE>mode, operands[1]);
1902 ix86_fixup_binary_operands_no_copy (<CODE>, <MODE>mode, operands);
1905 (define_insn "*<code><mode>3_finite<mask_name><round_saeonly_name>"
1906 [(set (match_operand:VF 0 "register_operand" "=x,v")
1908 (match_operand:VF 1 "<round_saeonly_nimm_predicate>" "%0,v")
1909 (match_operand:VF 2 "<round_saeonly_nimm_predicate>" "xm,<round_saeonly_constraint>")))]
1910 "TARGET_SSE && flag_finite_math_only
1911 && ix86_binary_operator_ok (<CODE>, <MODE>mode, operands)
1912 && <mask_mode512bit_condition> && <round_saeonly_mode512bit_condition>"
1914 <maxmin_float><ssemodesuffix>\t{%2, %0|%0, %2}
1915 v<maxmin_float><ssemodesuffix>\t{<round_saeonly_mask_op3>%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2<round_saeonly_mask_op3>}"
1916 [(set_attr "isa" "noavx,avx")
1917 (set_attr "type" "sseadd")
1918 (set_attr "btver2_sse_attr" "maxmin")
1919 (set_attr "prefix" "<mask_prefix3>")
1920 (set_attr "mode" "<MODE>")])
1922 (define_insn "*<code><mode>3<mask_name><round_saeonly_name>"
1923 [(set (match_operand:VF 0 "register_operand" "=x,v")
1925 (match_operand:VF 1 "register_operand" "0,v")
1926 (match_operand:VF 2 "<round_saeonly_nimm_predicate>" "xm,<round_saeonly_constraint>")))]
1927 "TARGET_SSE && !flag_finite_math_only
1928 && <mask_mode512bit_condition> && <round_saeonly_mode512bit_condition>"
1930 <maxmin_float><ssemodesuffix>\t{%2, %0|%0, %2}
1931 v<maxmin_float><ssemodesuffix>\t{<round_saeonly_mask_op3>%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2<round_saeonly_mask_op3>}"
1932 [(set_attr "isa" "noavx,avx")
1933 (set_attr "type" "sseadd")
1934 (set_attr "btver2_sse_attr" "maxmin")
1935 (set_attr "prefix" "<mask_prefix3>")
1936 (set_attr "mode" "<MODE>")])
1938 (define_insn "<sse>_vm<code><mode>3<round_saeonly_name>"
1939 [(set (match_operand:VF_128 0 "register_operand" "=x,v")
1942 (match_operand:VF_128 1 "register_operand" "0,v")
1943 (match_operand:VF_128 2 "nonimmediate_operand" "xm,<round_saeonly_constraint>"))
1948 <maxmin_float><ssescalarmodesuffix>\t{%2, %0|%0, %<iptr>2}
1949 v<maxmin_float><ssescalarmodesuffix>\t{<round_saeonly_op3>%2, %1, %0|%0, %1, %<iptr>2<round_saeonly_op3>}"
1950 [(set_attr "isa" "noavx,avx")
1951 (set_attr "type" "sse")
1952 (set_attr "btver2_sse_attr" "maxmin")
1953 (set_attr "prefix" "<round_saeonly_prefix>")
1954 (set_attr "mode" "<ssescalarmode>")])
1956 ;; These versions of the min/max patterns implement exactly the operations
1957 ;; min = (op1 < op2 ? op1 : op2)
1958 ;; max = (!(op1 < op2) ? op1 : op2)
1959 ;; Their operands are not commutative, and thus they may be used in the
1960 ;; presence of -0.0 and NaN.
1962 (define_insn "*ieee_smin<mode>3"
1963 [(set (match_operand:VF 0 "register_operand" "=v,v")
1965 [(match_operand:VF 1 "register_operand" "0,v")
1966 (match_operand:VF 2 "nonimmediate_operand" "vm,vm")]
1970 min<ssemodesuffix>\t{%2, %0|%0, %2}
1971 vmin<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
1972 [(set_attr "isa" "noavx,avx")
1973 (set_attr "type" "sseadd")
1974 (set_attr "prefix" "orig,vex")
1975 (set_attr "mode" "<MODE>")])
1977 (define_insn "*ieee_smax<mode>3"
1978 [(set (match_operand:VF 0 "register_operand" "=v,v")
1980 [(match_operand:VF 1 "register_operand" "0,v")
1981 (match_operand:VF 2 "nonimmediate_operand" "vm,vm")]
1985 max<ssemodesuffix>\t{%2, %0|%0, %2}
1986 vmax<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
1987 [(set_attr "isa" "noavx,avx")
1988 (set_attr "type" "sseadd")
1989 (set_attr "prefix" "orig,vex")
1990 (set_attr "mode" "<MODE>")])
1992 (define_insn "avx_addsubv4df3"
1993 [(set (match_operand:V4DF 0 "register_operand" "=x")
1996 (match_operand:V4DF 1 "register_operand" "x")
1997 (match_operand:V4DF 2 "nonimmediate_operand" "xm"))
1998 (minus:V4DF (match_dup 1) (match_dup 2))
2001 "vaddsubpd\t{%2, %1, %0|%0, %1, %2}"
2002 [(set_attr "type" "sseadd")
2003 (set_attr "prefix" "vex")
2004 (set_attr "mode" "V4DF")])
2006 (define_insn "sse3_addsubv2df3"
2007 [(set (match_operand:V2DF 0 "register_operand" "=x,x")
2010 (match_operand:V2DF 1 "register_operand" "0,x")
2011 (match_operand:V2DF 2 "nonimmediate_operand" "xm,xm"))
2012 (minus:V2DF (match_dup 1) (match_dup 2))
2016 addsubpd\t{%2, %0|%0, %2}
2017 vaddsubpd\t{%2, %1, %0|%0, %1, %2}"
2018 [(set_attr "isa" "noavx,avx")
2019 (set_attr "type" "sseadd")
2020 (set_attr "atom_unit" "complex")
2021 (set_attr "prefix" "orig,vex")
2022 (set_attr "mode" "V2DF")])
2024 (define_insn "avx_addsubv8sf3"
2025 [(set (match_operand:V8SF 0 "register_operand" "=x")
2028 (match_operand:V8SF 1 "register_operand" "x")
2029 (match_operand:V8SF 2 "nonimmediate_operand" "xm"))
2030 (minus:V8SF (match_dup 1) (match_dup 2))
2033 "vaddsubps\t{%2, %1, %0|%0, %1, %2}"
2034 [(set_attr "type" "sseadd")
2035 (set_attr "prefix" "vex")
2036 (set_attr "mode" "V8SF")])
2038 (define_insn "sse3_addsubv4sf3"
2039 [(set (match_operand:V4SF 0 "register_operand" "=x,x")
2042 (match_operand:V4SF 1 "register_operand" "0,x")
2043 (match_operand:V4SF 2 "nonimmediate_operand" "xm,xm"))
2044 (minus:V4SF (match_dup 1) (match_dup 2))
2048 addsubps\t{%2, %0|%0, %2}
2049 vaddsubps\t{%2, %1, %0|%0, %1, %2}"
2050 [(set_attr "isa" "noavx,avx")
2051 (set_attr "type" "sseadd")
2052 (set_attr "prefix" "orig,vex")
2053 (set_attr "prefix_rep" "1,*")
2054 (set_attr "mode" "V4SF")])
2056 (define_insn "avx_h<plusminus_insn>v4df3"
2057 [(set (match_operand:V4DF 0 "register_operand" "=x")
2062 (match_operand:V4DF 1 "register_operand" "x")
2063 (parallel [(const_int 0)]))
2064 (vec_select:DF (match_dup 1) (parallel [(const_int 1)])))
2067 (match_operand:V4DF 2 "nonimmediate_operand" "xm")
2068 (parallel [(const_int 0)]))
2069 (vec_select:DF (match_dup 2) (parallel [(const_int 1)]))))
2072 (vec_select:DF (match_dup 1) (parallel [(const_int 2)]))
2073 (vec_select:DF (match_dup 1) (parallel [(const_int 3)])))
2075 (vec_select:DF (match_dup 2) (parallel [(const_int 2)]))
2076 (vec_select:DF (match_dup 2) (parallel [(const_int 3)]))))))]
2078 "vh<plusminus_mnemonic>pd\t{%2, %1, %0|%0, %1, %2}"
2079 [(set_attr "type" "sseadd")
2080 (set_attr "prefix" "vex")
2081 (set_attr "mode" "V4DF")])
2083 (define_expand "sse3_haddv2df3"
2084 [(set (match_operand:V2DF 0 "register_operand")
2088 (match_operand:V2DF 1 "register_operand")
2089 (parallel [(const_int 0)]))
2090 (vec_select:DF (match_dup 1) (parallel [(const_int 1)])))
2093 (match_operand:V2DF 2 "nonimmediate_operand")
2094 (parallel [(const_int 0)]))
2095 (vec_select:DF (match_dup 2) (parallel [(const_int 1)])))))]
2098 (define_insn "*sse3_haddv2df3"
2099 [(set (match_operand:V2DF 0 "register_operand" "=x,x")
2103 (match_operand:V2DF 1 "register_operand" "0,x")
2104 (parallel [(match_operand:SI 3 "const_0_to_1_operand")]))
2107 (parallel [(match_operand:SI 4 "const_0_to_1_operand")])))
2110 (match_operand:V2DF 2 "nonimmediate_operand" "xm,xm")
2111 (parallel [(match_operand:SI 5 "const_0_to_1_operand")]))
2114 (parallel [(match_operand:SI 6 "const_0_to_1_operand")])))))]
2116 && INTVAL (operands[3]) != INTVAL (operands[4])
2117 && INTVAL (operands[5]) != INTVAL (operands[6])"
2119 haddpd\t{%2, %0|%0, %2}
2120 vhaddpd\t{%2, %1, %0|%0, %1, %2}"
2121 [(set_attr "isa" "noavx,avx")
2122 (set_attr "type" "sseadd")
2123 (set_attr "prefix" "orig,vex")
2124 (set_attr "mode" "V2DF")])
2126 (define_insn "sse3_hsubv2df3"
2127 [(set (match_operand:V2DF 0 "register_operand" "=x,x")
2131 (match_operand:V2DF 1 "register_operand" "0,x")
2132 (parallel [(const_int 0)]))
2133 (vec_select:DF (match_dup 1) (parallel [(const_int 1)])))
2136 (match_operand:V2DF 2 "nonimmediate_operand" "xm,xm")
2137 (parallel [(const_int 0)]))
2138 (vec_select:DF (match_dup 2) (parallel [(const_int 1)])))))]
2141 hsubpd\t{%2, %0|%0, %2}
2142 vhsubpd\t{%2, %1, %0|%0, %1, %2}"
2143 [(set_attr "isa" "noavx,avx")
2144 (set_attr "type" "sseadd")
2145 (set_attr "prefix" "orig,vex")
2146 (set_attr "mode" "V2DF")])
2148 (define_insn "*sse3_haddv2df3_low"
2149 [(set (match_operand:DF 0 "register_operand" "=x,x")
2152 (match_operand:V2DF 1 "register_operand" "0,x")
2153 (parallel [(match_operand:SI 2 "const_0_to_1_operand")]))
2156 (parallel [(match_operand:SI 3 "const_0_to_1_operand")]))))]
2158 && INTVAL (operands[2]) != INTVAL (operands[3])"
2160 haddpd\t{%0, %0|%0, %0}
2161 vhaddpd\t{%1, %1, %0|%0, %1, %1}"
2162 [(set_attr "isa" "noavx,avx")
2163 (set_attr "type" "sseadd1")
2164 (set_attr "prefix" "orig,vex")
2165 (set_attr "mode" "V2DF")])
2167 (define_insn "*sse3_hsubv2df3_low"
2168 [(set (match_operand:DF 0 "register_operand" "=x,x")
2171 (match_operand:V2DF 1 "register_operand" "0,x")
2172 (parallel [(const_int 0)]))
2175 (parallel [(const_int 1)]))))]
2178 hsubpd\t{%0, %0|%0, %0}
2179 vhsubpd\t{%1, %1, %0|%0, %1, %1}"
2180 [(set_attr "isa" "noavx,avx")
2181 (set_attr "type" "sseadd1")
2182 (set_attr "prefix" "orig,vex")
2183 (set_attr "mode" "V2DF")])
2185 (define_insn "avx_h<plusminus_insn>v8sf3"
2186 [(set (match_operand:V8SF 0 "register_operand" "=x")
2192 (match_operand:V8SF 1 "register_operand" "x")
2193 (parallel [(const_int 0)]))
2194 (vec_select:SF (match_dup 1) (parallel [(const_int 1)])))
2196 (vec_select:SF (match_dup 1) (parallel [(const_int 2)]))
2197 (vec_select:SF (match_dup 1) (parallel [(const_int 3)]))))
2201 (match_operand:V8SF 2 "nonimmediate_operand" "xm")
2202 (parallel [(const_int 0)]))
2203 (vec_select:SF (match_dup 2) (parallel [(const_int 1)])))
2205 (vec_select:SF (match_dup 2) (parallel [(const_int 2)]))
2206 (vec_select:SF (match_dup 2) (parallel [(const_int 3)])))))
2210 (vec_select:SF (match_dup 1) (parallel [(const_int 4)]))
2211 (vec_select:SF (match_dup 1) (parallel [(const_int 5)])))
2213 (vec_select:SF (match_dup 1) (parallel [(const_int 6)]))
2214 (vec_select:SF (match_dup 1) (parallel [(const_int 7)]))))
2217 (vec_select:SF (match_dup 2) (parallel [(const_int 4)]))
2218 (vec_select:SF (match_dup 2) (parallel [(const_int 5)])))
2220 (vec_select:SF (match_dup 2) (parallel [(const_int 6)]))
2221 (vec_select:SF (match_dup 2) (parallel [(const_int 7)])))))))]
2223 "vh<plusminus_mnemonic>ps\t{%2, %1, %0|%0, %1, %2}"
2224 [(set_attr "type" "sseadd")
2225 (set_attr "prefix" "vex")
2226 (set_attr "mode" "V8SF")])
2228 (define_insn "sse3_h<plusminus_insn>v4sf3"
2229 [(set (match_operand:V4SF 0 "register_operand" "=x,x")
2234 (match_operand:V4SF 1 "register_operand" "0,x")
2235 (parallel [(const_int 0)]))
2236 (vec_select:SF (match_dup 1) (parallel [(const_int 1)])))
2238 (vec_select:SF (match_dup 1) (parallel [(const_int 2)]))
2239 (vec_select:SF (match_dup 1) (parallel [(const_int 3)]))))
2243 (match_operand:V4SF 2 "nonimmediate_operand" "xm,xm")
2244 (parallel [(const_int 0)]))
2245 (vec_select:SF (match_dup 2) (parallel [(const_int 1)])))
2247 (vec_select:SF (match_dup 2) (parallel [(const_int 2)]))
2248 (vec_select:SF (match_dup 2) (parallel [(const_int 3)]))))))]
2251 h<plusminus_mnemonic>ps\t{%2, %0|%0, %2}
2252 vh<plusminus_mnemonic>ps\t{%2, %1, %0|%0, %1, %2}"
2253 [(set_attr "isa" "noavx,avx")
2254 (set_attr "type" "sseadd")
2255 (set_attr "atom_unit" "complex")
2256 (set_attr "prefix" "orig,vex")
2257 (set_attr "prefix_rep" "1,*")
2258 (set_attr "mode" "V4SF")])
2260 (define_expand "reduc_splus_v8df"
2261 [(match_operand:V8DF 0 "register_operand")
2262 (match_operand:V8DF 1 "register_operand")]
2265 ix86_expand_reduc (gen_addv8df3, operands[0], operands[1]);
2269 (define_expand "reduc_splus_v4df"
2270 [(match_operand:V4DF 0 "register_operand")
2271 (match_operand:V4DF 1 "register_operand")]
2274 rtx tmp = gen_reg_rtx (V4DFmode);
2275 rtx tmp2 = gen_reg_rtx (V4DFmode);
2276 emit_insn (gen_avx_haddv4df3 (tmp, operands[1], operands[1]));
2277 emit_insn (gen_avx_vperm2f128v4df3 (tmp2, tmp, tmp, GEN_INT (1)));
2278 emit_insn (gen_addv4df3 (operands[0], tmp, tmp2));
2282 (define_expand "reduc_splus_v2df"
2283 [(match_operand:V2DF 0 "register_operand")
2284 (match_operand:V2DF 1 "register_operand")]
2287 emit_insn (gen_sse3_haddv2df3 (operands[0], operands[1], operands[1]));
2291 (define_expand "reduc_splus_v16sf"
2292 [(match_operand:V16SF 0 "register_operand")
2293 (match_operand:V16SF 1 "register_operand")]
2296 ix86_expand_reduc (gen_addv16sf3, operands[0], operands[1]);
2300 (define_expand "reduc_splus_v8sf"
2301 [(match_operand:V8SF 0 "register_operand")
2302 (match_operand:V8SF 1 "register_operand")]
2305 rtx tmp = gen_reg_rtx (V8SFmode);
2306 rtx tmp2 = gen_reg_rtx (V8SFmode);
2307 emit_insn (gen_avx_haddv8sf3 (tmp, operands[1], operands[1]));
2308 emit_insn (gen_avx_haddv8sf3 (tmp2, tmp, tmp));
2309 emit_insn (gen_avx_vperm2f128v8sf3 (tmp, tmp2, tmp2, GEN_INT (1)));
2310 emit_insn (gen_addv8sf3 (operands[0], tmp, tmp2));
2314 (define_expand "reduc_splus_v4sf"
2315 [(match_operand:V4SF 0 "register_operand")
2316 (match_operand:V4SF 1 "register_operand")]
2321 rtx tmp = gen_reg_rtx (V4SFmode);
2322 emit_insn (gen_sse3_haddv4sf3 (tmp, operands[1], operands[1]));
2323 emit_insn (gen_sse3_haddv4sf3 (operands[0], tmp, tmp));
2326 ix86_expand_reduc (gen_addv4sf3, operands[0], operands[1]);
2330 ;; Modes handled by reduc_sm{in,ax}* patterns.
2331 (define_mode_iterator REDUC_SMINMAX_MODE
2332 [(V32QI "TARGET_AVX2") (V16HI "TARGET_AVX2")
2333 (V8SI "TARGET_AVX2") (V4DI "TARGET_AVX2")
2334 (V8SF "TARGET_AVX") (V4DF "TARGET_AVX")
2335 (V4SF "TARGET_SSE") (V64QI "TARGET_AVX512BW")
2336 (V32HI "TARGET_AVX512BW") (V16SI "TARGET_AVX512F")
2337 (V8DI "TARGET_AVX512F") (V16SF "TARGET_AVX512F")
2338 (V8DF "TARGET_AVX512F")])
2340 (define_expand "reduc_<code>_<mode>"
2341 [(smaxmin:REDUC_SMINMAX_MODE
2342 (match_operand:REDUC_SMINMAX_MODE 0 "register_operand")
2343 (match_operand:REDUC_SMINMAX_MODE 1 "register_operand"))]
2346 ix86_expand_reduc (gen_<code><mode>3, operands[0], operands[1]);
2350 (define_expand "reduc_<code>_<mode>"
2351 [(umaxmin:VI_AVX512BW
2352 (match_operand:VI_AVX512BW 0 "register_operand")
2353 (match_operand:VI_AVX512BW 1 "register_operand"))]
2356 ix86_expand_reduc (gen_<code><mode>3, operands[0], operands[1]);
2360 (define_expand "reduc_<code>_<mode>"
2362 (match_operand:VI_256 0 "register_operand")
2363 (match_operand:VI_256 1 "register_operand"))]
2366 ix86_expand_reduc (gen_<code><mode>3, operands[0], operands[1]);
2370 (define_expand "reduc_umin_v8hi"
2372 (match_operand:V8HI 0 "register_operand")
2373 (match_operand:V8HI 1 "register_operand"))]
2376 ix86_expand_reduc (gen_uminv8hi3, operands[0], operands[1]);
2380 (define_insn "<mask_codefor>reducep<mode><mask_name>"
2381 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v")
2383 [(match_operand:VF_AVX512VL 1 "nonimmediate_operand" "vm")
2384 (match_operand:SI 2 "const_0_to_255_operand")]
2387 "vreduce<ssemodesuffix>\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
2388 [(set_attr "type" "sse")
2389 (set_attr "prefix" "evex")
2390 (set_attr "mode" "<MODE>")])
2392 (define_insn "reduces<mode>"
2393 [(set (match_operand:VF_128 0 "register_operand" "=v")
2396 [(match_operand:VF_128 1 "register_operand" "v")
2397 (match_operand:VF_128 2 "nonimmediate_operand" "vm")
2398 (match_operand:SI 3 "const_0_to_255_operand")]
2403 "vreduce<ssescalarmodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
2404 [(set_attr "type" "sse")
2405 (set_attr "prefix" "evex")
2406 (set_attr "mode" "<MODE>")])
2408 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
2410 ;; Parallel floating point comparisons
2412 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
2414 (define_insn "avx_cmp<mode>3"
2415 [(set (match_operand:VF_128_256 0 "register_operand" "=x")
2417 [(match_operand:VF_128_256 1 "register_operand" "x")
2418 (match_operand:VF_128_256 2 "nonimmediate_operand" "xm")
2419 (match_operand:SI 3 "const_0_to_31_operand" "n")]
2422 "vcmp<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
2423 [(set_attr "type" "ssecmp")
2424 (set_attr "length_immediate" "1")
2425 (set_attr "prefix" "vex")
2426 (set_attr "mode" "<MODE>")])
2428 (define_insn "avx_vmcmp<mode>3"
2429 [(set (match_operand:VF_128 0 "register_operand" "=x")
2432 [(match_operand:VF_128 1 "register_operand" "x")
2433 (match_operand:VF_128 2 "nonimmediate_operand" "xm")
2434 (match_operand:SI 3 "const_0_to_31_operand" "n")]
2439 "vcmp<ssescalarmodesuffix>\t{%3, %2, %1, %0|%0, %1, %<iptr>2, %3}"
2440 [(set_attr "type" "ssecmp")
2441 (set_attr "length_immediate" "1")
2442 (set_attr "prefix" "vex")
2443 (set_attr "mode" "<ssescalarmode>")])
2445 (define_insn "*<sse>_maskcmp<mode>3_comm"
2446 [(set (match_operand:VF_128_256 0 "register_operand" "=x,x")
2447 (match_operator:VF_128_256 3 "sse_comparison_operator"
2448 [(match_operand:VF_128_256 1 "register_operand" "%0,x")
2449 (match_operand:VF_128_256 2 "nonimmediate_operand" "xm,xm")]))]
2451 && GET_RTX_CLASS (GET_CODE (operands[3])) == RTX_COMM_COMPARE"
2453 cmp%D3<ssemodesuffix>\t{%2, %0|%0, %2}
2454 vcmp%D3<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
2455 [(set_attr "isa" "noavx,avx")
2456 (set_attr "type" "ssecmp")
2457 (set_attr "length_immediate" "1")
2458 (set_attr "prefix" "orig,vex")
2459 (set_attr "mode" "<MODE>")])
2461 (define_insn "<sse>_maskcmp<mode>3"
2462 [(set (match_operand:VF_128_256 0 "register_operand" "=x,x")
2463 (match_operator:VF_128_256 3 "sse_comparison_operator"
2464 [(match_operand:VF_128_256 1 "register_operand" "0,x")
2465 (match_operand:VF_128_256 2 "nonimmediate_operand" "xm,xm")]))]
2468 cmp%D3<ssemodesuffix>\t{%2, %0|%0, %2}
2469 vcmp%D3<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
2470 [(set_attr "isa" "noavx,avx")
2471 (set_attr "type" "ssecmp")
2472 (set_attr "length_immediate" "1")
2473 (set_attr "prefix" "orig,vex")
2474 (set_attr "mode" "<MODE>")])
2476 (define_insn "<sse>_vmmaskcmp<mode>3"
2477 [(set (match_operand:VF_128 0 "register_operand" "=x,x")
2479 (match_operator:VF_128 3 "sse_comparison_operator"
2480 [(match_operand:VF_128 1 "register_operand" "0,x")
2481 (match_operand:VF_128 2 "nonimmediate_operand" "xm,xm")])
2486 cmp%D3<ssescalarmodesuffix>\t{%2, %0|%0, %<iptr>2}
2487 vcmp%D3<ssescalarmodesuffix>\t{%2, %1, %0|%0, %1, %<iptr>2}"
2488 [(set_attr "isa" "noavx,avx")
2489 (set_attr "type" "ssecmp")
2490 (set_attr "length_immediate" "1,*")
2491 (set_attr "prefix" "orig,vex")
2492 (set_attr "mode" "<ssescalarmode>")])
2494 (define_mode_attr cmp_imm_predicate
2495 [(V16SF "const_0_to_31_operand") (V8DF "const_0_to_31_operand")
2496 (V16SI "const_0_to_7_operand") (V8DI "const_0_to_7_operand")
2497 (V8SF "const_0_to_31_operand") (V4DF "const_0_to_31_operand")
2498 (V8SI "const_0_to_7_operand") (V4DI "const_0_to_7_operand")
2499 (V4SF "const_0_to_31_operand") (V2DF "const_0_to_31_operand")
2500 (V4SI "const_0_to_7_operand") (V2DI "const_0_to_7_operand")
2501 (V32HI "const_0_to_7_operand") (V64QI "const_0_to_7_operand")
2502 (V16HI "const_0_to_7_operand") (V32QI "const_0_to_7_operand")
2503 (V8HI "const_0_to_7_operand") (V16QI "const_0_to_7_operand")])
2505 (define_insn "<avx512>_cmp<mode>3<mask_scalar_merge_name><round_saeonly_name>"
2506 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=Yk")
2507 (unspec:<avx512fmaskmode>
2508 [(match_operand:V48_AVX512VL 1 "register_operand" "v")
2509 (match_operand:V48_AVX512VL 2 "nonimmediate_operand" "<round_saeonly_constraint>")
2510 (match_operand:SI 3 "<cmp_imm_predicate>" "n")]
2512 "TARGET_AVX512F && <round_saeonly_mode512bit_condition>"
2513 "v<sseintprefix>cmp<ssemodesuffix>\t{%3, <round_saeonly_mask_scalar_merge_op4>%2, %1, %0<mask_scalar_merge_operand4>|%0<mask_scalar_merge_operand4>, %1, %2<round_saeonly_mask_scalar_merge_op4>, %3}"
2514 [(set_attr "type" "ssecmp")
2515 (set_attr "length_immediate" "1")
2516 (set_attr "prefix" "evex")
2517 (set_attr "mode" "<sseinsnmode>")])
2519 (define_insn "<avx512>_cmp<mode>3<mask_scalar_merge_name>"
2520 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=Yk")
2521 (unspec:<avx512fmaskmode>
2522 [(match_operand:VI12_AVX512VL 1 "register_operand" "v")
2523 (match_operand:VI12_AVX512VL 2 "nonimmediate_operand" "vm")
2524 (match_operand:SI 3 "<cmp_imm_predicate>" "n")]
2527 "vpcmp<ssemodesuffix>\t{%3, %2, %1, %0<mask_scalar_merge_operand4>|%0<mask_scalar_merge_operand4>, %1, %2, %3}"
2528 [(set_attr "type" "ssecmp")
2529 (set_attr "length_immediate" "1")
2530 (set_attr "prefix" "evex")
2531 (set_attr "mode" "<sseinsnmode>")])
2533 (define_insn "<avx512>_ucmp<mode>3<mask_scalar_merge_name>"
2534 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=Yk")
2535 (unspec:<avx512fmaskmode>
2536 [(match_operand:VI12_AVX512VL 1 "register_operand" "v")
2537 (match_operand:VI12_AVX512VL 2 "nonimmediate_operand" "vm")
2538 (match_operand:SI 3 "const_0_to_7_operand" "n")]
2539 UNSPEC_UNSIGNED_PCMP))]
2541 "vpcmpu<ssemodesuffix>\t{%3, %2, %1, %0<mask_scalar_merge_operand4>|%0<mask_scalar_merge_operand4>, %1, %2, %3}"
2542 [(set_attr "type" "ssecmp")
2543 (set_attr "length_immediate" "1")
2544 (set_attr "prefix" "evex")
2545 (set_attr "mode" "<sseinsnmode>")])
2547 (define_insn "<avx512>_ucmp<mode>3<mask_scalar_merge_name>"
2548 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=Yk")
2549 (unspec:<avx512fmaskmode>
2550 [(match_operand:VI48_AVX512VL 1 "register_operand" "v")
2551 (match_operand:VI48_AVX512VL 2 "nonimmediate_operand" "vm")
2552 (match_operand:SI 3 "const_0_to_7_operand" "n")]
2553 UNSPEC_UNSIGNED_PCMP))]
2555 "vpcmpu<ssemodesuffix>\t{%3, %2, %1, %0<mask_scalar_merge_operand4>|%0<mask_scalar_merge_operand4>, %1, %2, %3}"
2556 [(set_attr "type" "ssecmp")
2557 (set_attr "length_immediate" "1")
2558 (set_attr "prefix" "evex")
2559 (set_attr "mode" "<sseinsnmode>")])
2561 (define_insn "avx512f_vmcmp<mode>3<round_saeonly_name>"
2562 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=Yk")
2563 (and:<avx512fmaskmode>
2564 (unspec:<avx512fmaskmode>
2565 [(match_operand:VF_128 1 "register_operand" "v")
2566 (match_operand:VF_128 2 "<round_saeonly_nimm_predicate>" "<round_saeonly_constraint>")
2567 (match_operand:SI 3 "const_0_to_31_operand" "n")]
2571 "vcmp<ssescalarmodesuffix>\t{%3, <round_saeonly_op4>%2, %1, %0|%0, %1, %2<round_saeonly_op4>, %3}"
2572 [(set_attr "type" "ssecmp")
2573 (set_attr "length_immediate" "1")
2574 (set_attr "prefix" "evex")
2575 (set_attr "mode" "<ssescalarmode>")])
2577 (define_insn "avx512f_vmcmp<mode>3_mask<round_saeonly_name>"
2578 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=Yk")
2579 (and:<avx512fmaskmode>
2580 (unspec:<avx512fmaskmode>
2581 [(match_operand:VF_128 1 "register_operand" "v")
2582 (match_operand:VF_128 2 "<round_saeonly_nimm_predicate>" "<round_saeonly_constraint>")
2583 (match_operand:SI 3 "const_0_to_31_operand" "n")]
2585 (and:<avx512fmaskmode>
2586 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk")
2589 "vcmp<ssescalarmodesuffix>\t{%3, <round_saeonly_op5>%2, %1, %0%{%4%}|%0%{%4%}, %1, %2<round_saeonly_op5>, %3}"
2590 [(set_attr "type" "ssecmp")
2591 (set_attr "length_immediate" "1")
2592 (set_attr "prefix" "evex")
2593 (set_attr "mode" "<ssescalarmode>")])
2595 (define_insn "avx512f_maskcmp<mode>3"
2596 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=Yk")
2597 (match_operator:<avx512fmaskmode> 3 "sse_comparison_operator"
2598 [(match_operand:VF 1 "register_operand" "v")
2599 (match_operand:VF 2 "nonimmediate_operand" "vm")]))]
2601 "vcmp%D3<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
2602 [(set_attr "type" "ssecmp")
2603 (set_attr "length_immediate" "1")
2604 (set_attr "prefix" "evex")
2605 (set_attr "mode" "<sseinsnmode>")])
2607 (define_insn "<sse>_comi<round_saeonly_name>"
2608 [(set (reg:CCFP FLAGS_REG)
2611 (match_operand:<ssevecmode> 0 "register_operand" "v")
2612 (parallel [(const_int 0)]))
2614 (match_operand:<ssevecmode> 1 "<round_saeonly_nimm_predicate>" "<round_saeonly_constraint>")
2615 (parallel [(const_int 0)]))))]
2616 "SSE_FLOAT_MODE_P (<MODE>mode)"
2617 "%vcomi<ssemodesuffix>\t{<round_saeonly_op2>%1, %0|%0, %<iptr>1<round_saeonly_op2>}"
2618 [(set_attr "type" "ssecomi")
2619 (set_attr "prefix" "maybe_vex")
2620 (set_attr "prefix_rep" "0")
2621 (set (attr "prefix_data16")
2622 (if_then_else (eq_attr "mode" "DF")
2624 (const_string "0")))
2625 (set_attr "mode" "<MODE>")])
2627 (define_insn "<sse>_ucomi<round_saeonly_name>"
2628 [(set (reg:CCFPU FLAGS_REG)
2631 (match_operand:<ssevecmode> 0 "register_operand" "v")
2632 (parallel [(const_int 0)]))
2634 (match_operand:<ssevecmode> 1 "<round_saeonly_nimm_predicate>" "<round_saeonly_constraint>")
2635 (parallel [(const_int 0)]))))]
2636 "SSE_FLOAT_MODE_P (<MODE>mode)"
2637 "%vucomi<ssemodesuffix>\t{<round_saeonly_op2>%1, %0|%0, %<iptr>1<round_saeonly_op2>}"
2638 [(set_attr "type" "ssecomi")
2639 (set_attr "prefix" "maybe_vex")
2640 (set_attr "prefix_rep" "0")
2641 (set (attr "prefix_data16")
2642 (if_then_else (eq_attr "mode" "DF")
2644 (const_string "0")))
2645 (set_attr "mode" "<MODE>")])
2647 (define_expand "vcond<V_512:mode><VF_512:mode>"
2648 [(set (match_operand:V_512 0 "register_operand")
2650 (match_operator 3 ""
2651 [(match_operand:VF_512 4 "nonimmediate_operand")
2652 (match_operand:VF_512 5 "nonimmediate_operand")])
2653 (match_operand:V_512 1 "general_operand")
2654 (match_operand:V_512 2 "general_operand")))]
2656 && (GET_MODE_NUNITS (<V_512:MODE>mode)
2657 == GET_MODE_NUNITS (<VF_512:MODE>mode))"
2659 bool ok = ix86_expand_fp_vcond (operands);
2664 (define_expand "vcond<V_256:mode><VF_256:mode>"
2665 [(set (match_operand:V_256 0 "register_operand")
2667 (match_operator 3 ""
2668 [(match_operand:VF_256 4 "nonimmediate_operand")
2669 (match_operand:VF_256 5 "nonimmediate_operand")])
2670 (match_operand:V_256 1 "general_operand")
2671 (match_operand:V_256 2 "general_operand")))]
2673 && (GET_MODE_NUNITS (<V_256:MODE>mode)
2674 == GET_MODE_NUNITS (<VF_256:MODE>mode))"
2676 bool ok = ix86_expand_fp_vcond (operands);
2681 (define_expand "vcond<V_128:mode><VF_128:mode>"
2682 [(set (match_operand:V_128 0 "register_operand")
2684 (match_operator 3 ""
2685 [(match_operand:VF_128 4 "nonimmediate_operand")
2686 (match_operand:VF_128 5 "nonimmediate_operand")])
2687 (match_operand:V_128 1 "general_operand")
2688 (match_operand:V_128 2 "general_operand")))]
2690 && (GET_MODE_NUNITS (<V_128:MODE>mode)
2691 == GET_MODE_NUNITS (<VF_128:MODE>mode))"
2693 bool ok = ix86_expand_fp_vcond (operands);
2698 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
2700 ;; Parallel floating point logical operations
2702 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
2704 (define_insn "<sse>_andnot<mode>3<mask_name>"
2705 [(set (match_operand:VF_128_256 0 "register_operand" "=x,v")
2708 (match_operand:VF_128_256 1 "register_operand" "0,v"))
2709 (match_operand:VF_128_256 2 "nonimmediate_operand" "xm,vm")))]
2710 "TARGET_SSE && <mask_avx512vl_condition>"
2712 static char buf[128];
2716 switch (get_attr_mode (insn))
2723 suffix = "<ssemodesuffix>";
2726 switch (which_alternative)
2729 ops = "andn%s\t{%%2, %%0|%%0, %%2}";
2732 ops = "vandn%s\t{%%2, %%1, %%0<mask_operand3_1>|%%0<mask_operand3_1>, %%1, %%2}";
2738 /* There is no vandnp[sd] in avx512f. Use vpandn[qd]. */
2739 if (<mask_applied> && !TARGET_AVX512DQ)
2741 suffix = GET_MODE_INNER (<MODE>mode) == DFmode ? "q" : "d";
2742 ops = "vpandn%s\t{%%2, %%1, %%0<mask_operand3_1>|%%0<mask_operand3_1>, %%1, %%2}";
2745 snprintf (buf, sizeof (buf), ops, suffix);
2748 [(set_attr "isa" "noavx,avx")
2749 (set_attr "type" "sselog")
2750 (set_attr "prefix" "orig,maybe_evex")
2752 (cond [(and (match_test "<MODE_SIZE> == 16")
2753 (match_test "TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL"))
2754 (const_string "<ssePSmode>")
2755 (match_test "TARGET_AVX")
2756 (const_string "<MODE>")
2757 (match_test "optimize_function_for_size_p (cfun)")
2758 (const_string "V4SF")
2760 (const_string "<MODE>")))])
2763 (define_insn "<sse>_andnot<mode>3<mask_name>"
2764 [(set (match_operand:VF_512 0 "register_operand" "=v")
2767 (match_operand:VF_512 1 "register_operand" "v"))
2768 (match_operand:VF_512 2 "nonimmediate_operand" "vm")))]
2771 static char buf[128];
2775 suffix = "<ssemodesuffix>";
2778 /* There is no vandnp[sd] in avx512f. Use vpandn[qd]. */
2779 if (!TARGET_AVX512DQ)
2781 suffix = GET_MODE_INNER (<MODE>mode) == DFmode ? "q" : "d";
2785 snprintf (buf, sizeof (buf),
2786 "v%sandn%s\t{%%2, %%1, %%0<mask_operand3_1>|%%0<mask_operand3_1>, %%1, %%2}",
2790 [(set_attr "type" "sselog")
2791 (set_attr "prefix" "evex")
2792 (set_attr "mode" "<sseinsnmode>")])
2794 (define_expand "<code><mode>3<mask_name>"
2795 [(set (match_operand:VF_128_256 0 "register_operand")
2796 (any_logic:VF_128_256
2797 (match_operand:VF_128_256 1 "nonimmediate_operand")
2798 (match_operand:VF_128_256 2 "nonimmediate_operand")))]
2799 "TARGET_SSE && <mask_avx512vl_condition>"
2800 "ix86_fixup_binary_operands_no_copy (<CODE>, <MODE>mode, operands);")
2802 (define_expand "<code><mode>3<mask_name>"
2803 [(set (match_operand:VF_512 0 "register_operand")
2805 (match_operand:VF_512 1 "nonimmediate_operand")
2806 (match_operand:VF_512 2 "nonimmediate_operand")))]
2808 "ix86_fixup_binary_operands_no_copy (<CODE>, <MODE>mode, operands);")
2810 (define_insn "*<code><mode>3<mask_name>"
2811 [(set (match_operand:VF_128_256 0 "register_operand" "=x,v")
2812 (any_logic:VF_128_256
2813 (match_operand:VF_128_256 1 "nonimmediate_operand" "%0,v")
2814 (match_operand:VF_128_256 2 "nonimmediate_operand" "xm,vm")))]
2815 "TARGET_SSE && <mask_avx512vl_condition>
2816 && ix86_binary_operator_ok (<CODE>, <MODE>mode, operands)"
2818 static char buf[128];
2822 switch (get_attr_mode (insn))
2829 suffix = "<ssemodesuffix>";
2832 switch (which_alternative)
2835 ops = "<logic>%s\t{%%2, %%0|%%0, %%2}";
2838 ops = "v<logic>%s\t{%%2, %%1, %%0<mask_operand3_1>|%%0<mask_operand3_1>, %%1, %%2}";
2844 /* There is no v<logic>p[sd] in avx512f. Use vp<logic>[dq]. */
2845 if (<mask_applied> && !TARGET_AVX512DQ)
2847 suffix = GET_MODE_INNER (<MODE>mode) == DFmode ? "q" : "d";
2848 ops = "vp<logic>%s\t{%%2, %%1, %%0<mask_operand3_1>|%%0<mask_operand3_1>, %%1, %%2}";
2851 snprintf (buf, sizeof (buf), ops, suffix);
2854 [(set_attr "isa" "noavx,avx")
2855 (set_attr "type" "sselog")
2856 (set_attr "prefix" "orig,maybe_evex")
2858 (cond [(and (match_test "<MODE_SIZE> == 16")
2859 (match_test "TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL"))
2860 (const_string "<ssePSmode>")
2861 (match_test "TARGET_AVX")
2862 (const_string "<MODE>")
2863 (match_test "optimize_function_for_size_p (cfun)")
2864 (const_string "V4SF")
2866 (const_string "<MODE>")))])
2868 (define_insn "*<code><mode>3<mask_name>"
2869 [(set (match_operand:VF_512 0 "register_operand" "=v")
2871 (match_operand:VF_512 1 "nonimmediate_operand" "%v")
2872 (match_operand:VF_512 2 "nonimmediate_operand" "vm")))]
2873 "TARGET_AVX512F && ix86_binary_operator_ok (<CODE>, <MODE>mode, operands)"
2875 static char buf[128];
2879 suffix = "<ssemodesuffix>";
2882 /* There is no v<logic>p[sd] in avx512f. Use vp<logic>[dq]. */
2883 if ((<MODE_SIZE> == 64 || <mask_applied>) && !TARGET_AVX512DQ)
2885 suffix = GET_MODE_INNER (<MODE>mode) == DFmode ? "q" : "d";
2889 snprintf (buf, sizeof (buf),
2890 "v%s<logic>%s\t{%%2, %%1, %%0<mask_operand3_1>|%%0<mask_operand3_1>, %%1, %%2}",
2894 [(set_attr "type" "sselog")
2895 (set_attr "prefix" "evex")
2896 (set_attr "mode" "<sseinsnmode>")])
2898 (define_expand "copysign<mode>3"
2901 (not:VF (match_dup 3))
2902 (match_operand:VF 1 "nonimmediate_operand")))
2904 (and:VF (match_dup 3)
2905 (match_operand:VF 2 "nonimmediate_operand")))
2906 (set (match_operand:VF 0 "register_operand")
2907 (ior:VF (match_dup 4) (match_dup 5)))]
2910 operands[3] = ix86_build_signbit_mask (<MODE>mode, 1, 0);
2912 operands[4] = gen_reg_rtx (<MODE>mode);
2913 operands[5] = gen_reg_rtx (<MODE>mode);
2916 ;; Also define scalar versions. These are used for abs, neg, and
2917 ;; conditional move. Using subregs into vector modes causes register
2918 ;; allocation lossage. These patterns do not allow memory operands
2919 ;; because the native instructions read the full 128-bits.
2921 (define_insn "*andnot<mode>3"
2922 [(set (match_operand:MODEF 0 "register_operand" "=x,x")
2925 (match_operand:MODEF 1 "register_operand" "0,x"))
2926 (match_operand:MODEF 2 "register_operand" "x,x")))]
2927 "SSE_FLOAT_MODE_P (<MODE>mode)"
2929 static char buf[32];
2932 = (get_attr_mode (insn) == MODE_V4SF) ? "ps" : "<ssevecmodesuffix>";
2934 switch (which_alternative)
2937 ops = "andn%s\t{%%2, %%0|%%0, %%2}";
2940 ops = "vandn%s\t{%%2, %%1, %%0|%%0, %%1, %%2}";
2946 snprintf (buf, sizeof (buf), ops, suffix);
2949 [(set_attr "isa" "noavx,avx")
2950 (set_attr "type" "sselog")
2951 (set_attr "prefix" "orig,vex")
2953 (cond [(and (match_test "<MODE_SIZE> == 16")
2954 (match_test "TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL"))
2955 (const_string "V4SF")
2956 (match_test "TARGET_AVX")
2957 (const_string "<ssevecmode>")
2958 (match_test "optimize_function_for_size_p (cfun)")
2959 (const_string "V4SF")
2961 (const_string "<ssevecmode>")))])
2963 (define_insn "*andnottf3"
2964 [(set (match_operand:TF 0 "register_operand" "=x,x")
2966 (not:TF (match_operand:TF 1 "register_operand" "0,x"))
2967 (match_operand:TF 2 "nonimmediate_operand" "xm,xm")))]
2970 static char buf[32];
2973 = (get_attr_mode (insn) == MODE_V4SF) ? "andnps" : "pandn";
2975 switch (which_alternative)
2978 ops = "%s\t{%%2, %%0|%%0, %%2}";
2981 ops = "v%s\t{%%2, %%1, %%0|%%0, %%1, %%2}";
2987 snprintf (buf, sizeof (buf), ops, tmp);
2990 [(set_attr "isa" "noavx,avx")
2991 (set_attr "type" "sselog")
2992 (set (attr "prefix_data16")
2994 (and (eq_attr "alternative" "0")
2995 (eq_attr "mode" "TI"))
2997 (const_string "*")))
2998 (set_attr "prefix" "orig,vex")
3000 (cond [(match_test "TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL")
3001 (const_string "V4SF")
3002 (match_test "TARGET_AVX")
3004 (ior (not (match_test "TARGET_SSE2"))
3005 (match_test "optimize_function_for_size_p (cfun)"))
3006 (const_string "V4SF")
3008 (const_string "TI")))])
3010 (define_insn "*<code><mode>3"
3011 [(set (match_operand:MODEF 0 "register_operand" "=x,x")
3013 (match_operand:MODEF 1 "register_operand" "%0,x")
3014 (match_operand:MODEF 2 "register_operand" "x,x")))]
3015 "SSE_FLOAT_MODE_P (<MODE>mode)"
3017 static char buf[32];
3020 = (get_attr_mode (insn) == MODE_V4SF) ? "ps" : "<ssevecmodesuffix>";
3022 switch (which_alternative)
3025 ops = "<logic>%s\t{%%2, %%0|%%0, %%2}";
3028 ops = "v<logic>%s\t{%%2, %%1, %%0|%%0, %%1, %%2}";
3034 snprintf (buf, sizeof (buf), ops, suffix);
3037 [(set_attr "isa" "noavx,avx")
3038 (set_attr "type" "sselog")
3039 (set_attr "prefix" "orig,vex")
3041 (cond [(and (match_test "<MODE_SIZE> == 16")
3042 (match_test "TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL"))
3043 (const_string "V4SF")
3044 (match_test "TARGET_AVX")
3045 (const_string "<ssevecmode>")
3046 (match_test "optimize_function_for_size_p (cfun)")
3047 (const_string "V4SF")
3049 (const_string "<ssevecmode>")))])
3051 (define_expand "<code>tf3"
3052 [(set (match_operand:TF 0 "register_operand")
3054 (match_operand:TF 1 "nonimmediate_operand")
3055 (match_operand:TF 2 "nonimmediate_operand")))]
3057 "ix86_fixup_binary_operands_no_copy (<CODE>, TFmode, operands);")
3059 (define_insn "*<code>tf3"
3060 [(set (match_operand:TF 0 "register_operand" "=x,x")
3062 (match_operand:TF 1 "nonimmediate_operand" "%0,x")
3063 (match_operand:TF 2 "nonimmediate_operand" "xm,xm")))]
3065 && ix86_binary_operator_ok (<CODE>, TFmode, operands)"
3067 static char buf[32];
3070 = (get_attr_mode (insn) == MODE_V4SF) ? "<logic>ps" : "p<logic>";
3072 switch (which_alternative)
3075 ops = "%s\t{%%2, %%0|%%0, %%2}";
3078 ops = "v%s\t{%%2, %%1, %%0|%%0, %%1, %%2}";
3084 snprintf (buf, sizeof (buf), ops, tmp);
3087 [(set_attr "isa" "noavx,avx")
3088 (set_attr "type" "sselog")
3089 (set (attr "prefix_data16")
3091 (and (eq_attr "alternative" "0")
3092 (eq_attr "mode" "TI"))
3094 (const_string "*")))
3095 (set_attr "prefix" "orig,vex")
3097 (cond [(match_test "TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL")
3098 (const_string "V4SF")
3099 (match_test "TARGET_AVX")
3101 (ior (not (match_test "TARGET_SSE2"))
3102 (match_test "optimize_function_for_size_p (cfun)"))
3103 (const_string "V4SF")
3105 (const_string "TI")))])
3107 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
3109 ;; FMA floating point multiply/accumulate instructions. These include
3110 ;; scalar versions of the instructions as well as vector versions.
3112 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
3114 ;; The standard names for scalar FMA are only available with SSE math enabled.
3115 ;; CPUID bit AVX512F enables evex encoded scalar and 512-bit fma. It doesn't
3116 ;; care about FMA bit, so we enable fma for TARGET_AVX512F even when TARGET_FMA
3117 ;; and TARGET_FMA4 are both false.
3118 ;; TODO: In theory AVX512F does not automatically imply FMA, and without FMA
3119 ;; one must force the EVEX encoding of the fma insns. Ideally we'd improve
3120 ;; GAS to allow proper prefix selection. However, for the moment all hardware
3121 ;; that supports AVX512F also supports FMA so we can ignore this for now.
3122 (define_mode_iterator FMAMODEM
3123 [(SF "TARGET_SSE_MATH && (TARGET_FMA || TARGET_FMA4 || TARGET_AVX512F)")
3124 (DF "TARGET_SSE_MATH && (TARGET_FMA || TARGET_FMA4 || TARGET_AVX512F)")
3125 (V4SF "TARGET_FMA || TARGET_FMA4 || TARGET_AVX512VL")
3126 (V2DF "TARGET_FMA || TARGET_FMA4 || TARGET_AVX512VL")
3127 (V8SF "TARGET_FMA || TARGET_FMA4 || TARGET_AVX512VL")
3128 (V4DF "TARGET_FMA || TARGET_FMA4 || TARGET_AVX512VL")
3129 (V16SF "TARGET_AVX512F")
3130 (V8DF "TARGET_AVX512F")])
3132 (define_expand "fma<mode>4"
3133 [(set (match_operand:FMAMODEM 0 "register_operand")
3135 (match_operand:FMAMODEM 1 "nonimmediate_operand")
3136 (match_operand:FMAMODEM 2 "nonimmediate_operand")
3137 (match_operand:FMAMODEM 3 "nonimmediate_operand")))])
3139 (define_expand "fms<mode>4"
3140 [(set (match_operand:FMAMODEM 0 "register_operand")
3142 (match_operand:FMAMODEM 1 "nonimmediate_operand")
3143 (match_operand:FMAMODEM 2 "nonimmediate_operand")
3144 (neg:FMAMODEM (match_operand:FMAMODEM 3 "nonimmediate_operand"))))])
3146 (define_expand "fnma<mode>4"
3147 [(set (match_operand:FMAMODEM 0 "register_operand")
3149 (neg:FMAMODEM (match_operand:FMAMODEM 1 "nonimmediate_operand"))
3150 (match_operand:FMAMODEM 2 "nonimmediate_operand")
3151 (match_operand:FMAMODEM 3 "nonimmediate_operand")))])
3153 (define_expand "fnms<mode>4"
3154 [(set (match_operand:FMAMODEM 0 "register_operand")
3156 (neg:FMAMODEM (match_operand:FMAMODEM 1 "nonimmediate_operand"))
3157 (match_operand:FMAMODEM 2 "nonimmediate_operand")
3158 (neg:FMAMODEM (match_operand:FMAMODEM 3 "nonimmediate_operand"))))])
3160 ;; The builtins for intrinsics are not constrained by SSE math enabled.
3161 (define_mode_iterator FMAMODE_AVX512
3162 [(SF "TARGET_FMA || TARGET_FMA4 || TARGET_AVX512F")
3163 (DF "TARGET_FMA || TARGET_FMA4 || TARGET_AVX512F")
3164 (V4SF "TARGET_FMA || TARGET_FMA4 || TARGET_AVX512VL")
3165 (V2DF "TARGET_FMA || TARGET_FMA4 || TARGET_AVX512VL")
3166 (V8SF "TARGET_FMA || TARGET_FMA4 || TARGET_AVX512VL")
3167 (V4DF "TARGET_FMA || TARGET_FMA4 || TARGET_AVX512VL")
3168 (V16SF "TARGET_AVX512F")
3169 (V8DF "TARGET_AVX512F")])
3171 (define_mode_iterator FMAMODE
3172 [SF DF V4SF V2DF V8SF V4DF])
3174 (define_expand "fma4i_fmadd_<mode>"
3175 [(set (match_operand:FMAMODE_AVX512 0 "register_operand")
3177 (match_operand:FMAMODE_AVX512 1 "nonimmediate_operand")
3178 (match_operand:FMAMODE_AVX512 2 "nonimmediate_operand")
3179 (match_operand:FMAMODE_AVX512 3 "nonimmediate_operand")))])
3181 (define_expand "<avx512>_fmadd_<mode>_maskz<round_expand_name>"
3182 [(match_operand:VF_AVX512VL 0 "register_operand")
3183 (match_operand:VF_AVX512VL 1 "<round_expand_nimm_predicate>")
3184 (match_operand:VF_AVX512VL 2 "<round_expand_nimm_predicate>")
3185 (match_operand:VF_AVX512VL 3 "<round_expand_nimm_predicate>")
3186 (match_operand:<avx512fmaskmode> 4 "register_operand")]
3187 "TARGET_AVX512F && <round_mode512bit_condition>"
3189 emit_insn (gen_fma_fmadd_<mode>_maskz_1<round_expand_name> (
3190 operands[0], operands[1], operands[2], operands[3],
3191 CONST0_RTX (<MODE>mode), operands[4]<round_expand_operand>));
3195 (define_insn "*fma_fmadd_<mode>"
3196 [(set (match_operand:FMAMODE 0 "register_operand" "=v,v,v,x,x")
3198 (match_operand:FMAMODE 1 "nonimmediate_operand" "%0,0,v,x,x")
3199 (match_operand:FMAMODE 2 "nonimmediate_operand" "vm,v,vm,x,m")
3200 (match_operand:FMAMODE 3 "nonimmediate_operand" "v,vm,0,xm,x")))]
3201 "TARGET_FMA || TARGET_FMA4"
3203 vfmadd132<ssemodesuffix>\t{%2, %3, %0|%0, %3, %2}
3204 vfmadd213<ssemodesuffix>\t{%3, %2, %0|%0, %2, %3}
3205 vfmadd231<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}
3206 vfmadd<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}
3207 vfmadd<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
3208 [(set_attr "isa" "fma,fma,fma,fma4,fma4")
3209 (set_attr "type" "ssemuladd")
3210 (set_attr "mode" "<MODE>")])
3212 ;; Suppose AVX-512F as baseline
3213 (define_mode_iterator VF_SF_AVX512VL
3214 [SF V16SF (V8SF "TARGET_AVX512VL") (V4SF "TARGET_AVX512VL")
3215 DF V8DF (V4DF "TARGET_AVX512VL") (V2DF "TARGET_AVX512VL")])
3217 (define_insn "<sd_mask_codefor>fma_fmadd_<mode><sd_maskz_name><round_name>"
3218 [(set (match_operand:VF_SF_AVX512VL 0 "register_operand" "=v,v,v")
3220 (match_operand:VF_SF_AVX512VL 1 "<round_nimm_predicate>" "%0,0,v")
3221 (match_operand:VF_SF_AVX512VL 2 "<round_nimm_predicate>" "<round_constraint>,v,<round_constraint>")
3222 (match_operand:VF_SF_AVX512VL 3 "<round_nimm_predicate>" "v,<round_constraint>,0")))]
3223 "TARGET_AVX512F && <sd_mask_mode512bit_condition> && <round_mode512bit_condition>"
3225 vfmadd132<ssemodesuffix>\t{<round_sd_mask_op4>%2, %3, %0<sd_mask_op4>|%0<sd_mask_op4>, %3, %2<round_sd_mask_op4>}
3226 vfmadd213<ssemodesuffix>\t{<round_sd_mask_op4>%3, %2, %0<sd_mask_op4>|%0<sd_mask_op4>, %2, %3<round_sd_mask_op4>}
3227 vfmadd231<ssemodesuffix>\t{<round_sd_mask_op4>%2, %1, %0<sd_mask_op4>|%0<sd_mask_op4>, %1, %2<round_sd_mask_op4>}"
3228 [(set_attr "type" "ssemuladd")
3229 (set_attr "mode" "<MODE>")])
3231 (define_insn "<avx512>_fmadd_<mode>_mask<round_name>"
3232 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v,v")
3233 (vec_merge:VF_AVX512VL
3235 (match_operand:VF_AVX512VL 1 "register_operand" "0,0")
3236 (match_operand:VF_AVX512VL 2 "nonimmediate_operand" "<round_constraint>,v")
3237 (match_operand:VF_AVX512VL 3 "nonimmediate_operand" "v,<round_constraint>"))
3239 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk,Yk")))]
3240 "TARGET_AVX512F && <round_mode512bit_condition>"
3242 vfmadd132<ssemodesuffix>\t{<round_op5>%2, %3, %0%{%4%}|%0%{%4%}, %3, %2<round_op5>}
3243 vfmadd213<ssemodesuffix>\t{<round_op5>%3, %2, %0%{%4%}|%0%{%4%}, %2, %3<round_op5>}"
3244 [(set_attr "isa" "fma_avx512f,fma_avx512f")
3245 (set_attr "type" "ssemuladd")
3246 (set_attr "mode" "<MODE>")])
3248 (define_insn "<avx512>_fmadd_<mode>_mask3<round_name>"
3249 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=x")
3250 (vec_merge:VF_AVX512VL
3252 (match_operand:VF_AVX512VL 1 "register_operand" "x")
3253 (match_operand:VF_AVX512VL 2 "nonimmediate_operand" "<round_constraint>")
3254 (match_operand:VF_AVX512VL 3 "register_operand" "0"))
3256 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk")))]
3258 "vfmadd231<ssemodesuffix>\t{<round_op5>%2, %1, %0%{%4%}|%0%{%4%}, %1, %2<round_op5>}"
3259 [(set_attr "isa" "fma_avx512f")
3260 (set_attr "type" "ssemuladd")
3261 (set_attr "mode" "<MODE>")])
3263 (define_insn "*fma_fmsub_<mode>"
3264 [(set (match_operand:FMAMODE 0 "register_operand" "=v,v,v,x,x")
3266 (match_operand:FMAMODE 1 "nonimmediate_operand" "%0,0,v,x,x")
3267 (match_operand:FMAMODE 2 "nonimmediate_operand" "vm,v,vm,x,m")
3269 (match_operand:FMAMODE 3 "nonimmediate_operand" "v,vm,0,xm,x"))))]
3270 "TARGET_FMA || TARGET_FMA4"
3272 vfmsub132<ssemodesuffix>\t{%2, %3, %0|%0, %3, %2}
3273 vfmsub213<ssemodesuffix>\t{%3, %2, %0|%0, %2, %3}
3274 vfmsub231<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}
3275 vfmsub<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}
3276 vfmsub<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
3277 [(set_attr "isa" "fma,fma,fma,fma4,fma4")
3278 (set_attr "type" "ssemuladd")
3279 (set_attr "mode" "<MODE>")])
3281 (define_insn "<sd_mask_codefor>fma_fmsub_<mode><sd_maskz_name><round_name>"
3282 [(set (match_operand:VF_SF_AVX512VL 0 "register_operand" "=v,v,v")
3284 (match_operand:VF_SF_AVX512VL 1 "<round_nimm_predicate>" "%0,0,v")
3285 (match_operand:VF_SF_AVX512VL 2 "<round_nimm_predicate>" "<round_constraint>,v,<round_constraint>")
3287 (match_operand:VF_SF_AVX512VL 3 "<round_nimm_predicate>" "v,<round_constraint>,0"))))]
3288 "TARGET_AVX512F && <sd_mask_mode512bit_condition> && <round_mode512bit_condition>"
3290 vfmsub132<ssemodesuffix>\t{<round_sd_mask_op4>%2, %3, %0<sd_mask_op4>|%0<sd_mask_op4>, %3, %2<round_sd_mask_op4>}
3291 vfmsub213<ssemodesuffix>\t{<round_sd_mask_op4>%3, %2, %0<sd_mask_op4>|%0<sd_mask_op4>, %2, %3<round_sd_mask_op4>}
3292 vfmsub231<ssemodesuffix>\t{<round_sd_mask_op4>%2, %1, %0<sd_mask_op4>|%0<sd_mask_op4>, %1, %2<round_sd_mask_op4>}"
3293 [(set_attr "type" "ssemuladd")
3294 (set_attr "mode" "<MODE>")])
3296 (define_insn "<avx512>_fmsub_<mode>_mask<round_name>"
3297 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v,v")
3298 (vec_merge:VF_AVX512VL
3300 (match_operand:VF_AVX512VL 1 "register_operand" "0,0")
3301 (match_operand:VF_AVX512VL 2 "nonimmediate_operand" "<round_constraint>,v")
3303 (match_operand:VF_AVX512VL 3 "nonimmediate_operand" "v,<round_constraint>")))
3305 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk,Yk")))]
3308 vfmsub132<ssemodesuffix>\t{<round_op5>%2, %3, %0%{%4%}|%0%{%4%}, %3, %2<round_op5>}
3309 vfmsub213<ssemodesuffix>\t{<round_op5>%3, %2, %0%{%4%}|%0%{%4%}, %2, %3<round_op5>}"
3310 [(set_attr "isa" "fma_avx512f,fma_avx512f")
3311 (set_attr "type" "ssemuladd")
3312 (set_attr "mode" "<MODE>")])
3314 (define_insn "<avx512>_fmsub_<mode>_mask3<round_name>"
3315 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v")
3316 (vec_merge:VF_AVX512VL
3318 (match_operand:VF_AVX512VL 1 "register_operand" "v")
3319 (match_operand:VF_AVX512VL 2 "nonimmediate_operand" "<round_constraint>")
3321 (match_operand:VF_AVX512VL 3 "register_operand" "0")))
3323 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk")))]
3324 "TARGET_AVX512F && <round_mode512bit_condition>"
3325 "vfmsub231<ssemodesuffix>\t{<round_op5>%2, %1, %0%{%4%}|%0%{%4%}, %1, %2<round_op5>}"
3326 [(set_attr "isa" "fma_avx512f")
3327 (set_attr "type" "ssemuladd")
3328 (set_attr "mode" "<MODE>")])
3330 (define_insn "*fma_fnmadd_<mode>"
3331 [(set (match_operand:FMAMODE 0 "register_operand" "=v,v,v,x,x")
3334 (match_operand:FMAMODE 1 "nonimmediate_operand" "%0,0,v,x,x"))
3335 (match_operand:FMAMODE 2 "nonimmediate_operand" "vm,v,vm,x,m")
3336 (match_operand:FMAMODE 3 "nonimmediate_operand" "v,vm,0,xm,x")))]
3337 "TARGET_FMA || TARGET_FMA4"
3339 vfnmadd132<ssemodesuffix>\t{%2, %3, %0|%0, %3, %2}
3340 vfnmadd213<ssemodesuffix>\t{%3, %2, %0|%0, %2, %3}
3341 vfnmadd231<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}
3342 vfnmadd<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}
3343 vfnmadd<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
3344 [(set_attr "isa" "fma,fma,fma,fma4,fma4")
3345 (set_attr "type" "ssemuladd")
3346 (set_attr "mode" "<MODE>")])
3348 (define_insn "<sd_mask_codefor>fma_fnmadd_<mode><sd_maskz_name><round_name>"
3349 [(set (match_operand:VF_SF_AVX512VL 0 "register_operand" "=v,v,v")
3352 (match_operand:VF_SF_AVX512VL 1 "<round_nimm_predicate>" "%0,0,v"))
3353 (match_operand:VF_SF_AVX512VL 2 "<round_nimm_predicate>" "<round_constraint>,v,<round_constraint>")
3354 (match_operand:VF_SF_AVX512VL 3 "<round_nimm_predicate>" "v,<round_constraint>,0")))]
3355 "TARGET_AVX512F && <sd_mask_mode512bit_condition> && <round_mode512bit_condition>"
3357 vfnmadd132<ssemodesuffix>\t{<round_sd_mask_op4>%2, %3, %0<sd_mask_op4>|%0<sd_mask_op4>, %3, %2<round_sd_mask_op4>}
3358 vfnmadd213<ssemodesuffix>\t{<round_sd_mask_op4>%3, %2, %0<sd_mask_op4>|%0<sd_mask_op4>, %2, %3<round_sd_mask_op4>}
3359 vfnmadd231<ssemodesuffix>\t{<round_sd_mask_op4>%2, %1, %0<sd_mask_op4>|%0<sd_mask_op4>, %1, %2<round_sd_mask_op4>}"
3360 [(set_attr "type" "ssemuladd")
3361 (set_attr "mode" "<MODE>")])
3363 (define_insn "<avx512>_fnmadd_<mode>_mask<round_name>"
3364 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v,v")
3365 (vec_merge:VF_AVX512VL
3368 (match_operand:VF_AVX512VL 1 "register_operand" "0,0"))
3369 (match_operand:VF_AVX512VL 2 "nonimmediate_operand" "<round_constraint>,v")
3370 (match_operand:VF_AVX512VL 3 "nonimmediate_operand" "v,<round_constraint>"))
3372 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk,Yk")))]
3373 "TARGET_AVX512F && <round_mode512bit_condition>"
3375 vfnmadd132<ssemodesuffix>\t{<round_op5>%2, %3, %0%{%4%}|%0%{%4%}, %3, %2<round_op5>}
3376 vfnmadd213<ssemodesuffix>\t{<round_op5>%3, %2, %0%{%4%}|%0%{%4%}, %2, %3<round_op5>}"
3377 [(set_attr "isa" "fma_avx512f,fma_avx512f")
3378 (set_attr "type" "ssemuladd")
3379 (set_attr "mode" "<MODE>")])
3381 (define_insn "<avx512>_fnmadd_<mode>_mask3<round_name>"
3382 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v")
3383 (vec_merge:VF_AVX512VL
3386 (match_operand:VF_AVX512VL 1 "register_operand" "v"))
3387 (match_operand:VF_AVX512VL 2 "nonimmediate_operand" "<round_constraint>")
3388 (match_operand:VF_AVX512VL 3 "register_operand" "0"))
3390 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk")))]
3391 "TARGET_AVX512F && <round_mode512bit_condition>"
3392 "vfnmadd231<ssemodesuffix>\t{<round_op5>%2, %1, %0%{%4%}|%0%{%4%}, %1, %2<round_op5>}"
3393 [(set_attr "isa" "fma_avx512f")
3394 (set_attr "type" "ssemuladd")
3395 (set_attr "mode" "<MODE>")])
3397 (define_insn "*fma_fnmsub_<mode>"
3398 [(set (match_operand:FMAMODE 0 "register_operand" "=v,v,v,x,x")
3401 (match_operand:FMAMODE 1 "nonimmediate_operand" "%0,0,v,x,x"))
3402 (match_operand:FMAMODE 2 "nonimmediate_operand" "vm,v,vm,x,m")
3404 (match_operand:FMAMODE 3 "nonimmediate_operand" "v,vm,0,xm,x"))))]
3405 "TARGET_FMA || TARGET_FMA4"
3407 vfnmsub132<ssemodesuffix>\t{<round_sd_mask_op4>%2, %3, %0<sd_mask_op4>|%0<sd_mask_op4>, %3, %2<round_sd_mask_op4>}
3408 vfnmsub213<ssemodesuffix>\t{<round_sd_mask_op4>%3, %2, %0<sd_mask_op4>|%0<sd_mask_op4>, %2, %3<round_sd_mask_op4>}
3409 vfnmsub231<ssemodesuffix>\t{<round_sd_mask_op4>%2, %1, %0<sd_mask_op4>|%0<sd_mask_op4>, %1, %2<round_sd_mask_op4>}
3410 vfnmsub<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}
3411 vfnmsub<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
3412 [(set_attr "isa" "fma,fma,fma,fma4,fma4")
3413 (set_attr "type" "ssemuladd")
3414 (set_attr "mode" "<MODE>")])
3416 (define_insn "<sd_mask_codefor>fma_fnmsub_<mode><sd_maskz_name><round_name>"
3417 [(set (match_operand:VF_SF_AVX512VL 0 "register_operand" "=v,v,v")
3420 (match_operand:VF_SF_AVX512VL 1 "<round_nimm_predicate>" "%0,0,v"))
3421 (match_operand:VF_SF_AVX512VL 2 "<round_nimm_predicate>" "<round_constraint>,v,<round_constraint>")
3423 (match_operand:VF_SF_AVX512VL 3 "<round_nimm_predicate>" "v,<round_constraint>,0"))))]
3424 "TARGET_AVX512F && <sd_mask_mode512bit_condition> && <round_mode512bit_condition>"
3426 vfnmsub132<ssemodesuffix>\t{<round_sd_mask_op4>%2, %3, %0<sd_mask_op4>|%0<sd_mask_op4>, %3, %2<round_sd_mask_op4>}
3427 vfnmsub213<ssemodesuffix>\t{<round_sd_mask_op4>%3, %2, %0<sd_mask_op4>|%0<sd_mask_op4>, %2, %3<round_sd_mask_op4>}
3428 vfnmsub231<ssemodesuffix>\t{<round_sd_mask_op4>%2, %1, %0<sd_mask_op4>|%0<sd_mask_op4>, %1, %2<round_sd_mask_op4>}"
3429 [(set_attr "type" "ssemuladd")
3430 (set_attr "mode" "<MODE>")])
3432 (define_insn "<avx512>_fnmsub_<mode>_mask<round_name>"
3433 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v,v")
3434 (vec_merge:VF_AVX512VL
3437 (match_operand:VF_AVX512VL 1 "register_operand" "0,0"))
3438 (match_operand:VF_AVX512VL 2 "nonimmediate_operand" "<round_constraint>,v")
3440 (match_operand:VF_AVX512VL 3 "nonimmediate_operand" "v,<round_constraint>")))
3442 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk,Yk")))]
3443 "TARGET_AVX512F && <round_mode512bit_condition>"
3445 vfnmsub132<ssemodesuffix>\t{<round_op5>%2, %3, %0%{%4%}|%0%{%4%}, %3, %2<round_op5>}
3446 vfnmsub213<ssemodesuffix>\t{<round_op5>%3, %2, %0%{%4%}|%0%{%4%}, %2, %3<round_op5>}"
3447 [(set_attr "isa" "fma_avx512f,fma_avx512f")
3448 (set_attr "type" "ssemuladd")
3449 (set_attr "mode" "<MODE>")])
3451 (define_insn "<avx512>_fnmsub_<mode>_mask3<round_name>"
3452 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v")
3453 (vec_merge:VF_AVX512VL
3456 (match_operand:VF_AVX512VL 1 "register_operand" "v"))
3457 (match_operand:VF_AVX512VL 2 "nonimmediate_operand" "<round_constraint>")
3459 (match_operand:VF_AVX512VL 3 "register_operand" "0")))
3461 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk")))]
3463 "vfnmsub231<ssemodesuffix>\t{<round_op5>%2, %1, %0%{%4%}|%0%{%4%}, %1, %2<round_op5>}"
3464 [(set_attr "isa" "fma_avx512f")
3465 (set_attr "type" "ssemuladd")
3466 (set_attr "mode" "<MODE>")])
3468 ;; FMA parallel floating point multiply addsub and subadd operations.
3470 ;; It would be possible to represent these without the UNSPEC as
3473 ;; (fma op1 op2 op3)
3474 ;; (fma op1 op2 (neg op3))
3477 ;; But this doesn't seem useful in practice.
3479 (define_expand "fmaddsub_<mode>"
3480 [(set (match_operand:VF 0 "register_operand")
3482 [(match_operand:VF 1 "nonimmediate_operand")
3483 (match_operand:VF 2 "nonimmediate_operand")
3484 (match_operand:VF 3 "nonimmediate_operand")]
3486 "TARGET_FMA || TARGET_FMA4 || TARGET_AVX512F")
3488 (define_expand "<avx512>_fmaddsub_<mode>_maskz<round_expand_name>"
3489 [(match_operand:VF_AVX512VL 0 "register_operand")
3490 (match_operand:VF_AVX512VL 1 "<round_expand_nimm_predicate>")
3491 (match_operand:VF_AVX512VL 2 "<round_expand_nimm_predicate>")
3492 (match_operand:VF_AVX512VL 3 "<round_expand_nimm_predicate>")
3493 (match_operand:<avx512fmaskmode> 4 "register_operand")]
3496 emit_insn (gen_fma_fmaddsub_<mode>_maskz_1<round_expand_name> (
3497 operands[0], operands[1], operands[2], operands[3],
3498 CONST0_RTX (<MODE>mode), operands[4]<round_expand_operand>));
3502 (define_insn "*fma_fmaddsub_<mode>"
3503 [(set (match_operand:VF_128_256 0 "register_operand" "=v,v,v,x,x")
3505 [(match_operand:VF_128_256 1 "nonimmediate_operand" "%0,0,v,x,x")
3506 (match_operand:VF_128_256 2 "nonimmediate_operand" "vm,v,vm,x,m")
3507 (match_operand:VF_128_256 3 "nonimmediate_operand" "v,vm,0,xm,x")]
3509 "TARGET_FMA || TARGET_FMA4"
3511 vfmaddsub132<ssemodesuffix>\t{%2, %3, %0|%0, %3, %2}
3512 vfmaddsub213<ssemodesuffix>\t{%3, %2, %0|%0, %2, %3}
3513 vfmaddsub231<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}
3514 vfmaddsub<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}
3515 vfmaddsub<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
3516 [(set_attr "isa" "fma,fma,fma,fma4,fma4")
3517 (set_attr "type" "ssemuladd")
3518 (set_attr "mode" "<MODE>")])
3520 (define_insn "<sd_mask_codefor>fma_fmaddsub_<mode><sd_maskz_name><round_name>"
3521 [(set (match_operand:VF_SF_AVX512VL 0 "register_operand" "=v,v,v")
3522 (unspec:VF_SF_AVX512VL
3523 [(match_operand:VF_SF_AVX512VL 1 "<round_nimm_predicate>" "%0,0,v")
3524 (match_operand:VF_SF_AVX512VL 2 "<round_nimm_predicate>" "<round_constraint>,v,<round_constraint>")
3525 (match_operand:VF_SF_AVX512VL 3 "<round_nimm_predicate>" "v,<round_constraint>,0")]
3527 "TARGET_AVX512F && <sd_mask_mode512bit_condition> && <round_mode512bit_condition>"
3529 vfmaddsub132<ssemodesuffix>\t{<round_sd_mask_op4>%2, %3, %0<sd_mask_op4>|%0<sd_mask_op4>, %3, %2<round_sd_mask_op4>}
3530 vfmaddsub213<ssemodesuffix>\t{<round_sd_mask_op4>%3, %2, %0<sd_mask_op4>|%0<sd_mask_op4>, %2, %3<round_sd_mask_op4>}
3531 vfmaddsub231<ssemodesuffix>\t{<round_sd_mask_op4>%2, %1, %0<sd_mask_op4>|%0<sd_mask_op4>, %1, %2<round_sd_mask_op4>}"
3532 [(set_attr "type" "ssemuladd")
3533 (set_attr "mode" "<MODE>")])
3535 (define_insn "<avx512>_fmaddsub_<mode>_mask<round_name>"
3536 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v,v")
3537 (vec_merge:VF_AVX512VL
3539 [(match_operand:VF_AVX512VL 1 "register_operand" "0,0")
3540 (match_operand:VF_AVX512VL 2 "nonimmediate_operand" "<round_constraint>,v")
3541 (match_operand:VF_AVX512VL 3 "nonimmediate_operand" "v,<round_constraint>")]
3544 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk,Yk")))]
3547 vfmaddsub132<ssemodesuffix>\t{<round_op5>%2, %3, %0%{%4%}|%0%{%4%}, %3, %2<round_op5>}
3548 vfmaddsub213<ssemodesuffix>\t{<round_op5>%3, %2, %0%{%4%}|%0%{%4%}, %2, %3<round_op5>}"
3549 [(set_attr "isa" "fma_avx512f,fma_avx512f")
3550 (set_attr "type" "ssemuladd")
3551 (set_attr "mode" "<MODE>")])
3553 (define_insn "<avx512>_fmaddsub_<mode>_mask3<round_name>"
3554 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v")
3555 (vec_merge:VF_AVX512VL
3557 [(match_operand:VF_AVX512VL 1 "register_operand" "v")
3558 (match_operand:VF_AVX512VL 2 "nonimmediate_operand" "<round_constraint>")
3559 (match_operand:VF_AVX512VL 3 "register_operand" "0")]
3562 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk")))]
3564 "vfmaddsub231<ssemodesuffix>\t{<round_op5>%2, %1, %0%{%4%}|%0%{%4%}, %1, %2<round_op5>}"
3565 [(set_attr "isa" "fma_avx512f")
3566 (set_attr "type" "ssemuladd")
3567 (set_attr "mode" "<MODE>")])
3569 (define_insn "*fma_fmsubadd_<mode>"
3570 [(set (match_operand:VF_128_256 0 "register_operand" "=v,v,v,x,x")
3572 [(match_operand:VF_128_256 1 "nonimmediate_operand" "%0,0,v,x,x")
3573 (match_operand:VF_128_256 2 "nonimmediate_operand" "vm,v,vm,x,m")
3575 (match_operand:VF_128_256 3 "nonimmediate_operand" "v,vm,0,xm,x"))]
3577 "TARGET_FMA || TARGET_FMA4"
3579 vfmsubadd132<ssemodesuffix>\t{%2, %3, %0|%0, %3, %2}
3580 vfmsubadd213<ssemodesuffix>\t{%3, %2, %0|%0, %2, %3}
3581 vfmsubadd231<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}
3582 vfmsubadd<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}
3583 vfmsubadd<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
3584 [(set_attr "isa" "fma,fma,fma,fma4,fma4")
3585 (set_attr "type" "ssemuladd")
3586 (set_attr "mode" "<MODE>")])
3588 (define_insn "<sd_mask_codefor>fma_fmsubadd_<mode><sd_maskz_name><round_name>"
3589 [(set (match_operand:VF_SF_AVX512VL 0 "register_operand" "=v,v,v")
3590 (unspec:VF_SF_AVX512VL
3591 [(match_operand:VF_SF_AVX512VL 1 "<round_nimm_predicate>" "%0,0,v")
3592 (match_operand:VF_SF_AVX512VL 2 "<round_nimm_predicate>" "<round_constraint>,v,<round_constraint>")
3594 (match_operand:VF_SF_AVX512VL 3 "<round_nimm_predicate>" "v,<round_constraint>,0"))]
3596 "TARGET_AVX512F && <sd_mask_mode512bit_condition> && <round_mode512bit_condition>"
3598 vfmsubadd132<ssemodesuffix>\t{<round_sd_mask_op4>%2, %3, %0<sd_mask_op4>|%0<sd_mask_op4>, %3, %2<round_sd_mask_op4>}
3599 vfmsubadd213<ssemodesuffix>\t{<round_sd_mask_op4>%3, %2, %0<sd_mask_op4>|%0<sd_mask_op4>, %2, %3<round_sd_mask_op4>}
3600 vfmsubadd231<ssemodesuffix>\t{<round_sd_mask_op4>%2, %1, %0<sd_mask_op4>|%0<sd_mask_op4>, %1, %2<round_sd_mask_op4>}"
3601 [(set_attr "type" "ssemuladd")
3602 (set_attr "mode" "<MODE>")])
3604 (define_insn "<avx512>_fmsubadd_<mode>_mask<round_name>"
3605 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v,v")
3606 (vec_merge:VF_AVX512VL
3608 [(match_operand:VF_AVX512VL 1 "register_operand" "0,0")
3609 (match_operand:VF_AVX512VL 2 "nonimmediate_operand" "<round_constraint>,v")
3611 (match_operand:VF_AVX512VL 3 "nonimmediate_operand" "v,<round_constraint>"))]
3614 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk,Yk")))]
3617 vfmsubadd132<ssemodesuffix>\t{<round_op5>%2, %3, %0%{%4%}|%0%{%4%}, %3, %2<round_op5>}
3618 vfmsubadd213<ssemodesuffix>\t{<round_op5>%3, %2, %0%{%4%}|%0%{%4%}, %2, %3<round_op5>}"
3619 [(set_attr "isa" "fma_avx512f,fma_avx512f")
3620 (set_attr "type" "ssemuladd")
3621 (set_attr "mode" "<MODE>")])
3623 (define_insn "<avx512>_fmsubadd_<mode>_mask3<round_name>"
3624 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v")
3625 (vec_merge:VF_AVX512VL
3627 [(match_operand:VF_AVX512VL 1 "register_operand" "v")
3628 (match_operand:VF_AVX512VL 2 "nonimmediate_operand" "<round_constraint>")
3630 (match_operand:VF_AVX512VL 3 "register_operand" "0"))]
3633 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk")))]
3635 "vfmsubadd231<ssemodesuffix>\t{<round_op5>%2, %1, %0%{%4%}|%0%{%4%}, %1, %2<round_op5>}"
3636 [(set_attr "isa" "fma_avx512f")
3637 (set_attr "type" "ssemuladd")
3638 (set_attr "mode" "<MODE>")])
3640 ;; FMA3 floating point scalar intrinsics. These merge result with
3641 ;; high-order elements from the destination register.
3643 (define_expand "fmai_vmfmadd_<mode><round_name>"
3644 [(set (match_operand:VF_128 0 "register_operand")
3647 (match_operand:VF_128 1 "<round_nimm_predicate>")
3648 (match_operand:VF_128 2 "<round_nimm_predicate>")
3649 (match_operand:VF_128 3 "<round_nimm_predicate>"))
3654 (define_insn "*fmai_fmadd_<mode>"
3655 [(set (match_operand:VF_128 0 "register_operand" "=v,v")
3658 (match_operand:VF_128 1 "<round_nimm_predicate>" " 0, 0")
3659 (match_operand:VF_128 2 "<round_nimm_predicate>" "<round_constraint>, v")
3660 (match_operand:VF_128 3 "<round_nimm_predicate>" " v,<round_constraint>"))
3663 "TARGET_FMA || TARGET_AVX512F"
3665 vfmadd132<ssescalarmodesuffix>\t{<round_op4>%2, %3, %0|%0, %<iptr>3, %<iptr>2<round_op4>}
3666 vfmadd213<ssescalarmodesuffix>\t{<round_op4>%3, %2, %0|%0, %<iptr>2, %<iptr>3<round_op4>}"
3667 [(set_attr "type" "ssemuladd")
3668 (set_attr "mode" "<MODE>")])
3670 (define_insn "*fmai_fmsub_<mode>"
3671 [(set (match_operand:VF_128 0 "register_operand" "=v,v")
3674 (match_operand:VF_128 1 "<round_nimm_predicate>" "0,0")
3675 (match_operand:VF_128 2 "<round_nimm_predicate>" "<round_constraint>,v")
3677 (match_operand:VF_128 3 "<round_nimm_predicate>" " v,<round_constraint>")))
3680 "TARGET_FMA || TARGET_AVX512F"
3682 vfmsub132<ssescalarmodesuffix>\t{<round_op4>%2, %3, %0|%0, %<iptr>3, %<iptr>2<round_op4>}
3683 vfmsub213<ssescalarmodesuffix>\t{<round_op4>%3, %2, %0|%0, %<iptr>2, %<iptr>3<round_op4>}"
3684 [(set_attr "type" "ssemuladd")
3685 (set_attr "mode" "<MODE>")])
3687 (define_insn "*fmai_fnmadd_<mode><round_name>"
3688 [(set (match_operand:VF_128 0 "register_operand" "=v,v")
3692 (match_operand:VF_128 2 "<round_nimm_predicate>" "<round_constraint>,v"))
3693 (match_operand:VF_128 1 "<round_nimm_predicate>" "0,0")
3694 (match_operand:VF_128 3 "<round_nimm_predicate>" "v,<round_constraint>"))
3697 "TARGET_FMA || TARGET_AVX512F"
3699 vfnmadd132<ssescalarmodesuffix>\t{<round_op4>%2, %3, %0|%0, %<iptr>3, %<iptr>2<round_op4>}
3700 vfnmadd213<ssescalarmodesuffix>\t{<round_op4>%3, %2, %0|%0, %<iptr>2, %<iptr>3<round_op4>}"
3701 [(set_attr "type" "ssemuladd")
3702 (set_attr "mode" "<MODE>")])
3704 (define_insn "*fmai_fnmsub_<mode><round_name>"
3705 [(set (match_operand:VF_128 0 "register_operand" "=v,v")
3709 (match_operand:VF_128 2 "<round_nimm_predicate>" "<round_constraint>, v"))
3710 (match_operand:VF_128 1 "<round_nimm_predicate>" " 0, 0")
3712 (match_operand:VF_128 3 "<round_nimm_predicate>" " v,<round_constraint>")))
3715 "TARGET_FMA || TARGET_AVX512F"
3717 vfnmsub132<ssescalarmodesuffix>\t{<round_op4>%2, %3, %0|%0, %<iptr>3, %<iptr>2<round_op4>}
3718 vfnmsub213<ssescalarmodesuffix>\t{<round_op4>%3, %2, %0|%0, %<iptr>2, %<iptr>3<round_op4>}"
3719 [(set_attr "type" "ssemuladd")
3720 (set_attr "mode" "<MODE>")])
3722 ;; FMA4 floating point scalar intrinsics. These write the
3723 ;; entire destination register, with the high-order elements zeroed.
3725 (define_expand "fma4i_vmfmadd_<mode>"
3726 [(set (match_operand:VF_128 0 "register_operand")
3729 (match_operand:VF_128 1 "nonimmediate_operand")
3730 (match_operand:VF_128 2 "nonimmediate_operand")
3731 (match_operand:VF_128 3 "nonimmediate_operand"))
3735 "operands[4] = CONST0_RTX (<MODE>mode);")
3737 (define_insn "*fma4i_vmfmadd_<mode>"
3738 [(set (match_operand:VF_128 0 "register_operand" "=x,x")
3741 (match_operand:VF_128 1 "nonimmediate_operand" "%x,x")
3742 (match_operand:VF_128 2 "nonimmediate_operand" " x,m")
3743 (match_operand:VF_128 3 "nonimmediate_operand" "xm,x"))
3744 (match_operand:VF_128 4 "const0_operand")
3747 "vfmadd<ssescalarmodesuffix>\t{%3, %2, %1, %0|%0, %1, %<iptr>2, %<iptr>3}"
3748 [(set_attr "type" "ssemuladd")
3749 (set_attr "mode" "<MODE>")])
3751 (define_insn "*fma4i_vmfmsub_<mode>"
3752 [(set (match_operand:VF_128 0 "register_operand" "=x,x")
3755 (match_operand:VF_128 1 "nonimmediate_operand" "%x,x")
3756 (match_operand:VF_128 2 "nonimmediate_operand" " x,m")
3758 (match_operand:VF_128 3 "nonimmediate_operand" "xm,x")))
3759 (match_operand:VF_128 4 "const0_operand")
3762 "vfmsub<ssescalarmodesuffix>\t{%3, %2, %1, %0|%0, %1, %<iptr>2, %<iptr>3}"
3763 [(set_attr "type" "ssemuladd")
3764 (set_attr "mode" "<MODE>")])
3766 (define_insn "*fma4i_vmfnmadd_<mode>"
3767 [(set (match_operand:VF_128 0 "register_operand" "=x,x")
3771 (match_operand:VF_128 1 "nonimmediate_operand" "%x,x"))
3772 (match_operand:VF_128 2 "nonimmediate_operand" " x,m")
3773 (match_operand:VF_128 3 "nonimmediate_operand" "xm,x"))
3774 (match_operand:VF_128 4 "const0_operand")
3777 "vfnmadd<ssescalarmodesuffix>\t{%3, %2, %1, %0|%0, %1, %<iptr>2, %<iptr>3}"
3778 [(set_attr "type" "ssemuladd")
3779 (set_attr "mode" "<MODE>")])
3781 (define_insn "*fma4i_vmfnmsub_<mode>"
3782 [(set (match_operand:VF_128 0 "register_operand" "=x,x")
3786 (match_operand:VF_128 1 "nonimmediate_operand" "%x,x"))
3787 (match_operand:VF_128 2 "nonimmediate_operand" " x,m")
3789 (match_operand:VF_128 3 "nonimmediate_operand" "xm,x")))
3790 (match_operand:VF_128 4 "const0_operand")
3793 "vfnmsub<ssescalarmodesuffix>\t{%3, %2, %1, %0|%0, %1, %<iptr>2, %<iptr>3}"
3794 [(set_attr "type" "ssemuladd")
3795 (set_attr "mode" "<MODE>")])
3797 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
3799 ;; Parallel single-precision floating point conversion operations
3801 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
3803 (define_insn "sse_cvtpi2ps"
3804 [(set (match_operand:V4SF 0 "register_operand" "=x")
3807 (float:V2SF (match_operand:V2SI 2 "nonimmediate_operand" "ym")))
3808 (match_operand:V4SF 1 "register_operand" "0")
3811 "cvtpi2ps\t{%2, %0|%0, %2}"
3812 [(set_attr "type" "ssecvt")
3813 (set_attr "mode" "V4SF")])
3815 (define_insn "sse_cvtps2pi"
3816 [(set (match_operand:V2SI 0 "register_operand" "=y")
3818 (unspec:V4SI [(match_operand:V4SF 1 "nonimmediate_operand" "xm")]
3820 (parallel [(const_int 0) (const_int 1)])))]
3822 "cvtps2pi\t{%1, %0|%0, %q1}"
3823 [(set_attr "type" "ssecvt")
3824 (set_attr "unit" "mmx")
3825 (set_attr "mode" "DI")])
3827 (define_insn "sse_cvttps2pi"
3828 [(set (match_operand:V2SI 0 "register_operand" "=y")
3830 (fix:V4SI (match_operand:V4SF 1 "nonimmediate_operand" "xm"))
3831 (parallel [(const_int 0) (const_int 1)])))]
3833 "cvttps2pi\t{%1, %0|%0, %q1}"
3834 [(set_attr "type" "ssecvt")
3835 (set_attr "unit" "mmx")
3836 (set_attr "prefix_rep" "0")
3837 (set_attr "mode" "SF")])
3839 (define_insn "sse_cvtsi2ss<round_name>"
3840 [(set (match_operand:V4SF 0 "register_operand" "=x,x,v")
3843 (float:SF (match_operand:SI 2 "<round_nimm_predicate>" "r,m,<round_constraint3>")))
3844 (match_operand:V4SF 1 "register_operand" "0,0,v")
3848 cvtsi2ss\t{%2, %0|%0, %2}
3849 cvtsi2ss\t{%2, %0|%0, %2}
3850 vcvtsi2ss\t{<round_op3>%2, %1, %0|%0, %1, %2<round_op3>}"
3851 [(set_attr "isa" "noavx,noavx,avx")
3852 (set_attr "type" "sseicvt")
3853 (set_attr "athlon_decode" "vector,double,*")
3854 (set_attr "amdfam10_decode" "vector,double,*")
3855 (set_attr "bdver1_decode" "double,direct,*")
3856 (set_attr "btver2_decode" "double,double,double")
3857 (set_attr "prefix" "orig,orig,maybe_evex")
3858 (set_attr "mode" "SF")])
3860 (define_insn "sse_cvtsi2ssq<round_name>"
3861 [(set (match_operand:V4SF 0 "register_operand" "=x,x,v")
3864 (float:SF (match_operand:DI 2 "<round_nimm_predicate>" "r,m,<round_constraint3>")))
3865 (match_operand:V4SF 1 "register_operand" "0,0,v")
3867 "TARGET_SSE && TARGET_64BIT"
3869 cvtsi2ssq\t{%2, %0|%0, %2}
3870 cvtsi2ssq\t{%2, %0|%0, %2}
3871 vcvtsi2ssq\t{<round_op3>%2, %1, %0|%0, %1, %2<round_op3>}"
3872 [(set_attr "isa" "noavx,noavx,avx")
3873 (set_attr "type" "sseicvt")
3874 (set_attr "athlon_decode" "vector,double,*")
3875 (set_attr "amdfam10_decode" "vector,double,*")
3876 (set_attr "bdver1_decode" "double,direct,*")
3877 (set_attr "btver2_decode" "double,double,double")
3878 (set_attr "length_vex" "*,*,4")
3879 (set_attr "prefix_rex" "1,1,*")
3880 (set_attr "prefix" "orig,orig,maybe_evex")
3881 (set_attr "mode" "SF")])
3883 (define_insn "sse_cvtss2si<round_name>"
3884 [(set (match_operand:SI 0 "register_operand" "=r,r")
3887 (match_operand:V4SF 1 "<round_nimm_predicate>" "v,<round_constraint2>")
3888 (parallel [(const_int 0)]))]
3889 UNSPEC_FIX_NOTRUNC))]
3891 "%vcvtss2si\t{<round_op2>%1, %0|%0, %k1<round_op2>}"
3892 [(set_attr "type" "sseicvt")
3893 (set_attr "athlon_decode" "double,vector")
3894 (set_attr "bdver1_decode" "double,double")
3895 (set_attr "prefix_rep" "1")
3896 (set_attr "prefix" "maybe_vex")
3897 (set_attr "mode" "SI")])
3899 (define_insn "sse_cvtss2si_2"
3900 [(set (match_operand:SI 0 "register_operand" "=r,r")
3901 (unspec:SI [(match_operand:SF 1 "nonimmediate_operand" "v,m")]
3902 UNSPEC_FIX_NOTRUNC))]
3904 "%vcvtss2si\t{%1, %0|%0, %k1}"
3905 [(set_attr "type" "sseicvt")
3906 (set_attr "athlon_decode" "double,vector")
3907 (set_attr "amdfam10_decode" "double,double")
3908 (set_attr "bdver1_decode" "double,double")
3909 (set_attr "prefix_rep" "1")
3910 (set_attr "prefix" "maybe_vex")
3911 (set_attr "mode" "SI")])
3913 (define_insn "sse_cvtss2siq<round_name>"
3914 [(set (match_operand:DI 0 "register_operand" "=r,r")
3917 (match_operand:V4SF 1 "<round_nimm_predicate>" "v,<round_constraint2>")
3918 (parallel [(const_int 0)]))]
3919 UNSPEC_FIX_NOTRUNC))]
3920 "TARGET_SSE && TARGET_64BIT"
3921 "%vcvtss2si{q}\t{<round_op2>%1, %0|%0, %k1<round_op2>}"
3922 [(set_attr "type" "sseicvt")
3923 (set_attr "athlon_decode" "double,vector")
3924 (set_attr "bdver1_decode" "double,double")
3925 (set_attr "prefix_rep" "1")
3926 (set_attr "prefix" "maybe_vex")
3927 (set_attr "mode" "DI")])
3929 (define_insn "sse_cvtss2siq_2"
3930 [(set (match_operand:DI 0 "register_operand" "=r,r")
3931 (unspec:DI [(match_operand:SF 1 "nonimmediate_operand" "v,m")]
3932 UNSPEC_FIX_NOTRUNC))]
3933 "TARGET_SSE && TARGET_64BIT"
3934 "%vcvtss2si{q}\t{%1, %0|%0, %k1}"
3935 [(set_attr "type" "sseicvt")
3936 (set_attr "athlon_decode" "double,vector")
3937 (set_attr "amdfam10_decode" "double,double")
3938 (set_attr "bdver1_decode" "double,double")
3939 (set_attr "prefix_rep" "1")
3940 (set_attr "prefix" "maybe_vex")
3941 (set_attr "mode" "DI")])
3943 (define_insn "sse_cvttss2si<round_saeonly_name>"
3944 [(set (match_operand:SI 0 "register_operand" "=r,r")
3947 (match_operand:V4SF 1 "<round_saeonly_nimm_predicate>" "v,<round_saeonly_constraint2>")
3948 (parallel [(const_int 0)]))))]
3950 "%vcvttss2si\t{<round_saeonly_op2>%1, %0|%0, %k1<round_saeonly_op2>}"
3951 [(set_attr "type" "sseicvt")
3952 (set_attr "athlon_decode" "double,vector")
3953 (set_attr "amdfam10_decode" "double,double")
3954 (set_attr "bdver1_decode" "double,double")
3955 (set_attr "prefix_rep" "1")
3956 (set_attr "prefix" "maybe_vex")
3957 (set_attr "mode" "SI")])
3959 (define_insn "sse_cvttss2siq<round_saeonly_name>"
3960 [(set (match_operand:DI 0 "register_operand" "=r,r")
3963 (match_operand:V4SF 1 "<round_saeonly_nimm_predicate>" "v,<round_saeonly_constraint>")
3964 (parallel [(const_int 0)]))))]
3965 "TARGET_SSE && TARGET_64BIT"
3966 "%vcvttss2si{q}\t{<round_saeonly_op2>%1, %0|%0, %k1<round_saeonly_op2>}"
3967 [(set_attr "type" "sseicvt")
3968 (set_attr "athlon_decode" "double,vector")
3969 (set_attr "amdfam10_decode" "double,double")
3970 (set_attr "bdver1_decode" "double,double")
3971 (set_attr "prefix_rep" "1")
3972 (set_attr "prefix" "maybe_vex")
3973 (set_attr "mode" "DI")])
3975 (define_insn "cvtusi2<ssescalarmodesuffix>32<round_name>"
3976 [(set (match_operand:VF_128 0 "register_operand" "=v")
3978 (vec_duplicate:VF_128
3979 (unsigned_float:<ssescalarmode>
3980 (match_operand:SI 2 "<round_nimm_predicate>" "<round_constraint3>")))
3981 (match_operand:VF_128 1 "register_operand" "v")
3983 "TARGET_AVX512F && <round_modev4sf_condition>"
3984 "vcvtusi2<ssescalarmodesuffix>\t{<round_op3>%2, %1, %0|%0, %1, %2<round_op3>}"
3985 [(set_attr "type" "sseicvt")
3986 (set_attr "prefix" "evex")
3987 (set_attr "mode" "<ssescalarmode>")])
3989 (define_insn "cvtusi2<ssescalarmodesuffix>64<round_name>"
3990 [(set (match_operand:VF_128 0 "register_operand" "=v")
3992 (vec_duplicate:VF_128
3993 (unsigned_float:<ssescalarmode>
3994 (match_operand:DI 2 "<round_nimm_predicate>" "<round_constraint3>")))
3995 (match_operand:VF_128 1 "register_operand" "v")
3997 "TARGET_AVX512F && TARGET_64BIT"
3998 "vcvtusi2<ssescalarmodesuffix>\t{<round_op3>%2, %1, %0|%0, %1, %2<round_op3>}"
3999 [(set_attr "type" "sseicvt")
4000 (set_attr "prefix" "evex")
4001 (set_attr "mode" "<ssescalarmode>")])
4003 (define_insn "float<sseintvecmodelower><mode>2<mask_name><round_name>"
4004 [(set (match_operand:VF1 0 "register_operand" "=v")
4006 (match_operand:<sseintvecmode> 1 "<round_nimm_predicate>" "<round_constraint>")))]
4007 "TARGET_SSE2 && <mask_mode512bit_condition> && <round_mode512bit_condition>"
4008 "%vcvtdq2ps\t{<round_mask_op2>%1, %0<mask_operand2>|%0<mask_operand2>, %1<round_mask_op2>}"
4009 [(set_attr "type" "ssecvt")
4010 (set_attr "prefix" "maybe_vex")
4011 (set_attr "mode" "<sseinsnmode>")])
4013 (define_insn "ufloat<sseintvecmodelower><mode>2<mask_name><round_name>"
4014 [(set (match_operand:VF1_AVX512VL 0 "register_operand" "=v")
4015 (unsigned_float:VF1_AVX512VL
4016 (match_operand:<sseintvecmode> 1 "nonimmediate_operand" "<round_constraint>")))]
4018 "vcvtudq2ps\t{<round_mask_op2>%1, %0<mask_operand2>|%0<mask_operand2>, %1<round_mask_op2>}"
4019 [(set_attr "type" "ssecvt")
4020 (set_attr "prefix" "evex")
4021 (set_attr "mode" "<MODE>")])
4023 (define_expand "floatuns<sseintvecmodelower><mode>2"
4024 [(match_operand:VF1 0 "register_operand")
4025 (match_operand:<sseintvecmode> 1 "register_operand")]
4026 "TARGET_SSE2 && (<MODE>mode == V4SFmode || TARGET_AVX2)"
4028 if (<MODE>mode == V16SFmode)
4029 emit_insn (gen_ufloatv16siv16sf2 (operands[0], operands[1]));
4031 if (TARGET_AVX512VL)
4033 if (<MODE>mode == V4SFmode)
4034 emit_insn (gen_ufloatv4siv4sf2 (operands[0], operands[1]));
4036 emit_insn (gen_ufloatv8siv8sf2 (operands[0], operands[1]));
4039 ix86_expand_vector_convert_uns_vsivsf (operands[0], operands[1]);
4045 ;; For <sse2_avx_avx512f>_fix_notrunc<sf2simodelower><mode> insn pattern
4046 (define_mode_attr sf2simodelower
4047 [(V16SI "v16sf") (V8SI "v8sf") (V4SI "v4sf")])
4049 (define_insn "<sse2_avx_avx512f>_fix_notrunc<sf2simodelower><mode><mask_name>"
4050 [(set (match_operand:VI4_AVX 0 "register_operand" "=v")
4052 [(match_operand:<ssePSmode> 1 "nonimmediate_operand" "vm")]
4053 UNSPEC_FIX_NOTRUNC))]
4054 "TARGET_SSE2 && <mask_mode512bit_condition>"
4055 "%vcvtps2dq\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
4056 [(set_attr "type" "ssecvt")
4057 (set (attr "prefix_data16")
4059 (match_test "TARGET_AVX")
4061 (const_string "1")))
4062 (set_attr "prefix" "maybe_vex")
4063 (set_attr "mode" "<sseinsnmode>")])
4065 (define_insn "<mask_codefor>avx512f_fix_notruncv16sfv16si<mask_name><round_name>"
4066 [(set (match_operand:V16SI 0 "register_operand" "=v")
4068 [(match_operand:V16SF 1 "<round_nimm_predicate>" "<round_constraint>")]
4069 UNSPEC_FIX_NOTRUNC))]
4071 "vcvtps2dq\t{<round_mask_op2>%1, %0<mask_operand2>|%0<mask_operand2>, %1<round_mask_op2>}"
4072 [(set_attr "type" "ssecvt")
4073 (set_attr "prefix" "evex")
4074 (set_attr "mode" "XI")])
4076 (define_insn "<mask_codefor><avx512>_ufix_notrunc<sf2simodelower><mode><mask_name><round_name>"
4077 [(set (match_operand:VI4_AVX512VL 0 "register_operand" "=v")
4078 (unspec:VI4_AVX512VL
4079 [(match_operand:<ssePSmode> 1 "nonimmediate_operand" "<round_constraint>")]
4080 UNSPEC_UNSIGNED_FIX_NOTRUNC))]
4082 "vcvtps2udq\t{<round_mask_op2>%1, %0<mask_operand2>|%0<mask_operand2>, %1<round_mask_op2>}"
4083 [(set_attr "type" "ssecvt")
4084 (set_attr "prefix" "evex")
4085 (set_attr "mode" "<sseinsnmode>")])
4087 (define_insn "<mask_codefor>avx512dq_cvtps2qq<mode><mask_name><round_name>"
4088 [(set (match_operand:VI8_256_512 0 "register_operand" "=v")
4089 (unspec:VI8_256_512 [(match_operand:<ssePSmode2> 1 "nonimmediate_operand" "<round_constraint>")]
4090 UNSPEC_FIX_NOTRUNC))]
4091 "TARGET_AVX512DQ && <round_mode512bit_condition>"
4092 "vcvtps2qq\t{<round_mask_op2>%1, %0<mask_operand2>|%0<mask_operand2>, %1<round_mask_op2>}"
4093 [(set_attr "type" "ssecvt")
4094 (set_attr "prefix" "evex")
4095 (set_attr "mode" "<sseinsnmode>")])
4097 (define_insn "<mask_codefor>avx512dq_cvtps2qqv2di<mask_name>"
4098 [(set (match_operand:V2DI 0 "register_operand" "=v")
4101 (match_operand:V4SF 1 "nonimmediate_operand" "vm")
4102 (parallel [(const_int 0) (const_int 1)]))]
4103 UNSPEC_FIX_NOTRUNC))]
4104 "TARGET_AVX512DQ && TARGET_AVX512VL"
4105 "vcvtps2qq\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
4106 [(set_attr "type" "ssecvt")
4107 (set_attr "prefix" "evex")
4108 (set_attr "mode" "TI")])
4110 (define_insn "<mask_codefor>avx512dq_cvtps2uqq<mode><mask_name><round_name>"
4111 [(set (match_operand:VI8_256_512 0 "register_operand" "=v")
4112 (unspec:VI8_256_512 [(match_operand:<ssePSmode2> 1 "nonimmediate_operand" "<round_constraint>")]
4113 UNSPEC_UNSIGNED_FIX_NOTRUNC))]
4114 "TARGET_AVX512DQ && <round_mode512bit_condition>"
4115 "vcvtps2uqq\t{<round_mask_op2>%1, %0<mask_operand2>|%0<mask_operand2>, %1<round_mask_op2>}"
4116 [(set_attr "type" "ssecvt")
4117 (set_attr "prefix" "evex")
4118 (set_attr "mode" "<sseinsnmode>")])
4120 (define_insn "<mask_codefor>avx512dq_cvtps2uqqv2di<mask_name>"
4121 [(set (match_operand:V2DI 0 "register_operand" "=v")
4124 (match_operand:V4SF 1 "nonimmediate_operand" "vm")
4125 (parallel [(const_int 0) (const_int 1)]))]
4126 UNSPEC_UNSIGNED_FIX_NOTRUNC))]
4127 "TARGET_AVX512DQ && TARGET_AVX512VL"
4128 "vcvtps2uqq\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
4129 [(set_attr "type" "ssecvt")
4130 (set_attr "prefix" "evex")
4131 (set_attr "mode" "TI")])
4133 (define_insn "<fixsuffix>fix_truncv16sfv16si2<mask_name><round_saeonly_name>"
4134 [(set (match_operand:V16SI 0 "register_operand" "=v")
4136 (match_operand:V16SF 1 "<round_saeonly_nimm_predicate>" "<round_saeonly_constraint>")))]
4138 "vcvttps2<fixsuffix>dq\t{<round_saeonly_mask_op2>%1, %0<mask_operand2>|%0<mask_operand2>, %1<round_saeonly_mask_op2>}"
4139 [(set_attr "type" "ssecvt")
4140 (set_attr "prefix" "evex")
4141 (set_attr "mode" "XI")])
4143 (define_insn "fix_truncv8sfv8si2<mask_name>"
4144 [(set (match_operand:V8SI 0 "register_operand" "=v")
4145 (fix:V8SI (match_operand:V8SF 1 "nonimmediate_operand" "vm")))]
4146 "TARGET_AVX && <mask_avx512vl_condition>"
4147 "vcvttps2dq\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
4148 [(set_attr "type" "ssecvt")
4149 (set_attr "prefix" "<mask_prefix>")
4150 (set_attr "mode" "OI")])
4152 (define_insn "fix_truncv4sfv4si2<mask_name>"
4153 [(set (match_operand:V4SI 0 "register_operand" "=v")
4154 (fix:V4SI (match_operand:V4SF 1 "nonimmediate_operand" "vm")))]
4155 "TARGET_SSE2 && <mask_avx512vl_condition>"
4156 "%vcvttps2dq\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
4157 [(set_attr "type" "ssecvt")
4158 (set (attr "prefix_rep")
4160 (match_test "TARGET_AVX")
4162 (const_string "1")))
4163 (set (attr "prefix_data16")
4165 (match_test "TARGET_AVX")
4167 (const_string "0")))
4168 (set_attr "prefix_data16" "0")
4169 (set_attr "prefix" "<mask_prefix2>")
4170 (set_attr "mode" "TI")])
4172 (define_expand "fixuns_trunc<mode><sseintvecmodelower>2"
4173 [(match_operand:<sseintvecmode> 0 "register_operand")
4174 (match_operand:VF1 1 "register_operand")]
4177 if (<MODE>mode == V16SFmode)
4178 emit_insn (gen_ufix_truncv16sfv16si2 (operands[0],
4183 tmp[0] = ix86_expand_adjust_ufix_to_sfix_si (operands[1], &tmp[2]);
4184 tmp[1] = gen_reg_rtx (<sseintvecmode>mode);
4185 emit_insn (gen_fix_trunc<mode><sseintvecmodelower>2 (tmp[1], tmp[0]));
4186 emit_insn (gen_xor<sseintvecmodelower>3 (operands[0], tmp[1], tmp[2]));
4191 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
4193 ;; Parallel double-precision floating point conversion operations
4195 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
4197 (define_insn "sse2_cvtpi2pd"
4198 [(set (match_operand:V2DF 0 "register_operand" "=x,x")
4199 (float:V2DF (match_operand:V2SI 1 "nonimmediate_operand" "y,m")))]
4201 "cvtpi2pd\t{%1, %0|%0, %1}"
4202 [(set_attr "type" "ssecvt")
4203 (set_attr "unit" "mmx,*")
4204 (set_attr "prefix_data16" "1,*")
4205 (set_attr "mode" "V2DF")])
4207 (define_insn "sse2_cvtpd2pi"
4208 [(set (match_operand:V2SI 0 "register_operand" "=y")
4209 (unspec:V2SI [(match_operand:V2DF 1 "nonimmediate_operand" "xm")]
4210 UNSPEC_FIX_NOTRUNC))]
4212 "cvtpd2pi\t{%1, %0|%0, %1}"
4213 [(set_attr "type" "ssecvt")
4214 (set_attr "unit" "mmx")
4215 (set_attr "bdver1_decode" "double")
4216 (set_attr "btver2_decode" "direct")
4217 (set_attr "prefix_data16" "1")
4218 (set_attr "mode" "DI")])
4220 (define_insn "sse2_cvttpd2pi"
4221 [(set (match_operand:V2SI 0 "register_operand" "=y")
4222 (fix:V2SI (match_operand:V2DF 1 "nonimmediate_operand" "xm")))]
4224 "cvttpd2pi\t{%1, %0|%0, %1}"
4225 [(set_attr "type" "ssecvt")
4226 (set_attr "unit" "mmx")
4227 (set_attr "bdver1_decode" "double")
4228 (set_attr "prefix_data16" "1")
4229 (set_attr "mode" "TI")])
4231 (define_insn "sse2_cvtsi2sd"
4232 [(set (match_operand:V2DF 0 "register_operand" "=x,x,x")
4235 (float:DF (match_operand:SI 2 "nonimmediate_operand" "r,m,rm")))
4236 (match_operand:V2DF 1 "register_operand" "0,0,x")
4240 cvtsi2sd\t{%2, %0|%0, %2}
4241 cvtsi2sd\t{%2, %0|%0, %2}
4242 vcvtsi2sd\t{%2, %1, %0|%0, %1, %2}"
4243 [(set_attr "isa" "noavx,noavx,avx")
4244 (set_attr "type" "sseicvt")
4245 (set_attr "athlon_decode" "double,direct,*")
4246 (set_attr "amdfam10_decode" "vector,double,*")
4247 (set_attr "bdver1_decode" "double,direct,*")
4248 (set_attr "btver2_decode" "double,double,double")
4249 (set_attr "prefix" "orig,orig,vex")
4250 (set_attr "mode" "DF")])
4252 (define_insn "sse2_cvtsi2sdq<round_name>"
4253 [(set (match_operand:V2DF 0 "register_operand" "=x,x,v")
4256 (float:DF (match_operand:DI 2 "<round_nimm_predicate>" "r,m,<round_constraint3>")))
4257 (match_operand:V2DF 1 "register_operand" "0,0,v")
4259 "TARGET_SSE2 && TARGET_64BIT"
4261 cvtsi2sdq\t{%2, %0|%0, %2}
4262 cvtsi2sdq\t{%2, %0|%0, %2}
4263 vcvtsi2sdq\t{<round_op3>%2, %1, %0|%0, %1, %2<round_op3>}"
4264 [(set_attr "isa" "noavx,noavx,avx")
4265 (set_attr "type" "sseicvt")
4266 (set_attr "athlon_decode" "double,direct,*")
4267 (set_attr "amdfam10_decode" "vector,double,*")
4268 (set_attr "bdver1_decode" "double,direct,*")
4269 (set_attr "length_vex" "*,*,4")
4270 (set_attr "prefix_rex" "1,1,*")
4271 (set_attr "prefix" "orig,orig,maybe_evex")
4272 (set_attr "mode" "DF")])
4274 (define_insn "avx512f_vcvtss2usi<round_name>"
4275 [(set (match_operand:SI 0 "register_operand" "=r")
4278 (match_operand:V4SF 1 "<round_nimm_predicate>" "<round_constraint>")
4279 (parallel [(const_int 0)]))]
4280 UNSPEC_UNSIGNED_FIX_NOTRUNC))]
4282 "vcvtss2usi\t{<round_op2>%1, %0|%0, %1<round_op2>}"
4283 [(set_attr "type" "sseicvt")
4284 (set_attr "prefix" "evex")
4285 (set_attr "mode" "SI")])
4287 (define_insn "avx512f_vcvtss2usiq<round_name>"
4288 [(set (match_operand:DI 0 "register_operand" "=r")
4291 (match_operand:V4SF 1 "<round_nimm_predicate>" "<round_constraint>")
4292 (parallel [(const_int 0)]))]
4293 UNSPEC_UNSIGNED_FIX_NOTRUNC))]
4294 "TARGET_AVX512F && TARGET_64BIT"
4295 "vcvtss2usi\t{<round_op2>%1, %0|%0, %1<round_op2>}"
4296 [(set_attr "type" "sseicvt")
4297 (set_attr "prefix" "evex")
4298 (set_attr "mode" "DI")])
4300 (define_insn "avx512f_vcvttss2usi<round_saeonly_name>"
4301 [(set (match_operand:SI 0 "register_operand" "=r")
4304 (match_operand:V4SF 1 "<round_saeonly_nimm_predicate>" "<round_saeonly_constraint>")
4305 (parallel [(const_int 0)]))))]
4307 "vcvttss2usi\t{<round_saeonly_op2>%1, %0|%0, %1<round_saeonly_op2>}"
4308 [(set_attr "type" "sseicvt")
4309 (set_attr "prefix" "evex")
4310 (set_attr "mode" "SI")])
4312 (define_insn "avx512f_vcvttss2usiq<round_saeonly_name>"
4313 [(set (match_operand:DI 0 "register_operand" "=r")
4316 (match_operand:V4SF 1 "<round_saeonly_nimm_predicate>" "<round_saeonly_constraint>")
4317 (parallel [(const_int 0)]))))]
4318 "TARGET_AVX512F && TARGET_64BIT"
4319 "vcvttss2usi\t{<round_saeonly_op2>%1, %0|%0, %1<round_saeonly_op2>}"
4320 [(set_attr "type" "sseicvt")
4321 (set_attr "prefix" "evex")
4322 (set_attr "mode" "DI")])
4324 (define_insn "avx512f_vcvtsd2usi<round_name>"
4325 [(set (match_operand:SI 0 "register_operand" "=r")
4328 (match_operand:V2DF 1 "<round_nimm_predicate>" "<round_constraint>")
4329 (parallel [(const_int 0)]))]
4330 UNSPEC_UNSIGNED_FIX_NOTRUNC))]
4332 "vcvtsd2usi\t{<round_op2>%1, %0|%0, %1<round_op2>}"
4333 [(set_attr "type" "sseicvt")
4334 (set_attr "prefix" "evex")
4335 (set_attr "mode" "SI")])
4337 (define_insn "avx512f_vcvtsd2usiq<round_name>"
4338 [(set (match_operand:DI 0 "register_operand" "=r")
4341 (match_operand:V2DF 1 "<round_nimm_predicate>" "<round_constraint>")
4342 (parallel [(const_int 0)]))]
4343 UNSPEC_UNSIGNED_FIX_NOTRUNC))]
4344 "TARGET_AVX512F && TARGET_64BIT"
4345 "vcvtsd2usi\t{<round_op2>%1, %0|%0, %1<round_op2>}"
4346 [(set_attr "type" "sseicvt")
4347 (set_attr "prefix" "evex")
4348 (set_attr "mode" "DI")])
4350 (define_insn "avx512f_vcvttsd2usi<round_saeonly_name>"
4351 [(set (match_operand:SI 0 "register_operand" "=r")
4354 (match_operand:V2DF 1 "<round_saeonly_nimm_predicate>" "<round_saeonly_constraint>")
4355 (parallel [(const_int 0)]))))]
4357 "vcvttsd2usi\t{<round_saeonly_op2>%1, %0|%0, %1<round_saeonly_op2>}"
4358 [(set_attr "type" "sseicvt")
4359 (set_attr "prefix" "evex")
4360 (set_attr "mode" "SI")])
4362 (define_insn "avx512f_vcvttsd2usiq<round_saeonly_name>"
4363 [(set (match_operand:DI 0 "register_operand" "=r")
4366 (match_operand:V2DF 1 "<round_saeonly_nimm_predicate>" "<round_saeonly_constraint>")
4367 (parallel [(const_int 0)]))))]
4368 "TARGET_AVX512F && TARGET_64BIT"
4369 "vcvttsd2usi\t{<round_saeonly_op2>%1, %0|%0, %1<round_saeonly_op2>}"
4370 [(set_attr "type" "sseicvt")
4371 (set_attr "prefix" "evex")
4372 (set_attr "mode" "DI")])
4374 (define_insn "sse2_cvtsd2si<round_name>"
4375 [(set (match_operand:SI 0 "register_operand" "=r,r")
4378 (match_operand:V2DF 1 "<round_nimm_predicate>" "v,<round_constraint2>")
4379 (parallel [(const_int 0)]))]
4380 UNSPEC_FIX_NOTRUNC))]
4382 "%vcvtsd2si\t{<round_op2>%1, %0|%0, %q1<round_op2>}"
4383 [(set_attr "type" "sseicvt")
4384 (set_attr "athlon_decode" "double,vector")
4385 (set_attr "bdver1_decode" "double,double")
4386 (set_attr "btver2_decode" "double,double")
4387 (set_attr "prefix_rep" "1")
4388 (set_attr "prefix" "maybe_vex")
4389 (set_attr "mode" "SI")])
4391 (define_insn "sse2_cvtsd2si_2"
4392 [(set (match_operand:SI 0 "register_operand" "=r,r")
4393 (unspec:SI [(match_operand:DF 1 "nonimmediate_operand" "v,m")]
4394 UNSPEC_FIX_NOTRUNC))]
4396 "%vcvtsd2si\t{%1, %0|%0, %q1}"
4397 [(set_attr "type" "sseicvt")
4398 (set_attr "athlon_decode" "double,vector")
4399 (set_attr "amdfam10_decode" "double,double")
4400 (set_attr "bdver1_decode" "double,double")
4401 (set_attr "prefix_rep" "1")
4402 (set_attr "prefix" "maybe_vex")
4403 (set_attr "mode" "SI")])
4405 (define_insn "sse2_cvtsd2siq<round_name>"
4406 [(set (match_operand:DI 0 "register_operand" "=r,r")
4409 (match_operand:V2DF 1 "<round_nimm_predicate>" "v,<round_constraint2>")
4410 (parallel [(const_int 0)]))]
4411 UNSPEC_FIX_NOTRUNC))]
4412 "TARGET_SSE2 && TARGET_64BIT"
4413 "%vcvtsd2si{q}\t{<round_op2>%1, %0|%0, %q1<round_op2>}"
4414 [(set_attr "type" "sseicvt")
4415 (set_attr "athlon_decode" "double,vector")
4416 (set_attr "bdver1_decode" "double,double")
4417 (set_attr "prefix_rep" "1")
4418 (set_attr "prefix" "maybe_vex")
4419 (set_attr "mode" "DI")])
4421 (define_insn "sse2_cvtsd2siq_2"
4422 [(set (match_operand:DI 0 "register_operand" "=r,r")
4423 (unspec:DI [(match_operand:DF 1 "nonimmediate_operand" "v,m")]
4424 UNSPEC_FIX_NOTRUNC))]
4425 "TARGET_SSE2 && TARGET_64BIT"
4426 "%vcvtsd2si{q}\t{%1, %0|%0, %q1}"
4427 [(set_attr "type" "sseicvt")
4428 (set_attr "athlon_decode" "double,vector")
4429 (set_attr "amdfam10_decode" "double,double")
4430 (set_attr "bdver1_decode" "double,double")
4431 (set_attr "prefix_rep" "1")
4432 (set_attr "prefix" "maybe_vex")
4433 (set_attr "mode" "DI")])
4435 (define_insn "sse2_cvttsd2si<round_saeonly_name>"
4436 [(set (match_operand:SI 0 "register_operand" "=r,r")
4439 (match_operand:V2DF 1 "<round_saeonly_nimm_predicate>" "v,<round_saeonly_constraint2>")
4440 (parallel [(const_int 0)]))))]
4442 "%vcvttsd2si\t{<round_saeonly_op2>%1, %0|%0, %q1<round_saeonly_op2>}"
4443 [(set_attr "type" "sseicvt")
4444 (set_attr "athlon_decode" "double,vector")
4445 (set_attr "amdfam10_decode" "double,double")
4446 (set_attr "bdver1_decode" "double,double")
4447 (set_attr "btver2_decode" "double,double")
4448 (set_attr "prefix_rep" "1")
4449 (set_attr "prefix" "maybe_vex")
4450 (set_attr "mode" "SI")])
4452 (define_insn "sse2_cvttsd2siq<round_saeonly_name>"
4453 [(set (match_operand:DI 0 "register_operand" "=r,r")
4456 (match_operand:V2DF 1 "<round_saeonly_nimm_predicate>" "v,<round_saeonly_constraint2>")
4457 (parallel [(const_int 0)]))))]
4458 "TARGET_SSE2 && TARGET_64BIT"
4459 "%vcvttsd2si{q}\t{<round_saeonly_op2>%1, %0|%0, %q1<round_saeonly_op2>}"
4460 [(set_attr "type" "sseicvt")
4461 (set_attr "athlon_decode" "double,vector")
4462 (set_attr "amdfam10_decode" "double,double")
4463 (set_attr "bdver1_decode" "double,double")
4464 (set_attr "prefix_rep" "1")
4465 (set_attr "prefix" "maybe_vex")
4466 (set_attr "mode" "DI")])
4468 ;; For float<si2dfmode><mode>2 insn pattern
4469 (define_mode_attr si2dfmode
4470 [(V8DF "V8SI") (V4DF "V4SI")])
4471 (define_mode_attr si2dfmodelower
4472 [(V8DF "v8si") (V4DF "v4si")])
4474 (define_insn "float<si2dfmodelower><mode>2<mask_name>"
4475 [(set (match_operand:VF2_512_256 0 "register_operand" "=v")
4476 (float:VF2_512_256 (match_operand:<si2dfmode> 1 "nonimmediate_operand" "vm")))]
4477 "TARGET_AVX && <mask_mode512bit_condition>"
4478 "vcvtdq2pd\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
4479 [(set_attr "type" "ssecvt")
4480 (set_attr "prefix" "maybe_vex")
4481 (set_attr "mode" "<MODE>")])
4483 (define_insn "<floatsuffix>float<sseintvecmodelower><mode>2<mask_name><round_name>"
4484 [(set (match_operand:VF2_AVX512VL 0 "register_operand" "=v")
4485 (any_float:VF2_AVX512VL
4486 (match_operand:<sseintvecmode> 1 "nonimmediate_operand" "vm")))]
4488 "vcvt<floatsuffix>qq2pd\t{<round_mask_op2>%1, %0<mask_operand2>|%0<mask_operand2>, %1<round_mask_op2>}"
4489 [(set_attr "type" "ssecvt")
4490 (set_attr "prefix" "evex")
4491 (set_attr "mode" "<MODE>")])
4493 ;; For <floatsuffix>float<sselondveclower><mode> insn patterns
4494 (define_mode_attr qq2pssuff
4495 [(V8SF "") (V4SF "{y}")])
4497 (define_mode_attr sselongvecmode
4498 [(V8SF "V8DI") (V4SF "V4DI")])
4500 (define_mode_attr sselongvecmodelower
4501 [(V8SF "v8di") (V4SF "v4di")])
4503 (define_mode_attr sseintvecmode3
4504 [(V8SF "XI") (V4SF "OI")
4505 (V8DF "OI") (V4DF "TI")])
4507 (define_insn "<floatsuffix>float<sselongvecmodelower><mode>2<mask_name><round_name>"
4508 [(set (match_operand:VF1_128_256VL 0 "register_operand" "=v")
4509 (any_float:VF1_128_256VL
4510 (match_operand:<sselongvecmode> 1 "nonimmediate_operand" "<round_constraint>")))]
4511 "TARGET_AVX512DQ && <round_modev8sf_condition>"
4512 "vcvt<floatsuffix>qq2ps<qq2pssuff>\t{<round_mask_op2>%1, %0<mask_operand2>|%0<mask_operand2>, %1<round_mask_op2>}"
4513 [(set_attr "type" "ssecvt")
4514 (set_attr "prefix" "evex")
4515 (set_attr "mode" "<MODE>")])
4517 (define_insn "*<floatsuffix>floatv2div2sf2"
4518 [(set (match_operand:V4SF 0 "register_operand" "=v")
4520 (any_float:V2SF (match_operand:V2DI 1 "nonimmediate_operand" "vm"))
4521 (const_vector:V2SF [(const_int 0) (const_int 0)])))]
4522 "TARGET_AVX512DQ && TARGET_AVX512VL"
4523 "vcvt<floatsuffix>qq2ps{x}\t{%1, %0|%0, %1}"
4524 [(set_attr "type" "ssecvt")
4525 (set_attr "prefix" "evex")
4526 (set_attr "mode" "V4SF")])
4528 (define_insn "<floatsuffix>floatv2div2sf2_mask"
4529 [(set (match_operand:V4SF 0 "register_operand" "=v")
4532 (any_float:V2SF (match_operand:V2DI 1 "nonimmediate_operand" "vm"))
4534 (match_operand:V4SF 2 "vector_move_operand" "0C")
4535 (parallel [(const_int 0) (const_int 1)]))
4536 (match_operand:QI 3 "register_operand" "Yk"))
4537 (const_vector:V2SF [(const_int 0) (const_int 0)])))]
4538 "TARGET_AVX512DQ && TARGET_AVX512VL"
4539 "vcvt<floatsuffix>qq2ps{x}\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"
4540 [(set_attr "type" "ssecvt")
4541 (set_attr "prefix" "evex")
4542 (set_attr "mode" "V4SF")])
4544 (define_insn "ufloat<si2dfmodelower><mode>2<mask_name>"
4545 [(set (match_operand:VF2_512_256VL 0 "register_operand" "=v")
4546 (unsigned_float:VF2_512_256VL
4547 (match_operand:<si2dfmode> 1 "nonimmediate_operand" "vm")))]
4549 "vcvtudq2pd\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
4550 [(set_attr "type" "ssecvt")
4551 (set_attr "prefix" "evex")
4552 (set_attr "mode" "<MODE>")])
4554 (define_insn "ufloatv2siv2df2<mask_name>"
4555 [(set (match_operand:V2DF 0 "register_operand" "=v")
4556 (unsigned_float:V2DF
4558 (match_operand:V4SI 1 "nonimmediate_operand" "vm")
4559 (parallel [(const_int 0) (const_int 1)]))))]
4561 "vcvtudq2pd\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
4562 [(set_attr "type" "ssecvt")
4563 (set_attr "prefix" "evex")
4564 (set_attr "mode" "V2DF")])
4566 (define_insn "avx512f_cvtdq2pd512_2"
4567 [(set (match_operand:V8DF 0 "register_operand" "=v")
4570 (match_operand:V16SI 1 "nonimmediate_operand" "vm")
4571 (parallel [(const_int 0) (const_int 1)
4572 (const_int 2) (const_int 3)
4573 (const_int 4) (const_int 5)
4574 (const_int 6) (const_int 7)]))))]
4576 "vcvtdq2pd\t{%t1, %0|%0, %t1}"
4577 [(set_attr "type" "ssecvt")
4578 (set_attr "prefix" "evex")
4579 (set_attr "mode" "V8DF")])
4581 (define_insn "avx_cvtdq2pd256_2"
4582 [(set (match_operand:V4DF 0 "register_operand" "=v")
4585 (match_operand:V8SI 1 "nonimmediate_operand" "vm")
4586 (parallel [(const_int 0) (const_int 1)
4587 (const_int 2) (const_int 3)]))))]
4589 "vcvtdq2pd\t{%x1, %0|%0, %x1}"
4590 [(set_attr "type" "ssecvt")
4591 (set_attr "prefix" "maybe_evex")
4592 (set_attr "mode" "V4DF")])
4594 (define_insn "sse2_cvtdq2pd<mask_name>"
4595 [(set (match_operand:V2DF 0 "register_operand" "=v")
4598 (match_operand:V4SI 1 "nonimmediate_operand" "vm")
4599 (parallel [(const_int 0) (const_int 1)]))))]
4600 "TARGET_SSE2 && <mask_avx512vl_condition>"
4601 "%vcvtdq2pd\t{%1, %0<mask_operand2>|%0<mask_operand2>, %q1}"
4602 [(set_attr "type" "ssecvt")
4603 (set_attr "prefix" "maybe_vex")
4604 (set_attr "ssememalign" "64")
4605 (set_attr "mode" "V2DF")])
4607 (define_insn "<mask_codefor>avx512f_cvtpd2dq512<mask_name><round_name>"
4608 [(set (match_operand:V8SI 0 "register_operand" "=v")
4610 [(match_operand:V8DF 1 "<round_nimm_predicate>" "<round_constraint>")]
4611 UNSPEC_FIX_NOTRUNC))]
4613 "vcvtpd2dq\t{<round_mask_op2>%1, %0<mask_operand2>|%0<mask_operand2>, %1<round_mask_op2>}"
4614 [(set_attr "type" "ssecvt")
4615 (set_attr "prefix" "evex")
4616 (set_attr "mode" "OI")])
4618 (define_insn "avx_cvtpd2dq256<mask_name>"
4619 [(set (match_operand:V4SI 0 "register_operand" "=v")
4620 (unspec:V4SI [(match_operand:V4DF 1 "nonimmediate_operand" "vm")]
4621 UNSPEC_FIX_NOTRUNC))]
4622 "TARGET_AVX && <mask_avx512vl_condition>"
4623 "vcvtpd2dq{y}\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
4624 [(set_attr "type" "ssecvt")
4625 (set_attr "prefix" "<mask_prefix>")
4626 (set_attr "mode" "OI")])
4628 (define_expand "avx_cvtpd2dq256_2"
4629 [(set (match_operand:V8SI 0 "register_operand")
4631 (unspec:V4SI [(match_operand:V4DF 1 "nonimmediate_operand")]
4635 "operands[2] = CONST0_RTX (V4SImode);")
4637 (define_insn "*avx_cvtpd2dq256_2"
4638 [(set (match_operand:V8SI 0 "register_operand" "=x")
4640 (unspec:V4SI [(match_operand:V4DF 1 "nonimmediate_operand" "xm")]
4642 (match_operand:V4SI 2 "const0_operand")))]
4644 "vcvtpd2dq{y}\t{%1, %x0|%x0, %1}"
4645 [(set_attr "type" "ssecvt")
4646 (set_attr "prefix" "vex")
4647 (set_attr "btver2_decode" "vector")
4648 (set_attr "mode" "OI")])
4650 (define_insn "sse2_cvtpd2dq<mask_name>"
4651 [(set (match_operand:V4SI 0 "register_operand" "=v")
4653 (unspec:V2SI [(match_operand:V2DF 1 "nonimmediate_operand" "vm")]
4655 (const_vector:V2SI [(const_int 0) (const_int 0)])))]
4656 "TARGET_SSE2 && <mask_avx512vl_condition>"
4659 return "vcvtpd2dq{x}\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}";
4661 return "cvtpd2dq\t{%1, %0|%0, %1}";
4663 [(set_attr "type" "ssecvt")
4664 (set_attr "prefix_rep" "1")
4665 (set_attr "prefix_data16" "0")
4666 (set_attr "prefix" "maybe_vex")
4667 (set_attr "mode" "TI")
4668 (set_attr "amdfam10_decode" "double")
4669 (set_attr "athlon_decode" "vector")
4670 (set_attr "bdver1_decode" "double")])
4672 ;; For ufix_notrunc* insn patterns
4673 (define_mode_attr pd2udqsuff
4674 [(V8DF "") (V4DF "{y}")])
4676 (define_insn "ufix_notrunc<mode><si2dfmodelower>2<mask_name><round_name>"
4677 [(set (match_operand:<si2dfmode> 0 "register_operand" "=v")
4679 [(match_operand:VF2_512_256VL 1 "nonimmediate_operand" "<round_constraint>")]
4680 UNSPEC_UNSIGNED_FIX_NOTRUNC))]
4682 "vcvtpd2udq<pd2udqsuff>\t{<round_mask_op2>%1, %0<mask_operand2>|%0<mask_operand2>, %1<round_mask_op2>}"
4683 [(set_attr "type" "ssecvt")
4684 (set_attr "prefix" "evex")
4685 (set_attr "mode" "<sseinsnmode>")])
4687 (define_insn "ufix_notruncv2dfv2si2<mask_name>"
4688 [(set (match_operand:V4SI 0 "register_operand" "=v")
4691 [(match_operand:V2DF 1 "nonimmediate_operand" "vm")]
4692 UNSPEC_UNSIGNED_FIX_NOTRUNC)
4693 (const_vector:V2SI [(const_int 0) (const_int 0)])))]
4695 "vcvtpd2udq{x}\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
4696 [(set_attr "type" "ssecvt")
4697 (set_attr "prefix" "evex")
4698 (set_attr "mode" "TI")])
4700 (define_insn "<fixsuffix>fix_truncv8dfv8si2<mask_name><round_saeonly_name>"
4701 [(set (match_operand:V8SI 0 "register_operand" "=v")
4703 (match_operand:V8DF 1 "<round_saeonly_nimm_predicate>" "<round_saeonly_constraint>")))]
4705 "vcvttpd2<fixsuffix>dq\t{<round_saeonly_mask_op2>%1, %0<mask_operand2>|%0<mask_operand2>, %1<round_saeonly_mask_op2>}"
4706 [(set_attr "type" "ssecvt")
4707 (set_attr "prefix" "evex")
4708 (set_attr "mode" "OI")])
4710 (define_insn "ufix_truncv2dfv2si2<mask_name>"
4711 [(set (match_operand:V4SI 0 "register_operand" "=v")
4713 (unsigned_fix:V2SI (match_operand:V2DF 1 "nonimmediate_operand" "vm"))
4714 (const_vector:V2SI [(const_int 0) (const_int 0)])))]
4716 "vcvttpd2udq{x}\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
4717 [(set_attr "type" "ssecvt")
4718 (set_attr "prefix" "evex")
4719 (set_attr "mode" "TI")])
4721 (define_insn "fix_truncv4dfv4si2<mask_name>"
4722 [(set (match_operand:V4SI 0 "register_operand" "=v")
4723 (fix:V4SI (match_operand:V4DF 1 "nonimmediate_operand" "vm")))]
4724 "TARGET_AVX || (TARGET_AVX512VL && TARGET_AVX512F)"
4725 "vcvttpd2dq{y}\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
4726 [(set_attr "type" "ssecvt")
4727 (set_attr "prefix" "maybe_evex")
4728 (set_attr "mode" "OI")])
4730 (define_insn "ufix_truncv4dfv4si2<mask_name>"
4731 [(set (match_operand:V4SI 0 "register_operand" "=v")
4732 (unsigned_fix:V4SI (match_operand:V4DF 1 "nonimmediate_operand" "vm")))]
4733 "TARGET_AVX512VL && TARGET_AVX512F"
4734 "vcvttpd2udq{y}\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
4735 [(set_attr "type" "ssecvt")
4736 (set_attr "prefix" "maybe_evex")
4737 (set_attr "mode" "OI")])
4739 (define_insn "<fixsuffix>fix_trunc<mode><sseintvecmodelower>2<mask_name><round_saeonly_name>"
4740 [(set (match_operand:<sseintvecmode> 0 "register_operand" "=v")
4741 (any_fix:<sseintvecmode>
4742 (match_operand:VF2_AVX512VL 1 "<round_saeonly_nimm_predicate>" "<round_saeonly_constraint>")))]
4743 "TARGET_AVX512DQ && <round_saeonly_mode512bit_condition>"
4744 "vcvttpd2<fixsuffix>qq\t{<round_saeonly_mask_op2>%1, %0<mask_operand2>|%0<mask_operand2>, %1<round_saeonly_mask_op2>}"
4745 [(set_attr "type" "ssecvt")
4746 (set_attr "prefix" "evex")
4747 (set_attr "mode" "<sseintvecmode2>")])
4749 (define_insn "fix_notrunc<mode><sseintvecmodelower>2<mask_name><round_name>"
4750 [(set (match_operand:<sseintvecmode> 0 "register_operand" "=v")
4751 (unspec:<sseintvecmode>
4752 [(match_operand:VF2_AVX512VL 1 "<round_nimm_predicate>" "<round_constraint>")]
4753 UNSPEC_FIX_NOTRUNC))]
4754 "TARGET_AVX512DQ && <round_mode512bit_condition>"
4755 "vcvtpd2qq\t{<round_mask_op2>%1, %0<mask_operand2>|%0<mask_operand2>, %1<round_mask_op2>}"
4756 [(set_attr "type" "ssecvt")
4757 (set_attr "prefix" "evex")
4758 (set_attr "mode" "<sseintvecmode2>")])
4760 (define_insn "ufix_notrunc<mode><sseintvecmodelower>2<mask_name><round_name>"
4761 [(set (match_operand:<sseintvecmode> 0 "register_operand" "=v")
4762 (unspec:<sseintvecmode>
4763 [(match_operand:VF2_AVX512VL 1 "nonimmediate_operand" "<round_constraint>")]
4764 UNSPEC_UNSIGNED_FIX_NOTRUNC))]
4765 "TARGET_AVX512DQ && <round_mode512bit_condition>"
4766 "vcvtpd2uqq\t{<round_mask_op2>%1, %0<mask_operand2>|%0<mask_operand2>, %1<round_mask_op2>}"
4767 [(set_attr "type" "ssecvt")
4768 (set_attr "prefix" "evex")
4769 (set_attr "mode" "<sseintvecmode2>")])
4771 (define_insn "<fixsuffix>fix_trunc<mode><sselongvecmodelower>2<mask_name><round_saeonly_name>"
4772 [(set (match_operand:<sselongvecmode> 0 "register_operand" "=v")
4773 (any_fix:<sselongvecmode>
4774 (match_operand:VF1_128_256VL 1 "<round_saeonly_nimm_predicate>" "<round_saeonly_constraint>")))]
4775 "TARGET_AVX512DQ && <round_saeonly_modev8sf_condition>"
4776 "vcvttps2<fixsuffix>qq\t{<round_saeonly_mask_op2>%1, %0<mask_operand2>|%0<mask_operand2>, %1<round_saeonly_mask_op2>}"
4777 [(set_attr "type" "ssecvt")
4778 (set_attr "prefix" "evex")
4779 (set_attr "mode" "<sseintvecmode3>")])
4781 (define_insn "<fixsuffix>fix_truncv2sfv2di2<mask_name>"
4782 [(set (match_operand:V2DI 0 "register_operand" "=v")
4785 (match_operand:V4SF 1 "nonimmediate_operand" "vm")
4786 (parallel [(const_int 0) (const_int 1)]))))]
4787 "TARGET_AVX512DQ && TARGET_AVX512VL"
4788 "vcvttps2<fixsuffix>qq\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
4789 [(set_attr "type" "ssecvt")
4790 (set_attr "prefix" "evex")
4791 (set_attr "mode" "TI")])
4793 (define_insn "ufix_trunc<mode><sseintvecmodelower>2<mask_name>"
4794 [(set (match_operand:<sseintvecmode> 0 "register_operand" "=v")
4795 (unsigned_fix:<sseintvecmode>
4796 (match_operand:VF1_128_256VL 1 "nonimmediate_operand" "vm")))]
4798 "vcvttps2udq\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
4799 [(set_attr "type" "ssecvt")
4800 (set_attr "prefix" "evex")
4801 (set_attr "mode" "<sseintvecmode2>")])
4803 (define_expand "avx_cvttpd2dq256_2"
4804 [(set (match_operand:V8SI 0 "register_operand")
4806 (fix:V4SI (match_operand:V4DF 1 "nonimmediate_operand"))
4809 "operands[2] = CONST0_RTX (V4SImode);")
4811 (define_insn "sse2_cvttpd2dq<mask_name>"
4812 [(set (match_operand:V4SI 0 "register_operand" "=v")
4814 (fix:V2SI (match_operand:V2DF 1 "nonimmediate_operand" "vm"))
4815 (const_vector:V2SI [(const_int 0) (const_int 0)])))]
4816 "TARGET_SSE2 && <mask_avx512vl_condition>"
4819 return "vcvttpd2dq{x}\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}";
4821 return "cvttpd2dq\t{%1, %0|%0, %1}";
4823 [(set_attr "type" "ssecvt")
4824 (set_attr "amdfam10_decode" "double")
4825 (set_attr "athlon_decode" "vector")
4826 (set_attr "bdver1_decode" "double")
4827 (set_attr "prefix" "maybe_vex")
4828 (set_attr "mode" "TI")])
4830 (define_insn "sse2_cvtsd2ss<round_name>"
4831 [(set (match_operand:V4SF 0 "register_operand" "=x,x,v")
4834 (float_truncate:V2SF
4835 (match_operand:V2DF 2 "nonimmediate_operand" "x,m,<round_constraint>")))
4836 (match_operand:V4SF 1 "register_operand" "0,0,v")
4840 cvtsd2ss\t{%2, %0|%0, %2}
4841 cvtsd2ss\t{%2, %0|%0, %q2}
4842 vcvtsd2ss\t{<round_op3>%2, %1, %0|%0, %1, %q2<round_op3>}"
4843 [(set_attr "isa" "noavx,noavx,avx")
4844 (set_attr "type" "ssecvt")
4845 (set_attr "athlon_decode" "vector,double,*")
4846 (set_attr "amdfam10_decode" "vector,double,*")
4847 (set_attr "bdver1_decode" "direct,direct,*")
4848 (set_attr "btver2_decode" "double,double,double")
4849 (set_attr "prefix" "orig,orig,<round_prefix>")
4850 (set_attr "mode" "SF")])
4852 (define_insn "sse2_cvtss2sd<round_saeonly_name>"
4853 [(set (match_operand:V2DF 0 "register_operand" "=x,x,v")
4857 (match_operand:V4SF 2 "<round_saeonly_nimm_predicate>" "x,m,<round_saeonly_constraint>")
4858 (parallel [(const_int 0) (const_int 1)])))
4859 (match_operand:V2DF 1 "register_operand" "0,0,v")
4863 cvtss2sd\t{%2, %0|%0, %2}
4864 cvtss2sd\t{%2, %0|%0, %k2}
4865 vcvtss2sd\t{<round_saeonly_op3>%2, %1, %0|%0, %1, %k2<round_saeonly_op3>}"
4866 [(set_attr "isa" "noavx,noavx,avx")
4867 (set_attr "type" "ssecvt")
4868 (set_attr "amdfam10_decode" "vector,double,*")
4869 (set_attr "athlon_decode" "direct,direct,*")
4870 (set_attr "bdver1_decode" "direct,direct,*")
4871 (set_attr "btver2_decode" "double,double,double")
4872 (set_attr "prefix" "orig,orig,<round_saeonly_prefix>")
4873 (set_attr "mode" "DF")])
4875 (define_insn "<mask_codefor>avx512f_cvtpd2ps512<mask_name><round_name>"
4876 [(set (match_operand:V8SF 0 "register_operand" "=v")
4877 (float_truncate:V8SF
4878 (match_operand:V8DF 1 "<round_nimm_predicate>" "<round_constraint>")))]
4880 "vcvtpd2ps\t{<round_mask_op2>%1, %0<mask_operand2>|%0<mask_operand2>, %1<round_mask_op2>}"
4881 [(set_attr "type" "ssecvt")
4882 (set_attr "prefix" "evex")
4883 (set_attr "mode" "V8SF")])
4885 (define_insn "avx_cvtpd2ps256<mask_name>"
4886 [(set (match_operand:V4SF 0 "register_operand" "=v")
4887 (float_truncate:V4SF
4888 (match_operand:V4DF 1 "nonimmediate_operand" "vm")))]
4889 "TARGET_AVX && <mask_avx512vl_condition>"
4890 "vcvtpd2ps{y}\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
4891 [(set_attr "type" "ssecvt")
4892 (set_attr "prefix" "maybe_evex")
4893 (set_attr "btver2_decode" "vector")
4894 (set_attr "mode" "V4SF")])
4896 (define_expand "sse2_cvtpd2ps"
4897 [(set (match_operand:V4SF 0 "register_operand")
4899 (float_truncate:V2SF
4900 (match_operand:V2DF 1 "nonimmediate_operand"))
4903 "operands[2] = CONST0_RTX (V2SFmode);")
4905 (define_expand "sse2_cvtpd2ps_mask"
4906 [(set (match_operand:V4SF 0 "register_operand")
4909 (float_truncate:V2SF
4910 (match_operand:V2DF 1 "nonimmediate_operand"))
4912 (match_operand:V4SF 2 "register_operand")
4913 (match_operand:QI 3 "register_operand")))]
4915 "operands[4] = CONST0_RTX (V2SFmode);")
4917 (define_insn "*sse2_cvtpd2ps<mask_name>"
4918 [(set (match_operand:V4SF 0 "register_operand" "=v")
4920 (float_truncate:V2SF
4921 (match_operand:V2DF 1 "nonimmediate_operand" "vm"))
4922 (match_operand:V2SF 2 "const0_operand")))]
4923 "TARGET_SSE2 && <mask_avx512vl_condition>"
4926 return "vcvtpd2ps{x}\t{%1, %0<mask_operand3>|%0<mask_operand3>, %1}";
4928 return "cvtpd2ps\t{%1, %0|%0, %1}";
4930 [(set_attr "type" "ssecvt")
4931 (set_attr "amdfam10_decode" "double")
4932 (set_attr "athlon_decode" "vector")
4933 (set_attr "bdver1_decode" "double")
4934 (set_attr "prefix_data16" "1")
4935 (set_attr "prefix" "maybe_vex")
4936 (set_attr "mode" "V4SF")])
4938 ;; For <sse2_avx_avx512f>_cvtps2pd<avxsizesuffix> insn pattern
4939 (define_mode_attr sf2dfmode
4940 [(V8DF "V8SF") (V4DF "V4SF")])
4942 (define_insn "<sse2_avx_avx512f>_cvtps2pd<avxsizesuffix><mask_name><round_saeonly_name>"
4943 [(set (match_operand:VF2_512_256 0 "register_operand" "=v")
4944 (float_extend:VF2_512_256
4945 (match_operand:<sf2dfmode> 1 "<round_saeonly_nimm_predicate>" "<round_saeonly_constraint>")))]
4946 "TARGET_AVX && <mask_mode512bit_condition> && <round_saeonly_mode512bit_condition>"
4947 "vcvtps2pd\t{<round_saeonly_mask_op2>%1, %0<mask_operand2>|%0<mask_operand2>, %1<round_saeonly_mask_op2>}"
4948 [(set_attr "type" "ssecvt")
4949 (set_attr "prefix" "maybe_vex")
4950 (set_attr "mode" "<MODE>")])
4952 (define_insn "*avx_cvtps2pd256_2"
4953 [(set (match_operand:V4DF 0 "register_operand" "=x")
4956 (match_operand:V8SF 1 "nonimmediate_operand" "xm")
4957 (parallel [(const_int 0) (const_int 1)
4958 (const_int 2) (const_int 3)]))))]
4960 "vcvtps2pd\t{%x1, %0|%0, %x1}"
4961 [(set_attr "type" "ssecvt")
4962 (set_attr "prefix" "vex")
4963 (set_attr "mode" "V4DF")])
4965 (define_insn "vec_unpacks_lo_v16sf"
4966 [(set (match_operand:V8DF 0 "register_operand" "=v")
4969 (match_operand:V16SF 1 "nonimmediate_operand" "vm")
4970 (parallel [(const_int 0) (const_int 1)
4971 (const_int 2) (const_int 3)
4972 (const_int 4) (const_int 5)
4973 (const_int 6) (const_int 7)]))))]
4975 "vcvtps2pd\t{%t1, %0|%0, %t1}"
4976 [(set_attr "type" "ssecvt")
4977 (set_attr "prefix" "evex")
4978 (set_attr "mode" "V8DF")])
4980 (define_insn "<avx512>_cvt<ssemodesuffix>2mask<mode>"
4981 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=Yk")
4982 (unspec:<avx512fmaskmode>
4983 [(match_operand:VI12_AVX512VL 1 "register_operand" "v")]
4984 UNSPEC_CVTINT2MASK))]
4986 "vpmov<ssemodesuffix>2m\t{%1, %0|%0, %1}"
4987 [(set_attr "prefix" "evex")
4988 (set_attr "mode" "<sseinsnmode>")])
4990 (define_insn "<avx512>_cvt<ssemodesuffix>2mask<mode>"
4991 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=Yk")
4992 (unspec:<avx512fmaskmode>
4993 [(match_operand:VI48_AVX512VL 1 "register_operand" "v")]
4994 UNSPEC_CVTINT2MASK))]
4996 "vpmov<ssemodesuffix>2m\t{%1, %0|%0, %1}"
4997 [(set_attr "prefix" "evex")
4998 (set_attr "mode" "<sseinsnmode>")])
5000 (define_expand "<avx512>_cvtmask2<ssemodesuffix><mode>"
5001 [(set (match_operand:VI12_AVX512VL 0 "register_operand")
5002 (vec_merge:VI12_AVX512VL
5005 (match_operand:<avx512fmaskmode> 1 "register_operand")))]
5008 operands[2] = CONSTM1_RTX (<MODE>mode);
5009 operands[3] = CONST0_RTX (<MODE>mode);
5012 (define_insn "*<avx512>_cvtmask2<ssemodesuffix><mode>"
5013 [(set (match_operand:VI12_AVX512VL 0 "register_operand" "=v")
5014 (vec_merge:VI12_AVX512VL
5015 (match_operand:VI12_AVX512VL 2 "constm1_operand")
5016 (match_operand:VI12_AVX512VL 3 "const0_operand")
5017 (match_operand:<avx512fmaskmode> 1 "register_operand" "Yk")))]
5019 "vpmovm2<ssemodesuffix>\t{%1, %0|%0, %1}"
5020 [(set_attr "prefix" "evex")
5021 (set_attr "mode" "<sseinsnmode>")])
5023 (define_expand "<avx512>_cvtmask2<ssemodesuffix><mode>"
5024 [(set (match_operand:VI48_AVX512VL 0 "register_operand")
5025 (vec_merge:VI48_AVX512VL
5028 (match_operand:<avx512fmaskmode> 1 "register_operand")))]
5031 operands[2] = CONSTM1_RTX (<MODE>mode);
5032 operands[3] = CONST0_RTX (<MODE>mode);
5035 (define_insn "*<avx512>_cvtmask2<ssemodesuffix><mode>"
5036 [(set (match_operand:VI48_AVX512VL 0 "register_operand" "=v")
5037 (vec_merge:VI48_AVX512VL
5038 (match_operand:VI48_AVX512VL 2 "constm1_operand")
5039 (match_operand:VI48_AVX512VL 3 "const0_operand")
5040 (match_operand:<avx512fmaskmode> 1 "register_operand" "Yk")))]
5042 "vpmovm2<ssemodesuffix>\t{%1, %0|%0, %1}"
5043 [(set_attr "prefix" "evex")
5044 (set_attr "mode" "<sseinsnmode>")])
5046 (define_insn "sse2_cvtps2pd<mask_name>"
5047 [(set (match_operand:V2DF 0 "register_operand" "=v")
5050 (match_operand:V4SF 1 "nonimmediate_operand" "vm")
5051 (parallel [(const_int 0) (const_int 1)]))))]
5052 "TARGET_SSE2 && <mask_avx512vl_condition>"
5053 "%vcvtps2pd\t{%1, %0<mask_operand2>|%0<mask_operand2>, %q1}"
5054 [(set_attr "type" "ssecvt")
5055 (set_attr "amdfam10_decode" "direct")
5056 (set_attr "athlon_decode" "double")
5057 (set_attr "bdver1_decode" "double")
5058 (set_attr "prefix_data16" "0")
5059 (set_attr "prefix" "maybe_vex")
5060 (set_attr "mode" "V2DF")])
5062 (define_expand "vec_unpacks_hi_v4sf"
5067 (match_operand:V4SF 1 "nonimmediate_operand"))
5068 (parallel [(const_int 6) (const_int 7)
5069 (const_int 2) (const_int 3)])))
5070 (set (match_operand:V2DF 0 "register_operand")
5074 (parallel [(const_int 0) (const_int 1)]))))]
5076 "operands[2] = gen_reg_rtx (V4SFmode);")
5078 (define_expand "vec_unpacks_hi_v8sf"
5081 (match_operand:V8SF 1 "register_operand")
5082 (parallel [(const_int 4) (const_int 5)
5083 (const_int 6) (const_int 7)])))
5084 (set (match_operand:V4DF 0 "register_operand")
5088 "operands[2] = gen_reg_rtx (V4SFmode);")
5090 (define_expand "vec_unpacks_hi_v16sf"
5093 (match_operand:V16SF 1 "register_operand")
5094 (parallel [(const_int 8) (const_int 9)
5095 (const_int 10) (const_int 11)
5096 (const_int 12) (const_int 13)
5097 (const_int 14) (const_int 15)])))
5098 (set (match_operand:V8DF 0 "register_operand")
5102 "operands[2] = gen_reg_rtx (V8SFmode);")
5104 (define_expand "vec_unpacks_lo_v4sf"
5105 [(set (match_operand:V2DF 0 "register_operand")
5108 (match_operand:V4SF 1 "nonimmediate_operand")
5109 (parallel [(const_int 0) (const_int 1)]))))]
5112 (define_expand "vec_unpacks_lo_v8sf"
5113 [(set (match_operand:V4DF 0 "register_operand")
5116 (match_operand:V8SF 1 "nonimmediate_operand")
5117 (parallel [(const_int 0) (const_int 1)
5118 (const_int 2) (const_int 3)]))))]
5121 (define_mode_attr sseunpackfltmode
5122 [(V8HI "V4SF") (V4SI "V2DF") (V16HI "V8SF")
5123 (V8SI "V4DF") (V32HI "V16SF") (V16SI "V8DF")])
5125 (define_expand "vec_unpacks_float_hi_<mode>"
5126 [(match_operand:<sseunpackfltmode> 0 "register_operand")
5127 (match_operand:VI2_AVX512F 1 "register_operand")]
5130 rtx tmp = gen_reg_rtx (<sseunpackmode>mode);
5132 emit_insn (gen_vec_unpacks_hi_<mode> (tmp, operands[1]));
5133 emit_insn (gen_rtx_SET (VOIDmode, operands[0],
5134 gen_rtx_FLOAT (<sseunpackfltmode>mode, tmp)));
5138 (define_expand "vec_unpacks_float_lo_<mode>"
5139 [(match_operand:<sseunpackfltmode> 0 "register_operand")
5140 (match_operand:VI2_AVX512F 1 "register_operand")]
5143 rtx tmp = gen_reg_rtx (<sseunpackmode>mode);
5145 emit_insn (gen_vec_unpacks_lo_<mode> (tmp, operands[1]));
5146 emit_insn (gen_rtx_SET (VOIDmode, operands[0],
5147 gen_rtx_FLOAT (<sseunpackfltmode>mode, tmp)));
5151 (define_expand "vec_unpacku_float_hi_<mode>"
5152 [(match_operand:<sseunpackfltmode> 0 "register_operand")
5153 (match_operand:VI2_AVX512F 1 "register_operand")]
5156 rtx tmp = gen_reg_rtx (<sseunpackmode>mode);
5158 emit_insn (gen_vec_unpacku_hi_<mode> (tmp, operands[1]));
5159 emit_insn (gen_rtx_SET (VOIDmode, operands[0],
5160 gen_rtx_FLOAT (<sseunpackfltmode>mode, tmp)));
5164 (define_expand "vec_unpacku_float_lo_<mode>"
5165 [(match_operand:<sseunpackfltmode> 0 "register_operand")
5166 (match_operand:VI2_AVX512F 1 "register_operand")]
5169 rtx tmp = gen_reg_rtx (<sseunpackmode>mode);
5171 emit_insn (gen_vec_unpacku_lo_<mode> (tmp, operands[1]));
5172 emit_insn (gen_rtx_SET (VOIDmode, operands[0],
5173 gen_rtx_FLOAT (<sseunpackfltmode>mode, tmp)));
5177 (define_expand "vec_unpacks_float_hi_v4si"
5180 (match_operand:V4SI 1 "nonimmediate_operand")
5181 (parallel [(const_int 2) (const_int 3)
5182 (const_int 2) (const_int 3)])))
5183 (set (match_operand:V2DF 0 "register_operand")
5187 (parallel [(const_int 0) (const_int 1)]))))]
5189 "operands[2] = gen_reg_rtx (V4SImode);")
5191 (define_expand "vec_unpacks_float_lo_v4si"
5192 [(set (match_operand:V2DF 0 "register_operand")
5195 (match_operand:V4SI 1 "nonimmediate_operand")
5196 (parallel [(const_int 0) (const_int 1)]))))]
5199 (define_expand "vec_unpacks_float_hi_v8si"
5202 (match_operand:V8SI 1 "nonimmediate_operand")
5203 (parallel [(const_int 4) (const_int 5)
5204 (const_int 6) (const_int 7)])))
5205 (set (match_operand:V4DF 0 "register_operand")
5209 "operands[2] = gen_reg_rtx (V4SImode);")
5211 (define_expand "vec_unpacks_float_lo_v8si"
5212 [(set (match_operand:V4DF 0 "register_operand")
5215 (match_operand:V8SI 1 "nonimmediate_operand")
5216 (parallel [(const_int 0) (const_int 1)
5217 (const_int 2) (const_int 3)]))))]
5220 (define_expand "vec_unpacks_float_hi_v16si"
5223 (match_operand:V16SI 1 "nonimmediate_operand")
5224 (parallel [(const_int 8) (const_int 9)
5225 (const_int 10) (const_int 11)
5226 (const_int 12) (const_int 13)
5227 (const_int 14) (const_int 15)])))
5228 (set (match_operand:V8DF 0 "register_operand")
5232 "operands[2] = gen_reg_rtx (V8SImode);")
5234 (define_expand "vec_unpacks_float_lo_v16si"
5235 [(set (match_operand:V8DF 0 "register_operand")
5238 (match_operand:V16SI 1 "nonimmediate_operand")
5239 (parallel [(const_int 0) (const_int 1)
5240 (const_int 2) (const_int 3)
5241 (const_int 4) (const_int 5)
5242 (const_int 6) (const_int 7)]))))]
5245 (define_expand "vec_unpacku_float_hi_v4si"
5248 (match_operand:V4SI 1 "nonimmediate_operand")
5249 (parallel [(const_int 2) (const_int 3)
5250 (const_int 2) (const_int 3)])))
5255 (parallel [(const_int 0) (const_int 1)]))))
5257 (lt:V2DF (match_dup 6) (match_dup 3)))
5259 (and:V2DF (match_dup 7) (match_dup 4)))
5260 (set (match_operand:V2DF 0 "register_operand")
5261 (plus:V2DF (match_dup 6) (match_dup 8)))]
5264 REAL_VALUE_TYPE TWO32r;
5268 real_ldexp (&TWO32r, &dconst1, 32);
5269 x = const_double_from_real_value (TWO32r, DFmode);
5271 operands[3] = force_reg (V2DFmode, CONST0_RTX (V2DFmode));
5272 operands[4] = force_reg (V2DFmode,
5273 ix86_build_const_vector (V2DFmode, 1, x));
5275 operands[5] = gen_reg_rtx (V4SImode);
5277 for (i = 6; i < 9; i++)
5278 operands[i] = gen_reg_rtx (V2DFmode);
5281 (define_expand "vec_unpacku_float_lo_v4si"
5285 (match_operand:V4SI 1 "nonimmediate_operand")
5286 (parallel [(const_int 0) (const_int 1)]))))
5288 (lt:V2DF (match_dup 5) (match_dup 3)))
5290 (and:V2DF (match_dup 6) (match_dup 4)))
5291 (set (match_operand:V2DF 0 "register_operand")
5292 (plus:V2DF (match_dup 5) (match_dup 7)))]
5295 REAL_VALUE_TYPE TWO32r;
5299 real_ldexp (&TWO32r, &dconst1, 32);
5300 x = const_double_from_real_value (TWO32r, DFmode);
5302 operands[3] = force_reg (V2DFmode, CONST0_RTX (V2DFmode));
5303 operands[4] = force_reg (V2DFmode,
5304 ix86_build_const_vector (V2DFmode, 1, x));
5306 for (i = 5; i < 8; i++)
5307 operands[i] = gen_reg_rtx (V2DFmode);
5310 (define_expand "vec_unpacku_float_hi_v8si"
5311 [(match_operand:V4DF 0 "register_operand")
5312 (match_operand:V8SI 1 "register_operand")]
5315 REAL_VALUE_TYPE TWO32r;
5319 real_ldexp (&TWO32r, &dconst1, 32);
5320 x = const_double_from_real_value (TWO32r, DFmode);
5322 tmp[0] = force_reg (V4DFmode, CONST0_RTX (V4DFmode));
5323 tmp[1] = force_reg (V4DFmode, ix86_build_const_vector (V4DFmode, 1, x));
5324 tmp[5] = gen_reg_rtx (V4SImode);
5326 for (i = 2; i < 5; i++)
5327 tmp[i] = gen_reg_rtx (V4DFmode);
5328 emit_insn (gen_vec_extract_hi_v8si (tmp[5], operands[1]));
5329 emit_insn (gen_floatv4siv4df2 (tmp[2], tmp[5]));
5330 emit_insn (gen_rtx_SET (VOIDmode, tmp[3],
5331 gen_rtx_LT (V4DFmode, tmp[2], tmp[0])));
5332 emit_insn (gen_andv4df3 (tmp[4], tmp[3], tmp[1]));
5333 emit_insn (gen_addv4df3 (operands[0], tmp[2], tmp[4]));
5337 (define_expand "vec_unpacku_float_hi_v16si"
5338 [(match_operand:V8DF 0 "register_operand")
5339 (match_operand:V16SI 1 "register_operand")]
5342 REAL_VALUE_TYPE TWO32r;
5345 real_ldexp (&TWO32r, &dconst1, 32);
5346 x = const_double_from_real_value (TWO32r, DFmode);
5348 tmp[0] = force_reg (V8DFmode, CONST0_RTX (V8DFmode));
5349 tmp[1] = force_reg (V8DFmode, ix86_build_const_vector (V8DFmode, 1, x));
5350 tmp[2] = gen_reg_rtx (V8DFmode);
5351 tmp[3] = gen_reg_rtx (V8SImode);
5352 k = gen_reg_rtx (QImode);
5354 emit_insn (gen_vec_extract_hi_v16si (tmp[3], operands[1]));
5355 emit_insn (gen_floatv8siv8df2 (tmp[2], tmp[3]));
5356 emit_insn (gen_rtx_SET (VOIDmode, k,
5357 gen_rtx_LT (QImode, tmp[2], tmp[0])));
5358 emit_insn (gen_addv8df3_mask (tmp[2], tmp[2], tmp[1], tmp[2], k));
5359 emit_move_insn (operands[0], tmp[2]);
5363 (define_expand "vec_unpacku_float_lo_v8si"
5364 [(match_operand:V4DF 0 "register_operand")
5365 (match_operand:V8SI 1 "nonimmediate_operand")]
5368 REAL_VALUE_TYPE TWO32r;
5372 real_ldexp (&TWO32r, &dconst1, 32);
5373 x = const_double_from_real_value (TWO32r, DFmode);
5375 tmp[0] = force_reg (V4DFmode, CONST0_RTX (V4DFmode));
5376 tmp[1] = force_reg (V4DFmode, ix86_build_const_vector (V4DFmode, 1, x));
5378 for (i = 2; i < 5; i++)
5379 tmp[i] = gen_reg_rtx (V4DFmode);
5380 emit_insn (gen_avx_cvtdq2pd256_2 (tmp[2], operands[1]));
5381 emit_insn (gen_rtx_SET (VOIDmode, tmp[3],
5382 gen_rtx_LT (V4DFmode, tmp[2], tmp[0])));
5383 emit_insn (gen_andv4df3 (tmp[4], tmp[3], tmp[1]));
5384 emit_insn (gen_addv4df3 (operands[0], tmp[2], tmp[4]));
5388 (define_expand "vec_unpacku_float_lo_v16si"
5389 [(match_operand:V8DF 0 "register_operand")
5390 (match_operand:V16SI 1 "nonimmediate_operand")]
5393 REAL_VALUE_TYPE TWO32r;
5396 real_ldexp (&TWO32r, &dconst1, 32);
5397 x = const_double_from_real_value (TWO32r, DFmode);
5399 tmp[0] = force_reg (V8DFmode, CONST0_RTX (V8DFmode));
5400 tmp[1] = force_reg (V8DFmode, ix86_build_const_vector (V8DFmode, 1, x));
5401 tmp[2] = gen_reg_rtx (V8DFmode);
5402 k = gen_reg_rtx (QImode);
5404 emit_insn (gen_avx512f_cvtdq2pd512_2 (tmp[2], operands[1]));
5405 emit_insn (gen_rtx_SET (VOIDmode, k,
5406 gen_rtx_LT (QImode, tmp[2], tmp[0])));
5407 emit_insn (gen_addv8df3_mask (tmp[2], tmp[2], tmp[1], tmp[2], k));
5408 emit_move_insn (operands[0], tmp[2]);
5412 (define_expand "vec_pack_trunc_<mode>"
5414 (float_truncate:<sf2dfmode>
5415 (match_operand:VF2_512_256 1 "nonimmediate_operand")))
5417 (float_truncate:<sf2dfmode>
5418 (match_operand:VF2_512_256 2 "nonimmediate_operand")))
5419 (set (match_operand:<ssePSmode> 0 "register_operand")
5420 (vec_concat:<ssePSmode>
5425 operands[3] = gen_reg_rtx (<sf2dfmode>mode);
5426 operands[4] = gen_reg_rtx (<sf2dfmode>mode);
5429 (define_expand "vec_pack_trunc_v2df"
5430 [(match_operand:V4SF 0 "register_operand")
5431 (match_operand:V2DF 1 "nonimmediate_operand")
5432 (match_operand:V2DF 2 "nonimmediate_operand")]
5437 if (TARGET_AVX && !TARGET_PREFER_AVX128 && optimize_insn_for_speed_p ())
5439 tmp0 = gen_reg_rtx (V4DFmode);
5440 tmp1 = force_reg (V2DFmode, operands[1]);
5442 emit_insn (gen_avx_vec_concatv4df (tmp0, tmp1, operands[2]));
5443 emit_insn (gen_avx_cvtpd2ps256 (operands[0], tmp0));
5447 tmp0 = gen_reg_rtx (V4SFmode);
5448 tmp1 = gen_reg_rtx (V4SFmode);
5450 emit_insn (gen_sse2_cvtpd2ps (tmp0, operands[1]));
5451 emit_insn (gen_sse2_cvtpd2ps (tmp1, operands[2]));
5452 emit_insn (gen_sse_movlhps (operands[0], tmp0, tmp1));
5457 (define_expand "vec_pack_sfix_trunc_v8df"
5458 [(match_operand:V16SI 0 "register_operand")
5459 (match_operand:V8DF 1 "nonimmediate_operand")
5460 (match_operand:V8DF 2 "nonimmediate_operand")]
5465 r1 = gen_reg_rtx (V8SImode);
5466 r2 = gen_reg_rtx (V8SImode);
5468 emit_insn (gen_fix_truncv8dfv8si2 (r1, operands[1]));
5469 emit_insn (gen_fix_truncv8dfv8si2 (r2, operands[2]));
5470 emit_insn (gen_avx_vec_concatv16si (operands[0], r1, r2));
5474 (define_expand "vec_pack_sfix_trunc_v4df"
5475 [(match_operand:V8SI 0 "register_operand")
5476 (match_operand:V4DF 1 "nonimmediate_operand")
5477 (match_operand:V4DF 2 "nonimmediate_operand")]
5482 r1 = gen_reg_rtx (V4SImode);
5483 r2 = gen_reg_rtx (V4SImode);
5485 emit_insn (gen_fix_truncv4dfv4si2 (r1, operands[1]));
5486 emit_insn (gen_fix_truncv4dfv4si2 (r2, operands[2]));
5487 emit_insn (gen_avx_vec_concatv8si (operands[0], r1, r2));
5491 (define_expand "vec_pack_sfix_trunc_v2df"
5492 [(match_operand:V4SI 0 "register_operand")
5493 (match_operand:V2DF 1 "nonimmediate_operand")
5494 (match_operand:V2DF 2 "nonimmediate_operand")]
5497 rtx tmp0, tmp1, tmp2;
5499 if (TARGET_AVX && !TARGET_PREFER_AVX128 && optimize_insn_for_speed_p ())
5501 tmp0 = gen_reg_rtx (V4DFmode);
5502 tmp1 = force_reg (V2DFmode, operands[1]);
5504 emit_insn (gen_avx_vec_concatv4df (tmp0, tmp1, operands[2]));
5505 emit_insn (gen_fix_truncv4dfv4si2 (operands[0], tmp0));
5509 tmp0 = gen_reg_rtx (V4SImode);
5510 tmp1 = gen_reg_rtx (V4SImode);
5511 tmp2 = gen_reg_rtx (V2DImode);
5513 emit_insn (gen_sse2_cvttpd2dq (tmp0, operands[1]));
5514 emit_insn (gen_sse2_cvttpd2dq (tmp1, operands[2]));
5515 emit_insn (gen_vec_interleave_lowv2di (tmp2,
5516 gen_lowpart (V2DImode, tmp0),
5517 gen_lowpart (V2DImode, tmp1)));
5518 emit_move_insn (operands[0], gen_lowpart (V4SImode, tmp2));
5523 (define_mode_attr ssepackfltmode
5524 [(V8DF "V16SI") (V4DF "V8SI") (V2DF "V4SI")])
5526 (define_expand "vec_pack_ufix_trunc_<mode>"
5527 [(match_operand:<ssepackfltmode> 0 "register_operand")
5528 (match_operand:VF2 1 "register_operand")
5529 (match_operand:VF2 2 "register_operand")]
5532 if (<MODE>mode == V8DFmode)
5536 r1 = gen_reg_rtx (V8SImode);
5537 r2 = gen_reg_rtx (V8SImode);
5539 emit_insn (gen_ufix_truncv8dfv8si2 (r1, operands[1]));
5540 emit_insn (gen_ufix_truncv8dfv8si2 (r2, operands[2]));
5541 emit_insn (gen_avx_vec_concatv16si (operands[0], r1, r2));
5546 tmp[0] = ix86_expand_adjust_ufix_to_sfix_si (operands[1], &tmp[2]);
5547 tmp[1] = ix86_expand_adjust_ufix_to_sfix_si (operands[2], &tmp[3]);
5548 tmp[4] = gen_reg_rtx (<ssepackfltmode>mode);
5549 emit_insn (gen_vec_pack_sfix_trunc_<mode> (tmp[4], tmp[0], tmp[1]));
5550 if (<ssepackfltmode>mode == V4SImode || TARGET_AVX2)
5552 tmp[5] = gen_reg_rtx (<ssepackfltmode>mode);
5553 ix86_expand_vec_extract_even_odd (tmp[5], tmp[2], tmp[3], 0);
5557 tmp[5] = gen_reg_rtx (V8SFmode);
5558 ix86_expand_vec_extract_even_odd (tmp[5], gen_lowpart (V8SFmode, tmp[2]),
5559 gen_lowpart (V8SFmode, tmp[3]), 0);
5560 tmp[5] = gen_lowpart (V8SImode, tmp[5]);
5562 tmp[6] = expand_simple_binop (<ssepackfltmode>mode, XOR, tmp[4], tmp[5],
5563 operands[0], 0, OPTAB_DIRECT);
5564 if (tmp[6] != operands[0])
5565 emit_move_insn (operands[0], tmp[6]);
5571 (define_expand "vec_pack_sfix_v4df"
5572 [(match_operand:V8SI 0 "register_operand")
5573 (match_operand:V4DF 1 "nonimmediate_operand")
5574 (match_operand:V4DF 2 "nonimmediate_operand")]
5579 r1 = gen_reg_rtx (V4SImode);
5580 r2 = gen_reg_rtx (V4SImode);
5582 emit_insn (gen_avx_cvtpd2dq256 (r1, operands[1]));
5583 emit_insn (gen_avx_cvtpd2dq256 (r2, operands[2]));
5584 emit_insn (gen_avx_vec_concatv8si (operands[0], r1, r2));
5588 (define_expand "vec_pack_sfix_v2df"
5589 [(match_operand:V4SI 0 "register_operand")
5590 (match_operand:V2DF 1 "nonimmediate_operand")
5591 (match_operand:V2DF 2 "nonimmediate_operand")]
5594 rtx tmp0, tmp1, tmp2;
5596 if (TARGET_AVX && !TARGET_PREFER_AVX128 && optimize_insn_for_speed_p ())
5598 tmp0 = gen_reg_rtx (V4DFmode);
5599 tmp1 = force_reg (V2DFmode, operands[1]);
5601 emit_insn (gen_avx_vec_concatv4df (tmp0, tmp1, operands[2]));
5602 emit_insn (gen_avx_cvtpd2dq256 (operands[0], tmp0));
5606 tmp0 = gen_reg_rtx (V4SImode);
5607 tmp1 = gen_reg_rtx (V4SImode);
5608 tmp2 = gen_reg_rtx (V2DImode);
5610 emit_insn (gen_sse2_cvtpd2dq (tmp0, operands[1]));
5611 emit_insn (gen_sse2_cvtpd2dq (tmp1, operands[2]));
5612 emit_insn (gen_vec_interleave_lowv2di (tmp2,
5613 gen_lowpart (V2DImode, tmp0),
5614 gen_lowpart (V2DImode, tmp1)));
5615 emit_move_insn (operands[0], gen_lowpart (V4SImode, tmp2));
5620 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
5622 ;; Parallel single-precision floating point element swizzling
5624 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
5626 (define_expand "sse_movhlps_exp"
5627 [(set (match_operand:V4SF 0 "nonimmediate_operand")
5630 (match_operand:V4SF 1 "nonimmediate_operand")
5631 (match_operand:V4SF 2 "nonimmediate_operand"))
5632 (parallel [(const_int 6)
5638 rtx dst = ix86_fixup_binary_operands (UNKNOWN, V4SFmode, operands);
5640 emit_insn (gen_sse_movhlps (dst, operands[1], operands[2]));
5642 /* Fix up the destination if needed. */
5643 if (dst != operands[0])
5644 emit_move_insn (operands[0], dst);
5649 (define_insn "sse_movhlps"
5650 [(set (match_operand:V4SF 0 "nonimmediate_operand" "=x,x,x,x,m")
5653 (match_operand:V4SF 1 "nonimmediate_operand" " 0,x,0,x,0")
5654 (match_operand:V4SF 2 "nonimmediate_operand" " x,x,o,o,x"))
5655 (parallel [(const_int 6)
5659 "TARGET_SSE && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
5661 movhlps\t{%2, %0|%0, %2}
5662 vmovhlps\t{%2, %1, %0|%0, %1, %2}
5663 movlps\t{%H2, %0|%0, %H2}
5664 vmovlps\t{%H2, %1, %0|%0, %1, %H2}
5665 %vmovhps\t{%2, %0|%q0, %2}"
5666 [(set_attr "isa" "noavx,avx,noavx,avx,*")
5667 (set_attr "type" "ssemov")
5668 (set_attr "ssememalign" "64")
5669 (set_attr "prefix" "orig,vex,orig,vex,maybe_vex")
5670 (set_attr "mode" "V4SF,V4SF,V2SF,V2SF,V2SF")])
5672 (define_expand "sse_movlhps_exp"
5673 [(set (match_operand:V4SF 0 "nonimmediate_operand")
5676 (match_operand:V4SF 1 "nonimmediate_operand")
5677 (match_operand:V4SF 2 "nonimmediate_operand"))
5678 (parallel [(const_int 0)
5684 rtx dst = ix86_fixup_binary_operands (UNKNOWN, V4SFmode, operands);
5686 emit_insn (gen_sse_movlhps (dst, operands[1], operands[2]));
5688 /* Fix up the destination if needed. */
5689 if (dst != operands[0])
5690 emit_move_insn (operands[0], dst);
5695 (define_insn "sse_movlhps"
5696 [(set (match_operand:V4SF 0 "nonimmediate_operand" "=x,x,x,x,o")
5699 (match_operand:V4SF 1 "nonimmediate_operand" " 0,x,0,x,0")
5700 (match_operand:V4SF 2 "nonimmediate_operand" " x,x,m,m,x"))
5701 (parallel [(const_int 0)
5705 "TARGET_SSE && ix86_binary_operator_ok (UNKNOWN, V4SFmode, operands)"
5707 movlhps\t{%2, %0|%0, %2}
5708 vmovlhps\t{%2, %1, %0|%0, %1, %2}
5709 movhps\t{%2, %0|%0, %q2}
5710 vmovhps\t{%2, %1, %0|%0, %1, %q2}
5711 %vmovlps\t{%2, %H0|%H0, %2}"
5712 [(set_attr "isa" "noavx,avx,noavx,avx,*")
5713 (set_attr "type" "ssemov")
5714 (set_attr "ssememalign" "64")
5715 (set_attr "prefix" "orig,vex,orig,vex,maybe_vex")
5716 (set_attr "mode" "V4SF,V4SF,V2SF,V2SF,V2SF")])
5718 (define_insn "<mask_codefor>avx512f_unpckhps512<mask_name>"
5719 [(set (match_operand:V16SF 0 "register_operand" "=v")
5722 (match_operand:V16SF 1 "register_operand" "v")
5723 (match_operand:V16SF 2 "nonimmediate_operand" "vm"))
5724 (parallel [(const_int 2) (const_int 18)
5725 (const_int 3) (const_int 19)
5726 (const_int 6) (const_int 22)
5727 (const_int 7) (const_int 23)
5728 (const_int 10) (const_int 26)
5729 (const_int 11) (const_int 27)
5730 (const_int 14) (const_int 30)
5731 (const_int 15) (const_int 31)])))]
5733 "vunpckhps\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
5734 [(set_attr "type" "sselog")
5735 (set_attr "prefix" "evex")
5736 (set_attr "mode" "V16SF")])
5738 ;; Recall that the 256-bit unpck insns only shuffle within their lanes.
5739 (define_insn "avx_unpckhps256<mask_name>"
5740 [(set (match_operand:V8SF 0 "register_operand" "=v")
5743 (match_operand:V8SF 1 "register_operand" "v")
5744 (match_operand:V8SF 2 "nonimmediate_operand" "vm"))
5745 (parallel [(const_int 2) (const_int 10)
5746 (const_int 3) (const_int 11)
5747 (const_int 6) (const_int 14)
5748 (const_int 7) (const_int 15)])))]
5749 "TARGET_AVX && <mask_avx512vl_condition>"
5750 "vunpckhps\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
5751 [(set_attr "type" "sselog")
5752 (set_attr "prefix" "vex")
5753 (set_attr "mode" "V8SF")])
5755 (define_expand "vec_interleave_highv8sf"
5759 (match_operand:V8SF 1 "register_operand" "x")
5760 (match_operand:V8SF 2 "nonimmediate_operand" "xm"))
5761 (parallel [(const_int 0) (const_int 8)
5762 (const_int 1) (const_int 9)
5763 (const_int 4) (const_int 12)
5764 (const_int 5) (const_int 13)])))
5770 (parallel [(const_int 2) (const_int 10)
5771 (const_int 3) (const_int 11)
5772 (const_int 6) (const_int 14)
5773 (const_int 7) (const_int 15)])))
5774 (set (match_operand:V8SF 0 "register_operand")
5779 (parallel [(const_int 4) (const_int 5)
5780 (const_int 6) (const_int 7)
5781 (const_int 12) (const_int 13)
5782 (const_int 14) (const_int 15)])))]
5785 operands[3] = gen_reg_rtx (V8SFmode);
5786 operands[4] = gen_reg_rtx (V8SFmode);
5789 (define_insn "vec_interleave_highv4sf<mask_name>"
5790 [(set (match_operand:V4SF 0 "register_operand" "=x,v")
5793 (match_operand:V4SF 1 "register_operand" "0,v")
5794 (match_operand:V4SF 2 "nonimmediate_operand" "xm,vm"))
5795 (parallel [(const_int 2) (const_int 6)
5796 (const_int 3) (const_int 7)])))]
5797 "TARGET_SSE && <mask_avx512vl_condition>"
5799 unpckhps\t{%2, %0|%0, %2}
5800 vunpckhps\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
5801 [(set_attr "isa" "noavx,avx")
5802 (set_attr "type" "sselog")
5803 (set_attr "prefix" "orig,vex")
5804 (set_attr "mode" "V4SF")])
5806 (define_insn "<mask_codefor>avx512f_unpcklps512<mask_name>"
5807 [(set (match_operand:V16SF 0 "register_operand" "=v")
5810 (match_operand:V16SF 1 "register_operand" "v")
5811 (match_operand:V16SF 2 "nonimmediate_operand" "vm"))
5812 (parallel [(const_int 0) (const_int 16)
5813 (const_int 1) (const_int 17)
5814 (const_int 4) (const_int 20)
5815 (const_int 5) (const_int 21)
5816 (const_int 8) (const_int 24)
5817 (const_int 9) (const_int 25)
5818 (const_int 12) (const_int 28)
5819 (const_int 13) (const_int 29)])))]
5821 "vunpcklps\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
5822 [(set_attr "type" "sselog")
5823 (set_attr "prefix" "evex")
5824 (set_attr "mode" "V16SF")])
5826 ;; Recall that the 256-bit unpck insns only shuffle within their lanes.
5827 (define_insn "avx_unpcklps256<mask_name>"
5828 [(set (match_operand:V8SF 0 "register_operand" "=v")
5831 (match_operand:V8SF 1 "register_operand" "v")
5832 (match_operand:V8SF 2 "nonimmediate_operand" "vm"))
5833 (parallel [(const_int 0) (const_int 8)
5834 (const_int 1) (const_int 9)
5835 (const_int 4) (const_int 12)
5836 (const_int 5) (const_int 13)])))]
5837 "TARGET_AVX && <mask_avx512vl_condition>"
5838 "vunpcklps\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
5839 [(set_attr "type" "sselog")
5840 (set_attr "prefix" "vex")
5841 (set_attr "mode" "V8SF")])
5843 (define_insn "unpcklps128_mask"
5844 [(set (match_operand:V4SF 0 "register_operand" "=v")
5848 (match_operand:V4SF 1 "register_operand" "v")
5849 (match_operand:V4SF 2 "nonimmediate_operand" "vm"))
5850 (parallel [(const_int 0) (const_int 4)
5851 (const_int 1) (const_int 5)]))
5852 (match_operand:V4SF 3 "vector_move_operand" "0C")
5853 (match_operand:QI 4 "register_operand" "Yk")))]
5855 "vunpcklps\t{%2, %1, %0%{%4%}%N3|%0%{%4%}%N3, %1, %2}"
5856 [(set_attr "type" "sselog")
5857 (set_attr "prefix" "evex")
5858 (set_attr "mode" "V4SF")])
5860 (define_expand "vec_interleave_lowv8sf"
5864 (match_operand:V8SF 1 "register_operand" "x")
5865 (match_operand:V8SF 2 "nonimmediate_operand" "xm"))
5866 (parallel [(const_int 0) (const_int 8)
5867 (const_int 1) (const_int 9)
5868 (const_int 4) (const_int 12)
5869 (const_int 5) (const_int 13)])))
5875 (parallel [(const_int 2) (const_int 10)
5876 (const_int 3) (const_int 11)
5877 (const_int 6) (const_int 14)
5878 (const_int 7) (const_int 15)])))
5879 (set (match_operand:V8SF 0 "register_operand")
5884 (parallel [(const_int 0) (const_int 1)
5885 (const_int 2) (const_int 3)
5886 (const_int 8) (const_int 9)
5887 (const_int 10) (const_int 11)])))]
5890 operands[3] = gen_reg_rtx (V8SFmode);
5891 operands[4] = gen_reg_rtx (V8SFmode);
5894 (define_insn "vec_interleave_lowv4sf"
5895 [(set (match_operand:V4SF 0 "register_operand" "=x,x")
5898 (match_operand:V4SF 1 "register_operand" "0,x")
5899 (match_operand:V4SF 2 "nonimmediate_operand" "xm,xm"))
5900 (parallel [(const_int 0) (const_int 4)
5901 (const_int 1) (const_int 5)])))]
5904 unpcklps\t{%2, %0|%0, %2}
5905 vunpcklps\t{%2, %1, %0|%0, %1, %2}"
5906 [(set_attr "isa" "noavx,avx")
5907 (set_attr "type" "sselog")
5908 (set_attr "prefix" "orig,vex")
5909 (set_attr "mode" "V4SF")])
5911 ;; These are modeled with the same vec_concat as the others so that we
5912 ;; capture users of shufps that can use the new instructions
5913 (define_insn "avx_movshdup256<mask_name>"
5914 [(set (match_operand:V8SF 0 "register_operand" "=v")
5917 (match_operand:V8SF 1 "nonimmediate_operand" "vm")
5919 (parallel [(const_int 1) (const_int 1)
5920 (const_int 3) (const_int 3)
5921 (const_int 5) (const_int 5)
5922 (const_int 7) (const_int 7)])))]
5923 "TARGET_AVX && <mask_avx512vl_condition>"
5924 "vmovshdup\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
5925 [(set_attr "type" "sse")
5926 (set_attr "prefix" "vex")
5927 (set_attr "mode" "V8SF")])
5929 (define_insn "sse3_movshdup<mask_name>"
5930 [(set (match_operand:V4SF 0 "register_operand" "=v")
5933 (match_operand:V4SF 1 "nonimmediate_operand" "vm")
5935 (parallel [(const_int 1)
5939 "TARGET_SSE3 && <mask_avx512vl_condition>"
5940 "%vmovshdup\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
5941 [(set_attr "type" "sse")
5942 (set_attr "prefix_rep" "1")
5943 (set_attr "prefix" "maybe_vex")
5944 (set_attr "mode" "V4SF")])
5946 (define_insn "<mask_codefor>avx512f_movshdup512<mask_name>"
5947 [(set (match_operand:V16SF 0 "register_operand" "=v")
5950 (match_operand:V16SF 1 "nonimmediate_operand" "vm")
5952 (parallel [(const_int 1) (const_int 1)
5953 (const_int 3) (const_int 3)
5954 (const_int 5) (const_int 5)
5955 (const_int 7) (const_int 7)
5956 (const_int 9) (const_int 9)
5957 (const_int 11) (const_int 11)
5958 (const_int 13) (const_int 13)
5959 (const_int 15) (const_int 15)])))]
5961 "vmovshdup\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
5962 [(set_attr "type" "sse")
5963 (set_attr "prefix" "evex")
5964 (set_attr "mode" "V16SF")])
5966 (define_insn "avx_movsldup256<mask_name>"
5967 [(set (match_operand:V8SF 0 "register_operand" "=v")
5970 (match_operand:V8SF 1 "nonimmediate_operand" "vm")
5972 (parallel [(const_int 0) (const_int 0)
5973 (const_int 2) (const_int 2)
5974 (const_int 4) (const_int 4)
5975 (const_int 6) (const_int 6)])))]
5976 "TARGET_AVX && <mask_avx512vl_condition>"
5977 "vmovsldup\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
5978 [(set_attr "type" "sse")
5979 (set_attr "prefix" "vex")
5980 (set_attr "mode" "V8SF")])
5982 (define_insn "sse3_movsldup<mask_name>"
5983 [(set (match_operand:V4SF 0 "register_operand" "=v")
5986 (match_operand:V4SF 1 "nonimmediate_operand" "vm")
5988 (parallel [(const_int 0)
5992 "TARGET_SSE3 && <mask_avx512vl_condition>"
5993 "%vmovsldup\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
5994 [(set_attr "type" "sse")
5995 (set_attr "prefix_rep" "1")
5996 (set_attr "prefix" "maybe_vex")
5997 (set_attr "mode" "V4SF")])
5999 (define_insn "<mask_codefor>avx512f_movsldup512<mask_name>"
6000 [(set (match_operand:V16SF 0 "register_operand" "=v")
6003 (match_operand:V16SF 1 "nonimmediate_operand" "vm")
6005 (parallel [(const_int 0) (const_int 0)
6006 (const_int 2) (const_int 2)
6007 (const_int 4) (const_int 4)
6008 (const_int 6) (const_int 6)
6009 (const_int 8) (const_int 8)
6010 (const_int 10) (const_int 10)
6011 (const_int 12) (const_int 12)
6012 (const_int 14) (const_int 14)])))]
6014 "vmovsldup\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
6015 [(set_attr "type" "sse")
6016 (set_attr "prefix" "evex")
6017 (set_attr "mode" "V16SF")])
6019 (define_expand "avx_shufps256<mask_expand4_name>"
6020 [(match_operand:V8SF 0 "register_operand")
6021 (match_operand:V8SF 1 "register_operand")
6022 (match_operand:V8SF 2 "nonimmediate_operand")
6023 (match_operand:SI 3 "const_int_operand")]
6026 int mask = INTVAL (operands[3]);
6027 emit_insn (gen_avx_shufps256_1<mask_expand4_name> (operands[0],
6030 GEN_INT ((mask >> 0) & 3),
6031 GEN_INT ((mask >> 2) & 3),
6032 GEN_INT (((mask >> 4) & 3) + 8),
6033 GEN_INT (((mask >> 6) & 3) + 8),
6034 GEN_INT (((mask >> 0) & 3) + 4),
6035 GEN_INT (((mask >> 2) & 3) + 4),
6036 GEN_INT (((mask >> 4) & 3) + 12),
6037 GEN_INT (((mask >> 6) & 3) + 12)
6038 <mask_expand4_args>));
6042 ;; One bit in mask selects 2 elements.
6043 (define_insn "avx_shufps256_1<mask_name>"
6044 [(set (match_operand:V8SF 0 "register_operand" "=v")
6047 (match_operand:V8SF 1 "register_operand" "v")
6048 (match_operand:V8SF 2 "nonimmediate_operand" "vm"))
6049 (parallel [(match_operand 3 "const_0_to_3_operand" )
6050 (match_operand 4 "const_0_to_3_operand" )
6051 (match_operand 5 "const_8_to_11_operand" )
6052 (match_operand 6 "const_8_to_11_operand" )
6053 (match_operand 7 "const_4_to_7_operand" )
6054 (match_operand 8 "const_4_to_7_operand" )
6055 (match_operand 9 "const_12_to_15_operand")
6056 (match_operand 10 "const_12_to_15_operand")])))]
6058 && <mask_avx512vl_condition>
6059 && (INTVAL (operands[3]) == (INTVAL (operands[7]) - 4)
6060 && INTVAL (operands[4]) == (INTVAL (operands[8]) - 4)
6061 && INTVAL (operands[5]) == (INTVAL (operands[9]) - 4)
6062 && INTVAL (operands[6]) == (INTVAL (operands[10]) - 4))"
6065 mask = INTVAL (operands[3]);
6066 mask |= INTVAL (operands[4]) << 2;
6067 mask |= (INTVAL (operands[5]) - 8) << 4;
6068 mask |= (INTVAL (operands[6]) - 8) << 6;
6069 operands[3] = GEN_INT (mask);
6071 return "vshufps\t{%3, %2, %1, %0<mask_operand11>|%0<mask_operand11>, %1, %2, %3}";
6073 [(set_attr "type" "sseshuf")
6074 (set_attr "length_immediate" "1")
6075 (set_attr "prefix" "<mask_prefix>")
6076 (set_attr "mode" "V8SF")])
6078 (define_expand "sse_shufps<mask_expand4_name>"
6079 [(match_operand:V4SF 0 "register_operand")
6080 (match_operand:V4SF 1 "register_operand")
6081 (match_operand:V4SF 2 "nonimmediate_operand")
6082 (match_operand:SI 3 "const_int_operand")]
6085 int mask = INTVAL (operands[3]);
6086 emit_insn (gen_sse_shufps_v4sf<mask_expand4_name> (operands[0],
6089 GEN_INT ((mask >> 0) & 3),
6090 GEN_INT ((mask >> 2) & 3),
6091 GEN_INT (((mask >> 4) & 3) + 4),
6092 GEN_INT (((mask >> 6) & 3) + 4)
6093 <mask_expand4_args>));
6097 (define_insn "sse_shufps_v4sf_mask"
6098 [(set (match_operand:V4SF 0 "register_operand" "=v")
6102 (match_operand:V4SF 1 "register_operand" "v")
6103 (match_operand:V4SF 2 "nonimmediate_operand" "vm"))
6104 (parallel [(match_operand 3 "const_0_to_3_operand")
6105 (match_operand 4 "const_0_to_3_operand")
6106 (match_operand 5 "const_4_to_7_operand")
6107 (match_operand 6 "const_4_to_7_operand")]))
6108 (match_operand:V4SF 7 "vector_move_operand" "0C")
6109 (match_operand:QI 8 "register_operand" "Yk")))]
6113 mask |= INTVAL (operands[3]) << 0;
6114 mask |= INTVAL (operands[4]) << 2;
6115 mask |= (INTVAL (operands[5]) - 4) << 4;
6116 mask |= (INTVAL (operands[6]) - 4) << 6;
6117 operands[3] = GEN_INT (mask);
6119 return "vshufps\t{%3, %2, %1, %0%{%8%}%N7|%0%{%8%}%N7, %1, %2, %3}";
6121 [(set_attr "type" "sseshuf")
6122 (set_attr "length_immediate" "1")
6123 (set_attr "prefix" "evex")
6124 (set_attr "mode" "V4SF")])
6126 (define_insn "sse_shufps_<mode>"
6127 [(set (match_operand:VI4F_128 0 "register_operand" "=x,x")
6128 (vec_select:VI4F_128
6129 (vec_concat:<ssedoublevecmode>
6130 (match_operand:VI4F_128 1 "register_operand" "0,x")
6131 (match_operand:VI4F_128 2 "nonimmediate_operand" "xm,xm"))
6132 (parallel [(match_operand 3 "const_0_to_3_operand")
6133 (match_operand 4 "const_0_to_3_operand")
6134 (match_operand 5 "const_4_to_7_operand")
6135 (match_operand 6 "const_4_to_7_operand")])))]
6139 mask |= INTVAL (operands[3]) << 0;
6140 mask |= INTVAL (operands[4]) << 2;
6141 mask |= (INTVAL (operands[5]) - 4) << 4;
6142 mask |= (INTVAL (operands[6]) - 4) << 6;
6143 operands[3] = GEN_INT (mask);
6145 switch (which_alternative)
6148 return "shufps\t{%3, %2, %0|%0, %2, %3}";
6150 return "vshufps\t{%3, %2, %1, %0|%0, %1, %2, %3}";
6155 [(set_attr "isa" "noavx,avx")
6156 (set_attr "type" "sseshuf")
6157 (set_attr "length_immediate" "1")
6158 (set_attr "prefix" "orig,vex")
6159 (set_attr "mode" "V4SF")])
6161 (define_insn "sse_storehps"
6162 [(set (match_operand:V2SF 0 "nonimmediate_operand" "=m,x,x")
6164 (match_operand:V4SF 1 "nonimmediate_operand" "x,x,o")
6165 (parallel [(const_int 2) (const_int 3)])))]
6168 %vmovhps\t{%1, %0|%q0, %1}
6169 %vmovhlps\t{%1, %d0|%d0, %1}
6170 %vmovlps\t{%H1, %d0|%d0, %H1}"
6171 [(set_attr "type" "ssemov")
6172 (set_attr "ssememalign" "64")
6173 (set_attr "prefix" "maybe_vex")
6174 (set_attr "mode" "V2SF,V4SF,V2SF")])
6176 (define_expand "sse_loadhps_exp"
6177 [(set (match_operand:V4SF 0 "nonimmediate_operand")
6180 (match_operand:V4SF 1 "nonimmediate_operand")
6181 (parallel [(const_int 0) (const_int 1)]))
6182 (match_operand:V2SF 2 "nonimmediate_operand")))]
6185 rtx dst = ix86_fixup_binary_operands (UNKNOWN, V4SFmode, operands);
6187 emit_insn (gen_sse_loadhps (dst, operands[1], operands[2]));
6189 /* Fix up the destination if needed. */
6190 if (dst != operands[0])
6191 emit_move_insn (operands[0], dst);
6196 (define_insn "sse_loadhps"
6197 [(set (match_operand:V4SF 0 "nonimmediate_operand" "=x,x,x,x,o")
6200 (match_operand:V4SF 1 "nonimmediate_operand" " 0,x,0,x,0")
6201 (parallel [(const_int 0) (const_int 1)]))
6202 (match_operand:V2SF 2 "nonimmediate_operand" " m,m,x,x,x")))]
6205 movhps\t{%2, %0|%0, %q2}
6206 vmovhps\t{%2, %1, %0|%0, %1, %q2}
6207 movlhps\t{%2, %0|%0, %2}
6208 vmovlhps\t{%2, %1, %0|%0, %1, %2}
6209 %vmovlps\t{%2, %H0|%H0, %2}"
6210 [(set_attr "isa" "noavx,avx,noavx,avx,*")
6211 (set_attr "type" "ssemov")
6212 (set_attr "ssememalign" "64")
6213 (set_attr "prefix" "orig,vex,orig,vex,maybe_vex")
6214 (set_attr "mode" "V2SF,V2SF,V4SF,V4SF,V2SF")])
6216 (define_insn "sse_storelps"
6217 [(set (match_operand:V2SF 0 "nonimmediate_operand" "=m,x,x")
6219 (match_operand:V4SF 1 "nonimmediate_operand" " x,x,m")
6220 (parallel [(const_int 0) (const_int 1)])))]
6223 %vmovlps\t{%1, %0|%q0, %1}
6224 %vmovaps\t{%1, %0|%0, %1}
6225 %vmovlps\t{%1, %d0|%d0, %q1}"
6226 [(set_attr "type" "ssemov")
6227 (set_attr "prefix" "maybe_vex")
6228 (set_attr "mode" "V2SF,V4SF,V2SF")])
6230 (define_expand "sse_loadlps_exp"
6231 [(set (match_operand:V4SF 0 "nonimmediate_operand")
6233 (match_operand:V2SF 2 "nonimmediate_operand")
6235 (match_operand:V4SF 1 "nonimmediate_operand")
6236 (parallel [(const_int 2) (const_int 3)]))))]
6239 rtx dst = ix86_fixup_binary_operands (UNKNOWN, V4SFmode, operands);
6241 emit_insn (gen_sse_loadlps (dst, operands[1], operands[2]));
6243 /* Fix up the destination if needed. */
6244 if (dst != operands[0])
6245 emit_move_insn (operands[0], dst);
6250 (define_insn "sse_loadlps"
6251 [(set (match_operand:V4SF 0 "nonimmediate_operand" "=x,x,x,x,m")
6253 (match_operand:V2SF 2 "nonimmediate_operand" " 0,x,m,m,x")
6255 (match_operand:V4SF 1 "nonimmediate_operand" " x,x,0,x,0")
6256 (parallel [(const_int 2) (const_int 3)]))))]
6259 shufps\t{$0xe4, %1, %0|%0, %1, 0xe4}
6260 vshufps\t{$0xe4, %1, %2, %0|%0, %2, %1, 0xe4}
6261 movlps\t{%2, %0|%0, %q2}
6262 vmovlps\t{%2, %1, %0|%0, %1, %q2}
6263 %vmovlps\t{%2, %0|%q0, %2}"
6264 [(set_attr "isa" "noavx,avx,noavx,avx,*")
6265 (set_attr "type" "sseshuf,sseshuf,ssemov,ssemov,ssemov")
6266 (set_attr "ssememalign" "64")
6267 (set_attr "length_immediate" "1,1,*,*,*")
6268 (set_attr "prefix" "orig,vex,orig,vex,maybe_vex")
6269 (set_attr "mode" "V4SF,V4SF,V2SF,V2SF,V2SF")])
6271 (define_insn "sse_movss"
6272 [(set (match_operand:V4SF 0 "register_operand" "=x,x")
6274 (match_operand:V4SF 2 "register_operand" " x,x")
6275 (match_operand:V4SF 1 "register_operand" " 0,x")
6279 movss\t{%2, %0|%0, %2}
6280 vmovss\t{%2, %1, %0|%0, %1, %2}"
6281 [(set_attr "isa" "noavx,avx")
6282 (set_attr "type" "ssemov")
6283 (set_attr "prefix" "orig,vex")
6284 (set_attr "mode" "SF")])
6286 (define_insn "avx2_vec_dup<mode>"
6287 [(set (match_operand:VF1_128_256 0 "register_operand" "=x")
6288 (vec_duplicate:VF1_128_256
6290 (match_operand:V4SF 1 "register_operand" "x")
6291 (parallel [(const_int 0)]))))]
6293 "vbroadcastss\t{%1, %0|%0, %1}"
6294 [(set_attr "type" "sselog1")
6295 (set_attr "prefix" "vex")
6296 (set_attr "mode" "<MODE>")])
6298 (define_insn "avx2_vec_dupv8sf_1"
6299 [(set (match_operand:V8SF 0 "register_operand" "=x")
6302 (match_operand:V8SF 1 "register_operand" "x")
6303 (parallel [(const_int 0)]))))]
6305 "vbroadcastss\t{%x1, %0|%0, %x1}"
6306 [(set_attr "type" "sselog1")
6307 (set_attr "prefix" "vex")
6308 (set_attr "mode" "V8SF")])
6310 (define_insn "avx512f_vec_dup<mode>_1"
6311 [(set (match_operand:VF_512 0 "register_operand" "=v")
6312 (vec_duplicate:VF_512
6313 (vec_select:<ssescalarmode>
6314 (match_operand:VF_512 1 "register_operand" "v")
6315 (parallel [(const_int 0)]))))]
6317 "vbroadcast<bcstscalarsuff>\t{%x1, %0|%0, %x1}"
6318 [(set_attr "type" "sselog1")
6319 (set_attr "prefix" "evex")
6320 (set_attr "mode" "<MODE>")])
6322 ;; Although insertps takes register source, we prefer
6323 ;; unpcklps with register source since it is shorter.
6324 (define_insn "*vec_concatv2sf_sse4_1"
6325 [(set (match_operand:V2SF 0 "register_operand" "=Yr,*x,x,Yr,*x,x,x,*y ,*y")
6327 (match_operand:SF 1 "nonimmediate_operand" " 0, 0,x, 0,0, x,m, 0 , m")
6328 (match_operand:SF 2 "vector_move_operand" " Yr,*x,x, m,m, m,C,*ym, C")))]
6331 unpcklps\t{%2, %0|%0, %2}
6332 unpcklps\t{%2, %0|%0, %2}
6333 vunpcklps\t{%2, %1, %0|%0, %1, %2}
6334 insertps\t{$0x10, %2, %0|%0, %2, 0x10}
6335 insertps\t{$0x10, %2, %0|%0, %2, 0x10}
6336 vinsertps\t{$0x10, %2, %1, %0|%0, %1, %2, 0x10}
6337 %vmovss\t{%1, %0|%0, %1}
6338 punpckldq\t{%2, %0|%0, %2}
6339 movd\t{%1, %0|%0, %1}"
6340 [(set_attr "isa" "noavx,noavx,avx,noavx,noavx,avx,*,*,*")
6341 (set_attr "type" "sselog,sselog,sselog,sselog,sselog,sselog,ssemov,mmxcvt,mmxmov")
6342 (set_attr "prefix_data16" "*,*,*,1,1,*,*,*,*")
6343 (set_attr "prefix_extra" "*,*,*,1,1,1,*,*,*")
6344 (set_attr "length_immediate" "*,*,*,1,1,1,*,*,*")
6345 (set_attr "prefix" "orig,orig,vex,orig,orig,vex,maybe_vex,orig,orig")
6346 (set_attr "mode" "V4SF,V4SF,V4SF,V4SF,V4SF,V4SF,SF,DI,DI")])
6348 ;; ??? In theory we can match memory for the MMX alternative, but allowing
6349 ;; nonimmediate_operand for operand 2 and *not* allowing memory for the SSE
6350 ;; alternatives pretty much forces the MMX alternative to be chosen.
6351 (define_insn "*vec_concatv2sf_sse"
6352 [(set (match_operand:V2SF 0 "register_operand" "=x,x,*y,*y")
6354 (match_operand:SF 1 "nonimmediate_operand" " 0,m, 0, m")
6355 (match_operand:SF 2 "reg_or_0_operand" " x,C,*y, C")))]
6358 unpcklps\t{%2, %0|%0, %2}
6359 movss\t{%1, %0|%0, %1}
6360 punpckldq\t{%2, %0|%0, %2}
6361 movd\t{%1, %0|%0, %1}"
6362 [(set_attr "type" "sselog,ssemov,mmxcvt,mmxmov")
6363 (set_attr "mode" "V4SF,SF,DI,DI")])
6365 (define_insn "*vec_concatv4sf"
6366 [(set (match_operand:V4SF 0 "register_operand" "=x,x,x,x")
6368 (match_operand:V2SF 1 "register_operand" " 0,x,0,x")
6369 (match_operand:V2SF 2 "nonimmediate_operand" " x,x,m,m")))]
6372 movlhps\t{%2, %0|%0, %2}
6373 vmovlhps\t{%2, %1, %0|%0, %1, %2}
6374 movhps\t{%2, %0|%0, %q2}
6375 vmovhps\t{%2, %1, %0|%0, %1, %q2}"
6376 [(set_attr "isa" "noavx,avx,noavx,avx")
6377 (set_attr "type" "ssemov")
6378 (set_attr "prefix" "orig,vex,orig,vex")
6379 (set_attr "mode" "V4SF,V4SF,V2SF,V2SF")])
6381 (define_expand "vec_init<mode>"
6382 [(match_operand:V_128 0 "register_operand")
6386 ix86_expand_vector_init (false, operands[0], operands[1]);
6390 ;; Avoid combining registers from different units in a single alternative,
6391 ;; see comment above inline_secondary_memory_needed function in i386.c
6392 (define_insn "vec_set<mode>_0"
6393 [(set (match_operand:VI4F_128 0 "nonimmediate_operand"
6394 "=Yr,*v,v,v ,x,x,v,Yr ,*x ,x ,m ,m ,m")
6396 (vec_duplicate:VI4F_128
6397 (match_operand:<ssescalarmode> 2 "general_operand"
6398 " Yr,*v,m,*r,m,x,v,*rm,*rm,*rm,!x,!*re,!*fF"))
6399 (match_operand:VI4F_128 1 "vector_move_operand"
6400 " C , C,C,C ,C,0,v,0 ,0 ,x ,0 ,0 ,0")
6404 %vinsertps\t{$0xe, %d2, %0|%0, %d2, 0xe}
6405 %vinsertps\t{$0xe, %d2, %0|%0, %d2, 0xe}
6406 %vmov<ssescalarmodesuffix>\t{%2, %0|%0, %2}
6407 %vmovd\t{%2, %0|%0, %2}
6408 movss\t{%2, %0|%0, %2}
6409 movss\t{%2, %0|%0, %2}
6410 vmovss\t{%2, %1, %0|%0, %1, %2}
6411 pinsrd\t{$0, %2, %0|%0, %2, 0}
6412 pinsrd\t{$0, %2, %0|%0, %2, 0}
6413 vpinsrd\t{$0, %2, %1, %0|%0, %1, %2, 0}
6417 [(set_attr "isa" "sse4,sse4,sse2,sse2,noavx,noavx,avx,sse4_noavx,sse4_noavx,avx,*,*,*")
6419 (cond [(eq_attr "alternative" "0,1,7,8,9")
6420 (const_string "sselog")
6421 (eq_attr "alternative" "11")
6422 (const_string "imov")
6423 (eq_attr "alternative" "12")
6424 (const_string "fmov")
6426 (const_string "ssemov")))
6427 (set_attr "prefix_extra" "*,*,*,*,*,*,*,1,1,1,*,*,*")
6428 (set_attr "length_immediate" "*,*,*,*,*,*,*,1,1,1,*,*,*")
6429 (set_attr "prefix" "maybe_vex,maybe_vex,maybe_vex,maybe_vex,orig,orig,vex,orig,orig,vex,*,*,*")
6430 (set_attr "mode" "SF,SF,<ssescalarmode>,SI,SF,SF,SF,TI,TI,TI,*,*,*")])
6432 ;; A subset is vec_setv4sf.
6433 (define_insn "*vec_setv4sf_sse4_1"
6434 [(set (match_operand:V4SF 0 "register_operand" "=Yr,*x,x")
6437 (match_operand:SF 2 "nonimmediate_operand" "Yrm,*xm,xm"))
6438 (match_operand:V4SF 1 "register_operand" "0,0,x")
6439 (match_operand:SI 3 "const_int_operand")))]
6441 && ((unsigned) exact_log2 (INTVAL (operands[3]))
6442 < GET_MODE_NUNITS (V4SFmode))"
6444 operands[3] = GEN_INT (exact_log2 (INTVAL (operands[3])) << 4);
6445 switch (which_alternative)
6449 return "insertps\t{%3, %2, %0|%0, %2, %3}";
6451 return "vinsertps\t{%3, %2, %1, %0|%0, %1, %2, %3}";
6456 [(set_attr "isa" "noavx,noavx,avx")
6457 (set_attr "type" "sselog")
6458 (set_attr "prefix_data16" "1,1,*")
6459 (set_attr "prefix_extra" "1")
6460 (set_attr "length_immediate" "1")
6461 (set_attr "prefix" "orig,orig,vex")
6462 (set_attr "mode" "V4SF")])
6464 (define_insn "sse4_1_insertps"
6465 [(set (match_operand:V4SF 0 "register_operand" "=Yr,*x,x")
6466 (unspec:V4SF [(match_operand:V4SF 2 "nonimmediate_operand" "Yrm,*xm,xm")
6467 (match_operand:V4SF 1 "register_operand" "0,0,x")
6468 (match_operand:SI 3 "const_0_to_255_operand" "n,n,n")]
6472 if (MEM_P (operands[2]))
6474 unsigned count_s = INTVAL (operands[3]) >> 6;
6476 operands[3] = GEN_INT (INTVAL (operands[3]) & 0x3f);
6477 operands[2] = adjust_address_nv (operands[2], SFmode, count_s * 4);
6479 switch (which_alternative)
6483 return "insertps\t{%3, %2, %0|%0, %2, %3}";
6485 return "vinsertps\t{%3, %2, %1, %0|%0, %1, %2, %3}";
6490 [(set_attr "isa" "noavx,noavx,avx")
6491 (set_attr "type" "sselog")
6492 (set_attr "prefix_data16" "1,1,*")
6493 (set_attr "prefix_extra" "1")
6494 (set_attr "length_immediate" "1")
6495 (set_attr "prefix" "orig,orig,vex")
6496 (set_attr "mode" "V4SF")])
6499 [(set (match_operand:VI4F_128 0 "memory_operand")
6501 (vec_duplicate:VI4F_128
6502 (match_operand:<ssescalarmode> 1 "nonmemory_operand"))
6505 "TARGET_SSE && reload_completed"
6506 [(set (match_dup 0) (match_dup 1))]
6507 "operands[0] = adjust_address (operands[0], <ssescalarmode>mode, 0);")
6509 (define_expand "vec_set<mode>"
6510 [(match_operand:V 0 "register_operand")
6511 (match_operand:<ssescalarmode> 1 "register_operand")
6512 (match_operand 2 "const_int_operand")]
6515 ix86_expand_vector_set (false, operands[0], operands[1],
6516 INTVAL (operands[2]));
6520 (define_insn_and_split "*vec_extractv4sf_0"
6521 [(set (match_operand:SF 0 "nonimmediate_operand" "=x,m,f,r")
6523 (match_operand:V4SF 1 "nonimmediate_operand" "xm,x,m,m")
6524 (parallel [(const_int 0)])))]
6525 "TARGET_SSE && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
6527 "&& reload_completed"
6528 [(set (match_dup 0) (match_dup 1))]
6530 if (REG_P (operands[1]))
6531 operands[1] = gen_rtx_REG (SFmode, REGNO (operands[1]));
6533 operands[1] = adjust_address (operands[1], SFmode, 0);
6536 (define_insn_and_split "*sse4_1_extractps"
6537 [(set (match_operand:SF 0 "nonimmediate_operand" "=rm,rm,x,x")
6539 (match_operand:V4SF 1 "register_operand" "Yr,*x,0,x")
6540 (parallel [(match_operand:SI 2 "const_0_to_3_operand" "n,n,n,n")])))]
6543 %vextractps\t{%2, %1, %0|%0, %1, %2}
6544 %vextractps\t{%2, %1, %0|%0, %1, %2}
6547 "&& reload_completed && SSE_REG_P (operands[0])"
6550 rtx dest = gen_rtx_REG (V4SFmode, REGNO (operands[0]));
6551 switch (INTVAL (operands[2]))
6555 emit_insn (gen_sse_shufps_v4sf (dest, operands[1], operands[1],
6556 operands[2], operands[2],
6557 GEN_INT (INTVAL (operands[2]) + 4),
6558 GEN_INT (INTVAL (operands[2]) + 4)));
6561 emit_insn (gen_vec_interleave_highv4sf (dest, operands[1], operands[1]));
6564 /* 0 should be handled by the *vec_extractv4sf_0 pattern above. */
6569 [(set_attr "isa" "*,*,noavx,avx")
6570 (set_attr "type" "sselog,sselog,*,*")
6571 (set_attr "prefix_data16" "1,1,*,*")
6572 (set_attr "prefix_extra" "1,1,*,*")
6573 (set_attr "length_immediate" "1,1,*,*")
6574 (set_attr "prefix" "maybe_vex,maybe_vex,*,*")
6575 (set_attr "mode" "V4SF,V4SF,*,*")])
6577 (define_insn_and_split "*vec_extractv4sf_mem"
6578 [(set (match_operand:SF 0 "register_operand" "=x,*r,f")
6580 (match_operand:V4SF 1 "memory_operand" "o,o,o")
6581 (parallel [(match_operand 2 "const_0_to_3_operand" "n,n,n")])))]
6584 "&& reload_completed"
6585 [(set (match_dup 0) (match_dup 1))]
6587 operands[1] = adjust_address (operands[1], SFmode, INTVAL (operands[2]) * 4);
6590 (define_mode_attr extract_type
6591 [(V16SF "avx512f") (V16SI "avx512f") (V8DF "avx512dq") (V8DI "avx512dq")])
6593 (define_mode_attr extract_suf
6594 [(V16SF "32x4") (V16SI "32x4") (V8DF "64x2") (V8DI "64x2")])
6596 (define_mode_iterator AVX512_VEC
6597 [(V8DF "TARGET_AVX512DQ") (V8DI "TARGET_AVX512DQ") V16SF V16SI])
6599 (define_expand "<extract_type>_vextract<shuffletype><extract_suf>_mask"
6600 [(match_operand:<ssequartermode> 0 "nonimmediate_operand")
6601 (match_operand:AVX512_VEC 1 "register_operand")
6602 (match_operand:SI 2 "const_0_to_3_operand")
6603 (match_operand:<ssequartermode> 3 "nonimmediate_operand")
6604 (match_operand:QI 4 "register_operand")]
6608 mask = INTVAL (operands[2]);
6610 if (MEM_P (operands[0]) && GET_CODE (operands[3]) == CONST_VECTOR)
6611 operands[0] = force_reg (<ssequartermode>mode, operands[0]);
6613 if (<MODE>mode == V16SImode || <MODE>mode == V16SFmode)
6614 emit_insn (gen_avx512f_vextract<shuffletype>32x4_1_mask (operands[0],
6615 operands[1], GEN_INT (mask * 4), GEN_INT (mask * 4 + 1),
6616 GEN_INT (mask * 4 + 2), GEN_INT (mask * 4 + 3), operands[3],
6619 emit_insn (gen_avx512dq_vextract<shuffletype>64x2_1_mask (operands[0],
6620 operands[1], GEN_INT (mask * 2), GEN_INT (mask * 2 + 1), operands[3],
6625 (define_insn "avx512dq_vextract<shuffletype>64x2_1_maskm"
6626 [(set (match_operand:<ssequartermode> 0 "memory_operand" "=m")
6627 (vec_merge:<ssequartermode>
6628 (vec_select:<ssequartermode>
6629 (match_operand:V8FI 1 "register_operand" "v")
6630 (parallel [(match_operand 2 "const_0_to_7_operand")
6631 (match_operand 3 "const_0_to_7_operand")]))
6632 (match_operand:<ssequartermode> 4 "memory_operand" "0")
6633 (match_operand:QI 5 "register_operand" "k")))]
6635 && (INTVAL (operands[2]) % 2 == 0)
6636 && (INTVAL (operands[2]) == INTVAL (operands[3]) - 1 )"
6638 operands[2] = GEN_INT ((INTVAL (operands[2])) >> 1);
6639 return "vextract<shuffletype>64x2\t{%2, %1, %0%{%5%}|%0%{%5%}, %1, %2}";
6641 [(set_attr "type" "sselog")
6642 (set_attr "prefix_extra" "1")
6643 (set_attr "length_immediate" "1")
6644 (set_attr "memory" "store")
6645 (set_attr "prefix" "evex")
6646 (set_attr "mode" "<sseinsnmode>")])
6648 (define_insn "avx512f_vextract<shuffletype>32x4_1_maskm"
6649 [(set (match_operand:<ssequartermode> 0 "memory_operand" "=m")
6650 (vec_merge:<ssequartermode>
6651 (vec_select:<ssequartermode>
6652 (match_operand:V16FI 1 "register_operand" "v")
6653 (parallel [(match_operand 2 "const_0_to_15_operand")
6654 (match_operand 3 "const_0_to_15_operand")
6655 (match_operand 4 "const_0_to_15_operand")
6656 (match_operand 5 "const_0_to_15_operand")]))
6657 (match_operand:<ssequartermode> 6 "memory_operand" "0")
6658 (match_operand:QI 7 "register_operand" "Yk")))]
6660 && ((INTVAL (operands[2]) % 4 == 0)
6661 && INTVAL (operands[2]) == (INTVAL (operands[3]) - 1)
6662 && INTVAL (operands[3]) == (INTVAL (operands[4]) - 1)
6663 && INTVAL (operands[4]) == (INTVAL (operands[5]) - 1))"
6665 operands[2] = GEN_INT ((INTVAL (operands[2])) >> 2);
6666 return "vextract<shuffletype>32x4\t{%2, %1, %0%{%7%}|%0%{%7%}, %1, %2}";
6668 [(set_attr "type" "sselog")
6669 (set_attr "prefix_extra" "1")
6670 (set_attr "length_immediate" "1")
6671 (set_attr "memory" "store")
6672 (set_attr "prefix" "evex")
6673 (set_attr "mode" "<sseinsnmode>")])
6675 (define_insn "<mask_codefor>avx512dq_vextract<shuffletype>64x2_1<mask_name>"
6676 [(set (match_operand:<ssequartermode> 0 "<store_mask_predicate>" "=<store_mask_constraint>")
6677 (vec_select:<ssequartermode>
6678 (match_operand:V8FI 1 "register_operand" "v")
6679 (parallel [(match_operand 2 "const_0_to_7_operand")
6680 (match_operand 3 "const_0_to_7_operand")])))]
6681 "TARGET_AVX512DQ && (INTVAL (operands[2]) == INTVAL (operands[3]) - 1)"
6683 operands[2] = GEN_INT ((INTVAL (operands[2])) >> 1);
6684 return "vextract<shuffletype>64x2\t{%2, %1, %0<mask_operand4>|%0<mask_operand4>, %1, %2}";
6686 [(set_attr "type" "sselog1")
6687 (set_attr "prefix_extra" "1")
6688 (set_attr "length_immediate" "1")
6689 (set_attr "prefix" "evex")
6690 (set_attr "mode" "<sseinsnmode>")])
6692 (define_insn "<mask_codefor>avx512f_vextract<shuffletype>32x4_1<mask_name>"
6693 [(set (match_operand:<ssequartermode> 0 "<store_mask_predicate>" "=<store_mask_constraint>")
6694 (vec_select:<ssequartermode>
6695 (match_operand:V16FI 1 "register_operand" "v")
6696 (parallel [(match_operand 2 "const_0_to_15_operand")
6697 (match_operand 3 "const_0_to_15_operand")
6698 (match_operand 4 "const_0_to_15_operand")
6699 (match_operand 5 "const_0_to_15_operand")])))]
6701 && (INTVAL (operands[2]) == (INTVAL (operands[3]) - 1)
6702 && INTVAL (operands[3]) == (INTVAL (operands[4]) - 1)
6703 && INTVAL (operands[4]) == (INTVAL (operands[5]) - 1))"
6705 operands[2] = GEN_INT ((INTVAL (operands[2])) >> 2);
6706 return "vextract<shuffletype>32x4\t{%2, %1, %0<mask_operand6>|%0<mask_operand6>, %1, %2}";
6708 [(set_attr "type" "sselog1")
6709 (set_attr "prefix_extra" "1")
6710 (set_attr "length_immediate" "1")
6711 (set_attr "prefix" "evex")
6712 (set_attr "mode" "<sseinsnmode>")])
6714 (define_mode_attr extract_type_2
6715 [(V16SF "avx512dq") (V16SI "avx512dq") (V8DF "avx512f") (V8DI "avx512f")])
6717 (define_mode_attr extract_suf_2
6718 [(V16SF "32x8") (V16SI "32x8") (V8DF "64x4") (V8DI "64x4")])
6720 (define_mode_iterator AVX512_VEC_2
6721 [(V16SF "TARGET_AVX512DQ") (V16SI "TARGET_AVX512DQ") V8DF V8DI])
6723 (define_expand "<extract_type_2>_vextract<shuffletype><extract_suf_2>_mask"
6724 [(match_operand:<ssehalfvecmode> 0 "nonimmediate_operand")
6725 (match_operand:AVX512_VEC_2 1 "register_operand")
6726 (match_operand:SI 2 "const_0_to_1_operand")
6727 (match_operand:<ssehalfvecmode> 3 "nonimmediate_operand")
6728 (match_operand:QI 4 "register_operand")]
6731 rtx (*insn)(rtx, rtx, rtx, rtx);
6733 if (MEM_P (operands[0]) && GET_CODE (operands[3]) == CONST_VECTOR)
6734 operands[0] = force_reg (<ssequartermode>mode, operands[0]);
6736 switch (INTVAL (operands[2]))
6739 insn = gen_vec_extract_lo_<mode>_mask;
6742 insn = gen_vec_extract_hi_<mode>_mask;
6748 emit_insn (insn (operands[0], operands[1], operands[3], operands[4]));
6753 [(set (match_operand:<ssehalfvecmode> 0 "nonimmediate_operand")
6754 (vec_select:<ssehalfvecmode>
6755 (match_operand:V8FI 1 "nonimmediate_operand")
6756 (parallel [(const_int 0) (const_int 1)
6757 (const_int 2) (const_int 3)])))]
6758 "TARGET_AVX512F && !(MEM_P (operands[0]) && MEM_P (operands[1]))
6759 && reload_completed"
6762 rtx op1 = operands[1];
6764 op1 = gen_rtx_REG (<ssehalfvecmode>mode, REGNO (op1));
6766 op1 = gen_lowpart (<ssehalfvecmode>mode, op1);
6767 emit_move_insn (operands[0], op1);
6771 (define_insn "vec_extract_lo_<mode>_maskm"
6772 [(set (match_operand:<ssehalfvecmode> 0 "memory_operand" "=m")
6773 (vec_merge:<ssehalfvecmode>
6774 (vec_select:<ssehalfvecmode>
6775 (match_operand:V8FI 1 "register_operand" "v")
6776 (parallel [(const_int 0) (const_int 1)
6777 (const_int 2) (const_int 3)]))
6778 (match_operand:<ssehalfvecmode> 2 "memory_operand" "0")
6779 (match_operand:QI 3 "register_operand" "Yk")))]
6781 "vextract<shuffletype>64x4\t{$0x0, %1, %0%{%3%}|%0%{%3%}, %1, 0x0}"
6782 [(set_attr "type" "sselog1")
6783 (set_attr "prefix_extra" "1")
6784 (set_attr "length_immediate" "1")
6785 (set_attr "prefix" "evex")
6786 (set_attr "mode" "<sseinsnmode>")])
6788 (define_insn "vec_extract_lo_<mode><mask_name>"
6789 [(set (match_operand:<ssehalfvecmode> 0 "<store_mask_predicate>" "=<store_mask_constraint>,v")
6790 (vec_select:<ssehalfvecmode>
6791 (match_operand:V8FI 1 "nonimmediate_operand" "v,m")
6792 (parallel [(const_int 0) (const_int 1)
6793 (const_int 2) (const_int 3)])))]
6794 "TARGET_AVX512F && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
6797 return "vextract<shuffletype>64x4\t{$0x0, %1, %0<mask_operand2>|%0<mask_operand2>, %1, 0x0}";
6801 [(set_attr "type" "sselog1")
6802 (set_attr "prefix_extra" "1")
6803 (set_attr "length_immediate" "1")
6804 (set_attr "prefix" "evex")
6805 (set_attr "mode" "<sseinsnmode>")])
6807 (define_insn "vec_extract_hi_<mode>_maskm"
6808 [(set (match_operand:<ssehalfvecmode> 0 "memory_operand" "=m")
6809 (vec_merge:<ssehalfvecmode>
6810 (vec_select:<ssehalfvecmode>
6811 (match_operand:V8FI 1 "register_operand" "v")
6812 (parallel [(const_int 4) (const_int 5)
6813 (const_int 6) (const_int 7)]))
6814 (match_operand:<ssehalfvecmode> 2 "memory_operand" "0")
6815 (match_operand:QI 3 "register_operand" "Yk")))]
6817 "vextract<shuffletype>64x4\t{$0x1, %1, %0%{%3%}|%0%{%3%}, %1, 0x1}"
6818 [(set_attr "type" "sselog")
6819 (set_attr "prefix_extra" "1")
6820 (set_attr "length_immediate" "1")
6821 (set_attr "memory" "store")
6822 (set_attr "prefix" "evex")
6823 (set_attr "mode" "<sseinsnmode>")])
6825 (define_insn "vec_extract_hi_<mode><mask_name>"
6826 [(set (match_operand:<ssehalfvecmode> 0 "<store_mask_predicate>" "=<store_mask_constraint>")
6827 (vec_select:<ssehalfvecmode>
6828 (match_operand:V8FI 1 "register_operand" "v")
6829 (parallel [(const_int 4) (const_int 5)
6830 (const_int 6) (const_int 7)])))]
6832 "vextract<shuffletype>64x4\t{$0x1, %1, %0<mask_operand2>|%0<mask_operand2>, %1, 0x1}"
6833 [(set_attr "type" "sselog1")
6834 (set_attr "prefix_extra" "1")
6835 (set_attr "length_immediate" "1")
6836 (set_attr "prefix" "evex")
6837 (set_attr "mode" "<sseinsnmode>")])
6839 (define_insn "vec_extract_hi_<mode>_maskm"
6840 [(set (match_operand:<ssehalfvecmode> 0 "memory_operand" "=m")
6841 (vec_merge:<ssehalfvecmode>
6842 (vec_select:<ssehalfvecmode>
6843 (match_operand:V16FI 1 "register_operand" "v")
6844 (parallel [(const_int 8) (const_int 9)
6845 (const_int 10) (const_int 11)
6846 (const_int 12) (const_int 13)
6847 (const_int 14) (const_int 15)]))
6848 (match_operand:<ssehalfvecmode> 2 "memory_operand" "0")
6849 (match_operand:QI 3 "register_operand" "k")))]
6851 "vextract<shuffletype>32x8\t{$0x1, %1, %0%{%3%}|%0%{%3%}, %1, 0x1}"
6852 [(set_attr "type" "sselog1")
6853 (set_attr "prefix_extra" "1")
6854 (set_attr "length_immediate" "1")
6855 (set_attr "prefix" "evex")
6856 (set_attr "mode" "<sseinsnmode>")])
6858 (define_insn "vec_extract_hi_<mode><mask_name>"
6859 [(set (match_operand:<ssehalfvecmode> 0 "<store_mask_predicate>" "=<store_mask_constraint>,vm")
6860 (vec_select:<ssehalfvecmode>
6861 (match_operand:V16FI 1 "register_operand" "v,v")
6862 (parallel [(const_int 8) (const_int 9)
6863 (const_int 10) (const_int 11)
6864 (const_int 12) (const_int 13)
6865 (const_int 14) (const_int 15)])))]
6866 "TARGET_AVX512F && <mask_avx512dq_condition>"
6868 vextract<shuffletype>32x8\t{$0x1, %1, %0<mask_operand2>|%0<mask_operand2>, %1, 0x1}
6869 vextracti64x4\t{$0x1, %1, %0|%0, %1, 0x1}"
6870 [(set_attr "type" "sselog1")
6871 (set_attr "prefix_extra" "1")
6872 (set_attr "isa" "avx512dq,noavx512dq")
6873 (set_attr "length_immediate" "1")
6874 (set_attr "prefix" "evex")
6875 (set_attr "mode" "<sseinsnmode>")])
6877 (define_expand "avx512vl_vextractf128<mode>"
6878 [(match_operand:<ssehalfvecmode> 0 "nonimmediate_operand")
6879 (match_operand:VI48F_256 1 "register_operand")
6880 (match_operand:SI 2 "const_0_to_1_operand")
6881 (match_operand:<ssehalfvecmode> 3 "vector_move_operand")
6882 (match_operand:QI 4 "register_operand")]
6883 "TARGET_AVX512DQ && TARGET_AVX512VL"
6885 rtx (*insn)(rtx, rtx, rtx, rtx);
6887 if (MEM_P (operands[0]) && GET_CODE (operands[3]) == CONST_VECTOR)
6888 operands[0] = force_reg (<ssehalfvecmode>mode, operands[0]);
6890 switch (INTVAL (operands[2]))
6893 insn = gen_vec_extract_lo_<mode>_mask;
6896 insn = gen_vec_extract_hi_<mode>_mask;
6902 emit_insn (insn (operands[0], operands[1], operands[3], operands[4]));
6906 (define_expand "avx_vextractf128<mode>"
6907 [(match_operand:<ssehalfvecmode> 0 "nonimmediate_operand")
6908 (match_operand:V_256 1 "register_operand")
6909 (match_operand:SI 2 "const_0_to_1_operand")]
6912 rtx (*insn)(rtx, rtx);
6914 switch (INTVAL (operands[2]))
6917 insn = gen_vec_extract_lo_<mode>;
6920 insn = gen_vec_extract_hi_<mode>;
6926 emit_insn (insn (operands[0], operands[1]));
6930 (define_insn "vec_extract_lo_<mode><mask_name>"
6931 [(set (match_operand:<ssehalfvecmode> 0 "nonimmediate_operand" "=v,m")
6932 (vec_select:<ssehalfvecmode>
6933 (match_operand:V16FI 1 "nonimmediate_operand" "vm,v")
6934 (parallel [(const_int 0) (const_int 1)
6935 (const_int 2) (const_int 3)
6936 (const_int 4) (const_int 5)
6937 (const_int 6) (const_int 7)])))]
6939 && <mask_mode512bit_condition>
6940 && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
6943 return "vextract<shuffletype>32x8\t{$0x0, %1, %0<mask_operand2>|%0<mask_operand2>, %1, 0x0}";
6949 [(set (match_operand:<ssehalfvecmode> 0 "nonimmediate_operand")
6950 (vec_select:<ssehalfvecmode>
6951 (match_operand:V16FI 1 "nonimmediate_operand")
6952 (parallel [(const_int 0) (const_int 1)
6953 (const_int 2) (const_int 3)
6954 (const_int 4) (const_int 5)
6955 (const_int 6) (const_int 7)])))]
6956 "TARGET_AVX512F && !(MEM_P (operands[0]) && MEM_P (operands[1]))
6957 && reload_completed"
6960 rtx op1 = operands[1];
6962 op1 = gen_rtx_REG (<ssehalfvecmode>mode, REGNO (op1));
6964 op1 = gen_lowpart (<ssehalfvecmode>mode, op1);
6965 emit_move_insn (operands[0], op1);
6969 (define_insn "vec_extract_lo_<mode><mask_name>"
6970 [(set (match_operand:<ssehalfvecmode> 0 "<store_mask_predicate>" "=v,m")
6971 (vec_select:<ssehalfvecmode>
6972 (match_operand:VI8F_256 1 "nonimmediate_operand" "vm,v")
6973 (parallel [(const_int 0) (const_int 1)])))]
6975 && <mask_avx512vl_condition> && <mask_avx512dq_condition>
6976 && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
6979 return "vextract<shuffletype>64x2\t{$0x0, %1, %0%{%3%}|%0%{%3%}, %1, 0x0}";
6983 [(set_attr "type" "sselog")
6984 (set_attr "prefix_extra" "1")
6985 (set_attr "length_immediate" "1")
6986 (set_attr "memory" "none,store")
6987 (set_attr "prefix" "evex")
6988 (set_attr "mode" "XI")])
6991 [(set (match_operand:<ssehalfvecmode> 0 "nonimmediate_operand")
6992 (vec_select:<ssehalfvecmode>
6993 (match_operand:VI8F_256 1 "nonimmediate_operand")
6994 (parallel [(const_int 0) (const_int 1)])))]
6995 "TARGET_AVX && !(MEM_P (operands[0]) && MEM_P (operands[1]))
6996 && reload_completed"
6999 rtx op1 = operands[1];
7001 op1 = gen_rtx_REG (<ssehalfvecmode>mode, REGNO (op1));
7003 op1 = gen_lowpart (<ssehalfvecmode>mode, op1);
7004 emit_move_insn (operands[0], op1);
7008 (define_insn "vec_extract_hi_<mode><mask_name>"
7009 [(set (match_operand:<ssehalfvecmode> 0 "<store_mask_predicate>" "=v,<store_mask_constraint>")
7010 (vec_select:<ssehalfvecmode>
7011 (match_operand:VI8F_256 1 "register_operand" "v,v")
7012 (parallel [(const_int 2) (const_int 3)])))]
7015 if (TARGET_AVX512DQ && TARGET_AVX512VL)
7016 return "vextract<shuffletype>64x2\t{$0x1, %1, %0<mask_operand2>|%0<mask_operand2>, %1, 0x1}";
7018 return "vextract<i128>\t{$0x1, %1, %0|%0, %1, 0x1}";
7020 [(set_attr "type" "sselog")
7021 (set_attr "prefix_extra" "1")
7022 (set_attr "length_immediate" "1")
7023 (set_attr "memory" "none,store")
7024 (set_attr "prefix" "vex")
7025 (set_attr "mode" "<sseinsnmode>")])
7028 [(set (match_operand:<ssehalfvecmode> 0 "nonimmediate_operand")
7029 (vec_select:<ssehalfvecmode>
7030 (match_operand:VI4F_256 1 "nonimmediate_operand")
7031 (parallel [(const_int 0) (const_int 1)
7032 (const_int 2) (const_int 3)])))]
7033 "TARGET_AVX && !(MEM_P (operands[0]) && MEM_P (operands[1])) && reload_completed"
7036 rtx op1 = operands[1];
7038 op1 = gen_rtx_REG (<ssehalfvecmode>mode, REGNO (op1));
7040 op1 = gen_lowpart (<ssehalfvecmode>mode, op1);
7041 emit_move_insn (operands[0], op1);
7046 (define_insn "vec_extract_lo_<mode><mask_name>"
7047 [(set (match_operand:<ssehalfvecmode> 0 "<store_mask_predicate>" "=<store_mask_constraint>")
7048 (vec_select:<ssehalfvecmode>
7049 (match_operand:VI4F_256 1 "nonimmediate_operand" "v")
7050 (parallel [(const_int 0) (const_int 1)
7051 (const_int 2) (const_int 3)])))]
7052 "TARGET_AVX && <mask_avx512vl_condition> && <mask_avx512dq_condition>"
7055 return "vextract<shuffletype>32x4\t{$0x0, %1, %0<mask_operand2>|%0<mask_operand2>, %1, 0x0}";
7059 [(set_attr "type" "sselog1")
7060 (set_attr "prefix_extra" "1")
7061 (set_attr "length_immediate" "1")
7062 (set_attr "prefix" "evex")
7063 (set_attr "mode" "<sseinsnmode>")])
7065 (define_insn "vec_extract_lo_<mode>_maskm"
7066 [(set (match_operand:<ssehalfvecmode> 0 "memory_operand" "=m")
7067 (vec_merge:<ssehalfvecmode>
7068 (vec_select:<ssehalfvecmode>
7069 (match_operand:VI4F_256 1 "register_operand" "v")
7070 (parallel [(const_int 0) (const_int 1)
7071 (const_int 2) (const_int 3)]))
7072 (match_operand:<ssehalfvecmode> 2 "memory_operand" "0")
7073 (match_operand:QI 3 "register_operand" "k")))]
7074 "TARGET_AVX512VL && TARGET_AVX512F"
7075 "vextract<shuffletype>32x4\t{$0x0, %1, %0%{3%}|%0%{%3%}, %1, 0x0}"
7076 [(set_attr "type" "sselog1")
7077 (set_attr "prefix_extra" "1")
7078 (set_attr "length_immediate" "1")
7079 (set_attr "prefix" "evex")
7080 (set_attr "mode" "<sseinsnmode>")])
7082 (define_insn "vec_extract_hi_<mode>_maskm"
7083 [(set (match_operand:<ssehalfvecmode> 0 "memory_operand" "=m")
7084 (vec_merge:<ssehalfvecmode>
7085 (vec_select:<ssehalfvecmode>
7086 (match_operand:VI4F_256 1 "register_operand" "v")
7087 (parallel [(const_int 4) (const_int 5)
7088 (const_int 6) (const_int 7)]))
7089 (match_operand:<ssehalfvecmode> 2 "memory_operand" "0")
7090 (match_operand:<ssehalfvecmode> 3 "register_operand" "k")))]
7091 "TARGET_AVX512F && TARGET_AVX512VL"
7093 return "vextract<shuffletype>32x4\t{$0x1, %1, %0%{%3%}|%0%{%3%}, %1, 0x1}";
7095 [(set_attr "type" "sselog1")
7096 (set_attr "prefix_extra" "1")
7097 (set_attr "length_immediate" "1")
7098 (set_attr "prefix" "evex")
7099 (set_attr "mode" "<sseinsnmode>")])
7101 (define_insn "vec_extract_hi_<mode><mask_name>"
7102 [(set (match_operand:<ssehalfvecmode> 0 "<store_mask_predicate>" "=<store_mask_constraint>")
7103 (vec_select:<ssehalfvecmode>
7104 (match_operand:VI4F_256 1 "register_operand" "v")
7105 (parallel [(const_int 4) (const_int 5)
7106 (const_int 6) (const_int 7)])))]
7107 "TARGET_AVX && <mask_avx512vl_condition>"
7109 if (TARGET_AVX512VL)
7110 return "vextract<shuffletype>32x4\t{$0x1, %1, %0<mask_operand2>|%0<mask_operand2>, %1, 0x1}";
7112 return "vextract<i128>\t{$0x1, %1, %0|%0, %1, 0x1}";
7114 [(set_attr "type" "sselog1")
7115 (set_attr "prefix_extra" "1")
7116 (set_attr "length_immediate" "1")
7117 (set (attr "prefix")
7119 (match_test "TARGET_AVX512VL")
7120 (const_string "evex")
7121 (const_string "vex")))
7122 (set_attr "mode" "<sseinsnmode>")])
7124 (define_insn_and_split "vec_extract_lo_v32hi"
7125 [(set (match_operand:V16HI 0 "nonimmediate_operand" "=v,m")
7127 (match_operand:V32HI 1 "nonimmediate_operand" "vm,v")
7128 (parallel [(const_int 0) (const_int 1)
7129 (const_int 2) (const_int 3)
7130 (const_int 4) (const_int 5)
7131 (const_int 6) (const_int 7)
7132 (const_int 8) (const_int 9)
7133 (const_int 10) (const_int 11)
7134 (const_int 12) (const_int 13)
7135 (const_int 14) (const_int 15)])))]
7136 "TARGET_AVX512F && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
7138 "&& reload_completed"
7139 [(set (match_dup 0) (match_dup 1))]
7141 if (REG_P (operands[1]))
7142 operands[1] = gen_rtx_REG (V16HImode, REGNO (operands[1]));
7144 operands[1] = adjust_address (operands[1], V16HImode, 0);
7147 (define_insn "vec_extract_hi_v32hi"
7148 [(set (match_operand:V16HI 0 "nonimmediate_operand" "=v,m")
7150 (match_operand:V32HI 1 "nonimmediate_operand" "v,v")
7151 (parallel [(const_int 16) (const_int 17)
7152 (const_int 18) (const_int 19)
7153 (const_int 20) (const_int 21)
7154 (const_int 22) (const_int 23)
7155 (const_int 24) (const_int 25)
7156 (const_int 26) (const_int 27)
7157 (const_int 28) (const_int 29)
7158 (const_int 30) (const_int 31)])))]
7160 "vextracti64x4\t{$0x1, %1, %0|%0, %1, 0x1}"
7161 [(set_attr "type" "sselog")
7162 (set_attr "prefix_extra" "1")
7163 (set_attr "length_immediate" "1")
7164 (set_attr "memory" "none,store")
7165 (set_attr "prefix" "evex")
7166 (set_attr "mode" "XI")])
7168 (define_insn_and_split "vec_extract_lo_v16hi"
7169 [(set (match_operand:V8HI 0 "nonimmediate_operand" "=x,m")
7171 (match_operand:V16HI 1 "nonimmediate_operand" "xm,x")
7172 (parallel [(const_int 0) (const_int 1)
7173 (const_int 2) (const_int 3)
7174 (const_int 4) (const_int 5)
7175 (const_int 6) (const_int 7)])))]
7176 "TARGET_AVX && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
7178 "&& reload_completed"
7179 [(set (match_dup 0) (match_dup 1))]
7181 if (REG_P (operands[1]))
7182 operands[1] = gen_rtx_REG (V8HImode, REGNO (operands[1]));
7184 operands[1] = adjust_address (operands[1], V8HImode, 0);
7187 (define_insn "vec_extract_hi_v16hi"
7188 [(set (match_operand:V8HI 0 "nonimmediate_operand" "=x,m")
7190 (match_operand:V16HI 1 "register_operand" "x,x")
7191 (parallel [(const_int 8) (const_int 9)
7192 (const_int 10) (const_int 11)
7193 (const_int 12) (const_int 13)
7194 (const_int 14) (const_int 15)])))]
7196 "vextract%~128\t{$0x1, %1, %0|%0, %1, 0x1}"
7197 [(set_attr "type" "sselog")
7198 (set_attr "prefix_extra" "1")
7199 (set_attr "length_immediate" "1")
7200 (set_attr "memory" "none,store")
7201 (set_attr "prefix" "vex")
7202 (set_attr "mode" "OI")])
7204 (define_insn_and_split "vec_extract_lo_v64qi"
7205 [(set (match_operand:V32QI 0 "nonimmediate_operand" "=v,m")
7207 (match_operand:V64QI 1 "nonimmediate_operand" "vm,v")
7208 (parallel [(const_int 0) (const_int 1)
7209 (const_int 2) (const_int 3)
7210 (const_int 4) (const_int 5)
7211 (const_int 6) (const_int 7)
7212 (const_int 8) (const_int 9)
7213 (const_int 10) (const_int 11)
7214 (const_int 12) (const_int 13)
7215 (const_int 14) (const_int 15)
7216 (const_int 16) (const_int 17)
7217 (const_int 18) (const_int 19)
7218 (const_int 20) (const_int 21)
7219 (const_int 22) (const_int 23)
7220 (const_int 24) (const_int 25)
7221 (const_int 26) (const_int 27)
7222 (const_int 28) (const_int 29)
7223 (const_int 30) (const_int 31)])))]
7224 "TARGET_AVX512F && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
7226 "&& reload_completed"
7227 [(set (match_dup 0) (match_dup 1))]
7229 if (REG_P (operands[1]))
7230 operands[1] = gen_rtx_REG (V32QImode, REGNO (operands[1]));
7232 operands[1] = adjust_address (operands[1], V32QImode, 0);
7235 (define_insn "vec_extract_hi_v64qi"
7236 [(set (match_operand:V32QI 0 "nonimmediate_operand" "=v,m")
7238 (match_operand:V64QI 1 "nonimmediate_operand" "v,v")
7239 (parallel [(const_int 32) (const_int 33)
7240 (const_int 34) (const_int 35)
7241 (const_int 36) (const_int 37)
7242 (const_int 38) (const_int 39)
7243 (const_int 40) (const_int 41)
7244 (const_int 42) (const_int 43)
7245 (const_int 44) (const_int 45)
7246 (const_int 46) (const_int 47)
7247 (const_int 48) (const_int 49)
7248 (const_int 50) (const_int 51)
7249 (const_int 52) (const_int 53)
7250 (const_int 54) (const_int 55)
7251 (const_int 56) (const_int 57)
7252 (const_int 58) (const_int 59)
7253 (const_int 60) (const_int 61)
7254 (const_int 62) (const_int 63)])))]
7256 "vextracti64x4\t{$0x1, %1, %0|%0, %1, 0x1}"
7257 [(set_attr "type" "sselog")
7258 (set_attr "prefix_extra" "1")
7259 (set_attr "length_immediate" "1")
7260 (set_attr "memory" "none,store")
7261 (set_attr "prefix" "evex")
7262 (set_attr "mode" "XI")])
7264 (define_insn_and_split "vec_extract_lo_v32qi"
7265 [(set (match_operand:V16QI 0 "nonimmediate_operand" "=x,m")
7267 (match_operand:V32QI 1 "nonimmediate_operand" "xm,x")
7268 (parallel [(const_int 0) (const_int 1)
7269 (const_int 2) (const_int 3)
7270 (const_int 4) (const_int 5)
7271 (const_int 6) (const_int 7)
7272 (const_int 8) (const_int 9)
7273 (const_int 10) (const_int 11)
7274 (const_int 12) (const_int 13)
7275 (const_int 14) (const_int 15)])))]
7276 "TARGET_AVX && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
7278 "&& reload_completed"
7279 [(set (match_dup 0) (match_dup 1))]
7281 if (REG_P (operands[1]))
7282 operands[1] = gen_rtx_REG (V16QImode, REGNO (operands[1]));
7284 operands[1] = adjust_address (operands[1], V16QImode, 0);
7287 (define_insn "vec_extract_hi_v32qi"
7288 [(set (match_operand:V16QI 0 "nonimmediate_operand" "=x,m")
7290 (match_operand:V32QI 1 "register_operand" "x,x")
7291 (parallel [(const_int 16) (const_int 17)
7292 (const_int 18) (const_int 19)
7293 (const_int 20) (const_int 21)
7294 (const_int 22) (const_int 23)
7295 (const_int 24) (const_int 25)
7296 (const_int 26) (const_int 27)
7297 (const_int 28) (const_int 29)
7298 (const_int 30) (const_int 31)])))]
7300 "vextract%~128\t{$0x1, %1, %0|%0, %1, 0x1}"
7301 [(set_attr "type" "sselog")
7302 (set_attr "prefix_extra" "1")
7303 (set_attr "length_immediate" "1")
7304 (set_attr "memory" "none,store")
7305 (set_attr "prefix" "vex")
7306 (set_attr "mode" "OI")])
7308 ;; Modes handled by vec_extract patterns.
7309 (define_mode_iterator VEC_EXTRACT_MODE
7310 [(V64QI "TARGET_AVX512BW") (V32QI "TARGET_AVX") V16QI
7311 (V32HI "TARGET_AVX512BW") (V16HI "TARGET_AVX") V8HI
7312 (V16SI "TARGET_AVX512F") (V8SI "TARGET_AVX") V4SI
7313 (V8DI "TARGET_AVX512F") (V4DI "TARGET_AVX") V2DI
7314 (V16SF "TARGET_AVX512F") (V8SF "TARGET_AVX") V4SF
7315 (V8DF "TARGET_AVX512F") (V4DF "TARGET_AVX") V2DF])
7317 (define_expand "vec_extract<mode>"
7318 [(match_operand:<ssescalarmode> 0 "register_operand")
7319 (match_operand:VEC_EXTRACT_MODE 1 "register_operand")
7320 (match_operand 2 "const_int_operand")]
7323 ix86_expand_vector_extract (false, operands[0], operands[1],
7324 INTVAL (operands[2]));
7328 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
7330 ;; Parallel double-precision floating point element swizzling
7332 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
7334 (define_insn "<mask_codefor>avx512f_unpckhpd512<mask_name>"
7335 [(set (match_operand:V8DF 0 "register_operand" "=v")
7338 (match_operand:V8DF 1 "nonimmediate_operand" "v")
7339 (match_operand:V8DF 2 "nonimmediate_operand" "vm"))
7340 (parallel [(const_int 1) (const_int 9)
7341 (const_int 3) (const_int 11)
7342 (const_int 5) (const_int 13)
7343 (const_int 7) (const_int 15)])))]
7345 "vunpckhpd\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
7346 [(set_attr "type" "sselog")
7347 (set_attr "prefix" "evex")
7348 (set_attr "mode" "V8DF")])
7350 ;; Recall that the 256-bit unpck insns only shuffle within their lanes.
7351 (define_insn "avx_unpckhpd256<mask_name>"
7352 [(set (match_operand:V4DF 0 "register_operand" "=v")
7355 (match_operand:V4DF 1 "register_operand" "v")
7356 (match_operand:V4DF 2 "nonimmediate_operand" "vm"))
7357 (parallel [(const_int 1) (const_int 5)
7358 (const_int 3) (const_int 7)])))]
7359 "TARGET_AVX && <mask_avx512vl_condition>"
7360 "vunpckhpd\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
7361 [(set_attr "type" "sselog")
7362 (set_attr "prefix" "vex")
7363 (set_attr "mode" "V4DF")])
7365 (define_expand "vec_interleave_highv4df"
7369 (match_operand:V4DF 1 "register_operand" "x")
7370 (match_operand:V4DF 2 "nonimmediate_operand" "xm"))
7371 (parallel [(const_int 0) (const_int 4)
7372 (const_int 2) (const_int 6)])))
7378 (parallel [(const_int 1) (const_int 5)
7379 (const_int 3) (const_int 7)])))
7380 (set (match_operand:V4DF 0 "register_operand")
7385 (parallel [(const_int 2) (const_int 3)
7386 (const_int 6) (const_int 7)])))]
7389 operands[3] = gen_reg_rtx (V4DFmode);
7390 operands[4] = gen_reg_rtx (V4DFmode);
7394 (define_insn "avx512vl_unpckhpd128_mask"
7395 [(set (match_operand:V2DF 0 "register_operand" "=v")
7399 (match_operand:V2DF 1 "register_operand" "v")
7400 (match_operand:V2DF 2 "nonimmediate_operand" "vm"))
7401 (parallel [(const_int 1) (const_int 3)]))
7402 (match_operand:V2DF 3 "vector_move_operand" "0C")
7403 (match_operand:QI 4 "register_operand" "Yk")))]
7405 "vunpckhpd\t{%2, %1, %0%{%4%}%N3|%0%{%4%}%N3, %1, %2}"
7406 [(set_attr "type" "sselog")
7407 (set_attr "prefix" "evex")
7408 (set_attr "mode" "V2DF")])
7410 (define_expand "vec_interleave_highv2df"
7411 [(set (match_operand:V2DF 0 "register_operand")
7414 (match_operand:V2DF 1 "nonimmediate_operand")
7415 (match_operand:V2DF 2 "nonimmediate_operand"))
7416 (parallel [(const_int 1)
7420 if (!ix86_vec_interleave_v2df_operator_ok (operands, 1))
7421 operands[2] = force_reg (V2DFmode, operands[2]);
7424 (define_insn "*vec_interleave_highv2df"
7425 [(set (match_operand:V2DF 0 "nonimmediate_operand" "=x,x,x,x,x,m")
7428 (match_operand:V2DF 1 "nonimmediate_operand" " 0,x,o,o,o,x")
7429 (match_operand:V2DF 2 "nonimmediate_operand" " x,x,1,0,x,0"))
7430 (parallel [(const_int 1)
7432 "TARGET_SSE2 && ix86_vec_interleave_v2df_operator_ok (operands, 1)"
7434 unpckhpd\t{%2, %0|%0, %2}
7435 vunpckhpd\t{%2, %1, %0|%0, %1, %2}
7436 %vmovddup\t{%H1, %0|%0, %H1}
7437 movlpd\t{%H1, %0|%0, %H1}
7438 vmovlpd\t{%H1, %2, %0|%0, %2, %H1}
7439 %vmovhpd\t{%1, %0|%q0, %1}"
7440 [(set_attr "isa" "noavx,avx,sse3,noavx,avx,*")
7441 (set_attr "type" "sselog,sselog,sselog,ssemov,ssemov,ssemov")
7442 (set_attr "ssememalign" "64")
7443 (set_attr "prefix_data16" "*,*,*,1,*,1")
7444 (set_attr "prefix" "orig,vex,maybe_vex,orig,vex,maybe_vex")
7445 (set_attr "mode" "V2DF,V2DF,DF,V1DF,V1DF,V1DF")])
7447 (define_expand "avx512f_movddup512<mask_name>"
7448 [(set (match_operand:V8DF 0 "register_operand")
7451 (match_operand:V8DF 1 "nonimmediate_operand")
7453 (parallel [(const_int 0) (const_int 8)
7454 (const_int 2) (const_int 10)
7455 (const_int 4) (const_int 12)
7456 (const_int 6) (const_int 14)])))]
7459 (define_expand "avx512f_unpcklpd512<mask_name>"
7460 [(set (match_operand:V8DF 0 "register_operand")
7463 (match_operand:V8DF 1 "register_operand")
7464 (match_operand:V8DF 2 "nonimmediate_operand"))
7465 (parallel [(const_int 0) (const_int 8)
7466 (const_int 2) (const_int 10)
7467 (const_int 4) (const_int 12)
7468 (const_int 6) (const_int 14)])))]
7471 (define_insn "*avx512f_unpcklpd512<mask_name>"
7472 [(set (match_operand:V8DF 0 "register_operand" "=v,v")
7475 (match_operand:V8DF 1 "nonimmediate_operand" "vm, v")
7476 (match_operand:V8DF 2 "nonimmediate_operand" "1 ,vm"))
7477 (parallel [(const_int 0) (const_int 8)
7478 (const_int 2) (const_int 10)
7479 (const_int 4) (const_int 12)
7480 (const_int 6) (const_int 14)])))]
7483 vmovddup\t{%1, %0<mask_operand3>|%0<mask_operand3>, %1}
7484 vunpcklpd\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
7485 [(set_attr "type" "sselog")
7486 (set_attr "prefix" "evex")
7487 (set_attr "mode" "V8DF")])
7489 ;; Recall that the 256-bit unpck insns only shuffle within their lanes.
7490 (define_expand "avx_movddup256<mask_name>"
7491 [(set (match_operand:V4DF 0 "register_operand")
7494 (match_operand:V4DF 1 "nonimmediate_operand")
7496 (parallel [(const_int 0) (const_int 4)
7497 (const_int 2) (const_int 6)])))]
7498 "TARGET_AVX && <mask_avx512vl_condition>")
7500 (define_expand "avx_unpcklpd256<mask_name>"
7501 [(set (match_operand:V4DF 0 "register_operand")
7504 (match_operand:V4DF 1 "register_operand")
7505 (match_operand:V4DF 2 "nonimmediate_operand"))
7506 (parallel [(const_int 0) (const_int 4)
7507 (const_int 2) (const_int 6)])))]
7508 "TARGET_AVX && <mask_avx512vl_condition>")
7510 (define_insn "*avx_unpcklpd256<mask_name>"
7511 [(set (match_operand:V4DF 0 "register_operand" "=v,v")
7514 (match_operand:V4DF 1 "nonimmediate_operand" " v,m")
7515 (match_operand:V4DF 2 "nonimmediate_operand" "vm,1"))
7516 (parallel [(const_int 0) (const_int 4)
7517 (const_int 2) (const_int 6)])))]
7518 "TARGET_AVX && <mask_avx512vl_condition>"
7520 vunpcklpd\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}
7521 vmovddup\t{%1, %0<mask_operand3>|%0<mask_operand3>, %1}"
7522 [(set_attr "type" "sselog")
7523 (set_attr "prefix" "vex")
7524 (set_attr "mode" "V4DF")])
7526 (define_expand "vec_interleave_lowv4df"
7530 (match_operand:V4DF 1 "register_operand" "x")
7531 (match_operand:V4DF 2 "nonimmediate_operand" "xm"))
7532 (parallel [(const_int 0) (const_int 4)
7533 (const_int 2) (const_int 6)])))
7539 (parallel [(const_int 1) (const_int 5)
7540 (const_int 3) (const_int 7)])))
7541 (set (match_operand:V4DF 0 "register_operand")
7546 (parallel [(const_int 0) (const_int 1)
7547 (const_int 4) (const_int 5)])))]
7550 operands[3] = gen_reg_rtx (V4DFmode);
7551 operands[4] = gen_reg_rtx (V4DFmode);
7554 (define_insn "avx512vl_unpcklpd128_mask"
7555 [(set (match_operand:V2DF 0 "register_operand" "=v")
7559 (match_operand:V2DF 1 "register_operand" "v")
7560 (match_operand:V2DF 2 "nonimmediate_operand" "vm"))
7561 (parallel [(const_int 0) (const_int 2)]))
7562 (match_operand:V2DF 3 "vector_move_operand" "0C")
7563 (match_operand:QI 4 "register_operand" "Yk")))]
7565 "vunpcklpd\t{%2, %1, %0%{%4%}%N3|%0%{%4%}%N3, %1, %2}"
7566 [(set_attr "type" "sselog")
7567 (set_attr "prefix" "evex")
7568 (set_attr "mode" "V2DF")])
7570 (define_expand "vec_interleave_lowv2df"
7571 [(set (match_operand:V2DF 0 "register_operand")
7574 (match_operand:V2DF 1 "nonimmediate_operand")
7575 (match_operand:V2DF 2 "nonimmediate_operand"))
7576 (parallel [(const_int 0)
7580 if (!ix86_vec_interleave_v2df_operator_ok (operands, 0))
7581 operands[1] = force_reg (V2DFmode, operands[1]);
7584 (define_insn "*vec_interleave_lowv2df"
7585 [(set (match_operand:V2DF 0 "nonimmediate_operand" "=x,x,x,x,x,o")
7588 (match_operand:V2DF 1 "nonimmediate_operand" " 0,x,m,0,x,0")
7589 (match_operand:V2DF 2 "nonimmediate_operand" " x,x,1,m,m,x"))
7590 (parallel [(const_int 0)
7592 "TARGET_SSE2 && ix86_vec_interleave_v2df_operator_ok (operands, 0)"
7594 unpcklpd\t{%2, %0|%0, %2}
7595 vunpcklpd\t{%2, %1, %0|%0, %1, %2}
7596 %vmovddup\t{%1, %0|%0, %q1}
7597 movhpd\t{%2, %0|%0, %q2}
7598 vmovhpd\t{%2, %1, %0|%0, %1, %q2}
7599 %vmovlpd\t{%2, %H0|%H0, %2}"
7600 [(set_attr "isa" "noavx,avx,sse3,noavx,avx,*")
7601 (set_attr "type" "sselog,sselog,sselog,ssemov,ssemov,ssemov")
7602 (set_attr "ssememalign" "64")
7603 (set_attr "prefix_data16" "*,*,*,1,*,1")
7604 (set_attr "prefix" "orig,vex,maybe_vex,orig,vex,maybe_vex")
7605 (set_attr "mode" "V2DF,V2DF,DF,V1DF,V1DF,V1DF")])
7608 [(set (match_operand:V2DF 0 "memory_operand")
7611 (match_operand:V2DF 1 "register_operand")
7613 (parallel [(const_int 0)
7615 "TARGET_SSE3 && reload_completed"
7618 rtx low = gen_rtx_REG (DFmode, REGNO (operands[1]));
7619 emit_move_insn (adjust_address (operands[0], DFmode, 0), low);
7620 emit_move_insn (adjust_address (operands[0], DFmode, 8), low);
7625 [(set (match_operand:V2DF 0 "register_operand")
7628 (match_operand:V2DF 1 "memory_operand")
7630 (parallel [(match_operand:SI 2 "const_0_to_1_operand")
7631 (match_operand:SI 3 "const_int_operand")])))]
7632 "TARGET_SSE3 && INTVAL (operands[2]) + 2 == INTVAL (operands[3])"
7633 [(set (match_dup 0) (vec_duplicate:V2DF (match_dup 1)))]
7635 operands[1] = adjust_address (operands[1], DFmode, INTVAL (operands[2]) * 8);
7638 (define_insn "avx512f_vmscalef<mode><round_name>"
7639 [(set (match_operand:VF_128 0 "register_operand" "=v")
7642 [(match_operand:VF_128 1 "register_operand" "v")
7643 (match_operand:VF_128 2 "<round_nimm_predicate>" "<round_constraint>")]
7648 "vscalef<ssescalarmodesuffix>\t{<round_op3>%2, %1, %0|%0, %1, %2<round_op3>}"
7649 [(set_attr "prefix" "evex")
7650 (set_attr "mode" "<ssescalarmode>")])
7652 (define_insn "<avx512>_scalef<mode><mask_name><round_name>"
7653 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v")
7655 [(match_operand:VF_AVX512VL 1 "register_operand" "v")
7656 (match_operand:VF_AVX512VL 2 "nonimmediate_operand" "<round_constraint>")]
7659 "vscalef<ssemodesuffix>\t{<round_mask_op3>%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2<round_mask_op3>}"
7660 [(set_attr "prefix" "evex")
7661 (set_attr "mode" "<MODE>")])
7663 (define_expand "<avx512>_vternlog<mode>_maskz"
7664 [(match_operand:VI48_AVX512VL 0 "register_operand")
7665 (match_operand:VI48_AVX512VL 1 "register_operand")
7666 (match_operand:VI48_AVX512VL 2 "register_operand")
7667 (match_operand:VI48_AVX512VL 3 "nonimmediate_operand")
7668 (match_operand:SI 4 "const_0_to_255_operand")
7669 (match_operand:<avx512fmaskmode> 5 "register_operand")]
7672 emit_insn (gen_<avx512>_vternlog<mode>_maskz_1 (
7673 operands[0], operands[1], operands[2], operands[3],
7674 operands[4], CONST0_RTX (<MODE>mode), operands[5]));
7678 (define_insn "<avx512>_vternlog<mode><sd_maskz_name>"
7679 [(set (match_operand:VI48_AVX512VL 0 "register_operand" "=v")
7680 (unspec:VI48_AVX512VL
7681 [(match_operand:VI48_AVX512VL 1 "register_operand" "0")
7682 (match_operand:VI48_AVX512VL 2 "register_operand" "v")
7683 (match_operand:VI48_AVX512VL 3 "nonimmediate_operand" "vm")
7684 (match_operand:SI 4 "const_0_to_255_operand")]
7687 "vpternlog<ssemodesuffix>\t{%4, %3, %2, %0<sd_mask_op5>|%0<sd_mask_op5>, %2, %3, %4}"
7688 [(set_attr "type" "sselog")
7689 (set_attr "prefix" "evex")
7690 (set_attr "mode" "<sseinsnmode>")])
7692 (define_insn "<avx512>_vternlog<mode>_mask"
7693 [(set (match_operand:VI48_AVX512VL 0 "register_operand" "=v")
7694 (vec_merge:VI48_AVX512VL
7695 (unspec:VI48_AVX512VL
7696 [(match_operand:VI48_AVX512VL 1 "register_operand" "0")
7697 (match_operand:VI48_AVX512VL 2 "register_operand" "v")
7698 (match_operand:VI48_AVX512VL 3 "nonimmediate_operand" "vm")
7699 (match_operand:SI 4 "const_0_to_255_operand")]
7702 (match_operand:<avx512fmaskmode> 5 "register_operand" "Yk")))]
7704 "vpternlog<ssemodesuffix>\t{%4, %3, %2, %0%{%5%}|%0%{%5%}, %2, %3, %4}"
7705 [(set_attr "type" "sselog")
7706 (set_attr "prefix" "evex")
7707 (set_attr "mode" "<sseinsnmode>")])
7709 (define_insn "<avx512>_getexp<mode><mask_name><round_saeonly_name>"
7710 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v")
7711 (unspec:VF_AVX512VL [(match_operand:VF_AVX512VL 1 "<round_saeonly_nimm_predicate>" "<round_saeonly_constraint>")]
7714 "vgetexp<ssemodesuffix>\t{<round_saeonly_mask_op2>%1, %0<mask_operand2>|%0<mask_operand2>, %1<round_saeonly_mask_op2>}";
7715 [(set_attr "prefix" "evex")
7716 (set_attr "mode" "<MODE>")])
7718 (define_insn "avx512f_sgetexp<mode><round_saeonly_name>"
7719 [(set (match_operand:VF_128 0 "register_operand" "=v")
7722 [(match_operand:VF_128 1 "register_operand" "v")
7723 (match_operand:VF_128 2 "<round_saeonly_nimm_predicate>" "<round_saeonly_constraint>")]
7728 "vgetexp<ssescalarmodesuffix>\t{<round_saeonly_op3>%2, %1, %0|%0, %1, %2<round_saeonly_op3>}";
7729 [(set_attr "prefix" "evex")
7730 (set_attr "mode" "<ssescalarmode>")])
7732 (define_insn "<mask_codefor><avx512>_align<mode><mask_name>"
7733 [(set (match_operand:VI48_AVX512VL 0 "register_operand" "=v")
7734 (unspec:VI48_AVX512VL [(match_operand:VI48_AVX512VL 1 "register_operand" "v")
7735 (match_operand:VI48_AVX512VL 2 "nonimmediate_operand" "vm")
7736 (match_operand:SI 3 "const_0_to_255_operand")]
7739 "valign<ssemodesuffix>\t{%3, %2, %1, %0<mask_operand4>|%0<mask_operand4>, %1, %2, %3}";
7740 [(set_attr "prefix" "evex")
7741 (set_attr "mode" "<sseinsnmode>")])
7743 (define_expand "avx512f_shufps512_mask"
7744 [(match_operand:V16SF 0 "register_operand")
7745 (match_operand:V16SF 1 "register_operand")
7746 (match_operand:V16SF 2 "nonimmediate_operand")
7747 (match_operand:SI 3 "const_0_to_255_operand")
7748 (match_operand:V16SF 4 "register_operand")
7749 (match_operand:HI 5 "register_operand")]
7752 int mask = INTVAL (operands[3]);
7753 emit_insn (gen_avx512f_shufps512_1_mask (operands[0], operands[1], operands[2],
7754 GEN_INT ((mask >> 0) & 3),
7755 GEN_INT ((mask >> 2) & 3),
7756 GEN_INT (((mask >> 4) & 3) + 16),
7757 GEN_INT (((mask >> 6) & 3) + 16),
7758 GEN_INT (((mask >> 0) & 3) + 4),
7759 GEN_INT (((mask >> 2) & 3) + 4),
7760 GEN_INT (((mask >> 4) & 3) + 20),
7761 GEN_INT (((mask >> 6) & 3) + 20),
7762 GEN_INT (((mask >> 0) & 3) + 8),
7763 GEN_INT (((mask >> 2) & 3) + 8),
7764 GEN_INT (((mask >> 4) & 3) + 24),
7765 GEN_INT (((mask >> 6) & 3) + 24),
7766 GEN_INT (((mask >> 0) & 3) + 12),
7767 GEN_INT (((mask >> 2) & 3) + 12),
7768 GEN_INT (((mask >> 4) & 3) + 28),
7769 GEN_INT (((mask >> 6) & 3) + 28),
7770 operands[4], operands[5]));
7775 (define_expand "<avx512>_fixupimm<mode>_maskz<round_saeonly_expand_name>"
7776 [(match_operand:VF_AVX512VL 0 "register_operand")
7777 (match_operand:VF_AVX512VL 1 "register_operand")
7778 (match_operand:VF_AVX512VL 2 "register_operand")
7779 (match_operand:<sseintvecmode> 3 "<round_saeonly_expand_nimm_predicate>")
7780 (match_operand:SI 4 "const_0_to_255_operand")
7781 (match_operand:<avx512fmaskmode> 5 "register_operand")]
7784 emit_insn (gen_<avx512>_fixupimm<mode>_maskz_1<round_saeonly_expand_name> (
7785 operands[0], operands[1], operands[2], operands[3],
7786 operands[4], CONST0_RTX (<MODE>mode), operands[5]
7787 <round_saeonly_expand_operand6>));
7791 (define_insn "<avx512>_fixupimm<mode><sd_maskz_name><round_saeonly_name>"
7792 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v")
7794 [(match_operand:VF_AVX512VL 1 "register_operand" "0")
7795 (match_operand:VF_AVX512VL 2 "register_operand" "v")
7796 (match_operand:<sseintvecmode> 3 "nonimmediate_operand" "<round_saeonly_constraint>")
7797 (match_operand:SI 4 "const_0_to_255_operand")]
7800 "vfixupimm<ssemodesuffix>\t{%4, <round_saeonly_sd_mask_op5>%3, %2, %0<sd_mask_op5>|%0<sd_mask_op5>, %2, %3<round_saeonly_sd_mask_op5>, %4}";
7801 [(set_attr "prefix" "evex")
7802 (set_attr "mode" "<MODE>")])
7804 (define_insn "<avx512>_fixupimm<mode>_mask<round_saeonly_name>"
7805 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v")
7806 (vec_merge:VF_AVX512VL
7808 [(match_operand:VF_AVX512VL 1 "register_operand" "0")
7809 (match_operand:VF_AVX512VL 2 "register_operand" "v")
7810 (match_operand:<sseintvecmode> 3 "nonimmediate_operand" "<round_saeonly_constraint>")
7811 (match_operand:SI 4 "const_0_to_255_operand")]
7814 (match_operand:<avx512fmaskmode> 5 "register_operand" "Yk")))]
7816 "vfixupimm<ssemodesuffix>\t{%4, <round_saeonly_op6>%3, %2, %0%{%5%}|%0%{%5%}, %2, %3<round_saeonly_op6>, %4}";
7817 [(set_attr "prefix" "evex")
7818 (set_attr "mode" "<MODE>")])
7820 (define_expand "avx512f_sfixupimm<mode>_maskz<round_saeonly_expand_name>"
7821 [(match_operand:VF_128 0 "register_operand")
7822 (match_operand:VF_128 1 "register_operand")
7823 (match_operand:VF_128 2 "register_operand")
7824 (match_operand:<sseintvecmode> 3 "<round_saeonly_expand_nimm_predicate>")
7825 (match_operand:SI 4 "const_0_to_255_operand")
7826 (match_operand:<avx512fmaskmode> 5 "register_operand")]
7829 emit_insn (gen_avx512f_sfixupimm<mode>_maskz_1<round_saeonly_expand_name> (
7830 operands[0], operands[1], operands[2], operands[3],
7831 operands[4], CONST0_RTX (<MODE>mode), operands[5]
7832 <round_saeonly_expand_operand6>));
7836 (define_insn "avx512f_sfixupimm<mode><sd_maskz_name><round_saeonly_name>"
7837 [(set (match_operand:VF_128 0 "register_operand" "=v")
7840 [(match_operand:VF_128 1 "register_operand" "0")
7841 (match_operand:VF_128 2 "register_operand" "v")
7842 (match_operand:<sseintvecmode> 3 "<round_saeonly_nimm_predicate>" "<round_saeonly_constraint>")
7843 (match_operand:SI 4 "const_0_to_255_operand")]
7848 "vfixupimm<ssescalarmodesuffix>\t{%4, <round_saeonly_sd_mask_op5>%3, %2, %0<sd_mask_op5>|%0<sd_mask_op5>, %2, %3<round_saeonly_sd_mask_op5>, %4}";
7849 [(set_attr "prefix" "evex")
7850 (set_attr "mode" "<ssescalarmode>")])
7852 (define_insn "avx512f_sfixupimm<mode>_mask<round_saeonly_name>"
7853 [(set (match_operand:VF_128 0 "register_operand" "=v")
7857 [(match_operand:VF_128 1 "register_operand" "0")
7858 (match_operand:VF_128 2 "register_operand" "v")
7859 (match_operand:<sseintvecmode> 3 "<round_saeonly_nimm_predicate>" "<round_saeonly_constraint>")
7860 (match_operand:SI 4 "const_0_to_255_operand")]
7865 (match_operand:<avx512fmaskmode> 5 "register_operand" "Yk")))]
7867 "vfixupimm<ssescalarmodesuffix>\t{%4, <round_saeonly_op6>%3, %2, %0%{%5%}|%0%{%5%}, %2, %3<round_saeonly_op6>, %4}";
7868 [(set_attr "prefix" "evex")
7869 (set_attr "mode" "<ssescalarmode>")])
7871 (define_insn "<avx512>_rndscale<mode><mask_name><round_saeonly_name>"
7872 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v")
7874 [(match_operand:VF_AVX512VL 1 "nonimmediate_operand" "<round_saeonly_constraint>")
7875 (match_operand:SI 2 "const_0_to_255_operand")]
7878 "vrndscale<ssemodesuffix>\t{%2, <round_saeonly_mask_op3>%1, %0<mask_operand3>|%0<mask_operand3>, %1<round_saeonly_mask_op3>, %2}"
7879 [(set_attr "length_immediate" "1")
7880 (set_attr "prefix" "evex")
7881 (set_attr "mode" "<MODE>")])
7883 (define_insn "avx512f_rndscale<mode><round_saeonly_name>"
7884 [(set (match_operand:VF_128 0 "register_operand" "=v")
7887 [(match_operand:VF_128 1 "register_operand" "v")
7888 (match_operand:VF_128 2 "<round_saeonly_nimm_predicate>" "<round_saeonly_constraint>")
7889 (match_operand:SI 3 "const_0_to_255_operand")]
7894 "vrndscale<ssescalarmodesuffix>\t{%3, <round_saeonly_op4>%2, %1, %0|%0, %1, %2<round_saeonly_op4>, %3}"
7895 [(set_attr "length_immediate" "1")
7896 (set_attr "prefix" "evex")
7897 (set_attr "mode" "<MODE>")])
7899 ;; One bit in mask selects 2 elements.
7900 (define_insn "avx512f_shufps512_1<mask_name>"
7901 [(set (match_operand:V16SF 0 "register_operand" "=v")
7904 (match_operand:V16SF 1 "register_operand" "v")
7905 (match_operand:V16SF 2 "nonimmediate_operand" "vm"))
7906 (parallel [(match_operand 3 "const_0_to_3_operand")
7907 (match_operand 4 "const_0_to_3_operand")
7908 (match_operand 5 "const_16_to_19_operand")
7909 (match_operand 6 "const_16_to_19_operand")
7910 (match_operand 7 "const_4_to_7_operand")
7911 (match_operand 8 "const_4_to_7_operand")
7912 (match_operand 9 "const_20_to_23_operand")
7913 (match_operand 10 "const_20_to_23_operand")
7914 (match_operand 11 "const_8_to_11_operand")
7915 (match_operand 12 "const_8_to_11_operand")
7916 (match_operand 13 "const_24_to_27_operand")
7917 (match_operand 14 "const_24_to_27_operand")
7918 (match_operand 15 "const_12_to_15_operand")
7919 (match_operand 16 "const_12_to_15_operand")
7920 (match_operand 17 "const_28_to_31_operand")
7921 (match_operand 18 "const_28_to_31_operand")])))]
7923 && (INTVAL (operands[3]) == (INTVAL (operands[7]) - 4)
7924 && INTVAL (operands[4]) == (INTVAL (operands[8]) - 4)
7925 && INTVAL (operands[5]) == (INTVAL (operands[9]) - 4)
7926 && INTVAL (operands[6]) == (INTVAL (operands[10]) - 4)
7927 && INTVAL (operands[3]) == (INTVAL (operands[11]) - 8)
7928 && INTVAL (operands[4]) == (INTVAL (operands[12]) - 8)
7929 && INTVAL (operands[5]) == (INTVAL (operands[13]) - 8)
7930 && INTVAL (operands[6]) == (INTVAL (operands[14]) - 8)
7931 && INTVAL (operands[3]) == (INTVAL (operands[15]) - 12)
7932 && INTVAL (operands[4]) == (INTVAL (operands[16]) - 12)
7933 && INTVAL (operands[5]) == (INTVAL (operands[17]) - 12)
7934 && INTVAL (operands[6]) == (INTVAL (operands[18]) - 12))"
7937 mask = INTVAL (operands[3]);
7938 mask |= INTVAL (operands[4]) << 2;
7939 mask |= (INTVAL (operands[5]) - 16) << 4;
7940 mask |= (INTVAL (operands[6]) - 16) << 6;
7941 operands[3] = GEN_INT (mask);
7943 return "vshufps\t{%3, %2, %1, %0<mask_operand19>|%0<mask_operand19>, %1, %2, %3}";
7945 [(set_attr "type" "sselog")
7946 (set_attr "length_immediate" "1")
7947 (set_attr "prefix" "evex")
7948 (set_attr "mode" "V16SF")])
7950 (define_expand "avx512f_shufpd512_mask"
7951 [(match_operand:V8DF 0 "register_operand")
7952 (match_operand:V8DF 1 "register_operand")
7953 (match_operand:V8DF 2 "nonimmediate_operand")
7954 (match_operand:SI 3 "const_0_to_255_operand")
7955 (match_operand:V8DF 4 "register_operand")
7956 (match_operand:QI 5 "register_operand")]
7959 int mask = INTVAL (operands[3]);
7960 emit_insn (gen_avx512f_shufpd512_1_mask (operands[0], operands[1], operands[2],
7962 GEN_INT (mask & 2 ? 9 : 8),
7963 GEN_INT (mask & 4 ? 3 : 2),
7964 GEN_INT (mask & 8 ? 11 : 10),
7965 GEN_INT (mask & 16 ? 5 : 4),
7966 GEN_INT (mask & 32 ? 13 : 12),
7967 GEN_INT (mask & 64 ? 7 : 6),
7968 GEN_INT (mask & 128 ? 15 : 14),
7969 operands[4], operands[5]));
7973 (define_insn "avx512f_shufpd512_1<mask_name>"
7974 [(set (match_operand:V8DF 0 "register_operand" "=v")
7977 (match_operand:V8DF 1 "register_operand" "v")
7978 (match_operand:V8DF 2 "nonimmediate_operand" "vm"))
7979 (parallel [(match_operand 3 "const_0_to_1_operand")
7980 (match_operand 4 "const_8_to_9_operand")
7981 (match_operand 5 "const_2_to_3_operand")
7982 (match_operand 6 "const_10_to_11_operand")
7983 (match_operand 7 "const_4_to_5_operand")
7984 (match_operand 8 "const_12_to_13_operand")
7985 (match_operand 9 "const_6_to_7_operand")
7986 (match_operand 10 "const_14_to_15_operand")])))]
7990 mask = INTVAL (operands[3]);
7991 mask |= (INTVAL (operands[4]) - 8) << 1;
7992 mask |= (INTVAL (operands[5]) - 2) << 2;
7993 mask |= (INTVAL (operands[6]) - 10) << 3;
7994 mask |= (INTVAL (operands[7]) - 4) << 4;
7995 mask |= (INTVAL (operands[8]) - 12) << 5;
7996 mask |= (INTVAL (operands[9]) - 6) << 6;
7997 mask |= (INTVAL (operands[10]) - 14) << 7;
7998 operands[3] = GEN_INT (mask);
8000 return "vshufpd\t{%3, %2, %1, %0<mask_operand11>|%0<mask_operand11>, %1, %2, %3}";
8002 [(set_attr "type" "sselog")
8003 (set_attr "length_immediate" "1")
8004 (set_attr "prefix" "evex")
8005 (set_attr "mode" "V8DF")])
8007 (define_expand "avx_shufpd256<mask_expand4_name>"
8008 [(match_operand:V4DF 0 "register_operand")
8009 (match_operand:V4DF 1 "register_operand")
8010 (match_operand:V4DF 2 "nonimmediate_operand")
8011 (match_operand:SI 3 "const_int_operand")]
8014 int mask = INTVAL (operands[3]);
8015 emit_insn (gen_avx_shufpd256_1<mask_expand4_name> (operands[0],
8019 GEN_INT (mask & 2 ? 5 : 4),
8020 GEN_INT (mask & 4 ? 3 : 2),
8021 GEN_INT (mask & 8 ? 7 : 6)
8022 <mask_expand4_args>));
8026 (define_insn "avx_shufpd256_1<mask_name>"
8027 [(set (match_operand:V4DF 0 "register_operand" "=v")
8030 (match_operand:V4DF 1 "register_operand" "v")
8031 (match_operand:V4DF 2 "nonimmediate_operand" "vm"))
8032 (parallel [(match_operand 3 "const_0_to_1_operand")
8033 (match_operand 4 "const_4_to_5_operand")
8034 (match_operand 5 "const_2_to_3_operand")
8035 (match_operand 6 "const_6_to_7_operand")])))]
8036 "TARGET_AVX && <mask_avx512vl_condition>"
8039 mask = INTVAL (operands[3]);
8040 mask |= (INTVAL (operands[4]) - 4) << 1;
8041 mask |= (INTVAL (operands[5]) - 2) << 2;
8042 mask |= (INTVAL (operands[6]) - 6) << 3;
8043 operands[3] = GEN_INT (mask);
8045 return "vshufpd\t{%3, %2, %1, %0<mask_operand7>|%0<mask_operand7>, %1, %2, %3}";
8047 [(set_attr "type" "sseshuf")
8048 (set_attr "length_immediate" "1")
8049 (set_attr "prefix" "vex")
8050 (set_attr "mode" "V4DF")])
8052 (define_expand "sse2_shufpd<mask_expand4_name>"
8053 [(match_operand:V2DF 0 "register_operand")
8054 (match_operand:V2DF 1 "register_operand")
8055 (match_operand:V2DF 2 "nonimmediate_operand")
8056 (match_operand:SI 3 "const_int_operand")]
8059 int mask = INTVAL (operands[3]);
8060 emit_insn (gen_sse2_shufpd_v2df<mask_expand4_name> (operands[0], operands[1],
8061 operands[2], GEN_INT (mask & 1),
8062 GEN_INT (mask & 2 ? 3 : 2)
8063 <mask_expand4_args>));
8067 (define_insn "sse2_shufpd_v2df_mask"
8068 [(set (match_operand:V2DF 0 "register_operand" "=v")
8072 (match_operand:V2DF 1 "register_operand" "v")
8073 (match_operand:V2DF 2 "nonimmediate_operand" "vm"))
8074 (parallel [(match_operand 3 "const_0_to_1_operand")
8075 (match_operand 4 "const_2_to_3_operand")]))
8076 (match_operand:V2DF 5 "vector_move_operand" "0C")
8077 (match_operand:QI 6 "register_operand" "Yk")))]
8081 mask = INTVAL (operands[3]);
8082 mask |= (INTVAL (operands[4]) - 2) << 1;
8083 operands[3] = GEN_INT (mask);
8085 return "vshufpd\t{%3, %2, %1, %0%{%6%}%N5|%0%{6%}%N5, %1, %2, %3}";
8087 [(set_attr "type" "sseshuf")
8088 (set_attr "length_immediate" "1")
8089 (set_attr "prefix" "evex")
8090 (set_attr "mode" "V2DF")])
8092 ;; punpcklqdq and punpckhqdq are shorter than shufpd.
8093 (define_insn "avx2_interleave_highv4di<mask_name>"
8094 [(set (match_operand:V4DI 0 "register_operand" "=v")
8097 (match_operand:V4DI 1 "register_operand" "v")
8098 (match_operand:V4DI 2 "nonimmediate_operand" "vm"))
8099 (parallel [(const_int 1)
8103 "TARGET_AVX2 && <mask_avx512vl_condition>"
8104 "vpunpckhqdq\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
8105 [(set_attr "type" "sselog")
8106 (set_attr "prefix" "vex")
8107 (set_attr "mode" "OI")])
8109 (define_insn "<mask_codefor>avx512f_interleave_highv8di<mask_name>"
8110 [(set (match_operand:V8DI 0 "register_operand" "=v")
8113 (match_operand:V8DI 1 "register_operand" "v")
8114 (match_operand:V8DI 2 "nonimmediate_operand" "vm"))
8115 (parallel [(const_int 1) (const_int 9)
8116 (const_int 3) (const_int 11)
8117 (const_int 5) (const_int 13)
8118 (const_int 7) (const_int 15)])))]
8120 "vpunpckhqdq\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
8121 [(set_attr "type" "sselog")
8122 (set_attr "prefix" "evex")
8123 (set_attr "mode" "XI")])
8125 (define_insn "vec_interleave_highv2di<mask_name>"
8126 [(set (match_operand:V2DI 0 "register_operand" "=x,v")
8129 (match_operand:V2DI 1 "register_operand" "0,v")
8130 (match_operand:V2DI 2 "nonimmediate_operand" "xm,vm"))
8131 (parallel [(const_int 1)
8133 "TARGET_SSE2 && <mask_avx512vl_condition>"
8135 punpckhqdq\t{%2, %0|%0, %2}
8136 vpunpckhqdq\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
8137 [(set_attr "isa" "noavx,avx")
8138 (set_attr "type" "sselog")
8139 (set_attr "prefix_data16" "1,*")
8140 (set_attr "prefix" "orig,<mask_prefix>")
8141 (set_attr "mode" "TI")])
8143 (define_insn "avx2_interleave_lowv4di<mask_name>"
8144 [(set (match_operand:V4DI 0 "register_operand" "=v")
8147 (match_operand:V4DI 1 "register_operand" "v")
8148 (match_operand:V4DI 2 "nonimmediate_operand" "vm"))
8149 (parallel [(const_int 0)
8153 "TARGET_AVX2 && <mask_avx512vl_condition>"
8154 "vpunpcklqdq\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
8155 [(set_attr "type" "sselog")
8156 (set_attr "prefix" "vex")
8157 (set_attr "mode" "OI")])
8159 (define_insn "<mask_codefor>avx512f_interleave_lowv8di<mask_name>"
8160 [(set (match_operand:V8DI 0 "register_operand" "=v")
8163 (match_operand:V8DI 1 "register_operand" "v")
8164 (match_operand:V8DI 2 "nonimmediate_operand" "vm"))
8165 (parallel [(const_int 0) (const_int 8)
8166 (const_int 2) (const_int 10)
8167 (const_int 4) (const_int 12)
8168 (const_int 6) (const_int 14)])))]
8170 "vpunpcklqdq\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
8171 [(set_attr "type" "sselog")
8172 (set_attr "prefix" "evex")
8173 (set_attr "mode" "XI")])
8175 (define_insn "vec_interleave_lowv2di<mask_name>"
8176 [(set (match_operand:V2DI 0 "register_operand" "=x,v")
8179 (match_operand:V2DI 1 "register_operand" "0,v")
8180 (match_operand:V2DI 2 "nonimmediate_operand" "xm,vm"))
8181 (parallel [(const_int 0)
8183 "TARGET_SSE2 && <mask_avx512vl_condition>"
8185 punpcklqdq\t{%2, %0|%0, %2}
8186 vpunpcklqdq\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
8187 [(set_attr "isa" "noavx,avx")
8188 (set_attr "type" "sselog")
8189 (set_attr "prefix_data16" "1,*")
8190 (set_attr "prefix" "orig,vex")
8191 (set_attr "mode" "TI")])
8193 (define_insn "sse2_shufpd_<mode>"
8194 [(set (match_operand:VI8F_128 0 "register_operand" "=x,x")
8195 (vec_select:VI8F_128
8196 (vec_concat:<ssedoublevecmode>
8197 (match_operand:VI8F_128 1 "register_operand" "0,x")
8198 (match_operand:VI8F_128 2 "nonimmediate_operand" "xm,xm"))
8199 (parallel [(match_operand 3 "const_0_to_1_operand")
8200 (match_operand 4 "const_2_to_3_operand")])))]
8204 mask = INTVAL (operands[3]);
8205 mask |= (INTVAL (operands[4]) - 2) << 1;
8206 operands[3] = GEN_INT (mask);
8208 switch (which_alternative)
8211 return "shufpd\t{%3, %2, %0|%0, %2, %3}";
8213 return "vshufpd\t{%3, %2, %1, %0|%0, %1, %2, %3}";
8218 [(set_attr "isa" "noavx,avx")
8219 (set_attr "type" "sseshuf")
8220 (set_attr "length_immediate" "1")
8221 (set_attr "prefix" "orig,vex")
8222 (set_attr "mode" "V2DF")])
8224 ;; Avoid combining registers from different units in a single alternative,
8225 ;; see comment above inline_secondary_memory_needed function in i386.c
8226 (define_insn "sse2_storehpd"
8227 [(set (match_operand:DF 0 "nonimmediate_operand" "=m,x,x,x,*f,r")
8229 (match_operand:V2DF 1 "nonimmediate_operand" " x,0,x,o,o,o")
8230 (parallel [(const_int 1)])))]
8231 "TARGET_SSE2 && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
8233 %vmovhpd\t{%1, %0|%0, %1}
8235 vunpckhpd\t{%d1, %0|%0, %d1}
8239 [(set_attr "isa" "*,noavx,avx,*,*,*")
8240 (set_attr "type" "ssemov,sselog1,sselog1,ssemov,fmov,imov")
8241 (set (attr "prefix_data16")
8243 (and (eq_attr "alternative" "0")
8244 (not (match_test "TARGET_AVX")))
8246 (const_string "*")))
8247 (set_attr "prefix" "maybe_vex,orig,vex,*,*,*")
8248 (set_attr "mode" "V1DF,V1DF,V2DF,DF,DF,DF")])
8251 [(set (match_operand:DF 0 "register_operand")
8253 (match_operand:V2DF 1 "memory_operand")
8254 (parallel [(const_int 1)])))]
8255 "TARGET_SSE2 && reload_completed"
8256 [(set (match_dup 0) (match_dup 1))]
8257 "operands[1] = adjust_address (operands[1], DFmode, 8);")
8259 (define_insn "*vec_extractv2df_1_sse"
8260 [(set (match_operand:DF 0 "nonimmediate_operand" "=m,x,x")
8262 (match_operand:V2DF 1 "nonimmediate_operand" "x,x,o")
8263 (parallel [(const_int 1)])))]
8264 "!TARGET_SSE2 && TARGET_SSE
8265 && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
8267 movhps\t{%1, %0|%q0, %1}
8268 movhlps\t{%1, %0|%0, %1}
8269 movlps\t{%H1, %0|%0, %H1}"
8270 [(set_attr "type" "ssemov")
8271 (set_attr "ssememalign" "64")
8272 (set_attr "mode" "V2SF,V4SF,V2SF")])
8274 ;; Avoid combining registers from different units in a single alternative,
8275 ;; see comment above inline_secondary_memory_needed function in i386.c
8276 (define_insn "sse2_storelpd"
8277 [(set (match_operand:DF 0 "nonimmediate_operand" "=m,x,x,*f,r")
8279 (match_operand:V2DF 1 "nonimmediate_operand" " x,x,m,m,m")
8280 (parallel [(const_int 0)])))]
8281 "TARGET_SSE2 && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
8283 %vmovlpd\t{%1, %0|%0, %1}
8288 [(set_attr "type" "ssemov,ssemov,ssemov,fmov,imov")
8289 (set_attr "prefix_data16" "1,*,*,*,*")
8290 (set_attr "prefix" "maybe_vex")
8291 (set_attr "mode" "V1DF,DF,DF,DF,DF")])
8294 [(set (match_operand:DF 0 "register_operand")
8296 (match_operand:V2DF 1 "nonimmediate_operand")
8297 (parallel [(const_int 0)])))]
8298 "TARGET_SSE2 && reload_completed"
8299 [(set (match_dup 0) (match_dup 1))]
8301 if (REG_P (operands[1]))
8302 operands[1] = gen_rtx_REG (DFmode, REGNO (operands[1]));
8304 operands[1] = adjust_address (operands[1], DFmode, 0);
8307 (define_insn "*vec_extractv2df_0_sse"
8308 [(set (match_operand:DF 0 "nonimmediate_operand" "=m,x,x")
8310 (match_operand:V2DF 1 "nonimmediate_operand" "x,x,m")
8311 (parallel [(const_int 0)])))]
8312 "!TARGET_SSE2 && TARGET_SSE
8313 && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
8315 movlps\t{%1, %0|%0, %1}
8316 movaps\t{%1, %0|%0, %1}
8317 movlps\t{%1, %0|%0, %q1}"
8318 [(set_attr "type" "ssemov")
8319 (set_attr "mode" "V2SF,V4SF,V2SF")])
8321 (define_expand "sse2_loadhpd_exp"
8322 [(set (match_operand:V2DF 0 "nonimmediate_operand")
8325 (match_operand:V2DF 1 "nonimmediate_operand")
8326 (parallel [(const_int 0)]))
8327 (match_operand:DF 2 "nonimmediate_operand")))]
8330 rtx dst = ix86_fixup_binary_operands (UNKNOWN, V2DFmode, operands);
8332 emit_insn (gen_sse2_loadhpd (dst, operands[1], operands[2]));
8334 /* Fix up the destination if needed. */
8335 if (dst != operands[0])
8336 emit_move_insn (operands[0], dst);
8341 ;; Avoid combining registers from different units in a single alternative,
8342 ;; see comment above inline_secondary_memory_needed function in i386.c
8343 (define_insn "sse2_loadhpd"
8344 [(set (match_operand:V2DF 0 "nonimmediate_operand"
8348 (match_operand:V2DF 1 "nonimmediate_operand"
8350 (parallel [(const_int 0)]))
8351 (match_operand:DF 2 "nonimmediate_operand"
8352 " m,m,x,x,x,*f,r")))]
8353 "TARGET_SSE2 && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
8355 movhpd\t{%2, %0|%0, %2}
8356 vmovhpd\t{%2, %1, %0|%0, %1, %2}
8357 unpcklpd\t{%2, %0|%0, %2}
8358 vunpcklpd\t{%2, %1, %0|%0, %1, %2}
8362 [(set_attr "isa" "noavx,avx,noavx,avx,*,*,*")
8363 (set_attr "type" "ssemov,ssemov,sselog,sselog,ssemov,fmov,imov")
8364 (set_attr "ssememalign" "64")
8365 (set_attr "prefix_data16" "1,*,*,*,*,*,*")
8366 (set_attr "prefix" "orig,vex,orig,vex,*,*,*")
8367 (set_attr "mode" "V1DF,V1DF,V2DF,V2DF,DF,DF,DF")])
8370 [(set (match_operand:V2DF 0 "memory_operand")
8372 (vec_select:DF (match_dup 0) (parallel [(const_int 0)]))
8373 (match_operand:DF 1 "register_operand")))]
8374 "TARGET_SSE2 && reload_completed"
8375 [(set (match_dup 0) (match_dup 1))]
8376 "operands[0] = adjust_address (operands[0], DFmode, 8);")
8378 (define_expand "sse2_loadlpd_exp"
8379 [(set (match_operand:V2DF 0 "nonimmediate_operand")
8381 (match_operand:DF 2 "nonimmediate_operand")
8383 (match_operand:V2DF 1 "nonimmediate_operand")
8384 (parallel [(const_int 1)]))))]
8387 rtx dst = ix86_fixup_binary_operands (UNKNOWN, V2DFmode, operands);
8389 emit_insn (gen_sse2_loadlpd (dst, operands[1], operands[2]));
8391 /* Fix up the destination if needed. */
8392 if (dst != operands[0])
8393 emit_move_insn (operands[0], dst);
8398 ;; Avoid combining registers from different units in a single alternative,
8399 ;; see comment above inline_secondary_memory_needed function in i386.c
8400 (define_insn "sse2_loadlpd"
8401 [(set (match_operand:V2DF 0 "nonimmediate_operand"
8402 "=x,x,x,x,x,x,x,x,m,m ,m")
8404 (match_operand:DF 2 "nonimmediate_operand"
8405 " m,m,m,x,x,0,0,x,x,*f,r")
8407 (match_operand:V2DF 1 "vector_move_operand"
8408 " C,0,x,0,x,x,o,o,0,0 ,0")
8409 (parallel [(const_int 1)]))))]
8410 "TARGET_SSE2 && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
8412 %vmovsd\t{%2, %0|%0, %2}
8413 movlpd\t{%2, %0|%0, %2}
8414 vmovlpd\t{%2, %1, %0|%0, %1, %2}
8415 movsd\t{%2, %0|%0, %2}
8416 vmovsd\t{%2, %1, %0|%0, %1, %2}
8417 shufpd\t{$2, %1, %0|%0, %1, 2}
8418 movhpd\t{%H1, %0|%0, %H1}
8419 vmovhpd\t{%H1, %2, %0|%0, %2, %H1}
8423 [(set_attr "isa" "*,noavx,avx,noavx,avx,noavx,noavx,avx,*,*,*")
8425 (cond [(eq_attr "alternative" "5")
8426 (const_string "sselog")
8427 (eq_attr "alternative" "9")
8428 (const_string "fmov")
8429 (eq_attr "alternative" "10")
8430 (const_string "imov")
8432 (const_string "ssemov")))
8433 (set_attr "ssememalign" "64")
8434 (set_attr "prefix_data16" "*,1,*,*,*,*,1,*,*,*,*")
8435 (set_attr "length_immediate" "*,*,*,*,*,1,*,*,*,*,*")
8436 (set_attr "prefix" "maybe_vex,orig,vex,orig,vex,orig,orig,vex,*,*,*")
8437 (set_attr "mode" "DF,V1DF,V1DF,V1DF,V1DF,V2DF,V1DF,V1DF,DF,DF,DF")])
8440 [(set (match_operand:V2DF 0 "memory_operand")
8442 (match_operand:DF 1 "register_operand")
8443 (vec_select:DF (match_dup 0) (parallel [(const_int 1)]))))]
8444 "TARGET_SSE2 && reload_completed"
8445 [(set (match_dup 0) (match_dup 1))]
8446 "operands[0] = adjust_address (operands[0], DFmode, 0);")
8448 (define_insn "sse2_movsd"
8449 [(set (match_operand:V2DF 0 "nonimmediate_operand" "=x,x,x,x,m,x,x,x,o")
8451 (match_operand:V2DF 2 "nonimmediate_operand" " x,x,m,m,x,0,0,x,0")
8452 (match_operand:V2DF 1 "nonimmediate_operand" " 0,x,0,x,0,x,o,o,x")
8456 movsd\t{%2, %0|%0, %2}
8457 vmovsd\t{%2, %1, %0|%0, %1, %2}
8458 movlpd\t{%2, %0|%0, %q2}
8459 vmovlpd\t{%2, %1, %0|%0, %1, %q2}
8460 %vmovlpd\t{%2, %0|%q0, %2}
8461 shufpd\t{$2, %1, %0|%0, %1, 2}
8462 movhps\t{%H1, %0|%0, %H1}
8463 vmovhps\t{%H1, %2, %0|%0, %2, %H1}
8464 %vmovhps\t{%1, %H0|%H0, %1}"
8465 [(set_attr "isa" "noavx,avx,noavx,avx,*,noavx,noavx,avx,*")
8468 (eq_attr "alternative" "5")
8469 (const_string "sselog")
8470 (const_string "ssemov")))
8471 (set (attr "prefix_data16")
8473 (and (eq_attr "alternative" "2,4")
8474 (not (match_test "TARGET_AVX")))
8476 (const_string "*")))
8477 (set_attr "length_immediate" "*,*,*,*,*,1,*,*,*")
8478 (set_attr "ssememalign" "64")
8479 (set_attr "prefix" "orig,vex,orig,vex,maybe_vex,orig,orig,vex,maybe_vex")
8480 (set_attr "mode" "DF,DF,V1DF,V1DF,V1DF,V2DF,V1DF,V1DF,V1DF")])
8482 (define_insn "vec_dupv2df<mask_name>"
8483 [(set (match_operand:V2DF 0 "register_operand" "=x,v")
8485 (match_operand:DF 1 "nonimmediate_operand" " 0,vm")))]
8486 "TARGET_SSE2 && <mask_avx512vl_condition>"
8489 %vmovddup\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
8490 [(set_attr "isa" "noavx,sse3")
8491 (set_attr "type" "sselog1")
8492 (set_attr "prefix" "orig,maybe_vex")
8493 (set_attr "mode" "V2DF,DF")])
8495 (define_insn "*vec_concatv2df"
8496 [(set (match_operand:V2DF 0 "register_operand" "=x,v,v,x,x,v,x,x")
8498 (match_operand:DF 1 "nonimmediate_operand" " 0,v,m,0,x,m,0,0")
8499 (match_operand:DF 2 "vector_move_operand" " x,v,1,m,m,C,x,m")))]
8502 unpcklpd\t{%2, %0|%0, %2}
8503 vunpcklpd\t{%2, %1, %0|%0, %1, %2}
8504 %vmovddup\t{%1, %0|%0, %1}
8505 movhpd\t{%2, %0|%0, %2}
8506 vmovhpd\t{%2, %1, %0|%0, %1, %2}
8507 %vmovsd\t{%1, %0|%0, %1}
8508 movlhps\t{%2, %0|%0, %2}
8509 movhps\t{%2, %0|%0, %2}"
8510 [(set_attr "isa" "sse2_noavx,avx,sse3,sse2_noavx,avx,sse2,noavx,noavx")
8513 (eq_attr "alternative" "0,1,2")
8514 (const_string "sselog")
8515 (const_string "ssemov")))
8516 (set_attr "prefix_data16" "*,*,*,1,*,*,*,*")
8517 (set_attr "prefix" "orig,vex,maybe_vex,orig,vex,maybe_vex,orig,orig")
8518 (set_attr "mode" "V2DF,V2DF,DF,V1DF,V1DF,DF,V4SF,V2SF")])
8520 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
8522 ;; Parallel integer down-conversion operations
8524 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
8526 (define_mode_iterator PMOV_DST_MODE_1 [V16QI V16HI V8SI V8HI])
8527 (define_mode_attr pmov_src_mode
8528 [(V16QI "V16SI") (V16HI "V16SI") (V8SI "V8DI") (V8HI "V8DI")])
8529 (define_mode_attr pmov_src_lower
8530 [(V16QI "v16si") (V16HI "v16si") (V8SI "v8di") (V8HI "v8di")])
8531 (define_mode_attr pmov_suff_1
8532 [(V16QI "db") (V16HI "dw") (V8SI "qd") (V8HI "qw")])
8534 (define_insn "*avx512f_<code><pmov_src_lower><mode>2"
8535 [(set (match_operand:PMOV_DST_MODE_1 0 "nonimmediate_operand" "=v,m")
8536 (any_truncate:PMOV_DST_MODE_1
8537 (match_operand:<pmov_src_mode> 1 "register_operand" "v,v")))]
8539 "vpmov<trunsuffix><pmov_suff_1>\t{%1, %0|%0, %1}"
8540 [(set_attr "type" "ssemov")
8541 (set_attr "memory" "none,store")
8542 (set_attr "prefix" "evex")
8543 (set_attr "mode" "<sseinsnmode>")])
8545 (define_insn "avx512f_<code><pmov_src_lower><mode>2_mask"
8546 [(set (match_operand:PMOV_DST_MODE_1 0 "nonimmediate_operand" "=v,m")
8547 (vec_merge:PMOV_DST_MODE_1
8548 (any_truncate:PMOV_DST_MODE_1
8549 (match_operand:<pmov_src_mode> 1 "register_operand" "v,v"))
8550 (match_operand:PMOV_DST_MODE_1 2 "vector_move_operand" "0C,0")
8551 (match_operand:<avx512fmaskmode> 3 "register_operand" "Yk,Yk")))]
8553 "vpmov<trunsuffix><pmov_suff_1>\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"
8554 [(set_attr "type" "ssemov")
8555 (set_attr "memory" "none,store")
8556 (set_attr "prefix" "evex")
8557 (set_attr "mode" "<sseinsnmode>")])
8559 (define_expand "avx512f_<code><pmov_src_lower><mode>2_mask_store"
8560 [(set (match_operand:PMOV_DST_MODE_1 0 "memory_operand")
8561 (vec_merge:PMOV_DST_MODE_1
8562 (any_truncate:PMOV_DST_MODE_1
8563 (match_operand:<pmov_src_mode> 1 "register_operand"))
8565 (match_operand:<avx512fmaskmode> 2 "register_operand")))]
8568 (define_insn "*avx512bw_<code>v32hiv32qi2"
8569 [(set (match_operand:V32QI 0 "nonimmediate_operand" "=v,m")
8571 (match_operand:V32HI 1 "register_operand" "v,v")))]
8573 "vpmov<trunsuffix>wb\t{%1, %0|%0, %1}"
8574 [(set_attr "type" "ssemov")
8575 (set_attr "memory" "none,store")
8576 (set_attr "prefix" "evex")
8577 (set_attr "mode" "XI")])
8579 (define_insn "avx512bw_<code>v32hiv32qi2_mask"
8580 [(set (match_operand:V32QI 0 "nonimmediate_operand" "=v,m")
8583 (match_operand:V32HI 1 "register_operand" "v,v"))
8584 (match_operand:V32QI 2 "vector_move_operand" "0C,0")
8585 (match_operand:SI 3 "register_operand" "Yk,Yk")))]
8587 "vpmov<trunsuffix>wb\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"
8588 [(set_attr "type" "ssemov")
8589 (set_attr "memory" "none,store")
8590 (set_attr "prefix" "evex")
8591 (set_attr "mode" "XI")])
8593 (define_expand "avx512bw_<code>v32hiv32qi2_mask_store"
8594 [(set (match_operand:V32QI 0 "nonimmediate_operand")
8597 (match_operand:V32HI 1 "register_operand"))
8599 (match_operand:SI 2 "register_operand")))]
8602 (define_mode_iterator PMOV_DST_MODE_2
8603 [V4SI V8HI (V16QI "TARGET_AVX512BW")])
8604 (define_mode_attr pmov_suff_2
8605 [(V16QI "wb") (V8HI "dw") (V4SI "qd")])
8607 (define_insn "*avx512vl_<code><ssedoublemodelower><mode>2"
8608 [(set (match_operand:PMOV_DST_MODE_2 0 "nonimmediate_operand" "=v,m")
8609 (any_truncate:PMOV_DST_MODE_2
8610 (match_operand:<ssedoublemode> 1 "register_operand" "v,v")))]
8612 "vpmov<trunsuffix><pmov_suff_2>\t{%1, %0|%0, %1}"
8613 [(set_attr "type" "ssemov")
8614 (set_attr "memory" "none,store")
8615 (set_attr "prefix" "evex")
8616 (set_attr "mode" "<sseinsnmode>")])
8618 (define_insn "<avx512>_<code><ssedoublemodelower><mode>2_mask"
8619 [(set (match_operand:PMOV_DST_MODE_2 0 "nonimmediate_operand" "=v,m")
8620 (vec_merge:PMOV_DST_MODE_2
8621 (any_truncate:PMOV_DST_MODE_2
8622 (match_operand:<ssedoublemode> 1 "register_operand" "v,v"))
8623 (match_operand:PMOV_DST_MODE_2 2 "vector_move_operand" "0C,0")
8624 (match_operand:<avx512fmaskmode> 3 "register_operand" "Yk,Yk")))]
8626 "vpmov<trunsuffix><pmov_suff_2>\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"
8627 [(set_attr "type" "ssemov")
8628 (set_attr "memory" "none,store")
8629 (set_attr "prefix" "evex")
8630 (set_attr "mode" "<sseinsnmode>")])
8632 (define_expand "<avx512>_<code><ssedoublemodelower><mode>2_mask_store"
8633 [(set (match_operand:PMOV_DST_MODE_2 0 "nonimmediate_operand")
8634 (vec_merge:PMOV_DST_MODE_2
8635 (any_truncate:PMOV_DST_MODE_2
8636 (match_operand:<ssedoublemode> 1 "register_operand"))
8638 (match_operand:<avx512fmaskmode> 2 "register_operand")))]
8641 (define_mode_iterator PMOV_SRC_MODE_3 [V4DI V2DI V8SI V4SI (V8HI "TARGET_AVX512BW")])
8642 (define_mode_attr pmov_dst_3
8643 [(V4DI "V4QI") (V2DI "V2QI") (V8SI "V8QI") (V4SI "V4QI") (V8HI "V8QI")])
8644 (define_mode_attr pmov_dst_zeroed_3
8645 [(V4DI "V12QI") (V2DI "V14QI") (V8SI "V8QI") (V4SI "V12QI") (V8HI "V8QI")])
8646 (define_mode_attr pmov_suff_3
8647 [(V4DI "qb") (V2DI "qb") (V8SI "db") (V4SI "db") (V8HI "wb")])
8649 (define_insn "*avx512vl_<code><mode>v<ssescalarnum>qi2"
8650 [(set (match_operand:V16QI 0 "register_operand" "=v")
8652 (any_truncate:<pmov_dst_3>
8653 (match_operand:PMOV_SRC_MODE_3 1 "register_operand" "v"))
8654 (match_operand:<pmov_dst_zeroed_3> 2 "const0_operand")))]
8656 "vpmov<trunsuffix><pmov_suff_3>\t{%1, %0|%0, %1}"
8657 [(set_attr "type" "ssemov")
8658 (set_attr "prefix" "evex")
8659 (set_attr "mode" "TI")])
8661 (define_insn "*avx512vl_<code>v2div2qi2_store"
8662 [(set (match_operand:V16QI 0 "memory_operand" "=m")
8665 (match_operand:V2DI 1 "register_operand" "v"))
8668 (parallel [(const_int 2) (const_int 3)
8669 (const_int 4) (const_int 5)
8670 (const_int 6) (const_int 7)
8671 (const_int 8) (const_int 9)
8672 (const_int 10) (const_int 11)
8673 (const_int 12) (const_int 13)
8674 (const_int 14) (const_int 15)]))))]
8676 "vpmov<trunsuffix>qb\t{%1, %0|%0, %1}"
8677 [(set_attr "type" "ssemov")
8678 (set_attr "memory" "store")
8679 (set_attr "prefix" "evex")
8680 (set_attr "mode" "TI")])
8682 (define_insn "avx512vl_<code>v2div2qi2_mask"
8683 [(set (match_operand:V16QI 0 "register_operand" "=v")
8687 (match_operand:V2DI 1 "register_operand" "v"))
8689 (match_operand:V16QI 2 "vector_move_operand" "0C")
8690 (parallel [(const_int 0) (const_int 1)]))
8691 (match_operand:QI 3 "register_operand" "Yk"))
8692 (const_vector:V14QI [(const_int 0) (const_int 0)
8693 (const_int 0) (const_int 0)
8694 (const_int 0) (const_int 0)
8695 (const_int 0) (const_int 0)
8696 (const_int 0) (const_int 0)
8697 (const_int 0) (const_int 0)
8698 (const_int 0) (const_int 0)])))]
8700 "vpmov<trunsuffix>qb\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"
8701 [(set_attr "type" "ssemov")
8702 (set_attr "prefix" "evex")
8703 (set_attr "mode" "TI")])
8705 (define_insn "avx512vl_<code>v2div2qi2_mask_store"
8706 [(set (match_operand:V16QI 0 "memory_operand" "=m")
8710 (match_operand:V2DI 1 "register_operand" "v"))
8713 (parallel [(const_int 0) (const_int 1)]))
8714 (match_operand:QI 2 "register_operand" "Yk"))
8717 (parallel [(const_int 2) (const_int 3)
8718 (const_int 4) (const_int 5)
8719 (const_int 6) (const_int 7)
8720 (const_int 8) (const_int 9)
8721 (const_int 10) (const_int 11)
8722 (const_int 12) (const_int 13)
8723 (const_int 14) (const_int 15)]))))]
8725 "vpmov<trunsuffix>qb\t{%1, %0%{%2%}|%0%{%2%}, %1}"
8726 [(set_attr "type" "ssemov")
8727 (set_attr "memory" "store")
8728 (set_attr "prefix" "evex")
8729 (set_attr "mode" "TI")])
8731 (define_insn "*avx512vl_<code><mode>v4qi2_store"
8732 [(set (match_operand:V16QI 0 "memory_operand" "=m")
8735 (match_operand:VI4_128_8_256 1 "register_operand" "v"))
8738 (parallel [(const_int 4) (const_int 5)
8739 (const_int 6) (const_int 7)
8740 (const_int 8) (const_int 9)
8741 (const_int 10) (const_int 11)
8742 (const_int 12) (const_int 13)
8743 (const_int 14) (const_int 15)]))))]
8745 "vpmov<trunsuffix><pmov_suff_3>\t{%1, %0|%0, %1}"
8746 [(set_attr "type" "ssemov")
8747 (set_attr "memory" "store")
8748 (set_attr "prefix" "evex")
8749 (set_attr "mode" "TI")])
8751 (define_insn "avx512vl_<code><mode>v4qi2_mask"
8752 [(set (match_operand:V16QI 0 "register_operand" "=v")
8756 (match_operand:VI4_128_8_256 1 "register_operand" "v"))
8758 (match_operand:V16QI 2 "vector_move_operand" "0C")
8759 (parallel [(const_int 0) (const_int 1)
8760 (const_int 2) (const_int 3)]))
8761 (match_operand:QI 3 "register_operand" "Yk"))
8762 (const_vector:V12QI [(const_int 0) (const_int 0)
8763 (const_int 0) (const_int 0)
8764 (const_int 0) (const_int 0)
8765 (const_int 0) (const_int 0)
8766 (const_int 0) (const_int 0)
8767 (const_int 0) (const_int 0)])))]
8769 "vpmov<trunsuffix><pmov_suff_3>\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"
8770 [(set_attr "type" "ssemov")
8771 (set_attr "prefix" "evex")
8772 (set_attr "mode" "TI")])
8774 (define_insn "avx512vl_<code><mode>v4qi2_mask_store"
8775 [(set (match_operand:V16QI 0 "memory_operand" "=m")
8779 (match_operand:VI4_128_8_256 1 "register_operand" "v"))
8782 (parallel [(const_int 0) (const_int 1)
8783 (const_int 2) (const_int 3)]))
8784 (match_operand:QI 2 "register_operand" "Yk"))
8787 (parallel [(const_int 4) (const_int 5)
8788 (const_int 6) (const_int 7)
8789 (const_int 8) (const_int 9)
8790 (const_int 10) (const_int 11)
8791 (const_int 12) (const_int 13)
8792 (const_int 14) (const_int 15)]))))]
8794 "vpmov<trunsuffix><pmov_suff_3>\t{%1, %0%{%2%}|%0%{%2%}, %1}"
8795 [(set_attr "type" "ssemov")
8796 (set_attr "memory" "store")
8797 (set_attr "prefix" "evex")
8798 (set_attr "mode" "TI")])
8800 (define_mode_iterator VI2_128_BW_4_256
8801 [(V8HI "TARGET_AVX512BW") V8SI])
8803 (define_insn "*avx512vl_<code><mode>v8qi2_store"
8804 [(set (match_operand:V16QI 0 "memory_operand" "=m")
8807 (match_operand:VI2_128_BW_4_256 1 "register_operand" "v"))
8810 (parallel [(const_int 8) (const_int 9)
8811 (const_int 10) (const_int 11)
8812 (const_int 12) (const_int 13)
8813 (const_int 14) (const_int 15)]))))]
8815 "vpmov<trunsuffix><pmov_suff_3>\t{%1, %0|%0, %1}"
8816 [(set_attr "type" "ssemov")
8817 (set_attr "memory" "store")
8818 (set_attr "prefix" "evex")
8819 (set_attr "mode" "TI")])
8821 (define_insn "avx512vl_<code><mode>v8qi2_mask"
8822 [(set (match_operand:V16QI 0 "register_operand" "=v")
8826 (match_operand:VI2_128_BW_4_256 1 "register_operand" "v"))
8828 (match_operand:V16QI 2 "vector_move_operand" "0C")
8829 (parallel [(const_int 0) (const_int 1)
8830 (const_int 2) (const_int 3)
8831 (const_int 4) (const_int 5)
8832 (const_int 6) (const_int 7)]))
8833 (match_operand:QI 3 "register_operand" "Yk"))
8834 (const_vector:V8QI [(const_int 0) (const_int 0)
8835 (const_int 0) (const_int 0)
8836 (const_int 0) (const_int 0)
8837 (const_int 0) (const_int 0)])))]
8839 "vpmov<trunsuffix><pmov_suff_3>\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"
8840 [(set_attr "type" "ssemov")
8841 (set_attr "prefix" "evex")
8842 (set_attr "mode" "TI")])
8844 (define_insn "avx512vl_<code><mode>v8qi2_mask_store"
8845 [(set (match_operand:V16QI 0 "memory_operand" "=m")
8849 (match_operand:VI2_128_BW_4_256 1 "register_operand" "v"))
8852 (parallel [(const_int 0) (const_int 1)
8853 (const_int 2) (const_int 3)
8854 (const_int 4) (const_int 5)
8855 (const_int 6) (const_int 7)]))
8856 (match_operand:QI 2 "register_operand" "Yk"))
8859 (parallel [(const_int 8) (const_int 9)
8860 (const_int 10) (const_int 11)
8861 (const_int 12) (const_int 13)
8862 (const_int 14) (const_int 15)]))))]
8864 "vpmov<trunsuffix><pmov_suff_3>\t{%1, %0%{%2%}|%0%{%2%}, %1}"
8865 [(set_attr "type" "ssemov")
8866 (set_attr "memory" "store")
8867 (set_attr "prefix" "evex")
8868 (set_attr "mode" "TI")])
8870 (define_mode_iterator PMOV_SRC_MODE_4 [V4DI V2DI V4SI])
8871 (define_mode_attr pmov_dst_4
8872 [(V4DI "V4HI") (V2DI "V2HI") (V4SI "V4HI")])
8873 (define_mode_attr pmov_dst_zeroed_4
8874 [(V4DI "V4HI") (V2DI "V6HI") (V4SI "V4HI")])
8875 (define_mode_attr pmov_suff_4
8876 [(V4DI "qw") (V2DI "qw") (V4SI "dw")])
8878 (define_insn "*avx512vl_<code><mode>v<ssescalarnum>hi2"
8879 [(set (match_operand:V8HI 0 "register_operand" "=v")
8881 (any_truncate:<pmov_dst_4>
8882 (match_operand:PMOV_SRC_MODE_4 1 "register_operand" "v"))
8883 (match_operand:<pmov_dst_zeroed_4> 2 "const0_operand")))]
8885 "vpmov<trunsuffix><pmov_suff_4>\t{%1, %0|%0, %1}"
8886 [(set_attr "type" "ssemov")
8887 (set_attr "prefix" "evex")
8888 (set_attr "mode" "TI")])
8890 (define_insn "*avx512vl_<code><mode>v4hi2_store"
8891 [(set (match_operand:V8HI 0 "memory_operand" "=m")
8894 (match_operand:VI4_128_8_256 1 "register_operand" "v"))
8897 (parallel [(const_int 4) (const_int 5)
8898 (const_int 6) (const_int 7)]))))]
8900 "vpmov<trunsuffix><pmov_suff_4>\t{%1, %0|%0, %1}"
8901 [(set_attr "type" "ssemov")
8902 (set_attr "memory" "store")
8903 (set_attr "prefix" "evex")
8904 (set_attr "mode" "TI")])
8906 (define_insn "avx512vl_<code><mode>v4hi2_mask"
8907 [(set (match_operand:V8HI 0 "register_operand" "=v")
8911 (match_operand:VI4_128_8_256 1 "register_operand" "v"))
8913 (match_operand:V8HI 2 "vector_move_operand" "0C")
8914 (parallel [(const_int 0) (const_int 1)
8915 (const_int 2) (const_int 3)]))
8916 (match_operand:QI 3 "register_operand" "Yk"))
8917 (const_vector:V4HI [(const_int 0) (const_int 0)
8918 (const_int 0) (const_int 0)])))]
8920 "vpmov<trunsuffix><pmov_suff_4>\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"
8921 [(set_attr "type" "ssemov")
8922 (set_attr "prefix" "evex")
8923 (set_attr "mode" "TI")])
8925 (define_insn "avx512vl_<code><mode>v4hi2_mask_store"
8926 [(set (match_operand:V8HI 0 "memory_operand" "=m")
8930 (match_operand:VI4_128_8_256 1 "register_operand" "v"))
8933 (parallel [(const_int 0) (const_int 1)
8934 (const_int 2) (const_int 3)]))
8935 (match_operand:QI 2 "register_operand" "Yk"))
8938 (parallel [(const_int 4) (const_int 5)
8939 (const_int 6) (const_int 7)]))))]
8941 "vpmov<trunsuffix><pmov_suff_4>\t{%1, %0%{%2%}|%0%{%2%}, %1}"
8942 [(set_attr "type" "ssemov")
8943 (set_attr "memory" "store")
8944 (set_attr "prefix" "evex")
8945 (set_attr "mode" "TI")])
8947 (define_insn "*avx512vl_<code>v2div2hi2_store"
8948 [(set (match_operand:V8HI 0 "memory_operand" "=m")
8951 (match_operand:V2DI 1 "register_operand" "v"))
8954 (parallel [(const_int 2) (const_int 3)
8955 (const_int 4) (const_int 5)
8956 (const_int 6) (const_int 7)]))))]
8958 "vpmov<trunsuffix>qw\t{%1, %0|%0, %1}"
8959 [(set_attr "type" "ssemov")
8960 (set_attr "memory" "store")
8961 (set_attr "prefix" "evex")
8962 (set_attr "mode" "TI")])
8964 (define_insn "avx512vl_<code>v2div2hi2_mask"
8965 [(set (match_operand:V8HI 0 "register_operand" "=v")
8969 (match_operand:V2DI 1 "register_operand" "v"))
8971 (match_operand:V8HI 2 "vector_move_operand" "0C")
8972 (parallel [(const_int 0) (const_int 1)]))
8973 (match_operand:QI 3 "register_operand" "Yk"))
8974 (const_vector:V6HI [(const_int 0) (const_int 0)
8975 (const_int 0) (const_int 0)
8976 (const_int 0) (const_int 0)])))]
8978 "vpmov<trunsuffix>qw\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"
8979 [(set_attr "type" "ssemov")
8980 (set_attr "prefix" "evex")
8981 (set_attr "mode" "TI")])
8983 (define_insn "avx512vl_<code>v2div2hi2_mask_store"
8984 [(set (match_operand:V8HI 0 "memory_operand" "=m")
8988 (match_operand:V2DI 1 "register_operand" "v"))
8991 (parallel [(const_int 0) (const_int 1)]))
8992 (match_operand:QI 2 "register_operand" "Yk"))
8995 (parallel [(const_int 2) (const_int 3)
8996 (const_int 4) (const_int 5)
8997 (const_int 6) (const_int 7)]))))]
8999 "vpmov<trunsuffix>qw\t{%1, %0%{%2%}|%0%{%2%}, %1}"
9000 [(set_attr "type" "ssemov")
9001 (set_attr "memory" "store")
9002 (set_attr "prefix" "evex")
9003 (set_attr "mode" "TI")])
9005 (define_insn "*avx512vl_<code>v2div2si2"
9006 [(set (match_operand:V4SI 0 "register_operand" "=v")
9009 (match_operand:V2DI 1 "register_operand" "v"))
9010 (match_operand:V2SI 2 "const0_operand")))]
9012 "vpmov<trunsuffix>qd\t{%1, %0|%0, %1}"
9013 [(set_attr "type" "ssemov")
9014 (set_attr "prefix" "evex")
9015 (set_attr "mode" "TI")])
9017 (define_insn "*avx512vl_<code>v2div2si2_store"
9018 [(set (match_operand:V4SI 0 "memory_operand" "=m")
9021 (match_operand:V2DI 1 "register_operand" "v"))
9024 (parallel [(const_int 2) (const_int 3)]))))]
9026 "vpmov<trunsuffix>qd\t{%1, %0|%0, %1}"
9027 [(set_attr "type" "ssemov")
9028 (set_attr "memory" "store")
9029 (set_attr "prefix" "evex")
9030 (set_attr "mode" "TI")])
9032 (define_insn "avx512vl_<code>v2div2si2_mask"
9033 [(set (match_operand:V4SI 0 "register_operand" "=v")
9037 (match_operand:V2DI 1 "register_operand" "v"))
9039 (match_operand:V4SI 2 "vector_move_operand" "0C")
9040 (parallel [(const_int 0) (const_int 1)]))
9041 (match_operand:QI 3 "register_operand" "Yk"))
9042 (const_vector:V2SI [(const_int 0) (const_int 0)])))]
9044 "vpmov<trunsuffix>qd\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"
9045 [(set_attr "type" "ssemov")
9046 (set_attr "prefix" "evex")
9047 (set_attr "mode" "TI")])
9049 (define_insn "avx512vl_<code>v2div2si2_mask_store"
9050 [(set (match_operand:V4SI 0 "memory_operand" "=m")
9054 (match_operand:V2DI 1 "register_operand" "v"))
9057 (parallel [(const_int 0) (const_int 1)]))
9058 (match_operand:QI 2 "register_operand" "Yk"))
9061 (parallel [(const_int 2) (const_int 3)]))))]
9063 "vpmov<trunsuffix>qd\t{%1, %0%{%2%}|%0%{%2%}, %1}"
9064 [(set_attr "type" "ssemov")
9065 (set_attr "memory" "store")
9066 (set_attr "prefix" "evex")
9067 (set_attr "mode" "TI")])
9069 (define_insn "*avx512f_<code>v8div16qi2"
9070 [(set (match_operand:V16QI 0 "register_operand" "=v")
9073 (match_operand:V8DI 1 "register_operand" "v"))
9074 (const_vector:V8QI [(const_int 0) (const_int 0)
9075 (const_int 0) (const_int 0)
9076 (const_int 0) (const_int 0)
9077 (const_int 0) (const_int 0)])))]
9079 "vpmov<trunsuffix>qb\t{%1, %0|%0, %1}"
9080 [(set_attr "type" "ssemov")
9081 (set_attr "prefix" "evex")
9082 (set_attr "mode" "TI")])
9084 (define_insn "*avx512f_<code>v8div16qi2_store"
9085 [(set (match_operand:V16QI 0 "memory_operand" "=m")
9088 (match_operand:V8DI 1 "register_operand" "v"))
9091 (parallel [(const_int 8) (const_int 9)
9092 (const_int 10) (const_int 11)
9093 (const_int 12) (const_int 13)
9094 (const_int 14) (const_int 15)]))))]
9096 "vpmov<trunsuffix>qb\t{%1, %0|%0, %1}"
9097 [(set_attr "type" "ssemov")
9098 (set_attr "memory" "store")
9099 (set_attr "prefix" "evex")
9100 (set_attr "mode" "TI")])
9102 (define_insn "avx512f_<code>v8div16qi2_mask"
9103 [(set (match_operand:V16QI 0 "register_operand" "=v")
9107 (match_operand:V8DI 1 "register_operand" "v"))
9109 (match_operand:V16QI 2 "vector_move_operand" "0C")
9110 (parallel [(const_int 0) (const_int 1)
9111 (const_int 2) (const_int 3)
9112 (const_int 4) (const_int 5)
9113 (const_int 6) (const_int 7)]))
9114 (match_operand:QI 3 "register_operand" "Yk"))
9115 (const_vector:V8QI [(const_int 0) (const_int 0)
9116 (const_int 0) (const_int 0)
9117 (const_int 0) (const_int 0)
9118 (const_int 0) (const_int 0)])))]
9120 "vpmov<trunsuffix>qb\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"
9121 [(set_attr "type" "ssemov")
9122 (set_attr "prefix" "evex")
9123 (set_attr "mode" "TI")])
9125 (define_insn "avx512f_<code>v8div16qi2_mask_store"
9126 [(set (match_operand:V16QI 0 "memory_operand" "=m")
9130 (match_operand:V8DI 1 "register_operand" "v"))
9133 (parallel [(const_int 0) (const_int 1)
9134 (const_int 2) (const_int 3)
9135 (const_int 4) (const_int 5)
9136 (const_int 6) (const_int 7)]))
9137 (match_operand:QI 2 "register_operand" "Yk"))
9140 (parallel [(const_int 8) (const_int 9)
9141 (const_int 10) (const_int 11)
9142 (const_int 12) (const_int 13)
9143 (const_int 14) (const_int 15)]))))]
9145 "vpmov<trunsuffix>qb\t{%1, %0%{%2%}|%0%{%2%}, %1}"
9146 [(set_attr "type" "ssemov")
9147 (set_attr "memory" "store")
9148 (set_attr "prefix" "evex")
9149 (set_attr "mode" "TI")])
9151 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
9153 ;; Parallel integral arithmetic
9155 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
9157 (define_expand "neg<mode>2"
9158 [(set (match_operand:VI_AVX2 0 "register_operand")
9161 (match_operand:VI_AVX2 1 "nonimmediate_operand")))]
9163 "operands[2] = force_reg (<MODE>mode, CONST0_RTX (<MODE>mode));")
9165 (define_expand "<plusminus_insn><mode>3"
9166 [(set (match_operand:VI_AVX2 0 "register_operand")
9168 (match_operand:VI_AVX2 1 "nonimmediate_operand")
9169 (match_operand:VI_AVX2 2 "nonimmediate_operand")))]
9171 "ix86_fixup_binary_operands_no_copy (<CODE>, <MODE>mode, operands);")
9173 (define_expand "<plusminus_insn><mode>3_mask"
9174 [(set (match_operand:VI48_AVX512VL 0 "register_operand")
9175 (vec_merge:VI48_AVX512VL
9176 (plusminus:VI48_AVX512VL
9177 (match_operand:VI48_AVX512VL 1 "nonimmediate_operand")
9178 (match_operand:VI48_AVX512VL 2 "nonimmediate_operand"))
9179 (match_operand:VI48_AVX512VL 3 "vector_move_operand")
9180 (match_operand:<avx512fmaskmode> 4 "register_operand")))]
9182 "ix86_fixup_binary_operands_no_copy (<CODE>, <MODE>mode, operands);")
9184 (define_expand "<plusminus_insn><mode>3_mask"
9185 [(set (match_operand:VI12_AVX512VL 0 "register_operand")
9186 (vec_merge:VI12_AVX512VL
9187 (plusminus:VI12_AVX512VL
9188 (match_operand:VI12_AVX512VL 1 "nonimmediate_operand")
9189 (match_operand:VI12_AVX512VL 2 "nonimmediate_operand"))
9190 (match_operand:VI12_AVX512VL 3 "vector_move_operand")
9191 (match_operand:<avx512fmaskmode> 4 "register_operand")))]
9193 "ix86_fixup_binary_operands_no_copy (<CODE>, <MODE>mode, operands);")
9195 (define_insn "*<plusminus_insn><mode>3"
9196 [(set (match_operand:VI_AVX2 0 "register_operand" "=x,v")
9198 (match_operand:VI_AVX2 1 "nonimmediate_operand" "<comm>0,v")
9199 (match_operand:VI_AVX2 2 "nonimmediate_operand" "xm,vm")))]
9201 && ix86_binary_operator_ok (<CODE>, <MODE>mode, operands)"
9203 p<plusminus_mnemonic><ssemodesuffix>\t{%2, %0|%0, %2}
9204 vp<plusminus_mnemonic><ssemodesuffix>\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
9205 [(set_attr "isa" "noavx,avx")
9206 (set_attr "type" "sseiadd")
9207 (set_attr "prefix_data16" "1,*")
9208 (set_attr "prefix" "<mask_prefix3>")
9209 (set_attr "mode" "<sseinsnmode>")])
9211 (define_insn "*<plusminus_insn><mode>3_mask"
9212 [(set (match_operand:VI48_AVX512VL 0 "register_operand" "=v")
9213 (vec_merge:VI48_AVX512VL
9214 (plusminus:VI48_AVX512VL
9215 (match_operand:VI48_AVX512VL 1 "nonimmediate_operand" "<comm>v")
9216 (match_operand:VI48_AVX512VL 2 "nonimmediate_operand" "vm"))
9217 (match_operand:VI48_AVX512VL 3 "vector_move_operand" "0C")
9218 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk")))]
9220 && ix86_binary_operator_ok (<CODE>, <MODE>mode, operands)"
9221 "vp<plusminus_mnemonic><ssemodesuffix>\t{%2, %1, %0%{%4%}%N3|%0%{%4%}%N3, %1, %2}"
9222 [(set_attr "type" "sseiadd")
9223 (set_attr "prefix" "evex")
9224 (set_attr "mode" "<sseinsnmode>")])
9226 (define_insn "*<plusminus_insn><mode>3_mask"
9227 [(set (match_operand:VI12_AVX512VL 0 "register_operand" "=v")
9228 (vec_merge:VI12_AVX512VL
9229 (plusminus:VI12_AVX512VL
9230 (match_operand:VI12_AVX512VL 1 "nonimmediate_operand" "<comm>v")
9231 (match_operand:VI12_AVX512VL 2 "nonimmediate_operand" "vm"))
9232 (match_operand:VI12_AVX512VL 3 "vector_move_operand" "0C")
9233 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk")))]
9234 "TARGET_AVX512BW && ix86_binary_operator_ok (<CODE>, <MODE>mode, operands)"
9235 "vp<plusminus_mnemonic><ssemodesuffix>\t{%2, %1, %0%{%4%}%N3|%0%{%4%}%N3, %1, %2}"
9236 [(set_attr "type" "sseiadd")
9237 (set_attr "prefix" "evex")
9238 (set_attr "mode" "<sseinsnmode>")])
9240 (define_expand "<sse2_avx2>_<plusminus_insn><mode>3<mask_name>"
9241 [(set (match_operand:VI12_AVX2 0 "register_operand")
9242 (sat_plusminus:VI12_AVX2
9243 (match_operand:VI12_AVX2 1 "nonimmediate_operand")
9244 (match_operand:VI12_AVX2 2 "nonimmediate_operand")))]
9245 "TARGET_SSE2 && <mask_mode512bit_condition> && <mask_avx512bw_condition>"
9246 "ix86_fixup_binary_operands_no_copy (<CODE>, <MODE>mode, operands);")
9248 (define_insn "*<sse2_avx2>_<plusminus_insn><mode>3<mask_name>"
9249 [(set (match_operand:VI12_AVX2 0 "register_operand" "=x,v")
9250 (sat_plusminus:VI12_AVX2
9251 (match_operand:VI12_AVX2 1 "nonimmediate_operand" "<comm>0,v")
9252 (match_operand:VI12_AVX2 2 "nonimmediate_operand" "xm,vm")))]
9253 "TARGET_SSE2 && <mask_mode512bit_condition> && <mask_avx512bw_condition>
9254 && ix86_binary_operator_ok (<CODE>, <MODE>mode, operands)"
9256 p<plusminus_mnemonic><ssemodesuffix>\t{%2, %0|%0, %2}
9257 vp<plusminus_mnemonic><ssemodesuffix>\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
9258 [(set_attr "isa" "noavx,avx")
9259 (set_attr "type" "sseiadd")
9260 (set_attr "prefix_data16" "1,*")
9261 (set_attr "prefix" "orig,maybe_evex")
9262 (set_attr "mode" "TI")])
9264 (define_expand "mul<mode>3<mask_name>"
9265 [(set (match_operand:VI1_AVX512 0 "register_operand")
9266 (mult:VI1_AVX512 (match_operand:VI1_AVX512 1 "register_operand")
9267 (match_operand:VI1_AVX512 2 "register_operand")))]
9268 "TARGET_SSE2 && <mask_mode512bit_condition> && <mask_avx512bw_condition>"
9270 ix86_expand_vecop_qihi (MULT, operands[0], operands[1], operands[2]);
9274 (define_expand "mul<mode>3<mask_name>"
9275 [(set (match_operand:VI2_AVX2 0 "register_operand")
9276 (mult:VI2_AVX2 (match_operand:VI2_AVX2 1 "nonimmediate_operand")
9277 (match_operand:VI2_AVX2 2 "nonimmediate_operand")))]
9278 "TARGET_SSE2 && <mask_mode512bit_condition> && <mask_avx512bw_condition>"
9279 "ix86_fixup_binary_operands_no_copy (MULT, <MODE>mode, operands);")
9281 (define_insn "*mul<mode>3<mask_name>"
9282 [(set (match_operand:VI2_AVX2 0 "register_operand" "=x,v")
9283 (mult:VI2_AVX2 (match_operand:VI2_AVX2 1 "nonimmediate_operand" "%0,v")
9284 (match_operand:VI2_AVX2 2 "nonimmediate_operand" "xm,vm")))]
9286 && ix86_binary_operator_ok (MULT, <MODE>mode, operands)
9287 && <mask_mode512bit_condition> && <mask_avx512bw_condition>"
9289 pmullw\t{%2, %0|%0, %2}
9290 vpmullw\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
9291 [(set_attr "isa" "noavx,avx")
9292 (set_attr "type" "sseimul")
9293 (set_attr "prefix_data16" "1,*")
9294 (set_attr "prefix" "orig,vex")
9295 (set_attr "mode" "<sseinsnmode>")])
9297 (define_expand "<s>mul<mode>3_highpart<mask_name>"
9298 [(set (match_operand:VI2_AVX2 0 "register_operand")
9300 (lshiftrt:<ssedoublemode>
9301 (mult:<ssedoublemode>
9302 (any_extend:<ssedoublemode>
9303 (match_operand:VI2_AVX2 1 "nonimmediate_operand"))
9304 (any_extend:<ssedoublemode>
9305 (match_operand:VI2_AVX2 2 "nonimmediate_operand")))
9308 && <mask_mode512bit_condition> && <mask_avx512bw_condition>"
9309 "ix86_fixup_binary_operands_no_copy (MULT, <MODE>mode, operands);")
9311 (define_insn "*<s>mul<mode>3_highpart<mask_name>"
9312 [(set (match_operand:VI2_AVX2 0 "register_operand" "=x,v")
9314 (lshiftrt:<ssedoublemode>
9315 (mult:<ssedoublemode>
9316 (any_extend:<ssedoublemode>
9317 (match_operand:VI2_AVX2 1 "nonimmediate_operand" "%0,v"))
9318 (any_extend:<ssedoublemode>
9319 (match_operand:VI2_AVX2 2 "nonimmediate_operand" "xm,vm")))
9322 && ix86_binary_operator_ok (MULT, <MODE>mode, operands)
9323 && <mask_mode512bit_condition> && <mask_avx512bw_condition>"
9325 pmulh<u>w\t{%2, %0|%0, %2}
9326 vpmulh<u>w\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
9327 [(set_attr "isa" "noavx,avx")
9328 (set_attr "type" "sseimul")
9329 (set_attr "prefix_data16" "1,*")
9330 (set_attr "prefix" "orig,vex")
9331 (set_attr "mode" "<sseinsnmode>")])
9333 (define_expand "vec_widen_umult_even_v16si<mask_name>"
9334 [(set (match_operand:V8DI 0 "register_operand")
9338 (match_operand:V16SI 1 "nonimmediate_operand")
9339 (parallel [(const_int 0) (const_int 2)
9340 (const_int 4) (const_int 6)
9341 (const_int 8) (const_int 10)
9342 (const_int 12) (const_int 14)])))
9345 (match_operand:V16SI 2 "nonimmediate_operand")
9346 (parallel [(const_int 0) (const_int 2)
9347 (const_int 4) (const_int 6)
9348 (const_int 8) (const_int 10)
9349 (const_int 12) (const_int 14)])))))]
9351 "ix86_fixup_binary_operands_no_copy (MULT, V16SImode, operands);")
9353 (define_insn "*vec_widen_umult_even_v16si<mask_name>"
9354 [(set (match_operand:V8DI 0 "register_operand" "=v")
9358 (match_operand:V16SI 1 "nonimmediate_operand" "%v")
9359 (parallel [(const_int 0) (const_int 2)
9360 (const_int 4) (const_int 6)
9361 (const_int 8) (const_int 10)
9362 (const_int 12) (const_int 14)])))
9365 (match_operand:V16SI 2 "nonimmediate_operand" "vm")
9366 (parallel [(const_int 0) (const_int 2)
9367 (const_int 4) (const_int 6)
9368 (const_int 8) (const_int 10)
9369 (const_int 12) (const_int 14)])))))]
9370 "TARGET_AVX512F && ix86_binary_operator_ok (MULT, V16SImode, operands)"
9371 "vpmuludq\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
9372 [(set_attr "isa" "avx512f")
9373 (set_attr "type" "sseimul")
9374 (set_attr "prefix_extra" "1")
9375 (set_attr "prefix" "evex")
9376 (set_attr "mode" "XI")])
9378 (define_expand "vec_widen_umult_even_v8si<mask_name>"
9379 [(set (match_operand:V4DI 0 "register_operand")
9383 (match_operand:V8SI 1 "nonimmediate_operand")
9384 (parallel [(const_int 0) (const_int 2)
9385 (const_int 4) (const_int 6)])))
9388 (match_operand:V8SI 2 "nonimmediate_operand")
9389 (parallel [(const_int 0) (const_int 2)
9390 (const_int 4) (const_int 6)])))))]
9391 "TARGET_AVX2 && <mask_avx512vl_condition>"
9392 "ix86_fixup_binary_operands_no_copy (MULT, V8SImode, operands);")
9394 (define_insn "*vec_widen_umult_even_v8si<mask_name>"
9395 [(set (match_operand:V4DI 0 "register_operand" "=v")
9399 (match_operand:V8SI 1 "nonimmediate_operand" "%v")
9400 (parallel [(const_int 0) (const_int 2)
9401 (const_int 4) (const_int 6)])))
9404 (match_operand:V8SI 2 "nonimmediate_operand" "vm")
9405 (parallel [(const_int 0) (const_int 2)
9406 (const_int 4) (const_int 6)])))))]
9407 "TARGET_AVX2 && <mask_avx512vl_condition>
9408 && ix86_binary_operator_ok (MULT, V8SImode, operands)"
9409 "vpmuludq\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
9410 [(set_attr "type" "sseimul")
9411 (set_attr "prefix" "maybe_evex")
9412 (set_attr "mode" "OI")])
9414 (define_expand "vec_widen_umult_even_v4si<mask_name>"
9415 [(set (match_operand:V2DI 0 "register_operand")
9419 (match_operand:V4SI 1 "nonimmediate_operand")
9420 (parallel [(const_int 0) (const_int 2)])))
9423 (match_operand:V4SI 2 "nonimmediate_operand")
9424 (parallel [(const_int 0) (const_int 2)])))))]
9425 "TARGET_SSE2 && <mask_avx512vl_condition>"
9426 "ix86_fixup_binary_operands_no_copy (MULT, V4SImode, operands);")
9428 (define_insn "*vec_widen_umult_even_v4si<mask_name>"
9429 [(set (match_operand:V2DI 0 "register_operand" "=x,v")
9433 (match_operand:V4SI 1 "nonimmediate_operand" "%0,v")
9434 (parallel [(const_int 0) (const_int 2)])))
9437 (match_operand:V4SI 2 "nonimmediate_operand" "xm,vm")
9438 (parallel [(const_int 0) (const_int 2)])))))]
9439 "TARGET_SSE2 && <mask_avx512vl_condition>
9440 && ix86_binary_operator_ok (MULT, V4SImode, operands)"
9442 pmuludq\t{%2, %0|%0, %2}
9443 vpmuludq\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
9444 [(set_attr "isa" "noavx,avx")
9445 (set_attr "type" "sseimul")
9446 (set_attr "prefix_data16" "1,*")
9447 (set_attr "prefix" "orig,maybe_evex")
9448 (set_attr "mode" "TI")])
9450 (define_expand "vec_widen_smult_even_v16si<mask_name>"
9451 [(set (match_operand:V8DI 0 "register_operand")
9455 (match_operand:V16SI 1 "nonimmediate_operand")
9456 (parallel [(const_int 0) (const_int 2)
9457 (const_int 4) (const_int 6)
9458 (const_int 8) (const_int 10)
9459 (const_int 12) (const_int 14)])))
9462 (match_operand:V16SI 2 "nonimmediate_operand")
9463 (parallel [(const_int 0) (const_int 2)
9464 (const_int 4) (const_int 6)
9465 (const_int 8) (const_int 10)
9466 (const_int 12) (const_int 14)])))))]
9468 "ix86_fixup_binary_operands_no_copy (MULT, V16SImode, operands);")
9470 (define_insn "*vec_widen_smult_even_v16si<mask_name>"
9471 [(set (match_operand:V8DI 0 "register_operand" "=v")
9475 (match_operand:V16SI 1 "nonimmediate_operand" "%v")
9476 (parallel [(const_int 0) (const_int 2)
9477 (const_int 4) (const_int 6)
9478 (const_int 8) (const_int 10)
9479 (const_int 12) (const_int 14)])))
9482 (match_operand:V16SI 2 "nonimmediate_operand" "vm")
9483 (parallel [(const_int 0) (const_int 2)
9484 (const_int 4) (const_int 6)
9485 (const_int 8) (const_int 10)
9486 (const_int 12) (const_int 14)])))))]
9487 "TARGET_AVX512F && ix86_binary_operator_ok (MULT, V16SImode, operands)"
9488 "vpmuldq\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
9489 [(set_attr "isa" "avx512f")
9490 (set_attr "type" "sseimul")
9491 (set_attr "prefix_extra" "1")
9492 (set_attr "prefix" "evex")
9493 (set_attr "mode" "XI")])
9495 (define_expand "vec_widen_smult_even_v8si<mask_name>"
9496 [(set (match_operand:V4DI 0 "register_operand")
9500 (match_operand:V8SI 1 "nonimmediate_operand")
9501 (parallel [(const_int 0) (const_int 2)
9502 (const_int 4) (const_int 6)])))
9505 (match_operand:V8SI 2 "nonimmediate_operand")
9506 (parallel [(const_int 0) (const_int 2)
9507 (const_int 4) (const_int 6)])))))]
9508 "TARGET_AVX2 && <mask_avx512vl_condition>"
9509 "ix86_fixup_binary_operands_no_copy (MULT, V8SImode, operands);")
9511 (define_insn "*vec_widen_smult_even_v8si<mask_name>"
9512 [(set (match_operand:V4DI 0 "register_operand" "=v")
9516 (match_operand:V8SI 1 "nonimmediate_operand" "v")
9517 (parallel [(const_int 0) (const_int 2)
9518 (const_int 4) (const_int 6)])))
9521 (match_operand:V8SI 2 "nonimmediate_operand" "vm")
9522 (parallel [(const_int 0) (const_int 2)
9523 (const_int 4) (const_int 6)])))))]
9525 && ix86_binary_operator_ok (MULT, V8SImode, operands)"
9526 "vpmuldq\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
9527 [(set_attr "type" "sseimul")
9528 (set_attr "prefix_extra" "1")
9529 (set_attr "prefix" "vex")
9530 (set_attr "mode" "OI")])
9532 (define_expand "sse4_1_mulv2siv2di3<mask_name>"
9533 [(set (match_operand:V2DI 0 "register_operand")
9537 (match_operand:V4SI 1 "nonimmediate_operand")
9538 (parallel [(const_int 0) (const_int 2)])))
9541 (match_operand:V4SI 2 "nonimmediate_operand")
9542 (parallel [(const_int 0) (const_int 2)])))))]
9543 "TARGET_SSE4_1 && <mask_avx512vl_condition>"
9544 "ix86_fixup_binary_operands_no_copy (MULT, V4SImode, operands);")
9546 (define_insn "*sse4_1_mulv2siv2di3<mask_name>"
9547 [(set (match_operand:V2DI 0 "register_operand" "=Yr,*x,v")
9551 (match_operand:V4SI 1 "nonimmediate_operand" "%0,0,v")
9552 (parallel [(const_int 0) (const_int 2)])))
9555 (match_operand:V4SI 2 "nonimmediate_operand" "Yrm,*xm,vm")
9556 (parallel [(const_int 0) (const_int 2)])))))]
9557 "TARGET_SSE4_1 && <mask_avx512vl_condition>
9558 && ix86_binary_operator_ok (MULT, V4SImode, operands)"
9560 pmuldq\t{%2, %0|%0, %2}
9561 pmuldq\t{%2, %0|%0, %2}
9562 vpmuldq\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
9563 [(set_attr "isa" "noavx,noavx,avx")
9564 (set_attr "type" "sseimul")
9565 (set_attr "prefix_data16" "1,1,*")
9566 (set_attr "prefix_extra" "1")
9567 (set_attr "prefix" "orig,orig,vex")
9568 (set_attr "mode" "TI")])
9570 (define_insn "avx512bw_pmaddwd512<mode><mask_name>"
9571 [(set (match_operand:<sseunpackmode> 0 "register_operand" "=v")
9572 (unspec:<sseunpackmode>
9573 [(match_operand:VI2_AVX2 1 "register_operand" "v")
9574 (match_operand:VI2_AVX2 2 "nonimmediate_operand" "vm")]
9575 UNSPEC_PMADDWD512))]
9576 "TARGET_AVX512BW && <mask_mode512bit_condition>"
9577 "vpmaddwd\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}";
9578 [(set_attr "type" "sseiadd")
9579 (set_attr "prefix" "evex")
9580 (set_attr "mode" "XI")])
9582 (define_expand "avx2_pmaddwd"
9583 [(set (match_operand:V8SI 0 "register_operand")
9588 (match_operand:V16HI 1 "nonimmediate_operand")
9589 (parallel [(const_int 0) (const_int 2)
9590 (const_int 4) (const_int 6)
9591 (const_int 8) (const_int 10)
9592 (const_int 12) (const_int 14)])))
9595 (match_operand:V16HI 2 "nonimmediate_operand")
9596 (parallel [(const_int 0) (const_int 2)
9597 (const_int 4) (const_int 6)
9598 (const_int 8) (const_int 10)
9599 (const_int 12) (const_int 14)]))))
9602 (vec_select:V8HI (match_dup 1)
9603 (parallel [(const_int 1) (const_int 3)
9604 (const_int 5) (const_int 7)
9605 (const_int 9) (const_int 11)
9606 (const_int 13) (const_int 15)])))
9608 (vec_select:V8HI (match_dup 2)
9609 (parallel [(const_int 1) (const_int 3)
9610 (const_int 5) (const_int 7)
9611 (const_int 9) (const_int 11)
9612 (const_int 13) (const_int 15)]))))))]
9614 "ix86_fixup_binary_operands_no_copy (MULT, V16HImode, operands);")
9616 (define_insn "*avx2_pmaddwd"
9617 [(set (match_operand:V8SI 0 "register_operand" "=x")
9622 (match_operand:V16HI 1 "nonimmediate_operand" "%x")
9623 (parallel [(const_int 0) (const_int 2)
9624 (const_int 4) (const_int 6)
9625 (const_int 8) (const_int 10)
9626 (const_int 12) (const_int 14)])))
9629 (match_operand:V16HI 2 "nonimmediate_operand" "xm")
9630 (parallel [(const_int 0) (const_int 2)
9631 (const_int 4) (const_int 6)
9632 (const_int 8) (const_int 10)
9633 (const_int 12) (const_int 14)]))))
9636 (vec_select:V8HI (match_dup 1)
9637 (parallel [(const_int 1) (const_int 3)
9638 (const_int 5) (const_int 7)
9639 (const_int 9) (const_int 11)
9640 (const_int 13) (const_int 15)])))
9642 (vec_select:V8HI (match_dup 2)
9643 (parallel [(const_int 1) (const_int 3)
9644 (const_int 5) (const_int 7)
9645 (const_int 9) (const_int 11)
9646 (const_int 13) (const_int 15)]))))))]
9647 "TARGET_AVX2 && ix86_binary_operator_ok (MULT, V16HImode, operands)"
9648 "vpmaddwd\t{%2, %1, %0|%0, %1, %2}"
9649 [(set_attr "type" "sseiadd")
9650 (set_attr "prefix" "vex")
9651 (set_attr "mode" "OI")])
9653 (define_expand "sse2_pmaddwd"
9654 [(set (match_operand:V4SI 0 "register_operand")
9659 (match_operand:V8HI 1 "nonimmediate_operand")
9660 (parallel [(const_int 0) (const_int 2)
9661 (const_int 4) (const_int 6)])))
9664 (match_operand:V8HI 2 "nonimmediate_operand")
9665 (parallel [(const_int 0) (const_int 2)
9666 (const_int 4) (const_int 6)]))))
9669 (vec_select:V4HI (match_dup 1)
9670 (parallel [(const_int 1) (const_int 3)
9671 (const_int 5) (const_int 7)])))
9673 (vec_select:V4HI (match_dup 2)
9674 (parallel [(const_int 1) (const_int 3)
9675 (const_int 5) (const_int 7)]))))))]
9677 "ix86_fixup_binary_operands_no_copy (MULT, V8HImode, operands);")
9679 (define_insn "*sse2_pmaddwd"
9680 [(set (match_operand:V4SI 0 "register_operand" "=x,x")
9685 (match_operand:V8HI 1 "nonimmediate_operand" "%0,x")
9686 (parallel [(const_int 0) (const_int 2)
9687 (const_int 4) (const_int 6)])))
9690 (match_operand:V8HI 2 "nonimmediate_operand" "xm,xm")
9691 (parallel [(const_int 0) (const_int 2)
9692 (const_int 4) (const_int 6)]))))
9695 (vec_select:V4HI (match_dup 1)
9696 (parallel [(const_int 1) (const_int 3)
9697 (const_int 5) (const_int 7)])))
9699 (vec_select:V4HI (match_dup 2)
9700 (parallel [(const_int 1) (const_int 3)
9701 (const_int 5) (const_int 7)]))))))]
9702 "TARGET_SSE2 && ix86_binary_operator_ok (MULT, V8HImode, operands)"
9704 pmaddwd\t{%2, %0|%0, %2}
9705 vpmaddwd\t{%2, %1, %0|%0, %1, %2}"
9706 [(set_attr "isa" "noavx,avx")
9707 (set_attr "type" "sseiadd")
9708 (set_attr "atom_unit" "simul")
9709 (set_attr "prefix_data16" "1,*")
9710 (set_attr "prefix" "orig,vex")
9711 (set_attr "mode" "TI")])
9713 (define_insn "avx512dq_mul<mode>3<mask_name>"
9714 [(set (match_operand:VI8 0 "register_operand" "=v")
9716 (match_operand:VI8 1 "register_operand" "v")
9717 (match_operand:VI8 2 "nonimmediate_operand" "vm")))]
9718 "TARGET_AVX512DQ && <mask_mode512bit_condition>"
9719 "vpmullq\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
9720 [(set_attr "type" "sseimul")
9721 (set_attr "prefix" "evex")
9722 (set_attr "mode" "<sseinsnmode>")])
9724 (define_expand "mul<mode>3<mask_name>"
9725 [(set (match_operand:VI4_AVX512F 0 "register_operand")
9727 (match_operand:VI4_AVX512F 1 "general_vector_operand")
9728 (match_operand:VI4_AVX512F 2 "general_vector_operand")))]
9729 "TARGET_SSE2 && <mask_mode512bit_condition>"
9733 if (!nonimmediate_operand (operands[1], <MODE>mode))
9734 operands[1] = force_reg (<MODE>mode, operands[1]);
9735 if (!nonimmediate_operand (operands[2], <MODE>mode))
9736 operands[2] = force_reg (<MODE>mode, operands[2]);
9737 ix86_fixup_binary_operands_no_copy (MULT, <MODE>mode, operands);
9741 ix86_expand_sse2_mulv4si3 (operands[0], operands[1], operands[2]);
9746 (define_insn "*<sse4_1_avx2>_mul<mode>3<mask_name>"
9747 [(set (match_operand:VI4_AVX512F 0 "register_operand" "=Yr,*x,v")
9749 (match_operand:VI4_AVX512F 1 "nonimmediate_operand" "%0,0,v")
9750 (match_operand:VI4_AVX512F 2 "nonimmediate_operand" "Yrm,*xm,vm")))]
9751 "TARGET_SSE4_1 && ix86_binary_operator_ok (MULT, <MODE>mode, operands) && <mask_mode512bit_condition>"
9753 pmulld\t{%2, %0|%0, %2}
9754 pmulld\t{%2, %0|%0, %2}
9755 vpmulld\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
9756 [(set_attr "isa" "noavx,noavx,avx")
9757 (set_attr "type" "sseimul")
9758 (set_attr "prefix_extra" "1")
9759 (set_attr "prefix" "<mask_prefix4>")
9760 (set_attr "btver2_decode" "vector,vector,vector")
9761 (set_attr "mode" "<sseinsnmode>")])
9763 (define_expand "mul<mode>3"
9764 [(set (match_operand:VI8_AVX2_AVX512F 0 "register_operand")
9765 (mult:VI8_AVX2_AVX512F
9766 (match_operand:VI8_AVX2_AVX512F 1 "register_operand")
9767 (match_operand:VI8_AVX2_AVX512F 2 "register_operand")))]
9770 ix86_expand_sse2_mulvxdi3 (operands[0], operands[1], operands[2]);
9774 (define_expand "vec_widen_<s>mult_hi_<mode>"
9775 [(match_operand:<sseunpackmode> 0 "register_operand")
9776 (any_extend:<sseunpackmode>
9777 (match_operand:VI124_AVX2 1 "register_operand"))
9778 (match_operand:VI124_AVX2 2 "register_operand")]
9781 ix86_expand_mul_widen_hilo (operands[0], operands[1], operands[2],
9786 (define_expand "vec_widen_<s>mult_lo_<mode>"
9787 [(match_operand:<sseunpackmode> 0 "register_operand")
9788 (any_extend:<sseunpackmode>
9789 (match_operand:VI124_AVX2 1 "register_operand"))
9790 (match_operand:VI124_AVX2 2 "register_operand")]
9793 ix86_expand_mul_widen_hilo (operands[0], operands[1], operands[2],
9798 ;; Most widen_<s>mult_even_<mode> can be handled directly from other
9799 ;; named patterns, but signed V4SI needs special help for plain SSE2.
9800 (define_expand "vec_widen_smult_even_v4si"
9801 [(match_operand:V2DI 0 "register_operand")
9802 (match_operand:V4SI 1 "nonimmediate_operand")
9803 (match_operand:V4SI 2 "nonimmediate_operand")]
9806 ix86_expand_mul_widen_evenodd (operands[0], operands[1], operands[2],
9811 (define_expand "vec_widen_<s>mult_odd_<mode>"
9812 [(match_operand:<sseunpackmode> 0 "register_operand")
9813 (any_extend:<sseunpackmode>
9814 (match_operand:VI4_AVX512F 1 "general_vector_operand"))
9815 (match_operand:VI4_AVX512F 2 "general_vector_operand")]
9818 ix86_expand_mul_widen_evenodd (operands[0], operands[1], operands[2],
9823 (define_mode_attr SDOT_PMADD_SUF
9824 [(V32HI "512v32hi") (V16HI "") (V8HI "")])
9826 (define_expand "sdot_prod<mode>"
9827 [(match_operand:<sseunpackmode> 0 "register_operand")
9828 (match_operand:VI2_AVX2 1 "register_operand")
9829 (match_operand:VI2_AVX2 2 "register_operand")
9830 (match_operand:<sseunpackmode> 3 "register_operand")]
9833 rtx t = gen_reg_rtx (<sseunpackmode>mode);
9834 emit_insn (gen_<sse2_avx2>_pmaddwd<SDOT_PMADD_SUF> (t, operands[1], operands[2]));
9835 emit_insn (gen_rtx_SET (VOIDmode, operands[0],
9836 gen_rtx_PLUS (<sseunpackmode>mode,
9841 ;; Normally we use widen_mul_even/odd, but combine can't quite get it all
9842 ;; back together when madd is available.
9843 (define_expand "sdot_prodv4si"
9844 [(match_operand:V2DI 0 "register_operand")
9845 (match_operand:V4SI 1 "register_operand")
9846 (match_operand:V4SI 2 "register_operand")
9847 (match_operand:V2DI 3 "register_operand")]
9850 rtx t = gen_reg_rtx (V2DImode);
9851 emit_insn (gen_xop_pmacsdqh (t, operands[1], operands[2], operands[3]));
9852 emit_insn (gen_xop_pmacsdql (operands[0], operands[1], operands[2], t));
9856 (define_expand "usadv16qi"
9857 [(match_operand:V4SI 0 "register_operand")
9858 (match_operand:V16QI 1 "register_operand")
9859 (match_operand:V16QI 2 "nonimmediate_operand")
9860 (match_operand:V4SI 3 "nonimmediate_operand")]
9863 rtx t1 = gen_reg_rtx (V2DImode);
9864 rtx t2 = gen_reg_rtx (V4SImode);
9865 emit_insn (gen_sse2_psadbw (t1, operands[1], operands[2]));
9866 convert_move (t2, t1, 0);
9867 emit_insn (gen_addv4si3 (operands[0], t2, operands[3]));
9871 (define_expand "usadv32qi"
9872 [(match_operand:V8SI 0 "register_operand")
9873 (match_operand:V32QI 1 "register_operand")
9874 (match_operand:V32QI 2 "nonimmediate_operand")
9875 (match_operand:V8SI 3 "nonimmediate_operand")]
9878 rtx t1 = gen_reg_rtx (V4DImode);
9879 rtx t2 = gen_reg_rtx (V8SImode);
9880 emit_insn (gen_avx2_psadbw (t1, operands[1], operands[2]));
9881 convert_move (t2, t1, 0);
9882 emit_insn (gen_addv8si3 (operands[0], t2, operands[3]));
9886 (define_insn "ashr<mode>3"
9887 [(set (match_operand:VI24_AVX2 0 "register_operand" "=x,x")
9889 (match_operand:VI24_AVX2 1 "register_operand" "0,x")
9890 (match_operand:SI 2 "nonmemory_operand" "xN,xN")))]
9893 psra<ssemodesuffix>\t{%2, %0|%0, %2}
9894 vpsra<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
9895 [(set_attr "isa" "noavx,avx")
9896 (set_attr "type" "sseishft")
9897 (set (attr "length_immediate")
9898 (if_then_else (match_operand 2 "const_int_operand")
9900 (const_string "0")))
9901 (set_attr "prefix_data16" "1,*")
9902 (set_attr "prefix" "orig,vex")
9903 (set_attr "mode" "<sseinsnmode>")])
9905 (define_insn "<mask_codefor>ashr<mode>3<mask_name>"
9906 [(set (match_operand:VI24_AVX512BW_1 0 "register_operand" "=v,v")
9907 (ashiftrt:VI24_AVX512BW_1
9908 (match_operand:VI24_AVX512BW_1 1 "nonimmediate_operand" "v,vm")
9909 (match_operand:SI 2 "nonmemory_operand" "v,N")))]
9911 "vpsra<ssemodesuffix>\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
9912 [(set_attr "type" "sseishft")
9913 (set (attr "length_immediate")
9914 (if_then_else (match_operand 2 "const_int_operand")
9916 (const_string "0")))
9917 (set_attr "mode" "<sseinsnmode>")])
9919 (define_insn "<mask_codefor>ashrv2di3<mask_name>"
9920 [(set (match_operand:V2DI 0 "register_operand" "=v,v")
9922 (match_operand:V2DI 1 "nonimmediate_operand" "v,vm")
9923 (match_operand:DI 2 "nonmemory_operand" "v,N")))]
9925 "vpsraq\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
9926 [(set_attr "type" "sseishft")
9927 (set (attr "length_immediate")
9928 (if_then_else (match_operand 2 "const_int_operand")
9930 (const_string "0")))
9931 (set_attr "mode" "TI")])
9933 (define_insn "ashr<mode>3<mask_name>"
9934 [(set (match_operand:VI248_AVX512BW_AVX512VL 0 "register_operand" "=v,v")
9935 (ashiftrt:VI248_AVX512BW_AVX512VL
9936 (match_operand:VI248_AVX512BW_AVX512VL 1 "nonimmediate_operand" "v,vm")
9937 (match_operand:SI 2 "nonmemory_operand" "v,N")))]
9939 "vpsra<ssemodesuffix>\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
9940 [(set_attr "type" "sseishft")
9941 (set (attr "length_immediate")
9942 (if_then_else (match_operand 2 "const_int_operand")
9944 (const_string "0")))
9945 (set_attr "mode" "<sseinsnmode>")])
9947 (define_insn "<shift_insn><mode>3<mask_name>"
9948 [(set (match_operand:VI2_AVX2_AVX512BW 0 "register_operand" "=x,v")
9949 (any_lshift:VI2_AVX2_AVX512BW
9950 (match_operand:VI2_AVX2_AVX512BW 1 "register_operand" "0,v")
9951 (match_operand:SI 2 "nonmemory_operand" "xN,vN")))]
9952 "TARGET_SSE2 && <mask_mode512bit_condition> && <mask_avx512bw_condition>"
9954 p<vshift><ssemodesuffix>\t{%2, %0|%0, %2}
9955 vp<vshift><ssemodesuffix>\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
9956 [(set_attr "isa" "noavx,avx")
9957 (set_attr "type" "sseishft")
9958 (set (attr "length_immediate")
9959 (if_then_else (match_operand 2 "const_int_operand")
9961 (const_string "0")))
9962 (set_attr "prefix_data16" "1,*")
9963 (set_attr "prefix" "orig,vex")
9964 (set_attr "mode" "<sseinsnmode>")])
9966 (define_insn "<shift_insn><mode>3<mask_name>"
9967 [(set (match_operand:VI48_AVX2 0 "register_operand" "=x,v")
9968 (any_lshift:VI48_AVX2
9969 (match_operand:VI48_AVX2 1 "register_operand" "0,v")
9970 (match_operand:SI 2 "nonmemory_operand" "xN,vN")))]
9971 "TARGET_SSE2 && <mask_mode512bit_condition>"
9973 p<vshift><ssemodesuffix>\t{%2, %0|%0, %2}
9974 vp<vshift><ssemodesuffix>\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
9975 [(set_attr "isa" "noavx,avx")
9976 (set_attr "type" "sseishft")
9977 (set (attr "length_immediate")
9978 (if_then_else (match_operand 2 "const_int_operand")
9980 (const_string "0")))
9981 (set_attr "prefix_data16" "1,*")
9982 (set_attr "prefix" "orig,vex")
9983 (set_attr "mode" "<sseinsnmode>")])
9985 (define_insn "<shift_insn><mode>3<mask_name>"
9986 [(set (match_operand:VI48_512 0 "register_operand" "=v,v")
9987 (any_lshift:VI48_512
9988 (match_operand:VI48_512 1 "nonimmediate_operand" "v,m")
9989 (match_operand:SI 2 "nonmemory_operand" "vN,N")))]
9990 "TARGET_AVX512F && <mask_mode512bit_condition>"
9991 "vp<vshift><ssemodesuffix>\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
9992 [(set_attr "isa" "avx512f")
9993 (set_attr "type" "sseishft")
9994 (set (attr "length_immediate")
9995 (if_then_else (match_operand 2 "const_int_operand")
9997 (const_string "0")))
9998 (set_attr "prefix" "evex")
9999 (set_attr "mode" "<sseinsnmode>")])
10002 (define_expand "vec_shl_<mode>"
10003 [(set (match_dup 3)
10005 (match_operand:VI_128 1 "register_operand")
10006 (match_operand:SI 2 "const_0_to_255_mul_8_operand")))
10007 (set (match_operand:VI_128 0 "register_operand") (match_dup 4))]
10010 operands[1] = gen_lowpart (V1TImode, operands[1]);
10011 operands[3] = gen_reg_rtx (V1TImode);
10012 operands[4] = gen_lowpart (<MODE>mode, operands[3]);
10015 (define_insn "<sse2_avx2>_ashl<mode>3"
10016 [(set (match_operand:VIMAX_AVX2 0 "register_operand" "=x,v")
10018 (match_operand:VIMAX_AVX2 1 "register_operand" "0,v")
10019 (match_operand:SI 2 "const_0_to_255_mul_8_operand" "n,n")))]
10022 operands[2] = GEN_INT (INTVAL (operands[2]) / 8);
10024 switch (which_alternative)
10027 return "pslldq\t{%2, %0|%0, %2}";
10029 return "vpslldq\t{%2, %1, %0|%0, %1, %2}";
10031 gcc_unreachable ();
10034 [(set_attr "isa" "noavx,avx")
10035 (set_attr "type" "sseishft")
10036 (set_attr "length_immediate" "1")
10037 (set_attr "prefix_data16" "1,*")
10038 (set_attr "prefix" "orig,vex")
10039 (set_attr "mode" "<sseinsnmode>")])
10041 (define_expand "vec_shr_<mode>"
10042 [(set (match_dup 3)
10044 (match_operand:VI_128 1 "register_operand")
10045 (match_operand:SI 2 "const_0_to_255_mul_8_operand")))
10046 (set (match_operand:VI_128 0 "register_operand") (match_dup 4))]
10049 operands[1] = gen_lowpart (V1TImode, operands[1]);
10050 operands[3] = gen_reg_rtx (V1TImode);
10051 operands[4] = gen_lowpart (<MODE>mode, operands[3]);
10054 (define_insn "<sse2_avx2>_lshr<mode>3"
10055 [(set (match_operand:VIMAX_AVX2 0 "register_operand" "=x,v")
10056 (lshiftrt:VIMAX_AVX2
10057 (match_operand:VIMAX_AVX2 1 "register_operand" "0,v")
10058 (match_operand:SI 2 "const_0_to_255_mul_8_operand" "n,n")))]
10061 operands[2] = GEN_INT (INTVAL (operands[2]) / 8);
10063 switch (which_alternative)
10066 return "psrldq\t{%2, %0|%0, %2}";
10068 return "vpsrldq\t{%2, %1, %0|%0, %1, %2}";
10070 gcc_unreachable ();
10073 [(set_attr "isa" "noavx,avx")
10074 (set_attr "type" "sseishft")
10075 (set_attr "length_immediate" "1")
10076 (set_attr "atom_unit" "sishuf")
10077 (set_attr "prefix_data16" "1,*")
10078 (set_attr "prefix" "orig,vex")
10079 (set_attr "mode" "<sseinsnmode>")])
10081 (define_insn "<avx512>_<rotate>v<mode><mask_name>"
10082 [(set (match_operand:VI48_AVX512VL 0 "register_operand" "=v")
10083 (any_rotate:VI48_AVX512VL
10084 (match_operand:VI48_AVX512VL 1 "register_operand" "v")
10085 (match_operand:VI48_AVX512VL 2 "nonimmediate_operand" "vm")))]
10087 "vp<rotate>v<ssemodesuffix>\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
10088 [(set_attr "prefix" "evex")
10089 (set_attr "mode" "<sseinsnmode>")])
10091 (define_insn "<avx512>_<rotate><mode><mask_name>"
10092 [(set (match_operand:VI48_AVX512VL 0 "register_operand" "=v")
10093 (any_rotate:VI48_AVX512VL
10094 (match_operand:VI48_AVX512VL 1 "nonimmediate_operand" "vm")
10095 (match_operand:SI 2 "const_0_to_255_operand")))]
10097 "vp<rotate><ssemodesuffix>\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
10098 [(set_attr "prefix" "evex")
10099 (set_attr "mode" "<sseinsnmode>")])
10101 (define_expand "<code><mode>3"
10102 [(set (match_operand:VI124_256_AVX512F_AVX512BW 0 "register_operand")
10103 (maxmin:VI124_256_AVX512F_AVX512BW
10104 (match_operand:VI124_256_AVX512F_AVX512BW 1 "nonimmediate_operand")
10105 (match_operand:VI124_256_AVX512F_AVX512BW 2 "nonimmediate_operand")))]
10107 "ix86_fixup_binary_operands_no_copy (<CODE>, <MODE>mode, operands);")
10109 (define_insn "*avx2_<code><mode>3"
10110 [(set (match_operand:VI124_256 0 "register_operand" "=v")
10112 (match_operand:VI124_256 1 "nonimmediate_operand" "%v")
10113 (match_operand:VI124_256 2 "nonimmediate_operand" "vm")))]
10114 "TARGET_AVX2 && ix86_binary_operator_ok (<CODE>, <MODE>mode, operands)"
10115 "vp<maxmin_int><ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
10116 [(set_attr "type" "sseiadd")
10117 (set_attr "prefix_extra" "1")
10118 (set_attr "prefix" "vex")
10119 (set_attr "mode" "OI")])
10121 (define_expand "<code><mode>3_mask"
10122 [(set (match_operand:VI48_AVX512VL 0 "register_operand")
10123 (vec_merge:VI48_AVX512VL
10124 (maxmin:VI48_AVX512VL
10125 (match_operand:VI48_AVX512VL 1 "nonimmediate_operand")
10126 (match_operand:VI48_AVX512VL 2 "nonimmediate_operand"))
10127 (match_operand:VI48_AVX512VL 3 "vector_move_operand")
10128 (match_operand:<avx512fmaskmode> 4 "register_operand")))]
10130 "ix86_fixup_binary_operands_no_copy (<CODE>, <MODE>mode, operands);")
10132 (define_insn "*avx512bw_<code><mode>3<mask_name>"
10133 [(set (match_operand:VI48_AVX512VL 0 "register_operand" "=v")
10134 (maxmin:VI48_AVX512VL
10135 (match_operand:VI48_AVX512VL 1 "nonimmediate_operand" "%v")
10136 (match_operand:VI48_AVX512VL 2 "nonimmediate_operand" "vm")))]
10137 "TARGET_AVX512F && ix86_binary_operator_ok (<CODE>, <MODE>mode, operands)"
10138 "vp<maxmin_int><ssemodesuffix>\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
10139 [(set_attr "type" "sseiadd")
10140 (set_attr "prefix_extra" "1")
10141 (set_attr "prefix" "maybe_evex")
10142 (set_attr "mode" "<sseinsnmode>")])
10144 (define_insn "<mask_codefor><code><mode>3<mask_name>"
10145 [(set (match_operand:VI12_AVX512VL 0 "register_operand" "=v")
10146 (maxmin:VI12_AVX512VL
10147 (match_operand:VI12_AVX512VL 1 "register_operand" "v")
10148 (match_operand:VI12_AVX512VL 2 "nonimmediate_operand" "vm")))]
10150 "vp<maxmin_int><ssemodesuffix>\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
10151 [(set_attr "type" "sseiadd")
10152 (set_attr "prefix" "evex")
10153 (set_attr "mode" "<sseinsnmode>")])
10155 (define_expand "<code><mode>3"
10156 [(set (match_operand:VI8_AVX2_AVX512BW 0 "register_operand")
10157 (maxmin:VI8_AVX2_AVX512BW
10158 (match_operand:VI8_AVX2_AVX512BW 1 "register_operand")
10159 (match_operand:VI8_AVX2_AVX512BW 2 "register_operand")))]
10163 && (<MODE>mode == V8DImode || TARGET_AVX512VL))
10164 ix86_fixup_binary_operands_no_copy (<CODE>, <MODE>mode, operands);
10167 enum rtx_code code;
10172 xops[0] = operands[0];
10174 if (<CODE> == SMAX || <CODE> == UMAX)
10176 xops[1] = operands[1];
10177 xops[2] = operands[2];
10181 xops[1] = operands[2];
10182 xops[2] = operands[1];
10185 code = (<CODE> == UMAX || <CODE> == UMIN) ? GTU : GT;
10187 xops[3] = gen_rtx_fmt_ee (code, VOIDmode, operands[1], operands[2]);
10188 xops[4] = operands[1];
10189 xops[5] = operands[2];
10191 ok = ix86_expand_int_vcond (xops);
10197 (define_expand "<code><mode>3"
10198 [(set (match_operand:VI124_128 0 "register_operand")
10200 (match_operand:VI124_128 1 "nonimmediate_operand")
10201 (match_operand:VI124_128 2 "nonimmediate_operand")))]
10204 if (TARGET_SSE4_1 || <MODE>mode == V8HImode)
10205 ix86_fixup_binary_operands_no_copy (<CODE>, <MODE>mode, operands);
10211 xops[0] = operands[0];
10212 operands[1] = force_reg (<MODE>mode, operands[1]);
10213 operands[2] = force_reg (<MODE>mode, operands[2]);
10215 if (<CODE> == SMAX)
10217 xops[1] = operands[1];
10218 xops[2] = operands[2];
10222 xops[1] = operands[2];
10223 xops[2] = operands[1];
10226 xops[3] = gen_rtx_GT (VOIDmode, operands[1], operands[2]);
10227 xops[4] = operands[1];
10228 xops[5] = operands[2];
10230 ok = ix86_expand_int_vcond (xops);
10236 (define_insn "*sse4_1_<code><mode>3<mask_name>"
10237 [(set (match_operand:VI14_128 0 "register_operand" "=Yr,*x,v")
10239 (match_operand:VI14_128 1 "nonimmediate_operand" "%0,0,v")
10240 (match_operand:VI14_128 2 "nonimmediate_operand" "Yrm,*xm,vm")))]
10242 && <mask_mode512bit_condition>
10243 && ix86_binary_operator_ok (<CODE>, <MODE>mode, operands)"
10245 p<maxmin_int><ssemodesuffix>\t{%2, %0|%0, %2}
10246 p<maxmin_int><ssemodesuffix>\t{%2, %0|%0, %2}
10247 vp<maxmin_int><ssemodesuffix>\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
10248 [(set_attr "isa" "noavx,noavx,avx")
10249 (set_attr "type" "sseiadd")
10250 (set_attr "prefix_extra" "1,1,*")
10251 (set_attr "prefix" "orig,orig,vex")
10252 (set_attr "mode" "TI")])
10254 (define_insn "*<code>v8hi3"
10255 [(set (match_operand:V8HI 0 "register_operand" "=x,x")
10257 (match_operand:V8HI 1 "nonimmediate_operand" "%0,x")
10258 (match_operand:V8HI 2 "nonimmediate_operand" "xm,xm")))]
10259 "TARGET_SSE2 && ix86_binary_operator_ok (<CODE>, V8HImode, operands)"
10261 p<maxmin_int>w\t{%2, %0|%0, %2}
10262 vp<maxmin_int>w\t{%2, %1, %0|%0, %1, %2}"
10263 [(set_attr "isa" "noavx,avx")
10264 (set_attr "type" "sseiadd")
10265 (set_attr "prefix_data16" "1,*")
10266 (set_attr "prefix_extra" "*,1")
10267 (set_attr "prefix" "orig,vex")
10268 (set_attr "mode" "TI")])
10270 (define_expand "<code><mode>3"
10271 [(set (match_operand:VI124_128 0 "register_operand")
10273 (match_operand:VI124_128 1 "nonimmediate_operand")
10274 (match_operand:VI124_128 2 "nonimmediate_operand")))]
10277 if (TARGET_SSE4_1 || <MODE>mode == V16QImode)
10278 ix86_fixup_binary_operands_no_copy (<CODE>, <MODE>mode, operands);
10279 else if (<CODE> == UMAX && <MODE>mode == V8HImode)
10281 rtx op0 = operands[0], op2 = operands[2], op3 = op0;
10282 operands[1] = force_reg (<MODE>mode, operands[1]);
10283 if (rtx_equal_p (op3, op2))
10284 op3 = gen_reg_rtx (V8HImode);
10285 emit_insn (gen_sse2_ussubv8hi3 (op3, operands[1], op2));
10286 emit_insn (gen_addv8hi3 (op0, op3, op2));
10294 operands[1] = force_reg (<MODE>mode, operands[1]);
10295 operands[2] = force_reg (<MODE>mode, operands[2]);
10297 xops[0] = operands[0];
10299 if (<CODE> == UMAX)
10301 xops[1] = operands[1];
10302 xops[2] = operands[2];
10306 xops[1] = operands[2];
10307 xops[2] = operands[1];
10310 xops[3] = gen_rtx_GTU (VOIDmode, operands[1], operands[2]);
10311 xops[4] = operands[1];
10312 xops[5] = operands[2];
10314 ok = ix86_expand_int_vcond (xops);
10320 (define_insn "*sse4_1_<code><mode>3<mask_name>"
10321 [(set (match_operand:VI24_128 0 "register_operand" "=Yr,*x,v")
10323 (match_operand:VI24_128 1 "nonimmediate_operand" "%0,0,v")
10324 (match_operand:VI24_128 2 "nonimmediate_operand" "Yrm,*xm,vm")))]
10326 && <mask_mode512bit_condition>
10327 && ix86_binary_operator_ok (<CODE>, <MODE>mode, operands)"
10329 p<maxmin_int><ssemodesuffix>\t{%2, %0|%0, %2}
10330 p<maxmin_int><ssemodesuffix>\t{%2, %0|%0, %2}
10331 vp<maxmin_int><ssemodesuffix>\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
10332 [(set_attr "isa" "noavx,noavx,avx")
10333 (set_attr "type" "sseiadd")
10334 (set_attr "prefix_extra" "1,1,*")
10335 (set_attr "prefix" "orig,orig,vex")
10336 (set_attr "mode" "TI")])
10338 (define_insn "*<code>v16qi3"
10339 [(set (match_operand:V16QI 0 "register_operand" "=x,x")
10341 (match_operand:V16QI 1 "nonimmediate_operand" "%0,x")
10342 (match_operand:V16QI 2 "nonimmediate_operand" "xm,xm")))]
10343 "TARGET_SSE2 && ix86_binary_operator_ok (<CODE>, V16QImode, operands)"
10345 p<maxmin_int>b\t{%2, %0|%0, %2}
10346 vp<maxmin_int>b\t{%2, %1, %0|%0, %1, %2}"
10347 [(set_attr "isa" "noavx,avx")
10348 (set_attr "type" "sseiadd")
10349 (set_attr "prefix_data16" "1,*")
10350 (set_attr "prefix_extra" "*,1")
10351 (set_attr "prefix" "orig,vex")
10352 (set_attr "mode" "TI")])
10354 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
10356 ;; Parallel integral comparisons
10358 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
10360 (define_expand "avx2_eq<mode>3"
10361 [(set (match_operand:VI_256 0 "register_operand")
10363 (match_operand:VI_256 1 "nonimmediate_operand")
10364 (match_operand:VI_256 2 "nonimmediate_operand")))]
10366 "ix86_fixup_binary_operands_no_copy (EQ, <MODE>mode, operands);")
10368 (define_insn "*avx2_eq<mode>3"
10369 [(set (match_operand:VI_256 0 "register_operand" "=x")
10371 (match_operand:VI_256 1 "nonimmediate_operand" "%x")
10372 (match_operand:VI_256 2 "nonimmediate_operand" "xm")))]
10373 "TARGET_AVX2 && ix86_binary_operator_ok (EQ, <MODE>mode, operands)"
10374 "vpcmpeq<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
10375 [(set_attr "type" "ssecmp")
10376 (set_attr "prefix_extra" "1")
10377 (set_attr "prefix" "vex")
10378 (set_attr "mode" "OI")])
10380 (define_expand "<avx512>_eq<mode>3<mask_scalar_merge_name>"
10381 [(set (match_operand:<avx512fmaskmode> 0 "register_operand")
10382 (unspec:<avx512fmaskmode>
10383 [(match_operand:VI12_AVX512VL 1 "register_operand")
10384 (match_operand:VI12_AVX512VL 2 "nonimmediate_operand")]
10385 UNSPEC_MASKED_EQ))]
10387 "ix86_fixup_binary_operands_no_copy (EQ, <MODE>mode, operands);")
10389 (define_expand "<avx512>_eq<mode>3<mask_scalar_merge_name>"
10390 [(set (match_operand:<avx512fmaskmode> 0 "register_operand")
10391 (unspec:<avx512fmaskmode>
10392 [(match_operand:VI48_AVX512VL 1 "register_operand")
10393 (match_operand:VI48_AVX512VL 2 "nonimmediate_operand")]
10394 UNSPEC_MASKED_EQ))]
10396 "ix86_fixup_binary_operands_no_copy (EQ, <MODE>mode, operands);")
10398 (define_insn "<avx512>_eq<mode>3<mask_scalar_merge_name>_1"
10399 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=Yk")
10400 (unspec:<avx512fmaskmode>
10401 [(match_operand:VI12_AVX512VL 1 "register_operand" "%v")
10402 (match_operand:VI12_AVX512VL 2 "nonimmediate_operand" "vm")]
10403 UNSPEC_MASKED_EQ))]
10404 "TARGET_AVX512F && ix86_binary_operator_ok (EQ, <MODE>mode, operands)"
10405 "vpcmpeq<ssemodesuffix>\t{%2, %1, %0<mask_scalar_merge_operand3>|%0<mask_scalar_merge_operand3>, %1, %2}"
10406 [(set_attr "type" "ssecmp")
10407 (set_attr "prefix_extra" "1")
10408 (set_attr "prefix" "evex")
10409 (set_attr "mode" "<sseinsnmode>")])
10411 (define_insn "<avx512>_eq<mode>3<mask_scalar_merge_name>_1"
10412 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=Yk")
10413 (unspec:<avx512fmaskmode>
10414 [(match_operand:VI48_AVX512VL 1 "register_operand" "%v")
10415 (match_operand:VI48_AVX512VL 2 "nonimmediate_operand" "vm")]
10416 UNSPEC_MASKED_EQ))]
10417 "TARGET_AVX512F && ix86_binary_operator_ok (EQ, <MODE>mode, operands)"
10418 "vpcmpeq<ssemodesuffix>\t{%2, %1, %0<mask_scalar_merge_operand3>|%0<mask_scalar_merge_operand3>, %1, %2}"
10419 [(set_attr "type" "ssecmp")
10420 (set_attr "prefix_extra" "1")
10421 (set_attr "prefix" "evex")
10422 (set_attr "mode" "<sseinsnmode>")])
10424 (define_insn "*sse4_1_eqv2di3"
10425 [(set (match_operand:V2DI 0 "register_operand" "=Yr,*x,x")
10427 (match_operand:V2DI 1 "nonimmediate_operand" "%0,0,x")
10428 (match_operand:V2DI 2 "nonimmediate_operand" "Yrm,*xm,xm")))]
10429 "TARGET_SSE4_1 && ix86_binary_operator_ok (EQ, V2DImode, operands)"
10431 pcmpeqq\t{%2, %0|%0, %2}
10432 pcmpeqq\t{%2, %0|%0, %2}
10433 vpcmpeqq\t{%2, %1, %0|%0, %1, %2}"
10434 [(set_attr "isa" "noavx,noavx,avx")
10435 (set_attr "type" "ssecmp")
10436 (set_attr "prefix_extra" "1")
10437 (set_attr "prefix" "orig,orig,vex")
10438 (set_attr "mode" "TI")])
10440 (define_insn "*sse2_eq<mode>3"
10441 [(set (match_operand:VI124_128 0 "register_operand" "=x,x")
10443 (match_operand:VI124_128 1 "nonimmediate_operand" "%0,x")
10444 (match_operand:VI124_128 2 "nonimmediate_operand" "xm,xm")))]
10445 "TARGET_SSE2 && !TARGET_XOP
10446 && ix86_binary_operator_ok (EQ, <MODE>mode, operands)"
10448 pcmpeq<ssemodesuffix>\t{%2, %0|%0, %2}
10449 vpcmpeq<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
10450 [(set_attr "isa" "noavx,avx")
10451 (set_attr "type" "ssecmp")
10452 (set_attr "prefix_data16" "1,*")
10453 (set_attr "prefix" "orig,vex")
10454 (set_attr "mode" "TI")])
10456 (define_expand "sse2_eq<mode>3"
10457 [(set (match_operand:VI124_128 0 "register_operand")
10459 (match_operand:VI124_128 1 "nonimmediate_operand")
10460 (match_operand:VI124_128 2 "nonimmediate_operand")))]
10461 "TARGET_SSE2 && !TARGET_XOP "
10462 "ix86_fixup_binary_operands_no_copy (EQ, <MODE>mode, operands);")
10464 (define_expand "sse4_1_eqv2di3"
10465 [(set (match_operand:V2DI 0 "register_operand")
10467 (match_operand:V2DI 1 "nonimmediate_operand")
10468 (match_operand:V2DI 2 "nonimmediate_operand")))]
10470 "ix86_fixup_binary_operands_no_copy (EQ, V2DImode, operands);")
10472 (define_insn "sse4_2_gtv2di3"
10473 [(set (match_operand:V2DI 0 "register_operand" "=Yr,*x,x")
10475 (match_operand:V2DI 1 "register_operand" "0,0,x")
10476 (match_operand:V2DI 2 "nonimmediate_operand" "Yrm,*xm,xm")))]
10479 pcmpgtq\t{%2, %0|%0, %2}
10480 pcmpgtq\t{%2, %0|%0, %2}
10481 vpcmpgtq\t{%2, %1, %0|%0, %1, %2}"
10482 [(set_attr "isa" "noavx,noavx,avx")
10483 (set_attr "type" "ssecmp")
10484 (set_attr "prefix_extra" "1")
10485 (set_attr "prefix" "orig,orig,vex")
10486 (set_attr "mode" "TI")])
10488 (define_insn "avx2_gt<mode>3"
10489 [(set (match_operand:VI_256 0 "register_operand" "=x")
10491 (match_operand:VI_256 1 "register_operand" "x")
10492 (match_operand:VI_256 2 "nonimmediate_operand" "xm")))]
10494 "vpcmpgt<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
10495 [(set_attr "type" "ssecmp")
10496 (set_attr "prefix_extra" "1")
10497 (set_attr "prefix" "vex")
10498 (set_attr "mode" "OI")])
10500 (define_insn "<avx512>_gt<mode>3<mask_scalar_merge_name>"
10501 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=Yk")
10502 (unspec:<avx512fmaskmode>
10503 [(match_operand:VI48_AVX512VL 1 "register_operand" "v")
10504 (match_operand:VI48_AVX512VL 2 "nonimmediate_operand" "vm")] UNSPEC_MASKED_GT))]
10506 "vpcmpgt<ssemodesuffix>\t{%2, %1, %0<mask_scalar_merge_operand3>|%0<mask_scalar_merge_operand3>, %1, %2}"
10507 [(set_attr "type" "ssecmp")
10508 (set_attr "prefix_extra" "1")
10509 (set_attr "prefix" "evex")
10510 (set_attr "mode" "<sseinsnmode>")])
10512 (define_insn "<avx512>_gt<mode>3<mask_scalar_merge_name>"
10513 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=Yk")
10514 (unspec:<avx512fmaskmode>
10515 [(match_operand:VI12_AVX512VL 1 "register_operand" "v")
10516 (match_operand:VI12_AVX512VL 2 "nonimmediate_operand" "vm")] UNSPEC_MASKED_GT))]
10518 "vpcmpgt<ssemodesuffix>\t{%2, %1, %0<mask_scalar_merge_operand3>|%0<mask_scalar_merge_operand3>, %1, %2}"
10519 [(set_attr "type" "ssecmp")
10520 (set_attr "prefix_extra" "1")
10521 (set_attr "prefix" "evex")
10522 (set_attr "mode" "<sseinsnmode>")])
10524 (define_insn "sse2_gt<mode>3"
10525 [(set (match_operand:VI124_128 0 "register_operand" "=x,x")
10527 (match_operand:VI124_128 1 "register_operand" "0,x")
10528 (match_operand:VI124_128 2 "nonimmediate_operand" "xm,xm")))]
10529 "TARGET_SSE2 && !TARGET_XOP"
10531 pcmpgt<ssemodesuffix>\t{%2, %0|%0, %2}
10532 vpcmpgt<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
10533 [(set_attr "isa" "noavx,avx")
10534 (set_attr "type" "ssecmp")
10535 (set_attr "prefix_data16" "1,*")
10536 (set_attr "prefix" "orig,vex")
10537 (set_attr "mode" "TI")])
10539 (define_expand "vcond<V_512:mode><VI_512:mode>"
10540 [(set (match_operand:V_512 0 "register_operand")
10541 (if_then_else:V_512
10542 (match_operator 3 ""
10543 [(match_operand:VI_512 4 "nonimmediate_operand")
10544 (match_operand:VI_512 5 "general_operand")])
10545 (match_operand:V_512 1)
10546 (match_operand:V_512 2)))]
10548 && (GET_MODE_NUNITS (<V_512:MODE>mode)
10549 == GET_MODE_NUNITS (<VI_512:MODE>mode))"
10551 bool ok = ix86_expand_int_vcond (operands);
10556 (define_expand "vcond<V_256:mode><VI_256:mode>"
10557 [(set (match_operand:V_256 0 "register_operand")
10558 (if_then_else:V_256
10559 (match_operator 3 ""
10560 [(match_operand:VI_256 4 "nonimmediate_operand")
10561 (match_operand:VI_256 5 "general_operand")])
10562 (match_operand:V_256 1)
10563 (match_operand:V_256 2)))]
10565 && (GET_MODE_NUNITS (<V_256:MODE>mode)
10566 == GET_MODE_NUNITS (<VI_256:MODE>mode))"
10568 bool ok = ix86_expand_int_vcond (operands);
10573 (define_expand "vcond<V_128:mode><VI124_128:mode>"
10574 [(set (match_operand:V_128 0 "register_operand")
10575 (if_then_else:V_128
10576 (match_operator 3 ""
10577 [(match_operand:VI124_128 4 "nonimmediate_operand")
10578 (match_operand:VI124_128 5 "general_operand")])
10579 (match_operand:V_128 1)
10580 (match_operand:V_128 2)))]
10582 && (GET_MODE_NUNITS (<V_128:MODE>mode)
10583 == GET_MODE_NUNITS (<VI124_128:MODE>mode))"
10585 bool ok = ix86_expand_int_vcond (operands);
10590 (define_expand "vcond<VI8F_128:mode>v2di"
10591 [(set (match_operand:VI8F_128 0 "register_operand")
10592 (if_then_else:VI8F_128
10593 (match_operator 3 ""
10594 [(match_operand:V2DI 4 "nonimmediate_operand")
10595 (match_operand:V2DI 5 "general_operand")])
10596 (match_operand:VI8F_128 1)
10597 (match_operand:VI8F_128 2)))]
10600 bool ok = ix86_expand_int_vcond (operands);
10605 (define_expand "vcondu<V_512:mode><VI_512:mode>"
10606 [(set (match_operand:V_512 0 "register_operand")
10607 (if_then_else:V_512
10608 (match_operator 3 ""
10609 [(match_operand:VI_512 4 "nonimmediate_operand")
10610 (match_operand:VI_512 5 "nonimmediate_operand")])
10611 (match_operand:V_512 1 "general_operand")
10612 (match_operand:V_512 2 "general_operand")))]
10614 && (GET_MODE_NUNITS (<V_512:MODE>mode)
10615 == GET_MODE_NUNITS (<VI_512:MODE>mode))"
10617 bool ok = ix86_expand_int_vcond (operands);
10622 (define_expand "vcondu<V_256:mode><VI_256:mode>"
10623 [(set (match_operand:V_256 0 "register_operand")
10624 (if_then_else:V_256
10625 (match_operator 3 ""
10626 [(match_operand:VI_256 4 "nonimmediate_operand")
10627 (match_operand:VI_256 5 "nonimmediate_operand")])
10628 (match_operand:V_256 1 "general_operand")
10629 (match_operand:V_256 2 "general_operand")))]
10631 && (GET_MODE_NUNITS (<V_256:MODE>mode)
10632 == GET_MODE_NUNITS (<VI_256:MODE>mode))"
10634 bool ok = ix86_expand_int_vcond (operands);
10639 (define_expand "vcondu<V_128:mode><VI124_128:mode>"
10640 [(set (match_operand:V_128 0 "register_operand")
10641 (if_then_else:V_128
10642 (match_operator 3 ""
10643 [(match_operand:VI124_128 4 "nonimmediate_operand")
10644 (match_operand:VI124_128 5 "nonimmediate_operand")])
10645 (match_operand:V_128 1 "general_operand")
10646 (match_operand:V_128 2 "general_operand")))]
10648 && (GET_MODE_NUNITS (<V_128:MODE>mode)
10649 == GET_MODE_NUNITS (<VI124_128:MODE>mode))"
10651 bool ok = ix86_expand_int_vcond (operands);
10656 (define_expand "vcondu<VI8F_128:mode>v2di"
10657 [(set (match_operand:VI8F_128 0 "register_operand")
10658 (if_then_else:VI8F_128
10659 (match_operator 3 ""
10660 [(match_operand:V2DI 4 "nonimmediate_operand")
10661 (match_operand:V2DI 5 "nonimmediate_operand")])
10662 (match_operand:VI8F_128 1 "general_operand")
10663 (match_operand:VI8F_128 2 "general_operand")))]
10666 bool ok = ix86_expand_int_vcond (operands);
10671 (define_mode_iterator VEC_PERM_AVX2
10672 [V16QI V8HI V4SI V2DI V4SF V2DF
10673 (V32QI "TARGET_AVX2") (V16HI "TARGET_AVX2")
10674 (V8SI "TARGET_AVX2") (V4DI "TARGET_AVX2")
10675 (V8SF "TARGET_AVX2") (V4DF "TARGET_AVX2")
10676 (V16SF "TARGET_AVX512F") (V8DF "TARGET_AVX512F")
10677 (V16SI "TARGET_AVX512F") (V8DI "TARGET_AVX512F")
10678 (V32HI "TARGET_AVX512BW") (V64QI "TARGET_AVX512VBMI")])
10680 (define_expand "vec_perm<mode>"
10681 [(match_operand:VEC_PERM_AVX2 0 "register_operand")
10682 (match_operand:VEC_PERM_AVX2 1 "register_operand")
10683 (match_operand:VEC_PERM_AVX2 2 "register_operand")
10684 (match_operand:<sseintvecmode> 3 "register_operand")]
10685 "TARGET_SSSE3 || TARGET_AVX || TARGET_XOP"
10687 ix86_expand_vec_perm (operands);
10691 (define_mode_iterator VEC_PERM_CONST
10692 [(V4SF "TARGET_SSE") (V4SI "TARGET_SSE")
10693 (V2DF "TARGET_SSE") (V2DI "TARGET_SSE")
10694 (V16QI "TARGET_SSE2") (V8HI "TARGET_SSE2")
10695 (V8SF "TARGET_AVX") (V4DF "TARGET_AVX")
10696 (V8SI "TARGET_AVX") (V4DI "TARGET_AVX")
10697 (V32QI "TARGET_AVX2") (V16HI "TARGET_AVX2")
10698 (V16SI "TARGET_AVX512F") (V8DI "TARGET_AVX512F")
10699 (V16SF "TARGET_AVX512F") (V8DF "TARGET_AVX512F")
10700 (V32HI "TARGET_AVX512BW") (V64QI "TARGET_AVX512BW")])
10702 (define_expand "vec_perm_const<mode>"
10703 [(match_operand:VEC_PERM_CONST 0 "register_operand")
10704 (match_operand:VEC_PERM_CONST 1 "register_operand")
10705 (match_operand:VEC_PERM_CONST 2 "register_operand")
10706 (match_operand:<sseintvecmode> 3)]
10709 if (ix86_expand_vec_perm_const (operands))
10715 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
10717 ;; Parallel bitwise logical operations
10719 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
10721 (define_expand "one_cmpl<mode>2"
10722 [(set (match_operand:VI 0 "register_operand")
10723 (xor:VI (match_operand:VI 1 "nonimmediate_operand")
10727 int i, n = GET_MODE_NUNITS (<MODE>mode);
10728 rtvec v = rtvec_alloc (n);
10730 for (i = 0; i < n; ++i)
10731 RTVEC_ELT (v, i) = constm1_rtx;
10733 operands[2] = force_reg (<MODE>mode, gen_rtx_CONST_VECTOR (<MODE>mode, v));
10736 (define_expand "<sse2_avx2>_andnot<mode>3"
10737 [(set (match_operand:VI_AVX2 0 "register_operand")
10739 (not:VI_AVX2 (match_operand:VI_AVX2 1 "register_operand"))
10740 (match_operand:VI_AVX2 2 "nonimmediate_operand")))]
10743 (define_expand "<sse2_avx2>_andnot<mode>3_mask"
10744 [(set (match_operand:VI48_AVX512VL 0 "register_operand")
10745 (vec_merge:VI48_AVX512VL
10748 (match_operand:VI48_AVX512VL 1 "register_operand"))
10749 (match_operand:VI48_AVX512VL 2 "nonimmediate_operand"))
10750 (match_operand:VI48_AVX512VL 3 "vector_move_operand")
10751 (match_operand:<avx512fmaskmode> 4 "register_operand")))]
10754 (define_expand "<sse2_avx2>_andnot<mode>3_mask"
10755 [(set (match_operand:VI12_AVX512VL 0 "register_operand")
10756 (vec_merge:VI12_AVX512VL
10759 (match_operand:VI12_AVX512VL 1 "register_operand"))
10760 (match_operand:VI12_AVX512VL 2 "nonimmediate_operand"))
10761 (match_operand:VI12_AVX512VL 3 "vector_move_operand")
10762 (match_operand:<avx512fmaskmode> 4 "register_operand")))]
10765 (define_insn "*andnot<mode>3"
10766 [(set (match_operand:VI 0 "register_operand" "=x,v")
10768 (not:VI (match_operand:VI 1 "register_operand" "0,v"))
10769 (match_operand:VI 2 "nonimmediate_operand" "xm,vm")))]
10772 static char buf[64];
10776 switch (get_attr_mode (insn))
10779 gcc_assert (TARGET_AVX512F);
10781 gcc_assert (TARGET_AVX2 || TARGET_AVX512VL);
10783 gcc_assert (TARGET_SSE2 || TARGET_AVX512VL);
10784 switch (<MODE>mode)
10788 if (TARGET_AVX512F)
10790 tmp = "pandn<ssemodesuffix>";
10797 if (TARGET_AVX512VL)
10799 tmp = "pandn<ssemodesuffix>";
10803 tmp = TARGET_AVX512VL ? "pandnq" : "pandn";
10808 gcc_assert (TARGET_AVX512F);
10810 gcc_assert (TARGET_AVX);
10812 gcc_assert (TARGET_SSE);
10818 gcc_unreachable ();
10821 switch (which_alternative)
10824 ops = "%s\t{%%2, %%0|%%0, %%2}";
10827 ops = "v%s\t{%%2, %%1, %%0<mask_operand3_1>|%%0<mask_operand3_1>, %%1, %%2}";
10830 gcc_unreachable ();
10833 snprintf (buf, sizeof (buf), ops, tmp);
10836 [(set_attr "isa" "noavx,avx")
10837 (set_attr "type" "sselog")
10838 (set (attr "prefix_data16")
10840 (and (eq_attr "alternative" "0")
10841 (eq_attr "mode" "TI"))
10843 (const_string "*")))
10844 (set_attr "prefix" "orig,vex")
10846 (cond [(and (match_test "<MODE_SIZE> == 16")
10847 (match_test "TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL"))
10848 (const_string "<ssePSmode>")
10849 (match_test "TARGET_AVX2")
10850 (const_string "<sseinsnmode>")
10851 (match_test "TARGET_AVX")
10853 (match_test "<MODE_SIZE> > 16")
10854 (const_string "V8SF")
10855 (const_string "<sseinsnmode>"))
10856 (ior (not (match_test "TARGET_SSE2"))
10857 (match_test "optimize_function_for_size_p (cfun)"))
10858 (const_string "V4SF")
10860 (const_string "<sseinsnmode>")))])
10862 (define_insn "*andnot<mode>3_mask"
10863 [(set (match_operand:VI48_AVX512VL 0 "register_operand" "=v")
10864 (vec_merge:VI48_AVX512VL
10867 (match_operand:VI48_AVX512VL 1 "register_operand" "v"))
10868 (match_operand:VI48_AVX512VL 2 "nonimmediate_operand" "vm"))
10869 (match_operand:VI48_AVX512VL 3 "vector_move_operand" "0C")
10870 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk")))]
10872 "vpandn<ssemodesuffix>\t{%2, %1, %0%{%4%}%N3|%0%{%4%}%N3, %1, %2}";
10873 [(set_attr "type" "sselog")
10874 (set_attr "prefix" "evex")
10875 (set_attr "mode" "<sseinsnmode>")])
10877 (define_insn "*andnot<mode>3_mask"
10878 [(set (match_operand:VI12_AVX512VL 0 "register_operand" "=v")
10879 (vec_merge:VI12_AVX512VL
10882 (match_operand:VI12_AVX512VL 1 "register_operand" "v"))
10883 (match_operand:VI12_AVX512VL 2 "nonimmediate_operand" "vm"))
10884 (match_operand:VI12_AVX512VL 3 "vector_move_operand" "0C")
10885 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk")))]
10887 "vpandn<ssemodesuffix>\t{%2, %1, %0%{%4%}%N3|%0%{%4%}%N3, %1, %2}";
10888 [(set_attr "type" "sselog")
10889 (set_attr "prefix" "evex")
10890 (set_attr "mode" "<sseinsnmode>")])
10892 (define_expand "<code><mode>3"
10893 [(set (match_operand:VI 0 "register_operand")
10895 (match_operand:VI 1 "nonimmediate_or_const_vector_operand")
10896 (match_operand:VI 2 "nonimmediate_or_const_vector_operand")))]
10899 ix86_expand_vector_logical_operator (<CODE>, <MODE>mode, operands);
10903 (define_insn "<mask_codefor><code><mode>3<mask_name>"
10904 [(set (match_operand:VI 0 "register_operand" "=x,v")
10906 (match_operand:VI 1 "nonimmediate_operand" "%0,v")
10907 (match_operand:VI 2 "nonimmediate_operand" "xm,vm")))]
10908 "TARGET_SSE && <mask_mode512bit_condition>
10909 && ix86_binary_operator_ok (<CODE>, <MODE>mode, operands)"
10911 static char buf[64];
10915 switch (get_attr_mode (insn))
10918 gcc_assert (TARGET_AVX512F);
10920 gcc_assert (TARGET_AVX2 || TARGET_AVX512VL);
10922 gcc_assert (TARGET_SSE2 || TARGET_AVX512VL);
10923 switch (<MODE>mode)
10927 if (TARGET_AVX512F)
10929 tmp = "p<logic><ssemodesuffix>";
10936 if (TARGET_AVX512VL)
10938 tmp = "p<logic><ssemodesuffix>";
10942 tmp = TARGET_AVX512VL ? "p<logic>q" : "p<logic>";
10947 gcc_assert (TARGET_AVX512F);
10949 gcc_assert (TARGET_AVX);
10951 gcc_assert (TARGET_SSE);
10957 gcc_unreachable ();
10960 switch (which_alternative)
10963 ops = "%s\t{%%2, %%0|%%0, %%2}";
10966 ops = "v%s\t{%%2, %%1, %%0<mask_operand3_1>|%%0<mask_operand3_1>, %%1, %%2}";
10969 gcc_unreachable ();
10972 snprintf (buf, sizeof (buf), ops, tmp);
10975 [(set_attr "isa" "noavx,avx")
10976 (set_attr "type" "sselog")
10977 (set (attr "prefix_data16")
10979 (and (eq_attr "alternative" "0")
10980 (eq_attr "mode" "TI"))
10982 (const_string "*")))
10983 (set_attr "prefix" "<mask_prefix3>")
10985 (cond [(and (match_test "<MODE_SIZE> == 16")
10986 (match_test "TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL"))
10987 (const_string "<ssePSmode>")
10988 (match_test "TARGET_AVX2")
10989 (const_string "<sseinsnmode>")
10990 (match_test "TARGET_AVX")
10992 (match_test "<MODE_SIZE> > 16")
10993 (const_string "V8SF")
10994 (const_string "<sseinsnmode>"))
10995 (ior (not (match_test "TARGET_SSE2"))
10996 (match_test "optimize_function_for_size_p (cfun)"))
10997 (const_string "V4SF")
10999 (const_string "<sseinsnmode>")))])
11001 (define_insn "<avx512>_testm<mode>3<mask_scalar_merge_name>"
11002 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=Yk")
11003 (unspec:<avx512fmaskmode>
11004 [(match_operand:VI12_AVX512VL 1 "register_operand" "v")
11005 (match_operand:VI12_AVX512VL 2 "nonimmediate_operand" "vm")]
11008 "vptestm<ssemodesuffix>\t{%2, %1, %0<mask_scalar_merge_operand3>|%0<mask_scalar_merge_operand3>, %1, %2}"
11009 [(set_attr "prefix" "evex")
11010 (set_attr "mode" "<sseinsnmode>")])
11012 (define_insn "<avx512>_testm<mode>3<mask_scalar_merge_name>"
11013 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=Yk")
11014 (unspec:<avx512fmaskmode>
11015 [(match_operand:VI48_AVX512VL 1 "register_operand" "v")
11016 (match_operand:VI48_AVX512VL 2 "nonimmediate_operand" "vm")]
11019 "vptestm<ssemodesuffix>\t{%2, %1, %0<mask_scalar_merge_operand3>|%0<mask_scalar_merge_operand3>, %1, %2}"
11020 [(set_attr "prefix" "evex")
11021 (set_attr "mode" "<sseinsnmode>")])
11023 (define_insn "<avx512>_testnm<mode>3<mask_scalar_merge_name>"
11024 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=Yk")
11025 (unspec:<avx512fmaskmode>
11026 [(match_operand:VI12_AVX512VL 1 "register_operand" "v")
11027 (match_operand:VI12_AVX512VL 2 "nonimmediate_operand" "vm")]
11030 "vptestnm<ssemodesuffix>\t{%2, %1, %0<mask_scalar_merge_operand3>|%0<mask_scalar_merge_operand3>, %1, %2}"
11031 [(set_attr "prefix" "evex")
11032 (set_attr "mode" "<sseinsnmode>")])
11034 (define_insn "<avx512>_testnm<mode>3<mask_scalar_merge_name>"
11035 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=Yk")
11036 (unspec:<avx512fmaskmode>
11037 [(match_operand:VI48_AVX512VL 1 "register_operand" "v")
11038 (match_operand:VI48_AVX512VL 2 "nonimmediate_operand" "vm")]
11041 "vptestnm<ssemodesuffix>\t{%2, %1, %0<mask_scalar_merge_operand3>|%0<mask_scalar_merge_operand3>, %1, %2}"
11042 [(set_attr "prefix" "evex")
11043 (set_attr "mode" "<sseinsnmode>")])
11045 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
11047 ;; Parallel integral element swizzling
11049 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
11051 (define_expand "vec_pack_trunc_<mode>"
11052 [(match_operand:<ssepackmode> 0 "register_operand")
11053 (match_operand:VI248_AVX2_8_AVX512F 1 "register_operand")
11054 (match_operand:VI248_AVX2_8_AVX512F 2 "register_operand")]
11057 rtx op1 = gen_lowpart (<ssepackmode>mode, operands[1]);
11058 rtx op2 = gen_lowpart (<ssepackmode>mode, operands[2]);
11059 ix86_expand_vec_extract_even_odd (operands[0], op1, op2, 0);
11063 (define_insn "<sse2_avx2>_packsswb<mask_name>"
11064 [(set (match_operand:VI1_AVX512 0 "register_operand" "=x,x")
11065 (vec_concat:VI1_AVX512
11066 (ss_truncate:<ssehalfvecmode>
11067 (match_operand:<sseunpackmode> 1 "register_operand" "0,v"))
11068 (ss_truncate:<ssehalfvecmode>
11069 (match_operand:<sseunpackmode> 2 "nonimmediate_operand" "xm,vm"))))]
11070 "TARGET_SSE2 && <mask_mode512bit_condition> && <mask_avx512bw_condition>"
11072 packsswb\t{%2, %0|%0, %2}
11073 vpacksswb\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
11074 [(set_attr "isa" "noavx,avx")
11075 (set_attr "type" "sselog")
11076 (set_attr "prefix_data16" "1,*")
11077 (set_attr "prefix" "orig,maybe_evex")
11078 (set_attr "mode" "<sseinsnmode>")])
11080 (define_insn "<sse2_avx2>_packssdw<mask_name>"
11081 [(set (match_operand:VI2_AVX2 0 "register_operand" "=x,v")
11082 (vec_concat:VI2_AVX2
11083 (ss_truncate:<ssehalfvecmode>
11084 (match_operand:<sseunpackmode> 1 "register_operand" "0,v"))
11085 (ss_truncate:<ssehalfvecmode>
11086 (match_operand:<sseunpackmode> 2 "nonimmediate_operand" "xm,vm"))))]
11087 "TARGET_SSE2 && <mask_mode512bit_condition> && <mask_avx512bw_condition>"
11089 packssdw\t{%2, %0|%0, %2}
11090 vpackssdw\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
11091 [(set_attr "isa" "noavx,avx")
11092 (set_attr "type" "sselog")
11093 (set_attr "prefix_data16" "1,*")
11094 (set_attr "prefix" "orig,vex")
11095 (set_attr "mode" "<sseinsnmode>")])
11097 (define_insn "<sse2_avx2>_packuswb<mask_name>"
11098 [(set (match_operand:VI1_AVX512 0 "register_operand" "=x,x")
11099 (vec_concat:VI1_AVX512
11100 (us_truncate:<ssehalfvecmode>
11101 (match_operand:<sseunpackmode> 1 "register_operand" "0,v"))
11102 (us_truncate:<ssehalfvecmode>
11103 (match_operand:<sseunpackmode> 2 "nonimmediate_operand" "xm,vm"))))]
11104 "TARGET_SSE2 && <mask_mode512bit_condition> && <mask_avx512bw_condition>"
11106 packuswb\t{%2, %0|%0, %2}
11107 vpackuswb\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
11108 [(set_attr "isa" "noavx,avx")
11109 (set_attr "type" "sselog")
11110 (set_attr "prefix_data16" "1,*")
11111 (set_attr "prefix" "orig,vex")
11112 (set_attr "mode" "<sseinsnmode>")])
11114 (define_insn "avx512bw_interleave_highv64qi<mask_name>"
11115 [(set (match_operand:V64QI 0 "register_operand" "=v")
11118 (match_operand:V64QI 1 "register_operand" "v")
11119 (match_operand:V64QI 2 "nonimmediate_operand" "vm"))
11120 (parallel [(const_int 8) (const_int 72)
11121 (const_int 9) (const_int 73)
11122 (const_int 10) (const_int 74)
11123 (const_int 11) (const_int 75)
11124 (const_int 12) (const_int 76)
11125 (const_int 13) (const_int 77)
11126 (const_int 14) (const_int 78)
11127 (const_int 15) (const_int 79)
11128 (const_int 24) (const_int 88)
11129 (const_int 25) (const_int 89)
11130 (const_int 26) (const_int 90)
11131 (const_int 27) (const_int 91)
11132 (const_int 28) (const_int 92)
11133 (const_int 29) (const_int 93)
11134 (const_int 30) (const_int 94)
11135 (const_int 31) (const_int 95)
11136 (const_int 40) (const_int 104)
11137 (const_int 41) (const_int 105)
11138 (const_int 42) (const_int 106)
11139 (const_int 43) (const_int 107)
11140 (const_int 44) (const_int 108)
11141 (const_int 45) (const_int 109)
11142 (const_int 46) (const_int 110)
11143 (const_int 47) (const_int 111)
11144 (const_int 56) (const_int 120)
11145 (const_int 57) (const_int 121)
11146 (const_int 58) (const_int 122)
11147 (const_int 59) (const_int 123)
11148 (const_int 60) (const_int 124)
11149 (const_int 61) (const_int 125)
11150 (const_int 62) (const_int 126)
11151 (const_int 63) (const_int 127)])))]
11153 "vpunpckhbw\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
11154 [(set_attr "type" "sselog")
11155 (set_attr "prefix" "evex")
11156 (set_attr "mode" "XI")])
11158 (define_insn "avx2_interleave_highv32qi<mask_name>"
11159 [(set (match_operand:V32QI 0 "register_operand" "=v")
11162 (match_operand:V32QI 1 "register_operand" "v")
11163 (match_operand:V32QI 2 "nonimmediate_operand" "vm"))
11164 (parallel [(const_int 8) (const_int 40)
11165 (const_int 9) (const_int 41)
11166 (const_int 10) (const_int 42)
11167 (const_int 11) (const_int 43)
11168 (const_int 12) (const_int 44)
11169 (const_int 13) (const_int 45)
11170 (const_int 14) (const_int 46)
11171 (const_int 15) (const_int 47)
11172 (const_int 24) (const_int 56)
11173 (const_int 25) (const_int 57)
11174 (const_int 26) (const_int 58)
11175 (const_int 27) (const_int 59)
11176 (const_int 28) (const_int 60)
11177 (const_int 29) (const_int 61)
11178 (const_int 30) (const_int 62)
11179 (const_int 31) (const_int 63)])))]
11180 "TARGET_AVX2 && <mask_avx512vl_condition>"
11181 "vpunpckhbw\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
11182 [(set_attr "type" "sselog")
11183 (set_attr "prefix" "<mask_prefix>")
11184 (set_attr "mode" "OI")])
11186 (define_insn "vec_interleave_highv16qi<mask_name>"
11187 [(set (match_operand:V16QI 0 "register_operand" "=x,v")
11190 (match_operand:V16QI 1 "register_operand" "0,v")
11191 (match_operand:V16QI 2 "nonimmediate_operand" "xm,vm"))
11192 (parallel [(const_int 8) (const_int 24)
11193 (const_int 9) (const_int 25)
11194 (const_int 10) (const_int 26)
11195 (const_int 11) (const_int 27)
11196 (const_int 12) (const_int 28)
11197 (const_int 13) (const_int 29)
11198 (const_int 14) (const_int 30)
11199 (const_int 15) (const_int 31)])))]
11200 "TARGET_SSE2 && <mask_avx512vl_condition>"
11202 punpckhbw\t{%2, %0|%0, %2}
11203 vpunpckhbw\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
11204 [(set_attr "isa" "noavx,avx")
11205 (set_attr "type" "sselog")
11206 (set_attr "prefix_data16" "1,*")
11207 (set_attr "prefix" "orig,<mask_prefix>")
11208 (set_attr "mode" "TI")])
11210 (define_insn "avx512bw_interleave_lowv64qi<mask_name>"
11211 [(set (match_operand:V64QI 0 "register_operand" "=v")
11214 (match_operand:V64QI 1 "register_operand" "v")
11215 (match_operand:V64QI 2 "nonimmediate_operand" "vm"))
11216 (parallel [(const_int 0) (const_int 64)
11217 (const_int 1) (const_int 65)
11218 (const_int 2) (const_int 66)
11219 (const_int 3) (const_int 67)
11220 (const_int 4) (const_int 68)
11221 (const_int 5) (const_int 69)
11222 (const_int 6) (const_int 70)
11223 (const_int 7) (const_int 71)
11224 (const_int 16) (const_int 80)
11225 (const_int 17) (const_int 81)
11226 (const_int 18) (const_int 82)
11227 (const_int 19) (const_int 83)
11228 (const_int 20) (const_int 84)
11229 (const_int 21) (const_int 85)
11230 (const_int 22) (const_int 86)
11231 (const_int 23) (const_int 87)
11232 (const_int 32) (const_int 96)
11233 (const_int 33) (const_int 97)
11234 (const_int 34) (const_int 98)
11235 (const_int 35) (const_int 99)
11236 (const_int 36) (const_int 100)
11237 (const_int 37) (const_int 101)
11238 (const_int 38) (const_int 102)
11239 (const_int 39) (const_int 103)
11240 (const_int 48) (const_int 112)
11241 (const_int 49) (const_int 113)
11242 (const_int 50) (const_int 114)
11243 (const_int 51) (const_int 115)
11244 (const_int 52) (const_int 116)
11245 (const_int 53) (const_int 117)
11246 (const_int 54) (const_int 118)
11247 (const_int 55) (const_int 119)])))]
11249 "vpunpcklbw\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
11250 [(set_attr "type" "sselog")
11251 (set_attr "prefix" "evex")
11252 (set_attr "mode" "XI")])
11254 (define_insn "avx2_interleave_lowv32qi<mask_name>"
11255 [(set (match_operand:V32QI 0 "register_operand" "=v")
11258 (match_operand:V32QI 1 "register_operand" "v")
11259 (match_operand:V32QI 2 "nonimmediate_operand" "vm"))
11260 (parallel [(const_int 0) (const_int 32)
11261 (const_int 1) (const_int 33)
11262 (const_int 2) (const_int 34)
11263 (const_int 3) (const_int 35)
11264 (const_int 4) (const_int 36)
11265 (const_int 5) (const_int 37)
11266 (const_int 6) (const_int 38)
11267 (const_int 7) (const_int 39)
11268 (const_int 16) (const_int 48)
11269 (const_int 17) (const_int 49)
11270 (const_int 18) (const_int 50)
11271 (const_int 19) (const_int 51)
11272 (const_int 20) (const_int 52)
11273 (const_int 21) (const_int 53)
11274 (const_int 22) (const_int 54)
11275 (const_int 23) (const_int 55)])))]
11276 "TARGET_AVX2 && <mask_avx512vl_condition> && <mask_avx512bw_condition>"
11277 "vpunpcklbw\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
11278 [(set_attr "type" "sselog")
11279 (set_attr "prefix" "maybe_vex")
11280 (set_attr "mode" "OI")])
11282 (define_insn "vec_interleave_lowv16qi<mask_name>"
11283 [(set (match_operand:V16QI 0 "register_operand" "=x,v")
11286 (match_operand:V16QI 1 "register_operand" "0,v")
11287 (match_operand:V16QI 2 "nonimmediate_operand" "xm,vm"))
11288 (parallel [(const_int 0) (const_int 16)
11289 (const_int 1) (const_int 17)
11290 (const_int 2) (const_int 18)
11291 (const_int 3) (const_int 19)
11292 (const_int 4) (const_int 20)
11293 (const_int 5) (const_int 21)
11294 (const_int 6) (const_int 22)
11295 (const_int 7) (const_int 23)])))]
11296 "TARGET_SSE2 && <mask_avx512vl_condition> && <mask_avx512bw_condition>"
11298 punpcklbw\t{%2, %0|%0, %2}
11299 vpunpcklbw\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
11300 [(set_attr "isa" "noavx,avx")
11301 (set_attr "type" "sselog")
11302 (set_attr "prefix_data16" "1,*")
11303 (set_attr "prefix" "orig,vex")
11304 (set_attr "mode" "TI")])
11306 (define_insn "avx512bw_interleave_highv32hi<mask_name>"
11307 [(set (match_operand:V32HI 0 "register_operand" "=v")
11310 (match_operand:V32HI 1 "register_operand" "v")
11311 (match_operand:V32HI 2 "nonimmediate_operand" "vm"))
11312 (parallel [(const_int 4) (const_int 36)
11313 (const_int 5) (const_int 37)
11314 (const_int 6) (const_int 38)
11315 (const_int 7) (const_int 39)
11316 (const_int 12) (const_int 44)
11317 (const_int 13) (const_int 45)
11318 (const_int 14) (const_int 46)
11319 (const_int 15) (const_int 47)
11320 (const_int 20) (const_int 52)
11321 (const_int 21) (const_int 53)
11322 (const_int 22) (const_int 54)
11323 (const_int 23) (const_int 55)
11324 (const_int 28) (const_int 60)
11325 (const_int 29) (const_int 61)
11326 (const_int 30) (const_int 62)
11327 (const_int 31) (const_int 63)])))]
11329 "vpunpckhwd\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
11330 [(set_attr "type" "sselog")
11331 (set_attr "prefix" "evex")
11332 (set_attr "mode" "XI")])
11334 (define_insn "avx2_interleave_highv16hi<mask_name>"
11335 [(set (match_operand:V16HI 0 "register_operand" "=v")
11338 (match_operand:V16HI 1 "register_operand" "v")
11339 (match_operand:V16HI 2 "nonimmediate_operand" "vm"))
11340 (parallel [(const_int 4) (const_int 20)
11341 (const_int 5) (const_int 21)
11342 (const_int 6) (const_int 22)
11343 (const_int 7) (const_int 23)
11344 (const_int 12) (const_int 28)
11345 (const_int 13) (const_int 29)
11346 (const_int 14) (const_int 30)
11347 (const_int 15) (const_int 31)])))]
11348 "TARGET_AVX2 && <mask_avx512vl_condition> && <mask_avx512bw_condition>"
11349 "vpunpckhwd\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
11350 [(set_attr "type" "sselog")
11351 (set_attr "prefix" "maybe_evex")
11352 (set_attr "mode" "OI")])
11354 (define_insn "vec_interleave_highv8hi<mask_name>"
11355 [(set (match_operand:V8HI 0 "register_operand" "=x,v")
11358 (match_operand:V8HI 1 "register_operand" "0,v")
11359 (match_operand:V8HI 2 "nonimmediate_operand" "xm,vm"))
11360 (parallel [(const_int 4) (const_int 12)
11361 (const_int 5) (const_int 13)
11362 (const_int 6) (const_int 14)
11363 (const_int 7) (const_int 15)])))]
11364 "TARGET_SSE2 && <mask_avx512vl_condition> && <mask_avx512bw_condition>"
11366 punpckhwd\t{%2, %0|%0, %2}
11367 vpunpckhwd\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
11368 [(set_attr "isa" "noavx,avx")
11369 (set_attr "type" "sselog")
11370 (set_attr "prefix_data16" "1,*")
11371 (set_attr "prefix" "orig,maybe_vex")
11372 (set_attr "mode" "TI")])
11374 (define_insn "<mask_codefor>avx512bw_interleave_lowv32hi<mask_name>"
11375 [(set (match_operand:V32HI 0 "register_operand" "=v")
11378 (match_operand:V32HI 1 "register_operand" "v")
11379 (match_operand:V32HI 2 "nonimmediate_operand" "vm"))
11380 (parallel [(const_int 0) (const_int 32)
11381 (const_int 1) (const_int 33)
11382 (const_int 2) (const_int 34)
11383 (const_int 3) (const_int 35)
11384 (const_int 8) (const_int 40)
11385 (const_int 9) (const_int 41)
11386 (const_int 10) (const_int 42)
11387 (const_int 11) (const_int 43)
11388 (const_int 16) (const_int 48)
11389 (const_int 17) (const_int 49)
11390 (const_int 18) (const_int 50)
11391 (const_int 19) (const_int 51)
11392 (const_int 24) (const_int 56)
11393 (const_int 25) (const_int 57)
11394 (const_int 26) (const_int 58)
11395 (const_int 27) (const_int 59)])))]
11397 "vpunpcklwd\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
11398 [(set_attr "type" "sselog")
11399 (set_attr "prefix" "evex")
11400 (set_attr "mode" "XI")])
11402 (define_insn "avx2_interleave_lowv16hi<mask_name>"
11403 [(set (match_operand:V16HI 0 "register_operand" "=v")
11406 (match_operand:V16HI 1 "register_operand" "v")
11407 (match_operand:V16HI 2 "nonimmediate_operand" "vm"))
11408 (parallel [(const_int 0) (const_int 16)
11409 (const_int 1) (const_int 17)
11410 (const_int 2) (const_int 18)
11411 (const_int 3) (const_int 19)
11412 (const_int 8) (const_int 24)
11413 (const_int 9) (const_int 25)
11414 (const_int 10) (const_int 26)
11415 (const_int 11) (const_int 27)])))]
11416 "TARGET_AVX2 && <mask_avx512vl_condition> && <mask_avx512bw_condition>"
11417 "vpunpcklwd\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
11418 [(set_attr "type" "sselog")
11419 (set_attr "prefix" "maybe_evex")
11420 (set_attr "mode" "OI")])
11422 (define_insn "vec_interleave_lowv8hi<mask_name>"
11423 [(set (match_operand:V8HI 0 "register_operand" "=x,v")
11426 (match_operand:V8HI 1 "register_operand" "0,v")
11427 (match_operand:V8HI 2 "nonimmediate_operand" "xm,vm"))
11428 (parallel [(const_int 0) (const_int 8)
11429 (const_int 1) (const_int 9)
11430 (const_int 2) (const_int 10)
11431 (const_int 3) (const_int 11)])))]
11432 "TARGET_SSE2 && <mask_avx512vl_condition> && <mask_avx512bw_condition>"
11434 punpcklwd\t{%2, %0|%0, %2}
11435 vpunpcklwd\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
11436 [(set_attr "isa" "noavx,avx")
11437 (set_attr "type" "sselog")
11438 (set_attr "prefix_data16" "1,*")
11439 (set_attr "prefix" "orig,maybe_evex")
11440 (set_attr "mode" "TI")])
11442 (define_insn "avx2_interleave_highv8si<mask_name>"
11443 [(set (match_operand:V8SI 0 "register_operand" "=v")
11446 (match_operand:V8SI 1 "register_operand" "v")
11447 (match_operand:V8SI 2 "nonimmediate_operand" "vm"))
11448 (parallel [(const_int 2) (const_int 10)
11449 (const_int 3) (const_int 11)
11450 (const_int 6) (const_int 14)
11451 (const_int 7) (const_int 15)])))]
11452 "TARGET_AVX2 && <mask_avx512vl_condition>"
11453 "vpunpckhdq\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
11454 [(set_attr "type" "sselog")
11455 (set_attr "prefix" "maybe_evex")
11456 (set_attr "mode" "OI")])
11458 (define_insn "<mask_codefor>avx512f_interleave_highv16si<mask_name>"
11459 [(set (match_operand:V16SI 0 "register_operand" "=v")
11462 (match_operand:V16SI 1 "register_operand" "v")
11463 (match_operand:V16SI 2 "nonimmediate_operand" "vm"))
11464 (parallel [(const_int 2) (const_int 18)
11465 (const_int 3) (const_int 19)
11466 (const_int 6) (const_int 22)
11467 (const_int 7) (const_int 23)
11468 (const_int 10) (const_int 26)
11469 (const_int 11) (const_int 27)
11470 (const_int 14) (const_int 30)
11471 (const_int 15) (const_int 31)])))]
11473 "vpunpckhdq\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
11474 [(set_attr "type" "sselog")
11475 (set_attr "prefix" "evex")
11476 (set_attr "mode" "XI")])
11479 (define_insn "vec_interleave_highv4si<mask_name>"
11480 [(set (match_operand:V4SI 0 "register_operand" "=x,v")
11483 (match_operand:V4SI 1 "register_operand" "0,v")
11484 (match_operand:V4SI 2 "nonimmediate_operand" "xm,vm"))
11485 (parallel [(const_int 2) (const_int 6)
11486 (const_int 3) (const_int 7)])))]
11487 "TARGET_SSE2 && <mask_avx512vl_condition>"
11489 punpckhdq\t{%2, %0|%0, %2}
11490 vpunpckhdq\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
11491 [(set_attr "isa" "noavx,avx")
11492 (set_attr "type" "sselog")
11493 (set_attr "prefix_data16" "1,*")
11494 (set_attr "prefix" "orig,maybe_vex")
11495 (set_attr "mode" "TI")])
11497 (define_insn "avx2_interleave_lowv8si<mask_name>"
11498 [(set (match_operand:V8SI 0 "register_operand" "=v")
11501 (match_operand:V8SI 1 "register_operand" "v")
11502 (match_operand:V8SI 2 "nonimmediate_operand" "vm"))
11503 (parallel [(const_int 0) (const_int 8)
11504 (const_int 1) (const_int 9)
11505 (const_int 4) (const_int 12)
11506 (const_int 5) (const_int 13)])))]
11507 "TARGET_AVX2 && <mask_avx512vl_condition>"
11508 "vpunpckldq\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
11509 [(set_attr "type" "sselog")
11510 (set_attr "prefix" "maybe_evex")
11511 (set_attr "mode" "OI")])
11513 (define_insn "<mask_codefor>avx512f_interleave_lowv16si<mask_name>"
11514 [(set (match_operand:V16SI 0 "register_operand" "=v")
11517 (match_operand:V16SI 1 "register_operand" "v")
11518 (match_operand:V16SI 2 "nonimmediate_operand" "vm"))
11519 (parallel [(const_int 0) (const_int 16)
11520 (const_int 1) (const_int 17)
11521 (const_int 4) (const_int 20)
11522 (const_int 5) (const_int 21)
11523 (const_int 8) (const_int 24)
11524 (const_int 9) (const_int 25)
11525 (const_int 12) (const_int 28)
11526 (const_int 13) (const_int 29)])))]
11528 "vpunpckldq\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
11529 [(set_attr "type" "sselog")
11530 (set_attr "prefix" "evex")
11531 (set_attr "mode" "XI")])
11533 (define_insn "vec_interleave_lowv4si<mask_name>"
11534 [(set (match_operand:V4SI 0 "register_operand" "=x,v")
11537 (match_operand:V4SI 1 "register_operand" "0,v")
11538 (match_operand:V4SI 2 "nonimmediate_operand" "xm,vm"))
11539 (parallel [(const_int 0) (const_int 4)
11540 (const_int 1) (const_int 5)])))]
11541 "TARGET_SSE2 && <mask_avx512vl_condition>"
11543 punpckldq\t{%2, %0|%0, %2}
11544 vpunpckldq\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
11545 [(set_attr "isa" "noavx,avx")
11546 (set_attr "type" "sselog")
11547 (set_attr "prefix_data16" "1,*")
11548 (set_attr "prefix" "orig,vex")
11549 (set_attr "mode" "TI")])
11551 (define_expand "vec_interleave_high<mode>"
11552 [(match_operand:VI_256 0 "register_operand" "=x")
11553 (match_operand:VI_256 1 "register_operand" "x")
11554 (match_operand:VI_256 2 "nonimmediate_operand" "xm")]
11557 rtx t1 = gen_reg_rtx (<MODE>mode);
11558 rtx t2 = gen_reg_rtx (<MODE>mode);
11559 rtx t3 = gen_reg_rtx (V4DImode);
11560 emit_insn (gen_avx2_interleave_low<mode> (t1, operands[1], operands[2]));
11561 emit_insn (gen_avx2_interleave_high<mode> (t2, operands[1], operands[2]));
11562 emit_insn (gen_avx2_permv2ti (t3, gen_lowpart (V4DImode, t1),
11563 gen_lowpart (V4DImode, t2),
11564 GEN_INT (1 + (3 << 4))));
11565 emit_move_insn (operands[0], gen_lowpart (<MODE>mode, t3));
11569 (define_expand "vec_interleave_low<mode>"
11570 [(match_operand:VI_256 0 "register_operand" "=x")
11571 (match_operand:VI_256 1 "register_operand" "x")
11572 (match_operand:VI_256 2 "nonimmediate_operand" "xm")]
11575 rtx t1 = gen_reg_rtx (<MODE>mode);
11576 rtx t2 = gen_reg_rtx (<MODE>mode);
11577 rtx t3 = gen_reg_rtx (V4DImode);
11578 emit_insn (gen_avx2_interleave_low<mode> (t1, operands[1], operands[2]));
11579 emit_insn (gen_avx2_interleave_high<mode> (t2, operands[1], operands[2]));
11580 emit_insn (gen_avx2_permv2ti (t3, gen_lowpart (V4DImode, t1),
11581 gen_lowpart (V4DImode, t2),
11582 GEN_INT (0 + (2 << 4))));
11583 emit_move_insn (operands[0], gen_lowpart (<MODE>mode, t3));
11587 ;; Modes handled by pinsr patterns.
11588 (define_mode_iterator PINSR_MODE
11589 [(V16QI "TARGET_SSE4_1") V8HI
11590 (V4SI "TARGET_SSE4_1")
11591 (V2DI "TARGET_SSE4_1 && TARGET_64BIT")])
11593 (define_mode_attr sse2p4_1
11594 [(V16QI "sse4_1") (V8HI "sse2")
11595 (V4SI "sse4_1") (V2DI "sse4_1")])
11597 ;; sse4_1_pinsrd must come before sse2_loadld since it is preferred.
11598 (define_insn "<sse2p4_1>_pinsr<ssemodesuffix>"
11599 [(set (match_operand:PINSR_MODE 0 "register_operand" "=x,x,x,x")
11600 (vec_merge:PINSR_MODE
11601 (vec_duplicate:PINSR_MODE
11602 (match_operand:<ssescalarmode> 2 "nonimmediate_operand" "r,m,r,m"))
11603 (match_operand:PINSR_MODE 1 "register_operand" "0,0,x,x")
11604 (match_operand:SI 3 "const_int_operand")))]
11606 && ((unsigned) exact_log2 (INTVAL (operands[3]))
11607 < GET_MODE_NUNITS (<MODE>mode))"
11609 operands[3] = GEN_INT (exact_log2 (INTVAL (operands[3])));
11611 switch (which_alternative)
11614 if (GET_MODE_SIZE (<ssescalarmode>mode) < GET_MODE_SIZE (SImode))
11615 return "pinsr<ssemodesuffix>\t{%3, %k2, %0|%0, %k2, %3}";
11618 return "pinsr<ssemodesuffix>\t{%3, %2, %0|%0, %2, %3}";
11620 if (GET_MODE_SIZE (<ssescalarmode>mode) < GET_MODE_SIZE (SImode))
11621 return "vpinsr<ssemodesuffix>\t{%3, %k2, %1, %0|%0, %1, %k2, %3}";
11624 return "vpinsr<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}";
11626 gcc_unreachable ();
11629 [(set_attr "isa" "noavx,noavx,avx,avx")
11630 (set_attr "type" "sselog")
11631 (set (attr "prefix_rex")
11633 (and (not (match_test "TARGET_AVX"))
11634 (eq (const_string "<MODE>mode") (const_string "V2DImode")))
11636 (const_string "*")))
11637 (set (attr "prefix_data16")
11639 (and (not (match_test "TARGET_AVX"))
11640 (eq (const_string "<MODE>mode") (const_string "V8HImode")))
11642 (const_string "*")))
11643 (set (attr "prefix_extra")
11645 (and (not (match_test "TARGET_AVX"))
11646 (eq (const_string "<MODE>mode") (const_string "V8HImode")))
11648 (const_string "1")))
11649 (set_attr "length_immediate" "1")
11650 (set_attr "prefix" "orig,orig,vex,vex")
11651 (set_attr "mode" "TI")])
11653 (define_expand "<extract_type>_vinsert<shuffletype><extract_suf>_mask"
11654 [(match_operand:AVX512_VEC 0 "register_operand")
11655 (match_operand:AVX512_VEC 1 "register_operand")
11656 (match_operand:<ssequartermode> 2 "nonimmediate_operand")
11657 (match_operand:SI 3 "const_0_to_3_operand")
11658 (match_operand:AVX512_VEC 4 "register_operand")
11659 (match_operand:<avx512fmaskmode> 5 "register_operand")]
11663 mask = INTVAL (operands[3]);
11664 selector = GET_MODE_SIZE (GET_MODE_INNER (<MODE>mode)) == 4 ?
11665 0xFFFF ^ (0xF000 >> mask * 4)
11666 : 0xFF ^ (0xC0 >> mask * 2);
11667 emit_insn (gen_<extract_type>_vinsert<shuffletype><extract_suf>_1_mask
11668 (operands[0], operands[1], operands[2], GEN_INT (selector),
11669 operands[4], operands[5]));
11673 (define_insn "<mask_codefor><extract_type>_vinsert<shuffletype><extract_suf>_1<mask_name>"
11674 [(set (match_operand:AVX512_VEC 0 "register_operand" "=v")
11675 (vec_merge:AVX512_VEC
11676 (match_operand:AVX512_VEC 1 "register_operand" "v")
11677 (vec_duplicate:AVX512_VEC
11678 (match_operand:<ssequartermode> 2 "nonimmediate_operand" "vm"))
11679 (match_operand:SI 3 "const_int_operand" "n")))]
11683 int selector = INTVAL (operands[3]);
11685 if (selector == 0xFFF || selector == 0x3F)
11687 else if ( selector == 0xF0FF || selector == 0xCF)
11689 else if ( selector == 0xFF0F || selector == 0xF3)
11691 else if ( selector == 0xFFF0 || selector == 0xFC)
11694 gcc_unreachable ();
11696 operands[3] = GEN_INT (mask);
11698 return "vinsert<shuffletype><extract_suf>\t{%3, %2, %1, %0<mask_operand4>|%0<mask_operand4>, %1, %2, %3}";
11700 [(set_attr "type" "sselog")
11701 (set_attr "length_immediate" "1")
11702 (set_attr "prefix" "evex")
11703 (set_attr "mode" "<sseinsnmode>")])
11705 (define_expand "<extract_type_2>_vinsert<shuffletype><extract_suf_2>_mask"
11706 [(match_operand:AVX512_VEC_2 0 "register_operand")
11707 (match_operand:AVX512_VEC_2 1 "register_operand")
11708 (match_operand:<ssehalfvecmode> 2 "nonimmediate_operand")
11709 (match_operand:SI 3 "const_0_to_1_operand")
11710 (match_operand:AVX512_VEC_2 4 "register_operand")
11711 (match_operand:<avx512fmaskmode> 5 "register_operand")]
11714 int mask = INTVAL (operands[3]);
11716 emit_insn (gen_vec_set_lo_<mode>_mask
11717 (operands[0], operands[1], operands[2],
11718 operands[4], operands[5]));
11720 emit_insn (gen_vec_set_hi_<mode>_mask
11721 (operands[0], operands[1], operands[2],
11722 operands[4], operands[5]));
11726 (define_insn "vec_set_lo_<mode><mask_name>"
11727 [(set (match_operand:V16FI 0 "register_operand" "=v")
11729 (match_operand:<ssehalfvecmode> 2 "nonimmediate_operand" "vm")
11730 (vec_select:<ssehalfvecmode>
11731 (match_operand:V16FI 1 "register_operand" "v")
11732 (parallel [(const_int 8) (const_int 9)
11733 (const_int 10) (const_int 11)
11734 (const_int 12) (const_int 13)
11735 (const_int 14) (const_int 15)]))))]
11737 "vinsert<shuffletype>32x8\t{$0x0, %2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2, $0x0}"
11738 [(set_attr "type" "sselog")
11739 (set_attr "length_immediate" "1")
11740 (set_attr "prefix" "evex")
11741 (set_attr "mode" "<sseinsnmode>")])
11743 (define_insn "vec_set_hi_<mode><mask_name>"
11744 [(set (match_operand:V16FI 0 "register_operand" "=v")
11746 (match_operand:<ssehalfvecmode> 2 "nonimmediate_operand" "vm")
11747 (vec_select:<ssehalfvecmode>
11748 (match_operand:V16FI 1 "register_operand" "v")
11749 (parallel [(const_int 0) (const_int 1)
11750 (const_int 2) (const_int 3)
11751 (const_int 4) (const_int 5)
11752 (const_int 6) (const_int 7)]))))]
11754 "vinsert<shuffletype>32x8\t{$0x1, %2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2, $0x1}"
11755 [(set_attr "type" "sselog")
11756 (set_attr "length_immediate" "1")
11757 (set_attr "prefix" "evex")
11758 (set_attr "mode" "<sseinsnmode>")])
11760 (define_insn "vec_set_lo_<mode><mask_name>"
11761 [(set (match_operand:V8FI 0 "register_operand" "=v")
11763 (match_operand:<ssehalfvecmode> 2 "nonimmediate_operand" "vm")
11764 (vec_select:<ssehalfvecmode>
11765 (match_operand:V8FI 1 "register_operand" "v")
11766 (parallel [(const_int 4) (const_int 5)
11767 (const_int 6) (const_int 7)]))))]
11769 "vinsert<shuffletype>64x4\t{$0x0, %2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2, $0x0}"
11770 [(set_attr "type" "sselog")
11771 (set_attr "length_immediate" "1")
11772 (set_attr "prefix" "evex")
11773 (set_attr "mode" "XI")])
11775 (define_insn "vec_set_hi_<mode><mask_name>"
11776 [(set (match_operand:V8FI 0 "register_operand" "=v")
11778 (match_operand:<ssehalfvecmode> 2 "nonimmediate_operand" "vm")
11779 (vec_select:<ssehalfvecmode>
11780 (match_operand:V8FI 1 "register_operand" "v")
11781 (parallel [(const_int 0) (const_int 1)
11782 (const_int 2) (const_int 3)]))))]
11784 "vinsert<shuffletype>64x4\t{$0x1, %2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2, $0x1}"
11785 [(set_attr "type" "sselog")
11786 (set_attr "length_immediate" "1")
11787 (set_attr "prefix" "evex")
11788 (set_attr "mode" "XI")])
11790 (define_expand "avx512dq_shuf_<shuffletype>64x2_mask"
11791 [(match_operand:VI8F_256 0 "register_operand")
11792 (match_operand:VI8F_256 1 "register_operand")
11793 (match_operand:VI8F_256 2 "nonimmediate_operand")
11794 (match_operand:SI 3 "const_0_to_3_operand")
11795 (match_operand:VI8F_256 4 "register_operand")
11796 (match_operand:QI 5 "register_operand")]
11799 int mask = INTVAL (operands[3]);
11800 emit_insn (gen_avx512dq_shuf_<shuffletype>64x2_1_mask
11801 (operands[0], operands[1], operands[2],
11802 GEN_INT (((mask >> 0) & 1) * 2 + 0),
11803 GEN_INT (((mask >> 0) & 1) * 2 + 1),
11804 GEN_INT (((mask >> 1) & 1) * 2 + 4),
11805 GEN_INT (((mask >> 1) & 1) * 2 + 5),
11806 operands[4], operands[5]));
11810 (define_insn "<mask_codefor>avx512dq_shuf_<shuffletype>64x2_1<mask_name>"
11811 [(set (match_operand:VI8F_256 0 "register_operand" "=v")
11812 (vec_select:VI8F_256
11813 (vec_concat:<ssedoublemode>
11814 (match_operand:VI8F_256 1 "register_operand" "v")
11815 (match_operand:VI8F_256 2 "nonimmediate_operand" "vm"))
11816 (parallel [(match_operand 3 "const_0_to_3_operand")
11817 (match_operand 4 "const_0_to_3_operand")
11818 (match_operand 5 "const_4_to_7_operand")
11819 (match_operand 6 "const_4_to_7_operand")])))]
11821 && (INTVAL (operands[3]) == (INTVAL (operands[4]) - 1)
11822 && INTVAL (operands[5]) == (INTVAL (operands[6]) - 1))"
11825 mask = INTVAL (operands[3]) / 2;
11826 mask |= (INTVAL (operands[5]) - 4) / 2 << 1;
11827 operands[3] = GEN_INT (mask);
11828 return "vshuf<shuffletype>64x2\t{%3, %2, %1, %0<mask_operand7>|%0<mask_operand7>, %1, %2, %3}";
11830 [(set_attr "type" "sselog")
11831 (set_attr "length_immediate" "1")
11832 (set_attr "prefix" "evex")
11833 (set_attr "mode" "XI")])
11835 (define_expand "avx512f_shuf_<shuffletype>64x2_mask"
11836 [(match_operand:V8FI 0 "register_operand")
11837 (match_operand:V8FI 1 "register_operand")
11838 (match_operand:V8FI 2 "nonimmediate_operand")
11839 (match_operand:SI 3 "const_0_to_255_operand")
11840 (match_operand:V8FI 4 "register_operand")
11841 (match_operand:QI 5 "register_operand")]
11844 int mask = INTVAL (operands[3]);
11845 emit_insn (gen_avx512f_shuf_<shuffletype>64x2_1_mask
11846 (operands[0], operands[1], operands[2],
11847 GEN_INT (((mask >> 0) & 3) * 2),
11848 GEN_INT (((mask >> 0) & 3) * 2 + 1),
11849 GEN_INT (((mask >> 2) & 3) * 2),
11850 GEN_INT (((mask >> 2) & 3) * 2 + 1),
11851 GEN_INT (((mask >> 4) & 3) * 2 + 8),
11852 GEN_INT (((mask >> 4) & 3) * 2 + 9),
11853 GEN_INT (((mask >> 6) & 3) * 2 + 8),
11854 GEN_INT (((mask >> 6) & 3) * 2 + 9),
11855 operands[4], operands[5]));
11859 (define_insn "avx512f_shuf_<shuffletype>64x2_1<mask_name>"
11860 [(set (match_operand:V8FI 0 "register_operand" "=v")
11862 (vec_concat:<ssedoublemode>
11863 (match_operand:V8FI 1 "register_operand" "v")
11864 (match_operand:V8FI 2 "nonimmediate_operand" "vm"))
11865 (parallel [(match_operand 3 "const_0_to_7_operand")
11866 (match_operand 4 "const_0_to_7_operand")
11867 (match_operand 5 "const_0_to_7_operand")
11868 (match_operand 6 "const_0_to_7_operand")
11869 (match_operand 7 "const_8_to_15_operand")
11870 (match_operand 8 "const_8_to_15_operand")
11871 (match_operand 9 "const_8_to_15_operand")
11872 (match_operand 10 "const_8_to_15_operand")])))]
11874 && (INTVAL (operands[3]) == (INTVAL (operands[4]) - 1)
11875 && INTVAL (operands[5]) == (INTVAL (operands[6]) - 1)
11876 && INTVAL (operands[7]) == (INTVAL (operands[8]) - 1)
11877 && INTVAL (operands[9]) == (INTVAL (operands[10]) - 1))"
11880 mask = INTVAL (operands[3]) / 2;
11881 mask |= INTVAL (operands[5]) / 2 << 2;
11882 mask |= (INTVAL (operands[7]) - 8) / 2 << 4;
11883 mask |= (INTVAL (operands[9]) - 8) / 2 << 6;
11884 operands[3] = GEN_INT (mask);
11886 return "vshuf<shuffletype>64x2\t{%3, %2, %1, %0<mask_operand11>|%0<mask_operand11>, %1, %2, %3}";
11888 [(set_attr "type" "sselog")
11889 (set_attr "length_immediate" "1")
11890 (set_attr "prefix" "evex")
11891 (set_attr "mode" "<sseinsnmode>")])
11893 (define_expand "avx512vl_shuf_<shuffletype>32x4_mask"
11894 [(match_operand:VI4F_256 0 "register_operand")
11895 (match_operand:VI4F_256 1 "register_operand")
11896 (match_operand:VI4F_256 2 "nonimmediate_operand")
11897 (match_operand:SI 3 "const_0_to_3_operand")
11898 (match_operand:VI4F_256 4 "register_operand")
11899 (match_operand:QI 5 "register_operand")]
11902 int mask = INTVAL (operands[3]);
11903 emit_insn (gen_avx512vl_shuf_<shuffletype>32x4_1_mask
11904 (operands[0], operands[1], operands[2],
11905 GEN_INT (((mask >> 0) & 1) * 4 + 0),
11906 GEN_INT (((mask >> 0) & 1) * 4 + 1),
11907 GEN_INT (((mask >> 0) & 1) * 4 + 2),
11908 GEN_INT (((mask >> 0) & 1) * 4 + 3),
11909 GEN_INT (((mask >> 1) & 1) * 4 + 8),
11910 GEN_INT (((mask >> 1) & 1) * 4 + 9),
11911 GEN_INT (((mask >> 1) & 1) * 4 + 10),
11912 GEN_INT (((mask >> 1) & 1) * 4 + 11),
11913 operands[4], operands[5]));
11917 (define_insn "<mask_codefor>avx512vl_shuf_<shuffletype>32x4_1<mask_name>"
11918 [(set (match_operand:VI4F_256 0 "register_operand" "=v")
11919 (vec_select:VI4F_256
11920 (vec_concat:<ssedoublemode>
11921 (match_operand:VI4F_256 1 "register_operand" "v")
11922 (match_operand:VI4F_256 2 "nonimmediate_operand" "vm"))
11923 (parallel [(match_operand 3 "const_0_to_7_operand")
11924 (match_operand 4 "const_0_to_7_operand")
11925 (match_operand 5 "const_0_to_7_operand")
11926 (match_operand 6 "const_0_to_7_operand")
11927 (match_operand 7 "const_8_to_15_operand")
11928 (match_operand 8 "const_8_to_15_operand")
11929 (match_operand 9 "const_8_to_15_operand")
11930 (match_operand 10 "const_8_to_15_operand")])))]
11932 && (INTVAL (operands[3]) == (INTVAL (operands[4]) - 1)
11933 && INTVAL (operands[3]) == (INTVAL (operands[5]) - 2)
11934 && INTVAL (operands[3]) == (INTVAL (operands[6]) - 3)
11935 && INTVAL (operands[7]) == (INTVAL (operands[8]) - 1)
11936 && INTVAL (operands[7]) == (INTVAL (operands[9]) - 2)
11937 && INTVAL (operands[7]) == (INTVAL (operands[10]) - 3))"
11940 mask = INTVAL (operands[3]) / 4;
11941 mask |= (INTVAL (operands[7]) - 8) / 4 << 1;
11942 operands[3] = GEN_INT (mask);
11944 return "vshuf<shuffletype>32x4\t{%3, %2, %1, %0<mask_operand11>|%0<mask_operand11>, %1, %2, %3}";
11946 [(set_attr "type" "sselog")
11947 (set_attr "length_immediate" "1")
11948 (set_attr "prefix" "evex")
11949 (set_attr "mode" "<sseinsnmode>")])
11951 (define_expand "avx512f_shuf_<shuffletype>32x4_mask"
11952 [(match_operand:V16FI 0 "register_operand")
11953 (match_operand:V16FI 1 "register_operand")
11954 (match_operand:V16FI 2 "nonimmediate_operand")
11955 (match_operand:SI 3 "const_0_to_255_operand")
11956 (match_operand:V16FI 4 "register_operand")
11957 (match_operand:HI 5 "register_operand")]
11960 int mask = INTVAL (operands[3]);
11961 emit_insn (gen_avx512f_shuf_<shuffletype>32x4_1_mask
11962 (operands[0], operands[1], operands[2],
11963 GEN_INT (((mask >> 0) & 3) * 4),
11964 GEN_INT (((mask >> 0) & 3) * 4 + 1),
11965 GEN_INT (((mask >> 0) & 3) * 4 + 2),
11966 GEN_INT (((mask >> 0) & 3) * 4 + 3),
11967 GEN_INT (((mask >> 2) & 3) * 4),
11968 GEN_INT (((mask >> 2) & 3) * 4 + 1),
11969 GEN_INT (((mask >> 2) & 3) * 4 + 2),
11970 GEN_INT (((mask >> 2) & 3) * 4 + 3),
11971 GEN_INT (((mask >> 4) & 3) * 4 + 16),
11972 GEN_INT (((mask >> 4) & 3) * 4 + 17),
11973 GEN_INT (((mask >> 4) & 3) * 4 + 18),
11974 GEN_INT (((mask >> 4) & 3) * 4 + 19),
11975 GEN_INT (((mask >> 6) & 3) * 4 + 16),
11976 GEN_INT (((mask >> 6) & 3) * 4 + 17),
11977 GEN_INT (((mask >> 6) & 3) * 4 + 18),
11978 GEN_INT (((mask >> 6) & 3) * 4 + 19),
11979 operands[4], operands[5]));
11983 (define_insn "avx512f_shuf_<shuffletype>32x4_1<mask_name>"
11984 [(set (match_operand:V16FI 0 "register_operand" "=v")
11986 (vec_concat:<ssedoublemode>
11987 (match_operand:V16FI 1 "register_operand" "v")
11988 (match_operand:V16FI 2 "nonimmediate_operand" "vm"))
11989 (parallel [(match_operand 3 "const_0_to_15_operand")
11990 (match_operand 4 "const_0_to_15_operand")
11991 (match_operand 5 "const_0_to_15_operand")
11992 (match_operand 6 "const_0_to_15_operand")
11993 (match_operand 7 "const_0_to_15_operand")
11994 (match_operand 8 "const_0_to_15_operand")
11995 (match_operand 9 "const_0_to_15_operand")
11996 (match_operand 10 "const_0_to_15_operand")
11997 (match_operand 11 "const_16_to_31_operand")
11998 (match_operand 12 "const_16_to_31_operand")
11999 (match_operand 13 "const_16_to_31_operand")
12000 (match_operand 14 "const_16_to_31_operand")
12001 (match_operand 15 "const_16_to_31_operand")
12002 (match_operand 16 "const_16_to_31_operand")
12003 (match_operand 17 "const_16_to_31_operand")
12004 (match_operand 18 "const_16_to_31_operand")])))]
12006 && (INTVAL (operands[3]) == (INTVAL (operands[4]) - 1)
12007 && INTVAL (operands[3]) == (INTVAL (operands[5]) - 2)
12008 && INTVAL (operands[3]) == (INTVAL (operands[6]) - 3)
12009 && INTVAL (operands[7]) == (INTVAL (operands[8]) - 1)
12010 && INTVAL (operands[7]) == (INTVAL (operands[9]) - 2)
12011 && INTVAL (operands[7]) == (INTVAL (operands[10]) - 3)
12012 && INTVAL (operands[11]) == (INTVAL (operands[12]) - 1)
12013 && INTVAL (operands[11]) == (INTVAL (operands[13]) - 2)
12014 && INTVAL (operands[11]) == (INTVAL (operands[14]) - 3)
12015 && INTVAL (operands[15]) == (INTVAL (operands[16]) - 1)
12016 && INTVAL (operands[15]) == (INTVAL (operands[17]) - 2)
12017 && INTVAL (operands[15]) == (INTVAL (operands[18]) - 3))"
12020 mask = INTVAL (operands[3]) / 4;
12021 mask |= INTVAL (operands[7]) / 4 << 2;
12022 mask |= (INTVAL (operands[11]) - 16) / 4 << 4;
12023 mask |= (INTVAL (operands[15]) - 16) / 4 << 6;
12024 operands[3] = GEN_INT (mask);
12026 return "vshuf<shuffletype>32x4\t{%3, %2, %1, %0<mask_operand19>|%0<mask_operand19>, %1, %2, %3}";
12028 [(set_attr "type" "sselog")
12029 (set_attr "length_immediate" "1")
12030 (set_attr "prefix" "evex")
12031 (set_attr "mode" "<sseinsnmode>")])
12033 (define_expand "avx512f_pshufdv3_mask"
12034 [(match_operand:V16SI 0 "register_operand")
12035 (match_operand:V16SI 1 "nonimmediate_operand")
12036 (match_operand:SI 2 "const_0_to_255_operand")
12037 (match_operand:V16SI 3 "register_operand")
12038 (match_operand:HI 4 "register_operand")]
12041 int mask = INTVAL (operands[2]);
12042 emit_insn (gen_avx512f_pshufd_1_mask (operands[0], operands[1],
12043 GEN_INT ((mask >> 0) & 3),
12044 GEN_INT ((mask >> 2) & 3),
12045 GEN_INT ((mask >> 4) & 3),
12046 GEN_INT ((mask >> 6) & 3),
12047 GEN_INT (((mask >> 0) & 3) + 4),
12048 GEN_INT (((mask >> 2) & 3) + 4),
12049 GEN_INT (((mask >> 4) & 3) + 4),
12050 GEN_INT (((mask >> 6) & 3) + 4),
12051 GEN_INT (((mask >> 0) & 3) + 8),
12052 GEN_INT (((mask >> 2) & 3) + 8),
12053 GEN_INT (((mask >> 4) & 3) + 8),
12054 GEN_INT (((mask >> 6) & 3) + 8),
12055 GEN_INT (((mask >> 0) & 3) + 12),
12056 GEN_INT (((mask >> 2) & 3) + 12),
12057 GEN_INT (((mask >> 4) & 3) + 12),
12058 GEN_INT (((mask >> 6) & 3) + 12),
12059 operands[3], operands[4]));
12063 (define_insn "avx512f_pshufd_1<mask_name>"
12064 [(set (match_operand:V16SI 0 "register_operand" "=v")
12066 (match_operand:V16SI 1 "nonimmediate_operand" "vm")
12067 (parallel [(match_operand 2 "const_0_to_3_operand")
12068 (match_operand 3 "const_0_to_3_operand")
12069 (match_operand 4 "const_0_to_3_operand")
12070 (match_operand 5 "const_0_to_3_operand")
12071 (match_operand 6 "const_4_to_7_operand")
12072 (match_operand 7 "const_4_to_7_operand")
12073 (match_operand 8 "const_4_to_7_operand")
12074 (match_operand 9 "const_4_to_7_operand")
12075 (match_operand 10 "const_8_to_11_operand")
12076 (match_operand 11 "const_8_to_11_operand")
12077 (match_operand 12 "const_8_to_11_operand")
12078 (match_operand 13 "const_8_to_11_operand")
12079 (match_operand 14 "const_12_to_15_operand")
12080 (match_operand 15 "const_12_to_15_operand")
12081 (match_operand 16 "const_12_to_15_operand")
12082 (match_operand 17 "const_12_to_15_operand")])))]
12084 && INTVAL (operands[2]) + 4 == INTVAL (operands[6])
12085 && INTVAL (operands[3]) + 4 == INTVAL (operands[7])
12086 && INTVAL (operands[4]) + 4 == INTVAL (operands[8])
12087 && INTVAL (operands[5]) + 4 == INTVAL (operands[9])
12088 && INTVAL (operands[2]) + 8 == INTVAL (operands[10])
12089 && INTVAL (operands[3]) + 8 == INTVAL (operands[11])
12090 && INTVAL (operands[4]) + 8 == INTVAL (operands[12])
12091 && INTVAL (operands[5]) + 8 == INTVAL (operands[13])
12092 && INTVAL (operands[2]) + 12 == INTVAL (operands[14])
12093 && INTVAL (operands[3]) + 12 == INTVAL (operands[15])
12094 && INTVAL (operands[4]) + 12 == INTVAL (operands[16])
12095 && INTVAL (operands[5]) + 12 == INTVAL (operands[17])"
12098 mask |= INTVAL (operands[2]) << 0;
12099 mask |= INTVAL (operands[3]) << 2;
12100 mask |= INTVAL (operands[4]) << 4;
12101 mask |= INTVAL (operands[5]) << 6;
12102 operands[2] = GEN_INT (mask);
12104 return "vpshufd\t{%2, %1, %0<mask_operand18>|%0<mask_operand18>, %1, %2}";
12106 [(set_attr "type" "sselog1")
12107 (set_attr "prefix" "evex")
12108 (set_attr "length_immediate" "1")
12109 (set_attr "mode" "XI")])
12111 (define_expand "avx512vl_pshufdv3_mask"
12112 [(match_operand:V8SI 0 "register_operand")
12113 (match_operand:V8SI 1 "nonimmediate_operand")
12114 (match_operand:SI 2 "const_0_to_255_operand")
12115 (match_operand:V8SI 3 "register_operand")
12116 (match_operand:QI 4 "register_operand")]
12119 int mask = INTVAL (operands[2]);
12120 emit_insn (gen_avx2_pshufd_1_mask (operands[0], operands[1],
12121 GEN_INT ((mask >> 0) & 3),
12122 GEN_INT ((mask >> 2) & 3),
12123 GEN_INT ((mask >> 4) & 3),
12124 GEN_INT ((mask >> 6) & 3),
12125 GEN_INT (((mask >> 0) & 3) + 4),
12126 GEN_INT (((mask >> 2) & 3) + 4),
12127 GEN_INT (((mask >> 4) & 3) + 4),
12128 GEN_INT (((mask >> 6) & 3) + 4),
12129 operands[3], operands[4]));
12133 (define_expand "avx2_pshufdv3"
12134 [(match_operand:V8SI 0 "register_operand")
12135 (match_operand:V8SI 1 "nonimmediate_operand")
12136 (match_operand:SI 2 "const_0_to_255_operand")]
12139 int mask = INTVAL (operands[2]);
12140 emit_insn (gen_avx2_pshufd_1 (operands[0], operands[1],
12141 GEN_INT ((mask >> 0) & 3),
12142 GEN_INT ((mask >> 2) & 3),
12143 GEN_INT ((mask >> 4) & 3),
12144 GEN_INT ((mask >> 6) & 3),
12145 GEN_INT (((mask >> 0) & 3) + 4),
12146 GEN_INT (((mask >> 2) & 3) + 4),
12147 GEN_INT (((mask >> 4) & 3) + 4),
12148 GEN_INT (((mask >> 6) & 3) + 4)));
12152 (define_insn "avx2_pshufd_1<mask_name>"
12153 [(set (match_operand:V8SI 0 "register_operand" "=v")
12155 (match_operand:V8SI 1 "nonimmediate_operand" "vm")
12156 (parallel [(match_operand 2 "const_0_to_3_operand")
12157 (match_operand 3 "const_0_to_3_operand")
12158 (match_operand 4 "const_0_to_3_operand")
12159 (match_operand 5 "const_0_to_3_operand")
12160 (match_operand 6 "const_4_to_7_operand")
12161 (match_operand 7 "const_4_to_7_operand")
12162 (match_operand 8 "const_4_to_7_operand")
12163 (match_operand 9 "const_4_to_7_operand")])))]
12165 && <mask_avx512vl_condition>
12166 && INTVAL (operands[2]) + 4 == INTVAL (operands[6])
12167 && INTVAL (operands[3]) + 4 == INTVAL (operands[7])
12168 && INTVAL (operands[4]) + 4 == INTVAL (operands[8])
12169 && INTVAL (operands[5]) + 4 == INTVAL (operands[9])"
12172 mask |= INTVAL (operands[2]) << 0;
12173 mask |= INTVAL (operands[3]) << 2;
12174 mask |= INTVAL (operands[4]) << 4;
12175 mask |= INTVAL (operands[5]) << 6;
12176 operands[2] = GEN_INT (mask);
12178 return "vpshufd\t{%2, %1, %0<mask_operand10>|%0<mask_operand10>, %1, %2}";
12180 [(set_attr "type" "sselog1")
12181 (set_attr "prefix" "maybe_evex")
12182 (set_attr "length_immediate" "1")
12183 (set_attr "mode" "OI")])
12185 (define_expand "avx512vl_pshufd_mask"
12186 [(match_operand:V4SI 0 "register_operand")
12187 (match_operand:V4SI 1 "nonimmediate_operand")
12188 (match_operand:SI 2 "const_0_to_255_operand")
12189 (match_operand:V4SI 3 "register_operand")
12190 (match_operand:QI 4 "register_operand")]
12193 int mask = INTVAL (operands[2]);
12194 emit_insn (gen_sse2_pshufd_1_mask (operands[0], operands[1],
12195 GEN_INT ((mask >> 0) & 3),
12196 GEN_INT ((mask >> 2) & 3),
12197 GEN_INT ((mask >> 4) & 3),
12198 GEN_INT ((mask >> 6) & 3),
12199 operands[3], operands[4]));
12203 (define_expand "sse2_pshufd"
12204 [(match_operand:V4SI 0 "register_operand")
12205 (match_operand:V4SI 1 "nonimmediate_operand")
12206 (match_operand:SI 2 "const_int_operand")]
12209 int mask = INTVAL (operands[2]);
12210 emit_insn (gen_sse2_pshufd_1 (operands[0], operands[1],
12211 GEN_INT ((mask >> 0) & 3),
12212 GEN_INT ((mask >> 2) & 3),
12213 GEN_INT ((mask >> 4) & 3),
12214 GEN_INT ((mask >> 6) & 3)));
12218 (define_insn "sse2_pshufd_1<mask_name>"
12219 [(set (match_operand:V4SI 0 "register_operand" "=v")
12221 (match_operand:V4SI 1 "nonimmediate_operand" "vm")
12222 (parallel [(match_operand 2 "const_0_to_3_operand")
12223 (match_operand 3 "const_0_to_3_operand")
12224 (match_operand 4 "const_0_to_3_operand")
12225 (match_operand 5 "const_0_to_3_operand")])))]
12226 "TARGET_SSE2 && <mask_avx512vl_condition>"
12229 mask |= INTVAL (operands[2]) << 0;
12230 mask |= INTVAL (operands[3]) << 2;
12231 mask |= INTVAL (operands[4]) << 4;
12232 mask |= INTVAL (operands[5]) << 6;
12233 operands[2] = GEN_INT (mask);
12235 return "%vpshufd\t{%2, %1, %0<mask_operand6>|%0<mask_operand6>, %1, %2}";
12237 [(set_attr "type" "sselog1")
12238 (set_attr "prefix_data16" "1")
12239 (set_attr "prefix" "<mask_prefix2>")
12240 (set_attr "length_immediate" "1")
12241 (set_attr "mode" "TI")])
12243 (define_insn "<mask_codefor>avx512bw_pshuflwv32hi<mask_name>"
12244 [(set (match_operand:V32HI 0 "register_operand" "=v")
12246 [(match_operand:V32HI 1 "nonimmediate_operand" "vm")
12247 (match_operand:SI 2 "const_0_to_255_operand" "n")]
12250 "vpshuflw\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
12251 [(set_attr "type" "sselog")
12252 (set_attr "prefix" "evex")
12253 (set_attr "mode" "XI")])
12255 (define_expand "avx512vl_pshuflwv3_mask"
12256 [(match_operand:V16HI 0 "register_operand")
12257 (match_operand:V16HI 1 "nonimmediate_operand")
12258 (match_operand:SI 2 "const_0_to_255_operand")
12259 (match_operand:V16HI 3 "register_operand")
12260 (match_operand:HI 4 "register_operand")]
12261 "TARGET_AVX512VL && TARGET_AVX512BW"
12263 int mask = INTVAL (operands[2]);
12264 emit_insn (gen_avx2_pshuflw_1_mask (operands[0], operands[1],
12265 GEN_INT ((mask >> 0) & 3),
12266 GEN_INT ((mask >> 2) & 3),
12267 GEN_INT ((mask >> 4) & 3),
12268 GEN_INT ((mask >> 6) & 3),
12269 GEN_INT (((mask >> 0) & 3) + 8),
12270 GEN_INT (((mask >> 2) & 3) + 8),
12271 GEN_INT (((mask >> 4) & 3) + 8),
12272 GEN_INT (((mask >> 6) & 3) + 8),
12273 operands[3], operands[4]));
12277 (define_expand "avx2_pshuflwv3"
12278 [(match_operand:V16HI 0 "register_operand")
12279 (match_operand:V16HI 1 "nonimmediate_operand")
12280 (match_operand:SI 2 "const_0_to_255_operand")]
12283 int mask = INTVAL (operands[2]);
12284 emit_insn (gen_avx2_pshuflw_1 (operands[0], operands[1],
12285 GEN_INT ((mask >> 0) & 3),
12286 GEN_INT ((mask >> 2) & 3),
12287 GEN_INT ((mask >> 4) & 3),
12288 GEN_INT ((mask >> 6) & 3),
12289 GEN_INT (((mask >> 0) & 3) + 8),
12290 GEN_INT (((mask >> 2) & 3) + 8),
12291 GEN_INT (((mask >> 4) & 3) + 8),
12292 GEN_INT (((mask >> 6) & 3) + 8)));
12296 (define_insn "avx2_pshuflw_1<mask_name>"
12297 [(set (match_operand:V16HI 0 "register_operand" "=v")
12299 (match_operand:V16HI 1 "nonimmediate_operand" "vm")
12300 (parallel [(match_operand 2 "const_0_to_3_operand")
12301 (match_operand 3 "const_0_to_3_operand")
12302 (match_operand 4 "const_0_to_3_operand")
12303 (match_operand 5 "const_0_to_3_operand")
12308 (match_operand 6 "const_8_to_11_operand")
12309 (match_operand 7 "const_8_to_11_operand")
12310 (match_operand 8 "const_8_to_11_operand")
12311 (match_operand 9 "const_8_to_11_operand")
12315 (const_int 15)])))]
12317 && <mask_avx512bw_condition> && <mask_avx512vl_condition>
12318 && INTVAL (operands[2]) + 8 == INTVAL (operands[6])
12319 && INTVAL (operands[3]) + 8 == INTVAL (operands[7])
12320 && INTVAL (operands[4]) + 8 == INTVAL (operands[8])
12321 && INTVAL (operands[5]) + 8 == INTVAL (operands[9])"
12324 mask |= INTVAL (operands[2]) << 0;
12325 mask |= INTVAL (operands[3]) << 2;
12326 mask |= INTVAL (operands[4]) << 4;
12327 mask |= INTVAL (operands[5]) << 6;
12328 operands[2] = GEN_INT (mask);
12330 return "vpshuflw\t{%2, %1, %0<mask_operand10>|%0<mask_operand10>, %1, %2}";
12332 [(set_attr "type" "sselog")
12333 (set_attr "prefix" "maybe_evex")
12334 (set_attr "length_immediate" "1")
12335 (set_attr "mode" "OI")])
12337 (define_expand "avx512vl_pshuflw_mask"
12338 [(match_operand:V8HI 0 "register_operand")
12339 (match_operand:V8HI 1 "nonimmediate_operand")
12340 (match_operand:SI 2 "const_0_to_255_operand")
12341 (match_operand:V8HI 3 "register_operand")
12342 (match_operand:QI 4 "register_operand")]
12343 "TARGET_AVX512VL && TARGET_AVX512BW"
12345 int mask = INTVAL (operands[2]);
12346 emit_insn (gen_sse2_pshuflw_1_mask (operands[0], operands[1],
12347 GEN_INT ((mask >> 0) & 3),
12348 GEN_INT ((mask >> 2) & 3),
12349 GEN_INT ((mask >> 4) & 3),
12350 GEN_INT ((mask >> 6) & 3),
12351 operands[3], operands[4]));
12355 (define_expand "sse2_pshuflw"
12356 [(match_operand:V8HI 0 "register_operand")
12357 (match_operand:V8HI 1 "nonimmediate_operand")
12358 (match_operand:SI 2 "const_int_operand")]
12361 int mask = INTVAL (operands[2]);
12362 emit_insn (gen_sse2_pshuflw_1 (operands[0], operands[1],
12363 GEN_INT ((mask >> 0) & 3),
12364 GEN_INT ((mask >> 2) & 3),
12365 GEN_INT ((mask >> 4) & 3),
12366 GEN_INT ((mask >> 6) & 3)));
12370 (define_insn "sse2_pshuflw_1<mask_name>"
12371 [(set (match_operand:V8HI 0 "register_operand" "=v")
12373 (match_operand:V8HI 1 "nonimmediate_operand" "vm")
12374 (parallel [(match_operand 2 "const_0_to_3_operand")
12375 (match_operand 3 "const_0_to_3_operand")
12376 (match_operand 4 "const_0_to_3_operand")
12377 (match_operand 5 "const_0_to_3_operand")
12382 "TARGET_SSE2 && <mask_avx512bw_condition> && <mask_avx512vl_condition>"
12385 mask |= INTVAL (operands[2]) << 0;
12386 mask |= INTVAL (operands[3]) << 2;
12387 mask |= INTVAL (operands[4]) << 4;
12388 mask |= INTVAL (operands[5]) << 6;
12389 operands[2] = GEN_INT (mask);
12391 return "%vpshuflw\t{%2, %1, %0<mask_operand6>|%0<mask_operand6>, %1, %2}";
12393 [(set_attr "type" "sselog")
12394 (set_attr "prefix_data16" "0")
12395 (set_attr "prefix_rep" "1")
12396 (set_attr "prefix" "maybe_vex")
12397 (set_attr "length_immediate" "1")
12398 (set_attr "mode" "TI")])
12400 (define_expand "avx2_pshufhwv3"
12401 [(match_operand:V16HI 0 "register_operand")
12402 (match_operand:V16HI 1 "nonimmediate_operand")
12403 (match_operand:SI 2 "const_0_to_255_operand")]
12406 int mask = INTVAL (operands[2]);
12407 emit_insn (gen_avx2_pshufhw_1 (operands[0], operands[1],
12408 GEN_INT (((mask >> 0) & 3) + 4),
12409 GEN_INT (((mask >> 2) & 3) + 4),
12410 GEN_INT (((mask >> 4) & 3) + 4),
12411 GEN_INT (((mask >> 6) & 3) + 4),
12412 GEN_INT (((mask >> 0) & 3) + 12),
12413 GEN_INT (((mask >> 2) & 3) + 12),
12414 GEN_INT (((mask >> 4) & 3) + 12),
12415 GEN_INT (((mask >> 6) & 3) + 12)));
12419 (define_insn "<mask_codefor>avx512bw_pshufhwv32hi<mask_name>"
12420 [(set (match_operand:V32HI 0 "register_operand" "=v")
12422 [(match_operand:V32HI 1 "nonimmediate_operand" "vm")
12423 (match_operand:SI 2 "const_0_to_255_operand" "n")]
12426 "vpshufhw\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
12427 [(set_attr "type" "sselog")
12428 (set_attr "prefix" "evex")
12429 (set_attr "mode" "XI")])
12431 (define_expand "avx512vl_pshufhwv3_mask"
12432 [(match_operand:V16HI 0 "register_operand")
12433 (match_operand:V16HI 1 "nonimmediate_operand")
12434 (match_operand:SI 2 "const_0_to_255_operand")
12435 (match_operand:V16HI 3 "register_operand")
12436 (match_operand:HI 4 "register_operand")]
12437 "TARGET_AVX512VL && TARGET_AVX512BW"
12439 int mask = INTVAL (operands[2]);
12440 emit_insn (gen_avx2_pshufhw_1_mask (operands[0], operands[1],
12441 GEN_INT (((mask >> 0) & 3) + 4),
12442 GEN_INT (((mask >> 2) & 3) + 4),
12443 GEN_INT (((mask >> 4) & 3) + 4),
12444 GEN_INT (((mask >> 6) & 3) + 4),
12445 GEN_INT (((mask >> 0) & 3) + 12),
12446 GEN_INT (((mask >> 2) & 3) + 12),
12447 GEN_INT (((mask >> 4) & 3) + 12),
12448 GEN_INT (((mask >> 6) & 3) + 12),
12449 operands[3], operands[4]));
12453 (define_insn "avx2_pshufhw_1<mask_name>"
12454 [(set (match_operand:V16HI 0 "register_operand" "=v")
12456 (match_operand:V16HI 1 "nonimmediate_operand" "vm")
12457 (parallel [(const_int 0)
12461 (match_operand 2 "const_4_to_7_operand")
12462 (match_operand 3 "const_4_to_7_operand")
12463 (match_operand 4 "const_4_to_7_operand")
12464 (match_operand 5 "const_4_to_7_operand")
12469 (match_operand 6 "const_12_to_15_operand")
12470 (match_operand 7 "const_12_to_15_operand")
12471 (match_operand 8 "const_12_to_15_operand")
12472 (match_operand 9 "const_12_to_15_operand")])))]
12474 && <mask_avx512bw_condition> && <mask_avx512vl_condition>
12475 && INTVAL (operands[2]) + 8 == INTVAL (operands[6])
12476 && INTVAL (operands[3]) + 8 == INTVAL (operands[7])
12477 && INTVAL (operands[4]) + 8 == INTVAL (operands[8])
12478 && INTVAL (operands[5]) + 8 == INTVAL (operands[9])"
12481 mask |= (INTVAL (operands[2]) - 4) << 0;
12482 mask |= (INTVAL (operands[3]) - 4) << 2;
12483 mask |= (INTVAL (operands[4]) - 4) << 4;
12484 mask |= (INTVAL (operands[5]) - 4) << 6;
12485 operands[2] = GEN_INT (mask);
12487 return "vpshufhw\t{%2, %1, %0<mask_operand10>|%0<mask_operand10>, %1, %2}";
12489 [(set_attr "type" "sselog")
12490 (set_attr "prefix" "maybe_evex")
12491 (set_attr "length_immediate" "1")
12492 (set_attr "mode" "OI")])
12494 (define_expand "avx512vl_pshufhw_mask"
12495 [(match_operand:V8HI 0 "register_operand")
12496 (match_operand:V8HI 1 "nonimmediate_operand")
12497 (match_operand:SI 2 "const_0_to_255_operand")
12498 (match_operand:V8HI 3 "register_operand")
12499 (match_operand:QI 4 "register_operand")]
12500 "TARGET_AVX512VL && TARGET_AVX512BW"
12502 int mask = INTVAL (operands[2]);
12503 emit_insn (gen_sse2_pshufhw_1_mask (operands[0], operands[1],
12504 GEN_INT (((mask >> 0) & 3) + 4),
12505 GEN_INT (((mask >> 2) & 3) + 4),
12506 GEN_INT (((mask >> 4) & 3) + 4),
12507 GEN_INT (((mask >> 6) & 3) + 4),
12508 operands[3], operands[4]));
12512 (define_expand "sse2_pshufhw"
12513 [(match_operand:V8HI 0 "register_operand")
12514 (match_operand:V8HI 1 "nonimmediate_operand")
12515 (match_operand:SI 2 "const_int_operand")]
12518 int mask = INTVAL (operands[2]);
12519 emit_insn (gen_sse2_pshufhw_1 (operands[0], operands[1],
12520 GEN_INT (((mask >> 0) & 3) + 4),
12521 GEN_INT (((mask >> 2) & 3) + 4),
12522 GEN_INT (((mask >> 4) & 3) + 4),
12523 GEN_INT (((mask >> 6) & 3) + 4)));
12527 (define_insn "sse2_pshufhw_1<mask_name>"
12528 [(set (match_operand:V8HI 0 "register_operand" "=v")
12530 (match_operand:V8HI 1 "nonimmediate_operand" "vm")
12531 (parallel [(const_int 0)
12535 (match_operand 2 "const_4_to_7_operand")
12536 (match_operand 3 "const_4_to_7_operand")
12537 (match_operand 4 "const_4_to_7_operand")
12538 (match_operand 5 "const_4_to_7_operand")])))]
12539 "TARGET_SSE2 && <mask_avx512bw_condition> && <mask_avx512vl_condition>"
12542 mask |= (INTVAL (operands[2]) - 4) << 0;
12543 mask |= (INTVAL (operands[3]) - 4) << 2;
12544 mask |= (INTVAL (operands[4]) - 4) << 4;
12545 mask |= (INTVAL (operands[5]) - 4) << 6;
12546 operands[2] = GEN_INT (mask);
12548 return "%vpshufhw\t{%2, %1, %0<mask_operand6>|%0<mask_operand6>, %1, %2}";
12550 [(set_attr "type" "sselog")
12551 (set_attr "prefix_rep" "1")
12552 (set_attr "prefix_data16" "0")
12553 (set_attr "prefix" "maybe_vex")
12554 (set_attr "length_immediate" "1")
12555 (set_attr "mode" "TI")])
12557 (define_expand "sse2_loadd"
12558 [(set (match_operand:V4SI 0 "register_operand")
12560 (vec_duplicate:V4SI
12561 (match_operand:SI 1 "nonimmediate_operand"))
12565 "operands[2] = CONST0_RTX (V4SImode);")
12567 (define_insn "sse2_loadld"
12568 [(set (match_operand:V4SI 0 "register_operand" "=x,Yi,x,x,x")
12570 (vec_duplicate:V4SI
12571 (match_operand:SI 2 "nonimmediate_operand" "m ,r ,m,x,x"))
12572 (match_operand:V4SI 1 "reg_or_0_operand" "C ,C ,C,0,x")
12576 %vmovd\t{%2, %0|%0, %2}
12577 %vmovd\t{%2, %0|%0, %2}
12578 movss\t{%2, %0|%0, %2}
12579 movss\t{%2, %0|%0, %2}
12580 vmovss\t{%2, %1, %0|%0, %1, %2}"
12581 [(set_attr "isa" "sse2,*,noavx,noavx,avx")
12582 (set_attr "type" "ssemov")
12583 (set_attr "prefix" "maybe_vex,maybe_vex,orig,orig,vex")
12584 (set_attr "mode" "TI,TI,V4SF,SF,SF")])
12586 (define_insn "*vec_extract<mode>"
12587 [(set (match_operand:<ssescalarmode> 0 "nonimmediate_operand" "=r,m")
12588 (vec_select:<ssescalarmode>
12589 (match_operand:VI12_128 1 "register_operand" "x,x")
12591 [(match_operand:SI 2 "const_0_to_<ssescalarnummask>_operand")])))]
12594 %vpextr<ssemodesuffix>\t{%2, %1, %k0|%k0, %1, %2}
12595 %vpextr<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
12596 [(set_attr "type" "sselog1")
12597 (set (attr "prefix_data16")
12599 (and (eq_attr "alternative" "0")
12600 (eq (const_string "<MODE>mode") (const_string "V8HImode")))
12602 (const_string "*")))
12603 (set (attr "prefix_extra")
12605 (and (eq_attr "alternative" "0")
12606 (eq (const_string "<MODE>mode") (const_string "V8HImode")))
12608 (const_string "1")))
12609 (set_attr "length_immediate" "1")
12610 (set_attr "prefix" "maybe_vex")
12611 (set_attr "mode" "TI")])
12613 (define_insn "*vec_extractv8hi_sse2"
12614 [(set (match_operand:HI 0 "register_operand" "=r")
12616 (match_operand:V8HI 1 "register_operand" "x")
12618 [(match_operand:SI 2 "const_0_to_7_operand")])))]
12619 "TARGET_SSE2 && !TARGET_SSE4_1"
12620 "pextrw\t{%2, %1, %k0|%k0, %1, %2}"
12621 [(set_attr "type" "sselog1")
12622 (set_attr "prefix_data16" "1")
12623 (set_attr "length_immediate" "1")
12624 (set_attr "mode" "TI")])
12626 (define_insn "*vec_extractv16qi_zext"
12627 [(set (match_operand:SWI48 0 "register_operand" "=r")
12630 (match_operand:V16QI 1 "register_operand" "x")
12632 [(match_operand:SI 2 "const_0_to_15_operand")]))))]
12634 "%vpextrb\t{%2, %1, %k0|%k0, %1, %2}"
12635 [(set_attr "type" "sselog1")
12636 (set_attr "prefix_extra" "1")
12637 (set_attr "length_immediate" "1")
12638 (set_attr "prefix" "maybe_vex")
12639 (set_attr "mode" "TI")])
12641 (define_insn "*vec_extractv8hi_zext"
12642 [(set (match_operand:SWI48 0 "register_operand" "=r")
12645 (match_operand:V8HI 1 "register_operand" "x")
12647 [(match_operand:SI 2 "const_0_to_7_operand")]))))]
12649 "%vpextrw\t{%2, %1, %k0|%k0, %1, %2}"
12650 [(set_attr "type" "sselog1")
12651 (set_attr "prefix_data16" "1")
12652 (set_attr "length_immediate" "1")
12653 (set_attr "prefix" "maybe_vex")
12654 (set_attr "mode" "TI")])
12656 (define_insn "*vec_extract<mode>_mem"
12657 [(set (match_operand:<ssescalarmode> 0 "register_operand" "=r")
12658 (vec_select:<ssescalarmode>
12659 (match_operand:VI12_128 1 "memory_operand" "o")
12661 [(match_operand 2 "const_0_to_<ssescalarnummask>_operand")])))]
12665 (define_insn "*vec_extract<ssevecmodelower>_0"
12666 [(set (match_operand:SWI48 0 "nonimmediate_operand" "=r ,r,x ,m")
12668 (match_operand:<ssevecmode> 1 "nonimmediate_operand" "mYj,x,xm,x")
12669 (parallel [(const_int 0)])))]
12670 "TARGET_SSE && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
12672 [(set_attr "isa" "*,sse4,*,*")])
12674 (define_insn_and_split "*vec_extractv4si_0_zext"
12675 [(set (match_operand:DI 0 "register_operand" "=r")
12678 (match_operand:V4SI 1 "register_operand" "x")
12679 (parallel [(const_int 0)]))))]
12680 "TARGET_64BIT && TARGET_SSE2 && TARGET_INTER_UNIT_MOVES_FROM_VEC"
12682 "&& reload_completed"
12683 [(set (match_dup 0) (zero_extend:DI (match_dup 1)))]
12684 "operands[1] = gen_rtx_REG (SImode, REGNO (operands[1]));")
12686 (define_insn "*vec_extractv2di_0_sse"
12687 [(set (match_operand:DI 0 "nonimmediate_operand" "=x,m")
12689 (match_operand:V2DI 1 "nonimmediate_operand" "xm,x")
12690 (parallel [(const_int 0)])))]
12691 "TARGET_SSE && !TARGET_64BIT
12692 && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
12696 [(set (match_operand:SWI48x 0 "nonimmediate_operand")
12698 (match_operand:<ssevecmode> 1 "register_operand")
12699 (parallel [(const_int 0)])))]
12700 "TARGET_SSE && reload_completed"
12701 [(set (match_dup 0) (match_dup 1))]
12702 "operands[1] = gen_rtx_REG (<MODE>mode, REGNO (operands[1]));")
12704 (define_insn "*vec_extractv4si"
12705 [(set (match_operand:SI 0 "nonimmediate_operand" "=rm,Yr,*x,x")
12707 (match_operand:V4SI 1 "register_operand" "x,0,0,x")
12708 (parallel [(match_operand:SI 2 "const_0_to_3_operand")])))]
12711 switch (which_alternative)
12714 return "%vpextrd\t{%2, %1, %0|%0, %1, %2}";
12718 operands [2] = GEN_INT (INTVAL (operands[2]) * 4);
12719 return "psrldq\t{%2, %0|%0, %2}";
12722 operands [2] = GEN_INT (INTVAL (operands[2]) * 4);
12723 return "vpsrldq\t{%2, %1, %0|%0, %1, %2}";
12726 gcc_unreachable ();
12729 [(set_attr "isa" "*,noavx,noavx,avx")
12730 (set_attr "type" "sselog1,sseishft1,sseishft1,sseishft1")
12731 (set_attr "prefix_extra" "1,*,*,*")
12732 (set_attr "length_immediate" "1")
12733 (set_attr "prefix" "maybe_vex,orig,orig,vex")
12734 (set_attr "mode" "TI")])
12736 (define_insn "*vec_extractv4si_zext"
12737 [(set (match_operand:DI 0 "register_operand" "=r")
12740 (match_operand:V4SI 1 "register_operand" "x")
12741 (parallel [(match_operand:SI 2 "const_0_to_3_operand")]))))]
12742 "TARGET_64BIT && TARGET_SSE4_1"
12743 "%vpextrd\t{%2, %1, %k0|%k0, %1, %2}"
12744 [(set_attr "type" "sselog1")
12745 (set_attr "prefix_extra" "1")
12746 (set_attr "length_immediate" "1")
12747 (set_attr "prefix" "maybe_vex")
12748 (set_attr "mode" "TI")])
12750 (define_insn "*vec_extractv4si_mem"
12751 [(set (match_operand:SI 0 "register_operand" "=x,r")
12753 (match_operand:V4SI 1 "memory_operand" "o,o")
12754 (parallel [(match_operand 2 "const_0_to_3_operand")])))]
12758 (define_insn_and_split "*vec_extractv4si_zext_mem"
12759 [(set (match_operand:DI 0 "register_operand" "=x,r")
12762 (match_operand:V4SI 1 "memory_operand" "o,o")
12763 (parallel [(match_operand:SI 2 "const_0_to_3_operand")]))))]
12764 "TARGET_64BIT && TARGET_SSE"
12766 "&& reload_completed"
12767 [(set (match_dup 0) (zero_extend:DI (match_dup 1)))]
12769 operands[1] = adjust_address (operands[1], SImode, INTVAL (operands[2]) * 4);
12772 (define_insn "*vec_extractv2di_1"
12773 [(set (match_operand:DI 0 "nonimmediate_operand" "=rm,m,x,x,x,x,r")
12775 (match_operand:V2DI 1 "nonimmediate_operand" "x ,x,0,x,x,o,o")
12776 (parallel [(const_int 1)])))]
12777 "TARGET_SSE && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
12779 %vpextrq\t{$1, %1, %0|%0, %1, 1}
12780 %vmovhps\t{%1, %0|%0, %1}
12781 psrldq\t{$8, %0|%0, 8}
12782 vpsrldq\t{$8, %1, %0|%0, %1, 8}
12783 movhlps\t{%1, %0|%0, %1}
12786 [(set_attr "isa" "x64_sse4,*,sse2_noavx,avx,noavx,*,x64")
12787 (set_attr "type" "sselog1,ssemov,sseishft1,sseishft1,ssemov,ssemov,imov")
12788 (set_attr "length_immediate" "1,*,1,1,*,*,*")
12789 (set_attr "prefix_rex" "1,*,*,*,*,*,*")
12790 (set_attr "prefix_extra" "1,*,*,*,*,*,*")
12791 (set_attr "prefix" "maybe_vex,maybe_vex,orig,vex,orig,*,*")
12792 (set_attr "mode" "TI,V2SF,TI,TI,V4SF,DI,DI")])
12795 [(set (match_operand:<ssescalarmode> 0 "register_operand")
12796 (vec_select:<ssescalarmode>
12797 (match_operand:VI_128 1 "memory_operand")
12799 [(match_operand 2 "const_0_to_<ssescalarnummask>_operand")])))]
12800 "TARGET_SSE && reload_completed"
12801 [(set (match_dup 0) (match_dup 1))]
12803 int offs = INTVAL (operands[2]) * GET_MODE_SIZE (<ssescalarmode>mode);
12805 operands[1] = adjust_address (operands[1], <ssescalarmode>mode, offs);
12808 (define_insn "*vec_concatv2si_sse4_1"
12809 [(set (match_operand:V2SI 0 "register_operand" "=Yr,*x,x, Yr,*x,x, x, *y,*y")
12811 (match_operand:SI 1 "nonimmediate_operand" " 0, 0,x, 0,0, x,rm, 0,rm")
12812 (match_operand:SI 2 "vector_move_operand" " rm,rm,rm,Yr,*x,x, C,*ym, C")))]
12815 pinsrd\t{$1, %2, %0|%0, %2, 1}
12816 pinsrd\t{$1, %2, %0|%0, %2, 1}
12817 vpinsrd\t{$1, %2, %1, %0|%0, %1, %2, 1}
12818 punpckldq\t{%2, %0|%0, %2}
12819 punpckldq\t{%2, %0|%0, %2}
12820 vpunpckldq\t{%2, %1, %0|%0, %1, %2}
12821 %vmovd\t{%1, %0|%0, %1}
12822 punpckldq\t{%2, %0|%0, %2}
12823 movd\t{%1, %0|%0, %1}"
12824 [(set_attr "isa" "noavx,noavx,avx,noavx,noavx,avx,*,*,*")
12825 (set_attr "type" "sselog,sselog,sselog,sselog,sselog,sselog,ssemov,mmxcvt,mmxmov")
12826 (set_attr "prefix_extra" "1,1,1,*,*,*,*,*,*")
12827 (set_attr "length_immediate" "1,1,1,*,*,*,*,*,*")
12828 (set_attr "prefix" "orig,orig,vex,orig,orig,vex,maybe_vex,orig,orig")
12829 (set_attr "mode" "TI,TI,TI,TI,TI,TI,TI,DI,DI")])
12831 ;; ??? In theory we can match memory for the MMX alternative, but allowing
12832 ;; nonimmediate_operand for operand 2 and *not* allowing memory for the SSE
12833 ;; alternatives pretty much forces the MMX alternative to be chosen.
12834 (define_insn "*vec_concatv2si"
12835 [(set (match_operand:V2SI 0 "register_operand" "=x,x ,*y,x,x,*y,*y")
12837 (match_operand:SI 1 "nonimmediate_operand" " 0,rm,rm,0,m, 0,*rm")
12838 (match_operand:SI 2 "reg_or_0_operand" " x,C ,C, x,C,*y,C")))]
12839 "TARGET_SSE && !TARGET_SSE4_1"
12841 punpckldq\t{%2, %0|%0, %2}
12842 movd\t{%1, %0|%0, %1}
12843 movd\t{%1, %0|%0, %1}
12844 unpcklps\t{%2, %0|%0, %2}
12845 movss\t{%1, %0|%0, %1}
12846 punpckldq\t{%2, %0|%0, %2}
12847 movd\t{%1, %0|%0, %1}"
12848 [(set_attr "isa" "sse2,sse2,sse2,*,*,*,*")
12849 (set_attr "type" "sselog,ssemov,mmxmov,sselog,ssemov,mmxcvt,mmxmov")
12850 (set_attr "mode" "TI,TI,DI,V4SF,SF,DI,DI")])
12852 (define_insn "*vec_concatv4si"
12853 [(set (match_operand:V4SI 0 "register_operand" "=x,x,x,x,x")
12855 (match_operand:V2SI 1 "register_operand" " 0,x,0,0,x")
12856 (match_operand:V2SI 2 "nonimmediate_operand" " x,x,x,m,m")))]
12859 punpcklqdq\t{%2, %0|%0, %2}
12860 vpunpcklqdq\t{%2, %1, %0|%0, %1, %2}
12861 movlhps\t{%2, %0|%0, %2}
12862 movhps\t{%2, %0|%0, %q2}
12863 vmovhps\t{%2, %1, %0|%0, %1, %q2}"
12864 [(set_attr "isa" "sse2_noavx,avx,noavx,noavx,avx")
12865 (set_attr "type" "sselog,sselog,ssemov,ssemov,ssemov")
12866 (set_attr "prefix" "orig,vex,orig,orig,vex")
12867 (set_attr "mode" "TI,TI,V4SF,V2SF,V2SF")])
12869 ;; movd instead of movq is required to handle broken assemblers.
12870 (define_insn "vec_concatv2di"
12871 [(set (match_operand:V2DI 0 "register_operand"
12872 "=Yr,*x,x ,Yi,x ,!x,x,x,x,x,x")
12874 (match_operand:DI 1 "nonimmediate_operand"
12875 " 0, 0,x ,r ,xm,*y,0,x,0,0,x")
12876 (match_operand:DI 2 "vector_move_operand"
12877 "*rm,rm,rm,C ,C ,C ,x,x,x,m,m")))]
12880 pinsrq\t{$1, %2, %0|%0, %2, 1}
12881 pinsrq\t{$1, %2, %0|%0, %2, 1}
12882 vpinsrq\t{$1, %2, %1, %0|%0, %1, %2, 1}
12883 * return HAVE_AS_IX86_INTERUNIT_MOVQ ? \"%vmovq\t{%1, %0|%0, %1}\" : \"%vmovd\t{%1, %0|%0, %1}\";
12884 %vmovq\t{%1, %0|%0, %1}
12885 movq2dq\t{%1, %0|%0, %1}
12886 punpcklqdq\t{%2, %0|%0, %2}
12887 vpunpcklqdq\t{%2, %1, %0|%0, %1, %2}
12888 movlhps\t{%2, %0|%0, %2}
12889 movhps\t{%2, %0|%0, %2}
12890 vmovhps\t{%2, %1, %0|%0, %1, %2}"
12891 [(set_attr "isa" "x64_sse4_noavx,x64_sse4_noavx,x64_avx,x64,sse2,sse2,sse2_noavx,avx,noavx,noavx,avx")
12894 (eq_attr "alternative" "0,1,2,6,7")
12895 (const_string "sselog")
12896 (const_string "ssemov")))
12897 (set_attr "prefix_rex" "1,1,1,1,*,*,*,*,*,*,*")
12898 (set_attr "prefix_extra" "1,1,1,*,*,*,*,*,*,*,*")
12899 (set_attr "length_immediate" "1,1,1,*,*,*,*,*,*,*,*")
12900 (set_attr "prefix" "orig,orig,vex,maybe_vex,maybe_vex,orig,orig,vex,orig,orig,vex")
12901 (set_attr "mode" "TI,TI,TI,TI,TI,TI,TI,TI,V4SF,V2SF,V2SF")])
12903 (define_expand "vec_unpacks_lo_<mode>"
12904 [(match_operand:<sseunpackmode> 0 "register_operand")
12905 (match_operand:VI124_AVX512F 1 "register_operand")]
12907 "ix86_expand_sse_unpack (operands[0], operands[1], false, false); DONE;")
12909 (define_expand "vec_unpacks_hi_<mode>"
12910 [(match_operand:<sseunpackmode> 0 "register_operand")
12911 (match_operand:VI124_AVX512F 1 "register_operand")]
12913 "ix86_expand_sse_unpack (operands[0], operands[1], false, true); DONE;")
12915 (define_expand "vec_unpacku_lo_<mode>"
12916 [(match_operand:<sseunpackmode> 0 "register_operand")
12917 (match_operand:VI124_AVX512F 1 "register_operand")]
12919 "ix86_expand_sse_unpack (operands[0], operands[1], true, false); DONE;")
12921 (define_expand "vec_unpacku_hi_<mode>"
12922 [(match_operand:<sseunpackmode> 0 "register_operand")
12923 (match_operand:VI124_AVX512F 1 "register_operand")]
12925 "ix86_expand_sse_unpack (operands[0], operands[1], true, true); DONE;")
12927 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
12931 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
12933 (define_expand "<sse2_avx2>_uavg<mode>3<mask_name>"
12934 [(set (match_operand:VI12_AVX2 0 "register_operand")
12935 (truncate:VI12_AVX2
12936 (lshiftrt:<ssedoublemode>
12937 (plus:<ssedoublemode>
12938 (plus:<ssedoublemode>
12939 (zero_extend:<ssedoublemode>
12940 (match_operand:VI12_AVX2 1 "nonimmediate_operand"))
12941 (zero_extend:<ssedoublemode>
12942 (match_operand:VI12_AVX2 2 "nonimmediate_operand")))
12943 (match_dup <mask_expand_op3>))
12945 "TARGET_SSE2 && <mask_mode512bit_condition> && <mask_avx512bw_condition>"
12948 if (<mask_applied>)
12950 operands[3] = CONST1_RTX(<MODE>mode);
12951 ix86_fixup_binary_operands_no_copy (PLUS, <MODE>mode, operands);
12953 if (<mask_applied>)
12955 operands[5] = operands[3];
12960 (define_insn "*<sse2_avx2>_uavg<mode>3<mask_name>"
12961 [(set (match_operand:VI12_AVX2 0 "register_operand" "=x,v")
12962 (truncate:VI12_AVX2
12963 (lshiftrt:<ssedoublemode>
12964 (plus:<ssedoublemode>
12965 (plus:<ssedoublemode>
12966 (zero_extend:<ssedoublemode>
12967 (match_operand:VI12_AVX2 1 "nonimmediate_operand" "%0,v"))
12968 (zero_extend:<ssedoublemode>
12969 (match_operand:VI12_AVX2 2 "nonimmediate_operand" "xm,vm")))
12970 (match_operand:VI12_AVX2 <mask_expand_op3> "const1_operand"))
12972 "TARGET_SSE2 && <mask_mode512bit_condition> && <mask_avx512bw_condition>
12973 && ix86_binary_operator_ok (PLUS, <MODE>mode, operands)"
12975 pavg<ssemodesuffix>\t{%2, %0|%0, %2}
12976 vpavg<ssemodesuffix>\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
12977 [(set_attr "isa" "noavx,avx")
12978 (set_attr "type" "sseiadd")
12979 (set_attr "prefix_data16" "1,*")
12980 (set_attr "prefix" "orig,<mask_prefix>")
12981 (set_attr "mode" "<sseinsnmode>")])
12983 ;; The correct representation for this is absolutely enormous, and
12984 ;; surely not generally useful.
12985 (define_insn "<sse2_avx2>_psadbw"
12986 [(set (match_operand:VI8_AVX2_AVX512BW 0 "register_operand" "=x,v")
12987 (unspec:VI8_AVX2_AVX512BW
12988 [(match_operand:<ssebytemode> 1 "register_operand" "0,v")
12989 (match_operand:<ssebytemode> 2 "nonimmediate_operand" "xm,vm")]
12993 psadbw\t{%2, %0|%0, %2}
12994 vpsadbw\t{%2, %1, %0|%0, %1, %2}"
12995 [(set_attr "isa" "noavx,avx")
12996 (set_attr "type" "sseiadd")
12997 (set_attr "atom_unit" "simul")
12998 (set_attr "prefix_data16" "1,*")
12999 (set_attr "prefix" "orig,maybe_evex")
13000 (set_attr "mode" "<sseinsnmode>")])
13002 (define_insn "<sse>_movmsk<ssemodesuffix><avxsizesuffix>"
13003 [(set (match_operand:SI 0 "register_operand" "=r")
13005 [(match_operand:VF_128_256 1 "register_operand" "x")]
13008 "%vmovmsk<ssemodesuffix>\t{%1, %0|%0, %1}"
13009 [(set_attr "type" "ssemov")
13010 (set_attr "prefix" "maybe_vex")
13011 (set_attr "mode" "<MODE>")])
13013 (define_insn "avx2_pmovmskb"
13014 [(set (match_operand:SI 0 "register_operand" "=r")
13015 (unspec:SI [(match_operand:V32QI 1 "register_operand" "x")]
13018 "vpmovmskb\t{%1, %0|%0, %1}"
13019 [(set_attr "type" "ssemov")
13020 (set_attr "prefix" "vex")
13021 (set_attr "mode" "DI")])
13023 (define_insn "sse2_pmovmskb"
13024 [(set (match_operand:SI 0 "register_operand" "=r")
13025 (unspec:SI [(match_operand:V16QI 1 "register_operand" "x")]
13028 "%vpmovmskb\t{%1, %0|%0, %1}"
13029 [(set_attr "type" "ssemov")
13030 (set_attr "prefix_data16" "1")
13031 (set_attr "prefix" "maybe_vex")
13032 (set_attr "mode" "SI")])
13034 (define_expand "sse2_maskmovdqu"
13035 [(set (match_operand:V16QI 0 "memory_operand")
13036 (unspec:V16QI [(match_operand:V16QI 1 "register_operand")
13037 (match_operand:V16QI 2 "register_operand")
13042 (define_insn "*sse2_maskmovdqu"
13043 [(set (mem:V16QI (match_operand:P 0 "register_operand" "D"))
13044 (unspec:V16QI [(match_operand:V16QI 1 "register_operand" "x")
13045 (match_operand:V16QI 2 "register_operand" "x")
13046 (mem:V16QI (match_dup 0))]
13050 /* We can't use %^ here due to ASM_OUTPUT_OPCODE processing
13051 that requires %v to be at the beginning of the opcode name. */
13052 if (Pmode != word_mode)
13053 fputs ("\taddr32", asm_out_file);
13054 return "%vmaskmovdqu\t{%2, %1|%1, %2}";
13056 [(set_attr "type" "ssemov")
13057 (set_attr "prefix_data16" "1")
13058 (set (attr "length_address")
13059 (symbol_ref ("Pmode != word_mode")))
13060 ;; The implicit %rdi operand confuses default length_vex computation.
13061 (set (attr "length_vex")
13062 (symbol_ref ("3 + REX_SSE_REGNO_P (REGNO (operands[2]))")))
13063 (set_attr "prefix" "maybe_vex")
13064 (set_attr "mode" "TI")])
13066 (define_insn "sse_ldmxcsr"
13067 [(unspec_volatile [(match_operand:SI 0 "memory_operand" "m")]
13071 [(set_attr "type" "sse")
13072 (set_attr "atom_sse_attr" "mxcsr")
13073 (set_attr "prefix" "maybe_vex")
13074 (set_attr "memory" "load")])
13076 (define_insn "sse_stmxcsr"
13077 [(set (match_operand:SI 0 "memory_operand" "=m")
13078 (unspec_volatile:SI [(const_int 0)] UNSPECV_STMXCSR))]
13081 [(set_attr "type" "sse")
13082 (set_attr "atom_sse_attr" "mxcsr")
13083 (set_attr "prefix" "maybe_vex")
13084 (set_attr "memory" "store")])
13086 (define_insn "sse2_clflush"
13087 [(unspec_volatile [(match_operand 0 "address_operand" "p")]
13091 [(set_attr "type" "sse")
13092 (set_attr "atom_sse_attr" "fence")
13093 (set_attr "memory" "unknown")])
13096 (define_insn "sse3_mwait"
13097 [(unspec_volatile [(match_operand:SI 0 "register_operand" "a")
13098 (match_operand:SI 1 "register_operand" "c")]
13101 ;; 64bit version is "mwait %rax,%rcx". But only lower 32bits are used.
13102 ;; Since 32bit register operands are implicitly zero extended to 64bit,
13103 ;; we only need to set up 32bit registers.
13105 [(set_attr "length" "3")])
13107 (define_insn "sse3_monitor_<mode>"
13108 [(unspec_volatile [(match_operand:P 0 "register_operand" "a")
13109 (match_operand:SI 1 "register_operand" "c")
13110 (match_operand:SI 2 "register_operand" "d")]
13113 ;; 64bit version is "monitor %rax,%rcx,%rdx". But only lower 32bits in
13114 ;; RCX and RDX are used. Since 32bit register operands are implicitly
13115 ;; zero extended to 64bit, we only need to set up 32bit registers.
13117 [(set (attr "length")
13118 (symbol_ref ("(Pmode != word_mode) + 3")))])
13120 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
13122 ;; SSSE3 instructions
13124 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
13126 (define_code_iterator ssse3_plusminus [plus ss_plus minus ss_minus])
13128 (define_insn "avx2_ph<plusminus_mnemonic>wv16hi3"
13129 [(set (match_operand:V16HI 0 "register_operand" "=x")
13134 (ssse3_plusminus:HI
13136 (match_operand:V16HI 1 "register_operand" "x")
13137 (parallel [(const_int 0)]))
13138 (vec_select:HI (match_dup 1) (parallel [(const_int 1)])))
13139 (ssse3_plusminus:HI
13140 (vec_select:HI (match_dup 1) (parallel [(const_int 2)]))
13141 (vec_select:HI (match_dup 1) (parallel [(const_int 3)]))))
13143 (ssse3_plusminus:HI
13144 (vec_select:HI (match_dup 1) (parallel [(const_int 4)]))
13145 (vec_select:HI (match_dup 1) (parallel [(const_int 5)])))
13146 (ssse3_plusminus:HI
13147 (vec_select:HI (match_dup 1) (parallel [(const_int 6)]))
13148 (vec_select:HI (match_dup 1) (parallel [(const_int 7)])))))
13151 (ssse3_plusminus:HI
13152 (vec_select:HI (match_dup 1) (parallel [(const_int 8)]))
13153 (vec_select:HI (match_dup 1) (parallel [(const_int 9)])))
13154 (ssse3_plusminus:HI
13155 (vec_select:HI (match_dup 1) (parallel [(const_int 10)]))
13156 (vec_select:HI (match_dup 1) (parallel [(const_int 11)]))))
13158 (ssse3_plusminus:HI
13159 (vec_select:HI (match_dup 1) (parallel [(const_int 12)]))
13160 (vec_select:HI (match_dup 1) (parallel [(const_int 13)])))
13161 (ssse3_plusminus:HI
13162 (vec_select:HI (match_dup 1) (parallel [(const_int 14)]))
13163 (vec_select:HI (match_dup 1) (parallel [(const_int 15)]))))))
13167 (ssse3_plusminus:HI
13169 (match_operand:V16HI 2 "nonimmediate_operand" "xm")
13170 (parallel [(const_int 0)]))
13171 (vec_select:HI (match_dup 2) (parallel [(const_int 1)])))
13172 (ssse3_plusminus:HI
13173 (vec_select:HI (match_dup 2) (parallel [(const_int 2)]))
13174 (vec_select:HI (match_dup 2) (parallel [(const_int 3)]))))
13176 (ssse3_plusminus:HI
13177 (vec_select:HI (match_dup 2) (parallel [(const_int 4)]))
13178 (vec_select:HI (match_dup 2) (parallel [(const_int 5)])))
13179 (ssse3_plusminus:HI
13180 (vec_select:HI (match_dup 2) (parallel [(const_int 6)]))
13181 (vec_select:HI (match_dup 2) (parallel [(const_int 7)])))))
13184 (ssse3_plusminus:HI
13185 (vec_select:HI (match_dup 2) (parallel [(const_int 8)]))
13186 (vec_select:HI (match_dup 2) (parallel [(const_int 9)])))
13187 (ssse3_plusminus:HI
13188 (vec_select:HI (match_dup 2) (parallel [(const_int 10)]))
13189 (vec_select:HI (match_dup 2) (parallel [(const_int 11)]))))
13191 (ssse3_plusminus:HI
13192 (vec_select:HI (match_dup 2) (parallel [(const_int 12)]))
13193 (vec_select:HI (match_dup 2) (parallel [(const_int 13)])))
13194 (ssse3_plusminus:HI
13195 (vec_select:HI (match_dup 2) (parallel [(const_int 14)]))
13196 (vec_select:HI (match_dup 2) (parallel [(const_int 15)]))))))))]
13198 "vph<plusminus_mnemonic>w\t{%2, %1, %0|%0, %1, %2}"
13199 [(set_attr "type" "sseiadd")
13200 (set_attr "prefix_extra" "1")
13201 (set_attr "prefix" "vex")
13202 (set_attr "mode" "OI")])
13204 (define_insn "ssse3_ph<plusminus_mnemonic>wv8hi3"
13205 [(set (match_operand:V8HI 0 "register_operand" "=x,x")
13209 (ssse3_plusminus:HI
13211 (match_operand:V8HI 1 "register_operand" "0,x")
13212 (parallel [(const_int 0)]))
13213 (vec_select:HI (match_dup 1) (parallel [(const_int 1)])))
13214 (ssse3_plusminus:HI
13215 (vec_select:HI (match_dup 1) (parallel [(const_int 2)]))
13216 (vec_select:HI (match_dup 1) (parallel [(const_int 3)]))))
13218 (ssse3_plusminus:HI
13219 (vec_select:HI (match_dup 1) (parallel [(const_int 4)]))
13220 (vec_select:HI (match_dup 1) (parallel [(const_int 5)])))
13221 (ssse3_plusminus:HI
13222 (vec_select:HI (match_dup 1) (parallel [(const_int 6)]))
13223 (vec_select:HI (match_dup 1) (parallel [(const_int 7)])))))
13226 (ssse3_plusminus:HI
13228 (match_operand:V8HI 2 "nonimmediate_operand" "xm,xm")
13229 (parallel [(const_int 0)]))
13230 (vec_select:HI (match_dup 2) (parallel [(const_int 1)])))
13231 (ssse3_plusminus:HI
13232 (vec_select:HI (match_dup 2) (parallel [(const_int 2)]))
13233 (vec_select:HI (match_dup 2) (parallel [(const_int 3)]))))
13235 (ssse3_plusminus:HI
13236 (vec_select:HI (match_dup 2) (parallel [(const_int 4)]))
13237 (vec_select:HI (match_dup 2) (parallel [(const_int 5)])))
13238 (ssse3_plusminus:HI
13239 (vec_select:HI (match_dup 2) (parallel [(const_int 6)]))
13240 (vec_select:HI (match_dup 2) (parallel [(const_int 7)])))))))]
13243 ph<plusminus_mnemonic>w\t{%2, %0|%0, %2}
13244 vph<plusminus_mnemonic>w\t{%2, %1, %0|%0, %1, %2}"
13245 [(set_attr "isa" "noavx,avx")
13246 (set_attr "type" "sseiadd")
13247 (set_attr "atom_unit" "complex")
13248 (set_attr "prefix_data16" "1,*")
13249 (set_attr "prefix_extra" "1")
13250 (set_attr "prefix" "orig,vex")
13251 (set_attr "mode" "TI")])
13253 (define_insn "ssse3_ph<plusminus_mnemonic>wv4hi3"
13254 [(set (match_operand:V4HI 0 "register_operand" "=y")
13257 (ssse3_plusminus:HI
13259 (match_operand:V4HI 1 "register_operand" "0")
13260 (parallel [(const_int 0)]))
13261 (vec_select:HI (match_dup 1) (parallel [(const_int 1)])))
13262 (ssse3_plusminus:HI
13263 (vec_select:HI (match_dup 1) (parallel [(const_int 2)]))
13264 (vec_select:HI (match_dup 1) (parallel [(const_int 3)]))))
13266 (ssse3_plusminus:HI
13268 (match_operand:V4HI 2 "nonimmediate_operand" "ym")
13269 (parallel [(const_int 0)]))
13270 (vec_select:HI (match_dup 2) (parallel [(const_int 1)])))
13271 (ssse3_plusminus:HI
13272 (vec_select:HI (match_dup 2) (parallel [(const_int 2)]))
13273 (vec_select:HI (match_dup 2) (parallel [(const_int 3)]))))))]
13275 "ph<plusminus_mnemonic>w\t{%2, %0|%0, %2}"
13276 [(set_attr "type" "sseiadd")
13277 (set_attr "atom_unit" "complex")
13278 (set_attr "prefix_extra" "1")
13279 (set (attr "prefix_rex") (symbol_ref "x86_extended_reg_mentioned_p (insn)"))
13280 (set_attr "mode" "DI")])
13282 (define_insn "avx2_ph<plusminus_mnemonic>dv8si3"
13283 [(set (match_operand:V8SI 0 "register_operand" "=x")
13289 (match_operand:V8SI 1 "register_operand" "x")
13290 (parallel [(const_int 0)]))
13291 (vec_select:SI (match_dup 1) (parallel [(const_int 1)])))
13293 (vec_select:SI (match_dup 1) (parallel [(const_int 2)]))
13294 (vec_select:SI (match_dup 1) (parallel [(const_int 3)]))))
13297 (vec_select:SI (match_dup 1) (parallel [(const_int 4)]))
13298 (vec_select:SI (match_dup 1) (parallel [(const_int 5)])))
13300 (vec_select:SI (match_dup 1) (parallel [(const_int 6)]))
13301 (vec_select:SI (match_dup 1) (parallel [(const_int 7)])))))
13306 (match_operand:V8SI 2 "nonimmediate_operand" "xm")
13307 (parallel [(const_int 0)]))
13308 (vec_select:SI (match_dup 2) (parallel [(const_int 1)])))
13310 (vec_select:SI (match_dup 2) (parallel [(const_int 2)]))
13311 (vec_select:SI (match_dup 2) (parallel [(const_int 3)]))))
13314 (vec_select:SI (match_dup 2) (parallel [(const_int 4)]))
13315 (vec_select:SI (match_dup 2) (parallel [(const_int 5)])))
13317 (vec_select:SI (match_dup 2) (parallel [(const_int 6)]))
13318 (vec_select:SI (match_dup 2) (parallel [(const_int 7)])))))))]
13320 "vph<plusminus_mnemonic>d\t{%2, %1, %0|%0, %1, %2}"
13321 [(set_attr "type" "sseiadd")
13322 (set_attr "prefix_extra" "1")
13323 (set_attr "prefix" "vex")
13324 (set_attr "mode" "OI")])
13326 (define_insn "ssse3_ph<plusminus_mnemonic>dv4si3"
13327 [(set (match_operand:V4SI 0 "register_operand" "=x,x")
13332 (match_operand:V4SI 1 "register_operand" "0,x")
13333 (parallel [(const_int 0)]))
13334 (vec_select:SI (match_dup 1) (parallel [(const_int 1)])))
13336 (vec_select:SI (match_dup 1) (parallel [(const_int 2)]))
13337 (vec_select:SI (match_dup 1) (parallel [(const_int 3)]))))
13341 (match_operand:V4SI 2 "nonimmediate_operand" "xm,xm")
13342 (parallel [(const_int 0)]))
13343 (vec_select:SI (match_dup 2) (parallel [(const_int 1)])))
13345 (vec_select:SI (match_dup 2) (parallel [(const_int 2)]))
13346 (vec_select:SI (match_dup 2) (parallel [(const_int 3)]))))))]
13349 ph<plusminus_mnemonic>d\t{%2, %0|%0, %2}
13350 vph<plusminus_mnemonic>d\t{%2, %1, %0|%0, %1, %2}"
13351 [(set_attr "isa" "noavx,avx")
13352 (set_attr "type" "sseiadd")
13353 (set_attr "atom_unit" "complex")
13354 (set_attr "prefix_data16" "1,*")
13355 (set_attr "prefix_extra" "1")
13356 (set_attr "prefix" "orig,vex")
13357 (set_attr "mode" "TI")])
13359 (define_insn "ssse3_ph<plusminus_mnemonic>dv2si3"
13360 [(set (match_operand:V2SI 0 "register_operand" "=y")
13364 (match_operand:V2SI 1 "register_operand" "0")
13365 (parallel [(const_int 0)]))
13366 (vec_select:SI (match_dup 1) (parallel [(const_int 1)])))
13369 (match_operand:V2SI 2 "nonimmediate_operand" "ym")
13370 (parallel [(const_int 0)]))
13371 (vec_select:SI (match_dup 2) (parallel [(const_int 1)])))))]
13373 "ph<plusminus_mnemonic>d\t{%2, %0|%0, %2}"
13374 [(set_attr "type" "sseiadd")
13375 (set_attr "atom_unit" "complex")
13376 (set_attr "prefix_extra" "1")
13377 (set (attr "prefix_rex") (symbol_ref "x86_extended_reg_mentioned_p (insn)"))
13378 (set_attr "mode" "DI")])
13380 (define_insn "avx2_pmaddubsw256"
13381 [(set (match_operand:V16HI 0 "register_operand" "=x")
13386 (match_operand:V32QI 1 "register_operand" "x")
13387 (parallel [(const_int 0) (const_int 2)
13388 (const_int 4) (const_int 6)
13389 (const_int 8) (const_int 10)
13390 (const_int 12) (const_int 14)
13391 (const_int 16) (const_int 18)
13392 (const_int 20) (const_int 22)
13393 (const_int 24) (const_int 26)
13394 (const_int 28) (const_int 30)])))
13397 (match_operand:V32QI 2 "nonimmediate_operand" "xm")
13398 (parallel [(const_int 0) (const_int 2)
13399 (const_int 4) (const_int 6)
13400 (const_int 8) (const_int 10)
13401 (const_int 12) (const_int 14)
13402 (const_int 16) (const_int 18)
13403 (const_int 20) (const_int 22)
13404 (const_int 24) (const_int 26)
13405 (const_int 28) (const_int 30)]))))
13408 (vec_select:V16QI (match_dup 1)
13409 (parallel [(const_int 1) (const_int 3)
13410 (const_int 5) (const_int 7)
13411 (const_int 9) (const_int 11)
13412 (const_int 13) (const_int 15)
13413 (const_int 17) (const_int 19)
13414 (const_int 21) (const_int 23)
13415 (const_int 25) (const_int 27)
13416 (const_int 29) (const_int 31)])))
13418 (vec_select:V16QI (match_dup 2)
13419 (parallel [(const_int 1) (const_int 3)
13420 (const_int 5) (const_int 7)
13421 (const_int 9) (const_int 11)
13422 (const_int 13) (const_int 15)
13423 (const_int 17) (const_int 19)
13424 (const_int 21) (const_int 23)
13425 (const_int 25) (const_int 27)
13426 (const_int 29) (const_int 31)]))))))]
13428 "vpmaddubsw\t{%2, %1, %0|%0, %1, %2}"
13429 [(set_attr "type" "sseiadd")
13430 (set_attr "prefix_extra" "1")
13431 (set_attr "prefix" "vex")
13432 (set_attr "mode" "OI")])
13434 ;; The correct representation for this is absolutely enormous, and
13435 ;; surely not generally useful.
13436 (define_insn "avx512bw_pmaddubsw512<mode><mask_name>"
13437 [(set (match_operand:VI2_AVX512VL 0 "register_operand" "=v")
13438 (unspec:VI2_AVX512VL
13439 [(match_operand:<dbpsadbwmode> 1 "register_operand" "v")
13440 (match_operand:<dbpsadbwmode> 2 "nonimmediate_operand" "vm")]
13441 UNSPEC_PMADDUBSW512))]
13443 "vpmaddubsw\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}";
13444 [(set_attr "type" "sseiadd")
13445 (set_attr "prefix" "evex")
13446 (set_attr "mode" "XI")])
13448 (define_insn "avx512bw_umulhrswv32hi3<mask_name>"
13449 [(set (match_operand:V32HI 0 "register_operand" "=v")
13456 (match_operand:V32HI 1 "nonimmediate_operand" "%v"))
13458 (match_operand:V32HI 2 "nonimmediate_operand" "vm")))
13460 (const_vector:V32HI [(const_int 1) (const_int 1)
13461 (const_int 1) (const_int 1)
13462 (const_int 1) (const_int 1)
13463 (const_int 1) (const_int 1)
13464 (const_int 1) (const_int 1)
13465 (const_int 1) (const_int 1)
13466 (const_int 1) (const_int 1)
13467 (const_int 1) (const_int 1)
13468 (const_int 1) (const_int 1)
13469 (const_int 1) (const_int 1)
13470 (const_int 1) (const_int 1)
13471 (const_int 1) (const_int 1)
13472 (const_int 1) (const_int 1)
13473 (const_int 1) (const_int 1)
13474 (const_int 1) (const_int 1)
13475 (const_int 1) (const_int 1)]))
13478 "vpmulhrsw\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
13479 [(set_attr "type" "sseimul")
13480 (set_attr "prefix" "evex")
13481 (set_attr "mode" "XI")])
13483 (define_insn "ssse3_pmaddubsw128"
13484 [(set (match_operand:V8HI 0 "register_operand" "=x,x")
13489 (match_operand:V16QI 1 "register_operand" "0,x")
13490 (parallel [(const_int 0) (const_int 2)
13491 (const_int 4) (const_int 6)
13492 (const_int 8) (const_int 10)
13493 (const_int 12) (const_int 14)])))
13496 (match_operand:V16QI 2 "nonimmediate_operand" "xm,xm")
13497 (parallel [(const_int 0) (const_int 2)
13498 (const_int 4) (const_int 6)
13499 (const_int 8) (const_int 10)
13500 (const_int 12) (const_int 14)]))))
13503 (vec_select:V8QI (match_dup 1)
13504 (parallel [(const_int 1) (const_int 3)
13505 (const_int 5) (const_int 7)
13506 (const_int 9) (const_int 11)
13507 (const_int 13) (const_int 15)])))
13509 (vec_select:V8QI (match_dup 2)
13510 (parallel [(const_int 1) (const_int 3)
13511 (const_int 5) (const_int 7)
13512 (const_int 9) (const_int 11)
13513 (const_int 13) (const_int 15)]))))))]
13516 pmaddubsw\t{%2, %0|%0, %2}
13517 vpmaddubsw\t{%2, %1, %0|%0, %1, %2}"
13518 [(set_attr "isa" "noavx,avx")
13519 (set_attr "type" "sseiadd")
13520 (set_attr "atom_unit" "simul")
13521 (set_attr "prefix_data16" "1,*")
13522 (set_attr "prefix_extra" "1")
13523 (set_attr "prefix" "orig,vex")
13524 (set_attr "mode" "TI")])
13526 (define_insn "ssse3_pmaddubsw"
13527 [(set (match_operand:V4HI 0 "register_operand" "=y")
13532 (match_operand:V8QI 1 "register_operand" "0")
13533 (parallel [(const_int 0) (const_int 2)
13534 (const_int 4) (const_int 6)])))
13537 (match_operand:V8QI 2 "nonimmediate_operand" "ym")
13538 (parallel [(const_int 0) (const_int 2)
13539 (const_int 4) (const_int 6)]))))
13542 (vec_select:V4QI (match_dup 1)
13543 (parallel [(const_int 1) (const_int 3)
13544 (const_int 5) (const_int 7)])))
13546 (vec_select:V4QI (match_dup 2)
13547 (parallel [(const_int 1) (const_int 3)
13548 (const_int 5) (const_int 7)]))))))]
13550 "pmaddubsw\t{%2, %0|%0, %2}"
13551 [(set_attr "type" "sseiadd")
13552 (set_attr "atom_unit" "simul")
13553 (set_attr "prefix_extra" "1")
13554 (set (attr "prefix_rex") (symbol_ref "x86_extended_reg_mentioned_p (insn)"))
13555 (set_attr "mode" "DI")])
13557 (define_mode_iterator PMULHRSW
13558 [V4HI V8HI (V16HI "TARGET_AVX2")])
13560 (define_expand "<ssse3_avx2>_pmulhrsw<mode>3_mask"
13561 [(set (match_operand:PMULHRSW 0 "register_operand")
13562 (vec_merge:PMULHRSW
13564 (lshiftrt:<ssedoublemode>
13565 (plus:<ssedoublemode>
13566 (lshiftrt:<ssedoublemode>
13567 (mult:<ssedoublemode>
13568 (sign_extend:<ssedoublemode>
13569 (match_operand:PMULHRSW 1 "nonimmediate_operand"))
13570 (sign_extend:<ssedoublemode>
13571 (match_operand:PMULHRSW 2 "nonimmediate_operand")))
13575 (match_operand:PMULHRSW 3 "register_operand")
13576 (match_operand:<avx512fmaskmode> 4 "register_operand")))]
13577 "TARGET_AVX512BW && TARGET_AVX512VL"
13579 operands[5] = CONST1_RTX(<MODE>mode);
13580 ix86_fixup_binary_operands_no_copy (MULT, <MODE>mode, operands);
13583 (define_expand "<ssse3_avx2>_pmulhrsw<mode>3"
13584 [(set (match_operand:PMULHRSW 0 "register_operand")
13586 (lshiftrt:<ssedoublemode>
13587 (plus:<ssedoublemode>
13588 (lshiftrt:<ssedoublemode>
13589 (mult:<ssedoublemode>
13590 (sign_extend:<ssedoublemode>
13591 (match_operand:PMULHRSW 1 "nonimmediate_operand"))
13592 (sign_extend:<ssedoublemode>
13593 (match_operand:PMULHRSW 2 "nonimmediate_operand")))
13599 operands[3] = CONST1_RTX(<MODE>mode);
13600 ix86_fixup_binary_operands_no_copy (MULT, <MODE>mode, operands);
13603 (define_insn "*<ssse3_avx2>_pmulhrsw<mode>3<mask_name>"
13604 [(set (match_operand:VI2_AVX2 0 "register_operand" "=x,v")
13606 (lshiftrt:<ssedoublemode>
13607 (plus:<ssedoublemode>
13608 (lshiftrt:<ssedoublemode>
13609 (mult:<ssedoublemode>
13610 (sign_extend:<ssedoublemode>
13611 (match_operand:VI2_AVX2 1 "nonimmediate_operand" "%0,v"))
13612 (sign_extend:<ssedoublemode>
13613 (match_operand:VI2_AVX2 2 "nonimmediate_operand" "xm,vm")))
13615 (match_operand:VI2_AVX2 3 "const1_operand"))
13617 "TARGET_SSSE3 && <mask_mode512bit_condition> && <mask_avx512bw_condition>
13618 && ix86_binary_operator_ok (MULT, <MODE>mode, operands)"
13620 pmulhrsw\t{%2, %0|%0, %2}
13621 vpmulhrsw\t{%2, %1, %0<mask_operand4>|%0<mask_operand4>, %1, %2}"
13622 [(set_attr "isa" "noavx,avx")
13623 (set_attr "type" "sseimul")
13624 (set_attr "prefix_data16" "1,*")
13625 (set_attr "prefix_extra" "1")
13626 (set_attr "prefix" "orig,maybe_evex")
13627 (set_attr "mode" "<sseinsnmode>")])
13629 (define_insn "*ssse3_pmulhrswv4hi3"
13630 [(set (match_operand:V4HI 0 "register_operand" "=y")
13637 (match_operand:V4HI 1 "nonimmediate_operand" "%0"))
13639 (match_operand:V4HI 2 "nonimmediate_operand" "ym")))
13641 (match_operand:V4HI 3 "const1_operand"))
13643 "TARGET_SSSE3 && ix86_binary_operator_ok (MULT, V4HImode, operands)"
13644 "pmulhrsw\t{%2, %0|%0, %2}"
13645 [(set_attr "type" "sseimul")
13646 (set_attr "prefix_extra" "1")
13647 (set (attr "prefix_rex") (symbol_ref "x86_extended_reg_mentioned_p (insn)"))
13648 (set_attr "mode" "DI")])
13650 (define_insn "<ssse3_avx2>_pshufb<mode>3<mask_name>"
13651 [(set (match_operand:VI1_AVX512 0 "register_operand" "=x,v")
13653 [(match_operand:VI1_AVX512 1 "register_operand" "0,v")
13654 (match_operand:VI1_AVX512 2 "nonimmediate_operand" "xm,vm")]
13656 "TARGET_SSSE3 && <mask_mode512bit_condition> && <mask_avx512bw_condition>"
13658 pshufb\t{%2, %0|%0, %2}
13659 vpshufb\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
13660 [(set_attr "isa" "noavx,avx")
13661 (set_attr "type" "sselog1")
13662 (set_attr "prefix_data16" "1,*")
13663 (set_attr "prefix_extra" "1")
13664 (set_attr "prefix" "orig,maybe_evex")
13665 (set_attr "btver2_decode" "vector,vector")
13666 (set_attr "mode" "<sseinsnmode>")])
13668 (define_insn "ssse3_pshufbv8qi3"
13669 [(set (match_operand:V8QI 0 "register_operand" "=y")
13670 (unspec:V8QI [(match_operand:V8QI 1 "register_operand" "0")
13671 (match_operand:V8QI 2 "nonimmediate_operand" "ym")]
13674 "pshufb\t{%2, %0|%0, %2}";
13675 [(set_attr "type" "sselog1")
13676 (set_attr "prefix_extra" "1")
13677 (set (attr "prefix_rex") (symbol_ref "x86_extended_reg_mentioned_p (insn)"))
13678 (set_attr "mode" "DI")])
13680 (define_insn "<ssse3_avx2>_psign<mode>3"
13681 [(set (match_operand:VI124_AVX2 0 "register_operand" "=x,x")
13683 [(match_operand:VI124_AVX2 1 "register_operand" "0,x")
13684 (match_operand:VI124_AVX2 2 "nonimmediate_operand" "xm,xm")]
13688 psign<ssemodesuffix>\t{%2, %0|%0, %2}
13689 vpsign<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
13690 [(set_attr "isa" "noavx,avx")
13691 (set_attr "type" "sselog1")
13692 (set_attr "prefix_data16" "1,*")
13693 (set_attr "prefix_extra" "1")
13694 (set_attr "prefix" "orig,vex")
13695 (set_attr "mode" "<sseinsnmode>")])
13697 (define_insn "ssse3_psign<mode>3"
13698 [(set (match_operand:MMXMODEI 0 "register_operand" "=y")
13700 [(match_operand:MMXMODEI 1 "register_operand" "0")
13701 (match_operand:MMXMODEI 2 "nonimmediate_operand" "ym")]
13704 "psign<mmxvecsize>\t{%2, %0|%0, %2}";
13705 [(set_attr "type" "sselog1")
13706 (set_attr "prefix_extra" "1")
13707 (set (attr "prefix_rex") (symbol_ref "x86_extended_reg_mentioned_p (insn)"))
13708 (set_attr "mode" "DI")])
13710 (define_insn "<ssse3_avx2>_palignr<mode>_mask"
13711 [(set (match_operand:VI1_AVX512 0 "register_operand" "=v")
13712 (vec_merge:VI1_AVX512
13714 [(match_operand:VI1_AVX512 1 "register_operand" "v")
13715 (match_operand:VI1_AVX512 2 "nonimmediate_operand" "vm")
13716 (match_operand:SI 3 "const_0_to_255_mul_8_operand" "n")]
13718 (match_operand:VI1_AVX512 4 "vector_move_operand" "0C")
13719 (match_operand:<avx512fmaskmode> 5 "register_operand" "Yk")))]
13720 "TARGET_AVX512BW && (<MODE_SIZE> == 64 || TARGET_AVX512VL)"
13722 operands[3] = GEN_INT (INTVAL (operands[3]) / 8);
13723 return "vpalignr\t{%3, %2, %1, %0%{%5%}%N4|%0%{%5%}%N4, %1, %2, %3}";
13725 [(set_attr "type" "sseishft")
13726 (set_attr "atom_unit" "sishuf")
13727 (set_attr "prefix_extra" "1")
13728 (set_attr "length_immediate" "1")
13729 (set_attr "prefix" "evex")
13730 (set_attr "mode" "<sseinsnmode>")])
13732 (define_insn "<ssse3_avx2>_palignr<mode>"
13733 [(set (match_operand:SSESCALARMODE 0 "register_operand" "=x,v")
13734 (unspec:SSESCALARMODE
13735 [(match_operand:SSESCALARMODE 1 "register_operand" "0,v")
13736 (match_operand:SSESCALARMODE 2 "nonimmediate_operand" "xm,vm")
13737 (match_operand:SI 3 "const_0_to_255_mul_8_operand" "n,n")]
13741 operands[3] = GEN_INT (INTVAL (operands[3]) / 8);
13743 switch (which_alternative)
13746 return "palignr\t{%3, %2, %0|%0, %2, %3}";
13748 return "vpalignr\t{%3, %2, %1, %0|%0, %1, %2, %3}";
13750 gcc_unreachable ();
13753 [(set_attr "isa" "noavx,avx")
13754 (set_attr "type" "sseishft")
13755 (set_attr "atom_unit" "sishuf")
13756 (set_attr "prefix_data16" "1,*")
13757 (set_attr "prefix_extra" "1")
13758 (set_attr "length_immediate" "1")
13759 (set_attr "prefix" "orig,vex")
13760 (set_attr "mode" "<sseinsnmode>")])
13762 (define_insn "ssse3_palignrdi"
13763 [(set (match_operand:DI 0 "register_operand" "=y")
13764 (unspec:DI [(match_operand:DI 1 "register_operand" "0")
13765 (match_operand:DI 2 "nonimmediate_operand" "ym")
13766 (match_operand:SI 3 "const_0_to_255_mul_8_operand" "n")]
13770 operands[3] = GEN_INT (INTVAL (operands[3]) / 8);
13771 return "palignr\t{%3, %2, %0|%0, %2, %3}";
13773 [(set_attr "type" "sseishft")
13774 (set_attr "atom_unit" "sishuf")
13775 (set_attr "prefix_extra" "1")
13776 (set_attr "length_immediate" "1")
13777 (set (attr "prefix_rex") (symbol_ref "x86_extended_reg_mentioned_p (insn)"))
13778 (set_attr "mode" "DI")])
13780 ;; Mode iterator to handle singularity w/ absence of V2DI and V4DI
13781 ;; modes for abs instruction on pre AVX-512 targets.
13782 (define_mode_iterator VI1248_AVX512VL_AVX512BW
13783 [(V64QI "TARGET_AVX512BW") (V32QI "TARGET_AVX2") V16QI
13784 (V32HI "TARGET_AVX512BW") (V16HI "TARGET_AVX2") V8HI
13785 (V16SI "TARGET_AVX512F") (V8SI "TARGET_AVX2") V4SI
13786 (V8DI "TARGET_AVX512F") (V4DI "TARGET_AVX512VL") (V2DI "TARGET_AVX512VL")])
13788 (define_insn "*abs<mode>2"
13789 [(set (match_operand:VI1248_AVX512VL_AVX512BW 0 "register_operand" "=v")
13790 (abs:VI1248_AVX512VL_AVX512BW
13791 (match_operand:VI1248_AVX512VL_AVX512BW 1 "nonimmediate_operand" "vm")))]
13793 "%vpabs<ssemodesuffix>\t{%1, %0|%0, %1}"
13794 [(set_attr "type" "sselog1")
13795 (set_attr "prefix_data16" "1")
13796 (set_attr "prefix_extra" "1")
13797 (set_attr "prefix" "maybe_vex")
13798 (set_attr "mode" "<sseinsnmode>")])
13800 (define_insn "abs<mode>2_mask"
13801 [(set (match_operand:VI48_AVX512VL 0 "register_operand" "=v")
13802 (vec_merge:VI48_AVX512VL
13804 (match_operand:VI48_AVX512VL 1 "nonimmediate_operand" "vm"))
13805 (match_operand:VI48_AVX512VL 2 "vector_move_operand" "0C")
13806 (match_operand:<avx512fmaskmode> 3 "register_operand" "Yk")))]
13808 "vpabs<ssemodesuffix>\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"
13809 [(set_attr "type" "sselog1")
13810 (set_attr "prefix" "evex")
13811 (set_attr "mode" "<sseinsnmode>")])
13813 (define_insn "abs<mode>2_mask"
13814 [(set (match_operand:VI12_AVX512VL 0 "register_operand" "=v")
13815 (vec_merge:VI12_AVX512VL
13817 (match_operand:VI12_AVX512VL 1 "nonimmediate_operand" "vm"))
13818 (match_operand:VI12_AVX512VL 2 "vector_move_operand" "0C")
13819 (match_operand:<avx512fmaskmode> 3 "register_operand" "Yk")))]
13821 "vpabs<ssemodesuffix>\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"
13822 [(set_attr "type" "sselog1")
13823 (set_attr "prefix" "evex")
13824 (set_attr "mode" "<sseinsnmode>")])
13826 (define_expand "abs<mode>2"
13827 [(set (match_operand:VI1248_AVX512VL_AVX512BW 0 "register_operand")
13828 (abs:VI1248_AVX512VL_AVX512BW
13829 (match_operand:VI1248_AVX512VL_AVX512BW 1 "nonimmediate_operand")))]
13834 ix86_expand_sse2_abs (operands[0], operands[1]);
13839 (define_insn "abs<mode>2"
13840 [(set (match_operand:MMXMODEI 0 "register_operand" "=y")
13842 (match_operand:MMXMODEI 1 "nonimmediate_operand" "ym")))]
13844 "pabs<mmxvecsize>\t{%1, %0|%0, %1}";
13845 [(set_attr "type" "sselog1")
13846 (set_attr "prefix_rep" "0")
13847 (set_attr "prefix_extra" "1")
13848 (set (attr "prefix_rex") (symbol_ref "x86_extended_reg_mentioned_p (insn)"))
13849 (set_attr "mode" "DI")])
13851 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
13853 ;; AMD SSE4A instructions
13855 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
13857 (define_insn "sse4a_movnt<mode>"
13858 [(set (match_operand:MODEF 0 "memory_operand" "=m")
13860 [(match_operand:MODEF 1 "register_operand" "x")]
13863 "movnt<ssemodesuffix>\t{%1, %0|%0, %1}"
13864 [(set_attr "type" "ssemov")
13865 (set_attr "mode" "<MODE>")])
13867 (define_insn "sse4a_vmmovnt<mode>"
13868 [(set (match_operand:<ssescalarmode> 0 "memory_operand" "=m")
13869 (unspec:<ssescalarmode>
13870 [(vec_select:<ssescalarmode>
13871 (match_operand:VF_128 1 "register_operand" "x")
13872 (parallel [(const_int 0)]))]
13875 "movnt<ssescalarmodesuffix>\t{%1, %0|%0, %1}"
13876 [(set_attr "type" "ssemov")
13877 (set_attr "mode" "<ssescalarmode>")])
13879 (define_insn "sse4a_extrqi"
13880 [(set (match_operand:V2DI 0 "register_operand" "=x")
13881 (unspec:V2DI [(match_operand:V2DI 1 "register_operand" "0")
13882 (match_operand 2 "const_0_to_255_operand")
13883 (match_operand 3 "const_0_to_255_operand")]
13886 "extrq\t{%3, %2, %0|%0, %2, %3}"
13887 [(set_attr "type" "sse")
13888 (set_attr "prefix_data16" "1")
13889 (set_attr "length_immediate" "2")
13890 (set_attr "mode" "TI")])
13892 (define_insn "sse4a_extrq"
13893 [(set (match_operand:V2DI 0 "register_operand" "=x")
13894 (unspec:V2DI [(match_operand:V2DI 1 "register_operand" "0")
13895 (match_operand:V16QI 2 "register_operand" "x")]
13898 "extrq\t{%2, %0|%0, %2}"
13899 [(set_attr "type" "sse")
13900 (set_attr "prefix_data16" "1")
13901 (set_attr "mode" "TI")])
13903 (define_insn "sse4a_insertqi"
13904 [(set (match_operand:V2DI 0 "register_operand" "=x")
13905 (unspec:V2DI [(match_operand:V2DI 1 "register_operand" "0")
13906 (match_operand:V2DI 2 "register_operand" "x")
13907 (match_operand 3 "const_0_to_255_operand")
13908 (match_operand 4 "const_0_to_255_operand")]
13911 "insertq\t{%4, %3, %2, %0|%0, %2, %3, %4}"
13912 [(set_attr "type" "sseins")
13913 (set_attr "prefix_data16" "0")
13914 (set_attr "prefix_rep" "1")
13915 (set_attr "length_immediate" "2")
13916 (set_attr "mode" "TI")])
13918 (define_insn "sse4a_insertq"
13919 [(set (match_operand:V2DI 0 "register_operand" "=x")
13920 (unspec:V2DI [(match_operand:V2DI 1 "register_operand" "0")
13921 (match_operand:V2DI 2 "register_operand" "x")]
13924 "insertq\t{%2, %0|%0, %2}"
13925 [(set_attr "type" "sseins")
13926 (set_attr "prefix_data16" "0")
13927 (set_attr "prefix_rep" "1")
13928 (set_attr "mode" "TI")])
13930 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
13932 ;; Intel SSE4.1 instructions
13934 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
13936 ;; Mapping of immediate bits for blend instructions
13937 (define_mode_attr blendbits
13938 [(V8SF "255") (V4SF "15") (V4DF "15") (V2DF "3")])
13940 (define_insn "<sse4_1>_blend<ssemodesuffix><avxsizesuffix>"
13941 [(set (match_operand:VF_128_256 0 "register_operand" "=Yr,*x,x")
13942 (vec_merge:VF_128_256
13943 (match_operand:VF_128_256 2 "nonimmediate_operand" "Yrm,*xm,xm")
13944 (match_operand:VF_128_256 1 "register_operand" "0,0,x")
13945 (match_operand:SI 3 "const_0_to_<blendbits>_operand")))]
13948 blend<ssemodesuffix>\t{%3, %2, %0|%0, %2, %3}
13949 blend<ssemodesuffix>\t{%3, %2, %0|%0, %2, %3}
13950 vblend<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
13951 [(set_attr "isa" "noavx,noavx,avx")
13952 (set_attr "type" "ssemov")
13953 (set_attr "length_immediate" "1")
13954 (set_attr "prefix_data16" "1,1,*")
13955 (set_attr "prefix_extra" "1")
13956 (set_attr "prefix" "orig,orig,vex")
13957 (set_attr "mode" "<MODE>")])
13959 (define_insn "<sse4_1>_blendv<ssemodesuffix><avxsizesuffix>"
13960 [(set (match_operand:VF_128_256 0 "register_operand" "=Yr,*x,x")
13962 [(match_operand:VF_128_256 1 "register_operand" "0,0,x")
13963 (match_operand:VF_128_256 2 "nonimmediate_operand" "Yrm,*xm,xm")
13964 (match_operand:VF_128_256 3 "register_operand" "Yz,Yz,x")]
13968 blendv<ssemodesuffix>\t{%3, %2, %0|%0, %2, %3}
13969 blendv<ssemodesuffix>\t{%3, %2, %0|%0, %2, %3}
13970 vblendv<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
13971 [(set_attr "isa" "noavx,noavx,avx")
13972 (set_attr "type" "ssemov")
13973 (set_attr "length_immediate" "1")
13974 (set_attr "prefix_data16" "1,1,*")
13975 (set_attr "prefix_extra" "1")
13976 (set_attr "prefix" "orig,orig,vex")
13977 (set_attr "btver2_decode" "vector,vector,vector")
13978 (set_attr "mode" "<MODE>")])
13980 (define_insn "<sse4_1>_dp<ssemodesuffix><avxsizesuffix>"
13981 [(set (match_operand:VF_128_256 0 "register_operand" "=Yr,*x,x")
13983 [(match_operand:VF_128_256 1 "nonimmediate_operand" "%0,0,x")
13984 (match_operand:VF_128_256 2 "nonimmediate_operand" "Yrm,*xm,xm")
13985 (match_operand:SI 3 "const_0_to_255_operand" "n,n,n")]
13989 dp<ssemodesuffix>\t{%3, %2, %0|%0, %2, %3}
13990 dp<ssemodesuffix>\t{%3, %2, %0|%0, %2, %3}
13991 vdp<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
13992 [(set_attr "isa" "noavx,noavx,avx")
13993 (set_attr "type" "ssemul")
13994 (set_attr "length_immediate" "1")
13995 (set_attr "prefix_data16" "1,1,*")
13996 (set_attr "prefix_extra" "1")
13997 (set_attr "prefix" "orig,orig,vex")
13998 (set_attr "btver2_decode" "vector,vector,vector")
13999 (set_attr "mode" "<MODE>")])
14001 ;; Mode attribute used by `vmovntdqa' pattern
14002 (define_mode_attr vi8_sse4_1_avx2_avx512
14003 [(V2DI "sse4_1") (V4DI "avx2") (V8DI "avx512f")])
14005 (define_insn "<vi8_sse4_1_avx2_avx512>_movntdqa"
14006 [(set (match_operand:VI8_AVX2_AVX512F 0 "register_operand" "=Yr,*x, v")
14007 (unspec:VI8_AVX2_AVX512F [(match_operand:VI8_AVX2_AVX512F 1 "memory_operand" "m, m, m")]
14010 "%vmovntdqa\t{%1, %0|%0, %1}"
14011 [(set_attr "type" "ssemov")
14012 (set_attr "prefix_extra" "1,1,*")
14013 (set_attr "prefix" "maybe_vex,maybe_vex,evex")
14014 (set_attr "mode" "<sseinsnmode>")])
14016 (define_insn "<sse4_1_avx2>_mpsadbw"
14017 [(set (match_operand:VI1_AVX2 0 "register_operand" "=Yr,*x,x")
14019 [(match_operand:VI1_AVX2 1 "register_operand" "0,0,x")
14020 (match_operand:VI1_AVX2 2 "nonimmediate_operand" "Yrm,*xm,xm")
14021 (match_operand:SI 3 "const_0_to_255_operand" "n,n,n")]
14025 mpsadbw\t{%3, %2, %0|%0, %2, %3}
14026 mpsadbw\t{%3, %2, %0|%0, %2, %3}
14027 vmpsadbw\t{%3, %2, %1, %0|%0, %1, %2, %3}"
14028 [(set_attr "isa" "noavx,noavx,avx")
14029 (set_attr "type" "sselog1")
14030 (set_attr "length_immediate" "1")
14031 (set_attr "prefix_extra" "1")
14032 (set_attr "prefix" "orig,orig,vex")
14033 (set_attr "btver2_decode" "vector,vector,vector")
14034 (set_attr "mode" "<sseinsnmode>")])
14036 (define_insn "<sse4_1_avx2>_packusdw<mask_name>"
14037 [(set (match_operand:VI2_AVX2 0 "register_operand" "=Yr,*x,v")
14038 (vec_concat:VI2_AVX2
14039 (us_truncate:<ssehalfvecmode>
14040 (match_operand:<sseunpackmode> 1 "register_operand" "0,0,v"))
14041 (us_truncate:<ssehalfvecmode>
14042 (match_operand:<sseunpackmode> 2 "nonimmediate_operand" "Yrm,*xm,vm"))))]
14043 "TARGET_SSE4_1 && <mask_mode512bit_condition> && <mask_avx512bw_condition>"
14045 packusdw\t{%2, %0|%0, %2}
14046 packusdw\t{%2, %0|%0, %2}
14047 vpackusdw\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
14048 [(set_attr "isa" "noavx,noavx,avx")
14049 (set_attr "type" "sselog")
14050 (set_attr "prefix_extra" "1")
14051 (set_attr "prefix" "orig,orig,maybe_evex")
14052 (set_attr "mode" "<sseinsnmode>")])
14054 (define_insn "<sse4_1_avx2>_pblendvb"
14055 [(set (match_operand:VI1_AVX2 0 "register_operand" "=Yr,*x,x")
14057 [(match_operand:VI1_AVX2 1 "register_operand" "0,0,x")
14058 (match_operand:VI1_AVX2 2 "nonimmediate_operand" "Yrm,*xm,xm")
14059 (match_operand:VI1_AVX2 3 "register_operand" "Yz,Yz,x")]
14063 pblendvb\t{%3, %2, %0|%0, %2, %3}
14064 pblendvb\t{%3, %2, %0|%0, %2, %3}
14065 vpblendvb\t{%3, %2, %1, %0|%0, %1, %2, %3}"
14066 [(set_attr "isa" "noavx,noavx,avx")
14067 (set_attr "type" "ssemov")
14068 (set_attr "prefix_extra" "1")
14069 (set_attr "length_immediate" "*,*,1")
14070 (set_attr "prefix" "orig,orig,vex")
14071 (set_attr "btver2_decode" "vector,vector,vector")
14072 (set_attr "mode" "<sseinsnmode>")])
14074 (define_insn "sse4_1_pblendw"
14075 [(set (match_operand:V8HI 0 "register_operand" "=Yr,*x,x")
14077 (match_operand:V8HI 2 "nonimmediate_operand" "Yrm,*xm,xm")
14078 (match_operand:V8HI 1 "register_operand" "0,0,x")
14079 (match_operand:SI 3 "const_0_to_255_operand" "n,n,n")))]
14082 pblendw\t{%3, %2, %0|%0, %2, %3}
14083 pblendw\t{%3, %2, %0|%0, %2, %3}
14084 vpblendw\t{%3, %2, %1, %0|%0, %1, %2, %3}"
14085 [(set_attr "isa" "noavx,noavx,avx")
14086 (set_attr "type" "ssemov")
14087 (set_attr "prefix_extra" "1")
14088 (set_attr "length_immediate" "1")
14089 (set_attr "prefix" "orig,orig,vex")
14090 (set_attr "mode" "TI")])
14092 ;; The builtin uses an 8-bit immediate. Expand that.
14093 (define_expand "avx2_pblendw"
14094 [(set (match_operand:V16HI 0 "register_operand")
14096 (match_operand:V16HI 2 "nonimmediate_operand")
14097 (match_operand:V16HI 1 "register_operand")
14098 (match_operand:SI 3 "const_0_to_255_operand")))]
14101 HOST_WIDE_INT val = INTVAL (operands[3]) & 0xff;
14102 operands[3] = GEN_INT (val << 8 | val);
14105 (define_insn "*avx2_pblendw"
14106 [(set (match_operand:V16HI 0 "register_operand" "=x")
14108 (match_operand:V16HI 2 "nonimmediate_operand" "xm")
14109 (match_operand:V16HI 1 "register_operand" "x")
14110 (match_operand:SI 3 "avx2_pblendw_operand" "n")))]
14113 operands[3] = GEN_INT (INTVAL (operands[3]) & 0xff);
14114 return "vpblendw\t{%3, %2, %1, %0|%0, %1, %2, %3}";
14116 [(set_attr "type" "ssemov")
14117 (set_attr "prefix_extra" "1")
14118 (set_attr "length_immediate" "1")
14119 (set_attr "prefix" "vex")
14120 (set_attr "mode" "OI")])
14122 (define_insn "avx2_pblendd<mode>"
14123 [(set (match_operand:VI4_AVX2 0 "register_operand" "=x")
14124 (vec_merge:VI4_AVX2
14125 (match_operand:VI4_AVX2 2 "nonimmediate_operand" "xm")
14126 (match_operand:VI4_AVX2 1 "register_operand" "x")
14127 (match_operand:SI 3 "const_0_to_255_operand" "n")))]
14129 "vpblendd\t{%3, %2, %1, %0|%0, %1, %2, %3}"
14130 [(set_attr "type" "ssemov")
14131 (set_attr "prefix_extra" "1")
14132 (set_attr "length_immediate" "1")
14133 (set_attr "prefix" "vex")
14134 (set_attr "mode" "<sseinsnmode>")])
14136 (define_insn "sse4_1_phminposuw"
14137 [(set (match_operand:V8HI 0 "register_operand" "=Yr,*x")
14138 (unspec:V8HI [(match_operand:V8HI 1 "nonimmediate_operand" "Yrm,*xm")]
14139 UNSPEC_PHMINPOSUW))]
14141 "%vphminposuw\t{%1, %0|%0, %1}"
14142 [(set_attr "type" "sselog1")
14143 (set_attr "prefix_extra" "1")
14144 (set_attr "prefix" "maybe_vex")
14145 (set_attr "mode" "TI")])
14147 (define_insn "avx2_<code>v16qiv16hi2<mask_name>"
14148 [(set (match_operand:V16HI 0 "register_operand" "=v")
14150 (match_operand:V16QI 1 "nonimmediate_operand" "vm")))]
14151 "TARGET_AVX2 && <mask_avx512bw_condition> && <mask_avx512vl_condition>"
14152 "vpmov<extsuffix>bw\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
14153 [(set_attr "type" "ssemov")
14154 (set_attr "prefix_extra" "1")
14155 (set_attr "prefix" "maybe_evex")
14156 (set_attr "mode" "OI")])
14158 (define_insn "avx512bw_<code>v32qiv32hi2<mask_name>"
14159 [(set (match_operand:V32HI 0 "register_operand" "=v")
14161 (match_operand:V32QI 1 "nonimmediate_operand" "vm")))]
14163 "vpmov<extsuffix>bw\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
14164 [(set_attr "type" "ssemov")
14165 (set_attr "prefix_extra" "1")
14166 (set_attr "prefix" "evex")
14167 (set_attr "mode" "XI")])
14169 (define_insn "sse4_1_<code>v8qiv8hi2<mask_name>"
14170 [(set (match_operand:V8HI 0 "register_operand" "=Yr,*v")
14173 (match_operand:V16QI 1 "nonimmediate_operand" "Yrm,*vm")
14174 (parallel [(const_int 0) (const_int 1)
14175 (const_int 2) (const_int 3)
14176 (const_int 4) (const_int 5)
14177 (const_int 6) (const_int 7)]))))]
14178 "TARGET_SSE4_1 && <mask_avx512bw_condition> && <mask_avx512vl_condition>"
14179 "%vpmov<extsuffix>bw\t{%1, %0<mask_operand2>|%0<mask_operand2>, %q1}"
14180 [(set_attr "type" "ssemov")
14181 (set_attr "ssememalign" "64")
14182 (set_attr "prefix_extra" "1")
14183 (set_attr "prefix" "maybe_vex")
14184 (set_attr "mode" "TI")])
14186 (define_insn "<mask_codefor>avx512f_<code>v16qiv16si2<mask_name>"
14187 [(set (match_operand:V16SI 0 "register_operand" "=v")
14189 (match_operand:V16QI 1 "nonimmediate_operand" "vm")))]
14191 "vpmov<extsuffix>bd\t{%1, %0<mask_operand2>|%0<mask_operand2>, %q1}"
14192 [(set_attr "type" "ssemov")
14193 (set_attr "prefix" "evex")
14194 (set_attr "mode" "XI")])
14196 (define_insn "avx2_<code>v8qiv8si2<mask_name>"
14197 [(set (match_operand:V8SI 0 "register_operand" "=v")
14200 (match_operand:V16QI 1 "nonimmediate_operand" "vm")
14201 (parallel [(const_int 0) (const_int 1)
14202 (const_int 2) (const_int 3)
14203 (const_int 4) (const_int 5)
14204 (const_int 6) (const_int 7)]))))]
14205 "TARGET_AVX2 && <mask_avx512vl_condition>"
14206 "vpmov<extsuffix>bd\t{%1, %0<mask_operand2>|%0<mask_operand2>, %q1}"
14207 [(set_attr "type" "ssemov")
14208 (set_attr "prefix_extra" "1")
14209 (set_attr "prefix" "maybe_evex")
14210 (set_attr "mode" "OI")])
14212 (define_insn "sse4_1_<code>v4qiv4si2<mask_name>"
14213 [(set (match_operand:V4SI 0 "register_operand" "=Yr,*v")
14216 (match_operand:V16QI 1 "nonimmediate_operand" "Yrm,*vm")
14217 (parallel [(const_int 0) (const_int 1)
14218 (const_int 2) (const_int 3)]))))]
14219 "TARGET_SSE4_1 && <mask_avx512vl_condition>"
14220 "%vpmov<extsuffix>bd\t{%1, %0<mask_operand2>|%0<mask_operand2>, %k1}"
14221 [(set_attr "type" "ssemov")
14222 (set_attr "ssememalign" "32")
14223 (set_attr "prefix_extra" "1")
14224 (set_attr "prefix" "maybe_vex")
14225 (set_attr "mode" "TI")])
14227 (define_insn "avx512f_<code>v16hiv16si2<mask_name>"
14228 [(set (match_operand:V16SI 0 "register_operand" "=v")
14230 (match_operand:V16HI 1 "nonimmediate_operand" "vm")))]
14232 "vpmov<extsuffix>wd\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
14233 [(set_attr "type" "ssemov")
14234 (set_attr "prefix" "evex")
14235 (set_attr "mode" "XI")])
14237 (define_insn "avx2_<code>v8hiv8si2<mask_name>"
14238 [(set (match_operand:V8SI 0 "register_operand" "=v")
14240 (match_operand:V8HI 1 "nonimmediate_operand" "vm")))]
14241 "TARGET_AVX2 && <mask_avx512vl_condition>"
14242 "vpmov<extsuffix>wd\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
14243 [(set_attr "type" "ssemov")
14244 (set_attr "prefix_extra" "1")
14245 (set_attr "prefix" "maybe_evex")
14246 (set_attr "mode" "OI")])
14248 (define_insn "sse4_1_<code>v4hiv4si2<mask_name>"
14249 [(set (match_operand:V4SI 0 "register_operand" "=Yr,*v")
14252 (match_operand:V8HI 1 "nonimmediate_operand" "Yrm,*vm")
14253 (parallel [(const_int 0) (const_int 1)
14254 (const_int 2) (const_int 3)]))))]
14255 "TARGET_SSE4_1 && <mask_avx512vl_condition>"
14256 "%vpmov<extsuffix>wd\t{%1, %0<mask_operand2>|%0<mask_operand2>, %q1}"
14257 [(set_attr "type" "ssemov")
14258 (set_attr "ssememalign" "64")
14259 (set_attr "prefix_extra" "1")
14260 (set_attr "prefix" "maybe_vex")
14261 (set_attr "mode" "TI")])
14263 (define_insn "avx512f_<code>v8qiv8di2<mask_name>"
14264 [(set (match_operand:V8DI 0 "register_operand" "=v")
14267 (match_operand:V16QI 1 "nonimmediate_operand" "vm")
14268 (parallel [(const_int 0) (const_int 1)
14269 (const_int 2) (const_int 3)
14270 (const_int 4) (const_int 5)
14271 (const_int 6) (const_int 7)]))))]
14273 "vpmov<extsuffix>bq\t{%1, %0<mask_operand2>|%0<mask_operand2>, %k1}"
14274 [(set_attr "type" "ssemov")
14275 (set_attr "prefix" "evex")
14276 (set_attr "mode" "XI")])
14278 (define_insn "avx2_<code>v4qiv4di2<mask_name>"
14279 [(set (match_operand:V4DI 0 "register_operand" "=v")
14282 (match_operand:V16QI 1 "nonimmediate_operand" "vm")
14283 (parallel [(const_int 0) (const_int 1)
14284 (const_int 2) (const_int 3)]))))]
14285 "TARGET_AVX2 && <mask_avx512vl_condition>"
14286 "vpmov<extsuffix>bq\t{%1, %0<mask_operand2>|%0<mask_operand2>, %k1}"
14287 [(set_attr "type" "ssemov")
14288 (set_attr "prefix_extra" "1")
14289 (set_attr "prefix" "maybe_evex")
14290 (set_attr "mode" "OI")])
14292 (define_insn "sse4_1_<code>v2qiv2di2<mask_name>"
14293 [(set (match_operand:V2DI 0 "register_operand" "=Yr,*v")
14296 (match_operand:V16QI 1 "nonimmediate_operand" "Yrm,*vm")
14297 (parallel [(const_int 0) (const_int 1)]))))]
14298 "TARGET_SSE4_1 && <mask_avx512vl_condition>"
14299 "%vpmov<extsuffix>bq\t{%1, %0<mask_operand2>|%0<mask_operand2>, %w1}"
14300 [(set_attr "type" "ssemov")
14301 (set_attr "ssememalign" "16")
14302 (set_attr "prefix_extra" "1")
14303 (set_attr "prefix" "maybe_vex")
14304 (set_attr "mode" "TI")])
14306 (define_insn "avx512f_<code>v8hiv8di2<mask_name>"
14307 [(set (match_operand:V8DI 0 "register_operand" "=v")
14309 (match_operand:V8HI 1 "nonimmediate_operand" "vm")))]
14311 "vpmov<extsuffix>wq\t{%1, %0<mask_operand2>|%0<mask_operand2>, %q1}"
14312 [(set_attr "type" "ssemov")
14313 (set_attr "prefix" "evex")
14314 (set_attr "mode" "XI")])
14316 (define_insn "avx2_<code>v4hiv4di2<mask_name>"
14317 [(set (match_operand:V4DI 0 "register_operand" "=v")
14320 (match_operand:V8HI 1 "nonimmediate_operand" "vm")
14321 (parallel [(const_int 0) (const_int 1)
14322 (const_int 2) (const_int 3)]))))]
14323 "TARGET_AVX2 && <mask_avx512vl_condition>"
14324 "vpmov<extsuffix>wq\t{%1, %0<mask_operand2>|%0<mask_operand2>, %q1}"
14325 [(set_attr "type" "ssemov")
14326 (set_attr "prefix_extra" "1")
14327 (set_attr "prefix" "maybe_evex")
14328 (set_attr "mode" "OI")])
14330 (define_insn "sse4_1_<code>v2hiv2di2<mask_name>"
14331 [(set (match_operand:V2DI 0 "register_operand" "=Yr,*v")
14334 (match_operand:V8HI 1 "nonimmediate_operand" "Yrm,*vm")
14335 (parallel [(const_int 0) (const_int 1)]))))]
14336 "TARGET_SSE4_1 && <mask_avx512vl_condition>"
14337 "%vpmov<extsuffix>wq\t{%1, %0<mask_operand2>|%0<mask_operand2>, %k1}"
14338 [(set_attr "type" "ssemov")
14339 (set_attr "ssememalign" "32")
14340 (set_attr "prefix_extra" "1")
14341 (set_attr "prefix" "maybe_vex")
14342 (set_attr "mode" "TI")])
14344 (define_insn "avx512f_<code>v8siv8di2<mask_name>"
14345 [(set (match_operand:V8DI 0 "register_operand" "=v")
14347 (match_operand:V8SI 1 "nonimmediate_operand" "vm")))]
14349 "vpmov<extsuffix>dq\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
14350 [(set_attr "type" "ssemov")
14351 (set_attr "prefix" "evex")
14352 (set_attr "mode" "XI")])
14354 (define_insn "avx2_<code>v4siv4di2<mask_name>"
14355 [(set (match_operand:V4DI 0 "register_operand" "=v")
14357 (match_operand:V4SI 1 "nonimmediate_operand" "vm")))]
14358 "TARGET_AVX2 && <mask_avx512vl_condition>"
14359 "vpmov<extsuffix>dq\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
14360 [(set_attr "type" "ssemov")
14361 (set_attr "prefix" "maybe_evex")
14362 (set_attr "prefix_extra" "1")
14363 (set_attr "mode" "OI")])
14365 (define_insn "sse4_1_<code>v2siv2di2<mask_name>"
14366 [(set (match_operand:V2DI 0 "register_operand" "=Yr,*v")
14369 (match_operand:V4SI 1 "nonimmediate_operand" "Yrm,*vm")
14370 (parallel [(const_int 0) (const_int 1)]))))]
14371 "TARGET_SSE4_1 && <mask_avx512vl_condition>"
14372 "%vpmov<extsuffix>dq\t{%1, %0<mask_operand2>|%0<mask_operand2>, %q1}"
14373 [(set_attr "type" "ssemov")
14374 (set_attr "ssememalign" "64")
14375 (set_attr "prefix_extra" "1")
14376 (set_attr "prefix" "maybe_vex")
14377 (set_attr "mode" "TI")])
14379 ;; ptestps/ptestpd are very similar to comiss and ucomiss when
14380 ;; setting FLAGS_REG. But it is not a really compare instruction.
14381 (define_insn "avx_vtest<ssemodesuffix><avxsizesuffix>"
14382 [(set (reg:CC FLAGS_REG)
14383 (unspec:CC [(match_operand:VF_128_256 0 "register_operand" "x")
14384 (match_operand:VF_128_256 1 "nonimmediate_operand" "xm")]
14387 "vtest<ssemodesuffix>\t{%1, %0|%0, %1}"
14388 [(set_attr "type" "ssecomi")
14389 (set_attr "prefix_extra" "1")
14390 (set_attr "prefix" "vex")
14391 (set_attr "mode" "<MODE>")])
14393 ;; ptest is very similar to comiss and ucomiss when setting FLAGS_REG.
14394 ;; But it is not a really compare instruction.
14395 (define_insn "avx_ptest256"
14396 [(set (reg:CC FLAGS_REG)
14397 (unspec:CC [(match_operand:V4DI 0 "register_operand" "x")
14398 (match_operand:V4DI 1 "nonimmediate_operand" "xm")]
14401 "vptest\t{%1, %0|%0, %1}"
14402 [(set_attr "type" "ssecomi")
14403 (set_attr "prefix_extra" "1")
14404 (set_attr "prefix" "vex")
14405 (set_attr "btver2_decode" "vector")
14406 (set_attr "mode" "OI")])
14408 (define_insn "sse4_1_ptest"
14409 [(set (reg:CC FLAGS_REG)
14410 (unspec:CC [(match_operand:V2DI 0 "register_operand" "Yr,*x")
14411 (match_operand:V2DI 1 "nonimmediate_operand" "Yrm,*xm")]
14414 "%vptest\t{%1, %0|%0, %1}"
14415 [(set_attr "type" "ssecomi")
14416 (set_attr "prefix_extra" "1")
14417 (set_attr "prefix" "maybe_vex")
14418 (set_attr "mode" "TI")])
14420 (define_insn "<sse4_1>_round<ssemodesuffix><avxsizesuffix>"
14421 [(set (match_operand:VF_128_256 0 "register_operand" "=Yr,*x")
14423 [(match_operand:VF_128_256 1 "nonimmediate_operand" "Yrm,*xm")
14424 (match_operand:SI 2 "const_0_to_15_operand" "n,n")]
14427 "%vround<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
14428 [(set_attr "type" "ssecvt")
14429 (set (attr "prefix_data16")
14431 (match_test "TARGET_AVX")
14433 (const_string "1")))
14434 (set_attr "prefix_extra" "1")
14435 (set_attr "length_immediate" "1")
14436 (set_attr "prefix" "maybe_vex")
14437 (set_attr "mode" "<MODE>")])
14439 (define_expand "<sse4_1>_round<ssemodesuffix>_sfix<avxsizesuffix>"
14440 [(match_operand:<sseintvecmode> 0 "register_operand")
14441 (match_operand:VF1_128_256 1 "nonimmediate_operand")
14442 (match_operand:SI 2 "const_0_to_15_operand")]
14445 rtx tmp = gen_reg_rtx (<MODE>mode);
14448 (gen_<sse4_1>_round<ssemodesuffix><avxsizesuffix> (tmp, operands[1],
14451 (gen_fix_trunc<mode><sseintvecmodelower>2 (operands[0], tmp));
14455 (define_expand "avx512f_roundpd512"
14456 [(match_operand:V8DF 0 "register_operand")
14457 (match_operand:V8DF 1 "nonimmediate_operand")
14458 (match_operand:SI 2 "const_0_to_15_operand")]
14461 emit_insn (gen_avx512f_rndscalev8df (operands[0], operands[1], operands[2]));
14465 (define_expand "<sse4_1>_round<ssemodesuffix>_vec_pack_sfix<avxsizesuffix>"
14466 [(match_operand:<ssepackfltmode> 0 "register_operand")
14467 (match_operand:VF2 1 "nonimmediate_operand")
14468 (match_operand:VF2 2 "nonimmediate_operand")
14469 (match_operand:SI 3 "const_0_to_15_operand")]
14474 if (<MODE>mode == V2DFmode
14475 && TARGET_AVX && !TARGET_PREFER_AVX128 && optimize_insn_for_speed_p ())
14477 rtx tmp2 = gen_reg_rtx (V4DFmode);
14479 tmp0 = gen_reg_rtx (V4DFmode);
14480 tmp1 = force_reg (V2DFmode, operands[1]);
14482 emit_insn (gen_avx_vec_concatv4df (tmp0, tmp1, operands[2]));
14483 emit_insn (gen_avx_roundpd256 (tmp2, tmp0, operands[3]));
14484 emit_insn (gen_fix_truncv4dfv4si2 (operands[0], tmp2));
14488 tmp0 = gen_reg_rtx (<MODE>mode);
14489 tmp1 = gen_reg_rtx (<MODE>mode);
14492 (gen_<sse4_1>_round<ssemodesuffix><avxsizesuffix> (tmp0, operands[1],
14495 (gen_<sse4_1>_round<ssemodesuffix><avxsizesuffix> (tmp1, operands[2],
14498 (gen_vec_pack_sfix_trunc_<mode> (operands[0], tmp0, tmp1));
14503 (define_insn "sse4_1_round<ssescalarmodesuffix>"
14504 [(set (match_operand:VF_128 0 "register_operand" "=Yr,*x,x")
14507 [(match_operand:VF_128 2 "register_operand" "Yr,*x,x")
14508 (match_operand:SI 3 "const_0_to_15_operand" "n,n,n")]
14510 (match_operand:VF_128 1 "register_operand" "0,0,x")
14514 round<ssescalarmodesuffix>\t{%3, %2, %0|%0, %2, %3}
14515 round<ssescalarmodesuffix>\t{%3, %2, %0|%0, %2, %3}
14516 vround<ssescalarmodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
14517 [(set_attr "isa" "noavx,noavx,avx")
14518 (set_attr "type" "ssecvt")
14519 (set_attr "length_immediate" "1")
14520 (set_attr "prefix_data16" "1,1,*")
14521 (set_attr "prefix_extra" "1")
14522 (set_attr "prefix" "orig,orig,vex")
14523 (set_attr "mode" "<MODE>")])
14525 (define_expand "round<mode>2"
14526 [(set (match_dup 4)
14528 (match_operand:VF 1 "register_operand")
14530 (set (match_operand:VF 0 "register_operand")
14532 [(match_dup 4) (match_dup 5)]
14534 "TARGET_ROUND && !flag_trapping_math"
14536 machine_mode scalar_mode;
14537 const struct real_format *fmt;
14538 REAL_VALUE_TYPE pred_half, half_minus_pred_half;
14539 rtx half, vec_half;
14541 scalar_mode = GET_MODE_INNER (<MODE>mode);
14543 /* load nextafter (0.5, 0.0) */
14544 fmt = REAL_MODE_FORMAT (scalar_mode);
14545 real_2expN (&half_minus_pred_half, -(fmt->p) - 1, scalar_mode);
14546 REAL_ARITHMETIC (pred_half, MINUS_EXPR, dconsthalf, half_minus_pred_half);
14547 half = const_double_from_real_value (pred_half, scalar_mode);
14549 vec_half = ix86_build_const_vector (<MODE>mode, true, half);
14550 vec_half = force_reg (<MODE>mode, vec_half);
14552 operands[3] = gen_reg_rtx (<MODE>mode);
14553 emit_insn (gen_copysign<mode>3 (operands[3], vec_half, operands[1]));
14555 operands[4] = gen_reg_rtx (<MODE>mode);
14556 operands[5] = GEN_INT (ROUND_TRUNC);
14559 (define_expand "round<mode>2_sfix"
14560 [(match_operand:<sseintvecmode> 0 "register_operand")
14561 (match_operand:VF1_128_256 1 "register_operand")]
14562 "TARGET_ROUND && !flag_trapping_math"
14564 rtx tmp = gen_reg_rtx (<MODE>mode);
14566 emit_insn (gen_round<mode>2 (tmp, operands[1]));
14569 (gen_fix_trunc<mode><sseintvecmodelower>2 (operands[0], tmp));
14573 (define_expand "round<mode>2_vec_pack_sfix"
14574 [(match_operand:<ssepackfltmode> 0 "register_operand")
14575 (match_operand:VF2 1 "register_operand")
14576 (match_operand:VF2 2 "register_operand")]
14577 "TARGET_ROUND && !flag_trapping_math"
14581 if (<MODE>mode == V2DFmode
14582 && TARGET_AVX && !TARGET_PREFER_AVX128 && optimize_insn_for_speed_p ())
14584 rtx tmp2 = gen_reg_rtx (V4DFmode);
14586 tmp0 = gen_reg_rtx (V4DFmode);
14587 tmp1 = force_reg (V2DFmode, operands[1]);
14589 emit_insn (gen_avx_vec_concatv4df (tmp0, tmp1, operands[2]));
14590 emit_insn (gen_roundv4df2 (tmp2, tmp0));
14591 emit_insn (gen_fix_truncv4dfv4si2 (operands[0], tmp2));
14595 tmp0 = gen_reg_rtx (<MODE>mode);
14596 tmp1 = gen_reg_rtx (<MODE>mode);
14598 emit_insn (gen_round<mode>2 (tmp0, operands[1]));
14599 emit_insn (gen_round<mode>2 (tmp1, operands[2]));
14602 (gen_vec_pack_sfix_trunc_<mode> (operands[0], tmp0, tmp1));
14607 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
14609 ;; Intel SSE4.2 string/text processing instructions
14611 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
14613 (define_insn_and_split "sse4_2_pcmpestr"
14614 [(set (match_operand:SI 0 "register_operand" "=c,c")
14616 [(match_operand:V16QI 2 "register_operand" "x,x")
14617 (match_operand:SI 3 "register_operand" "a,a")
14618 (match_operand:V16QI 4 "nonimmediate_operand" "x,m")
14619 (match_operand:SI 5 "register_operand" "d,d")
14620 (match_operand:SI 6 "const_0_to_255_operand" "n,n")]
14622 (set (match_operand:V16QI 1 "register_operand" "=Yz,Yz")
14630 (set (reg:CC FLAGS_REG)
14639 && can_create_pseudo_p ()"
14644 int ecx = !find_regno_note (curr_insn, REG_UNUSED, REGNO (operands[0]));
14645 int xmm0 = !find_regno_note (curr_insn, REG_UNUSED, REGNO (operands[1]));
14646 int flags = !find_regno_note (curr_insn, REG_UNUSED, FLAGS_REG);
14649 emit_insn (gen_sse4_2_pcmpestri (operands[0], operands[2],
14650 operands[3], operands[4],
14651 operands[5], operands[6]));
14653 emit_insn (gen_sse4_2_pcmpestrm (operands[1], operands[2],
14654 operands[3], operands[4],
14655 operands[5], operands[6]));
14656 if (flags && !(ecx || xmm0))
14657 emit_insn (gen_sse4_2_pcmpestr_cconly (NULL, NULL,
14658 operands[2], operands[3],
14659 operands[4], operands[5],
14661 if (!(flags || ecx || xmm0))
14662 emit_note (NOTE_INSN_DELETED);
14666 [(set_attr "type" "sselog")
14667 (set_attr "prefix_data16" "1")
14668 (set_attr "prefix_extra" "1")
14669 (set_attr "ssememalign" "8")
14670 (set_attr "length_immediate" "1")
14671 (set_attr "memory" "none,load")
14672 (set_attr "mode" "TI")])
14674 (define_insn_and_split "*sse4_2_pcmpestr_unaligned"
14675 [(set (match_operand:SI 0 "register_operand" "=c")
14677 [(match_operand:V16QI 2 "register_operand" "x")
14678 (match_operand:SI 3 "register_operand" "a")
14680 [(match_operand:V16QI 4 "memory_operand" "m")]
14682 (match_operand:SI 5 "register_operand" "d")
14683 (match_operand:SI 6 "const_0_to_255_operand" "n")]
14685 (set (match_operand:V16QI 1 "register_operand" "=Yz")
14689 (unspec:V16QI [(match_dup 4)] UNSPEC_LOADU)
14693 (set (reg:CC FLAGS_REG)
14697 (unspec:V16QI [(match_dup 4)] UNSPEC_LOADU)
14702 && can_create_pseudo_p ()"
14707 int ecx = !find_regno_note (curr_insn, REG_UNUSED, REGNO (operands[0]));
14708 int xmm0 = !find_regno_note (curr_insn, REG_UNUSED, REGNO (operands[1]));
14709 int flags = !find_regno_note (curr_insn, REG_UNUSED, FLAGS_REG);
14712 emit_insn (gen_sse4_2_pcmpestri (operands[0], operands[2],
14713 operands[3], operands[4],
14714 operands[5], operands[6]));
14716 emit_insn (gen_sse4_2_pcmpestrm (operands[1], operands[2],
14717 operands[3], operands[4],
14718 operands[5], operands[6]));
14719 if (flags && !(ecx || xmm0))
14720 emit_insn (gen_sse4_2_pcmpestr_cconly (NULL, NULL,
14721 operands[2], operands[3],
14722 operands[4], operands[5],
14724 if (!(flags || ecx || xmm0))
14725 emit_note (NOTE_INSN_DELETED);
14729 [(set_attr "type" "sselog")
14730 (set_attr "prefix_data16" "1")
14731 (set_attr "prefix_extra" "1")
14732 (set_attr "ssememalign" "8")
14733 (set_attr "length_immediate" "1")
14734 (set_attr "memory" "load")
14735 (set_attr "mode" "TI")])
14737 (define_insn "sse4_2_pcmpestri"
14738 [(set (match_operand:SI 0 "register_operand" "=c,c")
14740 [(match_operand:V16QI 1 "register_operand" "x,x")
14741 (match_operand:SI 2 "register_operand" "a,a")
14742 (match_operand:V16QI 3 "nonimmediate_operand" "x,m")
14743 (match_operand:SI 4 "register_operand" "d,d")
14744 (match_operand:SI 5 "const_0_to_255_operand" "n,n")]
14746 (set (reg:CC FLAGS_REG)
14755 "%vpcmpestri\t{%5, %3, %1|%1, %3, %5}"
14756 [(set_attr "type" "sselog")
14757 (set_attr "prefix_data16" "1")
14758 (set_attr "prefix_extra" "1")
14759 (set_attr "prefix" "maybe_vex")
14760 (set_attr "ssememalign" "8")
14761 (set_attr "length_immediate" "1")
14762 (set_attr "btver2_decode" "vector")
14763 (set_attr "memory" "none,load")
14764 (set_attr "mode" "TI")])
14766 (define_insn "sse4_2_pcmpestrm"
14767 [(set (match_operand:V16QI 0 "register_operand" "=Yz,Yz")
14769 [(match_operand:V16QI 1 "register_operand" "x,x")
14770 (match_operand:SI 2 "register_operand" "a,a")
14771 (match_operand:V16QI 3 "nonimmediate_operand" "x,m")
14772 (match_operand:SI 4 "register_operand" "d,d")
14773 (match_operand:SI 5 "const_0_to_255_operand" "n,n")]
14775 (set (reg:CC FLAGS_REG)
14784 "%vpcmpestrm\t{%5, %3, %1|%1, %3, %5}"
14785 [(set_attr "type" "sselog")
14786 (set_attr "prefix_data16" "1")
14787 (set_attr "prefix_extra" "1")
14788 (set_attr "ssememalign" "8")
14789 (set_attr "length_immediate" "1")
14790 (set_attr "prefix" "maybe_vex")
14791 (set_attr "btver2_decode" "vector")
14792 (set_attr "memory" "none,load")
14793 (set_attr "mode" "TI")])
14795 (define_insn "sse4_2_pcmpestr_cconly"
14796 [(set (reg:CC FLAGS_REG)
14798 [(match_operand:V16QI 2 "register_operand" "x,x,x,x")
14799 (match_operand:SI 3 "register_operand" "a,a,a,a")
14800 (match_operand:V16QI 4 "nonimmediate_operand" "x,m,x,m")
14801 (match_operand:SI 5 "register_operand" "d,d,d,d")
14802 (match_operand:SI 6 "const_0_to_255_operand" "n,n,n,n")]
14804 (clobber (match_scratch:V16QI 0 "=Yz,Yz,X,X"))
14805 (clobber (match_scratch:SI 1 "= X, X,c,c"))]
14808 %vpcmpestrm\t{%6, %4, %2|%2, %4, %6}
14809 %vpcmpestrm\t{%6, %4, %2|%2, %4, %6}
14810 %vpcmpestri\t{%6, %4, %2|%2, %4, %6}
14811 %vpcmpestri\t{%6, %4, %2|%2, %4, %6}"
14812 [(set_attr "type" "sselog")
14813 (set_attr "prefix_data16" "1")
14814 (set_attr "prefix_extra" "1")
14815 (set_attr "ssememalign" "8")
14816 (set_attr "length_immediate" "1")
14817 (set_attr "memory" "none,load,none,load")
14818 (set_attr "btver2_decode" "vector,vector,vector,vector")
14819 (set_attr "prefix" "maybe_vex")
14820 (set_attr "mode" "TI")])
14822 (define_insn_and_split "sse4_2_pcmpistr"
14823 [(set (match_operand:SI 0 "register_operand" "=c,c")
14825 [(match_operand:V16QI 2 "register_operand" "x,x")
14826 (match_operand:V16QI 3 "nonimmediate_operand" "x,m")
14827 (match_operand:SI 4 "const_0_to_255_operand" "n,n")]
14829 (set (match_operand:V16QI 1 "register_operand" "=Yz,Yz")
14835 (set (reg:CC FLAGS_REG)
14842 && can_create_pseudo_p ()"
14847 int ecx = !find_regno_note (curr_insn, REG_UNUSED, REGNO (operands[0]));
14848 int xmm0 = !find_regno_note (curr_insn, REG_UNUSED, REGNO (operands[1]));
14849 int flags = !find_regno_note (curr_insn, REG_UNUSED, FLAGS_REG);
14852 emit_insn (gen_sse4_2_pcmpistri (operands[0], operands[2],
14853 operands[3], operands[4]));
14855 emit_insn (gen_sse4_2_pcmpistrm (operands[1], operands[2],
14856 operands[3], operands[4]));
14857 if (flags && !(ecx || xmm0))
14858 emit_insn (gen_sse4_2_pcmpistr_cconly (NULL, NULL,
14859 operands[2], operands[3],
14861 if (!(flags || ecx || xmm0))
14862 emit_note (NOTE_INSN_DELETED);
14866 [(set_attr "type" "sselog")
14867 (set_attr "prefix_data16" "1")
14868 (set_attr "prefix_extra" "1")
14869 (set_attr "ssememalign" "8")
14870 (set_attr "length_immediate" "1")
14871 (set_attr "memory" "none,load")
14872 (set_attr "mode" "TI")])
14874 (define_insn_and_split "*sse4_2_pcmpistr_unaligned"
14875 [(set (match_operand:SI 0 "register_operand" "=c")
14877 [(match_operand:V16QI 2 "register_operand" "x")
14879 [(match_operand:V16QI 3 "memory_operand" "m")]
14881 (match_operand:SI 4 "const_0_to_255_operand" "n")]
14883 (set (match_operand:V16QI 1 "register_operand" "=Yz")
14886 (unspec:V16QI [(match_dup 3)] UNSPEC_LOADU)
14889 (set (reg:CC FLAGS_REG)
14892 (unspec:V16QI [(match_dup 3)] UNSPEC_LOADU)
14896 && can_create_pseudo_p ()"
14901 int ecx = !find_regno_note (curr_insn, REG_UNUSED, REGNO (operands[0]));
14902 int xmm0 = !find_regno_note (curr_insn, REG_UNUSED, REGNO (operands[1]));
14903 int flags = !find_regno_note (curr_insn, REG_UNUSED, FLAGS_REG);
14906 emit_insn (gen_sse4_2_pcmpistri (operands[0], operands[2],
14907 operands[3], operands[4]));
14909 emit_insn (gen_sse4_2_pcmpistrm (operands[1], operands[2],
14910 operands[3], operands[4]));
14911 if (flags && !(ecx || xmm0))
14912 emit_insn (gen_sse4_2_pcmpistr_cconly (NULL, NULL,
14913 operands[2], operands[3],
14915 if (!(flags || ecx || xmm0))
14916 emit_note (NOTE_INSN_DELETED);
14920 [(set_attr "type" "sselog")
14921 (set_attr "prefix_data16" "1")
14922 (set_attr "prefix_extra" "1")
14923 (set_attr "ssememalign" "8")
14924 (set_attr "length_immediate" "1")
14925 (set_attr "memory" "load")
14926 (set_attr "mode" "TI")])
14928 (define_insn "sse4_2_pcmpistri"
14929 [(set (match_operand:SI 0 "register_operand" "=c,c")
14931 [(match_operand:V16QI 1 "register_operand" "x,x")
14932 (match_operand:V16QI 2 "nonimmediate_operand" "x,m")
14933 (match_operand:SI 3 "const_0_to_255_operand" "n,n")]
14935 (set (reg:CC FLAGS_REG)
14942 "%vpcmpistri\t{%3, %2, %1|%1, %2, %3}"
14943 [(set_attr "type" "sselog")
14944 (set_attr "prefix_data16" "1")
14945 (set_attr "prefix_extra" "1")
14946 (set_attr "ssememalign" "8")
14947 (set_attr "length_immediate" "1")
14948 (set_attr "prefix" "maybe_vex")
14949 (set_attr "memory" "none,load")
14950 (set_attr "btver2_decode" "vector")
14951 (set_attr "mode" "TI")])
14953 (define_insn "sse4_2_pcmpistrm"
14954 [(set (match_operand:V16QI 0 "register_operand" "=Yz,Yz")
14956 [(match_operand:V16QI 1 "register_operand" "x,x")
14957 (match_operand:V16QI 2 "nonimmediate_operand" "x,m")
14958 (match_operand:SI 3 "const_0_to_255_operand" "n,n")]
14960 (set (reg:CC FLAGS_REG)
14967 "%vpcmpistrm\t{%3, %2, %1|%1, %2, %3}"
14968 [(set_attr "type" "sselog")
14969 (set_attr "prefix_data16" "1")
14970 (set_attr "prefix_extra" "1")
14971 (set_attr "ssememalign" "8")
14972 (set_attr "length_immediate" "1")
14973 (set_attr "prefix" "maybe_vex")
14974 (set_attr "memory" "none,load")
14975 (set_attr "btver2_decode" "vector")
14976 (set_attr "mode" "TI")])
14978 (define_insn "sse4_2_pcmpistr_cconly"
14979 [(set (reg:CC FLAGS_REG)
14981 [(match_operand:V16QI 2 "register_operand" "x,x,x,x")
14982 (match_operand:V16QI 3 "nonimmediate_operand" "x,m,x,m")
14983 (match_operand:SI 4 "const_0_to_255_operand" "n,n,n,n")]
14985 (clobber (match_scratch:V16QI 0 "=Yz,Yz,X,X"))
14986 (clobber (match_scratch:SI 1 "= X, X,c,c"))]
14989 %vpcmpistrm\t{%4, %3, %2|%2, %3, %4}
14990 %vpcmpistrm\t{%4, %3, %2|%2, %3, %4}
14991 %vpcmpistri\t{%4, %3, %2|%2, %3, %4}
14992 %vpcmpistri\t{%4, %3, %2|%2, %3, %4}"
14993 [(set_attr "type" "sselog")
14994 (set_attr "prefix_data16" "1")
14995 (set_attr "prefix_extra" "1")
14996 (set_attr "ssememalign" "8")
14997 (set_attr "length_immediate" "1")
14998 (set_attr "memory" "none,load,none,load")
14999 (set_attr "prefix" "maybe_vex")
15000 (set_attr "btver2_decode" "vector,vector,vector,vector")
15001 (set_attr "mode" "TI")])
15003 ;; Packed float variants
15004 (define_mode_attr GATHER_SCATTER_SF_MEM_MODE
15005 [(V8DI "V8SF") (V16SI "V16SF")])
15007 (define_expand "avx512pf_gatherpf<mode>sf"
15009 [(match_operand:<avx512fmaskmode> 0 "register_or_constm1_operand")
15010 (mem:<GATHER_SCATTER_SF_MEM_MODE>
15012 [(match_operand 2 "vsib_address_operand")
15013 (match_operand:VI48_512 1 "register_operand")
15014 (match_operand:SI 3 "const1248_operand")]))
15015 (match_operand:SI 4 "const_2_to_3_operand")]
15016 UNSPEC_GATHER_PREFETCH)]
15020 = gen_rtx_UNSPEC (Pmode, gen_rtvec (3, operands[2], operands[1],
15021 operands[3]), UNSPEC_VSIBADDR);
15024 (define_insn "*avx512pf_gatherpf<mode>sf_mask"
15026 [(match_operand:<avx512fmaskmode> 0 "register_operand" "Yk")
15027 (match_operator:<GATHER_SCATTER_SF_MEM_MODE> 5 "vsib_mem_operator"
15029 [(match_operand:P 2 "vsib_address_operand" "Tv")
15030 (match_operand:VI48_512 1 "register_operand" "v")
15031 (match_operand:SI 3 "const1248_operand" "n")]
15033 (match_operand:SI 4 "const_2_to_3_operand" "n")]
15034 UNSPEC_GATHER_PREFETCH)]
15037 switch (INTVAL (operands[4]))
15040 return "vgatherpf0<ssemodesuffix>ps\t{%5%{%0%}|%5%{%0%}}";
15042 return "vgatherpf1<ssemodesuffix>ps\t{%5%{%0%}|%5%{%0%}}";
15044 gcc_unreachable ();
15047 [(set_attr "type" "sse")
15048 (set_attr "prefix" "evex")
15049 (set_attr "mode" "XI")])
15051 (define_insn "*avx512pf_gatherpf<mode>sf"
15054 (match_operator:<GATHER_SCATTER_SF_MEM_MODE> 4 "vsib_mem_operator"
15056 [(match_operand:P 1 "vsib_address_operand" "Tv")
15057 (match_operand:VI48_512 0 "register_operand" "v")
15058 (match_operand:SI 2 "const1248_operand" "n")]
15060 (match_operand:SI 3 "const_2_to_3_operand" "n")]
15061 UNSPEC_GATHER_PREFETCH)]
15064 switch (INTVAL (operands[3]))
15067 return "vgatherpf0<ssemodesuffix>ps\t{%4|%4}";
15069 return "vgatherpf1<ssemodesuffix>ps\t{%4|%4}";
15071 gcc_unreachable ();
15074 [(set_attr "type" "sse")
15075 (set_attr "prefix" "evex")
15076 (set_attr "mode" "XI")])
15078 ;; Packed double variants
15079 (define_expand "avx512pf_gatherpf<mode>df"
15081 [(match_operand:<avx512fmaskmode> 0 "register_or_constm1_operand")
15084 [(match_operand 2 "vsib_address_operand")
15085 (match_operand:VI4_256_8_512 1 "register_operand")
15086 (match_operand:SI 3 "const1248_operand")]))
15087 (match_operand:SI 4 "const_2_to_3_operand")]
15088 UNSPEC_GATHER_PREFETCH)]
15092 = gen_rtx_UNSPEC (Pmode, gen_rtvec (3, operands[2], operands[1],
15093 operands[3]), UNSPEC_VSIBADDR);
15096 (define_insn "*avx512pf_gatherpf<mode>df_mask"
15098 [(match_operand:<avx512fmaskmode> 0 "register_operand" "Yk")
15099 (match_operator:V8DF 5 "vsib_mem_operator"
15101 [(match_operand:P 2 "vsib_address_operand" "Tv")
15102 (match_operand:VI4_256_8_512 1 "register_operand" "v")
15103 (match_operand:SI 3 "const1248_operand" "n")]
15105 (match_operand:SI 4 "const_2_to_3_operand" "n")]
15106 UNSPEC_GATHER_PREFETCH)]
15109 switch (INTVAL (operands[4]))
15112 return "vgatherpf0<ssemodesuffix>pd\t{%5%{%0%}|%5%{%0%}}";
15114 return "vgatherpf1<ssemodesuffix>pd\t{%5%{%0%}|%5%{%0%}}";
15116 gcc_unreachable ();
15119 [(set_attr "type" "sse")
15120 (set_attr "prefix" "evex")
15121 (set_attr "mode" "XI")])
15123 (define_insn "*avx512pf_gatherpf<mode>df"
15126 (match_operator:V8DF 4 "vsib_mem_operator"
15128 [(match_operand:P 1 "vsib_address_operand" "Tv")
15129 (match_operand:VI4_256_8_512 0 "register_operand" "v")
15130 (match_operand:SI 2 "const1248_operand" "n")]
15132 (match_operand:SI 3 "const_2_to_3_operand" "n")]
15133 UNSPEC_GATHER_PREFETCH)]
15136 switch (INTVAL (operands[3]))
15139 return "vgatherpf0<ssemodesuffix>pd\t{%4|%4}";
15141 return "vgatherpf1<ssemodesuffix>pd\t{%4|%4}";
15143 gcc_unreachable ();
15146 [(set_attr "type" "sse")
15147 (set_attr "prefix" "evex")
15148 (set_attr "mode" "XI")])
15150 ;; Packed float variants
15151 (define_expand "avx512pf_scatterpf<mode>sf"
15153 [(match_operand:<avx512fmaskmode> 0 "register_or_constm1_operand")
15154 (mem:<GATHER_SCATTER_SF_MEM_MODE>
15156 [(match_operand 2 "vsib_address_operand")
15157 (match_operand:VI48_512 1 "register_operand")
15158 (match_operand:SI 3 "const1248_operand")]))
15159 (match_operand:SI 4 "const2367_operand")]
15160 UNSPEC_SCATTER_PREFETCH)]
15164 = gen_rtx_UNSPEC (Pmode, gen_rtvec (3, operands[2], operands[1],
15165 operands[3]), UNSPEC_VSIBADDR);
15168 (define_insn "*avx512pf_scatterpf<mode>sf_mask"
15170 [(match_operand:<avx512fmaskmode> 0 "register_operand" "Yk")
15171 (match_operator:<GATHER_SCATTER_SF_MEM_MODE> 5 "vsib_mem_operator"
15173 [(match_operand:P 2 "vsib_address_operand" "Tv")
15174 (match_operand:VI48_512 1 "register_operand" "v")
15175 (match_operand:SI 3 "const1248_operand" "n")]
15177 (match_operand:SI 4 "const2367_operand" "n")]
15178 UNSPEC_SCATTER_PREFETCH)]
15181 switch (INTVAL (operands[4]))
15185 return "vscatterpf0<ssemodesuffix>ps\t{%5%{%0%}|%5%{%0%}}";
15188 return "vscatterpf1<ssemodesuffix>ps\t{%5%{%0%}|%5%{%0%}}";
15190 gcc_unreachable ();
15193 [(set_attr "type" "sse")
15194 (set_attr "prefix" "evex")
15195 (set_attr "mode" "XI")])
15197 (define_insn "*avx512pf_scatterpf<mode>sf"
15200 (match_operator:<GATHER_SCATTER_SF_MEM_MODE> 4 "vsib_mem_operator"
15202 [(match_operand:P 1 "vsib_address_operand" "Tv")
15203 (match_operand:VI48_512 0 "register_operand" "v")
15204 (match_operand:SI 2 "const1248_operand" "n")]
15206 (match_operand:SI 3 "const2367_operand" "n")]
15207 UNSPEC_SCATTER_PREFETCH)]
15210 switch (INTVAL (operands[3]))
15214 return "vscatterpf0<ssemodesuffix>ps\t{%4|%4}";
15217 return "vscatterpf1<ssemodesuffix>ps\t{%4|%4}";
15219 gcc_unreachable ();
15222 [(set_attr "type" "sse")
15223 (set_attr "prefix" "evex")
15224 (set_attr "mode" "XI")])
15226 ;; Packed double variants
15227 (define_expand "avx512pf_scatterpf<mode>df"
15229 [(match_operand:<avx512fmaskmode> 0 "register_or_constm1_operand")
15232 [(match_operand 2 "vsib_address_operand")
15233 (match_operand:VI4_256_8_512 1 "register_operand")
15234 (match_operand:SI 3 "const1248_operand")]))
15235 (match_operand:SI 4 "const2367_operand")]
15236 UNSPEC_SCATTER_PREFETCH)]
15240 = gen_rtx_UNSPEC (Pmode, gen_rtvec (3, operands[2], operands[1],
15241 operands[3]), UNSPEC_VSIBADDR);
15244 (define_insn "*avx512pf_scatterpf<mode>df_mask"
15246 [(match_operand:<avx512fmaskmode> 0 "register_operand" "Yk")
15247 (match_operator:V8DF 5 "vsib_mem_operator"
15249 [(match_operand:P 2 "vsib_address_operand" "Tv")
15250 (match_operand:VI4_256_8_512 1 "register_operand" "v")
15251 (match_operand:SI 3 "const1248_operand" "n")]
15253 (match_operand:SI 4 "const2367_operand" "n")]
15254 UNSPEC_SCATTER_PREFETCH)]
15257 switch (INTVAL (operands[4]))
15261 return "vscatterpf0<ssemodesuffix>pd\t{%5%{%0%}|%5%{%0%}}";
15264 return "vscatterpf1<ssemodesuffix>pd\t{%5%{%0%}|%5%{%0%}}";
15266 gcc_unreachable ();
15269 [(set_attr "type" "sse")
15270 (set_attr "prefix" "evex")
15271 (set_attr "mode" "XI")])
15273 (define_insn "*avx512pf_scatterpf<mode>df"
15276 (match_operator:V8DF 4 "vsib_mem_operator"
15278 [(match_operand:P 1 "vsib_address_operand" "Tv")
15279 (match_operand:VI4_256_8_512 0 "register_operand" "v")
15280 (match_operand:SI 2 "const1248_operand" "n")]
15282 (match_operand:SI 3 "const2367_operand" "n")]
15283 UNSPEC_SCATTER_PREFETCH)]
15286 switch (INTVAL (operands[3]))
15290 return "vscatterpf0<ssemodesuffix>pd\t{%4|%4}";
15293 return "vscatterpf1<ssemodesuffix>pd\t{%4|%4}";
15295 gcc_unreachable ();
15298 [(set_attr "type" "sse")
15299 (set_attr "prefix" "evex")
15300 (set_attr "mode" "XI")])
15302 (define_insn "avx512er_exp2<mode><mask_name><round_saeonly_name>"
15303 [(set (match_operand:VF_512 0 "register_operand" "=v")
15305 [(match_operand:VF_512 1 "<round_saeonly_nimm_predicate>" "<round_saeonly_constraint>")]
15308 "vexp2<ssemodesuffix>\t{<round_saeonly_mask_op2>%1, %0<mask_operand2>|%0<mask_operand2>, %1<round_saeonly_mask_op2>}"
15309 [(set_attr "prefix" "evex")
15310 (set_attr "type" "sse")
15311 (set_attr "mode" "<MODE>")])
15313 (define_insn "<mask_codefor>avx512er_rcp28<mode><mask_name><round_saeonly_name>"
15314 [(set (match_operand:VF_512 0 "register_operand" "=v")
15316 [(match_operand:VF_512 1 "<round_saeonly_nimm_predicate>" "<round_saeonly_constraint>")]
15319 "vrcp28<ssemodesuffix>\t{<round_saeonly_mask_op2>%1, %0<mask_operand2>|%0<mask_operand2>, %1<round_saeonly_mask_op2>}"
15320 [(set_attr "prefix" "evex")
15321 (set_attr "type" "sse")
15322 (set_attr "mode" "<MODE>")])
15324 (define_insn "avx512er_vmrcp28<mode><round_saeonly_name>"
15325 [(set (match_operand:VF_128 0 "register_operand" "=v")
15328 [(match_operand:VF_128 1 "<round_saeonly_nimm_predicate>" "<round_saeonly_constraint>")]
15330 (match_operand:VF_128 2 "register_operand" "v")
15333 "vrcp28<ssescalarmodesuffix>\t{<round_saeonly_op3>%1, %2, %0|%0, %2, %1<round_saeonly_op3>}"
15334 [(set_attr "length_immediate" "1")
15335 (set_attr "prefix" "evex")
15336 (set_attr "type" "sse")
15337 (set_attr "mode" "<MODE>")])
15339 (define_insn "<mask_codefor>avx512er_rsqrt28<mode><mask_name><round_saeonly_name>"
15340 [(set (match_operand:VF_512 0 "register_operand" "=v")
15342 [(match_operand:VF_512 1 "<round_saeonly_nimm_predicate>" "<round_saeonly_constraint>")]
15345 "vrsqrt28<ssemodesuffix>\t{<round_saeonly_mask_op2>%1, %0<mask_operand2>|%0<mask_operand2>, %1<round_saeonly_mask_op2>}"
15346 [(set_attr "prefix" "evex")
15347 (set_attr "type" "sse")
15348 (set_attr "mode" "<MODE>")])
15350 (define_insn "avx512er_vmrsqrt28<mode><round_saeonly_name>"
15351 [(set (match_operand:VF_128 0 "register_operand" "=v")
15354 [(match_operand:VF_128 1 "<round_saeonly_nimm_predicate>" "<round_saeonly_constraint>")]
15356 (match_operand:VF_128 2 "register_operand" "v")
15359 "vrsqrt28<ssescalarmodesuffix>\t{<round_saeonly_op3>%1, %2, %0|%0, %2, %1<round_saeonly_op3>}"
15360 [(set_attr "length_immediate" "1")
15361 (set_attr "type" "sse")
15362 (set_attr "prefix" "evex")
15363 (set_attr "mode" "<MODE>")])
15365 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
15367 ;; XOP instructions
15369 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
15371 (define_code_iterator xop_plus [plus ss_plus])
15373 (define_code_attr macs [(plus "macs") (ss_plus "macss")])
15374 (define_code_attr madcs [(plus "madcs") (ss_plus "madcss")])
15376 ;; XOP parallel integer multiply/add instructions.
15378 (define_insn "xop_p<macs><ssemodesuffix><ssemodesuffix>"
15379 [(set (match_operand:VI24_128 0 "register_operand" "=x")
15382 (match_operand:VI24_128 1 "nonimmediate_operand" "%x")
15383 (match_operand:VI24_128 2 "nonimmediate_operand" "xm"))
15384 (match_operand:VI24_128 3 "register_operand" "x")))]
15386 "vp<macs><ssemodesuffix><ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
15387 [(set_attr "type" "ssemuladd")
15388 (set_attr "mode" "TI")])
15390 (define_insn "xop_p<macs>dql"
15391 [(set (match_operand:V2DI 0 "register_operand" "=x")
15396 (match_operand:V4SI 1 "nonimmediate_operand" "%x")
15397 (parallel [(const_int 0) (const_int 2)])))
15400 (match_operand:V4SI 2 "nonimmediate_operand" "xm")
15401 (parallel [(const_int 0) (const_int 2)]))))
15402 (match_operand:V2DI 3 "register_operand" "x")))]
15404 "vp<macs>dql\t{%3, %2, %1, %0|%0, %1, %2, %3}"
15405 [(set_attr "type" "ssemuladd")
15406 (set_attr "mode" "TI")])
15408 (define_insn "xop_p<macs>dqh"
15409 [(set (match_operand:V2DI 0 "register_operand" "=x")
15414 (match_operand:V4SI 1 "nonimmediate_operand" "%x")
15415 (parallel [(const_int 1) (const_int 3)])))
15418 (match_operand:V4SI 2 "nonimmediate_operand" "xm")
15419 (parallel [(const_int 1) (const_int 3)]))))
15420 (match_operand:V2DI 3 "register_operand" "x")))]
15422 "vp<macs>dqh\t{%3, %2, %1, %0|%0, %1, %2, %3}"
15423 [(set_attr "type" "ssemuladd")
15424 (set_attr "mode" "TI")])
15426 ;; XOP parallel integer multiply/add instructions for the intrinisics
15427 (define_insn "xop_p<macs>wd"
15428 [(set (match_operand:V4SI 0 "register_operand" "=x")
15433 (match_operand:V8HI 1 "nonimmediate_operand" "%x")
15434 (parallel [(const_int 1) (const_int 3)
15435 (const_int 5) (const_int 7)])))
15438 (match_operand:V8HI 2 "nonimmediate_operand" "xm")
15439 (parallel [(const_int 1) (const_int 3)
15440 (const_int 5) (const_int 7)]))))
15441 (match_operand:V4SI 3 "register_operand" "x")))]
15443 "vp<macs>wd\t{%3, %2, %1, %0|%0, %1, %2, %3}"
15444 [(set_attr "type" "ssemuladd")
15445 (set_attr "mode" "TI")])
15447 (define_insn "xop_p<madcs>wd"
15448 [(set (match_operand:V4SI 0 "register_operand" "=x")
15454 (match_operand:V8HI 1 "nonimmediate_operand" "%x")
15455 (parallel [(const_int 0) (const_int 2)
15456 (const_int 4) (const_int 6)])))
15459 (match_operand:V8HI 2 "nonimmediate_operand" "xm")
15460 (parallel [(const_int 0) (const_int 2)
15461 (const_int 4) (const_int 6)]))))
15466 (parallel [(const_int 1) (const_int 3)
15467 (const_int 5) (const_int 7)])))
15471 (parallel [(const_int 1) (const_int 3)
15472 (const_int 5) (const_int 7)])))))
15473 (match_operand:V4SI 3 "register_operand" "x")))]
15475 "vp<madcs>wd\t{%3, %2, %1, %0|%0, %1, %2, %3}"
15476 [(set_attr "type" "ssemuladd")
15477 (set_attr "mode" "TI")])
15479 ;; XOP parallel XMM conditional moves
15480 (define_insn "xop_pcmov_<mode><avxsizesuffix>"
15481 [(set (match_operand:V 0 "register_operand" "=x,x")
15483 (match_operand:V 3 "nonimmediate_operand" "x,m")
15484 (match_operand:V 1 "register_operand" "x,x")
15485 (match_operand:V 2 "nonimmediate_operand" "xm,x")))]
15487 "vpcmov\t{%3, %2, %1, %0|%0, %1, %2, %3}"
15488 [(set_attr "type" "sse4arg")])
15490 ;; XOP horizontal add/subtract instructions
15491 (define_insn "xop_phadd<u>bw"
15492 [(set (match_operand:V8HI 0 "register_operand" "=x")
15496 (match_operand:V16QI 1 "nonimmediate_operand" "xm")
15497 (parallel [(const_int 0) (const_int 2)
15498 (const_int 4) (const_int 6)
15499 (const_int 8) (const_int 10)
15500 (const_int 12) (const_int 14)])))
15504 (parallel [(const_int 1) (const_int 3)
15505 (const_int 5) (const_int 7)
15506 (const_int 9) (const_int 11)
15507 (const_int 13) (const_int 15)])))))]
15509 "vphadd<u>bw\t{%1, %0|%0, %1}"
15510 [(set_attr "type" "sseiadd1")])
15512 (define_insn "xop_phadd<u>bd"
15513 [(set (match_operand:V4SI 0 "register_operand" "=x")
15518 (match_operand:V16QI 1 "nonimmediate_operand" "xm")
15519 (parallel [(const_int 0) (const_int 4)
15520 (const_int 8) (const_int 12)])))
15524 (parallel [(const_int 1) (const_int 5)
15525 (const_int 9) (const_int 13)]))))
15530 (parallel [(const_int 2) (const_int 6)
15531 (const_int 10) (const_int 14)])))
15535 (parallel [(const_int 3) (const_int 7)
15536 (const_int 11) (const_int 15)]))))))]
15538 "vphadd<u>bd\t{%1, %0|%0, %1}"
15539 [(set_attr "type" "sseiadd1")])
15541 (define_insn "xop_phadd<u>bq"
15542 [(set (match_operand:V2DI 0 "register_operand" "=x")
15548 (match_operand:V16QI 1 "nonimmediate_operand" "xm")
15549 (parallel [(const_int 0) (const_int 8)])))
15553 (parallel [(const_int 1) (const_int 9)]))))
15558 (parallel [(const_int 2) (const_int 10)])))
15562 (parallel [(const_int 3) (const_int 11)])))))
15568 (parallel [(const_int 4) (const_int 12)])))
15572 (parallel [(const_int 5) (const_int 13)]))))
15577 (parallel [(const_int 6) (const_int 14)])))
15581 (parallel [(const_int 7) (const_int 15)])))))))]
15583 "vphadd<u>bq\t{%1, %0|%0, %1}"
15584 [(set_attr "type" "sseiadd1")])
15586 (define_insn "xop_phadd<u>wd"
15587 [(set (match_operand:V4SI 0 "register_operand" "=x")
15591 (match_operand:V8HI 1 "nonimmediate_operand" "xm")
15592 (parallel [(const_int 0) (const_int 2)
15593 (const_int 4) (const_int 6)])))
15597 (parallel [(const_int 1) (const_int 3)
15598 (const_int 5) (const_int 7)])))))]
15600 "vphadd<u>wd\t{%1, %0|%0, %1}"
15601 [(set_attr "type" "sseiadd1")])
15603 (define_insn "xop_phadd<u>wq"
15604 [(set (match_operand:V2DI 0 "register_operand" "=x")
15609 (match_operand:V8HI 1 "nonimmediate_operand" "xm")
15610 (parallel [(const_int 0) (const_int 4)])))
15614 (parallel [(const_int 1) (const_int 5)]))))
15619 (parallel [(const_int 2) (const_int 6)])))
15623 (parallel [(const_int 3) (const_int 7)]))))))]
15625 "vphadd<u>wq\t{%1, %0|%0, %1}"
15626 [(set_attr "type" "sseiadd1")])
15628 (define_insn "xop_phadd<u>dq"
15629 [(set (match_operand:V2DI 0 "register_operand" "=x")
15633 (match_operand:V4SI 1 "nonimmediate_operand" "xm")
15634 (parallel [(const_int 0) (const_int 2)])))
15638 (parallel [(const_int 1) (const_int 3)])))))]
15640 "vphadd<u>dq\t{%1, %0|%0, %1}"
15641 [(set_attr "type" "sseiadd1")])
15643 (define_insn "xop_phsubbw"
15644 [(set (match_operand:V8HI 0 "register_operand" "=x")
15648 (match_operand:V16QI 1 "nonimmediate_operand" "xm")
15649 (parallel [(const_int 0) (const_int 2)
15650 (const_int 4) (const_int 6)
15651 (const_int 8) (const_int 10)
15652 (const_int 12) (const_int 14)])))
15656 (parallel [(const_int 1) (const_int 3)
15657 (const_int 5) (const_int 7)
15658 (const_int 9) (const_int 11)
15659 (const_int 13) (const_int 15)])))))]
15661 "vphsubbw\t{%1, %0|%0, %1}"
15662 [(set_attr "type" "sseiadd1")])
15664 (define_insn "xop_phsubwd"
15665 [(set (match_operand:V4SI 0 "register_operand" "=x")
15669 (match_operand:V8HI 1 "nonimmediate_operand" "xm")
15670 (parallel [(const_int 0) (const_int 2)
15671 (const_int 4) (const_int 6)])))
15675 (parallel [(const_int 1) (const_int 3)
15676 (const_int 5) (const_int 7)])))))]
15678 "vphsubwd\t{%1, %0|%0, %1}"
15679 [(set_attr "type" "sseiadd1")])
15681 (define_insn "xop_phsubdq"
15682 [(set (match_operand:V2DI 0 "register_operand" "=x")
15686 (match_operand:V4SI 1 "nonimmediate_operand" "xm")
15687 (parallel [(const_int 0) (const_int 2)])))
15691 (parallel [(const_int 1) (const_int 3)])))))]
15693 "vphsubdq\t{%1, %0|%0, %1}"
15694 [(set_attr "type" "sseiadd1")])
15696 ;; XOP permute instructions
15697 (define_insn "xop_pperm"
15698 [(set (match_operand:V16QI 0 "register_operand" "=x,x")
15700 [(match_operand:V16QI 1 "register_operand" "x,x")
15701 (match_operand:V16QI 2 "nonimmediate_operand" "x,m")
15702 (match_operand:V16QI 3 "nonimmediate_operand" "xm,x")]
15703 UNSPEC_XOP_PERMUTE))]
15704 "TARGET_XOP && !(MEM_P (operands[2]) && MEM_P (operands[3]))"
15705 "vpperm\t{%3, %2, %1, %0|%0, %1, %2, %3}"
15706 [(set_attr "type" "sse4arg")
15707 (set_attr "mode" "TI")])
15709 ;; XOP pack instructions that combine two vectors into a smaller vector
15710 (define_insn "xop_pperm_pack_v2di_v4si"
15711 [(set (match_operand:V4SI 0 "register_operand" "=x,x")
15714 (match_operand:V2DI 1 "register_operand" "x,x"))
15716 (match_operand:V2DI 2 "nonimmediate_operand" "x,m"))))
15717 (use (match_operand:V16QI 3 "nonimmediate_operand" "xm,x"))]
15718 "TARGET_XOP && !(MEM_P (operands[2]) && MEM_P (operands[3]))"
15719 "vpperm\t{%3, %2, %1, %0|%0, %1, %2, %3}"
15720 [(set_attr "type" "sse4arg")
15721 (set_attr "mode" "TI")])
15723 (define_insn "xop_pperm_pack_v4si_v8hi"
15724 [(set (match_operand:V8HI 0 "register_operand" "=x,x")
15727 (match_operand:V4SI 1 "register_operand" "x,x"))
15729 (match_operand:V4SI 2 "nonimmediate_operand" "x,m"))))
15730 (use (match_operand:V16QI 3 "nonimmediate_operand" "xm,x"))]
15731 "TARGET_XOP && !(MEM_P (operands[2]) && MEM_P (operands[3]))"
15732 "vpperm\t{%3, %2, %1, %0|%0, %1, %2, %3}"
15733 [(set_attr "type" "sse4arg")
15734 (set_attr "mode" "TI")])
15736 (define_insn "xop_pperm_pack_v8hi_v16qi"
15737 [(set (match_operand:V16QI 0 "register_operand" "=x,x")
15740 (match_operand:V8HI 1 "register_operand" "x,x"))
15742 (match_operand:V8HI 2 "nonimmediate_operand" "x,m"))))
15743 (use (match_operand:V16QI 3 "nonimmediate_operand" "xm,x"))]
15744 "TARGET_XOP && !(MEM_P (operands[2]) && MEM_P (operands[3]))"
15745 "vpperm\t{%3, %2, %1, %0|%0, %1, %2, %3}"
15746 [(set_attr "type" "sse4arg")
15747 (set_attr "mode" "TI")])
15749 ;; XOP packed rotate instructions
15750 (define_expand "rotl<mode>3"
15751 [(set (match_operand:VI_128 0 "register_operand")
15753 (match_operand:VI_128 1 "nonimmediate_operand")
15754 (match_operand:SI 2 "general_operand")))]
15757 /* If we were given a scalar, convert it to parallel */
15758 if (! const_0_to_<sserotatemax>_operand (operands[2], SImode))
15760 rtvec vs = rtvec_alloc (<ssescalarnum>);
15761 rtx par = gen_rtx_PARALLEL (<MODE>mode, vs);
15762 rtx reg = gen_reg_rtx (<MODE>mode);
15763 rtx op2 = operands[2];
15766 if (GET_MODE (op2) != <ssescalarmode>mode)
15768 op2 = gen_reg_rtx (<ssescalarmode>mode);
15769 convert_move (op2, operands[2], false);
15772 for (i = 0; i < <ssescalarnum>; i++)
15773 RTVEC_ELT (vs, i) = op2;
15775 emit_insn (gen_vec_init<mode> (reg, par));
15776 emit_insn (gen_xop_vrotl<mode>3 (operands[0], operands[1], reg));
15781 (define_expand "rotr<mode>3"
15782 [(set (match_operand:VI_128 0 "register_operand")
15784 (match_operand:VI_128 1 "nonimmediate_operand")
15785 (match_operand:SI 2 "general_operand")))]
15788 /* If we were given a scalar, convert it to parallel */
15789 if (! const_0_to_<sserotatemax>_operand (operands[2], SImode))
15791 rtvec vs = rtvec_alloc (<ssescalarnum>);
15792 rtx par = gen_rtx_PARALLEL (<MODE>mode, vs);
15793 rtx neg = gen_reg_rtx (<MODE>mode);
15794 rtx reg = gen_reg_rtx (<MODE>mode);
15795 rtx op2 = operands[2];
15798 if (GET_MODE (op2) != <ssescalarmode>mode)
15800 op2 = gen_reg_rtx (<ssescalarmode>mode);
15801 convert_move (op2, operands[2], false);
15804 for (i = 0; i < <ssescalarnum>; i++)
15805 RTVEC_ELT (vs, i) = op2;
15807 emit_insn (gen_vec_init<mode> (reg, par));
15808 emit_insn (gen_neg<mode>2 (neg, reg));
15809 emit_insn (gen_xop_vrotl<mode>3 (operands[0], operands[1], neg));
15814 (define_insn "xop_rotl<mode>3"
15815 [(set (match_operand:VI_128 0 "register_operand" "=x")
15817 (match_operand:VI_128 1 "nonimmediate_operand" "xm")
15818 (match_operand:SI 2 "const_0_to_<sserotatemax>_operand" "n")))]
15820 "vprot<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
15821 [(set_attr "type" "sseishft")
15822 (set_attr "length_immediate" "1")
15823 (set_attr "mode" "TI")])
15825 (define_insn "xop_rotr<mode>3"
15826 [(set (match_operand:VI_128 0 "register_operand" "=x")
15828 (match_operand:VI_128 1 "nonimmediate_operand" "xm")
15829 (match_operand:SI 2 "const_0_to_<sserotatemax>_operand" "n")))]
15833 = GEN_INT (GET_MODE_BITSIZE (<ssescalarmode>mode) - INTVAL (operands[2]));
15834 return \"vprot<ssemodesuffix>\t{%3, %1, %0|%0, %1, %3}\";
15836 [(set_attr "type" "sseishft")
15837 (set_attr "length_immediate" "1")
15838 (set_attr "mode" "TI")])
15840 (define_expand "vrotr<mode>3"
15841 [(match_operand:VI_128 0 "register_operand")
15842 (match_operand:VI_128 1 "register_operand")
15843 (match_operand:VI_128 2 "register_operand")]
15846 rtx reg = gen_reg_rtx (<MODE>mode);
15847 emit_insn (gen_neg<mode>2 (reg, operands[2]));
15848 emit_insn (gen_xop_vrotl<mode>3 (operands[0], operands[1], reg));
15852 (define_expand "vrotl<mode>3"
15853 [(match_operand:VI_128 0 "register_operand")
15854 (match_operand:VI_128 1 "register_operand")
15855 (match_operand:VI_128 2 "register_operand")]
15858 emit_insn (gen_xop_vrotl<mode>3 (operands[0], operands[1], operands[2]));
15862 (define_insn "xop_vrotl<mode>3"
15863 [(set (match_operand:VI_128 0 "register_operand" "=x,x")
15864 (if_then_else:VI_128
15866 (match_operand:VI_128 2 "nonimmediate_operand" "x,m")
15869 (match_operand:VI_128 1 "nonimmediate_operand" "xm,x")
15873 (neg:VI_128 (match_dup 2)))))]
15874 "TARGET_XOP && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
15875 "vprot<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
15876 [(set_attr "type" "sseishft")
15877 (set_attr "prefix_data16" "0")
15878 (set_attr "prefix_extra" "2")
15879 (set_attr "mode" "TI")])
15881 ;; XOP packed shift instructions.
15882 (define_expand "vlshr<mode>3"
15883 [(set (match_operand:VI12_128 0 "register_operand")
15885 (match_operand:VI12_128 1 "register_operand")
15886 (match_operand:VI12_128 2 "nonimmediate_operand")))]
15889 rtx neg = gen_reg_rtx (<MODE>mode);
15890 emit_insn (gen_neg<mode>2 (neg, operands[2]));
15891 emit_insn (gen_xop_shl<mode>3 (operands[0], operands[1], neg));
15895 (define_expand "vlshr<mode>3"
15896 [(set (match_operand:VI48_128 0 "register_operand")
15898 (match_operand:VI48_128 1 "register_operand")
15899 (match_operand:VI48_128 2 "nonimmediate_operand")))]
15900 "TARGET_AVX2 || TARGET_XOP"
15904 rtx neg = gen_reg_rtx (<MODE>mode);
15905 emit_insn (gen_neg<mode>2 (neg, operands[2]));
15906 emit_insn (gen_xop_shl<mode>3 (operands[0], operands[1], neg));
15911 (define_expand "vlshr<mode>3"
15912 [(set (match_operand:VI48_512 0 "register_operand")
15914 (match_operand:VI48_512 1 "register_operand")
15915 (match_operand:VI48_512 2 "nonimmediate_operand")))]
15918 (define_expand "vlshr<mode>3"
15919 [(set (match_operand:VI48_256 0 "register_operand")
15921 (match_operand:VI48_256 1 "register_operand")
15922 (match_operand:VI48_256 2 "nonimmediate_operand")))]
15925 (define_expand "vashrv8hi3<mask_name>"
15926 [(set (match_operand:V8HI 0 "register_operand")
15928 (match_operand:V8HI 1 "register_operand")
15929 (match_operand:V8HI 2 "nonimmediate_operand")))]
15930 "TARGET_XOP || (TARGET_AVX512BW && TARGET_AVX512VL)"
15934 rtx neg = gen_reg_rtx (V8HImode);
15935 emit_insn (gen_negv8hi2 (neg, operands[2]));
15936 emit_insn (gen_xop_shav8hi3 (operands[0], operands[1], neg));
15941 (define_expand "vashrv16qi3"
15942 [(set (match_operand:V16QI 0 "register_operand")
15944 (match_operand:V16QI 1 "register_operand")
15945 (match_operand:V16QI 2 "nonimmediate_operand")))]
15948 rtx neg = gen_reg_rtx (V16QImode);
15949 emit_insn (gen_negv16qi2 (neg, operands[2]));
15950 emit_insn (gen_xop_shav16qi3 (operands[0], operands[1], neg));
15954 (define_expand "vashrv2di3<mask_name>"
15955 [(set (match_operand:V2DI 0 "register_operand")
15957 (match_operand:V2DI 1 "register_operand")
15958 (match_operand:V2DI 2 "nonimmediate_operand")))]
15959 "TARGET_XOP || TARGET_AVX512VL"
15963 rtx neg = gen_reg_rtx (V2DImode);
15964 emit_insn (gen_negv2di2 (neg, operands[2]));
15965 emit_insn (gen_xop_shav2di3 (operands[0], operands[1], neg));
15970 (define_expand "vashrv4si3"
15971 [(set (match_operand:V4SI 0 "register_operand")
15972 (ashiftrt:V4SI (match_operand:V4SI 1 "register_operand")
15973 (match_operand:V4SI 2 "nonimmediate_operand")))]
15974 "TARGET_AVX2 || TARGET_XOP"
15978 rtx neg = gen_reg_rtx (V4SImode);
15979 emit_insn (gen_negv4si2 (neg, operands[2]));
15980 emit_insn (gen_xop_shav4si3 (operands[0], operands[1], neg));
15985 (define_expand "vashrv16si3"
15986 [(set (match_operand:V16SI 0 "register_operand")
15987 (ashiftrt:V16SI (match_operand:V16SI 1 "register_operand")
15988 (match_operand:V16SI 2 "nonimmediate_operand")))]
15991 (define_expand "vashrv8si3"
15992 [(set (match_operand:V8SI 0 "register_operand")
15993 (ashiftrt:V8SI (match_operand:V8SI 1 "register_operand")
15994 (match_operand:V8SI 2 "nonimmediate_operand")))]
15997 (define_expand "vashl<mode>3"
15998 [(set (match_operand:VI12_128 0 "register_operand")
16000 (match_operand:VI12_128 1 "register_operand")
16001 (match_operand:VI12_128 2 "nonimmediate_operand")))]
16004 emit_insn (gen_xop_sha<mode>3 (operands[0], operands[1], operands[2]));
16008 (define_expand "vashl<mode>3"
16009 [(set (match_operand:VI48_128 0 "register_operand")
16011 (match_operand:VI48_128 1 "register_operand")
16012 (match_operand:VI48_128 2 "nonimmediate_operand")))]
16013 "TARGET_AVX2 || TARGET_XOP"
16017 operands[2] = force_reg (<MODE>mode, operands[2]);
16018 emit_insn (gen_xop_sha<mode>3 (operands[0], operands[1], operands[2]));
16023 (define_expand "vashl<mode>3"
16024 [(set (match_operand:VI48_512 0 "register_operand")
16026 (match_operand:VI48_512 1 "register_operand")
16027 (match_operand:VI48_512 2 "nonimmediate_operand")))]
16030 (define_expand "vashl<mode>3"
16031 [(set (match_operand:VI48_256 0 "register_operand")
16033 (match_operand:VI48_256 1 "register_operand")
16034 (match_operand:VI48_256 2 "nonimmediate_operand")))]
16037 (define_insn "xop_sha<mode>3"
16038 [(set (match_operand:VI_128 0 "register_operand" "=x,x")
16039 (if_then_else:VI_128
16041 (match_operand:VI_128 2 "nonimmediate_operand" "x,m")
16044 (match_operand:VI_128 1 "nonimmediate_operand" "xm,x")
16048 (neg:VI_128 (match_dup 2)))))]
16049 "TARGET_XOP && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
16050 "vpsha<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
16051 [(set_attr "type" "sseishft")
16052 (set_attr "prefix_data16" "0")
16053 (set_attr "prefix_extra" "2")
16054 (set_attr "mode" "TI")])
16056 (define_insn "xop_shl<mode>3"
16057 [(set (match_operand:VI_128 0 "register_operand" "=x,x")
16058 (if_then_else:VI_128
16060 (match_operand:VI_128 2 "nonimmediate_operand" "x,m")
16063 (match_operand:VI_128 1 "nonimmediate_operand" "xm,x")
16067 (neg:VI_128 (match_dup 2)))))]
16068 "TARGET_XOP && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
16069 "vpshl<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
16070 [(set_attr "type" "sseishft")
16071 (set_attr "prefix_data16" "0")
16072 (set_attr "prefix_extra" "2")
16073 (set_attr "mode" "TI")])
16075 (define_expand "<shift_insn><mode>3"
16076 [(set (match_operand:VI1_AVX512 0 "register_operand")
16077 (any_shift:VI1_AVX512
16078 (match_operand:VI1_AVX512 1 "register_operand")
16079 (match_operand:SI 2 "nonmemory_operand")))]
16082 if (TARGET_XOP && <MODE>mode == V16QImode)
16084 bool negate = false;
16085 rtx (*gen) (rtx, rtx, rtx);
16089 if (<CODE> != ASHIFT)
16091 if (CONST_INT_P (operands[2]))
16092 operands[2] = GEN_INT (-INTVAL (operands[2]));
16096 par = gen_rtx_PARALLEL (V16QImode, rtvec_alloc (16));
16097 for (i = 0; i < 16; i++)
16098 XVECEXP (par, 0, i) = operands[2];
16100 tmp = gen_reg_rtx (V16QImode);
16101 emit_insn (gen_vec_initv16qi (tmp, par));
16104 emit_insn (gen_negv16qi2 (tmp, tmp));
16106 gen = (<CODE> == LSHIFTRT ? gen_xop_shlv16qi3 : gen_xop_shav16qi3);
16107 emit_insn (gen (operands[0], operands[1], tmp));
16110 ix86_expand_vecop_qihi (<CODE>, operands[0], operands[1], operands[2]);
16114 (define_expand "ashrv2di3"
16115 [(set (match_operand:V2DI 0 "register_operand")
16117 (match_operand:V2DI 1 "register_operand")
16118 (match_operand:DI 2 "nonmemory_operand")))]
16119 "TARGET_XOP || TARGET_AVX512VL"
16121 if (!TARGET_AVX512VL)
16123 rtx reg = gen_reg_rtx (V2DImode);
16125 bool negate = false;
16128 if (CONST_INT_P (operands[2]))
16129 operands[2] = GEN_INT (-INTVAL (operands[2]));
16133 par = gen_rtx_PARALLEL (V2DImode, rtvec_alloc (2));
16134 for (i = 0; i < 2; i++)
16135 XVECEXP (par, 0, i) = operands[2];
16137 emit_insn (gen_vec_initv2di (reg, par));
16140 emit_insn (gen_negv2di2 (reg, reg));
16142 emit_insn (gen_xop_shav2di3 (operands[0], operands[1], reg));
16147 ;; XOP FRCZ support
16148 (define_insn "xop_frcz<mode>2"
16149 [(set (match_operand:FMAMODE 0 "register_operand" "=x")
16151 [(match_operand:FMAMODE 1 "nonimmediate_operand" "xm")]
16154 "vfrcz<ssemodesuffix>\t{%1, %0|%0, %1}"
16155 [(set_attr "type" "ssecvt1")
16156 (set_attr "mode" "<MODE>")])
16158 (define_expand "xop_vmfrcz<mode>2"
16159 [(set (match_operand:VF_128 0 "register_operand")
16162 [(match_operand:VF_128 1 "nonimmediate_operand")]
16167 "operands[2] = CONST0_RTX (<MODE>mode);")
16169 (define_insn "*xop_vmfrcz<mode>2"
16170 [(set (match_operand:VF_128 0 "register_operand" "=x")
16173 [(match_operand:VF_128 1 "nonimmediate_operand" "xm")]
16175 (match_operand:VF_128 2 "const0_operand")
16178 "vfrcz<ssescalarmodesuffix>\t{%1, %0|%0, %<iptr>1}"
16179 [(set_attr "type" "ssecvt1")
16180 (set_attr "mode" "<MODE>")])
16182 (define_insn "xop_maskcmp<mode>3"
16183 [(set (match_operand:VI_128 0 "register_operand" "=x")
16184 (match_operator:VI_128 1 "ix86_comparison_int_operator"
16185 [(match_operand:VI_128 2 "register_operand" "x")
16186 (match_operand:VI_128 3 "nonimmediate_operand" "xm")]))]
16188 "vpcom%Y1<ssemodesuffix>\t{%3, %2, %0|%0, %2, %3}"
16189 [(set_attr "type" "sse4arg")
16190 (set_attr "prefix_data16" "0")
16191 (set_attr "prefix_rep" "0")
16192 (set_attr "prefix_extra" "2")
16193 (set_attr "length_immediate" "1")
16194 (set_attr "mode" "TI")])
16196 (define_insn "xop_maskcmp_uns<mode>3"
16197 [(set (match_operand:VI_128 0 "register_operand" "=x")
16198 (match_operator:VI_128 1 "ix86_comparison_uns_operator"
16199 [(match_operand:VI_128 2 "register_operand" "x")
16200 (match_operand:VI_128 3 "nonimmediate_operand" "xm")]))]
16202 "vpcom%Y1u<ssemodesuffix>\t{%3, %2, %0|%0, %2, %3}"
16203 [(set_attr "type" "ssecmp")
16204 (set_attr "prefix_data16" "0")
16205 (set_attr "prefix_rep" "0")
16206 (set_attr "prefix_extra" "2")
16207 (set_attr "length_immediate" "1")
16208 (set_attr "mode" "TI")])
16210 ;; Version of pcom*u* that is called from the intrinsics that allows pcomequ*
16211 ;; and pcomneu* not to be converted to the signed ones in case somebody needs
16212 ;; the exact instruction generated for the intrinsic.
16213 (define_insn "xop_maskcmp_uns2<mode>3"
16214 [(set (match_operand:VI_128 0 "register_operand" "=x")
16216 [(match_operator:VI_128 1 "ix86_comparison_uns_operator"
16217 [(match_operand:VI_128 2 "register_operand" "x")
16218 (match_operand:VI_128 3 "nonimmediate_operand" "xm")])]
16219 UNSPEC_XOP_UNSIGNED_CMP))]
16221 "vpcom%Y1u<ssemodesuffix>\t{%3, %2, %0|%0, %2, %3}"
16222 [(set_attr "type" "ssecmp")
16223 (set_attr "prefix_data16" "0")
16224 (set_attr "prefix_extra" "2")
16225 (set_attr "length_immediate" "1")
16226 (set_attr "mode" "TI")])
16228 ;; Pcomtrue and pcomfalse support. These are useless instructions, but are
16229 ;; being added here to be complete.
16230 (define_insn "xop_pcom_tf<mode>3"
16231 [(set (match_operand:VI_128 0 "register_operand" "=x")
16233 [(match_operand:VI_128 1 "register_operand" "x")
16234 (match_operand:VI_128 2 "nonimmediate_operand" "xm")
16235 (match_operand:SI 3 "const_int_operand" "n")]
16236 UNSPEC_XOP_TRUEFALSE))]
16239 return ((INTVAL (operands[3]) != 0)
16240 ? "vpcomtrue<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
16241 : "vpcomfalse<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}");
16243 [(set_attr "type" "ssecmp")
16244 (set_attr "prefix_data16" "0")
16245 (set_attr "prefix_extra" "2")
16246 (set_attr "length_immediate" "1")
16247 (set_attr "mode" "TI")])
16249 (define_insn "xop_vpermil2<mode>3"
16250 [(set (match_operand:VF_128_256 0 "register_operand" "=x")
16252 [(match_operand:VF_128_256 1 "register_operand" "x")
16253 (match_operand:VF_128_256 2 "nonimmediate_operand" "%x")
16254 (match_operand:<sseintvecmode> 3 "nonimmediate_operand" "xm")
16255 (match_operand:SI 4 "const_0_to_3_operand" "n")]
16258 "vpermil2<ssemodesuffix>\t{%4, %3, %2, %1, %0|%0, %1, %2, %3, %4}"
16259 [(set_attr "type" "sse4arg")
16260 (set_attr "length_immediate" "1")
16261 (set_attr "mode" "<MODE>")])
16263 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
16265 (define_insn "aesenc"
16266 [(set (match_operand:V2DI 0 "register_operand" "=x,x")
16267 (unspec:V2DI [(match_operand:V2DI 1 "register_operand" "0,x")
16268 (match_operand:V2DI 2 "nonimmediate_operand" "xm,xm")]
16272 aesenc\t{%2, %0|%0, %2}
16273 vaesenc\t{%2, %1, %0|%0, %1, %2}"
16274 [(set_attr "isa" "noavx,avx")
16275 (set_attr "type" "sselog1")
16276 (set_attr "prefix_extra" "1")
16277 (set_attr "prefix" "orig,vex")
16278 (set_attr "btver2_decode" "double,double")
16279 (set_attr "mode" "TI")])
16281 (define_insn "aesenclast"
16282 [(set (match_operand:V2DI 0 "register_operand" "=x,x")
16283 (unspec:V2DI [(match_operand:V2DI 1 "register_operand" "0,x")
16284 (match_operand:V2DI 2 "nonimmediate_operand" "xm,xm")]
16285 UNSPEC_AESENCLAST))]
16288 aesenclast\t{%2, %0|%0, %2}
16289 vaesenclast\t{%2, %1, %0|%0, %1, %2}"
16290 [(set_attr "isa" "noavx,avx")
16291 (set_attr "type" "sselog1")
16292 (set_attr "prefix_extra" "1")
16293 (set_attr "prefix" "orig,vex")
16294 (set_attr "btver2_decode" "double,double")
16295 (set_attr "mode" "TI")])
16297 (define_insn "aesdec"
16298 [(set (match_operand:V2DI 0 "register_operand" "=x,x")
16299 (unspec:V2DI [(match_operand:V2DI 1 "register_operand" "0,x")
16300 (match_operand:V2DI 2 "nonimmediate_operand" "xm,xm")]
16304 aesdec\t{%2, %0|%0, %2}
16305 vaesdec\t{%2, %1, %0|%0, %1, %2}"
16306 [(set_attr "isa" "noavx,avx")
16307 (set_attr "type" "sselog1")
16308 (set_attr "prefix_extra" "1")
16309 (set_attr "prefix" "orig,vex")
16310 (set_attr "btver2_decode" "double,double")
16311 (set_attr "mode" "TI")])
16313 (define_insn "aesdeclast"
16314 [(set (match_operand:V2DI 0 "register_operand" "=x,x")
16315 (unspec:V2DI [(match_operand:V2DI 1 "register_operand" "0,x")
16316 (match_operand:V2DI 2 "nonimmediate_operand" "xm,xm")]
16317 UNSPEC_AESDECLAST))]
16320 aesdeclast\t{%2, %0|%0, %2}
16321 vaesdeclast\t{%2, %1, %0|%0, %1, %2}"
16322 [(set_attr "isa" "noavx,avx")
16323 (set_attr "type" "sselog1")
16324 (set_attr "prefix_extra" "1")
16325 (set_attr "prefix" "orig,vex")
16326 (set_attr "btver2_decode" "double,double")
16327 (set_attr "mode" "TI")])
16329 (define_insn "aesimc"
16330 [(set (match_operand:V2DI 0 "register_operand" "=x")
16331 (unspec:V2DI [(match_operand:V2DI 1 "nonimmediate_operand" "xm")]
16334 "%vaesimc\t{%1, %0|%0, %1}"
16335 [(set_attr "type" "sselog1")
16336 (set_attr "prefix_extra" "1")
16337 (set_attr "prefix" "maybe_vex")
16338 (set_attr "mode" "TI")])
16340 (define_insn "aeskeygenassist"
16341 [(set (match_operand:V2DI 0 "register_operand" "=x")
16342 (unspec:V2DI [(match_operand:V2DI 1 "nonimmediate_operand" "xm")
16343 (match_operand:SI 2 "const_0_to_255_operand" "n")]
16344 UNSPEC_AESKEYGENASSIST))]
16346 "%vaeskeygenassist\t{%2, %1, %0|%0, %1, %2}"
16347 [(set_attr "type" "sselog1")
16348 (set_attr "prefix_extra" "1")
16349 (set_attr "length_immediate" "1")
16350 (set_attr "prefix" "maybe_vex")
16351 (set_attr "mode" "TI")])
16353 (define_insn "pclmulqdq"
16354 [(set (match_operand:V2DI 0 "register_operand" "=x,x")
16355 (unspec:V2DI [(match_operand:V2DI 1 "register_operand" "0,x")
16356 (match_operand:V2DI 2 "nonimmediate_operand" "xm,xm")
16357 (match_operand:SI 3 "const_0_to_255_operand" "n,n")]
16361 pclmulqdq\t{%3, %2, %0|%0, %2, %3}
16362 vpclmulqdq\t{%3, %2, %1, %0|%0, %1, %2, %3}"
16363 [(set_attr "isa" "noavx,avx")
16364 (set_attr "type" "sselog1")
16365 (set_attr "prefix_extra" "1")
16366 (set_attr "length_immediate" "1")
16367 (set_attr "prefix" "orig,vex")
16368 (set_attr "mode" "TI")])
16370 (define_expand "avx_vzeroall"
16371 [(match_par_dup 0 [(const_int 0)])]
16374 int nregs = TARGET_64BIT ? 16 : 8;
16377 operands[0] = gen_rtx_PARALLEL (VOIDmode, rtvec_alloc (nregs + 1));
16379 XVECEXP (operands[0], 0, 0)
16380 = gen_rtx_UNSPEC_VOLATILE (VOIDmode, gen_rtvec (1, const0_rtx),
16383 for (regno = 0; regno < nregs; regno++)
16384 XVECEXP (operands[0], 0, regno + 1)
16385 = gen_rtx_SET (VOIDmode,
16386 gen_rtx_REG (V8SImode, SSE_REGNO (regno)),
16387 CONST0_RTX (V8SImode));
16390 (define_insn "*avx_vzeroall"
16391 [(match_parallel 0 "vzeroall_operation"
16392 [(unspec_volatile [(const_int 0)] UNSPECV_VZEROALL)])]
16395 [(set_attr "type" "sse")
16396 (set_attr "modrm" "0")
16397 (set_attr "memory" "none")
16398 (set_attr "prefix" "vex")
16399 (set_attr "btver2_decode" "vector")
16400 (set_attr "mode" "OI")])
16402 ;; Clear the upper 128bits of AVX registers, equivalent to a NOP
16403 ;; if the upper 128bits are unused.
16404 (define_insn "avx_vzeroupper"
16405 [(unspec_volatile [(const_int 0)] UNSPECV_VZEROUPPER)]
16408 [(set_attr "type" "sse")
16409 (set_attr "modrm" "0")
16410 (set_attr "memory" "none")
16411 (set_attr "prefix" "vex")
16412 (set_attr "btver2_decode" "vector")
16413 (set_attr "mode" "OI")])
16415 (define_insn "avx2_pbroadcast<mode>"
16416 [(set (match_operand:VI 0 "register_operand" "=x")
16418 (vec_select:<ssescalarmode>
16419 (match_operand:<ssexmmmode> 1 "nonimmediate_operand" "xm")
16420 (parallel [(const_int 0)]))))]
16422 "vpbroadcast<ssemodesuffix>\t{%1, %0|%0, %<iptr>1}"
16423 [(set_attr "type" "ssemov")
16424 (set_attr "prefix_extra" "1")
16425 (set_attr "prefix" "vex")
16426 (set_attr "mode" "<sseinsnmode>")])
16428 (define_insn "avx2_pbroadcast<mode>_1"
16429 [(set (match_operand:VI_256 0 "register_operand" "=x,x")
16430 (vec_duplicate:VI_256
16431 (vec_select:<ssescalarmode>
16432 (match_operand:VI_256 1 "nonimmediate_operand" "m,x")
16433 (parallel [(const_int 0)]))))]
16436 vpbroadcast<ssemodesuffix>\t{%1, %0|%0, %<iptr>1}
16437 vpbroadcast<ssemodesuffix>\t{%x1, %0|%0, %x1}"
16438 [(set_attr "type" "ssemov")
16439 (set_attr "prefix_extra" "1")
16440 (set_attr "prefix" "vex")
16441 (set_attr "mode" "<sseinsnmode>")])
16443 (define_insn "<avx2_avx512>_permvar<mode><mask_name>"
16444 [(set (match_operand:VI48F_256_512 0 "register_operand" "=v")
16445 (unspec:VI48F_256_512
16446 [(match_operand:VI48F_256_512 1 "nonimmediate_operand" "vm")
16447 (match_operand:<sseintvecmode> 2 "register_operand" "v")]
16449 "TARGET_AVX2 && <mask_mode512bit_condition>"
16450 "vperm<ssemodesuffix>\t{%1, %2, %0<mask_operand3>|%0<mask_operand3>, %2, %1}"
16451 [(set_attr "type" "sselog")
16452 (set_attr "prefix" "<mask_prefix2>")
16453 (set_attr "mode" "<sseinsnmode>")])
16455 (define_insn "<avx512>_permvar<mode><mask_name>"
16456 [(set (match_operand:VI1_AVX512VL 0 "register_operand" "=v")
16457 (unspec:VI1_AVX512VL
16458 [(match_operand:VI1_AVX512VL 1 "nonimmediate_operand" "vm")
16459 (match_operand:<sseintvecmode> 2 "register_operand" "v")]
16461 "TARGET_AVX512VBMI && <mask_mode512bit_condition>"
16462 "vperm<ssemodesuffix>\t{%1, %2, %0<mask_operand3>|%0<mask_operand3>, %2, %1}"
16463 [(set_attr "type" "sselog")
16464 (set_attr "prefix" "<mask_prefix2>")
16465 (set_attr "mode" "<sseinsnmode>")])
16467 (define_insn "<avx512>_permvar<mode><mask_name>"
16468 [(set (match_operand:VI2_AVX512VL 0 "register_operand" "=v")
16469 (unspec:VI2_AVX512VL
16470 [(match_operand:VI2_AVX512VL 1 "nonimmediate_operand" "vm")
16471 (match_operand:<sseintvecmode> 2 "register_operand" "v")]
16473 "TARGET_AVX512BW && <mask_mode512bit_condition>"
16474 "vperm<ssemodesuffix>\t{%1, %2, %0<mask_operand3>|%0<mask_operand3>, %2, %1}"
16475 [(set_attr "type" "sselog")
16476 (set_attr "prefix" "<mask_prefix2>")
16477 (set_attr "mode" "<sseinsnmode>")])
16479 (define_expand "<avx2_avx512>_perm<mode>"
16480 [(match_operand:VI8F_256_512 0 "register_operand")
16481 (match_operand:VI8F_256_512 1 "nonimmediate_operand")
16482 (match_operand:SI 2 "const_0_to_255_operand")]
16485 int mask = INTVAL (operands[2]);
16486 emit_insn (gen_<avx2_avx512>_perm<mode>_1 (operands[0], operands[1],
16487 GEN_INT ((mask >> 0) & 3),
16488 GEN_INT ((mask >> 2) & 3),
16489 GEN_INT ((mask >> 4) & 3),
16490 GEN_INT ((mask >> 6) & 3)));
16494 (define_expand "<avx512>_perm<mode>_mask"
16495 [(match_operand:VI8F_256_512 0 "register_operand")
16496 (match_operand:VI8F_256_512 1 "nonimmediate_operand")
16497 (match_operand:SI 2 "const_0_to_255_operand")
16498 (match_operand:VI8F_256_512 3 "vector_move_operand")
16499 (match_operand:<avx512fmaskmode> 4 "register_operand")]
16502 int mask = INTVAL (operands[2]);
16503 emit_insn (gen_<avx2_avx512>_perm<mode>_1_mask (operands[0], operands[1],
16504 GEN_INT ((mask >> 0) & 3),
16505 GEN_INT ((mask >> 2) & 3),
16506 GEN_INT ((mask >> 4) & 3),
16507 GEN_INT ((mask >> 6) & 3),
16508 operands[3], operands[4]));
16512 (define_insn "<avx2_avx512>_perm<mode>_1<mask_name>"
16513 [(set (match_operand:VI8F_256_512 0 "register_operand" "=v")
16514 (vec_select:VI8F_256_512
16515 (match_operand:VI8F_256_512 1 "nonimmediate_operand" "vm")
16516 (parallel [(match_operand 2 "const_0_to_3_operand")
16517 (match_operand 3 "const_0_to_3_operand")
16518 (match_operand 4 "const_0_to_3_operand")
16519 (match_operand 5 "const_0_to_3_operand")])))]
16520 "TARGET_AVX2 && <mask_mode512bit_condition>"
16523 mask |= INTVAL (operands[2]) << 0;
16524 mask |= INTVAL (operands[3]) << 2;
16525 mask |= INTVAL (operands[4]) << 4;
16526 mask |= INTVAL (operands[5]) << 6;
16527 operands[2] = GEN_INT (mask);
16528 return "vperm<ssemodesuffix>\t{%2, %1, %0<mask_operand6>|%0<mask_operand6>, %1, %2}";
16530 [(set_attr "type" "sselog")
16531 (set_attr "prefix" "<mask_prefix2>")
16532 (set_attr "mode" "<sseinsnmode>")])
16534 (define_insn "avx2_permv2ti"
16535 [(set (match_operand:V4DI 0 "register_operand" "=x")
16537 [(match_operand:V4DI 1 "register_operand" "x")
16538 (match_operand:V4DI 2 "nonimmediate_operand" "xm")
16539 (match_operand:SI 3 "const_0_to_255_operand" "n")]
16542 "vperm2i128\t{%3, %2, %1, %0|%0, %1, %2, %3}"
16543 [(set_attr "type" "sselog")
16544 (set_attr "prefix" "vex")
16545 (set_attr "mode" "OI")])
16547 (define_insn "avx2_vec_dupv4df"
16548 [(set (match_operand:V4DF 0 "register_operand" "=x")
16549 (vec_duplicate:V4DF
16551 (match_operand:V2DF 1 "register_operand" "x")
16552 (parallel [(const_int 0)]))))]
16554 "vbroadcastsd\t{%1, %0|%0, %1}"
16555 [(set_attr "type" "sselog1")
16556 (set_attr "prefix" "vex")
16557 (set_attr "mode" "V4DF")])
16559 (define_insn "<avx512>_vec_dup<mode>_1"
16560 [(set (match_operand:VI_AVX512BW 0 "register_operand" "=v,v")
16561 (vec_duplicate:VI_AVX512BW
16562 (vec_select:VI_AVX512BW
16563 (match_operand:VI_AVX512BW 1 "nonimmediate_operand" "v,m")
16564 (parallel [(const_int 0)]))))]
16566 "vpbroadcast<ssemodesuffix>\t{%1, %0|%0, %<iptr>1}
16567 vpbroadcast<ssemodesuffix>\t{%x1, %0|%0, %x1}"
16568 [(set_attr "type" "ssemov")
16569 (set_attr "prefix" "evex")
16570 (set_attr "mode" "<sseinsnmode>")])
16572 (define_insn "<avx512>_vec_dup<mode><mask_name>"
16573 [(set (match_operand:V48_AVX512VL 0 "register_operand" "=v")
16574 (vec_duplicate:V48_AVX512VL
16575 (vec_select:<ssescalarmode>
16576 (match_operand:<ssexmmmode> 1 "nonimmediate_operand" "vm")
16577 (parallel [(const_int 0)]))))]
16579 "v<sseintprefix>broadcast<bcstscalarsuff>\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
16580 [(set_attr "type" "ssemov")
16581 (set_attr "prefix" "evex")
16582 (set_attr "mode" "<sseinsnmode>")])
16584 (define_insn "<avx512>_vec_dup<mode><mask_name>"
16585 [(set (match_operand:VI12_AVX512VL 0 "register_operand" "=v")
16586 (vec_duplicate:VI12_AVX512VL
16587 (vec_select:<ssescalarmode>
16588 (match_operand:<ssexmmmode> 1 "nonimmediate_operand" "vm")
16589 (parallel [(const_int 0)]))))]
16591 "vpbroadcast<bcstscalarsuff>\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
16592 [(set_attr "type" "ssemov")
16593 (set_attr "prefix" "evex")
16594 (set_attr "mode" "<sseinsnmode>")])
16596 (define_insn "<mask_codefor>avx512f_broadcast<mode><mask_name>"
16597 [(set (match_operand:V16FI 0 "register_operand" "=v,v")
16598 (vec_duplicate:V16FI
16599 (match_operand:<ssexmmmode> 1 "nonimmediate_operand" "v,m")))]
16602 vshuf<shuffletype>32x4\t{$0x0, %g1, %g1, %0<mask_operand2>|%0<mask_operand2>, %g1, %g1, 0x0}
16603 vbroadcast<shuffletype>32x4\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
16604 [(set_attr "type" "ssemov")
16605 (set_attr "prefix" "evex")
16606 (set_attr "mode" "<sseinsnmode>")])
16608 (define_insn "<mask_codefor>avx512f_broadcast<mode><mask_name>"
16609 [(set (match_operand:V8FI 0 "register_operand" "=v,v")
16610 (vec_duplicate:V8FI
16611 (match_operand:<ssehalfvecmode> 1 "nonimmediate_operand" "v,m")))]
16614 vshuf<shuffletype>64x2\t{$0x44, %g1, %g1, %0<mask_operand2>|%0<mask_operand2>, %g1, %g1, 0x44}
16615 vbroadcast<shuffletype>64x4\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
16616 [(set_attr "type" "ssemov")
16617 (set_attr "prefix" "evex")
16618 (set_attr "mode" "<sseinsnmode>")])
16620 (define_insn "<mask_codefor><avx512>_vec_dup_gpr<mode><mask_name>"
16621 [(set (match_operand:VI12_AVX512VL 0 "register_operand" "=v,v")
16622 (vec_duplicate:VI12_AVX512VL
16623 (match_operand:<ssescalarmode> 1 "nonimmediate_operand" "vm,r")))]
16626 vpbroadcast<bcstscalarsuff>\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}
16627 vpbroadcast<bcstscalarsuff>\t{%k1, %0<mask_operand2>|%0<mask_operand2>, %k1}"
16628 [(set_attr "type" "ssemov")
16629 (set_attr "prefix" "evex")
16630 (set_attr "mode" "<sseinsnmode>")])
16632 (define_insn "<mask_codefor><avx512>_vec_dup_gpr<mode><mask_name>"
16633 [(set (match_operand:V48_AVX512VL 0 "register_operand" "=v,v")
16634 (vec_duplicate:V48_AVX512VL
16635 (match_operand:<ssescalarmode> 1 "nonimmediate_operand" "vm,r")))]
16637 "v<sseintprefix>broadcast<bcstscalarsuff>\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
16638 [(set_attr "type" "ssemov")
16639 (set_attr "prefix" "evex")
16640 (set_attr "mode" "<sseinsnmode>")
16641 (set (attr "enabled")
16642 (if_then_else (eq_attr "alternative" "1")
16643 (symbol_ref "GET_MODE_CLASS (<ssescalarmode>mode) == MODE_INT
16644 && (<ssescalarmode>mode != DImode || TARGET_64BIT)")
16647 (define_insn "vec_dupv4sf"
16648 [(set (match_operand:V4SF 0 "register_operand" "=x,x,x")
16649 (vec_duplicate:V4SF
16650 (match_operand:SF 1 "nonimmediate_operand" "x,m,0")))]
16653 vshufps\t{$0, %1, %1, %0|%0, %1, %1, 0}
16654 vbroadcastss\t{%1, %0|%0, %1}
16655 shufps\t{$0, %0, %0|%0, %0, 0}"
16656 [(set_attr "isa" "avx,avx,noavx")
16657 (set_attr "type" "sseshuf1,ssemov,sseshuf1")
16658 (set_attr "length_immediate" "1,0,1")
16659 (set_attr "prefix_extra" "0,1,*")
16660 (set_attr "prefix" "vex,vex,orig")
16661 (set_attr "mode" "V4SF")])
16663 (define_insn "*vec_dupv4si"
16664 [(set (match_operand:V4SI 0 "register_operand" "=x,x,x")
16665 (vec_duplicate:V4SI
16666 (match_operand:SI 1 "nonimmediate_operand" " x,m,0")))]
16669 %vpshufd\t{$0, %1, %0|%0, %1, 0}
16670 vbroadcastss\t{%1, %0|%0, %1}
16671 shufps\t{$0, %0, %0|%0, %0, 0}"
16672 [(set_attr "isa" "sse2,avx,noavx")
16673 (set_attr "type" "sselog1,ssemov,sselog1")
16674 (set_attr "length_immediate" "1,0,1")
16675 (set_attr "prefix_extra" "0,1,*")
16676 (set_attr "prefix" "maybe_vex,vex,orig")
16677 (set_attr "mode" "TI,V4SF,V4SF")])
16679 (define_insn "*vec_dupv2di"
16680 [(set (match_operand:V2DI 0 "register_operand" "=x,x,x,x")
16681 (vec_duplicate:V2DI
16682 (match_operand:DI 1 "nonimmediate_operand" " 0,x,m,0")))]
16686 vpunpcklqdq\t{%d1, %0|%0, %d1}
16687 %vmovddup\t{%1, %0|%0, %1}
16689 [(set_attr "isa" "sse2_noavx,avx,sse3,noavx")
16690 (set_attr "type" "sselog1,sselog1,sselog1,ssemov")
16691 (set_attr "prefix" "orig,vex,maybe_vex,orig")
16692 (set_attr "mode" "TI,TI,DF,V4SF")])
16694 (define_insn "avx2_vbroadcasti128_<mode>"
16695 [(set (match_operand:VI_256 0 "register_operand" "=x")
16697 (match_operand:<ssehalfvecmode> 1 "memory_operand" "m")
16700 "vbroadcasti128\t{%1, %0|%0, %1}"
16701 [(set_attr "type" "ssemov")
16702 (set_attr "prefix_extra" "1")
16703 (set_attr "prefix" "vex")
16704 (set_attr "mode" "OI")])
16706 ;; Modes handled by AVX vec_dup patterns.
16707 (define_mode_iterator AVX_VEC_DUP_MODE
16708 [V8SI V8SF V4DI V4DF])
16709 ;; Modes handled by AVX2 vec_dup patterns.
16710 (define_mode_iterator AVX2_VEC_DUP_MODE
16711 [V32QI V16QI V16HI V8HI V8SI V4SI])
16713 (define_insn "*vec_dup<mode>"
16714 [(set (match_operand:AVX2_VEC_DUP_MODE 0 "register_operand" "=x,x,x")
16715 (vec_duplicate:AVX2_VEC_DUP_MODE
16716 (match_operand:<ssescalarmode> 1 "nonimmediate_operand" "m,x,$r")))]
16719 v<sseintprefix>broadcast<bcstscalarsuff>\t{%1, %0|%0, %1}
16720 v<sseintprefix>broadcast<bcstscalarsuff>\t{%x1, %0|%0, %x1}
16722 [(set_attr "type" "ssemov")
16723 (set_attr "prefix_extra" "1")
16724 (set_attr "prefix" "maybe_evex")
16725 (set_attr "mode" "<sseinsnmode>")])
16727 (define_insn "vec_dup<mode>"
16728 [(set (match_operand:AVX_VEC_DUP_MODE 0 "register_operand" "=x,x,v,x")
16729 (vec_duplicate:AVX_VEC_DUP_MODE
16730 (match_operand:<ssescalarmode> 1 "nonimmediate_operand" "m,m,v,?x")))]
16733 v<sseintprefix>broadcast<bcstscalarsuff>\t{%1, %0|%0, %1}
16734 vbroadcast<ssescalarmodesuffix>\t{%1, %0|%0, %1}
16735 v<sseintprefix>broadcast<bcstscalarsuff>\t{%x1, %0|%0, %x1}
16737 [(set_attr "type" "ssemov")
16738 (set_attr "prefix_extra" "1")
16739 (set_attr "prefix" "maybe_evex")
16740 (set_attr "isa" "avx2,noavx2,avx2,noavx2")
16741 (set_attr "mode" "<sseinsnmode>,V8SF,<sseinsnmode>,V8SF")])
16744 [(set (match_operand:AVX2_VEC_DUP_MODE 0 "register_operand")
16745 (vec_duplicate:AVX2_VEC_DUP_MODE
16746 (match_operand:<ssescalarmode> 1 "register_operand")))]
16748 /* Disable this splitter if avx512vl_vec_dup_gprv*[qhs]i insn is
16749 available, because then we can broadcast from GPRs directly.
16750 For V*[QH]I modes it requires both -mavx512vl and -mavx512bw,
16751 for V*SI mode it requires just -mavx512vl. */
16752 && !(TARGET_AVX512VL
16753 && (TARGET_AVX512BW || <ssescalarmode>mode == SImode))
16754 && reload_completed && GENERAL_REG_P (operands[1])"
16757 emit_insn (gen_vec_setv4si_0 (gen_lowpart (V4SImode, operands[0]),
16758 CONST0_RTX (V4SImode),
16759 gen_lowpart (SImode, operands[1])));
16760 emit_insn (gen_avx2_pbroadcast<mode> (operands[0],
16761 gen_lowpart (<ssexmmmode>mode,
16767 [(set (match_operand:AVX_VEC_DUP_MODE 0 "register_operand")
16768 (vec_duplicate:AVX_VEC_DUP_MODE
16769 (match_operand:<ssescalarmode> 1 "register_operand")))]
16770 "TARGET_AVX && !TARGET_AVX2 && reload_completed"
16771 [(set (match_dup 2)
16772 (vec_duplicate:<ssehalfvecmode> (match_dup 1)))
16774 (vec_concat:AVX_VEC_DUP_MODE (match_dup 2) (match_dup 2)))]
16775 "operands[2] = gen_rtx_REG (<ssehalfvecmode>mode, REGNO (operands[0]));")
16777 (define_insn "avx_vbroadcastf128_<mode>"
16778 [(set (match_operand:V_256 0 "register_operand" "=x,x,x")
16780 (match_operand:<ssehalfvecmode> 1 "nonimmediate_operand" "m,0,?x")
16784 vbroadcast<i128>\t{%1, %0|%0, %1}
16785 vinsert<i128>\t{$1, %1, %0, %0|%0, %0, %1, 1}
16786 vperm2<i128>\t{$0, %t1, %t1, %0|%0, %t1, %t1, 0}"
16787 [(set_attr "type" "ssemov,sselog1,sselog1")
16788 (set_attr "prefix_extra" "1")
16789 (set_attr "length_immediate" "0,1,1")
16790 (set_attr "prefix" "vex")
16791 (set_attr "mode" "<sseinsnmode>")])
16793 ;; For broadcast[i|f]32x2. Yes there is no v4sf version, only v4si.
16794 (define_mode_iterator VI4F_BRCST32x2
16795 [V16SI (V8SI "TARGET_AVX512VL") (V4SI "TARGET_AVX512VL")
16796 V16SF (V8SF "TARGET_AVX512VL")])
16798 (define_mode_attr 64x2mode
16799 [(V8DF "V2DF") (V8DI "V2DI") (V4DI "V2DI") (V4DF "V2DF")])
16801 (define_mode_attr 32x2mode
16802 [(V16SF "V2SF") (V16SI "V2SI") (V8SI "V2SI")
16803 (V8SF "V2SF") (V4SI "V2SI")])
16805 (define_insn "<mask_codefor>avx512dq_broadcast<mode><mask_name>"
16806 [(set (match_operand:VI4F_BRCST32x2 0 "register_operand" "=v")
16807 (vec_duplicate:VI4F_BRCST32x2
16808 (vec_select:<32x2mode>
16809 (match_operand:<ssexmmmode> 1 "nonimmediate_operand" "vm")
16810 (parallel [(const_int 0) (const_int 1)]))))]
16812 "vbroadcast<shuffletype>32x2\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
16813 [(set_attr "type" "ssemov")
16814 (set_attr "prefix_extra" "1")
16815 (set_attr "prefix" "evex")
16816 (set_attr "mode" "<sseinsnmode>")])
16818 (define_insn "<mask_codefor>avx512vl_broadcast<mode><mask_name>_1"
16819 [(set (match_operand:VI4F_256 0 "register_operand" "=v,v")
16820 (vec_duplicate:VI4F_256
16821 (match_operand:<ssexmmmode> 1 "nonimmediate_operand" "v,m")))]
16824 vshuf<shuffletype>32x4\t{$0x0, %t1, %t1, %0<mask_operand2>|%0<mask_operand2>, %t1, %t1, 0x0}
16825 vbroadcast<shuffletype>32x4\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
16826 [(set_attr "type" "ssemov")
16827 (set_attr "prefix_extra" "1")
16828 (set_attr "prefix" "evex")
16829 (set_attr "mode" "<sseinsnmode>")])
16831 (define_insn "<mask_codefor>avx512dq_broadcast<mode><mask_name>_1"
16832 [(set (match_operand:V16FI 0 "register_operand" "=v,v")
16833 (vec_duplicate:V16FI
16834 (match_operand:<ssehalfvecmode> 1 "nonimmediate_operand" "v,m")))]
16837 vshuf<shuffletype>32x4\t{$0x44, %g1, %g1, %0<mask_operand2>|%0<mask_operand2>, %g1, %g1, 0x44}
16838 vbroadcast<shuffletype>32x8\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
16839 [(set_attr "type" "ssemov")
16840 (set_attr "prefix_extra" "1")
16841 (set_attr "prefix" "evex")
16842 (set_attr "mode" "<sseinsnmode>")])
16844 ;; For broadcast[i|f]64x2
16845 (define_mode_iterator VI8F_BRCST64x2
16846 [V8DI V8DF (V4DI "TARGET_AVX512VL") (V4DF "TARGET_AVX512VL")])
16848 (define_insn "<mask_codefor>avx512dq_broadcast<mode><mask_name>_1"
16849 [(set (match_operand:VI8F_BRCST64x2 0 "register_operand" "=v,v")
16850 (vec_duplicate:VI8F_BRCST64x2
16851 (match_operand:<64x2mode> 1 "nonimmediate_operand" "v,m")))]
16854 vshuf<shuffletype>64x2\t{$0x0, %<concat_tg_mode>1, %<concat_tg_mode>1, %0<mask_operand2>|%0<mask_operand2>, %<concat_tg_mode>1, %<concat_tg_mode>1, 0x0}
16855 vbroadcast<shuffletype>64x2\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
16856 [(set_attr "type" "ssemov")
16857 (set_attr "prefix_extra" "1")
16858 (set_attr "prefix" "evex")
16859 (set_attr "mode" "<sseinsnmode>")])
16861 (define_insn "avx512cd_maskb_vec_dup<mode>"
16862 [(set (match_operand:VI8_AVX512VL 0 "register_operand" "=v")
16863 (vec_duplicate:VI8_AVX512VL
16865 (match_operand:QI 1 "register_operand" "Yk"))))]
16867 "vpbroadcastmb2q\t{%1, %0|%0, %1}"
16868 [(set_attr "type" "mskmov")
16869 (set_attr "prefix" "evex")
16870 (set_attr "mode" "XI")])
16872 (define_insn "avx512cd_maskw_vec_dup<mode>"
16873 [(set (match_operand:VI4_AVX512VL 0 "register_operand" "=v")
16874 (vec_duplicate:VI4_AVX512VL
16876 (match_operand:HI 1 "register_operand" "Yk"))))]
16878 "vpbroadcastmw2d\t{%1, %0|%0, %1}"
16879 [(set_attr "type" "mskmov")
16880 (set_attr "prefix" "evex")
16881 (set_attr "mode" "XI")])
16883 ;; Recognize broadcast as a vec_select as produced by builtin_vec_perm.
16884 ;; If it so happens that the input is in memory, use vbroadcast.
16885 ;; Otherwise use vpermilp (and in the case of 256-bit modes, vperm2f128).
16886 (define_insn "*avx_vperm_broadcast_v4sf"
16887 [(set (match_operand:V4SF 0 "register_operand" "=x,x,x")
16889 (match_operand:V4SF 1 "nonimmediate_operand" "m,o,x")
16890 (match_parallel 2 "avx_vbroadcast_operand"
16891 [(match_operand 3 "const_int_operand" "C,n,n")])))]
16894 int elt = INTVAL (operands[3]);
16895 switch (which_alternative)
16899 operands[1] = adjust_address_nv (operands[1], SFmode, elt * 4);
16900 return "vbroadcastss\t{%1, %0|%0, %k1}";
16902 operands[2] = GEN_INT (elt * 0x55);
16903 return "vpermilps\t{%2, %1, %0|%0, %1, %2}";
16905 gcc_unreachable ();
16908 [(set_attr "type" "ssemov,ssemov,sselog1")
16909 (set_attr "prefix_extra" "1")
16910 (set_attr "length_immediate" "0,0,1")
16911 (set_attr "prefix" "vex")
16912 (set_attr "mode" "SF,SF,V4SF")])
16914 (define_insn_and_split "*avx_vperm_broadcast_<mode>"
16915 [(set (match_operand:VF_256 0 "register_operand" "=x,x,x")
16917 (match_operand:VF_256 1 "nonimmediate_operand" "m,o,?x")
16918 (match_parallel 2 "avx_vbroadcast_operand"
16919 [(match_operand 3 "const_int_operand" "C,n,n")])))]
16922 "&& reload_completed && (<MODE>mode != V4DFmode || !TARGET_AVX2)"
16923 [(set (match_dup 0) (vec_duplicate:VF_256 (match_dup 1)))]
16925 rtx op0 = operands[0], op1 = operands[1];
16926 int elt = INTVAL (operands[3]);
16932 if (TARGET_AVX2 && elt == 0)
16934 emit_insn (gen_vec_dup<mode> (op0, gen_lowpart (<ssescalarmode>mode,
16939 /* Shuffle element we care about into all elements of the 128-bit lane.
16940 The other lane gets shuffled too, but we don't care. */
16941 if (<MODE>mode == V4DFmode)
16942 mask = (elt & 1 ? 15 : 0);
16944 mask = (elt & 3) * 0x55;
16945 emit_insn (gen_avx_vpermil<mode> (op0, op1, GEN_INT (mask)));
16947 /* Shuffle the lane we care about into both lanes of the dest. */
16948 mask = (elt / (<ssescalarnum> / 2)) * 0x11;
16949 emit_insn (gen_avx_vperm2f128<mode>3 (op0, op0, op0, GEN_INT (mask)));
16953 operands[1] = adjust_address (op1, <ssescalarmode>mode,
16954 elt * GET_MODE_SIZE (<ssescalarmode>mode));
16957 (define_expand "<sse2_avx_avx512f>_vpermil<mode><mask_name>"
16958 [(set (match_operand:VF2 0 "register_operand")
16960 (match_operand:VF2 1 "nonimmediate_operand")
16961 (match_operand:SI 2 "const_0_to_255_operand")))]
16962 "TARGET_AVX && <mask_mode512bit_condition>"
16964 int mask = INTVAL (operands[2]);
16965 rtx perm[<ssescalarnum>];
16968 for (i = 0; i < <ssescalarnum>; i = i + 2)
16970 perm[i] = GEN_INT (((mask >> i) & 1) + i);
16971 perm[i + 1] = GEN_INT (((mask >> (i + 1)) & 1) + i);
16975 = gen_rtx_PARALLEL (VOIDmode, gen_rtvec_v (<ssescalarnum>, perm));
16978 (define_expand "<sse2_avx_avx512f>_vpermil<mode><mask_name>"
16979 [(set (match_operand:VF1 0 "register_operand")
16981 (match_operand:VF1 1 "nonimmediate_operand")
16982 (match_operand:SI 2 "const_0_to_255_operand")))]
16983 "TARGET_AVX && <mask_mode512bit_condition>"
16985 int mask = INTVAL (operands[2]);
16986 rtx perm[<ssescalarnum>];
16989 for (i = 0; i < <ssescalarnum>; i = i + 4)
16991 perm[i] = GEN_INT (((mask >> 0) & 3) + i);
16992 perm[i + 1] = GEN_INT (((mask >> 2) & 3) + i);
16993 perm[i + 2] = GEN_INT (((mask >> 4) & 3) + i);
16994 perm[i + 3] = GEN_INT (((mask >> 6) & 3) + i);
16998 = gen_rtx_PARALLEL (VOIDmode, gen_rtvec_v (<ssescalarnum>, perm));
17001 (define_insn "*<sse2_avx_avx512f>_vpermilp<mode><mask_name>"
17002 [(set (match_operand:VF 0 "register_operand" "=v")
17004 (match_operand:VF 1 "nonimmediate_operand" "vm")
17005 (match_parallel 2 ""
17006 [(match_operand 3 "const_int_operand")])))]
17007 "TARGET_AVX && <mask_mode512bit_condition>
17008 && avx_vpermilp_parallel (operands[2], <MODE>mode)"
17010 int mask = avx_vpermilp_parallel (operands[2], <MODE>mode) - 1;
17011 operands[2] = GEN_INT (mask);
17012 return "vpermil<ssemodesuffix>\t{%2, %1, %0<mask_operand4>|%0<mask_operand4>, %1, %2}";
17014 [(set_attr "type" "sselog")
17015 (set_attr "prefix_extra" "1")
17016 (set_attr "length_immediate" "1")
17017 (set_attr "prefix" "<mask_prefix>")
17018 (set_attr "mode" "<sseinsnmode>")])
17020 (define_insn "<sse2_avx_avx512f>_vpermilvar<mode>3<mask_name>"
17021 [(set (match_operand:VF 0 "register_operand" "=v")
17023 [(match_operand:VF 1 "register_operand" "v")
17024 (match_operand:<sseintvecmode> 2 "nonimmediate_operand" "vm")]
17026 "TARGET_AVX && <mask_mode512bit_condition>"
17027 "vpermil<ssemodesuffix>\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
17028 [(set_attr "type" "sselog")
17029 (set_attr "prefix_extra" "1")
17030 (set_attr "btver2_decode" "vector")
17031 (set_attr "prefix" "<mask_prefix>")
17032 (set_attr "mode" "<sseinsnmode>")])
17034 (define_expand "<avx512>_vpermi2var<mode>3_maskz"
17035 [(match_operand:VI48F 0 "register_operand" "=v")
17036 (match_operand:VI48F 1 "register_operand" "v")
17037 (match_operand:<sseintvecmode> 2 "register_operand" "0")
17038 (match_operand:VI48F 3 "nonimmediate_operand" "vm")
17039 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk")]
17042 emit_insn (gen_<avx512>_vpermi2var<mode>3_maskz_1 (
17043 operands[0], operands[1], operands[2], operands[3],
17044 CONST0_RTX (<MODE>mode), operands[4]));
17048 (define_expand "<avx512>_vpermi2var<mode>3_maskz"
17049 [(match_operand:VI1_AVX512VL 0 "register_operand")
17050 (match_operand:VI1_AVX512VL 1 "register_operand")
17051 (match_operand:<sseintvecmode> 2 "register_operand")
17052 (match_operand:VI1_AVX512VL 3 "nonimmediate_operand")
17053 (match_operand:<avx512fmaskmode> 4 "register_operand")]
17054 "TARGET_AVX512VBMI"
17056 emit_insn (gen_<avx512>_vpermi2var<mode>3_maskz_1 (
17057 operands[0], operands[1], operands[2], operands[3],
17058 CONST0_RTX (<MODE>mode), operands[4]));
17062 (define_expand "<avx512>_vpermi2var<mode>3_maskz"
17063 [(match_operand:VI2_AVX512VL 0 "register_operand" "=v")
17064 (match_operand:VI2_AVX512VL 1 "register_operand" "v")
17065 (match_operand:<sseintvecmode> 2 "register_operand" "0")
17066 (match_operand:VI2_AVX512VL 3 "nonimmediate_operand" "vm")
17067 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk")]
17070 emit_insn (gen_<avx512>_vpermi2var<mode>3_maskz_1 (
17071 operands[0], operands[1], operands[2], operands[3],
17072 CONST0_RTX (<MODE>mode), operands[4]));
17076 (define_insn "<avx512>_vpermi2var<mode>3<sd_maskz_name>"
17077 [(set (match_operand:VI48F 0 "register_operand" "=v")
17079 [(match_operand:VI48F 1 "register_operand" "v")
17080 (match_operand:<sseintvecmode> 2 "register_operand" "0")
17081 (match_operand:VI48F 3 "nonimmediate_operand" "vm")]
17084 "vpermi2<ssemodesuffix>\t{%3, %1, %0<sd_mask_op4>|%0<sd_mask_op4>, %1, %3}"
17085 [(set_attr "type" "sselog")
17086 (set_attr "prefix" "evex")
17087 (set_attr "mode" "<sseinsnmode>")])
17089 (define_insn "<avx512>_vpermi2var<mode>3<sd_maskz_name>"
17090 [(set (match_operand:VI1_AVX512VL 0 "register_operand" "=v")
17091 (unspec:VI1_AVX512VL
17092 [(match_operand:VI1_AVX512VL 1 "register_operand" "v")
17093 (match_operand:<sseintvecmode> 2 "register_operand" "0")
17094 (match_operand:VI1_AVX512VL 3 "nonimmediate_operand" "vm")]
17096 "TARGET_AVX512VBMI"
17097 "vpermi2<ssemodesuffix>\t{%3, %1, %0<sd_mask_op4>|%0<sd_mask_op4>, %1, %3}"
17098 [(set_attr "type" "sselog")
17099 (set_attr "prefix" "evex")
17100 (set_attr "mode" "<sseinsnmode>")])
17102 (define_insn "<avx512>_vpermi2var<mode>3<sd_maskz_name>"
17103 [(set (match_operand:VI2_AVX512VL 0 "register_operand" "=v")
17104 (unspec:VI2_AVX512VL
17105 [(match_operand:VI2_AVX512VL 1 "register_operand" "v")
17106 (match_operand:<sseintvecmode> 2 "register_operand" "0")
17107 (match_operand:VI2_AVX512VL 3 "nonimmediate_operand" "vm")]
17110 "vpermi2<ssemodesuffix>\t{%3, %1, %0<sd_mask_op4>|%0<sd_mask_op4>, %1, %3}"
17111 [(set_attr "type" "sselog")
17112 (set_attr "prefix" "evex")
17113 (set_attr "mode" "<sseinsnmode>")])
17115 (define_insn "<avx512>_vpermi2var<mode>3_mask"
17116 [(set (match_operand:VI48F 0 "register_operand" "=v")
17119 [(match_operand:VI48F 1 "register_operand" "v")
17120 (match_operand:<sseintvecmode> 2 "register_operand" "0")
17121 (match_operand:VI48F 3 "nonimmediate_operand" "vm")]
17122 UNSPEC_VPERMI2_MASK)
17124 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk")))]
17126 "vpermi2<ssemodesuffix>\t{%3, %1, %0%{%4%}|%0%{%4%}, %1, %3}"
17127 [(set_attr "type" "sselog")
17128 (set_attr "prefix" "evex")
17129 (set_attr "mode" "<sseinsnmode>")])
17131 (define_insn "<avx512>_vpermi2var<mode>3_mask"
17132 [(set (match_operand:VI1_AVX512VL 0 "register_operand" "=v")
17133 (vec_merge:VI1_AVX512VL
17134 (unspec:VI1_AVX512VL
17135 [(match_operand:VI1_AVX512VL 1 "register_operand" "v")
17136 (match_operand:<sseintvecmode> 2 "register_operand" "0")
17137 (match_operand:VI1_AVX512VL 3 "nonimmediate_operand" "vm")]
17138 UNSPEC_VPERMI2_MASK)
17140 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk")))]
17141 "TARGET_AVX512VBMI"
17142 "vpermi2<ssemodesuffix>\t{%3, %1, %0%{%4%}|%0%{%4%}, %1, %3}"
17143 [(set_attr "type" "sselog")
17144 (set_attr "prefix" "evex")
17145 (set_attr "mode" "<sseinsnmode>")])
17147 (define_insn "<avx512>_vpermi2var<mode>3_mask"
17148 [(set (match_operand:VI2_AVX512VL 0 "register_operand" "=v")
17149 (vec_merge:VI2_AVX512VL
17150 (unspec:VI2_AVX512VL
17151 [(match_operand:VI2_AVX512VL 1 "register_operand" "v")
17152 (match_operand:<sseintvecmode> 2 "register_operand" "0")
17153 (match_operand:VI2_AVX512VL 3 "nonimmediate_operand" "vm")]
17154 UNSPEC_VPERMI2_MASK)
17156 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk")))]
17158 "vpermi2<ssemodesuffix>\t{%3, %1, %0%{%4%}|%0%{%4%}, %1, %3}"
17159 [(set_attr "type" "sselog")
17160 (set_attr "prefix" "evex")
17161 (set_attr "mode" "<sseinsnmode>")])
17163 (define_expand "<avx512>_vpermt2var<mode>3_maskz"
17164 [(match_operand:VI48F 0 "register_operand" "=v")
17165 (match_operand:<sseintvecmode> 1 "register_operand" "v")
17166 (match_operand:VI48F 2 "register_operand" "0")
17167 (match_operand:VI48F 3 "nonimmediate_operand" "vm")
17168 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk")]
17171 emit_insn (gen_<avx512>_vpermt2var<mode>3_maskz_1 (
17172 operands[0], operands[1], operands[2], operands[3],
17173 CONST0_RTX (<MODE>mode), operands[4]));
17177 (define_expand "<avx512>_vpermt2var<mode>3_maskz"
17178 [(match_operand:VI1_AVX512VL 0 "register_operand" "=v")
17179 (match_operand:<sseintvecmode> 1 "register_operand" "v")
17180 (match_operand:VI1_AVX512VL 2 "register_operand" "0")
17181 (match_operand:VI1_AVX512VL 3 "nonimmediate_operand" "vm")
17182 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk")]
17183 "TARGET_AVX512VBMI"
17185 emit_insn (gen_<avx512>_vpermt2var<mode>3_maskz_1 (
17186 operands[0], operands[1], operands[2], operands[3],
17187 CONST0_RTX (<MODE>mode), operands[4]));
17191 (define_expand "<avx512>_vpermt2var<mode>3_maskz"
17192 [(match_operand:VI2_AVX512VL 0 "register_operand" "=v")
17193 (match_operand:<sseintvecmode> 1 "register_operand" "v")
17194 (match_operand:VI2_AVX512VL 2 "register_operand" "0")
17195 (match_operand:VI2_AVX512VL 3 "nonimmediate_operand" "vm")
17196 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk")]
17199 emit_insn (gen_<avx512>_vpermt2var<mode>3_maskz_1 (
17200 operands[0], operands[1], operands[2], operands[3],
17201 CONST0_RTX (<MODE>mode), operands[4]));
17205 (define_insn "<avx512>_vpermt2var<mode>3<sd_maskz_name>"
17206 [(set (match_operand:VI48F 0 "register_operand" "=v")
17208 [(match_operand:<sseintvecmode> 1 "register_operand" "v")
17209 (match_operand:VI48F 2 "register_operand" "0")
17210 (match_operand:VI48F 3 "nonimmediate_operand" "vm")]
17213 "vpermt2<ssemodesuffix>\t{%3, %1, %0<sd_mask_op4>|%0<sd_mask_op4>, %1, %3}"
17214 [(set_attr "type" "sselog")
17215 (set_attr "prefix" "evex")
17216 (set_attr "mode" "<sseinsnmode>")])
17218 (define_insn "<avx512>_vpermt2var<mode>3<sd_maskz_name>"
17219 [(set (match_operand:VI1_AVX512VL 0 "register_operand" "=v")
17220 (unspec:VI1_AVX512VL
17221 [(match_operand:<sseintvecmode> 1 "register_operand" "v")
17222 (match_operand:VI1_AVX512VL 2 "register_operand" "0")
17223 (match_operand:VI1_AVX512VL 3 "nonimmediate_operand" "vm")]
17225 "TARGET_AVX512VBMI"
17226 "vpermt2<ssemodesuffix>\t{%3, %1, %0<sd_mask_op4>|%0<sd_mask_op4>, %1, %3}"
17227 [(set_attr "type" "sselog")
17228 (set_attr "prefix" "evex")
17229 (set_attr "mode" "<sseinsnmode>")])
17231 (define_insn "<avx512>_vpermt2var<mode>3<sd_maskz_name>"
17232 [(set (match_operand:VI2_AVX512VL 0 "register_operand" "=v")
17233 (unspec:VI2_AVX512VL
17234 [(match_operand:<sseintvecmode> 1 "register_operand" "v")
17235 (match_operand:VI2_AVX512VL 2 "register_operand" "0")
17236 (match_operand:VI2_AVX512VL 3 "nonimmediate_operand" "vm")]
17239 "vpermt2<ssemodesuffix>\t{%3, %1, %0<sd_mask_op4>|%0<sd_mask_op4>, %1, %3}"
17240 [(set_attr "type" "sselog")
17241 (set_attr "prefix" "evex")
17242 (set_attr "mode" "<sseinsnmode>")])
17244 (define_insn "<avx512>_vpermt2var<mode>3_mask"
17245 [(set (match_operand:VI48F 0 "register_operand" "=v")
17248 [(match_operand:<sseintvecmode> 1 "register_operand" "v")
17249 (match_operand:VI48F 2 "register_operand" "0")
17250 (match_operand:VI48F 3 "nonimmediate_operand" "vm")]
17253 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk")))]
17255 "vpermt2<ssemodesuffix>\t{%3, %1, %0%{%4%}|%0%{%4%}, %1, %3}"
17256 [(set_attr "type" "sselog")
17257 (set_attr "prefix" "evex")
17258 (set_attr "mode" "<sseinsnmode>")])
17260 (define_insn "<avx512>_vpermt2var<mode>3_mask"
17261 [(set (match_operand:VI1_AVX512VL 0 "register_operand" "=v")
17262 (vec_merge:VI1_AVX512VL
17263 (unspec:VI1_AVX512VL
17264 [(match_operand:<sseintvecmode> 1 "register_operand" "v")
17265 (match_operand:VI1_AVX512VL 2 "register_operand" "0")
17266 (match_operand:VI1_AVX512VL 3 "nonimmediate_operand" "vm")]
17269 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk")))]
17270 "TARGET_AVX512VBMI"
17271 "vpermt2<ssemodesuffix>\t{%3, %1, %0%{%4%}|%0%{%4%}, %1, %3}"
17272 [(set_attr "type" "sselog")
17273 (set_attr "prefix" "evex")
17274 (set_attr "mode" "<sseinsnmode>")])
17276 (define_insn "<avx512>_vpermt2var<mode>3_mask"
17277 [(set (match_operand:VI2_AVX512VL 0 "register_operand" "=v")
17278 (vec_merge:VI2_AVX512VL
17279 (unspec:VI2_AVX512VL
17280 [(match_operand:<sseintvecmode> 1 "register_operand" "v")
17281 (match_operand:VI2_AVX512VL 2 "register_operand" "0")
17282 (match_operand:VI2_AVX512VL 3 "nonimmediate_operand" "vm")]
17285 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk")))]
17287 "vpermt2<ssemodesuffix>\t{%3, %1, %0%{%4%}|%0%{%4%}, %1, %3}"
17288 [(set_attr "type" "sselog")
17289 (set_attr "prefix" "evex")
17290 (set_attr "mode" "<sseinsnmode>")])
17292 (define_expand "avx_vperm2f128<mode>3"
17293 [(set (match_operand:AVX256MODE2P 0 "register_operand")
17294 (unspec:AVX256MODE2P
17295 [(match_operand:AVX256MODE2P 1 "register_operand")
17296 (match_operand:AVX256MODE2P 2 "nonimmediate_operand")
17297 (match_operand:SI 3 "const_0_to_255_operand")]
17298 UNSPEC_VPERMIL2F128))]
17301 int mask = INTVAL (operands[3]);
17302 if ((mask & 0x88) == 0)
17304 rtx perm[<ssescalarnum>], t1, t2;
17305 int i, base, nelt = <ssescalarnum>, nelt2 = nelt / 2;
17307 base = (mask & 3) * nelt2;
17308 for (i = 0; i < nelt2; ++i)
17309 perm[i] = GEN_INT (base + i);
17311 base = ((mask >> 4) & 3) * nelt2;
17312 for (i = 0; i < nelt2; ++i)
17313 perm[i + nelt2] = GEN_INT (base + i);
17315 t2 = gen_rtx_VEC_CONCAT (<ssedoublevecmode>mode,
17316 operands[1], operands[2]);
17317 t1 = gen_rtx_PARALLEL (VOIDmode, gen_rtvec_v (nelt, perm));
17318 t2 = gen_rtx_VEC_SELECT (<MODE>mode, t2, t1);
17319 t2 = gen_rtx_SET (VOIDmode, operands[0], t2);
17325 ;; Note that bits 7 and 3 of the imm8 allow lanes to be zeroed, which
17326 ;; means that in order to represent this properly in rtl we'd have to
17327 ;; nest *another* vec_concat with a zero operand and do the select from
17328 ;; a 4x wide vector. That doesn't seem very nice.
17329 (define_insn "*avx_vperm2f128<mode>_full"
17330 [(set (match_operand:AVX256MODE2P 0 "register_operand" "=x")
17331 (unspec:AVX256MODE2P
17332 [(match_operand:AVX256MODE2P 1 "register_operand" "x")
17333 (match_operand:AVX256MODE2P 2 "nonimmediate_operand" "xm")
17334 (match_operand:SI 3 "const_0_to_255_operand" "n")]
17335 UNSPEC_VPERMIL2F128))]
17337 "vperm2<i128>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
17338 [(set_attr "type" "sselog")
17339 (set_attr "prefix_extra" "1")
17340 (set_attr "length_immediate" "1")
17341 (set_attr "prefix" "vex")
17342 (set_attr "mode" "<sseinsnmode>")])
17344 (define_insn "*avx_vperm2f128<mode>_nozero"
17345 [(set (match_operand:AVX256MODE2P 0 "register_operand" "=x")
17346 (vec_select:AVX256MODE2P
17347 (vec_concat:<ssedoublevecmode>
17348 (match_operand:AVX256MODE2P 1 "register_operand" "x")
17349 (match_operand:AVX256MODE2P 2 "nonimmediate_operand" "xm"))
17350 (match_parallel 3 ""
17351 [(match_operand 4 "const_int_operand")])))]
17353 && avx_vperm2f128_parallel (operands[3], <MODE>mode)"
17355 int mask = avx_vperm2f128_parallel (operands[3], <MODE>mode) - 1;
17357 return "vinsert<i128>\t{$0, %x2, %1, %0|%0, %1, %x2, 0}";
17359 return "vinsert<i128>\t{$1, %x2, %1, %0|%0, %1, %x2, 1}";
17360 operands[3] = GEN_INT (mask);
17361 return "vperm2<i128>\t{%3, %2, %1, %0|%0, %1, %2, %3}";
17363 [(set_attr "type" "sselog")
17364 (set_attr "prefix_extra" "1")
17365 (set_attr "length_immediate" "1")
17366 (set_attr "prefix" "vex")
17367 (set_attr "mode" "<sseinsnmode>")])
17369 (define_insn "*ssse3_palignr<mode>_perm"
17370 [(set (match_operand:V_128 0 "register_operand" "=x,x")
17372 (match_operand:V_128 1 "register_operand" "0,x")
17373 (match_parallel 2 "palignr_operand"
17374 [(match_operand 3 "const_int_operand" "n, n")])))]
17377 machine_mode imode = GET_MODE_INNER (GET_MODE (operands[0]));
17378 operands[2] = GEN_INT (INTVAL (operands[3]) * GET_MODE_SIZE (imode));
17380 switch (which_alternative)
17383 return "palignr\t{%2, %1, %0|%0, %1, %2}";
17385 return "vpalignr\t{%2, %1, %1, %0|%0, %1, %1, %2}";
17387 gcc_unreachable ();
17390 [(set_attr "isa" "noavx,avx")
17391 (set_attr "type" "sseishft")
17392 (set_attr "atom_unit" "sishuf")
17393 (set_attr "prefix_data16" "1,*")
17394 (set_attr "prefix_extra" "1")
17395 (set_attr "length_immediate" "1")
17396 (set_attr "prefix" "orig,vex")])
17398 (define_expand "avx512vl_vinsert<mode>"
17399 [(match_operand:VI48F_256 0 "register_operand")
17400 (match_operand:VI48F_256 1 "register_operand")
17401 (match_operand:<ssehalfvecmode> 2 "nonimmediate_operand")
17402 (match_operand:SI 3 "const_0_to_1_operand")
17403 (match_operand:VI48F_256 4 "register_operand")
17404 (match_operand:<avx512fmaskmode> 5 "register_operand")]
17407 rtx (*insn)(rtx, rtx, rtx, rtx, rtx);
17409 switch (INTVAL (operands[3]))
17412 insn = gen_vec_set_lo_<mode>_mask;
17415 insn = gen_vec_set_hi_<mode>_mask;
17418 gcc_unreachable ();
17421 emit_insn (insn (operands[0], operands[1], operands[2], operands[4],
17426 (define_expand "avx_vinsertf128<mode>"
17427 [(match_operand:V_256 0 "register_operand")
17428 (match_operand:V_256 1 "register_operand")
17429 (match_operand:<ssehalfvecmode> 2 "nonimmediate_operand")
17430 (match_operand:SI 3 "const_0_to_1_operand")]
17433 rtx (*insn)(rtx, rtx, rtx);
17435 switch (INTVAL (operands[3]))
17438 insn = gen_vec_set_lo_<mode>;
17441 insn = gen_vec_set_hi_<mode>;
17444 gcc_unreachable ();
17447 emit_insn (insn (operands[0], operands[1], operands[2]));
17451 (define_insn "vec_set_lo_<mode><mask_name>"
17452 [(set (match_operand:VI8F_256 0 "register_operand" "=v")
17453 (vec_concat:VI8F_256
17454 (match_operand:<ssehalfvecmode> 2 "nonimmediate_operand" "vm")
17455 (vec_select:<ssehalfvecmode>
17456 (match_operand:VI8F_256 1 "register_operand" "v")
17457 (parallel [(const_int 2) (const_int 3)]))))]
17460 if (TARGET_AVX512VL)
17461 return "vinsert<shuffletype>64x2\t{$0x0, %2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2, 0x0}";
17463 return "vinsert<i128>\t{$0x0, %2, %1, %0|%0, %1, %2, 0x0}";
17465 [(set_attr "type" "sselog")
17466 (set_attr "prefix_extra" "1")
17467 (set_attr "length_immediate" "1")
17468 (set_attr "prefix" "vex")
17469 (set_attr "mode" "<sseinsnmode>")])
17471 (define_insn "vec_set_hi_<mode><mask_name>"
17472 [(set (match_operand:VI8F_256 0 "register_operand" "=v")
17473 (vec_concat:VI8F_256
17474 (vec_select:<ssehalfvecmode>
17475 (match_operand:VI8F_256 1 "register_operand" "v")
17476 (parallel [(const_int 0) (const_int 1)]))
17477 (match_operand:<ssehalfvecmode> 2 "nonimmediate_operand" "vm")))]
17480 if (TARGET_AVX512VL)
17481 return "vinsert<shuffletype>64x2\t{$0x1, %2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2, 0x1}";
17483 return "vinsert<i128>\t{$0x1, %2, %1, %0|%0, %1, %2, 0x1}";
17485 [(set_attr "type" "sselog")
17486 (set_attr "prefix_extra" "1")
17487 (set_attr "length_immediate" "1")
17488 (set_attr "prefix" "vex")
17489 (set_attr "mode" "<sseinsnmode>")])
17491 (define_insn "vec_set_lo_<mode><mask_name>"
17492 [(set (match_operand:VI4F_256 0 "register_operand" "=v")
17493 (vec_concat:VI4F_256
17494 (match_operand:<ssehalfvecmode> 2 "nonimmediate_operand" "vm")
17495 (vec_select:<ssehalfvecmode>
17496 (match_operand:VI4F_256 1 "register_operand" "v")
17497 (parallel [(const_int 4) (const_int 5)
17498 (const_int 6) (const_int 7)]))))]
17501 if (TARGET_AVX512VL)
17502 return "vinsert<shuffletype>32x4\t{$0x0, %2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2, 0x0}";
17504 return "vinsert<i128>\t{$0x0, %2, %1, %0|%0, %1, %2, 0x0}";
17506 [(set_attr "type" "sselog")
17507 (set_attr "prefix_extra" "1")
17508 (set_attr "length_immediate" "1")
17509 (set_attr "prefix" "vex")
17510 (set_attr "mode" "<sseinsnmode>")])
17512 (define_insn "vec_set_hi_<mode><mask_name>"
17513 [(set (match_operand:VI4F_256 0 "register_operand" "=v")
17514 (vec_concat:VI4F_256
17515 (vec_select:<ssehalfvecmode>
17516 (match_operand:VI4F_256 1 "register_operand" "v")
17517 (parallel [(const_int 0) (const_int 1)
17518 (const_int 2) (const_int 3)]))
17519 (match_operand:<ssehalfvecmode> 2 "nonimmediate_operand" "vm")))]
17522 if (TARGET_AVX512VL)
17523 return "vinsert<shuffletype>32x4\t{$0x1, %2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2, 0x1}";
17525 return "vinsert<i128>\t{$0x1, %2, %1, %0|%0, %1, %2, 0x1}";
17527 [(set_attr "type" "sselog")
17528 (set_attr "prefix_extra" "1")
17529 (set_attr "length_immediate" "1")
17530 (set_attr "prefix" "vex")
17531 (set_attr "mode" "<sseinsnmode>")])
17533 (define_insn "vec_set_lo_v16hi"
17534 [(set (match_operand:V16HI 0 "register_operand" "=x")
17536 (match_operand:V8HI 2 "nonimmediate_operand" "xm")
17538 (match_operand:V16HI 1 "register_operand" "x")
17539 (parallel [(const_int 8) (const_int 9)
17540 (const_int 10) (const_int 11)
17541 (const_int 12) (const_int 13)
17542 (const_int 14) (const_int 15)]))))]
17544 "vinsert%~128\t{$0x0, %2, %1, %0|%0, %1, %2, 0x0}"
17545 [(set_attr "type" "sselog")
17546 (set_attr "prefix_extra" "1")
17547 (set_attr "length_immediate" "1")
17548 (set_attr "prefix" "vex")
17549 (set_attr "mode" "OI")])
17551 (define_insn "vec_set_hi_v16hi"
17552 [(set (match_operand:V16HI 0 "register_operand" "=x")
17555 (match_operand:V16HI 1 "register_operand" "x")
17556 (parallel [(const_int 0) (const_int 1)
17557 (const_int 2) (const_int 3)
17558 (const_int 4) (const_int 5)
17559 (const_int 6) (const_int 7)]))
17560 (match_operand:V8HI 2 "nonimmediate_operand" "xm")))]
17562 "vinsert%~128\t{$0x1, %2, %1, %0|%0, %1, %2, 0x1}"
17563 [(set_attr "type" "sselog")
17564 (set_attr "prefix_extra" "1")
17565 (set_attr "length_immediate" "1")
17566 (set_attr "prefix" "vex")
17567 (set_attr "mode" "OI")])
17569 (define_insn "vec_set_lo_v32qi"
17570 [(set (match_operand:V32QI 0 "register_operand" "=x")
17572 (match_operand:V16QI 2 "nonimmediate_operand" "xm")
17574 (match_operand:V32QI 1 "register_operand" "x")
17575 (parallel [(const_int 16) (const_int 17)
17576 (const_int 18) (const_int 19)
17577 (const_int 20) (const_int 21)
17578 (const_int 22) (const_int 23)
17579 (const_int 24) (const_int 25)
17580 (const_int 26) (const_int 27)
17581 (const_int 28) (const_int 29)
17582 (const_int 30) (const_int 31)]))))]
17584 "vinsert%~128\t{$0x0, %2, %1, %0|%0, %1, %2, 0x0}"
17585 [(set_attr "type" "sselog")
17586 (set_attr "prefix_extra" "1")
17587 (set_attr "length_immediate" "1")
17588 (set_attr "prefix" "vex")
17589 (set_attr "mode" "OI")])
17591 (define_insn "vec_set_hi_v32qi"
17592 [(set (match_operand:V32QI 0 "register_operand" "=x")
17595 (match_operand:V32QI 1 "register_operand" "x")
17596 (parallel [(const_int 0) (const_int 1)
17597 (const_int 2) (const_int 3)
17598 (const_int 4) (const_int 5)
17599 (const_int 6) (const_int 7)
17600 (const_int 8) (const_int 9)
17601 (const_int 10) (const_int 11)
17602 (const_int 12) (const_int 13)
17603 (const_int 14) (const_int 15)]))
17604 (match_operand:V16QI 2 "nonimmediate_operand" "xm")))]
17606 "vinsert%~128\t{$0x1, %2, %1, %0|%0, %1, %2, 0x1}"
17607 [(set_attr "type" "sselog")
17608 (set_attr "prefix_extra" "1")
17609 (set_attr "length_immediate" "1")
17610 (set_attr "prefix" "vex")
17611 (set_attr "mode" "OI")])
17613 (define_insn "<avx_avx2>_maskload<ssemodesuffix><avxsizesuffix>"
17614 [(set (match_operand:V48_AVX2 0 "register_operand" "=x")
17616 [(match_operand:<sseintvecmode> 2 "register_operand" "x")
17617 (match_operand:V48_AVX2 1 "memory_operand" "m")]
17620 "v<sseintprefix>maskmov<ssemodesuffix>\t{%1, %2, %0|%0, %2, %1}"
17621 [(set_attr "type" "sselog1")
17622 (set_attr "prefix_extra" "1")
17623 (set_attr "prefix" "vex")
17624 (set_attr "btver2_decode" "vector")
17625 (set_attr "mode" "<sseinsnmode>")])
17627 (define_insn "<avx_avx2>_maskstore<ssemodesuffix><avxsizesuffix>"
17628 [(set (match_operand:V48_AVX2 0 "memory_operand" "+m")
17630 [(match_operand:<sseintvecmode> 1 "register_operand" "x")
17631 (match_operand:V48_AVX2 2 "register_operand" "x")
17635 "v<sseintprefix>maskmov<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
17636 [(set_attr "type" "sselog1")
17637 (set_attr "prefix_extra" "1")
17638 (set_attr "prefix" "vex")
17639 (set_attr "btver2_decode" "vector")
17640 (set_attr "mode" "<sseinsnmode>")])
17642 (define_expand "maskload<mode>"
17643 [(set (match_operand:V48_AVX2 0 "register_operand")
17645 [(match_operand:<sseintvecmode> 2 "register_operand")
17646 (match_operand:V48_AVX2 1 "memory_operand")]
17650 (define_expand "maskstore<mode>"
17651 [(set (match_operand:V48_AVX2 0 "memory_operand")
17653 [(match_operand:<sseintvecmode> 2 "register_operand")
17654 (match_operand:V48_AVX2 1 "register_operand")
17659 (define_insn_and_split "avx_<castmode><avxsizesuffix>_<castmode>"
17660 [(set (match_operand:AVX256MODE2P 0 "nonimmediate_operand" "=x,m")
17661 (unspec:AVX256MODE2P
17662 [(match_operand:<ssehalfvecmode> 1 "nonimmediate_operand" "xm,x")]
17666 "&& reload_completed"
17669 rtx op0 = operands[0];
17670 rtx op1 = operands[1];
17672 op0 = gen_rtx_REG (<ssehalfvecmode>mode, REGNO (op0));
17674 op1 = gen_rtx_REG (<MODE>mode, REGNO (op1));
17675 emit_move_insn (op0, op1);
17679 (define_expand "vec_init<mode>"
17680 [(match_operand:V_256 0 "register_operand")
17684 ix86_expand_vector_init (false, operands[0], operands[1]);
17688 (define_expand "vec_init<mode>"
17689 [(match_operand:VF48_I1248 0 "register_operand")
17693 ix86_expand_vector_init (false, operands[0], operands[1]);
17697 (define_insn "<avx2_avx512>_ashrv<mode><mask_name>"
17698 [(set (match_operand:VI48_AVX512F_AVX512VL 0 "register_operand" "=v")
17699 (ashiftrt:VI48_AVX512F_AVX512VL
17700 (match_operand:VI48_AVX512F_AVX512VL 1 "register_operand" "v")
17701 (match_operand:VI48_AVX512F_AVX512VL 2 "nonimmediate_operand" "vm")))]
17702 "TARGET_AVX2 && <mask_mode512bit_condition>"
17703 "vpsrav<ssemodesuffix>\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
17704 [(set_attr "type" "sseishft")
17705 (set_attr "prefix" "maybe_evex")
17706 (set_attr "mode" "<sseinsnmode>")])
17708 (define_insn "<avx2_avx512>_ashrv<mode><mask_name>"
17709 [(set (match_operand:VI2_AVX512VL 0 "register_operand" "=v")
17710 (ashiftrt:VI2_AVX512VL
17711 (match_operand:VI2_AVX512VL 1 "register_operand" "v")
17712 (match_operand:VI2_AVX512VL 2 "nonimmediate_operand" "vm")))]
17714 "vpsravw\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
17715 [(set_attr "type" "sseishft")
17716 (set_attr "prefix" "maybe_evex")
17717 (set_attr "mode" "<sseinsnmode>")])
17719 (define_insn "<avx2_avx512>_<shift_insn>v<mode><mask_name>"
17720 [(set (match_operand:VI48_AVX512F 0 "register_operand" "=v")
17721 (any_lshift:VI48_AVX512F
17722 (match_operand:VI48_AVX512F 1 "register_operand" "v")
17723 (match_operand:VI48_AVX512F 2 "nonimmediate_operand" "vm")))]
17724 "TARGET_AVX2 && <mask_mode512bit_condition>"
17725 "vp<vshift>v<ssemodesuffix>\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
17726 [(set_attr "type" "sseishft")
17727 (set_attr "prefix" "maybe_evex")
17728 (set_attr "mode" "<sseinsnmode>")])
17730 (define_insn "<avx2_avx512>_<shift_insn>v<mode><mask_name>"
17731 [(set (match_operand:VI2_AVX512VL 0 "register_operand" "=v")
17732 (any_lshift:VI2_AVX512VL
17733 (match_operand:VI2_AVX512VL 1 "register_operand" "v")
17734 (match_operand:VI2_AVX512VL 2 "nonimmediate_operand" "vm")))]
17736 "vp<vshift>v<ssemodesuffix>\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
17737 [(set_attr "type" "sseishft")
17738 (set_attr "prefix" "maybe_evex")
17739 (set_attr "mode" "<sseinsnmode>")])
17741 (define_insn "avx_vec_concat<mode>"
17742 [(set (match_operand:V_256_512 0 "register_operand" "=x,x")
17743 (vec_concat:V_256_512
17744 (match_operand:<ssehalfvecmode> 1 "register_operand" "x,x")
17745 (match_operand:<ssehalfvecmode> 2 "vector_move_operand" "xm,C")))]
17748 switch (which_alternative)
17751 return "vinsert<i128>\t{$0x1, %2, %<concat_tg_mode>1, %0|%0, %<concat_tg_mode>1, %2, 0x1}";
17753 switch (get_attr_mode (insn))
17756 return "vmovaps\t{%1, %t0|%t0, %1}";
17758 return "vmovapd\t{%1, %t0|%t0, %1}";
17760 return "vmovaps\t{%1, %x0|%x0, %1}";
17762 return "vmovapd\t{%1, %x0|%x0, %1}";
17764 return "vmovdqa\t{%1, %t0|%t0, %1}";
17766 return "vmovdqa\t{%1, %x0|%x0, %1}";
17768 gcc_unreachable ();
17771 gcc_unreachable ();
17774 [(set_attr "type" "sselog,ssemov")
17775 (set_attr "prefix_extra" "1,*")
17776 (set_attr "length_immediate" "1,*")
17777 (set_attr "prefix" "maybe_evex")
17778 (set_attr "mode" "<sseinsnmode>")])
17780 (define_insn "vcvtph2ps<mask_name>"
17781 [(set (match_operand:V4SF 0 "register_operand" "=v")
17783 (unspec:V8SF [(match_operand:V8HI 1 "register_operand" "v")]
17785 (parallel [(const_int 0) (const_int 1)
17786 (const_int 2) (const_int 3)])))]
17787 "TARGET_F16C || TARGET_AVX512VL"
17788 "vcvtph2ps\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
17789 [(set_attr "type" "ssecvt")
17790 (set_attr "prefix" "maybe_evex")
17791 (set_attr "mode" "V4SF")])
17793 (define_insn "*vcvtph2ps_load<mask_name>"
17794 [(set (match_operand:V4SF 0 "register_operand" "=v")
17795 (unspec:V4SF [(match_operand:V4HI 1 "memory_operand" "m")]
17796 UNSPEC_VCVTPH2PS))]
17797 "TARGET_F16C || TARGET_AVX512VL"
17798 "vcvtph2ps\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
17799 [(set_attr "type" "ssecvt")
17800 (set_attr "prefix" "vex")
17801 (set_attr "mode" "V8SF")])
17803 (define_insn "vcvtph2ps256<mask_name>"
17804 [(set (match_operand:V8SF 0 "register_operand" "=v")
17805 (unspec:V8SF [(match_operand:V8HI 1 "nonimmediate_operand" "vm")]
17806 UNSPEC_VCVTPH2PS))]
17807 "TARGET_F16C || TARGET_AVX512VL"
17808 "vcvtph2ps\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
17809 [(set_attr "type" "ssecvt")
17810 (set_attr "prefix" "vex")
17811 (set_attr "btver2_decode" "double")
17812 (set_attr "mode" "V8SF")])
17814 (define_insn "<mask_codefor>avx512f_vcvtph2ps512<mask_name><round_saeonly_name>"
17815 [(set (match_operand:V16SF 0 "register_operand" "=v")
17817 [(match_operand:V16HI 1 "<round_saeonly_nimm_predicate>" "<round_saeonly_constraint>")]
17818 UNSPEC_VCVTPH2PS))]
17820 "vcvtph2ps\t{<round_saeonly_mask_op2>%1, %0<mask_operand2>|%0<mask_operand2>, %1<round_saeonly_mask_op2>}"
17821 [(set_attr "type" "ssecvt")
17822 (set_attr "prefix" "evex")
17823 (set_attr "mode" "V16SF")])
17825 (define_expand "vcvtps2ph_mask"
17826 [(set (match_operand:V8HI 0 "register_operand")
17829 (unspec:V4HI [(match_operand:V4SF 1 "register_operand")
17830 (match_operand:SI 2 "const_0_to_255_operand")]
17833 (match_operand:V8HI 3 "vector_move_operand")
17834 (match_operand:QI 4 "register_operand")))]
17836 "operands[5] = CONST0_RTX (V4HImode);")
17838 (define_expand "vcvtps2ph"
17839 [(set (match_operand:V8HI 0 "register_operand")
17841 (unspec:V4HI [(match_operand:V4SF 1 "register_operand")
17842 (match_operand:SI 2 "const_0_to_255_operand")]
17846 "operands[3] = CONST0_RTX (V4HImode);")
17848 (define_insn "*vcvtps2ph<mask_name>"
17849 [(set (match_operand:V8HI 0 "register_operand" "=v")
17851 (unspec:V4HI [(match_operand:V4SF 1 "register_operand" "v")
17852 (match_operand:SI 2 "const_0_to_255_operand" "N")]
17854 (match_operand:V4HI 3 "const0_operand")))]
17855 "(TARGET_F16C || TARGET_AVX512VL) && <mask_avx512vl_condition>"
17856 "vcvtps2ph\t{%2, %1, %0<mask_operand4>|%0<mask_operand4>, %1, %2}"
17857 [(set_attr "type" "ssecvt")
17858 (set_attr "prefix" "maybe_evex")
17859 (set_attr "mode" "V4SF")])
17861 (define_insn "*vcvtps2ph_store<mask_name>"
17862 [(set (match_operand:V4HI 0 "memory_operand" "=m")
17863 (unspec:V4HI [(match_operand:V4SF 1 "register_operand" "x")
17864 (match_operand:SI 2 "const_0_to_255_operand" "N")]
17865 UNSPEC_VCVTPS2PH))]
17866 "TARGET_F16C || TARGET_AVX512VL"
17867 "vcvtps2ph\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
17868 [(set_attr "type" "ssecvt")
17869 (set_attr "prefix" "maybe_evex")
17870 (set_attr "mode" "V4SF")])
17872 (define_insn "vcvtps2ph256<mask_name>"
17873 [(set (match_operand:V8HI 0 "nonimmediate_operand" "=xm")
17874 (unspec:V8HI [(match_operand:V8SF 1 "register_operand" "x")
17875 (match_operand:SI 2 "const_0_to_255_operand" "N")]
17876 UNSPEC_VCVTPS2PH))]
17877 "TARGET_F16C || TARGET_AVX512VL"
17878 "vcvtps2ph\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
17879 [(set_attr "type" "ssecvt")
17880 (set_attr "prefix" "maybe_evex")
17881 (set_attr "btver2_decode" "vector")
17882 (set_attr "mode" "V8SF")])
17884 (define_insn "<mask_codefor>avx512f_vcvtps2ph512<mask_name>"
17885 [(set (match_operand:V16HI 0 "nonimmediate_operand" "=vm")
17887 [(match_operand:V16SF 1 "register_operand" "v")
17888 (match_operand:SI 2 "const_0_to_255_operand" "N")]
17889 UNSPEC_VCVTPS2PH))]
17891 "vcvtps2ph\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
17892 [(set_attr "type" "ssecvt")
17893 (set_attr "prefix" "evex")
17894 (set_attr "mode" "V16SF")])
17896 ;; For gather* insn patterns
17897 (define_mode_iterator VEC_GATHER_MODE
17898 [V2DI V2DF V4DI V4DF V4SI V4SF V8SI V8SF])
17899 (define_mode_attr VEC_GATHER_IDXSI
17900 [(V2DI "V4SI") (V4DI "V4SI") (V8DI "V8SI")
17901 (V2DF "V4SI") (V4DF "V4SI") (V8DF "V8SI")
17902 (V4SI "V4SI") (V8SI "V8SI") (V16SI "V16SI")
17903 (V4SF "V4SI") (V8SF "V8SI") (V16SF "V16SI")])
17905 (define_mode_attr VEC_GATHER_IDXDI
17906 [(V2DI "V2DI") (V4DI "V4DI") (V8DI "V8DI")
17907 (V2DF "V2DI") (V4DF "V4DI") (V8DF "V8DI")
17908 (V4SI "V2DI") (V8SI "V4DI") (V16SI "V8DI")
17909 (V4SF "V2DI") (V8SF "V4DI") (V16SF "V8DI")])
17911 (define_mode_attr VEC_GATHER_SRCDI
17912 [(V2DI "V2DI") (V4DI "V4DI") (V8DI "V8DI")
17913 (V2DF "V2DF") (V4DF "V4DF") (V8DF "V8DF")
17914 (V4SI "V4SI") (V8SI "V4SI") (V16SI "V8SI")
17915 (V4SF "V4SF") (V8SF "V4SF") (V16SF "V8SF")])
17917 (define_expand "avx2_gathersi<mode>"
17918 [(parallel [(set (match_operand:VEC_GATHER_MODE 0 "register_operand")
17919 (unspec:VEC_GATHER_MODE
17920 [(match_operand:VEC_GATHER_MODE 1 "register_operand")
17921 (mem:<ssescalarmode>
17923 [(match_operand 2 "vsib_address_operand")
17924 (match_operand:<VEC_GATHER_IDXSI>
17925 3 "register_operand")
17926 (match_operand:SI 5 "const1248_operand ")]))
17927 (mem:BLK (scratch))
17928 (match_operand:VEC_GATHER_MODE 4 "register_operand")]
17930 (clobber (match_scratch:VEC_GATHER_MODE 6))])]
17934 = gen_rtx_UNSPEC (Pmode, gen_rtvec (3, operands[2], operands[3],
17935 operands[5]), UNSPEC_VSIBADDR);
17938 (define_insn "*avx2_gathersi<mode>"
17939 [(set (match_operand:VEC_GATHER_MODE 0 "register_operand" "=&x")
17940 (unspec:VEC_GATHER_MODE
17941 [(match_operand:VEC_GATHER_MODE 2 "register_operand" "0")
17942 (match_operator:<ssescalarmode> 7 "vsib_mem_operator"
17944 [(match_operand:P 3 "vsib_address_operand" "Tv")
17945 (match_operand:<VEC_GATHER_IDXSI> 4 "register_operand" "x")
17946 (match_operand:SI 6 "const1248_operand" "n")]
17948 (mem:BLK (scratch))
17949 (match_operand:VEC_GATHER_MODE 5 "register_operand" "1")]
17951 (clobber (match_scratch:VEC_GATHER_MODE 1 "=&x"))]
17953 "v<sseintprefix>gatherd<ssemodesuffix>\t{%1, %7, %0|%0, %7, %1}"
17954 [(set_attr "type" "ssemov")
17955 (set_attr "prefix" "vex")
17956 (set_attr "mode" "<sseinsnmode>")])
17958 (define_insn "*avx2_gathersi<mode>_2"
17959 [(set (match_operand:VEC_GATHER_MODE 0 "register_operand" "=&x")
17960 (unspec:VEC_GATHER_MODE
17962 (match_operator:<ssescalarmode> 6 "vsib_mem_operator"
17964 [(match_operand:P 2 "vsib_address_operand" "Tv")
17965 (match_operand:<VEC_GATHER_IDXSI> 3 "register_operand" "x")
17966 (match_operand:SI 5 "const1248_operand" "n")]
17968 (mem:BLK (scratch))
17969 (match_operand:VEC_GATHER_MODE 4 "register_operand" "1")]
17971 (clobber (match_scratch:VEC_GATHER_MODE 1 "=&x"))]
17973 "v<sseintprefix>gatherd<ssemodesuffix>\t{%1, %6, %0|%0, %6, %1}"
17974 [(set_attr "type" "ssemov")
17975 (set_attr "prefix" "vex")
17976 (set_attr "mode" "<sseinsnmode>")])
17978 (define_expand "avx2_gatherdi<mode>"
17979 [(parallel [(set (match_operand:VEC_GATHER_MODE 0 "register_operand")
17980 (unspec:VEC_GATHER_MODE
17981 [(match_operand:<VEC_GATHER_SRCDI> 1 "register_operand")
17982 (mem:<ssescalarmode>
17984 [(match_operand 2 "vsib_address_operand")
17985 (match_operand:<VEC_GATHER_IDXDI>
17986 3 "register_operand")
17987 (match_operand:SI 5 "const1248_operand ")]))
17988 (mem:BLK (scratch))
17989 (match_operand:<VEC_GATHER_SRCDI>
17990 4 "register_operand")]
17992 (clobber (match_scratch:VEC_GATHER_MODE 6))])]
17996 = gen_rtx_UNSPEC (Pmode, gen_rtvec (3, operands[2], operands[3],
17997 operands[5]), UNSPEC_VSIBADDR);
18000 (define_insn "*avx2_gatherdi<mode>"
18001 [(set (match_operand:VEC_GATHER_MODE 0 "register_operand" "=&x")
18002 (unspec:VEC_GATHER_MODE
18003 [(match_operand:<VEC_GATHER_SRCDI> 2 "register_operand" "0")
18004 (match_operator:<ssescalarmode> 7 "vsib_mem_operator"
18006 [(match_operand:P 3 "vsib_address_operand" "Tv")
18007 (match_operand:<VEC_GATHER_IDXDI> 4 "register_operand" "x")
18008 (match_operand:SI 6 "const1248_operand" "n")]
18010 (mem:BLK (scratch))
18011 (match_operand:<VEC_GATHER_SRCDI> 5 "register_operand" "1")]
18013 (clobber (match_scratch:VEC_GATHER_MODE 1 "=&x"))]
18015 "v<sseintprefix>gatherq<ssemodesuffix>\t{%5, %7, %2|%2, %7, %5}"
18016 [(set_attr "type" "ssemov")
18017 (set_attr "prefix" "vex")
18018 (set_attr "mode" "<sseinsnmode>")])
18020 (define_insn "*avx2_gatherdi<mode>_2"
18021 [(set (match_operand:VEC_GATHER_MODE 0 "register_operand" "=&x")
18022 (unspec:VEC_GATHER_MODE
18024 (match_operator:<ssescalarmode> 6 "vsib_mem_operator"
18026 [(match_operand:P 2 "vsib_address_operand" "Tv")
18027 (match_operand:<VEC_GATHER_IDXDI> 3 "register_operand" "x")
18028 (match_operand:SI 5 "const1248_operand" "n")]
18030 (mem:BLK (scratch))
18031 (match_operand:<VEC_GATHER_SRCDI> 4 "register_operand" "1")]
18033 (clobber (match_scratch:VEC_GATHER_MODE 1 "=&x"))]
18036 if (<MODE>mode != <VEC_GATHER_SRCDI>mode)
18037 return "v<sseintprefix>gatherq<ssemodesuffix>\t{%4, %6, %x0|%x0, %6, %4}";
18038 return "v<sseintprefix>gatherq<ssemodesuffix>\t{%4, %6, %0|%0, %6, %4}";
18040 [(set_attr "type" "ssemov")
18041 (set_attr "prefix" "vex")
18042 (set_attr "mode" "<sseinsnmode>")])
18044 (define_insn "*avx2_gatherdi<mode>_3"
18045 [(set (match_operand:<VEC_GATHER_SRCDI> 0 "register_operand" "=&x")
18046 (vec_select:<VEC_GATHER_SRCDI>
18048 [(match_operand:<VEC_GATHER_SRCDI> 2 "register_operand" "0")
18049 (match_operator:<ssescalarmode> 7 "vsib_mem_operator"
18051 [(match_operand:P 3 "vsib_address_operand" "Tv")
18052 (match_operand:<VEC_GATHER_IDXDI> 4 "register_operand" "x")
18053 (match_operand:SI 6 "const1248_operand" "n")]
18055 (mem:BLK (scratch))
18056 (match_operand:<VEC_GATHER_SRCDI> 5 "register_operand" "1")]
18058 (parallel [(const_int 0) (const_int 1)
18059 (const_int 2) (const_int 3)])))
18060 (clobber (match_scratch:VI4F_256 1 "=&x"))]
18062 "v<sseintprefix>gatherq<ssemodesuffix>\t{%5, %7, %0|%0, %7, %5}"
18063 [(set_attr "type" "ssemov")
18064 (set_attr "prefix" "vex")
18065 (set_attr "mode" "<sseinsnmode>")])
18067 (define_insn "*avx2_gatherdi<mode>_4"
18068 [(set (match_operand:<VEC_GATHER_SRCDI> 0 "register_operand" "=&x")
18069 (vec_select:<VEC_GATHER_SRCDI>
18072 (match_operator:<ssescalarmode> 6 "vsib_mem_operator"
18074 [(match_operand:P 2 "vsib_address_operand" "Tv")
18075 (match_operand:<VEC_GATHER_IDXDI> 3 "register_operand" "x")
18076 (match_operand:SI 5 "const1248_operand" "n")]
18078 (mem:BLK (scratch))
18079 (match_operand:<VEC_GATHER_SRCDI> 4 "register_operand" "1")]
18081 (parallel [(const_int 0) (const_int 1)
18082 (const_int 2) (const_int 3)])))
18083 (clobber (match_scratch:VI4F_256 1 "=&x"))]
18085 "v<sseintprefix>gatherq<ssemodesuffix>\t{%4, %6, %0|%0, %6, %4}"
18086 [(set_attr "type" "ssemov")
18087 (set_attr "prefix" "vex")
18088 (set_attr "mode" "<sseinsnmode>")])
18090 (define_expand "<avx512>_gathersi<mode>"
18091 [(parallel [(set (match_operand:VI48F 0 "register_operand")
18093 [(match_operand:VI48F 1 "register_operand")
18094 (match_operand:<avx512fmaskmode> 4 "register_operand")
18095 (mem:<ssescalarmode>
18097 [(match_operand 2 "vsib_address_operand")
18098 (match_operand:<VEC_GATHER_IDXSI> 3 "register_operand")
18099 (match_operand:SI 5 "const1248_operand")]))]
18101 (clobber (match_scratch:<avx512fmaskmode> 7))])]
18105 = gen_rtx_UNSPEC (Pmode, gen_rtvec (3, operands[2], operands[3],
18106 operands[5]), UNSPEC_VSIBADDR);
18109 (define_insn "*avx512f_gathersi<mode>"
18110 [(set (match_operand:VI48F 0 "register_operand" "=&v")
18112 [(match_operand:VI48F 1 "register_operand" "0")
18113 (match_operand:<avx512fmaskmode> 7 "register_operand" "2")
18114 (match_operator:<ssescalarmode> 6 "vsib_mem_operator"
18116 [(match_operand:P 4 "vsib_address_operand" "Tv")
18117 (match_operand:<VEC_GATHER_IDXSI> 3 "register_operand" "v")
18118 (match_operand:SI 5 "const1248_operand" "n")]
18119 UNSPEC_VSIBADDR)])]
18121 (clobber (match_scratch:<avx512fmaskmode> 2 "=&Yk"))]
18123 "v<sseintprefix>gatherd<ssemodesuffix>\t{%6, %0%{%2%}|%0%{%2%}, %g6}"
18124 [(set_attr "type" "ssemov")
18125 (set_attr "prefix" "evex")
18126 (set_attr "mode" "<sseinsnmode>")])
18128 (define_insn "*avx512f_gathersi<mode>_2"
18129 [(set (match_operand:VI48F 0 "register_operand" "=&v")
18132 (match_operand:<avx512fmaskmode> 6 "register_operand" "1")
18133 (match_operator:<ssescalarmode> 5 "vsib_mem_operator"
18135 [(match_operand:P 3 "vsib_address_operand" "Tv")
18136 (match_operand:<VEC_GATHER_IDXSI> 2 "register_operand" "v")
18137 (match_operand:SI 4 "const1248_operand" "n")]
18138 UNSPEC_VSIBADDR)])]
18140 (clobber (match_scratch:<avx512fmaskmode> 1 "=&Yk"))]
18142 "v<sseintprefix>gatherd<ssemodesuffix>\t{%5, %0%{%1%}|%0%{%1%}, %g5}"
18143 [(set_attr "type" "ssemov")
18144 (set_attr "prefix" "evex")
18145 (set_attr "mode" "<sseinsnmode>")])
18148 (define_expand "<avx512>_gatherdi<mode>"
18149 [(parallel [(set (match_operand:VI48F 0 "register_operand")
18151 [(match_operand:<VEC_GATHER_SRCDI> 1 "register_operand")
18152 (match_operand:QI 4 "register_operand")
18153 (mem:<ssescalarmode>
18155 [(match_operand 2 "vsib_address_operand")
18156 (match_operand:<VEC_GATHER_IDXDI> 3 "register_operand")
18157 (match_operand:SI 5 "const1248_operand")]))]
18159 (clobber (match_scratch:QI 7))])]
18163 = gen_rtx_UNSPEC (Pmode, gen_rtvec (3, operands[2], operands[3],
18164 operands[5]), UNSPEC_VSIBADDR);
18167 (define_insn "*avx512f_gatherdi<mode>"
18168 [(set (match_operand:VI48F 0 "register_operand" "=&v")
18170 [(match_operand:<VEC_GATHER_SRCDI> 1 "register_operand" "0")
18171 (match_operand:QI 7 "register_operand" "2")
18172 (match_operator:<ssescalarmode> 6 "vsib_mem_operator"
18174 [(match_operand:P 4 "vsib_address_operand" "Tv")
18175 (match_operand:<VEC_GATHER_IDXDI> 3 "register_operand" "v")
18176 (match_operand:SI 5 "const1248_operand" "n")]
18177 UNSPEC_VSIBADDR)])]
18179 (clobber (match_scratch:QI 2 "=&Yk"))]
18181 "v<sseintprefix>gatherq<ssemodesuffix>\t{%6, %1%{%2%}|%1%{%2%}, %g6}"
18182 [(set_attr "type" "ssemov")
18183 (set_attr "prefix" "evex")
18184 (set_attr "mode" "<sseinsnmode>")])
18186 (define_insn "*avx512f_gatherdi<mode>_2"
18187 [(set (match_operand:VI48F 0 "register_operand" "=&v")
18190 (match_operand:QI 6 "register_operand" "1")
18191 (match_operator:<ssescalarmode> 5 "vsib_mem_operator"
18193 [(match_operand:P 3 "vsib_address_operand" "Tv")
18194 (match_operand:<VEC_GATHER_IDXDI> 2 "register_operand" "v")
18195 (match_operand:SI 4 "const1248_operand" "n")]
18196 UNSPEC_VSIBADDR)])]
18198 (clobber (match_scratch:QI 1 "=&Yk"))]
18201 if (<MODE>mode != <VEC_GATHER_SRCDI>mode)
18203 if (<MODE_SIZE> != 64)
18204 return "v<sseintprefix>gatherq<ssemodesuffix>\t{%5, %x0%{%1%}|%t0%{%1%}, %g5}";
18206 return "v<sseintprefix>gatherq<ssemodesuffix>\t{%5, %t0%{%1%}|%t0%{%1%}, %g5}";
18208 return "v<sseintprefix>gatherq<ssemodesuffix>\t{%5, %0%{%1%}|%0%{%1%}, %g5}";
18210 [(set_attr "type" "ssemov")
18211 (set_attr "prefix" "evex")
18212 (set_attr "mode" "<sseinsnmode>")])
18214 (define_expand "<avx512>_scattersi<mode>"
18215 [(parallel [(set (mem:VI48F
18217 [(match_operand 0 "vsib_address_operand")
18218 (match_operand:<VEC_GATHER_IDXSI> 2 "register_operand")
18219 (match_operand:SI 4 "const1248_operand")]))
18221 [(match_operand:<avx512fmaskmode> 1 "register_operand")
18222 (match_operand:VI48F 3 "register_operand")]
18224 (clobber (match_scratch:<avx512fmaskmode> 6))])]
18228 = gen_rtx_UNSPEC (Pmode, gen_rtvec (3, operands[0], operands[2],
18229 operands[4]), UNSPEC_VSIBADDR);
18232 (define_insn "*avx512f_scattersi<mode>"
18233 [(set (match_operator:VI48F 5 "vsib_mem_operator"
18235 [(match_operand:P 0 "vsib_address_operand" "Tv")
18236 (match_operand:<VEC_GATHER_IDXSI> 2 "register_operand" "v")
18237 (match_operand:SI 4 "const1248_operand" "n")]
18240 [(match_operand:<avx512fmaskmode> 6 "register_operand" "1")
18241 (match_operand:VI48F 3 "register_operand" "v")]
18243 (clobber (match_scratch:<avx512fmaskmode> 1 "=&Yk"))]
18245 "v<sseintprefix>scatterd<ssemodesuffix>\t{%3, %5%{%1%}|%5%{%1%}, %3}"
18246 [(set_attr "type" "ssemov")
18247 (set_attr "prefix" "evex")
18248 (set_attr "mode" "<sseinsnmode>")])
18250 (define_expand "<avx512>_scatterdi<mode>"
18251 [(parallel [(set (mem:VI48F
18253 [(match_operand 0 "vsib_address_operand")
18254 (match_operand:<VEC_GATHER_IDXDI> 2 "register_operand")
18255 (match_operand:SI 4 "const1248_operand")]))
18257 [(match_operand:QI 1 "register_operand")
18258 (match_operand:<VEC_GATHER_SRCDI> 3 "register_operand")]
18260 (clobber (match_scratch:QI 6))])]
18264 = gen_rtx_UNSPEC (Pmode, gen_rtvec (3, operands[0], operands[2],
18265 operands[4]), UNSPEC_VSIBADDR);
18268 (define_insn "*avx512f_scatterdi<mode>"
18269 [(set (match_operator:VI48F 5 "vsib_mem_operator"
18271 [(match_operand:P 0 "vsib_address_operand" "Tv")
18272 (match_operand:<VEC_GATHER_IDXDI> 2 "register_operand" "v")
18273 (match_operand:SI 4 "const1248_operand" "n")]
18276 [(match_operand:QI 6 "register_operand" "1")
18277 (match_operand:<VEC_GATHER_SRCDI> 3 "register_operand" "v")]
18279 (clobber (match_scratch:QI 1 "=&Yk"))]
18281 "v<sseintprefix>scatterq<ssemodesuffix>\t{%3, %5%{%1%}|%5%{%1%}, %3}"
18282 [(set_attr "type" "ssemov")
18283 (set_attr "prefix" "evex")
18284 (set_attr "mode" "<sseinsnmode>")])
18286 (define_insn "<avx512>_compress<mode>_mask"
18287 [(set (match_operand:VI48F 0 "register_operand" "=v")
18289 [(match_operand:VI48F 1 "register_operand" "v")
18290 (match_operand:VI48F 2 "vector_move_operand" "0C")
18291 (match_operand:<avx512fmaskmode> 3 "register_operand" "Yk")]
18294 "v<sseintprefix>compress<ssemodesuffix>\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"
18295 [(set_attr "type" "ssemov")
18296 (set_attr "prefix" "evex")
18297 (set_attr "mode" "<sseinsnmode>")])
18299 (define_insn "<avx512>_compressstore<mode>_mask"
18300 [(set (match_operand:VI48F 0 "memory_operand" "=m")
18302 [(match_operand:VI48F 1 "register_operand" "x")
18304 (match_operand:<avx512fmaskmode> 2 "register_operand" "Yk")]
18305 UNSPEC_COMPRESS_STORE))]
18307 "v<sseintprefix>compress<ssemodesuffix>\t{%1, %0%{%2%}|%0%{%2%}, %1}"
18308 [(set_attr "type" "ssemov")
18309 (set_attr "prefix" "evex")
18310 (set_attr "memory" "store")
18311 (set_attr "mode" "<sseinsnmode>")])
18313 (define_expand "<avx512>_expand<mode>_maskz"
18314 [(set (match_operand:VI48F 0 "register_operand")
18316 [(match_operand:VI48F 1 "nonimmediate_operand")
18317 (match_operand:VI48F 2 "vector_move_operand")
18318 (match_operand:<avx512fmaskmode> 3 "register_operand")]
18321 "operands[2] = CONST0_RTX (<MODE>mode);")
18323 (define_insn "<avx512>_expand<mode>_mask"
18324 [(set (match_operand:VI48F 0 "register_operand" "=v,v")
18326 [(match_operand:VI48F 1 "nonimmediate_operand" "v,m")
18327 (match_operand:VI48F 2 "vector_move_operand" "0C,0C")
18328 (match_operand:<avx512fmaskmode> 3 "register_operand" "Yk,Yk")]
18331 "v<sseintprefix>expand<ssemodesuffix>\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"
18332 [(set_attr "type" "ssemov")
18333 (set_attr "prefix" "evex")
18334 (set_attr "memory" "none,load")
18335 (set_attr "mode" "<sseinsnmode>")])
18337 (define_insn "avx512dq_rangep<mode><mask_name><round_saeonly_name>"
18338 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v")
18339 (unspec:VF_AVX512VL
18340 [(match_operand:VF_AVX512VL 1 "register_operand" "v")
18341 (match_operand:VF_AVX512VL 2 "<round_saeonly_nimm_predicate>" "<round_saeonly_constraint>")
18342 (match_operand:SI 3 "const_0_to_15_operand")]
18344 "TARGET_AVX512DQ && <round_saeonly_mode512bit_condition>"
18345 "vrange<ssemodesuffix>\t{<round_saeonly_mask_op4>%3, %2, %1, %0<mask_operand4>|%0<mask_operand4>, %1, %2, %3<round_saeonly_mask_op4>}"
18346 [(set_attr "type" "sse")
18347 (set_attr "prefix" "evex")
18348 (set_attr "mode" "<MODE>")])
18350 (define_insn "avx512dq_ranges<mode><round_saeonly_name>"
18351 [(set (match_operand:VF_128 0 "register_operand" "=v")
18354 [(match_operand:VF_128 1 "register_operand" "v")
18355 (match_operand:VF_128 2 "<round_saeonly_nimm_predicate>" "<round_saeonly_constraint>")
18356 (match_operand:SI 3 "const_0_to_15_operand")]
18361 "vrange<ssescalarmodesuffix>\t{<round_saeonly_op4>%3, %2, %1, %0|%0, %1, %2, %3<round_saeonly_op4>}"
18362 [(set_attr "type" "sse")
18363 (set_attr "prefix" "evex")
18364 (set_attr "mode" "<MODE>")])
18366 (define_insn "avx512dq_fpclass<mode><mask_scalar_merge_name>"
18367 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=Yk")
18368 (unspec:<avx512fmaskmode>
18369 [(match_operand:VF_AVX512VL 1 "register_operand" "v")
18370 (match_operand:QI 2 "const_0_to_255_operand" "n")]
18373 "vfpclass<ssemodesuffix>\t{%2, %1, %0<mask_scalar_merge_operand3>|%0<mask_scalar_merge_operand3>, %1, %2}";
18374 [(set_attr "type" "sse")
18375 (set_attr "length_immediate" "1")
18376 (set_attr "prefix" "evex")
18377 (set_attr "mode" "<MODE>")])
18379 (define_insn "avx512dq_vmfpclass<mode>"
18380 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=Yk")
18381 (and:<avx512fmaskmode>
18382 (unspec:<avx512fmaskmode>
18383 [(match_operand:VF_128 1 "register_operand" "v")
18384 (match_operand:QI 2 "const_0_to_255_operand" "n")]
18388 "vfpclass<ssescalarmodesuffix>\t{%2, %1, %0|%0, %1, %2}";
18389 [(set_attr "type" "sse")
18390 (set_attr "length_immediate" "1")
18391 (set_attr "prefix" "evex")
18392 (set_attr "mode" "<MODE>")])
18394 (define_insn "<avx512>_getmant<mode><mask_name><round_saeonly_name>"
18395 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v")
18396 (unspec:VF_AVX512VL
18397 [(match_operand:VF_AVX512VL 1 "nonimmediate_operand" "<round_saeonly_constraint>")
18398 (match_operand:SI 2 "const_0_to_15_operand")]
18401 "vgetmant<ssemodesuffix>\t{%2, <round_saeonly_mask_op3>%1, %0<mask_operand3>|%0<mask_operand3>, %1<round_saeonly_mask_op3>, %2}";
18402 [(set_attr "prefix" "evex")
18403 (set_attr "mode" "<MODE>")])
18405 (define_insn "avx512f_vgetmant<mode><round_saeonly_name>"
18406 [(set (match_operand:VF_128 0 "register_operand" "=v")
18409 [(match_operand:VF_128 1 "register_operand" "v")
18410 (match_operand:VF_128 2 "<round_saeonly_nimm_predicate>" "<round_saeonly_constraint>")
18411 (match_operand:SI 3 "const_0_to_15_operand")]
18416 "vgetmant<ssescalarmodesuffix>\t{%3, <round_saeonly_op4>%2, %1, %0|%0, %1, %2<round_saeonly_op4>, %3}";
18417 [(set_attr "prefix" "evex")
18418 (set_attr "mode" "<ssescalarmode>")])
18420 ;; The correct representation for this is absolutely enormous, and
18421 ;; surely not generally useful.
18422 (define_insn "<mask_codefor>avx512bw_dbpsadbw<mode><mask_name>"
18423 [(set (match_operand:VI2_AVX512VL 0 "register_operand" "=v")
18424 (unspec:VI2_AVX512VL
18425 [(match_operand:<dbpsadbwmode> 1 "register_operand" "v")
18426 (match_operand:<dbpsadbwmode> 2 "nonimmediate_operand" "vm")
18427 (match_operand:SI 3 "const_0_to_255_operand")]
18430 "vdbpsadbw\t{%3, %2, %1, %0<mask_operand4>|%0<mask_operand4>, %1, %2, %3}"
18431 [(set_attr "isa" "avx")
18432 (set_attr "type" "sselog1")
18433 (set_attr "length_immediate" "1")
18434 (set_attr "prefix" "evex")
18435 (set_attr "mode" "<sseinsnmode>")])
18437 (define_insn "clz<mode>2<mask_name>"
18438 [(set (match_operand:VI48_AVX512VL 0 "register_operand" "=v")
18440 (match_operand:VI48_AVX512VL 1 "nonimmediate_operand" "vm")))]
18442 "vplzcnt<ssemodesuffix>\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
18443 [(set_attr "type" "sse")
18444 (set_attr "prefix" "evex")
18445 (set_attr "mode" "<sseinsnmode>")])
18447 (define_insn "<mask_codefor>conflict<mode><mask_name>"
18448 [(set (match_operand:VI48_AVX512VL 0 "register_operand" "=v")
18449 (unspec:VI48_AVX512VL
18450 [(match_operand:VI48_AVX512VL 1 "nonimmediate_operand" "vm")]
18453 "vpconflict<ssemodesuffix>\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
18454 [(set_attr "type" "sse")
18455 (set_attr "prefix" "evex")
18456 (set_attr "mode" "<sseinsnmode>")])
18458 (define_insn "sha1msg1"
18459 [(set (match_operand:V4SI 0 "register_operand" "=x")
18461 [(match_operand:V4SI 1 "register_operand" "0")
18462 (match_operand:V4SI 2 "nonimmediate_operand" "xm")]
18465 "sha1msg1\t{%2, %0|%0, %2}"
18466 [(set_attr "type" "sselog1")
18467 (set_attr "mode" "TI")])
18469 (define_insn "sha1msg2"
18470 [(set (match_operand:V4SI 0 "register_operand" "=x")
18472 [(match_operand:V4SI 1 "register_operand" "0")
18473 (match_operand:V4SI 2 "nonimmediate_operand" "xm")]
18476 "sha1msg2\t{%2, %0|%0, %2}"
18477 [(set_attr "type" "sselog1")
18478 (set_attr "mode" "TI")])
18480 (define_insn "sha1nexte"
18481 [(set (match_operand:V4SI 0 "register_operand" "=x")
18483 [(match_operand:V4SI 1 "register_operand" "0")
18484 (match_operand:V4SI 2 "nonimmediate_operand" "xm")]
18485 UNSPEC_SHA1NEXTE))]
18487 "sha1nexte\t{%2, %0|%0, %2}"
18488 [(set_attr "type" "sselog1")
18489 (set_attr "mode" "TI")])
18491 (define_insn "sha1rnds4"
18492 [(set (match_operand:V4SI 0 "register_operand" "=x")
18494 [(match_operand:V4SI 1 "register_operand" "0")
18495 (match_operand:V4SI 2 "nonimmediate_operand" "xm")
18496 (match_operand:SI 3 "const_0_to_3_operand" "n")]
18497 UNSPEC_SHA1RNDS4))]
18499 "sha1rnds4\t{%3, %2, %0|%0, %2, %3}"
18500 [(set_attr "type" "sselog1")
18501 (set_attr "length_immediate" "1")
18502 (set_attr "mode" "TI")])
18504 (define_insn "sha256msg1"
18505 [(set (match_operand:V4SI 0 "register_operand" "=x")
18507 [(match_operand:V4SI 1 "register_operand" "0")
18508 (match_operand:V4SI 2 "nonimmediate_operand" "xm")]
18509 UNSPEC_SHA256MSG1))]
18511 "sha256msg1\t{%2, %0|%0, %2}"
18512 [(set_attr "type" "sselog1")
18513 (set_attr "mode" "TI")])
18515 (define_insn "sha256msg2"
18516 [(set (match_operand:V4SI 0 "register_operand" "=x")
18518 [(match_operand:V4SI 1 "register_operand" "0")
18519 (match_operand:V4SI 2 "nonimmediate_operand" "xm")]
18520 UNSPEC_SHA256MSG2))]
18522 "sha256msg2\t{%2, %0|%0, %2}"
18523 [(set_attr "type" "sselog1")
18524 (set_attr "mode" "TI")])
18526 (define_insn "sha256rnds2"
18527 [(set (match_operand:V4SI 0 "register_operand" "=x")
18529 [(match_operand:V4SI 1 "register_operand" "0")
18530 (match_operand:V4SI 2 "nonimmediate_operand" "xm")
18531 (match_operand:V4SI 3 "register_operand" "Yz")]
18532 UNSPEC_SHA256RNDS2))]
18534 "sha256rnds2\t{%3, %2, %0|%0, %2, %3}"
18535 [(set_attr "type" "sselog1")
18536 (set_attr "length_immediate" "1")
18537 (set_attr "mode" "TI")])
18539 (define_insn_and_split "avx512f_<castmode><avxsizesuffix>_<castmode>"
18540 [(set (match_operand:AVX512MODE2P 0 "nonimmediate_operand" "=x,m")
18541 (unspec:AVX512MODE2P
18542 [(match_operand:<ssequartermode> 1 "nonimmediate_operand" "xm,x")]
18546 "&& reload_completed"
18549 rtx op0 = operands[0];
18550 rtx op1 = operands[1];
18552 op0 = gen_rtx_REG (<ssequartermode>mode, REGNO (op0));
18554 op1 = gen_rtx_REG (<MODE>mode, REGNO (op1));
18555 emit_move_insn (op0, op1);
18559 (define_insn_and_split "avx512f_<castmode><avxsizesuffix>_256<castmode>"
18560 [(set (match_operand:AVX512MODE2P 0 "nonimmediate_operand" "=x,m")
18561 (unspec:AVX512MODE2P
18562 [(match_operand:<ssehalfvecmode> 1 "nonimmediate_operand" "xm,x")]
18566 "&& reload_completed"
18569 rtx op0 = operands[0];
18570 rtx op1 = operands[1];
18572 op0 = gen_rtx_REG (<ssehalfvecmode>mode, REGNO (op0));
18574 op1 = gen_rtx_REG (<MODE>mode, REGNO (op1));
18575 emit_move_insn (op0, op1);
18579 (define_int_iterator VPMADD52
18580 [UNSPEC_VPMADD52LUQ
18581 UNSPEC_VPMADD52HUQ])
18583 (define_int_attr vpmadd52type
18584 [(UNSPEC_VPMADD52LUQ "luq") (UNSPEC_VPMADD52HUQ "huq")])
18586 (define_expand "vpamdd52huq<mode>_maskz"
18587 [(match_operand:VI8_AVX512VL 0 "register_operand")
18588 (match_operand:VI8_AVX512VL 1 "register_operand")
18589 (match_operand:VI8_AVX512VL 2 "register_operand")
18590 (match_operand:VI8_AVX512VL 3 "nonimmediate_operand")
18591 (match_operand:<avx512fmaskmode> 4 "register_operand")]
18592 "TARGET_AVX512IFMA"
18594 emit_insn (gen_vpamdd52huq<mode>_maskz_1 (
18595 operands[0], operands[1], operands[2], operands[3],
18596 CONST0_RTX (<MODE>mode), operands[4]));
18600 (define_expand "vpamdd52luq<mode>_maskz"
18601 [(match_operand:VI8_AVX512VL 0 "register_operand")
18602 (match_operand:VI8_AVX512VL 1 "register_operand")
18603 (match_operand:VI8_AVX512VL 2 "register_operand")
18604 (match_operand:VI8_AVX512VL 3 "nonimmediate_operand")
18605 (match_operand:<avx512fmaskmode> 4 "register_operand")]
18606 "TARGET_AVX512IFMA"
18608 emit_insn (gen_vpamdd52luq<mode>_maskz_1 (
18609 operands[0], operands[1], operands[2], operands[3],
18610 CONST0_RTX (<MODE>mode), operands[4]));
18614 (define_insn "vpamdd52<vpmadd52type><mode><sd_maskz_name>"
18615 [(set (match_operand:VI8_AVX512VL 0 "register_operand" "=v")
18616 (unspec:VI8_AVX512VL
18617 [(match_operand:VI8_AVX512VL 1 "register_operand" "0")
18618 (match_operand:VI8_AVX512VL 2 "register_operand" "v")
18619 (match_operand:VI8_AVX512VL 3 "nonimmediate_operand" "vm")]
18621 "TARGET_AVX512IFMA"
18622 "vpmadd52<vpmadd52type>\t{%3, %2, %0<sd_mask_op4>|%0<sd_mask_op4>, %2, %3}"
18623 [(set_attr "type" "ssemuladd")
18624 (set_attr "prefix" "evex")
18625 (set_attr "mode" "<sseinsnmode>")])
18627 (define_insn "vpamdd52<vpmadd52type><mode>_mask"
18628 [(set (match_operand:VI8_AVX512VL 0 "register_operand" "=v")
18629 (vec_merge:VI8_AVX512VL
18630 (unspec:VI8_AVX512VL
18631 [(match_operand:VI8_AVX512VL 1 "register_operand" "0")
18632 (match_operand:VI8_AVX512VL 2 "register_operand" "v")
18633 (match_operand:VI8_AVX512VL 3 "nonimmediate_operand" "vm")]
18636 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk")))]
18637 "TARGET_AVX512IFMA"
18638 "vpmadd52<vpmadd52type>\t{%3, %2, %0%{%4%}|%0%{%4%}, %2, %3}"
18639 [(set_attr "type" "ssemuladd")
18640 (set_attr "prefix" "evex")
18641 (set_attr "mode" "<sseinsnmode>")])
18643 (define_insn "vpmultishiftqb<mode><mask_name>"
18644 [(set (match_operand:VI1_AVX512VL 0 "register_operand" "=v")
18645 (unspec:VI1_AVX512VL
18646 [(match_operand:VI1_AVX512VL 1 "register_operand" "v")
18647 (match_operand:VI1_AVX512VL 2 "nonimmediate_operand" "vm")]
18648 UNSPEC_VPMULTISHIFT))]
18649 "TARGET_AVX512VBMI"
18650 "vpmultishiftqb\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
18651 [(set_attr "type" "sselog")
18652 (set_attr "prefix" "evex")
18653 (set_attr "mode" "<sseinsnmode>")])