1 ;; GCC machine description for SSE instructions
2 ;; Copyright (C) 2005-2018 Free Software Foundation, Inc.
4 ;; This file is part of GCC.
6 ;; GCC is free software; you can redistribute it and/or modify
7 ;; it under the terms of the GNU General Public License as published by
8 ;; the Free Software Foundation; either version 3, or (at your option)
11 ;; GCC is distributed in the hope that it will be useful,
12 ;; but WITHOUT ANY WARRANTY; without even the implied warranty of
13 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 ;; GNU General Public License for more details.
16 ;; You should have received a copy of the GNU General Public License
17 ;; along with GCC; see the file COPYING3. If not see
18 ;; <http://www.gnu.org/licenses/>.
20 (define_c_enum "unspec" [
53 UNSPEC_XOP_UNSIGNED_CMP
64 UNSPEC_AESKEYGENASSIST
85 ;; For AVX512F support
87 UNSPEC_UNSIGNED_FIX_NOTRUNC
102 UNSPEC_COMPRESS_STORE
112 ;; For embed. rounding feature
113 UNSPEC_EMBEDDED_ROUNDING
115 ;; For AVX512PF support
116 UNSPEC_GATHER_PREFETCH
117 UNSPEC_SCATTER_PREFETCH
119 ;; For AVX512ER support
133 ;; For AVX512BW support
141 ;; For AVX512DQ support
146 ;; For AVX512IFMA support
150 ;; For AVX512VBMI support
153 ;; For AVX5124FMAPS/AVX5124VNNIW support
160 UNSPEC_GF2P8AFFINEINV
164 ;; For AVX512VBMI2 support
170 ;; For AVX512VNNI support
171 UNSPEC_VPMADDUBSWACCD
172 UNSPEC_VPMADDUBSWACCSSD
174 UNSPEC_VPMADDWDACCSSD
182 ;; For VPCLMULQDQ support
185 ;; For AVX512BITALG support
189 (define_c_enum "unspecv" [
199 ;; All vector modes including V?TImode, used in move patterns.
200 (define_mode_iterator VMOVE
201 [(V64QI "TARGET_AVX512F") (V32QI "TARGET_AVX") V16QI
202 (V32HI "TARGET_AVX512F") (V16HI "TARGET_AVX") V8HI
203 (V16SI "TARGET_AVX512F") (V8SI "TARGET_AVX") V4SI
204 (V8DI "TARGET_AVX512F") (V4DI "TARGET_AVX") V2DI
205 (V4TI "TARGET_AVX512F") (V2TI "TARGET_AVX") V1TI
206 (V16SF "TARGET_AVX512F") (V8SF "TARGET_AVX") V4SF
207 (V8DF "TARGET_AVX512F") (V4DF "TARGET_AVX") V2DF])
209 ;; All AVX-512{F,VL} vector modes. Supposed TARGET_AVX512F baseline.
210 (define_mode_iterator V48_AVX512VL
211 [V16SI (V8SI "TARGET_AVX512VL") (V4SI "TARGET_AVX512VL")
212 V8DI (V4DI "TARGET_AVX512VL") (V2DI "TARGET_AVX512VL")
213 V16SF (V8SF "TARGET_AVX512VL") (V4SF "TARGET_AVX512VL")
214 V8DF (V4DF "TARGET_AVX512VL") (V2DF "TARGET_AVX512VL")])
216 ;; 1,2 byte AVX-512{BW,VL} vector modes. Supposed TARGET_AVX512BW baseline.
217 (define_mode_iterator VI12_AVX512VL
218 [V64QI (V16QI "TARGET_AVX512VL") (V32QI "TARGET_AVX512VL")
219 V32HI (V16HI "TARGET_AVX512VL") (V8HI "TARGET_AVX512VL")])
221 ;; Same iterator, but without supposed TARGET_AVX512BW
222 (define_mode_iterator VI12_AVX512VLBW
223 [(V64QI "TARGET_AVX512BW") (V16QI "TARGET_AVX512VL")
224 (V32QI "TARGET_AVX512VL && TARGET_AVX512BW") (V32HI "TARGET_AVX512BW")
225 (V16HI "TARGET_AVX512VL") (V8HI "TARGET_AVX512VL")])
227 (define_mode_iterator VI1_AVX512VL
228 [V64QI (V16QI "TARGET_AVX512VL") (V32QI "TARGET_AVX512VL")])
231 (define_mode_iterator V
232 [(V32QI "TARGET_AVX") V16QI
233 (V16HI "TARGET_AVX") V8HI
234 (V16SI "TARGET_AVX512F") (V8SI "TARGET_AVX") V4SI
235 (V8DI "TARGET_AVX512F") (V4DI "TARGET_AVX") V2DI
236 (V16SF "TARGET_AVX512F") (V8SF "TARGET_AVX") V4SF
237 (V8DF "TARGET_AVX512F") (V4DF "TARGET_AVX") (V2DF "TARGET_SSE2")])
239 ;; All 128bit vector modes
240 (define_mode_iterator V_128
241 [V16QI V8HI V4SI V2DI V4SF (V2DF "TARGET_SSE2")])
243 ;; All 256bit vector modes
244 (define_mode_iterator V_256
245 [V32QI V16HI V8SI V4DI V8SF V4DF])
247 ;; All 512bit vector modes
248 (define_mode_iterator V_512 [V64QI V32HI V16SI V8DI V16SF V8DF])
250 ;; All 256bit and 512bit vector modes
251 (define_mode_iterator V_256_512
252 [V32QI V16HI V8SI V4DI V8SF V4DF
253 (V64QI "TARGET_AVX512F") (V32HI "TARGET_AVX512F") (V16SI "TARGET_AVX512F")
254 (V8DI "TARGET_AVX512F") (V16SF "TARGET_AVX512F") (V8DF "TARGET_AVX512F")])
256 ;; All vector float modes
257 (define_mode_iterator VF
258 [(V16SF "TARGET_AVX512F") (V8SF "TARGET_AVX") V4SF
259 (V8DF "TARGET_AVX512F") (V4DF "TARGET_AVX") (V2DF "TARGET_SSE2")])
261 ;; 128- and 256-bit float vector modes
262 (define_mode_iterator VF_128_256
263 [(V8SF "TARGET_AVX") V4SF
264 (V4DF "TARGET_AVX") (V2DF "TARGET_SSE2")])
266 ;; All SFmode vector float modes
267 (define_mode_iterator VF1
268 [(V16SF "TARGET_AVX512F") (V8SF "TARGET_AVX") V4SF])
270 ;; 128- and 256-bit SF vector modes
271 (define_mode_iterator VF1_128_256
272 [(V8SF "TARGET_AVX") V4SF])
274 (define_mode_iterator VF1_128_256VL
275 [V8SF (V4SF "TARGET_AVX512VL")])
277 ;; All DFmode vector float modes
278 (define_mode_iterator VF2
279 [(V8DF "TARGET_AVX512F") (V4DF "TARGET_AVX") V2DF])
281 ;; 128- and 256-bit DF vector modes
282 (define_mode_iterator VF2_128_256
283 [(V4DF "TARGET_AVX") V2DF])
285 (define_mode_iterator VF2_512_256
286 [(V8DF "TARGET_AVX512F") V4DF])
288 (define_mode_iterator VF2_512_256VL
289 [V8DF (V4DF "TARGET_AVX512VL")])
291 ;; All 128bit vector float modes
292 (define_mode_iterator VF_128
293 [V4SF (V2DF "TARGET_SSE2")])
295 ;; All 256bit vector float modes
296 (define_mode_iterator VF_256
299 ;; All 512bit vector float modes
300 (define_mode_iterator VF_512
303 (define_mode_iterator VI48_AVX512VL
304 [V16SI (V8SI "TARGET_AVX512VL") (V4SI "TARGET_AVX512VL")
305 V8DI (V4DI "TARGET_AVX512VL") (V2DI "TARGET_AVX512VL")])
307 (define_mode_iterator VF_AVX512VL
308 [V16SF (V8SF "TARGET_AVX512VL") (V4SF "TARGET_AVX512VL")
309 V8DF (V4DF "TARGET_AVX512VL") (V2DF "TARGET_AVX512VL")])
311 (define_mode_iterator VF2_AVX512VL
312 [V8DF (V4DF "TARGET_AVX512VL") (V2DF "TARGET_AVX512VL")])
314 (define_mode_iterator VF1_AVX512VL
315 [V16SF (V8SF "TARGET_AVX512VL") (V4SF "TARGET_AVX512VL")])
317 ;; All vector integer modes
318 (define_mode_iterator VI
319 [(V16SI "TARGET_AVX512F") (V8DI "TARGET_AVX512F")
320 (V64QI "TARGET_AVX512BW") (V32QI "TARGET_AVX") V16QI
321 (V32HI "TARGET_AVX512BW") (V16HI "TARGET_AVX") V8HI
322 (V8SI "TARGET_AVX") V4SI
323 (V4DI "TARGET_AVX") V2DI])
325 (define_mode_iterator VI_AVX2
326 [(V64QI "TARGET_AVX512BW") (V32QI "TARGET_AVX2") V16QI
327 (V32HI "TARGET_AVX512BW") (V16HI "TARGET_AVX2") V8HI
328 (V16SI "TARGET_AVX512F") (V8SI "TARGET_AVX2") V4SI
329 (V8DI "TARGET_AVX512F") (V4DI "TARGET_AVX2") V2DI])
331 ;; All QImode vector integer modes
332 (define_mode_iterator VI1
333 [(V32QI "TARGET_AVX") V16QI])
335 ;; All DImode vector integer modes
336 (define_mode_iterator V_AVX
337 [V16QI V8HI V4SI V2DI V4SF V2DF
338 (V32QI "TARGET_AVX") (V16HI "TARGET_AVX")
339 (V8SI "TARGET_AVX") (V4DI "TARGET_AVX")
340 (V8SF "TARGET_AVX") (V4DF"TARGET_AVX")])
342 (define_mode_iterator VI48_AVX
344 (V8SI "TARGET_AVX") (V4DI "TARGET_AVX")])
346 (define_mode_iterator VI8
347 [(V8DI "TARGET_AVX512F") (V4DI "TARGET_AVX") V2DI])
349 (define_mode_iterator VI8_FVL
350 [(V8DI "TARGET_AVX512F") V4DI (V2DI "TARGET_AVX512VL")])
352 (define_mode_iterator VI8_AVX512VL
353 [V8DI (V4DI "TARGET_AVX512VL") (V2DI "TARGET_AVX512VL")])
355 (define_mode_iterator VI8_256_512
356 [V8DI (V4DI "TARGET_AVX512VL")])
358 (define_mode_iterator VI1_AVX2
359 [(V32QI "TARGET_AVX2") V16QI])
361 (define_mode_iterator VI1_AVX512
362 [(V64QI "TARGET_AVX512BW") (V32QI "TARGET_AVX2") V16QI])
364 (define_mode_iterator VI1_AVX512F
365 [(V64QI "TARGET_AVX512F") (V32QI "TARGET_AVX") V16QI])
367 (define_mode_iterator VI2_AVX2
368 [(V32HI "TARGET_AVX512BW") (V16HI "TARGET_AVX2") V8HI])
370 (define_mode_iterator VI2_AVX512F
371 [(V32HI "TARGET_AVX512F") (V16HI "TARGET_AVX2") V8HI])
373 (define_mode_iterator VI4_AVX
374 [(V8SI "TARGET_AVX") V4SI])
376 (define_mode_iterator VI4_AVX2
377 [(V8SI "TARGET_AVX2") V4SI])
379 (define_mode_iterator VI4_AVX512F
380 [(V16SI "TARGET_AVX512F") (V8SI "TARGET_AVX2") V4SI])
382 (define_mode_iterator VI4_AVX512VL
383 [V16SI (V8SI "TARGET_AVX512VL") (V4SI "TARGET_AVX512VL")])
385 (define_mode_iterator VI48_AVX512F_AVX512VL
386 [V4SI V8SI (V16SI "TARGET_AVX512F")
387 (V2DI "TARGET_AVX512VL") (V4DI "TARGET_AVX512VL") (V8DI "TARGET_AVX512F")])
389 (define_mode_iterator VI2_AVX512VL
390 [(V8HI "TARGET_AVX512VL") (V16HI "TARGET_AVX512VL") V32HI])
392 (define_mode_iterator VI1_AVX512VL_F
393 [V32QI (V16QI "TARGET_AVX512VL") (V64QI "TARGET_AVX512F")])
395 (define_mode_iterator VI8_AVX2_AVX512BW
396 [(V8DI "TARGET_AVX512BW") (V4DI "TARGET_AVX2") V2DI])
398 (define_mode_iterator VI8_AVX2
399 [(V4DI "TARGET_AVX2") V2DI])
401 (define_mode_iterator VI8_AVX2_AVX512F
402 [(V8DI "TARGET_AVX512F") (V4DI "TARGET_AVX2") V2DI])
404 (define_mode_iterator VI8_AVX_AVX512F
405 [(V8DI "TARGET_AVX512F") (V4DI "TARGET_AVX")])
407 (define_mode_iterator VI4_128_8_256
411 (define_mode_iterator V8FI
415 (define_mode_iterator V16FI
418 ;; ??? We should probably use TImode instead.
419 (define_mode_iterator VIMAX_AVX2_AVX512BW
420 [(V4TI "TARGET_AVX512BW") (V2TI "TARGET_AVX2") V1TI])
422 ;; Suppose TARGET_AVX512BW as baseline
423 (define_mode_iterator VIMAX_AVX512VL
424 [V4TI (V2TI "TARGET_AVX512VL") (V1TI "TARGET_AVX512VL")])
426 (define_mode_iterator VIMAX_AVX2
427 [(V2TI "TARGET_AVX2") V1TI])
429 ;; ??? This should probably be dropped in favor of VIMAX_AVX2_AVX512BW.
430 (define_mode_iterator SSESCALARMODE
431 [(V4TI "TARGET_AVX512BW") (V2TI "TARGET_AVX2") TI])
433 (define_mode_iterator VI12_AVX2
434 [(V64QI "TARGET_AVX512BW") (V32QI "TARGET_AVX2") V16QI
435 (V32HI "TARGET_AVX512BW") (V16HI "TARGET_AVX2") V8HI])
437 (define_mode_iterator VI24_AVX2
438 [(V16HI "TARGET_AVX2") V8HI
439 (V8SI "TARGET_AVX2") V4SI])
441 (define_mode_iterator VI124_AVX2_24_AVX512F_1_AVX512BW
442 [(V64QI "TARGET_AVX512BW") (V32QI "TARGET_AVX2") V16QI
443 (V32HI "TARGET_AVX512F") (V16HI "TARGET_AVX2") V8HI
444 (V16SI "TARGET_AVX512F") (V8SI "TARGET_AVX2") V4SI])
446 (define_mode_iterator VI124_AVX2
447 [(V32QI "TARGET_AVX2") V16QI
448 (V16HI "TARGET_AVX2") V8HI
449 (V8SI "TARGET_AVX2") V4SI])
451 (define_mode_iterator VI2_AVX2_AVX512BW
452 [(V32HI "TARGET_AVX512BW") (V16HI "TARGET_AVX2") V8HI])
454 (define_mode_iterator VI248_AVX512VL
456 (V16HI "TARGET_AVX512VL") (V8SI "TARGET_AVX512VL")
457 (V4DI "TARGET_AVX512VL") (V8HI "TARGET_AVX512VL")
458 (V4SI "TARGET_AVX512VL") (V2DI "TARGET_AVX512VL")])
460 (define_mode_iterator VI48_AVX2
461 [(V8SI "TARGET_AVX2") V4SI
462 (V4DI "TARGET_AVX2") V2DI])
464 (define_mode_iterator VI248_AVX2
465 [(V16HI "TARGET_AVX2") V8HI
466 (V8SI "TARGET_AVX2") V4SI
467 (V4DI "TARGET_AVX2") V2DI])
469 (define_mode_iterator VI248_AVX2_8_AVX512F_24_AVX512BW
470 [(V32HI "TARGET_AVX512BW") (V16HI "TARGET_AVX2") V8HI
471 (V16SI "TARGET_AVX512BW") (V8SI "TARGET_AVX2") V4SI
472 (V8DI "TARGET_AVX512F") (V4DI "TARGET_AVX2") V2DI])
474 (define_mode_iterator VI248_AVX512BW
475 [(V32HI "TARGET_AVX512BW") V16SI V8DI])
477 (define_mode_iterator VI248_AVX512BW_AVX512VL
478 [(V32HI "TARGET_AVX512BW")
479 (V4DI "TARGET_AVX512VL") V16SI V8DI])
481 ;; Suppose TARGET_AVX512VL as baseline
482 (define_mode_iterator VI248_AVX512BW_1
483 [(V16HI "TARGET_AVX512BW") (V8HI "TARGET_AVX512BW")
487 (define_mode_iterator VI248_AVX512BW_2
488 [(V16HI "TARGET_AVX512BW") (V8HI "TARGET_AVX512BW")
492 (define_mode_iterator VI48_AVX512F
493 [(V16SI "TARGET_AVX512F") V8SI V4SI
494 (V8DI "TARGET_AVX512F") V4DI V2DI])
496 (define_mode_iterator VI48_AVX_AVX512F
497 [(V16SI "TARGET_AVX512F") (V8SI "TARGET_AVX") V4SI
498 (V8DI "TARGET_AVX512F") (V4DI "TARGET_AVX") V2DI])
500 (define_mode_iterator VI12_AVX_AVX512F
501 [ (V64QI "TARGET_AVX512F") (V32QI "TARGET_AVX") V16QI
502 (V32HI "TARGET_AVX512F") (V16HI "TARGET_AVX") V8HI])
504 (define_mode_iterator V48_AVX2
507 (V4SI "TARGET_AVX2") (V2DI "TARGET_AVX2")
508 (V8SI "TARGET_AVX2") (V4DI "TARGET_AVX2")])
510 (define_mode_iterator VI1_AVX512VLBW
511 [(V64QI "TARGET_AVX512BW") (V32QI "TARGET_AVX512VL")
512 (V16QI "TARGET_AVX512VL")])
514 (define_mode_attr avx512
515 [(V16QI "avx512vl") (V32QI "avx512vl") (V64QI "avx512bw")
516 (V8HI "avx512vl") (V16HI "avx512vl") (V32HI "avx512bw")
517 (V4SI "avx512vl") (V8SI "avx512vl") (V16SI "avx512f")
518 (V2DI "avx512vl") (V4DI "avx512vl") (V8DI "avx512f")
519 (V4SF "avx512vl") (V8SF "avx512vl") (V16SF "avx512f")
520 (V2DF "avx512vl") (V4DF "avx512vl") (V8DF "avx512f")])
522 (define_mode_attr sse2_avx_avx512f
523 [(V16QI "sse2") (V32QI "avx") (V64QI "avx512f")
524 (V8HI "avx512vl") (V16HI "avx512vl") (V32HI "avx512bw")
525 (V4SI "sse2") (V8SI "avx") (V16SI "avx512f")
526 (V2DI "avx512vl") (V4DI "avx512vl") (V8DI "avx512f")
527 (V16SF "avx512f") (V8SF "avx") (V4SF "avx")
528 (V8DF "avx512f") (V4DF "avx") (V2DF "avx")])
530 (define_mode_attr sse2_avx2
531 [(V16QI "sse2") (V32QI "avx2") (V64QI "avx512bw")
532 (V8HI "sse2") (V16HI "avx2") (V32HI "avx512bw")
533 (V4SI "sse2") (V8SI "avx2") (V16SI "avx512f")
534 (V2DI "sse2") (V4DI "avx2") (V8DI "avx512f")
535 (V1TI "sse2") (V2TI "avx2") (V4TI "avx512bw")])
537 (define_mode_attr ssse3_avx2
538 [(V16QI "ssse3") (V32QI "avx2") (V64QI "avx512bw")
539 (V4HI "ssse3") (V8HI "ssse3") (V16HI "avx2") (V32HI "avx512bw")
540 (V4SI "ssse3") (V8SI "avx2")
541 (V2DI "ssse3") (V4DI "avx2")
542 (TI "ssse3") (V2TI "avx2") (V4TI "avx512bw")])
544 (define_mode_attr sse4_1_avx2
545 [(V16QI "sse4_1") (V32QI "avx2") (V64QI "avx512bw")
546 (V8HI "sse4_1") (V16HI "avx2") (V32HI "avx512bw")
547 (V4SI "sse4_1") (V8SI "avx2") (V16SI "avx512f")
548 (V2DI "sse4_1") (V4DI "avx2") (V8DI "avx512dq")])
550 (define_mode_attr avx_avx2
551 [(V4SF "avx") (V2DF "avx")
552 (V8SF "avx") (V4DF "avx")
553 (V4SI "avx2") (V2DI "avx2")
554 (V8SI "avx2") (V4DI "avx2")])
556 (define_mode_attr vec_avx2
557 [(V16QI "vec") (V32QI "avx2")
558 (V8HI "vec") (V16HI "avx2")
559 (V4SI "vec") (V8SI "avx2")
560 (V2DI "vec") (V4DI "avx2")])
562 (define_mode_attr avx2_avx512
563 [(V4SI "avx2") (V8SI "avx2") (V16SI "avx512f")
564 (V2DI "avx2") (V4DI "avx2") (V8DI "avx512f")
565 (V4SF "avx2") (V8SF "avx2") (V16SF "avx512f")
566 (V2DF "avx2") (V4DF "avx2") (V8DF "avx512f")
567 (V8HI "avx512vl") (V16HI "avx512vl") (V32HI "avx512bw")])
569 (define_mode_attr shuffletype
570 [(V16SF "f") (V16SI "i") (V8DF "f") (V8DI "i")
571 (V8SF "f") (V8SI "i") (V4DF "f") (V4DI "i")
572 (V4SF "f") (V4SI "i") (V2DF "f") (V2DI "i")
573 (V32HI "i") (V16HI "i") (V8HI "i")
574 (V64QI "i") (V32QI "i") (V16QI "i")
575 (V4TI "i") (V2TI "i") (V1TI "i")])
577 (define_mode_attr ssequartermode
578 [(V16SF "V4SF") (V8DF "V2DF") (V16SI "V4SI") (V8DI "V2DI")])
580 (define_mode_attr ssedoublemodelower
581 [(V16QI "v16hi") (V32QI "v32hi") (V64QI "v64hi")
582 (V8HI "v8si") (V16HI "v16si") (V32HI "v32si")
583 (V4SI "v4di") (V8SI "v8di") (V16SI "v16di")])
585 (define_mode_attr ssedoublemode
586 [(V4SF "V8SF") (V8SF "V16SF") (V16SF "V32SF")
587 (V2DF "V4DF") (V4DF "V8DF") (V8DF "V16DF")
588 (V16QI "V16HI") (V32QI "V32HI") (V64QI "V64HI")
589 (V4HI "V4SI") (V8HI "V8SI") (V16HI "V16SI") (V32HI "V32SI")
590 (V4SI "V4DI") (V8SI "V16SI") (V16SI "V32SI")
591 (V4DI "V8DI") (V8DI "V16DI")])
593 (define_mode_attr ssebytemode
594 [(V8DI "V64QI") (V4DI "V32QI") (V2DI "V16QI")])
596 ;; All 128bit vector integer modes
597 (define_mode_iterator VI_128 [V16QI V8HI V4SI V2DI])
599 ;; All 256bit vector integer modes
600 (define_mode_iterator VI_256 [V32QI V16HI V8SI V4DI])
602 ;; Various 128bit vector integer mode combinations
603 (define_mode_iterator VI12_128 [V16QI V8HI])
604 (define_mode_iterator VI14_128 [V16QI V4SI])
605 (define_mode_iterator VI124_128 [V16QI V8HI V4SI])
606 (define_mode_iterator VI24_128 [V8HI V4SI])
607 (define_mode_iterator VI248_128 [V8HI V4SI V2DI])
608 (define_mode_iterator VI48_128 [V4SI V2DI])
610 ;; Various 256bit and 512 vector integer mode combinations
611 (define_mode_iterator VI124_256 [V32QI V16HI V8SI])
612 (define_mode_iterator VI124_256_AVX512F_AVX512BW
614 (V64QI "TARGET_AVX512BW")
615 (V32HI "TARGET_AVX512BW")
616 (V16SI "TARGET_AVX512F")])
617 (define_mode_iterator VI48_256 [V8SI V4DI])
618 (define_mode_iterator VI48_512 [V16SI V8DI])
619 (define_mode_iterator VI4_256_8_512 [V8SI V8DI])
620 (define_mode_iterator VI_AVX512BW
621 [V16SI V8DI (V32HI "TARGET_AVX512BW") (V64QI "TARGET_AVX512BW")])
623 ;; Int-float size matches
624 (define_mode_iterator VI4F_128 [V4SI V4SF])
625 (define_mode_iterator VI8F_128 [V2DI V2DF])
626 (define_mode_iterator VI4F_256 [V8SI V8SF])
627 (define_mode_iterator VI8F_256 [V4DI V4DF])
628 (define_mode_iterator VI4F_256_512
630 (V16SI "TARGET_AVX512F") (V16SF "TARGET_AVX512F")])
631 (define_mode_iterator VI48F_256_512
633 (V16SI "TARGET_AVX512F") (V16SF "TARGET_AVX512F")
634 (V8DI "TARGET_AVX512F") (V8DF "TARGET_AVX512F")
635 (V4DI "TARGET_AVX512VL") (V4DF "TARGET_AVX512VL")])
636 (define_mode_iterator VF48_I1248
637 [V16SI V16SF V8DI V8DF V32HI V64QI])
638 (define_mode_iterator VI48F
639 [V16SI V16SF V8DI V8DF
640 (V8SI "TARGET_AVX512VL") (V8SF "TARGET_AVX512VL")
641 (V4DI "TARGET_AVX512VL") (V4DF "TARGET_AVX512VL")
642 (V4SI "TARGET_AVX512VL") (V4SF "TARGET_AVX512VL")
643 (V2DI "TARGET_AVX512VL") (V2DF "TARGET_AVX512VL")])
644 (define_mode_iterator VI48F_256 [V8SI V8SF V4DI V4DF])
646 ;; Mapping from float mode to required SSE level
647 (define_mode_attr sse
648 [(SF "sse") (DF "sse2")
649 (V4SF "sse") (V2DF "sse2")
650 (V16SF "avx512f") (V8SF "avx")
651 (V8DF "avx512f") (V4DF "avx")])
653 (define_mode_attr sse2
654 [(V16QI "sse2") (V32QI "avx") (V64QI "avx512f")
655 (V2DI "sse2") (V4DI "avx") (V8DI "avx512f")])
657 (define_mode_attr sse3
658 [(V16QI "sse3") (V32QI "avx")])
660 (define_mode_attr sse4_1
661 [(V4SF "sse4_1") (V2DF "sse4_1")
662 (V8SF "avx") (V4DF "avx")
664 (V4DI "avx") (V2DI "sse4_1")
665 (V8SI "avx") (V4SI "sse4_1")
666 (V16QI "sse4_1") (V32QI "avx")
667 (V8HI "sse4_1") (V16HI "avx")])
669 (define_mode_attr avxsizesuffix
670 [(V64QI "512") (V32HI "512") (V16SI "512") (V8DI "512")
671 (V32QI "256") (V16HI "256") (V8SI "256") (V4DI "256")
672 (V16QI "") (V8HI "") (V4SI "") (V2DI "")
673 (V16SF "512") (V8DF "512")
674 (V8SF "256") (V4DF "256")
675 (V4SF "") (V2DF "")])
677 ;; SSE instruction mode
678 (define_mode_attr sseinsnmode
679 [(V64QI "XI") (V32HI "XI") (V16SI "XI") (V8DI "XI") (V4TI "XI")
680 (V32QI "OI") (V16HI "OI") (V8SI "OI") (V4DI "OI") (V2TI "OI")
681 (V16QI "TI") (V8HI "TI") (V4SI "TI") (V2DI "TI") (V1TI "TI")
682 (V16SF "V16SF") (V8DF "V8DF")
683 (V8SF "V8SF") (V4DF "V4DF")
684 (V4SF "V4SF") (V2DF "V2DF")
687 ;; Mapping of vector modes to corresponding mask size
688 (define_mode_attr avx512fmaskmode
689 [(V64QI "DI") (V32QI "SI") (V16QI "HI")
690 (V32HI "SI") (V16HI "HI") (V8HI "QI") (V4HI "QI")
691 (V16SI "HI") (V8SI "QI") (V4SI "QI")
692 (V8DI "QI") (V4DI "QI") (V2DI "QI")
693 (V16SF "HI") (V8SF "QI") (V4SF "QI")
694 (V8DF "QI") (V4DF "QI") (V2DF "QI")])
696 ;; Mapping of vector modes to corresponding mask size
697 (define_mode_attr avx512fmaskmodelower
698 [(V64QI "di") (V32QI "si") (V16QI "hi")
699 (V32HI "si") (V16HI "hi") (V8HI "qi") (V4HI "qi")
700 (V16SI "hi") (V8SI "qi") (V4SI "qi")
701 (V8DI "qi") (V4DI "qi") (V2DI "qi")
702 (V16SF "hi") (V8SF "qi") (V4SF "qi")
703 (V8DF "qi") (V4DF "qi") (V2DF "qi")])
705 ;; Mapping of vector float modes to an integer mode of the same size
706 (define_mode_attr sseintvecmode
707 [(V16SF "V16SI") (V8DF "V8DI")
708 (V8SF "V8SI") (V4DF "V4DI")
709 (V4SF "V4SI") (V2DF "V2DI")
710 (V16SI "V16SI") (V8DI "V8DI")
711 (V8SI "V8SI") (V4DI "V4DI")
712 (V4SI "V4SI") (V2DI "V2DI")
713 (V16HI "V16HI") (V8HI "V8HI")
714 (V32HI "V32HI") (V64QI "V64QI")
715 (V32QI "V32QI") (V16QI "V16QI")])
717 (define_mode_attr sseintvecmode2
718 [(V8DF "XI") (V4DF "OI") (V2DF "TI")
719 (V8SF "OI") (V4SF "TI")])
721 (define_mode_attr sseintvecmodelower
722 [(V16SF "v16si") (V8DF "v8di")
723 (V8SF "v8si") (V4DF "v4di")
724 (V4SF "v4si") (V2DF "v2di")
725 (V8SI "v8si") (V4DI "v4di")
726 (V4SI "v4si") (V2DI "v2di")
727 (V16HI "v16hi") (V8HI "v8hi")
728 (V32QI "v32qi") (V16QI "v16qi")])
730 ;; Mapping of vector modes to a vector mode of double size
731 (define_mode_attr ssedoublevecmode
732 [(V32QI "V64QI") (V16HI "V32HI") (V8SI "V16SI") (V4DI "V8DI")
733 (V16QI "V32QI") (V8HI "V16HI") (V4SI "V8SI") (V2DI "V4DI")
734 (V8SF "V16SF") (V4DF "V8DF")
735 (V4SF "V8SF") (V2DF "V4DF")])
737 ;; Mapping of vector modes to a vector mode of half size
738 (define_mode_attr ssehalfvecmode
739 [(V64QI "V32QI") (V32HI "V16HI") (V16SI "V8SI") (V8DI "V4DI") (V4TI "V2TI")
740 (V32QI "V16QI") (V16HI "V8HI") (V8SI "V4SI") (V4DI "V2DI")
741 (V16QI "V8QI") (V8HI "V4HI") (V4SI "V2SI")
742 (V16SF "V8SF") (V8DF "V4DF")
743 (V8SF "V4SF") (V4DF "V2DF")
746 (define_mode_attr ssehalfvecmodelower
747 [(V64QI "v32qi") (V32HI "v16hi") (V16SI "v8si") (V8DI "v4di") (V4TI "v2ti")
748 (V32QI "v16qi") (V16HI "v8hi") (V8SI "v4si") (V4DI "v2di")
749 (V16QI "v8qi") (V8HI "v4hi") (V4SI "v2si")
750 (V16SF "v8sf") (V8DF "v4df")
751 (V8SF "v4sf") (V4DF "v2df")
754 ;; Mapping of vector modes ti packed single mode of the same size
755 (define_mode_attr ssePSmode
756 [(V16SI "V16SF") (V8DF "V16SF")
757 (V16SF "V16SF") (V8DI "V16SF")
758 (V64QI "V16SF") (V32QI "V8SF") (V16QI "V4SF")
759 (V32HI "V16SF") (V16HI "V8SF") (V8HI "V4SF")
760 (V8SI "V8SF") (V4SI "V4SF")
761 (V4DI "V8SF") (V2DI "V4SF")
762 (V4TI "V16SF") (V2TI "V8SF") (V1TI "V4SF")
763 (V8SF "V8SF") (V4SF "V4SF")
764 (V4DF "V8SF") (V2DF "V4SF")])
766 (define_mode_attr ssePSmode2
767 [(V8DI "V8SF") (V4DI "V4SF")])
769 ;; Mapping of vector modes back to the scalar modes
770 (define_mode_attr ssescalarmode
771 [(V64QI "QI") (V32QI "QI") (V16QI "QI")
772 (V32HI "HI") (V16HI "HI") (V8HI "HI")
773 (V16SI "SI") (V8SI "SI") (V4SI "SI")
774 (V8DI "DI") (V4DI "DI") (V2DI "DI")
775 (V16SF "SF") (V8SF "SF") (V4SF "SF")
776 (V8DF "DF") (V4DF "DF") (V2DF "DF")
777 (V4TI "TI") (V2TI "TI")])
779 ;; Mapping of vector modes back to the scalar modes
780 (define_mode_attr ssescalarmodelower
781 [(V64QI "qi") (V32QI "qi") (V16QI "qi")
782 (V32HI "hi") (V16HI "hi") (V8HI "hi")
783 (V16SI "si") (V8SI "si") (V4SI "si")
784 (V8DI "di") (V4DI "di") (V2DI "di")
785 (V16SF "sf") (V8SF "sf") (V4SF "sf")
786 (V8DF "df") (V4DF "df") (V2DF "df")
787 (V4TI "ti") (V2TI "ti")])
789 ;; Mapping of vector modes to the 128bit modes
790 (define_mode_attr ssexmmmode
791 [(V64QI "V16QI") (V32QI "V16QI") (V16QI "V16QI")
792 (V32HI "V8HI") (V16HI "V8HI") (V8HI "V8HI")
793 (V16SI "V4SI") (V8SI "V4SI") (V4SI "V4SI")
794 (V8DI "V2DI") (V4DI "V2DI") (V2DI "V2DI")
795 (V16SF "V4SF") (V8SF "V4SF") (V4SF "V4SF")
796 (V8DF "V2DF") (V4DF "V2DF") (V2DF "V2DF")])
798 ;; Pointer size override for scalar modes (Intel asm dialect)
799 (define_mode_attr iptr
800 [(V64QI "b") (V32HI "w") (V16SI "k") (V8DI "q")
801 (V32QI "b") (V16HI "w") (V8SI "k") (V4DI "q")
802 (V16QI "b") (V8HI "w") (V4SI "k") (V2DI "q")
803 (V8SF "k") (V4DF "q")
804 (V4SF "k") (V2DF "q")
807 ;; Number of scalar elements in each vector type
808 (define_mode_attr ssescalarnum
809 [(V64QI "64") (V16SI "16") (V8DI "8")
810 (V32QI "32") (V16HI "16") (V8SI "8") (V4DI "4")
811 (V16QI "16") (V8HI "8") (V4SI "4") (V2DI "2")
812 (V16SF "16") (V8DF "8")
813 (V8SF "8") (V4DF "4")
814 (V4SF "4") (V2DF "2")])
816 ;; Mask of scalar elements in each vector type
817 (define_mode_attr ssescalarnummask
818 [(V32QI "31") (V16HI "15") (V8SI "7") (V4DI "3")
819 (V16QI "15") (V8HI "7") (V4SI "3") (V2DI "1")
820 (V8SF "7") (V4DF "3")
821 (V4SF "3") (V2DF "1")])
823 (define_mode_attr ssescalarsize
824 [(V4TI "64") (V2TI "64") (V1TI "64")
825 (V8DI "64") (V4DI "64") (V2DI "64")
826 (V64QI "8") (V32QI "8") (V16QI "8")
827 (V32HI "16") (V16HI "16") (V8HI "16")
828 (V16SI "32") (V8SI "32") (V4SI "32")
829 (V16SF "32") (V8SF "32") (V4SF "32")
830 (V8DF "64") (V4DF "64") (V2DF "64")])
832 ;; SSE prefix for integer vector modes
833 (define_mode_attr sseintprefix
834 [(V2DI "p") (V2DF "")
839 (V16SI "p") (V16SF "")
840 (V16QI "p") (V8HI "p")
841 (V32QI "p") (V16HI "p")
842 (V64QI "p") (V32HI "p")])
844 ;; SSE scalar suffix for vector modes
845 (define_mode_attr ssescalarmodesuffix
847 (V16SF "ss") (V8DF "sd")
848 (V8SF "ss") (V4DF "sd")
849 (V4SF "ss") (V2DF "sd")
850 (V16SI "d") (V8DI "q")
851 (V8SI "d") (V4DI "q")
852 (V4SI "d") (V2DI "q")])
854 ;; Pack/unpack vector modes
855 (define_mode_attr sseunpackmode
856 [(V16QI "V8HI") (V8HI "V4SI") (V4SI "V2DI")
857 (V32QI "V16HI") (V16HI "V8SI") (V8SI "V4DI")
858 (V32HI "V16SI") (V64QI "V32HI") (V16SI "V8DI")])
860 (define_mode_attr ssepackmode
861 [(V8HI "V16QI") (V4SI "V8HI") (V2DI "V4SI")
862 (V16HI "V32QI") (V8SI "V16HI") (V4DI "V8SI")
863 (V32HI "V64QI") (V16SI "V32HI") (V8DI "V16SI")])
865 ;; Mapping of the max integer size for xop rotate immediate constraint
866 (define_mode_attr sserotatemax
867 [(V16QI "7") (V8HI "15") (V4SI "31") (V2DI "63")])
869 ;; Mapping of mode to cast intrinsic name
870 (define_mode_attr castmode
871 [(V8SI "si") (V8SF "ps") (V4DF "pd")
872 (V16SI "si") (V16SF "ps") (V8DF "pd")])
874 ;; Instruction suffix for sign and zero extensions.
875 (define_code_attr extsuffix [(sign_extend "sx") (zero_extend "zx")])
877 ;; i128 for integer vectors and TARGET_AVX2, f128 otherwise.
878 ;; i64x4 or f64x4 for 512bit modes.
879 (define_mode_attr i128
880 [(V16SF "f64x4") (V8SF "f128") (V8DF "f64x4") (V4DF "f128")
881 (V64QI "i64x4") (V32QI "%~128") (V32HI "i64x4") (V16HI "%~128")
882 (V16SI "i64x4") (V8SI "%~128") (V8DI "i64x4") (V4DI "%~128")])
884 ;; For 256-bit modes for TARGET_AVX512VL && TARGET_AVX512DQ
885 ;; i32x4, f32x4, i64x2 or f64x2 suffixes.
886 (define_mode_attr i128vldq
887 [(V8SF "f32x4") (V4DF "f64x2")
888 (V32QI "i32x4") (V16HI "i32x4") (V8SI "i32x4") (V4DI "i64x2")])
891 (define_mode_iterator AVX256MODE2P [V8SI V8SF V4DF])
892 (define_mode_iterator AVX512MODE2P [V16SI V16SF V8DF])
894 ;; Mapping for dbpsabbw modes
895 (define_mode_attr dbpsadbwmode
896 [(V32HI "V64QI") (V16HI "V32QI") (V8HI "V16QI")])
898 ;; Mapping suffixes for broadcast
899 (define_mode_attr bcstscalarsuff
900 [(V64QI "b") (V32QI "b") (V16QI "b")
901 (V32HI "w") (V16HI "w") (V8HI "w")
902 (V16SI "d") (V8SI "d") (V4SI "d")
903 (V8DI "q") (V4DI "q") (V2DI "q")
904 (V16SF "ss") (V8SF "ss") (V4SF "ss")
905 (V8DF "sd") (V4DF "sd") (V2DF "sd")])
907 ;; Tie mode of assembler operand to mode iterator
908 (define_mode_attr concat_tg_mode
909 [(V32QI "t") (V16HI "t") (V8SI "t") (V4DI "t") (V8SF "t") (V4DF "t")
910 (V64QI "g") (V32HI "g") (V16SI "g") (V8DI "g") (V16SF "g") (V8DF "g")])
912 ;; Tie mode of assembler operand to mode iterator
913 (define_mode_attr xtg_mode
914 [(V16QI "x") (V8HI "x") (V4SI "x") (V2DI "x") (V4SF "x") (V2DF "x")
915 (V32QI "t") (V16HI "t") (V8SI "t") (V4DI "t") (V8SF "t") (V4DF "t")
916 (V64QI "g") (V32HI "g") (V16SI "g") (V8DI "g") (V16SF "g") (V8DF "g")])
918 ;; Half mask mode for unpacks
919 (define_mode_attr HALFMASKMODE
920 [(DI "SI") (SI "HI")])
922 ;; Double mask mode for packs
923 (define_mode_attr DOUBLEMASKMODE
924 [(HI "SI") (SI "DI")])
927 ;; Include define_subst patterns for instructions with mask
930 ;; Patterns whose name begins with "sse{,2,3}_" are invoked by intrinsics.
932 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
936 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
938 ;; All of these patterns are enabled for SSE1 as well as SSE2.
939 ;; This is essential for maintaining stable calling conventions.
941 (define_expand "mov<mode>"
942 [(set (match_operand:VMOVE 0 "nonimmediate_operand")
943 (match_operand:VMOVE 1 "nonimmediate_operand"))]
946 ix86_expand_vector_move (<MODE>mode, operands);
950 (define_insn "mov<mode>_internal"
951 [(set (match_operand:VMOVE 0 "nonimmediate_operand"
953 (match_operand:VMOVE 1 "nonimmediate_or_sse_const_operand"
956 && (register_operand (operands[0], <MODE>mode)
957 || register_operand (operands[1], <MODE>mode))"
959 switch (get_attr_type (insn))
962 return standard_sse_constant_opcode (insn, operands);
965 /* There is no evex-encoded vmov* for sizes smaller than 64-bytes
966 in avx512f, so we need to use workarounds, to access sse registers
967 16-31, which are evex-only. In avx512vl we don't need workarounds. */
968 if (TARGET_AVX512F && <MODE_SIZE> < 64 && !TARGET_AVX512VL
969 && (EXT_REX_SSE_REG_P (operands[0])
970 || EXT_REX_SSE_REG_P (operands[1])))
972 if (memory_operand (operands[0], <MODE>mode))
974 if (<MODE_SIZE> == 32)
975 return "vextract<shuffletype>64x4\t{$0x0, %g1, %0|%0, %g1, 0x0}";
976 else if (<MODE_SIZE> == 16)
977 return "vextract<shuffletype>32x4\t{$0x0, %g1, %0|%0, %g1, 0x0}";
981 else if (memory_operand (operands[1], <MODE>mode))
983 if (<MODE_SIZE> == 32)
984 return "vbroadcast<shuffletype>64x4\t{%1, %g0|%g0, %1}";
985 else if (<MODE_SIZE> == 16)
986 return "vbroadcast<shuffletype>32x4\t{%1, %g0|%g0, %1}";
991 /* Reg -> reg move is always aligned. Just use wider move. */
992 switch (get_attr_mode (insn))
996 return "vmovaps\t{%g1, %g0|%g0, %g1}";
999 return "vmovapd\t{%g1, %g0|%g0, %g1}";
1002 return "vmovdqa64\t{%g1, %g0|%g0, %g1}";
1008 switch (get_attr_mode (insn))
1013 if (misaligned_operand (operands[0], <MODE>mode)
1014 || misaligned_operand (operands[1], <MODE>mode))
1015 return "%vmovups\t{%1, %0|%0, %1}";
1017 return "%vmovaps\t{%1, %0|%0, %1}";
1022 if (misaligned_operand (operands[0], <MODE>mode)
1023 || misaligned_operand (operands[1], <MODE>mode))
1024 return "%vmovupd\t{%1, %0|%0, %1}";
1026 return "%vmovapd\t{%1, %0|%0, %1}";
1030 if (misaligned_operand (operands[0], <MODE>mode)
1031 || misaligned_operand (operands[1], <MODE>mode))
1032 return TARGET_AVX512VL
1033 && (<MODE>mode == V4SImode
1034 || <MODE>mode == V2DImode
1035 || <MODE>mode == V8SImode
1036 || <MODE>mode == V4DImode
1038 ? "vmovdqu<ssescalarsize>\t{%1, %0|%0, %1}"
1039 : "%vmovdqu\t{%1, %0|%0, %1}";
1041 return TARGET_AVX512VL ? "vmovdqa64\t{%1, %0|%0, %1}"
1042 : "%vmovdqa\t{%1, %0|%0, %1}";
1044 if (misaligned_operand (operands[0], <MODE>mode)
1045 || misaligned_operand (operands[1], <MODE>mode))
1046 return (<MODE>mode == V16SImode
1047 || <MODE>mode == V8DImode
1049 ? "vmovdqu<ssescalarsize>\t{%1, %0|%0, %1}"
1050 : "vmovdqu64\t{%1, %0|%0, %1}";
1052 return "vmovdqa64\t{%1, %0|%0, %1}";
1062 [(set_attr "type" "sselog1,sselog1,ssemov,ssemov")
1063 (set_attr "prefix" "maybe_vex")
1065 (cond [(and (eq_attr "alternative" "1")
1066 (match_test "TARGET_AVX512VL"))
1067 (const_string "<sseinsnmode>")
1068 (and (match_test "<MODE_SIZE> == 16")
1069 (ior (match_test "TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL")
1070 (and (eq_attr "alternative" "3")
1071 (match_test "TARGET_SSE_TYPELESS_STORES"))))
1072 (const_string "<ssePSmode>")
1073 (match_test "TARGET_AVX")
1074 (const_string "<sseinsnmode>")
1075 (ior (not (match_test "TARGET_SSE2"))
1076 (match_test "optimize_function_for_size_p (cfun)"))
1077 (const_string "V4SF")
1078 (and (eq_attr "alternative" "0")
1079 (match_test "TARGET_SSE_LOAD0_BY_PXOR"))
1082 (const_string "<sseinsnmode>")))
1083 (set (attr "enabled")
1084 (cond [(and (match_test "<MODE_SIZE> == 16")
1085 (eq_attr "alternative" "1"))
1086 (symbol_ref "TARGET_SSE2")
1087 (and (match_test "<MODE_SIZE> == 32")
1088 (eq_attr "alternative" "1"))
1089 (symbol_ref "TARGET_AVX2")
1091 (symbol_ref "true")))])
1093 (define_insn "<avx512>_load<mode>_mask"
1094 [(set (match_operand:V48_AVX512VL 0 "register_operand" "=v,v")
1095 (vec_merge:V48_AVX512VL
1096 (match_operand:V48_AVX512VL 1 "nonimmediate_operand" "v,m")
1097 (match_operand:V48_AVX512VL 2 "vector_move_operand" "0C,0C")
1098 (match_operand:<avx512fmaskmode> 3 "register_operand" "Yk,Yk")))]
1101 if (FLOAT_MODE_P (GET_MODE_INNER (<MODE>mode)))
1103 if (misaligned_operand (operands[1], <MODE>mode))
1104 return "vmovu<ssemodesuffix>\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}";
1106 return "vmova<ssemodesuffix>\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}";
1110 if (misaligned_operand (operands[1], <MODE>mode))
1111 return "vmovdqu<ssescalarsize>\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}";
1113 return "vmovdqa<ssescalarsize>\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}";
1116 [(set_attr "type" "ssemov")
1117 (set_attr "prefix" "evex")
1118 (set_attr "memory" "none,load")
1119 (set_attr "mode" "<sseinsnmode>")])
1121 (define_insn "<avx512>_load<mode>_mask"
1122 [(set (match_operand:VI12_AVX512VL 0 "register_operand" "=v,v")
1123 (vec_merge:VI12_AVX512VL
1124 (match_operand:VI12_AVX512VL 1 "nonimmediate_operand" "v,m")
1125 (match_operand:VI12_AVX512VL 2 "vector_move_operand" "0C,0C")
1126 (match_operand:<avx512fmaskmode> 3 "register_operand" "Yk,Yk")))]
1128 "vmovdqu<ssescalarsize>\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"
1129 [(set_attr "type" "ssemov")
1130 (set_attr "prefix" "evex")
1131 (set_attr "memory" "none,load")
1132 (set_attr "mode" "<sseinsnmode>")])
1134 (define_insn "<avx512>_blendm<mode>"
1135 [(set (match_operand:V48_AVX512VL 0 "register_operand" "=v")
1136 (vec_merge:V48_AVX512VL
1137 (match_operand:V48_AVX512VL 2 "nonimmediate_operand" "vm")
1138 (match_operand:V48_AVX512VL 1 "register_operand" "v")
1139 (match_operand:<avx512fmaskmode> 3 "register_operand" "Yk")))]
1141 "vblendm<ssemodesuffix>\t{%2, %1, %0%{%3%}|%0%{%3%}, %1, %2}"
1142 [(set_attr "type" "ssemov")
1143 (set_attr "prefix" "evex")
1144 (set_attr "mode" "<sseinsnmode>")])
1146 (define_insn "<avx512>_blendm<mode>"
1147 [(set (match_operand:VI12_AVX512VL 0 "register_operand" "=v")
1148 (vec_merge:VI12_AVX512VL
1149 (match_operand:VI12_AVX512VL 2 "nonimmediate_operand" "vm")
1150 (match_operand:VI12_AVX512VL 1 "register_operand" "v")
1151 (match_operand:<avx512fmaskmode> 3 "register_operand" "Yk")))]
1153 "vpblendm<ssemodesuffix>\t{%2, %1, %0%{%3%}|%0%{%3%}, %1, %2}"
1154 [(set_attr "type" "ssemov")
1155 (set_attr "prefix" "evex")
1156 (set_attr "mode" "<sseinsnmode>")])
1158 (define_insn "<avx512>_store<mode>_mask"
1159 [(set (match_operand:V48_AVX512VL 0 "memory_operand" "=m")
1160 (vec_merge:V48_AVX512VL
1161 (match_operand:V48_AVX512VL 1 "register_operand" "v")
1163 (match_operand:<avx512fmaskmode> 2 "register_operand" "Yk")))]
1166 if (FLOAT_MODE_P (GET_MODE_INNER (<MODE>mode)))
1168 if (misaligned_operand (operands[0], <MODE>mode))
1169 return "vmovu<ssemodesuffix>\t{%1, %0%{%2%}|%0%{%2%}, %1}";
1171 return "vmova<ssemodesuffix>\t{%1, %0%{%2%}|%0%{%2%}, %1}";
1175 if (misaligned_operand (operands[0], <MODE>mode))
1176 return "vmovdqu<ssescalarsize>\t{%1, %0%{%2%}|%0%{%2%}, %1}";
1178 return "vmovdqa<ssescalarsize>\t{%1, %0%{%2%}|%0%{%2%}, %1}";
1181 [(set_attr "type" "ssemov")
1182 (set_attr "prefix" "evex")
1183 (set_attr "memory" "store")
1184 (set_attr "mode" "<sseinsnmode>")])
1186 (define_insn "<avx512>_store<mode>_mask"
1187 [(set (match_operand:VI12_AVX512VL 0 "memory_operand" "=m")
1188 (vec_merge:VI12_AVX512VL
1189 (match_operand:VI12_AVX512VL 1 "register_operand" "v")
1191 (match_operand:<avx512fmaskmode> 2 "register_operand" "Yk")))]
1193 "vmovdqu<ssescalarsize>\t{%1, %0%{%2%}|%0%{%2%}, %1}"
1194 [(set_attr "type" "ssemov")
1195 (set_attr "prefix" "evex")
1196 (set_attr "memory" "store")
1197 (set_attr "mode" "<sseinsnmode>")])
1199 (define_insn "sse2_movq128"
1200 [(set (match_operand:V2DI 0 "register_operand" "=v")
1203 (match_operand:V2DI 1 "nonimmediate_operand" "vm")
1204 (parallel [(const_int 0)]))
1207 "%vmovq\t{%1, %0|%0, %q1}"
1208 [(set_attr "type" "ssemov")
1209 (set_attr "prefix" "maybe_vex")
1210 (set_attr "mode" "TI")])
1212 ;; Move a DI from a 32-bit register pair (e.g. %edx:%eax) to an xmm.
1213 ;; We'd rather avoid this entirely; if the 32-bit reg pair was loaded
1214 ;; from memory, we'd prefer to load the memory directly into the %xmm
1215 ;; register. To facilitate this happy circumstance, this pattern won't
1216 ;; split until after register allocation. If the 64-bit value didn't
1217 ;; come from memory, this is the best we can do. This is much better
1218 ;; than storing %edx:%eax into a stack temporary and loading an %xmm
1221 (define_insn_and_split "movdi_to_sse"
1223 [(set (match_operand:V4SI 0 "register_operand" "=?x,x")
1224 (subreg:V4SI (match_operand:DI 1 "nonimmediate_operand" "r,m") 0))
1225 (clobber (match_scratch:V4SI 2 "=&x,X"))])]
1226 "!TARGET_64BIT && TARGET_SSE2 && TARGET_INTER_UNIT_MOVES_TO_VEC"
1228 "&& reload_completed"
1231 if (register_operand (operands[1], DImode))
1233 /* The DImode arrived in a pair of integral registers (e.g. %edx:%eax).
1234 Assemble the 64-bit DImode value in an xmm register. */
1235 emit_insn (gen_sse2_loadld (operands[0], CONST0_RTX (V4SImode),
1236 gen_lowpart (SImode, operands[1])));
1237 emit_insn (gen_sse2_loadld (operands[2], CONST0_RTX (V4SImode),
1238 gen_highpart (SImode, operands[1])));
1239 emit_insn (gen_vec_interleave_lowv4si (operands[0], operands[0],
1242 else if (memory_operand (operands[1], DImode))
1244 rtx tmp = gen_reg_rtx (V2DImode);
1245 emit_insn (gen_vec_concatv2di (tmp, operands[1], const0_rtx));
1246 emit_move_insn (operands[0], gen_lowpart (V4SImode, tmp));
1254 [(set (match_operand:V4SF 0 "register_operand")
1255 (match_operand:V4SF 1 "zero_extended_scalar_load_operand"))]
1256 "TARGET_SSE && reload_completed"
1259 (vec_duplicate:V4SF (match_dup 1))
1263 operands[1] = gen_lowpart (SFmode, operands[1]);
1264 operands[2] = CONST0_RTX (V4SFmode);
1268 [(set (match_operand:V2DF 0 "register_operand")
1269 (match_operand:V2DF 1 "zero_extended_scalar_load_operand"))]
1270 "TARGET_SSE2 && reload_completed"
1271 [(set (match_dup 0) (vec_concat:V2DF (match_dup 1) (match_dup 2)))]
1273 operands[1] = gen_lowpart (DFmode, operands[1]);
1274 operands[2] = CONST0_RTX (DFmode);
1277 (define_expand "movmisalign<mode>"
1278 [(set (match_operand:VMOVE 0 "nonimmediate_operand")
1279 (match_operand:VMOVE 1 "nonimmediate_operand"))]
1282 ix86_expand_vector_move_misalign (<MODE>mode, operands);
1286 ;; Merge movsd/movhpd to movupd for TARGET_SSE_UNALIGNED_LOAD_OPTIMAL targets.
1288 [(set (match_operand:V2DF 0 "sse_reg_operand")
1289 (vec_concat:V2DF (match_operand:DF 1 "memory_operand")
1290 (match_operand:DF 4 "const0_operand")))
1291 (set (match_operand:V2DF 2 "sse_reg_operand")
1292 (vec_concat:V2DF (vec_select:DF (match_dup 2)
1293 (parallel [(const_int 0)]))
1294 (match_operand:DF 3 "memory_operand")))]
1295 "TARGET_SSE2 && TARGET_SSE_UNALIGNED_LOAD_OPTIMAL
1296 && ix86_operands_ok_for_move_multiple (operands, true, DFmode)"
1297 [(set (match_dup 2) (match_dup 5))]
1298 "operands[5] = adjust_address (operands[1], V2DFmode, 0);")
1301 [(set (match_operand:DF 0 "sse_reg_operand")
1302 (match_operand:DF 1 "memory_operand"))
1303 (set (match_operand:V2DF 2 "sse_reg_operand")
1304 (vec_concat:V2DF (match_operand:DF 4 "sse_reg_operand")
1305 (match_operand:DF 3 "memory_operand")))]
1306 "TARGET_SSE2 && TARGET_SSE_UNALIGNED_LOAD_OPTIMAL
1307 && REGNO (operands[4]) == REGNO (operands[2])
1308 && ix86_operands_ok_for_move_multiple (operands, true, DFmode)"
1309 [(set (match_dup 2) (match_dup 5))]
1310 "operands[5] = adjust_address (operands[1], V2DFmode, 0);")
1312 ;; Merge movlpd/movhpd to movupd for TARGET_SSE_UNALIGNED_STORE_OPTIMAL targets.
1314 [(set (match_operand:DF 0 "memory_operand")
1315 (vec_select:DF (match_operand:V2DF 1 "sse_reg_operand")
1316 (parallel [(const_int 0)])))
1317 (set (match_operand:DF 2 "memory_operand")
1318 (vec_select:DF (match_operand:V2DF 3 "sse_reg_operand")
1319 (parallel [(const_int 1)])))]
1320 "TARGET_SSE2 && TARGET_SSE_UNALIGNED_STORE_OPTIMAL
1321 && ix86_operands_ok_for_move_multiple (operands, false, DFmode)"
1322 [(set (match_dup 4) (match_dup 1))]
1323 "operands[4] = adjust_address (operands[0], V2DFmode, 0);")
1325 (define_insn "<sse3>_lddqu<avxsizesuffix>"
1326 [(set (match_operand:VI1 0 "register_operand" "=x")
1327 (unspec:VI1 [(match_operand:VI1 1 "memory_operand" "m")]
1330 "%vlddqu\t{%1, %0|%0, %1}"
1331 [(set_attr "type" "ssemov")
1332 (set_attr "movu" "1")
1333 (set (attr "prefix_data16")
1335 (match_test "TARGET_AVX")
1337 (const_string "0")))
1338 (set (attr "prefix_rep")
1340 (match_test "TARGET_AVX")
1342 (const_string "1")))
1343 (set_attr "prefix" "maybe_vex")
1344 (set_attr "mode" "<sseinsnmode>")])
1346 (define_insn "sse2_movnti<mode>"
1347 [(set (match_operand:SWI48 0 "memory_operand" "=m")
1348 (unspec:SWI48 [(match_operand:SWI48 1 "register_operand" "r")]
1351 "movnti\t{%1, %0|%0, %1}"
1352 [(set_attr "type" "ssemov")
1353 (set_attr "prefix_data16" "0")
1354 (set_attr "mode" "<MODE>")])
1356 (define_insn "<sse>_movnt<mode>"
1357 [(set (match_operand:VF 0 "memory_operand" "=m")
1359 [(match_operand:VF 1 "register_operand" "v")]
1362 "%vmovnt<ssemodesuffix>\t{%1, %0|%0, %1}"
1363 [(set_attr "type" "ssemov")
1364 (set_attr "prefix" "maybe_vex")
1365 (set_attr "mode" "<MODE>")])
1367 (define_insn "<sse2>_movnt<mode>"
1368 [(set (match_operand:VI8 0 "memory_operand" "=m")
1369 (unspec:VI8 [(match_operand:VI8 1 "register_operand" "v")]
1372 "%vmovntdq\t{%1, %0|%0, %1}"
1373 [(set_attr "type" "ssecvt")
1374 (set (attr "prefix_data16")
1376 (match_test "TARGET_AVX")
1378 (const_string "1")))
1379 (set_attr "prefix" "maybe_vex")
1380 (set_attr "mode" "<sseinsnmode>")])
1382 ; Expand patterns for non-temporal stores. At the moment, only those
1383 ; that directly map to insns are defined; it would be possible to
1384 ; define patterns for other modes that would expand to several insns.
1386 ;; Modes handled by storent patterns.
1387 (define_mode_iterator STORENT_MODE
1388 [(DI "TARGET_SSE2 && TARGET_64BIT") (SI "TARGET_SSE2")
1389 (SF "TARGET_SSE4A") (DF "TARGET_SSE4A")
1390 (V8DI "TARGET_AVX512F") (V4DI "TARGET_AVX") (V2DI "TARGET_SSE2")
1391 (V16SF "TARGET_AVX512F") (V8SF "TARGET_AVX") V4SF
1392 (V8DF "TARGET_AVX512F") (V4DF "TARGET_AVX") (V2DF "TARGET_SSE2")])
1394 (define_expand "storent<mode>"
1395 [(set (match_operand:STORENT_MODE 0 "memory_operand")
1396 (unspec:STORENT_MODE
1397 [(match_operand:STORENT_MODE 1 "register_operand")]
1401 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
1405 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
1407 ;; All integer modes with AVX512BW/DQ.
1408 (define_mode_iterator SWI1248_AVX512BWDQ
1409 [(QI "TARGET_AVX512DQ") HI (SI "TARGET_AVX512BW") (DI "TARGET_AVX512BW")])
1411 ;; All integer modes with AVX512BW, where HImode operation
1412 ;; can be used instead of QImode.
1413 (define_mode_iterator SWI1248_AVX512BW
1414 [QI HI (SI "TARGET_AVX512BW") (DI "TARGET_AVX512BW")])
1416 ;; All integer modes with AVX512BW/DQ, even HImode requires DQ.
1417 (define_mode_iterator SWI1248_AVX512BWDQ2
1418 [(QI "TARGET_AVX512DQ") (HI "TARGET_AVX512DQ")
1419 (SI "TARGET_AVX512BW") (DI "TARGET_AVX512BW")])
1421 (define_expand "kmov<mskmodesuffix>"
1422 [(set (match_operand:SWI1248_AVX512BWDQ 0 "nonimmediate_operand")
1423 (match_operand:SWI1248_AVX512BWDQ 1 "nonimmediate_operand"))]
1425 && !(MEM_P (operands[0]) && MEM_P (operands[1]))")
1427 (define_insn "k<code><mode>"
1428 [(set (match_operand:SWI1248_AVX512BW 0 "register_operand" "=k")
1429 (any_logic:SWI1248_AVX512BW
1430 (match_operand:SWI1248_AVX512BW 1 "register_operand" "k")
1431 (match_operand:SWI1248_AVX512BW 2 "register_operand" "k")))
1432 (unspec [(const_int 0)] UNSPEC_MASKOP)]
1435 if (get_attr_mode (insn) == MODE_HI)
1436 return "k<logic>w\t{%2, %1, %0|%0, %1, %2}";
1438 return "k<logic><mskmodesuffix>\t{%2, %1, %0|%0, %1, %2}";
1440 [(set_attr "type" "msklog")
1441 (set_attr "prefix" "vex")
1443 (cond [(and (match_test "<MODE>mode == QImode")
1444 (not (match_test "TARGET_AVX512DQ")))
1447 (const_string "<MODE>")))])
1449 (define_insn "kandn<mode>"
1450 [(set (match_operand:SWI1248_AVX512BW 0 "register_operand" "=k")
1451 (and:SWI1248_AVX512BW
1452 (not:SWI1248_AVX512BW
1453 (match_operand:SWI1248_AVX512BW 1 "register_operand" "k"))
1454 (match_operand:SWI1248_AVX512BW 2 "register_operand" "k")))
1455 (unspec [(const_int 0)] UNSPEC_MASKOP)]
1458 if (get_attr_mode (insn) == MODE_HI)
1459 return "kandnw\t{%2, %1, %0|%0, %1, %2}";
1461 return "kandn<mskmodesuffix>\t{%2, %1, %0|%0, %1, %2}";
1463 [(set_attr "type" "msklog")
1464 (set_attr "prefix" "vex")
1466 (cond [(and (match_test "<MODE>mode == QImode")
1467 (not (match_test "TARGET_AVX512DQ")))
1470 (const_string "<MODE>")))])
1472 (define_insn "kxnor<mode>"
1473 [(set (match_operand:SWI1248_AVX512BW 0 "register_operand" "=k")
1474 (not:SWI1248_AVX512BW
1475 (xor:SWI1248_AVX512BW
1476 (match_operand:SWI1248_AVX512BW 1 "register_operand" "k")
1477 (match_operand:SWI1248_AVX512BW 2 "register_operand" "k"))))
1478 (unspec [(const_int 0)] UNSPEC_MASKOP)]
1481 if (get_attr_mode (insn) == MODE_HI)
1482 return "kxnorw\t{%2, %1, %0|%0, %1, %2}";
1484 return "kxnor<mskmodesuffix>\t{%2, %1, %0|%0, %1, %2}";
1486 [(set_attr "type" "msklog")
1487 (set_attr "prefix" "vex")
1489 (cond [(and (match_test "<MODE>mode == QImode")
1490 (not (match_test "TARGET_AVX512DQ")))
1493 (const_string "<MODE>")))])
1495 (define_insn "knot<mode>"
1496 [(set (match_operand:SWI1248_AVX512BW 0 "register_operand" "=k")
1497 (not:SWI1248_AVX512BW
1498 (match_operand:SWI1248_AVX512BW 1 "register_operand" "k")))
1499 (unspec [(const_int 0)] UNSPEC_MASKOP)]
1502 if (get_attr_mode (insn) == MODE_HI)
1503 return "knotw\t{%1, %0|%0, %1}";
1505 return "knot<mskmodesuffix>\t{%1, %0|%0, %1}";
1507 [(set_attr "type" "msklog")
1508 (set_attr "prefix" "vex")
1510 (cond [(and (match_test "<MODE>mode == QImode")
1511 (not (match_test "TARGET_AVX512DQ")))
1514 (const_string "<MODE>")))])
1516 (define_insn "kadd<mode>"
1517 [(set (match_operand:SWI1248_AVX512BWDQ2 0 "register_operand" "=k")
1518 (plus:SWI1248_AVX512BWDQ2
1519 (match_operand:SWI1248_AVX512BWDQ2 1 "register_operand" "k")
1520 (match_operand:SWI1248_AVX512BWDQ2 2 "register_operand" "k")))
1521 (unspec [(const_int 0)] UNSPEC_MASKOP)]
1523 "kadd<mskmodesuffix>\t{%2, %1, %0|%0, %1, %2}"
1524 [(set_attr "type" "msklog")
1525 (set_attr "prefix" "vex")
1526 (set_attr "mode" "<MODE>")])
1528 ;; Mask variant shift mnemonics
1529 (define_code_attr mshift [(ashift "shiftl") (lshiftrt "shiftr")])
1531 (define_insn "k<code><mode>"
1532 [(set (match_operand:SWI1248_AVX512BWDQ 0 "register_operand" "=k")
1533 (any_lshift:SWI1248_AVX512BWDQ
1534 (match_operand:SWI1248_AVX512BWDQ 1 "register_operand" "k")
1535 (match_operand:QI 2 "immediate_operand" "n")))
1536 (unspec [(const_int 0)] UNSPEC_MASKOP)]
1538 "k<mshift><mskmodesuffix>\t{%2, %1, %0|%0, %1, %2}"
1539 [(set_attr "type" "msklog")
1540 (set_attr "prefix" "vex")
1541 (set_attr "mode" "<MODE>")])
1543 (define_insn "ktest<mode>"
1544 [(set (reg:CC FLAGS_REG)
1546 [(match_operand:SWI1248_AVX512BWDQ2 0 "register_operand" "k")
1547 (match_operand:SWI1248_AVX512BWDQ2 1 "register_operand" "k")]
1550 "ktest<mskmodesuffix>\t{%1, %0|%0, %1}"
1551 [(set_attr "mode" "<MODE>")
1552 (set_attr "type" "msklog")
1553 (set_attr "prefix" "vex")])
1555 (define_insn "kortest<mode>"
1556 [(set (reg:CC FLAGS_REG)
1558 [(match_operand:SWI1248_AVX512BWDQ 0 "register_operand" "k")
1559 (match_operand:SWI1248_AVX512BWDQ 1 "register_operand" "k")]
1562 "kortest<mskmodesuffix>\t{%1, %0|%0, %1}"
1563 [(set_attr "mode" "<MODE>")
1564 (set_attr "type" "msklog")
1565 (set_attr "prefix" "vex")])
1567 (define_insn "kunpckhi"
1568 [(set (match_operand:HI 0 "register_operand" "=k")
1571 (zero_extend:HI (match_operand:QI 1 "register_operand" "k"))
1573 (zero_extend:HI (match_operand:QI 2 "register_operand" "k"))))]
1575 "kunpckbw\t{%2, %1, %0|%0, %1, %2}"
1576 [(set_attr "mode" "HI")
1577 (set_attr "type" "msklog")
1578 (set_attr "prefix" "vex")])
1580 (define_insn "kunpcksi"
1581 [(set (match_operand:SI 0 "register_operand" "=k")
1584 (zero_extend:SI (match_operand:HI 1 "register_operand" "k"))
1586 (zero_extend:SI (match_operand:HI 2 "register_operand" "k"))))]
1588 "kunpckwd\t{%2, %1, %0|%0, %1, %2}"
1589 [(set_attr "mode" "SI")])
1591 (define_insn "kunpckdi"
1592 [(set (match_operand:DI 0 "register_operand" "=k")
1595 (zero_extend:DI (match_operand:SI 1 "register_operand" "k"))
1597 (zero_extend:DI (match_operand:SI 2 "register_operand" "k"))))]
1599 "kunpckdq\t{%2, %1, %0|%0, %1, %2}"
1600 [(set_attr "mode" "DI")])
1603 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
1605 ;; Parallel floating point arithmetic
1607 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
1609 (define_expand "<code><mode>2"
1610 [(set (match_operand:VF 0 "register_operand")
1612 (match_operand:VF 1 "register_operand")))]
1614 "ix86_expand_fp_absneg_operator (<CODE>, <MODE>mode, operands); DONE;")
1616 (define_insn_and_split "*absneg<mode>2"
1617 [(set (match_operand:VF 0 "register_operand" "=x,x,v,v")
1618 (match_operator:VF 3 "absneg_operator"
1619 [(match_operand:VF 1 "vector_operand" "0, xBm,v, m")]))
1620 (use (match_operand:VF 2 "vector_operand" "xBm,0, vm,v"))]
1623 "&& reload_completed"
1626 enum rtx_code absneg_op;
1632 if (MEM_P (operands[1]))
1633 op1 = operands[2], op2 = operands[1];
1635 op1 = operands[1], op2 = operands[2];
1640 if (rtx_equal_p (operands[0], operands[1]))
1646 absneg_op = GET_CODE (operands[3]) == NEG ? XOR : AND;
1647 t = gen_rtx_fmt_ee (absneg_op, <MODE>mode, op1, op2);
1648 t = gen_rtx_SET (operands[0], t);
1652 [(set_attr "isa" "noavx,noavx,avx,avx")])
1654 (define_expand "<plusminus_insn><mode>3<mask_name><round_name>"
1655 [(set (match_operand:VF 0 "register_operand")
1657 (match_operand:VF 1 "<round_nimm_predicate>")
1658 (match_operand:VF 2 "<round_nimm_predicate>")))]
1659 "TARGET_SSE && <mask_mode512bit_condition> && <round_mode512bit_condition>"
1660 "ix86_fixup_binary_operands_no_copy (<CODE>, <MODE>mode, operands);")
1662 (define_insn "*<plusminus_insn><mode>3<mask_name><round_name>"
1663 [(set (match_operand:VF 0 "register_operand" "=x,v")
1665 (match_operand:VF 1 "<round_nimm_predicate>" "<comm>0,v")
1666 (match_operand:VF 2 "<round_nimm_predicate>" "xBm,<round_constraint>")))]
1667 "TARGET_SSE && ix86_binary_operator_ok (<CODE>, <MODE>mode, operands)
1668 && <mask_mode512bit_condition> && <round_mode512bit_condition>"
1670 <plusminus_mnemonic><ssemodesuffix>\t{%2, %0|%0, %2}
1671 v<plusminus_mnemonic><ssemodesuffix>\t{<round_mask_op3>%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2<round_mask_op3>}"
1672 [(set_attr "isa" "noavx,avx")
1673 (set_attr "type" "sseadd")
1674 (set_attr "prefix" "<mask_prefix3>")
1675 (set_attr "mode" "<MODE>")])
1677 (define_insn "<sse>_vm<plusminus_insn><mode>3<mask_scalar_name><round_scalar_name>"
1678 [(set (match_operand:VF_128 0 "register_operand" "=x,v")
1681 (match_operand:VF_128 1 "register_operand" "0,v")
1682 (match_operand:VF_128 2 "vector_operand" "xBm,<round_scalar_constraint>"))
1687 <plusminus_mnemonic><ssescalarmodesuffix>\t{%2, %0|%0, %<iptr>2}
1688 v<plusminus_mnemonic><ssescalarmodesuffix>\t{<round_scalar_mask_op3>%2, %1, %0<mask_scalar_operand3>|%0<mask_scalar_operand3>, %1, %<iptr>2<round_scalar_mask_op3>}"
1689 [(set_attr "isa" "noavx,avx")
1690 (set_attr "type" "sseadd")
1691 (set_attr "prefix" "<round_scalar_prefix>")
1692 (set_attr "mode" "<ssescalarmode>")])
1694 (define_expand "mul<mode>3<mask_name><round_name>"
1695 [(set (match_operand:VF 0 "register_operand")
1697 (match_operand:VF 1 "<round_nimm_predicate>")
1698 (match_operand:VF 2 "<round_nimm_predicate>")))]
1699 "TARGET_SSE && <mask_mode512bit_condition> && <round_mode512bit_condition>"
1700 "ix86_fixup_binary_operands_no_copy (MULT, <MODE>mode, operands);")
1702 (define_insn "*mul<mode>3<mask_name><round_name>"
1703 [(set (match_operand:VF 0 "register_operand" "=x,v")
1705 (match_operand:VF 1 "<round_nimm_predicate>" "%0,v")
1706 (match_operand:VF 2 "<round_nimm_predicate>" "xBm,<round_constraint>")))]
1708 && !(MEM_P (operands[1]) && MEM_P (operands[2]))
1709 && <mask_mode512bit_condition> && <round_mode512bit_condition>"
1711 mul<ssemodesuffix>\t{%2, %0|%0, %2}
1712 vmul<ssemodesuffix>\t{<round_mask_op3>%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2<round_mask_op3>}"
1713 [(set_attr "isa" "noavx,avx")
1714 (set_attr "type" "ssemul")
1715 (set_attr "prefix" "<mask_prefix3>")
1716 (set_attr "btver2_decode" "direct,double")
1717 (set_attr "mode" "<MODE>")])
1719 (define_insn "<sse>_vm<multdiv_mnemonic><mode>3<mask_scalar_name><round_scalar_name>"
1720 [(set (match_operand:VF_128 0 "register_operand" "=x,v")
1723 (match_operand:VF_128 1 "register_operand" "0,v")
1724 (match_operand:VF_128 2 "vector_operand" "xBm,<round_scalar_constraint>"))
1729 <multdiv_mnemonic><ssescalarmodesuffix>\t{%2, %0|%0, %<iptr>2}
1730 v<multdiv_mnemonic><ssescalarmodesuffix>\t{<round_scalar_mask_op3>%2, %1, %0<mask_scalar_operand3>|%0<mask_scalar_operand3>, %1, %<iptr>2<round_scalar_mask_op3>}"
1731 [(set_attr "isa" "noavx,avx")
1732 (set_attr "type" "sse<multdiv_mnemonic>")
1733 (set_attr "prefix" "<round_scalar_prefix>")
1734 (set_attr "btver2_decode" "direct,double")
1735 (set_attr "mode" "<ssescalarmode>")])
1737 (define_expand "div<mode>3"
1738 [(set (match_operand:VF2 0 "register_operand")
1739 (div:VF2 (match_operand:VF2 1 "register_operand")
1740 (match_operand:VF2 2 "vector_operand")))]
1742 "ix86_fixup_binary_operands_no_copy (DIV, <MODE>mode, operands);")
1744 (define_expand "div<mode>3"
1745 [(set (match_operand:VF1 0 "register_operand")
1746 (div:VF1 (match_operand:VF1 1 "register_operand")
1747 (match_operand:VF1 2 "vector_operand")))]
1750 ix86_fixup_binary_operands_no_copy (DIV, <MODE>mode, operands);
1753 && TARGET_RECIP_VEC_DIV
1754 && !optimize_insn_for_size_p ()
1755 && flag_finite_math_only && !flag_trapping_math
1756 && flag_unsafe_math_optimizations)
1758 ix86_emit_swdivsf (operands[0], operands[1], operands[2], <MODE>mode);
1763 (define_insn "<sse>_div<mode>3<mask_name><round_name>"
1764 [(set (match_operand:VF 0 "register_operand" "=x,v")
1766 (match_operand:VF 1 "register_operand" "0,v")
1767 (match_operand:VF 2 "<round_nimm_predicate>" "xBm,<round_constraint>")))]
1768 "TARGET_SSE && <mask_mode512bit_condition> && <round_mode512bit_condition>"
1770 div<ssemodesuffix>\t{%2, %0|%0, %2}
1771 vdiv<ssemodesuffix>\t{<round_mask_op3>%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2<round_mask_op3>}"
1772 [(set_attr "isa" "noavx,avx")
1773 (set_attr "type" "ssediv")
1774 (set_attr "prefix" "<mask_prefix3>")
1775 (set_attr "mode" "<MODE>")])
1777 (define_insn "<sse>_rcp<mode>2"
1778 [(set (match_operand:VF1_128_256 0 "register_operand" "=x")
1780 [(match_operand:VF1_128_256 1 "vector_operand" "xBm")] UNSPEC_RCP))]
1782 "%vrcpps\t{%1, %0|%0, %1}"
1783 [(set_attr "type" "sse")
1784 (set_attr "atom_sse_attr" "rcp")
1785 (set_attr "btver2_sse_attr" "rcp")
1786 (set_attr "prefix" "maybe_vex")
1787 (set_attr "mode" "<MODE>")])
1789 (define_insn "sse_vmrcpv4sf2"
1790 [(set (match_operand:V4SF 0 "register_operand" "=x,x")
1792 (unspec:V4SF [(match_operand:V4SF 1 "nonimmediate_operand" "xm,xm")]
1794 (match_operand:V4SF 2 "register_operand" "0,x")
1798 rcpss\t{%1, %0|%0, %k1}
1799 vrcpss\t{%1, %2, %0|%0, %2, %k1}"
1800 [(set_attr "isa" "noavx,avx")
1801 (set_attr "type" "sse")
1802 (set_attr "atom_sse_attr" "rcp")
1803 (set_attr "btver2_sse_attr" "rcp")
1804 (set_attr "prefix" "orig,vex")
1805 (set_attr "mode" "SF")])
1807 (define_insn "<mask_codefor>rcp14<mode><mask_name>"
1808 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v")
1810 [(match_operand:VF_AVX512VL 1 "nonimmediate_operand" "vm")]
1813 "vrcp14<ssemodesuffix>\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
1814 [(set_attr "type" "sse")
1815 (set_attr "prefix" "evex")
1816 (set_attr "mode" "<MODE>")])
1818 (define_insn "srcp14<mode>"
1819 [(set (match_operand:VF_128 0 "register_operand" "=v")
1822 [(match_operand:VF_128 1 "nonimmediate_operand" "vm")]
1824 (match_operand:VF_128 2 "register_operand" "v")
1827 "vrcp14<ssescalarmodesuffix>\t{%1, %2, %0|%0, %2, %<iptr>1}"
1828 [(set_attr "type" "sse")
1829 (set_attr "prefix" "evex")
1830 (set_attr "mode" "<MODE>")])
1832 (define_insn "srcp14<mode>_mask"
1833 [(set (match_operand:VF_128 0 "register_operand" "=v")
1837 [(match_operand:VF_128 1 "nonimmediate_operand" "vm")]
1839 (match_operand:VF_128 3 "vector_move_operand" "0C")
1840 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk"))
1841 (match_operand:VF_128 2 "register_operand" "v")
1844 "vrcp14<ssescalarmodesuffix>\t{%1, %2, %0%{%4%}%N3|%0%{%4%}%N3, %2, %<iptr>1}"
1845 [(set_attr "type" "sse")
1846 (set_attr "prefix" "evex")
1847 (set_attr "mode" "<MODE>")])
1849 (define_expand "sqrt<mode>2"
1850 [(set (match_operand:VF2 0 "register_operand")
1851 (sqrt:VF2 (match_operand:VF2 1 "vector_operand")))]
1854 (define_expand "sqrt<mode>2"
1855 [(set (match_operand:VF1 0 "register_operand")
1856 (sqrt:VF1 (match_operand:VF1 1 "vector_operand")))]
1860 && TARGET_RECIP_VEC_SQRT
1861 && !optimize_insn_for_size_p ()
1862 && flag_finite_math_only && !flag_trapping_math
1863 && flag_unsafe_math_optimizations)
1865 ix86_emit_swsqrtsf (operands[0], operands[1], <MODE>mode, false);
1870 (define_insn "<sse>_sqrt<mode>2<mask_name><round_name>"
1871 [(set (match_operand:VF 0 "register_operand" "=x,v")
1872 (sqrt:VF (match_operand:VF 1 "<round_nimm_predicate>" "xBm,<round_constraint>")))]
1873 "TARGET_SSE && <mask_mode512bit_condition> && <round_mode512bit_condition>"
1875 sqrt<ssemodesuffix>\t{%1, %0|%0, %1}
1876 vsqrt<ssemodesuffix>\t{<round_mask_op2>%1, %0<mask_operand2>|%0<mask_operand2>, %1<round_mask_op2>}"
1877 [(set_attr "isa" "noavx,avx")
1878 (set_attr "type" "sse")
1879 (set_attr "atom_sse_attr" "sqrt")
1880 (set_attr "btver2_sse_attr" "sqrt")
1881 (set_attr "prefix" "maybe_vex")
1882 (set_attr "mode" "<MODE>")])
1884 (define_insn "<sse>_vmsqrt<mode>2<mask_scalar_name><round_scalar_name>"
1885 [(set (match_operand:VF_128 0 "register_operand" "=x,v")
1888 (match_operand:VF_128 1 "vector_operand" "xBm,<round_scalar_constraint>"))
1889 (match_operand:VF_128 2 "register_operand" "0,v")
1893 sqrt<ssescalarmodesuffix>\t{%1, %0|%0, %<iptr>1}
1894 vsqrt<ssescalarmodesuffix>\t{<round_scalar_mask_op3>%1, %2, %0<mask_scalar_operand3>|%0<mask_scalar_operand3>, %2, %<iptr>1<round_scalar_mask_op3>}"
1895 [(set_attr "isa" "noavx,avx")
1896 (set_attr "type" "sse")
1897 (set_attr "atom_sse_attr" "sqrt")
1898 (set_attr "prefix" "<round_scalar_prefix>")
1899 (set_attr "btver2_sse_attr" "sqrt")
1900 (set_attr "mode" "<ssescalarmode>")])
1902 (define_expand "rsqrt<mode>2"
1903 [(set (match_operand:VF1_128_256 0 "register_operand")
1905 [(match_operand:VF1_128_256 1 "vector_operand")] UNSPEC_RSQRT))]
1908 ix86_emit_swsqrtsf (operands[0], operands[1], <MODE>mode, true);
1912 (define_expand "rsqrtv16sf2"
1913 [(set (match_operand:V16SF 0 "register_operand")
1915 [(match_operand:V16SF 1 "vector_operand")]
1917 "TARGET_SSE_MATH && TARGET_AVX512ER"
1919 ix86_emit_swsqrtsf (operands[0], operands[1], V16SFmode, true);
1923 (define_insn "<sse>_rsqrt<mode>2"
1924 [(set (match_operand:VF1_128_256 0 "register_operand" "=x")
1926 [(match_operand:VF1_128_256 1 "vector_operand" "xBm")] UNSPEC_RSQRT))]
1928 "%vrsqrtps\t{%1, %0|%0, %1}"
1929 [(set_attr "type" "sse")
1930 (set_attr "prefix" "maybe_vex")
1931 (set_attr "mode" "<MODE>")])
1933 (define_insn "<mask_codefor>rsqrt14<mode><mask_name>"
1934 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v")
1936 [(match_operand:VF_AVX512VL 1 "nonimmediate_operand" "vm")]
1939 "vrsqrt14<ssemodesuffix>\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
1940 [(set_attr "type" "sse")
1941 (set_attr "prefix" "evex")
1942 (set_attr "mode" "<MODE>")])
1944 (define_insn "rsqrt14<mode>"
1945 [(set (match_operand:VF_128 0 "register_operand" "=v")
1948 [(match_operand:VF_128 1 "nonimmediate_operand" "vm")]
1950 (match_operand:VF_128 2 "register_operand" "v")
1953 "vrsqrt14<ssescalarmodesuffix>\t{%1, %2, %0|%0, %2, %<iptr>1}"
1954 [(set_attr "type" "sse")
1955 (set_attr "prefix" "evex")
1956 (set_attr "mode" "<MODE>")])
1958 (define_insn "rsqrt14_<mode>_mask"
1959 [(set (match_operand:VF_128 0 "register_operand" "=v")
1963 [(match_operand:VF_128 1 "nonimmediate_operand" "vm")]
1965 (match_operand:VF_128 3 "vector_move_operand" "0C")
1966 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk"))
1967 (match_operand:VF_128 2 "register_operand" "v")
1970 "vrsqrt14<ssescalarmodesuffix>\t{%1, %2, %0%{%4%}%N3|%0%{%4%}%N3, %2, %<iptr>1}"
1971 [(set_attr "type" "sse")
1972 (set_attr "prefix" "evex")
1973 (set_attr "mode" "<MODE>")])
1975 (define_insn "sse_vmrsqrtv4sf2"
1976 [(set (match_operand:V4SF 0 "register_operand" "=x,x")
1978 (unspec:V4SF [(match_operand:V4SF 1 "nonimmediate_operand" "xm,xm")]
1980 (match_operand:V4SF 2 "register_operand" "0,x")
1984 rsqrtss\t{%1, %0|%0, %k1}
1985 vrsqrtss\t{%1, %2, %0|%0, %2, %k1}"
1986 [(set_attr "isa" "noavx,avx")
1987 (set_attr "type" "sse")
1988 (set_attr "prefix" "orig,vex")
1989 (set_attr "mode" "SF")])
1991 (define_expand "<code><mode>3<mask_name><round_saeonly_name>"
1992 [(set (match_operand:VF 0 "register_operand")
1994 (match_operand:VF 1 "<round_saeonly_nimm_predicate>")
1995 (match_operand:VF 2 "<round_saeonly_nimm_predicate>")))]
1996 "TARGET_SSE && <mask_mode512bit_condition> && <round_saeonly_mode512bit_condition>"
1998 if (!flag_finite_math_only || flag_signed_zeros)
2000 operands[1] = force_reg (<MODE>mode, operands[1]);
2001 emit_insn (gen_ieee_<maxmin_float><mode>3<mask_name><round_saeonly_name>
2002 (operands[0], operands[1], operands[2]
2003 <mask_operand_arg34>
2004 <round_saeonly_mask_arg3>));
2008 ix86_fixup_binary_operands_no_copy (<CODE>, <MODE>mode, operands);
2011 ;; These versions of the min/max patterns are intentionally ignorant of
2012 ;; their behavior wrt -0.0 and NaN (via the commutative operand mark).
2013 ;; Since both the tree-level MAX_EXPR and the rtl-level SMAX operator
2014 ;; are undefined in this condition, we're certain this is correct.
2016 (define_insn "*<code><mode>3<mask_name><round_saeonly_name>"
2017 [(set (match_operand:VF 0 "register_operand" "=x,v")
2019 (match_operand:VF 1 "<round_saeonly_nimm_predicate>" "%0,v")
2020 (match_operand:VF 2 "<round_saeonly_nimm_predicate>" "xBm,<round_saeonly_constraint>")))]
2022 && !(MEM_P (operands[1]) && MEM_P (operands[2]))
2023 && <mask_mode512bit_condition> && <round_saeonly_mode512bit_condition>"
2025 <maxmin_float><ssemodesuffix>\t{%2, %0|%0, %2}
2026 v<maxmin_float><ssemodesuffix>\t{<round_saeonly_mask_op3>%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2<round_saeonly_mask_op3>}"
2027 [(set_attr "isa" "noavx,avx")
2028 (set_attr "type" "sseadd")
2029 (set_attr "btver2_sse_attr" "maxmin")
2030 (set_attr "prefix" "<mask_prefix3>")
2031 (set_attr "mode" "<MODE>")])
2033 ;; These versions of the min/max patterns implement exactly the operations
2034 ;; min = (op1 < op2 ? op1 : op2)
2035 ;; max = (!(op1 < op2) ? op1 : op2)
2036 ;; Their operands are not commutative, and thus they may be used in the
2037 ;; presence of -0.0 and NaN.
2039 (define_insn "ieee_<ieee_maxmin><mode>3<mask_name><round_saeonly_name>"
2040 [(set (match_operand:VF 0 "register_operand" "=x,v")
2042 [(match_operand:VF 1 "register_operand" "0,v")
2043 (match_operand:VF 2 "<round_saeonly_nimm_predicate>" "xBm,<round_saeonly_constraint>")]
2046 && <mask_mode512bit_condition> && <round_saeonly_mode512bit_condition>"
2048 <ieee_maxmin><ssemodesuffix>\t{%2, %0|%0, %2}
2049 v<ieee_maxmin><ssemodesuffix>\t{<round_saeonly_mask_op3>%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2<round_saeonly_mask_op3>}"
2050 [(set_attr "isa" "noavx,avx")
2051 (set_attr "type" "sseadd")
2052 (set_attr "btver2_sse_attr" "maxmin")
2053 (set_attr "prefix" "<mask_prefix3>")
2054 (set_attr "mode" "<MODE>")])
2056 (define_insn "<sse>_vm<code><mode>3<mask_scalar_name><round_saeonly_scalar_name>"
2057 [(set (match_operand:VF_128 0 "register_operand" "=x,v")
2060 (match_operand:VF_128 1 "register_operand" "0,v")
2061 (match_operand:VF_128 2 "vector_operand" "xBm,<round_saeonly_scalar_constraint>"))
2066 <maxmin_float><ssescalarmodesuffix>\t{%2, %0|%0, %<iptr>2}
2067 v<maxmin_float><ssescalarmodesuffix>\t{<round_saeonly_scalar_mask_op3>%2, %1, %0<mask_scalar_operand3>|%0<mask_scalar_operand3>, %1, %<iptr>2<round_saeonly_scalar_mask_op3>}"
2068 [(set_attr "isa" "noavx,avx")
2069 (set_attr "type" "sse")
2070 (set_attr "btver2_sse_attr" "maxmin")
2071 (set_attr "prefix" "<round_saeonly_scalar_prefix>")
2072 (set_attr "mode" "<ssescalarmode>")])
2074 (define_insn "avx_addsubv4df3"
2075 [(set (match_operand:V4DF 0 "register_operand" "=x")
2078 (match_operand:V4DF 1 "register_operand" "x")
2079 (match_operand:V4DF 2 "nonimmediate_operand" "xm"))
2080 (plus:V4DF (match_dup 1) (match_dup 2))
2083 "vaddsubpd\t{%2, %1, %0|%0, %1, %2}"
2084 [(set_attr "type" "sseadd")
2085 (set_attr "prefix" "vex")
2086 (set_attr "mode" "V4DF")])
2088 (define_insn "sse3_addsubv2df3"
2089 [(set (match_operand:V2DF 0 "register_operand" "=x,x")
2092 (match_operand:V2DF 1 "register_operand" "0,x")
2093 (match_operand:V2DF 2 "vector_operand" "xBm,xm"))
2094 (plus:V2DF (match_dup 1) (match_dup 2))
2098 addsubpd\t{%2, %0|%0, %2}
2099 vaddsubpd\t{%2, %1, %0|%0, %1, %2}"
2100 [(set_attr "isa" "noavx,avx")
2101 (set_attr "type" "sseadd")
2102 (set_attr "atom_unit" "complex")
2103 (set_attr "prefix" "orig,vex")
2104 (set_attr "mode" "V2DF")])
2106 (define_insn "avx_addsubv8sf3"
2107 [(set (match_operand:V8SF 0 "register_operand" "=x")
2110 (match_operand:V8SF 1 "register_operand" "x")
2111 (match_operand:V8SF 2 "nonimmediate_operand" "xm"))
2112 (plus:V8SF (match_dup 1) (match_dup 2))
2115 "vaddsubps\t{%2, %1, %0|%0, %1, %2}"
2116 [(set_attr "type" "sseadd")
2117 (set_attr "prefix" "vex")
2118 (set_attr "mode" "V8SF")])
2120 (define_insn "sse3_addsubv4sf3"
2121 [(set (match_operand:V4SF 0 "register_operand" "=x,x")
2124 (match_operand:V4SF 1 "register_operand" "0,x")
2125 (match_operand:V4SF 2 "vector_operand" "xBm,xm"))
2126 (plus:V4SF (match_dup 1) (match_dup 2))
2130 addsubps\t{%2, %0|%0, %2}
2131 vaddsubps\t{%2, %1, %0|%0, %1, %2}"
2132 [(set_attr "isa" "noavx,avx")
2133 (set_attr "type" "sseadd")
2134 (set_attr "prefix" "orig,vex")
2135 (set_attr "prefix_rep" "1,*")
2136 (set_attr "mode" "V4SF")])
2139 [(set (match_operand:VF_128_256 0 "register_operand")
2140 (match_operator:VF_128_256 6 "addsub_vm_operator"
2142 (match_operand:VF_128_256 1 "register_operand")
2143 (match_operand:VF_128_256 2 "vector_operand"))
2145 (match_operand:VF_128_256 3 "vector_operand")
2146 (match_operand:VF_128_256 4 "vector_operand"))
2147 (match_operand 5 "const_int_operand")]))]
2149 && can_create_pseudo_p ()
2150 && ((rtx_equal_p (operands[1], operands[3])
2151 && rtx_equal_p (operands[2], operands[4]))
2152 || (rtx_equal_p (operands[1], operands[4])
2153 && rtx_equal_p (operands[2], operands[3])))"
2155 (vec_merge:VF_128_256
2156 (minus:VF_128_256 (match_dup 1) (match_dup 2))
2157 (plus:VF_128_256 (match_dup 1) (match_dup 2))
2161 [(set (match_operand:VF_128_256 0 "register_operand")
2162 (match_operator:VF_128_256 6 "addsub_vm_operator"
2164 (match_operand:VF_128_256 1 "vector_operand")
2165 (match_operand:VF_128_256 2 "vector_operand"))
2167 (match_operand:VF_128_256 3 "register_operand")
2168 (match_operand:VF_128_256 4 "vector_operand"))
2169 (match_operand 5 "const_int_operand")]))]
2171 && can_create_pseudo_p ()
2172 && ((rtx_equal_p (operands[1], operands[3])
2173 && rtx_equal_p (operands[2], operands[4]))
2174 || (rtx_equal_p (operands[1], operands[4])
2175 && rtx_equal_p (operands[2], operands[3])))"
2177 (vec_merge:VF_128_256
2178 (minus:VF_128_256 (match_dup 3) (match_dup 4))
2179 (plus:VF_128_256 (match_dup 3) (match_dup 4))
2182 /* Negate mask bits to compensate for swapped PLUS and MINUS RTXes. */
2184 = GEN_INT (~INTVAL (operands[5])
2185 & ((HOST_WIDE_INT_1U << GET_MODE_NUNITS (<MODE>mode)) - 1));
2189 [(set (match_operand:VF_128_256 0 "register_operand")
2190 (match_operator:VF_128_256 7 "addsub_vs_operator"
2191 [(vec_concat:<ssedoublemode>
2193 (match_operand:VF_128_256 1 "register_operand")
2194 (match_operand:VF_128_256 2 "vector_operand"))
2196 (match_operand:VF_128_256 3 "vector_operand")
2197 (match_operand:VF_128_256 4 "vector_operand")))
2198 (match_parallel 5 "addsub_vs_parallel"
2199 [(match_operand 6 "const_int_operand")])]))]
2201 && can_create_pseudo_p ()
2202 && ((rtx_equal_p (operands[1], operands[3])
2203 && rtx_equal_p (operands[2], operands[4]))
2204 || (rtx_equal_p (operands[1], operands[4])
2205 && rtx_equal_p (operands[2], operands[3])))"
2207 (vec_merge:VF_128_256
2208 (minus:VF_128_256 (match_dup 1) (match_dup 2))
2209 (plus:VF_128_256 (match_dup 1) (match_dup 2))
2212 int i, nelt = XVECLEN (operands[5], 0);
2213 HOST_WIDE_INT ival = 0;
2215 for (i = 0; i < nelt; i++)
2216 if (INTVAL (XVECEXP (operands[5], 0, i)) < GET_MODE_NUNITS (<MODE>mode))
2217 ival |= HOST_WIDE_INT_1 << i;
2219 operands[5] = GEN_INT (ival);
2223 [(set (match_operand:VF_128_256 0 "register_operand")
2224 (match_operator:VF_128_256 7 "addsub_vs_operator"
2225 [(vec_concat:<ssedoublemode>
2227 (match_operand:VF_128_256 1 "vector_operand")
2228 (match_operand:VF_128_256 2 "vector_operand"))
2230 (match_operand:VF_128_256 3 "register_operand")
2231 (match_operand:VF_128_256 4 "vector_operand")))
2232 (match_parallel 5 "addsub_vs_parallel"
2233 [(match_operand 6 "const_int_operand")])]))]
2235 && can_create_pseudo_p ()
2236 && ((rtx_equal_p (operands[1], operands[3])
2237 && rtx_equal_p (operands[2], operands[4]))
2238 || (rtx_equal_p (operands[1], operands[4])
2239 && rtx_equal_p (operands[2], operands[3])))"
2241 (vec_merge:VF_128_256
2242 (minus:VF_128_256 (match_dup 3) (match_dup 4))
2243 (plus:VF_128_256 (match_dup 3) (match_dup 4))
2246 int i, nelt = XVECLEN (operands[5], 0);
2247 HOST_WIDE_INT ival = 0;
2249 for (i = 0; i < nelt; i++)
2250 if (INTVAL (XVECEXP (operands[5], 0, i)) >= GET_MODE_NUNITS (<MODE>mode))
2251 ival |= HOST_WIDE_INT_1 << i;
2253 operands[5] = GEN_INT (ival);
2256 (define_insn "avx_h<plusminus_insn>v4df3"
2257 [(set (match_operand:V4DF 0 "register_operand" "=x")
2262 (match_operand:V4DF 1 "register_operand" "x")
2263 (parallel [(const_int 0)]))
2264 (vec_select:DF (match_dup 1) (parallel [(const_int 1)])))
2267 (match_operand:V4DF 2 "nonimmediate_operand" "xm")
2268 (parallel [(const_int 0)]))
2269 (vec_select:DF (match_dup 2) (parallel [(const_int 1)]))))
2272 (vec_select:DF (match_dup 1) (parallel [(const_int 2)]))
2273 (vec_select:DF (match_dup 1) (parallel [(const_int 3)])))
2275 (vec_select:DF (match_dup 2) (parallel [(const_int 2)]))
2276 (vec_select:DF (match_dup 2) (parallel [(const_int 3)]))))))]
2278 "vh<plusminus_mnemonic>pd\t{%2, %1, %0|%0, %1, %2}"
2279 [(set_attr "type" "sseadd")
2280 (set_attr "prefix" "vex")
2281 (set_attr "mode" "V4DF")])
2283 (define_expand "sse3_haddv2df3"
2284 [(set (match_operand:V2DF 0 "register_operand")
2288 (match_operand:V2DF 1 "register_operand")
2289 (parallel [(const_int 0)]))
2290 (vec_select:DF (match_dup 1) (parallel [(const_int 1)])))
2293 (match_operand:V2DF 2 "vector_operand")
2294 (parallel [(const_int 0)]))
2295 (vec_select:DF (match_dup 2) (parallel [(const_int 1)])))))]
2298 (define_insn "*sse3_haddv2df3"
2299 [(set (match_operand:V2DF 0 "register_operand" "=x,x")
2303 (match_operand:V2DF 1 "register_operand" "0,x")
2304 (parallel [(match_operand:SI 3 "const_0_to_1_operand")]))
2307 (parallel [(match_operand:SI 4 "const_0_to_1_operand")])))
2310 (match_operand:V2DF 2 "vector_operand" "xBm,xm")
2311 (parallel [(match_operand:SI 5 "const_0_to_1_operand")]))
2314 (parallel [(match_operand:SI 6 "const_0_to_1_operand")])))))]
2316 && INTVAL (operands[3]) != INTVAL (operands[4])
2317 && INTVAL (operands[5]) != INTVAL (operands[6])"
2319 haddpd\t{%2, %0|%0, %2}
2320 vhaddpd\t{%2, %1, %0|%0, %1, %2}"
2321 [(set_attr "isa" "noavx,avx")
2322 (set_attr "type" "sseadd")
2323 (set_attr "prefix" "orig,vex")
2324 (set_attr "mode" "V2DF")])
2326 (define_insn "sse3_hsubv2df3"
2327 [(set (match_operand:V2DF 0 "register_operand" "=x,x")
2331 (match_operand:V2DF 1 "register_operand" "0,x")
2332 (parallel [(const_int 0)]))
2333 (vec_select:DF (match_dup 1) (parallel [(const_int 1)])))
2336 (match_operand:V2DF 2 "vector_operand" "xBm,xm")
2337 (parallel [(const_int 0)]))
2338 (vec_select:DF (match_dup 2) (parallel [(const_int 1)])))))]
2341 hsubpd\t{%2, %0|%0, %2}
2342 vhsubpd\t{%2, %1, %0|%0, %1, %2}"
2343 [(set_attr "isa" "noavx,avx")
2344 (set_attr "type" "sseadd")
2345 (set_attr "prefix" "orig,vex")
2346 (set_attr "mode" "V2DF")])
2348 (define_insn "*sse3_haddv2df3_low"
2349 [(set (match_operand:DF 0 "register_operand" "=x,x")
2352 (match_operand:V2DF 1 "register_operand" "0,x")
2353 (parallel [(match_operand:SI 2 "const_0_to_1_operand")]))
2356 (parallel [(match_operand:SI 3 "const_0_to_1_operand")]))))]
2358 && INTVAL (operands[2]) != INTVAL (operands[3])"
2360 haddpd\t{%0, %0|%0, %0}
2361 vhaddpd\t{%1, %1, %0|%0, %1, %1}"
2362 [(set_attr "isa" "noavx,avx")
2363 (set_attr "type" "sseadd1")
2364 (set_attr "prefix" "orig,vex")
2365 (set_attr "mode" "V2DF")])
2367 (define_insn "*sse3_hsubv2df3_low"
2368 [(set (match_operand:DF 0 "register_operand" "=x,x")
2371 (match_operand:V2DF 1 "register_operand" "0,x")
2372 (parallel [(const_int 0)]))
2375 (parallel [(const_int 1)]))))]
2378 hsubpd\t{%0, %0|%0, %0}
2379 vhsubpd\t{%1, %1, %0|%0, %1, %1}"
2380 [(set_attr "isa" "noavx,avx")
2381 (set_attr "type" "sseadd1")
2382 (set_attr "prefix" "orig,vex")
2383 (set_attr "mode" "V2DF")])
2385 (define_insn "avx_h<plusminus_insn>v8sf3"
2386 [(set (match_operand:V8SF 0 "register_operand" "=x")
2392 (match_operand:V8SF 1 "register_operand" "x")
2393 (parallel [(const_int 0)]))
2394 (vec_select:SF (match_dup 1) (parallel [(const_int 1)])))
2396 (vec_select:SF (match_dup 1) (parallel [(const_int 2)]))
2397 (vec_select:SF (match_dup 1) (parallel [(const_int 3)]))))
2401 (match_operand:V8SF 2 "nonimmediate_operand" "xm")
2402 (parallel [(const_int 0)]))
2403 (vec_select:SF (match_dup 2) (parallel [(const_int 1)])))
2405 (vec_select:SF (match_dup 2) (parallel [(const_int 2)]))
2406 (vec_select:SF (match_dup 2) (parallel [(const_int 3)])))))
2410 (vec_select:SF (match_dup 1) (parallel [(const_int 4)]))
2411 (vec_select:SF (match_dup 1) (parallel [(const_int 5)])))
2413 (vec_select:SF (match_dup 1) (parallel [(const_int 6)]))
2414 (vec_select:SF (match_dup 1) (parallel [(const_int 7)]))))
2417 (vec_select:SF (match_dup 2) (parallel [(const_int 4)]))
2418 (vec_select:SF (match_dup 2) (parallel [(const_int 5)])))
2420 (vec_select:SF (match_dup 2) (parallel [(const_int 6)]))
2421 (vec_select:SF (match_dup 2) (parallel [(const_int 7)])))))))]
2423 "vh<plusminus_mnemonic>ps\t{%2, %1, %0|%0, %1, %2}"
2424 [(set_attr "type" "sseadd")
2425 (set_attr "prefix" "vex")
2426 (set_attr "mode" "V8SF")])
2428 (define_insn "sse3_h<plusminus_insn>v4sf3"
2429 [(set (match_operand:V4SF 0 "register_operand" "=x,x")
2434 (match_operand:V4SF 1 "register_operand" "0,x")
2435 (parallel [(const_int 0)]))
2436 (vec_select:SF (match_dup 1) (parallel [(const_int 1)])))
2438 (vec_select:SF (match_dup 1) (parallel [(const_int 2)]))
2439 (vec_select:SF (match_dup 1) (parallel [(const_int 3)]))))
2443 (match_operand:V4SF 2 "vector_operand" "xBm,xm")
2444 (parallel [(const_int 0)]))
2445 (vec_select:SF (match_dup 2) (parallel [(const_int 1)])))
2447 (vec_select:SF (match_dup 2) (parallel [(const_int 2)]))
2448 (vec_select:SF (match_dup 2) (parallel [(const_int 3)]))))))]
2451 h<plusminus_mnemonic>ps\t{%2, %0|%0, %2}
2452 vh<plusminus_mnemonic>ps\t{%2, %1, %0|%0, %1, %2}"
2453 [(set_attr "isa" "noavx,avx")
2454 (set_attr "type" "sseadd")
2455 (set_attr "atom_unit" "complex")
2456 (set_attr "prefix" "orig,vex")
2457 (set_attr "prefix_rep" "1,*")
2458 (set_attr "mode" "V4SF")])
2460 (define_expand "reduc_plus_scal_v8df"
2461 [(match_operand:DF 0 "register_operand")
2462 (match_operand:V8DF 1 "register_operand")]
2465 rtx tmp = gen_reg_rtx (V8DFmode);
2466 ix86_expand_reduc (gen_addv8df3, tmp, operands[1]);
2467 emit_insn (gen_vec_extractv8dfdf (operands[0], tmp, const0_rtx));
2471 (define_expand "reduc_plus_scal_v4df"
2472 [(match_operand:DF 0 "register_operand")
2473 (match_operand:V4DF 1 "register_operand")]
2476 rtx tmp = gen_reg_rtx (V4DFmode);
2477 rtx tmp2 = gen_reg_rtx (V4DFmode);
2478 rtx vec_res = gen_reg_rtx (V4DFmode);
2479 emit_insn (gen_avx_haddv4df3 (tmp, operands[1], operands[1]));
2480 emit_insn (gen_avx_vperm2f128v4df3 (tmp2, tmp, tmp, GEN_INT (1)));
2481 emit_insn (gen_addv4df3 (vec_res, tmp, tmp2));
2482 emit_insn (gen_vec_extractv4dfdf (operands[0], vec_res, const0_rtx));
2486 (define_expand "reduc_plus_scal_v2df"
2487 [(match_operand:DF 0 "register_operand")
2488 (match_operand:V2DF 1 "register_operand")]
2491 rtx tmp = gen_reg_rtx (V2DFmode);
2492 emit_insn (gen_sse3_haddv2df3 (tmp, operands[1], operands[1]));
2493 emit_insn (gen_vec_extractv2dfdf (operands[0], tmp, const0_rtx));
2497 (define_expand "reduc_plus_scal_v16sf"
2498 [(match_operand:SF 0 "register_operand")
2499 (match_operand:V16SF 1 "register_operand")]
2502 rtx tmp = gen_reg_rtx (V16SFmode);
2503 ix86_expand_reduc (gen_addv16sf3, tmp, operands[1]);
2504 emit_insn (gen_vec_extractv16sfsf (operands[0], tmp, const0_rtx));
2508 (define_expand "reduc_plus_scal_v8sf"
2509 [(match_operand:SF 0 "register_operand")
2510 (match_operand:V8SF 1 "register_operand")]
2513 rtx tmp = gen_reg_rtx (V8SFmode);
2514 rtx tmp2 = gen_reg_rtx (V8SFmode);
2515 rtx vec_res = gen_reg_rtx (V8SFmode);
2516 emit_insn (gen_avx_haddv8sf3 (tmp, operands[1], operands[1]));
2517 emit_insn (gen_avx_haddv8sf3 (tmp2, tmp, tmp));
2518 emit_insn (gen_avx_vperm2f128v8sf3 (tmp, tmp2, tmp2, GEN_INT (1)));
2519 emit_insn (gen_addv8sf3 (vec_res, tmp, tmp2));
2520 emit_insn (gen_vec_extractv8sfsf (operands[0], vec_res, const0_rtx));
2524 (define_expand "reduc_plus_scal_v4sf"
2525 [(match_operand:SF 0 "register_operand")
2526 (match_operand:V4SF 1 "register_operand")]
2529 rtx vec_res = gen_reg_rtx (V4SFmode);
2532 rtx tmp = gen_reg_rtx (V4SFmode);
2533 emit_insn (gen_sse3_haddv4sf3 (tmp, operands[1], operands[1]));
2534 emit_insn (gen_sse3_haddv4sf3 (vec_res, tmp, tmp));
2537 ix86_expand_reduc (gen_addv4sf3, vec_res, operands[1]);
2538 emit_insn (gen_vec_extractv4sfsf (operands[0], vec_res, const0_rtx));
2542 ;; Modes handled by reduc_sm{in,ax}* patterns.
2543 (define_mode_iterator REDUC_SMINMAX_MODE
2544 [(V32QI "TARGET_AVX2") (V16HI "TARGET_AVX2")
2545 (V8SI "TARGET_AVX2") (V4DI "TARGET_AVX2")
2546 (V8SF "TARGET_AVX") (V4DF "TARGET_AVX")
2547 (V4SF "TARGET_SSE") (V64QI "TARGET_AVX512BW")
2548 (V32HI "TARGET_AVX512BW") (V16SI "TARGET_AVX512F")
2549 (V8DI "TARGET_AVX512F") (V16SF "TARGET_AVX512F")
2550 (V8DF "TARGET_AVX512F")])
2552 (define_expand "reduc_<code>_scal_<mode>"
2553 [(smaxmin:REDUC_SMINMAX_MODE
2554 (match_operand:<ssescalarmode> 0 "register_operand")
2555 (match_operand:REDUC_SMINMAX_MODE 1 "register_operand"))]
2558 rtx tmp = gen_reg_rtx (<MODE>mode);
2559 ix86_expand_reduc (gen_<code><mode>3, tmp, operands[1]);
2560 emit_insn (gen_vec_extract<mode><ssescalarmodelower> (operands[0], tmp,
2565 (define_expand "reduc_<code>_scal_<mode>"
2566 [(umaxmin:VI_AVX512BW
2567 (match_operand:<ssescalarmode> 0 "register_operand")
2568 (match_operand:VI_AVX512BW 1 "register_operand"))]
2571 rtx tmp = gen_reg_rtx (<MODE>mode);
2572 ix86_expand_reduc (gen_<code><mode>3, tmp, operands[1]);
2573 emit_insn (gen_vec_extract<mode><ssescalarmodelower> (operands[0], tmp,
2578 (define_expand "reduc_<code>_scal_<mode>"
2580 (match_operand:<ssescalarmode> 0 "register_operand")
2581 (match_operand:VI_256 1 "register_operand"))]
2584 rtx tmp = gen_reg_rtx (<MODE>mode);
2585 ix86_expand_reduc (gen_<code><mode>3, tmp, operands[1]);
2586 emit_insn (gen_vec_extract<mode><ssescalarmodelower> (operands[0], tmp,
2591 (define_expand "reduc_umin_scal_v8hi"
2593 (match_operand:HI 0 "register_operand")
2594 (match_operand:V8HI 1 "register_operand"))]
2597 rtx tmp = gen_reg_rtx (V8HImode);
2598 ix86_expand_reduc (gen_uminv8hi3, tmp, operands[1]);
2599 emit_insn (gen_vec_extractv8hihi (operands[0], tmp, const0_rtx));
2603 (define_insn "<mask_codefor>reducep<mode><mask_name>"
2604 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v")
2606 [(match_operand:VF_AVX512VL 1 "nonimmediate_operand" "vm")
2607 (match_operand:SI 2 "const_0_to_255_operand")]
2610 "vreduce<ssemodesuffix>\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
2611 [(set_attr "type" "sse")
2612 (set_attr "prefix" "evex")
2613 (set_attr "mode" "<MODE>")])
2615 (define_insn "reduces<mode><mask_scalar_name>"
2616 [(set (match_operand:VF_128 0 "register_operand" "=v")
2619 [(match_operand:VF_128 1 "register_operand" "v")
2620 (match_operand:VF_128 2 "nonimmediate_operand" "vm")
2621 (match_operand:SI 3 "const_0_to_255_operand")]
2626 "vreduce<ssescalarmodesuffix>\t{%3, %2, %1, %0<mask_scalar_operand4>|%0<mask_scalar_operand4>, %1, %2, %3}"
2627 [(set_attr "type" "sse")
2628 (set_attr "prefix" "evex")
2629 (set_attr "mode" "<MODE>")])
2631 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
2633 ;; Parallel floating point comparisons
2635 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
2637 (define_insn "avx_cmp<mode>3"
2638 [(set (match_operand:VF_128_256 0 "register_operand" "=x")
2640 [(match_operand:VF_128_256 1 "register_operand" "x")
2641 (match_operand:VF_128_256 2 "nonimmediate_operand" "xm")
2642 (match_operand:SI 3 "const_0_to_31_operand" "n")]
2645 "vcmp<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
2646 [(set_attr "type" "ssecmp")
2647 (set_attr "length_immediate" "1")
2648 (set_attr "prefix" "vex")
2649 (set_attr "mode" "<MODE>")])
2651 (define_insn "avx_vmcmp<mode>3"
2652 [(set (match_operand:VF_128 0 "register_operand" "=x")
2655 [(match_operand:VF_128 1 "register_operand" "x")
2656 (match_operand:VF_128 2 "nonimmediate_operand" "xm")
2657 (match_operand:SI 3 "const_0_to_31_operand" "n")]
2662 "vcmp<ssescalarmodesuffix>\t{%3, %2, %1, %0|%0, %1, %<iptr>2, %3}"
2663 [(set_attr "type" "ssecmp")
2664 (set_attr "length_immediate" "1")
2665 (set_attr "prefix" "vex")
2666 (set_attr "mode" "<ssescalarmode>")])
2668 (define_insn "*<sse>_maskcmp<mode>3_comm"
2669 [(set (match_operand:VF_128_256 0 "register_operand" "=x,x")
2670 (match_operator:VF_128_256 3 "sse_comparison_operator"
2671 [(match_operand:VF_128_256 1 "register_operand" "%0,x")
2672 (match_operand:VF_128_256 2 "vector_operand" "xBm,xm")]))]
2674 && GET_RTX_CLASS (GET_CODE (operands[3])) == RTX_COMM_COMPARE"
2676 cmp%D3<ssemodesuffix>\t{%2, %0|%0, %2}
2677 vcmp%D3<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
2678 [(set_attr "isa" "noavx,avx")
2679 (set_attr "type" "ssecmp")
2680 (set_attr "length_immediate" "1")
2681 (set_attr "prefix" "orig,vex")
2682 (set_attr "mode" "<MODE>")])
2684 (define_insn "<sse>_maskcmp<mode>3"
2685 [(set (match_operand:VF_128_256 0 "register_operand" "=x,x")
2686 (match_operator:VF_128_256 3 "sse_comparison_operator"
2687 [(match_operand:VF_128_256 1 "register_operand" "0,x")
2688 (match_operand:VF_128_256 2 "vector_operand" "xBm,xm")]))]
2691 cmp%D3<ssemodesuffix>\t{%2, %0|%0, %2}
2692 vcmp%D3<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
2693 [(set_attr "isa" "noavx,avx")
2694 (set_attr "type" "ssecmp")
2695 (set_attr "length_immediate" "1")
2696 (set_attr "prefix" "orig,vex")
2697 (set_attr "mode" "<MODE>")])
2699 (define_insn "<sse>_vmmaskcmp<mode>3"
2700 [(set (match_operand:VF_128 0 "register_operand" "=x,x")
2702 (match_operator:VF_128 3 "sse_comparison_operator"
2703 [(match_operand:VF_128 1 "register_operand" "0,x")
2704 (match_operand:VF_128 2 "vector_operand" "xBm,xm")])
2709 cmp%D3<ssescalarmodesuffix>\t{%2, %0|%0, %<iptr>2}
2710 vcmp%D3<ssescalarmodesuffix>\t{%2, %1, %0|%0, %1, %<iptr>2}"
2711 [(set_attr "isa" "noavx,avx")
2712 (set_attr "type" "ssecmp")
2713 (set_attr "length_immediate" "1,*")
2714 (set_attr "prefix" "orig,vex")
2715 (set_attr "mode" "<ssescalarmode>")])
2717 (define_mode_attr cmp_imm_predicate
2718 [(V16SF "const_0_to_31_operand") (V8DF "const_0_to_31_operand")
2719 (V16SI "const_0_to_7_operand") (V8DI "const_0_to_7_operand")
2720 (V8SF "const_0_to_31_operand") (V4DF "const_0_to_31_operand")
2721 (V8SI "const_0_to_7_operand") (V4DI "const_0_to_7_operand")
2722 (V4SF "const_0_to_31_operand") (V2DF "const_0_to_31_operand")
2723 (V4SI "const_0_to_7_operand") (V2DI "const_0_to_7_operand")
2724 (V32HI "const_0_to_7_operand") (V64QI "const_0_to_7_operand")
2725 (V16HI "const_0_to_7_operand") (V32QI "const_0_to_7_operand")
2726 (V8HI "const_0_to_7_operand") (V16QI "const_0_to_7_operand")])
2728 (define_insn "<avx512>_cmp<mode>3<mask_scalar_merge_name><round_saeonly_name>"
2729 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=Yk")
2730 (unspec:<avx512fmaskmode>
2731 [(match_operand:V48_AVX512VL 1 "register_operand" "v")
2732 (match_operand:V48_AVX512VL 2 "nonimmediate_operand" "<round_saeonly_constraint>")
2733 (match_operand:SI 3 "<cmp_imm_predicate>" "n")]
2735 "TARGET_AVX512F && <round_saeonly_mode512bit_condition>"
2736 "v<sseintprefix>cmp<ssemodesuffix>\t{%3, <round_saeonly_mask_scalar_merge_op4>%2, %1, %0<mask_scalar_merge_operand4>|%0<mask_scalar_merge_operand4>, %1, %2<round_saeonly_mask_scalar_merge_op4>, %3}"
2737 [(set_attr "type" "ssecmp")
2738 (set_attr "length_immediate" "1")
2739 (set_attr "prefix" "evex")
2740 (set_attr "mode" "<sseinsnmode>")])
2742 (define_insn "<avx512>_cmp<mode>3<mask_scalar_merge_name>"
2743 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=Yk")
2744 (unspec:<avx512fmaskmode>
2745 [(match_operand:VI12_AVX512VL 1 "register_operand" "v")
2746 (match_operand:VI12_AVX512VL 2 "nonimmediate_operand" "vm")
2747 (match_operand:SI 3 "<cmp_imm_predicate>" "n")]
2750 "vpcmp<ssemodesuffix>\t{%3, %2, %1, %0<mask_scalar_merge_operand4>|%0<mask_scalar_merge_operand4>, %1, %2, %3}"
2751 [(set_attr "type" "ssecmp")
2752 (set_attr "length_immediate" "1")
2753 (set_attr "prefix" "evex")
2754 (set_attr "mode" "<sseinsnmode>")])
2756 (define_insn "<avx512>_ucmp<mode>3<mask_scalar_merge_name>"
2757 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=Yk")
2758 (unspec:<avx512fmaskmode>
2759 [(match_operand:VI12_AVX512VL 1 "register_operand" "v")
2760 (match_operand:VI12_AVX512VL 2 "nonimmediate_operand" "vm")
2761 (match_operand:SI 3 "const_0_to_7_operand" "n")]
2762 UNSPEC_UNSIGNED_PCMP))]
2764 "vpcmpu<ssemodesuffix>\t{%3, %2, %1, %0<mask_scalar_merge_operand4>|%0<mask_scalar_merge_operand4>, %1, %2, %3}"
2765 [(set_attr "type" "ssecmp")
2766 (set_attr "length_immediate" "1")
2767 (set_attr "prefix" "evex")
2768 (set_attr "mode" "<sseinsnmode>")])
2770 (define_insn "<avx512>_ucmp<mode>3<mask_scalar_merge_name>"
2771 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=Yk")
2772 (unspec:<avx512fmaskmode>
2773 [(match_operand:VI48_AVX512VL 1 "register_operand" "v")
2774 (match_operand:VI48_AVX512VL 2 "nonimmediate_operand" "vm")
2775 (match_operand:SI 3 "const_0_to_7_operand" "n")]
2776 UNSPEC_UNSIGNED_PCMP))]
2778 "vpcmpu<ssemodesuffix>\t{%3, %2, %1, %0<mask_scalar_merge_operand4>|%0<mask_scalar_merge_operand4>, %1, %2, %3}"
2779 [(set_attr "type" "ssecmp")
2780 (set_attr "length_immediate" "1")
2781 (set_attr "prefix" "evex")
2782 (set_attr "mode" "<sseinsnmode>")])
2784 (define_insn "avx512f_vmcmp<mode>3<round_saeonly_name>"
2785 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=Yk")
2786 (and:<avx512fmaskmode>
2787 (unspec:<avx512fmaskmode>
2788 [(match_operand:VF_128 1 "register_operand" "v")
2789 (match_operand:VF_128 2 "<round_saeonly_nimm_predicate>" "<round_saeonly_constraint>")
2790 (match_operand:SI 3 "const_0_to_31_operand" "n")]
2794 "vcmp<ssescalarmodesuffix>\t{%3, <round_saeonly_op4>%2, %1, %0|%0, %1, %2<round_saeonly_op4>, %3}"
2795 [(set_attr "type" "ssecmp")
2796 (set_attr "length_immediate" "1")
2797 (set_attr "prefix" "evex")
2798 (set_attr "mode" "<ssescalarmode>")])
2800 (define_insn "avx512f_vmcmp<mode>3_mask<round_saeonly_name>"
2801 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=Yk")
2802 (and:<avx512fmaskmode>
2803 (unspec:<avx512fmaskmode>
2804 [(match_operand:VF_128 1 "register_operand" "v")
2805 (match_operand:VF_128 2 "<round_saeonly_nimm_predicate>" "<round_saeonly_constraint>")
2806 (match_operand:SI 3 "const_0_to_31_operand" "n")]
2808 (and:<avx512fmaskmode>
2809 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk")
2812 "vcmp<ssescalarmodesuffix>\t{%3, <round_saeonly_op5>%2, %1, %0%{%4%}|%0%{%4%}, %1, %2<round_saeonly_op5>, %3}"
2813 [(set_attr "type" "ssecmp")
2814 (set_attr "length_immediate" "1")
2815 (set_attr "prefix" "evex")
2816 (set_attr "mode" "<ssescalarmode>")])
2818 (define_insn "avx512f_maskcmp<mode>3"
2819 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=Yk")
2820 (match_operator:<avx512fmaskmode> 3 "sse_comparison_operator"
2821 [(match_operand:VF 1 "register_operand" "v")
2822 (match_operand:VF 2 "nonimmediate_operand" "vm")]))]
2824 "vcmp%D3<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
2825 [(set_attr "type" "ssecmp")
2826 (set_attr "length_immediate" "1")
2827 (set_attr "prefix" "evex")
2828 (set_attr "mode" "<sseinsnmode>")])
2830 (define_insn "<sse>_<unord>comi<round_saeonly_name>"
2831 [(set (reg:CCFP FLAGS_REG)
2834 (match_operand:<ssevecmode> 0 "register_operand" "v")
2835 (parallel [(const_int 0)]))
2837 (match_operand:<ssevecmode> 1 "<round_saeonly_nimm_scalar_predicate>" "<round_saeonly_constraint>")
2838 (parallel [(const_int 0)]))))]
2839 "SSE_FLOAT_MODE_P (<MODE>mode)"
2840 "%v<unord>comi<ssemodesuffix>\t{<round_saeonly_op2>%1, %0|%0, %<iptr>1<round_saeonly_op2>}"
2841 [(set_attr "type" "ssecomi")
2842 (set_attr "prefix" "maybe_vex")
2843 (set_attr "prefix_rep" "0")
2844 (set (attr "prefix_data16")
2845 (if_then_else (eq_attr "mode" "DF")
2847 (const_string "0")))
2848 (set_attr "mode" "<MODE>")])
2850 (define_expand "vec_cmp<mode><avx512fmaskmodelower>"
2851 [(set (match_operand:<avx512fmaskmode> 0 "register_operand")
2852 (match_operator:<avx512fmaskmode> 1 ""
2853 [(match_operand:V48_AVX512VL 2 "register_operand")
2854 (match_operand:V48_AVX512VL 3 "nonimmediate_operand")]))]
2857 bool ok = ix86_expand_mask_vec_cmp (operands);
2862 (define_expand "vec_cmp<mode><avx512fmaskmodelower>"
2863 [(set (match_operand:<avx512fmaskmode> 0 "register_operand")
2864 (match_operator:<avx512fmaskmode> 1 ""
2865 [(match_operand:VI12_AVX512VL 2 "register_operand")
2866 (match_operand:VI12_AVX512VL 3 "nonimmediate_operand")]))]
2869 bool ok = ix86_expand_mask_vec_cmp (operands);
2874 (define_expand "vec_cmp<mode><sseintvecmodelower>"
2875 [(set (match_operand:<sseintvecmode> 0 "register_operand")
2876 (match_operator:<sseintvecmode> 1 ""
2877 [(match_operand:VI_256 2 "register_operand")
2878 (match_operand:VI_256 3 "nonimmediate_operand")]))]
2881 bool ok = ix86_expand_int_vec_cmp (operands);
2886 (define_expand "vec_cmp<mode><sseintvecmodelower>"
2887 [(set (match_operand:<sseintvecmode> 0 "register_operand")
2888 (match_operator:<sseintvecmode> 1 ""
2889 [(match_operand:VI124_128 2 "register_operand")
2890 (match_operand:VI124_128 3 "vector_operand")]))]
2893 bool ok = ix86_expand_int_vec_cmp (operands);
2898 (define_expand "vec_cmpv2div2di"
2899 [(set (match_operand:V2DI 0 "register_operand")
2900 (match_operator:V2DI 1 ""
2901 [(match_operand:V2DI 2 "register_operand")
2902 (match_operand:V2DI 3 "vector_operand")]))]
2905 bool ok = ix86_expand_int_vec_cmp (operands);
2910 (define_expand "vec_cmp<mode><sseintvecmodelower>"
2911 [(set (match_operand:<sseintvecmode> 0 "register_operand")
2912 (match_operator:<sseintvecmode> 1 ""
2913 [(match_operand:VF_256 2 "register_operand")
2914 (match_operand:VF_256 3 "nonimmediate_operand")]))]
2917 bool ok = ix86_expand_fp_vec_cmp (operands);
2922 (define_expand "vec_cmp<mode><sseintvecmodelower>"
2923 [(set (match_operand:<sseintvecmode> 0 "register_operand")
2924 (match_operator:<sseintvecmode> 1 ""
2925 [(match_operand:VF_128 2 "register_operand")
2926 (match_operand:VF_128 3 "vector_operand")]))]
2929 bool ok = ix86_expand_fp_vec_cmp (operands);
2934 (define_expand "vec_cmpu<mode><avx512fmaskmodelower>"
2935 [(set (match_operand:<avx512fmaskmode> 0 "register_operand")
2936 (match_operator:<avx512fmaskmode> 1 ""
2937 [(match_operand:VI48_AVX512VL 2 "register_operand")
2938 (match_operand:VI48_AVX512VL 3 "nonimmediate_operand")]))]
2941 bool ok = ix86_expand_mask_vec_cmp (operands);
2946 (define_expand "vec_cmpu<mode><avx512fmaskmodelower>"
2947 [(set (match_operand:<avx512fmaskmode> 0 "register_operand")
2948 (match_operator:<avx512fmaskmode> 1 ""
2949 [(match_operand:VI12_AVX512VL 2 "register_operand")
2950 (match_operand:VI12_AVX512VL 3 "nonimmediate_operand")]))]
2953 bool ok = ix86_expand_mask_vec_cmp (operands);
2958 (define_expand "vec_cmpu<mode><sseintvecmodelower>"
2959 [(set (match_operand:<sseintvecmode> 0 "register_operand")
2960 (match_operator:<sseintvecmode> 1 ""
2961 [(match_operand:VI_256 2 "register_operand")
2962 (match_operand:VI_256 3 "nonimmediate_operand")]))]
2965 bool ok = ix86_expand_int_vec_cmp (operands);
2970 (define_expand "vec_cmpu<mode><sseintvecmodelower>"
2971 [(set (match_operand:<sseintvecmode> 0 "register_operand")
2972 (match_operator:<sseintvecmode> 1 ""
2973 [(match_operand:VI124_128 2 "register_operand")
2974 (match_operand:VI124_128 3 "vector_operand")]))]
2977 bool ok = ix86_expand_int_vec_cmp (operands);
2982 (define_expand "vec_cmpuv2div2di"
2983 [(set (match_operand:V2DI 0 "register_operand")
2984 (match_operator:V2DI 1 ""
2985 [(match_operand:V2DI 2 "register_operand")
2986 (match_operand:V2DI 3 "vector_operand")]))]
2989 bool ok = ix86_expand_int_vec_cmp (operands);
2994 (define_expand "vec_cmpeqv2div2di"
2995 [(set (match_operand:V2DI 0 "register_operand")
2996 (match_operator:V2DI 1 ""
2997 [(match_operand:V2DI 2 "register_operand")
2998 (match_operand:V2DI 3 "vector_operand")]))]
3001 bool ok = ix86_expand_int_vec_cmp (operands);
3006 (define_expand "vcond<V_512:mode><VF_512:mode>"
3007 [(set (match_operand:V_512 0 "register_operand")
3009 (match_operator 3 ""
3010 [(match_operand:VF_512 4 "nonimmediate_operand")
3011 (match_operand:VF_512 5 "nonimmediate_operand")])
3012 (match_operand:V_512 1 "general_operand")
3013 (match_operand:V_512 2 "general_operand")))]
3015 && (GET_MODE_NUNITS (<V_512:MODE>mode)
3016 == GET_MODE_NUNITS (<VF_512:MODE>mode))"
3018 bool ok = ix86_expand_fp_vcond (operands);
3023 (define_expand "vcond<V_256:mode><VF_256:mode>"
3024 [(set (match_operand:V_256 0 "register_operand")
3026 (match_operator 3 ""
3027 [(match_operand:VF_256 4 "nonimmediate_operand")
3028 (match_operand:VF_256 5 "nonimmediate_operand")])
3029 (match_operand:V_256 1 "general_operand")
3030 (match_operand:V_256 2 "general_operand")))]
3032 && (GET_MODE_NUNITS (<V_256:MODE>mode)
3033 == GET_MODE_NUNITS (<VF_256:MODE>mode))"
3035 bool ok = ix86_expand_fp_vcond (operands);
3040 (define_expand "vcond<V_128:mode><VF_128:mode>"
3041 [(set (match_operand:V_128 0 "register_operand")
3043 (match_operator 3 ""
3044 [(match_operand:VF_128 4 "vector_operand")
3045 (match_operand:VF_128 5 "vector_operand")])
3046 (match_operand:V_128 1 "general_operand")
3047 (match_operand:V_128 2 "general_operand")))]
3049 && (GET_MODE_NUNITS (<V_128:MODE>mode)
3050 == GET_MODE_NUNITS (<VF_128:MODE>mode))"
3052 bool ok = ix86_expand_fp_vcond (operands);
3057 (define_expand "vcond_mask_<mode><avx512fmaskmodelower>"
3058 [(set (match_operand:V48_AVX512VL 0 "register_operand")
3059 (vec_merge:V48_AVX512VL
3060 (match_operand:V48_AVX512VL 1 "nonimmediate_operand")
3061 (match_operand:V48_AVX512VL 2 "vector_move_operand")
3062 (match_operand:<avx512fmaskmode> 3 "register_operand")))]
3065 (define_expand "vcond_mask_<mode><avx512fmaskmodelower>"
3066 [(set (match_operand:VI12_AVX512VL 0 "register_operand")
3067 (vec_merge:VI12_AVX512VL
3068 (match_operand:VI12_AVX512VL 1 "nonimmediate_operand")
3069 (match_operand:VI12_AVX512VL 2 "vector_move_operand")
3070 (match_operand:<avx512fmaskmode> 3 "register_operand")))]
3073 (define_expand "vcond_mask_<mode><sseintvecmodelower>"
3074 [(set (match_operand:VI_256 0 "register_operand")
3076 (match_operand:VI_256 1 "nonimmediate_operand")
3077 (match_operand:VI_256 2 "vector_move_operand")
3078 (match_operand:<sseintvecmode> 3 "register_operand")))]
3081 ix86_expand_sse_movcc (operands[0], operands[3],
3082 operands[1], operands[2]);
3086 (define_expand "vcond_mask_<mode><sseintvecmodelower>"
3087 [(set (match_operand:VI124_128 0 "register_operand")
3088 (vec_merge:VI124_128
3089 (match_operand:VI124_128 1 "vector_operand")
3090 (match_operand:VI124_128 2 "vector_move_operand")
3091 (match_operand:<sseintvecmode> 3 "register_operand")))]
3094 ix86_expand_sse_movcc (operands[0], operands[3],
3095 operands[1], operands[2]);
3099 (define_expand "vcond_mask_v2div2di"
3100 [(set (match_operand:V2DI 0 "register_operand")
3102 (match_operand:V2DI 1 "vector_operand")
3103 (match_operand:V2DI 2 "vector_move_operand")
3104 (match_operand:V2DI 3 "register_operand")))]
3107 ix86_expand_sse_movcc (operands[0], operands[3],
3108 operands[1], operands[2]);
3112 (define_expand "vcond_mask_<mode><sseintvecmodelower>"
3113 [(set (match_operand:VF_256 0 "register_operand")
3115 (match_operand:VF_256 1 "nonimmediate_operand")
3116 (match_operand:VF_256 2 "vector_move_operand")
3117 (match_operand:<sseintvecmode> 3 "register_operand")))]
3120 ix86_expand_sse_movcc (operands[0], operands[3],
3121 operands[1], operands[2]);
3125 (define_expand "vcond_mask_<mode><sseintvecmodelower>"
3126 [(set (match_operand:VF_128 0 "register_operand")
3128 (match_operand:VF_128 1 "vector_operand")
3129 (match_operand:VF_128 2 "vector_move_operand")
3130 (match_operand:<sseintvecmode> 3 "register_operand")))]
3133 ix86_expand_sse_movcc (operands[0], operands[3],
3134 operands[1], operands[2]);
3138 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
3140 ;; Parallel floating point logical operations
3142 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
3144 (define_insn "<sse>_andnot<mode>3<mask_name>"
3145 [(set (match_operand:VF_128_256 0 "register_operand" "=x,x,v,v")
3148 (match_operand:VF_128_256 1 "register_operand" "0,x,v,v"))
3149 (match_operand:VF_128_256 2 "vector_operand" "xBm,xm,vm,vm")))]
3150 "TARGET_SSE && <mask_avx512vl_condition>"
3152 static char buf[128];
3156 switch (which_alternative)
3159 ops = "andn%s\t{%%2, %%0|%%0, %%2}";
3164 ops = "vandn%s\t{%%2, %%1, %%0<mask_operand3_1>|%%0<mask_operand3_1>, %%1, %%2}";
3170 switch (get_attr_mode (insn))
3178 /* There is no vandnp[sd] in avx512f. Use vpandn[qd]. */
3179 suffix = GET_MODE_INNER (<MODE>mode) == DFmode ? "q" : "d";
3180 ops = "vpandn%s\t{%%2, %%1, %%0<mask_operand3_1>|%%0<mask_operand3_1>, %%1, %%2}";
3183 suffix = "<ssemodesuffix>";
3186 snprintf (buf, sizeof (buf), ops, suffix);
3189 [(set_attr "isa" "noavx,avx,avx512dq,avx512f")
3190 (set_attr "type" "sselog")
3191 (set_attr "prefix" "orig,maybe_vex,evex,evex")
3193 (cond [(and (match_test "<mask_applied>")
3194 (and (eq_attr "alternative" "1")
3195 (match_test "!TARGET_AVX512DQ")))
3196 (const_string "<sseintvecmode2>")
3197 (eq_attr "alternative" "3")
3198 (const_string "<sseintvecmode2>")
3199 (and (match_test "<MODE_SIZE> == 16")
3200 (match_test "TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL"))
3201 (const_string "<ssePSmode>")
3202 (match_test "TARGET_AVX")
3203 (const_string "<MODE>")
3204 (match_test "optimize_function_for_size_p (cfun)")
3205 (const_string "V4SF")
3207 (const_string "<MODE>")))])
3210 (define_insn "<sse>_andnot<mode>3<mask_name>"
3211 [(set (match_operand:VF_512 0 "register_operand" "=v")
3214 (match_operand:VF_512 1 "register_operand" "v"))
3215 (match_operand:VF_512 2 "nonimmediate_operand" "vm")))]
3218 static char buf[128];
3222 suffix = "<ssemodesuffix>";
3225 /* There is no vandnp[sd] in avx512f. Use vpandn[qd]. */
3226 if (!TARGET_AVX512DQ)
3228 suffix = GET_MODE_INNER (<MODE>mode) == DFmode ? "q" : "d";
3232 snprintf (buf, sizeof (buf),
3233 "v%sandn%s\t{%%2, %%1, %%0<mask_operand3_1>|%%0<mask_operand3_1>, %%1, %%2}",
3237 [(set_attr "type" "sselog")
3238 (set_attr "prefix" "evex")
3240 (if_then_else (match_test "TARGET_AVX512DQ")
3241 (const_string "<sseinsnmode>")
3242 (const_string "XI")))])
3244 (define_expand "<code><mode>3<mask_name>"
3245 [(set (match_operand:VF_128_256 0 "register_operand")
3246 (any_logic:VF_128_256
3247 (match_operand:VF_128_256 1 "vector_operand")
3248 (match_operand:VF_128_256 2 "vector_operand")))]
3249 "TARGET_SSE && <mask_avx512vl_condition>"
3250 "ix86_fixup_binary_operands_no_copy (<CODE>, <MODE>mode, operands);")
3252 (define_expand "<code><mode>3<mask_name>"
3253 [(set (match_operand:VF_512 0 "register_operand")
3255 (match_operand:VF_512 1 "nonimmediate_operand")
3256 (match_operand:VF_512 2 "nonimmediate_operand")))]
3258 "ix86_fixup_binary_operands_no_copy (<CODE>, <MODE>mode, operands);")
3260 (define_insn "*<code><mode>3<mask_name>"
3261 [(set (match_operand:VF_128_256 0 "register_operand" "=x,x,v,v")
3262 (any_logic:VF_128_256
3263 (match_operand:VF_128_256 1 "vector_operand" "%0,x,v,v")
3264 (match_operand:VF_128_256 2 "vector_operand" "xBm,xm,vm,vm")))]
3265 "TARGET_SSE && <mask_avx512vl_condition>
3266 && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
3268 static char buf[128];
3272 switch (which_alternative)
3275 ops = "<logic>%s\t{%%2, %%0|%%0, %%2}";
3280 ops = "v<logic>%s\t{%%2, %%1, %%0<mask_operand3_1>|%%0<mask_operand3_1>, %%1, %%2}";
3286 switch (get_attr_mode (insn))
3294 /* There is no v<logic>p[sd] in avx512f. Use vp<logic>[qd]. */
3295 suffix = GET_MODE_INNER (<MODE>mode) == DFmode ? "q" : "d";
3296 ops = "vp<logic>%s\t{%%2, %%1, %%0<mask_operand3_1>|%%0<mask_operand3_1>, %%1, %%2}";
3299 suffix = "<ssemodesuffix>";
3302 snprintf (buf, sizeof (buf), ops, suffix);
3305 [(set_attr "isa" "noavx,avx,avx512dq,avx512f")
3306 (set_attr "type" "sselog")
3307 (set_attr "prefix" "orig,maybe_evex,evex,evex")
3309 (cond [(and (match_test "<mask_applied>")
3310 (and (eq_attr "alternative" "1")
3311 (match_test "!TARGET_AVX512DQ")))
3312 (const_string "<sseintvecmode2>")
3313 (eq_attr "alternative" "3")
3314 (const_string "<sseintvecmode2>")
3315 (and (match_test "<MODE_SIZE> == 16")
3316 (match_test "TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL"))
3317 (const_string "<ssePSmode>")
3318 (match_test "TARGET_AVX")
3319 (const_string "<MODE>")
3320 (match_test "optimize_function_for_size_p (cfun)")
3321 (const_string "V4SF")
3323 (const_string "<MODE>")))])
3325 (define_insn "*<code><mode>3<mask_name>"
3326 [(set (match_operand:VF_512 0 "register_operand" "=v")
3328 (match_operand:VF_512 1 "nonimmediate_operand" "%v")
3329 (match_operand:VF_512 2 "nonimmediate_operand" "vm")))]
3330 "TARGET_AVX512F && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
3332 static char buf[128];
3336 suffix = "<ssemodesuffix>";
3339 /* There is no v<logic>p[sd] in avx512f. Use vp<logic>[dq]. */
3340 if (!TARGET_AVX512DQ)
3342 suffix = GET_MODE_INNER (<MODE>mode) == DFmode ? "q" : "d";
3346 snprintf (buf, sizeof (buf),
3347 "v%s<logic>%s\t{%%2, %%1, %%0<mask_operand3_1>|%%0<mask_operand3_1>, %%1, %%2}",
3351 [(set_attr "type" "sselog")
3352 (set_attr "prefix" "evex")
3354 (if_then_else (match_test "TARGET_AVX512DQ")
3355 (const_string "<sseinsnmode>")
3356 (const_string "XI")))])
3358 (define_expand "copysign<mode>3"
3361 (not:VF (match_dup 3))
3362 (match_operand:VF 1 "vector_operand")))
3364 (and:VF (match_dup 3)
3365 (match_operand:VF 2 "vector_operand")))
3366 (set (match_operand:VF 0 "register_operand")
3367 (ior:VF (match_dup 4) (match_dup 5)))]
3370 operands[3] = ix86_build_signbit_mask (<MODE>mode, 1, 0);
3372 operands[4] = gen_reg_rtx (<MODE>mode);
3373 operands[5] = gen_reg_rtx (<MODE>mode);
3376 ;; Also define scalar versions. These are used for abs, neg, and
3377 ;; conditional move. Using subregs into vector modes causes register
3378 ;; allocation lossage. These patterns do not allow memory operands
3379 ;; because the native instructions read the full 128-bits.
3381 (define_insn "*andnot<mode>3"
3382 [(set (match_operand:MODEF 0 "register_operand" "=x,x,v,v")
3385 (match_operand:MODEF 1 "register_operand" "0,x,v,v"))
3386 (match_operand:MODEF 2 "register_operand" "x,x,v,v")))]
3387 "SSE_FLOAT_MODE_P (<MODE>mode)"
3389 static char buf[128];
3392 = (get_attr_mode (insn) == MODE_V4SF) ? "ps" : "<ssevecmodesuffix>";
3394 switch (which_alternative)
3397 ops = "andn%s\t{%%2, %%0|%%0, %%2}";
3400 ops = "vandn%s\t{%%2, %%1, %%0|%%0, %%1, %%2}";
3403 if (TARGET_AVX512DQ)
3404 ops = "vandn%s\t{%%2, %%1, %%0|%%0, %%1, %%2}";
3407 suffix = <MODE>mode == DFmode ? "q" : "d";
3408 ops = "vpandn%s\t{%%2, %%1, %%0|%%0, %%1, %%2}";
3412 if (TARGET_AVX512DQ)
3413 ops = "vandn%s\t{%%g2, %%g1, %%g0|%%g0, %%g1, %%g2}";
3416 suffix = <MODE>mode == DFmode ? "q" : "d";
3417 ops = "vpandn%s\t{%%g2, %%g1, %%g0|%%g0, %%g1, %%g2}";
3424 snprintf (buf, sizeof (buf), ops, suffix);
3427 [(set_attr "isa" "noavx,avx,avx512vl,avx512f")
3428 (set_attr "type" "sselog")
3429 (set_attr "prefix" "orig,vex,evex,evex")
3431 (cond [(eq_attr "alternative" "2")
3432 (if_then_else (match_test "TARGET_AVX512DQ")
3433 (const_string "<ssevecmode>")
3434 (const_string "TI"))
3435 (eq_attr "alternative" "3")
3436 (if_then_else (match_test "TARGET_AVX512DQ")
3437 (const_string "<avx512fvecmode>")
3438 (const_string "XI"))
3439 (and (match_test "<MODE_SIZE> == 16")
3440 (match_test "TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL"))
3441 (const_string "V4SF")
3442 (match_test "TARGET_AVX")
3443 (const_string "<ssevecmode>")
3444 (match_test "optimize_function_for_size_p (cfun)")
3445 (const_string "V4SF")
3447 (const_string "<ssevecmode>")))])
3449 (define_insn "*andnottf3"
3450 [(set (match_operand:TF 0 "register_operand" "=x,x,v,v")
3452 (not:TF (match_operand:TF 1 "register_operand" "0,x,v,v"))
3453 (match_operand:TF 2 "vector_operand" "xBm,xm,vm,v")))]
3456 static char buf[128];
3459 = (which_alternative >= 2 ? "pandnq"
3460 : get_attr_mode (insn) == MODE_V4SF ? "andnps" : "pandn");
3462 switch (which_alternative)
3465 ops = "%s\t{%%2, %%0|%%0, %%2}";
3469 ops = "v%s\t{%%2, %%1, %%0|%%0, %%1, %%2}";
3472 ops = "v%s\t{%%g2, %%g1, %%g0|%%g0, %%g1, %%g2}";
3478 snprintf (buf, sizeof (buf), ops, tmp);
3481 [(set_attr "isa" "noavx,avx,avx512vl,avx512f")
3482 (set_attr "type" "sselog")
3483 (set (attr "prefix_data16")
3485 (and (eq_attr "alternative" "0")
3486 (eq_attr "mode" "TI"))
3488 (const_string "*")))
3489 (set_attr "prefix" "orig,vex,evex,evex")
3491 (cond [(eq_attr "alternative" "2")
3493 (eq_attr "alternative" "3")
3495 (match_test "TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL")
3496 (const_string "V4SF")
3497 (match_test "TARGET_AVX")
3499 (ior (not (match_test "TARGET_SSE2"))
3500 (match_test "optimize_function_for_size_p (cfun)"))
3501 (const_string "V4SF")
3503 (const_string "TI")))])
3505 (define_insn "*<code><mode>3"
3506 [(set (match_operand:MODEF 0 "register_operand" "=x,x,v,v")
3508 (match_operand:MODEF 1 "register_operand" "%0,x,v,v")
3509 (match_operand:MODEF 2 "register_operand" "x,x,v,v")))]
3510 "SSE_FLOAT_MODE_P (<MODE>mode)"
3512 static char buf[128];
3515 = (get_attr_mode (insn) == MODE_V4SF) ? "ps" : "<ssevecmodesuffix>";
3517 switch (which_alternative)
3520 ops = "<logic>%s\t{%%2, %%0|%%0, %%2}";
3523 if (!TARGET_AVX512DQ)
3525 suffix = <MODE>mode == DFmode ? "q" : "d";
3526 ops = "vp<logic>%s\t{%%2, %%1, %%0|%%0, %%1, %%2}";
3531 ops = "v<logic>%s\t{%%2, %%1, %%0|%%0, %%1, %%2}";
3534 if (TARGET_AVX512DQ)
3535 ops = "v<logic>%s\t{%%g2, %%g1, %%g0|%%g0, %%g1, %%g2}";
3538 suffix = <MODE>mode == DFmode ? "q" : "d";
3539 ops = "vp<logic>%s\t{%%g2, %%g1, %%g0|%%g0, %%g1, %%g2}";
3546 snprintf (buf, sizeof (buf), ops, suffix);
3549 [(set_attr "isa" "noavx,avx,avx512vl,avx512f")
3550 (set_attr "type" "sselog")
3551 (set_attr "prefix" "orig,vex,evex,evex")
3553 (cond [(eq_attr "alternative" "2")
3554 (if_then_else (match_test "TARGET_AVX512DQ")
3555 (const_string "<ssevecmode>")
3556 (const_string "TI"))
3557 (eq_attr "alternative" "3")
3558 (if_then_else (match_test "TARGET_AVX512DQ")
3559 (const_string "<avx512fvecmode>")
3560 (const_string "XI"))
3561 (and (match_test "<MODE_SIZE> == 16")
3562 (match_test "TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL"))
3563 (const_string "V4SF")
3564 (match_test "TARGET_AVX")
3565 (const_string "<ssevecmode>")
3566 (match_test "optimize_function_for_size_p (cfun)")
3567 (const_string "V4SF")
3569 (const_string "<ssevecmode>")))])
3571 (define_expand "<code>tf3"
3572 [(set (match_operand:TF 0 "register_operand")
3574 (match_operand:TF 1 "vector_operand")
3575 (match_operand:TF 2 "vector_operand")))]
3577 "ix86_fixup_binary_operands_no_copy (<CODE>, TFmode, operands);")
3579 (define_insn "*<code>tf3"
3580 [(set (match_operand:TF 0 "register_operand" "=x,x,v,v")
3582 (match_operand:TF 1 "vector_operand" "%0,x,v,v")
3583 (match_operand:TF 2 "vector_operand" "xBm,xm,vm,v")))]
3584 "TARGET_SSE && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
3586 static char buf[128];
3589 = (which_alternative >= 2 ? "p<logic>q"
3590 : get_attr_mode (insn) == MODE_V4SF ? "<logic>ps" : "p<logic>");
3592 switch (which_alternative)
3595 ops = "%s\t{%%2, %%0|%%0, %%2}";
3599 ops = "v%s\t{%%2, %%1, %%0|%%0, %%1, %%2}";
3602 ops = "v%s\t{%%g2, %%g1, %%g0|%%g0, %%g1, %%g2}";
3608 snprintf (buf, sizeof (buf), ops, tmp);
3611 [(set_attr "isa" "noavx,avx,avx512vl,avx512f")
3612 (set_attr "type" "sselog")
3613 (set (attr "prefix_data16")
3615 (and (eq_attr "alternative" "0")
3616 (eq_attr "mode" "TI"))
3618 (const_string "*")))
3619 (set_attr "prefix" "orig,vex,evex,evex")
3621 (cond [(eq_attr "alternative" "2")
3623 (eq_attr "alternative" "3")
3625 (match_test "TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL")
3626 (const_string "V4SF")
3627 (match_test "TARGET_AVX")
3629 (ior (not (match_test "TARGET_SSE2"))
3630 (match_test "optimize_function_for_size_p (cfun)"))
3631 (const_string "V4SF")
3633 (const_string "TI")))])
3635 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
3637 ;; FMA floating point multiply/accumulate instructions. These include
3638 ;; scalar versions of the instructions as well as vector versions.
3640 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
3642 ;; The standard names for scalar FMA are only available with SSE math enabled.
3643 ;; CPUID bit AVX512F enables evex encoded scalar and 512-bit fma. It doesn't
3644 ;; care about FMA bit, so we enable fma for TARGET_AVX512F even when TARGET_FMA
3645 ;; and TARGET_FMA4 are both false.
3646 ;; TODO: In theory AVX512F does not automatically imply FMA, and without FMA
3647 ;; one must force the EVEX encoding of the fma insns. Ideally we'd improve
3648 ;; GAS to allow proper prefix selection. However, for the moment all hardware
3649 ;; that supports AVX512F also supports FMA so we can ignore this for now.
3650 (define_mode_iterator FMAMODEM
3651 [(SF "TARGET_SSE_MATH && (TARGET_FMA || TARGET_FMA4 || TARGET_AVX512F)")
3652 (DF "TARGET_SSE_MATH && (TARGET_FMA || TARGET_FMA4 || TARGET_AVX512F)")
3653 (V4SF "TARGET_FMA || TARGET_FMA4 || TARGET_AVX512VL")
3654 (V2DF "TARGET_FMA || TARGET_FMA4 || TARGET_AVX512VL")
3655 (V8SF "TARGET_FMA || TARGET_FMA4 || TARGET_AVX512VL")
3656 (V4DF "TARGET_FMA || TARGET_FMA4 || TARGET_AVX512VL")
3657 (V16SF "TARGET_AVX512F")
3658 (V8DF "TARGET_AVX512F")])
3660 (define_expand "fma<mode>4"
3661 [(set (match_operand:FMAMODEM 0 "register_operand")
3663 (match_operand:FMAMODEM 1 "nonimmediate_operand")
3664 (match_operand:FMAMODEM 2 "nonimmediate_operand")
3665 (match_operand:FMAMODEM 3 "nonimmediate_operand")))])
3667 (define_expand "fms<mode>4"
3668 [(set (match_operand:FMAMODEM 0 "register_operand")
3670 (match_operand:FMAMODEM 1 "nonimmediate_operand")
3671 (match_operand:FMAMODEM 2 "nonimmediate_operand")
3672 (neg:FMAMODEM (match_operand:FMAMODEM 3 "nonimmediate_operand"))))])
3674 (define_expand "fnma<mode>4"
3675 [(set (match_operand:FMAMODEM 0 "register_operand")
3677 (neg:FMAMODEM (match_operand:FMAMODEM 1 "nonimmediate_operand"))
3678 (match_operand:FMAMODEM 2 "nonimmediate_operand")
3679 (match_operand:FMAMODEM 3 "nonimmediate_operand")))])
3681 (define_expand "fnms<mode>4"
3682 [(set (match_operand:FMAMODEM 0 "register_operand")
3684 (neg:FMAMODEM (match_operand:FMAMODEM 1 "nonimmediate_operand"))
3685 (match_operand:FMAMODEM 2 "nonimmediate_operand")
3686 (neg:FMAMODEM (match_operand:FMAMODEM 3 "nonimmediate_operand"))))])
3688 ;; The builtins for intrinsics are not constrained by SSE math enabled.
3689 (define_mode_iterator FMAMODE_AVX512
3690 [(SF "TARGET_FMA || TARGET_FMA4 || TARGET_AVX512F")
3691 (DF "TARGET_FMA || TARGET_FMA4 || TARGET_AVX512F")
3692 (V4SF "TARGET_FMA || TARGET_FMA4 || TARGET_AVX512VL")
3693 (V2DF "TARGET_FMA || TARGET_FMA4 || TARGET_AVX512VL")
3694 (V8SF "TARGET_FMA || TARGET_FMA4 || TARGET_AVX512VL")
3695 (V4DF "TARGET_FMA || TARGET_FMA4 || TARGET_AVX512VL")
3696 (V16SF "TARGET_AVX512F")
3697 (V8DF "TARGET_AVX512F")])
3699 (define_mode_iterator FMAMODE
3700 [SF DF V4SF V2DF V8SF V4DF])
3702 (define_expand "fma4i_fmadd_<mode>"
3703 [(set (match_operand:FMAMODE_AVX512 0 "register_operand")
3705 (match_operand:FMAMODE_AVX512 1 "nonimmediate_operand")
3706 (match_operand:FMAMODE_AVX512 2 "nonimmediate_operand")
3707 (match_operand:FMAMODE_AVX512 3 "nonimmediate_operand")))])
3709 (define_expand "<avx512>_fmadd_<mode>_maskz<round_expand_name>"
3710 [(match_operand:VF_AVX512VL 0 "register_operand")
3711 (match_operand:VF_AVX512VL 1 "<round_expand_nimm_predicate>")
3712 (match_operand:VF_AVX512VL 2 "<round_expand_nimm_predicate>")
3713 (match_operand:VF_AVX512VL 3 "<round_expand_nimm_predicate>")
3714 (match_operand:<avx512fmaskmode> 4 "register_operand")]
3715 "TARGET_AVX512F && <round_mode512bit_condition>"
3717 emit_insn (gen_fma_fmadd_<mode>_maskz_1<round_expand_name> (
3718 operands[0], operands[1], operands[2], operands[3],
3719 CONST0_RTX (<MODE>mode), operands[4]<round_expand_operand>));
3723 (define_insn "*fma_fmadd_<mode>"
3724 [(set (match_operand:FMAMODE 0 "register_operand" "=v,v,v,x,x")
3726 (match_operand:FMAMODE 1 "nonimmediate_operand" "%0,0,v,x,x")
3727 (match_operand:FMAMODE 2 "nonimmediate_operand" "vm,v,vm,x,m")
3728 (match_operand:FMAMODE 3 "nonimmediate_operand" "v,vm,0,xm,x")))]
3729 "TARGET_FMA || TARGET_FMA4"
3731 vfmadd132<ssemodesuffix>\t{%2, %3, %0|%0, %3, %2}
3732 vfmadd213<ssemodesuffix>\t{%3, %2, %0|%0, %2, %3}
3733 vfmadd231<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}
3734 vfmadd<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}
3735 vfmadd<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
3736 [(set_attr "isa" "fma,fma,fma,fma4,fma4")
3737 (set_attr "type" "ssemuladd")
3738 (set_attr "mode" "<MODE>")])
3740 ;; Suppose AVX-512F as baseline
3741 (define_mode_iterator VF_SF_AVX512VL
3742 [SF V16SF (V8SF "TARGET_AVX512VL") (V4SF "TARGET_AVX512VL")
3743 DF V8DF (V4DF "TARGET_AVX512VL") (V2DF "TARGET_AVX512VL")])
3745 (define_insn "<sd_mask_codefor>fma_fmadd_<mode><sd_maskz_name><round_name>"
3746 [(set (match_operand:VF_SF_AVX512VL 0 "register_operand" "=v,v,v")
3748 (match_operand:VF_SF_AVX512VL 1 "<round_nimm_predicate>" "%0,0,v")
3749 (match_operand:VF_SF_AVX512VL 2 "<round_nimm_predicate>" "<round_constraint>,v,<round_constraint>")
3750 (match_operand:VF_SF_AVX512VL 3 "<round_nimm_predicate>" "v,<round_constraint>,0")))]
3751 "TARGET_AVX512F && <sd_mask_mode512bit_condition> && <round_mode512bit_condition>"
3753 vfmadd132<ssemodesuffix>\t{<round_sd_mask_op4>%2, %3, %0<sd_mask_op4>|%0<sd_mask_op4>, %3, %2<round_sd_mask_op4>}
3754 vfmadd213<ssemodesuffix>\t{<round_sd_mask_op4>%3, %2, %0<sd_mask_op4>|%0<sd_mask_op4>, %2, %3<round_sd_mask_op4>}
3755 vfmadd231<ssemodesuffix>\t{<round_sd_mask_op4>%2, %1, %0<sd_mask_op4>|%0<sd_mask_op4>, %1, %2<round_sd_mask_op4>}"
3756 [(set_attr "type" "ssemuladd")
3757 (set_attr "mode" "<MODE>")])
3759 (define_insn "<avx512>_fmadd_<mode>_mask<round_name>"
3760 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v,v")
3761 (vec_merge:VF_AVX512VL
3763 (match_operand:VF_AVX512VL 1 "register_operand" "0,0")
3764 (match_operand:VF_AVX512VL 2 "nonimmediate_operand" "<round_constraint>,v")
3765 (match_operand:VF_AVX512VL 3 "nonimmediate_operand" "v,<round_constraint>"))
3767 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk,Yk")))]
3768 "TARGET_AVX512F && <round_mode512bit_condition>"
3770 vfmadd132<ssemodesuffix>\t{<round_op5>%2, %3, %0%{%4%}|%0%{%4%}, %3, %2<round_op5>}
3771 vfmadd213<ssemodesuffix>\t{<round_op5>%3, %2, %0%{%4%}|%0%{%4%}, %2, %3<round_op5>}"
3772 [(set_attr "type" "ssemuladd")
3773 (set_attr "mode" "<MODE>")])
3775 (define_insn "<avx512>_fmadd_<mode>_mask3<round_name>"
3776 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v")
3777 (vec_merge:VF_AVX512VL
3779 (match_operand:VF_AVX512VL 1 "register_operand" "v")
3780 (match_operand:VF_AVX512VL 2 "nonimmediate_operand" "<round_constraint>")
3781 (match_operand:VF_AVX512VL 3 "register_operand" "0"))
3783 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk")))]
3785 "vfmadd231<ssemodesuffix>\t{<round_op5>%2, %1, %0%{%4%}|%0%{%4%}, %1, %2<round_op5>}"
3786 [(set_attr "type" "ssemuladd")
3787 (set_attr "mode" "<MODE>")])
3789 (define_insn "*fma_fmsub_<mode>"
3790 [(set (match_operand:FMAMODE 0 "register_operand" "=v,v,v,x,x")
3792 (match_operand:FMAMODE 1 "nonimmediate_operand" "%0,0,v,x,x")
3793 (match_operand:FMAMODE 2 "nonimmediate_operand" "vm,v,vm,x,m")
3795 (match_operand:FMAMODE 3 "nonimmediate_operand" "v,vm,0,xm,x"))))]
3796 "TARGET_FMA || TARGET_FMA4"
3798 vfmsub132<ssemodesuffix>\t{%2, %3, %0|%0, %3, %2}
3799 vfmsub213<ssemodesuffix>\t{%3, %2, %0|%0, %2, %3}
3800 vfmsub231<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}
3801 vfmsub<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}
3802 vfmsub<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
3803 [(set_attr "isa" "fma,fma,fma,fma4,fma4")
3804 (set_attr "type" "ssemuladd")
3805 (set_attr "mode" "<MODE>")])
3807 (define_insn "<sd_mask_codefor>fma_fmsub_<mode><sd_maskz_name><round_name>"
3808 [(set (match_operand:VF_SF_AVX512VL 0 "register_operand" "=v,v,v")
3810 (match_operand:VF_SF_AVX512VL 1 "<round_nimm_predicate>" "%0,0,v")
3811 (match_operand:VF_SF_AVX512VL 2 "<round_nimm_predicate>" "<round_constraint>,v,<round_constraint>")
3813 (match_operand:VF_SF_AVX512VL 3 "<round_nimm_predicate>" "v,<round_constraint>,0"))))]
3814 "TARGET_AVX512F && <sd_mask_mode512bit_condition> && <round_mode512bit_condition>"
3816 vfmsub132<ssemodesuffix>\t{<round_sd_mask_op4>%2, %3, %0<sd_mask_op4>|%0<sd_mask_op4>, %3, %2<round_sd_mask_op4>}
3817 vfmsub213<ssemodesuffix>\t{<round_sd_mask_op4>%3, %2, %0<sd_mask_op4>|%0<sd_mask_op4>, %2, %3<round_sd_mask_op4>}
3818 vfmsub231<ssemodesuffix>\t{<round_sd_mask_op4>%2, %1, %0<sd_mask_op4>|%0<sd_mask_op4>, %1, %2<round_sd_mask_op4>}"
3819 [(set_attr "type" "ssemuladd")
3820 (set_attr "mode" "<MODE>")])
3822 (define_insn "<avx512>_fmsub_<mode>_mask<round_name>"
3823 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v,v")
3824 (vec_merge:VF_AVX512VL
3826 (match_operand:VF_AVX512VL 1 "register_operand" "0,0")
3827 (match_operand:VF_AVX512VL 2 "nonimmediate_operand" "<round_constraint>,v")
3829 (match_operand:VF_AVX512VL 3 "nonimmediate_operand" "v,<round_constraint>")))
3831 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk,Yk")))]
3834 vfmsub132<ssemodesuffix>\t{<round_op5>%2, %3, %0%{%4%}|%0%{%4%}, %3, %2<round_op5>}
3835 vfmsub213<ssemodesuffix>\t{<round_op5>%3, %2, %0%{%4%}|%0%{%4%}, %2, %3<round_op5>}"
3836 [(set_attr "type" "ssemuladd")
3837 (set_attr "mode" "<MODE>")])
3839 (define_insn "<avx512>_fmsub_<mode>_mask3<round_name>"
3840 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v")
3841 (vec_merge:VF_AVX512VL
3843 (match_operand:VF_AVX512VL 1 "register_operand" "v")
3844 (match_operand:VF_AVX512VL 2 "nonimmediate_operand" "<round_constraint>")
3846 (match_operand:VF_AVX512VL 3 "register_operand" "0")))
3848 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk")))]
3849 "TARGET_AVX512F && <round_mode512bit_condition>"
3850 "vfmsub231<ssemodesuffix>\t{<round_op5>%2, %1, %0%{%4%}|%0%{%4%}, %1, %2<round_op5>}"
3851 [(set_attr "type" "ssemuladd")
3852 (set_attr "mode" "<MODE>")])
3854 (define_insn "*fma_fnmadd_<mode>"
3855 [(set (match_operand:FMAMODE 0 "register_operand" "=v,v,v,x,x")
3858 (match_operand:FMAMODE 1 "nonimmediate_operand" "%0,0,v,x,x"))
3859 (match_operand:FMAMODE 2 "nonimmediate_operand" "vm,v,vm,x,m")
3860 (match_operand:FMAMODE 3 "nonimmediate_operand" "v,vm,0,xm,x")))]
3861 "TARGET_FMA || TARGET_FMA4"
3863 vfnmadd132<ssemodesuffix>\t{%2, %3, %0|%0, %3, %2}
3864 vfnmadd213<ssemodesuffix>\t{%3, %2, %0|%0, %2, %3}
3865 vfnmadd231<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}
3866 vfnmadd<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}
3867 vfnmadd<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
3868 [(set_attr "isa" "fma,fma,fma,fma4,fma4")
3869 (set_attr "type" "ssemuladd")
3870 (set_attr "mode" "<MODE>")])
3872 (define_insn "<sd_mask_codefor>fma_fnmadd_<mode><sd_maskz_name><round_name>"
3873 [(set (match_operand:VF_SF_AVX512VL 0 "register_operand" "=v,v,v")
3876 (match_operand:VF_SF_AVX512VL 1 "<round_nimm_predicate>" "%0,0,v"))
3877 (match_operand:VF_SF_AVX512VL 2 "<round_nimm_predicate>" "<round_constraint>,v,<round_constraint>")
3878 (match_operand:VF_SF_AVX512VL 3 "<round_nimm_predicate>" "v,<round_constraint>,0")))]
3879 "TARGET_AVX512F && <sd_mask_mode512bit_condition> && <round_mode512bit_condition>"
3881 vfnmadd132<ssemodesuffix>\t{<round_sd_mask_op4>%2, %3, %0<sd_mask_op4>|%0<sd_mask_op4>, %3, %2<round_sd_mask_op4>}
3882 vfnmadd213<ssemodesuffix>\t{<round_sd_mask_op4>%3, %2, %0<sd_mask_op4>|%0<sd_mask_op4>, %2, %3<round_sd_mask_op4>}
3883 vfnmadd231<ssemodesuffix>\t{<round_sd_mask_op4>%2, %1, %0<sd_mask_op4>|%0<sd_mask_op4>, %1, %2<round_sd_mask_op4>}"
3884 [(set_attr "type" "ssemuladd")
3885 (set_attr "mode" "<MODE>")])
3887 (define_insn "<avx512>_fnmadd_<mode>_mask<round_name>"
3888 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v,v")
3889 (vec_merge:VF_AVX512VL
3892 (match_operand:VF_AVX512VL 1 "register_operand" "0,0"))
3893 (match_operand:VF_AVX512VL 2 "nonimmediate_operand" "<round_constraint>,v")
3894 (match_operand:VF_AVX512VL 3 "nonimmediate_operand" "v,<round_constraint>"))
3896 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk,Yk")))]
3897 "TARGET_AVX512F && <round_mode512bit_condition>"
3899 vfnmadd132<ssemodesuffix>\t{<round_op5>%2, %3, %0%{%4%}|%0%{%4%}, %3, %2<round_op5>}
3900 vfnmadd213<ssemodesuffix>\t{<round_op5>%3, %2, %0%{%4%}|%0%{%4%}, %2, %3<round_op5>}"
3901 [(set_attr "type" "ssemuladd")
3902 (set_attr "mode" "<MODE>")])
3904 (define_insn "<avx512>_fnmadd_<mode>_mask3<round_name>"
3905 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v")
3906 (vec_merge:VF_AVX512VL
3909 (match_operand:VF_AVX512VL 1 "register_operand" "v"))
3910 (match_operand:VF_AVX512VL 2 "nonimmediate_operand" "<round_constraint>")
3911 (match_operand:VF_AVX512VL 3 "register_operand" "0"))
3913 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk")))]
3914 "TARGET_AVX512F && <round_mode512bit_condition>"
3915 "vfnmadd231<ssemodesuffix>\t{<round_op5>%2, %1, %0%{%4%}|%0%{%4%}, %1, %2<round_op5>}"
3916 [(set_attr "type" "ssemuladd")
3917 (set_attr "mode" "<MODE>")])
3919 (define_insn "*fma_fnmsub_<mode>"
3920 [(set (match_operand:FMAMODE 0 "register_operand" "=v,v,v,x,x")
3923 (match_operand:FMAMODE 1 "nonimmediate_operand" "%0,0,v,x,x"))
3924 (match_operand:FMAMODE 2 "nonimmediate_operand" "vm,v,vm,x,m")
3926 (match_operand:FMAMODE 3 "nonimmediate_operand" "v,vm,0,xm,x"))))]
3927 "TARGET_FMA || TARGET_FMA4"
3929 vfnmsub132<ssemodesuffix>\t{<round_sd_mask_op4>%2, %3, %0<sd_mask_op4>|%0<sd_mask_op4>, %3, %2<round_sd_mask_op4>}
3930 vfnmsub213<ssemodesuffix>\t{<round_sd_mask_op4>%3, %2, %0<sd_mask_op4>|%0<sd_mask_op4>, %2, %3<round_sd_mask_op4>}
3931 vfnmsub231<ssemodesuffix>\t{<round_sd_mask_op4>%2, %1, %0<sd_mask_op4>|%0<sd_mask_op4>, %1, %2<round_sd_mask_op4>}
3932 vfnmsub<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}
3933 vfnmsub<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
3934 [(set_attr "isa" "fma,fma,fma,fma4,fma4")
3935 (set_attr "type" "ssemuladd")
3936 (set_attr "mode" "<MODE>")])
3938 (define_insn "<sd_mask_codefor>fma_fnmsub_<mode><sd_maskz_name><round_name>"
3939 [(set (match_operand:VF_SF_AVX512VL 0 "register_operand" "=v,v,v")
3942 (match_operand:VF_SF_AVX512VL 1 "<round_nimm_predicate>" "%0,0,v"))
3943 (match_operand:VF_SF_AVX512VL 2 "<round_nimm_predicate>" "<round_constraint>,v,<round_constraint>")
3945 (match_operand:VF_SF_AVX512VL 3 "<round_nimm_predicate>" "v,<round_constraint>,0"))))]
3946 "TARGET_AVX512F && <sd_mask_mode512bit_condition> && <round_mode512bit_condition>"
3948 vfnmsub132<ssemodesuffix>\t{<round_sd_mask_op4>%2, %3, %0<sd_mask_op4>|%0<sd_mask_op4>, %3, %2<round_sd_mask_op4>}
3949 vfnmsub213<ssemodesuffix>\t{<round_sd_mask_op4>%3, %2, %0<sd_mask_op4>|%0<sd_mask_op4>, %2, %3<round_sd_mask_op4>}
3950 vfnmsub231<ssemodesuffix>\t{<round_sd_mask_op4>%2, %1, %0<sd_mask_op4>|%0<sd_mask_op4>, %1, %2<round_sd_mask_op4>}"
3951 [(set_attr "type" "ssemuladd")
3952 (set_attr "mode" "<MODE>")])
3954 (define_insn "<avx512>_fnmsub_<mode>_mask<round_name>"
3955 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v,v")
3956 (vec_merge:VF_AVX512VL
3959 (match_operand:VF_AVX512VL 1 "register_operand" "0,0"))
3960 (match_operand:VF_AVX512VL 2 "nonimmediate_operand" "<round_constraint>,v")
3962 (match_operand:VF_AVX512VL 3 "nonimmediate_operand" "v,<round_constraint>")))
3964 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk,Yk")))]
3965 "TARGET_AVX512F && <round_mode512bit_condition>"
3967 vfnmsub132<ssemodesuffix>\t{<round_op5>%2, %3, %0%{%4%}|%0%{%4%}, %3, %2<round_op5>}
3968 vfnmsub213<ssemodesuffix>\t{<round_op5>%3, %2, %0%{%4%}|%0%{%4%}, %2, %3<round_op5>}"
3969 [(set_attr "type" "ssemuladd")
3970 (set_attr "mode" "<MODE>")])
3972 (define_insn "<avx512>_fnmsub_<mode>_mask3<round_name>"
3973 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v")
3974 (vec_merge:VF_AVX512VL
3977 (match_operand:VF_AVX512VL 1 "register_operand" "v"))
3978 (match_operand:VF_AVX512VL 2 "nonimmediate_operand" "<round_constraint>")
3980 (match_operand:VF_AVX512VL 3 "register_operand" "0")))
3982 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk")))]
3984 "vfnmsub231<ssemodesuffix>\t{<round_op5>%2, %1, %0%{%4%}|%0%{%4%}, %1, %2<round_op5>}"
3985 [(set_attr "type" "ssemuladd")
3986 (set_attr "mode" "<MODE>")])
3988 ;; FMA parallel floating point multiply addsub and subadd operations.
3990 ;; It would be possible to represent these without the UNSPEC as
3993 ;; (fma op1 op2 op3)
3994 ;; (fma op1 op2 (neg op3))
3997 ;; But this doesn't seem useful in practice.
3999 (define_expand "fmaddsub_<mode>"
4000 [(set (match_operand:VF 0 "register_operand")
4002 [(match_operand:VF 1 "nonimmediate_operand")
4003 (match_operand:VF 2 "nonimmediate_operand")
4004 (match_operand:VF 3 "nonimmediate_operand")]
4006 "TARGET_FMA || TARGET_FMA4 || TARGET_AVX512F")
4008 (define_expand "<avx512>_fmaddsub_<mode>_maskz<round_expand_name>"
4009 [(match_operand:VF_AVX512VL 0 "register_operand")
4010 (match_operand:VF_AVX512VL 1 "<round_expand_nimm_predicate>")
4011 (match_operand:VF_AVX512VL 2 "<round_expand_nimm_predicate>")
4012 (match_operand:VF_AVX512VL 3 "<round_expand_nimm_predicate>")
4013 (match_operand:<avx512fmaskmode> 4 "register_operand")]
4016 emit_insn (gen_fma_fmaddsub_<mode>_maskz_1<round_expand_name> (
4017 operands[0], operands[1], operands[2], operands[3],
4018 CONST0_RTX (<MODE>mode), operands[4]<round_expand_operand>));
4022 (define_insn "*fma_fmaddsub_<mode>"
4023 [(set (match_operand:VF_128_256 0 "register_operand" "=v,v,v,x,x")
4025 [(match_operand:VF_128_256 1 "nonimmediate_operand" "%0,0,v,x,x")
4026 (match_operand:VF_128_256 2 "nonimmediate_operand" "vm,v,vm,x,m")
4027 (match_operand:VF_128_256 3 "nonimmediate_operand" "v,vm,0,xm,x")]
4029 "TARGET_FMA || TARGET_FMA4"
4031 vfmaddsub132<ssemodesuffix>\t{%2, %3, %0|%0, %3, %2}
4032 vfmaddsub213<ssemodesuffix>\t{%3, %2, %0|%0, %2, %3}
4033 vfmaddsub231<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}
4034 vfmaddsub<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}
4035 vfmaddsub<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
4036 [(set_attr "isa" "fma,fma,fma,fma4,fma4")
4037 (set_attr "type" "ssemuladd")
4038 (set_attr "mode" "<MODE>")])
4040 (define_insn "<sd_mask_codefor>fma_fmaddsub_<mode><sd_maskz_name><round_name>"
4041 [(set (match_operand:VF_SF_AVX512VL 0 "register_operand" "=v,v,v")
4042 (unspec:VF_SF_AVX512VL
4043 [(match_operand:VF_SF_AVX512VL 1 "<round_nimm_predicate>" "%0,0,v")
4044 (match_operand:VF_SF_AVX512VL 2 "<round_nimm_predicate>" "<round_constraint>,v,<round_constraint>")
4045 (match_operand:VF_SF_AVX512VL 3 "<round_nimm_predicate>" "v,<round_constraint>,0")]
4047 "TARGET_AVX512F && <sd_mask_mode512bit_condition> && <round_mode512bit_condition>"
4049 vfmaddsub132<ssemodesuffix>\t{<round_sd_mask_op4>%2, %3, %0<sd_mask_op4>|%0<sd_mask_op4>, %3, %2<round_sd_mask_op4>}
4050 vfmaddsub213<ssemodesuffix>\t{<round_sd_mask_op4>%3, %2, %0<sd_mask_op4>|%0<sd_mask_op4>, %2, %3<round_sd_mask_op4>}
4051 vfmaddsub231<ssemodesuffix>\t{<round_sd_mask_op4>%2, %1, %0<sd_mask_op4>|%0<sd_mask_op4>, %1, %2<round_sd_mask_op4>}"
4052 [(set_attr "type" "ssemuladd")
4053 (set_attr "mode" "<MODE>")])
4055 (define_insn "<avx512>_fmaddsub_<mode>_mask<round_name>"
4056 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v,v")
4057 (vec_merge:VF_AVX512VL
4059 [(match_operand:VF_AVX512VL 1 "register_operand" "0,0")
4060 (match_operand:VF_AVX512VL 2 "nonimmediate_operand" "<round_constraint>,v")
4061 (match_operand:VF_AVX512VL 3 "nonimmediate_operand" "v,<round_constraint>")]
4064 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk,Yk")))]
4067 vfmaddsub132<ssemodesuffix>\t{<round_op5>%2, %3, %0%{%4%}|%0%{%4%}, %3, %2<round_op5>}
4068 vfmaddsub213<ssemodesuffix>\t{<round_op5>%3, %2, %0%{%4%}|%0%{%4%}, %2, %3<round_op5>}"
4069 [(set_attr "type" "ssemuladd")
4070 (set_attr "mode" "<MODE>")])
4072 (define_insn "<avx512>_fmaddsub_<mode>_mask3<round_name>"
4073 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v")
4074 (vec_merge:VF_AVX512VL
4076 [(match_operand:VF_AVX512VL 1 "register_operand" "v")
4077 (match_operand:VF_AVX512VL 2 "nonimmediate_operand" "<round_constraint>")
4078 (match_operand:VF_AVX512VL 3 "register_operand" "0")]
4081 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk")))]
4083 "vfmaddsub231<ssemodesuffix>\t{<round_op5>%2, %1, %0%{%4%}|%0%{%4%}, %1, %2<round_op5>}"
4084 [(set_attr "type" "ssemuladd")
4085 (set_attr "mode" "<MODE>")])
4087 (define_insn "*fma_fmsubadd_<mode>"
4088 [(set (match_operand:VF_128_256 0 "register_operand" "=v,v,v,x,x")
4090 [(match_operand:VF_128_256 1 "nonimmediate_operand" "%0,0,v,x,x")
4091 (match_operand:VF_128_256 2 "nonimmediate_operand" "vm,v,vm,x,m")
4093 (match_operand:VF_128_256 3 "nonimmediate_operand" "v,vm,0,xm,x"))]
4095 "TARGET_FMA || TARGET_FMA4"
4097 vfmsubadd132<ssemodesuffix>\t{%2, %3, %0|%0, %3, %2}
4098 vfmsubadd213<ssemodesuffix>\t{%3, %2, %0|%0, %2, %3}
4099 vfmsubadd231<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}
4100 vfmsubadd<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}
4101 vfmsubadd<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
4102 [(set_attr "isa" "fma,fma,fma,fma4,fma4")
4103 (set_attr "type" "ssemuladd")
4104 (set_attr "mode" "<MODE>")])
4106 (define_insn "<sd_mask_codefor>fma_fmsubadd_<mode><sd_maskz_name><round_name>"
4107 [(set (match_operand:VF_SF_AVX512VL 0 "register_operand" "=v,v,v")
4108 (unspec:VF_SF_AVX512VL
4109 [(match_operand:VF_SF_AVX512VL 1 "<round_nimm_predicate>" "%0,0,v")
4110 (match_operand:VF_SF_AVX512VL 2 "<round_nimm_predicate>" "<round_constraint>,v,<round_constraint>")
4112 (match_operand:VF_SF_AVX512VL 3 "<round_nimm_predicate>" "v,<round_constraint>,0"))]
4114 "TARGET_AVX512F && <sd_mask_mode512bit_condition> && <round_mode512bit_condition>"
4116 vfmsubadd132<ssemodesuffix>\t{<round_sd_mask_op4>%2, %3, %0<sd_mask_op4>|%0<sd_mask_op4>, %3, %2<round_sd_mask_op4>}
4117 vfmsubadd213<ssemodesuffix>\t{<round_sd_mask_op4>%3, %2, %0<sd_mask_op4>|%0<sd_mask_op4>, %2, %3<round_sd_mask_op4>}
4118 vfmsubadd231<ssemodesuffix>\t{<round_sd_mask_op4>%2, %1, %0<sd_mask_op4>|%0<sd_mask_op4>, %1, %2<round_sd_mask_op4>}"
4119 [(set_attr "type" "ssemuladd")
4120 (set_attr "mode" "<MODE>")])
4122 (define_insn "<avx512>_fmsubadd_<mode>_mask<round_name>"
4123 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v,v")
4124 (vec_merge:VF_AVX512VL
4126 [(match_operand:VF_AVX512VL 1 "register_operand" "0,0")
4127 (match_operand:VF_AVX512VL 2 "nonimmediate_operand" "<round_constraint>,v")
4129 (match_operand:VF_AVX512VL 3 "nonimmediate_operand" "v,<round_constraint>"))]
4132 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk,Yk")))]
4135 vfmsubadd132<ssemodesuffix>\t{<round_op5>%2, %3, %0%{%4%}|%0%{%4%}, %3, %2<round_op5>}
4136 vfmsubadd213<ssemodesuffix>\t{<round_op5>%3, %2, %0%{%4%}|%0%{%4%}, %2, %3<round_op5>}"
4137 [(set_attr "type" "ssemuladd")
4138 (set_attr "mode" "<MODE>")])
4140 (define_insn "<avx512>_fmsubadd_<mode>_mask3<round_name>"
4141 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v")
4142 (vec_merge:VF_AVX512VL
4144 [(match_operand:VF_AVX512VL 1 "register_operand" "v")
4145 (match_operand:VF_AVX512VL 2 "nonimmediate_operand" "<round_constraint>")
4147 (match_operand:VF_AVX512VL 3 "register_operand" "0"))]
4150 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk")))]
4152 "vfmsubadd231<ssemodesuffix>\t{<round_op5>%2, %1, %0%{%4%}|%0%{%4%}, %1, %2<round_op5>}"
4153 [(set_attr "type" "ssemuladd")
4154 (set_attr "mode" "<MODE>")])
4156 ;; FMA3 floating point scalar intrinsics. These merge result with
4157 ;; high-order elements from the destination register.
4159 (define_expand "fmai_vmfmadd_<mode><round_name>"
4160 [(set (match_operand:VF_128 0 "register_operand")
4163 (match_operand:VF_128 1 "<round_nimm_predicate>")
4164 (match_operand:VF_128 2 "<round_nimm_predicate>")
4165 (match_operand:VF_128 3 "<round_nimm_predicate>"))
4170 (define_insn "*fmai_fmadd_<mode>"
4171 [(set (match_operand:VF_128 0 "register_operand" "=v,v")
4174 (match_operand:VF_128 1 "<round_nimm_predicate>" " 0, 0")
4175 (match_operand:VF_128 2 "<round_nimm_predicate>" "<round_constraint>, v")
4176 (match_operand:VF_128 3 "<round_nimm_predicate>" " v,<round_constraint>"))
4179 "TARGET_FMA || TARGET_AVX512F"
4181 vfmadd132<ssescalarmodesuffix>\t{<round_op4>%2, %3, %0|%0, %<iptr>3, %<iptr>2<round_op4>}
4182 vfmadd213<ssescalarmodesuffix>\t{<round_op4>%3, %2, %0|%0, %<iptr>2, %<iptr>3<round_op4>}"
4183 [(set_attr "type" "ssemuladd")
4184 (set_attr "mode" "<MODE>")])
4186 (define_insn "*fmai_fmsub_<mode>"
4187 [(set (match_operand:VF_128 0 "register_operand" "=v,v")
4190 (match_operand:VF_128 1 "<round_nimm_predicate>" "0,0")
4191 (match_operand:VF_128 2 "<round_nimm_predicate>" "<round_constraint>,v")
4193 (match_operand:VF_128 3 "<round_nimm_predicate>" " v,<round_constraint>")))
4196 "TARGET_FMA || TARGET_AVX512F"
4198 vfmsub132<ssescalarmodesuffix>\t{<round_op4>%2, %3, %0|%0, %<iptr>3, %<iptr>2<round_op4>}
4199 vfmsub213<ssescalarmodesuffix>\t{<round_op4>%3, %2, %0|%0, %<iptr>2, %<iptr>3<round_op4>}"
4200 [(set_attr "type" "ssemuladd")
4201 (set_attr "mode" "<MODE>")])
4203 (define_insn "*fmai_fnmadd_<mode><round_name>"
4204 [(set (match_operand:VF_128 0 "register_operand" "=v,v")
4208 (match_operand:VF_128 2 "<round_nimm_predicate>" "<round_constraint>,v"))
4209 (match_operand:VF_128 1 "<round_nimm_predicate>" "0,0")
4210 (match_operand:VF_128 3 "<round_nimm_predicate>" "v,<round_constraint>"))
4213 "TARGET_FMA || TARGET_AVX512F"
4215 vfnmadd132<ssescalarmodesuffix>\t{<round_op4>%2, %3, %0|%0, %<iptr>3, %<iptr>2<round_op4>}
4216 vfnmadd213<ssescalarmodesuffix>\t{<round_op4>%3, %2, %0|%0, %<iptr>2, %<iptr>3<round_op4>}"
4217 [(set_attr "type" "ssemuladd")
4218 (set_attr "mode" "<MODE>")])
4220 (define_insn "*fmai_fnmsub_<mode><round_name>"
4221 [(set (match_operand:VF_128 0 "register_operand" "=v,v")
4225 (match_operand:VF_128 2 "<round_nimm_predicate>" "<round_constraint>, v"))
4226 (match_operand:VF_128 1 "<round_nimm_predicate>" " 0, 0")
4228 (match_operand:VF_128 3 "<round_nimm_predicate>" " v,<round_constraint>")))
4231 "TARGET_FMA || TARGET_AVX512F"
4233 vfnmsub132<ssescalarmodesuffix>\t{<round_op4>%2, %3, %0|%0, %<iptr>3, %<iptr>2<round_op4>}
4234 vfnmsub213<ssescalarmodesuffix>\t{<round_op4>%3, %2, %0|%0, %<iptr>2, %<iptr>3<round_op4>}"
4235 [(set_attr "type" "ssemuladd")
4236 (set_attr "mode" "<MODE>")])
4238 ;; FMA4 floating point scalar intrinsics. These write the
4239 ;; entire destination register, with the high-order elements zeroed.
4241 (define_expand "fma4i_vmfmadd_<mode>"
4242 [(set (match_operand:VF_128 0 "register_operand")
4245 (match_operand:VF_128 1 "nonimmediate_operand")
4246 (match_operand:VF_128 2 "nonimmediate_operand")
4247 (match_operand:VF_128 3 "nonimmediate_operand"))
4251 "operands[4] = CONST0_RTX (<MODE>mode);")
4253 (define_insn "*fma4i_vmfmadd_<mode>"
4254 [(set (match_operand:VF_128 0 "register_operand" "=x,x")
4257 (match_operand:VF_128 1 "nonimmediate_operand" "%x,x")
4258 (match_operand:VF_128 2 "nonimmediate_operand" " x,m")
4259 (match_operand:VF_128 3 "nonimmediate_operand" "xm,x"))
4260 (match_operand:VF_128 4 "const0_operand")
4263 "vfmadd<ssescalarmodesuffix>\t{%3, %2, %1, %0|%0, %1, %<iptr>2, %<iptr>3}"
4264 [(set_attr "type" "ssemuladd")
4265 (set_attr "mode" "<MODE>")])
4267 (define_insn "*fma4i_vmfmsub_<mode>"
4268 [(set (match_operand:VF_128 0 "register_operand" "=x,x")
4271 (match_operand:VF_128 1 "nonimmediate_operand" "%x,x")
4272 (match_operand:VF_128 2 "nonimmediate_operand" " x,m")
4274 (match_operand:VF_128 3 "nonimmediate_operand" "xm,x")))
4275 (match_operand:VF_128 4 "const0_operand")
4278 "vfmsub<ssescalarmodesuffix>\t{%3, %2, %1, %0|%0, %1, %<iptr>2, %<iptr>3}"
4279 [(set_attr "type" "ssemuladd")
4280 (set_attr "mode" "<MODE>")])
4282 (define_insn "*fma4i_vmfnmadd_<mode>"
4283 [(set (match_operand:VF_128 0 "register_operand" "=x,x")
4287 (match_operand:VF_128 1 "nonimmediate_operand" "%x,x"))
4288 (match_operand:VF_128 2 "nonimmediate_operand" " x,m")
4289 (match_operand:VF_128 3 "nonimmediate_operand" "xm,x"))
4290 (match_operand:VF_128 4 "const0_operand")
4293 "vfnmadd<ssescalarmodesuffix>\t{%3, %2, %1, %0|%0, %1, %<iptr>2, %<iptr>3}"
4294 [(set_attr "type" "ssemuladd")
4295 (set_attr "mode" "<MODE>")])
4297 (define_insn "*fma4i_vmfnmsub_<mode>"
4298 [(set (match_operand:VF_128 0 "register_operand" "=x,x")
4302 (match_operand:VF_128 1 "nonimmediate_operand" "%x,x"))
4303 (match_operand:VF_128 2 "nonimmediate_operand" " x,m")
4305 (match_operand:VF_128 3 "nonimmediate_operand" "xm,x")))
4306 (match_operand:VF_128 4 "const0_operand")
4309 "vfnmsub<ssescalarmodesuffix>\t{%3, %2, %1, %0|%0, %1, %<iptr>2, %<iptr>3}"
4310 [(set_attr "type" "ssemuladd")
4311 (set_attr "mode" "<MODE>")])
4313 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
4315 ;; Parallel single-precision floating point conversion operations
4317 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
4319 (define_insn "sse_cvtpi2ps"
4320 [(set (match_operand:V4SF 0 "register_operand" "=x")
4323 (float:V2SF (match_operand:V2SI 2 "nonimmediate_operand" "ym")))
4324 (match_operand:V4SF 1 "register_operand" "0")
4327 "cvtpi2ps\t{%2, %0|%0, %2}"
4328 [(set_attr "type" "ssecvt")
4329 (set_attr "mode" "V4SF")])
4331 (define_insn "sse_cvtps2pi"
4332 [(set (match_operand:V2SI 0 "register_operand" "=y")
4334 (unspec:V4SI [(match_operand:V4SF 1 "nonimmediate_operand" "xm")]
4336 (parallel [(const_int 0) (const_int 1)])))]
4338 "cvtps2pi\t{%1, %0|%0, %q1}"
4339 [(set_attr "type" "ssecvt")
4340 (set_attr "unit" "mmx")
4341 (set_attr "mode" "DI")])
4343 (define_insn "sse_cvttps2pi"
4344 [(set (match_operand:V2SI 0 "register_operand" "=y")
4346 (fix:V4SI (match_operand:V4SF 1 "nonimmediate_operand" "xm"))
4347 (parallel [(const_int 0) (const_int 1)])))]
4349 "cvttps2pi\t{%1, %0|%0, %q1}"
4350 [(set_attr "type" "ssecvt")
4351 (set_attr "unit" "mmx")
4352 (set_attr "prefix_rep" "0")
4353 (set_attr "mode" "SF")])
4355 (define_insn "sse_cvtsi2ss<round_name>"
4356 [(set (match_operand:V4SF 0 "register_operand" "=x,x,v")
4359 (float:SF (match_operand:SI 2 "<round_nimm_scalar_predicate>" "r,m,<round_constraint3>")))
4360 (match_operand:V4SF 1 "register_operand" "0,0,v")
4364 cvtsi2ss\t{%2, %0|%0, %2}
4365 cvtsi2ss\t{%2, %0|%0, %2}
4366 vcvtsi2ss\t{%2, <round_op3>%1, %0|%0, %1<round_op3>, %2}"
4367 [(set_attr "isa" "noavx,noavx,avx")
4368 (set_attr "type" "sseicvt")
4369 (set_attr "athlon_decode" "vector,double,*")
4370 (set_attr "amdfam10_decode" "vector,double,*")
4371 (set_attr "bdver1_decode" "double,direct,*")
4372 (set_attr "btver2_decode" "double,double,double")
4373 (set_attr "znver1_decode" "double,double,double")
4374 (set_attr "prefix" "orig,orig,maybe_evex")
4375 (set_attr "mode" "SF")])
4377 (define_insn "sse_cvtsi2ssq<round_name>"
4378 [(set (match_operand:V4SF 0 "register_operand" "=x,x,v")
4381 (float:SF (match_operand:DI 2 "<round_nimm_scalar_predicate>" "r,m,<round_constraint3>")))
4382 (match_operand:V4SF 1 "register_operand" "0,0,v")
4384 "TARGET_SSE && TARGET_64BIT"
4386 cvtsi2ssq\t{%2, %0|%0, %2}
4387 cvtsi2ssq\t{%2, %0|%0, %2}
4388 vcvtsi2ssq\t{%2, <round_op3>%1, %0|%0, %1<round_op3>, %2}"
4389 [(set_attr "isa" "noavx,noavx,avx")
4390 (set_attr "type" "sseicvt")
4391 (set_attr "athlon_decode" "vector,double,*")
4392 (set_attr "amdfam10_decode" "vector,double,*")
4393 (set_attr "bdver1_decode" "double,direct,*")
4394 (set_attr "btver2_decode" "double,double,double")
4395 (set_attr "length_vex" "*,*,4")
4396 (set_attr "prefix_rex" "1,1,*")
4397 (set_attr "prefix" "orig,orig,maybe_evex")
4398 (set_attr "mode" "SF")])
4400 (define_insn "sse_cvtss2si<round_name>"
4401 [(set (match_operand:SI 0 "register_operand" "=r,r")
4404 (match_operand:V4SF 1 "<round_nimm_scalar_predicate>" "v,<round_constraint2>")
4405 (parallel [(const_int 0)]))]
4406 UNSPEC_FIX_NOTRUNC))]
4408 "%vcvtss2si\t{<round_op2>%1, %0|%0, %k1<round_op2>}"
4409 [(set_attr "type" "sseicvt")
4410 (set_attr "athlon_decode" "double,vector")
4411 (set_attr "bdver1_decode" "double,double")
4412 (set_attr "prefix_rep" "1")
4413 (set_attr "prefix" "maybe_vex")
4414 (set_attr "mode" "SI")])
4416 (define_insn "sse_cvtss2si_2"
4417 [(set (match_operand:SI 0 "register_operand" "=r,r")
4418 (unspec:SI [(match_operand:SF 1 "nonimmediate_operand" "v,m")]
4419 UNSPEC_FIX_NOTRUNC))]
4421 "%vcvtss2si\t{%1, %0|%0, %k1}"
4422 [(set_attr "type" "sseicvt")
4423 (set_attr "athlon_decode" "double,vector")
4424 (set_attr "amdfam10_decode" "double,double")
4425 (set_attr "bdver1_decode" "double,double")
4426 (set_attr "prefix_rep" "1")
4427 (set_attr "prefix" "maybe_vex")
4428 (set_attr "mode" "SI")])
4430 (define_insn "sse_cvtss2siq<round_name>"
4431 [(set (match_operand:DI 0 "register_operand" "=r,r")
4434 (match_operand:V4SF 1 "<round_nimm_scalar_predicate>" "v,<round_constraint2>")
4435 (parallel [(const_int 0)]))]
4436 UNSPEC_FIX_NOTRUNC))]
4437 "TARGET_SSE && TARGET_64BIT"
4438 "%vcvtss2si{q}\t{<round_op2>%1, %0|%0, %k1<round_op2>}"
4439 [(set_attr "type" "sseicvt")
4440 (set_attr "athlon_decode" "double,vector")
4441 (set_attr "bdver1_decode" "double,double")
4442 (set_attr "prefix_rep" "1")
4443 (set_attr "prefix" "maybe_vex")
4444 (set_attr "mode" "DI")])
4446 (define_insn "sse_cvtss2siq_2"
4447 [(set (match_operand:DI 0 "register_operand" "=r,r")
4448 (unspec:DI [(match_operand:SF 1 "nonimmediate_operand" "v,m")]
4449 UNSPEC_FIX_NOTRUNC))]
4450 "TARGET_SSE && TARGET_64BIT"
4451 "%vcvtss2si{q}\t{%1, %0|%0, %k1}"
4452 [(set_attr "type" "sseicvt")
4453 (set_attr "athlon_decode" "double,vector")
4454 (set_attr "amdfam10_decode" "double,double")
4455 (set_attr "bdver1_decode" "double,double")
4456 (set_attr "prefix_rep" "1")
4457 (set_attr "prefix" "maybe_vex")
4458 (set_attr "mode" "DI")])
4460 (define_insn "sse_cvttss2si<round_saeonly_name>"
4461 [(set (match_operand:SI 0 "register_operand" "=r,r")
4464 (match_operand:V4SF 1 "<round_saeonly_nimm_scalar_predicate>" "v,<round_saeonly_constraint2>")
4465 (parallel [(const_int 0)]))))]
4467 "%vcvttss2si\t{<round_saeonly_op2>%1, %0|%0, %k1<round_saeonly_op2>}"
4468 [(set_attr "type" "sseicvt")
4469 (set_attr "athlon_decode" "double,vector")
4470 (set_attr "amdfam10_decode" "double,double")
4471 (set_attr "bdver1_decode" "double,double")
4472 (set_attr "prefix_rep" "1")
4473 (set_attr "prefix" "maybe_vex")
4474 (set_attr "mode" "SI")])
4476 (define_insn "sse_cvttss2siq<round_saeonly_name>"
4477 [(set (match_operand:DI 0 "register_operand" "=r,r")
4480 (match_operand:V4SF 1 "<round_saeonly_nimm_scalar_predicate>" "v,<round_saeonly_constraint>")
4481 (parallel [(const_int 0)]))))]
4482 "TARGET_SSE && TARGET_64BIT"
4483 "%vcvttss2si{q}\t{<round_saeonly_op2>%1, %0|%0, %k1<round_saeonly_op2>}"
4484 [(set_attr "type" "sseicvt")
4485 (set_attr "athlon_decode" "double,vector")
4486 (set_attr "amdfam10_decode" "double,double")
4487 (set_attr "bdver1_decode" "double,double")
4488 (set_attr "prefix_rep" "1")
4489 (set_attr "prefix" "maybe_vex")
4490 (set_attr "mode" "DI")])
4492 (define_insn "cvtusi2<ssescalarmodesuffix>32<round_name>"
4493 [(set (match_operand:VF_128 0 "register_operand" "=v")
4495 (vec_duplicate:VF_128
4496 (unsigned_float:<ssescalarmode>
4497 (match_operand:SI 2 "<round_nimm_predicate>" "<round_constraint3>")))
4498 (match_operand:VF_128 1 "register_operand" "v")
4500 "TARGET_AVX512F && <round_modev4sf_condition>"
4501 "vcvtusi2<ssescalarmodesuffix>\t{%2, <round_op3>%1, %0|%0, %1<round_op3>, %2}"
4502 [(set_attr "type" "sseicvt")
4503 (set_attr "prefix" "evex")
4504 (set_attr "mode" "<ssescalarmode>")])
4506 (define_insn "cvtusi2<ssescalarmodesuffix>64<round_name>"
4507 [(set (match_operand:VF_128 0 "register_operand" "=v")
4509 (vec_duplicate:VF_128
4510 (unsigned_float:<ssescalarmode>
4511 (match_operand:DI 2 "<round_nimm_predicate>" "<round_constraint3>")))
4512 (match_operand:VF_128 1 "register_operand" "v")
4514 "TARGET_AVX512F && TARGET_64BIT"
4515 "vcvtusi2<ssescalarmodesuffix>\t{%2, <round_op3>%1, %0|%0, %1<round_op3>, %2}"
4516 [(set_attr "type" "sseicvt")
4517 (set_attr "prefix" "evex")
4518 (set_attr "mode" "<ssescalarmode>")])
4520 (define_insn "float<sseintvecmodelower><mode>2<mask_name><round_name>"
4521 [(set (match_operand:VF1 0 "register_operand" "=x,v")
4523 (match_operand:<sseintvecmode> 1 "<round_nimm_predicate>" "xBm,<round_constraint>")))]
4524 "TARGET_SSE2 && <mask_mode512bit_condition> && <round_mode512bit_condition>"
4526 cvtdq2ps\t{%1, %0|%0, %1}
4527 vcvtdq2ps\t{<round_mask_op2>%1, %0<mask_operand2>|%0<mask_operand2>, %1<round_mask_op2>}"
4528 [(set_attr "isa" "noavx,avx")
4529 (set_attr "type" "ssecvt")
4530 (set_attr "prefix" "maybe_vex")
4531 (set_attr "mode" "<sseinsnmode>")])
4533 (define_insn "ufloat<sseintvecmodelower><mode>2<mask_name><round_name>"
4534 [(set (match_operand:VF1_AVX512VL 0 "register_operand" "=v")
4535 (unsigned_float:VF1_AVX512VL
4536 (match_operand:<sseintvecmode> 1 "nonimmediate_operand" "<round_constraint>")))]
4538 "vcvtudq2ps\t{<round_mask_op2>%1, %0<mask_operand2>|%0<mask_operand2>, %1<round_mask_op2>}"
4539 [(set_attr "type" "ssecvt")
4540 (set_attr "prefix" "evex")
4541 (set_attr "mode" "<MODE>")])
4543 (define_expand "floatuns<sseintvecmodelower><mode>2"
4544 [(match_operand:VF1 0 "register_operand")
4545 (match_operand:<sseintvecmode> 1 "register_operand")]
4546 "TARGET_SSE2 && (<MODE>mode == V4SFmode || TARGET_AVX2)"
4548 if (<MODE>mode == V16SFmode)
4549 emit_insn (gen_ufloatv16siv16sf2 (operands[0], operands[1]));
4551 if (TARGET_AVX512VL)
4553 if (<MODE>mode == V4SFmode)
4554 emit_insn (gen_ufloatv4siv4sf2 (operands[0], operands[1]));
4556 emit_insn (gen_ufloatv8siv8sf2 (operands[0], operands[1]));
4559 ix86_expand_vector_convert_uns_vsivsf (operands[0], operands[1]);
4565 ;; For <sse2_avx_avx512f>_fix_notrunc<sf2simodelower><mode> insn pattern
4566 (define_mode_attr sf2simodelower
4567 [(V16SI "v16sf") (V8SI "v8sf") (V4SI "v4sf")])
4569 (define_insn "<sse2_avx_avx512f>_fix_notrunc<sf2simodelower><mode><mask_name>"
4570 [(set (match_operand:VI4_AVX 0 "register_operand" "=v")
4572 [(match_operand:<ssePSmode> 1 "vector_operand" "vBm")]
4573 UNSPEC_FIX_NOTRUNC))]
4574 "TARGET_SSE2 && <mask_mode512bit_condition>"
4575 "%vcvtps2dq\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
4576 [(set_attr "type" "ssecvt")
4577 (set (attr "prefix_data16")
4579 (match_test "TARGET_AVX")
4581 (const_string "1")))
4582 (set_attr "prefix" "maybe_vex")
4583 (set_attr "mode" "<sseinsnmode>")])
4585 (define_insn "avx512f_fix_notruncv16sfv16si<mask_name><round_name>"
4586 [(set (match_operand:V16SI 0 "register_operand" "=v")
4588 [(match_operand:V16SF 1 "<round_nimm_predicate>" "<round_constraint>")]
4589 UNSPEC_FIX_NOTRUNC))]
4591 "vcvtps2dq\t{<round_mask_op2>%1, %0<mask_operand2>|%0<mask_operand2>, %1<round_mask_op2>}"
4592 [(set_attr "type" "ssecvt")
4593 (set_attr "prefix" "evex")
4594 (set_attr "mode" "XI")])
4596 (define_insn "<mask_codefor><avx512>_ufix_notrunc<sf2simodelower><mode><mask_name><round_name>"
4597 [(set (match_operand:VI4_AVX512VL 0 "register_operand" "=v")
4598 (unspec:VI4_AVX512VL
4599 [(match_operand:<ssePSmode> 1 "nonimmediate_operand" "<round_constraint>")]
4600 UNSPEC_UNSIGNED_FIX_NOTRUNC))]
4602 "vcvtps2udq\t{<round_mask_op2>%1, %0<mask_operand2>|%0<mask_operand2>, %1<round_mask_op2>}"
4603 [(set_attr "type" "ssecvt")
4604 (set_attr "prefix" "evex")
4605 (set_attr "mode" "<sseinsnmode>")])
4607 (define_insn "<mask_codefor>avx512dq_cvtps2qq<mode><mask_name><round_name>"
4608 [(set (match_operand:VI8_256_512 0 "register_operand" "=v")
4609 (unspec:VI8_256_512 [(match_operand:<ssePSmode2> 1 "nonimmediate_operand" "<round_constraint>")]
4610 UNSPEC_FIX_NOTRUNC))]
4611 "TARGET_AVX512DQ && <round_mode512bit_condition>"
4612 "vcvtps2qq\t{<round_mask_op2>%1, %0<mask_operand2>|%0<mask_operand2>, %1<round_mask_op2>}"
4613 [(set_attr "type" "ssecvt")
4614 (set_attr "prefix" "evex")
4615 (set_attr "mode" "<sseinsnmode>")])
4617 (define_insn "<mask_codefor>avx512dq_cvtps2qqv2di<mask_name>"
4618 [(set (match_operand:V2DI 0 "register_operand" "=v")
4621 (match_operand:V4SF 1 "nonimmediate_operand" "vm")
4622 (parallel [(const_int 0) (const_int 1)]))]
4623 UNSPEC_FIX_NOTRUNC))]
4624 "TARGET_AVX512DQ && TARGET_AVX512VL"
4625 "vcvtps2qq\t{%1, %0<mask_operand2>|%0<mask_operand2>, %q1}"
4626 [(set_attr "type" "ssecvt")
4627 (set_attr "prefix" "evex")
4628 (set_attr "mode" "TI")])
4630 (define_insn "<mask_codefor>avx512dq_cvtps2uqq<mode><mask_name><round_name>"
4631 [(set (match_operand:VI8_256_512 0 "register_operand" "=v")
4632 (unspec:VI8_256_512 [(match_operand:<ssePSmode2> 1 "nonimmediate_operand" "<round_constraint>")]
4633 UNSPEC_UNSIGNED_FIX_NOTRUNC))]
4634 "TARGET_AVX512DQ && <round_mode512bit_condition>"
4635 "vcvtps2uqq\t{<round_mask_op2>%1, %0<mask_operand2>|%0<mask_operand2>, %1<round_mask_op2>}"
4636 [(set_attr "type" "ssecvt")
4637 (set_attr "prefix" "evex")
4638 (set_attr "mode" "<sseinsnmode>")])
4640 (define_insn "<mask_codefor>avx512dq_cvtps2uqqv2di<mask_name>"
4641 [(set (match_operand:V2DI 0 "register_operand" "=v")
4644 (match_operand:V4SF 1 "nonimmediate_operand" "vm")
4645 (parallel [(const_int 0) (const_int 1)]))]
4646 UNSPEC_UNSIGNED_FIX_NOTRUNC))]
4647 "TARGET_AVX512DQ && TARGET_AVX512VL"
4648 "vcvtps2uqq\t{%1, %0<mask_operand2>|%0<mask_operand2>, %q1}"
4649 [(set_attr "type" "ssecvt")
4650 (set_attr "prefix" "evex")
4651 (set_attr "mode" "TI")])
4653 (define_insn "<fixsuffix>fix_truncv16sfv16si2<mask_name><round_saeonly_name>"
4654 [(set (match_operand:V16SI 0 "register_operand" "=v")
4656 (match_operand:V16SF 1 "<round_saeonly_nimm_predicate>" "<round_saeonly_constraint>")))]
4658 "vcvttps2<fixsuffix>dq\t{<round_saeonly_mask_op2>%1, %0<mask_operand2>|%0<mask_operand2>, %1<round_saeonly_mask_op2>}"
4659 [(set_attr "type" "ssecvt")
4660 (set_attr "prefix" "evex")
4661 (set_attr "mode" "XI")])
4663 (define_insn "fix_truncv8sfv8si2<mask_name>"
4664 [(set (match_operand:V8SI 0 "register_operand" "=v")
4665 (fix:V8SI (match_operand:V8SF 1 "nonimmediate_operand" "vm")))]
4666 "TARGET_AVX && <mask_avx512vl_condition>"
4667 "vcvttps2dq\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
4668 [(set_attr "type" "ssecvt")
4669 (set_attr "prefix" "<mask_prefix>")
4670 (set_attr "mode" "OI")])
4672 (define_insn "fix_truncv4sfv4si2<mask_name>"
4673 [(set (match_operand:V4SI 0 "register_operand" "=v")
4674 (fix:V4SI (match_operand:V4SF 1 "vector_operand" "vBm")))]
4675 "TARGET_SSE2 && <mask_avx512vl_condition>"
4676 "%vcvttps2dq\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
4677 [(set_attr "type" "ssecvt")
4678 (set (attr "prefix_rep")
4680 (match_test "TARGET_AVX")
4682 (const_string "1")))
4683 (set (attr "prefix_data16")
4685 (match_test "TARGET_AVX")
4687 (const_string "0")))
4688 (set_attr "prefix_data16" "0")
4689 (set_attr "prefix" "<mask_prefix2>")
4690 (set_attr "mode" "TI")])
4692 (define_expand "fixuns_trunc<mode><sseintvecmodelower>2"
4693 [(match_operand:<sseintvecmode> 0 "register_operand")
4694 (match_operand:VF1 1 "register_operand")]
4697 if (<MODE>mode == V16SFmode)
4698 emit_insn (gen_ufix_truncv16sfv16si2 (operands[0],
4703 tmp[0] = ix86_expand_adjust_ufix_to_sfix_si (operands[1], &tmp[2]);
4704 tmp[1] = gen_reg_rtx (<sseintvecmode>mode);
4705 emit_insn (gen_fix_trunc<mode><sseintvecmodelower>2 (tmp[1], tmp[0]));
4706 emit_insn (gen_xor<sseintvecmodelower>3 (operands[0], tmp[1], tmp[2]));
4711 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
4713 ;; Parallel double-precision floating point conversion operations
4715 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
4717 (define_insn "sse2_cvtpi2pd"
4718 [(set (match_operand:V2DF 0 "register_operand" "=x,x")
4719 (float:V2DF (match_operand:V2SI 1 "nonimmediate_operand" "y,m")))]
4721 "cvtpi2pd\t{%1, %0|%0, %1}"
4722 [(set_attr "type" "ssecvt")
4723 (set_attr "unit" "mmx,*")
4724 (set_attr "prefix_data16" "1,*")
4725 (set_attr "mode" "V2DF")])
4727 (define_insn "sse2_cvtpd2pi"
4728 [(set (match_operand:V2SI 0 "register_operand" "=y")
4729 (unspec:V2SI [(match_operand:V2DF 1 "nonimmediate_operand" "xm")]
4730 UNSPEC_FIX_NOTRUNC))]
4732 "cvtpd2pi\t{%1, %0|%0, %1}"
4733 [(set_attr "type" "ssecvt")
4734 (set_attr "unit" "mmx")
4735 (set_attr "bdver1_decode" "double")
4736 (set_attr "btver2_decode" "direct")
4737 (set_attr "prefix_data16" "1")
4738 (set_attr "mode" "DI")])
4740 (define_insn "sse2_cvttpd2pi"
4741 [(set (match_operand:V2SI 0 "register_operand" "=y")
4742 (fix:V2SI (match_operand:V2DF 1 "nonimmediate_operand" "xm")))]
4744 "cvttpd2pi\t{%1, %0|%0, %1}"
4745 [(set_attr "type" "ssecvt")
4746 (set_attr "unit" "mmx")
4747 (set_attr "bdver1_decode" "double")
4748 (set_attr "prefix_data16" "1")
4749 (set_attr "mode" "TI")])
4751 (define_insn "sse2_cvtsi2sd"
4752 [(set (match_operand:V2DF 0 "register_operand" "=x,x,v")
4755 (float:DF (match_operand:SI 2 "nonimmediate_operand" "r,m,rm")))
4756 (match_operand:V2DF 1 "register_operand" "0,0,v")
4760 cvtsi2sd\t{%2, %0|%0, %2}
4761 cvtsi2sd\t{%2, %0|%0, %2}
4762 vcvtsi2sd\t{%2, %1, %0|%0, %1, %2}"
4763 [(set_attr "isa" "noavx,noavx,avx")
4764 (set_attr "type" "sseicvt")
4765 (set_attr "athlon_decode" "double,direct,*")
4766 (set_attr "amdfam10_decode" "vector,double,*")
4767 (set_attr "bdver1_decode" "double,direct,*")
4768 (set_attr "btver2_decode" "double,double,double")
4769 (set_attr "znver1_decode" "double,double,double")
4770 (set_attr "prefix" "orig,orig,maybe_evex")
4771 (set_attr "mode" "DF")])
4773 (define_insn "sse2_cvtsi2sdq<round_name>"
4774 [(set (match_operand:V2DF 0 "register_operand" "=x,x,v")
4777 (float:DF (match_operand:DI 2 "<round_nimm_scalar_predicate>" "r,m,<round_constraint3>")))
4778 (match_operand:V2DF 1 "register_operand" "0,0,v")
4780 "TARGET_SSE2 && TARGET_64BIT"
4782 cvtsi2sdq\t{%2, %0|%0, %2}
4783 cvtsi2sdq\t{%2, %0|%0, %2}
4784 vcvtsi2sdq\t{%2, <round_op3>%1, %0|%0, %1<round_op3>, %2}"
4785 [(set_attr "isa" "noavx,noavx,avx")
4786 (set_attr "type" "sseicvt")
4787 (set_attr "athlon_decode" "double,direct,*")
4788 (set_attr "amdfam10_decode" "vector,double,*")
4789 (set_attr "bdver1_decode" "double,direct,*")
4790 (set_attr "length_vex" "*,*,4")
4791 (set_attr "prefix_rex" "1,1,*")
4792 (set_attr "prefix" "orig,orig,maybe_evex")
4793 (set_attr "mode" "DF")])
4795 (define_insn "avx512f_vcvtss2usi<round_name>"
4796 [(set (match_operand:SI 0 "register_operand" "=r")
4799 (match_operand:V4SF 1 "<round_nimm_predicate>" "<round_constraint>")
4800 (parallel [(const_int 0)]))]
4801 UNSPEC_UNSIGNED_FIX_NOTRUNC))]
4803 "vcvtss2usi\t{<round_op2>%1, %0|%0, %1<round_op2>}"
4804 [(set_attr "type" "sseicvt")
4805 (set_attr "prefix" "evex")
4806 (set_attr "mode" "SI")])
4808 (define_insn "avx512f_vcvtss2usiq<round_name>"
4809 [(set (match_operand:DI 0 "register_operand" "=r")
4812 (match_operand:V4SF 1 "<round_nimm_predicate>" "<round_constraint>")
4813 (parallel [(const_int 0)]))]
4814 UNSPEC_UNSIGNED_FIX_NOTRUNC))]
4815 "TARGET_AVX512F && TARGET_64BIT"
4816 "vcvtss2usi\t{<round_op2>%1, %0|%0, %1<round_op2>}"
4817 [(set_attr "type" "sseicvt")
4818 (set_attr "prefix" "evex")
4819 (set_attr "mode" "DI")])
4821 (define_insn "avx512f_vcvttss2usi<round_saeonly_name>"
4822 [(set (match_operand:SI 0 "register_operand" "=r")
4825 (match_operand:V4SF 1 "<round_saeonly_nimm_scalar_predicate>" "<round_saeonly_constraint>")
4826 (parallel [(const_int 0)]))))]
4828 "vcvttss2usi\t{<round_saeonly_op2>%1, %0|%0, %1<round_saeonly_op2>}"
4829 [(set_attr "type" "sseicvt")
4830 (set_attr "prefix" "evex")
4831 (set_attr "mode" "SI")])
4833 (define_insn "avx512f_vcvttss2usiq<round_saeonly_name>"
4834 [(set (match_operand:DI 0 "register_operand" "=r")
4837 (match_operand:V4SF 1 "<round_saeonly_nimm_scalar_predicate>" "<round_saeonly_constraint>")
4838 (parallel [(const_int 0)]))))]
4839 "TARGET_AVX512F && TARGET_64BIT"
4840 "vcvttss2usi\t{<round_saeonly_op2>%1, %0|%0, %1<round_saeonly_op2>}"
4841 [(set_attr "type" "sseicvt")
4842 (set_attr "prefix" "evex")
4843 (set_attr "mode" "DI")])
4845 (define_insn "avx512f_vcvtsd2usi<round_name>"
4846 [(set (match_operand:SI 0 "register_operand" "=r")
4849 (match_operand:V2DF 1 "<round_nimm_predicate>" "<round_constraint>")
4850 (parallel [(const_int 0)]))]
4851 UNSPEC_UNSIGNED_FIX_NOTRUNC))]
4853 "vcvtsd2usi\t{<round_op2>%1, %0|%0, %1<round_op2>}"
4854 [(set_attr "type" "sseicvt")
4855 (set_attr "prefix" "evex")
4856 (set_attr "mode" "SI")])
4858 (define_insn "avx512f_vcvtsd2usiq<round_name>"
4859 [(set (match_operand:DI 0 "register_operand" "=r")
4862 (match_operand:V2DF 1 "<round_nimm_predicate>" "<round_constraint>")
4863 (parallel [(const_int 0)]))]
4864 UNSPEC_UNSIGNED_FIX_NOTRUNC))]
4865 "TARGET_AVX512F && TARGET_64BIT"
4866 "vcvtsd2usi\t{<round_op2>%1, %0|%0, %1<round_op2>}"
4867 [(set_attr "type" "sseicvt")
4868 (set_attr "prefix" "evex")
4869 (set_attr "mode" "DI")])
4871 (define_insn "avx512f_vcvttsd2usi<round_saeonly_name>"
4872 [(set (match_operand:SI 0 "register_operand" "=r")
4875 (match_operand:V2DF 1 "<round_saeonly_nimm_scalar_predicate>" "<round_saeonly_constraint>")
4876 (parallel [(const_int 0)]))))]
4878 "vcvttsd2usi\t{<round_saeonly_op2>%1, %0|%0, %1<round_saeonly_op2>}"
4879 [(set_attr "type" "sseicvt")
4880 (set_attr "prefix" "evex")
4881 (set_attr "mode" "SI")])
4883 (define_insn "avx512f_vcvttsd2usiq<round_saeonly_name>"
4884 [(set (match_operand:DI 0 "register_operand" "=r")
4887 (match_operand:V2DF 1 "<round_saeonly_nimm_scalar_predicate>" "<round_saeonly_constraint>")
4888 (parallel [(const_int 0)]))))]
4889 "TARGET_AVX512F && TARGET_64BIT"
4890 "vcvttsd2usi\t{<round_saeonly_op2>%1, %0|%0, %1<round_saeonly_op2>}"
4891 [(set_attr "type" "sseicvt")
4892 (set_attr "prefix" "evex")
4893 (set_attr "mode" "DI")])
4895 (define_insn "sse2_cvtsd2si<round_name>"
4896 [(set (match_operand:SI 0 "register_operand" "=r,r")
4899 (match_operand:V2DF 1 "<round_nimm_scalar_predicate>" "v,<round_constraint2>")
4900 (parallel [(const_int 0)]))]
4901 UNSPEC_FIX_NOTRUNC))]
4903 "%vcvtsd2si\t{<round_op2>%1, %0|%0, %q1<round_op2>}"
4904 [(set_attr "type" "sseicvt")
4905 (set_attr "athlon_decode" "double,vector")
4906 (set_attr "bdver1_decode" "double,double")
4907 (set_attr "btver2_decode" "double,double")
4908 (set_attr "prefix_rep" "1")
4909 (set_attr "prefix" "maybe_vex")
4910 (set_attr "mode" "SI")])
4912 (define_insn "sse2_cvtsd2si_2"
4913 [(set (match_operand:SI 0 "register_operand" "=r,r")
4914 (unspec:SI [(match_operand:DF 1 "nonimmediate_operand" "v,m")]
4915 UNSPEC_FIX_NOTRUNC))]
4917 "%vcvtsd2si\t{%1, %0|%0, %q1}"
4918 [(set_attr "type" "sseicvt")
4919 (set_attr "athlon_decode" "double,vector")
4920 (set_attr "amdfam10_decode" "double,double")
4921 (set_attr "bdver1_decode" "double,double")
4922 (set_attr "prefix_rep" "1")
4923 (set_attr "prefix" "maybe_vex")
4924 (set_attr "mode" "SI")])
4926 (define_insn "sse2_cvtsd2siq<round_name>"
4927 [(set (match_operand:DI 0 "register_operand" "=r,r")
4930 (match_operand:V2DF 1 "<round_nimm_scalar_predicate>" "v,<round_constraint2>")
4931 (parallel [(const_int 0)]))]
4932 UNSPEC_FIX_NOTRUNC))]
4933 "TARGET_SSE2 && TARGET_64BIT"
4934 "%vcvtsd2si{q}\t{<round_op2>%1, %0|%0, %q1<round_op2>}"
4935 [(set_attr "type" "sseicvt")
4936 (set_attr "athlon_decode" "double,vector")
4937 (set_attr "bdver1_decode" "double,double")
4938 (set_attr "prefix_rep" "1")
4939 (set_attr "prefix" "maybe_vex")
4940 (set_attr "mode" "DI")])
4942 (define_insn "sse2_cvtsd2siq_2"
4943 [(set (match_operand:DI 0 "register_operand" "=r,r")
4944 (unspec:DI [(match_operand:DF 1 "nonimmediate_operand" "v,m")]
4945 UNSPEC_FIX_NOTRUNC))]
4946 "TARGET_SSE2 && TARGET_64BIT"
4947 "%vcvtsd2si{q}\t{%1, %0|%0, %q1}"
4948 [(set_attr "type" "sseicvt")
4949 (set_attr "athlon_decode" "double,vector")
4950 (set_attr "amdfam10_decode" "double,double")
4951 (set_attr "bdver1_decode" "double,double")
4952 (set_attr "prefix_rep" "1")
4953 (set_attr "prefix" "maybe_vex")
4954 (set_attr "mode" "DI")])
4956 (define_insn "sse2_cvttsd2si<round_saeonly_name>"
4957 [(set (match_operand:SI 0 "register_operand" "=r,r")
4960 (match_operand:V2DF 1 "<round_saeonly_nimm_scalar_predicate>" "v,<round_saeonly_constraint2>")
4961 (parallel [(const_int 0)]))))]
4963 "%vcvttsd2si\t{<round_saeonly_op2>%1, %0|%0, %q1<round_saeonly_op2>}"
4964 [(set_attr "type" "sseicvt")
4965 (set_attr "athlon_decode" "double,vector")
4966 (set_attr "amdfam10_decode" "double,double")
4967 (set_attr "bdver1_decode" "double,double")
4968 (set_attr "btver2_decode" "double,double")
4969 (set_attr "prefix_rep" "1")
4970 (set_attr "prefix" "maybe_vex")
4971 (set_attr "mode" "SI")])
4973 (define_insn "sse2_cvttsd2siq<round_saeonly_name>"
4974 [(set (match_operand:DI 0 "register_operand" "=r,r")
4977 (match_operand:V2DF 1 "<round_saeonly_nimm_scalar_predicate>" "v,<round_saeonly_constraint2>")
4978 (parallel [(const_int 0)]))))]
4979 "TARGET_SSE2 && TARGET_64BIT"
4980 "%vcvttsd2si{q}\t{<round_saeonly_op2>%1, %0|%0, %q1<round_saeonly_op2>}"
4981 [(set_attr "type" "sseicvt")
4982 (set_attr "athlon_decode" "double,vector")
4983 (set_attr "amdfam10_decode" "double,double")
4984 (set_attr "bdver1_decode" "double,double")
4985 (set_attr "prefix_rep" "1")
4986 (set_attr "prefix" "maybe_vex")
4987 (set_attr "mode" "DI")])
4989 ;; For float<si2dfmode><mode>2 insn pattern
4990 (define_mode_attr si2dfmode
4991 [(V8DF "V8SI") (V4DF "V4SI")])
4992 (define_mode_attr si2dfmodelower
4993 [(V8DF "v8si") (V4DF "v4si")])
4995 (define_insn "float<si2dfmodelower><mode>2<mask_name>"
4996 [(set (match_operand:VF2_512_256 0 "register_operand" "=v")
4997 (float:VF2_512_256 (match_operand:<si2dfmode> 1 "nonimmediate_operand" "vm")))]
4998 "TARGET_AVX && <mask_mode512bit_condition>"
4999 "vcvtdq2pd\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
5000 [(set_attr "type" "ssecvt")
5001 (set_attr "prefix" "maybe_vex")
5002 (set_attr "mode" "<MODE>")])
5004 (define_insn "<floatsuffix>float<sseintvecmodelower><mode>2<mask_name><round_name>"
5005 [(set (match_operand:VF2_AVX512VL 0 "register_operand" "=v")
5006 (any_float:VF2_AVX512VL
5007 (match_operand:<sseintvecmode> 1 "nonimmediate_operand" "<round_constraint>")))]
5009 "vcvt<floatsuffix>qq2pd\t{<round_mask_op2>%1, %0<mask_operand2>|%0<mask_operand2>, %1<round_mask_op2>}"
5010 [(set_attr "type" "ssecvt")
5011 (set_attr "prefix" "evex")
5012 (set_attr "mode" "<MODE>")])
5014 ;; For <floatsuffix>float<sselondveclower><mode> insn patterns
5015 (define_mode_attr qq2pssuff
5016 [(V8SF "") (V4SF "{y}")])
5018 (define_mode_attr sselongvecmode
5019 [(V8SF "V8DI") (V4SF "V4DI")])
5021 (define_mode_attr sselongvecmodelower
5022 [(V8SF "v8di") (V4SF "v4di")])
5024 (define_mode_attr sseintvecmode3
5025 [(V8SF "XI") (V4SF "OI")
5026 (V8DF "OI") (V4DF "TI")])
5028 (define_insn "<floatsuffix>float<sselongvecmodelower><mode>2<mask_name><round_name>"
5029 [(set (match_operand:VF1_128_256VL 0 "register_operand" "=v")
5030 (any_float:VF1_128_256VL
5031 (match_operand:<sselongvecmode> 1 "nonimmediate_operand" "<round_constraint>")))]
5032 "TARGET_AVX512DQ && <round_modev8sf_condition>"
5033 "vcvt<floatsuffix>qq2ps<qq2pssuff>\t{<round_mask_op2>%1, %0<mask_operand2>|%0<mask_operand2>, %1<round_mask_op2>}"
5034 [(set_attr "type" "ssecvt")
5035 (set_attr "prefix" "evex")
5036 (set_attr "mode" "<MODE>")])
5038 (define_insn "*<floatsuffix>floatv2div2sf2"
5039 [(set (match_operand:V4SF 0 "register_operand" "=v")
5041 (any_float:V2SF (match_operand:V2DI 1 "nonimmediate_operand" "vm"))
5042 (const_vector:V2SF [(const_int 0) (const_int 0)])))]
5043 "TARGET_AVX512DQ && TARGET_AVX512VL"
5044 "vcvt<floatsuffix>qq2ps{x}\t{%1, %0|%0, %1}"
5045 [(set_attr "type" "ssecvt")
5046 (set_attr "prefix" "evex")
5047 (set_attr "mode" "V4SF")])
5049 (define_insn "<floatsuffix>floatv2div2sf2_mask"
5050 [(set (match_operand:V4SF 0 "register_operand" "=v")
5053 (any_float:V2SF (match_operand:V2DI 1 "nonimmediate_operand" "vm"))
5055 (match_operand:V4SF 2 "vector_move_operand" "0C")
5056 (parallel [(const_int 0) (const_int 1)]))
5057 (match_operand:QI 3 "register_operand" "Yk"))
5058 (const_vector:V2SF [(const_int 0) (const_int 0)])))]
5059 "TARGET_AVX512DQ && TARGET_AVX512VL"
5060 "vcvt<floatsuffix>qq2ps{x}\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"
5061 [(set_attr "type" "ssecvt")
5062 (set_attr "prefix" "evex")
5063 (set_attr "mode" "V4SF")])
5065 (define_insn "*<floatsuffix>floatv2div2sf2_mask_1"
5066 [(set (match_operand:V4SF 0 "register_operand" "=v")
5069 (any_float:V2SF (match_operand:V2DI 1
5070 "nonimmediate_operand" "vm"))
5071 (const_vector:V2SF [(const_int 0) (const_int 0)])
5072 (match_operand:QI 2 "register_operand" "Yk"))
5073 (const_vector:V2SF [(const_int 0) (const_int 0)])))]
5074 "TARGET_AVX512DQ && TARGET_AVX512VL"
5075 "vcvt<floatsuffix>qq2ps{x}\t{%1, %0%{%2%}%{z%}|%0%{%2%}%{z%}, %1}"
5076 [(set_attr "type" "ssecvt")
5077 (set_attr "prefix" "evex")
5078 (set_attr "mode" "V4SF")])
5080 (define_insn "ufloat<si2dfmodelower><mode>2<mask_name>"
5081 [(set (match_operand:VF2_512_256VL 0 "register_operand" "=v")
5082 (unsigned_float:VF2_512_256VL
5083 (match_operand:<si2dfmode> 1 "nonimmediate_operand" "vm")))]
5085 "vcvtudq2pd\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
5086 [(set_attr "type" "ssecvt")
5087 (set_attr "prefix" "evex")
5088 (set_attr "mode" "<MODE>")])
5090 (define_insn "ufloatv2siv2df2<mask_name>"
5091 [(set (match_operand:V2DF 0 "register_operand" "=v")
5092 (unsigned_float:V2DF
5094 (match_operand:V4SI 1 "nonimmediate_operand" "vm")
5095 (parallel [(const_int 0) (const_int 1)]))))]
5097 "vcvtudq2pd\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
5098 [(set_attr "type" "ssecvt")
5099 (set_attr "prefix" "evex")
5100 (set_attr "mode" "V2DF")])
5102 (define_insn "avx512f_cvtdq2pd512_2"
5103 [(set (match_operand:V8DF 0 "register_operand" "=v")
5106 (match_operand:V16SI 1 "nonimmediate_operand" "vm")
5107 (parallel [(const_int 0) (const_int 1)
5108 (const_int 2) (const_int 3)
5109 (const_int 4) (const_int 5)
5110 (const_int 6) (const_int 7)]))))]
5112 "vcvtdq2pd\t{%t1, %0|%0, %t1}"
5113 [(set_attr "type" "ssecvt")
5114 (set_attr "prefix" "evex")
5115 (set_attr "mode" "V8DF")])
5117 (define_insn "avx_cvtdq2pd256_2"
5118 [(set (match_operand:V4DF 0 "register_operand" "=v")
5121 (match_operand:V8SI 1 "nonimmediate_operand" "vm")
5122 (parallel [(const_int 0) (const_int 1)
5123 (const_int 2) (const_int 3)]))))]
5125 "vcvtdq2pd\t{%x1, %0|%0, %x1}"
5126 [(set_attr "type" "ssecvt")
5127 (set_attr "prefix" "maybe_evex")
5128 (set_attr "mode" "V4DF")])
5130 (define_insn "sse2_cvtdq2pd<mask_name>"
5131 [(set (match_operand:V2DF 0 "register_operand" "=v")
5134 (match_operand:V4SI 1 "nonimmediate_operand" "vm")
5135 (parallel [(const_int 0) (const_int 1)]))))]
5136 "TARGET_SSE2 && <mask_avx512vl_condition>"
5137 "%vcvtdq2pd\t{%1, %0<mask_operand2>|%0<mask_operand2>, %q1}"
5138 [(set_attr "type" "ssecvt")
5139 (set_attr "prefix" "maybe_vex")
5140 (set_attr "mode" "V2DF")])
5142 (define_insn "avx512f_cvtpd2dq512<mask_name><round_name>"
5143 [(set (match_operand:V8SI 0 "register_operand" "=v")
5145 [(match_operand:V8DF 1 "<round_nimm_predicate>" "<round_constraint>")]
5146 UNSPEC_FIX_NOTRUNC))]
5148 "vcvtpd2dq\t{<round_mask_op2>%1, %0<mask_operand2>|%0<mask_operand2>, %1<round_mask_op2>}"
5149 [(set_attr "type" "ssecvt")
5150 (set_attr "prefix" "evex")
5151 (set_attr "mode" "OI")])
5153 (define_insn "avx_cvtpd2dq256<mask_name>"
5154 [(set (match_operand:V4SI 0 "register_operand" "=v")
5155 (unspec:V4SI [(match_operand:V4DF 1 "nonimmediate_operand" "vm")]
5156 UNSPEC_FIX_NOTRUNC))]
5157 "TARGET_AVX && <mask_avx512vl_condition>"
5158 "vcvtpd2dq{y}\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
5159 [(set_attr "type" "ssecvt")
5160 (set_attr "prefix" "<mask_prefix>")
5161 (set_attr "mode" "OI")])
5163 (define_expand "avx_cvtpd2dq256_2"
5164 [(set (match_operand:V8SI 0 "register_operand")
5166 (unspec:V4SI [(match_operand:V4DF 1 "nonimmediate_operand")]
5170 "operands[2] = CONST0_RTX (V4SImode);")
5172 (define_insn "*avx_cvtpd2dq256_2"
5173 [(set (match_operand:V8SI 0 "register_operand" "=v")
5175 (unspec:V4SI [(match_operand:V4DF 1 "nonimmediate_operand" "vm")]
5177 (match_operand:V4SI 2 "const0_operand")))]
5179 "vcvtpd2dq{y}\t{%1, %x0|%x0, %1}"
5180 [(set_attr "type" "ssecvt")
5181 (set_attr "prefix" "vex")
5182 (set_attr "btver2_decode" "vector")
5183 (set_attr "mode" "OI")])
5185 (define_insn "sse2_cvtpd2dq<mask_name>"
5186 [(set (match_operand:V4SI 0 "register_operand" "=v")
5188 (unspec:V2SI [(match_operand:V2DF 1 "vector_operand" "vBm")]
5190 (const_vector:V2SI [(const_int 0) (const_int 0)])))]
5191 "TARGET_SSE2 && <mask_avx512vl_condition>"
5194 return "vcvtpd2dq{x}\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}";
5196 return "cvtpd2dq\t{%1, %0|%0, %1}";
5198 [(set_attr "type" "ssecvt")
5199 (set_attr "prefix_rep" "1")
5200 (set_attr "prefix_data16" "0")
5201 (set_attr "prefix" "maybe_vex")
5202 (set_attr "mode" "TI")
5203 (set_attr "amdfam10_decode" "double")
5204 (set_attr "athlon_decode" "vector")
5205 (set_attr "bdver1_decode" "double")])
5207 ;; For ufix_notrunc* insn patterns
5208 (define_mode_attr pd2udqsuff
5209 [(V8DF "") (V4DF "{y}")])
5211 (define_insn "ufix_notrunc<mode><si2dfmodelower>2<mask_name><round_name>"
5212 [(set (match_operand:<si2dfmode> 0 "register_operand" "=v")
5214 [(match_operand:VF2_512_256VL 1 "nonimmediate_operand" "<round_constraint>")]
5215 UNSPEC_UNSIGNED_FIX_NOTRUNC))]
5217 "vcvtpd2udq<pd2udqsuff>\t{<round_mask_op2>%1, %0<mask_operand2>|%0<mask_operand2>, %1<round_mask_op2>}"
5218 [(set_attr "type" "ssecvt")
5219 (set_attr "prefix" "evex")
5220 (set_attr "mode" "<sseinsnmode>")])
5222 (define_insn "ufix_notruncv2dfv2si2<mask_name>"
5223 [(set (match_operand:V4SI 0 "register_operand" "=v")
5226 [(match_operand:V2DF 1 "nonimmediate_operand" "vm")]
5227 UNSPEC_UNSIGNED_FIX_NOTRUNC)
5228 (const_vector:V2SI [(const_int 0) (const_int 0)])))]
5230 "vcvtpd2udq{x}\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
5231 [(set_attr "type" "ssecvt")
5232 (set_attr "prefix" "evex")
5233 (set_attr "mode" "TI")])
5235 (define_insn "<fixsuffix>fix_truncv8dfv8si2<mask_name><round_saeonly_name>"
5236 [(set (match_operand:V8SI 0 "register_operand" "=v")
5238 (match_operand:V8DF 1 "<round_saeonly_nimm_predicate>" "<round_saeonly_constraint>")))]
5240 "vcvttpd2<fixsuffix>dq\t{<round_saeonly_mask_op2>%1, %0<mask_operand2>|%0<mask_operand2>, %1<round_saeonly_mask_op2>}"
5241 [(set_attr "type" "ssecvt")
5242 (set_attr "prefix" "evex")
5243 (set_attr "mode" "OI")])
5245 (define_insn "ufix_truncv2dfv2si2<mask_name>"
5246 [(set (match_operand:V4SI 0 "register_operand" "=v")
5248 (unsigned_fix:V2SI (match_operand:V2DF 1 "nonimmediate_operand" "vm"))
5249 (const_vector:V2SI [(const_int 0) (const_int 0)])))]
5251 "vcvttpd2udq{x}\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
5252 [(set_attr "type" "ssecvt")
5253 (set_attr "prefix" "evex")
5254 (set_attr "mode" "TI")])
5256 (define_insn "fix_truncv4dfv4si2<mask_name>"
5257 [(set (match_operand:V4SI 0 "register_operand" "=v")
5258 (fix:V4SI (match_operand:V4DF 1 "nonimmediate_operand" "vm")))]
5259 "TARGET_AVX || (TARGET_AVX512VL && TARGET_AVX512F)"
5260 "vcvttpd2dq{y}\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
5261 [(set_attr "type" "ssecvt")
5262 (set_attr "prefix" "maybe_evex")
5263 (set_attr "mode" "OI")])
5265 (define_insn "ufix_truncv4dfv4si2<mask_name>"
5266 [(set (match_operand:V4SI 0 "register_operand" "=v")
5267 (unsigned_fix:V4SI (match_operand:V4DF 1 "nonimmediate_operand" "vm")))]
5268 "TARGET_AVX512VL && TARGET_AVX512F"
5269 "vcvttpd2udq{y}\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
5270 [(set_attr "type" "ssecvt")
5271 (set_attr "prefix" "maybe_evex")
5272 (set_attr "mode" "OI")])
5274 (define_insn "<fixsuffix>fix_trunc<mode><sseintvecmodelower>2<mask_name><round_saeonly_name>"
5275 [(set (match_operand:<sseintvecmode> 0 "register_operand" "=v")
5276 (any_fix:<sseintvecmode>
5277 (match_operand:VF2_AVX512VL 1 "<round_saeonly_nimm_predicate>" "<round_saeonly_constraint>")))]
5278 "TARGET_AVX512DQ && <round_saeonly_mode512bit_condition>"
5279 "vcvttpd2<fixsuffix>qq\t{<round_saeonly_mask_op2>%1, %0<mask_operand2>|%0<mask_operand2>, %1<round_saeonly_mask_op2>}"
5280 [(set_attr "type" "ssecvt")
5281 (set_attr "prefix" "evex")
5282 (set_attr "mode" "<sseintvecmode2>")])
5284 (define_insn "fix_notrunc<mode><sseintvecmodelower>2<mask_name><round_name>"
5285 [(set (match_operand:<sseintvecmode> 0 "register_operand" "=v")
5286 (unspec:<sseintvecmode>
5287 [(match_operand:VF2_AVX512VL 1 "<round_nimm_predicate>" "<round_constraint>")]
5288 UNSPEC_FIX_NOTRUNC))]
5289 "TARGET_AVX512DQ && <round_mode512bit_condition>"
5290 "vcvtpd2qq\t{<round_mask_op2>%1, %0<mask_operand2>|%0<mask_operand2>, %1<round_mask_op2>}"
5291 [(set_attr "type" "ssecvt")
5292 (set_attr "prefix" "evex")
5293 (set_attr "mode" "<sseintvecmode2>")])
5295 (define_insn "ufix_notrunc<mode><sseintvecmodelower>2<mask_name><round_name>"
5296 [(set (match_operand:<sseintvecmode> 0 "register_operand" "=v")
5297 (unspec:<sseintvecmode>
5298 [(match_operand:VF2_AVX512VL 1 "nonimmediate_operand" "<round_constraint>")]
5299 UNSPEC_UNSIGNED_FIX_NOTRUNC))]
5300 "TARGET_AVX512DQ && <round_mode512bit_condition>"
5301 "vcvtpd2uqq\t{<round_mask_op2>%1, %0<mask_operand2>|%0<mask_operand2>, %1<round_mask_op2>}"
5302 [(set_attr "type" "ssecvt")
5303 (set_attr "prefix" "evex")
5304 (set_attr "mode" "<sseintvecmode2>")])
5306 (define_insn "<fixsuffix>fix_trunc<mode><sselongvecmodelower>2<mask_name><round_saeonly_name>"
5307 [(set (match_operand:<sselongvecmode> 0 "register_operand" "=v")
5308 (any_fix:<sselongvecmode>
5309 (match_operand:VF1_128_256VL 1 "<round_saeonly_nimm_predicate>" "<round_saeonly_constraint>")))]
5310 "TARGET_AVX512DQ && <round_saeonly_modev8sf_condition>"
5311 "vcvttps2<fixsuffix>qq\t{<round_saeonly_mask_op2>%1, %0<mask_operand2>|%0<mask_operand2>, %1<round_saeonly_mask_op2>}"
5312 [(set_attr "type" "ssecvt")
5313 (set_attr "prefix" "evex")
5314 (set_attr "mode" "<sseintvecmode3>")])
5316 (define_insn "<fixsuffix>fix_truncv2sfv2di2<mask_name>"
5317 [(set (match_operand:V2DI 0 "register_operand" "=v")
5320 (match_operand:V4SF 1 "nonimmediate_operand" "vm")
5321 (parallel [(const_int 0) (const_int 1)]))))]
5322 "TARGET_AVX512DQ && TARGET_AVX512VL"
5323 "vcvttps2<fixsuffix>qq\t{%1, %0<mask_operand2>|%0<mask_operand2>, %q1}"
5324 [(set_attr "type" "ssecvt")
5325 (set_attr "prefix" "evex")
5326 (set_attr "mode" "TI")])
5328 (define_insn "ufix_trunc<mode><sseintvecmodelower>2<mask_name>"
5329 [(set (match_operand:<sseintvecmode> 0 "register_operand" "=v")
5330 (unsigned_fix:<sseintvecmode>
5331 (match_operand:VF1_128_256VL 1 "nonimmediate_operand" "vm")))]
5333 "vcvttps2udq\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
5334 [(set_attr "type" "ssecvt")
5335 (set_attr "prefix" "evex")
5336 (set_attr "mode" "<sseintvecmode2>")])
5338 (define_expand "avx_cvttpd2dq256_2"
5339 [(set (match_operand:V8SI 0 "register_operand")
5341 (fix:V4SI (match_operand:V4DF 1 "nonimmediate_operand"))
5344 "operands[2] = CONST0_RTX (V4SImode);")
5346 (define_insn "sse2_cvttpd2dq<mask_name>"
5347 [(set (match_operand:V4SI 0 "register_operand" "=v")
5349 (fix:V2SI (match_operand:V2DF 1 "vector_operand" "vBm"))
5350 (const_vector:V2SI [(const_int 0) (const_int 0)])))]
5351 "TARGET_SSE2 && <mask_avx512vl_condition>"
5354 return "vcvttpd2dq{x}\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}";
5356 return "cvttpd2dq\t{%1, %0|%0, %1}";
5358 [(set_attr "type" "ssecvt")
5359 (set_attr "amdfam10_decode" "double")
5360 (set_attr "athlon_decode" "vector")
5361 (set_attr "bdver1_decode" "double")
5362 (set_attr "prefix" "maybe_vex")
5363 (set_attr "mode" "TI")])
5365 (define_insn "sse2_cvtsd2ss<round_name>"
5366 [(set (match_operand:V4SF 0 "register_operand" "=x,x,v")
5369 (float_truncate:V2SF
5370 (match_operand:V2DF 2 "nonimmediate_operand" "x,m,<round_constraint>")))
5371 (match_operand:V4SF 1 "register_operand" "0,0,v")
5375 cvtsd2ss\t{%2, %0|%0, %2}
5376 cvtsd2ss\t{%2, %0|%0, %q2}
5377 vcvtsd2ss\t{<round_op3>%2, %1, %0|%0, %1, %q2<round_op3>}"
5378 [(set_attr "isa" "noavx,noavx,avx")
5379 (set_attr "type" "ssecvt")
5380 (set_attr "athlon_decode" "vector,double,*")
5381 (set_attr "amdfam10_decode" "vector,double,*")
5382 (set_attr "bdver1_decode" "direct,direct,*")
5383 (set_attr "btver2_decode" "double,double,double")
5384 (set_attr "prefix" "orig,orig,<round_prefix>")
5385 (set_attr "mode" "SF")])
5387 (define_insn "*sse2_vd_cvtsd2ss"
5388 [(set (match_operand:V4SF 0 "register_operand" "=x,x,v")
5391 (float_truncate:SF (match_operand:DF 2 "nonimmediate_operand" "x,m,vm")))
5392 (match_operand:V4SF 1 "register_operand" "0,0,v")
5396 cvtsd2ss\t{%2, %0|%0, %2}
5397 cvtsd2ss\t{%2, %0|%0, %2}
5398 vcvtsd2ss\t{%2, %1, %0|%0, %1, %2}"
5399 [(set_attr "isa" "noavx,noavx,avx")
5400 (set_attr "type" "ssecvt")
5401 (set_attr "athlon_decode" "vector,double,*")
5402 (set_attr "amdfam10_decode" "vector,double,*")
5403 (set_attr "bdver1_decode" "direct,direct,*")
5404 (set_attr "btver2_decode" "double,double,double")
5405 (set_attr "prefix" "orig,orig,vex")
5406 (set_attr "mode" "SF")])
5408 (define_insn "sse2_cvtss2sd<round_saeonly_name>"
5409 [(set (match_operand:V2DF 0 "register_operand" "=x,x,v")
5413 (match_operand:V4SF 2 "<round_saeonly_nimm_scalar_predicate>" "x,m,<round_saeonly_constraint>")
5414 (parallel [(const_int 0) (const_int 1)])))
5415 (match_operand:V2DF 1 "register_operand" "0,0,v")
5419 cvtss2sd\t{%2, %0|%0, %2}
5420 cvtss2sd\t{%2, %0|%0, %k2}
5421 vcvtss2sd\t{<round_saeonly_op3>%2, %1, %0|%0, %1, %k2<round_saeonly_op3>}"
5422 [(set_attr "isa" "noavx,noavx,avx")
5423 (set_attr "type" "ssecvt")
5424 (set_attr "amdfam10_decode" "vector,double,*")
5425 (set_attr "athlon_decode" "direct,direct,*")
5426 (set_attr "bdver1_decode" "direct,direct,*")
5427 (set_attr "btver2_decode" "double,double,double")
5428 (set_attr "prefix" "orig,orig,<round_saeonly_prefix>")
5429 (set_attr "mode" "DF")])
5431 (define_insn "*sse2_vd_cvtss2sd"
5432 [(set (match_operand:V2DF 0 "register_operand" "=x,x,v")
5435 (float_extend:DF (match_operand:SF 2 "nonimmediate_operand" "x,m,vm")))
5436 (match_operand:V2DF 1 "register_operand" "0,0,v")
5440 cvtss2sd\t{%2, %0|%0, %2}
5441 cvtss2sd\t{%2, %0|%0, %2}
5442 vcvtss2sd\t{%2, %1, %0|%0, %1, %2}"
5443 [(set_attr "isa" "noavx,noavx,avx")
5444 (set_attr "type" "ssecvt")
5445 (set_attr "amdfam10_decode" "vector,double,*")
5446 (set_attr "athlon_decode" "direct,direct,*")
5447 (set_attr "bdver1_decode" "direct,direct,*")
5448 (set_attr "btver2_decode" "double,double,double")
5449 (set_attr "prefix" "orig,orig,vex")
5450 (set_attr "mode" "DF")])
5452 (define_insn "<mask_codefor>avx512f_cvtpd2ps512<mask_name><round_name>"
5453 [(set (match_operand:V8SF 0 "register_operand" "=v")
5454 (float_truncate:V8SF
5455 (match_operand:V8DF 1 "<round_nimm_predicate>" "<round_constraint>")))]
5457 "vcvtpd2ps\t{<round_mask_op2>%1, %0<mask_operand2>|%0<mask_operand2>, %1<round_mask_op2>}"
5458 [(set_attr "type" "ssecvt")
5459 (set_attr "prefix" "evex")
5460 (set_attr "mode" "V8SF")])
5462 (define_insn "avx_cvtpd2ps256<mask_name>"
5463 [(set (match_operand:V4SF 0 "register_operand" "=v")
5464 (float_truncate:V4SF
5465 (match_operand:V4DF 1 "nonimmediate_operand" "vm")))]
5466 "TARGET_AVX && <mask_avx512vl_condition>"
5467 "vcvtpd2ps{y}\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
5468 [(set_attr "type" "ssecvt")
5469 (set_attr "prefix" "maybe_evex")
5470 (set_attr "btver2_decode" "vector")
5471 (set_attr "mode" "V4SF")])
5473 (define_expand "sse2_cvtpd2ps"
5474 [(set (match_operand:V4SF 0 "register_operand")
5476 (float_truncate:V2SF
5477 (match_operand:V2DF 1 "vector_operand"))
5480 "operands[2] = CONST0_RTX (V2SFmode);")
5482 (define_expand "sse2_cvtpd2ps_mask"
5483 [(set (match_operand:V4SF 0 "register_operand")
5486 (float_truncate:V2SF
5487 (match_operand:V2DF 1 "vector_operand"))
5489 (match_operand:V4SF 2 "register_operand")
5490 (match_operand:QI 3 "register_operand")))]
5492 "operands[4] = CONST0_RTX (V2SFmode);")
5494 (define_insn "*sse2_cvtpd2ps<mask_name>"
5495 [(set (match_operand:V4SF 0 "register_operand" "=v")
5497 (float_truncate:V2SF
5498 (match_operand:V2DF 1 "vector_operand" "vBm"))
5499 (match_operand:V2SF 2 "const0_operand")))]
5500 "TARGET_SSE2 && <mask_avx512vl_condition>"
5503 return "vcvtpd2ps{x}\t{%1, %0<mask_operand3>|%0<mask_operand3>, %1}";
5505 return "cvtpd2ps\t{%1, %0|%0, %1}";
5507 [(set_attr "type" "ssecvt")
5508 (set_attr "amdfam10_decode" "double")
5509 (set_attr "athlon_decode" "vector")
5510 (set_attr "bdver1_decode" "double")
5511 (set_attr "prefix_data16" "1")
5512 (set_attr "prefix" "maybe_vex")
5513 (set_attr "mode" "V4SF")])
5515 ;; For <sse2_avx_avx512f>_cvtps2pd<avxsizesuffix> insn pattern
5516 (define_mode_attr sf2dfmode
5517 [(V8DF "V8SF") (V4DF "V4SF")])
5519 (define_insn "<sse2_avx_avx512f>_cvtps2pd<avxsizesuffix><mask_name><round_saeonly_name>"
5520 [(set (match_operand:VF2_512_256 0 "register_operand" "=v")
5521 (float_extend:VF2_512_256
5522 (match_operand:<sf2dfmode> 1 "<round_saeonly_nimm_predicate>" "<round_saeonly_constraint>")))]
5523 "TARGET_AVX && <mask_mode512bit_condition> && <round_saeonly_mode512bit_condition>"
5524 "vcvtps2pd\t{<round_saeonly_mask_op2>%1, %0<mask_operand2>|%0<mask_operand2>, %1<round_saeonly_mask_op2>}"
5525 [(set_attr "type" "ssecvt")
5526 (set_attr "prefix" "maybe_vex")
5527 (set_attr "mode" "<MODE>")])
5529 (define_insn "*avx_cvtps2pd256_2"
5530 [(set (match_operand:V4DF 0 "register_operand" "=v")
5533 (match_operand:V8SF 1 "nonimmediate_operand" "vm")
5534 (parallel [(const_int 0) (const_int 1)
5535 (const_int 2) (const_int 3)]))))]
5537 "vcvtps2pd\t{%x1, %0|%0, %x1}"
5538 [(set_attr "type" "ssecvt")
5539 (set_attr "prefix" "vex")
5540 (set_attr "mode" "V4DF")])
5542 (define_insn "vec_unpacks_lo_v16sf"
5543 [(set (match_operand:V8DF 0 "register_operand" "=v")
5546 (match_operand:V16SF 1 "nonimmediate_operand" "vm")
5547 (parallel [(const_int 0) (const_int 1)
5548 (const_int 2) (const_int 3)
5549 (const_int 4) (const_int 5)
5550 (const_int 6) (const_int 7)]))))]
5552 "vcvtps2pd\t{%t1, %0|%0, %t1}"
5553 [(set_attr "type" "ssecvt")
5554 (set_attr "prefix" "evex")
5555 (set_attr "mode" "V8DF")])
5557 (define_insn "<avx512>_cvt<ssemodesuffix>2mask<mode>"
5558 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=Yk")
5559 (unspec:<avx512fmaskmode>
5560 [(match_operand:VI12_AVX512VL 1 "register_operand" "v")]
5561 UNSPEC_CVTINT2MASK))]
5563 "vpmov<ssemodesuffix>2m\t{%1, %0|%0, %1}"
5564 [(set_attr "prefix" "evex")
5565 (set_attr "mode" "<sseinsnmode>")])
5567 (define_insn "<avx512>_cvt<ssemodesuffix>2mask<mode>"
5568 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=Yk")
5569 (unspec:<avx512fmaskmode>
5570 [(match_operand:VI48_AVX512VL 1 "register_operand" "v")]
5571 UNSPEC_CVTINT2MASK))]
5573 "vpmov<ssemodesuffix>2m\t{%1, %0|%0, %1}"
5574 [(set_attr "prefix" "evex")
5575 (set_attr "mode" "<sseinsnmode>")])
5577 (define_expand "<avx512>_cvtmask2<ssemodesuffix><mode>"
5578 [(set (match_operand:VI12_AVX512VL 0 "register_operand")
5579 (vec_merge:VI12_AVX512VL
5582 (match_operand:<avx512fmaskmode> 1 "register_operand")))]
5585 operands[2] = CONSTM1_RTX (<MODE>mode);
5586 operands[3] = CONST0_RTX (<MODE>mode);
5589 (define_insn "*<avx512>_cvtmask2<ssemodesuffix><mode>"
5590 [(set (match_operand:VI12_AVX512VL 0 "register_operand" "=v")
5591 (vec_merge:VI12_AVX512VL
5592 (match_operand:VI12_AVX512VL 2 "vector_all_ones_operand")
5593 (match_operand:VI12_AVX512VL 3 "const0_operand")
5594 (match_operand:<avx512fmaskmode> 1 "register_operand" "Yk")))]
5596 "vpmovm2<ssemodesuffix>\t{%1, %0|%0, %1}"
5597 [(set_attr "prefix" "evex")
5598 (set_attr "mode" "<sseinsnmode>")])
5600 (define_expand "<avx512>_cvtmask2<ssemodesuffix><mode>"
5601 [(set (match_operand:VI48_AVX512VL 0 "register_operand")
5602 (vec_merge:VI48_AVX512VL
5605 (match_operand:<avx512fmaskmode> 1 "register_operand")))]
5608 operands[2] = CONSTM1_RTX (<MODE>mode);
5609 operands[3] = CONST0_RTX (<MODE>mode);
5612 (define_insn "*<avx512>_cvtmask2<ssemodesuffix><mode>"
5613 [(set (match_operand:VI48_AVX512VL 0 "register_operand" "=v")
5614 (vec_merge:VI48_AVX512VL
5615 (match_operand:VI48_AVX512VL 2 "vector_all_ones_operand")
5616 (match_operand:VI48_AVX512VL 3 "const0_operand")
5617 (match_operand:<avx512fmaskmode> 1 "register_operand" "Yk")))]
5619 "vpmovm2<ssemodesuffix>\t{%1, %0|%0, %1}"
5620 [(set_attr "prefix" "evex")
5621 (set_attr "mode" "<sseinsnmode>")])
5623 (define_insn "sse2_cvtps2pd<mask_name>"
5624 [(set (match_operand:V2DF 0 "register_operand" "=v")
5627 (match_operand:V4SF 1 "vector_operand" "vm")
5628 (parallel [(const_int 0) (const_int 1)]))))]
5629 "TARGET_SSE2 && <mask_avx512vl_condition>"
5630 "%vcvtps2pd\t{%1, %0<mask_operand2>|%0<mask_operand2>, %q1}"
5631 [(set_attr "type" "ssecvt")
5632 (set_attr "amdfam10_decode" "direct")
5633 (set_attr "athlon_decode" "double")
5634 (set_attr "bdver1_decode" "double")
5635 (set_attr "prefix_data16" "0")
5636 (set_attr "prefix" "maybe_vex")
5637 (set_attr "mode" "V2DF")])
5639 (define_expand "vec_unpacks_hi_v4sf"
5644 (match_operand:V4SF 1 "vector_operand"))
5645 (parallel [(const_int 6) (const_int 7)
5646 (const_int 2) (const_int 3)])))
5647 (set (match_operand:V2DF 0 "register_operand")
5651 (parallel [(const_int 0) (const_int 1)]))))]
5653 "operands[2] = gen_reg_rtx (V4SFmode);")
5655 (define_expand "vec_unpacks_hi_v8sf"
5658 (match_operand:V8SF 1 "register_operand")
5659 (parallel [(const_int 4) (const_int 5)
5660 (const_int 6) (const_int 7)])))
5661 (set (match_operand:V4DF 0 "register_operand")
5665 "operands[2] = gen_reg_rtx (V4SFmode);")
5667 (define_expand "vec_unpacks_hi_v16sf"
5670 (match_operand:V16SF 1 "register_operand")
5671 (parallel [(const_int 8) (const_int 9)
5672 (const_int 10) (const_int 11)
5673 (const_int 12) (const_int 13)
5674 (const_int 14) (const_int 15)])))
5675 (set (match_operand:V8DF 0 "register_operand")
5679 "operands[2] = gen_reg_rtx (V8SFmode);")
5681 (define_expand "vec_unpacks_lo_v4sf"
5682 [(set (match_operand:V2DF 0 "register_operand")
5685 (match_operand:V4SF 1 "vector_operand")
5686 (parallel [(const_int 0) (const_int 1)]))))]
5689 (define_expand "vec_unpacks_lo_v8sf"
5690 [(set (match_operand:V4DF 0 "register_operand")
5693 (match_operand:V8SF 1 "nonimmediate_operand")
5694 (parallel [(const_int 0) (const_int 1)
5695 (const_int 2) (const_int 3)]))))]
5698 (define_mode_attr sseunpackfltmode
5699 [(V8HI "V4SF") (V4SI "V2DF") (V16HI "V8SF")
5700 (V8SI "V4DF") (V32HI "V16SF") (V16SI "V8DF")])
5702 (define_expand "vec_unpacks_float_hi_<mode>"
5703 [(match_operand:<sseunpackfltmode> 0 "register_operand")
5704 (match_operand:VI2_AVX512F 1 "register_operand")]
5707 rtx tmp = gen_reg_rtx (<sseunpackmode>mode);
5709 emit_insn (gen_vec_unpacks_hi_<mode> (tmp, operands[1]));
5710 emit_insn (gen_rtx_SET (operands[0],
5711 gen_rtx_FLOAT (<sseunpackfltmode>mode, tmp)));
5715 (define_expand "vec_unpacks_float_lo_<mode>"
5716 [(match_operand:<sseunpackfltmode> 0 "register_operand")
5717 (match_operand:VI2_AVX512F 1 "register_operand")]
5720 rtx tmp = gen_reg_rtx (<sseunpackmode>mode);
5722 emit_insn (gen_vec_unpacks_lo_<mode> (tmp, operands[1]));
5723 emit_insn (gen_rtx_SET (operands[0],
5724 gen_rtx_FLOAT (<sseunpackfltmode>mode, tmp)));
5728 (define_expand "vec_unpacku_float_hi_<mode>"
5729 [(match_operand:<sseunpackfltmode> 0 "register_operand")
5730 (match_operand:VI2_AVX512F 1 "register_operand")]
5733 rtx tmp = gen_reg_rtx (<sseunpackmode>mode);
5735 emit_insn (gen_vec_unpacku_hi_<mode> (tmp, operands[1]));
5736 emit_insn (gen_rtx_SET (operands[0],
5737 gen_rtx_FLOAT (<sseunpackfltmode>mode, tmp)));
5741 (define_expand "vec_unpacku_float_lo_<mode>"
5742 [(match_operand:<sseunpackfltmode> 0 "register_operand")
5743 (match_operand:VI2_AVX512F 1 "register_operand")]
5746 rtx tmp = gen_reg_rtx (<sseunpackmode>mode);
5748 emit_insn (gen_vec_unpacku_lo_<mode> (tmp, operands[1]));
5749 emit_insn (gen_rtx_SET (operands[0],
5750 gen_rtx_FLOAT (<sseunpackfltmode>mode, tmp)));
5754 (define_expand "vec_unpacks_float_hi_v4si"
5757 (match_operand:V4SI 1 "vector_operand")
5758 (parallel [(const_int 2) (const_int 3)
5759 (const_int 2) (const_int 3)])))
5760 (set (match_operand:V2DF 0 "register_operand")
5764 (parallel [(const_int 0) (const_int 1)]))))]
5766 "operands[2] = gen_reg_rtx (V4SImode);")
5768 (define_expand "vec_unpacks_float_lo_v4si"
5769 [(set (match_operand:V2DF 0 "register_operand")
5772 (match_operand:V4SI 1 "vector_operand")
5773 (parallel [(const_int 0) (const_int 1)]))))]
5776 (define_expand "vec_unpacks_float_hi_v8si"
5779 (match_operand:V8SI 1 "vector_operand")
5780 (parallel [(const_int 4) (const_int 5)
5781 (const_int 6) (const_int 7)])))
5782 (set (match_operand:V4DF 0 "register_operand")
5786 "operands[2] = gen_reg_rtx (V4SImode);")
5788 (define_expand "vec_unpacks_float_lo_v8si"
5789 [(set (match_operand:V4DF 0 "register_operand")
5792 (match_operand:V8SI 1 "nonimmediate_operand")
5793 (parallel [(const_int 0) (const_int 1)
5794 (const_int 2) (const_int 3)]))))]
5797 (define_expand "vec_unpacks_float_hi_v16si"
5800 (match_operand:V16SI 1 "nonimmediate_operand")
5801 (parallel [(const_int 8) (const_int 9)
5802 (const_int 10) (const_int 11)
5803 (const_int 12) (const_int 13)
5804 (const_int 14) (const_int 15)])))
5805 (set (match_operand:V8DF 0 "register_operand")
5809 "operands[2] = gen_reg_rtx (V8SImode);")
5811 (define_expand "vec_unpacks_float_lo_v16si"
5812 [(set (match_operand:V8DF 0 "register_operand")
5815 (match_operand:V16SI 1 "nonimmediate_operand")
5816 (parallel [(const_int 0) (const_int 1)
5817 (const_int 2) (const_int 3)
5818 (const_int 4) (const_int 5)
5819 (const_int 6) (const_int 7)]))))]
5822 (define_expand "vec_unpacku_float_hi_v4si"
5825 (match_operand:V4SI 1 "vector_operand")
5826 (parallel [(const_int 2) (const_int 3)
5827 (const_int 2) (const_int 3)])))
5832 (parallel [(const_int 0) (const_int 1)]))))
5834 (lt:V2DF (match_dup 6) (match_dup 3)))
5836 (and:V2DF (match_dup 7) (match_dup 4)))
5837 (set (match_operand:V2DF 0 "register_operand")
5838 (plus:V2DF (match_dup 6) (match_dup 8)))]
5841 REAL_VALUE_TYPE TWO32r;
5845 real_ldexp (&TWO32r, &dconst1, 32);
5846 x = const_double_from_real_value (TWO32r, DFmode);
5848 operands[3] = force_reg (V2DFmode, CONST0_RTX (V2DFmode));
5849 operands[4] = force_reg (V2DFmode,
5850 ix86_build_const_vector (V2DFmode, 1, x));
5852 operands[5] = gen_reg_rtx (V4SImode);
5854 for (i = 6; i < 9; i++)
5855 operands[i] = gen_reg_rtx (V2DFmode);
5858 (define_expand "vec_unpacku_float_lo_v4si"
5862 (match_operand:V4SI 1 "vector_operand")
5863 (parallel [(const_int 0) (const_int 1)]))))
5865 (lt:V2DF (match_dup 5) (match_dup 3)))
5867 (and:V2DF (match_dup 6) (match_dup 4)))
5868 (set (match_operand:V2DF 0 "register_operand")
5869 (plus:V2DF (match_dup 5) (match_dup 7)))]
5872 REAL_VALUE_TYPE TWO32r;
5876 real_ldexp (&TWO32r, &dconst1, 32);
5877 x = const_double_from_real_value (TWO32r, DFmode);
5879 operands[3] = force_reg (V2DFmode, CONST0_RTX (V2DFmode));
5880 operands[4] = force_reg (V2DFmode,
5881 ix86_build_const_vector (V2DFmode, 1, x));
5883 for (i = 5; i < 8; i++)
5884 operands[i] = gen_reg_rtx (V2DFmode);
5887 (define_expand "vec_unpacku_float_hi_v8si"
5888 [(match_operand:V4DF 0 "register_operand")
5889 (match_operand:V8SI 1 "register_operand")]
5892 REAL_VALUE_TYPE TWO32r;
5896 real_ldexp (&TWO32r, &dconst1, 32);
5897 x = const_double_from_real_value (TWO32r, DFmode);
5899 tmp[0] = force_reg (V4DFmode, CONST0_RTX (V4DFmode));
5900 tmp[1] = force_reg (V4DFmode, ix86_build_const_vector (V4DFmode, 1, x));
5901 tmp[5] = gen_reg_rtx (V4SImode);
5903 for (i = 2; i < 5; i++)
5904 tmp[i] = gen_reg_rtx (V4DFmode);
5905 emit_insn (gen_vec_extract_hi_v8si (tmp[5], operands[1]));
5906 emit_insn (gen_floatv4siv4df2 (tmp[2], tmp[5]));
5907 emit_insn (gen_rtx_SET (tmp[3], gen_rtx_LT (V4DFmode, tmp[2], tmp[0])));
5908 emit_insn (gen_andv4df3 (tmp[4], tmp[3], tmp[1]));
5909 emit_insn (gen_addv4df3 (operands[0], tmp[2], tmp[4]));
5913 (define_expand "vec_unpacku_float_hi_v16si"
5914 [(match_operand:V8DF 0 "register_operand")
5915 (match_operand:V16SI 1 "register_operand")]
5918 REAL_VALUE_TYPE TWO32r;
5921 real_ldexp (&TWO32r, &dconst1, 32);
5922 x = const_double_from_real_value (TWO32r, DFmode);
5924 tmp[0] = force_reg (V8DFmode, CONST0_RTX (V8DFmode));
5925 tmp[1] = force_reg (V8DFmode, ix86_build_const_vector (V8DFmode, 1, x));
5926 tmp[2] = gen_reg_rtx (V8DFmode);
5927 tmp[3] = gen_reg_rtx (V8SImode);
5928 k = gen_reg_rtx (QImode);
5930 emit_insn (gen_vec_extract_hi_v16si (tmp[3], operands[1]));
5931 emit_insn (gen_floatv8siv8df2 (tmp[2], tmp[3]));
5932 emit_insn (gen_rtx_SET (k, gen_rtx_LT (QImode, tmp[2], tmp[0])));
5933 emit_insn (gen_addv8df3_mask (tmp[2], tmp[2], tmp[1], tmp[2], k));
5934 emit_move_insn (operands[0], tmp[2]);
5938 (define_expand "vec_unpacku_float_lo_v8si"
5939 [(match_operand:V4DF 0 "register_operand")
5940 (match_operand:V8SI 1 "nonimmediate_operand")]
5943 REAL_VALUE_TYPE TWO32r;
5947 real_ldexp (&TWO32r, &dconst1, 32);
5948 x = const_double_from_real_value (TWO32r, DFmode);
5950 tmp[0] = force_reg (V4DFmode, CONST0_RTX (V4DFmode));
5951 tmp[1] = force_reg (V4DFmode, ix86_build_const_vector (V4DFmode, 1, x));
5953 for (i = 2; i < 5; i++)
5954 tmp[i] = gen_reg_rtx (V4DFmode);
5955 emit_insn (gen_avx_cvtdq2pd256_2 (tmp[2], operands[1]));
5956 emit_insn (gen_rtx_SET (tmp[3], gen_rtx_LT (V4DFmode, tmp[2], tmp[0])));
5957 emit_insn (gen_andv4df3 (tmp[4], tmp[3], tmp[1]));
5958 emit_insn (gen_addv4df3 (operands[0], tmp[2], tmp[4]));
5962 (define_expand "vec_unpacku_float_lo_v16si"
5963 [(match_operand:V8DF 0 "register_operand")
5964 (match_operand:V16SI 1 "nonimmediate_operand")]
5967 REAL_VALUE_TYPE TWO32r;
5970 real_ldexp (&TWO32r, &dconst1, 32);
5971 x = const_double_from_real_value (TWO32r, DFmode);
5973 tmp[0] = force_reg (V8DFmode, CONST0_RTX (V8DFmode));
5974 tmp[1] = force_reg (V8DFmode, ix86_build_const_vector (V8DFmode, 1, x));
5975 tmp[2] = gen_reg_rtx (V8DFmode);
5976 k = gen_reg_rtx (QImode);
5978 emit_insn (gen_avx512f_cvtdq2pd512_2 (tmp[2], operands[1]));
5979 emit_insn (gen_rtx_SET (k, gen_rtx_LT (QImode, tmp[2], tmp[0])));
5980 emit_insn (gen_addv8df3_mask (tmp[2], tmp[2], tmp[1], tmp[2], k));
5981 emit_move_insn (operands[0], tmp[2]);
5985 (define_expand "vec_pack_trunc_<mode>"
5987 (float_truncate:<sf2dfmode>
5988 (match_operand:VF2_512_256 1 "nonimmediate_operand")))
5990 (float_truncate:<sf2dfmode>
5991 (match_operand:VF2_512_256 2 "nonimmediate_operand")))
5992 (set (match_operand:<ssePSmode> 0 "register_operand")
5993 (vec_concat:<ssePSmode>
5998 operands[3] = gen_reg_rtx (<sf2dfmode>mode);
5999 operands[4] = gen_reg_rtx (<sf2dfmode>mode);
6002 (define_expand "vec_pack_trunc_v2df"
6003 [(match_operand:V4SF 0 "register_operand")
6004 (match_operand:V2DF 1 "vector_operand")
6005 (match_operand:V2DF 2 "vector_operand")]
6010 if (TARGET_AVX && !TARGET_PREFER_AVX128 && optimize_insn_for_speed_p ())
6012 tmp0 = gen_reg_rtx (V4DFmode);
6013 tmp1 = force_reg (V2DFmode, operands[1]);
6015 emit_insn (gen_avx_vec_concatv4df (tmp0, tmp1, operands[2]));
6016 emit_insn (gen_avx_cvtpd2ps256 (operands[0], tmp0));
6020 tmp0 = gen_reg_rtx (V4SFmode);
6021 tmp1 = gen_reg_rtx (V4SFmode);
6023 emit_insn (gen_sse2_cvtpd2ps (tmp0, operands[1]));
6024 emit_insn (gen_sse2_cvtpd2ps (tmp1, operands[2]));
6025 emit_insn (gen_sse_movlhps (operands[0], tmp0, tmp1));
6030 (define_expand "vec_pack_sfix_trunc_v8df"
6031 [(match_operand:V16SI 0 "register_operand")
6032 (match_operand:V8DF 1 "nonimmediate_operand")
6033 (match_operand:V8DF 2 "nonimmediate_operand")]
6038 r1 = gen_reg_rtx (V8SImode);
6039 r2 = gen_reg_rtx (V8SImode);
6041 emit_insn (gen_fix_truncv8dfv8si2 (r1, operands[1]));
6042 emit_insn (gen_fix_truncv8dfv8si2 (r2, operands[2]));
6043 emit_insn (gen_avx_vec_concatv16si (operands[0], r1, r2));
6047 (define_expand "vec_pack_sfix_trunc_v4df"
6048 [(match_operand:V8SI 0 "register_operand")
6049 (match_operand:V4DF 1 "nonimmediate_operand")
6050 (match_operand:V4DF 2 "nonimmediate_operand")]
6055 r1 = gen_reg_rtx (V4SImode);
6056 r2 = gen_reg_rtx (V4SImode);
6058 emit_insn (gen_fix_truncv4dfv4si2 (r1, operands[1]));
6059 emit_insn (gen_fix_truncv4dfv4si2 (r2, operands[2]));
6060 emit_insn (gen_avx_vec_concatv8si (operands[0], r1, r2));
6064 (define_expand "vec_pack_sfix_trunc_v2df"
6065 [(match_operand:V4SI 0 "register_operand")
6066 (match_operand:V2DF 1 "vector_operand")
6067 (match_operand:V2DF 2 "vector_operand")]
6070 rtx tmp0, tmp1, tmp2;
6072 if (TARGET_AVX && !TARGET_PREFER_AVX128 && optimize_insn_for_speed_p ())
6074 tmp0 = gen_reg_rtx (V4DFmode);
6075 tmp1 = force_reg (V2DFmode, operands[1]);
6077 emit_insn (gen_avx_vec_concatv4df (tmp0, tmp1, operands[2]));
6078 emit_insn (gen_fix_truncv4dfv4si2 (operands[0], tmp0));
6082 tmp0 = gen_reg_rtx (V4SImode);
6083 tmp1 = gen_reg_rtx (V4SImode);
6084 tmp2 = gen_reg_rtx (V2DImode);
6086 emit_insn (gen_sse2_cvttpd2dq (tmp0, operands[1]));
6087 emit_insn (gen_sse2_cvttpd2dq (tmp1, operands[2]));
6088 emit_insn (gen_vec_interleave_lowv2di (tmp2,
6089 gen_lowpart (V2DImode, tmp0),
6090 gen_lowpart (V2DImode, tmp1)));
6091 emit_move_insn (operands[0], gen_lowpart (V4SImode, tmp2));
6096 (define_mode_attr ssepackfltmode
6097 [(V8DF "V16SI") (V4DF "V8SI") (V2DF "V4SI")])
6099 (define_expand "vec_pack_ufix_trunc_<mode>"
6100 [(match_operand:<ssepackfltmode> 0 "register_operand")
6101 (match_operand:VF2 1 "register_operand")
6102 (match_operand:VF2 2 "register_operand")]
6105 if (<MODE>mode == V8DFmode)
6109 r1 = gen_reg_rtx (V8SImode);
6110 r2 = gen_reg_rtx (V8SImode);
6112 emit_insn (gen_ufix_truncv8dfv8si2 (r1, operands[1]));
6113 emit_insn (gen_ufix_truncv8dfv8si2 (r2, operands[2]));
6114 emit_insn (gen_avx_vec_concatv16si (operands[0], r1, r2));
6119 tmp[0] = ix86_expand_adjust_ufix_to_sfix_si (operands[1], &tmp[2]);
6120 tmp[1] = ix86_expand_adjust_ufix_to_sfix_si (operands[2], &tmp[3]);
6121 tmp[4] = gen_reg_rtx (<ssepackfltmode>mode);
6122 emit_insn (gen_vec_pack_sfix_trunc_<mode> (tmp[4], tmp[0], tmp[1]));
6123 if (<ssepackfltmode>mode == V4SImode || TARGET_AVX2)
6125 tmp[5] = gen_reg_rtx (<ssepackfltmode>mode);
6126 ix86_expand_vec_extract_even_odd (tmp[5], tmp[2], tmp[3], 0);
6130 tmp[5] = gen_reg_rtx (V8SFmode);
6131 ix86_expand_vec_extract_even_odd (tmp[5],
6132 gen_lowpart (V8SFmode, tmp[2]),
6133 gen_lowpart (V8SFmode, tmp[3]), 0);
6134 tmp[5] = gen_lowpart (V8SImode, tmp[5]);
6136 tmp[6] = expand_simple_binop (<ssepackfltmode>mode, XOR, tmp[4], tmp[5],
6137 operands[0], 0, OPTAB_DIRECT);
6138 if (tmp[6] != operands[0])
6139 emit_move_insn (operands[0], tmp[6]);
6145 (define_expand "avx512f_vec_pack_sfix_v8df"
6146 [(match_operand:V16SI 0 "register_operand")
6147 (match_operand:V8DF 1 "nonimmediate_operand")
6148 (match_operand:V8DF 2 "nonimmediate_operand")]
6153 r1 = gen_reg_rtx (V8SImode);
6154 r2 = gen_reg_rtx (V8SImode);
6156 emit_insn (gen_avx512f_cvtpd2dq512 (r1, operands[1]));
6157 emit_insn (gen_avx512f_cvtpd2dq512 (r2, operands[2]));
6158 emit_insn (gen_avx_vec_concatv16si (operands[0], r1, r2));
6162 (define_expand "vec_pack_sfix_v4df"
6163 [(match_operand:V8SI 0 "register_operand")
6164 (match_operand:V4DF 1 "nonimmediate_operand")
6165 (match_operand:V4DF 2 "nonimmediate_operand")]
6170 r1 = gen_reg_rtx (V4SImode);
6171 r2 = gen_reg_rtx (V4SImode);
6173 emit_insn (gen_avx_cvtpd2dq256 (r1, operands[1]));
6174 emit_insn (gen_avx_cvtpd2dq256 (r2, operands[2]));
6175 emit_insn (gen_avx_vec_concatv8si (operands[0], r1, r2));
6179 (define_expand "vec_pack_sfix_v2df"
6180 [(match_operand:V4SI 0 "register_operand")
6181 (match_operand:V2DF 1 "vector_operand")
6182 (match_operand:V2DF 2 "vector_operand")]
6185 rtx tmp0, tmp1, tmp2;
6187 if (TARGET_AVX && !TARGET_PREFER_AVX128 && optimize_insn_for_speed_p ())
6189 tmp0 = gen_reg_rtx (V4DFmode);
6190 tmp1 = force_reg (V2DFmode, operands[1]);
6192 emit_insn (gen_avx_vec_concatv4df (tmp0, tmp1, operands[2]));
6193 emit_insn (gen_avx_cvtpd2dq256 (operands[0], tmp0));
6197 tmp0 = gen_reg_rtx (V4SImode);
6198 tmp1 = gen_reg_rtx (V4SImode);
6199 tmp2 = gen_reg_rtx (V2DImode);
6201 emit_insn (gen_sse2_cvtpd2dq (tmp0, operands[1]));
6202 emit_insn (gen_sse2_cvtpd2dq (tmp1, operands[2]));
6203 emit_insn (gen_vec_interleave_lowv2di (tmp2,
6204 gen_lowpart (V2DImode, tmp0),
6205 gen_lowpart (V2DImode, tmp1)));
6206 emit_move_insn (operands[0], gen_lowpart (V4SImode, tmp2));
6211 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
6213 ;; Parallel single-precision floating point element swizzling
6215 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
6217 (define_expand "sse_movhlps_exp"
6218 [(set (match_operand:V4SF 0 "nonimmediate_operand")
6221 (match_operand:V4SF 1 "nonimmediate_operand")
6222 (match_operand:V4SF 2 "nonimmediate_operand"))
6223 (parallel [(const_int 6)
6229 rtx dst = ix86_fixup_binary_operands (UNKNOWN, V4SFmode, operands);
6231 emit_insn (gen_sse_movhlps (dst, operands[1], operands[2]));
6233 /* Fix up the destination if needed. */
6234 if (dst != operands[0])
6235 emit_move_insn (operands[0], dst);
6240 (define_insn "sse_movhlps"
6241 [(set (match_operand:V4SF 0 "nonimmediate_operand" "=x,v,x,v,m")
6244 (match_operand:V4SF 1 "nonimmediate_operand" " 0,v,0,v,0")
6245 (match_operand:V4SF 2 "nonimmediate_operand" " x,v,o,o,v"))
6246 (parallel [(const_int 6)
6250 "TARGET_SSE && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
6252 movhlps\t{%2, %0|%0, %2}
6253 vmovhlps\t{%2, %1, %0|%0, %1, %2}
6254 movlps\t{%H2, %0|%0, %H2}
6255 vmovlps\t{%H2, %1, %0|%0, %1, %H2}
6256 %vmovhps\t{%2, %0|%q0, %2}"
6257 [(set_attr "isa" "noavx,avx,noavx,avx,*")
6258 (set_attr "type" "ssemov")
6259 (set_attr "prefix" "orig,maybe_evex,orig,maybe_evex,maybe_vex")
6260 (set_attr "mode" "V4SF,V4SF,V2SF,V2SF,V2SF")])
6262 (define_expand "sse_movlhps_exp"
6263 [(set (match_operand:V4SF 0 "nonimmediate_operand")
6266 (match_operand:V4SF 1 "nonimmediate_operand")
6267 (match_operand:V4SF 2 "nonimmediate_operand"))
6268 (parallel [(const_int 0)
6274 rtx dst = ix86_fixup_binary_operands (UNKNOWN, V4SFmode, operands);
6276 emit_insn (gen_sse_movlhps (dst, operands[1], operands[2]));
6278 /* Fix up the destination if needed. */
6279 if (dst != operands[0])
6280 emit_move_insn (operands[0], dst);
6285 (define_insn "sse_movlhps"
6286 [(set (match_operand:V4SF 0 "nonimmediate_operand" "=x,v,x,v,o")
6289 (match_operand:V4SF 1 "nonimmediate_operand" " 0,v,0,v,0")
6290 (match_operand:V4SF 2 "nonimmediate_operand" " x,v,m,v,v"))
6291 (parallel [(const_int 0)
6295 "TARGET_SSE && ix86_binary_operator_ok (UNKNOWN, V4SFmode, operands)"
6297 movlhps\t{%2, %0|%0, %2}
6298 vmovlhps\t{%2, %1, %0|%0, %1, %2}
6299 movhps\t{%2, %0|%0, %q2}
6300 vmovhps\t{%2, %1, %0|%0, %1, %q2}
6301 %vmovlps\t{%2, %H0|%H0, %2}"
6302 [(set_attr "isa" "noavx,avx,noavx,avx,*")
6303 (set_attr "type" "ssemov")
6304 (set_attr "prefix" "orig,maybe_evex,orig,maybe_evex,maybe_vex")
6305 (set_attr "mode" "V4SF,V4SF,V2SF,V2SF,V2SF")])
6307 (define_insn "<mask_codefor>avx512f_unpckhps512<mask_name>"
6308 [(set (match_operand:V16SF 0 "register_operand" "=v")
6311 (match_operand:V16SF 1 "register_operand" "v")
6312 (match_operand:V16SF 2 "nonimmediate_operand" "vm"))
6313 (parallel [(const_int 2) (const_int 18)
6314 (const_int 3) (const_int 19)
6315 (const_int 6) (const_int 22)
6316 (const_int 7) (const_int 23)
6317 (const_int 10) (const_int 26)
6318 (const_int 11) (const_int 27)
6319 (const_int 14) (const_int 30)
6320 (const_int 15) (const_int 31)])))]
6322 "vunpckhps\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
6323 [(set_attr "type" "sselog")
6324 (set_attr "prefix" "evex")
6325 (set_attr "mode" "V16SF")])
6327 ;; Recall that the 256-bit unpck insns only shuffle within their lanes.
6328 (define_insn "avx_unpckhps256<mask_name>"
6329 [(set (match_operand:V8SF 0 "register_operand" "=v")
6332 (match_operand:V8SF 1 "register_operand" "v")
6333 (match_operand:V8SF 2 "nonimmediate_operand" "vm"))
6334 (parallel [(const_int 2) (const_int 10)
6335 (const_int 3) (const_int 11)
6336 (const_int 6) (const_int 14)
6337 (const_int 7) (const_int 15)])))]
6338 "TARGET_AVX && <mask_avx512vl_condition>"
6339 "vunpckhps\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
6340 [(set_attr "type" "sselog")
6341 (set_attr "prefix" "vex")
6342 (set_attr "mode" "V8SF")])
6344 (define_expand "vec_interleave_highv8sf"
6348 (match_operand:V8SF 1 "register_operand")
6349 (match_operand:V8SF 2 "nonimmediate_operand"))
6350 (parallel [(const_int 0) (const_int 8)
6351 (const_int 1) (const_int 9)
6352 (const_int 4) (const_int 12)
6353 (const_int 5) (const_int 13)])))
6359 (parallel [(const_int 2) (const_int 10)
6360 (const_int 3) (const_int 11)
6361 (const_int 6) (const_int 14)
6362 (const_int 7) (const_int 15)])))
6363 (set (match_operand:V8SF 0 "register_operand")
6368 (parallel [(const_int 4) (const_int 5)
6369 (const_int 6) (const_int 7)
6370 (const_int 12) (const_int 13)
6371 (const_int 14) (const_int 15)])))]
6374 operands[3] = gen_reg_rtx (V8SFmode);
6375 operands[4] = gen_reg_rtx (V8SFmode);
6378 (define_insn "vec_interleave_highv4sf<mask_name>"
6379 [(set (match_operand:V4SF 0 "register_operand" "=x,v")
6382 (match_operand:V4SF 1 "register_operand" "0,v")
6383 (match_operand:V4SF 2 "vector_operand" "xBm,vm"))
6384 (parallel [(const_int 2) (const_int 6)
6385 (const_int 3) (const_int 7)])))]
6386 "TARGET_SSE && <mask_avx512vl_condition>"
6388 unpckhps\t{%2, %0|%0, %2}
6389 vunpckhps\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
6390 [(set_attr "isa" "noavx,avx")
6391 (set_attr "type" "sselog")
6392 (set_attr "prefix" "orig,vex")
6393 (set_attr "mode" "V4SF")])
6395 (define_insn "<mask_codefor>avx512f_unpcklps512<mask_name>"
6396 [(set (match_operand:V16SF 0 "register_operand" "=v")
6399 (match_operand:V16SF 1 "register_operand" "v")
6400 (match_operand:V16SF 2 "nonimmediate_operand" "vm"))
6401 (parallel [(const_int 0) (const_int 16)
6402 (const_int 1) (const_int 17)
6403 (const_int 4) (const_int 20)
6404 (const_int 5) (const_int 21)
6405 (const_int 8) (const_int 24)
6406 (const_int 9) (const_int 25)
6407 (const_int 12) (const_int 28)
6408 (const_int 13) (const_int 29)])))]
6410 "vunpcklps\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
6411 [(set_attr "type" "sselog")
6412 (set_attr "prefix" "evex")
6413 (set_attr "mode" "V16SF")])
6415 ;; Recall that the 256-bit unpck insns only shuffle within their lanes.
6416 (define_insn "avx_unpcklps256<mask_name>"
6417 [(set (match_operand:V8SF 0 "register_operand" "=v")
6420 (match_operand:V8SF 1 "register_operand" "v")
6421 (match_operand:V8SF 2 "nonimmediate_operand" "vm"))
6422 (parallel [(const_int 0) (const_int 8)
6423 (const_int 1) (const_int 9)
6424 (const_int 4) (const_int 12)
6425 (const_int 5) (const_int 13)])))]
6426 "TARGET_AVX && <mask_avx512vl_condition>"
6427 "vunpcklps\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
6428 [(set_attr "type" "sselog")
6429 (set_attr "prefix" "vex")
6430 (set_attr "mode" "V8SF")])
6432 (define_insn "unpcklps128_mask"
6433 [(set (match_operand:V4SF 0 "register_operand" "=v")
6437 (match_operand:V4SF 1 "register_operand" "v")
6438 (match_operand:V4SF 2 "nonimmediate_operand" "vm"))
6439 (parallel [(const_int 0) (const_int 4)
6440 (const_int 1) (const_int 5)]))
6441 (match_operand:V4SF 3 "vector_move_operand" "0C")
6442 (match_operand:QI 4 "register_operand" "Yk")))]
6444 "vunpcklps\t{%2, %1, %0%{%4%}%N3|%0%{%4%}%N3, %1, %2}"
6445 [(set_attr "type" "sselog")
6446 (set_attr "prefix" "evex")
6447 (set_attr "mode" "V4SF")])
6449 (define_expand "vec_interleave_lowv8sf"
6453 (match_operand:V8SF 1 "register_operand")
6454 (match_operand:V8SF 2 "nonimmediate_operand"))
6455 (parallel [(const_int 0) (const_int 8)
6456 (const_int 1) (const_int 9)
6457 (const_int 4) (const_int 12)
6458 (const_int 5) (const_int 13)])))
6464 (parallel [(const_int 2) (const_int 10)
6465 (const_int 3) (const_int 11)
6466 (const_int 6) (const_int 14)
6467 (const_int 7) (const_int 15)])))
6468 (set (match_operand:V8SF 0 "register_operand")
6473 (parallel [(const_int 0) (const_int 1)
6474 (const_int 2) (const_int 3)
6475 (const_int 8) (const_int 9)
6476 (const_int 10) (const_int 11)])))]
6479 operands[3] = gen_reg_rtx (V8SFmode);
6480 operands[4] = gen_reg_rtx (V8SFmode);
6483 (define_insn "vec_interleave_lowv4sf"
6484 [(set (match_operand:V4SF 0 "register_operand" "=x,v")
6487 (match_operand:V4SF 1 "register_operand" "0,v")
6488 (match_operand:V4SF 2 "vector_operand" "xBm,vm"))
6489 (parallel [(const_int 0) (const_int 4)
6490 (const_int 1) (const_int 5)])))]
6493 unpcklps\t{%2, %0|%0, %2}
6494 vunpcklps\t{%2, %1, %0|%0, %1, %2}"
6495 [(set_attr "isa" "noavx,avx")
6496 (set_attr "type" "sselog")
6497 (set_attr "prefix" "orig,maybe_evex")
6498 (set_attr "mode" "V4SF")])
6500 ;; These are modeled with the same vec_concat as the others so that we
6501 ;; capture users of shufps that can use the new instructions
6502 (define_insn "avx_movshdup256<mask_name>"
6503 [(set (match_operand:V8SF 0 "register_operand" "=v")
6506 (match_operand:V8SF 1 "nonimmediate_operand" "vm")
6508 (parallel [(const_int 1) (const_int 1)
6509 (const_int 3) (const_int 3)
6510 (const_int 5) (const_int 5)
6511 (const_int 7) (const_int 7)])))]
6512 "TARGET_AVX && <mask_avx512vl_condition>"
6513 "vmovshdup\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
6514 [(set_attr "type" "sse")
6515 (set_attr "prefix" "vex")
6516 (set_attr "mode" "V8SF")])
6518 (define_insn "sse3_movshdup<mask_name>"
6519 [(set (match_operand:V4SF 0 "register_operand" "=v")
6522 (match_operand:V4SF 1 "vector_operand" "vBm")
6524 (parallel [(const_int 1)
6528 "TARGET_SSE3 && <mask_avx512vl_condition>"
6529 "%vmovshdup\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
6530 [(set_attr "type" "sse")
6531 (set_attr "prefix_rep" "1")
6532 (set_attr "prefix" "maybe_vex")
6533 (set_attr "mode" "V4SF")])
6535 (define_insn "<mask_codefor>avx512f_movshdup512<mask_name>"
6536 [(set (match_operand:V16SF 0 "register_operand" "=v")
6539 (match_operand:V16SF 1 "nonimmediate_operand" "vm")
6541 (parallel [(const_int 1) (const_int 1)
6542 (const_int 3) (const_int 3)
6543 (const_int 5) (const_int 5)
6544 (const_int 7) (const_int 7)
6545 (const_int 9) (const_int 9)
6546 (const_int 11) (const_int 11)
6547 (const_int 13) (const_int 13)
6548 (const_int 15) (const_int 15)])))]
6550 "vmovshdup\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
6551 [(set_attr "type" "sse")
6552 (set_attr "prefix" "evex")
6553 (set_attr "mode" "V16SF")])
6555 (define_insn "avx_movsldup256<mask_name>"
6556 [(set (match_operand:V8SF 0 "register_operand" "=v")
6559 (match_operand:V8SF 1 "nonimmediate_operand" "vm")
6561 (parallel [(const_int 0) (const_int 0)
6562 (const_int 2) (const_int 2)
6563 (const_int 4) (const_int 4)
6564 (const_int 6) (const_int 6)])))]
6565 "TARGET_AVX && <mask_avx512vl_condition>"
6566 "vmovsldup\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
6567 [(set_attr "type" "sse")
6568 (set_attr "prefix" "vex")
6569 (set_attr "mode" "V8SF")])
6571 (define_insn "sse3_movsldup<mask_name>"
6572 [(set (match_operand:V4SF 0 "register_operand" "=v")
6575 (match_operand:V4SF 1 "vector_operand" "vBm")
6577 (parallel [(const_int 0)
6581 "TARGET_SSE3 && <mask_avx512vl_condition>"
6582 "%vmovsldup\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
6583 [(set_attr "type" "sse")
6584 (set_attr "prefix_rep" "1")
6585 (set_attr "prefix" "maybe_vex")
6586 (set_attr "mode" "V4SF")])
6588 (define_insn "<mask_codefor>avx512f_movsldup512<mask_name>"
6589 [(set (match_operand:V16SF 0 "register_operand" "=v")
6592 (match_operand:V16SF 1 "nonimmediate_operand" "vm")
6594 (parallel [(const_int 0) (const_int 0)
6595 (const_int 2) (const_int 2)
6596 (const_int 4) (const_int 4)
6597 (const_int 6) (const_int 6)
6598 (const_int 8) (const_int 8)
6599 (const_int 10) (const_int 10)
6600 (const_int 12) (const_int 12)
6601 (const_int 14) (const_int 14)])))]
6603 "vmovsldup\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
6604 [(set_attr "type" "sse")
6605 (set_attr "prefix" "evex")
6606 (set_attr "mode" "V16SF")])
6608 (define_expand "avx_shufps256<mask_expand4_name>"
6609 [(match_operand:V8SF 0 "register_operand")
6610 (match_operand:V8SF 1 "register_operand")
6611 (match_operand:V8SF 2 "nonimmediate_operand")
6612 (match_operand:SI 3 "const_int_operand")]
6615 int mask = INTVAL (operands[3]);
6616 emit_insn (gen_avx_shufps256_1<mask_expand4_name> (operands[0],
6619 GEN_INT ((mask >> 0) & 3),
6620 GEN_INT ((mask >> 2) & 3),
6621 GEN_INT (((mask >> 4) & 3) + 8),
6622 GEN_INT (((mask >> 6) & 3) + 8),
6623 GEN_INT (((mask >> 0) & 3) + 4),
6624 GEN_INT (((mask >> 2) & 3) + 4),
6625 GEN_INT (((mask >> 4) & 3) + 12),
6626 GEN_INT (((mask >> 6) & 3) + 12)
6627 <mask_expand4_args>));
6631 ;; One bit in mask selects 2 elements.
6632 (define_insn "avx_shufps256_1<mask_name>"
6633 [(set (match_operand:V8SF 0 "register_operand" "=v")
6636 (match_operand:V8SF 1 "register_operand" "v")
6637 (match_operand:V8SF 2 "nonimmediate_operand" "vm"))
6638 (parallel [(match_operand 3 "const_0_to_3_operand" )
6639 (match_operand 4 "const_0_to_3_operand" )
6640 (match_operand 5 "const_8_to_11_operand" )
6641 (match_operand 6 "const_8_to_11_operand" )
6642 (match_operand 7 "const_4_to_7_operand" )
6643 (match_operand 8 "const_4_to_7_operand" )
6644 (match_operand 9 "const_12_to_15_operand")
6645 (match_operand 10 "const_12_to_15_operand")])))]
6647 && <mask_avx512vl_condition>
6648 && (INTVAL (operands[3]) == (INTVAL (operands[7]) - 4)
6649 && INTVAL (operands[4]) == (INTVAL (operands[8]) - 4)
6650 && INTVAL (operands[5]) == (INTVAL (operands[9]) - 4)
6651 && INTVAL (operands[6]) == (INTVAL (operands[10]) - 4))"
6654 mask = INTVAL (operands[3]);
6655 mask |= INTVAL (operands[4]) << 2;
6656 mask |= (INTVAL (operands[5]) - 8) << 4;
6657 mask |= (INTVAL (operands[6]) - 8) << 6;
6658 operands[3] = GEN_INT (mask);
6660 return "vshufps\t{%3, %2, %1, %0<mask_operand11>|%0<mask_operand11>, %1, %2, %3}";
6662 [(set_attr "type" "sseshuf")
6663 (set_attr "length_immediate" "1")
6664 (set_attr "prefix" "<mask_prefix>")
6665 (set_attr "mode" "V8SF")])
6667 (define_expand "sse_shufps<mask_expand4_name>"
6668 [(match_operand:V4SF 0 "register_operand")
6669 (match_operand:V4SF 1 "register_operand")
6670 (match_operand:V4SF 2 "vector_operand")
6671 (match_operand:SI 3 "const_int_operand")]
6674 int mask = INTVAL (operands[3]);
6675 emit_insn (gen_sse_shufps_v4sf<mask_expand4_name> (operands[0],
6678 GEN_INT ((mask >> 0) & 3),
6679 GEN_INT ((mask >> 2) & 3),
6680 GEN_INT (((mask >> 4) & 3) + 4),
6681 GEN_INT (((mask >> 6) & 3) + 4)
6682 <mask_expand4_args>));
6686 (define_insn "sse_shufps_v4sf_mask"
6687 [(set (match_operand:V4SF 0 "register_operand" "=v")
6691 (match_operand:V4SF 1 "register_operand" "v")
6692 (match_operand:V4SF 2 "nonimmediate_operand" "vm"))
6693 (parallel [(match_operand 3 "const_0_to_3_operand")
6694 (match_operand 4 "const_0_to_3_operand")
6695 (match_operand 5 "const_4_to_7_operand")
6696 (match_operand 6 "const_4_to_7_operand")]))
6697 (match_operand:V4SF 7 "vector_move_operand" "0C")
6698 (match_operand:QI 8 "register_operand" "Yk")))]
6702 mask |= INTVAL (operands[3]) << 0;
6703 mask |= INTVAL (operands[4]) << 2;
6704 mask |= (INTVAL (operands[5]) - 4) << 4;
6705 mask |= (INTVAL (operands[6]) - 4) << 6;
6706 operands[3] = GEN_INT (mask);
6708 return "vshufps\t{%3, %2, %1, %0%{%8%}%N7|%0%{%8%}%N7, %1, %2, %3}";
6710 [(set_attr "type" "sseshuf")
6711 (set_attr "length_immediate" "1")
6712 (set_attr "prefix" "evex")
6713 (set_attr "mode" "V4SF")])
6715 (define_insn "sse_shufps_<mode>"
6716 [(set (match_operand:VI4F_128 0 "register_operand" "=x,v")
6717 (vec_select:VI4F_128
6718 (vec_concat:<ssedoublevecmode>
6719 (match_operand:VI4F_128 1 "register_operand" "0,v")
6720 (match_operand:VI4F_128 2 "vector_operand" "xBm,vm"))
6721 (parallel [(match_operand 3 "const_0_to_3_operand")
6722 (match_operand 4 "const_0_to_3_operand")
6723 (match_operand 5 "const_4_to_7_operand")
6724 (match_operand 6 "const_4_to_7_operand")])))]
6728 mask |= INTVAL (operands[3]) << 0;
6729 mask |= INTVAL (operands[4]) << 2;
6730 mask |= (INTVAL (operands[5]) - 4) << 4;
6731 mask |= (INTVAL (operands[6]) - 4) << 6;
6732 operands[3] = GEN_INT (mask);
6734 switch (which_alternative)
6737 return "shufps\t{%3, %2, %0|%0, %2, %3}";
6739 return "vshufps\t{%3, %2, %1, %0|%0, %1, %2, %3}";
6744 [(set_attr "isa" "noavx,avx")
6745 (set_attr "type" "sseshuf")
6746 (set_attr "length_immediate" "1")
6747 (set_attr "prefix" "orig,maybe_evex")
6748 (set_attr "mode" "V4SF")])
6750 (define_insn "sse_storehps"
6751 [(set (match_operand:V2SF 0 "nonimmediate_operand" "=m,v,v")
6753 (match_operand:V4SF 1 "nonimmediate_operand" "v,v,o")
6754 (parallel [(const_int 2) (const_int 3)])))]
6755 "TARGET_SSE && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
6757 %vmovhps\t{%1, %0|%q0, %1}
6758 %vmovhlps\t{%1, %d0|%d0, %1}
6759 %vmovlps\t{%H1, %d0|%d0, %H1}"
6760 [(set_attr "type" "ssemov")
6761 (set_attr "prefix" "maybe_vex")
6762 (set_attr "mode" "V2SF,V4SF,V2SF")])
6764 (define_expand "sse_loadhps_exp"
6765 [(set (match_operand:V4SF 0 "nonimmediate_operand")
6768 (match_operand:V4SF 1 "nonimmediate_operand")
6769 (parallel [(const_int 0) (const_int 1)]))
6770 (match_operand:V2SF 2 "nonimmediate_operand")))]
6773 rtx dst = ix86_fixup_binary_operands (UNKNOWN, V4SFmode, operands);
6775 emit_insn (gen_sse_loadhps (dst, operands[1], operands[2]));
6777 /* Fix up the destination if needed. */
6778 if (dst != operands[0])
6779 emit_move_insn (operands[0], dst);
6784 (define_insn "sse_loadhps"
6785 [(set (match_operand:V4SF 0 "nonimmediate_operand" "=x,v,x,v,o")
6788 (match_operand:V4SF 1 "nonimmediate_operand" " 0,v,0,v,0")
6789 (parallel [(const_int 0) (const_int 1)]))
6790 (match_operand:V2SF 2 "nonimmediate_operand" " m,m,x,v,v")))]
6793 movhps\t{%2, %0|%0, %q2}
6794 vmovhps\t{%2, %1, %0|%0, %1, %q2}
6795 movlhps\t{%2, %0|%0, %2}
6796 vmovlhps\t{%2, %1, %0|%0, %1, %2}
6797 %vmovlps\t{%2, %H0|%H0, %2}"
6798 [(set_attr "isa" "noavx,avx,noavx,avx,*")
6799 (set_attr "type" "ssemov")
6800 (set_attr "prefix" "orig,maybe_evex,orig,maybe_evex,maybe_vex")
6801 (set_attr "mode" "V2SF,V2SF,V4SF,V4SF,V2SF")])
6803 (define_insn "sse_storelps"
6804 [(set (match_operand:V2SF 0 "nonimmediate_operand" "=m,v,v")
6806 (match_operand:V4SF 1 "nonimmediate_operand" " v,v,m")
6807 (parallel [(const_int 0) (const_int 1)])))]
6808 "TARGET_SSE && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
6810 %vmovlps\t{%1, %0|%q0, %1}
6811 %vmovaps\t{%1, %0|%0, %1}
6812 %vmovlps\t{%1, %d0|%d0, %q1}"
6813 [(set_attr "type" "ssemov")
6814 (set_attr "prefix" "maybe_vex")
6815 (set_attr "mode" "V2SF,V4SF,V2SF")])
6817 (define_expand "sse_loadlps_exp"
6818 [(set (match_operand:V4SF 0 "nonimmediate_operand")
6820 (match_operand:V2SF 2 "nonimmediate_operand")
6822 (match_operand:V4SF 1 "nonimmediate_operand")
6823 (parallel [(const_int 2) (const_int 3)]))))]
6826 rtx dst = ix86_fixup_binary_operands (UNKNOWN, V4SFmode, operands);
6828 emit_insn (gen_sse_loadlps (dst, operands[1], operands[2]));
6830 /* Fix up the destination if needed. */
6831 if (dst != operands[0])
6832 emit_move_insn (operands[0], dst);
6837 (define_insn "sse_loadlps"
6838 [(set (match_operand:V4SF 0 "nonimmediate_operand" "=x,v,x,v,m")
6840 (match_operand:V2SF 2 "nonimmediate_operand" " 0,v,m,m,v")
6842 (match_operand:V4SF 1 "nonimmediate_operand" " x,v,0,v,0")
6843 (parallel [(const_int 2) (const_int 3)]))))]
6846 shufps\t{$0xe4, %1, %0|%0, %1, 0xe4}
6847 vshufps\t{$0xe4, %1, %2, %0|%0, %2, %1, 0xe4}
6848 movlps\t{%2, %0|%0, %q2}
6849 vmovlps\t{%2, %1, %0|%0, %1, %q2}
6850 %vmovlps\t{%2, %0|%q0, %2}"
6851 [(set_attr "isa" "noavx,avx,noavx,avx,*")
6852 (set_attr "type" "sseshuf,sseshuf,ssemov,ssemov,ssemov")
6853 (set (attr "length_immediate")
6854 (if_then_else (eq_attr "alternative" "0,1")
6856 (const_string "*")))
6857 (set_attr "prefix" "orig,maybe_evex,orig,maybe_evex,maybe_vex")
6858 (set_attr "mode" "V4SF,V4SF,V2SF,V2SF,V2SF")])
6860 (define_insn "sse_movss"
6861 [(set (match_operand:V4SF 0 "register_operand" "=x,v")
6863 (match_operand:V4SF 2 "register_operand" " x,v")
6864 (match_operand:V4SF 1 "register_operand" " 0,v")
6868 movss\t{%2, %0|%0, %2}
6869 vmovss\t{%2, %1, %0|%0, %1, %2}"
6870 [(set_attr "isa" "noavx,avx")
6871 (set_attr "type" "ssemov")
6872 (set_attr "prefix" "orig,maybe_evex")
6873 (set_attr "mode" "SF")])
6875 (define_insn "avx2_vec_dup<mode>"
6876 [(set (match_operand:VF1_128_256 0 "register_operand" "=v")
6877 (vec_duplicate:VF1_128_256
6879 (match_operand:V4SF 1 "register_operand" "v")
6880 (parallel [(const_int 0)]))))]
6882 "vbroadcastss\t{%1, %0|%0, %1}"
6883 [(set_attr "type" "sselog1")
6884 (set_attr "prefix" "maybe_evex")
6885 (set_attr "mode" "<MODE>")])
6887 (define_insn "avx2_vec_dupv8sf_1"
6888 [(set (match_operand:V8SF 0 "register_operand" "=v")
6891 (match_operand:V8SF 1 "register_operand" "v")
6892 (parallel [(const_int 0)]))))]
6894 "vbroadcastss\t{%x1, %0|%0, %x1}"
6895 [(set_attr "type" "sselog1")
6896 (set_attr "prefix" "maybe_evex")
6897 (set_attr "mode" "V8SF")])
6899 (define_insn "avx512f_vec_dup<mode>_1"
6900 [(set (match_operand:VF_512 0 "register_operand" "=v")
6901 (vec_duplicate:VF_512
6902 (vec_select:<ssescalarmode>
6903 (match_operand:VF_512 1 "register_operand" "v")
6904 (parallel [(const_int 0)]))))]
6906 "vbroadcast<bcstscalarsuff>\t{%x1, %0|%0, %x1}"
6907 [(set_attr "type" "sselog1")
6908 (set_attr "prefix" "evex")
6909 (set_attr "mode" "<MODE>")])
6911 ;; Although insertps takes register source, we prefer
6912 ;; unpcklps with register source since it is shorter.
6913 (define_insn "*vec_concatv2sf_sse4_1"
6914 [(set (match_operand:V2SF 0 "register_operand"
6915 "=Yr,*x, v,Yr,*x,v,v,*y ,*y")
6917 (match_operand:SF 1 "nonimmediate_operand"
6918 " 0, 0,Yv, 0,0, v,m, 0 , m")
6919 (match_operand:SF 2 "vector_move_operand"
6920 " Yr,*x,Yv, m,m, m,C,*ym, C")))]
6921 "TARGET_SSE4_1 && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
6923 unpcklps\t{%2, %0|%0, %2}
6924 unpcklps\t{%2, %0|%0, %2}
6925 vunpcklps\t{%2, %1, %0|%0, %1, %2}
6926 insertps\t{$0x10, %2, %0|%0, %2, 0x10}
6927 insertps\t{$0x10, %2, %0|%0, %2, 0x10}
6928 vinsertps\t{$0x10, %2, %1, %0|%0, %1, %2, 0x10}
6929 %vmovss\t{%1, %0|%0, %1}
6930 punpckldq\t{%2, %0|%0, %2}
6931 movd\t{%1, %0|%0, %1}"
6933 (cond [(eq_attr "alternative" "0,1,3,4")
6934 (const_string "noavx")
6935 (eq_attr "alternative" "2,5")
6936 (const_string "avx")
6938 (const_string "*")))
6940 (cond [(eq_attr "alternative" "6")
6941 (const_string "ssemov")
6942 (eq_attr "alternative" "7")
6943 (const_string "mmxcvt")
6944 (eq_attr "alternative" "8")
6945 (const_string "mmxmov")
6947 (const_string "sselog")))
6948 (set (attr "prefix_data16")
6949 (if_then_else (eq_attr "alternative" "3,4")
6951 (const_string "*")))
6952 (set (attr "prefix_extra")
6953 (if_then_else (eq_attr "alternative" "3,4,5")
6955 (const_string "*")))
6956 (set (attr "length_immediate")
6957 (if_then_else (eq_attr "alternative" "3,4,5")
6959 (const_string "*")))
6960 (set (attr "prefix")
6961 (cond [(eq_attr "alternative" "2,5")
6962 (const_string "maybe_evex")
6963 (eq_attr "alternative" "6")
6964 (const_string "maybe_vex")
6966 (const_string "orig")))
6967 (set_attr "mode" "V4SF,V4SF,V4SF,V4SF,V4SF,V4SF,SF,DI,DI")])
6969 ;; ??? In theory we can match memory for the MMX alternative, but allowing
6970 ;; vector_operand for operand 2 and *not* allowing memory for the SSE
6971 ;; alternatives pretty much forces the MMX alternative to be chosen.
6972 (define_insn "*vec_concatv2sf_sse"
6973 [(set (match_operand:V2SF 0 "register_operand" "=x,x,*y,*y")
6975 (match_operand:SF 1 "nonimmediate_operand" " 0,m, 0, m")
6976 (match_operand:SF 2 "reg_or_0_operand" " x,C,*y, C")))]
6979 unpcklps\t{%2, %0|%0, %2}
6980 movss\t{%1, %0|%0, %1}
6981 punpckldq\t{%2, %0|%0, %2}
6982 movd\t{%1, %0|%0, %1}"
6983 [(set_attr "type" "sselog,ssemov,mmxcvt,mmxmov")
6984 (set_attr "mode" "V4SF,SF,DI,DI")])
6986 (define_insn "*vec_concatv4sf"
6987 [(set (match_operand:V4SF 0 "register_operand" "=x,v,x,v")
6989 (match_operand:V2SF 1 "register_operand" " 0,v,0,v")
6990 (match_operand:V2SF 2 "nonimmediate_operand" " x,v,m,m")))]
6993 movlhps\t{%2, %0|%0, %2}
6994 vmovlhps\t{%2, %1, %0|%0, %1, %2}
6995 movhps\t{%2, %0|%0, %q2}
6996 vmovhps\t{%2, %1, %0|%0, %1, %q2}"
6997 [(set_attr "isa" "noavx,avx,noavx,avx")
6998 (set_attr "type" "ssemov")
6999 (set_attr "prefix" "orig,maybe_evex,orig,maybe_evex")
7000 (set_attr "mode" "V4SF,V4SF,V2SF,V2SF")])
7002 ;; Avoid combining registers from different units in a single alternative,
7003 ;; see comment above inline_secondary_memory_needed function in i386.c
7004 (define_insn "vec_set<mode>_0"
7005 [(set (match_operand:VI4F_128 0 "nonimmediate_operand"
7006 "=Yr,*x,v,v,Yi,x,x,v,Yr ,*x ,x ,m ,m ,m")
7008 (vec_duplicate:VI4F_128
7009 (match_operand:<ssescalarmode> 2 "general_operand"
7010 " Yr,*x,v,m,r ,m,x,v,*rm,*rm,*rm,!x,!*re,!*fF"))
7011 (match_operand:VI4F_128 1 "vector_move_operand"
7012 " C , C,C,C,C ,C,0,v,0 ,0 ,x ,0 ,0 ,0")
7016 insertps\t{$0xe, %2, %0|%0, %2, 0xe}
7017 insertps\t{$0xe, %2, %0|%0, %2, 0xe}
7018 vinsertps\t{$0xe, %2, %2, %0|%0, %2, %2, 0xe}
7019 %vmov<ssescalarmodesuffix>\t{%2, %0|%0, %2}
7020 %vmovd\t{%2, %0|%0, %2}
7021 movss\t{%2, %0|%0, %2}
7022 movss\t{%2, %0|%0, %2}
7023 vmovss\t{%2, %1, %0|%0, %1, %2}
7024 pinsrd\t{$0, %2, %0|%0, %2, 0}
7025 pinsrd\t{$0, %2, %0|%0, %2, 0}
7026 vpinsrd\t{$0, %2, %1, %0|%0, %1, %2, 0}
7031 (cond [(eq_attr "alternative" "0,1,8,9")
7032 (const_string "sse4_noavx")
7033 (eq_attr "alternative" "2,7,10")
7034 (const_string "avx")
7035 (eq_attr "alternative" "3,4")
7036 (const_string "sse2")
7037 (eq_attr "alternative" "5,6")
7038 (const_string "noavx")
7040 (const_string "*")))
7042 (cond [(eq_attr "alternative" "0,1,2,8,9,10")
7043 (const_string "sselog")
7044 (eq_attr "alternative" "12")
7045 (const_string "imov")
7046 (eq_attr "alternative" "13")
7047 (const_string "fmov")
7049 (const_string "ssemov")))
7050 (set (attr "prefix_extra")
7051 (if_then_else (eq_attr "alternative" "8,9,10")
7053 (const_string "*")))
7054 (set (attr "length_immediate")
7055 (if_then_else (eq_attr "alternative" "8,9,10")
7057 (const_string "*")))
7058 (set (attr "prefix")
7059 (cond [(eq_attr "alternative" "0,1,5,6,8,9")
7060 (const_string "orig")
7061 (eq_attr "alternative" "2")
7062 (const_string "maybe_evex")
7063 (eq_attr "alternative" "3,4")
7064 (const_string "maybe_vex")
7065 (eq_attr "alternative" "7,10")
7066 (const_string "vex")
7068 (const_string "*")))
7069 (set_attr "mode" "SF,SF,SF,<ssescalarmode>,SI,SF,SF,SF,TI,TI,TI,*,*,*")])
7071 ;; A subset is vec_setv4sf.
7072 (define_insn "*vec_setv4sf_sse4_1"
7073 [(set (match_operand:V4SF 0 "register_operand" "=Yr,*x,v")
7076 (match_operand:SF 2 "nonimmediate_operand" "Yrm,*xm,vm"))
7077 (match_operand:V4SF 1 "register_operand" "0,0,v")
7078 (match_operand:SI 3 "const_int_operand")))]
7080 && ((unsigned) exact_log2 (INTVAL (operands[3]))
7081 < GET_MODE_NUNITS (V4SFmode))"
7083 operands[3] = GEN_INT (exact_log2 (INTVAL (operands[3])) << 4);
7084 switch (which_alternative)
7088 return "insertps\t{%3, %2, %0|%0, %2, %3}";
7090 return "vinsertps\t{%3, %2, %1, %0|%0, %1, %2, %3}";
7095 [(set_attr "isa" "noavx,noavx,avx")
7096 (set_attr "type" "sselog")
7097 (set_attr "prefix_data16" "1,1,*")
7098 (set_attr "prefix_extra" "1")
7099 (set_attr "length_immediate" "1")
7100 (set_attr "prefix" "orig,orig,maybe_evex")
7101 (set_attr "mode" "V4SF")])
7103 ;; All of vinsertps, vmovss, vmovd clear also the higher bits.
7104 (define_insn "vec_set<mode>_0"
7105 [(set (match_operand:VI4F_256_512 0 "register_operand" "=v,v,Yi")
7106 (vec_merge:VI4F_256_512
7107 (vec_duplicate:VI4F_256_512
7108 (match_operand:<ssescalarmode> 2 "general_operand" "v,m,r"))
7109 (match_operand:VI4F_256_512 1 "const0_operand" "C,C,C")
7113 vinsertps\t{$0xe, %2, %2, %x0|%x0, %2, %2, 0xe}
7114 vmov<ssescalarmodesuffix>\t{%x2, %x0|%x0, %2}
7115 vmovd\t{%2, %x0|%x0, %2}"
7117 (if_then_else (eq_attr "alternative" "0")
7118 (const_string "sselog")
7119 (const_string "ssemov")))
7120 (set_attr "prefix" "maybe_evex")
7121 (set_attr "mode" "SF,<ssescalarmode>,SI")])
7123 (define_insn "sse4_1_insertps"
7124 [(set (match_operand:V4SF 0 "register_operand" "=Yr,*x,v")
7125 (unspec:V4SF [(match_operand:V4SF 2 "nonimmediate_operand" "Yrm,*xm,vm")
7126 (match_operand:V4SF 1 "register_operand" "0,0,v")
7127 (match_operand:SI 3 "const_0_to_255_operand" "n,n,n")]
7131 if (MEM_P (operands[2]))
7133 unsigned count_s = INTVAL (operands[3]) >> 6;
7135 operands[3] = GEN_INT (INTVAL (operands[3]) & 0x3f);
7136 operands[2] = adjust_address_nv (operands[2], SFmode, count_s * 4);
7138 switch (which_alternative)
7142 return "insertps\t{%3, %2, %0|%0, %2, %3}";
7144 return "vinsertps\t{%3, %2, %1, %0|%0, %1, %2, %3}";
7149 [(set_attr "isa" "noavx,noavx,avx")
7150 (set_attr "type" "sselog")
7151 (set_attr "prefix_data16" "1,1,*")
7152 (set_attr "prefix_extra" "1")
7153 (set_attr "length_immediate" "1")
7154 (set_attr "prefix" "orig,orig,maybe_evex")
7155 (set_attr "mode" "V4SF")])
7158 [(set (match_operand:VI4F_128 0 "memory_operand")
7160 (vec_duplicate:VI4F_128
7161 (match_operand:<ssescalarmode> 1 "nonmemory_operand"))
7164 "TARGET_SSE && reload_completed"
7165 [(set (match_dup 0) (match_dup 1))]
7166 "operands[0] = adjust_address (operands[0], <ssescalarmode>mode, 0);")
7168 (define_expand "vec_set<mode>"
7169 [(match_operand:V 0 "register_operand")
7170 (match_operand:<ssescalarmode> 1 "register_operand")
7171 (match_operand 2 "const_int_operand")]
7174 ix86_expand_vector_set (false, operands[0], operands[1],
7175 INTVAL (operands[2]));
7179 (define_insn_and_split "*vec_extractv4sf_0"
7180 [(set (match_operand:SF 0 "nonimmediate_operand" "=v,m,f,r")
7182 (match_operand:V4SF 1 "nonimmediate_operand" "vm,v,m,m")
7183 (parallel [(const_int 0)])))]
7184 "TARGET_SSE && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
7186 "&& reload_completed"
7187 [(set (match_dup 0) (match_dup 1))]
7188 "operands[1] = gen_lowpart (SFmode, operands[1]);")
7190 (define_insn_and_split "*sse4_1_extractps"
7191 [(set (match_operand:SF 0 "nonimmediate_operand" "=rm,rm,rm,Yv,Yv")
7193 (match_operand:V4SF 1 "register_operand" "Yr,*x,v,0,v")
7194 (parallel [(match_operand:SI 2 "const_0_to_3_operand" "n,n,n,n,n")])))]
7197 extractps\t{%2, %1, %0|%0, %1, %2}
7198 extractps\t{%2, %1, %0|%0, %1, %2}
7199 vextractps\t{%2, %1, %0|%0, %1, %2}
7202 "&& reload_completed && SSE_REG_P (operands[0])"
7205 rtx dest = lowpart_subreg (V4SFmode, operands[0], SFmode);
7206 switch (INTVAL (operands[2]))
7210 emit_insn (gen_sse_shufps_v4sf (dest, operands[1], operands[1],
7211 operands[2], operands[2],
7212 GEN_INT (INTVAL (operands[2]) + 4),
7213 GEN_INT (INTVAL (operands[2]) + 4)));
7216 emit_insn (gen_vec_interleave_highv4sf (dest, operands[1], operands[1]));
7219 /* 0 should be handled by the *vec_extractv4sf_0 pattern above. */
7224 [(set_attr "isa" "noavx,noavx,avx,noavx,avx")
7225 (set_attr "type" "sselog,sselog,sselog,*,*")
7226 (set_attr "prefix_data16" "1,1,1,*,*")
7227 (set_attr "prefix_extra" "1,1,1,*,*")
7228 (set_attr "length_immediate" "1,1,1,*,*")
7229 (set_attr "prefix" "orig,orig,maybe_evex,*,*")
7230 (set_attr "mode" "V4SF,V4SF,V4SF,*,*")])
7232 (define_insn_and_split "*vec_extractv4sf_mem"
7233 [(set (match_operand:SF 0 "register_operand" "=v,*r,f")
7235 (match_operand:V4SF 1 "memory_operand" "o,o,o")
7236 (parallel [(match_operand 2 "const_0_to_3_operand" "n,n,n")])))]
7239 "&& reload_completed"
7240 [(set (match_dup 0) (match_dup 1))]
7242 operands[1] = adjust_address (operands[1], SFmode, INTVAL (operands[2]) * 4);
7245 (define_mode_attr extract_type
7246 [(V16SF "avx512f") (V16SI "avx512f") (V8DF "avx512dq") (V8DI "avx512dq")])
7248 (define_mode_attr extract_suf
7249 [(V16SF "32x4") (V16SI "32x4") (V8DF "64x2") (V8DI "64x2")])
7251 (define_mode_iterator AVX512_VEC
7252 [(V8DF "TARGET_AVX512DQ") (V8DI "TARGET_AVX512DQ") V16SF V16SI])
7254 (define_expand "<extract_type>_vextract<shuffletype><extract_suf>_mask"
7255 [(match_operand:<ssequartermode> 0 "nonimmediate_operand")
7256 (match_operand:AVX512_VEC 1 "register_operand")
7257 (match_operand:SI 2 "const_0_to_3_operand")
7258 (match_operand:<ssequartermode> 3 "nonimmediate_operand")
7259 (match_operand:QI 4 "register_operand")]
7263 mask = INTVAL (operands[2]);
7264 rtx dest = operands[0];
7266 if (MEM_P (operands[0]) && !rtx_equal_p (operands[0], operands[3]))
7267 dest = gen_reg_rtx (<ssequartermode>mode);
7269 if (<MODE>mode == V16SImode || <MODE>mode == V16SFmode)
7270 emit_insn (gen_avx512f_vextract<shuffletype>32x4_1_mask (dest,
7271 operands[1], GEN_INT (mask * 4), GEN_INT (mask * 4 + 1),
7272 GEN_INT (mask * 4 + 2), GEN_INT (mask * 4 + 3), operands[3],
7275 emit_insn (gen_avx512dq_vextract<shuffletype>64x2_1_mask (dest,
7276 operands[1], GEN_INT (mask * 2), GEN_INT (mask * 2 + 1), operands[3],
7278 if (dest != operands[0])
7279 emit_move_insn (operands[0], dest);
7283 (define_insn "avx512dq_vextract<shuffletype>64x2_1_maskm"
7284 [(set (match_operand:<ssequartermode> 0 "memory_operand" "=m")
7285 (vec_merge:<ssequartermode>
7286 (vec_select:<ssequartermode>
7287 (match_operand:V8FI 1 "register_operand" "v")
7288 (parallel [(match_operand 2 "const_0_to_7_operand")
7289 (match_operand 3 "const_0_to_7_operand")]))
7290 (match_operand:<ssequartermode> 4 "memory_operand" "0")
7291 (match_operand:QI 5 "register_operand" "Yk")))]
7293 && INTVAL (operands[2]) % 2 == 0
7294 && INTVAL (operands[2]) == INTVAL (operands[3]) - 1
7295 && rtx_equal_p (operands[4], operands[0])"
7297 operands[2] = GEN_INT ((INTVAL (operands[2])) >> 1);
7298 return "vextract<shuffletype>64x2\t{%2, %1, %0%{%5%}|%0%{%5%}, %1, %2}";
7300 [(set_attr "type" "sselog")
7301 (set_attr "prefix_extra" "1")
7302 (set_attr "length_immediate" "1")
7303 (set_attr "memory" "store")
7304 (set_attr "prefix" "evex")
7305 (set_attr "mode" "<sseinsnmode>")])
7307 (define_insn "avx512f_vextract<shuffletype>32x4_1_maskm"
7308 [(set (match_operand:<ssequartermode> 0 "memory_operand" "=m")
7309 (vec_merge:<ssequartermode>
7310 (vec_select:<ssequartermode>
7311 (match_operand:V16FI 1 "register_operand" "v")
7312 (parallel [(match_operand 2 "const_0_to_15_operand")
7313 (match_operand 3 "const_0_to_15_operand")
7314 (match_operand 4 "const_0_to_15_operand")
7315 (match_operand 5 "const_0_to_15_operand")]))
7316 (match_operand:<ssequartermode> 6 "memory_operand" "0")
7317 (match_operand:QI 7 "register_operand" "Yk")))]
7319 && INTVAL (operands[2]) % 4 == 0
7320 && INTVAL (operands[2]) == INTVAL (operands[3]) - 1
7321 && INTVAL (operands[3]) == INTVAL (operands[4]) - 1
7322 && INTVAL (operands[4]) == INTVAL (operands[5]) - 1
7323 && rtx_equal_p (operands[6], operands[0])"
7325 operands[2] = GEN_INT (INTVAL (operands[2]) >> 2);
7326 return "vextract<shuffletype>32x4\t{%2, %1, %0%{%7%}|%0%{%7%}, %1, %2}";
7328 [(set_attr "type" "sselog")
7329 (set_attr "prefix_extra" "1")
7330 (set_attr "length_immediate" "1")
7331 (set_attr "memory" "store")
7332 (set_attr "prefix" "evex")
7333 (set_attr "mode" "<sseinsnmode>")])
7335 (define_insn "<mask_codefor>avx512dq_vextract<shuffletype>64x2_1<mask_name>"
7336 [(set (match_operand:<ssequartermode> 0 "<store_mask_predicate>" "=<store_mask_constraint>")
7337 (vec_select:<ssequartermode>
7338 (match_operand:V8FI 1 "register_operand" "v")
7339 (parallel [(match_operand 2 "const_0_to_7_operand")
7340 (match_operand 3 "const_0_to_7_operand")])))]
7342 && INTVAL (operands[2]) % 2 == 0
7343 && INTVAL (operands[2]) == INTVAL (operands[3]) - 1"
7345 operands[2] = GEN_INT (INTVAL (operands[2]) >> 1);
7346 return "vextract<shuffletype>64x2\t{%2, %1, %0<mask_operand4>|%0<mask_operand4>, %1, %2}";
7348 [(set_attr "type" "sselog1")
7349 (set_attr "prefix_extra" "1")
7350 (set_attr "length_immediate" "1")
7351 (set_attr "prefix" "evex")
7352 (set_attr "mode" "<sseinsnmode>")])
7354 (define_insn "<mask_codefor>avx512f_vextract<shuffletype>32x4_1<mask_name>"
7355 [(set (match_operand:<ssequartermode> 0 "<store_mask_predicate>" "=<store_mask_constraint>")
7356 (vec_select:<ssequartermode>
7357 (match_operand:V16FI 1 "register_operand" "v")
7358 (parallel [(match_operand 2 "const_0_to_15_operand")
7359 (match_operand 3 "const_0_to_15_operand")
7360 (match_operand 4 "const_0_to_15_operand")
7361 (match_operand 5 "const_0_to_15_operand")])))]
7363 && INTVAL (operands[2]) % 4 == 0
7364 && INTVAL (operands[2]) == INTVAL (operands[3]) - 1
7365 && INTVAL (operands[3]) == INTVAL (operands[4]) - 1
7366 && INTVAL (operands[4]) == INTVAL (operands[5]) - 1"
7368 operands[2] = GEN_INT (INTVAL (operands[2]) >> 2);
7369 return "vextract<shuffletype>32x4\t{%2, %1, %0<mask_operand6>|%0<mask_operand6>, %1, %2}";
7371 [(set_attr "type" "sselog1")
7372 (set_attr "prefix_extra" "1")
7373 (set_attr "length_immediate" "1")
7374 (set_attr "prefix" "evex")
7375 (set_attr "mode" "<sseinsnmode>")])
7377 (define_mode_attr extract_type_2
7378 [(V16SF "avx512dq") (V16SI "avx512dq") (V8DF "avx512f") (V8DI "avx512f")])
7380 (define_mode_attr extract_suf_2
7381 [(V16SF "32x8") (V16SI "32x8") (V8DF "64x4") (V8DI "64x4")])
7383 (define_mode_iterator AVX512_VEC_2
7384 [(V16SF "TARGET_AVX512DQ") (V16SI "TARGET_AVX512DQ") V8DF V8DI])
7386 (define_expand "<extract_type_2>_vextract<shuffletype><extract_suf_2>_mask"
7387 [(match_operand:<ssehalfvecmode> 0 "nonimmediate_operand")
7388 (match_operand:AVX512_VEC_2 1 "register_operand")
7389 (match_operand:SI 2 "const_0_to_1_operand")
7390 (match_operand:<ssehalfvecmode> 3 "nonimmediate_operand")
7391 (match_operand:QI 4 "register_operand")]
7394 rtx (*insn)(rtx, rtx, rtx, rtx);
7395 rtx dest = operands[0];
7397 if (MEM_P (dest) && !rtx_equal_p (dest, operands[3]))
7398 dest = gen_reg_rtx (<ssehalfvecmode>mode);
7400 switch (INTVAL (operands[2]))
7403 insn = gen_vec_extract_lo_<mode>_mask;
7406 insn = gen_vec_extract_hi_<mode>_mask;
7412 emit_insn (insn (dest, operands[1], operands[3], operands[4]));
7413 if (dest != operands[0])
7414 emit_move_insn (operands[0], dest);
7419 [(set (match_operand:<ssehalfvecmode> 0 "nonimmediate_operand")
7420 (vec_select:<ssehalfvecmode>
7421 (match_operand:V8FI 1 "nonimmediate_operand")
7422 (parallel [(const_int 0) (const_int 1)
7423 (const_int 2) (const_int 3)])))]
7424 "TARGET_AVX512F && !(MEM_P (operands[0]) && MEM_P (operands[1]))
7427 || (REG_P (operands[0]) && !EXT_REX_SSE_REG_P (operands[1])))"
7428 [(set (match_dup 0) (match_dup 1))]
7429 "operands[1] = gen_lowpart (<ssehalfvecmode>mode, operands[1]);")
7431 (define_insn "vec_extract_lo_<mode>_maskm"
7432 [(set (match_operand:<ssehalfvecmode> 0 "memory_operand" "=m")
7433 (vec_merge:<ssehalfvecmode>
7434 (vec_select:<ssehalfvecmode>
7435 (match_operand:V8FI 1 "register_operand" "v")
7436 (parallel [(const_int 0) (const_int 1)
7437 (const_int 2) (const_int 3)]))
7438 (match_operand:<ssehalfvecmode> 2 "memory_operand" "0")
7439 (match_operand:QI 3 "register_operand" "Yk")))]
7441 && rtx_equal_p (operands[2], operands[0])"
7442 "vextract<shuffletype>64x4\t{$0x0, %1, %0%{%3%}|%0%{%3%}, %1, 0x0}"
7443 [(set_attr "type" "sselog1")
7444 (set_attr "prefix_extra" "1")
7445 (set_attr "length_immediate" "1")
7446 (set_attr "prefix" "evex")
7447 (set_attr "mode" "<sseinsnmode>")])
7449 (define_insn "vec_extract_lo_<mode><mask_name>"
7450 [(set (match_operand:<ssehalfvecmode> 0 "<store_mask_predicate>" "=<store_mask_constraint>,v")
7451 (vec_select:<ssehalfvecmode>
7452 (match_operand:V8FI 1 "<store_mask_predicate>" "v,<store_mask_constraint>")
7453 (parallel [(const_int 0) (const_int 1)
7454 (const_int 2) (const_int 3)])))]
7456 && (<mask_applied> || !(MEM_P (operands[0]) && MEM_P (operands[1])))"
7458 if (<mask_applied> || (!TARGET_AVX512VL && !MEM_P (operands[1])))
7459 return "vextract<shuffletype>64x4\t{$0x0, %1, %0<mask_operand2>|%0<mask_operand2>, %1, 0x0}";
7463 [(set_attr "type" "sselog1")
7464 (set_attr "prefix_extra" "1")
7465 (set_attr "length_immediate" "1")
7466 (set_attr "prefix" "evex")
7467 (set_attr "mode" "<sseinsnmode>")])
7469 (define_insn "vec_extract_hi_<mode>_maskm"
7470 [(set (match_operand:<ssehalfvecmode> 0 "memory_operand" "=m")
7471 (vec_merge:<ssehalfvecmode>
7472 (vec_select:<ssehalfvecmode>
7473 (match_operand:V8FI 1 "register_operand" "v")
7474 (parallel [(const_int 4) (const_int 5)
7475 (const_int 6) (const_int 7)]))
7476 (match_operand:<ssehalfvecmode> 2 "memory_operand" "0")
7477 (match_operand:QI 3 "register_operand" "Yk")))]
7479 && rtx_equal_p (operands[2], operands[0])"
7480 "vextract<shuffletype>64x4\t{$0x1, %1, %0%{%3%}|%0%{%3%}, %1, 0x1}"
7481 [(set_attr "type" "sselog")
7482 (set_attr "prefix_extra" "1")
7483 (set_attr "length_immediate" "1")
7484 (set_attr "memory" "store")
7485 (set_attr "prefix" "evex")
7486 (set_attr "mode" "<sseinsnmode>")])
7488 (define_insn "vec_extract_hi_<mode><mask_name>"
7489 [(set (match_operand:<ssehalfvecmode> 0 "<store_mask_predicate>" "=<store_mask_constraint>")
7490 (vec_select:<ssehalfvecmode>
7491 (match_operand:V8FI 1 "register_operand" "v")
7492 (parallel [(const_int 4) (const_int 5)
7493 (const_int 6) (const_int 7)])))]
7495 "vextract<shuffletype>64x4\t{$0x1, %1, %0<mask_operand2>|%0<mask_operand2>, %1, 0x1}"
7496 [(set_attr "type" "sselog1")
7497 (set_attr "prefix_extra" "1")
7498 (set_attr "length_immediate" "1")
7499 (set_attr "prefix" "evex")
7500 (set_attr "mode" "<sseinsnmode>")])
7502 (define_insn "vec_extract_hi_<mode>_maskm"
7503 [(set (match_operand:<ssehalfvecmode> 0 "memory_operand" "=m")
7504 (vec_merge:<ssehalfvecmode>
7505 (vec_select:<ssehalfvecmode>
7506 (match_operand:V16FI 1 "register_operand" "v")
7507 (parallel [(const_int 8) (const_int 9)
7508 (const_int 10) (const_int 11)
7509 (const_int 12) (const_int 13)
7510 (const_int 14) (const_int 15)]))
7511 (match_operand:<ssehalfvecmode> 2 "memory_operand" "0")
7512 (match_operand:QI 3 "register_operand" "Yk")))]
7514 && rtx_equal_p (operands[2], operands[0])"
7515 "vextract<shuffletype>32x8\t{$0x1, %1, %0%{%3%}|%0%{%3%}, %1, 0x1}"
7516 [(set_attr "type" "sselog1")
7517 (set_attr "prefix_extra" "1")
7518 (set_attr "length_immediate" "1")
7519 (set_attr "prefix" "evex")
7520 (set_attr "mode" "<sseinsnmode>")])
7522 (define_insn "vec_extract_hi_<mode><mask_name>"
7523 [(set (match_operand:<ssehalfvecmode> 0 "<store_mask_predicate>" "=<store_mask_constraint>,vm")
7524 (vec_select:<ssehalfvecmode>
7525 (match_operand:V16FI 1 "register_operand" "v,v")
7526 (parallel [(const_int 8) (const_int 9)
7527 (const_int 10) (const_int 11)
7528 (const_int 12) (const_int 13)
7529 (const_int 14) (const_int 15)])))]
7530 "TARGET_AVX512F && <mask_avx512dq_condition>"
7532 vextract<shuffletype>32x8\t{$0x1, %1, %0<mask_operand2>|%0<mask_operand2>, %1, 0x1}
7533 vextracti64x4\t{$0x1, %1, %0|%0, %1, 0x1}"
7534 [(set_attr "type" "sselog1")
7535 (set_attr "prefix_extra" "1")
7536 (set_attr "isa" "avx512dq,noavx512dq")
7537 (set_attr "length_immediate" "1")
7538 (set_attr "prefix" "evex")
7539 (set_attr "mode" "<sseinsnmode>")])
7541 (define_expand "avx512vl_vextractf128<mode>"
7542 [(match_operand:<ssehalfvecmode> 0 "nonimmediate_operand")
7543 (match_operand:VI48F_256 1 "register_operand")
7544 (match_operand:SI 2 "const_0_to_1_operand")
7545 (match_operand:<ssehalfvecmode> 3 "vector_move_operand")
7546 (match_operand:QI 4 "register_operand")]
7547 "TARGET_AVX512DQ && TARGET_AVX512VL"
7549 rtx (*insn)(rtx, rtx, rtx, rtx);
7550 rtx dest = operands[0];
7553 && (GET_MODE_SIZE (GET_MODE_INNER (<MODE>mode)) == 4
7554 /* For V8S[IF]mode there are maskm insns with =m and 0
7556 ? !rtx_equal_p (dest, operands[3])
7557 /* For V4D[IF]mode, hi insns don't allow memory, and
7558 lo insns have =m and 0C constraints. */
7559 : (operands[2] != const0_rtx
7560 || (!rtx_equal_p (dest, operands[3])
7561 && GET_CODE (operands[3]) != CONST_VECTOR))))
7562 dest = gen_reg_rtx (<ssehalfvecmode>mode);
7563 switch (INTVAL (operands[2]))
7566 insn = gen_vec_extract_lo_<mode>_mask;
7569 insn = gen_vec_extract_hi_<mode>_mask;
7575 emit_insn (insn (dest, operands[1], operands[3], operands[4]));
7576 if (dest != operands[0])
7577 emit_move_insn (operands[0], dest);
7581 (define_expand "avx_vextractf128<mode>"
7582 [(match_operand:<ssehalfvecmode> 0 "nonimmediate_operand")
7583 (match_operand:V_256 1 "register_operand")
7584 (match_operand:SI 2 "const_0_to_1_operand")]
7587 rtx (*insn)(rtx, rtx);
7589 switch (INTVAL (operands[2]))
7592 insn = gen_vec_extract_lo_<mode>;
7595 insn = gen_vec_extract_hi_<mode>;
7601 emit_insn (insn (operands[0], operands[1]));
7605 (define_insn "vec_extract_lo_<mode><mask_name>"
7606 [(set (match_operand:<ssehalfvecmode> 0 "nonimmediate_operand" "=v,m")
7607 (vec_select:<ssehalfvecmode>
7608 (match_operand:V16FI 1 "<store_mask_predicate>"
7609 "<store_mask_constraint>,v")
7610 (parallel [(const_int 0) (const_int 1)
7611 (const_int 2) (const_int 3)
7612 (const_int 4) (const_int 5)
7613 (const_int 6) (const_int 7)])))]
7615 && <mask_mode512bit_condition>
7616 && (<mask_applied> || !(MEM_P (operands[0]) && MEM_P (operands[1])))"
7619 return "vextract<shuffletype>32x8\t{$0x0, %1, %0<mask_operand2>|%0<mask_operand2>, %1, 0x0}";
7625 [(set (match_operand:<ssehalfvecmode> 0 "nonimmediate_operand")
7626 (vec_select:<ssehalfvecmode>
7627 (match_operand:V16FI 1 "nonimmediate_operand")
7628 (parallel [(const_int 0) (const_int 1)
7629 (const_int 2) (const_int 3)
7630 (const_int 4) (const_int 5)
7631 (const_int 6) (const_int 7)])))]
7632 "TARGET_AVX512F && !(MEM_P (operands[0]) && MEM_P (operands[1]))
7633 && reload_completed"
7634 [(set (match_dup 0) (match_dup 1))]
7635 "operands[1] = gen_lowpart (<ssehalfvecmode>mode, operands[1]);")
7637 (define_insn "vec_extract_lo_<mode><mask_name>"
7638 [(set (match_operand:<ssehalfvecmode> 0 "<store_mask_predicate>" "=v,m")
7639 (vec_select:<ssehalfvecmode>
7640 (match_operand:VI8F_256 1 "<store_mask_predicate>"
7641 "<store_mask_constraint>,v")
7642 (parallel [(const_int 0) (const_int 1)])))]
7644 && <mask_avx512vl_condition> && <mask_avx512dq_condition>
7645 && (<mask_applied> || !(MEM_P (operands[0]) && MEM_P (operands[1])))"
7648 return "vextract<shuffletype>64x2\t{$0x0, %1, %0%{%3%}|%0%{%3%}, %1, 0x0}";
7652 [(set_attr "type" "sselog")
7653 (set_attr "prefix_extra" "1")
7654 (set_attr "length_immediate" "1")
7655 (set_attr "memory" "none,store")
7656 (set_attr "prefix" "evex")
7657 (set_attr "mode" "XI")])
7660 [(set (match_operand:<ssehalfvecmode> 0 "nonimmediate_operand")
7661 (vec_select:<ssehalfvecmode>
7662 (match_operand:VI8F_256 1 "nonimmediate_operand")
7663 (parallel [(const_int 0) (const_int 1)])))]
7664 "TARGET_AVX && !(MEM_P (operands[0]) && MEM_P (operands[1]))
7665 && reload_completed"
7666 [(set (match_dup 0) (match_dup 1))]
7667 "operands[1] = gen_lowpart (<ssehalfvecmode>mode, operands[1]);")
7669 (define_insn "vec_extract_hi_<mode><mask_name>"
7670 [(set (match_operand:<ssehalfvecmode> 0 "<store_mask_predicate>" "=v,<store_mask_constraint>")
7671 (vec_select:<ssehalfvecmode>
7672 (match_operand:VI8F_256 1 "register_operand" "v,v")
7673 (parallel [(const_int 2) (const_int 3)])))]
7674 "TARGET_AVX && <mask_avx512vl_condition> && <mask_avx512dq_condition>"
7676 if (TARGET_AVX512VL)
7678 if (TARGET_AVX512DQ)
7679 return "vextract<shuffletype>64x2\t{$0x1, %1, %0<mask_operand2>|%0<mask_operand2>, %1, 0x1}";
7681 return "vextract<shuffletype>32x4\t{$0x1, %1, %0|%0, %1, 0x1}";
7684 return "vextract<i128>\t{$0x1, %1, %0|%0, %1, 0x1}";
7686 [(set_attr "type" "sselog")
7687 (set_attr "prefix_extra" "1")
7688 (set_attr "length_immediate" "1")
7689 (set_attr "memory" "none,store")
7690 (set_attr "prefix" "vex")
7691 (set_attr "mode" "<sseinsnmode>")])
7694 [(set (match_operand:<ssehalfvecmode> 0 "nonimmediate_operand")
7695 (vec_select:<ssehalfvecmode>
7696 (match_operand:VI4F_256 1 "nonimmediate_operand")
7697 (parallel [(const_int 0) (const_int 1)
7698 (const_int 2) (const_int 3)])))]
7699 "TARGET_AVX && !(MEM_P (operands[0]) && MEM_P (operands[1]))
7700 && reload_completed"
7701 [(set (match_dup 0) (match_dup 1))]
7702 "operands[1] = gen_lowpart (<ssehalfvecmode>mode, operands[1]);")
7704 (define_insn "vec_extract_lo_<mode><mask_name>"
7705 [(set (match_operand:<ssehalfvecmode> 0 "<store_mask_predicate>"
7706 "=<store_mask_constraint>,v")
7707 (vec_select:<ssehalfvecmode>
7708 (match_operand:VI4F_256 1 "<store_mask_predicate>"
7709 "v,<store_mask_constraint>")
7710 (parallel [(const_int 0) (const_int 1)
7711 (const_int 2) (const_int 3)])))]
7713 && <mask_avx512vl_condition> && <mask_avx512dq_condition>
7714 && (<mask_applied> || !(MEM_P (operands[0]) && MEM_P (operands[1])))"
7717 return "vextract<shuffletype>32x4\t{$0x0, %1, %0<mask_operand2>|%0<mask_operand2>, %1, 0x0}";
7721 [(set_attr "type" "sselog1")
7722 (set_attr "prefix_extra" "1")
7723 (set_attr "length_immediate" "1")
7724 (set_attr "prefix" "evex")
7725 (set_attr "mode" "<sseinsnmode>")])
7727 (define_insn "vec_extract_lo_<mode>_maskm"
7728 [(set (match_operand:<ssehalfvecmode> 0 "memory_operand" "=m")
7729 (vec_merge:<ssehalfvecmode>
7730 (vec_select:<ssehalfvecmode>
7731 (match_operand:VI4F_256 1 "register_operand" "v")
7732 (parallel [(const_int 0) (const_int 1)
7733 (const_int 2) (const_int 3)]))
7734 (match_operand:<ssehalfvecmode> 2 "memory_operand" "0")
7735 (match_operand:QI 3 "register_operand" "Yk")))]
7736 "TARGET_AVX512VL && TARGET_AVX512F
7737 && rtx_equal_p (operands[2], operands[0])"
7738 "vextract<shuffletype>32x4\t{$0x0, %1, %0%{%3%}|%0%{%3%}, %1, 0x0}"
7739 [(set_attr "type" "sselog1")
7740 (set_attr "prefix_extra" "1")
7741 (set_attr "length_immediate" "1")
7742 (set_attr "prefix" "evex")
7743 (set_attr "mode" "<sseinsnmode>")])
7745 (define_insn "vec_extract_hi_<mode>_maskm"
7746 [(set (match_operand:<ssehalfvecmode> 0 "memory_operand" "=m")
7747 (vec_merge:<ssehalfvecmode>
7748 (vec_select:<ssehalfvecmode>
7749 (match_operand:VI4F_256 1 "register_operand" "v")
7750 (parallel [(const_int 4) (const_int 5)
7751 (const_int 6) (const_int 7)]))
7752 (match_operand:<ssehalfvecmode> 2 "memory_operand" "0")
7753 (match_operand:<ssehalfvecmode> 3 "register_operand" "Yk")))]
7754 "TARGET_AVX512F && TARGET_AVX512VL
7755 && rtx_equal_p (operands[2], operands[0])"
7756 "vextract<shuffletype>32x4\t{$0x1, %1, %0%{%3%}|%0%{%3%}, %1, 0x1}"
7757 [(set_attr "type" "sselog1")
7758 (set_attr "length_immediate" "1")
7759 (set_attr "prefix" "evex")
7760 (set_attr "mode" "<sseinsnmode>")])
7762 (define_insn "vec_extract_hi_<mode>_mask"
7763 [(set (match_operand:<ssehalfvecmode> 0 "register_operand" "=v")
7764 (vec_merge:<ssehalfvecmode>
7765 (vec_select:<ssehalfvecmode>
7766 (match_operand:VI4F_256 1 "register_operand" "v")
7767 (parallel [(const_int 4) (const_int 5)
7768 (const_int 6) (const_int 7)]))
7769 (match_operand:<ssehalfvecmode> 2 "vector_move_operand" "0C")
7770 (match_operand:<avx512fmaskmode> 3 "register_operand" "Yk")))]
7772 "vextract<shuffletype>32x4\t{$0x1, %1, %0%{%3%}%N2|%0%{%3%}%N2, %1, 0x1}"
7773 [(set_attr "type" "sselog1")
7774 (set_attr "length_immediate" "1")
7775 (set_attr "prefix" "evex")
7776 (set_attr "mode" "<sseinsnmode>")])
7778 (define_insn "vec_extract_hi_<mode>"
7779 [(set (match_operand:<ssehalfvecmode> 0 "nonimmediate_operand" "=xm, vm")
7780 (vec_select:<ssehalfvecmode>
7781 (match_operand:VI4F_256 1 "register_operand" "x, v")
7782 (parallel [(const_int 4) (const_int 5)
7783 (const_int 6) (const_int 7)])))]
7786 vextract<i128>\t{$0x1, %1, %0|%0, %1, 0x1}
7787 vextract<shuffletype>32x4\t{$0x1, %1, %0|%0, %1, 0x1}"
7788 [(set_attr "isa" "*, avx512vl")
7789 (set_attr "prefix" "vex, evex")
7790 (set_attr "type" "sselog1")
7791 (set_attr "length_immediate" "1")
7792 (set_attr "mode" "<sseinsnmode>")])
7794 (define_insn_and_split "vec_extract_lo_v32hi"
7795 [(set (match_operand:V16HI 0 "nonimmediate_operand" "=v,m")
7797 (match_operand:V32HI 1 "nonimmediate_operand" "vm,v")
7798 (parallel [(const_int 0) (const_int 1)
7799 (const_int 2) (const_int 3)
7800 (const_int 4) (const_int 5)
7801 (const_int 6) (const_int 7)
7802 (const_int 8) (const_int 9)
7803 (const_int 10) (const_int 11)
7804 (const_int 12) (const_int 13)
7805 (const_int 14) (const_int 15)])))]
7806 "TARGET_AVX512F && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
7808 "&& reload_completed"
7809 [(set (match_dup 0) (match_dup 1))]
7810 "operands[1] = gen_lowpart (V16HImode, operands[1]);")
7812 (define_insn "vec_extract_hi_v32hi"
7813 [(set (match_operand:V16HI 0 "nonimmediate_operand" "=v,m")
7815 (match_operand:V32HI 1 "register_operand" "v,v")
7816 (parallel [(const_int 16) (const_int 17)
7817 (const_int 18) (const_int 19)
7818 (const_int 20) (const_int 21)
7819 (const_int 22) (const_int 23)
7820 (const_int 24) (const_int 25)
7821 (const_int 26) (const_int 27)
7822 (const_int 28) (const_int 29)
7823 (const_int 30) (const_int 31)])))]
7825 "vextracti64x4\t{$0x1, %1, %0|%0, %1, 0x1}"
7826 [(set_attr "type" "sselog")
7827 (set_attr "prefix_extra" "1")
7828 (set_attr "length_immediate" "1")
7829 (set_attr "memory" "none,store")
7830 (set_attr "prefix" "evex")
7831 (set_attr "mode" "XI")])
7833 (define_insn_and_split "vec_extract_lo_v16hi"
7834 [(set (match_operand:V8HI 0 "nonimmediate_operand" "=v,m")
7836 (match_operand:V16HI 1 "nonimmediate_operand" "vm,v")
7837 (parallel [(const_int 0) (const_int 1)
7838 (const_int 2) (const_int 3)
7839 (const_int 4) (const_int 5)
7840 (const_int 6) (const_int 7)])))]
7841 "TARGET_AVX && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
7843 "&& reload_completed"
7844 [(set (match_dup 0) (match_dup 1))]
7845 "operands[1] = gen_lowpart (V8HImode, operands[1]);")
7847 (define_insn "vec_extract_hi_v16hi"
7848 [(set (match_operand:V8HI 0 "nonimmediate_operand" "=x,m,v,m,v,m")
7850 (match_operand:V16HI 1 "register_operand" "x,x,v,v,v,v")
7851 (parallel [(const_int 8) (const_int 9)
7852 (const_int 10) (const_int 11)
7853 (const_int 12) (const_int 13)
7854 (const_int 14) (const_int 15)])))]
7857 vextract%~128\t{$0x1, %1, %0|%0, %1, 0x1}
7858 vextract%~128\t{$0x1, %1, %0|%0, %1, 0x1}
7859 vextracti32x4\t{$0x1, %1, %0|%0, %1, 0x1}
7860 vextracti32x4\t{$0x1, %1, %0|%0, %1, 0x1}
7861 vextracti32x4\t{$0x1, %g1, %0|%0, %g1, 0x1}
7862 vextracti32x4\t{$0x1, %g1, %0|%0, %g1, 0x1}"
7863 [(set_attr "type" "sselog")
7864 (set_attr "prefix_extra" "1")
7865 (set_attr "length_immediate" "1")
7866 (set_attr "isa" "*,*,avx512dq,avx512dq,avx512f,avx512f")
7867 (set_attr "memory" "none,store,none,store,none,store")
7868 (set_attr "prefix" "vex,vex,evex,evex,evex,evex")
7869 (set_attr "mode" "OI")])
7871 (define_insn_and_split "vec_extract_lo_v64qi"
7872 [(set (match_operand:V32QI 0 "nonimmediate_operand" "=v,m")
7874 (match_operand:V64QI 1 "nonimmediate_operand" "vm,v")
7875 (parallel [(const_int 0) (const_int 1)
7876 (const_int 2) (const_int 3)
7877 (const_int 4) (const_int 5)
7878 (const_int 6) (const_int 7)
7879 (const_int 8) (const_int 9)
7880 (const_int 10) (const_int 11)
7881 (const_int 12) (const_int 13)
7882 (const_int 14) (const_int 15)
7883 (const_int 16) (const_int 17)
7884 (const_int 18) (const_int 19)
7885 (const_int 20) (const_int 21)
7886 (const_int 22) (const_int 23)
7887 (const_int 24) (const_int 25)
7888 (const_int 26) (const_int 27)
7889 (const_int 28) (const_int 29)
7890 (const_int 30) (const_int 31)])))]
7891 "TARGET_AVX512F && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
7893 "&& reload_completed"
7894 [(set (match_dup 0) (match_dup 1))]
7895 "operands[1] = gen_lowpart (V32QImode, operands[1]);")
7897 (define_insn "vec_extract_hi_v64qi"
7898 [(set (match_operand:V32QI 0 "nonimmediate_operand" "=v,m")
7900 (match_operand:V64QI 1 "register_operand" "v,v")
7901 (parallel [(const_int 32) (const_int 33)
7902 (const_int 34) (const_int 35)
7903 (const_int 36) (const_int 37)
7904 (const_int 38) (const_int 39)
7905 (const_int 40) (const_int 41)
7906 (const_int 42) (const_int 43)
7907 (const_int 44) (const_int 45)
7908 (const_int 46) (const_int 47)
7909 (const_int 48) (const_int 49)
7910 (const_int 50) (const_int 51)
7911 (const_int 52) (const_int 53)
7912 (const_int 54) (const_int 55)
7913 (const_int 56) (const_int 57)
7914 (const_int 58) (const_int 59)
7915 (const_int 60) (const_int 61)
7916 (const_int 62) (const_int 63)])))]
7918 "vextracti64x4\t{$0x1, %1, %0|%0, %1, 0x1}"
7919 [(set_attr "type" "sselog")
7920 (set_attr "prefix_extra" "1")
7921 (set_attr "length_immediate" "1")
7922 (set_attr "memory" "none,store")
7923 (set_attr "prefix" "evex")
7924 (set_attr "mode" "XI")])
7926 (define_insn_and_split "vec_extract_lo_v32qi"
7927 [(set (match_operand:V16QI 0 "nonimmediate_operand" "=v,m")
7929 (match_operand:V32QI 1 "nonimmediate_operand" "vm,v")
7930 (parallel [(const_int 0) (const_int 1)
7931 (const_int 2) (const_int 3)
7932 (const_int 4) (const_int 5)
7933 (const_int 6) (const_int 7)
7934 (const_int 8) (const_int 9)
7935 (const_int 10) (const_int 11)
7936 (const_int 12) (const_int 13)
7937 (const_int 14) (const_int 15)])))]
7938 "TARGET_AVX && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
7940 "&& reload_completed"
7941 [(set (match_dup 0) (match_dup 1))]
7942 "operands[1] = gen_lowpart (V16QImode, operands[1]);")
7944 (define_insn "vec_extract_hi_v32qi"
7945 [(set (match_operand:V16QI 0 "nonimmediate_operand" "=x,m,v,m,v,m")
7947 (match_operand:V32QI 1 "register_operand" "x,x,v,v,v,v")
7948 (parallel [(const_int 16) (const_int 17)
7949 (const_int 18) (const_int 19)
7950 (const_int 20) (const_int 21)
7951 (const_int 22) (const_int 23)
7952 (const_int 24) (const_int 25)
7953 (const_int 26) (const_int 27)
7954 (const_int 28) (const_int 29)
7955 (const_int 30) (const_int 31)])))]
7958 vextract%~128\t{$0x1, %1, %0|%0, %1, 0x1}
7959 vextract%~128\t{$0x1, %1, %0|%0, %1, 0x1}
7960 vextracti32x4\t{$0x1, %1, %0|%0, %1, 0x1}
7961 vextracti32x4\t{$0x1, %1, %0|%0, %1, 0x1}
7962 vextracti32x4\t{$0x1, %g1, %0|%0, %g1, 0x1}
7963 vextracti32x4\t{$0x1, %g1, %0|%0, %g1, 0x1}"
7964 [(set_attr "type" "sselog")
7965 (set_attr "prefix_extra" "1")
7966 (set_attr "length_immediate" "1")
7967 (set_attr "isa" "*,*,avx512dq,avx512dq,avx512f,avx512f")
7968 (set_attr "memory" "none,store,none,store,none,store")
7969 (set_attr "prefix" "vex,vex,evex,evex,evex,evex")
7970 (set_attr "mode" "OI")])
7972 ;; Modes handled by vec_extract patterns.
7973 (define_mode_iterator VEC_EXTRACT_MODE
7974 [(V64QI "TARGET_AVX512BW") (V32QI "TARGET_AVX") V16QI
7975 (V32HI "TARGET_AVX512BW") (V16HI "TARGET_AVX") V8HI
7976 (V16SI "TARGET_AVX512F") (V8SI "TARGET_AVX") V4SI
7977 (V8DI "TARGET_AVX512F") (V4DI "TARGET_AVX") V2DI
7978 (V16SF "TARGET_AVX512F") (V8SF "TARGET_AVX") V4SF
7979 (V8DF "TARGET_AVX512F") (V4DF "TARGET_AVX") V2DF
7980 (V4TI "TARGET_AVX512F") (V2TI "TARGET_AVX")])
7982 (define_expand "vec_extract<mode><ssescalarmodelower>"
7983 [(match_operand:<ssescalarmode> 0 "register_operand")
7984 (match_operand:VEC_EXTRACT_MODE 1 "register_operand")
7985 (match_operand 2 "const_int_operand")]
7988 ix86_expand_vector_extract (false, operands[0], operands[1],
7989 INTVAL (operands[2]));
7993 (define_expand "vec_extract<mode><ssehalfvecmodelower>"
7994 [(match_operand:<ssehalfvecmode> 0 "nonimmediate_operand")
7995 (match_operand:V_512 1 "register_operand")
7996 (match_operand 2 "const_0_to_1_operand")]
7999 if (INTVAL (operands[2]))
8000 emit_insn (gen_vec_extract_hi_<mode> (operands[0], operands[1]));
8002 emit_insn (gen_vec_extract_lo_<mode> (operands[0], operands[1]));
8006 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
8008 ;; Parallel double-precision floating point element swizzling
8010 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
8012 (define_insn "<mask_codefor>avx512f_unpckhpd512<mask_name>"
8013 [(set (match_operand:V8DF 0 "register_operand" "=v")
8016 (match_operand:V8DF 1 "register_operand" "v")
8017 (match_operand:V8DF 2 "nonimmediate_operand" "vm"))
8018 (parallel [(const_int 1) (const_int 9)
8019 (const_int 3) (const_int 11)
8020 (const_int 5) (const_int 13)
8021 (const_int 7) (const_int 15)])))]
8023 "vunpckhpd\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
8024 [(set_attr "type" "sselog")
8025 (set_attr "prefix" "evex")
8026 (set_attr "mode" "V8DF")])
8028 ;; Recall that the 256-bit unpck insns only shuffle within their lanes.
8029 (define_insn "avx_unpckhpd256<mask_name>"
8030 [(set (match_operand:V4DF 0 "register_operand" "=v")
8033 (match_operand:V4DF 1 "register_operand" "v")
8034 (match_operand:V4DF 2 "nonimmediate_operand" "vm"))
8035 (parallel [(const_int 1) (const_int 5)
8036 (const_int 3) (const_int 7)])))]
8037 "TARGET_AVX && <mask_avx512vl_condition>"
8038 "vunpckhpd\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
8039 [(set_attr "type" "sselog")
8040 (set_attr "prefix" "vex")
8041 (set_attr "mode" "V4DF")])
8043 (define_expand "vec_interleave_highv4df"
8047 (match_operand:V4DF 1 "register_operand")
8048 (match_operand:V4DF 2 "nonimmediate_operand"))
8049 (parallel [(const_int 0) (const_int 4)
8050 (const_int 2) (const_int 6)])))
8056 (parallel [(const_int 1) (const_int 5)
8057 (const_int 3) (const_int 7)])))
8058 (set (match_operand:V4DF 0 "register_operand")
8063 (parallel [(const_int 2) (const_int 3)
8064 (const_int 6) (const_int 7)])))]
8067 operands[3] = gen_reg_rtx (V4DFmode);
8068 operands[4] = gen_reg_rtx (V4DFmode);
8072 (define_insn "avx512vl_unpckhpd128_mask"
8073 [(set (match_operand:V2DF 0 "register_operand" "=v")
8077 (match_operand:V2DF 1 "register_operand" "v")
8078 (match_operand:V2DF 2 "nonimmediate_operand" "vm"))
8079 (parallel [(const_int 1) (const_int 3)]))
8080 (match_operand:V2DF 3 "vector_move_operand" "0C")
8081 (match_operand:QI 4 "register_operand" "Yk")))]
8083 "vunpckhpd\t{%2, %1, %0%{%4%}%N3|%0%{%4%}%N3, %1, %2}"
8084 [(set_attr "type" "sselog")
8085 (set_attr "prefix" "evex")
8086 (set_attr "mode" "V2DF")])
8088 (define_expand "vec_interleave_highv2df"
8089 [(set (match_operand:V2DF 0 "register_operand")
8092 (match_operand:V2DF 1 "nonimmediate_operand")
8093 (match_operand:V2DF 2 "nonimmediate_operand"))
8094 (parallel [(const_int 1)
8098 if (!ix86_vec_interleave_v2df_operator_ok (operands, 1))
8099 operands[2] = force_reg (V2DFmode, operands[2]);
8102 (define_insn "*vec_interleave_highv2df"
8103 [(set (match_operand:V2DF 0 "nonimmediate_operand" "=x,v,v,x,v,m")
8106 (match_operand:V2DF 1 "nonimmediate_operand" " 0,v,o,o,o,v")
8107 (match_operand:V2DF 2 "nonimmediate_operand" " x,v,1,0,v,0"))
8108 (parallel [(const_int 1)
8110 "TARGET_SSE2 && ix86_vec_interleave_v2df_operator_ok (operands, 1)"
8112 unpckhpd\t{%2, %0|%0, %2}
8113 vunpckhpd\t{%2, %1, %0|%0, %1, %2}
8114 %vmovddup\t{%H1, %0|%0, %H1}
8115 movlpd\t{%H1, %0|%0, %H1}
8116 vmovlpd\t{%H1, %2, %0|%0, %2, %H1}
8117 %vmovhpd\t{%1, %0|%q0, %1}"
8118 [(set_attr "isa" "noavx,avx,sse3,noavx,avx,*")
8119 (set_attr "type" "sselog,sselog,sselog,ssemov,ssemov,ssemov")
8120 (set (attr "prefix_data16")
8121 (if_then_else (eq_attr "alternative" "3,5")
8123 (const_string "*")))
8124 (set_attr "prefix" "orig,maybe_evex,maybe_vex,orig,maybe_evex,maybe_vex")
8125 (set_attr "mode" "V2DF,V2DF,DF,V1DF,V1DF,V1DF")])
8127 (define_expand "avx512f_movddup512<mask_name>"
8128 [(set (match_operand:V8DF 0 "register_operand")
8131 (match_operand:V8DF 1 "nonimmediate_operand")
8133 (parallel [(const_int 0) (const_int 8)
8134 (const_int 2) (const_int 10)
8135 (const_int 4) (const_int 12)
8136 (const_int 6) (const_int 14)])))]
8139 (define_expand "avx512f_unpcklpd512<mask_name>"
8140 [(set (match_operand:V8DF 0 "register_operand")
8143 (match_operand:V8DF 1 "register_operand")
8144 (match_operand:V8DF 2 "nonimmediate_operand"))
8145 (parallel [(const_int 0) (const_int 8)
8146 (const_int 2) (const_int 10)
8147 (const_int 4) (const_int 12)
8148 (const_int 6) (const_int 14)])))]
8151 (define_insn "*avx512f_unpcklpd512<mask_name>"
8152 [(set (match_operand:V8DF 0 "register_operand" "=v,v")
8155 (match_operand:V8DF 1 "nonimmediate_operand" "vm, v")
8156 (match_operand:V8DF 2 "nonimmediate_operand" "1 ,vm"))
8157 (parallel [(const_int 0) (const_int 8)
8158 (const_int 2) (const_int 10)
8159 (const_int 4) (const_int 12)
8160 (const_int 6) (const_int 14)])))]
8163 vmovddup\t{%1, %0<mask_operand3>|%0<mask_operand3>, %1}
8164 vunpcklpd\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
8165 [(set_attr "type" "sselog")
8166 (set_attr "prefix" "evex")
8167 (set_attr "mode" "V8DF")])
8169 ;; Recall that the 256-bit unpck insns only shuffle within their lanes.
8170 (define_expand "avx_movddup256<mask_name>"
8171 [(set (match_operand:V4DF 0 "register_operand")
8174 (match_operand:V4DF 1 "nonimmediate_operand")
8176 (parallel [(const_int 0) (const_int 4)
8177 (const_int 2) (const_int 6)])))]
8178 "TARGET_AVX && <mask_avx512vl_condition>")
8180 (define_expand "avx_unpcklpd256<mask_name>"
8181 [(set (match_operand:V4DF 0 "register_operand")
8184 (match_operand:V4DF 1 "register_operand")
8185 (match_operand:V4DF 2 "nonimmediate_operand"))
8186 (parallel [(const_int 0) (const_int 4)
8187 (const_int 2) (const_int 6)])))]
8188 "TARGET_AVX && <mask_avx512vl_condition>")
8190 (define_insn "*avx_unpcklpd256<mask_name>"
8191 [(set (match_operand:V4DF 0 "register_operand" "=v,v")
8194 (match_operand:V4DF 1 "nonimmediate_operand" " v,m")
8195 (match_operand:V4DF 2 "nonimmediate_operand" "vm,1"))
8196 (parallel [(const_int 0) (const_int 4)
8197 (const_int 2) (const_int 6)])))]
8198 "TARGET_AVX && <mask_avx512vl_condition>"
8200 vunpcklpd\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}
8201 vmovddup\t{%1, %0<mask_operand3>|%0<mask_operand3>, %1}"
8202 [(set_attr "type" "sselog")
8203 (set_attr "prefix" "vex")
8204 (set_attr "mode" "V4DF")])
8206 (define_expand "vec_interleave_lowv4df"
8210 (match_operand:V4DF 1 "register_operand")
8211 (match_operand:V4DF 2 "nonimmediate_operand"))
8212 (parallel [(const_int 0) (const_int 4)
8213 (const_int 2) (const_int 6)])))
8219 (parallel [(const_int 1) (const_int 5)
8220 (const_int 3) (const_int 7)])))
8221 (set (match_operand:V4DF 0 "register_operand")
8226 (parallel [(const_int 0) (const_int 1)
8227 (const_int 4) (const_int 5)])))]
8230 operands[3] = gen_reg_rtx (V4DFmode);
8231 operands[4] = gen_reg_rtx (V4DFmode);
8234 (define_insn "avx512vl_unpcklpd128_mask"
8235 [(set (match_operand:V2DF 0 "register_operand" "=v")
8239 (match_operand:V2DF 1 "register_operand" "v")
8240 (match_operand:V2DF 2 "nonimmediate_operand" "vm"))
8241 (parallel [(const_int 0) (const_int 2)]))
8242 (match_operand:V2DF 3 "vector_move_operand" "0C")
8243 (match_operand:QI 4 "register_operand" "Yk")))]
8245 "vunpcklpd\t{%2, %1, %0%{%4%}%N3|%0%{%4%}%N3, %1, %2}"
8246 [(set_attr "type" "sselog")
8247 (set_attr "prefix" "evex")
8248 (set_attr "mode" "V2DF")])
8250 (define_expand "vec_interleave_lowv2df"
8251 [(set (match_operand:V2DF 0 "register_operand")
8254 (match_operand:V2DF 1 "nonimmediate_operand")
8255 (match_operand:V2DF 2 "nonimmediate_operand"))
8256 (parallel [(const_int 0)
8260 if (!ix86_vec_interleave_v2df_operator_ok (operands, 0))
8261 operands[1] = force_reg (V2DFmode, operands[1]);
8264 (define_insn "*vec_interleave_lowv2df"
8265 [(set (match_operand:V2DF 0 "nonimmediate_operand" "=x,v,v,x,v,o")
8268 (match_operand:V2DF 1 "nonimmediate_operand" " 0,v,m,0,v,0")
8269 (match_operand:V2DF 2 "nonimmediate_operand" " x,v,1,m,m,v"))
8270 (parallel [(const_int 0)
8272 "TARGET_SSE2 && ix86_vec_interleave_v2df_operator_ok (operands, 0)"
8274 unpcklpd\t{%2, %0|%0, %2}
8275 vunpcklpd\t{%2, %1, %0|%0, %1, %2}
8276 %vmovddup\t{%1, %0|%0, %q1}
8277 movhpd\t{%2, %0|%0, %q2}
8278 vmovhpd\t{%2, %1, %0|%0, %1, %q2}
8279 %vmovlpd\t{%2, %H0|%H0, %2}"
8280 [(set_attr "isa" "noavx,avx,sse3,noavx,avx,*")
8281 (set_attr "type" "sselog,sselog,sselog,ssemov,ssemov,ssemov")
8282 (set (attr "prefix_data16")
8283 (if_then_else (eq_attr "alternative" "3,5")
8285 (const_string "*")))
8286 (set_attr "prefix" "orig,maybe_evex,maybe_vex,orig,maybe_evex,maybe_vex")
8287 (set_attr "mode" "V2DF,V2DF,DF,V1DF,V1DF,V1DF")])
8290 [(set (match_operand:V2DF 0 "memory_operand")
8293 (match_operand:V2DF 1 "register_operand")
8295 (parallel [(const_int 0)
8297 "TARGET_SSE3 && reload_completed"
8300 rtx low = gen_lowpart (DFmode, operands[1]);
8302 emit_move_insn (adjust_address (operands[0], DFmode, 0), low);
8303 emit_move_insn (adjust_address (operands[0], DFmode, 8), low);
8308 [(set (match_operand:V2DF 0 "register_operand")
8311 (match_operand:V2DF 1 "memory_operand")
8313 (parallel [(match_operand:SI 2 "const_0_to_1_operand")
8314 (match_operand:SI 3 "const_int_operand")])))]
8315 "TARGET_SSE3 && INTVAL (operands[2]) + 2 == INTVAL (operands[3])"
8316 [(set (match_dup 0) (vec_duplicate:V2DF (match_dup 1)))]
8318 operands[1] = adjust_address (operands[1], DFmode, INTVAL (operands[2]) * 8);
8321 (define_insn "avx512f_vmscalef<mode><mask_scalar_name><round_scalar_name>"
8322 [(set (match_operand:VF_128 0 "register_operand" "=v")
8325 [(match_operand:VF_128 1 "register_operand" "v")
8326 (match_operand:VF_128 2 "<round_scalar_nimm_predicate>" "<round_scalar_constraint>")]
8331 "vscalef<ssescalarmodesuffix>\t{<round_scalar_mask_op3>%2, %1, %0<mask_scalar_operand3>|%0<mask_scalar_operand3>, %1, %2<round_scalar_mask_op3>}"
8332 [(set_attr "prefix" "evex")
8333 (set_attr "mode" "<ssescalarmode>")])
8335 (define_insn "<avx512>_scalef<mode><mask_name><round_name>"
8336 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v")
8338 [(match_operand:VF_AVX512VL 1 "register_operand" "v")
8339 (match_operand:VF_AVX512VL 2 "nonimmediate_operand" "<round_constraint>")]
8342 "vscalef<ssemodesuffix>\t{<round_mask_op3>%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2<round_mask_op3>}"
8343 [(set_attr "prefix" "evex")
8344 (set_attr "mode" "<MODE>")])
8346 (define_expand "<avx512>_vternlog<mode>_maskz"
8347 [(match_operand:VI48_AVX512VL 0 "register_operand")
8348 (match_operand:VI48_AVX512VL 1 "register_operand")
8349 (match_operand:VI48_AVX512VL 2 "register_operand")
8350 (match_operand:VI48_AVX512VL 3 "nonimmediate_operand")
8351 (match_operand:SI 4 "const_0_to_255_operand")
8352 (match_operand:<avx512fmaskmode> 5 "register_operand")]
8355 emit_insn (gen_<avx512>_vternlog<mode>_maskz_1 (
8356 operands[0], operands[1], operands[2], operands[3],
8357 operands[4], CONST0_RTX (<MODE>mode), operands[5]));
8361 (define_insn "<avx512>_vternlog<mode><sd_maskz_name>"
8362 [(set (match_operand:VI48_AVX512VL 0 "register_operand" "=v")
8363 (unspec:VI48_AVX512VL
8364 [(match_operand:VI48_AVX512VL 1 "register_operand" "0")
8365 (match_operand:VI48_AVX512VL 2 "register_operand" "v")
8366 (match_operand:VI48_AVX512VL 3 "nonimmediate_operand" "vm")
8367 (match_operand:SI 4 "const_0_to_255_operand")]
8370 "vpternlog<ssemodesuffix>\t{%4, %3, %2, %0<sd_mask_op5>|%0<sd_mask_op5>, %2, %3, %4}"
8371 [(set_attr "type" "sselog")
8372 (set_attr "prefix" "evex")
8373 (set_attr "mode" "<sseinsnmode>")])
8375 (define_insn "<avx512>_vternlog<mode>_mask"
8376 [(set (match_operand:VI48_AVX512VL 0 "register_operand" "=v")
8377 (vec_merge:VI48_AVX512VL
8378 (unspec:VI48_AVX512VL
8379 [(match_operand:VI48_AVX512VL 1 "register_operand" "0")
8380 (match_operand:VI48_AVX512VL 2 "register_operand" "v")
8381 (match_operand:VI48_AVX512VL 3 "nonimmediate_operand" "vm")
8382 (match_operand:SI 4 "const_0_to_255_operand")]
8385 (match_operand:<avx512fmaskmode> 5 "register_operand" "Yk")))]
8387 "vpternlog<ssemodesuffix>\t{%4, %3, %2, %0%{%5%}|%0%{%5%}, %2, %3, %4}"
8388 [(set_attr "type" "sselog")
8389 (set_attr "prefix" "evex")
8390 (set_attr "mode" "<sseinsnmode>")])
8392 (define_insn "<avx512>_getexp<mode><mask_name><round_saeonly_name>"
8393 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v")
8394 (unspec:VF_AVX512VL [(match_operand:VF_AVX512VL 1 "<round_saeonly_nimm_predicate>" "<round_saeonly_constraint>")]
8397 "vgetexp<ssemodesuffix>\t{<round_saeonly_mask_op2>%1, %0<mask_operand2>|%0<mask_operand2>, %1<round_saeonly_mask_op2>}";
8398 [(set_attr "prefix" "evex")
8399 (set_attr "mode" "<MODE>")])
8401 (define_insn "avx512f_sgetexp<mode><mask_scalar_name><round_saeonly_scalar_name>"
8402 [(set (match_operand:VF_128 0 "register_operand" "=v")
8405 [(match_operand:VF_128 1 "register_operand" "v")
8406 (match_operand:VF_128 2 "<round_saeonly_scalar_nimm_predicate>" "<round_saeonly_scalar_constraint>")]
8411 "vgetexp<ssescalarmodesuffix>\t{<round_saeonly_scalar_mask_op3>%2, %1, %0<mask_scalar_operand3>|%0<mask_scalar_operand3>, %1, %2<round_saeonly_scalar_mask_op3>}";
8412 [(set_attr "prefix" "evex")
8413 (set_attr "mode" "<ssescalarmode>")])
8415 (define_insn "<mask_codefor><avx512>_align<mode><mask_name>"
8416 [(set (match_operand:VI48_AVX512VL 0 "register_operand" "=v")
8417 (unspec:VI48_AVX512VL [(match_operand:VI48_AVX512VL 1 "register_operand" "v")
8418 (match_operand:VI48_AVX512VL 2 "nonimmediate_operand" "vm")
8419 (match_operand:SI 3 "const_0_to_255_operand")]
8422 "valign<ssemodesuffix>\t{%3, %2, %1, %0<mask_operand4>|%0<mask_operand4>, %1, %2, %3}";
8423 [(set_attr "prefix" "evex")
8424 (set_attr "mode" "<sseinsnmode>")])
8426 (define_expand "avx512f_shufps512_mask"
8427 [(match_operand:V16SF 0 "register_operand")
8428 (match_operand:V16SF 1 "register_operand")
8429 (match_operand:V16SF 2 "nonimmediate_operand")
8430 (match_operand:SI 3 "const_0_to_255_operand")
8431 (match_operand:V16SF 4 "register_operand")
8432 (match_operand:HI 5 "register_operand")]
8435 int mask = INTVAL (operands[3]);
8436 emit_insn (gen_avx512f_shufps512_1_mask (operands[0], operands[1], operands[2],
8437 GEN_INT ((mask >> 0) & 3),
8438 GEN_INT ((mask >> 2) & 3),
8439 GEN_INT (((mask >> 4) & 3) + 16),
8440 GEN_INT (((mask >> 6) & 3) + 16),
8441 GEN_INT (((mask >> 0) & 3) + 4),
8442 GEN_INT (((mask >> 2) & 3) + 4),
8443 GEN_INT (((mask >> 4) & 3) + 20),
8444 GEN_INT (((mask >> 6) & 3) + 20),
8445 GEN_INT (((mask >> 0) & 3) + 8),
8446 GEN_INT (((mask >> 2) & 3) + 8),
8447 GEN_INT (((mask >> 4) & 3) + 24),
8448 GEN_INT (((mask >> 6) & 3) + 24),
8449 GEN_INT (((mask >> 0) & 3) + 12),
8450 GEN_INT (((mask >> 2) & 3) + 12),
8451 GEN_INT (((mask >> 4) & 3) + 28),
8452 GEN_INT (((mask >> 6) & 3) + 28),
8453 operands[4], operands[5]));
8458 (define_expand "<avx512>_fixupimm<mode>_maskz<round_saeonly_expand_name>"
8459 [(match_operand:VF_AVX512VL 0 "register_operand")
8460 (match_operand:VF_AVX512VL 1 "register_operand")
8461 (match_operand:VF_AVX512VL 2 "register_operand")
8462 (match_operand:<sseintvecmode> 3 "<round_saeonly_expand_nimm_predicate>")
8463 (match_operand:SI 4 "const_0_to_255_operand")
8464 (match_operand:<avx512fmaskmode> 5 "register_operand")]
8467 emit_insn (gen_<avx512>_fixupimm<mode>_maskz_1<round_saeonly_expand_name> (
8468 operands[0], operands[1], operands[2], operands[3],
8469 operands[4], CONST0_RTX (<MODE>mode), operands[5]
8470 <round_saeonly_expand_operand6>));
8474 (define_insn "<avx512>_fixupimm<mode><sd_maskz_name><round_saeonly_name>"
8475 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v")
8477 [(match_operand:VF_AVX512VL 1 "register_operand" "0")
8478 (match_operand:VF_AVX512VL 2 "register_operand" "v")
8479 (match_operand:<sseintvecmode> 3 "nonimmediate_operand" "<round_saeonly_constraint>")
8480 (match_operand:SI 4 "const_0_to_255_operand")]
8483 "vfixupimm<ssemodesuffix>\t{%4, <round_saeonly_sd_mask_op5>%3, %2, %0<sd_mask_op5>|%0<sd_mask_op5>, %2, %3<round_saeonly_sd_mask_op5>, %4}";
8484 [(set_attr "prefix" "evex")
8485 (set_attr "mode" "<MODE>")])
8487 (define_insn "<avx512>_fixupimm<mode>_mask<round_saeonly_name>"
8488 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v")
8489 (vec_merge:VF_AVX512VL
8491 [(match_operand:VF_AVX512VL 1 "register_operand" "0")
8492 (match_operand:VF_AVX512VL 2 "register_operand" "v")
8493 (match_operand:<sseintvecmode> 3 "nonimmediate_operand" "<round_saeonly_constraint>")
8494 (match_operand:SI 4 "const_0_to_255_operand")]
8497 (match_operand:<avx512fmaskmode> 5 "register_operand" "Yk")))]
8499 "vfixupimm<ssemodesuffix>\t{%4, <round_saeonly_op6>%3, %2, %0%{%5%}|%0%{%5%}, %2, %3<round_saeonly_op6>, %4}";
8500 [(set_attr "prefix" "evex")
8501 (set_attr "mode" "<MODE>")])
8503 (define_expand "avx512f_sfixupimm<mode>_maskz<round_saeonly_expand_name>"
8504 [(match_operand:VF_128 0 "register_operand")
8505 (match_operand:VF_128 1 "register_operand")
8506 (match_operand:VF_128 2 "register_operand")
8507 (match_operand:<sseintvecmode> 3 "<round_saeonly_expand_nimm_predicate>")
8508 (match_operand:SI 4 "const_0_to_255_operand")
8509 (match_operand:<avx512fmaskmode> 5 "register_operand")]
8512 emit_insn (gen_avx512f_sfixupimm<mode>_maskz_1<round_saeonly_expand_name> (
8513 operands[0], operands[1], operands[2], operands[3],
8514 operands[4], CONST0_RTX (<MODE>mode), operands[5]
8515 <round_saeonly_expand_operand6>));
8519 (define_insn "avx512f_sfixupimm<mode><sd_maskz_name><round_saeonly_name>"
8520 [(set (match_operand:VF_128 0 "register_operand" "=v")
8523 [(match_operand:VF_128 1 "register_operand" "0")
8524 (match_operand:VF_128 2 "register_operand" "v")
8525 (match_operand:<sseintvecmode> 3 "<round_saeonly_nimm_predicate>" "<round_saeonly_constraint>")
8526 (match_operand:SI 4 "const_0_to_255_operand")]
8531 "vfixupimm<ssescalarmodesuffix>\t{%4, <round_saeonly_sd_mask_op5>%3, %2, %0<sd_mask_op5>|%0<sd_mask_op5>, %2, %3<round_saeonly_sd_mask_op5>, %4}";
8532 [(set_attr "prefix" "evex")
8533 (set_attr "mode" "<ssescalarmode>")])
8535 (define_insn "avx512f_sfixupimm<mode>_mask<round_saeonly_name>"
8536 [(set (match_operand:VF_128 0 "register_operand" "=v")
8540 [(match_operand:VF_128 1 "register_operand" "0")
8541 (match_operand:VF_128 2 "register_operand" "v")
8542 (match_operand:<sseintvecmode> 3 "<round_saeonly_nimm_predicate>" "<round_saeonly_constraint>")
8543 (match_operand:SI 4 "const_0_to_255_operand")]
8548 (match_operand:<avx512fmaskmode> 5 "register_operand" "Yk")))]
8550 "vfixupimm<ssescalarmodesuffix>\t{%4, <round_saeonly_op6>%3, %2, %0%{%5%}|%0%{%5%}, %2, %3<round_saeonly_op6>, %4}";
8551 [(set_attr "prefix" "evex")
8552 (set_attr "mode" "<ssescalarmode>")])
8554 (define_insn "<avx512>_rndscale<mode><mask_name><round_saeonly_name>"
8555 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v")
8557 [(match_operand:VF_AVX512VL 1 "nonimmediate_operand" "<round_saeonly_constraint>")
8558 (match_operand:SI 2 "const_0_to_255_operand")]
8561 "vrndscale<ssemodesuffix>\t{%2, <round_saeonly_mask_op3>%1, %0<mask_operand3>|%0<mask_operand3>, %1<round_saeonly_mask_op3>, %2}"
8562 [(set_attr "length_immediate" "1")
8563 (set_attr "prefix" "evex")
8564 (set_attr "mode" "<MODE>")])
8566 (define_insn "avx512f_rndscale<mode><round_saeonly_name>"
8567 [(set (match_operand:VF_128 0 "register_operand" "=v")
8570 [(match_operand:VF_128 1 "register_operand" "v")
8571 (match_operand:VF_128 2 "<round_saeonly_nimm_predicate>" "<round_saeonly_constraint>")
8572 (match_operand:SI 3 "const_0_to_255_operand")]
8577 "vrndscale<ssescalarmodesuffix>\t{%3, <round_saeonly_op4>%2, %1, %0|%0, %1, %2<round_saeonly_op4>, %3}"
8578 [(set_attr "length_immediate" "1")
8579 (set_attr "prefix" "evex")
8580 (set_attr "mode" "<MODE>")])
8582 ;; One bit in mask selects 2 elements.
8583 (define_insn "avx512f_shufps512_1<mask_name>"
8584 [(set (match_operand:V16SF 0 "register_operand" "=v")
8587 (match_operand:V16SF 1 "register_operand" "v")
8588 (match_operand:V16SF 2 "nonimmediate_operand" "vm"))
8589 (parallel [(match_operand 3 "const_0_to_3_operand")
8590 (match_operand 4 "const_0_to_3_operand")
8591 (match_operand 5 "const_16_to_19_operand")
8592 (match_operand 6 "const_16_to_19_operand")
8593 (match_operand 7 "const_4_to_7_operand")
8594 (match_operand 8 "const_4_to_7_operand")
8595 (match_operand 9 "const_20_to_23_operand")
8596 (match_operand 10 "const_20_to_23_operand")
8597 (match_operand 11 "const_8_to_11_operand")
8598 (match_operand 12 "const_8_to_11_operand")
8599 (match_operand 13 "const_24_to_27_operand")
8600 (match_operand 14 "const_24_to_27_operand")
8601 (match_operand 15 "const_12_to_15_operand")
8602 (match_operand 16 "const_12_to_15_operand")
8603 (match_operand 17 "const_28_to_31_operand")
8604 (match_operand 18 "const_28_to_31_operand")])))]
8606 && (INTVAL (operands[3]) == (INTVAL (operands[7]) - 4)
8607 && INTVAL (operands[4]) == (INTVAL (operands[8]) - 4)
8608 && INTVAL (operands[5]) == (INTVAL (operands[9]) - 4)
8609 && INTVAL (operands[6]) == (INTVAL (operands[10]) - 4)
8610 && INTVAL (operands[3]) == (INTVAL (operands[11]) - 8)
8611 && INTVAL (operands[4]) == (INTVAL (operands[12]) - 8)
8612 && INTVAL (operands[5]) == (INTVAL (operands[13]) - 8)
8613 && INTVAL (operands[6]) == (INTVAL (operands[14]) - 8)
8614 && INTVAL (operands[3]) == (INTVAL (operands[15]) - 12)
8615 && INTVAL (operands[4]) == (INTVAL (operands[16]) - 12)
8616 && INTVAL (operands[5]) == (INTVAL (operands[17]) - 12)
8617 && INTVAL (operands[6]) == (INTVAL (operands[18]) - 12))"
8620 mask = INTVAL (operands[3]);
8621 mask |= INTVAL (operands[4]) << 2;
8622 mask |= (INTVAL (operands[5]) - 16) << 4;
8623 mask |= (INTVAL (operands[6]) - 16) << 6;
8624 operands[3] = GEN_INT (mask);
8626 return "vshufps\t{%3, %2, %1, %0<mask_operand19>|%0<mask_operand19>, %1, %2, %3}";
8628 [(set_attr "type" "sselog")
8629 (set_attr "length_immediate" "1")
8630 (set_attr "prefix" "evex")
8631 (set_attr "mode" "V16SF")])
8633 (define_expand "avx512f_shufpd512_mask"
8634 [(match_operand:V8DF 0 "register_operand")
8635 (match_operand:V8DF 1 "register_operand")
8636 (match_operand:V8DF 2 "nonimmediate_operand")
8637 (match_operand:SI 3 "const_0_to_255_operand")
8638 (match_operand:V8DF 4 "register_operand")
8639 (match_operand:QI 5 "register_operand")]
8642 int mask = INTVAL (operands[3]);
8643 emit_insn (gen_avx512f_shufpd512_1_mask (operands[0], operands[1], operands[2],
8645 GEN_INT (mask & 2 ? 9 : 8),
8646 GEN_INT (mask & 4 ? 3 : 2),
8647 GEN_INT (mask & 8 ? 11 : 10),
8648 GEN_INT (mask & 16 ? 5 : 4),
8649 GEN_INT (mask & 32 ? 13 : 12),
8650 GEN_INT (mask & 64 ? 7 : 6),
8651 GEN_INT (mask & 128 ? 15 : 14),
8652 operands[4], operands[5]));
8656 (define_insn "avx512f_shufpd512_1<mask_name>"
8657 [(set (match_operand:V8DF 0 "register_operand" "=v")
8660 (match_operand:V8DF 1 "register_operand" "v")
8661 (match_operand:V8DF 2 "nonimmediate_operand" "vm"))
8662 (parallel [(match_operand 3 "const_0_to_1_operand")
8663 (match_operand 4 "const_8_to_9_operand")
8664 (match_operand 5 "const_2_to_3_operand")
8665 (match_operand 6 "const_10_to_11_operand")
8666 (match_operand 7 "const_4_to_5_operand")
8667 (match_operand 8 "const_12_to_13_operand")
8668 (match_operand 9 "const_6_to_7_operand")
8669 (match_operand 10 "const_14_to_15_operand")])))]
8673 mask = INTVAL (operands[3]);
8674 mask |= (INTVAL (operands[4]) - 8) << 1;
8675 mask |= (INTVAL (operands[5]) - 2) << 2;
8676 mask |= (INTVAL (operands[6]) - 10) << 3;
8677 mask |= (INTVAL (operands[7]) - 4) << 4;
8678 mask |= (INTVAL (operands[8]) - 12) << 5;
8679 mask |= (INTVAL (operands[9]) - 6) << 6;
8680 mask |= (INTVAL (operands[10]) - 14) << 7;
8681 operands[3] = GEN_INT (mask);
8683 return "vshufpd\t{%3, %2, %1, %0<mask_operand11>|%0<mask_operand11>, %1, %2, %3}";
8685 [(set_attr "type" "sselog")
8686 (set_attr "length_immediate" "1")
8687 (set_attr "prefix" "evex")
8688 (set_attr "mode" "V8DF")])
8690 (define_expand "avx_shufpd256<mask_expand4_name>"
8691 [(match_operand:V4DF 0 "register_operand")
8692 (match_operand:V4DF 1 "register_operand")
8693 (match_operand:V4DF 2 "nonimmediate_operand")
8694 (match_operand:SI 3 "const_int_operand")]
8697 int mask = INTVAL (operands[3]);
8698 emit_insn (gen_avx_shufpd256_1<mask_expand4_name> (operands[0],
8702 GEN_INT (mask & 2 ? 5 : 4),
8703 GEN_INT (mask & 4 ? 3 : 2),
8704 GEN_INT (mask & 8 ? 7 : 6)
8705 <mask_expand4_args>));
8709 (define_insn "avx_shufpd256_1<mask_name>"
8710 [(set (match_operand:V4DF 0 "register_operand" "=v")
8713 (match_operand:V4DF 1 "register_operand" "v")
8714 (match_operand:V4DF 2 "nonimmediate_operand" "vm"))
8715 (parallel [(match_operand 3 "const_0_to_1_operand")
8716 (match_operand 4 "const_4_to_5_operand")
8717 (match_operand 5 "const_2_to_3_operand")
8718 (match_operand 6 "const_6_to_7_operand")])))]
8719 "TARGET_AVX && <mask_avx512vl_condition>"
8722 mask = INTVAL (operands[3]);
8723 mask |= (INTVAL (operands[4]) - 4) << 1;
8724 mask |= (INTVAL (operands[5]) - 2) << 2;
8725 mask |= (INTVAL (operands[6]) - 6) << 3;
8726 operands[3] = GEN_INT (mask);
8728 return "vshufpd\t{%3, %2, %1, %0<mask_operand7>|%0<mask_operand7>, %1, %2, %3}";
8730 [(set_attr "type" "sseshuf")
8731 (set_attr "length_immediate" "1")
8732 (set_attr "prefix" "vex")
8733 (set_attr "mode" "V4DF")])
8735 (define_expand "sse2_shufpd<mask_expand4_name>"
8736 [(match_operand:V2DF 0 "register_operand")
8737 (match_operand:V2DF 1 "register_operand")
8738 (match_operand:V2DF 2 "vector_operand")
8739 (match_operand:SI 3 "const_int_operand")]
8742 int mask = INTVAL (operands[3]);
8743 emit_insn (gen_sse2_shufpd_v2df<mask_expand4_name> (operands[0], operands[1],
8744 operands[2], GEN_INT (mask & 1),
8745 GEN_INT (mask & 2 ? 3 : 2)
8746 <mask_expand4_args>));
8750 (define_insn "sse2_shufpd_v2df_mask"
8751 [(set (match_operand:V2DF 0 "register_operand" "=v")
8755 (match_operand:V2DF 1 "register_operand" "v")
8756 (match_operand:V2DF 2 "nonimmediate_operand" "vm"))
8757 (parallel [(match_operand 3 "const_0_to_1_operand")
8758 (match_operand 4 "const_2_to_3_operand")]))
8759 (match_operand:V2DF 5 "vector_move_operand" "0C")
8760 (match_operand:QI 6 "register_operand" "Yk")))]
8764 mask = INTVAL (operands[3]);
8765 mask |= (INTVAL (operands[4]) - 2) << 1;
8766 operands[3] = GEN_INT (mask);
8768 return "vshufpd\t{%3, %2, %1, %0%{%6%}%N5|%0%{6%}%N5, %1, %2, %3}";
8770 [(set_attr "type" "sseshuf")
8771 (set_attr "length_immediate" "1")
8772 (set_attr "prefix" "evex")
8773 (set_attr "mode" "V2DF")])
8775 ;; punpcklqdq and punpckhqdq are shorter than shufpd.
8776 (define_insn "avx2_interleave_highv4di<mask_name>"
8777 [(set (match_operand:V4DI 0 "register_operand" "=v")
8780 (match_operand:V4DI 1 "register_operand" "v")
8781 (match_operand:V4DI 2 "nonimmediate_operand" "vm"))
8782 (parallel [(const_int 1)
8786 "TARGET_AVX2 && <mask_avx512vl_condition>"
8787 "vpunpckhqdq\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
8788 [(set_attr "type" "sselog")
8789 (set_attr "prefix" "vex")
8790 (set_attr "mode" "OI")])
8792 (define_insn "<mask_codefor>avx512f_interleave_highv8di<mask_name>"
8793 [(set (match_operand:V8DI 0 "register_operand" "=v")
8796 (match_operand:V8DI 1 "register_operand" "v")
8797 (match_operand:V8DI 2 "nonimmediate_operand" "vm"))
8798 (parallel [(const_int 1) (const_int 9)
8799 (const_int 3) (const_int 11)
8800 (const_int 5) (const_int 13)
8801 (const_int 7) (const_int 15)])))]
8803 "vpunpckhqdq\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
8804 [(set_attr "type" "sselog")
8805 (set_attr "prefix" "evex")
8806 (set_attr "mode" "XI")])
8808 (define_insn "vec_interleave_highv2di<mask_name>"
8809 [(set (match_operand:V2DI 0 "register_operand" "=x,v")
8812 (match_operand:V2DI 1 "register_operand" "0,v")
8813 (match_operand:V2DI 2 "vector_operand" "xBm,vm"))
8814 (parallel [(const_int 1)
8816 "TARGET_SSE2 && <mask_avx512vl_condition>"
8818 punpckhqdq\t{%2, %0|%0, %2}
8819 vpunpckhqdq\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
8820 [(set_attr "isa" "noavx,avx")
8821 (set_attr "type" "sselog")
8822 (set_attr "prefix_data16" "1,*")
8823 (set_attr "prefix" "orig,<mask_prefix>")
8824 (set_attr "mode" "TI")])
8826 (define_insn "avx2_interleave_lowv4di<mask_name>"
8827 [(set (match_operand:V4DI 0 "register_operand" "=v")
8830 (match_operand:V4DI 1 "register_operand" "v")
8831 (match_operand:V4DI 2 "nonimmediate_operand" "vm"))
8832 (parallel [(const_int 0)
8836 "TARGET_AVX2 && <mask_avx512vl_condition>"
8837 "vpunpcklqdq\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
8838 [(set_attr "type" "sselog")
8839 (set_attr "prefix" "vex")
8840 (set_attr "mode" "OI")])
8842 (define_insn "<mask_codefor>avx512f_interleave_lowv8di<mask_name>"
8843 [(set (match_operand:V8DI 0 "register_operand" "=v")
8846 (match_operand:V8DI 1 "register_operand" "v")
8847 (match_operand:V8DI 2 "nonimmediate_operand" "vm"))
8848 (parallel [(const_int 0) (const_int 8)
8849 (const_int 2) (const_int 10)
8850 (const_int 4) (const_int 12)
8851 (const_int 6) (const_int 14)])))]
8853 "vpunpcklqdq\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
8854 [(set_attr "type" "sselog")
8855 (set_attr "prefix" "evex")
8856 (set_attr "mode" "XI")])
8858 (define_insn "vec_interleave_lowv2di<mask_name>"
8859 [(set (match_operand:V2DI 0 "register_operand" "=x,v")
8862 (match_operand:V2DI 1 "register_operand" "0,v")
8863 (match_operand:V2DI 2 "vector_operand" "xBm,vm"))
8864 (parallel [(const_int 0)
8866 "TARGET_SSE2 && <mask_avx512vl_condition>"
8868 punpcklqdq\t{%2, %0|%0, %2}
8869 vpunpcklqdq\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
8870 [(set_attr "isa" "noavx,avx")
8871 (set_attr "type" "sselog")
8872 (set_attr "prefix_data16" "1,*")
8873 (set_attr "prefix" "orig,vex")
8874 (set_attr "mode" "TI")])
8876 (define_insn "sse2_shufpd_<mode>"
8877 [(set (match_operand:VI8F_128 0 "register_operand" "=x,v")
8878 (vec_select:VI8F_128
8879 (vec_concat:<ssedoublevecmode>
8880 (match_operand:VI8F_128 1 "register_operand" "0,v")
8881 (match_operand:VI8F_128 2 "vector_operand" "xBm,vm"))
8882 (parallel [(match_operand 3 "const_0_to_1_operand")
8883 (match_operand 4 "const_2_to_3_operand")])))]
8887 mask = INTVAL (operands[3]);
8888 mask |= (INTVAL (operands[4]) - 2) << 1;
8889 operands[3] = GEN_INT (mask);
8891 switch (which_alternative)
8894 return "shufpd\t{%3, %2, %0|%0, %2, %3}";
8896 return "vshufpd\t{%3, %2, %1, %0|%0, %1, %2, %3}";
8901 [(set_attr "isa" "noavx,avx")
8902 (set_attr "type" "sseshuf")
8903 (set_attr "length_immediate" "1")
8904 (set_attr "prefix" "orig,maybe_evex")
8905 (set_attr "mode" "V2DF")])
8907 ;; Avoid combining registers from different units in a single alternative,
8908 ;; see comment above inline_secondary_memory_needed function in i386.c
8909 (define_insn "sse2_storehpd"
8910 [(set (match_operand:DF 0 "nonimmediate_operand" "=m,x,Yv,x,*f,r")
8912 (match_operand:V2DF 1 "nonimmediate_operand" " v,0, v,o,o,o")
8913 (parallel [(const_int 1)])))]
8914 "TARGET_SSE2 && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
8916 %vmovhpd\t{%1, %0|%0, %1}
8918 vunpckhpd\t{%d1, %0|%0, %d1}
8922 [(set_attr "isa" "*,noavx,avx,*,*,*")
8923 (set_attr "type" "ssemov,sselog1,sselog1,ssemov,fmov,imov")
8924 (set (attr "prefix_data16")
8926 (and (eq_attr "alternative" "0")
8927 (not (match_test "TARGET_AVX")))
8929 (const_string "*")))
8930 (set_attr "prefix" "maybe_vex,orig,maybe_evex,*,*,*")
8931 (set_attr "mode" "V1DF,V1DF,V2DF,DF,DF,DF")])
8934 [(set (match_operand:DF 0 "register_operand")
8936 (match_operand:V2DF 1 "memory_operand")
8937 (parallel [(const_int 1)])))]
8938 "TARGET_SSE2 && reload_completed"
8939 [(set (match_dup 0) (match_dup 1))]
8940 "operands[1] = adjust_address (operands[1], DFmode, 8);")
8942 (define_insn "*vec_extractv2df_1_sse"
8943 [(set (match_operand:DF 0 "nonimmediate_operand" "=m,x,x")
8945 (match_operand:V2DF 1 "nonimmediate_operand" "x,x,o")
8946 (parallel [(const_int 1)])))]
8947 "!TARGET_SSE2 && TARGET_SSE
8948 && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
8950 movhps\t{%1, %0|%q0, %1}
8951 movhlps\t{%1, %0|%0, %1}
8952 movlps\t{%H1, %0|%0, %H1}"
8953 [(set_attr "type" "ssemov")
8954 (set_attr "mode" "V2SF,V4SF,V2SF")])
8956 ;; Avoid combining registers from different units in a single alternative,
8957 ;; see comment above inline_secondary_memory_needed function in i386.c
8958 (define_insn "sse2_storelpd"
8959 [(set (match_operand:DF 0 "nonimmediate_operand" "=m,x,x,*f,r")
8961 (match_operand:V2DF 1 "nonimmediate_operand" " v,x,m,m,m")
8962 (parallel [(const_int 0)])))]
8963 "TARGET_SSE2 && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
8965 %vmovlpd\t{%1, %0|%0, %1}
8970 [(set_attr "type" "ssemov,ssemov,ssemov,fmov,imov")
8971 (set (attr "prefix_data16")
8972 (if_then_else (eq_attr "alternative" "0")
8974 (const_string "*")))
8975 (set_attr "prefix" "maybe_vex")
8976 (set_attr "mode" "V1DF,DF,DF,DF,DF")])
8979 [(set (match_operand:DF 0 "register_operand")
8981 (match_operand:V2DF 1 "nonimmediate_operand")
8982 (parallel [(const_int 0)])))]
8983 "TARGET_SSE2 && reload_completed"
8984 [(set (match_dup 0) (match_dup 1))]
8985 "operands[1] = gen_lowpart (DFmode, operands[1]);")
8987 (define_insn "*vec_extractv2df_0_sse"
8988 [(set (match_operand:DF 0 "nonimmediate_operand" "=m,x,x")
8990 (match_operand:V2DF 1 "nonimmediate_operand" "x,x,m")
8991 (parallel [(const_int 0)])))]
8992 "!TARGET_SSE2 && TARGET_SSE
8993 && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
8995 movlps\t{%1, %0|%0, %1}
8996 movaps\t{%1, %0|%0, %1}
8997 movlps\t{%1, %0|%0, %q1}"
8998 [(set_attr "type" "ssemov")
8999 (set_attr "mode" "V2SF,V4SF,V2SF")])
9001 (define_expand "sse2_loadhpd_exp"
9002 [(set (match_operand:V2DF 0 "nonimmediate_operand")
9005 (match_operand:V2DF 1 "nonimmediate_operand")
9006 (parallel [(const_int 0)]))
9007 (match_operand:DF 2 "nonimmediate_operand")))]
9010 rtx dst = ix86_fixup_binary_operands (UNKNOWN, V2DFmode, operands);
9012 emit_insn (gen_sse2_loadhpd (dst, operands[1], operands[2]));
9014 /* Fix up the destination if needed. */
9015 if (dst != operands[0])
9016 emit_move_insn (operands[0], dst);
9021 ;; Avoid combining registers from different units in a single alternative,
9022 ;; see comment above inline_secondary_memory_needed function in i386.c
9023 (define_insn "sse2_loadhpd"
9024 [(set (match_operand:V2DF 0 "nonimmediate_operand"
9028 (match_operand:V2DF 1 "nonimmediate_operand"
9030 (parallel [(const_int 0)]))
9031 (match_operand:DF 2 "nonimmediate_operand"
9032 " m,m,x,v,x,*f,r")))]
9033 "TARGET_SSE2 && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
9035 movhpd\t{%2, %0|%0, %2}
9036 vmovhpd\t{%2, %1, %0|%0, %1, %2}
9037 unpcklpd\t{%2, %0|%0, %2}
9038 vunpcklpd\t{%2, %1, %0|%0, %1, %2}
9042 [(set_attr "isa" "noavx,avx,noavx,avx,*,*,*")
9043 (set_attr "type" "ssemov,ssemov,sselog,sselog,ssemov,fmov,imov")
9044 (set (attr "prefix_data16")
9045 (if_then_else (eq_attr "alternative" "0")
9047 (const_string "*")))
9048 (set_attr "prefix" "orig,maybe_evex,orig,maybe_evex,*,*,*")
9049 (set_attr "mode" "V1DF,V1DF,V2DF,V2DF,DF,DF,DF")])
9052 [(set (match_operand:V2DF 0 "memory_operand")
9054 (vec_select:DF (match_dup 0) (parallel [(const_int 0)]))
9055 (match_operand:DF 1 "register_operand")))]
9056 "TARGET_SSE2 && reload_completed"
9057 [(set (match_dup 0) (match_dup 1))]
9058 "operands[0] = adjust_address (operands[0], DFmode, 8);")
9060 (define_expand "sse2_loadlpd_exp"
9061 [(set (match_operand:V2DF 0 "nonimmediate_operand")
9063 (match_operand:DF 2 "nonimmediate_operand")
9065 (match_operand:V2DF 1 "nonimmediate_operand")
9066 (parallel [(const_int 1)]))))]
9069 rtx dst = ix86_fixup_binary_operands (UNKNOWN, V2DFmode, operands);
9071 emit_insn (gen_sse2_loadlpd (dst, operands[1], operands[2]));
9073 /* Fix up the destination if needed. */
9074 if (dst != operands[0])
9075 emit_move_insn (operands[0], dst);
9080 ;; Avoid combining registers from different units in a single alternative,
9081 ;; see comment above inline_secondary_memory_needed function in i386.c
9082 (define_insn "sse2_loadlpd"
9083 [(set (match_operand:V2DF 0 "nonimmediate_operand"
9084 "=v,x,v,x,v,x,x,v,m,m ,m")
9086 (match_operand:DF 2 "nonimmediate_operand"
9087 "vm,m,m,x,v,0,0,v,x,*f,r")
9089 (match_operand:V2DF 1 "vector_move_operand"
9090 " C,0,v,0,v,x,o,o,0,0 ,0")
9091 (parallel [(const_int 1)]))))]
9092 "TARGET_SSE2 && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
9094 %vmovq\t{%2, %0|%0, %2}
9095 movlpd\t{%2, %0|%0, %2}
9096 vmovlpd\t{%2, %1, %0|%0, %1, %2}
9097 movsd\t{%2, %0|%0, %2}
9098 vmovsd\t{%2, %1, %0|%0, %1, %2}
9099 shufpd\t{$2, %1, %0|%0, %1, 2}
9100 movhpd\t{%H1, %0|%0, %H1}
9101 vmovhpd\t{%H1, %2, %0|%0, %2, %H1}
9105 [(set_attr "isa" "*,noavx,avx,noavx,avx,noavx,noavx,avx,*,*,*")
9107 (cond [(eq_attr "alternative" "5")
9108 (const_string "sselog")
9109 (eq_attr "alternative" "9")
9110 (const_string "fmov")
9111 (eq_attr "alternative" "10")
9112 (const_string "imov")
9114 (const_string "ssemov")))
9115 (set (attr "prefix_data16")
9116 (if_then_else (eq_attr "alternative" "1,6")
9118 (const_string "*")))
9119 (set (attr "length_immediate")
9120 (if_then_else (eq_attr "alternative" "5")
9122 (const_string "*")))
9123 (set (attr "prefix")
9124 (cond [(eq_attr "alternative" "0")
9125 (const_string "maybe_vex")
9126 (eq_attr "alternative" "1,3,5,6")
9127 (const_string "orig")
9128 (eq_attr "alternative" "2,4,7")
9129 (const_string "maybe_evex")
9131 (const_string "*")))
9132 (set_attr "mode" "DF,V1DF,V1DF,V1DF,V1DF,V2DF,V1DF,V1DF,DF,DF,DF")])
9135 [(set (match_operand:V2DF 0 "memory_operand")
9137 (match_operand:DF 1 "register_operand")
9138 (vec_select:DF (match_dup 0) (parallel [(const_int 1)]))))]
9139 "TARGET_SSE2 && reload_completed"
9140 [(set (match_dup 0) (match_dup 1))]
9141 "operands[0] = adjust_address (operands[0], DFmode, 0);")
9143 (define_insn "sse2_movsd"
9144 [(set (match_operand:V2DF 0 "nonimmediate_operand" "=x,v,x,v,m,x,x,v,o")
9146 (match_operand:V2DF 2 "nonimmediate_operand" " x,v,m,m,v,0,0,v,0")
9147 (match_operand:V2DF 1 "nonimmediate_operand" " 0,v,0,v,0,x,o,o,v")
9151 movsd\t{%2, %0|%0, %2}
9152 vmovsd\t{%2, %1, %0|%0, %1, %2}
9153 movlpd\t{%2, %0|%0, %q2}
9154 vmovlpd\t{%2, %1, %0|%0, %1, %q2}
9155 %vmovlpd\t{%2, %0|%q0, %2}
9156 shufpd\t{$2, %1, %0|%0, %1, 2}
9157 movhps\t{%H1, %0|%0, %H1}
9158 vmovhps\t{%H1, %2, %0|%0, %2, %H1}
9159 %vmovhps\t{%1, %H0|%H0, %1}"
9160 [(set_attr "isa" "noavx,avx,noavx,avx,*,noavx,noavx,avx,*")
9163 (eq_attr "alternative" "5")
9164 (const_string "sselog")
9165 (const_string "ssemov")))
9166 (set (attr "prefix_data16")
9168 (and (eq_attr "alternative" "2,4")
9169 (not (match_test "TARGET_AVX")))
9171 (const_string "*")))
9172 (set (attr "length_immediate")
9173 (if_then_else (eq_attr "alternative" "5")
9175 (const_string "*")))
9176 (set (attr "prefix")
9177 (cond [(eq_attr "alternative" "1,3,7")
9178 (const_string "maybe_evex")
9179 (eq_attr "alternative" "4,8")
9180 (const_string "maybe_vex")
9182 (const_string "orig")))
9183 (set_attr "mode" "DF,DF,V1DF,V1DF,V1DF,V2DF,V1DF,V1DF,V1DF")])
9185 (define_insn "vec_dupv2df<mask_name>"
9186 [(set (match_operand:V2DF 0 "register_operand" "=x,x,v")
9188 (match_operand:DF 1 "nonimmediate_operand" " 0,xm,vm")))]
9189 "TARGET_SSE2 && <mask_avx512vl_condition>"
9192 %vmovddup\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}
9193 vmovddup\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
9194 [(set_attr "isa" "noavx,sse3,avx512vl")
9195 (set_attr "type" "sselog1")
9196 (set_attr "prefix" "orig,maybe_vex,evex")
9197 (set_attr "mode" "V2DF,DF,DF")])
9199 (define_insn "vec_concatv2df"
9200 [(set (match_operand:V2DF 0 "register_operand" "=x,x,v,x,v,x,x, v,x,x")
9202 (match_operand:DF 1 "nonimmediate_operand" " 0,x,v,m,m,0,x,xm,0,0")
9203 (match_operand:DF 2 "vector_move_operand" " x,x,v,1,1,m,m, C,x,m")))]
9205 && (!(MEM_P (operands[1]) && MEM_P (operands[2]))
9206 || (TARGET_SSE3 && rtx_equal_p (operands[1], operands[2])))"
9208 unpcklpd\t{%2, %0|%0, %2}
9209 vunpcklpd\t{%2, %1, %0|%0, %1, %2}
9210 vunpcklpd\t{%2, %1, %0|%0, %1, %2}
9211 %vmovddup\t{%1, %0|%0, %1}
9212 vmovddup\t{%1, %0|%0, %1}
9213 movhpd\t{%2, %0|%0, %2}
9214 vmovhpd\t{%2, %1, %0|%0, %1, %2}
9215 %vmovq\t{%1, %0|%0, %1}
9216 movlhps\t{%2, %0|%0, %2}
9217 movhps\t{%2, %0|%0, %2}"
9219 (cond [(eq_attr "alternative" "0,5")
9220 (const_string "sse2_noavx")
9221 (eq_attr "alternative" "1,6")
9222 (const_string "avx")
9223 (eq_attr "alternative" "2,4")
9224 (const_string "avx512vl")
9225 (eq_attr "alternative" "3")
9226 (const_string "sse3")
9227 (eq_attr "alternative" "7")
9228 (const_string "sse2")
9230 (const_string "noavx")))
9233 (eq_attr "alternative" "0,1,2,3,4")
9234 (const_string "sselog")
9235 (const_string "ssemov")))
9236 (set (attr "prefix_data16")
9237 (if_then_else (eq_attr "alternative" "5")
9239 (const_string "*")))
9240 (set (attr "prefix")
9241 (cond [(eq_attr "alternative" "1,6")
9242 (const_string "vex")
9243 (eq_attr "alternative" "2,4")
9244 (const_string "evex")
9245 (eq_attr "alternative" "3,7")
9246 (const_string "maybe_vex")
9248 (const_string "orig")))
9249 (set_attr "mode" "V2DF,V2DF,V2DF, DF, DF, V1DF,V1DF,DF,V4SF,V2SF")])
9251 ;; vmovq clears also the higher bits.
9252 (define_insn "vec_set<mode>_0"
9253 [(set (match_operand:VF2_512_256 0 "register_operand" "=v")
9254 (vec_merge:VF2_512_256
9255 (vec_duplicate:VF2_512_256
9256 (match_operand:<ssescalarmode> 2 "general_operand" "xm"))
9257 (match_operand:VF2_512_256 1 "const0_operand" "C")
9260 "vmovq\t{%2, %x0|%x0, %2}"
9261 [(set_attr "type" "ssemov")
9262 (set_attr "prefix" "maybe_evex")
9263 (set_attr "mode" "DF")])
9265 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
9267 ;; Parallel integer down-conversion operations
9269 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
9271 (define_mode_iterator PMOV_DST_MODE_1 [V16QI V16HI V8SI V8HI])
9272 (define_mode_attr pmov_src_mode
9273 [(V16QI "V16SI") (V16HI "V16SI") (V8SI "V8DI") (V8HI "V8DI")])
9274 (define_mode_attr pmov_src_lower
9275 [(V16QI "v16si") (V16HI "v16si") (V8SI "v8di") (V8HI "v8di")])
9276 (define_mode_attr pmov_suff_1
9277 [(V16QI "db") (V16HI "dw") (V8SI "qd") (V8HI "qw")])
9279 (define_insn "*avx512f_<code><pmov_src_lower><mode>2"
9280 [(set (match_operand:PMOV_DST_MODE_1 0 "nonimmediate_operand" "=v,m")
9281 (any_truncate:PMOV_DST_MODE_1
9282 (match_operand:<pmov_src_mode> 1 "register_operand" "v,v")))]
9284 "vpmov<trunsuffix><pmov_suff_1>\t{%1, %0|%0, %1}"
9285 [(set_attr "type" "ssemov")
9286 (set_attr "memory" "none,store")
9287 (set_attr "prefix" "evex")
9288 (set_attr "mode" "<sseinsnmode>")])
9290 (define_insn "avx512f_<code><pmov_src_lower><mode>2_mask"
9291 [(set (match_operand:PMOV_DST_MODE_1 0 "nonimmediate_operand" "=v,m")
9292 (vec_merge:PMOV_DST_MODE_1
9293 (any_truncate:PMOV_DST_MODE_1
9294 (match_operand:<pmov_src_mode> 1 "register_operand" "v,v"))
9295 (match_operand:PMOV_DST_MODE_1 2 "vector_move_operand" "0C,0")
9296 (match_operand:<avx512fmaskmode> 3 "register_operand" "Yk,Yk")))]
9298 "vpmov<trunsuffix><pmov_suff_1>\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"
9299 [(set_attr "type" "ssemov")
9300 (set_attr "memory" "none,store")
9301 (set_attr "prefix" "evex")
9302 (set_attr "mode" "<sseinsnmode>")])
9304 (define_expand "avx512f_<code><pmov_src_lower><mode>2_mask_store"
9305 [(set (match_operand:PMOV_DST_MODE_1 0 "memory_operand")
9306 (vec_merge:PMOV_DST_MODE_1
9307 (any_truncate:PMOV_DST_MODE_1
9308 (match_operand:<pmov_src_mode> 1 "register_operand"))
9310 (match_operand:<avx512fmaskmode> 2 "register_operand")))]
9313 (define_insn "avx512bw_<code>v32hiv32qi2"
9314 [(set (match_operand:V32QI 0 "nonimmediate_operand" "=v,m")
9316 (match_operand:V32HI 1 "register_operand" "v,v")))]
9318 "vpmov<trunsuffix>wb\t{%1, %0|%0, %1}"
9319 [(set_attr "type" "ssemov")
9320 (set_attr "memory" "none,store")
9321 (set_attr "prefix" "evex")
9322 (set_attr "mode" "XI")])
9324 (define_insn "avx512bw_<code>v32hiv32qi2_mask"
9325 [(set (match_operand:V32QI 0 "nonimmediate_operand" "=v,m")
9328 (match_operand:V32HI 1 "register_operand" "v,v"))
9329 (match_operand:V32QI 2 "vector_move_operand" "0C,0")
9330 (match_operand:SI 3 "register_operand" "Yk,Yk")))]
9332 "vpmov<trunsuffix>wb\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"
9333 [(set_attr "type" "ssemov")
9334 (set_attr "memory" "none,store")
9335 (set_attr "prefix" "evex")
9336 (set_attr "mode" "XI")])
9338 (define_expand "avx512bw_<code>v32hiv32qi2_mask_store"
9339 [(set (match_operand:V32QI 0 "nonimmediate_operand")
9342 (match_operand:V32HI 1 "register_operand"))
9344 (match_operand:SI 2 "register_operand")))]
9347 (define_mode_iterator PMOV_DST_MODE_2
9348 [V4SI V8HI (V16QI "TARGET_AVX512BW")])
9349 (define_mode_attr pmov_suff_2
9350 [(V16QI "wb") (V8HI "dw") (V4SI "qd")])
9352 (define_insn "*avx512vl_<code><ssedoublemodelower><mode>2"
9353 [(set (match_operand:PMOV_DST_MODE_2 0 "nonimmediate_operand" "=v,m")
9354 (any_truncate:PMOV_DST_MODE_2
9355 (match_operand:<ssedoublemode> 1 "register_operand" "v,v")))]
9357 "vpmov<trunsuffix><pmov_suff_2>\t{%1, %0|%0, %1}"
9358 [(set_attr "type" "ssemov")
9359 (set_attr "memory" "none,store")
9360 (set_attr "prefix" "evex")
9361 (set_attr "mode" "<sseinsnmode>")])
9363 (define_insn "<avx512>_<code><ssedoublemodelower><mode>2_mask"
9364 [(set (match_operand:PMOV_DST_MODE_2 0 "nonimmediate_operand" "=v,m")
9365 (vec_merge:PMOV_DST_MODE_2
9366 (any_truncate:PMOV_DST_MODE_2
9367 (match_operand:<ssedoublemode> 1 "register_operand" "v,v"))
9368 (match_operand:PMOV_DST_MODE_2 2 "vector_move_operand" "0C,0")
9369 (match_operand:<avx512fmaskmode> 3 "register_operand" "Yk,Yk")))]
9371 "vpmov<trunsuffix><pmov_suff_2>\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"
9372 [(set_attr "type" "ssemov")
9373 (set_attr "memory" "none,store")
9374 (set_attr "prefix" "evex")
9375 (set_attr "mode" "<sseinsnmode>")])
9377 (define_expand "<avx512>_<code><ssedoublemodelower><mode>2_mask_store"
9378 [(set (match_operand:PMOV_DST_MODE_2 0 "nonimmediate_operand")
9379 (vec_merge:PMOV_DST_MODE_2
9380 (any_truncate:PMOV_DST_MODE_2
9381 (match_operand:<ssedoublemode> 1 "register_operand"))
9383 (match_operand:<avx512fmaskmode> 2 "register_operand")))]
9386 (define_mode_iterator PMOV_SRC_MODE_3 [V4DI V2DI V8SI V4SI (V8HI "TARGET_AVX512BW")])
9387 (define_mode_attr pmov_dst_3
9388 [(V4DI "V4QI") (V2DI "V2QI") (V8SI "V8QI") (V4SI "V4QI") (V8HI "V8QI")])
9389 (define_mode_attr pmov_dst_zeroed_3
9390 [(V4DI "V12QI") (V2DI "V14QI") (V8SI "V8QI") (V4SI "V12QI") (V8HI "V8QI")])
9391 (define_mode_attr pmov_suff_3
9392 [(V4DI "qb") (V2DI "qb") (V8SI "db") (V4SI "db") (V8HI "wb")])
9394 (define_insn "*avx512vl_<code><mode>v<ssescalarnum>qi2"
9395 [(set (match_operand:V16QI 0 "register_operand" "=v")
9397 (any_truncate:<pmov_dst_3>
9398 (match_operand:PMOV_SRC_MODE_3 1 "register_operand" "v"))
9399 (match_operand:<pmov_dst_zeroed_3> 2 "const0_operand")))]
9401 "vpmov<trunsuffix><pmov_suff_3>\t{%1, %0|%0, %1}"
9402 [(set_attr "type" "ssemov")
9403 (set_attr "prefix" "evex")
9404 (set_attr "mode" "TI")])
9406 (define_insn "*avx512vl_<code>v2div2qi2_store"
9407 [(set (match_operand:V16QI 0 "memory_operand" "=m")
9410 (match_operand:V2DI 1 "register_operand" "v"))
9413 (parallel [(const_int 2) (const_int 3)
9414 (const_int 4) (const_int 5)
9415 (const_int 6) (const_int 7)
9416 (const_int 8) (const_int 9)
9417 (const_int 10) (const_int 11)
9418 (const_int 12) (const_int 13)
9419 (const_int 14) (const_int 15)]))))]
9421 "vpmov<trunsuffix>qb\t{%1, %0|%0, %1}"
9422 [(set_attr "type" "ssemov")
9423 (set_attr "memory" "store")
9424 (set_attr "prefix" "evex")
9425 (set_attr "mode" "TI")])
9427 (define_insn "avx512vl_<code>v2div2qi2_mask"
9428 [(set (match_operand:V16QI 0 "register_operand" "=v")
9432 (match_operand:V2DI 1 "register_operand" "v"))
9434 (match_operand:V16QI 2 "vector_move_operand" "0C")
9435 (parallel [(const_int 0) (const_int 1)]))
9436 (match_operand:QI 3 "register_operand" "Yk"))
9437 (const_vector:V14QI [(const_int 0) (const_int 0)
9438 (const_int 0) (const_int 0)
9439 (const_int 0) (const_int 0)
9440 (const_int 0) (const_int 0)
9441 (const_int 0) (const_int 0)
9442 (const_int 0) (const_int 0)
9443 (const_int 0) (const_int 0)])))]
9445 "vpmov<trunsuffix>qb\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"
9446 [(set_attr "type" "ssemov")
9447 (set_attr "prefix" "evex")
9448 (set_attr "mode" "TI")])
9450 (define_insn "*avx512vl_<code>v2div2qi2_mask_1"
9451 [(set (match_operand:V16QI 0 "register_operand" "=v")
9455 (match_operand:V2DI 1 "register_operand" "v"))
9456 (const_vector:V2QI [(const_int 0) (const_int 0)])
9457 (match_operand:QI 2 "register_operand" "Yk"))
9458 (const_vector:V14QI [(const_int 0) (const_int 0)
9459 (const_int 0) (const_int 0)
9460 (const_int 0) (const_int 0)
9461 (const_int 0) (const_int 0)
9462 (const_int 0) (const_int 0)
9463 (const_int 0) (const_int 0)
9464 (const_int 0) (const_int 0)])))]
9466 "vpmov<trunsuffix>qb\t{%1, %0%{%2%}%{z%}|%0%{%2%}%{z%}, %1}"
9467 [(set_attr "type" "ssemov")
9468 (set_attr "prefix" "evex")
9469 (set_attr "mode" "TI")])
9471 (define_insn "avx512vl_<code>v2div2qi2_mask_store"
9472 [(set (match_operand:V16QI 0 "memory_operand" "=m")
9476 (match_operand:V2DI 1 "register_operand" "v"))
9479 (parallel [(const_int 0) (const_int 1)]))
9480 (match_operand:QI 2 "register_operand" "Yk"))
9483 (parallel [(const_int 2) (const_int 3)
9484 (const_int 4) (const_int 5)
9485 (const_int 6) (const_int 7)
9486 (const_int 8) (const_int 9)
9487 (const_int 10) (const_int 11)
9488 (const_int 12) (const_int 13)
9489 (const_int 14) (const_int 15)]))))]
9491 "vpmov<trunsuffix>qb\t{%1, %0%{%2%}|%w0%{%2%}, %1}"
9492 [(set_attr "type" "ssemov")
9493 (set_attr "memory" "store")
9494 (set_attr "prefix" "evex")
9495 (set_attr "mode" "TI")])
9497 (define_insn "*avx512vl_<code><mode>v4qi2_store"
9498 [(set (match_operand:V16QI 0 "memory_operand" "=m")
9501 (match_operand:VI4_128_8_256 1 "register_operand" "v"))
9504 (parallel [(const_int 4) (const_int 5)
9505 (const_int 6) (const_int 7)
9506 (const_int 8) (const_int 9)
9507 (const_int 10) (const_int 11)
9508 (const_int 12) (const_int 13)
9509 (const_int 14) (const_int 15)]))))]
9511 "vpmov<trunsuffix><pmov_suff_3>\t{%1, %0|%0, %1}"
9512 [(set_attr "type" "ssemov")
9513 (set_attr "memory" "store")
9514 (set_attr "prefix" "evex")
9515 (set_attr "mode" "TI")])
9517 (define_insn "avx512vl_<code><mode>v4qi2_mask"
9518 [(set (match_operand:V16QI 0 "register_operand" "=v")
9522 (match_operand:VI4_128_8_256 1 "register_operand" "v"))
9524 (match_operand:V16QI 2 "vector_move_operand" "0C")
9525 (parallel [(const_int 0) (const_int 1)
9526 (const_int 2) (const_int 3)]))
9527 (match_operand:QI 3 "register_operand" "Yk"))
9528 (const_vector:V12QI [(const_int 0) (const_int 0)
9529 (const_int 0) (const_int 0)
9530 (const_int 0) (const_int 0)
9531 (const_int 0) (const_int 0)
9532 (const_int 0) (const_int 0)
9533 (const_int 0) (const_int 0)])))]
9535 "vpmov<trunsuffix><pmov_suff_3>\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"
9536 [(set_attr "type" "ssemov")
9537 (set_attr "prefix" "evex")
9538 (set_attr "mode" "TI")])
9540 (define_insn "*avx512vl_<code><mode>v4qi2_mask_1"
9541 [(set (match_operand:V16QI 0 "register_operand" "=v")
9545 (match_operand:VI4_128_8_256 1 "register_operand" "v"))
9546 (const_vector:V4QI [(const_int 0) (const_int 0)
9547 (const_int 0) (const_int 0)])
9548 (match_operand:QI 2 "register_operand" "Yk"))
9549 (const_vector:V12QI [(const_int 0) (const_int 0)
9550 (const_int 0) (const_int 0)
9551 (const_int 0) (const_int 0)
9552 (const_int 0) (const_int 0)
9553 (const_int 0) (const_int 0)
9554 (const_int 0) (const_int 0)])))]
9556 "vpmov<trunsuffix><pmov_suff_3>\t{%1, %0%{%2%}%{z%}|%0%{%2%}%{z%}, %1}"
9557 [(set_attr "type" "ssemov")
9558 (set_attr "prefix" "evex")
9559 (set_attr "mode" "TI")])
9561 (define_insn "avx512vl_<code><mode>v4qi2_mask_store"
9562 [(set (match_operand:V16QI 0 "memory_operand" "=m")
9566 (match_operand:VI4_128_8_256 1 "register_operand" "v"))
9569 (parallel [(const_int 0) (const_int 1)
9570 (const_int 2) (const_int 3)]))
9571 (match_operand:QI 2 "register_operand" "Yk"))
9574 (parallel [(const_int 4) (const_int 5)
9575 (const_int 6) (const_int 7)
9576 (const_int 8) (const_int 9)
9577 (const_int 10) (const_int 11)
9578 (const_int 12) (const_int 13)
9579 (const_int 14) (const_int 15)]))))]
9582 if (GET_MODE_SIZE (GET_MODE_INNER (<MODE>mode)) == 8)
9583 return "vpmov<trunsuffix><pmov_suff_3>\t{%1, %0%{%2%}|%k0%{%2%}, %1}";
9584 return "vpmov<trunsuffix><pmov_suff_3>\t{%1, %0%{%2%}|%0%{%2%}, %g1}";
9586 [(set_attr "type" "ssemov")
9587 (set_attr "memory" "store")
9588 (set_attr "prefix" "evex")
9589 (set_attr "mode" "TI")])
9591 (define_mode_iterator VI2_128_BW_4_256
9592 [(V8HI "TARGET_AVX512BW") V8SI])
9594 (define_insn "*avx512vl_<code><mode>v8qi2_store"
9595 [(set (match_operand:V16QI 0 "memory_operand" "=m")
9598 (match_operand:VI2_128_BW_4_256 1 "register_operand" "v"))
9601 (parallel [(const_int 8) (const_int 9)
9602 (const_int 10) (const_int 11)
9603 (const_int 12) (const_int 13)
9604 (const_int 14) (const_int 15)]))))]
9606 "vpmov<trunsuffix><pmov_suff_3>\t{%1, %0|%0, %1}"
9607 [(set_attr "type" "ssemov")
9608 (set_attr "memory" "store")
9609 (set_attr "prefix" "evex")
9610 (set_attr "mode" "TI")])
9612 (define_insn "avx512vl_<code><mode>v8qi2_mask"
9613 [(set (match_operand:V16QI 0 "register_operand" "=v")
9617 (match_operand:VI2_128_BW_4_256 1 "register_operand" "v"))
9619 (match_operand:V16QI 2 "vector_move_operand" "0C")
9620 (parallel [(const_int 0) (const_int 1)
9621 (const_int 2) (const_int 3)
9622 (const_int 4) (const_int 5)
9623 (const_int 6) (const_int 7)]))
9624 (match_operand:QI 3 "register_operand" "Yk"))
9625 (const_vector:V8QI [(const_int 0) (const_int 0)
9626 (const_int 0) (const_int 0)
9627 (const_int 0) (const_int 0)
9628 (const_int 0) (const_int 0)])))]
9630 "vpmov<trunsuffix><pmov_suff_3>\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"
9631 [(set_attr "type" "ssemov")
9632 (set_attr "prefix" "evex")
9633 (set_attr "mode" "TI")])
9635 (define_insn "*avx512vl_<code><mode>v8qi2_mask_1"
9636 [(set (match_operand:V16QI 0 "register_operand" "=v")
9640 (match_operand:VI2_128_BW_4_256 1 "register_operand" "v"))
9641 (const_vector:V8QI [(const_int 0) (const_int 0)
9642 (const_int 0) (const_int 0)
9643 (const_int 0) (const_int 0)
9644 (const_int 0) (const_int 0)])
9645 (match_operand:QI 2 "register_operand" "Yk"))
9646 (const_vector:V8QI [(const_int 0) (const_int 0)
9647 (const_int 0) (const_int 0)
9648 (const_int 0) (const_int 0)
9649 (const_int 0) (const_int 0)])))]
9651 "vpmov<trunsuffix><pmov_suff_3>\t{%1, %0%{%2%}%{z%}|%0%{%2%}%{z%}, %1}"
9652 [(set_attr "type" "ssemov")
9653 (set_attr "prefix" "evex")
9654 (set_attr "mode" "TI")])
9656 (define_insn "avx512vl_<code><mode>v8qi2_mask_store"
9657 [(set (match_operand:V16QI 0 "memory_operand" "=m")
9661 (match_operand:VI2_128_BW_4_256 1 "register_operand" "v"))
9664 (parallel [(const_int 0) (const_int 1)
9665 (const_int 2) (const_int 3)
9666 (const_int 4) (const_int 5)
9667 (const_int 6) (const_int 7)]))
9668 (match_operand:QI 2 "register_operand" "Yk"))
9671 (parallel [(const_int 8) (const_int 9)
9672 (const_int 10) (const_int 11)
9673 (const_int 12) (const_int 13)
9674 (const_int 14) (const_int 15)]))))]
9677 if (GET_MODE_SIZE (GET_MODE_INNER (<MODE>mode)) == 4)
9678 return "vpmov<trunsuffix><pmov_suff_3>\t{%1, %0%{%2%}|%0%{%2%}, %g1}";
9679 return "vpmov<trunsuffix><pmov_suff_3>\t{%1, %0%{%2%}|%0%{%2%}, %1}";
9681 [(set_attr "type" "ssemov")
9682 (set_attr "memory" "store")
9683 (set_attr "prefix" "evex")
9684 (set_attr "mode" "TI")])
9686 (define_mode_iterator PMOV_SRC_MODE_4 [V4DI V2DI V4SI])
9687 (define_mode_attr pmov_dst_4
9688 [(V4DI "V4HI") (V2DI "V2HI") (V4SI "V4HI")])
9689 (define_mode_attr pmov_dst_zeroed_4
9690 [(V4DI "V4HI") (V2DI "V6HI") (V4SI "V4HI")])
9691 (define_mode_attr pmov_suff_4
9692 [(V4DI "qw") (V2DI "qw") (V4SI "dw")])
9694 (define_insn "*avx512vl_<code><mode>v<ssescalarnum>hi2"
9695 [(set (match_operand:V8HI 0 "register_operand" "=v")
9697 (any_truncate:<pmov_dst_4>
9698 (match_operand:PMOV_SRC_MODE_4 1 "register_operand" "v"))
9699 (match_operand:<pmov_dst_zeroed_4> 2 "const0_operand")))]
9701 "vpmov<trunsuffix><pmov_suff_4>\t{%1, %0|%0, %1}"
9702 [(set_attr "type" "ssemov")
9703 (set_attr "prefix" "evex")
9704 (set_attr "mode" "TI")])
9706 (define_insn "*avx512vl_<code><mode>v4hi2_store"
9707 [(set (match_operand:V8HI 0 "memory_operand" "=m")
9710 (match_operand:VI4_128_8_256 1 "register_operand" "v"))
9713 (parallel [(const_int 4) (const_int 5)
9714 (const_int 6) (const_int 7)]))))]
9716 "vpmov<trunsuffix><pmov_suff_4>\t{%1, %0|%0, %1}"
9717 [(set_attr "type" "ssemov")
9718 (set_attr "memory" "store")
9719 (set_attr "prefix" "evex")
9720 (set_attr "mode" "TI")])
9722 (define_insn "avx512vl_<code><mode>v4hi2_mask"
9723 [(set (match_operand:V8HI 0 "register_operand" "=v")
9727 (match_operand:VI4_128_8_256 1 "register_operand" "v"))
9729 (match_operand:V8HI 2 "vector_move_operand" "0C")
9730 (parallel [(const_int 0) (const_int 1)
9731 (const_int 2) (const_int 3)]))
9732 (match_operand:QI 3 "register_operand" "Yk"))
9733 (const_vector:V4HI [(const_int 0) (const_int 0)
9734 (const_int 0) (const_int 0)])))]
9736 "vpmov<trunsuffix><pmov_suff_4>\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"
9737 [(set_attr "type" "ssemov")
9738 (set_attr "prefix" "evex")
9739 (set_attr "mode" "TI")])
9741 (define_insn "*avx512vl_<code><mode>v4hi2_mask_1"
9742 [(set (match_operand:V8HI 0 "register_operand" "=v")
9746 (match_operand:VI4_128_8_256 1 "register_operand" "v"))
9747 (const_vector:V4HI [(const_int 0) (const_int 0)
9748 (const_int 0) (const_int 0)])
9749 (match_operand:QI 2 "register_operand" "Yk"))
9750 (const_vector:V4HI [(const_int 0) (const_int 0)
9751 (const_int 0) (const_int 0)])))]
9753 "vpmov<trunsuffix><pmov_suff_4>\t{%1, %0%{%2%}%{z%}|%0%{%2%}%{z%}, %1}"
9754 [(set_attr "type" "ssemov")
9755 (set_attr "prefix" "evex")
9756 (set_attr "mode" "TI")])
9758 (define_insn "avx512vl_<code><mode>v4hi2_mask_store"
9759 [(set (match_operand:V8HI 0 "memory_operand" "=m")
9763 (match_operand:VI4_128_8_256 1 "register_operand" "v"))
9766 (parallel [(const_int 0) (const_int 1)
9767 (const_int 2) (const_int 3)]))
9768 (match_operand:QI 2 "register_operand" "Yk"))
9771 (parallel [(const_int 4) (const_int 5)
9772 (const_int 6) (const_int 7)]))))]
9775 if (GET_MODE_SIZE (GET_MODE_INNER (<MODE>mode)) == 4)
9776 return "vpmov<trunsuffix><pmov_suff_4>\t{%1, %0%{%2%}|%0%{%2%}, %t1}";
9777 return "vpmov<trunsuffix><pmov_suff_4>\t{%1, %0%{%2%}|%0%{%2%}, %g1}";
9779 [(set_attr "type" "ssemov")
9780 (set_attr "memory" "store")
9781 (set_attr "prefix" "evex")
9782 (set_attr "mode" "TI")])
9784 (define_insn "*avx512vl_<code>v2div2hi2_store"
9785 [(set (match_operand:V8HI 0 "memory_operand" "=m")
9788 (match_operand:V2DI 1 "register_operand" "v"))
9791 (parallel [(const_int 2) (const_int 3)
9792 (const_int 4) (const_int 5)
9793 (const_int 6) (const_int 7)]))))]
9795 "vpmov<trunsuffix>qw\t{%1, %0|%0, %1}"
9796 [(set_attr "type" "ssemov")
9797 (set_attr "memory" "store")
9798 (set_attr "prefix" "evex")
9799 (set_attr "mode" "TI")])
9801 (define_insn "avx512vl_<code>v2div2hi2_mask"
9802 [(set (match_operand:V8HI 0 "register_operand" "=v")
9806 (match_operand:V2DI 1 "register_operand" "v"))
9808 (match_operand:V8HI 2 "vector_move_operand" "0C")
9809 (parallel [(const_int 0) (const_int 1)]))
9810 (match_operand:QI 3 "register_operand" "Yk"))
9811 (const_vector:V6HI [(const_int 0) (const_int 0)
9812 (const_int 0) (const_int 0)
9813 (const_int 0) (const_int 0)])))]
9815 "vpmov<trunsuffix>qw\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"
9816 [(set_attr "type" "ssemov")
9817 (set_attr "prefix" "evex")
9818 (set_attr "mode" "TI")])
9820 (define_insn "*avx512vl_<code>v2div2hi2_mask_1"
9821 [(set (match_operand:V8HI 0 "register_operand" "=v")
9825 (match_operand:V2DI 1 "register_operand" "v"))
9826 (const_vector:V2HI [(const_int 0) (const_int 0)])
9827 (match_operand:QI 2 "register_operand" "Yk"))
9828 (const_vector:V6HI [(const_int 0) (const_int 0)
9829 (const_int 0) (const_int 0)
9830 (const_int 0) (const_int 0)])))]
9832 "vpmov<trunsuffix>qw\t{%1, %0%{%2%}%{z%}|%0%{%2%}%{z%}, %1}"
9833 [(set_attr "type" "ssemov")
9834 (set_attr "prefix" "evex")
9835 (set_attr "mode" "TI")])
9837 (define_insn "avx512vl_<code>v2div2hi2_mask_store"
9838 [(set (match_operand:V8HI 0 "memory_operand" "=m")
9842 (match_operand:V2DI 1 "register_operand" "v"))
9845 (parallel [(const_int 0) (const_int 1)]))
9846 (match_operand:QI 2 "register_operand" "Yk"))
9849 (parallel [(const_int 2) (const_int 3)
9850 (const_int 4) (const_int 5)
9851 (const_int 6) (const_int 7)]))))]
9853 "vpmov<trunsuffix>qw\t{%1, %0%{%2%}|%0%{%2%}, %g1}"
9854 [(set_attr "type" "ssemov")
9855 (set_attr "memory" "store")
9856 (set_attr "prefix" "evex")
9857 (set_attr "mode" "TI")])
9859 (define_insn "*avx512vl_<code>v2div2si2"
9860 [(set (match_operand:V4SI 0 "register_operand" "=v")
9863 (match_operand:V2DI 1 "register_operand" "v"))
9864 (match_operand:V2SI 2 "const0_operand")))]
9866 "vpmov<trunsuffix>qd\t{%1, %0|%0, %1}"
9867 [(set_attr "type" "ssemov")
9868 (set_attr "prefix" "evex")
9869 (set_attr "mode" "TI")])
9871 (define_insn "*avx512vl_<code>v2div2si2_store"
9872 [(set (match_operand:V4SI 0 "memory_operand" "=m")
9875 (match_operand:V2DI 1 "register_operand" "v"))
9878 (parallel [(const_int 2) (const_int 3)]))))]
9880 "vpmov<trunsuffix>qd\t{%1, %0|%0, %1}"
9881 [(set_attr "type" "ssemov")
9882 (set_attr "memory" "store")
9883 (set_attr "prefix" "evex")
9884 (set_attr "mode" "TI")])
9886 (define_insn "avx512vl_<code>v2div2si2_mask"
9887 [(set (match_operand:V4SI 0 "register_operand" "=v")
9891 (match_operand:V2DI 1 "register_operand" "v"))
9893 (match_operand:V4SI 2 "vector_move_operand" "0C")
9894 (parallel [(const_int 0) (const_int 1)]))
9895 (match_operand:QI 3 "register_operand" "Yk"))
9896 (const_vector:V2SI [(const_int 0) (const_int 0)])))]
9898 "vpmov<trunsuffix>qd\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"
9899 [(set_attr "type" "ssemov")
9900 (set_attr "prefix" "evex")
9901 (set_attr "mode" "TI")])
9903 (define_insn "*avx512vl_<code>v2div2si2_mask_1"
9904 [(set (match_operand:V4SI 0 "register_operand" "=v")
9908 (match_operand:V2DI 1 "register_operand" "v"))
9909 (const_vector:V2SI [(const_int 0) (const_int 0)])
9910 (match_operand:QI 2 "register_operand" "Yk"))
9911 (const_vector:V2SI [(const_int 0) (const_int 0)])))]
9913 "vpmov<trunsuffix>qd\t{%1, %0%{%2%}%{z%}|%0%{%2%}%{z%}, %1}"
9914 [(set_attr "type" "ssemov")
9915 (set_attr "prefix" "evex")
9916 (set_attr "mode" "TI")])
9918 (define_insn "avx512vl_<code>v2div2si2_mask_store"
9919 [(set (match_operand:V4SI 0 "memory_operand" "=m")
9923 (match_operand:V2DI 1 "register_operand" "v"))
9926 (parallel [(const_int 0) (const_int 1)]))
9927 (match_operand:QI 2 "register_operand" "Yk"))
9930 (parallel [(const_int 2) (const_int 3)]))))]
9932 "vpmov<trunsuffix>qd\t{%1, %0%{%2%}|%0%{%2%}, %t1}"
9933 [(set_attr "type" "ssemov")
9934 (set_attr "memory" "store")
9935 (set_attr "prefix" "evex")
9936 (set_attr "mode" "TI")])
9938 (define_insn "*avx512f_<code>v8div16qi2"
9939 [(set (match_operand:V16QI 0 "register_operand" "=v")
9942 (match_operand:V8DI 1 "register_operand" "v"))
9943 (const_vector:V8QI [(const_int 0) (const_int 0)
9944 (const_int 0) (const_int 0)
9945 (const_int 0) (const_int 0)
9946 (const_int 0) (const_int 0)])))]
9948 "vpmov<trunsuffix>qb\t{%1, %0|%0, %1}"
9949 [(set_attr "type" "ssemov")
9950 (set_attr "prefix" "evex")
9951 (set_attr "mode" "TI")])
9953 (define_insn "*avx512f_<code>v8div16qi2_store"
9954 [(set (match_operand:V16QI 0 "memory_operand" "=m")
9957 (match_operand:V8DI 1 "register_operand" "v"))
9960 (parallel [(const_int 8) (const_int 9)
9961 (const_int 10) (const_int 11)
9962 (const_int 12) (const_int 13)
9963 (const_int 14) (const_int 15)]))))]
9965 "vpmov<trunsuffix>qb\t{%1, %0|%0, %1}"
9966 [(set_attr "type" "ssemov")
9967 (set_attr "memory" "store")
9968 (set_attr "prefix" "evex")
9969 (set_attr "mode" "TI")])
9971 (define_insn "avx512f_<code>v8div16qi2_mask"
9972 [(set (match_operand:V16QI 0 "register_operand" "=v")
9976 (match_operand:V8DI 1 "register_operand" "v"))
9978 (match_operand:V16QI 2 "vector_move_operand" "0C")
9979 (parallel [(const_int 0) (const_int 1)
9980 (const_int 2) (const_int 3)
9981 (const_int 4) (const_int 5)
9982 (const_int 6) (const_int 7)]))
9983 (match_operand:QI 3 "register_operand" "Yk"))
9984 (const_vector:V8QI [(const_int 0) (const_int 0)
9985 (const_int 0) (const_int 0)
9986 (const_int 0) (const_int 0)
9987 (const_int 0) (const_int 0)])))]
9989 "vpmov<trunsuffix>qb\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"
9990 [(set_attr "type" "ssemov")
9991 (set_attr "prefix" "evex")
9992 (set_attr "mode" "TI")])
9994 (define_insn "*avx512f_<code>v8div16qi2_mask_1"
9995 [(set (match_operand:V16QI 0 "register_operand" "=v")
9999 (match_operand:V8DI 1 "register_operand" "v"))
10000 (const_vector:V8QI [(const_int 0) (const_int 0)
10001 (const_int 0) (const_int 0)
10002 (const_int 0) (const_int 0)
10003 (const_int 0) (const_int 0)])
10004 (match_operand:QI 2 "register_operand" "Yk"))
10005 (const_vector:V8QI [(const_int 0) (const_int 0)
10006 (const_int 0) (const_int 0)
10007 (const_int 0) (const_int 0)
10008 (const_int 0) (const_int 0)])))]
10010 "vpmov<trunsuffix>qb\t{%1, %0%{%2%}%{z%}|%0%{%2%}%{z%}, %1}"
10011 [(set_attr "type" "ssemov")
10012 (set_attr "prefix" "evex")
10013 (set_attr "mode" "TI")])
10015 (define_insn "avx512f_<code>v8div16qi2_mask_store"
10016 [(set (match_operand:V16QI 0 "memory_operand" "=m")
10020 (match_operand:V8DI 1 "register_operand" "v"))
10023 (parallel [(const_int 0) (const_int 1)
10024 (const_int 2) (const_int 3)
10025 (const_int 4) (const_int 5)
10026 (const_int 6) (const_int 7)]))
10027 (match_operand:QI 2 "register_operand" "Yk"))
10030 (parallel [(const_int 8) (const_int 9)
10031 (const_int 10) (const_int 11)
10032 (const_int 12) (const_int 13)
10033 (const_int 14) (const_int 15)]))))]
10035 "vpmov<trunsuffix>qb\t{%1, %0%{%2%}|%q0%{%2%}, %1}"
10036 [(set_attr "type" "ssemov")
10037 (set_attr "memory" "store")
10038 (set_attr "prefix" "evex")
10039 (set_attr "mode" "TI")])
10041 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
10043 ;; Parallel integral arithmetic
10045 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
10047 (define_expand "neg<mode>2"
10048 [(set (match_operand:VI_AVX2 0 "register_operand")
10051 (match_operand:VI_AVX2 1 "vector_operand")))]
10053 "operands[2] = force_reg (<MODE>mode, CONST0_RTX (<MODE>mode));")
10055 (define_expand "<plusminus_insn><mode>3"
10056 [(set (match_operand:VI_AVX2 0 "register_operand")
10058 (match_operand:VI_AVX2 1 "vector_operand")
10059 (match_operand:VI_AVX2 2 "vector_operand")))]
10061 "ix86_fixup_binary_operands_no_copy (<CODE>, <MODE>mode, operands);")
10063 (define_expand "<plusminus_insn><mode>3_mask"
10064 [(set (match_operand:VI48_AVX512VL 0 "register_operand")
10065 (vec_merge:VI48_AVX512VL
10066 (plusminus:VI48_AVX512VL
10067 (match_operand:VI48_AVX512VL 1 "nonimmediate_operand")
10068 (match_operand:VI48_AVX512VL 2 "nonimmediate_operand"))
10069 (match_operand:VI48_AVX512VL 3 "vector_move_operand")
10070 (match_operand:<avx512fmaskmode> 4 "register_operand")))]
10072 "ix86_fixup_binary_operands_no_copy (<CODE>, <MODE>mode, operands);")
10074 (define_expand "<plusminus_insn><mode>3_mask"
10075 [(set (match_operand:VI12_AVX512VL 0 "register_operand")
10076 (vec_merge:VI12_AVX512VL
10077 (plusminus:VI12_AVX512VL
10078 (match_operand:VI12_AVX512VL 1 "nonimmediate_operand")
10079 (match_operand:VI12_AVX512VL 2 "nonimmediate_operand"))
10080 (match_operand:VI12_AVX512VL 3 "vector_move_operand")
10081 (match_operand:<avx512fmaskmode> 4 "register_operand")))]
10083 "ix86_fixup_binary_operands_no_copy (<CODE>, <MODE>mode, operands);")
10085 (define_insn "*<plusminus_insn><mode>3"
10086 [(set (match_operand:VI_AVX2 0 "register_operand" "=x,v")
10088 (match_operand:VI_AVX2 1 "vector_operand" "<comm>0,v")
10089 (match_operand:VI_AVX2 2 "vector_operand" "xBm,vm")))]
10090 "TARGET_SSE2 && ix86_binary_operator_ok (<CODE>, <MODE>mode, operands)"
10092 p<plusminus_mnemonic><ssemodesuffix>\t{%2, %0|%0, %2}
10093 vp<plusminus_mnemonic><ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
10094 [(set_attr "isa" "noavx,avx")
10095 (set_attr "type" "sseiadd")
10096 (set_attr "prefix_data16" "1,*")
10097 (set_attr "prefix" "orig,vex")
10098 (set_attr "mode" "<sseinsnmode>")])
10100 (define_insn "*<plusminus_insn><mode>3_mask"
10101 [(set (match_operand:VI48_AVX512VL 0 "register_operand" "=v")
10102 (vec_merge:VI48_AVX512VL
10103 (plusminus:VI48_AVX512VL
10104 (match_operand:VI48_AVX512VL 1 "nonimmediate_operand" "<comm>v")
10105 (match_operand:VI48_AVX512VL 2 "nonimmediate_operand" "vm"))
10106 (match_operand:VI48_AVX512VL 3 "vector_move_operand" "0C")
10107 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk")))]
10108 "TARGET_AVX512F && ix86_binary_operator_ok (<CODE>, <MODE>mode, operands)"
10109 "vp<plusminus_mnemonic><ssemodesuffix>\t{%2, %1, %0%{%4%}%N3|%0%{%4%}%N3, %1, %2}"
10110 [(set_attr "type" "sseiadd")
10111 (set_attr "prefix" "evex")
10112 (set_attr "mode" "<sseinsnmode>")])
10114 (define_insn "*<plusminus_insn><mode>3_mask"
10115 [(set (match_operand:VI12_AVX512VL 0 "register_operand" "=v")
10116 (vec_merge:VI12_AVX512VL
10117 (plusminus:VI12_AVX512VL
10118 (match_operand:VI12_AVX512VL 1 "nonimmediate_operand" "<comm>v")
10119 (match_operand:VI12_AVX512VL 2 "nonimmediate_operand" "vm"))
10120 (match_operand:VI12_AVX512VL 3 "vector_move_operand" "0C")
10121 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk")))]
10122 "TARGET_AVX512BW && ix86_binary_operator_ok (<CODE>, <MODE>mode, operands)"
10123 "vp<plusminus_mnemonic><ssemodesuffix>\t{%2, %1, %0%{%4%}%N3|%0%{%4%}%N3, %1, %2}"
10124 [(set_attr "type" "sseiadd")
10125 (set_attr "prefix" "evex")
10126 (set_attr "mode" "<sseinsnmode>")])
10128 (define_expand "<sse2_avx2>_<plusminus_insn><mode>3<mask_name>"
10129 [(set (match_operand:VI12_AVX2 0 "register_operand")
10130 (sat_plusminus:VI12_AVX2
10131 (match_operand:VI12_AVX2 1 "vector_operand")
10132 (match_operand:VI12_AVX2 2 "vector_operand")))]
10133 "TARGET_SSE2 && <mask_mode512bit_condition> && <mask_avx512bw_condition>"
10134 "ix86_fixup_binary_operands_no_copy (<CODE>, <MODE>mode, operands);")
10136 (define_insn "*<sse2_avx2>_<plusminus_insn><mode>3<mask_name>"
10137 [(set (match_operand:VI12_AVX2 0 "register_operand" "=x,v")
10138 (sat_plusminus:VI12_AVX2
10139 (match_operand:VI12_AVX2 1 "vector_operand" "<comm>0,v")
10140 (match_operand:VI12_AVX2 2 "vector_operand" "xBm,vm")))]
10141 "TARGET_SSE2 && <mask_mode512bit_condition> && <mask_avx512bw_condition>
10142 && ix86_binary_operator_ok (<CODE>, <MODE>mode, operands)"
10144 p<plusminus_mnemonic><ssemodesuffix>\t{%2, %0|%0, %2}
10145 vp<plusminus_mnemonic><ssemodesuffix>\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
10146 [(set_attr "isa" "noavx,avx")
10147 (set_attr "type" "sseiadd")
10148 (set_attr "prefix_data16" "1,*")
10149 (set_attr "prefix" "orig,maybe_evex")
10150 (set_attr "mode" "TI")])
10152 (define_expand "mul<mode>3<mask_name>"
10153 [(set (match_operand:VI1_AVX512 0 "register_operand")
10154 (mult:VI1_AVX512 (match_operand:VI1_AVX512 1 "register_operand")
10155 (match_operand:VI1_AVX512 2 "register_operand")))]
10156 "TARGET_SSE2 && <mask_mode512bit_condition> && <mask_avx512bw_condition>"
10158 ix86_expand_vecop_qihi (MULT, operands[0], operands[1], operands[2]);
10162 (define_expand "mul<mode>3<mask_name>"
10163 [(set (match_operand:VI2_AVX2 0 "register_operand")
10164 (mult:VI2_AVX2 (match_operand:VI2_AVX2 1 "vector_operand")
10165 (match_operand:VI2_AVX2 2 "vector_operand")))]
10166 "TARGET_SSE2 && <mask_mode512bit_condition> && <mask_avx512bw_condition>"
10167 "ix86_fixup_binary_operands_no_copy (MULT, <MODE>mode, operands);")
10169 (define_insn "*mul<mode>3<mask_name>"
10170 [(set (match_operand:VI2_AVX2 0 "register_operand" "=x,v")
10171 (mult:VI2_AVX2 (match_operand:VI2_AVX2 1 "vector_operand" "%0,v")
10172 (match_operand:VI2_AVX2 2 "vector_operand" "xBm,vm")))]
10173 "TARGET_SSE2 && !(MEM_P (operands[1]) && MEM_P (operands[2]))
10174 && <mask_mode512bit_condition> && <mask_avx512bw_condition>"
10176 pmullw\t{%2, %0|%0, %2}
10177 vpmullw\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
10178 [(set_attr "isa" "noavx,avx")
10179 (set_attr "type" "sseimul")
10180 (set_attr "prefix_data16" "1,*")
10181 (set_attr "prefix" "orig,vex")
10182 (set_attr "mode" "<sseinsnmode>")])
10184 (define_expand "<s>mul<mode>3_highpart<mask_name>"
10185 [(set (match_operand:VI2_AVX2 0 "register_operand")
10187 (lshiftrt:<ssedoublemode>
10188 (mult:<ssedoublemode>
10189 (any_extend:<ssedoublemode>
10190 (match_operand:VI2_AVX2 1 "vector_operand"))
10191 (any_extend:<ssedoublemode>
10192 (match_operand:VI2_AVX2 2 "vector_operand")))
10195 && <mask_mode512bit_condition> && <mask_avx512bw_condition>"
10196 "ix86_fixup_binary_operands_no_copy (MULT, <MODE>mode, operands);")
10198 (define_insn "*<s>mul<mode>3_highpart<mask_name>"
10199 [(set (match_operand:VI2_AVX2 0 "register_operand" "=x,v")
10201 (lshiftrt:<ssedoublemode>
10202 (mult:<ssedoublemode>
10203 (any_extend:<ssedoublemode>
10204 (match_operand:VI2_AVX2 1 "vector_operand" "%0,v"))
10205 (any_extend:<ssedoublemode>
10206 (match_operand:VI2_AVX2 2 "vector_operand" "xBm,vm")))
10208 "TARGET_SSE2 && !(MEM_P (operands[1]) && MEM_P (operands[2]))
10209 && <mask_mode512bit_condition> && <mask_avx512bw_condition>"
10211 pmulh<u>w\t{%2, %0|%0, %2}
10212 vpmulh<u>w\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
10213 [(set_attr "isa" "noavx,avx")
10214 (set_attr "type" "sseimul")
10215 (set_attr "prefix_data16" "1,*")
10216 (set_attr "prefix" "orig,vex")
10217 (set_attr "mode" "<sseinsnmode>")])
10219 (define_expand "vec_widen_umult_even_v16si<mask_name>"
10220 [(set (match_operand:V8DI 0 "register_operand")
10224 (match_operand:V16SI 1 "nonimmediate_operand")
10225 (parallel [(const_int 0) (const_int 2)
10226 (const_int 4) (const_int 6)
10227 (const_int 8) (const_int 10)
10228 (const_int 12) (const_int 14)])))
10231 (match_operand:V16SI 2 "nonimmediate_operand")
10232 (parallel [(const_int 0) (const_int 2)
10233 (const_int 4) (const_int 6)
10234 (const_int 8) (const_int 10)
10235 (const_int 12) (const_int 14)])))))]
10237 "ix86_fixup_binary_operands_no_copy (MULT, V16SImode, operands);")
10239 (define_insn "*vec_widen_umult_even_v16si<mask_name>"
10240 [(set (match_operand:V8DI 0 "register_operand" "=v")
10244 (match_operand:V16SI 1 "nonimmediate_operand" "%v")
10245 (parallel [(const_int 0) (const_int 2)
10246 (const_int 4) (const_int 6)
10247 (const_int 8) (const_int 10)
10248 (const_int 12) (const_int 14)])))
10251 (match_operand:V16SI 2 "nonimmediate_operand" "vm")
10252 (parallel [(const_int 0) (const_int 2)
10253 (const_int 4) (const_int 6)
10254 (const_int 8) (const_int 10)
10255 (const_int 12) (const_int 14)])))))]
10256 "TARGET_AVX512F && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
10257 "vpmuludq\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
10258 [(set_attr "type" "sseimul")
10259 (set_attr "prefix_extra" "1")
10260 (set_attr "prefix" "evex")
10261 (set_attr "mode" "XI")])
10263 (define_expand "vec_widen_umult_even_v8si<mask_name>"
10264 [(set (match_operand:V4DI 0 "register_operand")
10268 (match_operand:V8SI 1 "nonimmediate_operand")
10269 (parallel [(const_int 0) (const_int 2)
10270 (const_int 4) (const_int 6)])))
10273 (match_operand:V8SI 2 "nonimmediate_operand")
10274 (parallel [(const_int 0) (const_int 2)
10275 (const_int 4) (const_int 6)])))))]
10276 "TARGET_AVX2 && <mask_avx512vl_condition>"
10277 "ix86_fixup_binary_operands_no_copy (MULT, V8SImode, operands);")
10279 (define_insn "*vec_widen_umult_even_v8si<mask_name>"
10280 [(set (match_operand:V4DI 0 "register_operand" "=v")
10284 (match_operand:V8SI 1 "nonimmediate_operand" "%v")
10285 (parallel [(const_int 0) (const_int 2)
10286 (const_int 4) (const_int 6)])))
10289 (match_operand:V8SI 2 "nonimmediate_operand" "vm")
10290 (parallel [(const_int 0) (const_int 2)
10291 (const_int 4) (const_int 6)])))))]
10292 "TARGET_AVX2 && <mask_avx512vl_condition>
10293 && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
10294 "vpmuludq\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
10295 [(set_attr "type" "sseimul")
10296 (set_attr "prefix" "maybe_evex")
10297 (set_attr "mode" "OI")])
10299 (define_expand "vec_widen_umult_even_v4si<mask_name>"
10300 [(set (match_operand:V2DI 0 "register_operand")
10304 (match_operand:V4SI 1 "vector_operand")
10305 (parallel [(const_int 0) (const_int 2)])))
10308 (match_operand:V4SI 2 "vector_operand")
10309 (parallel [(const_int 0) (const_int 2)])))))]
10310 "TARGET_SSE2 && <mask_avx512vl_condition>"
10311 "ix86_fixup_binary_operands_no_copy (MULT, V4SImode, operands);")
10313 (define_insn "*vec_widen_umult_even_v4si<mask_name>"
10314 [(set (match_operand:V2DI 0 "register_operand" "=x,v")
10318 (match_operand:V4SI 1 "vector_operand" "%0,v")
10319 (parallel [(const_int 0) (const_int 2)])))
10322 (match_operand:V4SI 2 "vector_operand" "xBm,vm")
10323 (parallel [(const_int 0) (const_int 2)])))))]
10324 "TARGET_SSE2 && <mask_avx512vl_condition>
10325 && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
10327 pmuludq\t{%2, %0|%0, %2}
10328 vpmuludq\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
10329 [(set_attr "isa" "noavx,avx")
10330 (set_attr "type" "sseimul")
10331 (set_attr "prefix_data16" "1,*")
10332 (set_attr "prefix" "orig,maybe_evex")
10333 (set_attr "mode" "TI")])
10335 (define_expand "vec_widen_smult_even_v16si<mask_name>"
10336 [(set (match_operand:V8DI 0 "register_operand")
10340 (match_operand:V16SI 1 "nonimmediate_operand")
10341 (parallel [(const_int 0) (const_int 2)
10342 (const_int 4) (const_int 6)
10343 (const_int 8) (const_int 10)
10344 (const_int 12) (const_int 14)])))
10347 (match_operand:V16SI 2 "nonimmediate_operand")
10348 (parallel [(const_int 0) (const_int 2)
10349 (const_int 4) (const_int 6)
10350 (const_int 8) (const_int 10)
10351 (const_int 12) (const_int 14)])))))]
10353 "ix86_fixup_binary_operands_no_copy (MULT, V16SImode, operands);")
10355 (define_insn "*vec_widen_smult_even_v16si<mask_name>"
10356 [(set (match_operand:V8DI 0 "register_operand" "=v")
10360 (match_operand:V16SI 1 "nonimmediate_operand" "%v")
10361 (parallel [(const_int 0) (const_int 2)
10362 (const_int 4) (const_int 6)
10363 (const_int 8) (const_int 10)
10364 (const_int 12) (const_int 14)])))
10367 (match_operand:V16SI 2 "nonimmediate_operand" "vm")
10368 (parallel [(const_int 0) (const_int 2)
10369 (const_int 4) (const_int 6)
10370 (const_int 8) (const_int 10)
10371 (const_int 12) (const_int 14)])))))]
10372 "TARGET_AVX512F && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
10373 "vpmuldq\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
10374 [(set_attr "type" "sseimul")
10375 (set_attr "prefix_extra" "1")
10376 (set_attr "prefix" "evex")
10377 (set_attr "mode" "XI")])
10379 (define_expand "vec_widen_smult_even_v8si<mask_name>"
10380 [(set (match_operand:V4DI 0 "register_operand")
10384 (match_operand:V8SI 1 "nonimmediate_operand")
10385 (parallel [(const_int 0) (const_int 2)
10386 (const_int 4) (const_int 6)])))
10389 (match_operand:V8SI 2 "nonimmediate_operand")
10390 (parallel [(const_int 0) (const_int 2)
10391 (const_int 4) (const_int 6)])))))]
10392 "TARGET_AVX2 && <mask_avx512vl_condition>"
10393 "ix86_fixup_binary_operands_no_copy (MULT, V8SImode, operands);")
10395 (define_insn "*vec_widen_smult_even_v8si<mask_name>"
10396 [(set (match_operand:V4DI 0 "register_operand" "=v")
10400 (match_operand:V8SI 1 "nonimmediate_operand" "%v")
10401 (parallel [(const_int 0) (const_int 2)
10402 (const_int 4) (const_int 6)])))
10405 (match_operand:V8SI 2 "nonimmediate_operand" "vm")
10406 (parallel [(const_int 0) (const_int 2)
10407 (const_int 4) (const_int 6)])))))]
10408 "TARGET_AVX2 && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
10409 "vpmuldq\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
10410 [(set_attr "type" "sseimul")
10411 (set_attr "prefix_extra" "1")
10412 (set_attr "prefix" "vex")
10413 (set_attr "mode" "OI")])
10415 (define_expand "sse4_1_mulv2siv2di3<mask_name>"
10416 [(set (match_operand:V2DI 0 "register_operand")
10420 (match_operand:V4SI 1 "vector_operand")
10421 (parallel [(const_int 0) (const_int 2)])))
10424 (match_operand:V4SI 2 "vector_operand")
10425 (parallel [(const_int 0) (const_int 2)])))))]
10426 "TARGET_SSE4_1 && <mask_avx512vl_condition>"
10427 "ix86_fixup_binary_operands_no_copy (MULT, V4SImode, operands);")
10429 (define_insn "*sse4_1_mulv2siv2di3<mask_name>"
10430 [(set (match_operand:V2DI 0 "register_operand" "=Yr,*x,v")
10434 (match_operand:V4SI 1 "vector_operand" "%0,0,v")
10435 (parallel [(const_int 0) (const_int 2)])))
10438 (match_operand:V4SI 2 "vector_operand" "YrBm,*xBm,vm")
10439 (parallel [(const_int 0) (const_int 2)])))))]
10440 "TARGET_SSE4_1 && <mask_avx512vl_condition>
10441 && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
10443 pmuldq\t{%2, %0|%0, %2}
10444 pmuldq\t{%2, %0|%0, %2}
10445 vpmuldq\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
10446 [(set_attr "isa" "noavx,noavx,avx")
10447 (set_attr "type" "sseimul")
10448 (set_attr "prefix_data16" "1,1,*")
10449 (set_attr "prefix_extra" "1")
10450 (set_attr "prefix" "orig,orig,vex")
10451 (set_attr "mode" "TI")])
10453 (define_insn "avx512bw_pmaddwd512<mode><mask_name>"
10454 [(set (match_operand:<sseunpackmode> 0 "register_operand" "=v")
10455 (unspec:<sseunpackmode>
10456 [(match_operand:VI2_AVX2 1 "register_operand" "v")
10457 (match_operand:VI2_AVX2 2 "nonimmediate_operand" "vm")]
10458 UNSPEC_PMADDWD512))]
10459 "TARGET_AVX512BW && <mask_mode512bit_condition>"
10460 "vpmaddwd\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}";
10461 [(set_attr "type" "sseiadd")
10462 (set_attr "prefix" "evex")
10463 (set_attr "mode" "XI")])
10465 (define_expand "avx2_pmaddwd"
10466 [(set (match_operand:V8SI 0 "register_operand")
10471 (match_operand:V16HI 1 "nonimmediate_operand")
10472 (parallel [(const_int 0) (const_int 2)
10473 (const_int 4) (const_int 6)
10474 (const_int 8) (const_int 10)
10475 (const_int 12) (const_int 14)])))
10478 (match_operand:V16HI 2 "nonimmediate_operand")
10479 (parallel [(const_int 0) (const_int 2)
10480 (const_int 4) (const_int 6)
10481 (const_int 8) (const_int 10)
10482 (const_int 12) (const_int 14)]))))
10485 (vec_select:V8HI (match_dup 1)
10486 (parallel [(const_int 1) (const_int 3)
10487 (const_int 5) (const_int 7)
10488 (const_int 9) (const_int 11)
10489 (const_int 13) (const_int 15)])))
10491 (vec_select:V8HI (match_dup 2)
10492 (parallel [(const_int 1) (const_int 3)
10493 (const_int 5) (const_int 7)
10494 (const_int 9) (const_int 11)
10495 (const_int 13) (const_int 15)]))))))]
10497 "ix86_fixup_binary_operands_no_copy (MULT, V16HImode, operands);")
10499 (define_insn "*avx2_pmaddwd"
10500 [(set (match_operand:V8SI 0 "register_operand" "=x,v")
10505 (match_operand:V16HI 1 "nonimmediate_operand" "%x,v")
10506 (parallel [(const_int 0) (const_int 2)
10507 (const_int 4) (const_int 6)
10508 (const_int 8) (const_int 10)
10509 (const_int 12) (const_int 14)])))
10512 (match_operand:V16HI 2 "nonimmediate_operand" "xm,vm")
10513 (parallel [(const_int 0) (const_int 2)
10514 (const_int 4) (const_int 6)
10515 (const_int 8) (const_int 10)
10516 (const_int 12) (const_int 14)]))))
10519 (vec_select:V8HI (match_dup 1)
10520 (parallel [(const_int 1) (const_int 3)
10521 (const_int 5) (const_int 7)
10522 (const_int 9) (const_int 11)
10523 (const_int 13) (const_int 15)])))
10525 (vec_select:V8HI (match_dup 2)
10526 (parallel [(const_int 1) (const_int 3)
10527 (const_int 5) (const_int 7)
10528 (const_int 9) (const_int 11)
10529 (const_int 13) (const_int 15)]))))))]
10530 "TARGET_AVX2 && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
10531 "vpmaddwd\t{%2, %1, %0|%0, %1, %2}"
10532 [(set_attr "type" "sseiadd")
10533 (set_attr "isa" "*,avx512bw")
10534 (set_attr "prefix" "vex,evex")
10535 (set_attr "mode" "OI")])
10537 (define_expand "sse2_pmaddwd"
10538 [(set (match_operand:V4SI 0 "register_operand")
10543 (match_operand:V8HI 1 "vector_operand")
10544 (parallel [(const_int 0) (const_int 2)
10545 (const_int 4) (const_int 6)])))
10548 (match_operand:V8HI 2 "vector_operand")
10549 (parallel [(const_int 0) (const_int 2)
10550 (const_int 4) (const_int 6)]))))
10553 (vec_select:V4HI (match_dup 1)
10554 (parallel [(const_int 1) (const_int 3)
10555 (const_int 5) (const_int 7)])))
10557 (vec_select:V4HI (match_dup 2)
10558 (parallel [(const_int 1) (const_int 3)
10559 (const_int 5) (const_int 7)]))))))]
10561 "ix86_fixup_binary_operands_no_copy (MULT, V8HImode, operands);")
10563 (define_insn "*sse2_pmaddwd"
10564 [(set (match_operand:V4SI 0 "register_operand" "=x,x,v")
10569 (match_operand:V8HI 1 "vector_operand" "%0,x,v")
10570 (parallel [(const_int 0) (const_int 2)
10571 (const_int 4) (const_int 6)])))
10574 (match_operand:V8HI 2 "vector_operand" "xBm,xm,vm")
10575 (parallel [(const_int 0) (const_int 2)
10576 (const_int 4) (const_int 6)]))))
10579 (vec_select:V4HI (match_dup 1)
10580 (parallel [(const_int 1) (const_int 3)
10581 (const_int 5) (const_int 7)])))
10583 (vec_select:V4HI (match_dup 2)
10584 (parallel [(const_int 1) (const_int 3)
10585 (const_int 5) (const_int 7)]))))))]
10586 "TARGET_SSE2 && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
10588 pmaddwd\t{%2, %0|%0, %2}
10589 vpmaddwd\t{%2, %1, %0|%0, %1, %2}
10590 vpmaddwd\t{%2, %1, %0|%0, %1, %2}"
10591 [(set_attr "isa" "noavx,avx,avx512bw")
10592 (set_attr "type" "sseiadd")
10593 (set_attr "atom_unit" "simul")
10594 (set_attr "prefix_data16" "1,*,*")
10595 (set_attr "prefix" "orig,vex,evex")
10596 (set_attr "mode" "TI")])
10598 (define_insn "avx512dq_mul<mode>3<mask_name>"
10599 [(set (match_operand:VI8 0 "register_operand" "=v")
10601 (match_operand:VI8 1 "register_operand" "v")
10602 (match_operand:VI8 2 "nonimmediate_operand" "vm")))]
10603 "TARGET_AVX512DQ && <mask_mode512bit_condition>"
10604 "vpmullq\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
10605 [(set_attr "type" "sseimul")
10606 (set_attr "prefix" "evex")
10607 (set_attr "mode" "<sseinsnmode>")])
10609 (define_expand "mul<mode>3<mask_name>"
10610 [(set (match_operand:VI4_AVX512F 0 "register_operand")
10612 (match_operand:VI4_AVX512F 1 "general_vector_operand")
10613 (match_operand:VI4_AVX512F 2 "general_vector_operand")))]
10614 "TARGET_SSE2 && <mask_mode512bit_condition>"
10618 if (!vector_operand (operands[1], <MODE>mode))
10619 operands[1] = force_reg (<MODE>mode, operands[1]);
10620 if (!vector_operand (operands[2], <MODE>mode))
10621 operands[2] = force_reg (<MODE>mode, operands[2]);
10622 ix86_fixup_binary_operands_no_copy (MULT, <MODE>mode, operands);
10626 ix86_expand_sse2_mulv4si3 (operands[0], operands[1], operands[2]);
10631 (define_insn "*<sse4_1_avx2>_mul<mode>3<mask_name>"
10632 [(set (match_operand:VI4_AVX512F 0 "register_operand" "=Yr,*x,v")
10634 (match_operand:VI4_AVX512F 1 "vector_operand" "%0,0,v")
10635 (match_operand:VI4_AVX512F 2 "vector_operand" "YrBm,*xBm,vm")))]
10636 "TARGET_SSE4_1 && !(MEM_P (operands[1]) && MEM_P (operands[2]))
10637 && <mask_mode512bit_condition>"
10639 pmulld\t{%2, %0|%0, %2}
10640 pmulld\t{%2, %0|%0, %2}
10641 vpmulld\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
10642 [(set_attr "isa" "noavx,noavx,avx")
10643 (set_attr "type" "sseimul")
10644 (set_attr "prefix_extra" "1")
10645 (set_attr "prefix" "<mask_prefix4>")
10646 (set_attr "btver2_decode" "vector,vector,vector")
10647 (set_attr "mode" "<sseinsnmode>")])
10649 (define_expand "mul<mode>3"
10650 [(set (match_operand:VI8_AVX2_AVX512F 0 "register_operand")
10651 (mult:VI8_AVX2_AVX512F
10652 (match_operand:VI8_AVX2_AVX512F 1 "register_operand")
10653 (match_operand:VI8_AVX2_AVX512F 2 "register_operand")))]
10656 ix86_expand_sse2_mulvxdi3 (operands[0], operands[1], operands[2]);
10660 (define_expand "vec_widen_<s>mult_hi_<mode>"
10661 [(match_operand:<sseunpackmode> 0 "register_operand")
10662 (any_extend:<sseunpackmode>
10663 (match_operand:VI124_AVX2 1 "register_operand"))
10664 (match_operand:VI124_AVX2 2 "register_operand")]
10667 ix86_expand_mul_widen_hilo (operands[0], operands[1], operands[2],
10672 (define_expand "vec_widen_<s>mult_lo_<mode>"
10673 [(match_operand:<sseunpackmode> 0 "register_operand")
10674 (any_extend:<sseunpackmode>
10675 (match_operand:VI124_AVX2 1 "register_operand"))
10676 (match_operand:VI124_AVX2 2 "register_operand")]
10679 ix86_expand_mul_widen_hilo (operands[0], operands[1], operands[2],
10684 ;; Most widen_<s>mult_even_<mode> can be handled directly from other
10685 ;; named patterns, but signed V4SI needs special help for plain SSE2.
10686 (define_expand "vec_widen_smult_even_v4si"
10687 [(match_operand:V2DI 0 "register_operand")
10688 (match_operand:V4SI 1 "vector_operand")
10689 (match_operand:V4SI 2 "vector_operand")]
10692 ix86_expand_mul_widen_evenodd (operands[0], operands[1], operands[2],
10697 (define_expand "vec_widen_<s>mult_odd_<mode>"
10698 [(match_operand:<sseunpackmode> 0 "register_operand")
10699 (any_extend:<sseunpackmode>
10700 (match_operand:VI4_AVX512F 1 "general_vector_operand"))
10701 (match_operand:VI4_AVX512F 2 "general_vector_operand")]
10704 ix86_expand_mul_widen_evenodd (operands[0], operands[1], operands[2],
10709 (define_mode_attr SDOT_PMADD_SUF
10710 [(V32HI "512v32hi") (V16HI "") (V8HI "")])
10712 (define_expand "sdot_prod<mode>"
10713 [(match_operand:<sseunpackmode> 0 "register_operand")
10714 (match_operand:VI2_AVX2 1 "register_operand")
10715 (match_operand:VI2_AVX2 2 "register_operand")
10716 (match_operand:<sseunpackmode> 3 "register_operand")]
10719 rtx t = gen_reg_rtx (<sseunpackmode>mode);
10720 emit_insn (gen_<sse2_avx2>_pmaddwd<SDOT_PMADD_SUF> (t, operands[1], operands[2]));
10721 emit_insn (gen_rtx_SET (operands[0],
10722 gen_rtx_PLUS (<sseunpackmode>mode,
10727 ;; Normally we use widen_mul_even/odd, but combine can't quite get it all
10728 ;; back together when madd is available.
10729 (define_expand "sdot_prodv4si"
10730 [(match_operand:V2DI 0 "register_operand")
10731 (match_operand:V4SI 1 "register_operand")
10732 (match_operand:V4SI 2 "register_operand")
10733 (match_operand:V2DI 3 "register_operand")]
10736 rtx t = gen_reg_rtx (V2DImode);
10737 emit_insn (gen_xop_pmacsdqh (t, operands[1], operands[2], operands[3]));
10738 emit_insn (gen_xop_pmacsdql (operands[0], operands[1], operands[2], t));
10742 (define_expand "usadv16qi"
10743 [(match_operand:V4SI 0 "register_operand")
10744 (match_operand:V16QI 1 "register_operand")
10745 (match_operand:V16QI 2 "vector_operand")
10746 (match_operand:V4SI 3 "vector_operand")]
10749 rtx t1 = gen_reg_rtx (V2DImode);
10750 rtx t2 = gen_reg_rtx (V4SImode);
10751 emit_insn (gen_sse2_psadbw (t1, operands[1], operands[2]));
10752 convert_move (t2, t1, 0);
10753 emit_insn (gen_addv4si3 (operands[0], t2, operands[3]));
10757 (define_expand "usadv32qi"
10758 [(match_operand:V8SI 0 "register_operand")
10759 (match_operand:V32QI 1 "register_operand")
10760 (match_operand:V32QI 2 "nonimmediate_operand")
10761 (match_operand:V8SI 3 "nonimmediate_operand")]
10764 rtx t1 = gen_reg_rtx (V4DImode);
10765 rtx t2 = gen_reg_rtx (V8SImode);
10766 emit_insn (gen_avx2_psadbw (t1, operands[1], operands[2]));
10767 convert_move (t2, t1, 0);
10768 emit_insn (gen_addv8si3 (operands[0], t2, operands[3]));
10772 (define_insn "<mask_codefor>ashr<mode>3<mask_name>"
10773 [(set (match_operand:VI248_AVX512BW_1 0 "register_operand" "=v,v")
10774 (ashiftrt:VI248_AVX512BW_1
10775 (match_operand:VI248_AVX512BW_1 1 "nonimmediate_operand" "v,vm")
10776 (match_operand:DI 2 "nonmemory_operand" "v,N")))]
10778 "vpsra<ssemodesuffix>\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
10779 [(set_attr "type" "sseishft")
10780 (set (attr "length_immediate")
10781 (if_then_else (match_operand 2 "const_int_operand")
10783 (const_string "0")))
10784 (set_attr "mode" "<sseinsnmode>")])
10786 (define_insn "ashr<mode>3"
10787 [(set (match_operand:VI24_AVX2 0 "register_operand" "=x,x")
10788 (ashiftrt:VI24_AVX2
10789 (match_operand:VI24_AVX2 1 "register_operand" "0,x")
10790 (match_operand:DI 2 "nonmemory_operand" "xN,xN")))]
10793 psra<ssemodesuffix>\t{%2, %0|%0, %2}
10794 vpsra<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
10795 [(set_attr "isa" "noavx,avx")
10796 (set_attr "type" "sseishft")
10797 (set (attr "length_immediate")
10798 (if_then_else (match_operand 2 "const_int_operand")
10800 (const_string "0")))
10801 (set_attr "prefix_data16" "1,*")
10802 (set_attr "prefix" "orig,vex")
10803 (set_attr "mode" "<sseinsnmode>")])
10805 (define_insn "ashr<mode>3<mask_name>"
10806 [(set (match_operand:VI248_AVX512BW_AVX512VL 0 "register_operand" "=v,v")
10807 (ashiftrt:VI248_AVX512BW_AVX512VL
10808 (match_operand:VI248_AVX512BW_AVX512VL 1 "nonimmediate_operand" "v,vm")
10809 (match_operand:DI 2 "nonmemory_operand" "v,N")))]
10811 "vpsra<ssemodesuffix>\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
10812 [(set_attr "type" "sseishft")
10813 (set (attr "length_immediate")
10814 (if_then_else (match_operand 2 "const_int_operand")
10816 (const_string "0")))
10817 (set_attr "mode" "<sseinsnmode>")])
10819 (define_insn "<mask_codefor><shift_insn><mode>3<mask_name>"
10820 [(set (match_operand:VI248_AVX512BW_2 0 "register_operand" "=v,v")
10821 (any_lshift:VI248_AVX512BW_2
10822 (match_operand:VI248_AVX512BW_2 1 "nonimmediate_operand" "v,vm")
10823 (match_operand:DI 2 "nonmemory_operand" "v,N")))]
10825 "vp<vshift><ssemodesuffix>\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
10826 [(set_attr "type" "sseishft")
10827 (set (attr "length_immediate")
10828 (if_then_else (match_operand 2 "const_int_operand")
10830 (const_string "0")))
10831 (set_attr "mode" "<sseinsnmode>")])
10833 (define_insn "<shift_insn><mode>3"
10834 [(set (match_operand:VI248_AVX2 0 "register_operand" "=x,x")
10835 (any_lshift:VI248_AVX2
10836 (match_operand:VI248_AVX2 1 "register_operand" "0,x")
10837 (match_operand:DI 2 "nonmemory_operand" "xN,xN")))]
10840 p<vshift><ssemodesuffix>\t{%2, %0|%0, %2}
10841 vp<vshift><ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
10842 [(set_attr "isa" "noavx,avx")
10843 (set_attr "type" "sseishft")
10844 (set (attr "length_immediate")
10845 (if_then_else (match_operand 2 "const_int_operand")
10847 (const_string "0")))
10848 (set_attr "prefix_data16" "1,*")
10849 (set_attr "prefix" "orig,vex")
10850 (set_attr "mode" "<sseinsnmode>")])
10852 (define_insn "<shift_insn><mode>3<mask_name>"
10853 [(set (match_operand:VI248_AVX512BW 0 "register_operand" "=v,v")
10854 (any_lshift:VI248_AVX512BW
10855 (match_operand:VI248_AVX512BW 1 "nonimmediate_operand" "v,m")
10856 (match_operand:DI 2 "nonmemory_operand" "vN,N")))]
10858 "vp<vshift><ssemodesuffix>\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
10859 [(set_attr "type" "sseishft")
10860 (set (attr "length_immediate")
10861 (if_then_else (match_operand 2 "const_int_operand")
10863 (const_string "0")))
10864 (set_attr "mode" "<sseinsnmode>")])
10867 (define_expand "vec_shr_<mode>"
10868 [(set (match_dup 3)
10870 (match_operand:VI_128 1 "register_operand")
10871 (match_operand:SI 2 "const_0_to_255_mul_8_operand")))
10872 (set (match_operand:VI_128 0 "register_operand") (match_dup 4))]
10875 operands[1] = gen_lowpart (V1TImode, operands[1]);
10876 operands[3] = gen_reg_rtx (V1TImode);
10877 operands[4] = gen_lowpart (<MODE>mode, operands[3]);
10880 (define_insn "avx512bw_<shift_insn><mode>3"
10881 [(set (match_operand:VIMAX_AVX512VL 0 "register_operand" "=v")
10882 (any_lshift:VIMAX_AVX512VL
10883 (match_operand:VIMAX_AVX512VL 1 "nonimmediate_operand" "vm")
10884 (match_operand:SI 2 "const_0_to_255_mul_8_operand" "n")))]
10887 operands[2] = GEN_INT (INTVAL (operands[2]) / 8);
10888 return "vp<vshift>dq\t{%2, %1, %0|%0, %1, %2}";
10890 [(set_attr "type" "sseishft")
10891 (set_attr "length_immediate" "1")
10892 (set_attr "prefix" "maybe_evex")
10893 (set_attr "mode" "<sseinsnmode>")])
10895 (define_insn "<sse2_avx2>_<shift_insn><mode>3"
10896 [(set (match_operand:VIMAX_AVX2 0 "register_operand" "=x,v")
10897 (any_lshift:VIMAX_AVX2
10898 (match_operand:VIMAX_AVX2 1 "register_operand" "0,v")
10899 (match_operand:SI 2 "const_0_to_255_mul_8_operand" "n,n")))]
10902 operands[2] = GEN_INT (INTVAL (operands[2]) / 8);
10904 switch (which_alternative)
10907 return "p<vshift>dq\t{%2, %0|%0, %2}";
10909 return "vp<vshift>dq\t{%2, %1, %0|%0, %1, %2}";
10911 gcc_unreachable ();
10914 [(set_attr "isa" "noavx,avx")
10915 (set_attr "type" "sseishft")
10916 (set_attr "length_immediate" "1")
10917 (set_attr "atom_unit" "sishuf")
10918 (set_attr "prefix_data16" "1,*")
10919 (set_attr "prefix" "orig,vex")
10920 (set_attr "mode" "<sseinsnmode>")])
10922 (define_insn "<avx512>_<rotate>v<mode><mask_name>"
10923 [(set (match_operand:VI48_AVX512VL 0 "register_operand" "=v")
10924 (any_rotate:VI48_AVX512VL
10925 (match_operand:VI48_AVX512VL 1 "register_operand" "v")
10926 (match_operand:VI48_AVX512VL 2 "nonimmediate_operand" "vm")))]
10928 "vp<rotate>v<ssemodesuffix>\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
10929 [(set_attr "prefix" "evex")
10930 (set_attr "mode" "<sseinsnmode>")])
10932 (define_insn "<avx512>_<rotate><mode><mask_name>"
10933 [(set (match_operand:VI48_AVX512VL 0 "register_operand" "=v")
10934 (any_rotate:VI48_AVX512VL
10935 (match_operand:VI48_AVX512VL 1 "nonimmediate_operand" "vm")
10936 (match_operand:SI 2 "const_0_to_255_operand")))]
10938 "vp<rotate><ssemodesuffix>\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
10939 [(set_attr "prefix" "evex")
10940 (set_attr "mode" "<sseinsnmode>")])
10942 (define_expand "<code><mode>3"
10943 [(set (match_operand:VI124_256_AVX512F_AVX512BW 0 "register_operand")
10944 (maxmin:VI124_256_AVX512F_AVX512BW
10945 (match_operand:VI124_256_AVX512F_AVX512BW 1 "nonimmediate_operand")
10946 (match_operand:VI124_256_AVX512F_AVX512BW 2 "nonimmediate_operand")))]
10948 "ix86_fixup_binary_operands_no_copy (<CODE>, <MODE>mode, operands);")
10950 (define_insn "*avx2_<code><mode>3"
10951 [(set (match_operand:VI124_256 0 "register_operand" "=v")
10953 (match_operand:VI124_256 1 "nonimmediate_operand" "%v")
10954 (match_operand:VI124_256 2 "nonimmediate_operand" "vm")))]
10955 "TARGET_AVX2 && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
10956 "vp<maxmin_int><ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
10957 [(set_attr "type" "sseiadd")
10958 (set_attr "prefix_extra" "1")
10959 (set_attr "prefix" "vex")
10960 (set_attr "mode" "OI")])
10962 (define_expand "<code><mode>3_mask"
10963 [(set (match_operand:VI48_AVX512VL 0 "register_operand")
10964 (vec_merge:VI48_AVX512VL
10965 (maxmin:VI48_AVX512VL
10966 (match_operand:VI48_AVX512VL 1 "nonimmediate_operand")
10967 (match_operand:VI48_AVX512VL 2 "nonimmediate_operand"))
10968 (match_operand:VI48_AVX512VL 3 "vector_move_operand")
10969 (match_operand:<avx512fmaskmode> 4 "register_operand")))]
10971 "ix86_fixup_binary_operands_no_copy (<CODE>, <MODE>mode, operands);")
10973 (define_insn "*avx512f_<code><mode>3<mask_name>"
10974 [(set (match_operand:VI48_AVX512VL 0 "register_operand" "=v")
10975 (maxmin:VI48_AVX512VL
10976 (match_operand:VI48_AVX512VL 1 "nonimmediate_operand" "%v")
10977 (match_operand:VI48_AVX512VL 2 "nonimmediate_operand" "vm")))]
10978 "TARGET_AVX512F && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
10979 "vp<maxmin_int><ssemodesuffix>\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
10980 [(set_attr "type" "sseiadd")
10981 (set_attr "prefix_extra" "1")
10982 (set_attr "prefix" "maybe_evex")
10983 (set_attr "mode" "<sseinsnmode>")])
10985 (define_insn "<mask_codefor><code><mode>3<mask_name>"
10986 [(set (match_operand:VI12_AVX512VL 0 "register_operand" "=v")
10987 (maxmin:VI12_AVX512VL
10988 (match_operand:VI12_AVX512VL 1 "register_operand" "v")
10989 (match_operand:VI12_AVX512VL 2 "nonimmediate_operand" "vm")))]
10991 "vp<maxmin_int><ssemodesuffix>\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
10992 [(set_attr "type" "sseiadd")
10993 (set_attr "prefix" "evex")
10994 (set_attr "mode" "<sseinsnmode>")])
10996 (define_expand "<code><mode>3"
10997 [(set (match_operand:VI8_AVX2_AVX512F 0 "register_operand")
10998 (maxmin:VI8_AVX2_AVX512F
10999 (match_operand:VI8_AVX2_AVX512F 1 "register_operand")
11000 (match_operand:VI8_AVX2_AVX512F 2 "register_operand")))]
11004 && (<MODE>mode == V8DImode || TARGET_AVX512VL))
11005 ix86_fixup_binary_operands_no_copy (<CODE>, <MODE>mode, operands);
11008 enum rtx_code code;
11013 xops[0] = operands[0];
11015 if (<CODE> == SMAX || <CODE> == UMAX)
11017 xops[1] = operands[1];
11018 xops[2] = operands[2];
11022 xops[1] = operands[2];
11023 xops[2] = operands[1];
11026 code = (<CODE> == UMAX || <CODE> == UMIN) ? GTU : GT;
11028 xops[3] = gen_rtx_fmt_ee (code, VOIDmode, operands[1], operands[2]);
11029 xops[4] = operands[1];
11030 xops[5] = operands[2];
11032 ok = ix86_expand_int_vcond (xops);
11038 (define_expand "<code><mode>3"
11039 [(set (match_operand:VI124_128 0 "register_operand")
11041 (match_operand:VI124_128 1 "vector_operand")
11042 (match_operand:VI124_128 2 "vector_operand")))]
11045 if (TARGET_SSE4_1 || <MODE>mode == V8HImode)
11046 ix86_fixup_binary_operands_no_copy (<CODE>, <MODE>mode, operands);
11052 xops[0] = operands[0];
11053 operands[1] = force_reg (<MODE>mode, operands[1]);
11054 operands[2] = force_reg (<MODE>mode, operands[2]);
11056 if (<CODE> == SMAX)
11058 xops[1] = operands[1];
11059 xops[2] = operands[2];
11063 xops[1] = operands[2];
11064 xops[2] = operands[1];
11067 xops[3] = gen_rtx_GT (VOIDmode, operands[1], operands[2]);
11068 xops[4] = operands[1];
11069 xops[5] = operands[2];
11071 ok = ix86_expand_int_vcond (xops);
11077 (define_insn "*sse4_1_<code><mode>3<mask_name>"
11078 [(set (match_operand:VI14_128 0 "register_operand" "=Yr,*x,v")
11080 (match_operand:VI14_128 1 "vector_operand" "%0,0,v")
11081 (match_operand:VI14_128 2 "vector_operand" "YrBm,*xBm,vm")))]
11083 && <mask_mode512bit_condition>
11084 && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
11086 p<maxmin_int><ssemodesuffix>\t{%2, %0|%0, %2}
11087 p<maxmin_int><ssemodesuffix>\t{%2, %0|%0, %2}
11088 vp<maxmin_int><ssemodesuffix>\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
11089 [(set_attr "isa" "noavx,noavx,avx")
11090 (set_attr "type" "sseiadd")
11091 (set_attr "prefix_extra" "1,1,*")
11092 (set_attr "prefix" "orig,orig,vex")
11093 (set_attr "mode" "TI")])
11095 (define_insn "*<code>v8hi3"
11096 [(set (match_operand:V8HI 0 "register_operand" "=x,x,v")
11098 (match_operand:V8HI 1 "vector_operand" "%0,x,v")
11099 (match_operand:V8HI 2 "vector_operand" "xBm,xm,vm")))]
11100 "TARGET_SSE2 && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
11102 p<maxmin_int>w\t{%2, %0|%0, %2}
11103 vp<maxmin_int>w\t{%2, %1, %0|%0, %1, %2}
11104 vp<maxmin_int>w\t{%2, %1, %0|%0, %1, %2}"
11105 [(set_attr "isa" "noavx,avx,avx512bw")
11106 (set_attr "type" "sseiadd")
11107 (set_attr "prefix_data16" "1,*,*")
11108 (set_attr "prefix_extra" "*,1,1")
11109 (set_attr "prefix" "orig,vex,evex")
11110 (set_attr "mode" "TI")])
11112 (define_expand "<code><mode>3"
11113 [(set (match_operand:VI124_128 0 "register_operand")
11115 (match_operand:VI124_128 1 "vector_operand")
11116 (match_operand:VI124_128 2 "vector_operand")))]
11119 if (TARGET_SSE4_1 || <MODE>mode == V16QImode)
11120 ix86_fixup_binary_operands_no_copy (<CODE>, <MODE>mode, operands);
11121 else if (<CODE> == UMAX && <MODE>mode == V8HImode)
11123 rtx op0 = operands[0], op2 = operands[2], op3 = op0;
11124 operands[1] = force_reg (<MODE>mode, operands[1]);
11125 if (rtx_equal_p (op3, op2))
11126 op3 = gen_reg_rtx (V8HImode);
11127 emit_insn (gen_sse2_ussubv8hi3 (op3, operands[1], op2));
11128 emit_insn (gen_addv8hi3 (op0, op3, op2));
11136 operands[1] = force_reg (<MODE>mode, operands[1]);
11137 operands[2] = force_reg (<MODE>mode, operands[2]);
11139 xops[0] = operands[0];
11141 if (<CODE> == UMAX)
11143 xops[1] = operands[1];
11144 xops[2] = operands[2];
11148 xops[1] = operands[2];
11149 xops[2] = operands[1];
11152 xops[3] = gen_rtx_GTU (VOIDmode, operands[1], operands[2]);
11153 xops[4] = operands[1];
11154 xops[5] = operands[2];
11156 ok = ix86_expand_int_vcond (xops);
11162 (define_insn "*sse4_1_<code><mode>3<mask_name>"
11163 [(set (match_operand:VI24_128 0 "register_operand" "=Yr,*x,v")
11165 (match_operand:VI24_128 1 "vector_operand" "%0,0,v")
11166 (match_operand:VI24_128 2 "vector_operand" "YrBm,*xBm,vm")))]
11168 && <mask_mode512bit_condition>
11169 && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
11171 p<maxmin_int><ssemodesuffix>\t{%2, %0|%0, %2}
11172 p<maxmin_int><ssemodesuffix>\t{%2, %0|%0, %2}
11173 vp<maxmin_int><ssemodesuffix>\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
11174 [(set_attr "isa" "noavx,noavx,avx")
11175 (set_attr "type" "sseiadd")
11176 (set_attr "prefix_extra" "1,1,*")
11177 (set_attr "prefix" "orig,orig,vex")
11178 (set_attr "mode" "TI")])
11180 (define_insn "*<code>v16qi3"
11181 [(set (match_operand:V16QI 0 "register_operand" "=x,x,v")
11183 (match_operand:V16QI 1 "vector_operand" "%0,x,v")
11184 (match_operand:V16QI 2 "vector_operand" "xBm,xm,vm")))]
11185 "TARGET_SSE2 && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
11187 p<maxmin_int>b\t{%2, %0|%0, %2}
11188 vp<maxmin_int>b\t{%2, %1, %0|%0, %1, %2}
11189 vp<maxmin_int>b\t{%2, %1, %0|%0, %1, %2}"
11190 [(set_attr "isa" "noavx,avx,avx512bw")
11191 (set_attr "type" "sseiadd")
11192 (set_attr "prefix_data16" "1,*,*")
11193 (set_attr "prefix_extra" "*,1,1")
11194 (set_attr "prefix" "orig,vex,evex")
11195 (set_attr "mode" "TI")])
11197 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
11199 ;; Parallel integral comparisons
11201 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
11203 (define_expand "avx2_eq<mode>3"
11204 [(set (match_operand:VI_256 0 "register_operand")
11206 (match_operand:VI_256 1 "nonimmediate_operand")
11207 (match_operand:VI_256 2 "nonimmediate_operand")))]
11209 "ix86_fixup_binary_operands_no_copy (EQ, <MODE>mode, operands);")
11211 (define_insn "*avx2_eq<mode>3"
11212 [(set (match_operand:VI_256 0 "register_operand" "=x")
11214 (match_operand:VI_256 1 "nonimmediate_operand" "%x")
11215 (match_operand:VI_256 2 "nonimmediate_operand" "xm")))]
11216 "TARGET_AVX2 && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
11217 "vpcmpeq<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
11218 [(set_attr "type" "ssecmp")
11219 (set_attr "prefix_extra" "1")
11220 (set_attr "prefix" "vex")
11221 (set_attr "mode" "OI")])
11223 (define_expand "<avx512>_eq<mode>3<mask_scalar_merge_name>"
11224 [(set (match_operand:<avx512fmaskmode> 0 "register_operand")
11225 (unspec:<avx512fmaskmode>
11226 [(match_operand:VI12_AVX512VL 1 "nonimmediate_operand")
11227 (match_operand:VI12_AVX512VL 2 "nonimmediate_operand")]
11228 UNSPEC_MASKED_EQ))]
11230 "ix86_fixup_binary_operands_no_copy (EQ, <MODE>mode, operands);")
11232 (define_expand "<avx512>_eq<mode>3<mask_scalar_merge_name>"
11233 [(set (match_operand:<avx512fmaskmode> 0 "register_operand")
11234 (unspec:<avx512fmaskmode>
11235 [(match_operand:VI48_AVX512VL 1 "nonimmediate_operand")
11236 (match_operand:VI48_AVX512VL 2 "nonimmediate_operand")]
11237 UNSPEC_MASKED_EQ))]
11239 "ix86_fixup_binary_operands_no_copy (EQ, <MODE>mode, operands);")
11241 (define_insn "<avx512>_eq<mode>3<mask_scalar_merge_name>_1"
11242 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=Yk")
11243 (unspec:<avx512fmaskmode>
11244 [(match_operand:VI12_AVX512VL 1 "nonimmediate_operand" "%v")
11245 (match_operand:VI12_AVX512VL 2 "nonimmediate_operand" "vm")]
11246 UNSPEC_MASKED_EQ))]
11247 "TARGET_AVX512F && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
11248 "vpcmpeq<ssemodesuffix>\t{%2, %1, %0<mask_scalar_merge_operand3>|%0<mask_scalar_merge_operand3>, %1, %2}"
11249 [(set_attr "type" "ssecmp")
11250 (set_attr "prefix_extra" "1")
11251 (set_attr "prefix" "evex")
11252 (set_attr "mode" "<sseinsnmode>")])
11254 (define_insn "<avx512>_eq<mode>3<mask_scalar_merge_name>_1"
11255 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=Yk")
11256 (unspec:<avx512fmaskmode>
11257 [(match_operand:VI48_AVX512VL 1 "nonimmediate_operand" "%v")
11258 (match_operand:VI48_AVX512VL 2 "nonimmediate_operand" "vm")]
11259 UNSPEC_MASKED_EQ))]
11260 "TARGET_AVX512F && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
11261 "vpcmpeq<ssemodesuffix>\t{%2, %1, %0<mask_scalar_merge_operand3>|%0<mask_scalar_merge_operand3>, %1, %2}"
11262 [(set_attr "type" "ssecmp")
11263 (set_attr "prefix_extra" "1")
11264 (set_attr "prefix" "evex")
11265 (set_attr "mode" "<sseinsnmode>")])
11267 (define_insn "*sse4_1_eqv2di3"
11268 [(set (match_operand:V2DI 0 "register_operand" "=Yr,*x,x")
11270 (match_operand:V2DI 1 "vector_operand" "%0,0,x")
11271 (match_operand:V2DI 2 "vector_operand" "YrBm,*xBm,xm")))]
11272 "TARGET_SSE4_1 && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
11274 pcmpeqq\t{%2, %0|%0, %2}
11275 pcmpeqq\t{%2, %0|%0, %2}
11276 vpcmpeqq\t{%2, %1, %0|%0, %1, %2}"
11277 [(set_attr "isa" "noavx,noavx,avx")
11278 (set_attr "type" "ssecmp")
11279 (set_attr "prefix_extra" "1")
11280 (set_attr "prefix" "orig,orig,vex")
11281 (set_attr "mode" "TI")])
11283 (define_insn "*sse2_eq<mode>3"
11284 [(set (match_operand:VI124_128 0 "register_operand" "=x,x")
11286 (match_operand:VI124_128 1 "vector_operand" "%0,x")
11287 (match_operand:VI124_128 2 "vector_operand" "xBm,xm")))]
11288 "TARGET_SSE2 && !TARGET_XOP
11289 && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
11291 pcmpeq<ssemodesuffix>\t{%2, %0|%0, %2}
11292 vpcmpeq<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
11293 [(set_attr "isa" "noavx,avx")
11294 (set_attr "type" "ssecmp")
11295 (set_attr "prefix_data16" "1,*")
11296 (set_attr "prefix" "orig,vex")
11297 (set_attr "mode" "TI")])
11299 (define_expand "sse2_eq<mode>3"
11300 [(set (match_operand:VI124_128 0 "register_operand")
11302 (match_operand:VI124_128 1 "vector_operand")
11303 (match_operand:VI124_128 2 "vector_operand")))]
11304 "TARGET_SSE2 && !TARGET_XOP "
11305 "ix86_fixup_binary_operands_no_copy (EQ, <MODE>mode, operands);")
11307 (define_expand "sse4_1_eqv2di3"
11308 [(set (match_operand:V2DI 0 "register_operand")
11310 (match_operand:V2DI 1 "vector_operand")
11311 (match_operand:V2DI 2 "vector_operand")))]
11313 "ix86_fixup_binary_operands_no_copy (EQ, V2DImode, operands);")
11315 (define_insn "sse4_2_gtv2di3"
11316 [(set (match_operand:V2DI 0 "register_operand" "=Yr,*x,x")
11318 (match_operand:V2DI 1 "register_operand" "0,0,x")
11319 (match_operand:V2DI 2 "vector_operand" "YrBm,*xBm,xm")))]
11322 pcmpgtq\t{%2, %0|%0, %2}
11323 pcmpgtq\t{%2, %0|%0, %2}
11324 vpcmpgtq\t{%2, %1, %0|%0, %1, %2}"
11325 [(set_attr "isa" "noavx,noavx,avx")
11326 (set_attr "type" "ssecmp")
11327 (set_attr "prefix_extra" "1")
11328 (set_attr "prefix" "orig,orig,vex")
11329 (set_attr "mode" "TI")])
11331 (define_insn "avx2_gt<mode>3"
11332 [(set (match_operand:VI_256 0 "register_operand" "=x")
11334 (match_operand:VI_256 1 "register_operand" "x")
11335 (match_operand:VI_256 2 "nonimmediate_operand" "xm")))]
11337 "vpcmpgt<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
11338 [(set_attr "type" "ssecmp")
11339 (set_attr "prefix_extra" "1")
11340 (set_attr "prefix" "vex")
11341 (set_attr "mode" "OI")])
11343 (define_insn "<avx512>_gt<mode>3<mask_scalar_merge_name>"
11344 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=Yk")
11345 (unspec:<avx512fmaskmode>
11346 [(match_operand:VI48_AVX512VL 1 "register_operand" "v")
11347 (match_operand:VI48_AVX512VL 2 "nonimmediate_operand" "vm")] UNSPEC_MASKED_GT))]
11349 "vpcmpgt<ssemodesuffix>\t{%2, %1, %0<mask_scalar_merge_operand3>|%0<mask_scalar_merge_operand3>, %1, %2}"
11350 [(set_attr "type" "ssecmp")
11351 (set_attr "prefix_extra" "1")
11352 (set_attr "prefix" "evex")
11353 (set_attr "mode" "<sseinsnmode>")])
11355 (define_insn "<avx512>_gt<mode>3<mask_scalar_merge_name>"
11356 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=Yk")
11357 (unspec:<avx512fmaskmode>
11358 [(match_operand:VI12_AVX512VL 1 "register_operand" "v")
11359 (match_operand:VI12_AVX512VL 2 "nonimmediate_operand" "vm")] UNSPEC_MASKED_GT))]
11361 "vpcmpgt<ssemodesuffix>\t{%2, %1, %0<mask_scalar_merge_operand3>|%0<mask_scalar_merge_operand3>, %1, %2}"
11362 [(set_attr "type" "ssecmp")
11363 (set_attr "prefix_extra" "1")
11364 (set_attr "prefix" "evex")
11365 (set_attr "mode" "<sseinsnmode>")])
11367 (define_insn "sse2_gt<mode>3"
11368 [(set (match_operand:VI124_128 0 "register_operand" "=x,x")
11370 (match_operand:VI124_128 1 "register_operand" "0,x")
11371 (match_operand:VI124_128 2 "vector_operand" "xBm,xm")))]
11372 "TARGET_SSE2 && !TARGET_XOP"
11374 pcmpgt<ssemodesuffix>\t{%2, %0|%0, %2}
11375 vpcmpgt<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
11376 [(set_attr "isa" "noavx,avx")
11377 (set_attr "type" "ssecmp")
11378 (set_attr "prefix_data16" "1,*")
11379 (set_attr "prefix" "orig,vex")
11380 (set_attr "mode" "TI")])
11382 (define_expand "vcond<V_512:mode><VI_AVX512BW:mode>"
11383 [(set (match_operand:V_512 0 "register_operand")
11384 (if_then_else:V_512
11385 (match_operator 3 ""
11386 [(match_operand:VI_AVX512BW 4 "nonimmediate_operand")
11387 (match_operand:VI_AVX512BW 5 "general_operand")])
11388 (match_operand:V_512 1)
11389 (match_operand:V_512 2)))]
11391 && (GET_MODE_NUNITS (<V_512:MODE>mode)
11392 == GET_MODE_NUNITS (<VI_AVX512BW:MODE>mode))"
11394 bool ok = ix86_expand_int_vcond (operands);
11399 (define_expand "vcond<V_256:mode><VI_256:mode>"
11400 [(set (match_operand:V_256 0 "register_operand")
11401 (if_then_else:V_256
11402 (match_operator 3 ""
11403 [(match_operand:VI_256 4 "nonimmediate_operand")
11404 (match_operand:VI_256 5 "general_operand")])
11405 (match_operand:V_256 1)
11406 (match_operand:V_256 2)))]
11408 && (GET_MODE_NUNITS (<V_256:MODE>mode)
11409 == GET_MODE_NUNITS (<VI_256:MODE>mode))"
11411 bool ok = ix86_expand_int_vcond (operands);
11416 (define_expand "vcond<V_128:mode><VI124_128:mode>"
11417 [(set (match_operand:V_128 0 "register_operand")
11418 (if_then_else:V_128
11419 (match_operator 3 ""
11420 [(match_operand:VI124_128 4 "vector_operand")
11421 (match_operand:VI124_128 5 "general_operand")])
11422 (match_operand:V_128 1)
11423 (match_operand:V_128 2)))]
11425 && (GET_MODE_NUNITS (<V_128:MODE>mode)
11426 == GET_MODE_NUNITS (<VI124_128:MODE>mode))"
11428 bool ok = ix86_expand_int_vcond (operands);
11433 (define_expand "vcond<VI8F_128:mode>v2di"
11434 [(set (match_operand:VI8F_128 0 "register_operand")
11435 (if_then_else:VI8F_128
11436 (match_operator 3 ""
11437 [(match_operand:V2DI 4 "vector_operand")
11438 (match_operand:V2DI 5 "general_operand")])
11439 (match_operand:VI8F_128 1)
11440 (match_operand:VI8F_128 2)))]
11443 bool ok = ix86_expand_int_vcond (operands);
11448 (define_expand "vcondu<V_512:mode><VI_AVX512BW:mode>"
11449 [(set (match_operand:V_512 0 "register_operand")
11450 (if_then_else:V_512
11451 (match_operator 3 ""
11452 [(match_operand:VI_AVX512BW 4 "nonimmediate_operand")
11453 (match_operand:VI_AVX512BW 5 "nonimmediate_operand")])
11454 (match_operand:V_512 1 "general_operand")
11455 (match_operand:V_512 2 "general_operand")))]
11457 && (GET_MODE_NUNITS (<V_512:MODE>mode)
11458 == GET_MODE_NUNITS (<VI_AVX512BW:MODE>mode))"
11460 bool ok = ix86_expand_int_vcond (operands);
11465 (define_expand "vcondu<V_256:mode><VI_256:mode>"
11466 [(set (match_operand:V_256 0 "register_operand")
11467 (if_then_else:V_256
11468 (match_operator 3 ""
11469 [(match_operand:VI_256 4 "nonimmediate_operand")
11470 (match_operand:VI_256 5 "nonimmediate_operand")])
11471 (match_operand:V_256 1 "general_operand")
11472 (match_operand:V_256 2 "general_operand")))]
11474 && (GET_MODE_NUNITS (<V_256:MODE>mode)
11475 == GET_MODE_NUNITS (<VI_256:MODE>mode))"
11477 bool ok = ix86_expand_int_vcond (operands);
11482 (define_expand "vcondu<V_128:mode><VI124_128:mode>"
11483 [(set (match_operand:V_128 0 "register_operand")
11484 (if_then_else:V_128
11485 (match_operator 3 ""
11486 [(match_operand:VI124_128 4 "vector_operand")
11487 (match_operand:VI124_128 5 "vector_operand")])
11488 (match_operand:V_128 1 "general_operand")
11489 (match_operand:V_128 2 "general_operand")))]
11491 && (GET_MODE_NUNITS (<V_128:MODE>mode)
11492 == GET_MODE_NUNITS (<VI124_128:MODE>mode))"
11494 bool ok = ix86_expand_int_vcond (operands);
11499 (define_expand "vcondu<VI8F_128:mode>v2di"
11500 [(set (match_operand:VI8F_128 0 "register_operand")
11501 (if_then_else:VI8F_128
11502 (match_operator 3 ""
11503 [(match_operand:V2DI 4 "vector_operand")
11504 (match_operand:V2DI 5 "vector_operand")])
11505 (match_operand:VI8F_128 1 "general_operand")
11506 (match_operand:VI8F_128 2 "general_operand")))]
11509 bool ok = ix86_expand_int_vcond (operands);
11514 (define_expand "vcondeq<VI8F_128:mode>v2di"
11515 [(set (match_operand:VI8F_128 0 "register_operand")
11516 (if_then_else:VI8F_128
11517 (match_operator 3 ""
11518 [(match_operand:V2DI 4 "vector_operand")
11519 (match_operand:V2DI 5 "general_operand")])
11520 (match_operand:VI8F_128 1)
11521 (match_operand:VI8F_128 2)))]
11524 bool ok = ix86_expand_int_vcond (operands);
11529 (define_mode_iterator VEC_PERM_AVX2
11530 [V16QI V8HI V4SI V2DI V4SF V2DF
11531 (V32QI "TARGET_AVX2") (V16HI "TARGET_AVX2")
11532 (V8SI "TARGET_AVX2") (V4DI "TARGET_AVX2")
11533 (V8SF "TARGET_AVX2") (V4DF "TARGET_AVX2")
11534 (V16SF "TARGET_AVX512F") (V8DF "TARGET_AVX512F")
11535 (V16SI "TARGET_AVX512F") (V8DI "TARGET_AVX512F")
11536 (V32HI "TARGET_AVX512BW") (V64QI "TARGET_AVX512VBMI")])
11538 (define_expand "vec_perm<mode>"
11539 [(match_operand:VEC_PERM_AVX2 0 "register_operand")
11540 (match_operand:VEC_PERM_AVX2 1 "register_operand")
11541 (match_operand:VEC_PERM_AVX2 2 "register_operand")
11542 (match_operand:<sseintvecmode> 3 "register_operand")]
11543 "TARGET_SSSE3 || TARGET_AVX || TARGET_XOP"
11545 ix86_expand_vec_perm (operands);
11549 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
11551 ;; Parallel bitwise logical operations
11553 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
11555 (define_expand "one_cmpl<mode>2"
11556 [(set (match_operand:VI 0 "register_operand")
11557 (xor:VI (match_operand:VI 1 "vector_operand")
11561 operands[2] = force_reg (<MODE>mode, CONSTM1_RTX (<MODE>mode));
11564 (define_expand "<sse2_avx2>_andnot<mode>3"
11565 [(set (match_operand:VI_AVX2 0 "register_operand")
11567 (not:VI_AVX2 (match_operand:VI_AVX2 1 "register_operand"))
11568 (match_operand:VI_AVX2 2 "vector_operand")))]
11571 (define_expand "<sse2_avx2>_andnot<mode>3_mask"
11572 [(set (match_operand:VI48_AVX512VL 0 "register_operand")
11573 (vec_merge:VI48_AVX512VL
11576 (match_operand:VI48_AVX512VL 1 "register_operand"))
11577 (match_operand:VI48_AVX512VL 2 "nonimmediate_operand"))
11578 (match_operand:VI48_AVX512VL 3 "vector_move_operand")
11579 (match_operand:<avx512fmaskmode> 4 "register_operand")))]
11582 (define_expand "<sse2_avx2>_andnot<mode>3_mask"
11583 [(set (match_operand:VI12_AVX512VL 0 "register_operand")
11584 (vec_merge:VI12_AVX512VL
11587 (match_operand:VI12_AVX512VL 1 "register_operand"))
11588 (match_operand:VI12_AVX512VL 2 "nonimmediate_operand"))
11589 (match_operand:VI12_AVX512VL 3 "vector_move_operand")
11590 (match_operand:<avx512fmaskmode> 4 "register_operand")))]
11593 (define_insn "*andnot<mode>3"
11594 [(set (match_operand:VI 0 "register_operand" "=x,x,v")
11596 (not:VI (match_operand:VI 1 "register_operand" "0,x,v"))
11597 (match_operand:VI 2 "vector_operand" "xBm,xm,vm")))]
11600 static char buf[64];
11603 const char *ssesuffix;
11605 switch (get_attr_mode (insn))
11608 gcc_assert (TARGET_AVX512F);
11611 gcc_assert (TARGET_AVX2);
11614 gcc_assert (TARGET_SSE2);
11616 switch (<MODE>mode)
11620 /* There is no vpandnb or vpandnw instruction, nor vpandn for
11621 512-bit vectors. Use vpandnq instead. */
11626 ssesuffix = "<ssemodesuffix>";
11632 ssesuffix = (TARGET_AVX512VL && which_alternative == 2
11633 ? "<ssemodesuffix>" : "");
11636 ssesuffix = TARGET_AVX512VL && which_alternative == 2 ? "q" : "";
11641 gcc_assert (TARGET_AVX512F);
11644 gcc_assert (TARGET_AVX);
11647 gcc_assert (TARGET_SSE);
11653 gcc_unreachable ();
11656 switch (which_alternative)
11659 ops = "%s%s\t{%%2, %%0|%%0, %%2}";
11663 ops = "v%s%s\t{%%2, %%1, %%0|%%0, %%1, %%2}";
11666 gcc_unreachable ();
11669 snprintf (buf, sizeof (buf), ops, tmp, ssesuffix);
11672 [(set_attr "isa" "noavx,avx,avx")
11673 (set_attr "type" "sselog")
11674 (set (attr "prefix_data16")
11676 (and (eq_attr "alternative" "0")
11677 (eq_attr "mode" "TI"))
11679 (const_string "*")))
11680 (set_attr "prefix" "orig,vex,evex")
11682 (cond [(and (match_test "<MODE_SIZE> == 16")
11683 (match_test "TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL"))
11684 (const_string "<ssePSmode>")
11685 (match_test "TARGET_AVX2")
11686 (const_string "<sseinsnmode>")
11687 (match_test "TARGET_AVX")
11689 (match_test "<MODE_SIZE> > 16")
11690 (const_string "V8SF")
11691 (const_string "<sseinsnmode>"))
11692 (ior (not (match_test "TARGET_SSE2"))
11693 (match_test "optimize_function_for_size_p (cfun)"))
11694 (const_string "V4SF")
11696 (const_string "<sseinsnmode>")))])
11698 (define_insn "*andnot<mode>3_mask"
11699 [(set (match_operand:VI48_AVX512VL 0 "register_operand" "=v")
11700 (vec_merge:VI48_AVX512VL
11703 (match_operand:VI48_AVX512VL 1 "register_operand" "v"))
11704 (match_operand:VI48_AVX512VL 2 "nonimmediate_operand" "vm"))
11705 (match_operand:VI48_AVX512VL 3 "vector_move_operand" "0C")
11706 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk")))]
11708 "vpandn<ssemodesuffix>\t{%2, %1, %0%{%4%}%N3|%0%{%4%}%N3, %1, %2}";
11709 [(set_attr "type" "sselog")
11710 (set_attr "prefix" "evex")
11711 (set_attr "mode" "<sseinsnmode>")])
11713 (define_expand "<code><mode>3"
11714 [(set (match_operand:VI 0 "register_operand")
11716 (match_operand:VI 1 "nonimmediate_or_const_vector_operand")
11717 (match_operand:VI 2 "nonimmediate_or_const_vector_operand")))]
11720 ix86_expand_vector_logical_operator (<CODE>, <MODE>mode, operands);
11724 (define_insn "<mask_codefor><code><mode>3<mask_name>"
11725 [(set (match_operand:VI48_AVX_AVX512F 0 "register_operand" "=x,x,v")
11726 (any_logic:VI48_AVX_AVX512F
11727 (match_operand:VI48_AVX_AVX512F 1 "vector_operand" "%0,x,v")
11728 (match_operand:VI48_AVX_AVX512F 2 "vector_operand" "xBm,xm,vm")))]
11729 "TARGET_SSE && <mask_mode512bit_condition>
11730 && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
11732 static char buf[64];
11735 const char *ssesuffix;
11737 switch (get_attr_mode (insn))
11740 gcc_assert (TARGET_AVX512F);
11743 gcc_assert (TARGET_AVX2);
11746 gcc_assert (TARGET_SSE2);
11748 switch (<MODE>mode)
11752 ssesuffix = "<ssemodesuffix>";
11758 ssesuffix = (TARGET_AVX512VL
11759 && (<mask_applied> || which_alternative == 2)
11760 ? "<ssemodesuffix>" : "");
11763 gcc_unreachable ();
11768 gcc_assert (TARGET_AVX);
11771 gcc_assert (TARGET_SSE);
11777 gcc_unreachable ();
11780 switch (which_alternative)
11783 if (<mask_applied>)
11784 ops = "v%s%s\t{%%2, %%0, %%0<mask_operand3_1>|%%0<mask_operand3_1>, %%0, %%2}";
11786 ops = "%s%s\t{%%2, %%0|%%0, %%2}";
11790 ops = "v%s%s\t{%%2, %%1, %%0<mask_operand3_1>|%%0<mask_operand3_1>, %%1, %%2}";
11793 gcc_unreachable ();
11796 snprintf (buf, sizeof (buf), ops, tmp, ssesuffix);
11799 [(set_attr "isa" "noavx,avx,avx")
11800 (set_attr "type" "sselog")
11801 (set (attr "prefix_data16")
11803 (and (eq_attr "alternative" "0")
11804 (eq_attr "mode" "TI"))
11806 (const_string "*")))
11807 (set_attr "prefix" "<mask_prefix3>,evex")
11809 (cond [(and (match_test "<MODE_SIZE> == 16")
11810 (match_test "TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL"))
11811 (const_string "<ssePSmode>")
11812 (match_test "TARGET_AVX2")
11813 (const_string "<sseinsnmode>")
11814 (match_test "TARGET_AVX")
11816 (match_test "<MODE_SIZE> > 16")
11817 (const_string "V8SF")
11818 (const_string "<sseinsnmode>"))
11819 (ior (not (match_test "TARGET_SSE2"))
11820 (match_test "optimize_function_for_size_p (cfun)"))
11821 (const_string "V4SF")
11823 (const_string "<sseinsnmode>")))])
11825 (define_insn "*<code><mode>3"
11826 [(set (match_operand:VI12_AVX_AVX512F 0 "register_operand" "=x,x,v")
11827 (any_logic:VI12_AVX_AVX512F
11828 (match_operand:VI12_AVX_AVX512F 1 "vector_operand" "%0,x,v")
11829 (match_operand:VI12_AVX_AVX512F 2 "vector_operand" "xBm,xm,vm")))]
11830 "TARGET_SSE && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
11832 static char buf[64];
11835 const char *ssesuffix;
11837 switch (get_attr_mode (insn))
11840 gcc_assert (TARGET_AVX512F);
11843 gcc_assert (TARGET_AVX2);
11846 gcc_assert (TARGET_SSE2);
11848 switch (<MODE>mode)
11858 ssesuffix = TARGET_AVX512VL && which_alternative == 2 ? "q" : "";
11861 gcc_unreachable ();
11866 gcc_assert (TARGET_AVX);
11869 gcc_assert (TARGET_SSE);
11875 gcc_unreachable ();
11878 switch (which_alternative)
11881 ops = "%s%s\t{%%2, %%0|%%0, %%2}";
11885 ops = "v%s%s\t{%%2, %%1, %%0|%%0, %%1, %%2}";
11888 gcc_unreachable ();
11891 snprintf (buf, sizeof (buf), ops, tmp, ssesuffix);
11894 [(set_attr "isa" "noavx,avx,avx")
11895 (set_attr "type" "sselog")
11896 (set (attr "prefix_data16")
11898 (and (eq_attr "alternative" "0")
11899 (eq_attr "mode" "TI"))
11901 (const_string "*")))
11902 (set_attr "prefix" "orig,vex,evex")
11904 (cond [(and (match_test "<MODE_SIZE> == 16")
11905 (match_test "TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL"))
11906 (const_string "<ssePSmode>")
11907 (match_test "TARGET_AVX2")
11908 (const_string "<sseinsnmode>")
11909 (match_test "TARGET_AVX")
11911 (match_test "<MODE_SIZE> > 16")
11912 (const_string "V8SF")
11913 (const_string "<sseinsnmode>"))
11914 (ior (not (match_test "TARGET_SSE2"))
11915 (match_test "optimize_function_for_size_p (cfun)"))
11916 (const_string "V4SF")
11918 (const_string "<sseinsnmode>")))])
11920 (define_insn "<avx512>_testm<mode>3<mask_scalar_merge_name>"
11921 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=Yk")
11922 (unspec:<avx512fmaskmode>
11923 [(match_operand:VI12_AVX512VL 1 "register_operand" "v")
11924 (match_operand:VI12_AVX512VL 2 "nonimmediate_operand" "vm")]
11927 "vptestm<ssemodesuffix>\t{%2, %1, %0<mask_scalar_merge_operand3>|%0<mask_scalar_merge_operand3>, %1, %2}"
11928 [(set_attr "prefix" "evex")
11929 (set_attr "mode" "<sseinsnmode>")])
11931 (define_insn "<avx512>_testm<mode>3<mask_scalar_merge_name>"
11932 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=Yk")
11933 (unspec:<avx512fmaskmode>
11934 [(match_operand:VI48_AVX512VL 1 "register_operand" "v")
11935 (match_operand:VI48_AVX512VL 2 "nonimmediate_operand" "vm")]
11938 "vptestm<ssemodesuffix>\t{%2, %1, %0<mask_scalar_merge_operand3>|%0<mask_scalar_merge_operand3>, %1, %2}"
11939 [(set_attr "prefix" "evex")
11940 (set_attr "mode" "<sseinsnmode>")])
11942 (define_insn "<avx512>_testnm<mode>3<mask_scalar_merge_name>"
11943 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=Yk")
11944 (unspec:<avx512fmaskmode>
11945 [(match_operand:VI12_AVX512VL 1 "register_operand" "v")
11946 (match_operand:VI12_AVX512VL 2 "nonimmediate_operand" "vm")]
11949 "vptestnm<ssemodesuffix>\t{%2, %1, %0<mask_scalar_merge_operand3>|%0<mask_scalar_merge_operand3>, %1, %2}"
11950 [(set_attr "prefix" "evex")
11951 (set_attr "mode" "<sseinsnmode>")])
11953 (define_insn "<avx512>_testnm<mode>3<mask_scalar_merge_name>"
11954 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=Yk")
11955 (unspec:<avx512fmaskmode>
11956 [(match_operand:VI48_AVX512VL 1 "register_operand" "v")
11957 (match_operand:VI48_AVX512VL 2 "nonimmediate_operand" "vm")]
11960 "vptestnm<ssemodesuffix>\t{%2, %1, %0<mask_scalar_merge_operand3>|%0<mask_scalar_merge_operand3>, %1, %2}"
11961 [(set_attr "prefix" "evex")
11962 (set_attr "mode" "<sseinsnmode>")])
11964 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
11966 ;; Parallel integral element swizzling
11968 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
11970 (define_expand "vec_pack_trunc_<mode>"
11971 [(match_operand:<ssepackmode> 0 "register_operand")
11972 (match_operand:VI248_AVX2_8_AVX512F_24_AVX512BW 1 "register_operand")
11973 (match_operand:VI248_AVX2_8_AVX512F_24_AVX512BW 2 "register_operand")]
11976 rtx op1 = gen_lowpart (<ssepackmode>mode, operands[1]);
11977 rtx op2 = gen_lowpart (<ssepackmode>mode, operands[2]);
11978 ix86_expand_vec_extract_even_odd (operands[0], op1, op2, 0);
11982 (define_expand "vec_pack_trunc_qi"
11983 [(set (match_operand:HI 0 ("register_operand"))
11984 (ior:HI (ashift:HI (zero_extend:HI (match_operand:QI 2 ("register_operand")))
11986 (zero_extend:HI (match_operand:QI 1 ("register_operand")))))]
11989 (define_expand "vec_pack_trunc_<mode>"
11990 [(set (match_operand:<DOUBLEMASKMODE> 0 ("register_operand"))
11991 (ior:<DOUBLEMASKMODE> (ashift:<DOUBLEMASKMODE> (zero_extend:<DOUBLEMASKMODE> (match_operand:SWI24 2 ("register_operand")))
11993 (zero_extend:<DOUBLEMASKMODE> (match_operand:SWI24 1 ("register_operand")))))]
11996 operands[3] = GEN_INT (GET_MODE_BITSIZE (<MODE>mode));
11999 (define_insn "<sse2_avx2>_packsswb<mask_name>"
12000 [(set (match_operand:VI1_AVX512 0 "register_operand" "=x,x,v")
12001 (vec_concat:VI1_AVX512
12002 (ss_truncate:<ssehalfvecmode>
12003 (match_operand:<sseunpackmode> 1 "register_operand" "0,x,v"))
12004 (ss_truncate:<ssehalfvecmode>
12005 (match_operand:<sseunpackmode> 2 "vector_operand" "xBm,xm,vm"))))]
12006 "TARGET_SSE2 && <mask_mode512bit_condition> && <mask_avx512bw_condition>"
12008 packsswb\t{%2, %0|%0, %2}
12009 vpacksswb\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}
12010 vpacksswb\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
12011 [(set_attr "isa" "noavx,avx,avx512bw")
12012 (set_attr "type" "sselog")
12013 (set_attr "prefix_data16" "1,*,*")
12014 (set_attr "prefix" "orig,<mask_prefix>,evex")
12015 (set_attr "mode" "<sseinsnmode>")])
12017 (define_insn "<sse2_avx2>_packssdw<mask_name>"
12018 [(set (match_operand:VI2_AVX2 0 "register_operand" "=x,x,v")
12019 (vec_concat:VI2_AVX2
12020 (ss_truncate:<ssehalfvecmode>
12021 (match_operand:<sseunpackmode> 1 "register_operand" "0,x,v"))
12022 (ss_truncate:<ssehalfvecmode>
12023 (match_operand:<sseunpackmode> 2 "vector_operand" "xBm,xm,vm"))))]
12024 "TARGET_SSE2 && <mask_mode512bit_condition> && <mask_avx512bw_condition>"
12026 packssdw\t{%2, %0|%0, %2}
12027 vpackssdw\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}
12028 vpackssdw\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
12029 [(set_attr "isa" "noavx,avx,avx512bw")
12030 (set_attr "type" "sselog")
12031 (set_attr "prefix_data16" "1,*,*")
12032 (set_attr "prefix" "orig,<mask_prefix>,evex")
12033 (set_attr "mode" "<sseinsnmode>")])
12035 (define_insn "<sse2_avx2>_packuswb<mask_name>"
12036 [(set (match_operand:VI1_AVX512 0 "register_operand" "=x,x,v")
12037 (vec_concat:VI1_AVX512
12038 (us_truncate:<ssehalfvecmode>
12039 (match_operand:<sseunpackmode> 1 "register_operand" "0,x,v"))
12040 (us_truncate:<ssehalfvecmode>
12041 (match_operand:<sseunpackmode> 2 "vector_operand" "xBm,xm,vm"))))]
12042 "TARGET_SSE2 && <mask_mode512bit_condition> && <mask_avx512bw_condition>"
12044 packuswb\t{%2, %0|%0, %2}
12045 vpackuswb\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}
12046 vpackuswb\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
12047 [(set_attr "isa" "noavx,avx,avx512bw")
12048 (set_attr "type" "sselog")
12049 (set_attr "prefix_data16" "1,*,*")
12050 (set_attr "prefix" "orig,<mask_prefix>,evex")
12051 (set_attr "mode" "<sseinsnmode>")])
12053 (define_insn "avx512bw_interleave_highv64qi<mask_name>"
12054 [(set (match_operand:V64QI 0 "register_operand" "=v")
12057 (match_operand:V64QI 1 "register_operand" "v")
12058 (match_operand:V64QI 2 "nonimmediate_operand" "vm"))
12059 (parallel [(const_int 8) (const_int 72)
12060 (const_int 9) (const_int 73)
12061 (const_int 10) (const_int 74)
12062 (const_int 11) (const_int 75)
12063 (const_int 12) (const_int 76)
12064 (const_int 13) (const_int 77)
12065 (const_int 14) (const_int 78)
12066 (const_int 15) (const_int 79)
12067 (const_int 24) (const_int 88)
12068 (const_int 25) (const_int 89)
12069 (const_int 26) (const_int 90)
12070 (const_int 27) (const_int 91)
12071 (const_int 28) (const_int 92)
12072 (const_int 29) (const_int 93)
12073 (const_int 30) (const_int 94)
12074 (const_int 31) (const_int 95)
12075 (const_int 40) (const_int 104)
12076 (const_int 41) (const_int 105)
12077 (const_int 42) (const_int 106)
12078 (const_int 43) (const_int 107)
12079 (const_int 44) (const_int 108)
12080 (const_int 45) (const_int 109)
12081 (const_int 46) (const_int 110)
12082 (const_int 47) (const_int 111)
12083 (const_int 56) (const_int 120)
12084 (const_int 57) (const_int 121)
12085 (const_int 58) (const_int 122)
12086 (const_int 59) (const_int 123)
12087 (const_int 60) (const_int 124)
12088 (const_int 61) (const_int 125)
12089 (const_int 62) (const_int 126)
12090 (const_int 63) (const_int 127)])))]
12092 "vpunpckhbw\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
12093 [(set_attr "type" "sselog")
12094 (set_attr "prefix" "evex")
12095 (set_attr "mode" "XI")])
12097 (define_insn "avx2_interleave_highv32qi<mask_name>"
12098 [(set (match_operand:V32QI 0 "register_operand" "=v")
12101 (match_operand:V32QI 1 "register_operand" "v")
12102 (match_operand:V32QI 2 "nonimmediate_operand" "vm"))
12103 (parallel [(const_int 8) (const_int 40)
12104 (const_int 9) (const_int 41)
12105 (const_int 10) (const_int 42)
12106 (const_int 11) (const_int 43)
12107 (const_int 12) (const_int 44)
12108 (const_int 13) (const_int 45)
12109 (const_int 14) (const_int 46)
12110 (const_int 15) (const_int 47)
12111 (const_int 24) (const_int 56)
12112 (const_int 25) (const_int 57)
12113 (const_int 26) (const_int 58)
12114 (const_int 27) (const_int 59)
12115 (const_int 28) (const_int 60)
12116 (const_int 29) (const_int 61)
12117 (const_int 30) (const_int 62)
12118 (const_int 31) (const_int 63)])))]
12119 "TARGET_AVX2 && <mask_avx512vl_condition>"
12120 "vpunpckhbw\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
12121 [(set_attr "type" "sselog")
12122 (set_attr "prefix" "<mask_prefix>")
12123 (set_attr "mode" "OI")])
12125 (define_insn "vec_interleave_highv16qi<mask_name>"
12126 [(set (match_operand:V16QI 0 "register_operand" "=x,v")
12129 (match_operand:V16QI 1 "register_operand" "0,v")
12130 (match_operand:V16QI 2 "vector_operand" "xBm,vm"))
12131 (parallel [(const_int 8) (const_int 24)
12132 (const_int 9) (const_int 25)
12133 (const_int 10) (const_int 26)
12134 (const_int 11) (const_int 27)
12135 (const_int 12) (const_int 28)
12136 (const_int 13) (const_int 29)
12137 (const_int 14) (const_int 30)
12138 (const_int 15) (const_int 31)])))]
12139 "TARGET_SSE2 && <mask_avx512vl_condition>"
12141 punpckhbw\t{%2, %0|%0, %2}
12142 vpunpckhbw\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
12143 [(set_attr "isa" "noavx,avx")
12144 (set_attr "type" "sselog")
12145 (set_attr "prefix_data16" "1,*")
12146 (set_attr "prefix" "orig,<mask_prefix>")
12147 (set_attr "mode" "TI")])
12149 (define_insn "avx512bw_interleave_lowv64qi<mask_name>"
12150 [(set (match_operand:V64QI 0 "register_operand" "=v")
12153 (match_operand:V64QI 1 "register_operand" "v")
12154 (match_operand:V64QI 2 "nonimmediate_operand" "vm"))
12155 (parallel [(const_int 0) (const_int 64)
12156 (const_int 1) (const_int 65)
12157 (const_int 2) (const_int 66)
12158 (const_int 3) (const_int 67)
12159 (const_int 4) (const_int 68)
12160 (const_int 5) (const_int 69)
12161 (const_int 6) (const_int 70)
12162 (const_int 7) (const_int 71)
12163 (const_int 16) (const_int 80)
12164 (const_int 17) (const_int 81)
12165 (const_int 18) (const_int 82)
12166 (const_int 19) (const_int 83)
12167 (const_int 20) (const_int 84)
12168 (const_int 21) (const_int 85)
12169 (const_int 22) (const_int 86)
12170 (const_int 23) (const_int 87)
12171 (const_int 32) (const_int 96)
12172 (const_int 33) (const_int 97)
12173 (const_int 34) (const_int 98)
12174 (const_int 35) (const_int 99)
12175 (const_int 36) (const_int 100)
12176 (const_int 37) (const_int 101)
12177 (const_int 38) (const_int 102)
12178 (const_int 39) (const_int 103)
12179 (const_int 48) (const_int 112)
12180 (const_int 49) (const_int 113)
12181 (const_int 50) (const_int 114)
12182 (const_int 51) (const_int 115)
12183 (const_int 52) (const_int 116)
12184 (const_int 53) (const_int 117)
12185 (const_int 54) (const_int 118)
12186 (const_int 55) (const_int 119)])))]
12188 "vpunpcklbw\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
12189 [(set_attr "type" "sselog")
12190 (set_attr "prefix" "evex")
12191 (set_attr "mode" "XI")])
12193 (define_insn "avx2_interleave_lowv32qi<mask_name>"
12194 [(set (match_operand:V32QI 0 "register_operand" "=v")
12197 (match_operand:V32QI 1 "register_operand" "v")
12198 (match_operand:V32QI 2 "nonimmediate_operand" "vm"))
12199 (parallel [(const_int 0) (const_int 32)
12200 (const_int 1) (const_int 33)
12201 (const_int 2) (const_int 34)
12202 (const_int 3) (const_int 35)
12203 (const_int 4) (const_int 36)
12204 (const_int 5) (const_int 37)
12205 (const_int 6) (const_int 38)
12206 (const_int 7) (const_int 39)
12207 (const_int 16) (const_int 48)
12208 (const_int 17) (const_int 49)
12209 (const_int 18) (const_int 50)
12210 (const_int 19) (const_int 51)
12211 (const_int 20) (const_int 52)
12212 (const_int 21) (const_int 53)
12213 (const_int 22) (const_int 54)
12214 (const_int 23) (const_int 55)])))]
12215 "TARGET_AVX2 && <mask_avx512vl_condition> && <mask_avx512bw_condition>"
12216 "vpunpcklbw\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
12217 [(set_attr "type" "sselog")
12218 (set_attr "prefix" "maybe_vex")
12219 (set_attr "mode" "OI")])
12221 (define_insn "vec_interleave_lowv16qi<mask_name>"
12222 [(set (match_operand:V16QI 0 "register_operand" "=x,v")
12225 (match_operand:V16QI 1 "register_operand" "0,v")
12226 (match_operand:V16QI 2 "vector_operand" "xBm,vm"))
12227 (parallel [(const_int 0) (const_int 16)
12228 (const_int 1) (const_int 17)
12229 (const_int 2) (const_int 18)
12230 (const_int 3) (const_int 19)
12231 (const_int 4) (const_int 20)
12232 (const_int 5) (const_int 21)
12233 (const_int 6) (const_int 22)
12234 (const_int 7) (const_int 23)])))]
12235 "TARGET_SSE2 && <mask_avx512vl_condition> && <mask_avx512bw_condition>"
12237 punpcklbw\t{%2, %0|%0, %2}
12238 vpunpcklbw\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
12239 [(set_attr "isa" "noavx,avx")
12240 (set_attr "type" "sselog")
12241 (set_attr "prefix_data16" "1,*")
12242 (set_attr "prefix" "orig,vex")
12243 (set_attr "mode" "TI")])
12245 (define_insn "avx512bw_interleave_highv32hi<mask_name>"
12246 [(set (match_operand:V32HI 0 "register_operand" "=v")
12249 (match_operand:V32HI 1 "register_operand" "v")
12250 (match_operand:V32HI 2 "nonimmediate_operand" "vm"))
12251 (parallel [(const_int 4) (const_int 36)
12252 (const_int 5) (const_int 37)
12253 (const_int 6) (const_int 38)
12254 (const_int 7) (const_int 39)
12255 (const_int 12) (const_int 44)
12256 (const_int 13) (const_int 45)
12257 (const_int 14) (const_int 46)
12258 (const_int 15) (const_int 47)
12259 (const_int 20) (const_int 52)
12260 (const_int 21) (const_int 53)
12261 (const_int 22) (const_int 54)
12262 (const_int 23) (const_int 55)
12263 (const_int 28) (const_int 60)
12264 (const_int 29) (const_int 61)
12265 (const_int 30) (const_int 62)
12266 (const_int 31) (const_int 63)])))]
12268 "vpunpckhwd\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
12269 [(set_attr "type" "sselog")
12270 (set_attr "prefix" "evex")
12271 (set_attr "mode" "XI")])
12273 (define_insn "avx2_interleave_highv16hi<mask_name>"
12274 [(set (match_operand:V16HI 0 "register_operand" "=v")
12277 (match_operand:V16HI 1 "register_operand" "v")
12278 (match_operand:V16HI 2 "nonimmediate_operand" "vm"))
12279 (parallel [(const_int 4) (const_int 20)
12280 (const_int 5) (const_int 21)
12281 (const_int 6) (const_int 22)
12282 (const_int 7) (const_int 23)
12283 (const_int 12) (const_int 28)
12284 (const_int 13) (const_int 29)
12285 (const_int 14) (const_int 30)
12286 (const_int 15) (const_int 31)])))]
12287 "TARGET_AVX2 && <mask_avx512vl_condition> && <mask_avx512bw_condition>"
12288 "vpunpckhwd\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
12289 [(set_attr "type" "sselog")
12290 (set_attr "prefix" "maybe_evex")
12291 (set_attr "mode" "OI")])
12293 (define_insn "vec_interleave_highv8hi<mask_name>"
12294 [(set (match_operand:V8HI 0 "register_operand" "=x,v")
12297 (match_operand:V8HI 1 "register_operand" "0,v")
12298 (match_operand:V8HI 2 "vector_operand" "xBm,vm"))
12299 (parallel [(const_int 4) (const_int 12)
12300 (const_int 5) (const_int 13)
12301 (const_int 6) (const_int 14)
12302 (const_int 7) (const_int 15)])))]
12303 "TARGET_SSE2 && <mask_avx512vl_condition> && <mask_avx512bw_condition>"
12305 punpckhwd\t{%2, %0|%0, %2}
12306 vpunpckhwd\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
12307 [(set_attr "isa" "noavx,avx")
12308 (set_attr "type" "sselog")
12309 (set_attr "prefix_data16" "1,*")
12310 (set_attr "prefix" "orig,maybe_vex")
12311 (set_attr "mode" "TI")])
12313 (define_insn "<mask_codefor>avx512bw_interleave_lowv32hi<mask_name>"
12314 [(set (match_operand:V32HI 0 "register_operand" "=v")
12317 (match_operand:V32HI 1 "register_operand" "v")
12318 (match_operand:V32HI 2 "nonimmediate_operand" "vm"))
12319 (parallel [(const_int 0) (const_int 32)
12320 (const_int 1) (const_int 33)
12321 (const_int 2) (const_int 34)
12322 (const_int 3) (const_int 35)
12323 (const_int 8) (const_int 40)
12324 (const_int 9) (const_int 41)
12325 (const_int 10) (const_int 42)
12326 (const_int 11) (const_int 43)
12327 (const_int 16) (const_int 48)
12328 (const_int 17) (const_int 49)
12329 (const_int 18) (const_int 50)
12330 (const_int 19) (const_int 51)
12331 (const_int 24) (const_int 56)
12332 (const_int 25) (const_int 57)
12333 (const_int 26) (const_int 58)
12334 (const_int 27) (const_int 59)])))]
12336 "vpunpcklwd\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
12337 [(set_attr "type" "sselog")
12338 (set_attr "prefix" "evex")
12339 (set_attr "mode" "XI")])
12341 (define_insn "avx2_interleave_lowv16hi<mask_name>"
12342 [(set (match_operand:V16HI 0 "register_operand" "=v")
12345 (match_operand:V16HI 1 "register_operand" "v")
12346 (match_operand:V16HI 2 "nonimmediate_operand" "vm"))
12347 (parallel [(const_int 0) (const_int 16)
12348 (const_int 1) (const_int 17)
12349 (const_int 2) (const_int 18)
12350 (const_int 3) (const_int 19)
12351 (const_int 8) (const_int 24)
12352 (const_int 9) (const_int 25)
12353 (const_int 10) (const_int 26)
12354 (const_int 11) (const_int 27)])))]
12355 "TARGET_AVX2 && <mask_avx512vl_condition> && <mask_avx512bw_condition>"
12356 "vpunpcklwd\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
12357 [(set_attr "type" "sselog")
12358 (set_attr "prefix" "maybe_evex")
12359 (set_attr "mode" "OI")])
12361 (define_insn "vec_interleave_lowv8hi<mask_name>"
12362 [(set (match_operand:V8HI 0 "register_operand" "=x,v")
12365 (match_operand:V8HI 1 "register_operand" "0,v")
12366 (match_operand:V8HI 2 "vector_operand" "xBm,vm"))
12367 (parallel [(const_int 0) (const_int 8)
12368 (const_int 1) (const_int 9)
12369 (const_int 2) (const_int 10)
12370 (const_int 3) (const_int 11)])))]
12371 "TARGET_SSE2 && <mask_avx512vl_condition> && <mask_avx512bw_condition>"
12373 punpcklwd\t{%2, %0|%0, %2}
12374 vpunpcklwd\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
12375 [(set_attr "isa" "noavx,avx")
12376 (set_attr "type" "sselog")
12377 (set_attr "prefix_data16" "1,*")
12378 (set_attr "prefix" "orig,maybe_evex")
12379 (set_attr "mode" "TI")])
12381 (define_insn "avx2_interleave_highv8si<mask_name>"
12382 [(set (match_operand:V8SI 0 "register_operand" "=v")
12385 (match_operand:V8SI 1 "register_operand" "v")
12386 (match_operand:V8SI 2 "nonimmediate_operand" "vm"))
12387 (parallel [(const_int 2) (const_int 10)
12388 (const_int 3) (const_int 11)
12389 (const_int 6) (const_int 14)
12390 (const_int 7) (const_int 15)])))]
12391 "TARGET_AVX2 && <mask_avx512vl_condition>"
12392 "vpunpckhdq\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
12393 [(set_attr "type" "sselog")
12394 (set_attr "prefix" "maybe_evex")
12395 (set_attr "mode" "OI")])
12397 (define_insn "<mask_codefor>avx512f_interleave_highv16si<mask_name>"
12398 [(set (match_operand:V16SI 0 "register_operand" "=v")
12401 (match_operand:V16SI 1 "register_operand" "v")
12402 (match_operand:V16SI 2 "nonimmediate_operand" "vm"))
12403 (parallel [(const_int 2) (const_int 18)
12404 (const_int 3) (const_int 19)
12405 (const_int 6) (const_int 22)
12406 (const_int 7) (const_int 23)
12407 (const_int 10) (const_int 26)
12408 (const_int 11) (const_int 27)
12409 (const_int 14) (const_int 30)
12410 (const_int 15) (const_int 31)])))]
12412 "vpunpckhdq\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
12413 [(set_attr "type" "sselog")
12414 (set_attr "prefix" "evex")
12415 (set_attr "mode" "XI")])
12418 (define_insn "vec_interleave_highv4si<mask_name>"
12419 [(set (match_operand:V4SI 0 "register_operand" "=x,v")
12422 (match_operand:V4SI 1 "register_operand" "0,v")
12423 (match_operand:V4SI 2 "vector_operand" "xBm,vm"))
12424 (parallel [(const_int 2) (const_int 6)
12425 (const_int 3) (const_int 7)])))]
12426 "TARGET_SSE2 && <mask_avx512vl_condition>"
12428 punpckhdq\t{%2, %0|%0, %2}
12429 vpunpckhdq\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
12430 [(set_attr "isa" "noavx,avx")
12431 (set_attr "type" "sselog")
12432 (set_attr "prefix_data16" "1,*")
12433 (set_attr "prefix" "orig,maybe_vex")
12434 (set_attr "mode" "TI")])
12436 (define_insn "avx2_interleave_lowv8si<mask_name>"
12437 [(set (match_operand:V8SI 0 "register_operand" "=v")
12440 (match_operand:V8SI 1 "register_operand" "v")
12441 (match_operand:V8SI 2 "nonimmediate_operand" "vm"))
12442 (parallel [(const_int 0) (const_int 8)
12443 (const_int 1) (const_int 9)
12444 (const_int 4) (const_int 12)
12445 (const_int 5) (const_int 13)])))]
12446 "TARGET_AVX2 && <mask_avx512vl_condition>"
12447 "vpunpckldq\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
12448 [(set_attr "type" "sselog")
12449 (set_attr "prefix" "maybe_evex")
12450 (set_attr "mode" "OI")])
12452 (define_insn "<mask_codefor>avx512f_interleave_lowv16si<mask_name>"
12453 [(set (match_operand:V16SI 0 "register_operand" "=v")
12456 (match_operand:V16SI 1 "register_operand" "v")
12457 (match_operand:V16SI 2 "nonimmediate_operand" "vm"))
12458 (parallel [(const_int 0) (const_int 16)
12459 (const_int 1) (const_int 17)
12460 (const_int 4) (const_int 20)
12461 (const_int 5) (const_int 21)
12462 (const_int 8) (const_int 24)
12463 (const_int 9) (const_int 25)
12464 (const_int 12) (const_int 28)
12465 (const_int 13) (const_int 29)])))]
12467 "vpunpckldq\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
12468 [(set_attr "type" "sselog")
12469 (set_attr "prefix" "evex")
12470 (set_attr "mode" "XI")])
12472 (define_insn "vec_interleave_lowv4si<mask_name>"
12473 [(set (match_operand:V4SI 0 "register_operand" "=x,v")
12476 (match_operand:V4SI 1 "register_operand" "0,v")
12477 (match_operand:V4SI 2 "vector_operand" "xBm,vm"))
12478 (parallel [(const_int 0) (const_int 4)
12479 (const_int 1) (const_int 5)])))]
12480 "TARGET_SSE2 && <mask_avx512vl_condition>"
12482 punpckldq\t{%2, %0|%0, %2}
12483 vpunpckldq\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
12484 [(set_attr "isa" "noavx,avx")
12485 (set_attr "type" "sselog")
12486 (set_attr "prefix_data16" "1,*")
12487 (set_attr "prefix" "orig,vex")
12488 (set_attr "mode" "TI")])
12490 (define_expand "vec_interleave_high<mode>"
12491 [(match_operand:VI_256 0 "register_operand")
12492 (match_operand:VI_256 1 "register_operand")
12493 (match_operand:VI_256 2 "nonimmediate_operand")]
12496 rtx t1 = gen_reg_rtx (<MODE>mode);
12497 rtx t2 = gen_reg_rtx (<MODE>mode);
12498 rtx t3 = gen_reg_rtx (V4DImode);
12499 emit_insn (gen_avx2_interleave_low<mode> (t1, operands[1], operands[2]));
12500 emit_insn (gen_avx2_interleave_high<mode> (t2, operands[1], operands[2]));
12501 emit_insn (gen_avx2_permv2ti (t3, gen_lowpart (V4DImode, t1),
12502 gen_lowpart (V4DImode, t2),
12503 GEN_INT (1 + (3 << 4))));
12504 emit_move_insn (operands[0], gen_lowpart (<MODE>mode, t3));
12508 (define_expand "vec_interleave_low<mode>"
12509 [(match_operand:VI_256 0 "register_operand")
12510 (match_operand:VI_256 1 "register_operand")
12511 (match_operand:VI_256 2 "nonimmediate_operand")]
12514 rtx t1 = gen_reg_rtx (<MODE>mode);
12515 rtx t2 = gen_reg_rtx (<MODE>mode);
12516 rtx t3 = gen_reg_rtx (V4DImode);
12517 emit_insn (gen_avx2_interleave_low<mode> (t1, operands[1], operands[2]));
12518 emit_insn (gen_avx2_interleave_high<mode> (t2, operands[1], operands[2]));
12519 emit_insn (gen_avx2_permv2ti (t3, gen_lowpart (V4DImode, t1),
12520 gen_lowpart (V4DImode, t2),
12521 GEN_INT (0 + (2 << 4))));
12522 emit_move_insn (operands[0], gen_lowpart (<MODE>mode, t3));
12526 ;; Modes handled by pinsr patterns.
12527 (define_mode_iterator PINSR_MODE
12528 [(V16QI "TARGET_SSE4_1") V8HI
12529 (V4SI "TARGET_SSE4_1")
12530 (V2DI "TARGET_SSE4_1 && TARGET_64BIT")])
12532 (define_mode_attr sse2p4_1
12533 [(V16QI "sse4_1") (V8HI "sse2")
12534 (V4SI "sse4_1") (V2DI "sse4_1")])
12536 (define_mode_attr pinsr_evex_isa
12537 [(V16QI "avx512bw") (V8HI "avx512bw")
12538 (V4SI "avx512dq") (V2DI "avx512dq")])
12540 ;; sse4_1_pinsrd must come before sse2_loadld since it is preferred.
12541 (define_insn "<sse2p4_1>_pinsr<ssemodesuffix>"
12542 [(set (match_operand:PINSR_MODE 0 "register_operand" "=x,x,x,x,v,v")
12543 (vec_merge:PINSR_MODE
12544 (vec_duplicate:PINSR_MODE
12545 (match_operand:<ssescalarmode> 2 "nonimmediate_operand" "r,m,r,m,r,m"))
12546 (match_operand:PINSR_MODE 1 "register_operand" "0,0,x,x,v,v")
12547 (match_operand:SI 3 "const_int_operand")))]
12549 && ((unsigned) exact_log2 (INTVAL (operands[3]))
12550 < GET_MODE_NUNITS (<MODE>mode))"
12552 operands[3] = GEN_INT (exact_log2 (INTVAL (operands[3])));
12554 switch (which_alternative)
12557 if (GET_MODE_SIZE (<ssescalarmode>mode) < GET_MODE_SIZE (SImode))
12558 return "pinsr<ssemodesuffix>\t{%3, %k2, %0|%0, %k2, %3}";
12561 return "pinsr<ssemodesuffix>\t{%3, %2, %0|%0, %2, %3}";
12564 if (GET_MODE_SIZE (<ssescalarmode>mode) < GET_MODE_SIZE (SImode))
12565 return "vpinsr<ssemodesuffix>\t{%3, %k2, %1, %0|%0, %1, %k2, %3}";
12569 return "vpinsr<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}";
12571 gcc_unreachable ();
12574 [(set_attr "isa" "noavx,noavx,avx,avx,<pinsr_evex_isa>,<pinsr_evex_isa>")
12575 (set_attr "type" "sselog")
12576 (set (attr "prefix_rex")
12578 (and (not (match_test "TARGET_AVX"))
12579 (eq (const_string "<MODE>mode") (const_string "V2DImode")))
12581 (const_string "*")))
12582 (set (attr "prefix_data16")
12584 (and (not (match_test "TARGET_AVX"))
12585 (eq (const_string "<MODE>mode") (const_string "V8HImode")))
12587 (const_string "*")))
12588 (set (attr "prefix_extra")
12590 (and (not (match_test "TARGET_AVX"))
12591 (eq (const_string "<MODE>mode") (const_string "V8HImode")))
12593 (const_string "1")))
12594 (set_attr "length_immediate" "1")
12595 (set_attr "prefix" "orig,orig,vex,vex,evex,evex")
12596 (set_attr "mode" "TI")])
12598 (define_expand "<extract_type>_vinsert<shuffletype><extract_suf>_mask"
12599 [(match_operand:AVX512_VEC 0 "register_operand")
12600 (match_operand:AVX512_VEC 1 "register_operand")
12601 (match_operand:<ssequartermode> 2 "nonimmediate_operand")
12602 (match_operand:SI 3 "const_0_to_3_operand")
12603 (match_operand:AVX512_VEC 4 "register_operand")
12604 (match_operand:<avx512fmaskmode> 5 "register_operand")]
12608 mask = INTVAL (operands[3]);
12609 selector = GET_MODE_UNIT_SIZE (<MODE>mode) == 4 ?
12610 0xFFFF ^ (0xF000 >> mask * 4)
12611 : 0xFF ^ (0xC0 >> mask * 2);
12612 emit_insn (gen_<extract_type>_vinsert<shuffletype><extract_suf>_1_mask
12613 (operands[0], operands[1], operands[2], GEN_INT (selector),
12614 operands[4], operands[5]));
12618 (define_insn "<mask_codefor><extract_type>_vinsert<shuffletype><extract_suf>_1<mask_name>"
12619 [(set (match_operand:AVX512_VEC 0 "register_operand" "=v")
12620 (vec_merge:AVX512_VEC
12621 (match_operand:AVX512_VEC 1 "register_operand" "v")
12622 (vec_duplicate:AVX512_VEC
12623 (match_operand:<ssequartermode> 2 "nonimmediate_operand" "vm"))
12624 (match_operand:SI 3 "const_int_operand" "n")))]
12628 int selector = INTVAL (operands[3]);
12630 if (selector == 0xFFF || selector == 0x3F)
12632 else if ( selector == 0xF0FF || selector == 0xCF)
12634 else if ( selector == 0xFF0F || selector == 0xF3)
12636 else if ( selector == 0xFFF0 || selector == 0xFC)
12639 gcc_unreachable ();
12641 operands[3] = GEN_INT (mask);
12643 return "vinsert<shuffletype><extract_suf>\t{%3, %2, %1, %0<mask_operand4>|%0<mask_operand4>, %1, %2, %3}";
12645 [(set_attr "type" "sselog")
12646 (set_attr "length_immediate" "1")
12647 (set_attr "prefix" "evex")
12648 (set_attr "mode" "<sseinsnmode>")])
12650 (define_expand "<extract_type_2>_vinsert<shuffletype><extract_suf_2>_mask"
12651 [(match_operand:AVX512_VEC_2 0 "register_operand")
12652 (match_operand:AVX512_VEC_2 1 "register_operand")
12653 (match_operand:<ssehalfvecmode> 2 "nonimmediate_operand")
12654 (match_operand:SI 3 "const_0_to_1_operand")
12655 (match_operand:AVX512_VEC_2 4 "register_operand")
12656 (match_operand:<avx512fmaskmode> 5 "register_operand")]
12659 int mask = INTVAL (operands[3]);
12661 emit_insn (gen_vec_set_lo_<mode>_mask (operands[0], operands[1],
12662 operands[2], operands[4],
12665 emit_insn (gen_vec_set_hi_<mode>_mask (operands[0], operands[1],
12666 operands[2], operands[4],
12671 (define_insn "vec_set_lo_<mode><mask_name>"
12672 [(set (match_operand:V16FI 0 "register_operand" "=v")
12674 (match_operand:<ssehalfvecmode> 2 "nonimmediate_operand" "vm")
12675 (vec_select:<ssehalfvecmode>
12676 (match_operand:V16FI 1 "register_operand" "v")
12677 (parallel [(const_int 8) (const_int 9)
12678 (const_int 10) (const_int 11)
12679 (const_int 12) (const_int 13)
12680 (const_int 14) (const_int 15)]))))]
12682 "vinsert<shuffletype>32x8\t{$0x0, %2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2, 0x0}"
12683 [(set_attr "type" "sselog")
12684 (set_attr "length_immediate" "1")
12685 (set_attr "prefix" "evex")
12686 (set_attr "mode" "<sseinsnmode>")])
12688 (define_insn "vec_set_hi_<mode><mask_name>"
12689 [(set (match_operand:V16FI 0 "register_operand" "=v")
12691 (vec_select:<ssehalfvecmode>
12692 (match_operand:V16FI 1 "register_operand" "v")
12693 (parallel [(const_int 0) (const_int 1)
12694 (const_int 2) (const_int 3)
12695 (const_int 4) (const_int 5)
12696 (const_int 6) (const_int 7)]))
12697 (match_operand:<ssehalfvecmode> 2 "nonimmediate_operand" "vm")))]
12699 "vinsert<shuffletype>32x8\t{$0x1, %2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2, 0x1}"
12700 [(set_attr "type" "sselog")
12701 (set_attr "length_immediate" "1")
12702 (set_attr "prefix" "evex")
12703 (set_attr "mode" "<sseinsnmode>")])
12705 (define_insn "vec_set_lo_<mode><mask_name>"
12706 [(set (match_operand:V8FI 0 "register_operand" "=v")
12708 (match_operand:<ssehalfvecmode> 2 "nonimmediate_operand" "vm")
12709 (vec_select:<ssehalfvecmode>
12710 (match_operand:V8FI 1 "register_operand" "v")
12711 (parallel [(const_int 4) (const_int 5)
12712 (const_int 6) (const_int 7)]))))]
12714 "vinsert<shuffletype>64x4\t{$0x0, %2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2, 0x0}"
12715 [(set_attr "type" "sselog")
12716 (set_attr "length_immediate" "1")
12717 (set_attr "prefix" "evex")
12718 (set_attr "mode" "XI")])
12720 (define_insn "vec_set_hi_<mode><mask_name>"
12721 [(set (match_operand:V8FI 0 "register_operand" "=v")
12723 (vec_select:<ssehalfvecmode>
12724 (match_operand:V8FI 1 "register_operand" "v")
12725 (parallel [(const_int 0) (const_int 1)
12726 (const_int 2) (const_int 3)]))
12727 (match_operand:<ssehalfvecmode> 2 "nonimmediate_operand" "vm")))]
12729 "vinsert<shuffletype>64x4\t{$0x1, %2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2, 0x1}"
12730 [(set_attr "type" "sselog")
12731 (set_attr "length_immediate" "1")
12732 (set_attr "prefix" "evex")
12733 (set_attr "mode" "XI")])
12735 (define_expand "avx512dq_shuf_<shuffletype>64x2_mask"
12736 [(match_operand:VI8F_256 0 "register_operand")
12737 (match_operand:VI8F_256 1 "register_operand")
12738 (match_operand:VI8F_256 2 "nonimmediate_operand")
12739 (match_operand:SI 3 "const_0_to_3_operand")
12740 (match_operand:VI8F_256 4 "register_operand")
12741 (match_operand:QI 5 "register_operand")]
12744 int mask = INTVAL (operands[3]);
12745 emit_insn (gen_avx512dq_shuf_<shuffletype>64x2_1_mask
12746 (operands[0], operands[1], operands[2],
12747 GEN_INT (((mask >> 0) & 1) * 2 + 0),
12748 GEN_INT (((mask >> 0) & 1) * 2 + 1),
12749 GEN_INT (((mask >> 1) & 1) * 2 + 4),
12750 GEN_INT (((mask >> 1) & 1) * 2 + 5),
12751 operands[4], operands[5]));
12755 (define_insn "<mask_codefor>avx512dq_shuf_<shuffletype>64x2_1<mask_name>"
12756 [(set (match_operand:VI8F_256 0 "register_operand" "=v")
12757 (vec_select:VI8F_256
12758 (vec_concat:<ssedoublemode>
12759 (match_operand:VI8F_256 1 "register_operand" "v")
12760 (match_operand:VI8F_256 2 "nonimmediate_operand" "vm"))
12761 (parallel [(match_operand 3 "const_0_to_3_operand")
12762 (match_operand 4 "const_0_to_3_operand")
12763 (match_operand 5 "const_4_to_7_operand")
12764 (match_operand 6 "const_4_to_7_operand")])))]
12766 && (INTVAL (operands[3]) == (INTVAL (operands[4]) - 1)
12767 && INTVAL (operands[5]) == (INTVAL (operands[6]) - 1))"
12770 mask = INTVAL (operands[3]) / 2;
12771 mask |= (INTVAL (operands[5]) - 4) / 2 << 1;
12772 operands[3] = GEN_INT (mask);
12773 return "vshuf<shuffletype>64x2\t{%3, %2, %1, %0<mask_operand7>|%0<mask_operand7>, %1, %2, %3}";
12775 [(set_attr "type" "sselog")
12776 (set_attr "length_immediate" "1")
12777 (set_attr "prefix" "evex")
12778 (set_attr "mode" "XI")])
12780 (define_expand "avx512f_shuf_<shuffletype>64x2_mask"
12781 [(match_operand:V8FI 0 "register_operand")
12782 (match_operand:V8FI 1 "register_operand")
12783 (match_operand:V8FI 2 "nonimmediate_operand")
12784 (match_operand:SI 3 "const_0_to_255_operand")
12785 (match_operand:V8FI 4 "register_operand")
12786 (match_operand:QI 5 "register_operand")]
12789 int mask = INTVAL (operands[3]);
12790 emit_insn (gen_avx512f_shuf_<shuffletype>64x2_1_mask
12791 (operands[0], operands[1], operands[2],
12792 GEN_INT (((mask >> 0) & 3) * 2),
12793 GEN_INT (((mask >> 0) & 3) * 2 + 1),
12794 GEN_INT (((mask >> 2) & 3) * 2),
12795 GEN_INT (((mask >> 2) & 3) * 2 + 1),
12796 GEN_INT (((mask >> 4) & 3) * 2 + 8),
12797 GEN_INT (((mask >> 4) & 3) * 2 + 9),
12798 GEN_INT (((mask >> 6) & 3) * 2 + 8),
12799 GEN_INT (((mask >> 6) & 3) * 2 + 9),
12800 operands[4], operands[5]));
12804 (define_insn "avx512f_shuf_<shuffletype>64x2_1<mask_name>"
12805 [(set (match_operand:V8FI 0 "register_operand" "=v")
12807 (vec_concat:<ssedoublemode>
12808 (match_operand:V8FI 1 "register_operand" "v")
12809 (match_operand:V8FI 2 "nonimmediate_operand" "vm"))
12810 (parallel [(match_operand 3 "const_0_to_7_operand")
12811 (match_operand 4 "const_0_to_7_operand")
12812 (match_operand 5 "const_0_to_7_operand")
12813 (match_operand 6 "const_0_to_7_operand")
12814 (match_operand 7 "const_8_to_15_operand")
12815 (match_operand 8 "const_8_to_15_operand")
12816 (match_operand 9 "const_8_to_15_operand")
12817 (match_operand 10 "const_8_to_15_operand")])))]
12819 && (INTVAL (operands[3]) == (INTVAL (operands[4]) - 1)
12820 && INTVAL (operands[5]) == (INTVAL (operands[6]) - 1)
12821 && INTVAL (operands[7]) == (INTVAL (operands[8]) - 1)
12822 && INTVAL (operands[9]) == (INTVAL (operands[10]) - 1))"
12825 mask = INTVAL (operands[3]) / 2;
12826 mask |= INTVAL (operands[5]) / 2 << 2;
12827 mask |= (INTVAL (operands[7]) - 8) / 2 << 4;
12828 mask |= (INTVAL (operands[9]) - 8) / 2 << 6;
12829 operands[3] = GEN_INT (mask);
12831 return "vshuf<shuffletype>64x2\t{%3, %2, %1, %0<mask_operand11>|%0<mask_operand11>, %1, %2, %3}";
12833 [(set_attr "type" "sselog")
12834 (set_attr "length_immediate" "1")
12835 (set_attr "prefix" "evex")
12836 (set_attr "mode" "<sseinsnmode>")])
12838 (define_expand "avx512vl_shuf_<shuffletype>32x4_mask"
12839 [(match_operand:VI4F_256 0 "register_operand")
12840 (match_operand:VI4F_256 1 "register_operand")
12841 (match_operand:VI4F_256 2 "nonimmediate_operand")
12842 (match_operand:SI 3 "const_0_to_3_operand")
12843 (match_operand:VI4F_256 4 "register_operand")
12844 (match_operand:QI 5 "register_operand")]
12847 int mask = INTVAL (operands[3]);
12848 emit_insn (gen_avx512vl_shuf_<shuffletype>32x4_1_mask
12849 (operands[0], operands[1], operands[2],
12850 GEN_INT (((mask >> 0) & 1) * 4 + 0),
12851 GEN_INT (((mask >> 0) & 1) * 4 + 1),
12852 GEN_INT (((mask >> 0) & 1) * 4 + 2),
12853 GEN_INT (((mask >> 0) & 1) * 4 + 3),
12854 GEN_INT (((mask >> 1) & 1) * 4 + 8),
12855 GEN_INT (((mask >> 1) & 1) * 4 + 9),
12856 GEN_INT (((mask >> 1) & 1) * 4 + 10),
12857 GEN_INT (((mask >> 1) & 1) * 4 + 11),
12858 operands[4], operands[5]));
12862 (define_insn "avx512vl_shuf_<shuffletype>32x4_1<mask_name>"
12863 [(set (match_operand:VI4F_256 0 "register_operand" "=v")
12864 (vec_select:VI4F_256
12865 (vec_concat:<ssedoublemode>
12866 (match_operand:VI4F_256 1 "register_operand" "v")
12867 (match_operand:VI4F_256 2 "nonimmediate_operand" "vm"))
12868 (parallel [(match_operand 3 "const_0_to_7_operand")
12869 (match_operand 4 "const_0_to_7_operand")
12870 (match_operand 5 "const_0_to_7_operand")
12871 (match_operand 6 "const_0_to_7_operand")
12872 (match_operand 7 "const_8_to_15_operand")
12873 (match_operand 8 "const_8_to_15_operand")
12874 (match_operand 9 "const_8_to_15_operand")
12875 (match_operand 10 "const_8_to_15_operand")])))]
12877 && (INTVAL (operands[3]) == (INTVAL (operands[4]) - 1)
12878 && INTVAL (operands[3]) == (INTVAL (operands[5]) - 2)
12879 && INTVAL (operands[3]) == (INTVAL (operands[6]) - 3)
12880 && INTVAL (operands[7]) == (INTVAL (operands[8]) - 1)
12881 && INTVAL (operands[7]) == (INTVAL (operands[9]) - 2)
12882 && INTVAL (operands[7]) == (INTVAL (operands[10]) - 3))"
12885 mask = INTVAL (operands[3]) / 4;
12886 mask |= (INTVAL (operands[7]) - 8) / 4 << 1;
12887 operands[3] = GEN_INT (mask);
12889 return "vshuf<shuffletype>32x4\t{%3, %2, %1, %0<mask_operand11>|%0<mask_operand11>, %1, %2, %3}";
12891 [(set_attr "type" "sselog")
12892 (set_attr "length_immediate" "1")
12893 (set_attr "prefix" "evex")
12894 (set_attr "mode" "<sseinsnmode>")])
12896 (define_expand "avx512f_shuf_<shuffletype>32x4_mask"
12897 [(match_operand:V16FI 0 "register_operand")
12898 (match_operand:V16FI 1 "register_operand")
12899 (match_operand:V16FI 2 "nonimmediate_operand")
12900 (match_operand:SI 3 "const_0_to_255_operand")
12901 (match_operand:V16FI 4 "register_operand")
12902 (match_operand:HI 5 "register_operand")]
12905 int mask = INTVAL (operands[3]);
12906 emit_insn (gen_avx512f_shuf_<shuffletype>32x4_1_mask
12907 (operands[0], operands[1], operands[2],
12908 GEN_INT (((mask >> 0) & 3) * 4),
12909 GEN_INT (((mask >> 0) & 3) * 4 + 1),
12910 GEN_INT (((mask >> 0) & 3) * 4 + 2),
12911 GEN_INT (((mask >> 0) & 3) * 4 + 3),
12912 GEN_INT (((mask >> 2) & 3) * 4),
12913 GEN_INT (((mask >> 2) & 3) * 4 + 1),
12914 GEN_INT (((mask >> 2) & 3) * 4 + 2),
12915 GEN_INT (((mask >> 2) & 3) * 4 + 3),
12916 GEN_INT (((mask >> 4) & 3) * 4 + 16),
12917 GEN_INT (((mask >> 4) & 3) * 4 + 17),
12918 GEN_INT (((mask >> 4) & 3) * 4 + 18),
12919 GEN_INT (((mask >> 4) & 3) * 4 + 19),
12920 GEN_INT (((mask >> 6) & 3) * 4 + 16),
12921 GEN_INT (((mask >> 6) & 3) * 4 + 17),
12922 GEN_INT (((mask >> 6) & 3) * 4 + 18),
12923 GEN_INT (((mask >> 6) & 3) * 4 + 19),
12924 operands[4], operands[5]));
12928 (define_insn "avx512f_shuf_<shuffletype>32x4_1<mask_name>"
12929 [(set (match_operand:V16FI 0 "register_operand" "=v")
12931 (vec_concat:<ssedoublemode>
12932 (match_operand:V16FI 1 "register_operand" "v")
12933 (match_operand:V16FI 2 "nonimmediate_operand" "vm"))
12934 (parallel [(match_operand 3 "const_0_to_15_operand")
12935 (match_operand 4 "const_0_to_15_operand")
12936 (match_operand 5 "const_0_to_15_operand")
12937 (match_operand 6 "const_0_to_15_operand")
12938 (match_operand 7 "const_0_to_15_operand")
12939 (match_operand 8 "const_0_to_15_operand")
12940 (match_operand 9 "const_0_to_15_operand")
12941 (match_operand 10 "const_0_to_15_operand")
12942 (match_operand 11 "const_16_to_31_operand")
12943 (match_operand 12 "const_16_to_31_operand")
12944 (match_operand 13 "const_16_to_31_operand")
12945 (match_operand 14 "const_16_to_31_operand")
12946 (match_operand 15 "const_16_to_31_operand")
12947 (match_operand 16 "const_16_to_31_operand")
12948 (match_operand 17 "const_16_to_31_operand")
12949 (match_operand 18 "const_16_to_31_operand")])))]
12951 && (INTVAL (operands[3]) == (INTVAL (operands[4]) - 1)
12952 && INTVAL (operands[3]) == (INTVAL (operands[5]) - 2)
12953 && INTVAL (operands[3]) == (INTVAL (operands[6]) - 3)
12954 && INTVAL (operands[7]) == (INTVAL (operands[8]) - 1)
12955 && INTVAL (operands[7]) == (INTVAL (operands[9]) - 2)
12956 && INTVAL (operands[7]) == (INTVAL (operands[10]) - 3)
12957 && INTVAL (operands[11]) == (INTVAL (operands[12]) - 1)
12958 && INTVAL (operands[11]) == (INTVAL (operands[13]) - 2)
12959 && INTVAL (operands[11]) == (INTVAL (operands[14]) - 3)
12960 && INTVAL (operands[15]) == (INTVAL (operands[16]) - 1)
12961 && INTVAL (operands[15]) == (INTVAL (operands[17]) - 2)
12962 && INTVAL (operands[15]) == (INTVAL (operands[18]) - 3))"
12965 mask = INTVAL (operands[3]) / 4;
12966 mask |= INTVAL (operands[7]) / 4 << 2;
12967 mask |= (INTVAL (operands[11]) - 16) / 4 << 4;
12968 mask |= (INTVAL (operands[15]) - 16) / 4 << 6;
12969 operands[3] = GEN_INT (mask);
12971 return "vshuf<shuffletype>32x4\t{%3, %2, %1, %0<mask_operand19>|%0<mask_operand19>, %1, %2, %3}";
12973 [(set_attr "type" "sselog")
12974 (set_attr "length_immediate" "1")
12975 (set_attr "prefix" "evex")
12976 (set_attr "mode" "<sseinsnmode>")])
12978 (define_expand "avx512f_pshufdv3_mask"
12979 [(match_operand:V16SI 0 "register_operand")
12980 (match_operand:V16SI 1 "nonimmediate_operand")
12981 (match_operand:SI 2 "const_0_to_255_operand")
12982 (match_operand:V16SI 3 "register_operand")
12983 (match_operand:HI 4 "register_operand")]
12986 int mask = INTVAL (operands[2]);
12987 emit_insn (gen_avx512f_pshufd_1_mask (operands[0], operands[1],
12988 GEN_INT ((mask >> 0) & 3),
12989 GEN_INT ((mask >> 2) & 3),
12990 GEN_INT ((mask >> 4) & 3),
12991 GEN_INT ((mask >> 6) & 3),
12992 GEN_INT (((mask >> 0) & 3) + 4),
12993 GEN_INT (((mask >> 2) & 3) + 4),
12994 GEN_INT (((mask >> 4) & 3) + 4),
12995 GEN_INT (((mask >> 6) & 3) + 4),
12996 GEN_INT (((mask >> 0) & 3) + 8),
12997 GEN_INT (((mask >> 2) & 3) + 8),
12998 GEN_INT (((mask >> 4) & 3) + 8),
12999 GEN_INT (((mask >> 6) & 3) + 8),
13000 GEN_INT (((mask >> 0) & 3) + 12),
13001 GEN_INT (((mask >> 2) & 3) + 12),
13002 GEN_INT (((mask >> 4) & 3) + 12),
13003 GEN_INT (((mask >> 6) & 3) + 12),
13004 operands[3], operands[4]));
13008 (define_insn "avx512f_pshufd_1<mask_name>"
13009 [(set (match_operand:V16SI 0 "register_operand" "=v")
13011 (match_operand:V16SI 1 "nonimmediate_operand" "vm")
13012 (parallel [(match_operand 2 "const_0_to_3_operand")
13013 (match_operand 3 "const_0_to_3_operand")
13014 (match_operand 4 "const_0_to_3_operand")
13015 (match_operand 5 "const_0_to_3_operand")
13016 (match_operand 6 "const_4_to_7_operand")
13017 (match_operand 7 "const_4_to_7_operand")
13018 (match_operand 8 "const_4_to_7_operand")
13019 (match_operand 9 "const_4_to_7_operand")
13020 (match_operand 10 "const_8_to_11_operand")
13021 (match_operand 11 "const_8_to_11_operand")
13022 (match_operand 12 "const_8_to_11_operand")
13023 (match_operand 13 "const_8_to_11_operand")
13024 (match_operand 14 "const_12_to_15_operand")
13025 (match_operand 15 "const_12_to_15_operand")
13026 (match_operand 16 "const_12_to_15_operand")
13027 (match_operand 17 "const_12_to_15_operand")])))]
13029 && INTVAL (operands[2]) + 4 == INTVAL (operands[6])
13030 && INTVAL (operands[3]) + 4 == INTVAL (operands[7])
13031 && INTVAL (operands[4]) + 4 == INTVAL (operands[8])
13032 && INTVAL (operands[5]) + 4 == INTVAL (operands[9])
13033 && INTVAL (operands[2]) + 8 == INTVAL (operands[10])
13034 && INTVAL (operands[3]) + 8 == INTVAL (operands[11])
13035 && INTVAL (operands[4]) + 8 == INTVAL (operands[12])
13036 && INTVAL (operands[5]) + 8 == INTVAL (operands[13])
13037 && INTVAL (operands[2]) + 12 == INTVAL (operands[14])
13038 && INTVAL (operands[3]) + 12 == INTVAL (operands[15])
13039 && INTVAL (operands[4]) + 12 == INTVAL (operands[16])
13040 && INTVAL (operands[5]) + 12 == INTVAL (operands[17])"
13043 mask |= INTVAL (operands[2]) << 0;
13044 mask |= INTVAL (operands[3]) << 2;
13045 mask |= INTVAL (operands[4]) << 4;
13046 mask |= INTVAL (operands[5]) << 6;
13047 operands[2] = GEN_INT (mask);
13049 return "vpshufd\t{%2, %1, %0<mask_operand18>|%0<mask_operand18>, %1, %2}";
13051 [(set_attr "type" "sselog1")
13052 (set_attr "prefix" "evex")
13053 (set_attr "length_immediate" "1")
13054 (set_attr "mode" "XI")])
13056 (define_expand "avx512vl_pshufdv3_mask"
13057 [(match_operand:V8SI 0 "register_operand")
13058 (match_operand:V8SI 1 "nonimmediate_operand")
13059 (match_operand:SI 2 "const_0_to_255_operand")
13060 (match_operand:V8SI 3 "register_operand")
13061 (match_operand:QI 4 "register_operand")]
13064 int mask = INTVAL (operands[2]);
13065 emit_insn (gen_avx2_pshufd_1_mask (operands[0], operands[1],
13066 GEN_INT ((mask >> 0) & 3),
13067 GEN_INT ((mask >> 2) & 3),
13068 GEN_INT ((mask >> 4) & 3),
13069 GEN_INT ((mask >> 6) & 3),
13070 GEN_INT (((mask >> 0) & 3) + 4),
13071 GEN_INT (((mask >> 2) & 3) + 4),
13072 GEN_INT (((mask >> 4) & 3) + 4),
13073 GEN_INT (((mask >> 6) & 3) + 4),
13074 operands[3], operands[4]));
13078 (define_expand "avx2_pshufdv3"
13079 [(match_operand:V8SI 0 "register_operand")
13080 (match_operand:V8SI 1 "nonimmediate_operand")
13081 (match_operand:SI 2 "const_0_to_255_operand")]
13084 int mask = INTVAL (operands[2]);
13085 emit_insn (gen_avx2_pshufd_1 (operands[0], operands[1],
13086 GEN_INT ((mask >> 0) & 3),
13087 GEN_INT ((mask >> 2) & 3),
13088 GEN_INT ((mask >> 4) & 3),
13089 GEN_INT ((mask >> 6) & 3),
13090 GEN_INT (((mask >> 0) & 3) + 4),
13091 GEN_INT (((mask >> 2) & 3) + 4),
13092 GEN_INT (((mask >> 4) & 3) + 4),
13093 GEN_INT (((mask >> 6) & 3) + 4)));
13097 (define_insn "avx2_pshufd_1<mask_name>"
13098 [(set (match_operand:V8SI 0 "register_operand" "=v")
13100 (match_operand:V8SI 1 "nonimmediate_operand" "vm")
13101 (parallel [(match_operand 2 "const_0_to_3_operand")
13102 (match_operand 3 "const_0_to_3_operand")
13103 (match_operand 4 "const_0_to_3_operand")
13104 (match_operand 5 "const_0_to_3_operand")
13105 (match_operand 6 "const_4_to_7_operand")
13106 (match_operand 7 "const_4_to_7_operand")
13107 (match_operand 8 "const_4_to_7_operand")
13108 (match_operand 9 "const_4_to_7_operand")])))]
13110 && <mask_avx512vl_condition>
13111 && INTVAL (operands[2]) + 4 == INTVAL (operands[6])
13112 && INTVAL (operands[3]) + 4 == INTVAL (operands[7])
13113 && INTVAL (operands[4]) + 4 == INTVAL (operands[8])
13114 && INTVAL (operands[5]) + 4 == INTVAL (operands[9])"
13117 mask |= INTVAL (operands[2]) << 0;
13118 mask |= INTVAL (operands[3]) << 2;
13119 mask |= INTVAL (operands[4]) << 4;
13120 mask |= INTVAL (operands[5]) << 6;
13121 operands[2] = GEN_INT (mask);
13123 return "vpshufd\t{%2, %1, %0<mask_operand10>|%0<mask_operand10>, %1, %2}";
13125 [(set_attr "type" "sselog1")
13126 (set_attr "prefix" "maybe_evex")
13127 (set_attr "length_immediate" "1")
13128 (set_attr "mode" "OI")])
13130 (define_expand "avx512vl_pshufd_mask"
13131 [(match_operand:V4SI 0 "register_operand")
13132 (match_operand:V4SI 1 "nonimmediate_operand")
13133 (match_operand:SI 2 "const_0_to_255_operand")
13134 (match_operand:V4SI 3 "register_operand")
13135 (match_operand:QI 4 "register_operand")]
13138 int mask = INTVAL (operands[2]);
13139 emit_insn (gen_sse2_pshufd_1_mask (operands[0], operands[1],
13140 GEN_INT ((mask >> 0) & 3),
13141 GEN_INT ((mask >> 2) & 3),
13142 GEN_INT ((mask >> 4) & 3),
13143 GEN_INT ((mask >> 6) & 3),
13144 operands[3], operands[4]));
13148 (define_expand "sse2_pshufd"
13149 [(match_operand:V4SI 0 "register_operand")
13150 (match_operand:V4SI 1 "vector_operand")
13151 (match_operand:SI 2 "const_int_operand")]
13154 int mask = INTVAL (operands[2]);
13155 emit_insn (gen_sse2_pshufd_1 (operands[0], operands[1],
13156 GEN_INT ((mask >> 0) & 3),
13157 GEN_INT ((mask >> 2) & 3),
13158 GEN_INT ((mask >> 4) & 3),
13159 GEN_INT ((mask >> 6) & 3)));
13163 (define_insn "sse2_pshufd_1<mask_name>"
13164 [(set (match_operand:V4SI 0 "register_operand" "=v")
13166 (match_operand:V4SI 1 "vector_operand" "vBm")
13167 (parallel [(match_operand 2 "const_0_to_3_operand")
13168 (match_operand 3 "const_0_to_3_operand")
13169 (match_operand 4 "const_0_to_3_operand")
13170 (match_operand 5 "const_0_to_3_operand")])))]
13171 "TARGET_SSE2 && <mask_avx512vl_condition>"
13174 mask |= INTVAL (operands[2]) << 0;
13175 mask |= INTVAL (operands[3]) << 2;
13176 mask |= INTVAL (operands[4]) << 4;
13177 mask |= INTVAL (operands[5]) << 6;
13178 operands[2] = GEN_INT (mask);
13180 return "%vpshufd\t{%2, %1, %0<mask_operand6>|%0<mask_operand6>, %1, %2}";
13182 [(set_attr "type" "sselog1")
13183 (set_attr "prefix_data16" "1")
13184 (set_attr "prefix" "<mask_prefix2>")
13185 (set_attr "length_immediate" "1")
13186 (set_attr "mode" "TI")])
13188 (define_insn "<mask_codefor>avx512bw_pshuflwv32hi<mask_name>"
13189 [(set (match_operand:V32HI 0 "register_operand" "=v")
13191 [(match_operand:V32HI 1 "nonimmediate_operand" "vm")
13192 (match_operand:SI 2 "const_0_to_255_operand" "n")]
13195 "vpshuflw\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
13196 [(set_attr "type" "sselog")
13197 (set_attr "prefix" "evex")
13198 (set_attr "mode" "XI")])
13200 (define_expand "avx512vl_pshuflwv3_mask"
13201 [(match_operand:V16HI 0 "register_operand")
13202 (match_operand:V16HI 1 "nonimmediate_operand")
13203 (match_operand:SI 2 "const_0_to_255_operand")
13204 (match_operand:V16HI 3 "register_operand")
13205 (match_operand:HI 4 "register_operand")]
13206 "TARGET_AVX512VL && TARGET_AVX512BW"
13208 int mask = INTVAL (operands[2]);
13209 emit_insn (gen_avx2_pshuflw_1_mask (operands[0], operands[1],
13210 GEN_INT ((mask >> 0) & 3),
13211 GEN_INT ((mask >> 2) & 3),
13212 GEN_INT ((mask >> 4) & 3),
13213 GEN_INT ((mask >> 6) & 3),
13214 GEN_INT (((mask >> 0) & 3) + 8),
13215 GEN_INT (((mask >> 2) & 3) + 8),
13216 GEN_INT (((mask >> 4) & 3) + 8),
13217 GEN_INT (((mask >> 6) & 3) + 8),
13218 operands[3], operands[4]));
13222 (define_expand "avx2_pshuflwv3"
13223 [(match_operand:V16HI 0 "register_operand")
13224 (match_operand:V16HI 1 "nonimmediate_operand")
13225 (match_operand:SI 2 "const_0_to_255_operand")]
13228 int mask = INTVAL (operands[2]);
13229 emit_insn (gen_avx2_pshuflw_1 (operands[0], operands[1],
13230 GEN_INT ((mask >> 0) & 3),
13231 GEN_INT ((mask >> 2) & 3),
13232 GEN_INT ((mask >> 4) & 3),
13233 GEN_INT ((mask >> 6) & 3),
13234 GEN_INT (((mask >> 0) & 3) + 8),
13235 GEN_INT (((mask >> 2) & 3) + 8),
13236 GEN_INT (((mask >> 4) & 3) + 8),
13237 GEN_INT (((mask >> 6) & 3) + 8)));
13241 (define_insn "avx2_pshuflw_1<mask_name>"
13242 [(set (match_operand:V16HI 0 "register_operand" "=v")
13244 (match_operand:V16HI 1 "nonimmediate_operand" "vm")
13245 (parallel [(match_operand 2 "const_0_to_3_operand")
13246 (match_operand 3 "const_0_to_3_operand")
13247 (match_operand 4 "const_0_to_3_operand")
13248 (match_operand 5 "const_0_to_3_operand")
13253 (match_operand 6 "const_8_to_11_operand")
13254 (match_operand 7 "const_8_to_11_operand")
13255 (match_operand 8 "const_8_to_11_operand")
13256 (match_operand 9 "const_8_to_11_operand")
13260 (const_int 15)])))]
13262 && <mask_avx512bw_condition> && <mask_avx512vl_condition>
13263 && INTVAL (operands[2]) + 8 == INTVAL (operands[6])
13264 && INTVAL (operands[3]) + 8 == INTVAL (operands[7])
13265 && INTVAL (operands[4]) + 8 == INTVAL (operands[8])
13266 && INTVAL (operands[5]) + 8 == INTVAL (operands[9])"
13269 mask |= INTVAL (operands[2]) << 0;
13270 mask |= INTVAL (operands[3]) << 2;
13271 mask |= INTVAL (operands[4]) << 4;
13272 mask |= INTVAL (operands[5]) << 6;
13273 operands[2] = GEN_INT (mask);
13275 return "vpshuflw\t{%2, %1, %0<mask_operand10>|%0<mask_operand10>, %1, %2}";
13277 [(set_attr "type" "sselog")
13278 (set_attr "prefix" "maybe_evex")
13279 (set_attr "length_immediate" "1")
13280 (set_attr "mode" "OI")])
13282 (define_expand "avx512vl_pshuflw_mask"
13283 [(match_operand:V8HI 0 "register_operand")
13284 (match_operand:V8HI 1 "nonimmediate_operand")
13285 (match_operand:SI 2 "const_0_to_255_operand")
13286 (match_operand:V8HI 3 "register_operand")
13287 (match_operand:QI 4 "register_operand")]
13288 "TARGET_AVX512VL && TARGET_AVX512BW"
13290 int mask = INTVAL (operands[2]);
13291 emit_insn (gen_sse2_pshuflw_1_mask (operands[0], operands[1],
13292 GEN_INT ((mask >> 0) & 3),
13293 GEN_INT ((mask >> 2) & 3),
13294 GEN_INT ((mask >> 4) & 3),
13295 GEN_INT ((mask >> 6) & 3),
13296 operands[3], operands[4]));
13300 (define_expand "sse2_pshuflw"
13301 [(match_operand:V8HI 0 "register_operand")
13302 (match_operand:V8HI 1 "vector_operand")
13303 (match_operand:SI 2 "const_int_operand")]
13306 int mask = INTVAL (operands[2]);
13307 emit_insn (gen_sse2_pshuflw_1 (operands[0], operands[1],
13308 GEN_INT ((mask >> 0) & 3),
13309 GEN_INT ((mask >> 2) & 3),
13310 GEN_INT ((mask >> 4) & 3),
13311 GEN_INT ((mask >> 6) & 3)));
13315 (define_insn "sse2_pshuflw_1<mask_name>"
13316 [(set (match_operand:V8HI 0 "register_operand" "=v")
13318 (match_operand:V8HI 1 "vector_operand" "vBm")
13319 (parallel [(match_operand 2 "const_0_to_3_operand")
13320 (match_operand 3 "const_0_to_3_operand")
13321 (match_operand 4 "const_0_to_3_operand")
13322 (match_operand 5 "const_0_to_3_operand")
13327 "TARGET_SSE2 && <mask_avx512bw_condition> && <mask_avx512vl_condition>"
13330 mask |= INTVAL (operands[2]) << 0;
13331 mask |= INTVAL (operands[3]) << 2;
13332 mask |= INTVAL (operands[4]) << 4;
13333 mask |= INTVAL (operands[5]) << 6;
13334 operands[2] = GEN_INT (mask);
13336 return "%vpshuflw\t{%2, %1, %0<mask_operand6>|%0<mask_operand6>, %1, %2}";
13338 [(set_attr "type" "sselog")
13339 (set_attr "prefix_data16" "0")
13340 (set_attr "prefix_rep" "1")
13341 (set_attr "prefix" "maybe_vex")
13342 (set_attr "length_immediate" "1")
13343 (set_attr "mode" "TI")])
13345 (define_expand "avx2_pshufhwv3"
13346 [(match_operand:V16HI 0 "register_operand")
13347 (match_operand:V16HI 1 "nonimmediate_operand")
13348 (match_operand:SI 2 "const_0_to_255_operand")]
13351 int mask = INTVAL (operands[2]);
13352 emit_insn (gen_avx2_pshufhw_1 (operands[0], operands[1],
13353 GEN_INT (((mask >> 0) & 3) + 4),
13354 GEN_INT (((mask >> 2) & 3) + 4),
13355 GEN_INT (((mask >> 4) & 3) + 4),
13356 GEN_INT (((mask >> 6) & 3) + 4),
13357 GEN_INT (((mask >> 0) & 3) + 12),
13358 GEN_INT (((mask >> 2) & 3) + 12),
13359 GEN_INT (((mask >> 4) & 3) + 12),
13360 GEN_INT (((mask >> 6) & 3) + 12)));
13364 (define_insn "<mask_codefor>avx512bw_pshufhwv32hi<mask_name>"
13365 [(set (match_operand:V32HI 0 "register_operand" "=v")
13367 [(match_operand:V32HI 1 "nonimmediate_operand" "vm")
13368 (match_operand:SI 2 "const_0_to_255_operand" "n")]
13371 "vpshufhw\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
13372 [(set_attr "type" "sselog")
13373 (set_attr "prefix" "evex")
13374 (set_attr "mode" "XI")])
13376 (define_expand "avx512vl_pshufhwv3_mask"
13377 [(match_operand:V16HI 0 "register_operand")
13378 (match_operand:V16HI 1 "nonimmediate_operand")
13379 (match_operand:SI 2 "const_0_to_255_operand")
13380 (match_operand:V16HI 3 "register_operand")
13381 (match_operand:HI 4 "register_operand")]
13382 "TARGET_AVX512VL && TARGET_AVX512BW"
13384 int mask = INTVAL (operands[2]);
13385 emit_insn (gen_avx2_pshufhw_1_mask (operands[0], operands[1],
13386 GEN_INT (((mask >> 0) & 3) + 4),
13387 GEN_INT (((mask >> 2) & 3) + 4),
13388 GEN_INT (((mask >> 4) & 3) + 4),
13389 GEN_INT (((mask >> 6) & 3) + 4),
13390 GEN_INT (((mask >> 0) & 3) + 12),
13391 GEN_INT (((mask >> 2) & 3) + 12),
13392 GEN_INT (((mask >> 4) & 3) + 12),
13393 GEN_INT (((mask >> 6) & 3) + 12),
13394 operands[3], operands[4]));
13398 (define_insn "avx2_pshufhw_1<mask_name>"
13399 [(set (match_operand:V16HI 0 "register_operand" "=v")
13401 (match_operand:V16HI 1 "nonimmediate_operand" "vm")
13402 (parallel [(const_int 0)
13406 (match_operand 2 "const_4_to_7_operand")
13407 (match_operand 3 "const_4_to_7_operand")
13408 (match_operand 4 "const_4_to_7_operand")
13409 (match_operand 5 "const_4_to_7_operand")
13414 (match_operand 6 "const_12_to_15_operand")
13415 (match_operand 7 "const_12_to_15_operand")
13416 (match_operand 8 "const_12_to_15_operand")
13417 (match_operand 9 "const_12_to_15_operand")])))]
13419 && <mask_avx512bw_condition> && <mask_avx512vl_condition>
13420 && INTVAL (operands[2]) + 8 == INTVAL (operands[6])
13421 && INTVAL (operands[3]) + 8 == INTVAL (operands[7])
13422 && INTVAL (operands[4]) + 8 == INTVAL (operands[8])
13423 && INTVAL (operands[5]) + 8 == INTVAL (operands[9])"
13426 mask |= (INTVAL (operands[2]) - 4) << 0;
13427 mask |= (INTVAL (operands[3]) - 4) << 2;
13428 mask |= (INTVAL (operands[4]) - 4) << 4;
13429 mask |= (INTVAL (operands[5]) - 4) << 6;
13430 operands[2] = GEN_INT (mask);
13432 return "vpshufhw\t{%2, %1, %0<mask_operand10>|%0<mask_operand10>, %1, %2}";
13434 [(set_attr "type" "sselog")
13435 (set_attr "prefix" "maybe_evex")
13436 (set_attr "length_immediate" "1")
13437 (set_attr "mode" "OI")])
13439 (define_expand "avx512vl_pshufhw_mask"
13440 [(match_operand:V8HI 0 "register_operand")
13441 (match_operand:V8HI 1 "nonimmediate_operand")
13442 (match_operand:SI 2 "const_0_to_255_operand")
13443 (match_operand:V8HI 3 "register_operand")
13444 (match_operand:QI 4 "register_operand")]
13445 "TARGET_AVX512VL && TARGET_AVX512BW"
13447 int mask = INTVAL (operands[2]);
13448 emit_insn (gen_sse2_pshufhw_1_mask (operands[0], operands[1],
13449 GEN_INT (((mask >> 0) & 3) + 4),
13450 GEN_INT (((mask >> 2) & 3) + 4),
13451 GEN_INT (((mask >> 4) & 3) + 4),
13452 GEN_INT (((mask >> 6) & 3) + 4),
13453 operands[3], operands[4]));
13457 (define_expand "sse2_pshufhw"
13458 [(match_operand:V8HI 0 "register_operand")
13459 (match_operand:V8HI 1 "vector_operand")
13460 (match_operand:SI 2 "const_int_operand")]
13463 int mask = INTVAL (operands[2]);
13464 emit_insn (gen_sse2_pshufhw_1 (operands[0], operands[1],
13465 GEN_INT (((mask >> 0) & 3) + 4),
13466 GEN_INT (((mask >> 2) & 3) + 4),
13467 GEN_INT (((mask >> 4) & 3) + 4),
13468 GEN_INT (((mask >> 6) & 3) + 4)));
13472 (define_insn "sse2_pshufhw_1<mask_name>"
13473 [(set (match_operand:V8HI 0 "register_operand" "=v")
13475 (match_operand:V8HI 1 "vector_operand" "vBm")
13476 (parallel [(const_int 0)
13480 (match_operand 2 "const_4_to_7_operand")
13481 (match_operand 3 "const_4_to_7_operand")
13482 (match_operand 4 "const_4_to_7_operand")
13483 (match_operand 5 "const_4_to_7_operand")])))]
13484 "TARGET_SSE2 && <mask_avx512bw_condition> && <mask_avx512vl_condition>"
13487 mask |= (INTVAL (operands[2]) - 4) << 0;
13488 mask |= (INTVAL (operands[3]) - 4) << 2;
13489 mask |= (INTVAL (operands[4]) - 4) << 4;
13490 mask |= (INTVAL (operands[5]) - 4) << 6;
13491 operands[2] = GEN_INT (mask);
13493 return "%vpshufhw\t{%2, %1, %0<mask_operand6>|%0<mask_operand6>, %1, %2}";
13495 [(set_attr "type" "sselog")
13496 (set_attr "prefix_rep" "1")
13497 (set_attr "prefix_data16" "0")
13498 (set_attr "prefix" "maybe_vex")
13499 (set_attr "length_immediate" "1")
13500 (set_attr "mode" "TI")])
13502 (define_expand "sse2_loadd"
13503 [(set (match_operand:V4SI 0 "register_operand")
13505 (vec_duplicate:V4SI
13506 (match_operand:SI 1 "nonimmediate_operand"))
13510 "operands[2] = CONST0_RTX (V4SImode);")
13512 (define_insn "sse2_loadld"
13513 [(set (match_operand:V4SI 0 "register_operand" "=v,Yi,x,x,v")
13515 (vec_duplicate:V4SI
13516 (match_operand:SI 2 "nonimmediate_operand" "m ,r ,m,x,v"))
13517 (match_operand:V4SI 1 "reg_or_0_operand" "C ,C ,C,0,v")
13521 %vmovd\t{%2, %0|%0, %2}
13522 %vmovd\t{%2, %0|%0, %2}
13523 movss\t{%2, %0|%0, %2}
13524 movss\t{%2, %0|%0, %2}
13525 vmovss\t{%2, %1, %0|%0, %1, %2}"
13526 [(set_attr "isa" "sse2,sse2,noavx,noavx,avx")
13527 (set_attr "type" "ssemov")
13528 (set_attr "prefix" "maybe_vex,maybe_vex,orig,orig,maybe_evex")
13529 (set_attr "mode" "TI,TI,V4SF,SF,SF")])
13531 ;; QI and HI modes handled by pextr patterns.
13532 (define_mode_iterator PEXTR_MODE12
13533 [(V16QI "TARGET_SSE4_1") V8HI])
13535 (define_insn "*vec_extract<mode>"
13536 [(set (match_operand:<ssescalarmode> 0 "register_sse4nonimm_operand" "=r,m,r,m")
13537 (vec_select:<ssescalarmode>
13538 (match_operand:PEXTR_MODE12 1 "register_operand" "x,x,v,v")
13540 [(match_operand:SI 2 "const_0_to_<ssescalarnummask>_operand")])))]
13543 %vpextr<ssemodesuffix>\t{%2, %1, %k0|%k0, %1, %2}
13544 %vpextr<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}
13545 vpextr<ssemodesuffix>\t{%2, %1, %k0|%k0, %1, %2}
13546 vpextr<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
13547 [(set_attr "isa" "*,sse4,avx512bw,avx512bw")
13548 (set_attr "type" "sselog1")
13549 (set_attr "prefix_data16" "1")
13550 (set (attr "prefix_extra")
13552 (and (eq_attr "alternative" "0,2")
13553 (eq (const_string "<MODE>mode") (const_string "V8HImode")))
13555 (const_string "1")))
13556 (set_attr "length_immediate" "1")
13557 (set_attr "prefix" "maybe_vex,maybe_vex,evex,evex")
13558 (set_attr "mode" "TI")])
13560 (define_insn "*vec_extract<PEXTR_MODE12:mode>_zext"
13561 [(set (match_operand:SWI48 0 "register_operand" "=r,r")
13563 (vec_select:<PEXTR_MODE12:ssescalarmode>
13564 (match_operand:PEXTR_MODE12 1 "register_operand" "x,v")
13566 [(match_operand:SI 2
13567 "const_0_to_<PEXTR_MODE12:ssescalarnummask>_operand")]))))]
13570 %vpextr<PEXTR_MODE12:ssemodesuffix>\t{%2, %1, %k0|%k0, %1, %2}
13571 vpextr<PEXTR_MODE12:ssemodesuffix>\t{%2, %1, %k0|%k0, %1, %2}"
13572 [(set_attr "isa" "*,avx512bw")
13573 (set_attr "type" "sselog1")
13574 (set_attr "prefix_data16" "1")
13575 (set (attr "prefix_extra")
13577 (eq (const_string "<PEXTR_MODE12:MODE>mode") (const_string "V8HImode"))
13579 (const_string "1")))
13580 (set_attr "length_immediate" "1")
13581 (set_attr "prefix" "maybe_vex")
13582 (set_attr "mode" "TI")])
13584 (define_insn "*vec_extract<mode>_mem"
13585 [(set (match_operand:<ssescalarmode> 0 "register_operand" "=r")
13586 (vec_select:<ssescalarmode>
13587 (match_operand:VI12_128 1 "memory_operand" "o")
13589 [(match_operand 2 "const_0_to_<ssescalarnummask>_operand")])))]
13593 (define_insn "*vec_extract<ssevecmodelower>_0"
13594 [(set (match_operand:SWI48 0 "nonimmediate_operand" "=r ,v ,m")
13596 (match_operand:<ssevecmode> 1 "nonimmediate_operand" "mYj,vm,v")
13597 (parallel [(const_int 0)])))]
13598 "TARGET_SSE && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
13601 (define_insn "*vec_extractv2di_0_sse"
13602 [(set (match_operand:DI 0 "nonimmediate_operand" "=v,m")
13604 (match_operand:V2DI 1 "nonimmediate_operand" "vm,v")
13605 (parallel [(const_int 0)])))]
13606 "TARGET_SSE && !TARGET_64BIT
13607 && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
13611 [(set (match_operand:SWI48x 0 "nonimmediate_operand")
13613 (match_operand:<ssevecmode> 1 "register_operand")
13614 (parallel [(const_int 0)])))]
13615 "TARGET_SSE && reload_completed"
13616 [(set (match_dup 0) (match_dup 1))]
13617 "operands[1] = gen_lowpart (<MODE>mode, operands[1]);")
13619 (define_insn "*vec_extractv4si_0_zext_sse4"
13620 [(set (match_operand:DI 0 "register_operand" "=r,x,v")
13623 (match_operand:V4SI 1 "register_operand" "Yj,x,v")
13624 (parallel [(const_int 0)]))))]
13627 [(set_attr "isa" "x64,*,avx512f")])
13629 (define_insn "*vec_extractv4si_0_zext"
13630 [(set (match_operand:DI 0 "register_operand" "=r")
13633 (match_operand:V4SI 1 "register_operand" "x")
13634 (parallel [(const_int 0)]))))]
13635 "TARGET_64BIT && TARGET_SSE2 && TARGET_INTER_UNIT_MOVES_FROM_VEC"
13639 [(set (match_operand:DI 0 "register_operand")
13642 (match_operand:V4SI 1 "register_operand")
13643 (parallel [(const_int 0)]))))]
13644 "TARGET_SSE2 && reload_completed"
13645 [(set (match_dup 0) (zero_extend:DI (match_dup 1)))]
13646 "operands[1] = gen_lowpart (SImode, operands[1]);")
13648 (define_insn "*vec_extractv4si"
13649 [(set (match_operand:SI 0 "nonimmediate_operand" "=rm,rm,Yr,*x,x,Yv")
13651 (match_operand:V4SI 1 "register_operand" "x,v,0,0,x,v")
13652 (parallel [(match_operand:SI 2 "const_0_to_3_operand")])))]
13655 switch (which_alternative)
13659 return "%vpextrd\t{%2, %1, %0|%0, %1, %2}";
13663 operands[2] = GEN_INT (INTVAL (operands[2]) * 4);
13664 return "psrldq\t{%2, %0|%0, %2}";
13668 operands[2] = GEN_INT (INTVAL (operands[2]) * 4);
13669 return "vpsrldq\t{%2, %1, %0|%0, %1, %2}";
13672 gcc_unreachable ();
13675 [(set_attr "isa" "*,avx512dq,noavx,noavx,avx,avx512bw")
13676 (set_attr "type" "sselog1,sselog1,sseishft1,sseishft1,sseishft1,sseishft1")
13677 (set (attr "prefix_extra")
13678 (if_then_else (eq_attr "alternative" "0,1")
13680 (const_string "*")))
13681 (set_attr "length_immediate" "1")
13682 (set_attr "prefix" "maybe_vex,evex,orig,orig,vex,evex")
13683 (set_attr "mode" "TI")])
13685 (define_insn "*vec_extractv4si_zext"
13686 [(set (match_operand:DI 0 "register_operand" "=r,r")
13689 (match_operand:V4SI 1 "register_operand" "x,v")
13690 (parallel [(match_operand:SI 2 "const_0_to_3_operand")]))))]
13691 "TARGET_64BIT && TARGET_SSE4_1"
13692 "%vpextrd\t{%2, %1, %k0|%k0, %1, %2}"
13693 [(set_attr "isa" "*,avx512dq")
13694 (set_attr "type" "sselog1")
13695 (set_attr "prefix_extra" "1")
13696 (set_attr "length_immediate" "1")
13697 (set_attr "prefix" "maybe_vex")
13698 (set_attr "mode" "TI")])
13700 (define_insn "*vec_extractv4si_mem"
13701 [(set (match_operand:SI 0 "register_operand" "=x,r")
13703 (match_operand:V4SI 1 "memory_operand" "o,o")
13704 (parallel [(match_operand 2 "const_0_to_3_operand")])))]
13708 (define_insn_and_split "*vec_extractv4si_zext_mem"
13709 [(set (match_operand:DI 0 "register_operand" "=x,r")
13712 (match_operand:V4SI 1 "memory_operand" "o,o")
13713 (parallel [(match_operand:SI 2 "const_0_to_3_operand")]))))]
13714 "TARGET_64BIT && TARGET_SSE"
13716 "&& reload_completed"
13717 [(set (match_dup 0) (zero_extend:DI (match_dup 1)))]
13719 operands[1] = adjust_address (operands[1], SImode, INTVAL (operands[2]) * 4);
13722 (define_insn "*vec_extractv2di_1"
13723 [(set (match_operand:DI 0 "nonimmediate_operand" "=rm,rm,m,x,x,Yv,x,v,r")
13725 (match_operand:V2DI 1 "nonimmediate_operand" "x ,v ,v,0,x, v,x,o,o")
13726 (parallel [(const_int 1)])))]
13727 "TARGET_SSE && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
13729 %vpextrq\t{$1, %1, %0|%0, %1, 1}
13730 vpextrq\t{$1, %1, %0|%0, %1, 1}
13731 %vmovhps\t{%1, %0|%0, %1}
13732 psrldq\t{$8, %0|%0, 8}
13733 vpsrldq\t{$8, %1, %0|%0, %1, 8}
13734 vpsrldq\t{$8, %1, %0|%0, %1, 8}
13735 movhlps\t{%1, %0|%0, %1}
13739 (cond [(eq_attr "alternative" "0")
13740 (const_string "x64_sse4")
13741 (eq_attr "alternative" "1")
13742 (const_string "x64_avx512dq")
13743 (eq_attr "alternative" "3")
13744 (const_string "sse2_noavx")
13745 (eq_attr "alternative" "4")
13746 (const_string "avx")
13747 (eq_attr "alternative" "5")
13748 (const_string "avx512bw")
13749 (eq_attr "alternative" "6")
13750 (const_string "noavx")
13751 (eq_attr "alternative" "8")
13752 (const_string "x64")
13754 (const_string "*")))
13756 (cond [(eq_attr "alternative" "2,6,7")
13757 (const_string "ssemov")
13758 (eq_attr "alternative" "3,4,5")
13759 (const_string "sseishft1")
13760 (eq_attr "alternative" "8")
13761 (const_string "imov")
13763 (const_string "sselog1")))
13764 (set (attr "length_immediate")
13765 (if_then_else (eq_attr "alternative" "0,1,3,4,5")
13767 (const_string "*")))
13768 (set (attr "prefix_rex")
13769 (if_then_else (eq_attr "alternative" "0,1")
13771 (const_string "*")))
13772 (set (attr "prefix_extra")
13773 (if_then_else (eq_attr "alternative" "0,1")
13775 (const_string "*")))
13776 (set_attr "prefix" "maybe_vex,evex,maybe_vex,orig,vex,evex,orig,*,*")
13777 (set_attr "mode" "TI,TI,V2SF,TI,TI,TI,V4SF,DI,DI")])
13780 [(set (match_operand:<ssescalarmode> 0 "register_operand")
13781 (vec_select:<ssescalarmode>
13782 (match_operand:VI_128 1 "memory_operand")
13784 [(match_operand 2 "const_0_to_<ssescalarnummask>_operand")])))]
13785 "TARGET_SSE && reload_completed"
13786 [(set (match_dup 0) (match_dup 1))]
13788 int offs = INTVAL (operands[2]) * GET_MODE_SIZE (<ssescalarmode>mode);
13790 operands[1] = adjust_address (operands[1], <ssescalarmode>mode, offs);
13793 (define_insn "*vec_extractv2ti"
13794 [(set (match_operand:TI 0 "nonimmediate_operand" "=xm,vm")
13796 (match_operand:V2TI 1 "register_operand" "x,v")
13798 [(match_operand:SI 2 "const_0_to_1_operand")])))]
13801 vextract%~128\t{%2, %1, %0|%0, %1, %2}
13802 vextracti32x4\t{%2, %g1, %0|%0, %g1, %2}"
13803 [(set_attr "type" "sselog")
13804 (set_attr "prefix_extra" "1")
13805 (set_attr "length_immediate" "1")
13806 (set_attr "prefix" "vex,evex")
13807 (set_attr "mode" "OI")])
13809 (define_insn "*vec_extractv4ti"
13810 [(set (match_operand:TI 0 "nonimmediate_operand" "=vm")
13812 (match_operand:V4TI 1 "register_operand" "v")
13814 [(match_operand:SI 2 "const_0_to_3_operand")])))]
13816 "vextracti32x4\t{%2, %1, %0|%0, %1, %2}"
13817 [(set_attr "type" "sselog")
13818 (set_attr "prefix_extra" "1")
13819 (set_attr "length_immediate" "1")
13820 (set_attr "prefix" "evex")
13821 (set_attr "mode" "XI")])
13823 (define_mode_iterator VEXTRACTI128_MODE
13824 [(V4TI "TARGET_AVX512F") V2TI])
13827 [(set (match_operand:TI 0 "nonimmediate_operand")
13829 (match_operand:VEXTRACTI128_MODE 1 "register_operand")
13830 (parallel [(const_int 0)])))]
13832 && reload_completed
13833 && (TARGET_AVX512VL || !EXT_REX_SSE_REG_P (operands[1]))"
13834 [(set (match_dup 0) (match_dup 1))]
13835 "operands[1] = gen_lowpart (TImode, operands[1]);")
13837 ;; Turn SImode or DImode extraction from arbitrary SSE/AVX/AVX512F
13838 ;; vector modes into vec_extract*.
13840 [(set (match_operand:SWI48x 0 "nonimmediate_operand")
13841 (subreg:SWI48x (match_operand 1 "register_operand") 0))]
13842 "can_create_pseudo_p ()
13843 && REG_P (operands[1])
13844 && VECTOR_MODE_P (GET_MODE (operands[1]))
13845 && ((TARGET_SSE && GET_MODE_SIZE (GET_MODE (operands[1])) == 16)
13846 || (TARGET_AVX && GET_MODE_SIZE (GET_MODE (operands[1])) == 32)
13847 || (TARGET_AVX512F && GET_MODE_SIZE (GET_MODE (operands[1])) == 64))
13848 && (<MODE>mode == SImode || TARGET_64BIT || MEM_P (operands[0]))"
13849 [(set (match_dup 0) (vec_select:SWI48x (match_dup 1)
13850 (parallel [(const_int 0)])))]
13854 switch (GET_MODE_SIZE (GET_MODE (operands[1])))
13857 if (<MODE>mode == SImode)
13859 tmp = gen_reg_rtx (V8SImode);
13860 emit_insn (gen_vec_extract_lo_v16si (tmp,
13861 gen_lowpart (V16SImode,
13866 tmp = gen_reg_rtx (V4DImode);
13867 emit_insn (gen_vec_extract_lo_v8di (tmp,
13868 gen_lowpart (V8DImode,
13874 tmp = gen_reg_rtx (<ssevecmode>mode);
13875 if (<MODE>mode == SImode)
13876 emit_insn (gen_vec_extract_lo_v8si (tmp, gen_lowpart (V8SImode,
13879 emit_insn (gen_vec_extract_lo_v4di (tmp, gen_lowpart (V4DImode,
13884 operands[1] = gen_lowpart (<ssevecmode>mode, operands[1]);
13889 (define_insn "*vec_concatv2si_sse4_1"
13890 [(set (match_operand:V2SI 0 "register_operand"
13891 "=Yr,*x, x, v,Yr,*x, v, v, *y,*y")
13893 (match_operand:SI 1 "nonimmediate_operand"
13894 " 0, 0, x,Yv, 0, 0,Yv,rm, 0,rm")
13895 (match_operand:SI 2 "vector_move_operand"
13896 " rm,rm,rm,rm,Yr,*x,Yv, C,*ym, C")))]
13897 "TARGET_SSE4_1 && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
13899 pinsrd\t{$1, %2, %0|%0, %2, 1}
13900 pinsrd\t{$1, %2, %0|%0, %2, 1}
13901 vpinsrd\t{$1, %2, %1, %0|%0, %1, %2, 1}
13902 vpinsrd\t{$1, %2, %1, %0|%0, %1, %2, 1}
13903 punpckldq\t{%2, %0|%0, %2}
13904 punpckldq\t{%2, %0|%0, %2}
13905 vpunpckldq\t{%2, %1, %0|%0, %1, %2}
13906 %vmovd\t{%1, %0|%0, %1}
13907 punpckldq\t{%2, %0|%0, %2}
13908 movd\t{%1, %0|%0, %1}"
13909 [(set_attr "isa" "noavx,noavx,avx,avx512dq,noavx,noavx,avx,*,*,*")
13911 (cond [(eq_attr "alternative" "7")
13912 (const_string "ssemov")
13913 (eq_attr "alternative" "8")
13914 (const_string "mmxcvt")
13915 (eq_attr "alternative" "9")
13916 (const_string "mmxmov")
13918 (const_string "sselog")))
13919 (set (attr "prefix_extra")
13920 (if_then_else (eq_attr "alternative" "0,1,2,3")
13922 (const_string "*")))
13923 (set (attr "length_immediate")
13924 (if_then_else (eq_attr "alternative" "0,1,2,3")
13926 (const_string "*")))
13927 (set_attr "prefix" "orig,orig,vex,evex,orig,orig,maybe_evex,maybe_vex,orig,orig")
13928 (set_attr "mode" "TI,TI,TI,TI,TI,TI,TI,TI,DI,DI")])
13930 ;; ??? In theory we can match memory for the MMX alternative, but allowing
13931 ;; nonimmediate_operand for operand 2 and *not* allowing memory for the SSE
13932 ;; alternatives pretty much forces the MMX alternative to be chosen.
13933 (define_insn "*vec_concatv2si"
13934 [(set (match_operand:V2SI 0 "register_operand" "=x,x ,*y,x,x,*y,*y")
13936 (match_operand:SI 1 "nonimmediate_operand" " 0,rm,rm,0,m, 0,*rm")
13937 (match_operand:SI 2 "reg_or_0_operand" " x,C ,C, x,C,*y,C")))]
13938 "TARGET_SSE && !TARGET_SSE4_1"
13940 punpckldq\t{%2, %0|%0, %2}
13941 movd\t{%1, %0|%0, %1}
13942 movd\t{%1, %0|%0, %1}
13943 unpcklps\t{%2, %0|%0, %2}
13944 movss\t{%1, %0|%0, %1}
13945 punpckldq\t{%2, %0|%0, %2}
13946 movd\t{%1, %0|%0, %1}"
13947 [(set_attr "isa" "sse2,sse2,sse2,*,*,*,*")
13948 (set_attr "type" "sselog,ssemov,mmxmov,sselog,ssemov,mmxcvt,mmxmov")
13949 (set_attr "mode" "TI,TI,DI,V4SF,SF,DI,DI")])
13951 (define_insn "*vec_concatv4si"
13952 [(set (match_operand:V4SI 0 "register_operand" "=x,v,x,x,v")
13954 (match_operand:V2SI 1 "register_operand" " 0,v,0,0,v")
13955 (match_operand:V2SI 2 "nonimmediate_operand" " x,v,x,m,m")))]
13958 punpcklqdq\t{%2, %0|%0, %2}
13959 vpunpcklqdq\t{%2, %1, %0|%0, %1, %2}
13960 movlhps\t{%2, %0|%0, %2}
13961 movhps\t{%2, %0|%0, %q2}
13962 vmovhps\t{%2, %1, %0|%0, %1, %q2}"
13963 [(set_attr "isa" "sse2_noavx,avx,noavx,noavx,avx")
13964 (set_attr "type" "sselog,sselog,ssemov,ssemov,ssemov")
13965 (set_attr "prefix" "orig,maybe_evex,orig,orig,maybe_evex")
13966 (set_attr "mode" "TI,TI,V4SF,V2SF,V2SF")])
13968 ;; movd instead of movq is required to handle broken assemblers.
13969 (define_insn "vec_concatv2di"
13970 [(set (match_operand:V2DI 0 "register_operand"
13971 "=Yr,*x,x ,v ,Yi,v ,x ,x,v ,x,x,v")
13973 (match_operand:DI 1 "nonimmediate_operand"
13974 " 0, 0,x ,Yv,r ,vm,?!*Yn,0,Yv,0,0,v")
13975 (match_operand:DI 2 "vector_move_operand"
13976 " rm,rm,rm,rm,C ,C ,C ,x,Yv,x,m,m")))]
13979 pinsrq\t{$1, %2, %0|%0, %2, 1}
13980 pinsrq\t{$1, %2, %0|%0, %2, 1}
13981 vpinsrq\t{$1, %2, %1, %0|%0, %1, %2, 1}
13982 vpinsrq\t{$1, %2, %1, %0|%0, %1, %2, 1}
13983 * return HAVE_AS_IX86_INTERUNIT_MOVQ ? \"%vmovq\t{%1, %0|%0, %1}\" : \"%vmovd\t{%1, %0|%0, %1}\";
13984 %vmovq\t{%1, %0|%0, %1}
13985 movq2dq\t{%1, %0|%0, %1}
13986 punpcklqdq\t{%2, %0|%0, %2}
13987 vpunpcklqdq\t{%2, %1, %0|%0, %1, %2}
13988 movlhps\t{%2, %0|%0, %2}
13989 movhps\t{%2, %0|%0, %2}
13990 vmovhps\t{%2, %1, %0|%0, %1, %2}"
13992 (cond [(eq_attr "alternative" "0,1")
13993 (const_string "x64_sse4_noavx")
13994 (eq_attr "alternative" "2")
13995 (const_string "x64_avx")
13996 (eq_attr "alternative" "3")
13997 (const_string "x64_avx512dq")
13998 (eq_attr "alternative" "4")
13999 (const_string "x64")
14000 (eq_attr "alternative" "5,6")
14001 (const_string "sse2")
14002 (eq_attr "alternative" "7")
14003 (const_string "sse2_noavx")
14004 (eq_attr "alternative" "8,11")
14005 (const_string "avx")
14007 (const_string "noavx")))
14010 (eq_attr "alternative" "0,1,2,3,7,8")
14011 (const_string "sselog")
14012 (const_string "ssemov")))
14013 (set (attr "prefix_rex")
14014 (if_then_else (eq_attr "alternative" "0,1,2,3,4")
14016 (const_string "*")))
14017 (set (attr "prefix_extra")
14018 (if_then_else (eq_attr "alternative" "0,1,2,3")
14020 (const_string "*")))
14021 (set (attr "length_immediate")
14022 (if_then_else (eq_attr "alternative" "0,1,2,3")
14024 (const_string "*")))
14025 (set (attr "prefix")
14026 (cond [(eq_attr "alternative" "2")
14027 (const_string "vex")
14028 (eq_attr "alternative" "3")
14029 (const_string "evex")
14030 (eq_attr "alternative" "4,5")
14031 (const_string "maybe_vex")
14032 (eq_attr "alternative" "8,11")
14033 (const_string "maybe_evex")
14035 (const_string "orig")))
14036 (set_attr "mode" "TI,TI,TI,TI,TI,TI,TI,TI,TI,V4SF,V2SF,V2SF")])
14038 ;; vmovq clears also the higher bits.
14039 (define_insn "vec_set<mode>_0"
14040 [(set (match_operand:VI8_AVX_AVX512F 0 "register_operand" "=Yi,v")
14041 (vec_merge:VI8_AVX_AVX512F
14042 (vec_duplicate:VI8_AVX_AVX512F
14043 (match_operand:<ssescalarmode> 2 "general_operand" "r,vm"))
14044 (match_operand:VI8_AVX_AVX512F 1 "const0_operand" "C,C")
14047 "vmovq\t{%2, %x0|%x0, %2}"
14048 [(set_attr "isa" "x64,*")
14049 (set_attr "type" "ssemov")
14050 (set_attr "prefix_rex" "1,*")
14051 (set_attr "prefix" "maybe_evex")
14052 (set_attr "mode" "TI")])
14054 (define_expand "vec_unpacks_lo_<mode>"
14055 [(match_operand:<sseunpackmode> 0 "register_operand")
14056 (match_operand:VI124_AVX2_24_AVX512F_1_AVX512BW 1 "register_operand")]
14058 "ix86_expand_sse_unpack (operands[0], operands[1], false, false); DONE;")
14060 (define_expand "vec_unpacks_hi_<mode>"
14061 [(match_operand:<sseunpackmode> 0 "register_operand")
14062 (match_operand:VI124_AVX2_24_AVX512F_1_AVX512BW 1 "register_operand")]
14064 "ix86_expand_sse_unpack (operands[0], operands[1], false, true); DONE;")
14066 (define_expand "vec_unpacku_lo_<mode>"
14067 [(match_operand:<sseunpackmode> 0 "register_operand")
14068 (match_operand:VI124_AVX2_24_AVX512F_1_AVX512BW 1 "register_operand")]
14070 "ix86_expand_sse_unpack (operands[0], operands[1], true, false); DONE;")
14072 (define_expand "vec_unpacks_lo_hi"
14073 [(set (subreg:HI (match_operand:QI 0 "register_operand") 0)
14074 (match_operand:HI 1 "register_operand"))]
14077 (define_expand "vec_unpacks_lo_si"
14078 [(set (match_operand:HI 0 "register_operand")
14079 (subreg:HI (match_operand:SI 1 "register_operand") 0))]
14082 (define_expand "vec_unpacks_lo_di"
14083 [(set (match_operand:SI 0 "register_operand")
14084 (subreg:SI (match_operand:DI 1 "register_operand") 0))]
14087 (define_expand "vec_unpacku_hi_<mode>"
14088 [(match_operand:<sseunpackmode> 0 "register_operand")
14089 (match_operand:VI124_AVX2_24_AVX512F_1_AVX512BW 1 "register_operand")]
14091 "ix86_expand_sse_unpack (operands[0], operands[1], true, true); DONE;")
14093 (define_expand "vec_unpacks_hi_hi"
14095 [(set (subreg:HI (match_operand:QI 0 "register_operand") 0)
14096 (lshiftrt:HI (match_operand:HI 1 "register_operand")
14098 (unspec [(const_int 0)] UNSPEC_MASKOP)])]
14101 (define_expand "vec_unpacks_hi_<mode>"
14103 [(set (subreg:SWI48x
14104 (match_operand:<HALFMASKMODE> 0 "register_operand") 0)
14105 (lshiftrt:SWI48x (match_operand:SWI48x 1 "register_operand")
14107 (unspec [(const_int 0)] UNSPEC_MASKOP)])]
14109 "operands[2] = GEN_INT (GET_MODE_BITSIZE (<HALFMASKMODE>mode));")
14111 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
14115 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
14117 (define_expand "<sse2_avx2>_uavg<mode>3<mask_name>"
14118 [(set (match_operand:VI12_AVX2 0 "register_operand")
14119 (truncate:VI12_AVX2
14120 (lshiftrt:<ssedoublemode>
14121 (plus:<ssedoublemode>
14122 (plus:<ssedoublemode>
14123 (zero_extend:<ssedoublemode>
14124 (match_operand:VI12_AVX2 1 "vector_operand"))
14125 (zero_extend:<ssedoublemode>
14126 (match_operand:VI12_AVX2 2 "vector_operand")))
14127 (match_dup <mask_expand_op3>))
14129 "TARGET_SSE2 && <mask_mode512bit_condition> && <mask_avx512bw_condition>"
14132 if (<mask_applied>)
14134 operands[3] = CONST1_RTX(<MODE>mode);
14135 ix86_fixup_binary_operands_no_copy (PLUS, <MODE>mode, operands);
14137 if (<mask_applied>)
14139 operands[5] = operands[3];
14144 (define_insn "*<sse2_avx2>_uavg<mode>3<mask_name>"
14145 [(set (match_operand:VI12_AVX2 0 "register_operand" "=x,v")
14146 (truncate:VI12_AVX2
14147 (lshiftrt:<ssedoublemode>
14148 (plus:<ssedoublemode>
14149 (plus:<ssedoublemode>
14150 (zero_extend:<ssedoublemode>
14151 (match_operand:VI12_AVX2 1 "vector_operand" "%0,v"))
14152 (zero_extend:<ssedoublemode>
14153 (match_operand:VI12_AVX2 2 "vector_operand" "xBm,vm")))
14154 (match_operand:VI12_AVX2 <mask_expand_op3> "const1_operand"))
14156 "TARGET_SSE2 && <mask_mode512bit_condition> && <mask_avx512bw_condition>
14157 && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
14159 pavg<ssemodesuffix>\t{%2, %0|%0, %2}
14160 vpavg<ssemodesuffix>\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
14161 [(set_attr "isa" "noavx,avx")
14162 (set_attr "type" "sseiadd")
14163 (set_attr "prefix_data16" "1,*")
14164 (set_attr "prefix" "orig,<mask_prefix>")
14165 (set_attr "mode" "<sseinsnmode>")])
14167 ;; The correct representation for this is absolutely enormous, and
14168 ;; surely not generally useful.
14169 (define_insn "<sse2_avx2>_psadbw"
14170 [(set (match_operand:VI8_AVX2_AVX512BW 0 "register_operand" "=x,v")
14171 (unspec:VI8_AVX2_AVX512BW
14172 [(match_operand:<ssebytemode> 1 "register_operand" "0,v")
14173 (match_operand:<ssebytemode> 2 "vector_operand" "xBm,vm")]
14177 psadbw\t{%2, %0|%0, %2}
14178 vpsadbw\t{%2, %1, %0|%0, %1, %2}"
14179 [(set_attr "isa" "noavx,avx")
14180 (set_attr "type" "sseiadd")
14181 (set_attr "atom_unit" "simul")
14182 (set_attr "prefix_data16" "1,*")
14183 (set_attr "prefix" "orig,maybe_evex")
14184 (set_attr "mode" "<sseinsnmode>")])
14186 (define_insn "<sse>_movmsk<ssemodesuffix><avxsizesuffix>"
14187 [(set (match_operand:SI 0 "register_operand" "=r")
14189 [(match_operand:VF_128_256 1 "register_operand" "x")]
14192 "%vmovmsk<ssemodesuffix>\t{%1, %0|%0, %1}"
14193 [(set_attr "type" "ssemov")
14194 (set_attr "prefix" "maybe_vex")
14195 (set_attr "mode" "<MODE>")])
14197 (define_insn "*<sse>_movmsk<ssemodesuffix><avxsizesuffix>_zext"
14198 [(set (match_operand:DI 0 "register_operand" "=r")
14201 [(match_operand:VF_128_256 1 "register_operand" "x")]
14203 "TARGET_64BIT && TARGET_SSE"
14204 "%vmovmsk<ssemodesuffix>\t{%1, %k0|%k0, %1}"
14205 [(set_attr "type" "ssemov")
14206 (set_attr "prefix" "maybe_vex")
14207 (set_attr "mode" "<MODE>")])
14209 (define_insn "<sse2_avx2>_pmovmskb"
14210 [(set (match_operand:SI 0 "register_operand" "=r")
14212 [(match_operand:VI1_AVX2 1 "register_operand" "x")]
14215 "%vpmovmskb\t{%1, %0|%0, %1}"
14216 [(set_attr "type" "ssemov")
14217 (set (attr "prefix_data16")
14219 (match_test "TARGET_AVX")
14221 (const_string "1")))
14222 (set_attr "prefix" "maybe_vex")
14223 (set_attr "mode" "SI")])
14225 (define_insn "*<sse2_avx2>_pmovmskb_zext"
14226 [(set (match_operand:DI 0 "register_operand" "=r")
14229 [(match_operand:VI1_AVX2 1 "register_operand" "x")]
14231 "TARGET_64BIT && TARGET_SSE2"
14232 "%vpmovmskb\t{%1, %k0|%k0, %1}"
14233 [(set_attr "type" "ssemov")
14234 (set (attr "prefix_data16")
14236 (match_test "TARGET_AVX")
14238 (const_string "1")))
14239 (set_attr "prefix" "maybe_vex")
14240 (set_attr "mode" "SI")])
14242 (define_expand "sse2_maskmovdqu"
14243 [(set (match_operand:V16QI 0 "memory_operand")
14244 (unspec:V16QI [(match_operand:V16QI 1 "register_operand")
14245 (match_operand:V16QI 2 "register_operand")
14250 (define_insn "*sse2_maskmovdqu"
14251 [(set (mem:V16QI (match_operand:P 0 "register_operand" "D"))
14252 (unspec:V16QI [(match_operand:V16QI 1 "register_operand" "x")
14253 (match_operand:V16QI 2 "register_operand" "x")
14254 (mem:V16QI (match_dup 0))]
14258 /* We can't use %^ here due to ASM_OUTPUT_OPCODE processing
14259 that requires %v to be at the beginning of the opcode name. */
14260 if (Pmode != word_mode)
14261 fputs ("\taddr32", asm_out_file);
14262 return "%vmaskmovdqu\t{%2, %1|%1, %2}";
14264 [(set_attr "type" "ssemov")
14265 (set_attr "prefix_data16" "1")
14266 (set (attr "length_address")
14267 (symbol_ref ("Pmode != word_mode")))
14268 ;; The implicit %rdi operand confuses default length_vex computation.
14269 (set (attr "length_vex")
14270 (symbol_ref ("3 + REX_SSE_REGNO_P (REGNO (operands[2]))")))
14271 (set_attr "prefix" "maybe_vex")
14272 (set_attr "znver1_decode" "vector")
14273 (set_attr "mode" "TI")])
14275 (define_insn "sse_ldmxcsr"
14276 [(unspec_volatile [(match_operand:SI 0 "memory_operand" "m")]
14280 [(set_attr "type" "sse")
14281 (set_attr "atom_sse_attr" "mxcsr")
14282 (set_attr "prefix" "maybe_vex")
14283 (set_attr "memory" "load")])
14285 (define_insn "sse_stmxcsr"
14286 [(set (match_operand:SI 0 "memory_operand" "=m")
14287 (unspec_volatile:SI [(const_int 0)] UNSPECV_STMXCSR))]
14290 [(set_attr "type" "sse")
14291 (set_attr "atom_sse_attr" "mxcsr")
14292 (set_attr "prefix" "maybe_vex")
14293 (set_attr "memory" "store")])
14295 (define_insn "sse2_clflush"
14296 [(unspec_volatile [(match_operand 0 "address_operand" "p")]
14300 [(set_attr "type" "sse")
14301 (set_attr "atom_sse_attr" "fence")
14302 (set_attr "memory" "unknown")])
14304 ;; As per AMD and Intel ISA manuals, the first operand is extensions
14305 ;; and it goes to %ecx. The second operand received is hints and it goes
14307 (define_insn "sse3_mwait"
14308 [(unspec_volatile [(match_operand:SI 0 "register_operand" "c")
14309 (match_operand:SI 1 "register_operand" "a")]
14312 ;; 64bit version is "mwait %rax,%rcx". But only lower 32bits are used.
14313 ;; Since 32bit register operands are implicitly zero extended to 64bit,
14314 ;; we only need to set up 32bit registers.
14316 [(set_attr "length" "3")])
14318 (define_insn "sse3_monitor_<mode>"
14319 [(unspec_volatile [(match_operand:P 0 "register_operand" "a")
14320 (match_operand:SI 1 "register_operand" "c")
14321 (match_operand:SI 2 "register_operand" "d")]
14324 ;; 64bit version is "monitor %rax,%rcx,%rdx". But only lower 32bits in
14325 ;; RCX and RDX are used. Since 32bit register operands are implicitly
14326 ;; zero extended to 64bit, we only need to set up 32bit registers.
14328 [(set (attr "length")
14329 (symbol_ref ("(Pmode != word_mode) + 3")))])
14331 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
14333 ;; SSSE3 instructions
14335 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
14337 (define_code_iterator ssse3_plusminus [plus ss_plus minus ss_minus])
14339 (define_insn "avx2_ph<plusminus_mnemonic>wv16hi3"
14340 [(set (match_operand:V16HI 0 "register_operand" "=x")
14345 (ssse3_plusminus:HI
14347 (match_operand:V16HI 1 "register_operand" "x")
14348 (parallel [(const_int 0)]))
14349 (vec_select:HI (match_dup 1) (parallel [(const_int 1)])))
14350 (ssse3_plusminus:HI
14351 (vec_select:HI (match_dup 1) (parallel [(const_int 2)]))
14352 (vec_select:HI (match_dup 1) (parallel [(const_int 3)]))))
14354 (ssse3_plusminus:HI
14355 (vec_select:HI (match_dup 1) (parallel [(const_int 4)]))
14356 (vec_select:HI (match_dup 1) (parallel [(const_int 5)])))
14357 (ssse3_plusminus:HI
14358 (vec_select:HI (match_dup 1) (parallel [(const_int 6)]))
14359 (vec_select:HI (match_dup 1) (parallel [(const_int 7)])))))
14362 (ssse3_plusminus:HI
14363 (vec_select:HI (match_dup 1) (parallel [(const_int 8)]))
14364 (vec_select:HI (match_dup 1) (parallel [(const_int 9)])))
14365 (ssse3_plusminus:HI
14366 (vec_select:HI (match_dup 1) (parallel [(const_int 10)]))
14367 (vec_select:HI (match_dup 1) (parallel [(const_int 11)]))))
14369 (ssse3_plusminus:HI
14370 (vec_select:HI (match_dup 1) (parallel [(const_int 12)]))
14371 (vec_select:HI (match_dup 1) (parallel [(const_int 13)])))
14372 (ssse3_plusminus:HI
14373 (vec_select:HI (match_dup 1) (parallel [(const_int 14)]))
14374 (vec_select:HI (match_dup 1) (parallel [(const_int 15)]))))))
14378 (ssse3_plusminus:HI
14380 (match_operand:V16HI 2 "nonimmediate_operand" "xm")
14381 (parallel [(const_int 0)]))
14382 (vec_select:HI (match_dup 2) (parallel [(const_int 1)])))
14383 (ssse3_plusminus:HI
14384 (vec_select:HI (match_dup 2) (parallel [(const_int 2)]))
14385 (vec_select:HI (match_dup 2) (parallel [(const_int 3)]))))
14387 (ssse3_plusminus:HI
14388 (vec_select:HI (match_dup 2) (parallel [(const_int 4)]))
14389 (vec_select:HI (match_dup 2) (parallel [(const_int 5)])))
14390 (ssse3_plusminus:HI
14391 (vec_select:HI (match_dup 2) (parallel [(const_int 6)]))
14392 (vec_select:HI (match_dup 2) (parallel [(const_int 7)])))))
14395 (ssse3_plusminus:HI
14396 (vec_select:HI (match_dup 2) (parallel [(const_int 8)]))
14397 (vec_select:HI (match_dup 2) (parallel [(const_int 9)])))
14398 (ssse3_plusminus:HI
14399 (vec_select:HI (match_dup 2) (parallel [(const_int 10)]))
14400 (vec_select:HI (match_dup 2) (parallel [(const_int 11)]))))
14402 (ssse3_plusminus:HI
14403 (vec_select:HI (match_dup 2) (parallel [(const_int 12)]))
14404 (vec_select:HI (match_dup 2) (parallel [(const_int 13)])))
14405 (ssse3_plusminus:HI
14406 (vec_select:HI (match_dup 2) (parallel [(const_int 14)]))
14407 (vec_select:HI (match_dup 2) (parallel [(const_int 15)]))))))))]
14409 "vph<plusminus_mnemonic>w\t{%2, %1, %0|%0, %1, %2}"
14410 [(set_attr "type" "sseiadd")
14411 (set_attr "prefix_extra" "1")
14412 (set_attr "prefix" "vex")
14413 (set_attr "mode" "OI")])
14415 (define_insn "ssse3_ph<plusminus_mnemonic>wv8hi3"
14416 [(set (match_operand:V8HI 0 "register_operand" "=x,x")
14420 (ssse3_plusminus:HI
14422 (match_operand:V8HI 1 "register_operand" "0,x")
14423 (parallel [(const_int 0)]))
14424 (vec_select:HI (match_dup 1) (parallel [(const_int 1)])))
14425 (ssse3_plusminus:HI
14426 (vec_select:HI (match_dup 1) (parallel [(const_int 2)]))
14427 (vec_select:HI (match_dup 1) (parallel [(const_int 3)]))))
14429 (ssse3_plusminus:HI
14430 (vec_select:HI (match_dup 1) (parallel [(const_int 4)]))
14431 (vec_select:HI (match_dup 1) (parallel [(const_int 5)])))
14432 (ssse3_plusminus:HI
14433 (vec_select:HI (match_dup 1) (parallel [(const_int 6)]))
14434 (vec_select:HI (match_dup 1) (parallel [(const_int 7)])))))
14437 (ssse3_plusminus:HI
14439 (match_operand:V8HI 2 "vector_operand" "xBm,xm")
14440 (parallel [(const_int 0)]))
14441 (vec_select:HI (match_dup 2) (parallel [(const_int 1)])))
14442 (ssse3_plusminus:HI
14443 (vec_select:HI (match_dup 2) (parallel [(const_int 2)]))
14444 (vec_select:HI (match_dup 2) (parallel [(const_int 3)]))))
14446 (ssse3_plusminus:HI
14447 (vec_select:HI (match_dup 2) (parallel [(const_int 4)]))
14448 (vec_select:HI (match_dup 2) (parallel [(const_int 5)])))
14449 (ssse3_plusminus:HI
14450 (vec_select:HI (match_dup 2) (parallel [(const_int 6)]))
14451 (vec_select:HI (match_dup 2) (parallel [(const_int 7)])))))))]
14454 ph<plusminus_mnemonic>w\t{%2, %0|%0, %2}
14455 vph<plusminus_mnemonic>w\t{%2, %1, %0|%0, %1, %2}"
14456 [(set_attr "isa" "noavx,avx")
14457 (set_attr "type" "sseiadd")
14458 (set_attr "atom_unit" "complex")
14459 (set_attr "prefix_data16" "1,*")
14460 (set_attr "prefix_extra" "1")
14461 (set_attr "prefix" "orig,vex")
14462 (set_attr "mode" "TI")])
14464 (define_insn "ssse3_ph<plusminus_mnemonic>wv4hi3"
14465 [(set (match_operand:V4HI 0 "register_operand" "=y")
14468 (ssse3_plusminus:HI
14470 (match_operand:V4HI 1 "register_operand" "0")
14471 (parallel [(const_int 0)]))
14472 (vec_select:HI (match_dup 1) (parallel [(const_int 1)])))
14473 (ssse3_plusminus:HI
14474 (vec_select:HI (match_dup 1) (parallel [(const_int 2)]))
14475 (vec_select:HI (match_dup 1) (parallel [(const_int 3)]))))
14477 (ssse3_plusminus:HI
14479 (match_operand:V4HI 2 "nonimmediate_operand" "ym")
14480 (parallel [(const_int 0)]))
14481 (vec_select:HI (match_dup 2) (parallel [(const_int 1)])))
14482 (ssse3_plusminus:HI
14483 (vec_select:HI (match_dup 2) (parallel [(const_int 2)]))
14484 (vec_select:HI (match_dup 2) (parallel [(const_int 3)]))))))]
14486 "ph<plusminus_mnemonic>w\t{%2, %0|%0, %2}"
14487 [(set_attr "type" "sseiadd")
14488 (set_attr "atom_unit" "complex")
14489 (set_attr "prefix_extra" "1")
14490 (set (attr "prefix_rex") (symbol_ref "x86_extended_reg_mentioned_p (insn)"))
14491 (set_attr "mode" "DI")])
14493 (define_insn "avx2_ph<plusminus_mnemonic>dv8si3"
14494 [(set (match_operand:V8SI 0 "register_operand" "=x")
14500 (match_operand:V8SI 1 "register_operand" "x")
14501 (parallel [(const_int 0)]))
14502 (vec_select:SI (match_dup 1) (parallel [(const_int 1)])))
14504 (vec_select:SI (match_dup 1) (parallel [(const_int 2)]))
14505 (vec_select:SI (match_dup 1) (parallel [(const_int 3)]))))
14508 (vec_select:SI (match_dup 1) (parallel [(const_int 4)]))
14509 (vec_select:SI (match_dup 1) (parallel [(const_int 5)])))
14511 (vec_select:SI (match_dup 1) (parallel [(const_int 6)]))
14512 (vec_select:SI (match_dup 1) (parallel [(const_int 7)])))))
14517 (match_operand:V8SI 2 "nonimmediate_operand" "xm")
14518 (parallel [(const_int 0)]))
14519 (vec_select:SI (match_dup 2) (parallel [(const_int 1)])))
14521 (vec_select:SI (match_dup 2) (parallel [(const_int 2)]))
14522 (vec_select:SI (match_dup 2) (parallel [(const_int 3)]))))
14525 (vec_select:SI (match_dup 2) (parallel [(const_int 4)]))
14526 (vec_select:SI (match_dup 2) (parallel [(const_int 5)])))
14528 (vec_select:SI (match_dup 2) (parallel [(const_int 6)]))
14529 (vec_select:SI (match_dup 2) (parallel [(const_int 7)])))))))]
14531 "vph<plusminus_mnemonic>d\t{%2, %1, %0|%0, %1, %2}"
14532 [(set_attr "type" "sseiadd")
14533 (set_attr "prefix_extra" "1")
14534 (set_attr "prefix" "vex")
14535 (set_attr "mode" "OI")])
14537 (define_insn "ssse3_ph<plusminus_mnemonic>dv4si3"
14538 [(set (match_operand:V4SI 0 "register_operand" "=x,x")
14543 (match_operand:V4SI 1 "register_operand" "0,x")
14544 (parallel [(const_int 0)]))
14545 (vec_select:SI (match_dup 1) (parallel [(const_int 1)])))
14547 (vec_select:SI (match_dup 1) (parallel [(const_int 2)]))
14548 (vec_select:SI (match_dup 1) (parallel [(const_int 3)]))))
14552 (match_operand:V4SI 2 "vector_operand" "xBm,xm")
14553 (parallel [(const_int 0)]))
14554 (vec_select:SI (match_dup 2) (parallel [(const_int 1)])))
14556 (vec_select:SI (match_dup 2) (parallel [(const_int 2)]))
14557 (vec_select:SI (match_dup 2) (parallel [(const_int 3)]))))))]
14560 ph<plusminus_mnemonic>d\t{%2, %0|%0, %2}
14561 vph<plusminus_mnemonic>d\t{%2, %1, %0|%0, %1, %2}"
14562 [(set_attr "isa" "noavx,avx")
14563 (set_attr "type" "sseiadd")
14564 (set_attr "atom_unit" "complex")
14565 (set_attr "prefix_data16" "1,*")
14566 (set_attr "prefix_extra" "1")
14567 (set_attr "prefix" "orig,vex")
14568 (set_attr "mode" "TI")])
14570 (define_insn "ssse3_ph<plusminus_mnemonic>dv2si3"
14571 [(set (match_operand:V2SI 0 "register_operand" "=y")
14575 (match_operand:V2SI 1 "register_operand" "0")
14576 (parallel [(const_int 0)]))
14577 (vec_select:SI (match_dup 1) (parallel [(const_int 1)])))
14580 (match_operand:V2SI 2 "nonimmediate_operand" "ym")
14581 (parallel [(const_int 0)]))
14582 (vec_select:SI (match_dup 2) (parallel [(const_int 1)])))))]
14584 "ph<plusminus_mnemonic>d\t{%2, %0|%0, %2}"
14585 [(set_attr "type" "sseiadd")
14586 (set_attr "atom_unit" "complex")
14587 (set_attr "prefix_extra" "1")
14588 (set (attr "prefix_rex") (symbol_ref "x86_extended_reg_mentioned_p (insn)"))
14589 (set_attr "mode" "DI")])
14591 (define_insn "avx2_pmaddubsw256"
14592 [(set (match_operand:V16HI 0 "register_operand" "=x,v")
14597 (match_operand:V32QI 1 "register_operand" "x,v")
14598 (parallel [(const_int 0) (const_int 2)
14599 (const_int 4) (const_int 6)
14600 (const_int 8) (const_int 10)
14601 (const_int 12) (const_int 14)
14602 (const_int 16) (const_int 18)
14603 (const_int 20) (const_int 22)
14604 (const_int 24) (const_int 26)
14605 (const_int 28) (const_int 30)])))
14608 (match_operand:V32QI 2 "nonimmediate_operand" "xm,vm")
14609 (parallel [(const_int 0) (const_int 2)
14610 (const_int 4) (const_int 6)
14611 (const_int 8) (const_int 10)
14612 (const_int 12) (const_int 14)
14613 (const_int 16) (const_int 18)
14614 (const_int 20) (const_int 22)
14615 (const_int 24) (const_int 26)
14616 (const_int 28) (const_int 30)]))))
14619 (vec_select:V16QI (match_dup 1)
14620 (parallel [(const_int 1) (const_int 3)
14621 (const_int 5) (const_int 7)
14622 (const_int 9) (const_int 11)
14623 (const_int 13) (const_int 15)
14624 (const_int 17) (const_int 19)
14625 (const_int 21) (const_int 23)
14626 (const_int 25) (const_int 27)
14627 (const_int 29) (const_int 31)])))
14629 (vec_select:V16QI (match_dup 2)
14630 (parallel [(const_int 1) (const_int 3)
14631 (const_int 5) (const_int 7)
14632 (const_int 9) (const_int 11)
14633 (const_int 13) (const_int 15)
14634 (const_int 17) (const_int 19)
14635 (const_int 21) (const_int 23)
14636 (const_int 25) (const_int 27)
14637 (const_int 29) (const_int 31)]))))))]
14639 "vpmaddubsw\t{%2, %1, %0|%0, %1, %2}"
14640 [(set_attr "isa" "*,avx512bw")
14641 (set_attr "type" "sseiadd")
14642 (set_attr "prefix_extra" "1")
14643 (set_attr "prefix" "vex,evex")
14644 (set_attr "mode" "OI")])
14646 ;; The correct representation for this is absolutely enormous, and
14647 ;; surely not generally useful.
14648 (define_insn "avx512bw_pmaddubsw512<mode><mask_name>"
14649 [(set (match_operand:VI2_AVX512VL 0 "register_operand" "=v")
14650 (unspec:VI2_AVX512VL
14651 [(match_operand:<dbpsadbwmode> 1 "register_operand" "v")
14652 (match_operand:<dbpsadbwmode> 2 "nonimmediate_operand" "vm")]
14653 UNSPEC_PMADDUBSW512))]
14655 "vpmaddubsw\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}";
14656 [(set_attr "type" "sseiadd")
14657 (set_attr "prefix" "evex")
14658 (set_attr "mode" "XI")])
14660 (define_insn "avx512bw_umulhrswv32hi3<mask_name>"
14661 [(set (match_operand:V32HI 0 "register_operand" "=v")
14668 (match_operand:V32HI 1 "nonimmediate_operand" "%v"))
14670 (match_operand:V32HI 2 "nonimmediate_operand" "vm")))
14672 (const_vector:V32HI [(const_int 1) (const_int 1)
14673 (const_int 1) (const_int 1)
14674 (const_int 1) (const_int 1)
14675 (const_int 1) (const_int 1)
14676 (const_int 1) (const_int 1)
14677 (const_int 1) (const_int 1)
14678 (const_int 1) (const_int 1)
14679 (const_int 1) (const_int 1)
14680 (const_int 1) (const_int 1)
14681 (const_int 1) (const_int 1)
14682 (const_int 1) (const_int 1)
14683 (const_int 1) (const_int 1)
14684 (const_int 1) (const_int 1)
14685 (const_int 1) (const_int 1)
14686 (const_int 1) (const_int 1)
14687 (const_int 1) (const_int 1)]))
14690 "vpmulhrsw\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
14691 [(set_attr "type" "sseimul")
14692 (set_attr "prefix" "evex")
14693 (set_attr "mode" "XI")])
14695 (define_insn "ssse3_pmaddubsw128"
14696 [(set (match_operand:V8HI 0 "register_operand" "=x,x,v")
14701 (match_operand:V16QI 1 "register_operand" "0,x,v")
14702 (parallel [(const_int 0) (const_int 2)
14703 (const_int 4) (const_int 6)
14704 (const_int 8) (const_int 10)
14705 (const_int 12) (const_int 14)])))
14708 (match_operand:V16QI 2 "vector_operand" "xBm,xm,vm")
14709 (parallel [(const_int 0) (const_int 2)
14710 (const_int 4) (const_int 6)
14711 (const_int 8) (const_int 10)
14712 (const_int 12) (const_int 14)]))))
14715 (vec_select:V8QI (match_dup 1)
14716 (parallel [(const_int 1) (const_int 3)
14717 (const_int 5) (const_int 7)
14718 (const_int 9) (const_int 11)
14719 (const_int 13) (const_int 15)])))
14721 (vec_select:V8QI (match_dup 2)
14722 (parallel [(const_int 1) (const_int 3)
14723 (const_int 5) (const_int 7)
14724 (const_int 9) (const_int 11)
14725 (const_int 13) (const_int 15)]))))))]
14728 pmaddubsw\t{%2, %0|%0, %2}
14729 vpmaddubsw\t{%2, %1, %0|%0, %1, %2}
14730 vpmaddubsw\t{%2, %1, %0|%0, %1, %2}"
14731 [(set_attr "isa" "noavx,avx,avx512bw")
14732 (set_attr "type" "sseiadd")
14733 (set_attr "atom_unit" "simul")
14734 (set_attr "prefix_data16" "1,*,*")
14735 (set_attr "prefix_extra" "1")
14736 (set_attr "prefix" "orig,vex,evex")
14737 (set_attr "mode" "TI")])
14739 (define_insn "ssse3_pmaddubsw"
14740 [(set (match_operand:V4HI 0 "register_operand" "=y")
14745 (match_operand:V8QI 1 "register_operand" "0")
14746 (parallel [(const_int 0) (const_int 2)
14747 (const_int 4) (const_int 6)])))
14750 (match_operand:V8QI 2 "nonimmediate_operand" "ym")
14751 (parallel [(const_int 0) (const_int 2)
14752 (const_int 4) (const_int 6)]))))
14755 (vec_select:V4QI (match_dup 1)
14756 (parallel [(const_int 1) (const_int 3)
14757 (const_int 5) (const_int 7)])))
14759 (vec_select:V4QI (match_dup 2)
14760 (parallel [(const_int 1) (const_int 3)
14761 (const_int 5) (const_int 7)]))))))]
14763 "pmaddubsw\t{%2, %0|%0, %2}"
14764 [(set_attr "type" "sseiadd")
14765 (set_attr "atom_unit" "simul")
14766 (set_attr "prefix_extra" "1")
14767 (set (attr "prefix_rex") (symbol_ref "x86_extended_reg_mentioned_p (insn)"))
14768 (set_attr "mode" "DI")])
14770 (define_mode_iterator PMULHRSW
14771 [V4HI V8HI (V16HI "TARGET_AVX2")])
14773 (define_expand "<ssse3_avx2>_pmulhrsw<mode>3_mask"
14774 [(set (match_operand:PMULHRSW 0 "register_operand")
14775 (vec_merge:PMULHRSW
14777 (lshiftrt:<ssedoublemode>
14778 (plus:<ssedoublemode>
14779 (lshiftrt:<ssedoublemode>
14780 (mult:<ssedoublemode>
14781 (sign_extend:<ssedoublemode>
14782 (match_operand:PMULHRSW 1 "nonimmediate_operand"))
14783 (sign_extend:<ssedoublemode>
14784 (match_operand:PMULHRSW 2 "nonimmediate_operand")))
14788 (match_operand:PMULHRSW 3 "register_operand")
14789 (match_operand:<avx512fmaskmode> 4 "register_operand")))]
14790 "TARGET_AVX512BW && TARGET_AVX512VL"
14792 operands[5] = CONST1_RTX(<MODE>mode);
14793 ix86_fixup_binary_operands_no_copy (MULT, <MODE>mode, operands);
14796 (define_expand "<ssse3_avx2>_pmulhrsw<mode>3"
14797 [(set (match_operand:PMULHRSW 0 "register_operand")
14799 (lshiftrt:<ssedoublemode>
14800 (plus:<ssedoublemode>
14801 (lshiftrt:<ssedoublemode>
14802 (mult:<ssedoublemode>
14803 (sign_extend:<ssedoublemode>
14804 (match_operand:PMULHRSW 1 "nonimmediate_operand"))
14805 (sign_extend:<ssedoublemode>
14806 (match_operand:PMULHRSW 2 "nonimmediate_operand")))
14812 operands[3] = CONST1_RTX(<MODE>mode);
14813 ix86_fixup_binary_operands_no_copy (MULT, <MODE>mode, operands);
14816 (define_insn "*<ssse3_avx2>_pmulhrsw<mode>3<mask_name>"
14817 [(set (match_operand:VI2_AVX2 0 "register_operand" "=x,x,v")
14819 (lshiftrt:<ssedoublemode>
14820 (plus:<ssedoublemode>
14821 (lshiftrt:<ssedoublemode>
14822 (mult:<ssedoublemode>
14823 (sign_extend:<ssedoublemode>
14824 (match_operand:VI2_AVX2 1 "vector_operand" "%0,x,v"))
14825 (sign_extend:<ssedoublemode>
14826 (match_operand:VI2_AVX2 2 "vector_operand" "xBm,xm,vm")))
14828 (match_operand:VI2_AVX2 3 "const1_operand"))
14830 "TARGET_SSSE3 && <mask_mode512bit_condition> && <mask_avx512bw_condition>
14831 && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
14833 pmulhrsw\t{%2, %0|%0, %2}
14834 vpmulhrsw\t{%2, %1, %0<mask_operand4>|%0<mask_operand4>, %1, %2}
14835 vpmulhrsw\t{%2, %1, %0<mask_operand4>|%0<mask_operand4>, %1, %2}"
14836 [(set_attr "isa" "noavx,avx,avx512bw")
14837 (set_attr "type" "sseimul")
14838 (set_attr "prefix_data16" "1,*,*")
14839 (set_attr "prefix_extra" "1")
14840 (set_attr "prefix" "orig,maybe_evex,evex")
14841 (set_attr "mode" "<sseinsnmode>")])
14843 (define_insn "*ssse3_pmulhrswv4hi3"
14844 [(set (match_operand:V4HI 0 "register_operand" "=y")
14851 (match_operand:V4HI 1 "nonimmediate_operand" "%0"))
14853 (match_operand:V4HI 2 "nonimmediate_operand" "ym")))
14855 (match_operand:V4HI 3 "const1_operand"))
14857 "TARGET_SSSE3 && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
14858 "pmulhrsw\t{%2, %0|%0, %2}"
14859 [(set_attr "type" "sseimul")
14860 (set_attr "prefix_extra" "1")
14861 (set (attr "prefix_rex") (symbol_ref "x86_extended_reg_mentioned_p (insn)"))
14862 (set_attr "mode" "DI")])
14864 (define_insn "<ssse3_avx2>_pshufb<mode>3<mask_name>"
14865 [(set (match_operand:VI1_AVX512 0 "register_operand" "=x,x,v")
14867 [(match_operand:VI1_AVX512 1 "register_operand" "0,x,v")
14868 (match_operand:VI1_AVX512 2 "vector_operand" "xBm,xm,vm")]
14870 "TARGET_SSSE3 && <mask_mode512bit_condition> && <mask_avx512bw_condition>"
14872 pshufb\t{%2, %0|%0, %2}
14873 vpshufb\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}
14874 vpshufb\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
14875 [(set_attr "isa" "noavx,avx,avx512bw")
14876 (set_attr "type" "sselog1")
14877 (set_attr "prefix_data16" "1,*,*")
14878 (set_attr "prefix_extra" "1")
14879 (set_attr "prefix" "orig,maybe_evex,evex")
14880 (set_attr "btver2_decode" "vector")
14881 (set_attr "mode" "<sseinsnmode>")])
14883 (define_insn "ssse3_pshufbv8qi3"
14884 [(set (match_operand:V8QI 0 "register_operand" "=y")
14885 (unspec:V8QI [(match_operand:V8QI 1 "register_operand" "0")
14886 (match_operand:V8QI 2 "nonimmediate_operand" "ym")]
14889 "pshufb\t{%2, %0|%0, %2}";
14890 [(set_attr "type" "sselog1")
14891 (set_attr "prefix_extra" "1")
14892 (set (attr "prefix_rex") (symbol_ref "x86_extended_reg_mentioned_p (insn)"))
14893 (set_attr "mode" "DI")])
14895 (define_insn "<ssse3_avx2>_psign<mode>3"
14896 [(set (match_operand:VI124_AVX2 0 "register_operand" "=x,x")
14898 [(match_operand:VI124_AVX2 1 "register_operand" "0,x")
14899 (match_operand:VI124_AVX2 2 "vector_operand" "xBm,xm")]
14903 psign<ssemodesuffix>\t{%2, %0|%0, %2}
14904 vpsign<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
14905 [(set_attr "isa" "noavx,avx")
14906 (set_attr "type" "sselog1")
14907 (set_attr "prefix_data16" "1,*")
14908 (set_attr "prefix_extra" "1")
14909 (set_attr "prefix" "orig,vex")
14910 (set_attr "mode" "<sseinsnmode>")])
14912 (define_insn "ssse3_psign<mode>3"
14913 [(set (match_operand:MMXMODEI 0 "register_operand" "=y")
14915 [(match_operand:MMXMODEI 1 "register_operand" "0")
14916 (match_operand:MMXMODEI 2 "nonimmediate_operand" "ym")]
14919 "psign<mmxvecsize>\t{%2, %0|%0, %2}";
14920 [(set_attr "type" "sselog1")
14921 (set_attr "prefix_extra" "1")
14922 (set (attr "prefix_rex") (symbol_ref "x86_extended_reg_mentioned_p (insn)"))
14923 (set_attr "mode" "DI")])
14925 (define_insn "<ssse3_avx2>_palignr<mode>_mask"
14926 [(set (match_operand:VI1_AVX512 0 "register_operand" "=v")
14927 (vec_merge:VI1_AVX512
14929 [(match_operand:VI1_AVX512 1 "register_operand" "v")
14930 (match_operand:VI1_AVX512 2 "nonimmediate_operand" "vm")
14931 (match_operand:SI 3 "const_0_to_255_mul_8_operand" "n")]
14933 (match_operand:VI1_AVX512 4 "vector_move_operand" "0C")
14934 (match_operand:<avx512fmaskmode> 5 "register_operand" "Yk")))]
14935 "TARGET_AVX512BW && (<MODE_SIZE> == 64 || TARGET_AVX512VL)"
14937 operands[3] = GEN_INT (INTVAL (operands[3]) / 8);
14938 return "vpalignr\t{%3, %2, %1, %0%{%5%}%N4|%0%{%5%}%N4, %1, %2, %3}";
14940 [(set_attr "type" "sseishft")
14941 (set_attr "atom_unit" "sishuf")
14942 (set_attr "prefix_extra" "1")
14943 (set_attr "length_immediate" "1")
14944 (set_attr "prefix" "evex")
14945 (set_attr "mode" "<sseinsnmode>")])
14947 (define_insn "<ssse3_avx2>_palignr<mode>"
14948 [(set (match_operand:SSESCALARMODE 0 "register_operand" "=x,x,v")
14949 (unspec:SSESCALARMODE
14950 [(match_operand:SSESCALARMODE 1 "register_operand" "0,x,v")
14951 (match_operand:SSESCALARMODE 2 "vector_operand" "xBm,xm,vm")
14952 (match_operand:SI 3 "const_0_to_255_mul_8_operand" "n,n,n")]
14956 operands[3] = GEN_INT (INTVAL (operands[3]) / 8);
14958 switch (which_alternative)
14961 return "palignr\t{%3, %2, %0|%0, %2, %3}";
14964 return "vpalignr\t{%3, %2, %1, %0|%0, %1, %2, %3}";
14966 gcc_unreachable ();
14969 [(set_attr "isa" "noavx,avx,avx512bw")
14970 (set_attr "type" "sseishft")
14971 (set_attr "atom_unit" "sishuf")
14972 (set_attr "prefix_data16" "1,*,*")
14973 (set_attr "prefix_extra" "1")
14974 (set_attr "length_immediate" "1")
14975 (set_attr "prefix" "orig,vex,evex")
14976 (set_attr "mode" "<sseinsnmode>")])
14978 (define_insn "ssse3_palignrdi"
14979 [(set (match_operand:DI 0 "register_operand" "=y")
14980 (unspec:DI [(match_operand:DI 1 "register_operand" "0")
14981 (match_operand:DI 2 "nonimmediate_operand" "ym")
14982 (match_operand:SI 3 "const_0_to_255_mul_8_operand" "n")]
14986 operands[3] = GEN_INT (INTVAL (operands[3]) / 8);
14987 return "palignr\t{%3, %2, %0|%0, %2, %3}";
14989 [(set_attr "type" "sseishft")
14990 (set_attr "atom_unit" "sishuf")
14991 (set_attr "prefix_extra" "1")
14992 (set_attr "length_immediate" "1")
14993 (set (attr "prefix_rex") (symbol_ref "x86_extended_reg_mentioned_p (insn)"))
14994 (set_attr "mode" "DI")])
14996 ;; Mode iterator to handle singularity w/ absence of V2DI and V4DI
14997 ;; modes for abs instruction on pre AVX-512 targets.
14998 (define_mode_iterator VI1248_AVX512VL_AVX512BW
14999 [(V64QI "TARGET_AVX512BW") (V32QI "TARGET_AVX2") V16QI
15000 (V32HI "TARGET_AVX512BW") (V16HI "TARGET_AVX2") V8HI
15001 (V16SI "TARGET_AVX512F") (V8SI "TARGET_AVX2") V4SI
15002 (V8DI "TARGET_AVX512F") (V4DI "TARGET_AVX512VL") (V2DI "TARGET_AVX512VL")])
15004 (define_insn "*abs<mode>2"
15005 [(set (match_operand:VI1248_AVX512VL_AVX512BW 0 "register_operand" "=v")
15006 (abs:VI1248_AVX512VL_AVX512BW
15007 (match_operand:VI1248_AVX512VL_AVX512BW 1 "vector_operand" "vBm")))]
15009 "%vpabs<ssemodesuffix>\t{%1, %0|%0, %1}"
15010 [(set_attr "type" "sselog1")
15011 (set_attr "prefix_data16" "1")
15012 (set_attr "prefix_extra" "1")
15013 (set_attr "prefix" "maybe_vex")
15014 (set_attr "mode" "<sseinsnmode>")])
15016 (define_insn "abs<mode>2_mask"
15017 [(set (match_operand:VI48_AVX512VL 0 "register_operand" "=v")
15018 (vec_merge:VI48_AVX512VL
15020 (match_operand:VI48_AVX512VL 1 "nonimmediate_operand" "vm"))
15021 (match_operand:VI48_AVX512VL 2 "vector_move_operand" "0C")
15022 (match_operand:<avx512fmaskmode> 3 "register_operand" "Yk")))]
15024 "vpabs<ssemodesuffix>\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"
15025 [(set_attr "type" "sselog1")
15026 (set_attr "prefix" "evex")
15027 (set_attr "mode" "<sseinsnmode>")])
15029 (define_insn "abs<mode>2_mask"
15030 [(set (match_operand:VI12_AVX512VL 0 "register_operand" "=v")
15031 (vec_merge:VI12_AVX512VL
15033 (match_operand:VI12_AVX512VL 1 "nonimmediate_operand" "vm"))
15034 (match_operand:VI12_AVX512VL 2 "vector_move_operand" "0C")
15035 (match_operand:<avx512fmaskmode> 3 "register_operand" "Yk")))]
15037 "vpabs<ssemodesuffix>\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"
15038 [(set_attr "type" "sselog1")
15039 (set_attr "prefix" "evex")
15040 (set_attr "mode" "<sseinsnmode>")])
15042 (define_expand "abs<mode>2"
15043 [(set (match_operand:VI1248_AVX512VL_AVX512BW 0 "register_operand")
15044 (abs:VI1248_AVX512VL_AVX512BW
15045 (match_operand:VI1248_AVX512VL_AVX512BW 1 "vector_operand")))]
15050 ix86_expand_sse2_abs (operands[0], operands[1]);
15055 (define_insn "abs<mode>2"
15056 [(set (match_operand:MMXMODEI 0 "register_operand" "=y")
15058 (match_operand:MMXMODEI 1 "nonimmediate_operand" "ym")))]
15060 "pabs<mmxvecsize>\t{%1, %0|%0, %1}";
15061 [(set_attr "type" "sselog1")
15062 (set_attr "prefix_rep" "0")
15063 (set_attr "prefix_extra" "1")
15064 (set (attr "prefix_rex") (symbol_ref "x86_extended_reg_mentioned_p (insn)"))
15065 (set_attr "mode" "DI")])
15067 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
15069 ;; AMD SSE4A instructions
15071 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
15073 (define_insn "sse4a_movnt<mode>"
15074 [(set (match_operand:MODEF 0 "memory_operand" "=m")
15076 [(match_operand:MODEF 1 "register_operand" "x")]
15079 "movnt<ssemodesuffix>\t{%1, %0|%0, %1}"
15080 [(set_attr "type" "ssemov")
15081 (set_attr "mode" "<MODE>")])
15083 (define_insn "sse4a_vmmovnt<mode>"
15084 [(set (match_operand:<ssescalarmode> 0 "memory_operand" "=m")
15085 (unspec:<ssescalarmode>
15086 [(vec_select:<ssescalarmode>
15087 (match_operand:VF_128 1 "register_operand" "x")
15088 (parallel [(const_int 0)]))]
15091 "movnt<ssescalarmodesuffix>\t{%1, %0|%0, %1}"
15092 [(set_attr "type" "ssemov")
15093 (set_attr "mode" "<ssescalarmode>")])
15095 (define_insn "sse4a_extrqi"
15096 [(set (match_operand:V2DI 0 "register_operand" "=x")
15097 (unspec:V2DI [(match_operand:V2DI 1 "register_operand" "0")
15098 (match_operand 2 "const_0_to_255_operand")
15099 (match_operand 3 "const_0_to_255_operand")]
15102 "extrq\t{%3, %2, %0|%0, %2, %3}"
15103 [(set_attr "type" "sse")
15104 (set_attr "prefix_data16" "1")
15105 (set_attr "length_immediate" "2")
15106 (set_attr "mode" "TI")])
15108 (define_insn "sse4a_extrq"
15109 [(set (match_operand:V2DI 0 "register_operand" "=x")
15110 (unspec:V2DI [(match_operand:V2DI 1 "register_operand" "0")
15111 (match_operand:V16QI 2 "register_operand" "x")]
15114 "extrq\t{%2, %0|%0, %2}"
15115 [(set_attr "type" "sse")
15116 (set_attr "prefix_data16" "1")
15117 (set_attr "mode" "TI")])
15119 (define_insn "sse4a_insertqi"
15120 [(set (match_operand:V2DI 0 "register_operand" "=x")
15121 (unspec:V2DI [(match_operand:V2DI 1 "register_operand" "0")
15122 (match_operand:V2DI 2 "register_operand" "x")
15123 (match_operand 3 "const_0_to_255_operand")
15124 (match_operand 4 "const_0_to_255_operand")]
15127 "insertq\t{%4, %3, %2, %0|%0, %2, %3, %4}"
15128 [(set_attr "type" "sseins")
15129 (set_attr "prefix_data16" "0")
15130 (set_attr "prefix_rep" "1")
15131 (set_attr "length_immediate" "2")
15132 (set_attr "mode" "TI")])
15134 (define_insn "sse4a_insertq"
15135 [(set (match_operand:V2DI 0 "register_operand" "=x")
15136 (unspec:V2DI [(match_operand:V2DI 1 "register_operand" "0")
15137 (match_operand:V2DI 2 "register_operand" "x")]
15140 "insertq\t{%2, %0|%0, %2}"
15141 [(set_attr "type" "sseins")
15142 (set_attr "prefix_data16" "0")
15143 (set_attr "prefix_rep" "1")
15144 (set_attr "mode" "TI")])
15146 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
15148 ;; Intel SSE4.1 instructions
15150 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
15152 ;; Mapping of immediate bits for blend instructions
15153 (define_mode_attr blendbits
15154 [(V8SF "255") (V4SF "15") (V4DF "15") (V2DF "3")])
15156 (define_insn "<sse4_1>_blend<ssemodesuffix><avxsizesuffix>"
15157 [(set (match_operand:VF_128_256 0 "register_operand" "=Yr,*x,x")
15158 (vec_merge:VF_128_256
15159 (match_operand:VF_128_256 2 "vector_operand" "YrBm,*xBm,xm")
15160 (match_operand:VF_128_256 1 "register_operand" "0,0,x")
15161 (match_operand:SI 3 "const_0_to_<blendbits>_operand")))]
15164 blend<ssemodesuffix>\t{%3, %2, %0|%0, %2, %3}
15165 blend<ssemodesuffix>\t{%3, %2, %0|%0, %2, %3}
15166 vblend<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
15167 [(set_attr "isa" "noavx,noavx,avx")
15168 (set_attr "type" "ssemov")
15169 (set_attr "length_immediate" "1")
15170 (set_attr "prefix_data16" "1,1,*")
15171 (set_attr "prefix_extra" "1")
15172 (set_attr "prefix" "orig,orig,vex")
15173 (set_attr "mode" "<MODE>")])
15175 (define_insn "<sse4_1>_blendv<ssemodesuffix><avxsizesuffix>"
15176 [(set (match_operand:VF_128_256 0 "register_operand" "=Yr,*x,x")
15178 [(match_operand:VF_128_256 1 "register_operand" "0,0,x")
15179 (match_operand:VF_128_256 2 "vector_operand" "YrBm,*xBm,xm")
15180 (match_operand:VF_128_256 3 "register_operand" "Yz,Yz,x")]
15184 blendv<ssemodesuffix>\t{%3, %2, %0|%0, %2, %3}
15185 blendv<ssemodesuffix>\t{%3, %2, %0|%0, %2, %3}
15186 vblendv<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
15187 [(set_attr "isa" "noavx,noavx,avx")
15188 (set_attr "type" "ssemov")
15189 (set_attr "length_immediate" "1")
15190 (set_attr "prefix_data16" "1,1,*")
15191 (set_attr "prefix_extra" "1")
15192 (set_attr "prefix" "orig,orig,vex")
15193 (set_attr "btver2_decode" "vector,vector,vector")
15194 (set_attr "mode" "<MODE>")])
15196 (define_insn "<sse4_1>_dp<ssemodesuffix><avxsizesuffix>"
15197 [(set (match_operand:VF_128_256 0 "register_operand" "=Yr,*x,x")
15199 [(match_operand:VF_128_256 1 "vector_operand" "%0,0,x")
15200 (match_operand:VF_128_256 2 "vector_operand" "YrBm,*xBm,xm")
15201 (match_operand:SI 3 "const_0_to_255_operand" "n,n,n")]
15205 dp<ssemodesuffix>\t{%3, %2, %0|%0, %2, %3}
15206 dp<ssemodesuffix>\t{%3, %2, %0|%0, %2, %3}
15207 vdp<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
15208 [(set_attr "isa" "noavx,noavx,avx")
15209 (set_attr "type" "ssemul")
15210 (set_attr "length_immediate" "1")
15211 (set_attr "prefix_data16" "1,1,*")
15212 (set_attr "prefix_extra" "1")
15213 (set_attr "prefix" "orig,orig,vex")
15214 (set_attr "btver2_decode" "vector,vector,vector")
15215 (set_attr "znver1_decode" "vector,vector,vector")
15216 (set_attr "mode" "<MODE>")])
15218 ;; Mode attribute used by `vmovntdqa' pattern
15219 (define_mode_attr vi8_sse4_1_avx2_avx512
15220 [(V2DI "sse4_1") (V4DI "avx2") (V8DI "avx512f")])
15222 (define_insn "<vi8_sse4_1_avx2_avx512>_movntdqa"
15223 [(set (match_operand:VI8_AVX2_AVX512F 0 "register_operand" "=Yr,*x,v")
15224 (unspec:VI8_AVX2_AVX512F [(match_operand:VI8_AVX2_AVX512F 1 "memory_operand" "m,m,m")]
15227 "%vmovntdqa\t{%1, %0|%0, %1}"
15228 [(set_attr "isa" "noavx,noavx,avx")
15229 (set_attr "type" "ssemov")
15230 (set_attr "prefix_extra" "1,1,*")
15231 (set_attr "prefix" "orig,orig,maybe_evex")
15232 (set_attr "mode" "<sseinsnmode>")])
15234 (define_insn "<sse4_1_avx2>_mpsadbw"
15235 [(set (match_operand:VI1_AVX2 0 "register_operand" "=Yr,*x,x")
15237 [(match_operand:VI1_AVX2 1 "register_operand" "0,0,x")
15238 (match_operand:VI1_AVX2 2 "vector_operand" "YrBm,*xBm,xm")
15239 (match_operand:SI 3 "const_0_to_255_operand" "n,n,n")]
15243 mpsadbw\t{%3, %2, %0|%0, %2, %3}
15244 mpsadbw\t{%3, %2, %0|%0, %2, %3}
15245 vmpsadbw\t{%3, %2, %1, %0|%0, %1, %2, %3}"
15246 [(set_attr "isa" "noavx,noavx,avx")
15247 (set_attr "type" "sselog1")
15248 (set_attr "length_immediate" "1")
15249 (set_attr "prefix_extra" "1")
15250 (set_attr "prefix" "orig,orig,vex")
15251 (set_attr "btver2_decode" "vector,vector,vector")
15252 (set_attr "znver1_decode" "vector,vector,vector")
15253 (set_attr "mode" "<sseinsnmode>")])
15255 (define_insn "<sse4_1_avx2>_packusdw<mask_name>"
15256 [(set (match_operand:VI2_AVX2 0 "register_operand" "=Yr,*x,x,v")
15257 (vec_concat:VI2_AVX2
15258 (us_truncate:<ssehalfvecmode>
15259 (match_operand:<sseunpackmode> 1 "register_operand" "0,0,x,v"))
15260 (us_truncate:<ssehalfvecmode>
15261 (match_operand:<sseunpackmode> 2 "vector_operand" "YrBm,*xBm,xm,vm"))))]
15262 "TARGET_SSE4_1 && <mask_mode512bit_condition> && <mask_avx512bw_condition>"
15264 packusdw\t{%2, %0|%0, %2}
15265 packusdw\t{%2, %0|%0, %2}
15266 vpackusdw\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}
15267 vpackusdw\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
15268 [(set_attr "isa" "noavx,noavx,avx,avx512bw")
15269 (set_attr "type" "sselog")
15270 (set_attr "prefix_extra" "1")
15271 (set_attr "prefix" "orig,orig,<mask_prefix>,evex")
15272 (set_attr "mode" "<sseinsnmode>")])
15274 (define_insn "<sse4_1_avx2>_pblendvb"
15275 [(set (match_operand:VI1_AVX2 0 "register_operand" "=Yr,*x,x")
15277 [(match_operand:VI1_AVX2 1 "register_operand" "0,0,x")
15278 (match_operand:VI1_AVX2 2 "vector_operand" "YrBm,*xBm,xm")
15279 (match_operand:VI1_AVX2 3 "register_operand" "Yz,Yz,x")]
15283 pblendvb\t{%3, %2, %0|%0, %2, %3}
15284 pblendvb\t{%3, %2, %0|%0, %2, %3}
15285 vpblendvb\t{%3, %2, %1, %0|%0, %1, %2, %3}"
15286 [(set_attr "isa" "noavx,noavx,avx")
15287 (set_attr "type" "ssemov")
15288 (set_attr "prefix_extra" "1")
15289 (set_attr "length_immediate" "*,*,1")
15290 (set_attr "prefix" "orig,orig,vex")
15291 (set_attr "btver2_decode" "vector,vector,vector")
15292 (set_attr "mode" "<sseinsnmode>")])
15294 (define_insn "sse4_1_pblendw"
15295 [(set (match_operand:V8HI 0 "register_operand" "=Yr,*x,x")
15297 (match_operand:V8HI 2 "vector_operand" "YrBm,*xBm,xm")
15298 (match_operand:V8HI 1 "register_operand" "0,0,x")
15299 (match_operand:SI 3 "const_0_to_255_operand" "n,n,n")))]
15302 pblendw\t{%3, %2, %0|%0, %2, %3}
15303 pblendw\t{%3, %2, %0|%0, %2, %3}
15304 vpblendw\t{%3, %2, %1, %0|%0, %1, %2, %3}"
15305 [(set_attr "isa" "noavx,noavx,avx")
15306 (set_attr "type" "ssemov")
15307 (set_attr "prefix_extra" "1")
15308 (set_attr "length_immediate" "1")
15309 (set_attr "prefix" "orig,orig,vex")
15310 (set_attr "mode" "TI")])
15312 ;; The builtin uses an 8-bit immediate. Expand that.
15313 (define_expand "avx2_pblendw"
15314 [(set (match_operand:V16HI 0 "register_operand")
15316 (match_operand:V16HI 2 "nonimmediate_operand")
15317 (match_operand:V16HI 1 "register_operand")
15318 (match_operand:SI 3 "const_0_to_255_operand")))]
15321 HOST_WIDE_INT val = INTVAL (operands[3]) & 0xff;
15322 operands[3] = GEN_INT (val << 8 | val);
15325 (define_insn "*avx2_pblendw"
15326 [(set (match_operand:V16HI 0 "register_operand" "=x")
15328 (match_operand:V16HI 2 "nonimmediate_operand" "xm")
15329 (match_operand:V16HI 1 "register_operand" "x")
15330 (match_operand:SI 3 "avx2_pblendw_operand" "n")))]
15333 operands[3] = GEN_INT (INTVAL (operands[3]) & 0xff);
15334 return "vpblendw\t{%3, %2, %1, %0|%0, %1, %2, %3}";
15336 [(set_attr "type" "ssemov")
15337 (set_attr "prefix_extra" "1")
15338 (set_attr "length_immediate" "1")
15339 (set_attr "prefix" "vex")
15340 (set_attr "mode" "OI")])
15342 (define_insn "avx2_pblendd<mode>"
15343 [(set (match_operand:VI4_AVX2 0 "register_operand" "=x")
15344 (vec_merge:VI4_AVX2
15345 (match_operand:VI4_AVX2 2 "nonimmediate_operand" "xm")
15346 (match_operand:VI4_AVX2 1 "register_operand" "x")
15347 (match_operand:SI 3 "const_0_to_255_operand" "n")))]
15349 "vpblendd\t{%3, %2, %1, %0|%0, %1, %2, %3}"
15350 [(set_attr "type" "ssemov")
15351 (set_attr "prefix_extra" "1")
15352 (set_attr "length_immediate" "1")
15353 (set_attr "prefix" "vex")
15354 (set_attr "mode" "<sseinsnmode>")])
15356 (define_insn "sse4_1_phminposuw"
15357 [(set (match_operand:V8HI 0 "register_operand" "=Yr,*x,x")
15358 (unspec:V8HI [(match_operand:V8HI 1 "vector_operand" "YrBm,*xBm,xm")]
15359 UNSPEC_PHMINPOSUW))]
15361 "%vphminposuw\t{%1, %0|%0, %1}"
15362 [(set_attr "isa" "noavx,noavx,avx")
15363 (set_attr "type" "sselog1")
15364 (set_attr "prefix_extra" "1")
15365 (set_attr "prefix" "orig,orig,vex")
15366 (set_attr "mode" "TI")])
15368 (define_insn "avx2_<code>v16qiv16hi2<mask_name>"
15369 [(set (match_operand:V16HI 0 "register_operand" "=v")
15371 (match_operand:V16QI 1 "nonimmediate_operand" "vm")))]
15372 "TARGET_AVX2 && <mask_avx512bw_condition> && <mask_avx512vl_condition>"
15373 "vpmov<extsuffix>bw\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
15374 [(set_attr "type" "ssemov")
15375 (set_attr "prefix_extra" "1")
15376 (set_attr "prefix" "maybe_evex")
15377 (set_attr "mode" "OI")])
15379 (define_insn "avx512bw_<code>v32qiv32hi2<mask_name>"
15380 [(set (match_operand:V32HI 0 "register_operand" "=v")
15382 (match_operand:V32QI 1 "nonimmediate_operand" "vm")))]
15384 "vpmov<extsuffix>bw\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
15385 [(set_attr "type" "ssemov")
15386 (set_attr "prefix_extra" "1")
15387 (set_attr "prefix" "evex")
15388 (set_attr "mode" "XI")])
15390 (define_insn "sse4_1_<code>v8qiv8hi2<mask_name>"
15391 [(set (match_operand:V8HI 0 "register_operand" "=Yr,*x,v")
15394 (match_operand:V16QI 1 "nonimmediate_operand" "Yrm,*xm,vm")
15395 (parallel [(const_int 0) (const_int 1)
15396 (const_int 2) (const_int 3)
15397 (const_int 4) (const_int 5)
15398 (const_int 6) (const_int 7)]))))]
15399 "TARGET_SSE4_1 && <mask_avx512bw_condition> && <mask_avx512vl_condition>"
15400 "%vpmov<extsuffix>bw\t{%1, %0<mask_operand2>|%0<mask_operand2>, %q1}"
15401 [(set_attr "isa" "noavx,noavx,avx")
15402 (set_attr "type" "ssemov")
15403 (set_attr "prefix_extra" "1")
15404 (set_attr "prefix" "orig,orig,maybe_evex")
15405 (set_attr "mode" "TI")])
15407 (define_insn "<mask_codefor>avx512f_<code>v16qiv16si2<mask_name>"
15408 [(set (match_operand:V16SI 0 "register_operand" "=v")
15410 (match_operand:V16QI 1 "nonimmediate_operand" "vm")))]
15412 "vpmov<extsuffix>bd\t{%1, %0<mask_operand2>|%0<mask_operand2>, %q1}"
15413 [(set_attr "type" "ssemov")
15414 (set_attr "prefix" "evex")
15415 (set_attr "mode" "XI")])
15417 (define_insn "avx2_<code>v8qiv8si2<mask_name>"
15418 [(set (match_operand:V8SI 0 "register_operand" "=v")
15421 (match_operand:V16QI 1 "nonimmediate_operand" "vm")
15422 (parallel [(const_int 0) (const_int 1)
15423 (const_int 2) (const_int 3)
15424 (const_int 4) (const_int 5)
15425 (const_int 6) (const_int 7)]))))]
15426 "TARGET_AVX2 && <mask_avx512vl_condition>"
15427 "vpmov<extsuffix>bd\t{%1, %0<mask_operand2>|%0<mask_operand2>, %q1}"
15428 [(set_attr "type" "ssemov")
15429 (set_attr "prefix_extra" "1")
15430 (set_attr "prefix" "maybe_evex")
15431 (set_attr "mode" "OI")])
15433 (define_insn "sse4_1_<code>v4qiv4si2<mask_name>"
15434 [(set (match_operand:V4SI 0 "register_operand" "=Yr,*x,v")
15437 (match_operand:V16QI 1 "nonimmediate_operand" "Yrm,*xm,vm")
15438 (parallel [(const_int 0) (const_int 1)
15439 (const_int 2) (const_int 3)]))))]
15440 "TARGET_SSE4_1 && <mask_avx512vl_condition>"
15441 "%vpmov<extsuffix>bd\t{%1, %0<mask_operand2>|%0<mask_operand2>, %k1}"
15442 [(set_attr "isa" "noavx,noavx,avx")
15443 (set_attr "type" "ssemov")
15444 (set_attr "prefix_extra" "1")
15445 (set_attr "prefix" "orig,orig,maybe_evex")
15446 (set_attr "mode" "TI")])
15448 (define_insn "avx512f_<code>v16hiv16si2<mask_name>"
15449 [(set (match_operand:V16SI 0 "register_operand" "=v")
15451 (match_operand:V16HI 1 "nonimmediate_operand" "vm")))]
15453 "vpmov<extsuffix>wd\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
15454 [(set_attr "type" "ssemov")
15455 (set_attr "prefix" "evex")
15456 (set_attr "mode" "XI")])
15458 (define_insn "avx2_<code>v8hiv8si2<mask_name>"
15459 [(set (match_operand:V8SI 0 "register_operand" "=v")
15461 (match_operand:V8HI 1 "nonimmediate_operand" "vm")))]
15462 "TARGET_AVX2 && <mask_avx512vl_condition>"
15463 "vpmov<extsuffix>wd\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
15464 [(set_attr "type" "ssemov")
15465 (set_attr "prefix_extra" "1")
15466 (set_attr "prefix" "maybe_evex")
15467 (set_attr "mode" "OI")])
15469 (define_insn "sse4_1_<code>v4hiv4si2<mask_name>"
15470 [(set (match_operand:V4SI 0 "register_operand" "=Yr,*x,v")
15473 (match_operand:V8HI 1 "nonimmediate_operand" "Yrm,*xm,vm")
15474 (parallel [(const_int 0) (const_int 1)
15475 (const_int 2) (const_int 3)]))))]
15476 "TARGET_SSE4_1 && <mask_avx512vl_condition>"
15477 "%vpmov<extsuffix>wd\t{%1, %0<mask_operand2>|%0<mask_operand2>, %q1}"
15478 [(set_attr "isa" "noavx,noavx,avx")
15479 (set_attr "type" "ssemov")
15480 (set_attr "prefix_extra" "1")
15481 (set_attr "prefix" "orig,orig,maybe_evex")
15482 (set_attr "mode" "TI")])
15484 (define_insn "avx512f_<code>v8qiv8di2<mask_name>"
15485 [(set (match_operand:V8DI 0 "register_operand" "=v")
15488 (match_operand:V16QI 1 "nonimmediate_operand" "vm")
15489 (parallel [(const_int 0) (const_int 1)
15490 (const_int 2) (const_int 3)
15491 (const_int 4) (const_int 5)
15492 (const_int 6) (const_int 7)]))))]
15494 "vpmov<extsuffix>bq\t{%1, %0<mask_operand2>|%0<mask_operand2>, %k1}"
15495 [(set_attr "type" "ssemov")
15496 (set_attr "prefix" "evex")
15497 (set_attr "mode" "XI")])
15499 (define_insn "avx2_<code>v4qiv4di2<mask_name>"
15500 [(set (match_operand:V4DI 0 "register_operand" "=v")
15503 (match_operand:V16QI 1 "nonimmediate_operand" "vm")
15504 (parallel [(const_int 0) (const_int 1)
15505 (const_int 2) (const_int 3)]))))]
15506 "TARGET_AVX2 && <mask_avx512vl_condition>"
15507 "vpmov<extsuffix>bq\t{%1, %0<mask_operand2>|%0<mask_operand2>, %k1}"
15508 [(set_attr "type" "ssemov")
15509 (set_attr "prefix_extra" "1")
15510 (set_attr "prefix" "maybe_evex")
15511 (set_attr "mode" "OI")])
15513 (define_insn "sse4_1_<code>v2qiv2di2<mask_name>"
15514 [(set (match_operand:V2DI 0 "register_operand" "=Yr,*x,v")
15517 (match_operand:V16QI 1 "nonimmediate_operand" "Yrm,*xm,vm")
15518 (parallel [(const_int 0) (const_int 1)]))))]
15519 "TARGET_SSE4_1 && <mask_avx512vl_condition>"
15520 "%vpmov<extsuffix>bq\t{%1, %0<mask_operand2>|%0<mask_operand2>, %w1}"
15521 [(set_attr "isa" "noavx,noavx,avx")
15522 (set_attr "type" "ssemov")
15523 (set_attr "prefix_extra" "1")
15524 (set_attr "prefix" "orig,orig,maybe_evex")
15525 (set_attr "mode" "TI")])
15527 (define_insn "avx512f_<code>v8hiv8di2<mask_name>"
15528 [(set (match_operand:V8DI 0 "register_operand" "=v")
15530 (match_operand:V8HI 1 "nonimmediate_operand" "vm")))]
15532 "vpmov<extsuffix>wq\t{%1, %0<mask_operand2>|%0<mask_operand2>, %q1}"
15533 [(set_attr "type" "ssemov")
15534 (set_attr "prefix" "evex")
15535 (set_attr "mode" "XI")])
15537 (define_insn "avx2_<code>v4hiv4di2<mask_name>"
15538 [(set (match_operand:V4DI 0 "register_operand" "=v")
15541 (match_operand:V8HI 1 "nonimmediate_operand" "vm")
15542 (parallel [(const_int 0) (const_int 1)
15543 (const_int 2) (const_int 3)]))))]
15544 "TARGET_AVX2 && <mask_avx512vl_condition>"
15545 "vpmov<extsuffix>wq\t{%1, %0<mask_operand2>|%0<mask_operand2>, %q1}"
15546 [(set_attr "type" "ssemov")
15547 (set_attr "prefix_extra" "1")
15548 (set_attr "prefix" "maybe_evex")
15549 (set_attr "mode" "OI")])
15551 (define_insn "sse4_1_<code>v2hiv2di2<mask_name>"
15552 [(set (match_operand:V2DI 0 "register_operand" "=Yr,*x,v")
15555 (match_operand:V8HI 1 "nonimmediate_operand" "Yrm,*xm,vm")
15556 (parallel [(const_int 0) (const_int 1)]))))]
15557 "TARGET_SSE4_1 && <mask_avx512vl_condition>"
15558 "%vpmov<extsuffix>wq\t{%1, %0<mask_operand2>|%0<mask_operand2>, %k1}"
15559 [(set_attr "isa" "noavx,noavx,avx")
15560 (set_attr "type" "ssemov")
15561 (set_attr "prefix_extra" "1")
15562 (set_attr "prefix" "orig,orig,maybe_evex")
15563 (set_attr "mode" "TI")])
15565 (define_insn "avx512f_<code>v8siv8di2<mask_name>"
15566 [(set (match_operand:V8DI 0 "register_operand" "=v")
15568 (match_operand:V8SI 1 "nonimmediate_operand" "vm")))]
15570 "vpmov<extsuffix>dq\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
15571 [(set_attr "type" "ssemov")
15572 (set_attr "prefix" "evex")
15573 (set_attr "mode" "XI")])
15575 (define_insn "avx2_<code>v4siv4di2<mask_name>"
15576 [(set (match_operand:V4DI 0 "register_operand" "=v")
15578 (match_operand:V4SI 1 "nonimmediate_operand" "vm")))]
15579 "TARGET_AVX2 && <mask_avx512vl_condition>"
15580 "vpmov<extsuffix>dq\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
15581 [(set_attr "type" "ssemov")
15582 (set_attr "prefix" "maybe_evex")
15583 (set_attr "prefix_extra" "1")
15584 (set_attr "mode" "OI")])
15586 (define_insn "sse4_1_<code>v2siv2di2<mask_name>"
15587 [(set (match_operand:V2DI 0 "register_operand" "=Yr,*x,v")
15590 (match_operand:V4SI 1 "nonimmediate_operand" "Yrm,*xm,vm")
15591 (parallel [(const_int 0) (const_int 1)]))))]
15592 "TARGET_SSE4_1 && <mask_avx512vl_condition>"
15593 "%vpmov<extsuffix>dq\t{%1, %0<mask_operand2>|%0<mask_operand2>, %q1}"
15594 [(set_attr "isa" "noavx,noavx,avx")
15595 (set_attr "type" "ssemov")
15596 (set_attr "prefix_extra" "1")
15597 (set_attr "prefix" "orig,orig,maybe_evex")
15598 (set_attr "mode" "TI")])
15600 ;; ptestps/ptestpd are very similar to comiss and ucomiss when
15601 ;; setting FLAGS_REG. But it is not a really compare instruction.
15602 (define_insn "avx_vtest<ssemodesuffix><avxsizesuffix>"
15603 [(set (reg:CC FLAGS_REG)
15604 (unspec:CC [(match_operand:VF_128_256 0 "register_operand" "x")
15605 (match_operand:VF_128_256 1 "nonimmediate_operand" "xm")]
15608 "vtest<ssemodesuffix>\t{%1, %0|%0, %1}"
15609 [(set_attr "type" "ssecomi")
15610 (set_attr "prefix_extra" "1")
15611 (set_attr "prefix" "vex")
15612 (set_attr "mode" "<MODE>")])
15614 ;; ptest is very similar to comiss and ucomiss when setting FLAGS_REG.
15615 ;; But it is not a really compare instruction.
15616 (define_insn "<sse4_1>_ptest<mode>"
15617 [(set (reg:CC FLAGS_REG)
15618 (unspec:CC [(match_operand:V_AVX 0 "register_operand" "Yr, *x, x")
15619 (match_operand:V_AVX 1 "vector_operand" "YrBm, *xBm, xm")]
15622 "%vptest\t{%1, %0|%0, %1}"
15623 [(set_attr "isa" "noavx,noavx,avx")
15624 (set_attr "type" "ssecomi")
15625 (set_attr "prefix_extra" "1")
15626 (set_attr "prefix" "orig,orig,vex")
15627 (set (attr "btver2_decode")
15629 (match_test "<sseinsnmode>mode==OImode")
15630 (const_string "vector")
15631 (const_string "*")))
15632 (set_attr "mode" "<sseinsnmode>")])
15634 (define_insn "ptesttf2"
15635 [(set (reg:CC FLAGS_REG)
15636 (unspec:CC [(match_operand:TF 0 "register_operand" "Yr, *x, x")
15637 (match_operand:TF 1 "vector_operand" "YrBm, *xBm, xm")]
15640 "%vptest\t{%1, %0|%0, %1}"
15641 [(set_attr "isa" "noavx,noavx,avx")
15642 (set_attr "type" "ssecomi")
15643 (set_attr "prefix_extra" "1")
15644 (set_attr "prefix" "orig,orig,vex")
15645 (set_attr "mode" "TI")])
15647 (define_insn "<sse4_1>_round<ssemodesuffix><avxsizesuffix>"
15648 [(set (match_operand:VF_128_256 0 "register_operand" "=Yr,*x,x")
15650 [(match_operand:VF_128_256 1 "vector_operand" "YrBm,*xBm,xm")
15651 (match_operand:SI 2 "const_0_to_15_operand" "n,n,n")]
15654 "%vround<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
15655 [(set_attr "isa" "noavx,noavx,avx")
15656 (set_attr "type" "ssecvt")
15657 (set_attr "prefix_data16" "1,1,*")
15658 (set_attr "prefix_extra" "1")
15659 (set_attr "length_immediate" "1")
15660 (set_attr "prefix" "orig,orig,vex")
15661 (set_attr "mode" "<MODE>")])
15663 (define_expand "<sse4_1>_round<ssemodesuffix>_sfix<avxsizesuffix>"
15664 [(match_operand:<sseintvecmode> 0 "register_operand")
15665 (match_operand:VF1_128_256 1 "vector_operand")
15666 (match_operand:SI 2 "const_0_to_15_operand")]
15669 rtx tmp = gen_reg_rtx (<MODE>mode);
15672 (gen_<sse4_1>_round<ssemodesuffix><avxsizesuffix> (tmp, operands[1],
15675 (gen_fix_trunc<mode><sseintvecmodelower>2 (operands[0], tmp));
15679 (define_expand "avx512f_round<castmode>512"
15680 [(match_operand:VF_512 0 "register_operand")
15681 (match_operand:VF_512 1 "nonimmediate_operand")
15682 (match_operand:SI 2 "const_0_to_15_operand")]
15685 emit_insn (gen_avx512f_rndscale<mode> (operands[0], operands[1], operands[2]));
15689 (define_expand "avx512f_roundps512_sfix"
15690 [(match_operand:V16SI 0 "register_operand")
15691 (match_operand:V16SF 1 "nonimmediate_operand")
15692 (match_operand:SI 2 "const_0_to_15_operand")]
15695 rtx tmp = gen_reg_rtx (V16SFmode);
15696 emit_insn (gen_avx512f_rndscalev16sf (tmp, operands[1], operands[2]));
15697 emit_insn (gen_fix_truncv16sfv16si2 (operands[0], tmp));
15701 (define_expand "<sse4_1>_round<ssemodesuffix>_vec_pack_sfix<avxsizesuffix>"
15702 [(match_operand:<ssepackfltmode> 0 "register_operand")
15703 (match_operand:VF2 1 "vector_operand")
15704 (match_operand:VF2 2 "vector_operand")
15705 (match_operand:SI 3 "const_0_to_15_operand")]
15710 if (<MODE>mode == V2DFmode
15711 && TARGET_AVX && !TARGET_PREFER_AVX128 && optimize_insn_for_speed_p ())
15713 rtx tmp2 = gen_reg_rtx (V4DFmode);
15715 tmp0 = gen_reg_rtx (V4DFmode);
15716 tmp1 = force_reg (V2DFmode, operands[1]);
15718 emit_insn (gen_avx_vec_concatv4df (tmp0, tmp1, operands[2]));
15719 emit_insn (gen_avx_roundpd256 (tmp2, tmp0, operands[3]));
15720 emit_insn (gen_fix_truncv4dfv4si2 (operands[0], tmp2));
15724 tmp0 = gen_reg_rtx (<MODE>mode);
15725 tmp1 = gen_reg_rtx (<MODE>mode);
15728 (gen_<sse4_1>_round<ssemodesuffix><avxsizesuffix> (tmp0, operands[1],
15731 (gen_<sse4_1>_round<ssemodesuffix><avxsizesuffix> (tmp1, operands[2],
15734 (gen_vec_pack_sfix_trunc_<mode> (operands[0], tmp0, tmp1));
15739 (define_insn "sse4_1_round<ssescalarmodesuffix>"
15740 [(set (match_operand:VF_128 0 "register_operand" "=Yr,*x,x,v")
15743 [(match_operand:VF_128 2 "register_operand" "Yr,*x,x,v")
15744 (match_operand:SI 3 "const_0_to_15_operand" "n,n,n,n")]
15746 (match_operand:VF_128 1 "register_operand" "0,0,x,v")
15750 round<ssescalarmodesuffix>\t{%3, %2, %0|%0, %2, %3}
15751 round<ssescalarmodesuffix>\t{%3, %2, %0|%0, %2, %3}
15752 vround<ssescalarmodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}
15753 vrndscale<ssescalarmodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
15754 [(set_attr "isa" "noavx,noavx,avx,avx512f")
15755 (set_attr "type" "ssecvt")
15756 (set_attr "length_immediate" "1")
15757 (set_attr "prefix_data16" "1,1,*,*")
15758 (set_attr "prefix_extra" "1")
15759 (set_attr "prefix" "orig,orig,vex,evex")
15760 (set_attr "mode" "<MODE>")])
15762 (define_expand "round<mode>2"
15763 [(set (match_dup 3)
15765 (match_operand:VF 1 "register_operand")
15767 (set (match_operand:VF 0 "register_operand")
15769 [(match_dup 3) (match_dup 4)]
15771 "TARGET_SSE4_1 && !flag_trapping_math"
15773 machine_mode scalar_mode;
15774 const struct real_format *fmt;
15775 REAL_VALUE_TYPE pred_half, half_minus_pred_half;
15776 rtx half, vec_half;
15778 scalar_mode = GET_MODE_INNER (<MODE>mode);
15780 /* load nextafter (0.5, 0.0) */
15781 fmt = REAL_MODE_FORMAT (scalar_mode);
15782 real_2expN (&half_minus_pred_half, -(fmt->p) - 1, scalar_mode);
15783 real_arithmetic (&pred_half, MINUS_EXPR, &dconsthalf, &half_minus_pred_half);
15784 half = const_double_from_real_value (pred_half, scalar_mode);
15786 vec_half = ix86_build_const_vector (<MODE>mode, true, half);
15787 vec_half = force_reg (<MODE>mode, vec_half);
15789 operands[2] = gen_reg_rtx (<MODE>mode);
15790 emit_insn (gen_copysign<mode>3 (operands[2], vec_half, operands[1]));
15792 operands[3] = gen_reg_rtx (<MODE>mode);
15793 operands[4] = GEN_INT (ROUND_TRUNC);
15796 (define_expand "round<mode>2_sfix"
15797 [(match_operand:<sseintvecmode> 0 "register_operand")
15798 (match_operand:VF1 1 "register_operand")]
15799 "TARGET_SSE4_1 && !flag_trapping_math"
15801 rtx tmp = gen_reg_rtx (<MODE>mode);
15803 emit_insn (gen_round<mode>2 (tmp, operands[1]));
15806 (gen_fix_trunc<mode><sseintvecmodelower>2 (operands[0], tmp));
15810 (define_expand "round<mode>2_vec_pack_sfix"
15811 [(match_operand:<ssepackfltmode> 0 "register_operand")
15812 (match_operand:VF2 1 "register_operand")
15813 (match_operand:VF2 2 "register_operand")]
15814 "TARGET_SSE4_1 && !flag_trapping_math"
15818 if (<MODE>mode == V2DFmode
15819 && TARGET_AVX && !TARGET_PREFER_AVX128 && optimize_insn_for_speed_p ())
15821 rtx tmp2 = gen_reg_rtx (V4DFmode);
15823 tmp0 = gen_reg_rtx (V4DFmode);
15824 tmp1 = force_reg (V2DFmode, operands[1]);
15826 emit_insn (gen_avx_vec_concatv4df (tmp0, tmp1, operands[2]));
15827 emit_insn (gen_roundv4df2 (tmp2, tmp0));
15828 emit_insn (gen_fix_truncv4dfv4si2 (operands[0], tmp2));
15832 tmp0 = gen_reg_rtx (<MODE>mode);
15833 tmp1 = gen_reg_rtx (<MODE>mode);
15835 emit_insn (gen_round<mode>2 (tmp0, operands[1]));
15836 emit_insn (gen_round<mode>2 (tmp1, operands[2]));
15839 (gen_vec_pack_sfix_trunc_<mode> (operands[0], tmp0, tmp1));
15844 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
15846 ;; Intel SSE4.2 string/text processing instructions
15848 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
15850 (define_insn_and_split "sse4_2_pcmpestr"
15851 [(set (match_operand:SI 0 "register_operand" "=c,c")
15853 [(match_operand:V16QI 2 "register_operand" "x,x")
15854 (match_operand:SI 3 "register_operand" "a,a")
15855 (match_operand:V16QI 4 "nonimmediate_operand" "x,m")
15856 (match_operand:SI 5 "register_operand" "d,d")
15857 (match_operand:SI 6 "const_0_to_255_operand" "n,n")]
15859 (set (match_operand:V16QI 1 "register_operand" "=Yz,Yz")
15867 (set (reg:CC FLAGS_REG)
15876 && can_create_pseudo_p ()"
15881 int ecx = !find_regno_note (curr_insn, REG_UNUSED, REGNO (operands[0]));
15882 int xmm0 = !find_regno_note (curr_insn, REG_UNUSED, REGNO (operands[1]));
15883 int flags = !find_regno_note (curr_insn, REG_UNUSED, FLAGS_REG);
15886 emit_insn (gen_sse4_2_pcmpestri (operands[0], operands[2],
15887 operands[3], operands[4],
15888 operands[5], operands[6]));
15890 emit_insn (gen_sse4_2_pcmpestrm (operands[1], operands[2],
15891 operands[3], operands[4],
15892 operands[5], operands[6]));
15893 if (flags && !(ecx || xmm0))
15894 emit_insn (gen_sse4_2_pcmpestr_cconly (NULL, NULL,
15895 operands[2], operands[3],
15896 operands[4], operands[5],
15898 if (!(flags || ecx || xmm0))
15899 emit_note (NOTE_INSN_DELETED);
15903 [(set_attr "type" "sselog")
15904 (set_attr "prefix_data16" "1")
15905 (set_attr "prefix_extra" "1")
15906 (set_attr "length_immediate" "1")
15907 (set_attr "memory" "none,load")
15908 (set_attr "mode" "TI")])
15910 (define_insn "sse4_2_pcmpestri"
15911 [(set (match_operand:SI 0 "register_operand" "=c,c")
15913 [(match_operand:V16QI 1 "register_operand" "x,x")
15914 (match_operand:SI 2 "register_operand" "a,a")
15915 (match_operand:V16QI 3 "nonimmediate_operand" "x,m")
15916 (match_operand:SI 4 "register_operand" "d,d")
15917 (match_operand:SI 5 "const_0_to_255_operand" "n,n")]
15919 (set (reg:CC FLAGS_REG)
15928 "%vpcmpestri\t{%5, %3, %1|%1, %3, %5}"
15929 [(set_attr "type" "sselog")
15930 (set_attr "prefix_data16" "1")
15931 (set_attr "prefix_extra" "1")
15932 (set_attr "prefix" "maybe_vex")
15933 (set_attr "length_immediate" "1")
15934 (set_attr "btver2_decode" "vector")
15935 (set_attr "memory" "none,load")
15936 (set_attr "mode" "TI")])
15938 (define_insn "sse4_2_pcmpestrm"
15939 [(set (match_operand:V16QI 0 "register_operand" "=Yz,Yz")
15941 [(match_operand:V16QI 1 "register_operand" "x,x")
15942 (match_operand:SI 2 "register_operand" "a,a")
15943 (match_operand:V16QI 3 "nonimmediate_operand" "x,m")
15944 (match_operand:SI 4 "register_operand" "d,d")
15945 (match_operand:SI 5 "const_0_to_255_operand" "n,n")]
15947 (set (reg:CC FLAGS_REG)
15956 "%vpcmpestrm\t{%5, %3, %1|%1, %3, %5}"
15957 [(set_attr "type" "sselog")
15958 (set_attr "prefix_data16" "1")
15959 (set_attr "prefix_extra" "1")
15960 (set_attr "length_immediate" "1")
15961 (set_attr "prefix" "maybe_vex")
15962 (set_attr "btver2_decode" "vector")
15963 (set_attr "memory" "none,load")
15964 (set_attr "mode" "TI")])
15966 (define_insn "sse4_2_pcmpestr_cconly"
15967 [(set (reg:CC FLAGS_REG)
15969 [(match_operand:V16QI 2 "register_operand" "x,x,x,x")
15970 (match_operand:SI 3 "register_operand" "a,a,a,a")
15971 (match_operand:V16QI 4 "nonimmediate_operand" "x,m,x,m")
15972 (match_operand:SI 5 "register_operand" "d,d,d,d")
15973 (match_operand:SI 6 "const_0_to_255_operand" "n,n,n,n")]
15975 (clobber (match_scratch:V16QI 0 "=Yz,Yz,X,X"))
15976 (clobber (match_scratch:SI 1 "= X, X,c,c"))]
15979 %vpcmpestrm\t{%6, %4, %2|%2, %4, %6}
15980 %vpcmpestrm\t{%6, %4, %2|%2, %4, %6}
15981 %vpcmpestri\t{%6, %4, %2|%2, %4, %6}
15982 %vpcmpestri\t{%6, %4, %2|%2, %4, %6}"
15983 [(set_attr "type" "sselog")
15984 (set_attr "prefix_data16" "1")
15985 (set_attr "prefix_extra" "1")
15986 (set_attr "length_immediate" "1")
15987 (set_attr "memory" "none,load,none,load")
15988 (set_attr "btver2_decode" "vector,vector,vector,vector")
15989 (set_attr "prefix" "maybe_vex")
15990 (set_attr "mode" "TI")])
15992 (define_insn_and_split "sse4_2_pcmpistr"
15993 [(set (match_operand:SI 0 "register_operand" "=c,c")
15995 [(match_operand:V16QI 2 "register_operand" "x,x")
15996 (match_operand:V16QI 3 "nonimmediate_operand" "x,m")
15997 (match_operand:SI 4 "const_0_to_255_operand" "n,n")]
15999 (set (match_operand:V16QI 1 "register_operand" "=Yz,Yz")
16005 (set (reg:CC FLAGS_REG)
16012 && can_create_pseudo_p ()"
16017 int ecx = !find_regno_note (curr_insn, REG_UNUSED, REGNO (operands[0]));
16018 int xmm0 = !find_regno_note (curr_insn, REG_UNUSED, REGNO (operands[1]));
16019 int flags = !find_regno_note (curr_insn, REG_UNUSED, FLAGS_REG);
16022 emit_insn (gen_sse4_2_pcmpistri (operands[0], operands[2],
16023 operands[3], operands[4]));
16025 emit_insn (gen_sse4_2_pcmpistrm (operands[1], operands[2],
16026 operands[3], operands[4]));
16027 if (flags && !(ecx || xmm0))
16028 emit_insn (gen_sse4_2_pcmpistr_cconly (NULL, NULL,
16029 operands[2], operands[3],
16031 if (!(flags || ecx || xmm0))
16032 emit_note (NOTE_INSN_DELETED);
16036 [(set_attr "type" "sselog")
16037 (set_attr "prefix_data16" "1")
16038 (set_attr "prefix_extra" "1")
16039 (set_attr "length_immediate" "1")
16040 (set_attr "memory" "none,load")
16041 (set_attr "mode" "TI")])
16043 (define_insn "sse4_2_pcmpistri"
16044 [(set (match_operand:SI 0 "register_operand" "=c,c")
16046 [(match_operand:V16QI 1 "register_operand" "x,x")
16047 (match_operand:V16QI 2 "nonimmediate_operand" "x,m")
16048 (match_operand:SI 3 "const_0_to_255_operand" "n,n")]
16050 (set (reg:CC FLAGS_REG)
16057 "%vpcmpistri\t{%3, %2, %1|%1, %2, %3}"
16058 [(set_attr "type" "sselog")
16059 (set_attr "prefix_data16" "1")
16060 (set_attr "prefix_extra" "1")
16061 (set_attr "length_immediate" "1")
16062 (set_attr "prefix" "maybe_vex")
16063 (set_attr "memory" "none,load")
16064 (set_attr "btver2_decode" "vector")
16065 (set_attr "mode" "TI")])
16067 (define_insn "sse4_2_pcmpistrm"
16068 [(set (match_operand:V16QI 0 "register_operand" "=Yz,Yz")
16070 [(match_operand:V16QI 1 "register_operand" "x,x")
16071 (match_operand:V16QI 2 "nonimmediate_operand" "x,m")
16072 (match_operand:SI 3 "const_0_to_255_operand" "n,n")]
16074 (set (reg:CC FLAGS_REG)
16081 "%vpcmpistrm\t{%3, %2, %1|%1, %2, %3}"
16082 [(set_attr "type" "sselog")
16083 (set_attr "prefix_data16" "1")
16084 (set_attr "prefix_extra" "1")
16085 (set_attr "length_immediate" "1")
16086 (set_attr "prefix" "maybe_vex")
16087 (set_attr "memory" "none,load")
16088 (set_attr "btver2_decode" "vector")
16089 (set_attr "mode" "TI")])
16091 (define_insn "sse4_2_pcmpistr_cconly"
16092 [(set (reg:CC FLAGS_REG)
16094 [(match_operand:V16QI 2 "register_operand" "x,x,x,x")
16095 (match_operand:V16QI 3 "nonimmediate_operand" "x,m,x,m")
16096 (match_operand:SI 4 "const_0_to_255_operand" "n,n,n,n")]
16098 (clobber (match_scratch:V16QI 0 "=Yz,Yz,X,X"))
16099 (clobber (match_scratch:SI 1 "= X, X,c,c"))]
16102 %vpcmpistrm\t{%4, %3, %2|%2, %3, %4}
16103 %vpcmpistrm\t{%4, %3, %2|%2, %3, %4}
16104 %vpcmpistri\t{%4, %3, %2|%2, %3, %4}
16105 %vpcmpistri\t{%4, %3, %2|%2, %3, %4}"
16106 [(set_attr "type" "sselog")
16107 (set_attr "prefix_data16" "1")
16108 (set_attr "prefix_extra" "1")
16109 (set_attr "length_immediate" "1")
16110 (set_attr "memory" "none,load,none,load")
16111 (set_attr "prefix" "maybe_vex")
16112 (set_attr "btver2_decode" "vector,vector,vector,vector")
16113 (set_attr "mode" "TI")])
16115 ;; Packed float variants
16116 (define_mode_attr GATHER_SCATTER_SF_MEM_MODE
16117 [(V8DI "V8SF") (V16SI "V16SF")])
16119 (define_expand "avx512pf_gatherpf<mode>sf"
16121 [(match_operand:<avx512fmaskmode> 0 "register_operand")
16122 (mem:<GATHER_SCATTER_SF_MEM_MODE>
16124 [(match_operand 2 "vsib_address_operand")
16125 (match_operand:VI48_512 1 "register_operand")
16126 (match_operand:SI 3 "const1248_operand")]))
16127 (match_operand:SI 4 "const_2_to_3_operand")]
16128 UNSPEC_GATHER_PREFETCH)]
16132 = gen_rtx_UNSPEC (Pmode, gen_rtvec (3, operands[2], operands[1],
16133 operands[3]), UNSPEC_VSIBADDR);
16136 (define_insn "*avx512pf_gatherpf<mode>sf_mask"
16138 [(match_operand:<avx512fmaskmode> 0 "register_operand" "Yk")
16139 (match_operator:<GATHER_SCATTER_SF_MEM_MODE> 5 "vsib_mem_operator"
16141 [(match_operand:P 2 "vsib_address_operand" "Tv")
16142 (match_operand:VI48_512 1 "register_operand" "v")
16143 (match_operand:SI 3 "const1248_operand" "n")]
16145 (match_operand:SI 4 "const_2_to_3_operand" "n")]
16146 UNSPEC_GATHER_PREFETCH)]
16149 switch (INTVAL (operands[4]))
16152 return "vgatherpf0<ssemodesuffix>ps\t{%5%{%0%}|%5%{%0%}}";
16154 return "vgatherpf1<ssemodesuffix>ps\t{%5%{%0%}|%5%{%0%}}";
16156 gcc_unreachable ();
16159 [(set_attr "type" "sse")
16160 (set_attr "prefix" "evex")
16161 (set_attr "mode" "XI")])
16163 ;; Packed double variants
16164 (define_expand "avx512pf_gatherpf<mode>df"
16166 [(match_operand:<avx512fmaskmode> 0 "register_operand")
16169 [(match_operand 2 "vsib_address_operand")
16170 (match_operand:VI4_256_8_512 1 "register_operand")
16171 (match_operand:SI 3 "const1248_operand")]))
16172 (match_operand:SI 4 "const_2_to_3_operand")]
16173 UNSPEC_GATHER_PREFETCH)]
16177 = gen_rtx_UNSPEC (Pmode, gen_rtvec (3, operands[2], operands[1],
16178 operands[3]), UNSPEC_VSIBADDR);
16181 (define_insn "*avx512pf_gatherpf<mode>df_mask"
16183 [(match_operand:<avx512fmaskmode> 0 "register_operand" "Yk")
16184 (match_operator:V8DF 5 "vsib_mem_operator"
16186 [(match_operand:P 2 "vsib_address_operand" "Tv")
16187 (match_operand:VI4_256_8_512 1 "register_operand" "v")
16188 (match_operand:SI 3 "const1248_operand" "n")]
16190 (match_operand:SI 4 "const_2_to_3_operand" "n")]
16191 UNSPEC_GATHER_PREFETCH)]
16194 switch (INTVAL (operands[4]))
16197 return "vgatherpf0<ssemodesuffix>pd\t{%5%{%0%}|%5%{%0%}}";
16199 return "vgatherpf1<ssemodesuffix>pd\t{%5%{%0%}|%5%{%0%}}";
16201 gcc_unreachable ();
16204 [(set_attr "type" "sse")
16205 (set_attr "prefix" "evex")
16206 (set_attr "mode" "XI")])
16208 ;; Packed float variants
16209 (define_expand "avx512pf_scatterpf<mode>sf"
16211 [(match_operand:<avx512fmaskmode> 0 "register_operand")
16212 (mem:<GATHER_SCATTER_SF_MEM_MODE>
16214 [(match_operand 2 "vsib_address_operand")
16215 (match_operand:VI48_512 1 "register_operand")
16216 (match_operand:SI 3 "const1248_operand")]))
16217 (match_operand:SI 4 "const2367_operand")]
16218 UNSPEC_SCATTER_PREFETCH)]
16222 = gen_rtx_UNSPEC (Pmode, gen_rtvec (3, operands[2], operands[1],
16223 operands[3]), UNSPEC_VSIBADDR);
16226 (define_insn "*avx512pf_scatterpf<mode>sf_mask"
16228 [(match_operand:<avx512fmaskmode> 0 "register_operand" "Yk")
16229 (match_operator:<GATHER_SCATTER_SF_MEM_MODE> 5 "vsib_mem_operator"
16231 [(match_operand:P 2 "vsib_address_operand" "Tv")
16232 (match_operand:VI48_512 1 "register_operand" "v")
16233 (match_operand:SI 3 "const1248_operand" "n")]
16235 (match_operand:SI 4 "const2367_operand" "n")]
16236 UNSPEC_SCATTER_PREFETCH)]
16239 switch (INTVAL (operands[4]))
16243 return "vscatterpf0<ssemodesuffix>ps\t{%5%{%0%}|%5%{%0%}}";
16246 return "vscatterpf1<ssemodesuffix>ps\t{%5%{%0%}|%5%{%0%}}";
16248 gcc_unreachable ();
16251 [(set_attr "type" "sse")
16252 (set_attr "prefix" "evex")
16253 (set_attr "mode" "XI")])
16255 ;; Packed double variants
16256 (define_expand "avx512pf_scatterpf<mode>df"
16258 [(match_operand:<avx512fmaskmode> 0 "register_operand")
16261 [(match_operand 2 "vsib_address_operand")
16262 (match_operand:VI4_256_8_512 1 "register_operand")
16263 (match_operand:SI 3 "const1248_operand")]))
16264 (match_operand:SI 4 "const2367_operand")]
16265 UNSPEC_SCATTER_PREFETCH)]
16269 = gen_rtx_UNSPEC (Pmode, gen_rtvec (3, operands[2], operands[1],
16270 operands[3]), UNSPEC_VSIBADDR);
16273 (define_insn "*avx512pf_scatterpf<mode>df_mask"
16275 [(match_operand:<avx512fmaskmode> 0 "register_operand" "Yk")
16276 (match_operator:V8DF 5 "vsib_mem_operator"
16278 [(match_operand:P 2 "vsib_address_operand" "Tv")
16279 (match_operand:VI4_256_8_512 1 "register_operand" "v")
16280 (match_operand:SI 3 "const1248_operand" "n")]
16282 (match_operand:SI 4 "const2367_operand" "n")]
16283 UNSPEC_SCATTER_PREFETCH)]
16286 switch (INTVAL (operands[4]))
16290 return "vscatterpf0<ssemodesuffix>pd\t{%5%{%0%}|%5%{%0%}}";
16293 return "vscatterpf1<ssemodesuffix>pd\t{%5%{%0%}|%5%{%0%}}";
16295 gcc_unreachable ();
16298 [(set_attr "type" "sse")
16299 (set_attr "prefix" "evex")
16300 (set_attr "mode" "XI")])
16302 (define_insn "avx512er_exp2<mode><mask_name><round_saeonly_name>"
16303 [(set (match_operand:VF_512 0 "register_operand" "=v")
16305 [(match_operand:VF_512 1 "<round_saeonly_nimm_predicate>" "<round_saeonly_constraint>")]
16308 "vexp2<ssemodesuffix>\t{<round_saeonly_mask_op2>%1, %0<mask_operand2>|%0<mask_operand2>, %1<round_saeonly_mask_op2>}"
16309 [(set_attr "prefix" "evex")
16310 (set_attr "type" "sse")
16311 (set_attr "mode" "<MODE>")])
16313 (define_insn "<mask_codefor>avx512er_rcp28<mode><mask_name><round_saeonly_name>"
16314 [(set (match_operand:VF_512 0 "register_operand" "=v")
16316 [(match_operand:VF_512 1 "<round_saeonly_nimm_predicate>" "<round_saeonly_constraint>")]
16319 "vrcp28<ssemodesuffix>\t{<round_saeonly_mask_op2>%1, %0<mask_operand2>|%0<mask_operand2>, %1<round_saeonly_mask_op2>}"
16320 [(set_attr "prefix" "evex")
16321 (set_attr "type" "sse")
16322 (set_attr "mode" "<MODE>")])
16324 (define_insn "avx512er_vmrcp28<mode><round_saeonly_name>"
16325 [(set (match_operand:VF_128 0 "register_operand" "=v")
16328 [(match_operand:VF_128 1 "<round_saeonly_nimm_predicate>" "<round_saeonly_constraint>")]
16330 (match_operand:VF_128 2 "register_operand" "v")
16333 "vrcp28<ssescalarmodesuffix>\t{<round_saeonly_op3>%1, %2, %0|%0, %2, %1<round_saeonly_op3>}"
16334 [(set_attr "length_immediate" "1")
16335 (set_attr "prefix" "evex")
16336 (set_attr "type" "sse")
16337 (set_attr "mode" "<MODE>")])
16339 (define_insn "<mask_codefor>avx512er_rsqrt28<mode><mask_name><round_saeonly_name>"
16340 [(set (match_operand:VF_512 0 "register_operand" "=v")
16342 [(match_operand:VF_512 1 "<round_saeonly_nimm_predicate>" "<round_saeonly_constraint>")]
16345 "vrsqrt28<ssemodesuffix>\t{<round_saeonly_mask_op2>%1, %0<mask_operand2>|%0<mask_operand2>, %1<round_saeonly_mask_op2>}"
16346 [(set_attr "prefix" "evex")
16347 (set_attr "type" "sse")
16348 (set_attr "mode" "<MODE>")])
16350 (define_insn "avx512er_vmrsqrt28<mode><round_saeonly_name>"
16351 [(set (match_operand:VF_128 0 "register_operand" "=v")
16354 [(match_operand:VF_128 1 "<round_saeonly_nimm_predicate>" "<round_saeonly_constraint>")]
16356 (match_operand:VF_128 2 "register_operand" "v")
16359 "vrsqrt28<ssescalarmodesuffix>\t{<round_saeonly_op3>%1, %2, %0|%0, %2, %1<round_saeonly_op3>}"
16360 [(set_attr "length_immediate" "1")
16361 (set_attr "type" "sse")
16362 (set_attr "prefix" "evex")
16363 (set_attr "mode" "<MODE>")])
16365 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
16367 ;; XOP instructions
16369 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
16371 (define_code_iterator xop_plus [plus ss_plus])
16373 (define_code_attr macs [(plus "macs") (ss_plus "macss")])
16374 (define_code_attr madcs [(plus "madcs") (ss_plus "madcss")])
16376 ;; XOP parallel integer multiply/add instructions.
16378 (define_insn "xop_p<macs><ssemodesuffix><ssemodesuffix>"
16379 [(set (match_operand:VI24_128 0 "register_operand" "=x")
16382 (match_operand:VI24_128 1 "nonimmediate_operand" "%x")
16383 (match_operand:VI24_128 2 "nonimmediate_operand" "xm"))
16384 (match_operand:VI24_128 3 "register_operand" "x")))]
16386 "vp<macs><ssemodesuffix><ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
16387 [(set_attr "type" "ssemuladd")
16388 (set_attr "mode" "TI")])
16390 (define_insn "xop_p<macs>dql"
16391 [(set (match_operand:V2DI 0 "register_operand" "=x")
16396 (match_operand:V4SI 1 "nonimmediate_operand" "%x")
16397 (parallel [(const_int 0) (const_int 2)])))
16400 (match_operand:V4SI 2 "nonimmediate_operand" "xm")
16401 (parallel [(const_int 0) (const_int 2)]))))
16402 (match_operand:V2DI 3 "register_operand" "x")))]
16404 "vp<macs>dql\t{%3, %2, %1, %0|%0, %1, %2, %3}"
16405 [(set_attr "type" "ssemuladd")
16406 (set_attr "mode" "TI")])
16408 (define_insn "xop_p<macs>dqh"
16409 [(set (match_operand:V2DI 0 "register_operand" "=x")
16414 (match_operand:V4SI 1 "nonimmediate_operand" "%x")
16415 (parallel [(const_int 1) (const_int 3)])))
16418 (match_operand:V4SI 2 "nonimmediate_operand" "xm")
16419 (parallel [(const_int 1) (const_int 3)]))))
16420 (match_operand:V2DI 3 "register_operand" "x")))]
16422 "vp<macs>dqh\t{%3, %2, %1, %0|%0, %1, %2, %3}"
16423 [(set_attr "type" "ssemuladd")
16424 (set_attr "mode" "TI")])
16426 ;; XOP parallel integer multiply/add instructions for the intrinisics
16427 (define_insn "xop_p<macs>wd"
16428 [(set (match_operand:V4SI 0 "register_operand" "=x")
16433 (match_operand:V8HI 1 "nonimmediate_operand" "%x")
16434 (parallel [(const_int 1) (const_int 3)
16435 (const_int 5) (const_int 7)])))
16438 (match_operand:V8HI 2 "nonimmediate_operand" "xm")
16439 (parallel [(const_int 1) (const_int 3)
16440 (const_int 5) (const_int 7)]))))
16441 (match_operand:V4SI 3 "register_operand" "x")))]
16443 "vp<macs>wd\t{%3, %2, %1, %0|%0, %1, %2, %3}"
16444 [(set_attr "type" "ssemuladd")
16445 (set_attr "mode" "TI")])
16447 (define_insn "xop_p<madcs>wd"
16448 [(set (match_operand:V4SI 0 "register_operand" "=x")
16454 (match_operand:V8HI 1 "nonimmediate_operand" "%x")
16455 (parallel [(const_int 0) (const_int 2)
16456 (const_int 4) (const_int 6)])))
16459 (match_operand:V8HI 2 "nonimmediate_operand" "xm")
16460 (parallel [(const_int 0) (const_int 2)
16461 (const_int 4) (const_int 6)]))))
16466 (parallel [(const_int 1) (const_int 3)
16467 (const_int 5) (const_int 7)])))
16471 (parallel [(const_int 1) (const_int 3)
16472 (const_int 5) (const_int 7)])))))
16473 (match_operand:V4SI 3 "register_operand" "x")))]
16475 "vp<madcs>wd\t{%3, %2, %1, %0|%0, %1, %2, %3}"
16476 [(set_attr "type" "ssemuladd")
16477 (set_attr "mode" "TI")])
16479 ;; XOP parallel XMM conditional moves
16480 (define_insn "xop_pcmov_<mode><avxsizesuffix>"
16481 [(set (match_operand:V 0 "register_operand" "=x,x")
16483 (match_operand:V 3 "nonimmediate_operand" "x,m")
16484 (match_operand:V 1 "register_operand" "x,x")
16485 (match_operand:V 2 "nonimmediate_operand" "xm,x")))]
16487 "vpcmov\t{%3, %2, %1, %0|%0, %1, %2, %3}"
16488 [(set_attr "type" "sse4arg")])
16490 ;; XOP horizontal add/subtract instructions
16491 (define_insn "xop_phadd<u>bw"
16492 [(set (match_operand:V8HI 0 "register_operand" "=x")
16496 (match_operand:V16QI 1 "nonimmediate_operand" "xm")
16497 (parallel [(const_int 0) (const_int 2)
16498 (const_int 4) (const_int 6)
16499 (const_int 8) (const_int 10)
16500 (const_int 12) (const_int 14)])))
16504 (parallel [(const_int 1) (const_int 3)
16505 (const_int 5) (const_int 7)
16506 (const_int 9) (const_int 11)
16507 (const_int 13) (const_int 15)])))))]
16509 "vphadd<u>bw\t{%1, %0|%0, %1}"
16510 [(set_attr "type" "sseiadd1")])
16512 (define_insn "xop_phadd<u>bd"
16513 [(set (match_operand:V4SI 0 "register_operand" "=x")
16518 (match_operand:V16QI 1 "nonimmediate_operand" "xm")
16519 (parallel [(const_int 0) (const_int 4)
16520 (const_int 8) (const_int 12)])))
16524 (parallel [(const_int 1) (const_int 5)
16525 (const_int 9) (const_int 13)]))))
16530 (parallel [(const_int 2) (const_int 6)
16531 (const_int 10) (const_int 14)])))
16535 (parallel [(const_int 3) (const_int 7)
16536 (const_int 11) (const_int 15)]))))))]
16538 "vphadd<u>bd\t{%1, %0|%0, %1}"
16539 [(set_attr "type" "sseiadd1")])
16541 (define_insn "xop_phadd<u>bq"
16542 [(set (match_operand:V2DI 0 "register_operand" "=x")
16548 (match_operand:V16QI 1 "nonimmediate_operand" "xm")
16549 (parallel [(const_int 0) (const_int 8)])))
16553 (parallel [(const_int 1) (const_int 9)]))))
16558 (parallel [(const_int 2) (const_int 10)])))
16562 (parallel [(const_int 3) (const_int 11)])))))
16568 (parallel [(const_int 4) (const_int 12)])))
16572 (parallel [(const_int 5) (const_int 13)]))))
16577 (parallel [(const_int 6) (const_int 14)])))
16581 (parallel [(const_int 7) (const_int 15)])))))))]
16583 "vphadd<u>bq\t{%1, %0|%0, %1}"
16584 [(set_attr "type" "sseiadd1")])
16586 (define_insn "xop_phadd<u>wd"
16587 [(set (match_operand:V4SI 0 "register_operand" "=x")
16591 (match_operand:V8HI 1 "nonimmediate_operand" "xm")
16592 (parallel [(const_int 0) (const_int 2)
16593 (const_int 4) (const_int 6)])))
16597 (parallel [(const_int 1) (const_int 3)
16598 (const_int 5) (const_int 7)])))))]
16600 "vphadd<u>wd\t{%1, %0|%0, %1}"
16601 [(set_attr "type" "sseiadd1")])
16603 (define_insn "xop_phadd<u>wq"
16604 [(set (match_operand:V2DI 0 "register_operand" "=x")
16609 (match_operand:V8HI 1 "nonimmediate_operand" "xm")
16610 (parallel [(const_int 0) (const_int 4)])))
16614 (parallel [(const_int 1) (const_int 5)]))))
16619 (parallel [(const_int 2) (const_int 6)])))
16623 (parallel [(const_int 3) (const_int 7)]))))))]
16625 "vphadd<u>wq\t{%1, %0|%0, %1}"
16626 [(set_attr "type" "sseiadd1")])
16628 (define_insn "xop_phadd<u>dq"
16629 [(set (match_operand:V2DI 0 "register_operand" "=x")
16633 (match_operand:V4SI 1 "nonimmediate_operand" "xm")
16634 (parallel [(const_int 0) (const_int 2)])))
16638 (parallel [(const_int 1) (const_int 3)])))))]
16640 "vphadd<u>dq\t{%1, %0|%0, %1}"
16641 [(set_attr "type" "sseiadd1")])
16643 (define_insn "xop_phsubbw"
16644 [(set (match_operand:V8HI 0 "register_operand" "=x")
16648 (match_operand:V16QI 1 "nonimmediate_operand" "xm")
16649 (parallel [(const_int 0) (const_int 2)
16650 (const_int 4) (const_int 6)
16651 (const_int 8) (const_int 10)
16652 (const_int 12) (const_int 14)])))
16656 (parallel [(const_int 1) (const_int 3)
16657 (const_int 5) (const_int 7)
16658 (const_int 9) (const_int 11)
16659 (const_int 13) (const_int 15)])))))]
16661 "vphsubbw\t{%1, %0|%0, %1}"
16662 [(set_attr "type" "sseiadd1")])
16664 (define_insn "xop_phsubwd"
16665 [(set (match_operand:V4SI 0 "register_operand" "=x")
16669 (match_operand:V8HI 1 "nonimmediate_operand" "xm")
16670 (parallel [(const_int 0) (const_int 2)
16671 (const_int 4) (const_int 6)])))
16675 (parallel [(const_int 1) (const_int 3)
16676 (const_int 5) (const_int 7)])))))]
16678 "vphsubwd\t{%1, %0|%0, %1}"
16679 [(set_attr "type" "sseiadd1")])
16681 (define_insn "xop_phsubdq"
16682 [(set (match_operand:V2DI 0 "register_operand" "=x")
16686 (match_operand:V4SI 1 "nonimmediate_operand" "xm")
16687 (parallel [(const_int 0) (const_int 2)])))
16691 (parallel [(const_int 1) (const_int 3)])))))]
16693 "vphsubdq\t{%1, %0|%0, %1}"
16694 [(set_attr "type" "sseiadd1")])
16696 ;; XOP permute instructions
16697 (define_insn "xop_pperm"
16698 [(set (match_operand:V16QI 0 "register_operand" "=x,x")
16700 [(match_operand:V16QI 1 "register_operand" "x,x")
16701 (match_operand:V16QI 2 "nonimmediate_operand" "x,m")
16702 (match_operand:V16QI 3 "nonimmediate_operand" "xm,x")]
16703 UNSPEC_XOP_PERMUTE))]
16704 "TARGET_XOP && !(MEM_P (operands[2]) && MEM_P (operands[3]))"
16705 "vpperm\t{%3, %2, %1, %0|%0, %1, %2, %3}"
16706 [(set_attr "type" "sse4arg")
16707 (set_attr "mode" "TI")])
16709 ;; XOP pack instructions that combine two vectors into a smaller vector
16710 (define_insn "xop_pperm_pack_v2di_v4si"
16711 [(set (match_operand:V4SI 0 "register_operand" "=x,x")
16714 (match_operand:V2DI 1 "register_operand" "x,x"))
16716 (match_operand:V2DI 2 "nonimmediate_operand" "x,m"))))
16717 (use (match_operand:V16QI 3 "nonimmediate_operand" "xm,x"))]
16718 "TARGET_XOP && !(MEM_P (operands[2]) && MEM_P (operands[3]))"
16719 "vpperm\t{%3, %2, %1, %0|%0, %1, %2, %3}"
16720 [(set_attr "type" "sse4arg")
16721 (set_attr "mode" "TI")])
16723 (define_insn "xop_pperm_pack_v4si_v8hi"
16724 [(set (match_operand:V8HI 0 "register_operand" "=x,x")
16727 (match_operand:V4SI 1 "register_operand" "x,x"))
16729 (match_operand:V4SI 2 "nonimmediate_operand" "x,m"))))
16730 (use (match_operand:V16QI 3 "nonimmediate_operand" "xm,x"))]
16731 "TARGET_XOP && !(MEM_P (operands[2]) && MEM_P (operands[3]))"
16732 "vpperm\t{%3, %2, %1, %0|%0, %1, %2, %3}"
16733 [(set_attr "type" "sse4arg")
16734 (set_attr "mode" "TI")])
16736 (define_insn "xop_pperm_pack_v8hi_v16qi"
16737 [(set (match_operand:V16QI 0 "register_operand" "=x,x")
16740 (match_operand:V8HI 1 "register_operand" "x,x"))
16742 (match_operand:V8HI 2 "nonimmediate_operand" "x,m"))))
16743 (use (match_operand:V16QI 3 "nonimmediate_operand" "xm,x"))]
16744 "TARGET_XOP && !(MEM_P (operands[2]) && MEM_P (operands[3]))"
16745 "vpperm\t{%3, %2, %1, %0|%0, %1, %2, %3}"
16746 [(set_attr "type" "sse4arg")
16747 (set_attr "mode" "TI")])
16749 ;; XOP packed rotate instructions
16750 (define_expand "rotl<mode>3"
16751 [(set (match_operand:VI_128 0 "register_operand")
16753 (match_operand:VI_128 1 "nonimmediate_operand")
16754 (match_operand:SI 2 "general_operand")))]
16757 /* If we were given a scalar, convert it to parallel */
16758 if (! const_0_to_<sserotatemax>_operand (operands[2], SImode))
16760 rtvec vs = rtvec_alloc (<ssescalarnum>);
16761 rtx par = gen_rtx_PARALLEL (<MODE>mode, vs);
16762 rtx reg = gen_reg_rtx (<MODE>mode);
16763 rtx op2 = operands[2];
16766 if (GET_MODE (op2) != <ssescalarmode>mode)
16768 op2 = gen_reg_rtx (<ssescalarmode>mode);
16769 convert_move (op2, operands[2], false);
16772 for (i = 0; i < <ssescalarnum>; i++)
16773 RTVEC_ELT (vs, i) = op2;
16775 emit_insn (gen_vec_init<mode><ssescalarmodelower> (reg, par));
16776 emit_insn (gen_xop_vrotl<mode>3 (operands[0], operands[1], reg));
16781 (define_expand "rotr<mode>3"
16782 [(set (match_operand:VI_128 0 "register_operand")
16784 (match_operand:VI_128 1 "nonimmediate_operand")
16785 (match_operand:SI 2 "general_operand")))]
16788 /* If we were given a scalar, convert it to parallel */
16789 if (! const_0_to_<sserotatemax>_operand (operands[2], SImode))
16791 rtvec vs = rtvec_alloc (<ssescalarnum>);
16792 rtx par = gen_rtx_PARALLEL (<MODE>mode, vs);
16793 rtx neg = gen_reg_rtx (<MODE>mode);
16794 rtx reg = gen_reg_rtx (<MODE>mode);
16795 rtx op2 = operands[2];
16798 if (GET_MODE (op2) != <ssescalarmode>mode)
16800 op2 = gen_reg_rtx (<ssescalarmode>mode);
16801 convert_move (op2, operands[2], false);
16804 for (i = 0; i < <ssescalarnum>; i++)
16805 RTVEC_ELT (vs, i) = op2;
16807 emit_insn (gen_vec_init<mode><ssescalarmodelower> (reg, par));
16808 emit_insn (gen_neg<mode>2 (neg, reg));
16809 emit_insn (gen_xop_vrotl<mode>3 (operands[0], operands[1], neg));
16814 (define_insn "xop_rotl<mode>3"
16815 [(set (match_operand:VI_128 0 "register_operand" "=x")
16817 (match_operand:VI_128 1 "nonimmediate_operand" "xm")
16818 (match_operand:SI 2 "const_0_to_<sserotatemax>_operand" "n")))]
16820 "vprot<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
16821 [(set_attr "type" "sseishft")
16822 (set_attr "length_immediate" "1")
16823 (set_attr "mode" "TI")])
16825 (define_insn "xop_rotr<mode>3"
16826 [(set (match_operand:VI_128 0 "register_operand" "=x")
16828 (match_operand:VI_128 1 "nonimmediate_operand" "xm")
16829 (match_operand:SI 2 "const_0_to_<sserotatemax>_operand" "n")))]
16833 = GEN_INT (GET_MODE_BITSIZE (<ssescalarmode>mode) - INTVAL (operands[2]));
16834 return \"vprot<ssemodesuffix>\t{%3, %1, %0|%0, %1, %3}\";
16836 [(set_attr "type" "sseishft")
16837 (set_attr "length_immediate" "1")
16838 (set_attr "mode" "TI")])
16840 (define_expand "vrotr<mode>3"
16841 [(match_operand:VI_128 0 "register_operand")
16842 (match_operand:VI_128 1 "register_operand")
16843 (match_operand:VI_128 2 "register_operand")]
16846 rtx reg = gen_reg_rtx (<MODE>mode);
16847 emit_insn (gen_neg<mode>2 (reg, operands[2]));
16848 emit_insn (gen_xop_vrotl<mode>3 (operands[0], operands[1], reg));
16852 (define_expand "vrotl<mode>3"
16853 [(match_operand:VI_128 0 "register_operand")
16854 (match_operand:VI_128 1 "register_operand")
16855 (match_operand:VI_128 2 "register_operand")]
16858 emit_insn (gen_xop_vrotl<mode>3 (operands[0], operands[1], operands[2]));
16862 (define_insn "xop_vrotl<mode>3"
16863 [(set (match_operand:VI_128 0 "register_operand" "=x,x")
16864 (if_then_else:VI_128
16866 (match_operand:VI_128 2 "nonimmediate_operand" "x,m")
16869 (match_operand:VI_128 1 "nonimmediate_operand" "xm,x")
16873 (neg:VI_128 (match_dup 2)))))]
16874 "TARGET_XOP && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
16875 "vprot<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
16876 [(set_attr "type" "sseishft")
16877 (set_attr "prefix_data16" "0")
16878 (set_attr "prefix_extra" "2")
16879 (set_attr "mode" "TI")])
16881 ;; XOP packed shift instructions.
16882 (define_expand "vlshr<mode>3"
16883 [(set (match_operand:VI12_128 0 "register_operand")
16885 (match_operand:VI12_128 1 "register_operand")
16886 (match_operand:VI12_128 2 "nonimmediate_operand")))]
16889 rtx neg = gen_reg_rtx (<MODE>mode);
16890 emit_insn (gen_neg<mode>2 (neg, operands[2]));
16891 emit_insn (gen_xop_shl<mode>3 (operands[0], operands[1], neg));
16895 (define_expand "vlshr<mode>3"
16896 [(set (match_operand:VI48_128 0 "register_operand")
16898 (match_operand:VI48_128 1 "register_operand")
16899 (match_operand:VI48_128 2 "nonimmediate_operand")))]
16900 "TARGET_AVX2 || TARGET_XOP"
16904 rtx neg = gen_reg_rtx (<MODE>mode);
16905 emit_insn (gen_neg<mode>2 (neg, operands[2]));
16906 emit_insn (gen_xop_shl<mode>3 (operands[0], operands[1], neg));
16911 (define_expand "vlshr<mode>3"
16912 [(set (match_operand:VI48_512 0 "register_operand")
16914 (match_operand:VI48_512 1 "register_operand")
16915 (match_operand:VI48_512 2 "nonimmediate_operand")))]
16918 (define_expand "vlshr<mode>3"
16919 [(set (match_operand:VI48_256 0 "register_operand")
16921 (match_operand:VI48_256 1 "register_operand")
16922 (match_operand:VI48_256 2 "nonimmediate_operand")))]
16925 (define_expand "vashrv8hi3<mask_name>"
16926 [(set (match_operand:V8HI 0 "register_operand")
16928 (match_operand:V8HI 1 "register_operand")
16929 (match_operand:V8HI 2 "nonimmediate_operand")))]
16930 "TARGET_XOP || (TARGET_AVX512BW && TARGET_AVX512VL)"
16934 rtx neg = gen_reg_rtx (V8HImode);
16935 emit_insn (gen_negv8hi2 (neg, operands[2]));
16936 emit_insn (gen_xop_shav8hi3 (operands[0], operands[1], neg));
16941 (define_expand "vashrv16qi3"
16942 [(set (match_operand:V16QI 0 "register_operand")
16944 (match_operand:V16QI 1 "register_operand")
16945 (match_operand:V16QI 2 "nonimmediate_operand")))]
16948 rtx neg = gen_reg_rtx (V16QImode);
16949 emit_insn (gen_negv16qi2 (neg, operands[2]));
16950 emit_insn (gen_xop_shav16qi3 (operands[0], operands[1], neg));
16954 (define_expand "vashrv2di3<mask_name>"
16955 [(set (match_operand:V2DI 0 "register_operand")
16957 (match_operand:V2DI 1 "register_operand")
16958 (match_operand:V2DI 2 "nonimmediate_operand")))]
16959 "TARGET_XOP || TARGET_AVX512VL"
16963 rtx neg = gen_reg_rtx (V2DImode);
16964 emit_insn (gen_negv2di2 (neg, operands[2]));
16965 emit_insn (gen_xop_shav2di3 (operands[0], operands[1], neg));
16970 (define_expand "vashrv4si3"
16971 [(set (match_operand:V4SI 0 "register_operand")
16972 (ashiftrt:V4SI (match_operand:V4SI 1 "register_operand")
16973 (match_operand:V4SI 2 "nonimmediate_operand")))]
16974 "TARGET_AVX2 || TARGET_XOP"
16978 rtx neg = gen_reg_rtx (V4SImode);
16979 emit_insn (gen_negv4si2 (neg, operands[2]));
16980 emit_insn (gen_xop_shav4si3 (operands[0], operands[1], neg));
16985 (define_expand "vashrv16si3"
16986 [(set (match_operand:V16SI 0 "register_operand")
16987 (ashiftrt:V16SI (match_operand:V16SI 1 "register_operand")
16988 (match_operand:V16SI 2 "nonimmediate_operand")))]
16991 (define_expand "vashrv8si3"
16992 [(set (match_operand:V8SI 0 "register_operand")
16993 (ashiftrt:V8SI (match_operand:V8SI 1 "register_operand")
16994 (match_operand:V8SI 2 "nonimmediate_operand")))]
16997 (define_expand "vashl<mode>3"
16998 [(set (match_operand:VI12_128 0 "register_operand")
17000 (match_operand:VI12_128 1 "register_operand")
17001 (match_operand:VI12_128 2 "nonimmediate_operand")))]
17004 emit_insn (gen_xop_sha<mode>3 (operands[0], operands[1], operands[2]));
17008 (define_expand "vashl<mode>3"
17009 [(set (match_operand:VI48_128 0 "register_operand")
17011 (match_operand:VI48_128 1 "register_operand")
17012 (match_operand:VI48_128 2 "nonimmediate_operand")))]
17013 "TARGET_AVX2 || TARGET_XOP"
17017 operands[2] = force_reg (<MODE>mode, operands[2]);
17018 emit_insn (gen_xop_sha<mode>3 (operands[0], operands[1], operands[2]));
17023 (define_expand "vashl<mode>3"
17024 [(set (match_operand:VI48_512 0 "register_operand")
17026 (match_operand:VI48_512 1 "register_operand")
17027 (match_operand:VI48_512 2 "nonimmediate_operand")))]
17030 (define_expand "vashl<mode>3"
17031 [(set (match_operand:VI48_256 0 "register_operand")
17033 (match_operand:VI48_256 1 "register_operand")
17034 (match_operand:VI48_256 2 "nonimmediate_operand")))]
17037 (define_insn "xop_sha<mode>3"
17038 [(set (match_operand:VI_128 0 "register_operand" "=x,x")
17039 (if_then_else:VI_128
17041 (match_operand:VI_128 2 "nonimmediate_operand" "x,m")
17044 (match_operand:VI_128 1 "nonimmediate_operand" "xm,x")
17048 (neg:VI_128 (match_dup 2)))))]
17049 "TARGET_XOP && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
17050 "vpsha<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
17051 [(set_attr "type" "sseishft")
17052 (set_attr "prefix_data16" "0")
17053 (set_attr "prefix_extra" "2")
17054 (set_attr "mode" "TI")])
17056 (define_insn "xop_shl<mode>3"
17057 [(set (match_operand:VI_128 0 "register_operand" "=x,x")
17058 (if_then_else:VI_128
17060 (match_operand:VI_128 2 "nonimmediate_operand" "x,m")
17063 (match_operand:VI_128 1 "nonimmediate_operand" "xm,x")
17067 (neg:VI_128 (match_dup 2)))))]
17068 "TARGET_XOP && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
17069 "vpshl<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
17070 [(set_attr "type" "sseishft")
17071 (set_attr "prefix_data16" "0")
17072 (set_attr "prefix_extra" "2")
17073 (set_attr "mode" "TI")])
17075 (define_expand "<shift_insn><mode>3"
17076 [(set (match_operand:VI1_AVX512 0 "register_operand")
17077 (any_shift:VI1_AVX512
17078 (match_operand:VI1_AVX512 1 "register_operand")
17079 (match_operand:SI 2 "nonmemory_operand")))]
17082 if (TARGET_XOP && <MODE>mode == V16QImode)
17084 bool negate = false;
17085 rtx (*gen) (rtx, rtx, rtx);
17089 if (<CODE> != ASHIFT)
17091 if (CONST_INT_P (operands[2]))
17092 operands[2] = GEN_INT (-INTVAL (operands[2]));
17096 par = gen_rtx_PARALLEL (V16QImode, rtvec_alloc (16));
17097 for (i = 0; i < 16; i++)
17098 XVECEXP (par, 0, i) = operands[2];
17100 tmp = gen_reg_rtx (V16QImode);
17101 emit_insn (gen_vec_initv16qiqi (tmp, par));
17104 emit_insn (gen_negv16qi2 (tmp, tmp));
17106 gen = (<CODE> == LSHIFTRT ? gen_xop_shlv16qi3 : gen_xop_shav16qi3);
17107 emit_insn (gen (operands[0], operands[1], tmp));
17110 ix86_expand_vecop_qihi (<CODE>, operands[0], operands[1], operands[2]);
17114 (define_expand "ashrv2di3"
17115 [(set (match_operand:V2DI 0 "register_operand")
17117 (match_operand:V2DI 1 "register_operand")
17118 (match_operand:DI 2 "nonmemory_operand")))]
17119 "TARGET_XOP || TARGET_AVX512VL"
17121 if (!TARGET_AVX512VL)
17123 rtx reg = gen_reg_rtx (V2DImode);
17125 bool negate = false;
17128 if (CONST_INT_P (operands[2]))
17129 operands[2] = GEN_INT (-INTVAL (operands[2]));
17133 par = gen_rtx_PARALLEL (V2DImode, rtvec_alloc (2));
17134 for (i = 0; i < 2; i++)
17135 XVECEXP (par, 0, i) = operands[2];
17137 emit_insn (gen_vec_initv2didi (reg, par));
17140 emit_insn (gen_negv2di2 (reg, reg));
17142 emit_insn (gen_xop_shav2di3 (operands[0], operands[1], reg));
17147 ;; XOP FRCZ support
17148 (define_insn "xop_frcz<mode>2"
17149 [(set (match_operand:FMAMODE 0 "register_operand" "=x")
17151 [(match_operand:FMAMODE 1 "nonimmediate_operand" "xm")]
17154 "vfrcz<ssemodesuffix>\t{%1, %0|%0, %1}"
17155 [(set_attr "type" "ssecvt1")
17156 (set_attr "mode" "<MODE>")])
17158 (define_expand "xop_vmfrcz<mode>2"
17159 [(set (match_operand:VF_128 0 "register_operand")
17162 [(match_operand:VF_128 1 "nonimmediate_operand")]
17167 "operands[2] = CONST0_RTX (<MODE>mode);")
17169 (define_insn "*xop_vmfrcz<mode>2"
17170 [(set (match_operand:VF_128 0 "register_operand" "=x")
17173 [(match_operand:VF_128 1 "nonimmediate_operand" "xm")]
17175 (match_operand:VF_128 2 "const0_operand")
17178 "vfrcz<ssescalarmodesuffix>\t{%1, %0|%0, %<iptr>1}"
17179 [(set_attr "type" "ssecvt1")
17180 (set_attr "mode" "<MODE>")])
17182 (define_insn "xop_maskcmp<mode>3"
17183 [(set (match_operand:VI_128 0 "register_operand" "=x")
17184 (match_operator:VI_128 1 "ix86_comparison_int_operator"
17185 [(match_operand:VI_128 2 "register_operand" "x")
17186 (match_operand:VI_128 3 "nonimmediate_operand" "xm")]))]
17188 "vpcom%Y1<ssemodesuffix>\t{%3, %2, %0|%0, %2, %3}"
17189 [(set_attr "type" "sse4arg")
17190 (set_attr "prefix_data16" "0")
17191 (set_attr "prefix_rep" "0")
17192 (set_attr "prefix_extra" "2")
17193 (set_attr "length_immediate" "1")
17194 (set_attr "mode" "TI")])
17196 (define_insn "xop_maskcmp_uns<mode>3"
17197 [(set (match_operand:VI_128 0 "register_operand" "=x")
17198 (match_operator:VI_128 1 "ix86_comparison_uns_operator"
17199 [(match_operand:VI_128 2 "register_operand" "x")
17200 (match_operand:VI_128 3 "nonimmediate_operand" "xm")]))]
17202 "vpcom%Y1u<ssemodesuffix>\t{%3, %2, %0|%0, %2, %3}"
17203 [(set_attr "type" "ssecmp")
17204 (set_attr "prefix_data16" "0")
17205 (set_attr "prefix_rep" "0")
17206 (set_attr "prefix_extra" "2")
17207 (set_attr "length_immediate" "1")
17208 (set_attr "mode" "TI")])
17210 ;; Version of pcom*u* that is called from the intrinsics that allows pcomequ*
17211 ;; and pcomneu* not to be converted to the signed ones in case somebody needs
17212 ;; the exact instruction generated for the intrinsic.
17213 (define_insn "xop_maskcmp_uns2<mode>3"
17214 [(set (match_operand:VI_128 0 "register_operand" "=x")
17216 [(match_operator:VI_128 1 "ix86_comparison_uns_operator"
17217 [(match_operand:VI_128 2 "register_operand" "x")
17218 (match_operand:VI_128 3 "nonimmediate_operand" "xm")])]
17219 UNSPEC_XOP_UNSIGNED_CMP))]
17221 "vpcom%Y1u<ssemodesuffix>\t{%3, %2, %0|%0, %2, %3}"
17222 [(set_attr "type" "ssecmp")
17223 (set_attr "prefix_data16" "0")
17224 (set_attr "prefix_extra" "2")
17225 (set_attr "length_immediate" "1")
17226 (set_attr "mode" "TI")])
17228 ;; Pcomtrue and pcomfalse support. These are useless instructions, but are
17229 ;; being added here to be complete.
17230 (define_insn "xop_pcom_tf<mode>3"
17231 [(set (match_operand:VI_128 0 "register_operand" "=x")
17233 [(match_operand:VI_128 1 "register_operand" "x")
17234 (match_operand:VI_128 2 "nonimmediate_operand" "xm")
17235 (match_operand:SI 3 "const_int_operand" "n")]
17236 UNSPEC_XOP_TRUEFALSE))]
17239 return ((INTVAL (operands[3]) != 0)
17240 ? "vpcomtrue<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
17241 : "vpcomfalse<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}");
17243 [(set_attr "type" "ssecmp")
17244 (set_attr "prefix_data16" "0")
17245 (set_attr "prefix_extra" "2")
17246 (set_attr "length_immediate" "1")
17247 (set_attr "mode" "TI")])
17249 (define_insn "xop_vpermil2<mode>3"
17250 [(set (match_operand:VF_128_256 0 "register_operand" "=x,x")
17252 [(match_operand:VF_128_256 1 "register_operand" "x,x")
17253 (match_operand:VF_128_256 2 "nonimmediate_operand" "x,m")
17254 (match_operand:<sseintvecmode> 3 "nonimmediate_operand" "xm,x")
17255 (match_operand:SI 4 "const_0_to_3_operand" "n,n")]
17258 "vpermil2<ssemodesuffix>\t{%4, %3, %2, %1, %0|%0, %1, %2, %3, %4}"
17259 [(set_attr "type" "sse4arg")
17260 (set_attr "length_immediate" "1")
17261 (set_attr "mode" "<MODE>")])
17263 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
17265 (define_insn "aesenc"
17266 [(set (match_operand:V2DI 0 "register_operand" "=x,x")
17267 (unspec:V2DI [(match_operand:V2DI 1 "register_operand" "0,x")
17268 (match_operand:V2DI 2 "vector_operand" "xBm,xm")]
17272 aesenc\t{%2, %0|%0, %2}
17273 vaesenc\t{%2, %1, %0|%0, %1, %2}"
17274 [(set_attr "isa" "noavx,avx")
17275 (set_attr "type" "sselog1")
17276 (set_attr "prefix_extra" "1")
17277 (set_attr "prefix" "orig,vex")
17278 (set_attr "btver2_decode" "double,double")
17279 (set_attr "mode" "TI")])
17281 (define_insn "aesenclast"
17282 [(set (match_operand:V2DI 0 "register_operand" "=x,x")
17283 (unspec:V2DI [(match_operand:V2DI 1 "register_operand" "0,x")
17284 (match_operand:V2DI 2 "vector_operand" "xBm,xm")]
17285 UNSPEC_AESENCLAST))]
17288 aesenclast\t{%2, %0|%0, %2}
17289 vaesenclast\t{%2, %1, %0|%0, %1, %2}"
17290 [(set_attr "isa" "noavx,avx")
17291 (set_attr "type" "sselog1")
17292 (set_attr "prefix_extra" "1")
17293 (set_attr "prefix" "orig,vex")
17294 (set_attr "btver2_decode" "double,double")
17295 (set_attr "mode" "TI")])
17297 (define_insn "aesdec"
17298 [(set (match_operand:V2DI 0 "register_operand" "=x,x")
17299 (unspec:V2DI [(match_operand:V2DI 1 "register_operand" "0,x")
17300 (match_operand:V2DI 2 "vector_operand" "xBm,xm")]
17304 aesdec\t{%2, %0|%0, %2}
17305 vaesdec\t{%2, %1, %0|%0, %1, %2}"
17306 [(set_attr "isa" "noavx,avx")
17307 (set_attr "type" "sselog1")
17308 (set_attr "prefix_extra" "1")
17309 (set_attr "prefix" "orig,vex")
17310 (set_attr "btver2_decode" "double,double")
17311 (set_attr "mode" "TI")])
17313 (define_insn "aesdeclast"
17314 [(set (match_operand:V2DI 0 "register_operand" "=x,x")
17315 (unspec:V2DI [(match_operand:V2DI 1 "register_operand" "0,x")
17316 (match_operand:V2DI 2 "vector_operand" "xBm,xm")]
17317 UNSPEC_AESDECLAST))]
17320 aesdeclast\t{%2, %0|%0, %2}
17321 vaesdeclast\t{%2, %1, %0|%0, %1, %2}"
17322 [(set_attr "isa" "noavx,avx")
17323 (set_attr "type" "sselog1")
17324 (set_attr "prefix_extra" "1")
17325 (set_attr "prefix" "orig,vex")
17326 (set_attr "btver2_decode" "double,double")
17327 (set_attr "mode" "TI")])
17329 (define_insn "aesimc"
17330 [(set (match_operand:V2DI 0 "register_operand" "=x")
17331 (unspec:V2DI [(match_operand:V2DI 1 "vector_operand" "xBm")]
17334 "%vaesimc\t{%1, %0|%0, %1}"
17335 [(set_attr "type" "sselog1")
17336 (set_attr "prefix_extra" "1")
17337 (set_attr "prefix" "maybe_vex")
17338 (set_attr "mode" "TI")])
17340 (define_insn "aeskeygenassist"
17341 [(set (match_operand:V2DI 0 "register_operand" "=x")
17342 (unspec:V2DI [(match_operand:V2DI 1 "vector_operand" "xBm")
17343 (match_operand:SI 2 "const_0_to_255_operand" "n")]
17344 UNSPEC_AESKEYGENASSIST))]
17346 "%vaeskeygenassist\t{%2, %1, %0|%0, %1, %2}"
17347 [(set_attr "type" "sselog1")
17348 (set_attr "prefix_extra" "1")
17349 (set_attr "length_immediate" "1")
17350 (set_attr "prefix" "maybe_vex")
17351 (set_attr "mode" "TI")])
17353 (define_insn "pclmulqdq"
17354 [(set (match_operand:V2DI 0 "register_operand" "=x,x")
17355 (unspec:V2DI [(match_operand:V2DI 1 "register_operand" "0,x")
17356 (match_operand:V2DI 2 "vector_operand" "xBm,xm")
17357 (match_operand:SI 3 "const_0_to_255_operand" "n,n")]
17361 pclmulqdq\t{%3, %2, %0|%0, %2, %3}
17362 vpclmulqdq\t{%3, %2, %1, %0|%0, %1, %2, %3}"
17363 [(set_attr "isa" "noavx,avx")
17364 (set_attr "type" "sselog1")
17365 (set_attr "prefix_extra" "1")
17366 (set_attr "length_immediate" "1")
17367 (set_attr "prefix" "orig,vex")
17368 (set_attr "mode" "TI")])
17370 (define_expand "avx_vzeroall"
17371 [(match_par_dup 0 [(const_int 0)])]
17374 int nregs = TARGET_64BIT ? 16 : 8;
17377 operands[0] = gen_rtx_PARALLEL (VOIDmode, rtvec_alloc (nregs + 1));
17379 XVECEXP (operands[0], 0, 0)
17380 = gen_rtx_UNSPEC_VOLATILE (VOIDmode, gen_rtvec (1, const0_rtx),
17383 for (regno = 0; regno < nregs; regno++)
17384 XVECEXP (operands[0], 0, regno + 1)
17385 = gen_rtx_SET (gen_rtx_REG (V8SImode, SSE_REGNO (regno)),
17386 CONST0_RTX (V8SImode));
17389 (define_insn "*avx_vzeroall"
17390 [(match_parallel 0 "vzeroall_operation"
17391 [(unspec_volatile [(const_int 0)] UNSPECV_VZEROALL)])]
17394 [(set_attr "type" "sse")
17395 (set_attr "modrm" "0")
17396 (set_attr "memory" "none")
17397 (set_attr "prefix" "vex")
17398 (set_attr "btver2_decode" "vector")
17399 (set_attr "mode" "OI")])
17401 ;; Clear the upper 128bits of AVX registers, equivalent to a NOP
17402 ;; if the upper 128bits are unused.
17403 (define_insn "avx_vzeroupper"
17404 [(unspec_volatile [(const_int 0)] UNSPECV_VZEROUPPER)]
17407 [(set_attr "type" "sse")
17408 (set_attr "modrm" "0")
17409 (set_attr "memory" "none")
17410 (set_attr "prefix" "vex")
17411 (set_attr "btver2_decode" "vector")
17412 (set_attr "mode" "OI")])
17414 (define_mode_attr pbroadcast_evex_isa
17415 [(V64QI "avx512bw") (V32QI "avx512bw") (V16QI "avx512bw")
17416 (V32HI "avx512bw") (V16HI "avx512bw") (V8HI "avx512bw")
17417 (V16SI "avx512f") (V8SI "avx512f") (V4SI "avx512f")
17418 (V8DI "avx512f") (V4DI "avx512f") (V2DI "avx512f")])
17420 (define_insn "avx2_pbroadcast<mode>"
17421 [(set (match_operand:VI 0 "register_operand" "=x,v")
17423 (vec_select:<ssescalarmode>
17424 (match_operand:<ssexmmmode> 1 "nonimmediate_operand" "xm,vm")
17425 (parallel [(const_int 0)]))))]
17427 "vpbroadcast<ssemodesuffix>\t{%1, %0|%0, %<iptr>1}"
17428 [(set_attr "isa" "*,<pbroadcast_evex_isa>")
17429 (set_attr "type" "ssemov")
17430 (set_attr "prefix_extra" "1")
17431 (set_attr "prefix" "vex,evex")
17432 (set_attr "mode" "<sseinsnmode>")])
17434 (define_insn "avx2_pbroadcast<mode>_1"
17435 [(set (match_operand:VI_256 0 "register_operand" "=x,x,v,v")
17436 (vec_duplicate:VI_256
17437 (vec_select:<ssescalarmode>
17438 (match_operand:VI_256 1 "nonimmediate_operand" "m,x,m,v")
17439 (parallel [(const_int 0)]))))]
17442 vpbroadcast<ssemodesuffix>\t{%1, %0|%0, %<iptr>1}
17443 vpbroadcast<ssemodesuffix>\t{%x1, %0|%0, %x1}
17444 vpbroadcast<ssemodesuffix>\t{%1, %0|%0, %<iptr>1}
17445 vpbroadcast<ssemodesuffix>\t{%x1, %0|%0, %x1}"
17446 [(set_attr "isa" "*,*,<pbroadcast_evex_isa>,<pbroadcast_evex_isa>")
17447 (set_attr "type" "ssemov")
17448 (set_attr "prefix_extra" "1")
17449 (set_attr "prefix" "vex")
17450 (set_attr "mode" "<sseinsnmode>")])
17452 (define_insn "<avx2_avx512>_permvar<mode><mask_name>"
17453 [(set (match_operand:VI48F_256_512 0 "register_operand" "=v")
17454 (unspec:VI48F_256_512
17455 [(match_operand:VI48F_256_512 1 "nonimmediate_operand" "vm")
17456 (match_operand:<sseintvecmode> 2 "register_operand" "v")]
17458 "TARGET_AVX2 && <mask_mode512bit_condition>"
17459 "vperm<ssemodesuffix>\t{%1, %2, %0<mask_operand3>|%0<mask_operand3>, %2, %1}"
17460 [(set_attr "type" "sselog")
17461 (set_attr "prefix" "<mask_prefix2>")
17462 (set_attr "mode" "<sseinsnmode>")])
17464 (define_insn "<avx512>_permvar<mode><mask_name>"
17465 [(set (match_operand:VI1_AVX512VL 0 "register_operand" "=v")
17466 (unspec:VI1_AVX512VL
17467 [(match_operand:VI1_AVX512VL 1 "nonimmediate_operand" "vm")
17468 (match_operand:<sseintvecmode> 2 "register_operand" "v")]
17470 "TARGET_AVX512VBMI && <mask_mode512bit_condition>"
17471 "vperm<ssemodesuffix>\t{%1, %2, %0<mask_operand3>|%0<mask_operand3>, %2, %1}"
17472 [(set_attr "type" "sselog")
17473 (set_attr "prefix" "<mask_prefix2>")
17474 (set_attr "mode" "<sseinsnmode>")])
17476 (define_insn "<avx512>_permvar<mode><mask_name>"
17477 [(set (match_operand:VI2_AVX512VL 0 "register_operand" "=v")
17478 (unspec:VI2_AVX512VL
17479 [(match_operand:VI2_AVX512VL 1 "nonimmediate_operand" "vm")
17480 (match_operand:<sseintvecmode> 2 "register_operand" "v")]
17482 "TARGET_AVX512BW && <mask_mode512bit_condition>"
17483 "vperm<ssemodesuffix>\t{%1, %2, %0<mask_operand3>|%0<mask_operand3>, %2, %1}"
17484 [(set_attr "type" "sselog")
17485 (set_attr "prefix" "<mask_prefix2>")
17486 (set_attr "mode" "<sseinsnmode>")])
17488 (define_expand "avx2_perm<mode>"
17489 [(match_operand:VI8F_256 0 "register_operand")
17490 (match_operand:VI8F_256 1 "nonimmediate_operand")
17491 (match_operand:SI 2 "const_0_to_255_operand")]
17494 int mask = INTVAL (operands[2]);
17495 emit_insn (gen_avx2_perm<mode>_1 (operands[0], operands[1],
17496 GEN_INT ((mask >> 0) & 3),
17497 GEN_INT ((mask >> 2) & 3),
17498 GEN_INT ((mask >> 4) & 3),
17499 GEN_INT ((mask >> 6) & 3)));
17503 (define_expand "avx512vl_perm<mode>_mask"
17504 [(match_operand:VI8F_256 0 "register_operand")
17505 (match_operand:VI8F_256 1 "nonimmediate_operand")
17506 (match_operand:SI 2 "const_0_to_255_operand")
17507 (match_operand:VI8F_256 3 "vector_move_operand")
17508 (match_operand:<avx512fmaskmode> 4 "register_operand")]
17511 int mask = INTVAL (operands[2]);
17512 emit_insn (gen_<avx2_avx512>_perm<mode>_1_mask (operands[0], operands[1],
17513 GEN_INT ((mask >> 0) & 3),
17514 GEN_INT ((mask >> 2) & 3),
17515 GEN_INT ((mask >> 4) & 3),
17516 GEN_INT ((mask >> 6) & 3),
17517 operands[3], operands[4]));
17521 (define_insn "avx2_perm<mode>_1<mask_name>"
17522 [(set (match_operand:VI8F_256 0 "register_operand" "=v")
17523 (vec_select:VI8F_256
17524 (match_operand:VI8F_256 1 "nonimmediate_operand" "vm")
17525 (parallel [(match_operand 2 "const_0_to_3_operand")
17526 (match_operand 3 "const_0_to_3_operand")
17527 (match_operand 4 "const_0_to_3_operand")
17528 (match_operand 5 "const_0_to_3_operand")])))]
17529 "TARGET_AVX2 && <mask_mode512bit_condition>"
17532 mask |= INTVAL (operands[2]) << 0;
17533 mask |= INTVAL (operands[3]) << 2;
17534 mask |= INTVAL (operands[4]) << 4;
17535 mask |= INTVAL (operands[5]) << 6;
17536 operands[2] = GEN_INT (mask);
17537 return "vperm<ssemodesuffix>\t{%2, %1, %0<mask_operand6>|%0<mask_operand6>, %1, %2}";
17539 [(set_attr "type" "sselog")
17540 (set_attr "prefix" "<mask_prefix2>")
17541 (set_attr "mode" "<sseinsnmode>")])
17543 (define_expand "avx512f_perm<mode>"
17544 [(match_operand:V8FI 0 "register_operand")
17545 (match_operand:V8FI 1 "nonimmediate_operand")
17546 (match_operand:SI 2 "const_0_to_255_operand")]
17549 int mask = INTVAL (operands[2]);
17550 emit_insn (gen_avx512f_perm<mode>_1 (operands[0], operands[1],
17551 GEN_INT ((mask >> 0) & 3),
17552 GEN_INT ((mask >> 2) & 3),
17553 GEN_INT ((mask >> 4) & 3),
17554 GEN_INT ((mask >> 6) & 3),
17555 GEN_INT (((mask >> 0) & 3) + 4),
17556 GEN_INT (((mask >> 2) & 3) + 4),
17557 GEN_INT (((mask >> 4) & 3) + 4),
17558 GEN_INT (((mask >> 6) & 3) + 4)));
17562 (define_expand "avx512f_perm<mode>_mask"
17563 [(match_operand:V8FI 0 "register_operand")
17564 (match_operand:V8FI 1 "nonimmediate_operand")
17565 (match_operand:SI 2 "const_0_to_255_operand")
17566 (match_operand:V8FI 3 "vector_move_operand")
17567 (match_operand:<avx512fmaskmode> 4 "register_operand")]
17570 int mask = INTVAL (operands[2]);
17571 emit_insn (gen_avx512f_perm<mode>_1_mask (operands[0], operands[1],
17572 GEN_INT ((mask >> 0) & 3),
17573 GEN_INT ((mask >> 2) & 3),
17574 GEN_INT ((mask >> 4) & 3),
17575 GEN_INT ((mask >> 6) & 3),
17576 GEN_INT (((mask >> 0) & 3) + 4),
17577 GEN_INT (((mask >> 2) & 3) + 4),
17578 GEN_INT (((mask >> 4) & 3) + 4),
17579 GEN_INT (((mask >> 6) & 3) + 4),
17580 operands[3], operands[4]));
17584 (define_insn "avx512f_perm<mode>_1<mask_name>"
17585 [(set (match_operand:V8FI 0 "register_operand" "=v")
17587 (match_operand:V8FI 1 "nonimmediate_operand" "vm")
17588 (parallel [(match_operand 2 "const_0_to_3_operand")
17589 (match_operand 3 "const_0_to_3_operand")
17590 (match_operand 4 "const_0_to_3_operand")
17591 (match_operand 5 "const_0_to_3_operand")
17592 (match_operand 6 "const_4_to_7_operand")
17593 (match_operand 7 "const_4_to_7_operand")
17594 (match_operand 8 "const_4_to_7_operand")
17595 (match_operand 9 "const_4_to_7_operand")])))]
17596 "TARGET_AVX512F && <mask_mode512bit_condition>
17597 && (INTVAL (operands[2]) == (INTVAL (operands[6]) - 4)
17598 && INTVAL (operands[3]) == (INTVAL (operands[7]) - 4)
17599 && INTVAL (operands[4]) == (INTVAL (operands[8]) - 4)
17600 && INTVAL (operands[5]) == (INTVAL (operands[9]) - 4))"
17603 mask |= INTVAL (operands[2]) << 0;
17604 mask |= INTVAL (operands[3]) << 2;
17605 mask |= INTVAL (operands[4]) << 4;
17606 mask |= INTVAL (operands[5]) << 6;
17607 operands[2] = GEN_INT (mask);
17608 return "vperm<ssemodesuffix>\t{%2, %1, %0<mask_operand10>|%0<mask_operand10>, %1, %2}";
17610 [(set_attr "type" "sselog")
17611 (set_attr "prefix" "<mask_prefix2>")
17612 (set_attr "mode" "<sseinsnmode>")])
17614 (define_insn "avx2_permv2ti"
17615 [(set (match_operand:V4DI 0 "register_operand" "=x")
17617 [(match_operand:V4DI 1 "register_operand" "x")
17618 (match_operand:V4DI 2 "nonimmediate_operand" "xm")
17619 (match_operand:SI 3 "const_0_to_255_operand" "n")]
17622 "vperm2i128\t{%3, %2, %1, %0|%0, %1, %2, %3}"
17623 [(set_attr "type" "sselog")
17624 (set_attr "prefix" "vex")
17625 (set_attr "mode" "OI")])
17627 (define_insn "avx2_vec_dupv4df"
17628 [(set (match_operand:V4DF 0 "register_operand" "=v")
17629 (vec_duplicate:V4DF
17631 (match_operand:V2DF 1 "register_operand" "v")
17632 (parallel [(const_int 0)]))))]
17634 "vbroadcastsd\t{%1, %0|%0, %1}"
17635 [(set_attr "type" "sselog1")
17636 (set_attr "prefix" "maybe_evex")
17637 (set_attr "mode" "V4DF")])
17639 (define_insn "<avx512>_vec_dup<mode>_1"
17640 [(set (match_operand:VI_AVX512BW 0 "register_operand" "=v,v")
17641 (vec_duplicate:VI_AVX512BW
17642 (vec_select:<ssescalarmode>
17643 (match_operand:VI_AVX512BW 1 "nonimmediate_operand" "v,m")
17644 (parallel [(const_int 0)]))))]
17647 vpbroadcast<ssemodesuffix>\t{%x1, %0|%0, %x1}
17648 vpbroadcast<ssemodesuffix>\t{%x1, %0|%0, %<iptr>1}"
17649 [(set_attr "type" "ssemov")
17650 (set_attr "prefix" "evex")
17651 (set_attr "mode" "<sseinsnmode>")])
17653 (define_insn "<avx512>_vec_dup<mode><mask_name>"
17654 [(set (match_operand:V48_AVX512VL 0 "register_operand" "=v")
17655 (vec_duplicate:V48_AVX512VL
17656 (vec_select:<ssescalarmode>
17657 (match_operand:<ssexmmmode> 1 "nonimmediate_operand" "vm")
17658 (parallel [(const_int 0)]))))]
17661 /* There is no DF broadcast (in AVX-512*) to 128b register.
17662 Mimic it with integer variant. */
17663 if (<MODE>mode == V2DFmode)
17664 return "vpbroadcastq\t{%1, %0<mask_operand2>|%0<mask_operand2>, %q1}";
17666 if (GET_MODE_SIZE (GET_MODE_INNER (<MODE>mode)) == 4)
17667 return "v<sseintprefix>broadcast<bcstscalarsuff>\t{%1, %0<mask_operand2>|%0<mask_operand2>, %k1}";
17669 return "v<sseintprefix>broadcast<bcstscalarsuff>\t{%1, %0<mask_operand2>|%0<mask_operand2>, %q1}";
17671 [(set_attr "type" "ssemov")
17672 (set_attr "prefix" "evex")
17673 (set_attr "mode" "<sseinsnmode>")])
17675 (define_insn "<avx512>_vec_dup<mode><mask_name>"
17676 [(set (match_operand:VI12_AVX512VL 0 "register_operand" "=v")
17677 (vec_duplicate:VI12_AVX512VL
17678 (vec_select:<ssescalarmode>
17679 (match_operand:<ssexmmmode> 1 "nonimmediate_operand" "vm")
17680 (parallel [(const_int 0)]))))]
17682 "vpbroadcast<bcstscalarsuff>\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
17683 [(set_attr "type" "ssemov")
17684 (set_attr "prefix" "evex")
17685 (set_attr "mode" "<sseinsnmode>")])
17687 (define_insn "<mask_codefor>avx512f_broadcast<mode><mask_name>"
17688 [(set (match_operand:V16FI 0 "register_operand" "=v,v")
17689 (vec_duplicate:V16FI
17690 (match_operand:<ssexmmmode> 1 "nonimmediate_operand" "v,m")))]
17693 vshuf<shuffletype>32x4\t{$0x0, %g1, %g1, %0<mask_operand2>|%0<mask_operand2>, %g1, %g1, 0x0}
17694 vbroadcast<shuffletype>32x4\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
17695 [(set_attr "type" "ssemov")
17696 (set_attr "prefix" "evex")
17697 (set_attr "mode" "<sseinsnmode>")])
17699 (define_insn "<mask_codefor>avx512f_broadcast<mode><mask_name>"
17700 [(set (match_operand:V8FI 0 "register_operand" "=v,v")
17701 (vec_duplicate:V8FI
17702 (match_operand:<ssehalfvecmode> 1 "nonimmediate_operand" "v,m")))]
17705 vshuf<shuffletype>64x2\t{$0x44, %g1, %g1, %0<mask_operand2>|%0<mask_operand2>, %g1, %g1, 0x44}
17706 vbroadcast<shuffletype>64x4\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
17707 [(set_attr "type" "ssemov")
17708 (set_attr "prefix" "evex")
17709 (set_attr "mode" "<sseinsnmode>")])
17711 (define_insn "<mask_codefor><avx512>_vec_dup_gpr<mode><mask_name>"
17712 [(set (match_operand:VI12_AVX512VL 0 "register_operand" "=v,v")
17713 (vec_duplicate:VI12_AVX512VL
17714 (match_operand:<ssescalarmode> 1 "nonimmediate_operand" "vm,r")))]
17717 vpbroadcast<bcstscalarsuff>\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}
17718 vpbroadcast<bcstscalarsuff>\t{%k1, %0<mask_operand2>|%0<mask_operand2>, %k1}"
17719 [(set_attr "type" "ssemov")
17720 (set_attr "prefix" "evex")
17721 (set_attr "mode" "<sseinsnmode>")])
17723 (define_insn "<mask_codefor><avx512>_vec_dup_gpr<mode><mask_name>"
17724 [(set (match_operand:V48_AVX512VL 0 "register_operand" "=v,v")
17725 (vec_duplicate:V48_AVX512VL
17726 (match_operand:<ssescalarmode> 1 "nonimmediate_operand" "vm,r")))]
17728 "v<sseintprefix>broadcast<bcstscalarsuff>\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
17729 [(set_attr "type" "ssemov")
17730 (set_attr "prefix" "evex")
17731 (set_attr "mode" "<sseinsnmode>")
17732 (set (attr "enabled")
17733 (if_then_else (eq_attr "alternative" "1")
17734 (symbol_ref "GET_MODE_CLASS (<ssescalarmode>mode) == MODE_INT
17735 && (<ssescalarmode>mode != DImode || TARGET_64BIT)")
17738 (define_insn "vec_dupv4sf"
17739 [(set (match_operand:V4SF 0 "register_operand" "=v,v,x")
17740 (vec_duplicate:V4SF
17741 (match_operand:SF 1 "nonimmediate_operand" "Yv,m,0")))]
17744 vshufps\t{$0, %1, %1, %0|%0, %1, %1, 0}
17745 vbroadcastss\t{%1, %0|%0, %1}
17746 shufps\t{$0, %0, %0|%0, %0, 0}"
17747 [(set_attr "isa" "avx,avx,noavx")
17748 (set_attr "type" "sseshuf1,ssemov,sseshuf1")
17749 (set_attr "length_immediate" "1,0,1")
17750 (set_attr "prefix_extra" "0,1,*")
17751 (set_attr "prefix" "maybe_evex,maybe_evex,orig")
17752 (set_attr "mode" "V4SF")])
17754 (define_insn "*vec_dupv4si"
17755 [(set (match_operand:V4SI 0 "register_operand" "=v,v,x")
17756 (vec_duplicate:V4SI
17757 (match_operand:SI 1 "nonimmediate_operand" "Yv,m,0")))]
17760 %vpshufd\t{$0, %1, %0|%0, %1, 0}
17761 vbroadcastss\t{%1, %0|%0, %1}
17762 shufps\t{$0, %0, %0|%0, %0, 0}"
17763 [(set_attr "isa" "sse2,avx,noavx")
17764 (set_attr "type" "sselog1,ssemov,sselog1")
17765 (set_attr "length_immediate" "1,0,1")
17766 (set_attr "prefix_extra" "0,1,*")
17767 (set_attr "prefix" "maybe_vex,maybe_evex,orig")
17768 (set_attr "mode" "TI,V4SF,V4SF")])
17770 (define_insn "*vec_dupv2di"
17771 [(set (match_operand:V2DI 0 "register_operand" "=x,v,v,x")
17772 (vec_duplicate:V2DI
17773 (match_operand:DI 1 "nonimmediate_operand" " 0,Yv,m,0")))]
17777 vpunpcklqdq\t{%d1, %0|%0, %d1}
17778 %vmovddup\t{%1, %0|%0, %1}
17780 [(set_attr "isa" "sse2_noavx,avx,sse3,noavx")
17781 (set_attr "type" "sselog1,sselog1,sselog1,ssemov")
17782 (set_attr "prefix" "orig,maybe_evex,maybe_vex,orig")
17783 (set_attr "mode" "TI,TI,DF,V4SF")])
17785 (define_insn "avx2_vbroadcasti128_<mode>"
17786 [(set (match_operand:VI_256 0 "register_operand" "=x,v,v")
17788 (match_operand:<ssehalfvecmode> 1 "memory_operand" "m,m,m")
17792 vbroadcasti128\t{%1, %0|%0, %1}
17793 vbroadcast<i128vldq>\t{%1, %0|%0, %1}
17794 vbroadcast<shuffletype>32x4\t{%1, %0|%0, %1}"
17795 [(set_attr "isa" "*,avx512dq,avx512vl")
17796 (set_attr "type" "ssemov")
17797 (set_attr "prefix_extra" "1")
17798 (set_attr "prefix" "vex,evex,evex")
17799 (set_attr "mode" "OI")])
17801 ;; Modes handled by AVX vec_dup patterns.
17802 (define_mode_iterator AVX_VEC_DUP_MODE
17803 [V8SI V8SF V4DI V4DF])
17804 (define_mode_attr vecdupssescalarmodesuffix
17805 [(V8SF "ss") (V4DF "sd") (V8SI "ss") (V4DI "sd")])
17806 ;; Modes handled by AVX2 vec_dup patterns.
17807 (define_mode_iterator AVX2_VEC_DUP_MODE
17808 [V32QI V16QI V16HI V8HI V8SI V4SI])
17810 (define_insn "*vec_dup<mode>"
17811 [(set (match_operand:AVX2_VEC_DUP_MODE 0 "register_operand" "=x,x,Yi")
17812 (vec_duplicate:AVX2_VEC_DUP_MODE
17813 (match_operand:<ssescalarmode> 1 "nonimmediate_operand" "m,x,$r")))]
17816 v<sseintprefix>broadcast<bcstscalarsuff>\t{%1, %0|%0, %1}
17817 v<sseintprefix>broadcast<bcstscalarsuff>\t{%x1, %0|%0, %x1}
17819 [(set_attr "isa" "*,*,noavx512vl")
17820 (set_attr "type" "ssemov")
17821 (set_attr "prefix_extra" "1")
17822 (set_attr "prefix" "maybe_evex")
17823 (set_attr "mode" "<sseinsnmode>")])
17825 (define_insn "vec_dup<mode>"
17826 [(set (match_operand:AVX_VEC_DUP_MODE 0 "register_operand" "=x,x,x,v,x")
17827 (vec_duplicate:AVX_VEC_DUP_MODE
17828 (match_operand:<ssescalarmode> 1 "nonimmediate_operand" "m,m,x,v,?x")))]
17831 v<sseintprefix>broadcast<bcstscalarsuff>\t{%1, %0|%0, %1}
17832 vbroadcast<vecdupssescalarmodesuffix>\t{%1, %0|%0, %1}
17833 v<sseintprefix>broadcast<bcstscalarsuff>\t{%x1, %0|%0, %x1}
17834 v<sseintprefix>broadcast<bcstscalarsuff>\t{%x1, %g0|%g0, %x1}
17836 [(set_attr "type" "ssemov")
17837 (set_attr "prefix_extra" "1")
17838 (set_attr "prefix" "maybe_evex")
17839 (set_attr "isa" "avx2,noavx2,avx2,avx512f,noavx2")
17840 (set_attr "mode" "<sseinsnmode>,V8SF,<sseinsnmode>,<sseinsnmode>,V8SF")])
17843 [(set (match_operand:AVX2_VEC_DUP_MODE 0 "register_operand")
17844 (vec_duplicate:AVX2_VEC_DUP_MODE
17845 (match_operand:<ssescalarmode> 1 "register_operand")))]
17847 /* Disable this splitter if avx512vl_vec_dup_gprv*[qhs]i insn is
17848 available, because then we can broadcast from GPRs directly.
17849 For V*[QH]I modes it requires both -mavx512vl and -mavx512bw,
17850 for V*SI mode it requires just -mavx512vl. */
17851 && !(TARGET_AVX512VL
17852 && (TARGET_AVX512BW || <ssescalarmode>mode == SImode))
17853 && reload_completed && GENERAL_REG_P (operands[1])"
17856 emit_insn (gen_vec_setv4si_0 (gen_lowpart (V4SImode, operands[0]),
17857 CONST0_RTX (V4SImode),
17858 gen_lowpart (SImode, operands[1])));
17859 emit_insn (gen_avx2_pbroadcast<mode> (operands[0],
17860 gen_lowpart (<ssexmmmode>mode,
17866 [(set (match_operand:AVX_VEC_DUP_MODE 0 "register_operand")
17867 (vec_duplicate:AVX_VEC_DUP_MODE
17868 (match_operand:<ssescalarmode> 1 "register_operand")))]
17869 "TARGET_AVX && !TARGET_AVX2 && reload_completed"
17870 [(set (match_dup 2)
17871 (vec_duplicate:<ssehalfvecmode> (match_dup 1)))
17873 (vec_concat:AVX_VEC_DUP_MODE (match_dup 2) (match_dup 2)))]
17874 "operands[2] = gen_lowpart (<ssehalfvecmode>mode, operands[0]);")
17876 (define_insn "avx_vbroadcastf128_<mode>"
17877 [(set (match_operand:V_256 0 "register_operand" "=x,x,x,v,v,v,v")
17879 (match_operand:<ssehalfvecmode> 1 "nonimmediate_operand" "m,0,?x,m,0,m,0")
17883 vbroadcast<i128>\t{%1, %0|%0, %1}
17884 vinsert<i128>\t{$1, %1, %0, %0|%0, %0, %1, 1}
17885 vperm2<i128>\t{$0, %t1, %t1, %0|%0, %t1, %t1, 0}
17886 vbroadcast<i128vldq>\t{%1, %0|%0, %1}
17887 vinsert<i128vldq>\t{$1, %1, %0, %0|%0, %0, %1, 1}
17888 vbroadcast<shuffletype>32x4\t{%1, %0|%0, %1}
17889 vinsert<shuffletype>32x4\t{$1, %1, %0, %0|%0, %0, %1, 1}"
17890 [(set_attr "isa" "*,*,*,avx512dq,avx512dq,avx512vl,avx512vl")
17891 (set_attr "type" "ssemov,sselog1,sselog1,ssemov,sselog1,ssemov,sselog1")
17892 (set_attr "prefix_extra" "1")
17893 (set_attr "length_immediate" "0,1,1,0,1,0,1")
17894 (set_attr "prefix" "vex,vex,vex,evex,evex,evex,evex")
17895 (set_attr "mode" "<sseinsnmode>")])
17897 ;; For broadcast[i|f]32x2. Yes there is no v4sf version, only v4si.
17898 (define_mode_iterator VI4F_BRCST32x2
17899 [V16SI (V8SI "TARGET_AVX512VL") (V4SI "TARGET_AVX512VL")
17900 V16SF (V8SF "TARGET_AVX512VL")])
17902 (define_mode_attr 64x2mode
17903 [(V8DF "V2DF") (V8DI "V2DI") (V4DI "V2DI") (V4DF "V2DF")])
17905 (define_mode_attr 32x2mode
17906 [(V16SF "V2SF") (V16SI "V2SI") (V8SI "V2SI")
17907 (V8SF "V2SF") (V4SI "V2SI")])
17909 (define_insn "<mask_codefor>avx512dq_broadcast<mode><mask_name>"
17910 [(set (match_operand:VI4F_BRCST32x2 0 "register_operand" "=v")
17911 (vec_duplicate:VI4F_BRCST32x2
17912 (vec_select:<32x2mode>
17913 (match_operand:<ssexmmmode> 1 "nonimmediate_operand" "vm")
17914 (parallel [(const_int 0) (const_int 1)]))))]
17916 "vbroadcast<shuffletype>32x2\t{%1, %0<mask_operand2>|%0<mask_operand2>, %q1}"
17917 [(set_attr "type" "ssemov")
17918 (set_attr "prefix_extra" "1")
17919 (set_attr "prefix" "evex")
17920 (set_attr "mode" "<sseinsnmode>")])
17922 (define_insn "<mask_codefor>avx512vl_broadcast<mode><mask_name>_1"
17923 [(set (match_operand:VI4F_256 0 "register_operand" "=v,v")
17924 (vec_duplicate:VI4F_256
17925 (match_operand:<ssexmmmode> 1 "nonimmediate_operand" "v,m")))]
17928 vshuf<shuffletype>32x4\t{$0x0, %t1, %t1, %0<mask_operand2>|%0<mask_operand2>, %t1, %t1, 0x0}
17929 vbroadcast<shuffletype>32x4\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
17930 [(set_attr "type" "ssemov")
17931 (set_attr "prefix_extra" "1")
17932 (set_attr "prefix" "evex")
17933 (set_attr "mode" "<sseinsnmode>")])
17935 (define_insn "<mask_codefor>avx512dq_broadcast<mode><mask_name>_1"
17936 [(set (match_operand:V16FI 0 "register_operand" "=v,v")
17937 (vec_duplicate:V16FI
17938 (match_operand:<ssehalfvecmode> 1 "nonimmediate_operand" "v,m")))]
17941 vshuf<shuffletype>32x4\t{$0x44, %g1, %g1, %0<mask_operand2>|%0<mask_operand2>, %g1, %g1, 0x44}
17942 vbroadcast<shuffletype>32x8\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
17943 [(set_attr "type" "ssemov")
17944 (set_attr "prefix_extra" "1")
17945 (set_attr "prefix" "evex")
17946 (set_attr "mode" "<sseinsnmode>")])
17948 ;; For broadcast[i|f]64x2
17949 (define_mode_iterator VI8F_BRCST64x2
17950 [V8DI V8DF (V4DI "TARGET_AVX512VL") (V4DF "TARGET_AVX512VL")])
17952 (define_insn "<mask_codefor>avx512dq_broadcast<mode><mask_name>_1"
17953 [(set (match_operand:VI8F_BRCST64x2 0 "register_operand" "=v,v")
17954 (vec_duplicate:VI8F_BRCST64x2
17955 (match_operand:<64x2mode> 1 "nonimmediate_operand" "v,m")))]
17958 vshuf<shuffletype>64x2\t{$0x0, %<concat_tg_mode>1, %<concat_tg_mode>1, %0<mask_operand2>|%0<mask_operand2>, %<concat_tg_mode>1, %<concat_tg_mode>1, 0x0}
17959 vbroadcast<shuffletype>64x2\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
17960 [(set_attr "type" "ssemov")
17961 (set_attr "prefix_extra" "1")
17962 (set_attr "prefix" "evex")
17963 (set_attr "mode" "<sseinsnmode>")])
17965 (define_insn "avx512cd_maskb_vec_dup<mode>"
17966 [(set (match_operand:VI8_AVX512VL 0 "register_operand" "=v")
17967 (vec_duplicate:VI8_AVX512VL
17969 (match_operand:QI 1 "register_operand" "Yk"))))]
17971 "vpbroadcastmb2q\t{%1, %0|%0, %1}"
17972 [(set_attr "type" "mskmov")
17973 (set_attr "prefix" "evex")
17974 (set_attr "mode" "XI")])
17976 (define_insn "avx512cd_maskw_vec_dup<mode>"
17977 [(set (match_operand:VI4_AVX512VL 0 "register_operand" "=v")
17978 (vec_duplicate:VI4_AVX512VL
17980 (match_operand:HI 1 "register_operand" "Yk"))))]
17982 "vpbroadcastmw2d\t{%1, %0|%0, %1}"
17983 [(set_attr "type" "mskmov")
17984 (set_attr "prefix" "evex")
17985 (set_attr "mode" "XI")])
17987 ;; Recognize broadcast as a vec_select as produced by builtin_vec_perm.
17988 ;; If it so happens that the input is in memory, use vbroadcast.
17989 ;; Otherwise use vpermilp (and in the case of 256-bit modes, vperm2f128).
17990 (define_insn "*avx_vperm_broadcast_v4sf"
17991 [(set (match_operand:V4SF 0 "register_operand" "=v,v,v")
17993 (match_operand:V4SF 1 "nonimmediate_operand" "m,o,v")
17994 (match_parallel 2 "avx_vbroadcast_operand"
17995 [(match_operand 3 "const_int_operand" "C,n,n")])))]
17998 int elt = INTVAL (operands[3]);
17999 switch (which_alternative)
18003 operands[1] = adjust_address_nv (operands[1], SFmode, elt * 4);
18004 return "vbroadcastss\t{%1, %0|%0, %k1}";
18006 operands[2] = GEN_INT (elt * 0x55);
18007 return "vpermilps\t{%2, %1, %0|%0, %1, %2}";
18009 gcc_unreachable ();
18012 [(set_attr "type" "ssemov,ssemov,sselog1")
18013 (set_attr "prefix_extra" "1")
18014 (set_attr "length_immediate" "0,0,1")
18015 (set_attr "prefix" "maybe_evex")
18016 (set_attr "mode" "SF,SF,V4SF")])
18018 (define_insn_and_split "*avx_vperm_broadcast_<mode>"
18019 [(set (match_operand:VF_256 0 "register_operand" "=v,v,v")
18021 (match_operand:VF_256 1 "nonimmediate_operand" "m,o,?v")
18022 (match_parallel 2 "avx_vbroadcast_operand"
18023 [(match_operand 3 "const_int_operand" "C,n,n")])))]
18026 "&& reload_completed && (<MODE>mode != V4DFmode || !TARGET_AVX2)"
18027 [(set (match_dup 0) (vec_duplicate:VF_256 (match_dup 1)))]
18029 rtx op0 = operands[0], op1 = operands[1];
18030 int elt = INTVAL (operands[3]);
18036 if (TARGET_AVX2 && elt == 0)
18038 emit_insn (gen_vec_dup<mode> (op0, gen_lowpart (<ssescalarmode>mode,
18043 /* Shuffle element we care about into all elements of the 128-bit lane.
18044 The other lane gets shuffled too, but we don't care. */
18045 if (<MODE>mode == V4DFmode)
18046 mask = (elt & 1 ? 15 : 0);
18048 mask = (elt & 3) * 0x55;
18049 emit_insn (gen_avx_vpermil<mode> (op0, op1, GEN_INT (mask)));
18051 /* Shuffle the lane we care about into both lanes of the dest. */
18052 mask = (elt / (<ssescalarnum> / 2)) * 0x11;
18053 if (EXT_REX_SSE_REG_P (op0))
18055 /* There is no EVEX VPERM2F128, but we can use either VBROADCASTSS
18057 gcc_assert (<MODE>mode == V8SFmode);
18058 if ((mask & 1) == 0)
18059 emit_insn (gen_avx2_vec_dupv8sf (op0,
18060 gen_lowpart (V4SFmode, op0)));
18062 emit_insn (gen_avx512vl_shuf_f32x4_1 (op0, op0, op0,
18063 GEN_INT (4), GEN_INT (5),
18064 GEN_INT (6), GEN_INT (7),
18065 GEN_INT (12), GEN_INT (13),
18066 GEN_INT (14), GEN_INT (15)));
18070 emit_insn (gen_avx_vperm2f128<mode>3 (op0, op0, op0, GEN_INT (mask)));
18074 operands[1] = adjust_address (op1, <ssescalarmode>mode,
18075 elt * GET_MODE_SIZE (<ssescalarmode>mode));
18078 (define_expand "<sse2_avx_avx512f>_vpermil<mode><mask_name>"
18079 [(set (match_operand:VF2 0 "register_operand")
18081 (match_operand:VF2 1 "nonimmediate_operand")
18082 (match_operand:SI 2 "const_0_to_255_operand")))]
18083 "TARGET_AVX && <mask_mode512bit_condition>"
18085 int mask = INTVAL (operands[2]);
18086 rtx perm[<ssescalarnum>];
18089 for (i = 0; i < <ssescalarnum>; i = i + 2)
18091 perm[i] = GEN_INT (((mask >> i) & 1) + i);
18092 perm[i + 1] = GEN_INT (((mask >> (i + 1)) & 1) + i);
18096 = gen_rtx_PARALLEL (VOIDmode, gen_rtvec_v (<ssescalarnum>, perm));
18099 (define_expand "<sse2_avx_avx512f>_vpermil<mode><mask_name>"
18100 [(set (match_operand:VF1 0 "register_operand")
18102 (match_operand:VF1 1 "nonimmediate_operand")
18103 (match_operand:SI 2 "const_0_to_255_operand")))]
18104 "TARGET_AVX && <mask_mode512bit_condition>"
18106 int mask = INTVAL (operands[2]);
18107 rtx perm[<ssescalarnum>];
18110 for (i = 0; i < <ssescalarnum>; i = i + 4)
18112 perm[i] = GEN_INT (((mask >> 0) & 3) + i);
18113 perm[i + 1] = GEN_INT (((mask >> 2) & 3) + i);
18114 perm[i + 2] = GEN_INT (((mask >> 4) & 3) + i);
18115 perm[i + 3] = GEN_INT (((mask >> 6) & 3) + i);
18119 = gen_rtx_PARALLEL (VOIDmode, gen_rtvec_v (<ssescalarnum>, perm));
18122 (define_insn "*<sse2_avx_avx512f>_vpermilp<mode><mask_name>"
18123 [(set (match_operand:VF 0 "register_operand" "=v")
18125 (match_operand:VF 1 "nonimmediate_operand" "vm")
18126 (match_parallel 2 ""
18127 [(match_operand 3 "const_int_operand")])))]
18128 "TARGET_AVX && <mask_mode512bit_condition>
18129 && avx_vpermilp_parallel (operands[2], <MODE>mode)"
18131 int mask = avx_vpermilp_parallel (operands[2], <MODE>mode) - 1;
18132 operands[2] = GEN_INT (mask);
18133 return "vpermil<ssemodesuffix>\t{%2, %1, %0<mask_operand4>|%0<mask_operand4>, %1, %2}";
18135 [(set_attr "type" "sselog")
18136 (set_attr "prefix_extra" "1")
18137 (set_attr "length_immediate" "1")
18138 (set_attr "prefix" "<mask_prefix>")
18139 (set_attr "mode" "<sseinsnmode>")])
18141 (define_insn "<sse2_avx_avx512f>_vpermilvar<mode>3<mask_name>"
18142 [(set (match_operand:VF 0 "register_operand" "=v")
18144 [(match_operand:VF 1 "register_operand" "v")
18145 (match_operand:<sseintvecmode> 2 "nonimmediate_operand" "vm")]
18147 "TARGET_AVX && <mask_mode512bit_condition>"
18148 "vpermil<ssemodesuffix>\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
18149 [(set_attr "type" "sselog")
18150 (set_attr "prefix_extra" "1")
18151 (set_attr "btver2_decode" "vector")
18152 (set_attr "prefix" "<mask_prefix>")
18153 (set_attr "mode" "<sseinsnmode>")])
18155 (define_mode_iterator VPERMI2
18156 [V16SI V16SF V8DI V8DF
18157 (V8SI "TARGET_AVX512VL") (V8SF "TARGET_AVX512VL")
18158 (V4DI "TARGET_AVX512VL") (V4DF "TARGET_AVX512VL")
18159 (V4SI "TARGET_AVX512VL") (V4SF "TARGET_AVX512VL")
18160 (V2DI "TARGET_AVX512VL") (V2DF "TARGET_AVX512VL")
18161 (V32HI "TARGET_AVX512BW") (V16HI "TARGET_AVX512BW && TARGET_AVX512VL")
18162 (V8HI "TARGET_AVX512BW && TARGET_AVX512VL")
18163 (V64QI "TARGET_AVX512VBMI") (V32QI "TARGET_AVX512VBMI && TARGET_AVX512VL")
18164 (V16QI "TARGET_AVX512VBMI && TARGET_AVX512VL")])
18166 (define_mode_iterator VPERMI2I
18168 (V8SI "TARGET_AVX512VL") (V4SI "TARGET_AVX512VL")
18169 (V4DI "TARGET_AVX512VL") (V2DI "TARGET_AVX512VL")
18170 (V32HI "TARGET_AVX512BW") (V16HI "TARGET_AVX512BW && TARGET_AVX512VL")
18171 (V8HI "TARGET_AVX512BW && TARGET_AVX512VL")
18172 (V64QI "TARGET_AVX512VBMI") (V32QI "TARGET_AVX512VBMI && TARGET_AVX512VL")
18173 (V16QI "TARGET_AVX512VBMI && TARGET_AVX512VL")])
18175 (define_expand "<avx512>_vpermi2var<mode>3_mask"
18176 [(set (match_operand:VPERMI2 0 "register_operand")
18179 [(match_operand:<sseintvecmode> 2 "register_operand")
18180 (match_operand:VPERMI2 1 "register_operand")
18181 (match_operand:VPERMI2 3 "nonimmediate_operand")]
18184 (match_operand:<avx512fmaskmode> 4 "register_operand")))]
18187 operands[2] = force_reg (<sseintvecmode>mode, operands[2]);
18188 operands[5] = gen_lowpart (<MODE>mode, operands[2]);
18191 (define_insn "*<avx512>_vpermi2var<mode>3_mask"
18192 [(set (match_operand:VPERMI2I 0 "register_operand" "=v")
18193 (vec_merge:VPERMI2I
18195 [(match_operand:<sseintvecmode> 2 "register_operand" "0")
18196 (match_operand:VPERMI2I 1 "register_operand" "v")
18197 (match_operand:VPERMI2I 3 "nonimmediate_operand" "vm")]
18200 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk")))]
18202 "vpermi2<ssemodesuffix>\t{%3, %1, %0%{%4%}|%0%{%4%}, %1, %3}"
18203 [(set_attr "type" "sselog")
18204 (set_attr "prefix" "evex")
18205 (set_attr "mode" "<sseinsnmode>")])
18207 (define_insn "*<avx512>_vpermi2var<mode>3_mask"
18208 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v")
18209 (vec_merge:VF_AVX512VL
18210 (unspec:VF_AVX512VL
18211 [(match_operand:<sseintvecmode> 2 "register_operand" "0")
18212 (match_operand:VF_AVX512VL 1 "register_operand" "v")
18213 (match_operand:VF_AVX512VL 3 "nonimmediate_operand" "vm")]
18215 (subreg:VF_AVX512VL (match_dup 2) 0)
18216 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk")))]
18218 "vpermi2<ssemodesuffix>\t{%3, %1, %0%{%4%}|%0%{%4%}, %1, %3}"
18219 [(set_attr "type" "sselog")
18220 (set_attr "prefix" "evex")
18221 (set_attr "mode" "<sseinsnmode>")])
18223 (define_expand "<avx512>_vpermt2var<mode>3_maskz"
18224 [(match_operand:VPERMI2 0 "register_operand")
18225 (match_operand:<sseintvecmode> 1 "register_operand")
18226 (match_operand:VPERMI2 2 "register_operand")
18227 (match_operand:VPERMI2 3 "nonimmediate_operand")
18228 (match_operand:<avx512fmaskmode> 4 "register_operand")]
18231 emit_insn (gen_<avx512>_vpermt2var<mode>3_maskz_1 (
18232 operands[0], operands[1], operands[2], operands[3],
18233 CONST0_RTX (<MODE>mode), operands[4]));
18237 (define_insn "<avx512>_vpermt2var<mode>3<sd_maskz_name>"
18238 [(set (match_operand:VPERMI2 0 "register_operand" "=v,v")
18240 [(match_operand:<sseintvecmode> 1 "register_operand" "v,0")
18241 (match_operand:VPERMI2 2 "register_operand" "0,v")
18242 (match_operand:VPERMI2 3 "nonimmediate_operand" "vm,vm")]
18246 vpermt2<ssemodesuffix>\t{%3, %1, %0<sd_mask_op4>|%0<sd_mask_op4>, %1, %3}
18247 vpermi2<ssemodesuffix>\t{%3, %2, %0<sd_mask_op4>|%0<sd_mask_op4>, %2, %3}"
18248 [(set_attr "type" "sselog")
18249 (set_attr "prefix" "evex")
18250 (set_attr "mode" "<sseinsnmode>")])
18252 (define_insn "<avx512>_vpermt2var<mode>3_mask"
18253 [(set (match_operand:VPERMI2 0 "register_operand" "=v")
18256 [(match_operand:<sseintvecmode> 1 "register_operand" "v")
18257 (match_operand:VPERMI2 2 "register_operand" "0")
18258 (match_operand:VPERMI2 3 "nonimmediate_operand" "vm")]
18261 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk")))]
18263 "vpermt2<ssemodesuffix>\t{%3, %1, %0%{%4%}|%0%{%4%}, %1, %3}"
18264 [(set_attr "type" "sselog")
18265 (set_attr "prefix" "evex")
18266 (set_attr "mode" "<sseinsnmode>")])
18268 (define_expand "avx_vperm2f128<mode>3"
18269 [(set (match_operand:AVX256MODE2P 0 "register_operand")
18270 (unspec:AVX256MODE2P
18271 [(match_operand:AVX256MODE2P 1 "register_operand")
18272 (match_operand:AVX256MODE2P 2 "nonimmediate_operand")
18273 (match_operand:SI 3 "const_0_to_255_operand")]
18274 UNSPEC_VPERMIL2F128))]
18277 int mask = INTVAL (operands[3]);
18278 if ((mask & 0x88) == 0)
18280 rtx perm[<ssescalarnum>], t1, t2;
18281 int i, base, nelt = <ssescalarnum>, nelt2 = nelt / 2;
18283 base = (mask & 3) * nelt2;
18284 for (i = 0; i < nelt2; ++i)
18285 perm[i] = GEN_INT (base + i);
18287 base = ((mask >> 4) & 3) * nelt2;
18288 for (i = 0; i < nelt2; ++i)
18289 perm[i + nelt2] = GEN_INT (base + i);
18291 t2 = gen_rtx_VEC_CONCAT (<ssedoublevecmode>mode,
18292 operands[1], operands[2]);
18293 t1 = gen_rtx_PARALLEL (VOIDmode, gen_rtvec_v (nelt, perm));
18294 t2 = gen_rtx_VEC_SELECT (<MODE>mode, t2, t1);
18295 t2 = gen_rtx_SET (operands[0], t2);
18301 ;; Note that bits 7 and 3 of the imm8 allow lanes to be zeroed, which
18302 ;; means that in order to represent this properly in rtl we'd have to
18303 ;; nest *another* vec_concat with a zero operand and do the select from
18304 ;; a 4x wide vector. That doesn't seem very nice.
18305 (define_insn "*avx_vperm2f128<mode>_full"
18306 [(set (match_operand:AVX256MODE2P 0 "register_operand" "=x")
18307 (unspec:AVX256MODE2P
18308 [(match_operand:AVX256MODE2P 1 "register_operand" "x")
18309 (match_operand:AVX256MODE2P 2 "nonimmediate_operand" "xm")
18310 (match_operand:SI 3 "const_0_to_255_operand" "n")]
18311 UNSPEC_VPERMIL2F128))]
18313 "vperm2<i128>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
18314 [(set_attr "type" "sselog")
18315 (set_attr "prefix_extra" "1")
18316 (set_attr "length_immediate" "1")
18317 (set_attr "prefix" "vex")
18318 (set_attr "mode" "<sseinsnmode>")])
18320 (define_insn "*avx_vperm2f128<mode>_nozero"
18321 [(set (match_operand:AVX256MODE2P 0 "register_operand" "=x")
18322 (vec_select:AVX256MODE2P
18323 (vec_concat:<ssedoublevecmode>
18324 (match_operand:AVX256MODE2P 1 "register_operand" "x")
18325 (match_operand:AVX256MODE2P 2 "nonimmediate_operand" "xm"))
18326 (match_parallel 3 ""
18327 [(match_operand 4 "const_int_operand")])))]
18329 && avx_vperm2f128_parallel (operands[3], <MODE>mode)"
18331 int mask = avx_vperm2f128_parallel (operands[3], <MODE>mode) - 1;
18333 return "vinsert<i128>\t{$0, %x2, %1, %0|%0, %1, %x2, 0}";
18335 return "vinsert<i128>\t{$1, %x2, %1, %0|%0, %1, %x2, 1}";
18336 operands[3] = GEN_INT (mask);
18337 return "vperm2<i128>\t{%3, %2, %1, %0|%0, %1, %2, %3}";
18339 [(set_attr "type" "sselog")
18340 (set_attr "prefix_extra" "1")
18341 (set_attr "length_immediate" "1")
18342 (set_attr "prefix" "vex")
18343 (set_attr "mode" "<sseinsnmode>")])
18345 (define_insn "*ssse3_palignr<mode>_perm"
18346 [(set (match_operand:V_128 0 "register_operand" "=x,x,v")
18348 (match_operand:V_128 1 "register_operand" "0,x,v")
18349 (match_parallel 2 "palignr_operand"
18350 [(match_operand 3 "const_int_operand" "n,n,n")])))]
18353 operands[2] = (GEN_INT (INTVAL (operands[3])
18354 * GET_MODE_UNIT_SIZE (GET_MODE (operands[0]))));
18356 switch (which_alternative)
18359 return "palignr\t{%2, %1, %0|%0, %1, %2}";
18362 return "vpalignr\t{%2, %1, %1, %0|%0, %1, %1, %2}";
18364 gcc_unreachable ();
18367 [(set_attr "isa" "noavx,avx,avx512bw")
18368 (set_attr "type" "sseishft")
18369 (set_attr "atom_unit" "sishuf")
18370 (set_attr "prefix_data16" "1,*,*")
18371 (set_attr "prefix_extra" "1")
18372 (set_attr "length_immediate" "1")
18373 (set_attr "prefix" "orig,vex,evex")])
18375 (define_expand "avx512vl_vinsert<mode>"
18376 [(match_operand:VI48F_256 0 "register_operand")
18377 (match_operand:VI48F_256 1 "register_operand")
18378 (match_operand:<ssehalfvecmode> 2 "nonimmediate_operand")
18379 (match_operand:SI 3 "const_0_to_1_operand")
18380 (match_operand:VI48F_256 4 "register_operand")
18381 (match_operand:<avx512fmaskmode> 5 "register_operand")]
18384 rtx (*insn)(rtx, rtx, rtx, rtx, rtx);
18386 switch (INTVAL (operands[3]))
18389 insn = gen_vec_set_lo_<mode>_mask;
18392 insn = gen_vec_set_hi_<mode>_mask;
18395 gcc_unreachable ();
18398 emit_insn (insn (operands[0], operands[1], operands[2], operands[4],
18403 (define_expand "avx_vinsertf128<mode>"
18404 [(match_operand:V_256 0 "register_operand")
18405 (match_operand:V_256 1 "register_operand")
18406 (match_operand:<ssehalfvecmode> 2 "nonimmediate_operand")
18407 (match_operand:SI 3 "const_0_to_1_operand")]
18410 rtx (*insn)(rtx, rtx, rtx);
18412 switch (INTVAL (operands[3]))
18415 insn = gen_vec_set_lo_<mode>;
18418 insn = gen_vec_set_hi_<mode>;
18421 gcc_unreachable ();
18424 emit_insn (insn (operands[0], operands[1], operands[2]));
18428 (define_insn "vec_set_lo_<mode><mask_name>"
18429 [(set (match_operand:VI8F_256 0 "register_operand" "=v")
18430 (vec_concat:VI8F_256
18431 (match_operand:<ssehalfvecmode> 2 "nonimmediate_operand" "vm")
18432 (vec_select:<ssehalfvecmode>
18433 (match_operand:VI8F_256 1 "register_operand" "v")
18434 (parallel [(const_int 2) (const_int 3)]))))]
18435 "TARGET_AVX && <mask_avx512dq_condition>"
18437 if (TARGET_AVX512DQ)
18438 return "vinsert<shuffletype>64x2\t{$0x0, %2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2, 0x0}";
18439 else if (TARGET_AVX512VL)
18440 return "vinsert<shuffletype>32x4\t{$0x0, %2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2, 0x0}";
18442 return "vinsert<i128>\t{$0x0, %2, %1, %0|%0, %1, %2, 0x0}";
18444 [(set_attr "type" "sselog")
18445 (set_attr "prefix_extra" "1")
18446 (set_attr "length_immediate" "1")
18447 (set_attr "prefix" "vex")
18448 (set_attr "mode" "<sseinsnmode>")])
18450 (define_insn "vec_set_hi_<mode><mask_name>"
18451 [(set (match_operand:VI8F_256 0 "register_operand" "=v")
18452 (vec_concat:VI8F_256
18453 (vec_select:<ssehalfvecmode>
18454 (match_operand:VI8F_256 1 "register_operand" "v")
18455 (parallel [(const_int 0) (const_int 1)]))
18456 (match_operand:<ssehalfvecmode> 2 "nonimmediate_operand" "vm")))]
18457 "TARGET_AVX && <mask_avx512dq_condition>"
18459 if (TARGET_AVX512DQ)
18460 return "vinsert<shuffletype>64x2\t{$0x1, %2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2, 0x1}";
18461 else if (TARGET_AVX512VL)
18462 return "vinsert<shuffletype>32x4\t{$0x1, %2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2, 0x1}";
18464 return "vinsert<i128>\t{$0x1, %2, %1, %0|%0, %1, %2, 0x1}";
18466 [(set_attr "type" "sselog")
18467 (set_attr "prefix_extra" "1")
18468 (set_attr "length_immediate" "1")
18469 (set_attr "prefix" "vex")
18470 (set_attr "mode" "<sseinsnmode>")])
18472 (define_insn "vec_set_lo_<mode><mask_name>"
18473 [(set (match_operand:VI4F_256 0 "register_operand" "=v")
18474 (vec_concat:VI4F_256
18475 (match_operand:<ssehalfvecmode> 2 "nonimmediate_operand" "vm")
18476 (vec_select:<ssehalfvecmode>
18477 (match_operand:VI4F_256 1 "register_operand" "v")
18478 (parallel [(const_int 4) (const_int 5)
18479 (const_int 6) (const_int 7)]))))]
18482 if (TARGET_AVX512VL)
18483 return "vinsert<shuffletype>32x4\t{$0x0, %2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2, 0x0}";
18485 return "vinsert<i128>\t{$0x0, %2, %1, %0|%0, %1, %2, 0x0}";
18487 [(set_attr "type" "sselog")
18488 (set_attr "prefix_extra" "1")
18489 (set_attr "length_immediate" "1")
18490 (set_attr "prefix" "vex")
18491 (set_attr "mode" "<sseinsnmode>")])
18493 (define_insn "vec_set_hi_<mode><mask_name>"
18494 [(set (match_operand:VI4F_256 0 "register_operand" "=v")
18495 (vec_concat:VI4F_256
18496 (vec_select:<ssehalfvecmode>
18497 (match_operand:VI4F_256 1 "register_operand" "v")
18498 (parallel [(const_int 0) (const_int 1)
18499 (const_int 2) (const_int 3)]))
18500 (match_operand:<ssehalfvecmode> 2 "nonimmediate_operand" "vm")))]
18503 if (TARGET_AVX512VL)
18504 return "vinsert<shuffletype>32x4\t{$0x1, %2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2, 0x1}";
18506 return "vinsert<i128>\t{$0x1, %2, %1, %0|%0, %1, %2, 0x1}";
18508 [(set_attr "type" "sselog")
18509 (set_attr "prefix_extra" "1")
18510 (set_attr "length_immediate" "1")
18511 (set_attr "prefix" "vex")
18512 (set_attr "mode" "<sseinsnmode>")])
18514 (define_insn "vec_set_lo_v16hi"
18515 [(set (match_operand:V16HI 0 "register_operand" "=x,v")
18517 (match_operand:V8HI 2 "nonimmediate_operand" "xm,vm")
18519 (match_operand:V16HI 1 "register_operand" "x,v")
18520 (parallel [(const_int 8) (const_int 9)
18521 (const_int 10) (const_int 11)
18522 (const_int 12) (const_int 13)
18523 (const_int 14) (const_int 15)]))))]
18526 vinsert%~128\t{$0x0, %2, %1, %0|%0, %1, %2, 0x0}
18527 vinserti32x4\t{$0x0, %2, %1, %0|%0, %1, %2, 0x0}"
18528 [(set_attr "type" "sselog")
18529 (set_attr "prefix_extra" "1")
18530 (set_attr "length_immediate" "1")
18531 (set_attr "prefix" "vex,evex")
18532 (set_attr "mode" "OI")])
18534 (define_insn "vec_set_hi_v16hi"
18535 [(set (match_operand:V16HI 0 "register_operand" "=x,v")
18538 (match_operand:V16HI 1 "register_operand" "x,v")
18539 (parallel [(const_int 0) (const_int 1)
18540 (const_int 2) (const_int 3)
18541 (const_int 4) (const_int 5)
18542 (const_int 6) (const_int 7)]))
18543 (match_operand:V8HI 2 "nonimmediate_operand" "xm,vm")))]
18546 vinsert%~128\t{$0x1, %2, %1, %0|%0, %1, %2, 0x1}
18547 vinserti32x4\t{$0x1, %2, %1, %0|%0, %1, %2, 0x1}"
18548 [(set_attr "type" "sselog")
18549 (set_attr "prefix_extra" "1")
18550 (set_attr "length_immediate" "1")
18551 (set_attr "prefix" "vex,evex")
18552 (set_attr "mode" "OI")])
18554 (define_insn "vec_set_lo_v32qi"
18555 [(set (match_operand:V32QI 0 "register_operand" "=x,v")
18557 (match_operand:V16QI 2 "nonimmediate_operand" "xm,v")
18559 (match_operand:V32QI 1 "register_operand" "x,v")
18560 (parallel [(const_int 16) (const_int 17)
18561 (const_int 18) (const_int 19)
18562 (const_int 20) (const_int 21)
18563 (const_int 22) (const_int 23)
18564 (const_int 24) (const_int 25)
18565 (const_int 26) (const_int 27)
18566 (const_int 28) (const_int 29)
18567 (const_int 30) (const_int 31)]))))]
18570 vinsert%~128\t{$0x0, %2, %1, %0|%0, %1, %2, 0x0}
18571 vinserti32x4\t{$0x0, %2, %1, %0|%0, %1, %2, 0x0}"
18572 [(set_attr "type" "sselog")
18573 (set_attr "prefix_extra" "1")
18574 (set_attr "length_immediate" "1")
18575 (set_attr "prefix" "vex,evex")
18576 (set_attr "mode" "OI")])
18578 (define_insn "vec_set_hi_v32qi"
18579 [(set (match_operand:V32QI 0 "register_operand" "=x,v")
18582 (match_operand:V32QI 1 "register_operand" "x,v")
18583 (parallel [(const_int 0) (const_int 1)
18584 (const_int 2) (const_int 3)
18585 (const_int 4) (const_int 5)
18586 (const_int 6) (const_int 7)
18587 (const_int 8) (const_int 9)
18588 (const_int 10) (const_int 11)
18589 (const_int 12) (const_int 13)
18590 (const_int 14) (const_int 15)]))
18591 (match_operand:V16QI 2 "nonimmediate_operand" "xm,vm")))]
18594 vinsert%~128\t{$0x1, %2, %1, %0|%0, %1, %2, 0x1}
18595 vinserti32x4\t{$0x1, %2, %1, %0|%0, %1, %2, 0x1}"
18596 [(set_attr "type" "sselog")
18597 (set_attr "prefix_extra" "1")
18598 (set_attr "length_immediate" "1")
18599 (set_attr "prefix" "vex,evex")
18600 (set_attr "mode" "OI")])
18602 (define_insn "<avx_avx2>_maskload<ssemodesuffix><avxsizesuffix>"
18603 [(set (match_operand:V48_AVX2 0 "register_operand" "=x")
18605 [(match_operand:<sseintvecmode> 2 "register_operand" "x")
18606 (match_operand:V48_AVX2 1 "memory_operand" "m")]
18609 "v<sseintprefix>maskmov<ssemodesuffix>\t{%1, %2, %0|%0, %2, %1}"
18610 [(set_attr "type" "sselog1")
18611 (set_attr "prefix_extra" "1")
18612 (set_attr "prefix" "vex")
18613 (set_attr "btver2_decode" "vector")
18614 (set_attr "mode" "<sseinsnmode>")])
18616 (define_insn "<avx_avx2>_maskstore<ssemodesuffix><avxsizesuffix>"
18617 [(set (match_operand:V48_AVX2 0 "memory_operand" "+m")
18619 [(match_operand:<sseintvecmode> 1 "register_operand" "x")
18620 (match_operand:V48_AVX2 2 "register_operand" "x")
18624 "v<sseintprefix>maskmov<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
18625 [(set_attr "type" "sselog1")
18626 (set_attr "prefix_extra" "1")
18627 (set_attr "prefix" "vex")
18628 (set_attr "btver2_decode" "vector")
18629 (set_attr "mode" "<sseinsnmode>")])
18631 (define_expand "maskload<mode><sseintvecmodelower>"
18632 [(set (match_operand:V48_AVX2 0 "register_operand")
18634 [(match_operand:<sseintvecmode> 2 "register_operand")
18635 (match_operand:V48_AVX2 1 "memory_operand")]
18639 (define_expand "maskload<mode><avx512fmaskmodelower>"
18640 [(set (match_operand:V48_AVX512VL 0 "register_operand")
18641 (vec_merge:V48_AVX512VL
18642 (match_operand:V48_AVX512VL 1 "memory_operand")
18644 (match_operand:<avx512fmaskmode> 2 "register_operand")))]
18647 (define_expand "maskload<mode><avx512fmaskmodelower>"
18648 [(set (match_operand:VI12_AVX512VL 0 "register_operand")
18649 (vec_merge:VI12_AVX512VL
18650 (match_operand:VI12_AVX512VL 1 "memory_operand")
18652 (match_operand:<avx512fmaskmode> 2 "register_operand")))]
18655 (define_expand "maskstore<mode><sseintvecmodelower>"
18656 [(set (match_operand:V48_AVX2 0 "memory_operand")
18658 [(match_operand:<sseintvecmode> 2 "register_operand")
18659 (match_operand:V48_AVX2 1 "register_operand")
18664 (define_expand "maskstore<mode><avx512fmaskmodelower>"
18665 [(set (match_operand:V48_AVX512VL 0 "memory_operand")
18666 (vec_merge:V48_AVX512VL
18667 (match_operand:V48_AVX512VL 1 "register_operand")
18669 (match_operand:<avx512fmaskmode> 2 "register_operand")))]
18672 (define_expand "maskstore<mode><avx512fmaskmodelower>"
18673 [(set (match_operand:VI12_AVX512VL 0 "memory_operand")
18674 (vec_merge:VI12_AVX512VL
18675 (match_operand:VI12_AVX512VL 1 "register_operand")
18677 (match_operand:<avx512fmaskmode> 2 "register_operand")))]
18680 (define_expand "cbranch<mode>4"
18681 [(set (reg:CC FLAGS_REG)
18682 (compare:CC (match_operand:VI48_AVX 1 "register_operand")
18683 (match_operand:VI48_AVX 2 "nonimmediate_operand")))
18684 (set (pc) (if_then_else
18685 (match_operator 0 "bt_comparison_operator"
18686 [(reg:CC FLAGS_REG) (const_int 0)])
18687 (label_ref (match_operand 3))
18691 ix86_expand_branch (GET_CODE (operands[0]),
18692 operands[1], operands[2], operands[3]);
18697 (define_insn_and_split "avx_<castmode><avxsizesuffix>_<castmode>"
18698 [(set (match_operand:AVX256MODE2P 0 "nonimmediate_operand" "=x,m")
18699 (unspec:AVX256MODE2P
18700 [(match_operand:<ssehalfvecmode> 1 "nonimmediate_operand" "xm,x")]
18702 "TARGET_AVX && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
18704 "&& reload_completed"
18705 [(set (match_dup 0) (match_dup 1))]
18707 if (REG_P (operands[0]))
18708 operands[0] = gen_lowpart (<ssehalfvecmode>mode, operands[0]);
18710 operands[1] = lowpart_subreg (<MODE>mode, operands[1],
18711 <ssehalfvecmode>mode);
18714 ;; Modes handled by vec_init expanders.
18715 (define_mode_iterator VEC_INIT_MODE
18716 [(V64QI "TARGET_AVX512F") (V32QI "TARGET_AVX") V16QI
18717 (V32HI "TARGET_AVX512F") (V16HI "TARGET_AVX") V8HI
18718 (V16SI "TARGET_AVX512F") (V8SI "TARGET_AVX") V4SI
18719 (V8DI "TARGET_AVX512F") (V4DI "TARGET_AVX") V2DI
18720 (V16SF "TARGET_AVX512F") (V8SF "TARGET_AVX") V4SF
18721 (V8DF "TARGET_AVX512F") (V4DF "TARGET_AVX") (V2DF "TARGET_SSE2")
18722 (V4TI "TARGET_AVX512F") (V2TI "TARGET_AVX")])
18724 ;; Likewise, but for initialization from half sized vectors.
18725 ;; Thus, these are all VEC_INIT_MODE modes except V2??.
18726 (define_mode_iterator VEC_INIT_HALF_MODE
18727 [(V64QI "TARGET_AVX512F") (V32QI "TARGET_AVX") V16QI
18728 (V32HI "TARGET_AVX512F") (V16HI "TARGET_AVX") V8HI
18729 (V16SI "TARGET_AVX512F") (V8SI "TARGET_AVX") V4SI
18730 (V8DI "TARGET_AVX512F") (V4DI "TARGET_AVX")
18731 (V16SF "TARGET_AVX512F") (V8SF "TARGET_AVX") V4SF
18732 (V8DF "TARGET_AVX512F") (V4DF "TARGET_AVX")
18733 (V4TI "TARGET_AVX512F")])
18735 (define_expand "vec_init<mode><ssescalarmodelower>"
18736 [(match_operand:VEC_INIT_MODE 0 "register_operand")
18740 ix86_expand_vector_init (false, operands[0], operands[1]);
18744 (define_expand "vec_init<mode><ssehalfvecmodelower>"
18745 [(match_operand:VEC_INIT_HALF_MODE 0 "register_operand")
18749 ix86_expand_vector_init (false, operands[0], operands[1]);
18753 (define_insn "<avx2_avx512>_ashrv<mode><mask_name>"
18754 [(set (match_operand:VI48_AVX512F_AVX512VL 0 "register_operand" "=v")
18755 (ashiftrt:VI48_AVX512F_AVX512VL
18756 (match_operand:VI48_AVX512F_AVX512VL 1 "register_operand" "v")
18757 (match_operand:VI48_AVX512F_AVX512VL 2 "nonimmediate_operand" "vm")))]
18758 "TARGET_AVX2 && <mask_mode512bit_condition>"
18759 "vpsrav<ssemodesuffix>\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
18760 [(set_attr "type" "sseishft")
18761 (set_attr "prefix" "maybe_evex")
18762 (set_attr "mode" "<sseinsnmode>")])
18764 (define_insn "<avx2_avx512>_ashrv<mode><mask_name>"
18765 [(set (match_operand:VI2_AVX512VL 0 "register_operand" "=v")
18766 (ashiftrt:VI2_AVX512VL
18767 (match_operand:VI2_AVX512VL 1 "register_operand" "v")
18768 (match_operand:VI2_AVX512VL 2 "nonimmediate_operand" "vm")))]
18770 "vpsravw\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
18771 [(set_attr "type" "sseishft")
18772 (set_attr "prefix" "maybe_evex")
18773 (set_attr "mode" "<sseinsnmode>")])
18775 (define_insn "<avx2_avx512>_<shift_insn>v<mode><mask_name>"
18776 [(set (match_operand:VI48_AVX512F 0 "register_operand" "=v")
18777 (any_lshift:VI48_AVX512F
18778 (match_operand:VI48_AVX512F 1 "register_operand" "v")
18779 (match_operand:VI48_AVX512F 2 "nonimmediate_operand" "vm")))]
18780 "TARGET_AVX2 && <mask_mode512bit_condition>"
18781 "vp<vshift>v<ssemodesuffix>\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
18782 [(set_attr "type" "sseishft")
18783 (set_attr "prefix" "maybe_evex")
18784 (set_attr "mode" "<sseinsnmode>")])
18786 (define_insn "<avx2_avx512>_<shift_insn>v<mode><mask_name>"
18787 [(set (match_operand:VI2_AVX512VL 0 "register_operand" "=v")
18788 (any_lshift:VI2_AVX512VL
18789 (match_operand:VI2_AVX512VL 1 "register_operand" "v")
18790 (match_operand:VI2_AVX512VL 2 "nonimmediate_operand" "vm")))]
18792 "vp<vshift>v<ssemodesuffix>\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
18793 [(set_attr "type" "sseishft")
18794 (set_attr "prefix" "maybe_evex")
18795 (set_attr "mode" "<sseinsnmode>")])
18797 (define_insn "avx_vec_concat<mode>"
18798 [(set (match_operand:V_256_512 0 "register_operand" "=x,v,x,Yv")
18799 (vec_concat:V_256_512
18800 (match_operand:<ssehalfvecmode> 1 "register_operand" "x,v,x,v")
18801 (match_operand:<ssehalfvecmode> 2 "vector_move_operand" "xm,vm,C,C")))]
18804 switch (which_alternative)
18807 return "vinsert<i128>\t{$0x1, %2, %<concat_tg_mode>1, %0|%0, %<concat_tg_mode>1, %2, 0x1}";
18809 if (<MODE_SIZE> == 64)
18811 if (TARGET_AVX512DQ && GET_MODE_SIZE (<ssescalarmode>mode) == 4)
18812 return "vinsert<shuffletype>32x8\t{$0x1, %2, %<concat_tg_mode>1, %0|%0, %<concat_tg_mode>1, %2, 0x1}";
18814 return "vinsert<shuffletype>64x4\t{$0x1, %2, %<concat_tg_mode>1, %0|%0, %<concat_tg_mode>1, %2, 0x1}";
18818 if (TARGET_AVX512DQ && GET_MODE_SIZE (<ssescalarmode>mode) == 8)
18819 return "vinsert<shuffletype>64x2\t{$0x1, %2, %<concat_tg_mode>1, %0|%0, %<concat_tg_mode>1, %2, 0x1}";
18821 return "vinsert<shuffletype>32x4\t{$0x1, %2, %<concat_tg_mode>1, %0|%0, %<concat_tg_mode>1, %2, 0x1}";
18825 switch (get_attr_mode (insn))
18828 return "vmovaps\t{%1, %t0|%t0, %1}";
18830 return "vmovapd\t{%1, %t0|%t0, %1}";
18832 return "vmovaps\t{%1, %x0|%x0, %1}";
18834 return "vmovapd\t{%1, %x0|%x0, %1}";
18836 if (which_alternative == 2)
18837 return "vmovdqa\t{%1, %t0|%t0, %1}";
18838 else if (GET_MODE_SIZE (<ssescalarmode>mode) == 8)
18839 return "vmovdqa64\t{%1, %t0|%t0, %1}";
18841 return "vmovdqa32\t{%1, %t0|%t0, %1}";
18843 if (which_alternative == 2)
18844 return "vmovdqa\t{%1, %x0|%x0, %1}";
18845 else if (GET_MODE_SIZE (<ssescalarmode>mode) == 8)
18846 return "vmovdqa64\t{%1, %x0|%x0, %1}";
18848 return "vmovdqa32\t{%1, %x0|%x0, %1}";
18850 gcc_unreachable ();
18853 gcc_unreachable ();
18856 [(set_attr "type" "sselog,sselog,ssemov,ssemov")
18857 (set_attr "prefix_extra" "1,1,*,*")
18858 (set_attr "length_immediate" "1,1,*,*")
18859 (set_attr "prefix" "maybe_evex")
18860 (set_attr "mode" "<sseinsnmode>")])
18862 (define_insn "vcvtph2ps<mask_name>"
18863 [(set (match_operand:V4SF 0 "register_operand" "=v")
18865 (unspec:V8SF [(match_operand:V8HI 1 "register_operand" "v")]
18867 (parallel [(const_int 0) (const_int 1)
18868 (const_int 2) (const_int 3)])))]
18869 "TARGET_F16C || TARGET_AVX512VL"
18870 "vcvtph2ps\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
18871 [(set_attr "type" "ssecvt")
18872 (set_attr "prefix" "maybe_evex")
18873 (set_attr "mode" "V4SF")])
18875 (define_insn "*vcvtph2ps_load<mask_name>"
18876 [(set (match_operand:V4SF 0 "register_operand" "=v")
18877 (unspec:V4SF [(match_operand:V4HI 1 "memory_operand" "m")]
18878 UNSPEC_VCVTPH2PS))]
18879 "TARGET_F16C || TARGET_AVX512VL"
18880 "vcvtph2ps\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
18881 [(set_attr "type" "ssecvt")
18882 (set_attr "prefix" "vex")
18883 (set_attr "mode" "V8SF")])
18885 (define_insn "vcvtph2ps256<mask_name>"
18886 [(set (match_operand:V8SF 0 "register_operand" "=v")
18887 (unspec:V8SF [(match_operand:V8HI 1 "nonimmediate_operand" "vm")]
18888 UNSPEC_VCVTPH2PS))]
18889 "TARGET_F16C || TARGET_AVX512VL"
18890 "vcvtph2ps\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
18891 [(set_attr "type" "ssecvt")
18892 (set_attr "prefix" "vex")
18893 (set_attr "btver2_decode" "double")
18894 (set_attr "mode" "V8SF")])
18896 (define_insn "<mask_codefor>avx512f_vcvtph2ps512<mask_name><round_saeonly_name>"
18897 [(set (match_operand:V16SF 0 "register_operand" "=v")
18899 [(match_operand:V16HI 1 "<round_saeonly_nimm_predicate>" "<round_saeonly_constraint>")]
18900 UNSPEC_VCVTPH2PS))]
18902 "vcvtph2ps\t{<round_saeonly_mask_op2>%1, %0<mask_operand2>|%0<mask_operand2>, %1<round_saeonly_mask_op2>}"
18903 [(set_attr "type" "ssecvt")
18904 (set_attr "prefix" "evex")
18905 (set_attr "mode" "V16SF")])
18907 (define_expand "vcvtps2ph_mask"
18908 [(set (match_operand:V8HI 0 "register_operand")
18911 (unspec:V4HI [(match_operand:V4SF 1 "register_operand")
18912 (match_operand:SI 2 "const_0_to_255_operand")]
18915 (match_operand:V8HI 3 "vector_move_operand")
18916 (match_operand:QI 4 "register_operand")))]
18918 "operands[5] = CONST0_RTX (V4HImode);")
18920 (define_expand "vcvtps2ph"
18921 [(set (match_operand:V8HI 0 "register_operand")
18923 (unspec:V4HI [(match_operand:V4SF 1 "register_operand")
18924 (match_operand:SI 2 "const_0_to_255_operand")]
18928 "operands[3] = CONST0_RTX (V4HImode);")
18930 (define_insn "*vcvtps2ph<mask_name>"
18931 [(set (match_operand:V8HI 0 "register_operand" "=v")
18933 (unspec:V4HI [(match_operand:V4SF 1 "register_operand" "v")
18934 (match_operand:SI 2 "const_0_to_255_operand" "N")]
18936 (match_operand:V4HI 3 "const0_operand")))]
18937 "(TARGET_F16C || TARGET_AVX512VL) && <mask_avx512vl_condition>"
18938 "vcvtps2ph\t{%2, %1, %0<mask_operand4>|%0<mask_operand4>, %1, %2}"
18939 [(set_attr "type" "ssecvt")
18940 (set_attr "prefix" "maybe_evex")
18941 (set_attr "mode" "V4SF")])
18943 (define_insn "*vcvtps2ph_store<mask_name>"
18944 [(set (match_operand:V4HI 0 "memory_operand" "=m")
18945 (unspec:V4HI [(match_operand:V4SF 1 "register_operand" "v")
18946 (match_operand:SI 2 "const_0_to_255_operand" "N")]
18947 UNSPEC_VCVTPS2PH))]
18948 "TARGET_F16C || TARGET_AVX512VL"
18949 "vcvtps2ph\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
18950 [(set_attr "type" "ssecvt")
18951 (set_attr "prefix" "maybe_evex")
18952 (set_attr "mode" "V4SF")])
18954 (define_insn "vcvtps2ph256<mask_name>"
18955 [(set (match_operand:V8HI 0 "nonimmediate_operand" "=vm")
18956 (unspec:V8HI [(match_operand:V8SF 1 "register_operand" "v")
18957 (match_operand:SI 2 "const_0_to_255_operand" "N")]
18958 UNSPEC_VCVTPS2PH))]
18959 "TARGET_F16C || TARGET_AVX512VL"
18960 "vcvtps2ph\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
18961 [(set_attr "type" "ssecvt")
18962 (set_attr "prefix" "maybe_evex")
18963 (set_attr "btver2_decode" "vector")
18964 (set_attr "mode" "V8SF")])
18966 (define_insn "<mask_codefor>avx512f_vcvtps2ph512<mask_name>"
18967 [(set (match_operand:V16HI 0 "nonimmediate_operand" "=vm")
18969 [(match_operand:V16SF 1 "register_operand" "v")
18970 (match_operand:SI 2 "const_0_to_255_operand" "N")]
18971 UNSPEC_VCVTPS2PH))]
18973 "vcvtps2ph\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
18974 [(set_attr "type" "ssecvt")
18975 (set_attr "prefix" "evex")
18976 (set_attr "mode" "V16SF")])
18978 ;; For gather* insn patterns
18979 (define_mode_iterator VEC_GATHER_MODE
18980 [V2DI V2DF V4DI V4DF V4SI V4SF V8SI V8SF])
18981 (define_mode_attr VEC_GATHER_IDXSI
18982 [(V2DI "V4SI") (V4DI "V4SI") (V8DI "V8SI")
18983 (V2DF "V4SI") (V4DF "V4SI") (V8DF "V8SI")
18984 (V4SI "V4SI") (V8SI "V8SI") (V16SI "V16SI")
18985 (V4SF "V4SI") (V8SF "V8SI") (V16SF "V16SI")])
18987 (define_mode_attr VEC_GATHER_IDXDI
18988 [(V2DI "V2DI") (V4DI "V4DI") (V8DI "V8DI")
18989 (V2DF "V2DI") (V4DF "V4DI") (V8DF "V8DI")
18990 (V4SI "V2DI") (V8SI "V4DI") (V16SI "V8DI")
18991 (V4SF "V2DI") (V8SF "V4DI") (V16SF "V8DI")])
18993 (define_mode_attr VEC_GATHER_SRCDI
18994 [(V2DI "V2DI") (V4DI "V4DI") (V8DI "V8DI")
18995 (V2DF "V2DF") (V4DF "V4DF") (V8DF "V8DF")
18996 (V4SI "V4SI") (V8SI "V4SI") (V16SI "V8SI")
18997 (V4SF "V4SF") (V8SF "V4SF") (V16SF "V8SF")])
18999 (define_expand "avx2_gathersi<mode>"
19000 [(parallel [(set (match_operand:VEC_GATHER_MODE 0 "register_operand")
19001 (unspec:VEC_GATHER_MODE
19002 [(match_operand:VEC_GATHER_MODE 1 "register_operand")
19003 (mem:<ssescalarmode>
19005 [(match_operand 2 "vsib_address_operand")
19006 (match_operand:<VEC_GATHER_IDXSI>
19007 3 "register_operand")
19008 (match_operand:SI 5 "const1248_operand ")]))
19009 (mem:BLK (scratch))
19010 (match_operand:VEC_GATHER_MODE 4 "register_operand")]
19012 (clobber (match_scratch:VEC_GATHER_MODE 7))])]
19016 = gen_rtx_UNSPEC (Pmode, gen_rtvec (3, operands[2], operands[3],
19017 operands[5]), UNSPEC_VSIBADDR);
19020 (define_insn "*avx2_gathersi<mode>"
19021 [(set (match_operand:VEC_GATHER_MODE 0 "register_operand" "=&x")
19022 (unspec:VEC_GATHER_MODE
19023 [(match_operand:VEC_GATHER_MODE 2 "register_operand" "0")
19024 (match_operator:<ssescalarmode> 7 "vsib_mem_operator"
19026 [(match_operand:P 3 "vsib_address_operand" "Tv")
19027 (match_operand:<VEC_GATHER_IDXSI> 4 "register_operand" "x")
19028 (match_operand:SI 6 "const1248_operand" "n")]
19030 (mem:BLK (scratch))
19031 (match_operand:VEC_GATHER_MODE 5 "register_operand" "1")]
19033 (clobber (match_scratch:VEC_GATHER_MODE 1 "=&x"))]
19035 "v<sseintprefix>gatherd<ssemodesuffix>\t{%1, %7, %0|%0, %7, %1}"
19036 [(set_attr "type" "ssemov")
19037 (set_attr "prefix" "vex")
19038 (set_attr "mode" "<sseinsnmode>")])
19040 (define_insn "*avx2_gathersi<mode>_2"
19041 [(set (match_operand:VEC_GATHER_MODE 0 "register_operand" "=&x")
19042 (unspec:VEC_GATHER_MODE
19044 (match_operator:<ssescalarmode> 6 "vsib_mem_operator"
19046 [(match_operand:P 2 "vsib_address_operand" "Tv")
19047 (match_operand:<VEC_GATHER_IDXSI> 3 "register_operand" "x")
19048 (match_operand:SI 5 "const1248_operand" "n")]
19050 (mem:BLK (scratch))
19051 (match_operand:VEC_GATHER_MODE 4 "register_operand" "1")]
19053 (clobber (match_scratch:VEC_GATHER_MODE 1 "=&x"))]
19055 "v<sseintprefix>gatherd<ssemodesuffix>\t{%1, %6, %0|%0, %6, %1}"
19056 [(set_attr "type" "ssemov")
19057 (set_attr "prefix" "vex")
19058 (set_attr "mode" "<sseinsnmode>")])
19060 (define_expand "avx2_gatherdi<mode>"
19061 [(parallel [(set (match_operand:VEC_GATHER_MODE 0 "register_operand")
19062 (unspec:VEC_GATHER_MODE
19063 [(match_operand:<VEC_GATHER_SRCDI> 1 "register_operand")
19064 (mem:<ssescalarmode>
19066 [(match_operand 2 "vsib_address_operand")
19067 (match_operand:<VEC_GATHER_IDXDI>
19068 3 "register_operand")
19069 (match_operand:SI 5 "const1248_operand ")]))
19070 (mem:BLK (scratch))
19071 (match_operand:<VEC_GATHER_SRCDI> 4 "register_operand")]
19073 (clobber (match_scratch:VEC_GATHER_MODE 7))])]
19077 = gen_rtx_UNSPEC (Pmode, gen_rtvec (3, operands[2], operands[3],
19078 operands[5]), UNSPEC_VSIBADDR);
19081 (define_insn "*avx2_gatherdi<mode>"
19082 [(set (match_operand:VEC_GATHER_MODE 0 "register_operand" "=&x")
19083 (unspec:VEC_GATHER_MODE
19084 [(match_operand:<VEC_GATHER_SRCDI> 2 "register_operand" "0")
19085 (match_operator:<ssescalarmode> 7 "vsib_mem_operator"
19087 [(match_operand:P 3 "vsib_address_operand" "Tv")
19088 (match_operand:<VEC_GATHER_IDXDI> 4 "register_operand" "x")
19089 (match_operand:SI 6 "const1248_operand" "n")]
19091 (mem:BLK (scratch))
19092 (match_operand:<VEC_GATHER_SRCDI> 5 "register_operand" "1")]
19094 (clobber (match_scratch:VEC_GATHER_MODE 1 "=&x"))]
19096 "v<sseintprefix>gatherq<ssemodesuffix>\t{%5, %7, %2|%2, %7, %5}"
19097 [(set_attr "type" "ssemov")
19098 (set_attr "prefix" "vex")
19099 (set_attr "mode" "<sseinsnmode>")])
19101 (define_insn "*avx2_gatherdi<mode>_2"
19102 [(set (match_operand:VEC_GATHER_MODE 0 "register_operand" "=&x")
19103 (unspec:VEC_GATHER_MODE
19105 (match_operator:<ssescalarmode> 6 "vsib_mem_operator"
19107 [(match_operand:P 2 "vsib_address_operand" "Tv")
19108 (match_operand:<VEC_GATHER_IDXDI> 3 "register_operand" "x")
19109 (match_operand:SI 5 "const1248_operand" "n")]
19111 (mem:BLK (scratch))
19112 (match_operand:<VEC_GATHER_SRCDI> 4 "register_operand" "1")]
19114 (clobber (match_scratch:VEC_GATHER_MODE 1 "=&x"))]
19117 if (<MODE>mode != <VEC_GATHER_SRCDI>mode)
19118 return "v<sseintprefix>gatherq<ssemodesuffix>\t{%4, %6, %x0|%x0, %6, %4}";
19119 return "v<sseintprefix>gatherq<ssemodesuffix>\t{%4, %6, %0|%0, %6, %4}";
19121 [(set_attr "type" "ssemov")
19122 (set_attr "prefix" "vex")
19123 (set_attr "mode" "<sseinsnmode>")])
19125 (define_insn "*avx2_gatherdi<mode>_3"
19126 [(set (match_operand:<VEC_GATHER_SRCDI> 0 "register_operand" "=&x")
19127 (vec_select:<VEC_GATHER_SRCDI>
19129 [(match_operand:<VEC_GATHER_SRCDI> 2 "register_operand" "0")
19130 (match_operator:<ssescalarmode> 7 "vsib_mem_operator"
19132 [(match_operand:P 3 "vsib_address_operand" "Tv")
19133 (match_operand:<VEC_GATHER_IDXDI> 4 "register_operand" "x")
19134 (match_operand:SI 6 "const1248_operand" "n")]
19136 (mem:BLK (scratch))
19137 (match_operand:<VEC_GATHER_SRCDI> 5 "register_operand" "1")]
19139 (parallel [(const_int 0) (const_int 1)
19140 (const_int 2) (const_int 3)])))
19141 (clobber (match_scratch:VI4F_256 1 "=&x"))]
19143 "v<sseintprefix>gatherq<ssemodesuffix>\t{%5, %7, %0|%0, %7, %5}"
19144 [(set_attr "type" "ssemov")
19145 (set_attr "prefix" "vex")
19146 (set_attr "mode" "<sseinsnmode>")])
19148 (define_insn "*avx2_gatherdi<mode>_4"
19149 [(set (match_operand:<VEC_GATHER_SRCDI> 0 "register_operand" "=&x")
19150 (vec_select:<VEC_GATHER_SRCDI>
19153 (match_operator:<ssescalarmode> 6 "vsib_mem_operator"
19155 [(match_operand:P 2 "vsib_address_operand" "Tv")
19156 (match_operand:<VEC_GATHER_IDXDI> 3 "register_operand" "x")
19157 (match_operand:SI 5 "const1248_operand" "n")]
19159 (mem:BLK (scratch))
19160 (match_operand:<VEC_GATHER_SRCDI> 4 "register_operand" "1")]
19162 (parallel [(const_int 0) (const_int 1)
19163 (const_int 2) (const_int 3)])))
19164 (clobber (match_scratch:VI4F_256 1 "=&x"))]
19166 "v<sseintprefix>gatherq<ssemodesuffix>\t{%4, %6, %0|%0, %6, %4}"
19167 [(set_attr "type" "ssemov")
19168 (set_attr "prefix" "vex")
19169 (set_attr "mode" "<sseinsnmode>")])
19171 ;; Memory operand override for -masm=intel of the v*gatherq* patterns.
19172 (define_mode_attr gatherq_mode
19173 [(V4SI "q") (V2DI "x") (V4SF "q") (V2DF "x")
19174 (V8SI "x") (V4DI "t") (V8SF "x") (V4DF "t")
19175 (V16SI "t") (V8DI "g") (V16SF "t") (V8DF "g")])
19177 (define_expand "<avx512>_gathersi<mode>"
19178 [(parallel [(set (match_operand:VI48F 0 "register_operand")
19180 [(match_operand:VI48F 1 "register_operand")
19181 (match_operand:<avx512fmaskmode> 4 "register_operand")
19182 (mem:<ssescalarmode>
19184 [(match_operand 2 "vsib_address_operand")
19185 (match_operand:<VEC_GATHER_IDXSI> 3 "register_operand")
19186 (match_operand:SI 5 "const1248_operand")]))]
19188 (clobber (match_scratch:<avx512fmaskmode> 7))])]
19192 = gen_rtx_UNSPEC (Pmode, gen_rtvec (3, operands[2], operands[3],
19193 operands[5]), UNSPEC_VSIBADDR);
19196 (define_insn "*avx512f_gathersi<mode>"
19197 [(set (match_operand:VI48F 0 "register_operand" "=&v")
19199 [(match_operand:VI48F 1 "register_operand" "0")
19200 (match_operand:<avx512fmaskmode> 7 "register_operand" "2")
19201 (match_operator:<ssescalarmode> 6 "vsib_mem_operator"
19203 [(match_operand:P 4 "vsib_address_operand" "Tv")
19204 (match_operand:<VEC_GATHER_IDXSI> 3 "register_operand" "v")
19205 (match_operand:SI 5 "const1248_operand" "n")]
19206 UNSPEC_VSIBADDR)])]
19208 (clobber (match_scratch:<avx512fmaskmode> 2 "=&Yk"))]
19210 "v<sseintprefix>gatherd<ssemodesuffix>\t{%6, %0%{%2%}|%0%{%2%}, %<xtg_mode>6}"
19211 [(set_attr "type" "ssemov")
19212 (set_attr "prefix" "evex")
19213 (set_attr "mode" "<sseinsnmode>")])
19215 (define_insn "*avx512f_gathersi<mode>_2"
19216 [(set (match_operand:VI48F 0 "register_operand" "=&v")
19219 (match_operand:<avx512fmaskmode> 6 "register_operand" "1")
19220 (match_operator:<ssescalarmode> 5 "vsib_mem_operator"
19222 [(match_operand:P 3 "vsib_address_operand" "Tv")
19223 (match_operand:<VEC_GATHER_IDXSI> 2 "register_operand" "v")
19224 (match_operand:SI 4 "const1248_operand" "n")]
19225 UNSPEC_VSIBADDR)])]
19227 (clobber (match_scratch:<avx512fmaskmode> 1 "=&Yk"))]
19229 "v<sseintprefix>gatherd<ssemodesuffix>\t{%5, %0%{%1%}|%0%{%1%}, %<xtg_mode>5}"
19230 [(set_attr "type" "ssemov")
19231 (set_attr "prefix" "evex")
19232 (set_attr "mode" "<sseinsnmode>")])
19235 (define_expand "<avx512>_gatherdi<mode>"
19236 [(parallel [(set (match_operand:VI48F 0 "register_operand")
19238 [(match_operand:<VEC_GATHER_SRCDI> 1 "register_operand")
19239 (match_operand:QI 4 "register_operand")
19240 (mem:<ssescalarmode>
19242 [(match_operand 2 "vsib_address_operand")
19243 (match_operand:<VEC_GATHER_IDXDI> 3 "register_operand")
19244 (match_operand:SI 5 "const1248_operand")]))]
19246 (clobber (match_scratch:QI 7))])]
19250 = gen_rtx_UNSPEC (Pmode, gen_rtvec (3, operands[2], operands[3],
19251 operands[5]), UNSPEC_VSIBADDR);
19254 (define_insn "*avx512f_gatherdi<mode>"
19255 [(set (match_operand:VI48F 0 "register_operand" "=&v")
19257 [(match_operand:<VEC_GATHER_SRCDI> 1 "register_operand" "0")
19258 (match_operand:QI 7 "register_operand" "2")
19259 (match_operator:<ssescalarmode> 6 "vsib_mem_operator"
19261 [(match_operand:P 4 "vsib_address_operand" "Tv")
19262 (match_operand:<VEC_GATHER_IDXDI> 3 "register_operand" "v")
19263 (match_operand:SI 5 "const1248_operand" "n")]
19264 UNSPEC_VSIBADDR)])]
19266 (clobber (match_scratch:QI 2 "=&Yk"))]
19269 return "v<sseintprefix>gatherq<ssemodesuffix>\t{%6, %1%{%2%}|%1%{%2%}, %<gatherq_mode>6}";
19271 [(set_attr "type" "ssemov")
19272 (set_attr "prefix" "evex")
19273 (set_attr "mode" "<sseinsnmode>")])
19275 (define_insn "*avx512f_gatherdi<mode>_2"
19276 [(set (match_operand:VI48F 0 "register_operand" "=&v")
19279 (match_operand:QI 6 "register_operand" "1")
19280 (match_operator:<ssescalarmode> 5 "vsib_mem_operator"
19282 [(match_operand:P 3 "vsib_address_operand" "Tv")
19283 (match_operand:<VEC_GATHER_IDXDI> 2 "register_operand" "v")
19284 (match_operand:SI 4 "const1248_operand" "n")]
19285 UNSPEC_VSIBADDR)])]
19287 (clobber (match_scratch:QI 1 "=&Yk"))]
19290 if (<MODE>mode != <VEC_GATHER_SRCDI>mode)
19292 if (<MODE_SIZE> != 64)
19293 return "v<sseintprefix>gatherq<ssemodesuffix>\t{%5, %x0%{%1%}|%x0%{%1%}, %<gatherq_mode>5}";
19295 return "v<sseintprefix>gatherq<ssemodesuffix>\t{%5, %t0%{%1%}|%t0%{%1%}, %t5}";
19297 return "v<sseintprefix>gatherq<ssemodesuffix>\t{%5, %0%{%1%}|%0%{%1%}, %<gatherq_mode>5}";
19299 [(set_attr "type" "ssemov")
19300 (set_attr "prefix" "evex")
19301 (set_attr "mode" "<sseinsnmode>")])
19303 (define_expand "<avx512>_scattersi<mode>"
19304 [(parallel [(set (mem:VI48F
19306 [(match_operand 0 "vsib_address_operand")
19307 (match_operand:<VEC_GATHER_IDXSI> 2 "register_operand")
19308 (match_operand:SI 4 "const1248_operand")]))
19310 [(match_operand:<avx512fmaskmode> 1 "register_operand")
19311 (match_operand:VI48F 3 "register_operand")]
19313 (clobber (match_scratch:<avx512fmaskmode> 6))])]
19317 = gen_rtx_UNSPEC (Pmode, gen_rtvec (3, operands[0], operands[2],
19318 operands[4]), UNSPEC_VSIBADDR);
19321 (define_insn "*avx512f_scattersi<mode>"
19322 [(set (match_operator:VI48F 5 "vsib_mem_operator"
19324 [(match_operand:P 0 "vsib_address_operand" "Tv")
19325 (match_operand:<VEC_GATHER_IDXSI> 2 "register_operand" "v")
19326 (match_operand:SI 4 "const1248_operand" "n")]
19329 [(match_operand:<avx512fmaskmode> 6 "register_operand" "1")
19330 (match_operand:VI48F 3 "register_operand" "v")]
19332 (clobber (match_scratch:<avx512fmaskmode> 1 "=&Yk"))]
19334 "v<sseintprefix>scatterd<ssemodesuffix>\t{%3, %5%{%1%}|%5%{%1%}, %3}"
19335 [(set_attr "type" "ssemov")
19336 (set_attr "prefix" "evex")
19337 (set_attr "mode" "<sseinsnmode>")])
19339 (define_expand "<avx512>_scatterdi<mode>"
19340 [(parallel [(set (mem:VI48F
19342 [(match_operand 0 "vsib_address_operand")
19343 (match_operand:<VEC_GATHER_IDXDI> 2 "register_operand")
19344 (match_operand:SI 4 "const1248_operand")]))
19346 [(match_operand:QI 1 "register_operand")
19347 (match_operand:<VEC_GATHER_SRCDI> 3 "register_operand")]
19349 (clobber (match_scratch:QI 6))])]
19353 = gen_rtx_UNSPEC (Pmode, gen_rtvec (3, operands[0], operands[2],
19354 operands[4]), UNSPEC_VSIBADDR);
19357 (define_insn "*avx512f_scatterdi<mode>"
19358 [(set (match_operator:VI48F 5 "vsib_mem_operator"
19360 [(match_operand:P 0 "vsib_address_operand" "Tv")
19361 (match_operand:<VEC_GATHER_IDXDI> 2 "register_operand" "v")
19362 (match_operand:SI 4 "const1248_operand" "n")]
19365 [(match_operand:QI 6 "register_operand" "1")
19366 (match_operand:<VEC_GATHER_SRCDI> 3 "register_operand" "v")]
19368 (clobber (match_scratch:QI 1 "=&Yk"))]
19371 if (GET_MODE_SIZE (GET_MODE_INNER (<MODE>mode)) == 8)
19372 return "v<sseintprefix>scatterq<ssemodesuffix>\t{%3, %5%{%1%}|%5%{%1%}, %3}";
19373 return "v<sseintprefix>scatterq<ssemodesuffix>\t{%3, %5%{%1%}|%t5%{%1%}, %3}";
19375 [(set_attr "type" "ssemov")
19376 (set_attr "prefix" "evex")
19377 (set_attr "mode" "<sseinsnmode>")])
19379 (define_insn "<avx512>_compress<mode>_mask"
19380 [(set (match_operand:VI48F 0 "register_operand" "=v")
19382 [(match_operand:VI48F 1 "register_operand" "v")
19383 (match_operand:VI48F 2 "vector_move_operand" "0C")
19384 (match_operand:<avx512fmaskmode> 3 "register_operand" "Yk")]
19387 "v<sseintprefix>compress<ssemodesuffix>\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"
19388 [(set_attr "type" "ssemov")
19389 (set_attr "prefix" "evex")
19390 (set_attr "mode" "<sseinsnmode>")])
19392 (define_insn "compress<mode>_mask"
19393 [(set (match_operand:VI12_AVX512VLBW 0 "register_operand" "=v")
19394 (unspec:VI12_AVX512VLBW
19395 [(match_operand:VI12_AVX512VLBW 1 "register_operand" "v")
19396 (match_operand:VI12_AVX512VLBW 2 "vector_move_operand" "0C")
19397 (match_operand:<avx512fmaskmode> 3 "register_operand" "Yk")]
19399 "TARGET_AVX512VBMI2"
19400 "vpcompress<ssemodesuffix>\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"
19401 [(set_attr "type" "ssemov")
19402 (set_attr "prefix" "evex")
19403 (set_attr "mode" "<sseinsnmode>")])
19405 (define_insn "<avx512>_compressstore<mode>_mask"
19406 [(set (match_operand:VI48F 0 "memory_operand" "=m")
19408 [(match_operand:VI48F 1 "register_operand" "x")
19410 (match_operand:<avx512fmaskmode> 2 "register_operand" "Yk")]
19411 UNSPEC_COMPRESS_STORE))]
19413 "v<sseintprefix>compress<ssemodesuffix>\t{%1, %0%{%2%}|%0%{%2%}, %1}"
19414 [(set_attr "type" "ssemov")
19415 (set_attr "prefix" "evex")
19416 (set_attr "memory" "store")
19417 (set_attr "mode" "<sseinsnmode>")])
19419 (define_insn "compressstore<mode>_mask"
19420 [(set (match_operand:VI12_AVX512VLBW 0 "memory_operand" "=m")
19421 (unspec:VI12_AVX512VLBW
19422 [(match_operand:VI12_AVX512VLBW 1 "register_operand" "x")
19424 (match_operand:<avx512fmaskmode> 2 "register_operand" "Yk")]
19425 UNSPEC_COMPRESS_STORE))]
19426 "TARGET_AVX512VBMI2"
19427 "vpcompress<ssemodesuffix>\t{%1, %0%{%2%}|%0%{%2%}, %1}"
19428 [(set_attr "type" "ssemov")
19429 (set_attr "prefix" "evex")
19430 (set_attr "memory" "store")
19431 (set_attr "mode" "<sseinsnmode>")])
19433 (define_expand "<avx512>_expand<mode>_maskz"
19434 [(set (match_operand:VI48F 0 "register_operand")
19436 [(match_operand:VI48F 1 "nonimmediate_operand")
19437 (match_operand:VI48F 2 "vector_move_operand")
19438 (match_operand:<avx512fmaskmode> 3 "register_operand")]
19441 "operands[2] = CONST0_RTX (<MODE>mode);")
19443 (define_insn "<avx512>_expand<mode>_mask"
19444 [(set (match_operand:VI48F 0 "register_operand" "=v,v")
19446 [(match_operand:VI48F 1 "nonimmediate_operand" "v,m")
19447 (match_operand:VI48F 2 "vector_move_operand" "0C,0C")
19448 (match_operand:<avx512fmaskmode> 3 "register_operand" "Yk,Yk")]
19451 "v<sseintprefix>expand<ssemodesuffix>\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"
19452 [(set_attr "type" "ssemov")
19453 (set_attr "prefix" "evex")
19454 (set_attr "memory" "none,load")
19455 (set_attr "mode" "<sseinsnmode>")])
19457 (define_insn "expand<mode>_mask"
19458 [(set (match_operand:VI12_AVX512VLBW 0 "register_operand" "=v,v")
19459 (unspec:VI12_AVX512VLBW
19460 [(match_operand:VI12_AVX512VLBW 1 "nonimmediate_operand" "v,m")
19461 (match_operand:VI12_AVX512VLBW 2 "vector_move_operand" "0C,0C")
19462 (match_operand:<avx512fmaskmode> 3 "register_operand" "Yk,Yk")]
19464 "TARGET_AVX512VBMI2"
19465 "v<sseintprefix>expand<ssemodesuffix>\t{%1, %0%{%3%}%N2|%0%{%3%}%N2, %1}"
19466 [(set_attr "type" "ssemov")
19467 (set_attr "prefix" "evex")
19468 (set_attr "memory" "none,load")
19469 (set_attr "mode" "<sseinsnmode>")])
19471 (define_expand "expand<mode>_maskz"
19472 [(set (match_operand:VI12_AVX512VLBW 0 "register_operand")
19473 (unspec:VI12_AVX512VLBW
19474 [(match_operand:VI12_AVX512VLBW 1 "nonimmediate_operand")
19475 (match_operand:VI12_AVX512VLBW 2 "vector_move_operand")
19476 (match_operand:<avx512fmaskmode> 3 "register_operand")]
19478 "TARGET_AVX512VBMI2"
19479 "operands[2] = CONST0_RTX (<MODE>mode);")
19481 (define_insn "avx512dq_rangep<mode><mask_name><round_saeonly_name>"
19482 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v")
19483 (unspec:VF_AVX512VL
19484 [(match_operand:VF_AVX512VL 1 "register_operand" "v")
19485 (match_operand:VF_AVX512VL 2 "<round_saeonly_nimm_predicate>" "<round_saeonly_constraint>")
19486 (match_operand:SI 3 "const_0_to_15_operand")]
19488 "TARGET_AVX512DQ && <round_saeonly_mode512bit_condition>"
19489 "vrange<ssemodesuffix>\t{%3, <round_saeonly_mask_op4>%2, %1, %0<mask_operand4>|%0<mask_operand4>, %1, %2<round_saeonly_mask_op4>, %3}"
19490 [(set_attr "type" "sse")
19491 (set_attr "prefix" "evex")
19492 (set_attr "mode" "<MODE>")])
19494 (define_insn "avx512dq_ranges<mode><mask_scalar_name><round_saeonly_scalar_name>"
19495 [(set (match_operand:VF_128 0 "register_operand" "=v")
19498 [(match_operand:VF_128 1 "register_operand" "v")
19499 (match_operand:VF_128 2 "<round_saeonly_scalar_nimm_predicate>" "<round_saeonly_scalar_constraint>")
19500 (match_operand:SI 3 "const_0_to_15_operand")]
19505 "vrange<ssescalarmodesuffix>\t{%3, <round_saeonly_scalar_mask_op4>%2, %1, %0<mask_scalar_operand4>|%0<mask_scalar_operand4>, %1, %2<round_saeonly_scalar_mask_op4>, %3}"
19506 [(set_attr "type" "sse")
19507 (set_attr "prefix" "evex")
19508 (set_attr "mode" "<MODE>")])
19510 (define_insn "avx512dq_fpclass<mode><mask_scalar_merge_name>"
19511 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=Yk")
19512 (unspec:<avx512fmaskmode>
19513 [(match_operand:VF_AVX512VL 1 "register_operand" "v")
19514 (match_operand:QI 2 "const_0_to_255_operand" "n")]
19517 "vfpclass<ssemodesuffix>\t{%2, %1, %0<mask_scalar_merge_operand3>|%0<mask_scalar_merge_operand3>, %1, %2}";
19518 [(set_attr "type" "sse")
19519 (set_attr "length_immediate" "1")
19520 (set_attr "prefix" "evex")
19521 (set_attr "mode" "<MODE>")])
19523 (define_insn "avx512dq_vmfpclass<mode>"
19524 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=Yk")
19525 (and:<avx512fmaskmode>
19526 (unspec:<avx512fmaskmode>
19527 [(match_operand:VF_128 1 "register_operand" "v")
19528 (match_operand:QI 2 "const_0_to_255_operand" "n")]
19532 "vfpclass<ssescalarmodesuffix>\t{%2, %1, %0|%0, %1, %2}";
19533 [(set_attr "type" "sse")
19534 (set_attr "length_immediate" "1")
19535 (set_attr "prefix" "evex")
19536 (set_attr "mode" "<MODE>")])
19538 (define_insn "<avx512>_getmant<mode><mask_name><round_saeonly_name>"
19539 [(set (match_operand:VF_AVX512VL 0 "register_operand" "=v")
19540 (unspec:VF_AVX512VL
19541 [(match_operand:VF_AVX512VL 1 "nonimmediate_operand" "<round_saeonly_constraint>")
19542 (match_operand:SI 2 "const_0_to_15_operand")]
19545 "vgetmant<ssemodesuffix>\t{%2, <round_saeonly_mask_op3>%1, %0<mask_operand3>|%0<mask_operand3>, %1<round_saeonly_mask_op3>, %2}";
19546 [(set_attr "prefix" "evex")
19547 (set_attr "mode" "<MODE>")])
19549 (define_insn "avx512f_vgetmant<mode><mask_scalar_name><round_saeonly_scalar_name>"
19550 [(set (match_operand:VF_128 0 "register_operand" "=v")
19553 [(match_operand:VF_128 1 "register_operand" "v")
19554 (match_operand:VF_128 2 "<round_saeonly_scalar_nimm_predicate>" "<round_saeonly_scalar_constraint>")
19555 (match_operand:SI 3 "const_0_to_15_operand")]
19560 "vgetmant<ssescalarmodesuffix>\t{%3, <round_saeonly_scalar_mask_op4>%2, %1, %0<mask_scalar_operand4>|%0<mask_scalar_operand4>, %1, %2<round_saeonly_scalar_mask_op4>, %3}";
19561 [(set_attr "prefix" "evex")
19562 (set_attr "mode" "<ssescalarmode>")])
19564 ;; The correct representation for this is absolutely enormous, and
19565 ;; surely not generally useful.
19566 (define_insn "<mask_codefor>avx512bw_dbpsadbw<mode><mask_name>"
19567 [(set (match_operand:VI2_AVX512VL 0 "register_operand" "=v")
19568 (unspec:VI2_AVX512VL
19569 [(match_operand:<dbpsadbwmode> 1 "register_operand" "v")
19570 (match_operand:<dbpsadbwmode> 2 "nonimmediate_operand" "vm")
19571 (match_operand:SI 3 "const_0_to_255_operand")]
19574 "vdbpsadbw\t{%3, %2, %1, %0<mask_operand4>|%0<mask_operand4>, %1, %2, %3}"
19575 [(set_attr "type" "sselog1")
19576 (set_attr "length_immediate" "1")
19577 (set_attr "prefix" "evex")
19578 (set_attr "mode" "<sseinsnmode>")])
19580 (define_insn "clz<mode>2<mask_name>"
19581 [(set (match_operand:VI48_AVX512VL 0 "register_operand" "=v")
19583 (match_operand:VI48_AVX512VL 1 "nonimmediate_operand" "vm")))]
19585 "vplzcnt<ssemodesuffix>\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
19586 [(set_attr "type" "sse")
19587 (set_attr "prefix" "evex")
19588 (set_attr "mode" "<sseinsnmode>")])
19590 (define_insn "<mask_codefor>conflict<mode><mask_name>"
19591 [(set (match_operand:VI48_AVX512VL 0 "register_operand" "=v")
19592 (unspec:VI48_AVX512VL
19593 [(match_operand:VI48_AVX512VL 1 "nonimmediate_operand" "vm")]
19596 "vpconflict<ssemodesuffix>\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}"
19597 [(set_attr "type" "sse")
19598 (set_attr "prefix" "evex")
19599 (set_attr "mode" "<sseinsnmode>")])
19601 (define_insn "sha1msg1"
19602 [(set (match_operand:V4SI 0 "register_operand" "=x")
19604 [(match_operand:V4SI 1 "register_operand" "0")
19605 (match_operand:V4SI 2 "vector_operand" "xBm")]
19608 "sha1msg1\t{%2, %0|%0, %2}"
19609 [(set_attr "type" "sselog1")
19610 (set_attr "mode" "TI")])
19612 (define_insn "sha1msg2"
19613 [(set (match_operand:V4SI 0 "register_operand" "=x")
19615 [(match_operand:V4SI 1 "register_operand" "0")
19616 (match_operand:V4SI 2 "vector_operand" "xBm")]
19619 "sha1msg2\t{%2, %0|%0, %2}"
19620 [(set_attr "type" "sselog1")
19621 (set_attr "mode" "TI")])
19623 (define_insn "sha1nexte"
19624 [(set (match_operand:V4SI 0 "register_operand" "=x")
19626 [(match_operand:V4SI 1 "register_operand" "0")
19627 (match_operand:V4SI 2 "vector_operand" "xBm")]
19628 UNSPEC_SHA1NEXTE))]
19630 "sha1nexte\t{%2, %0|%0, %2}"
19631 [(set_attr "type" "sselog1")
19632 (set_attr "mode" "TI")])
19634 (define_insn "sha1rnds4"
19635 [(set (match_operand:V4SI 0 "register_operand" "=x")
19637 [(match_operand:V4SI 1 "register_operand" "0")
19638 (match_operand:V4SI 2 "vector_operand" "xBm")
19639 (match_operand:SI 3 "const_0_to_3_operand" "n")]
19640 UNSPEC_SHA1RNDS4))]
19642 "sha1rnds4\t{%3, %2, %0|%0, %2, %3}"
19643 [(set_attr "type" "sselog1")
19644 (set_attr "length_immediate" "1")
19645 (set_attr "mode" "TI")])
19647 (define_insn "sha256msg1"
19648 [(set (match_operand:V4SI 0 "register_operand" "=x")
19650 [(match_operand:V4SI 1 "register_operand" "0")
19651 (match_operand:V4SI 2 "vector_operand" "xBm")]
19652 UNSPEC_SHA256MSG1))]
19654 "sha256msg1\t{%2, %0|%0, %2}"
19655 [(set_attr "type" "sselog1")
19656 (set_attr "mode" "TI")])
19658 (define_insn "sha256msg2"
19659 [(set (match_operand:V4SI 0 "register_operand" "=x")
19661 [(match_operand:V4SI 1 "register_operand" "0")
19662 (match_operand:V4SI 2 "vector_operand" "xBm")]
19663 UNSPEC_SHA256MSG2))]
19665 "sha256msg2\t{%2, %0|%0, %2}"
19666 [(set_attr "type" "sselog1")
19667 (set_attr "mode" "TI")])
19669 (define_insn "sha256rnds2"
19670 [(set (match_operand:V4SI 0 "register_operand" "=x")
19672 [(match_operand:V4SI 1 "register_operand" "0")
19673 (match_operand:V4SI 2 "vector_operand" "xBm")
19674 (match_operand:V4SI 3 "register_operand" "Yz")]
19675 UNSPEC_SHA256RNDS2))]
19677 "sha256rnds2\t{%3, %2, %0|%0, %2, %3}"
19678 [(set_attr "type" "sselog1")
19679 (set_attr "length_immediate" "1")
19680 (set_attr "mode" "TI")])
19682 (define_insn_and_split "avx512f_<castmode><avxsizesuffix>_<castmode>"
19683 [(set (match_operand:AVX512MODE2P 0 "nonimmediate_operand" "=x,m")
19684 (unspec:AVX512MODE2P
19685 [(match_operand:<ssequartermode> 1 "nonimmediate_operand" "xm,x")]
19687 "TARGET_AVX512F && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
19689 "&& reload_completed"
19690 [(set (match_dup 0) (match_dup 1))]
19692 if (REG_P (operands[0]))
19693 operands[0] = gen_lowpart (<ssequartermode>mode, operands[0]);
19695 operands[1] = lowpart_subreg (<MODE>mode, operands[1],
19696 <ssequartermode>mode);
19699 (define_insn_and_split "avx512f_<castmode><avxsizesuffix>_256<castmode>"
19700 [(set (match_operand:AVX512MODE2P 0 "nonimmediate_operand" "=x,m")
19701 (unspec:AVX512MODE2P
19702 [(match_operand:<ssehalfvecmode> 1 "nonimmediate_operand" "xm,x")]
19704 "TARGET_AVX512F && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
19706 "&& reload_completed"
19707 [(set (match_dup 0) (match_dup 1))]
19709 if (REG_P (operands[0]))
19710 operands[0] = gen_lowpart (<ssehalfvecmode>mode, operands[0]);
19712 operands[1] = lowpart_subreg (<MODE>mode, operands[1],
19713 <ssehalfvecmode>mode);
19716 (define_int_iterator VPMADD52
19717 [UNSPEC_VPMADD52LUQ
19718 UNSPEC_VPMADD52HUQ])
19720 (define_int_attr vpmadd52type
19721 [(UNSPEC_VPMADD52LUQ "luq") (UNSPEC_VPMADD52HUQ "huq")])
19723 (define_expand "vpamdd52huq<mode>_maskz"
19724 [(match_operand:VI8_AVX512VL 0 "register_operand")
19725 (match_operand:VI8_AVX512VL 1 "register_operand")
19726 (match_operand:VI8_AVX512VL 2 "register_operand")
19727 (match_operand:VI8_AVX512VL 3 "nonimmediate_operand")
19728 (match_operand:<avx512fmaskmode> 4 "register_operand")]
19729 "TARGET_AVX512IFMA"
19731 emit_insn (gen_vpamdd52huq<mode>_maskz_1 (
19732 operands[0], operands[1], operands[2], operands[3],
19733 CONST0_RTX (<MODE>mode), operands[4]));
19737 (define_expand "vpamdd52luq<mode>_maskz"
19738 [(match_operand:VI8_AVX512VL 0 "register_operand")
19739 (match_operand:VI8_AVX512VL 1 "register_operand")
19740 (match_operand:VI8_AVX512VL 2 "register_operand")
19741 (match_operand:VI8_AVX512VL 3 "nonimmediate_operand")
19742 (match_operand:<avx512fmaskmode> 4 "register_operand")]
19743 "TARGET_AVX512IFMA"
19745 emit_insn (gen_vpamdd52luq<mode>_maskz_1 (
19746 operands[0], operands[1], operands[2], operands[3],
19747 CONST0_RTX (<MODE>mode), operands[4]));
19751 (define_insn "vpamdd52<vpmadd52type><mode><sd_maskz_name>"
19752 [(set (match_operand:VI8_AVX512VL 0 "register_operand" "=v")
19753 (unspec:VI8_AVX512VL
19754 [(match_operand:VI8_AVX512VL 1 "register_operand" "0")
19755 (match_operand:VI8_AVX512VL 2 "register_operand" "v")
19756 (match_operand:VI8_AVX512VL 3 "nonimmediate_operand" "vm")]
19758 "TARGET_AVX512IFMA"
19759 "vpmadd52<vpmadd52type>\t{%3, %2, %0<sd_mask_op4>|%0<sd_mask_op4>, %2, %3}"
19760 [(set_attr "type" "ssemuladd")
19761 (set_attr "prefix" "evex")
19762 (set_attr "mode" "<sseinsnmode>")])
19764 (define_insn "vpamdd52<vpmadd52type><mode>_mask"
19765 [(set (match_operand:VI8_AVX512VL 0 "register_operand" "=v")
19766 (vec_merge:VI8_AVX512VL
19767 (unspec:VI8_AVX512VL
19768 [(match_operand:VI8_AVX512VL 1 "register_operand" "0")
19769 (match_operand:VI8_AVX512VL 2 "register_operand" "v")
19770 (match_operand:VI8_AVX512VL 3 "nonimmediate_operand" "vm")]
19773 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk")))]
19774 "TARGET_AVX512IFMA"
19775 "vpmadd52<vpmadd52type>\t{%3, %2, %0%{%4%}|%0%{%4%}, %2, %3}"
19776 [(set_attr "type" "ssemuladd")
19777 (set_attr "prefix" "evex")
19778 (set_attr "mode" "<sseinsnmode>")])
19780 (define_insn "vpmultishiftqb<mode><mask_name>"
19781 [(set (match_operand:VI1_AVX512VL 0 "register_operand" "=v")
19782 (unspec:VI1_AVX512VL
19783 [(match_operand:VI1_AVX512VL 1 "register_operand" "v")
19784 (match_operand:VI1_AVX512VL 2 "nonimmediate_operand" "vm")]
19785 UNSPEC_VPMULTISHIFT))]
19786 "TARGET_AVX512VBMI"
19787 "vpmultishiftqb\t{%2, %1, %0<mask_operand3>|%0<mask_operand3>, %1, %2}"
19788 [(set_attr "type" "sselog")
19789 (set_attr "prefix" "evex")
19790 (set_attr "mode" "<sseinsnmode>")])
19792 (define_mode_iterator IMOD4
19793 [(V64SF "TARGET_AVX5124FMAPS") (V64SI "TARGET_AVX5124VNNIW")])
19795 (define_mode_attr imod4_narrow
19796 [(V64SF "V16SF") (V64SI "V16SI")])
19798 (define_expand "mov<mode>"
19799 [(set (match_operand:IMOD4 0 "nonimmediate_operand")
19800 (match_operand:IMOD4 1 "vector_move_operand"))]
19803 ix86_expand_vector_move (<MODE>mode, operands);
19807 (define_insn_and_split "*mov<mode>_internal"
19808 [(set (match_operand:IMOD4 0 "nonimmediate_operand" "=v,v ,m")
19809 (match_operand:IMOD4 1 "vector_move_operand" " C,vm,v"))]
19811 && (register_operand (operands[0], <MODE>mode)
19812 || register_operand (operands[1], <MODE>mode))"
19814 "&& reload_completed"
19820 for (i = 0; i < 4; i++)
19822 op0 = simplify_subreg
19823 (<imod4_narrow>mode, operands[0], <MODE>mode, i * 64);
19824 op1 = simplify_subreg
19825 (<imod4_narrow>mode, operands[1], <MODE>mode, i * 64);
19826 emit_move_insn (op0, op1);
19831 (define_insn "avx5124fmaddps_4fmaddps"
19832 [(set (match_operand:V16SF 0 "register_operand" "=v")
19834 [(match_operand:V16SF 1 "register_operand" "0")
19835 (match_operand:V64SF 2 "register_operand" "Yh")
19836 (match_operand:V4SF 3 "memory_operand" "m")] UNSPEC_VP4FMADD))]
19837 "TARGET_AVX5124FMAPS"
19838 "v4fmaddps\t{%3, %g2, %0|%0, %g2, %3}"
19839 [(set_attr ("type") ("ssemuladd"))
19840 (set_attr ("prefix") ("evex"))
19841 (set_attr ("mode") ("V16SF"))])
19843 (define_insn "avx5124fmaddps_4fmaddps_mask"
19844 [(set (match_operand:V16SF 0 "register_operand" "=v")
19847 [(match_operand:V64SF 1 "register_operand" "Yh")
19848 (match_operand:V4SF 2 "memory_operand" "m")] UNSPEC_VP4FMADD)
19849 (match_operand:V16SF 3 "register_operand" "0")
19850 (match_operand:HI 4 "register_operand" "Yk")))]
19851 "TARGET_AVX5124FMAPS"
19852 "v4fmaddps\t{%2, %g1, %0%{%4%}|%{%4%}%0, %g1, %2}"
19853 [(set_attr ("type") ("ssemuladd"))
19854 (set_attr ("prefix") ("evex"))
19855 (set_attr ("mode") ("V16SF"))])
19857 (define_insn "avx5124fmaddps_4fmaddps_maskz"
19858 [(set (match_operand:V16SF 0 "register_operand" "=v")
19861 [(match_operand:V16SF 1 "register_operand" "0")
19862 (match_operand:V64SF 2 "register_operand" "Yh")
19863 (match_operand:V4SF 3 "memory_operand" "m")] UNSPEC_VP4FMADD)
19864 (match_operand:V16SF 4 "const0_operand" "C")
19865 (match_operand:HI 5 "register_operand" "Yk")))]
19866 "TARGET_AVX5124FMAPS"
19867 "v4fmaddps\t{%3, %g2, %0%{%5%}%{z%}|%{%5%}%{z%}%0, %g2, %3}"
19868 [(set_attr ("type") ("ssemuladd"))
19869 (set_attr ("prefix") ("evex"))
19870 (set_attr ("mode") ("V16SF"))])
19872 (define_insn "avx5124fmaddps_4fmaddss"
19873 [(set (match_operand:V4SF 0 "register_operand" "=v")
19875 [(match_operand:V4SF 1 "register_operand" "0")
19876 (match_operand:V64SF 2 "register_operand" "Yh")
19877 (match_operand:V4SF 3 "memory_operand" "m")] UNSPEC_VP4FMADD))]
19878 "TARGET_AVX5124FMAPS"
19879 "v4fmaddss\t{%3, %x2, %0|%0, %x2, %3}"
19880 [(set_attr ("type") ("ssemuladd"))
19881 (set_attr ("prefix") ("evex"))
19882 (set_attr ("mode") ("SF"))])
19884 (define_insn "avx5124fmaddps_4fmaddss_mask"
19885 [(set (match_operand:V4SF 0 "register_operand" "=v")
19888 [(match_operand:V64SF 1 "register_operand" "Yh")
19889 (match_operand:V4SF 2 "memory_operand" "m")] UNSPEC_VP4FMADD)
19890 (match_operand:V4SF 3 "register_operand" "0")
19891 (match_operand:QI 4 "register_operand" "Yk")))]
19892 "TARGET_AVX5124FMAPS"
19893 "v4fmaddss\t{%2, %x1, %0%{%4%}|%{%4%}%0, %x1, %2}"
19894 [(set_attr ("type") ("ssemuladd"))
19895 (set_attr ("prefix") ("evex"))
19896 (set_attr ("mode") ("SF"))])
19898 (define_insn "avx5124fmaddps_4fmaddss_maskz"
19899 [(set (match_operand:V4SF 0 "register_operand" "=v")
19902 [(match_operand:V4SF 1 "register_operand" "0")
19903 (match_operand:V64SF 2 "register_operand" "Yh")
19904 (match_operand:V4SF 3 "memory_operand" "m")] UNSPEC_VP4FMADD)
19905 (match_operand:V4SF 4 "const0_operand" "C")
19906 (match_operand:QI 5 "register_operand" "Yk")))]
19907 "TARGET_AVX5124FMAPS"
19908 "v4fmaddss\t{%3, %x2, %0%{%5%}%{z%}|%{%5%}%{z%}%0, %x2, %3}"
19909 [(set_attr ("type") ("ssemuladd"))
19910 (set_attr ("prefix") ("evex"))
19911 (set_attr ("mode") ("SF"))])
19913 (define_insn "avx5124fmaddps_4fnmaddps"
19914 [(set (match_operand:V16SF 0 "register_operand" "=v")
19916 [(match_operand:V16SF 1 "register_operand" "0")
19917 (match_operand:V64SF 2 "register_operand" "Yh")
19918 (match_operand:V4SF 3 "memory_operand" "m")] UNSPEC_VP4FNMADD))]
19919 "TARGET_AVX5124FMAPS"
19920 "v4fnmaddps\t{%3, %g2, %0|%0, %g2, %3}"
19921 [(set_attr ("type") ("ssemuladd"))
19922 (set_attr ("prefix") ("evex"))
19923 (set_attr ("mode") ("V16SF"))])
19925 (define_insn "avx5124fmaddps_4fnmaddps_mask"
19926 [(set (match_operand:V16SF 0 "register_operand" "=v")
19929 [(match_operand:V64SF 1 "register_operand" "Yh")
19930 (match_operand:V4SF 2 "memory_operand" "m")] UNSPEC_VP4FNMADD)
19931 (match_operand:V16SF 3 "register_operand" "0")
19932 (match_operand:HI 4 "register_operand" "Yk")))]
19933 "TARGET_AVX5124FMAPS"
19934 "v4fnmaddps\t{%2, %g1, %0%{%4%}|%{%4%}%0, %g1, %2}"
19935 [(set_attr ("type") ("ssemuladd"))
19936 (set_attr ("prefix") ("evex"))
19937 (set_attr ("mode") ("V16SF"))])
19939 (define_insn "avx5124fmaddps_4fnmaddps_maskz"
19940 [(set (match_operand:V16SF 0 "register_operand" "=v")
19943 [(match_operand:V16SF 1 "register_operand" "0")
19944 (match_operand:V64SF 2 "register_operand" "Yh")
19945 (match_operand:V4SF 3 "memory_operand" "m")] UNSPEC_VP4FNMADD)
19946 (match_operand:V16SF 4 "const0_operand" "C")
19947 (match_operand:HI 5 "register_operand" "Yk")))]
19948 "TARGET_AVX5124FMAPS"
19949 "v4fnmaddps\t{%3, %g2, %0%{%5%}%{z%}|%{%5%}%{z%}%0, %g2, %3}"
19950 [(set_attr ("type") ("ssemuladd"))
19951 (set_attr ("prefix") ("evex"))
19952 (set_attr ("mode") ("V16SF"))])
19954 (define_insn "avx5124fmaddps_4fnmaddss"
19955 [(set (match_operand:V4SF 0 "register_operand" "=v")
19957 [(match_operand:V4SF 1 "register_operand" "0")
19958 (match_operand:V64SF 2 "register_operand" "Yh")
19959 (match_operand:V4SF 3 "memory_operand" "m")] UNSPEC_VP4FNMADD))]
19960 "TARGET_AVX5124FMAPS"
19961 "v4fnmaddss\t{%3, %x2, %0|%0, %x2, %3}"
19962 [(set_attr ("type") ("ssemuladd"))
19963 (set_attr ("prefix") ("evex"))
19964 (set_attr ("mode") ("SF"))])
19966 (define_insn "avx5124fmaddps_4fnmaddss_mask"
19967 [(set (match_operand:V4SF 0 "register_operand" "=v")
19970 [(match_operand:V64SF 1 "register_operand" "Yh")
19971 (match_operand:V4SF 2 "memory_operand" "m")] UNSPEC_VP4FNMADD)
19972 (match_operand:V4SF 3 "register_operand" "0")
19973 (match_operand:QI 4 "register_operand" "Yk")))]
19974 "TARGET_AVX5124FMAPS"
19975 "v4fnmaddss\t{%2, %x1, %0%{%4%}|%{%4%}%0, %x1, %2}"
19976 [(set_attr ("type") ("ssemuladd"))
19977 (set_attr ("prefix") ("evex"))
19978 (set_attr ("mode") ("SF"))])
19980 (define_insn "avx5124fmaddps_4fnmaddss_maskz"
19981 [(set (match_operand:V4SF 0 "register_operand" "=v")
19984 [(match_operand:V4SF 1 "register_operand" "0")
19985 (match_operand:V64SF 2 "register_operand" "Yh")
19986 (match_operand:V4SF 3 "memory_operand" "m")] UNSPEC_VP4FNMADD)
19987 (match_operand:V4SF 4 "const0_operand" "C")
19988 (match_operand:QI 5 "register_operand" "Yk")))]
19989 "TARGET_AVX5124FMAPS"
19990 "v4fnmaddss\t{%3, %x2, %0%{%5%}%{z%}|%{%5%}%{z%}%0, %x2, %3}"
19991 [(set_attr ("type") ("ssemuladd"))
19992 (set_attr ("prefix") ("evex"))
19993 (set_attr ("mode") ("SF"))])
19995 (define_insn "avx5124vnniw_vp4dpwssd"
19996 [(set (match_operand:V16SI 0 "register_operand" "=v")
19998 [(match_operand:V16SI 1 "register_operand" "0")
19999 (match_operand:V64SI 2 "register_operand" "Yh")
20000 (match_operand:V4SI 3 "memory_operand" "m")] UNSPEC_VP4DPWSSD))]
20001 "TARGET_AVX5124VNNIW"
20002 "vp4dpwssd\t{%3, %g2, %0|%0, %g2, %3}"
20003 [(set_attr ("type") ("ssemuladd"))
20004 (set_attr ("prefix") ("evex"))
20005 (set_attr ("mode") ("TI"))])
20007 (define_insn "avx5124vnniw_vp4dpwssd_mask"
20008 [(set (match_operand:V16SI 0 "register_operand" "=v")
20011 [(match_operand:V64SI 1 "register_operand" "Yh")
20012 (match_operand:V4SI 2 "memory_operand" "m")] UNSPEC_VP4DPWSSD)
20013 (match_operand:V16SI 3 "register_operand" "0")
20014 (match_operand:HI 4 "register_operand" "Yk")))]
20015 "TARGET_AVX5124VNNIW"
20016 "vp4dpwssd\t{%2, %g1, %0%{%4%}|%{%4%}%0, %g1, %2}"
20017 [(set_attr ("type") ("ssemuladd"))
20018 (set_attr ("prefix") ("evex"))
20019 (set_attr ("mode") ("TI"))])
20021 (define_insn "avx5124vnniw_vp4dpwssd_maskz"
20022 [(set (match_operand:V16SI 0 "register_operand" "=v")
20025 [(match_operand:V16SI 1 "register_operand" "0")
20026 (match_operand:V64SI 2 "register_operand" "Yh")
20027 (match_operand:V4SI 3 "memory_operand" "m")] UNSPEC_VP4DPWSSD)
20028 (match_operand:V16SI 4 "const0_operand" "C")
20029 (match_operand:HI 5 "register_operand" "Yk")))]
20030 "TARGET_AVX5124VNNIW"
20031 "vp4dpwssd\t{%3, %g2, %0%{%5%}%{z%}|%{%5%}%{z%}%0, %g2, %3}"
20032 [(set_attr ("type") ("ssemuladd"))
20033 (set_attr ("prefix") ("evex"))
20034 (set_attr ("mode") ("TI"))])
20036 (define_insn "avx5124vnniw_vp4dpwssds"
20037 [(set (match_operand:V16SI 0 "register_operand" "=v")
20039 [(match_operand:V16SI 1 "register_operand" "0")
20040 (match_operand:V64SI 2 "register_operand" "Yh")
20041 (match_operand:V4SI 3 "memory_operand" "m")] UNSPEC_VP4DPWSSDS))]
20042 "TARGET_AVX5124VNNIW"
20043 "vp4dpwssds\t{%3, %g2, %0|%0, %g2, %3}"
20044 [(set_attr ("type") ("ssemuladd"))
20045 (set_attr ("prefix") ("evex"))
20046 (set_attr ("mode") ("TI"))])
20048 (define_insn "avx5124vnniw_vp4dpwssds_mask"
20049 [(set (match_operand:V16SI 0 "register_operand" "=v")
20052 [(match_operand:V64SI 1 "register_operand" "Yh")
20053 (match_operand:V4SI 2 "memory_operand" "m")] UNSPEC_VP4DPWSSDS)
20054 (match_operand:V16SI 3 "register_operand" "0")
20055 (match_operand:HI 4 "register_operand" "Yk")))]
20056 "TARGET_AVX5124VNNIW"
20057 "vp4dpwssds\t{%2, %g1, %0%{%4%}|%{%4%}%0, %g1, %2}"
20058 [(set_attr ("type") ("ssemuladd"))
20059 (set_attr ("prefix") ("evex"))
20060 (set_attr ("mode") ("TI"))])
20062 (define_insn "avx5124vnniw_vp4dpwssds_maskz"
20063 [(set (match_operand:V16SI 0 "register_operand" "=v")
20066 [(match_operand:V16SI 1 "register_operand" "0")
20067 (match_operand:V64SI 2 "register_operand" "Yh")
20068 (match_operand:V4SI 3 "memory_operand" "m")] UNSPEC_VP4DPWSSDS)
20069 (match_operand:V16SI 4 "const0_operand" "C")
20070 (match_operand:HI 5 "register_operand" "Yk")))]
20071 "TARGET_AVX5124VNNIW"
20072 "vp4dpwssds\t{%3, %g2, %0%{%5%}%{z%}|%{%5%}%{z%}%0, %g2, %3}"
20073 [(set_attr ("type") ("ssemuladd"))
20074 (set_attr ("prefix") ("evex"))
20075 (set_attr ("mode") ("TI"))])
20077 (define_insn "vpopcount<mode><mask_name>"
20078 [(set (match_operand:VI48_AVX512VL 0 "register_operand" "=v")
20079 (popcount:VI48_AVX512VL
20080 (match_operand:VI48_AVX512VL 1 "nonimmediate_operand" "vm")))]
20081 "TARGET_AVX512VPOPCNTDQ"
20082 "vpopcnt<ssemodesuffix>\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}")
20084 ;; Save multiple registers out-of-line.
20085 (define_insn "save_multiple<mode>"
20086 [(match_parallel 0 "save_multiple"
20087 [(use (match_operand:P 1 "symbol_operand"))])]
20088 "TARGET_SSE && TARGET_64BIT"
20091 ;; Restore multiple registers out-of-line.
20092 (define_insn "restore_multiple<mode>"
20093 [(match_parallel 0 "restore_multiple"
20094 [(use (match_operand:P 1 "symbol_operand"))])]
20095 "TARGET_SSE && TARGET_64BIT"
20098 ;; Restore multiple registers out-of-line and return.
20099 (define_insn "restore_multiple_and_return<mode>"
20100 [(match_parallel 0 "restore_multiple"
20102 (use (match_operand:P 1 "symbol_operand"))
20103 (set (reg:DI SP_REG) (reg:DI R10_REG))
20105 "TARGET_SSE && TARGET_64BIT"
20108 ;; Restore multiple registers out-of-line when hard frame pointer is used,
20109 ;; perform the leave operation prior to returning (from the function).
20110 (define_insn "restore_multiple_leave_return<mode>"
20111 [(match_parallel 0 "restore_multiple"
20113 (use (match_operand:P 1 "symbol_operand"))
20114 (set (reg:DI SP_REG) (plus:DI (reg:DI BP_REG) (const_int 8)))
20115 (set (reg:DI BP_REG) (mem:DI (reg:DI BP_REG)))
20116 (clobber (mem:BLK (scratch)))
20118 "TARGET_SSE && TARGET_64BIT"
20121 (define_insn "vpopcount<mode><mask_name>"
20122 [(set (match_operand:VI12_AVX512VL 0 "register_operand" "=v")
20123 (popcount:VI12_AVX512VL
20124 (match_operand:VI12_AVX512VL 1 "nonimmediate_operand" "vm")))]
20125 "TARGET_AVX512BITALG"
20126 "vpopcnt<ssemodesuffix>\t{%1, %0<mask_operand2>|%0<mask_operand2>, %1}")
20128 (define_insn "vgf2p8affineinvqb_<mode><mask_name>"
20129 [(set (match_operand:VI1_AVX512F 0 "register_operand" "=x,x,v")
20130 (unspec:VI1_AVX512F
20131 [(match_operand:VI1_AVX512F 1 "register_operand" "%0,x,v")
20132 (match_operand:VI1_AVX512F 2 "nonimmediate_operand" "xBm,xm,vm")
20133 (match_operand:QI 3 "const_0_to_255_operand" "n,n,n")]
20134 UNSPEC_GF2P8AFFINEINV))]
20137 gf2p8affineinvqb\t{%3, %2, %0| %0, %2, %3}
20138 vgf2p8affineinvqb\t{%3, %2, %1, %0<mask_operand4>| %0<mask_operand4>, %1, %2, %3}
20139 vgf2p8affineinvqb\t{%3, %2, %1, %0<mask_operand4>| %0<mask_operand4>, %1, %2, %3}"
20140 [(set_attr "isa" "noavx,avx,avx512f")
20141 (set_attr "prefix_data16" "1,*,*")
20142 (set_attr "prefix_extra" "1")
20143 (set_attr "prefix" "orig,maybe_evex,evex")
20144 (set_attr "mode" "<sseinsnmode>")])
20146 (define_insn "vgf2p8affineqb_<mode><mask_name>"
20147 [(set (match_operand:VI1_AVX512F 0 "register_operand" "=x,x,v")
20148 (unspec:VI1_AVX512F
20149 [(match_operand:VI1_AVX512F 1 "register_operand" "%0,x,v")
20150 (match_operand:VI1_AVX512F 2 "nonimmediate_operand" "xBm,xm,vm")
20151 (match_operand:QI 3 "const_0_to_255_operand" "n,n,n")]
20152 UNSPEC_GF2P8AFFINE))]
20155 gf2p8affineqb\t{%3, %2, %0| %0, %2, %3}
20156 vgf2p8affineqb\t{%3, %2, %1, %0<mask_operand4>| %0<mask_operand4>, %1, %2, %3}
20157 vgf2p8affineqb\t{%3, %2, %1, %0<mask_operand4>| %0<mask_operand4>, %1, %2, %3}"
20158 [(set_attr "isa" "noavx,avx,avx512f")
20159 (set_attr "prefix_data16" "1,*,*")
20160 (set_attr "prefix_extra" "1")
20161 (set_attr "prefix" "orig,maybe_evex,evex")
20162 (set_attr "mode" "<sseinsnmode>")])
20164 (define_insn "vgf2p8mulb_<mode><mask_name>"
20165 [(set (match_operand:VI1_AVX512F 0 "register_operand" "=x,x,v")
20166 (unspec:VI1_AVX512F
20167 [(match_operand:VI1_AVX512F 1 "register_operand" "%0,x,v")
20168 (match_operand:VI1_AVX512F 2 "nonimmediate_operand" "xBm,xm,vm")]
20172 gf2p8mulb\t{%2, %0| %0, %2}
20173 vgf2p8mulb\t{%2, %1, %0<mask_operand3>| %0<mask_operand3>, %1, %2}
20174 vgf2p8mulb\t{%2, %1, %0<mask_operand3>| %0<mask_operand3>, %1, %2}"
20175 [(set_attr "isa" "noavx,avx,avx512f")
20176 (set_attr "prefix_data16" "1,*,*")
20177 (set_attr "prefix_extra" "1")
20178 (set_attr "prefix" "orig,maybe_evex,evex")
20179 (set_attr "mode" "<sseinsnmode>")])
20181 (define_insn "vpshrd_<mode><mask_name>"
20182 [(set (match_operand:VI248_AVX512VL 0 "register_operand" "=v")
20183 (unspec:VI248_AVX512VL
20184 [(match_operand:VI248_AVX512VL 1 "register_operand" "v")
20185 (match_operand:VI248_AVX512VL 2 "nonimmediate_operand" "vm")
20186 (match_operand:SI 3 "const_0_to_255_operand" "n")]
20188 "TARGET_AVX512VBMI2"
20189 "vpshrd<ssemodesuffix>\t{%3, %2, %1, %0<mask_operand4>|%0<mask_operand4>, %1, %2, %3 }"
20190 [(set_attr ("prefix") ("evex"))])
20192 (define_insn "vpshld_<mode><mask_name>"
20193 [(set (match_operand:VI248_AVX512VL 0 "register_operand" "=v")
20194 (unspec:VI248_AVX512VL
20195 [(match_operand:VI248_AVX512VL 1 "register_operand" "v")
20196 (match_operand:VI248_AVX512VL 2 "nonimmediate_operand" "vm")
20197 (match_operand:SI 3 "const_0_to_255_operand" "n")]
20199 "TARGET_AVX512VBMI2"
20200 "vpshld<ssemodesuffix>\t{%3, %2, %1, %0<mask_operand4>|%0<mask_operand4>, %1, %2, %3 }"
20201 [(set_attr ("prefix") ("evex"))])
20203 (define_insn "vpshrdv_<mode>"
20204 [(set (match_operand:VI248_AVX512VL 0 "register_operand" "=v")
20205 (unspec:VI248_AVX512VL
20206 [(match_operand:VI248_AVX512VL 1 "register_operand" "0")
20207 (match_operand:VI248_AVX512VL 2 "register_operand" "v")
20208 (match_operand:VI248_AVX512VL 3 "nonimmediate_operand" "vm")]
20210 "TARGET_AVX512VBMI2"
20211 "vpshrdv<ssemodesuffix>\t{%3, %2, %0|%0, %2, %3 }"
20212 [(set_attr ("prefix") ("evex"))
20213 (set_attr "mode" "<sseinsnmode>")])
20215 (define_insn "vpshrdv_<mode>_mask"
20216 [(set (match_operand:VI248_AVX512VL 0 "register_operand" "=v")
20217 (vec_merge:VI248_AVX512VL
20218 (unspec:VI248_AVX512VL
20219 [(match_operand:VI248_AVX512VL 1 "register_operand" "0")
20220 (match_operand:VI248_AVX512VL 2 "register_operand" "v")
20221 (match_operand:VI248_AVX512VL 3 "nonimmediate_operand" "vm")]
20224 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk")))]
20225 "TARGET_AVX512VBMI2"
20226 "vpshrdv<ssemodesuffix>\t{%3, %2, %0%{%4%}|%0%{%4%}, %2, %3 }"
20227 [(set_attr ("prefix") ("evex"))
20228 (set_attr "mode" "<sseinsnmode>")])
20230 (define_expand "vpshrdv_<mode>_maskz"
20231 [(match_operand:VI248_AVX512VL 0 "register_operand")
20232 (match_operand:VI248_AVX512VL 1 "register_operand")
20233 (match_operand:VI248_AVX512VL 2 "register_operand")
20234 (match_operand:VI248_AVX512VL 3 "nonimmediate_operand")
20235 (match_operand:<avx512fmaskmode> 4 "register_operand")]
20236 "TARGET_AVX512VBMI2"
20238 emit_insn (gen_vpshrdv_<mode>_maskz_1 (operands[0], operands[1],
20239 operands[2], operands[3],
20240 CONST0_RTX (<MODE>mode),
20245 (define_insn "vpshrdv_<mode>_maskz_1"
20246 [(set (match_operand:VI248_AVX512VL 0 "register_operand" "=v")
20247 (vec_merge:VI248_AVX512VL
20248 (unspec:VI248_AVX512VL
20249 [(match_operand:VI248_AVX512VL 1 "register_operand" "0")
20250 (match_operand:VI248_AVX512VL 2 "register_operand" "v")
20251 (match_operand:VI248_AVX512VL 3 "nonimmediate_operand" "vm")]
20253 (match_operand:VI248_AVX512VL 4 "const0_operand" "C")
20254 (match_operand:<avx512fmaskmode> 5 "register_operand" "Yk")))]
20255 "TARGET_AVX512VBMI2"
20256 "vpshrdv<ssemodesuffix>\t{%3, %2, %0%{%5%}%{z%}|%0%{%5%}%{z%}, %2, %3 }"
20257 [(set_attr ("prefix") ("evex"))
20258 (set_attr "mode" "<sseinsnmode>")])
20260 (define_insn "vpshldv_<mode>"
20261 [(set (match_operand:VI248_AVX512VL 0 "register_operand" "=v")
20262 (unspec:VI248_AVX512VL
20263 [(match_operand:VI248_AVX512VL 1 "register_operand" "0")
20264 (match_operand:VI248_AVX512VL 2 "register_operand" "v")
20265 (match_operand:VI248_AVX512VL 3 "nonimmediate_operand" "vm")]
20267 "TARGET_AVX512VBMI2"
20268 "vpshldv<ssemodesuffix>\t{%3, %2, %0|%0, %2, %3 }"
20269 [(set_attr ("prefix") ("evex"))
20270 (set_attr "mode" "<sseinsnmode>")])
20272 (define_insn "vpshldv_<mode>_mask"
20273 [(set (match_operand:VI248_AVX512VL 0 "register_operand" "=v")
20274 (vec_merge:VI248_AVX512VL
20275 (unspec:VI248_AVX512VL
20276 [(match_operand:VI248_AVX512VL 1 "register_operand" "0")
20277 (match_operand:VI248_AVX512VL 2 "register_operand" "v")
20278 (match_operand:VI248_AVX512VL 3 "nonimmediate_operand" "vm")]
20281 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk")))]
20282 "TARGET_AVX512VBMI2"
20283 "vpshldv<ssemodesuffix>\t{%3, %2, %0%{%4%}|%0%{%4%}, %2, %3 }"
20284 [(set_attr ("prefix") ("evex"))
20285 (set_attr "mode" "<sseinsnmode>")])
20287 (define_expand "vpshldv_<mode>_maskz"
20288 [(match_operand:VI248_AVX512VL 0 "register_operand")
20289 (match_operand:VI248_AVX512VL 1 "register_operand")
20290 (match_operand:VI248_AVX512VL 2 "register_operand")
20291 (match_operand:VI248_AVX512VL 3 "nonimmediate_operand")
20292 (match_operand:<avx512fmaskmode> 4 "register_operand")]
20293 "TARGET_AVX512VBMI2"
20295 emit_insn (gen_vpshldv_<mode>_maskz_1 (operands[0], operands[1],
20296 operands[2], operands[3],
20297 CONST0_RTX (<MODE>mode),
20302 (define_insn "vpshldv_<mode>_maskz_1"
20303 [(set (match_operand:VI248_AVX512VL 0 "register_operand" "=v")
20304 (vec_merge:VI248_AVX512VL
20305 (unspec:VI248_AVX512VL
20306 [(match_operand:VI248_AVX512VL 1 "register_operand" "0")
20307 (match_operand:VI248_AVX512VL 2 "register_operand" "v")
20308 (match_operand:VI248_AVX512VL 3 "nonimmediate_operand" "vm")]
20310 (match_operand:VI248_AVX512VL 4 "const0_operand" "C")
20311 (match_operand:<avx512fmaskmode> 5 "register_operand" "Yk")))]
20312 "TARGET_AVX512VBMI2"
20313 "vpshldv<ssemodesuffix>\t{%3, %2, %0%{%5%}%{z%}|%0%{%5%}%{z%}, %2, %3 }"
20314 [(set_attr ("prefix") ("evex"))
20315 (set_attr "mode" "<sseinsnmode>")])
20317 (define_insn "vpdpbusd_<mode>"
20318 [(set (match_operand:VI4_AVX512VL 0 "register_operand" "=v")
20319 (unspec:VI4_AVX512VL
20320 [(match_operand:VI4_AVX512VL 1 "register_operand" "0")
20321 (match_operand:VI4_AVX512VL 2 "register_operand" "v")
20322 (match_operand:VI4_AVX512VL 3 "nonimmediate_operand" "vm")]
20323 UNSPEC_VPMADDUBSWACCD))]
20324 "TARGET_AVX512VNNI"
20325 "vpdpbusd\t{%3, %2, %0|%0, %2, %3 }"
20326 [(set_attr ("prefix") ("evex"))])
20328 (define_insn "vpdpbusd_<mode>_mask"
20329 [(set (match_operand:VI4_AVX512VL 0 "register_operand" "=v")
20330 (vec_merge:VI4_AVX512VL
20331 (unspec:VI4_AVX512VL
20332 [(match_operand:VI4_AVX512VL 1 "register_operand" "0")
20333 (match_operand:VI4_AVX512VL 2 "register_operand" "v")
20334 (match_operand:VI4_AVX512VL 3 "nonimmediate_operand" "vm")]
20335 UNSPEC_VPMADDUBSWACCD)
20337 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk")))]
20338 "TARGET_AVX512VNNI"
20339 "vpdpbusd\t{%3, %2, %0%{%4%}|%0%{%4%}, %2, %3 }"
20340 [(set_attr ("prefix") ("evex"))])
20342 (define_expand "vpdpbusd_<mode>_maskz"
20343 [(match_operand:VI4_AVX512VL 0 "register_operand")
20344 (match_operand:VI4_AVX512VL 1 "register_operand")
20345 (match_operand:VI4_AVX512VL 2 "register_operand")
20346 (match_operand:VI4_AVX512VL 3 "nonimmediate_operand")
20347 (match_operand:<avx512fmaskmode> 4 "register_operand")]
20348 "TARGET_AVX512VNNI"
20350 emit_insn (gen_vpdpbusd_<mode>_maskz_1 (operands[0], operands[1],
20351 operands[2], operands[3],
20352 CONST0_RTX (<MODE>mode),
20357 (define_insn "vpdpbusd_<mode>_maskz_1"
20358 [(set (match_operand:VI4_AVX512VL 0 "register_operand" "=v")
20359 (vec_merge:VI4_AVX512VL
20360 (unspec:VI4_AVX512VL
20361 [(match_operand:VI4_AVX512VL 1 "register_operand" "0")
20362 (match_operand:VI4_AVX512VL 2 "register_operand" "v")
20363 (match_operand:VI4_AVX512VL 3 "nonimmediate_operand" "vm")
20364 ] UNSPEC_VPMADDUBSWACCD)
20365 (match_operand:VI4_AVX512VL 4 "const0_operand" "C")
20366 (match_operand:<avx512fmaskmode> 5 "register_operand" "Yk")))]
20367 "TARGET_AVX512VNNI"
20368 "vpdpbusd\t{%3, %2, %0%{%5%}%{z%}|%0%{%5%}%{z%}, %2, %3 }"
20369 [(set_attr ("prefix") ("evex"))])
20372 (define_insn "vpdpbusds_<mode>"
20373 [(set (match_operand:VI4_AVX512VL 0 "register_operand" "=v")
20374 (unspec:VI4_AVX512VL
20375 [(match_operand:VI4_AVX512VL 1 "register_operand" "0")
20376 (match_operand:VI4_AVX512VL 2 "register_operand" "v")
20377 (match_operand:VI4_AVX512VL 3 "nonimmediate_operand" "vm")]
20378 UNSPEC_VPMADDUBSWACCSSD))]
20379 "TARGET_AVX512VNNI"
20380 "vpdpbusds\t{%3, %2, %0|%0, %2, %3 }"
20381 [(set_attr ("prefix") ("evex"))])
20383 (define_insn "vpdpbusds_<mode>_mask"
20384 [(set (match_operand:VI4_AVX512VL 0 "register_operand" "=v")
20385 (vec_merge:VI4_AVX512VL
20386 (unspec:VI4_AVX512VL
20387 [(match_operand:VI4_AVX512VL 1 "register_operand" "0")
20388 (match_operand:VI4_AVX512VL 2 "register_operand" "v")
20389 (match_operand:VI4_AVX512VL 3 "nonimmediate_operand" "vm")]
20390 UNSPEC_VPMADDUBSWACCSSD)
20392 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk")))]
20393 "TARGET_AVX512VNNI"
20394 "vpdpbusds\t{%3, %2, %0%{%4%}|%0%{%4%}, %2, %3 }"
20395 [(set_attr ("prefix") ("evex"))])
20397 (define_expand "vpdpbusds_<mode>_maskz"
20398 [(match_operand:VI4_AVX512VL 0 "register_operand")
20399 (match_operand:VI4_AVX512VL 1 "register_operand")
20400 (match_operand:VI4_AVX512VL 2 "register_operand")
20401 (match_operand:VI4_AVX512VL 3 "nonimmediate_operand")
20402 (match_operand:<avx512fmaskmode> 4 "register_operand")]
20403 "TARGET_AVX512VNNI"
20405 emit_insn (gen_vpdpbusds_<mode>_maskz_1 (operands[0], operands[1],
20406 operands[2], operands[3],
20407 CONST0_RTX (<MODE>mode),
20412 (define_insn "vpdpbusds_<mode>_maskz_1"
20413 [(set (match_operand:VI4_AVX512VL 0 "register_operand" "=v")
20414 (vec_merge:VI4_AVX512VL
20415 (unspec:VI4_AVX512VL
20416 [(match_operand:VI4_AVX512VL 1 "register_operand" "0")
20417 (match_operand:VI4_AVX512VL 2 "register_operand" "v")
20418 (match_operand:VI4_AVX512VL 3 "nonimmediate_operand" "vm")]
20419 UNSPEC_VPMADDUBSWACCSSD)
20420 (match_operand:VI4_AVX512VL 4 "const0_operand" "C")
20421 (match_operand:<avx512fmaskmode> 5 "register_operand" "Yk")))]
20422 "TARGET_AVX512VNNI"
20423 "vpdpbusds\t{%3, %2, %0%{%5%}%{z%}|%0%{%5%}%{z%}, %2, %3 }"
20424 [(set_attr ("prefix") ("evex"))])
20427 (define_insn "vpdpwssd_<mode>"
20428 [(set (match_operand:VI4_AVX512VL 0 "register_operand" "=v")
20429 (unspec:VI4_AVX512VL
20430 [(match_operand:VI4_AVX512VL 1 "register_operand" "0")
20431 (match_operand:VI4_AVX512VL 2 "register_operand" "v")
20432 (match_operand:VI4_AVX512VL 3 "nonimmediate_operand" "vm")]
20433 UNSPEC_VPMADDWDACCD))]
20434 "TARGET_AVX512VNNI"
20435 "vpdpwssd\t{%3, %2, %0|%0, %2, %3 }"
20436 [(set_attr ("prefix") ("evex"))])
20438 (define_insn "vpdpwssd_<mode>_mask"
20439 [(set (match_operand:VI4_AVX512VL 0 "register_operand" "=v")
20440 (vec_merge:VI4_AVX512VL
20441 (unspec:VI4_AVX512VL
20442 [(match_operand:VI4_AVX512VL 1 "register_operand" "0")
20443 (match_operand:VI4_AVX512VL 2 "register_operand" "v")
20444 (match_operand:VI4_AVX512VL 3 "nonimmediate_operand" "vm")]
20445 UNSPEC_VPMADDWDACCD)
20447 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk")))]
20448 "TARGET_AVX512VNNI"
20449 "vpdpwssd\t{%3, %2, %0%{%4%}|%0%{%4%}, %2, %3 }"
20450 [(set_attr ("prefix") ("evex"))])
20452 (define_expand "vpdpwssd_<mode>_maskz"
20453 [(match_operand:VI4_AVX512VL 0 "register_operand")
20454 (match_operand:VI4_AVX512VL 1 "register_operand")
20455 (match_operand:VI4_AVX512VL 2 "register_operand")
20456 (match_operand:VI4_AVX512VL 3 "nonimmediate_operand")
20457 (match_operand:<avx512fmaskmode> 4 "register_operand")]
20458 "TARGET_AVX512VNNI"
20460 emit_insn (gen_vpdpwssd_<mode>_maskz_1 (operands[0], operands[1],
20461 operands[2], operands[3],
20462 CONST0_RTX (<MODE>mode),
20467 (define_insn "vpdpwssd_<mode>_maskz_1"
20468 [(set (match_operand:VI4_AVX512VL 0 "register_operand" "=v")
20469 (vec_merge:VI4_AVX512VL
20470 (unspec:VI4_AVX512VL
20471 [(match_operand:VI4_AVX512VL 1 "register_operand" "0")
20472 (match_operand:VI4_AVX512VL 2 "register_operand" "v")
20473 (match_operand:VI4_AVX512VL 3 "nonimmediate_operand" "vm")]
20474 UNSPEC_VPMADDWDACCD)
20475 (match_operand:VI4_AVX512VL 4 "const0_operand" "C")
20476 (match_operand:<avx512fmaskmode> 5 "register_operand" "Yk")))]
20477 "TARGET_AVX512VNNI"
20478 "vpdpwssd\t{%3, %2, %0%{%5%}%{z%}|%0%{%5%}%{z%}, %2, %3 }"
20479 [(set_attr ("prefix") ("evex"))])
20482 (define_insn "vpdpwssds_<mode>"
20483 [(set (match_operand:VI4_AVX512VL 0 "register_operand" "=v")
20484 (unspec:VI4_AVX512VL
20485 [(match_operand:VI4_AVX512VL 1 "register_operand" "0")
20486 (match_operand:VI4_AVX512VL 2 "register_operand" "v")
20487 (match_operand:VI4_AVX512VL 3 "nonimmediate_operand" "vm")]
20488 UNSPEC_VPMADDWDACCSSD))]
20489 "TARGET_AVX512VNNI"
20490 "vpdpwssds\t{%3, %2, %0|%0, %2, %3 }"
20491 [(set_attr ("prefix") ("evex"))])
20493 (define_insn "vpdpwssds_<mode>_mask"
20494 [(set (match_operand:VI4_AVX512VL 0 "register_operand" "=v")
20495 (vec_merge:VI4_AVX512VL
20496 (unspec:VI4_AVX512VL
20497 [(match_operand:VI4_AVX512VL 1 "register_operand" "0")
20498 (match_operand:VI4_AVX512VL 2 "register_operand" "v")
20499 (match_operand:VI4_AVX512VL 3 "nonimmediate_operand" "vm")]
20500 UNSPEC_VPMADDWDACCSSD)
20502 (match_operand:<avx512fmaskmode> 4 "register_operand" "Yk")))]
20503 "TARGET_AVX512VNNI"
20504 "vpdpwssds\t{%3, %2, %0%{%4%}|%0%{%4%}, %2, %3 }"
20505 [(set_attr ("prefix") ("evex"))])
20507 (define_expand "vpdpwssds_<mode>_maskz"
20508 [(match_operand:VI4_AVX512VL 0 "register_operand")
20509 (match_operand:VI4_AVX512VL 1 "register_operand")
20510 (match_operand:VI4_AVX512VL 2 "register_operand")
20511 (match_operand:VI4_AVX512VL 3 "nonimmediate_operand")
20512 (match_operand:<avx512fmaskmode> 4 "register_operand")]
20513 "TARGET_AVX512VNNI"
20515 emit_insn (gen_vpdpwssds_<mode>_maskz_1 (operands[0], operands[1],
20516 operands[2], operands[3],
20517 CONST0_RTX (<MODE>mode),
20522 (define_insn "vpdpwssds_<mode>_maskz_1"
20523 [(set (match_operand:VI4_AVX512VL 0 "register_operand" "=v")
20524 (vec_merge:VI4_AVX512VL
20525 (unspec:VI4_AVX512VL
20526 [(match_operand:VI4_AVX512VL 1 "register_operand" "0")
20527 (match_operand:VI4_AVX512VL 2 "register_operand" "v")
20528 (match_operand:VI4_AVX512VL 3 "nonimmediate_operand" "vm")]
20529 UNSPEC_VPMADDWDACCSSD)
20530 (match_operand:VI4_AVX512VL 4 "const0_operand" "C")
20531 (match_operand:<avx512fmaskmode> 5 "register_operand" "Yk")))]
20532 "TARGET_AVX512VNNI"
20533 "vpdpwssds\t{%3, %2, %0%{%5%}%{z%}|%0%{%5%}%{z%}, %2, %3 }"
20534 [(set_attr ("prefix") ("evex"))])
20536 (define_insn "vaesdec_<mode>"
20537 [(set (match_operand:VI1_AVX512VL_F 0 "register_operand" "=v")
20538 (unspec:VI1_AVX512VL_F
20539 [(match_operand:VI1_AVX512VL_F 1 "register_operand" "v")
20540 (match_operand:VI1_AVX512VL_F 2 "vector_operand" "v")]
20543 "vaesdec\t{%2, %1, %0|%0, %1, %2}"
20546 (define_insn "vaesdeclast_<mode>"
20547 [(set (match_operand:VI1_AVX512VL_F 0 "register_operand" "=v")
20548 (unspec:VI1_AVX512VL_F
20549 [(match_operand:VI1_AVX512VL_F 1 "register_operand" "v")
20550 (match_operand:VI1_AVX512VL_F 2 "vector_operand" "v")]
20551 UNSPEC_VAESDECLAST))]
20553 "vaesdeclast\t{%2, %1, %0|%0, %1, %2}"
20556 (define_insn "vaesenc_<mode>"
20557 [(set (match_operand:VI1_AVX512VL_F 0 "register_operand" "=v")
20558 (unspec:VI1_AVX512VL_F
20559 [(match_operand:VI1_AVX512VL_F 1 "register_operand" "v")
20560 (match_operand:VI1_AVX512VL_F 2 "vector_operand" "vm")]
20563 "vaesenc\t{%2, %1, %0|%0, %1, %2}"
20566 (define_insn "vaesenclast_<mode>"
20567 [(set (match_operand:VI1_AVX512VL_F 0 "register_operand" "=v")
20568 (unspec:VI1_AVX512VL_F
20569 [(match_operand:VI1_AVX512VL_F 1 "register_operand" "v")
20570 (match_operand:VI1_AVX512VL_F 2 "vector_operand" "vm")]
20571 UNSPEC_VAESENCLAST))]
20573 "vaesenclast\t{%2, %1, %0|%0, %1, %2}"
20576 (define_insn "vpclmulqdq_<mode>"
20577 [(set (match_operand:VI8_FVL 0 "register_operand" "=v")
20578 (unspec:VI8_FVL [(match_operand:VI8_FVL 1 "register_operand" "v")
20579 (match_operand:VI8_FVL 2 "vector_operand" "vm")
20580 (match_operand:SI 3 "const_0_to_255_operand" "n")]
20581 UNSPEC_VPCLMULQDQ))]
20582 "TARGET_VPCLMULQDQ"
20583 "vpclmulqdq\t{%3, %2, %1, %0|%0, %1, %2, %3}"
20584 [(set_attr "mode" "DI")])
20586 (define_insn "avx512vl_vpshufbitqmb<mode><mask_scalar_merge_name>"
20587 [(set (match_operand:<avx512fmaskmode> 0 "register_operand" "=Yk")
20588 (unspec:<avx512fmaskmode>
20589 [(match_operand:VI1_AVX512VLBW 1 "register_operand" "v")
20590 (match_operand:VI1_AVX512VLBW 2 "nonimmediate_operand" "vm")]
20591 UNSPEC_VPSHUFBIT))]
20592 "TARGET_AVX512BITALG"
20593 "vpshufbitqmb\t{%2, %1, %0<mask_scalar_merge_operand3>|%0<mask_scalar_merge_operand3>, %1, %2}"
20594 [(set_attr "prefix" "evex")
20595 (set_attr "mode" "<sseinsnmode>")])