1 ;; GCC machine description for SSE instructions
2 ;; Copyright (C) 2005-2013 Free Software Foundation, Inc.
4 ;; This file is part of GCC.
6 ;; GCC is free software; you can redistribute it and/or modify
7 ;; it under the terms of the GNU General Public License as published by
8 ;; the Free Software Foundation; either version 3, or (at your option)
11 ;; GCC is distributed in the hope that it will be useful,
12 ;; but WITHOUT ANY WARRANTY; without even the implied warranty of
13 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 ;; GNU General Public License for more details.
16 ;; You should have received a copy of the GNU General Public License
17 ;; along with GCC; see the file COPYING3. If not see
18 ;; <http://www.gnu.org/licenses/>.
20 (define_c_enum "unspec" [
55 UNSPEC_XOP_UNSIGNED_CMP
66 UNSPEC_AESKEYGENASSIST
88 (define_c_enum "unspecv" [
98 ;; All vector modes including V?TImode, used in move patterns.
99 (define_mode_iterator V16
100 [(V32QI "TARGET_AVX") V16QI
101 (V16HI "TARGET_AVX") V8HI
102 (V8SI "TARGET_AVX") V4SI
103 (V4DI "TARGET_AVX") V2DI
104 (V2TI "TARGET_AVX") V1TI
105 (V8SF "TARGET_AVX") V4SF
106 (V4DF "TARGET_AVX") V2DF])
109 (define_mode_iterator V
110 [(V32QI "TARGET_AVX") V16QI
111 (V16HI "TARGET_AVX") V8HI
112 (V8SI "TARGET_AVX") V4SI
113 (V4DI "TARGET_AVX") V2DI
114 (V8SF "TARGET_AVX") V4SF
115 (V4DF "TARGET_AVX") (V2DF "TARGET_SSE2")])
117 ;; All 128bit vector modes
118 (define_mode_iterator V_128
119 [V16QI V8HI V4SI V2DI V4SF (V2DF "TARGET_SSE2")])
121 ;; All 256bit vector modes
122 (define_mode_iterator V_256
123 [V32QI V16HI V8SI V4DI V8SF V4DF])
125 ;; All vector float modes
126 (define_mode_iterator VF
127 [(V8SF "TARGET_AVX") V4SF
128 (V4DF "TARGET_AVX") (V2DF "TARGET_SSE2")])
130 ;; All SFmode vector float modes
131 (define_mode_iterator VF1
132 [(V8SF "TARGET_AVX") V4SF])
134 ;; All DFmode vector float modes
135 (define_mode_iterator VF2
136 [(V4DF "TARGET_AVX") V2DF])
138 ;; All 128bit vector float modes
139 (define_mode_iterator VF_128
140 [V4SF (V2DF "TARGET_SSE2")])
142 ;; All 256bit vector float modes
143 (define_mode_iterator VF_256
146 ;; All vector integer modes
147 (define_mode_iterator VI
148 [(V32QI "TARGET_AVX") V16QI
149 (V16HI "TARGET_AVX") V8HI
150 (V8SI "TARGET_AVX") V4SI
151 (V4DI "TARGET_AVX") V2DI])
153 (define_mode_iterator VI_AVX2
154 [(V32QI "TARGET_AVX2") V16QI
155 (V16HI "TARGET_AVX2") V8HI
156 (V8SI "TARGET_AVX2") V4SI
157 (V4DI "TARGET_AVX2") V2DI])
159 ;; All QImode vector integer modes
160 (define_mode_iterator VI1
161 [(V32QI "TARGET_AVX") V16QI])
163 ;; All DImode vector integer modes
164 (define_mode_iterator VI8
165 [(V4DI "TARGET_AVX") V2DI])
167 (define_mode_iterator VI1_AVX2
168 [(V32QI "TARGET_AVX2") V16QI])
170 (define_mode_iterator VI2_AVX2
171 [(V16HI "TARGET_AVX2") V8HI])
173 (define_mode_iterator VI4_AVX2
174 [(V8SI "TARGET_AVX2") V4SI])
176 (define_mode_iterator VI8_AVX2
177 [(V4DI "TARGET_AVX2") V2DI])
179 ;; ??? We should probably use TImode instead.
180 (define_mode_iterator VIMAX_AVX2
181 [(V2TI "TARGET_AVX2") V1TI])
183 ;; ??? This should probably be dropped in favor of VIMAX_AVX2.
184 (define_mode_iterator SSESCALARMODE
185 [(V2TI "TARGET_AVX2") TI])
187 (define_mode_iterator VI12_AVX2
188 [(V32QI "TARGET_AVX2") V16QI
189 (V16HI "TARGET_AVX2") V8HI])
191 (define_mode_iterator VI24_AVX2
192 [(V16HI "TARGET_AVX2") V8HI
193 (V8SI "TARGET_AVX2") V4SI])
195 (define_mode_iterator VI124_AVX2
196 [(V32QI "TARGET_AVX2") V16QI
197 (V16HI "TARGET_AVX2") V8HI
198 (V8SI "TARGET_AVX2") V4SI])
200 (define_mode_iterator VI248_AVX2
201 [(V16HI "TARGET_AVX2") V8HI
202 (V8SI "TARGET_AVX2") V4SI
203 (V4DI "TARGET_AVX2") V2DI])
205 (define_mode_iterator VI48_AVX2
206 [(V8SI "TARGET_AVX2") V4SI
207 (V4DI "TARGET_AVX2") V2DI])
209 (define_mode_iterator V48_AVX2
212 (V4SI "TARGET_AVX2") (V2DI "TARGET_AVX2")
213 (V8SI "TARGET_AVX2") (V4DI "TARGET_AVX2")])
215 (define_mode_attr sse2_avx2
216 [(V16QI "sse2") (V32QI "avx2")
217 (V8HI "sse2") (V16HI "avx2")
218 (V4SI "sse2") (V8SI "avx2")
219 (V2DI "sse2") (V4DI "avx2")
220 (V1TI "sse2") (V2TI "avx2")])
222 (define_mode_attr ssse3_avx2
223 [(V16QI "ssse3") (V32QI "avx2")
224 (V4HI "ssse3") (V8HI "ssse3") (V16HI "avx2")
225 (V4SI "ssse3") (V8SI "avx2")
226 (V2DI "ssse3") (V4DI "avx2")
227 (TI "ssse3") (V2TI "avx2")])
229 (define_mode_attr sse4_1_avx2
230 [(V16QI "sse4_1") (V32QI "avx2")
231 (V8HI "sse4_1") (V16HI "avx2")
232 (V4SI "sse4_1") (V8SI "avx2")
233 (V2DI "sse4_1") (V4DI "avx2")])
235 (define_mode_attr avx_avx2
236 [(V4SF "avx") (V2DF "avx")
237 (V8SF "avx") (V4DF "avx")
238 (V4SI "avx2") (V2DI "avx2")
239 (V8SI "avx2") (V4DI "avx2")])
241 (define_mode_attr vec_avx2
242 [(V16QI "vec") (V32QI "avx2")
243 (V8HI "vec") (V16HI "avx2")
244 (V4SI "vec") (V8SI "avx2")
245 (V2DI "vec") (V4DI "avx2")])
247 (define_mode_attr ssedoublemode
248 [(V16HI "V16SI") (V8HI "V8SI") (V4HI "V4SI")
249 (V32QI "V32HI") (V16QI "V16HI")])
251 (define_mode_attr ssebytemode
252 [(V4DI "V32QI") (V2DI "V16QI")])
254 ;; All 128bit vector integer modes
255 (define_mode_iterator VI_128 [V16QI V8HI V4SI V2DI])
257 ;; All 256bit vector integer modes
258 (define_mode_iterator VI_256 [V32QI V16HI V8SI V4DI])
260 ;; Random 128bit vector integer mode combinations
261 (define_mode_iterator VI12_128 [V16QI V8HI])
262 (define_mode_iterator VI14_128 [V16QI V4SI])
263 (define_mode_iterator VI124_128 [V16QI V8HI V4SI])
264 (define_mode_iterator VI128_128 [V16QI V8HI V2DI])
265 (define_mode_iterator VI24_128 [V8HI V4SI])
266 (define_mode_iterator VI248_128 [V8HI V4SI V2DI])
267 (define_mode_iterator VI48_128 [V4SI V2DI])
269 ;; Random 256bit vector integer mode combinations
270 (define_mode_iterator VI124_256 [V32QI V16HI V8SI])
271 (define_mode_iterator VI48_256 [V8SI V4DI])
273 ;; Int-float size matches
274 (define_mode_iterator VI4F_128 [V4SI V4SF])
275 (define_mode_iterator VI8F_128 [V2DI V2DF])
276 (define_mode_iterator VI4F_256 [V8SI V8SF])
277 (define_mode_iterator VI8F_256 [V4DI V4DF])
279 ;; Mapping from float mode to required SSE level
280 (define_mode_attr sse
281 [(SF "sse") (DF "sse2")
282 (V4SF "sse") (V2DF "sse2")
283 (V8SF "avx") (V4DF "avx")])
285 (define_mode_attr sse2
286 [(V16QI "sse2") (V32QI "avx")
287 (V2DI "sse2") (V4DI "avx")])
289 (define_mode_attr sse3
290 [(V16QI "sse3") (V32QI "avx")])
292 (define_mode_attr sse4_1
293 [(V4SF "sse4_1") (V2DF "sse4_1")
294 (V8SF "avx") (V4DF "avx")])
296 (define_mode_attr avxsizesuffix
297 [(V32QI "256") (V16HI "256") (V8SI "256") (V4DI "256")
298 (V16QI "") (V8HI "") (V4SI "") (V2DI "")
299 (V8SF "256") (V4DF "256")
300 (V4SF "") (V2DF "")])
302 ;; SSE instruction mode
303 (define_mode_attr sseinsnmode
304 [(V32QI "OI") (V16HI "OI") (V8SI "OI") (V4DI "OI") (V2TI "OI")
305 (V16QI "TI") (V8HI "TI") (V4SI "TI") (V2DI "TI") (V1TI "TI")
306 (V8SF "V8SF") (V4DF "V4DF")
307 (V4SF "V4SF") (V2DF "V2DF")
310 ;; Mapping of vector float modes to an integer mode of the same size
311 (define_mode_attr sseintvecmode
312 [(V8SF "V8SI") (V4DF "V4DI")
313 (V4SF "V4SI") (V2DF "V2DI")
314 (V8SI "V8SI") (V4DI "V4DI")
315 (V4SI "V4SI") (V2DI "V2DI")
316 (V16HI "V16HI") (V8HI "V8HI")
317 (V32QI "V32QI") (V16QI "V16QI")])
319 (define_mode_attr sseintvecmodelower
320 [(V8SF "v8si") (V4DF "v4di")
321 (V4SF "v4si") (V2DF "v2di")
322 (V8SI "v8si") (V4DI "v4di")
323 (V4SI "v4si") (V2DI "v2di")
324 (V16HI "v16hi") (V8HI "v8hi")
325 (V32QI "v32qi") (V16QI "v16qi")])
327 ;; Mapping of vector modes to a vector mode of double size
328 (define_mode_attr ssedoublevecmode
329 [(V32QI "V64QI") (V16HI "V32HI") (V8SI "V16SI") (V4DI "V8DI")
330 (V16QI "V32QI") (V8HI "V16HI") (V4SI "V8SI") (V2DI "V4DI")
331 (V8SF "V16SF") (V4DF "V8DF")
332 (V4SF "V8SF") (V2DF "V4DF")])
334 ;; Mapping of vector modes to a vector mode of half size
335 (define_mode_attr ssehalfvecmode
336 [(V32QI "V16QI") (V16HI "V8HI") (V8SI "V4SI") (V4DI "V2DI")
337 (V16QI "V8QI") (V8HI "V4HI") (V4SI "V2SI")
338 (V8SF "V4SF") (V4DF "V2DF")
341 ;; Mapping of vector modes ti packed single mode of the same size
342 (define_mode_attr ssePSmode
343 [(V32QI "V8SF") (V16QI "V4SF")
344 (V16HI "V8SF") (V8HI "V4SF")
345 (V8SI "V8SF") (V4SI "V4SF")
346 (V4DI "V8SF") (V2DI "V4SF")
347 (V2TI "V8SF") (V1TI "V4SF")
348 (V8SF "V8SF") (V4SF "V4SF")
349 (V4DF "V8SF") (V2DF "V4SF")])
351 ;; Mapping of vector modes back to the scalar modes
352 (define_mode_attr ssescalarmode
353 [(V32QI "QI") (V16HI "HI") (V8SI "SI") (V4DI "DI")
354 (V16QI "QI") (V8HI "HI") (V4SI "SI") (V2DI "DI")
355 (V8SF "SF") (V4DF "DF")
356 (V4SF "SF") (V2DF "DF")])
358 ;; Number of scalar elements in each vector type
359 (define_mode_attr ssescalarnum
360 [(V32QI "32") (V16HI "16") (V8SI "8") (V4DI "4")
361 (V16QI "16") (V8HI "8") (V4SI "4") (V2DI "2")
362 (V8SF "8") (V4DF "4")
363 (V4SF "4") (V2DF "2")])
365 ;; Mask of scalar elements in each vector type
366 (define_mode_attr ssescalarnummask
367 [(V32QI "31") (V16HI "15") (V8SI "7") (V4DI "3")
368 (V16QI "15") (V8HI "7") (V4SI "3") (V2DI "1")
369 (V8SF "7") (V4DF "3")
370 (V4SF "3") (V2DF "1")])
372 ;; SSE prefix for integer vector modes
373 (define_mode_attr sseintprefix
374 [(V2DI "p") (V2DF "")
377 (V8SI "p") (V8SF "")])
379 ;; SSE scalar suffix for vector modes
380 (define_mode_attr ssescalarmodesuffix
382 (V8SF "ss") (V4DF "sd")
383 (V4SF "ss") (V2DF "sd")
384 (V8SI "ss") (V4DI "sd")
387 ;; Pack/unpack vector modes
388 (define_mode_attr sseunpackmode
389 [(V16QI "V8HI") (V8HI "V4SI") (V4SI "V2DI")
390 (V32QI "V16HI") (V16HI "V8SI") (V8SI "V4DI")])
392 (define_mode_attr ssepackmode
393 [(V8HI "V16QI") (V4SI "V8HI") (V2DI "V4SI")
394 (V16HI "V32QI") (V8SI "V16HI") (V4DI "V8SI")])
396 ;; Mapping of the max integer size for xop rotate immediate constraint
397 (define_mode_attr sserotatemax
398 [(V16QI "7") (V8HI "15") (V4SI "31") (V2DI "63")])
400 ;; Mapping of mode to cast intrinsic name
401 (define_mode_attr castmode [(V8SI "si") (V8SF "ps") (V4DF "pd")])
403 ;; Instruction suffix for sign and zero extensions.
404 (define_code_attr extsuffix [(sign_extend "sx") (zero_extend "zx")])
406 ;; i128 for integer vectors and TARGET_AVX2, f128 otherwise.
407 (define_mode_attr i128
408 [(V8SF "f128") (V4DF "f128") (V32QI "%~128") (V16HI "%~128")
409 (V8SI "%~128") (V4DI "%~128")])
412 (define_mode_iterator AVX256MODE2P [V8SI V8SF V4DF])
414 ;; Mapping of immediate bits for blend instructions
415 (define_mode_attr blendbits
416 [(V8SF "255") (V4SF "15") (V4DF "15") (V2DF "3")])
418 ;; Patterns whose name begins with "sse{,2,3}_" are invoked by intrinsics.
420 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
424 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
426 ;; All of these patterns are enabled for SSE1 as well as SSE2.
427 ;; This is essential for maintaining stable calling conventions.
429 (define_expand "mov<mode>"
430 [(set (match_operand:V16 0 "nonimmediate_operand")
431 (match_operand:V16 1 "nonimmediate_operand"))]
434 ix86_expand_vector_move (<MODE>mode, operands);
438 (define_insn "*mov<mode>_internal"
439 [(set (match_operand:V16 0 "nonimmediate_operand" "=x,x ,m")
440 (match_operand:V16 1 "nonimmediate_or_sse_const_operand" "C ,xm,x"))]
442 && (register_operand (operands[0], <MODE>mode)
443 || register_operand (operands[1], <MODE>mode))"
445 switch (which_alternative)
448 return standard_sse_constant_opcode (insn, operands[1]);
451 switch (get_attr_mode (insn))
456 && (misaligned_operand (operands[0], <MODE>mode)
457 || misaligned_operand (operands[1], <MODE>mode)))
458 return "vmovups\t{%1, %0|%0, %1}";
460 return "%vmovaps\t{%1, %0|%0, %1}";
465 && (misaligned_operand (operands[0], <MODE>mode)
466 || misaligned_operand (operands[1], <MODE>mode)))
467 return "vmovupd\t{%1, %0|%0, %1}";
469 return "%vmovapd\t{%1, %0|%0, %1}";
474 && (misaligned_operand (operands[0], <MODE>mode)
475 || misaligned_operand (operands[1], <MODE>mode)))
476 return "vmovdqu\t{%1, %0|%0, %1}";
478 return "%vmovdqa\t{%1, %0|%0, %1}";
487 [(set_attr "type" "sselog1,ssemov,ssemov")
488 (set_attr "prefix" "maybe_vex")
490 (cond [(match_test "TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL")
491 (const_string "<ssePSmode>")
492 (and (eq_attr "alternative" "2")
493 (match_test "TARGET_SSE_TYPELESS_STORES"))
494 (const_string "<ssePSmode>")
495 (match_test "TARGET_AVX")
496 (const_string "<sseinsnmode>")
497 (ior (not (match_test "TARGET_SSE2"))
498 (match_test "optimize_function_for_size_p (cfun)"))
499 (const_string "V4SF")
500 (and (eq_attr "alternative" "0")
501 (match_test "TARGET_SSE_LOAD0_BY_PXOR"))
504 (const_string "<sseinsnmode>")))])
506 (define_insn "sse2_movq128"
507 [(set (match_operand:V2DI 0 "register_operand" "=x")
510 (match_operand:V2DI 1 "nonimmediate_operand" "xm")
511 (parallel [(const_int 0)]))
514 "%vmovq\t{%1, %0|%0, %1}"
515 [(set_attr "type" "ssemov")
516 (set_attr "prefix" "maybe_vex")
517 (set_attr "mode" "TI")])
519 ;; Move a DI from a 32-bit register pair (e.g. %edx:%eax) to an xmm.
520 ;; We'd rather avoid this entirely; if the 32-bit reg pair was loaded
521 ;; from memory, we'd prefer to load the memory directly into the %xmm
522 ;; register. To facilitate this happy circumstance, this pattern won't
523 ;; split until after register allocation. If the 64-bit value didn't
524 ;; come from memory, this is the best we can do. This is much better
525 ;; than storing %edx:%eax into a stack temporary and loading an %xmm
528 (define_insn_and_split "movdi_to_sse"
530 [(set (match_operand:V4SI 0 "register_operand" "=?x,x")
531 (subreg:V4SI (match_operand:DI 1 "nonimmediate_operand" "r,m") 0))
532 (clobber (match_scratch:V4SI 2 "=&x,X"))])]
533 "!TARGET_64BIT && TARGET_SSE2 && TARGET_INTER_UNIT_MOVES_TO_VEC"
535 "&& reload_completed"
538 if (register_operand (operands[1], DImode))
540 /* The DImode arrived in a pair of integral registers (e.g. %edx:%eax).
541 Assemble the 64-bit DImode value in an xmm register. */
542 emit_insn (gen_sse2_loadld (operands[0], CONST0_RTX (V4SImode),
543 gen_rtx_SUBREG (SImode, operands[1], 0)));
544 emit_insn (gen_sse2_loadld (operands[2], CONST0_RTX (V4SImode),
545 gen_rtx_SUBREG (SImode, operands[1], 4)));
546 emit_insn (gen_vec_interleave_lowv4si (operands[0], operands[0],
549 else if (memory_operand (operands[1], DImode))
550 emit_insn (gen_vec_concatv2di (gen_lowpart (V2DImode, operands[0]),
551 operands[1], const0_rtx));
557 [(set (match_operand:V4SF 0 "register_operand")
558 (match_operand:V4SF 1 "zero_extended_scalar_load_operand"))]
559 "TARGET_SSE && reload_completed"
562 (vec_duplicate:V4SF (match_dup 1))
566 operands[1] = simplify_gen_subreg (SFmode, operands[1], V4SFmode, 0);
567 operands[2] = CONST0_RTX (V4SFmode);
571 [(set (match_operand:V2DF 0 "register_operand")
572 (match_operand:V2DF 1 "zero_extended_scalar_load_operand"))]
573 "TARGET_SSE2 && reload_completed"
574 [(set (match_dup 0) (vec_concat:V2DF (match_dup 1) (match_dup 2)))]
576 operands[1] = simplify_gen_subreg (DFmode, operands[1], V2DFmode, 0);
577 operands[2] = CONST0_RTX (DFmode);
580 (define_expand "push<mode>1"
581 [(match_operand:V16 0 "register_operand")]
584 ix86_expand_push (<MODE>mode, operands[0]);
588 (define_expand "movmisalign<mode>"
589 [(set (match_operand:V16 0 "nonimmediate_operand")
590 (match_operand:V16 1 "nonimmediate_operand"))]
593 ix86_expand_vector_move_misalign (<MODE>mode, operands);
597 (define_insn "<sse>_loadu<ssemodesuffix><avxsizesuffix>"
598 [(set (match_operand:VF 0 "register_operand" "=x")
600 [(match_operand:VF 1 "memory_operand" "m")]
604 switch (get_attr_mode (insn))
608 return "%vmovups\t{%1, %0|%0, %1}";
610 return "%vmovu<ssemodesuffix>\t{%1, %0|%0, %1}";
613 [(set_attr "type" "ssemov")
614 (set_attr "movu" "1")
615 (set_attr "prefix" "maybe_vex")
617 (cond [(match_test "TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL")
618 (const_string "<ssePSmode>")
619 (match_test "TARGET_AVX")
620 (const_string "<MODE>")
621 (match_test "optimize_function_for_size_p (cfun)")
622 (const_string "V4SF")
624 (const_string "<MODE>")))])
626 (define_insn "<sse>_storeu<ssemodesuffix><avxsizesuffix>"
627 [(set (match_operand:VF 0 "memory_operand" "=m")
629 [(match_operand:VF 1 "register_operand" "x")]
633 switch (get_attr_mode (insn))
637 return "%vmovups\t{%1, %0|%0, %1}";
639 return "%vmovu<ssemodesuffix>\t{%1, %0|%0, %1}";
642 [(set_attr "type" "ssemov")
643 (set_attr "movu" "1")
644 (set_attr "prefix" "maybe_vex")
646 (cond [(ior (match_test "TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL")
647 (match_test "TARGET_SSE_TYPELESS_STORES"))
648 (const_string "<ssePSmode>")
649 (match_test "TARGET_AVX")
650 (const_string "<MODE>")
651 (match_test "optimize_function_for_size_p (cfun)")
652 (const_string "V4SF")
654 (const_string "<MODE>")))])
656 (define_insn "<sse2>_loaddqu<avxsizesuffix>"
657 [(set (match_operand:VI1 0 "register_operand" "=x")
658 (unspec:VI1 [(match_operand:VI1 1 "memory_operand" "m")]
662 switch (get_attr_mode (insn))
666 return "%vmovups\t{%1, %0|%0, %1}";
668 return "%vmovdqu\t{%1, %0|%0, %1}";
671 [(set_attr "type" "ssemov")
672 (set_attr "movu" "1")
673 (set (attr "prefix_data16")
675 (match_test "TARGET_AVX")
678 (set_attr "prefix" "maybe_vex")
680 (cond [(match_test "TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL")
681 (const_string "<ssePSmode>")
682 (match_test "TARGET_AVX")
683 (const_string "<sseinsnmode>")
684 (match_test "optimize_function_for_size_p (cfun)")
685 (const_string "V4SF")
687 (const_string "<sseinsnmode>")))])
689 (define_insn "<sse2>_storedqu<avxsizesuffix>"
690 [(set (match_operand:VI1 0 "memory_operand" "=m")
691 (unspec:VI1 [(match_operand:VI1 1 "register_operand" "x")]
695 switch (get_attr_mode (insn))
699 return "%vmovups\t{%1, %0|%0, %1}";
701 return "%vmovdqu\t{%1, %0|%0, %1}";
704 [(set_attr "type" "ssemov")
705 (set_attr "movu" "1")
706 (set (attr "prefix_data16")
708 (match_test "TARGET_AVX")
711 (set_attr "prefix" "maybe_vex")
713 (cond [(ior (match_test "TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL")
714 (match_test "TARGET_SSE_TYPELESS_STORES"))
715 (const_string "<ssePSmode>")
716 (match_test "TARGET_AVX")
717 (const_string "<sseinsnmode>")
718 (match_test "optimize_function_for_size_p (cfun)")
719 (const_string "V4SF")
721 (const_string "<sseinsnmode>")))])
723 (define_insn "<sse3>_lddqu<avxsizesuffix>"
724 [(set (match_operand:VI1 0 "register_operand" "=x")
725 (unspec:VI1 [(match_operand:VI1 1 "memory_operand" "m")]
728 "%vlddqu\t{%1, %0|%0, %1}"
729 [(set_attr "type" "ssemov")
730 (set_attr "movu" "1")
731 (set (attr "prefix_data16")
733 (match_test "TARGET_AVX")
736 (set (attr "prefix_rep")
738 (match_test "TARGET_AVX")
741 (set_attr "prefix" "maybe_vex")
742 (set_attr "mode" "<sseinsnmode>")])
744 (define_insn "sse2_movnti<mode>"
745 [(set (match_operand:SWI48 0 "memory_operand" "=m")
746 (unspec:SWI48 [(match_operand:SWI48 1 "register_operand" "r")]
749 "movnti\t{%1, %0|%0, %1}"
750 [(set_attr "type" "ssemov")
751 (set_attr "prefix_data16" "0")
752 (set_attr "mode" "<MODE>")])
754 (define_insn "<sse>_movnt<mode>"
755 [(set (match_operand:VF 0 "memory_operand" "=m")
756 (unspec:VF [(match_operand:VF 1 "register_operand" "x")]
759 "%vmovnt<ssemodesuffix>\t{%1, %0|%0, %1}"
760 [(set_attr "type" "ssemov")
761 (set_attr "prefix" "maybe_vex")
762 (set_attr "mode" "<MODE>")])
764 (define_insn "<sse2>_movnt<mode>"
765 [(set (match_operand:VI8 0 "memory_operand" "=m")
766 (unspec:VI8 [(match_operand:VI8 1 "register_operand" "x")]
769 "%vmovntdq\t{%1, %0|%0, %1}"
770 [(set_attr "type" "ssecvt")
771 (set (attr "prefix_data16")
773 (match_test "TARGET_AVX")
776 (set_attr "prefix" "maybe_vex")
777 (set_attr "mode" "<sseinsnmode>")])
779 ; Expand patterns for non-temporal stores. At the moment, only those
780 ; that directly map to insns are defined; it would be possible to
781 ; define patterns for other modes that would expand to several insns.
783 ;; Modes handled by storent patterns.
784 (define_mode_iterator STORENT_MODE
785 [(DI "TARGET_SSE2 && TARGET_64BIT") (SI "TARGET_SSE2")
786 (SF "TARGET_SSE4A") (DF "TARGET_SSE4A")
787 (V4DI "TARGET_AVX") (V2DI "TARGET_SSE2")
788 (V8SF "TARGET_AVX") V4SF
789 (V4DF "TARGET_AVX") (V2DF "TARGET_SSE2")])
791 (define_expand "storent<mode>"
792 [(set (match_operand:STORENT_MODE 0 "memory_operand")
794 [(match_operand:STORENT_MODE 1 "register_operand")]
798 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
800 ;; Parallel floating point arithmetic
802 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
804 (define_expand "<code><mode>2"
805 [(set (match_operand:VF 0 "register_operand")
807 (match_operand:VF 1 "register_operand")))]
809 "ix86_expand_fp_absneg_operator (<CODE>, <MODE>mode, operands); DONE;")
811 (define_insn_and_split "*absneg<mode>2"
812 [(set (match_operand:VF 0 "register_operand" "=x,x,x,x")
813 (match_operator:VF 3 "absneg_operator"
814 [(match_operand:VF 1 "nonimmediate_operand" "0, xm,x, m")]))
815 (use (match_operand:VF 2 "nonimmediate_operand" "xm,0, xm,x"))]
818 "&& reload_completed"
821 enum rtx_code absneg_op;
827 if (MEM_P (operands[1]))
828 op1 = operands[2], op2 = operands[1];
830 op1 = operands[1], op2 = operands[2];
835 if (rtx_equal_p (operands[0], operands[1]))
841 absneg_op = GET_CODE (operands[3]) == NEG ? XOR : AND;
842 t = gen_rtx_fmt_ee (absneg_op, <MODE>mode, op1, op2);
843 t = gen_rtx_SET (VOIDmode, operands[0], t);
847 [(set_attr "isa" "noavx,noavx,avx,avx")])
849 (define_expand "<plusminus_insn><mode>3"
850 [(set (match_operand:VF 0 "register_operand")
852 (match_operand:VF 1 "nonimmediate_operand")
853 (match_operand:VF 2 "nonimmediate_operand")))]
855 "ix86_fixup_binary_operands_no_copy (<CODE>, <MODE>mode, operands);")
857 (define_insn "*<plusminus_insn><mode>3"
858 [(set (match_operand:VF 0 "register_operand" "=x,x")
860 (match_operand:VF 1 "nonimmediate_operand" "<comm>0,x")
861 (match_operand:VF 2 "nonimmediate_operand" "xm,xm")))]
862 "TARGET_SSE && ix86_binary_operator_ok (<CODE>, <MODE>mode, operands)"
864 <plusminus_mnemonic><ssemodesuffix>\t{%2, %0|%0, %2}
865 v<plusminus_mnemonic><ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
866 [(set_attr "isa" "noavx,avx")
867 (set_attr "type" "sseadd")
868 (set_attr "prefix" "orig,vex")
869 (set_attr "mode" "<MODE>")])
871 (define_insn "<sse>_vm<plusminus_insn><mode>3"
872 [(set (match_operand:VF_128 0 "register_operand" "=x,x")
875 (match_operand:VF_128 1 "register_operand" "0,x")
876 (match_operand:VF_128 2 "nonimmediate_operand" "xm,xm"))
881 <plusminus_mnemonic><ssescalarmodesuffix>\t{%2, %0|%0, %2}
882 v<plusminus_mnemonic><ssescalarmodesuffix>\t{%2, %1, %0|%0, %1, %2}"
883 [(set_attr "isa" "noavx,avx")
884 (set_attr "type" "sseadd")
885 (set_attr "prefix" "orig,vex")
886 (set_attr "mode" "<ssescalarmode>")])
888 (define_expand "mul<mode>3"
889 [(set (match_operand:VF 0 "register_operand")
891 (match_operand:VF 1 "nonimmediate_operand")
892 (match_operand:VF 2 "nonimmediate_operand")))]
894 "ix86_fixup_binary_operands_no_copy (MULT, <MODE>mode, operands);")
896 (define_insn "*mul<mode>3"
897 [(set (match_operand:VF 0 "register_operand" "=x,x")
899 (match_operand:VF 1 "nonimmediate_operand" "%0,x")
900 (match_operand:VF 2 "nonimmediate_operand" "xm,xm")))]
901 "TARGET_SSE && ix86_binary_operator_ok (MULT, <MODE>mode, operands)"
903 mul<ssemodesuffix>\t{%2, %0|%0, %2}
904 vmul<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
905 [(set_attr "isa" "noavx,avx")
906 (set_attr "type" "ssemul")
907 (set_attr "prefix" "orig,vex")
908 (set_attr "btver2_decode" "direct,double")
909 (set_attr "mode" "<MODE>")])
911 (define_insn "<sse>_vmmul<mode>3"
912 [(set (match_operand:VF_128 0 "register_operand" "=x,x")
915 (match_operand:VF_128 1 "register_operand" "0,x")
916 (match_operand:VF_128 2 "nonimmediate_operand" "xm,xm"))
921 mul<ssescalarmodesuffix>\t{%2, %0|%0, %2}
922 vmul<ssescalarmodesuffix>\t{%2, %1, %0|%0, %1, %2}"
923 [(set_attr "isa" "noavx,avx")
924 (set_attr "type" "ssemul")
925 (set_attr "prefix" "orig,vex")
926 (set_attr "mode" "<ssescalarmode>")])
928 (define_expand "div<mode>3"
929 [(set (match_operand:VF2 0 "register_operand")
930 (div:VF2 (match_operand:VF2 1 "register_operand")
931 (match_operand:VF2 2 "nonimmediate_operand")))]
933 "ix86_fixup_binary_operands_no_copy (DIV, <MODE>mode, operands);")
935 (define_expand "div<mode>3"
936 [(set (match_operand:VF1 0 "register_operand")
937 (div:VF1 (match_operand:VF1 1 "register_operand")
938 (match_operand:VF1 2 "nonimmediate_operand")))]
941 ix86_fixup_binary_operands_no_copy (DIV, <MODE>mode, operands);
944 && TARGET_RECIP_VEC_DIV
945 && !optimize_insn_for_size_p ()
946 && flag_finite_math_only && !flag_trapping_math
947 && flag_unsafe_math_optimizations)
949 ix86_emit_swdivsf (operands[0], operands[1], operands[2], <MODE>mode);
954 (define_insn "<sse>_div<mode>3"
955 [(set (match_operand:VF 0 "register_operand" "=x,x")
957 (match_operand:VF 1 "register_operand" "0,x")
958 (match_operand:VF 2 "nonimmediate_operand" "xm,xm")))]
961 div<ssemodesuffix>\t{%2, %0|%0, %2}
962 vdiv<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
963 [(set_attr "isa" "noavx,avx")
964 (set_attr "type" "ssediv")
965 (set_attr "prefix" "orig,vex")
966 (set_attr "mode" "<MODE>")])
968 (define_insn "<sse>_vmdiv<mode>3"
969 [(set (match_operand:VF_128 0 "register_operand" "=x,x")
972 (match_operand:VF_128 1 "register_operand" "0,x")
973 (match_operand:VF_128 2 "nonimmediate_operand" "xm,xm"))
978 div<ssescalarmodesuffix>\t{%2, %0|%0, %2}
979 vdiv<ssescalarmodesuffix>\t{%2, %1, %0|%0, %1, %2}"
980 [(set_attr "isa" "noavx,avx")
981 (set_attr "type" "ssediv")
982 (set_attr "prefix" "orig,vex")
983 (set_attr "btver2_decode" "direct,double")
984 (set_attr "mode" "<ssescalarmode>")])
986 (define_insn "<sse>_rcp<mode>2"
987 [(set (match_operand:VF1 0 "register_operand" "=x")
989 [(match_operand:VF1 1 "nonimmediate_operand" "xm")] UNSPEC_RCP))]
991 "%vrcpps\t{%1, %0|%0, %1}"
992 [(set_attr "type" "sse")
993 (set_attr "atom_sse_attr" "rcp")
994 (set_attr "btver2_sse_attr" "rcp")
995 (set_attr "prefix" "maybe_vex")
996 (set_attr "mode" "<MODE>")])
998 (define_insn "sse_vmrcpv4sf2"
999 [(set (match_operand:V4SF 0 "register_operand" "=x,x")
1001 (unspec:V4SF [(match_operand:V4SF 1 "nonimmediate_operand" "xm,xm")]
1003 (match_operand:V4SF 2 "register_operand" "0,x")
1007 rcpss\t{%1, %0|%0, %1}
1008 vrcpss\t{%1, %2, %0|%0, %2, %1}"
1009 [(set_attr "isa" "noavx,avx")
1010 (set_attr "type" "sse")
1011 (set_attr "atom_sse_attr" "rcp")
1012 (set_attr "btver2_sse_attr" "rcp")
1013 (set_attr "prefix" "orig,vex")
1014 (set_attr "mode" "SF")])
1016 (define_expand "sqrt<mode>2"
1017 [(set (match_operand:VF2 0 "register_operand")
1018 (sqrt:VF2 (match_operand:VF2 1 "nonimmediate_operand")))]
1021 (define_expand "sqrt<mode>2"
1022 [(set (match_operand:VF1 0 "register_operand")
1023 (sqrt:VF1 (match_operand:VF1 1 "nonimmediate_operand")))]
1027 && TARGET_RECIP_VEC_SQRT
1028 && !optimize_insn_for_size_p ()
1029 && flag_finite_math_only && !flag_trapping_math
1030 && flag_unsafe_math_optimizations)
1032 ix86_emit_swsqrtsf (operands[0], operands[1], <MODE>mode, false);
1037 (define_insn "<sse>_sqrt<mode>2"
1038 [(set (match_operand:VF 0 "register_operand" "=x")
1039 (sqrt:VF (match_operand:VF 1 "nonimmediate_operand" "xm")))]
1041 "%vsqrt<ssemodesuffix>\t{%1, %0|%0, %1}"
1042 [(set_attr "type" "sse")
1043 (set_attr "atom_sse_attr" "sqrt")
1044 (set_attr "btver2_sse_attr" "sqrt")
1045 (set_attr "prefix" "maybe_vex")
1046 (set_attr "mode" "<MODE>")])
1048 (define_insn "<sse>_vmsqrt<mode>2"
1049 [(set (match_operand:VF_128 0 "register_operand" "=x,x")
1052 (match_operand:VF_128 1 "nonimmediate_operand" "xm,xm"))
1053 (match_operand:VF_128 2 "register_operand" "0,x")
1057 sqrt<ssescalarmodesuffix>\t{%1, %0|%0, %1}
1058 vsqrt<ssescalarmodesuffix>\t{%1, %2, %0|%0, %2, %1}"
1059 [(set_attr "isa" "noavx,avx")
1060 (set_attr "type" "sse")
1061 (set_attr "atom_sse_attr" "sqrt")
1062 (set_attr "btver2_sse_attr" "sqrt")
1063 (set_attr "prefix" "orig,vex")
1064 (set_attr "mode" "<ssescalarmode>")])
1066 (define_expand "rsqrt<mode>2"
1067 [(set (match_operand:VF1 0 "register_operand")
1069 [(match_operand:VF1 1 "nonimmediate_operand")] UNSPEC_RSQRT))]
1072 ix86_emit_swsqrtsf (operands[0], operands[1], <MODE>mode, true);
1076 (define_insn "<sse>_rsqrt<mode>2"
1077 [(set (match_operand:VF1 0 "register_operand" "=x")
1079 [(match_operand:VF1 1 "nonimmediate_operand" "xm")] UNSPEC_RSQRT))]
1081 "%vrsqrtps\t{%1, %0|%0, %1}"
1082 [(set_attr "type" "sse")
1083 (set_attr "prefix" "maybe_vex")
1084 (set_attr "mode" "<MODE>")])
1086 (define_insn "sse_vmrsqrtv4sf2"
1087 [(set (match_operand:V4SF 0 "register_operand" "=x,x")
1089 (unspec:V4SF [(match_operand:V4SF 1 "nonimmediate_operand" "xm,xm")]
1091 (match_operand:V4SF 2 "register_operand" "0,x")
1095 rsqrtss\t{%1, %0|%0, %1}
1096 vrsqrtss\t{%1, %2, %0|%0, %2, %1}"
1097 [(set_attr "isa" "noavx,avx")
1098 (set_attr "type" "sse")
1099 (set_attr "prefix" "orig,vex")
1100 (set_attr "mode" "SF")])
1102 ;; ??? For !flag_finite_math_only, the representation with SMIN/SMAX
1103 ;; isn't really correct, as those rtl operators aren't defined when
1104 ;; applied to NaNs. Hopefully the optimizers won't get too smart on us.
1106 (define_expand "<code><mode>3"
1107 [(set (match_operand:VF 0 "register_operand")
1109 (match_operand:VF 1 "nonimmediate_operand")
1110 (match_operand:VF 2 "nonimmediate_operand")))]
1113 if (!flag_finite_math_only)
1114 operands[1] = force_reg (<MODE>mode, operands[1]);
1115 ix86_fixup_binary_operands_no_copy (<CODE>, <MODE>mode, operands);
1118 (define_insn "*<code><mode>3_finite"
1119 [(set (match_operand:VF 0 "register_operand" "=x,x")
1121 (match_operand:VF 1 "nonimmediate_operand" "%0,x")
1122 (match_operand:VF 2 "nonimmediate_operand" "xm,xm")))]
1123 "TARGET_SSE && flag_finite_math_only
1124 && ix86_binary_operator_ok (<CODE>, <MODE>mode, operands)"
1126 <maxmin_float><ssemodesuffix>\t{%2, %0|%0, %2}
1127 v<maxmin_float><ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
1128 [(set_attr "isa" "noavx,avx")
1129 (set_attr "type" "sseadd")
1130 (set_attr "btver2_sse_attr" "maxmin")
1131 (set_attr "prefix" "orig,vex")
1132 (set_attr "mode" "<MODE>")])
1134 (define_insn "*<code><mode>3"
1135 [(set (match_operand:VF 0 "register_operand" "=x,x")
1137 (match_operand:VF 1 "register_operand" "0,x")
1138 (match_operand:VF 2 "nonimmediate_operand" "xm,xm")))]
1139 "TARGET_SSE && !flag_finite_math_only"
1141 <maxmin_float><ssemodesuffix>\t{%2, %0|%0, %2}
1142 v<maxmin_float><ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
1143 [(set_attr "isa" "noavx,avx")
1144 (set_attr "type" "sseadd")
1145 (set_attr "btver2_sse_attr" "maxmin")
1146 (set_attr "prefix" "orig,vex")
1147 (set_attr "mode" "<MODE>")])
1149 (define_insn "<sse>_vm<code><mode>3"
1150 [(set (match_operand:VF_128 0 "register_operand" "=x,x")
1153 (match_operand:VF_128 1 "register_operand" "0,x")
1154 (match_operand:VF_128 2 "nonimmediate_operand" "xm,xm"))
1159 <maxmin_float><ssescalarmodesuffix>\t{%2, %0|%0, %2}
1160 v<maxmin_float><ssescalarmodesuffix>\t{%2, %1, %0|%0, %1, %2}"
1161 [(set_attr "isa" "noavx,avx")
1162 (set_attr "type" "sse")
1163 (set_attr "btver2_sse_attr" "maxmin")
1164 (set_attr "prefix" "orig,vex")
1165 (set_attr "mode" "<ssescalarmode>")])
1167 ;; These versions of the min/max patterns implement exactly the operations
1168 ;; min = (op1 < op2 ? op1 : op2)
1169 ;; max = (!(op1 < op2) ? op1 : op2)
1170 ;; Their operands are not commutative, and thus they may be used in the
1171 ;; presence of -0.0 and NaN.
1173 (define_insn "*ieee_smin<mode>3"
1174 [(set (match_operand:VF 0 "register_operand" "=x,x")
1176 [(match_operand:VF 1 "register_operand" "0,x")
1177 (match_operand:VF 2 "nonimmediate_operand" "xm,xm")]
1181 min<ssemodesuffix>\t{%2, %0|%0, %2}
1182 vmin<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
1183 [(set_attr "isa" "noavx,avx")
1184 (set_attr "type" "sseadd")
1185 (set_attr "prefix" "orig,vex")
1186 (set_attr "mode" "<MODE>")])
1188 (define_insn "*ieee_smax<mode>3"
1189 [(set (match_operand:VF 0 "register_operand" "=x,x")
1191 [(match_operand:VF 1 "register_operand" "0,x")
1192 (match_operand:VF 2 "nonimmediate_operand" "xm,xm")]
1196 max<ssemodesuffix>\t{%2, %0|%0, %2}
1197 vmax<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
1198 [(set_attr "isa" "noavx,avx")
1199 (set_attr "type" "sseadd")
1200 (set_attr "prefix" "orig,vex")
1201 (set_attr "mode" "<MODE>")])
1203 (define_insn "avx_addsubv4df3"
1204 [(set (match_operand:V4DF 0 "register_operand" "=x")
1207 (match_operand:V4DF 1 "register_operand" "x")
1208 (match_operand:V4DF 2 "nonimmediate_operand" "xm"))
1209 (minus:V4DF (match_dup 1) (match_dup 2))
1212 "vaddsubpd\t{%2, %1, %0|%0, %1, %2}"
1213 [(set_attr "type" "sseadd")
1214 (set_attr "prefix" "vex")
1215 (set_attr "mode" "V4DF")])
1217 (define_insn "sse3_addsubv2df3"
1218 [(set (match_operand:V2DF 0 "register_operand" "=x,x")
1221 (match_operand:V2DF 1 "register_operand" "0,x")
1222 (match_operand:V2DF 2 "nonimmediate_operand" "xm,xm"))
1223 (minus:V2DF (match_dup 1) (match_dup 2))
1227 addsubpd\t{%2, %0|%0, %2}
1228 vaddsubpd\t{%2, %1, %0|%0, %1, %2}"
1229 [(set_attr "isa" "noavx,avx")
1230 (set_attr "type" "sseadd")
1231 (set_attr "atom_unit" "complex")
1232 (set_attr "prefix" "orig,vex")
1233 (set_attr "mode" "V2DF")])
1235 (define_insn "avx_addsubv8sf3"
1236 [(set (match_operand:V8SF 0 "register_operand" "=x")
1239 (match_operand:V8SF 1 "register_operand" "x")
1240 (match_operand:V8SF 2 "nonimmediate_operand" "xm"))
1241 (minus:V8SF (match_dup 1) (match_dup 2))
1244 "vaddsubps\t{%2, %1, %0|%0, %1, %2}"
1245 [(set_attr "type" "sseadd")
1246 (set_attr "prefix" "vex")
1247 (set_attr "mode" "V8SF")])
1249 (define_insn "sse3_addsubv4sf3"
1250 [(set (match_operand:V4SF 0 "register_operand" "=x,x")
1253 (match_operand:V4SF 1 "register_operand" "0,x")
1254 (match_operand:V4SF 2 "nonimmediate_operand" "xm,xm"))
1255 (minus:V4SF (match_dup 1) (match_dup 2))
1259 addsubps\t{%2, %0|%0, %2}
1260 vaddsubps\t{%2, %1, %0|%0, %1, %2}"
1261 [(set_attr "isa" "noavx,avx")
1262 (set_attr "type" "sseadd")
1263 (set_attr "prefix" "orig,vex")
1264 (set_attr "prefix_rep" "1,*")
1265 (set_attr "mode" "V4SF")])
1267 (define_insn "avx_h<plusminus_insn>v4df3"
1268 [(set (match_operand:V4DF 0 "register_operand" "=x")
1273 (match_operand:V4DF 1 "register_operand" "x")
1274 (parallel [(const_int 0)]))
1275 (vec_select:DF (match_dup 1) (parallel [(const_int 1)])))
1278 (match_operand:V4DF 2 "nonimmediate_operand" "xm")
1279 (parallel [(const_int 0)]))
1280 (vec_select:DF (match_dup 2) (parallel [(const_int 1)]))))
1283 (vec_select:DF (match_dup 1) (parallel [(const_int 2)]))
1284 (vec_select:DF (match_dup 1) (parallel [(const_int 3)])))
1286 (vec_select:DF (match_dup 2) (parallel [(const_int 2)]))
1287 (vec_select:DF (match_dup 2) (parallel [(const_int 3)]))))))]
1289 "vh<plusminus_mnemonic>pd\t{%2, %1, %0|%0, %1, %2}"
1290 [(set_attr "type" "sseadd")
1291 (set_attr "prefix" "vex")
1292 (set_attr "mode" "V4DF")])
1294 (define_expand "sse3_haddv2df3"
1295 [(set (match_operand:V2DF 0 "register_operand")
1299 (match_operand:V2DF 1 "register_operand")
1300 (parallel [(const_int 0)]))
1301 (vec_select:DF (match_dup 1) (parallel [(const_int 1)])))
1304 (match_operand:V2DF 2 "nonimmediate_operand")
1305 (parallel [(const_int 0)]))
1306 (vec_select:DF (match_dup 2) (parallel [(const_int 1)])))))]
1309 (define_insn "*sse3_haddv2df3"
1310 [(set (match_operand:V2DF 0 "register_operand" "=x,x")
1314 (match_operand:V2DF 1 "register_operand" "0,x")
1315 (parallel [(match_operand:SI 3 "const_0_to_1_operand")]))
1318 (parallel [(match_operand:SI 4 "const_0_to_1_operand")])))
1321 (match_operand:V2DF 2 "nonimmediate_operand" "xm,xm")
1322 (parallel [(match_operand:SI 5 "const_0_to_1_operand")]))
1325 (parallel [(match_operand:SI 6 "const_0_to_1_operand")])))))]
1327 && INTVAL (operands[3]) != INTVAL (operands[4])
1328 && INTVAL (operands[5]) != INTVAL (operands[6])"
1330 haddpd\t{%2, %0|%0, %2}
1331 vhaddpd\t{%2, %1, %0|%0, %1, %2}"
1332 [(set_attr "isa" "noavx,avx")
1333 (set_attr "type" "sseadd")
1334 (set_attr "prefix" "orig,vex")
1335 (set_attr "mode" "V2DF")])
1337 (define_insn "sse3_hsubv2df3"
1338 [(set (match_operand:V2DF 0 "register_operand" "=x,x")
1342 (match_operand:V2DF 1 "register_operand" "0,x")
1343 (parallel [(const_int 0)]))
1344 (vec_select:DF (match_dup 1) (parallel [(const_int 1)])))
1347 (match_operand:V2DF 2 "nonimmediate_operand" "xm,xm")
1348 (parallel [(const_int 0)]))
1349 (vec_select:DF (match_dup 2) (parallel [(const_int 1)])))))]
1352 hsubpd\t{%2, %0|%0, %2}
1353 vhsubpd\t{%2, %1, %0|%0, %1, %2}"
1354 [(set_attr "isa" "noavx,avx")
1355 (set_attr "type" "sseadd")
1356 (set_attr "prefix" "orig,vex")
1357 (set_attr "mode" "V2DF")])
1359 (define_insn "*sse3_haddv2df3_low"
1360 [(set (match_operand:DF 0 "register_operand" "=x,x")
1363 (match_operand:V2DF 1 "register_operand" "0,x")
1364 (parallel [(match_operand:SI 2 "const_0_to_1_operand")]))
1367 (parallel [(match_operand:SI 3 "const_0_to_1_operand")]))))]
1369 && INTVAL (operands[2]) != INTVAL (operands[3])"
1371 haddpd\t{%0, %0|%0, %0}
1372 vhaddpd\t{%1, %1, %0|%0, %1, %1}"
1373 [(set_attr "isa" "noavx,avx")
1374 (set_attr "type" "sseadd1")
1375 (set_attr "prefix" "orig,vex")
1376 (set_attr "mode" "V2DF")])
1378 (define_insn "*sse3_hsubv2df3_low"
1379 [(set (match_operand:DF 0 "register_operand" "=x,x")
1382 (match_operand:V2DF 1 "register_operand" "0,x")
1383 (parallel [(const_int 0)]))
1386 (parallel [(const_int 1)]))))]
1389 hsubpd\t{%0, %0|%0, %0}
1390 vhsubpd\t{%1, %1, %0|%0, %1, %1}"
1391 [(set_attr "isa" "noavx,avx")
1392 (set_attr "type" "sseadd1")
1393 (set_attr "prefix" "orig,vex")
1394 (set_attr "mode" "V2DF")])
1396 (define_insn "avx_h<plusminus_insn>v8sf3"
1397 [(set (match_operand:V8SF 0 "register_operand" "=x")
1403 (match_operand:V8SF 1 "register_operand" "x")
1404 (parallel [(const_int 0)]))
1405 (vec_select:SF (match_dup 1) (parallel [(const_int 1)])))
1407 (vec_select:SF (match_dup 1) (parallel [(const_int 2)]))
1408 (vec_select:SF (match_dup 1) (parallel [(const_int 3)]))))
1412 (match_operand:V8SF 2 "nonimmediate_operand" "xm")
1413 (parallel [(const_int 0)]))
1414 (vec_select:SF (match_dup 2) (parallel [(const_int 1)])))
1416 (vec_select:SF (match_dup 2) (parallel [(const_int 2)]))
1417 (vec_select:SF (match_dup 2) (parallel [(const_int 3)])))))
1421 (vec_select:SF (match_dup 1) (parallel [(const_int 4)]))
1422 (vec_select:SF (match_dup 1) (parallel [(const_int 5)])))
1424 (vec_select:SF (match_dup 1) (parallel [(const_int 6)]))
1425 (vec_select:SF (match_dup 1) (parallel [(const_int 7)]))))
1428 (vec_select:SF (match_dup 2) (parallel [(const_int 4)]))
1429 (vec_select:SF (match_dup 2) (parallel [(const_int 5)])))
1431 (vec_select:SF (match_dup 2) (parallel [(const_int 6)]))
1432 (vec_select:SF (match_dup 2) (parallel [(const_int 7)])))))))]
1434 "vh<plusminus_mnemonic>ps\t{%2, %1, %0|%0, %1, %2}"
1435 [(set_attr "type" "sseadd")
1436 (set_attr "prefix" "vex")
1437 (set_attr "mode" "V8SF")])
1439 (define_insn "sse3_h<plusminus_insn>v4sf3"
1440 [(set (match_operand:V4SF 0 "register_operand" "=x,x")
1445 (match_operand:V4SF 1 "register_operand" "0,x")
1446 (parallel [(const_int 0)]))
1447 (vec_select:SF (match_dup 1) (parallel [(const_int 1)])))
1449 (vec_select:SF (match_dup 1) (parallel [(const_int 2)]))
1450 (vec_select:SF (match_dup 1) (parallel [(const_int 3)]))))
1454 (match_operand:V4SF 2 "nonimmediate_operand" "xm,xm")
1455 (parallel [(const_int 0)]))
1456 (vec_select:SF (match_dup 2) (parallel [(const_int 1)])))
1458 (vec_select:SF (match_dup 2) (parallel [(const_int 2)]))
1459 (vec_select:SF (match_dup 2) (parallel [(const_int 3)]))))))]
1462 h<plusminus_mnemonic>ps\t{%2, %0|%0, %2}
1463 vh<plusminus_mnemonic>ps\t{%2, %1, %0|%0, %1, %2}"
1464 [(set_attr "isa" "noavx,avx")
1465 (set_attr "type" "sseadd")
1466 (set_attr "atom_unit" "complex")
1467 (set_attr "prefix" "orig,vex")
1468 (set_attr "prefix_rep" "1,*")
1469 (set_attr "mode" "V4SF")])
1471 (define_expand "reduc_splus_v4df"
1472 [(match_operand:V4DF 0 "register_operand")
1473 (match_operand:V4DF 1 "register_operand")]
1476 rtx tmp = gen_reg_rtx (V4DFmode);
1477 rtx tmp2 = gen_reg_rtx (V4DFmode);
1478 emit_insn (gen_avx_haddv4df3 (tmp, operands[1], operands[1]));
1479 emit_insn (gen_avx_vperm2f128v4df3 (tmp2, tmp, tmp, GEN_INT (1)));
1480 emit_insn (gen_addv4df3 (operands[0], tmp, tmp2));
1484 (define_expand "reduc_splus_v2df"
1485 [(match_operand:V2DF 0 "register_operand")
1486 (match_operand:V2DF 1 "register_operand")]
1489 emit_insn (gen_sse3_haddv2df3 (operands[0], operands[1], operands[1]));
1493 (define_expand "reduc_splus_v8sf"
1494 [(match_operand:V8SF 0 "register_operand")
1495 (match_operand:V8SF 1 "register_operand")]
1498 rtx tmp = gen_reg_rtx (V8SFmode);
1499 rtx tmp2 = gen_reg_rtx (V8SFmode);
1500 emit_insn (gen_avx_haddv8sf3 (tmp, operands[1], operands[1]));
1501 emit_insn (gen_avx_haddv8sf3 (tmp2, tmp, tmp));
1502 emit_insn (gen_avx_vperm2f128v8sf3 (tmp, tmp2, tmp2, GEN_INT (1)));
1503 emit_insn (gen_addv8sf3 (operands[0], tmp, tmp2));
1507 (define_expand "reduc_splus_v4sf"
1508 [(match_operand:V4SF 0 "register_operand")
1509 (match_operand:V4SF 1 "register_operand")]
1514 rtx tmp = gen_reg_rtx (V4SFmode);
1515 emit_insn (gen_sse3_haddv4sf3 (tmp, operands[1], operands[1]));
1516 emit_insn (gen_sse3_haddv4sf3 (operands[0], tmp, tmp));
1519 ix86_expand_reduc (gen_addv4sf3, operands[0], operands[1]);
1523 ;; Modes handled by reduc_sm{in,ax}* patterns.
1524 (define_mode_iterator REDUC_SMINMAX_MODE
1525 [(V32QI "TARGET_AVX2") (V16HI "TARGET_AVX2")
1526 (V8SI "TARGET_AVX2") (V4DI "TARGET_AVX2")
1527 (V8SF "TARGET_AVX") (V4DF "TARGET_AVX")
1528 (V4SF "TARGET_SSE")])
1530 (define_expand "reduc_<code>_<mode>"
1531 [(smaxmin:REDUC_SMINMAX_MODE
1532 (match_operand:REDUC_SMINMAX_MODE 0 "register_operand")
1533 (match_operand:REDUC_SMINMAX_MODE 1 "register_operand"))]
1536 ix86_expand_reduc (gen_<code><mode>3, operands[0], operands[1]);
1540 (define_expand "reduc_<code>_<mode>"
1542 (match_operand:VI_256 0 "register_operand")
1543 (match_operand:VI_256 1 "register_operand"))]
1546 ix86_expand_reduc (gen_<code><mode>3, operands[0], operands[1]);
1550 (define_expand "reduc_umin_v8hi"
1552 (match_operand:V8HI 0 "register_operand")
1553 (match_operand:V8HI 1 "register_operand"))]
1556 ix86_expand_reduc (gen_uminv8hi3, operands[0], operands[1]);
1560 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
1562 ;; Parallel floating point comparisons
1564 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
1566 (define_insn "avx_cmp<mode>3"
1567 [(set (match_operand:VF 0 "register_operand" "=x")
1569 [(match_operand:VF 1 "register_operand" "x")
1570 (match_operand:VF 2 "nonimmediate_operand" "xm")
1571 (match_operand:SI 3 "const_0_to_31_operand" "n")]
1574 "vcmp<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
1575 [(set_attr "type" "ssecmp")
1576 (set_attr "length_immediate" "1")
1577 (set_attr "prefix" "vex")
1578 (set_attr "mode" "<MODE>")])
1580 (define_insn "avx_vmcmp<mode>3"
1581 [(set (match_operand:VF_128 0 "register_operand" "=x")
1584 [(match_operand:VF_128 1 "register_operand" "x")
1585 (match_operand:VF_128 2 "nonimmediate_operand" "xm")
1586 (match_operand:SI 3 "const_0_to_31_operand" "n")]
1591 "vcmp<ssescalarmodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
1592 [(set_attr "type" "ssecmp")
1593 (set_attr "length_immediate" "1")
1594 (set_attr "prefix" "vex")
1595 (set_attr "mode" "<ssescalarmode>")])
1597 (define_insn "*<sse>_maskcmp<mode>3_comm"
1598 [(set (match_operand:VF 0 "register_operand" "=x,x")
1599 (match_operator:VF 3 "sse_comparison_operator"
1600 [(match_operand:VF 1 "register_operand" "%0,x")
1601 (match_operand:VF 2 "nonimmediate_operand" "xm,xm")]))]
1603 && GET_RTX_CLASS (GET_CODE (operands[3])) == RTX_COMM_COMPARE"
1605 cmp%D3<ssemodesuffix>\t{%2, %0|%0, %2}
1606 vcmp%D3<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
1607 [(set_attr "isa" "noavx,avx")
1608 (set_attr "type" "ssecmp")
1609 (set_attr "length_immediate" "1")
1610 (set_attr "prefix" "orig,vex")
1611 (set_attr "mode" "<MODE>")])
1613 (define_insn "<sse>_maskcmp<mode>3"
1614 [(set (match_operand:VF 0 "register_operand" "=x,x")
1615 (match_operator:VF 3 "sse_comparison_operator"
1616 [(match_operand:VF 1 "register_operand" "0,x")
1617 (match_operand:VF 2 "nonimmediate_operand" "xm,xm")]))]
1620 cmp%D3<ssemodesuffix>\t{%2, %0|%0, %2}
1621 vcmp%D3<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
1622 [(set_attr "isa" "noavx,avx")
1623 (set_attr "type" "ssecmp")
1624 (set_attr "length_immediate" "1")
1625 (set_attr "prefix" "orig,vex")
1626 (set_attr "mode" "<MODE>")])
1628 (define_insn "<sse>_vmmaskcmp<mode>3"
1629 [(set (match_operand:VF_128 0 "register_operand" "=x,x")
1631 (match_operator:VF_128 3 "sse_comparison_operator"
1632 [(match_operand:VF_128 1 "register_operand" "0,x")
1633 (match_operand:VF_128 2 "nonimmediate_operand" "xm,xm")])
1638 cmp%D3<ssescalarmodesuffix>\t{%2, %0|%0, %2}
1639 vcmp%D3<ssescalarmodesuffix>\t{%2, %1, %0|%0, %1, %2}"
1640 [(set_attr "isa" "noavx,avx")
1641 (set_attr "type" "ssecmp")
1642 (set_attr "length_immediate" "1,*")
1643 (set_attr "prefix" "orig,vex")
1644 (set_attr "mode" "<ssescalarmode>")])
1646 (define_insn "<sse>_comi"
1647 [(set (reg:CCFP FLAGS_REG)
1650 (match_operand:<ssevecmode> 0 "register_operand" "x")
1651 (parallel [(const_int 0)]))
1653 (match_operand:<ssevecmode> 1 "nonimmediate_operand" "xm")
1654 (parallel [(const_int 0)]))))]
1655 "SSE_FLOAT_MODE_P (<MODE>mode)"
1656 "%vcomi<ssemodesuffix>\t{%1, %0|%0, %1}"
1657 [(set_attr "type" "ssecomi")
1658 (set_attr "prefix" "maybe_vex")
1659 (set_attr "prefix_rep" "0")
1660 (set (attr "prefix_data16")
1661 (if_then_else (eq_attr "mode" "DF")
1663 (const_string "0")))
1664 (set_attr "mode" "<MODE>")])
1666 (define_insn "<sse>_ucomi"
1667 [(set (reg:CCFPU FLAGS_REG)
1670 (match_operand:<ssevecmode> 0 "register_operand" "x")
1671 (parallel [(const_int 0)]))
1673 (match_operand:<ssevecmode> 1 "nonimmediate_operand" "xm")
1674 (parallel [(const_int 0)]))))]
1675 "SSE_FLOAT_MODE_P (<MODE>mode)"
1676 "%vucomi<ssemodesuffix>\t{%1, %0|%0, %1}"
1677 [(set_attr "type" "ssecomi")
1678 (set_attr "prefix" "maybe_vex")
1679 (set_attr "prefix_rep" "0")
1680 (set (attr "prefix_data16")
1681 (if_then_else (eq_attr "mode" "DF")
1683 (const_string "0")))
1684 (set_attr "mode" "<MODE>")])
1686 (define_expand "vcond<V_256:mode><VF_256:mode>"
1687 [(set (match_operand:V_256 0 "register_operand")
1689 (match_operator 3 ""
1690 [(match_operand:VF_256 4 "nonimmediate_operand")
1691 (match_operand:VF_256 5 "nonimmediate_operand")])
1692 (match_operand:V_256 1 "general_operand")
1693 (match_operand:V_256 2 "general_operand")))]
1695 && (GET_MODE_NUNITS (<V_256:MODE>mode)
1696 == GET_MODE_NUNITS (<VF_256:MODE>mode))"
1698 bool ok = ix86_expand_fp_vcond (operands);
1703 (define_expand "vcond<V_128:mode><VF_128:mode>"
1704 [(set (match_operand:V_128 0 "register_operand")
1706 (match_operator 3 ""
1707 [(match_operand:VF_128 4 "nonimmediate_operand")
1708 (match_operand:VF_128 5 "nonimmediate_operand")])
1709 (match_operand:V_128 1 "general_operand")
1710 (match_operand:V_128 2 "general_operand")))]
1712 && (GET_MODE_NUNITS (<V_128:MODE>mode)
1713 == GET_MODE_NUNITS (<VF_128:MODE>mode))"
1715 bool ok = ix86_expand_fp_vcond (operands);
1720 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
1722 ;; Parallel floating point logical operations
1724 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
1726 (define_insn "<sse>_andnot<mode>3"
1727 [(set (match_operand:VF 0 "register_operand" "=x,x")
1730 (match_operand:VF 1 "register_operand" "0,x"))
1731 (match_operand:VF 2 "nonimmediate_operand" "xm,xm")))]
1734 static char buf[32];
1738 switch (get_attr_mode (insn))
1745 suffix = "<ssemodesuffix>";
1748 switch (which_alternative)
1751 ops = "andn%s\t{%%2, %%0|%%0, %%2}";
1754 ops = "vandn%s\t{%%2, %%1, %%0|%%0, %%1, %%2}";
1760 snprintf (buf, sizeof (buf), ops, suffix);
1763 [(set_attr "isa" "noavx,avx")
1764 (set_attr "type" "sselog")
1765 (set_attr "prefix" "orig,vex")
1767 (cond [(match_test "TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL")
1768 (const_string "<ssePSmode>")
1769 (match_test "TARGET_AVX")
1770 (const_string "<MODE>")
1771 (match_test "optimize_function_for_size_p (cfun)")
1772 (const_string "V4SF")
1774 (const_string "<MODE>")))])
1776 (define_expand "<code><mode>3"
1777 [(set (match_operand:VF 0 "register_operand")
1779 (match_operand:VF 1 "nonimmediate_operand")
1780 (match_operand:VF 2 "nonimmediate_operand")))]
1782 "ix86_fixup_binary_operands_no_copy (<CODE>, <MODE>mode, operands);")
1784 (define_insn "*<code><mode>3"
1785 [(set (match_operand:VF 0 "register_operand" "=x,x")
1787 (match_operand:VF 1 "nonimmediate_operand" "%0,x")
1788 (match_operand:VF 2 "nonimmediate_operand" "xm,xm")))]
1789 "TARGET_SSE && ix86_binary_operator_ok (<CODE>, <MODE>mode, operands)"
1791 static char buf[32];
1795 switch (get_attr_mode (insn))
1802 suffix = "<ssemodesuffix>";
1805 switch (which_alternative)
1808 ops = "<logic>%s\t{%%2, %%0|%%0, %%2}";
1811 ops = "v<logic>%s\t{%%2, %%1, %%0|%%0, %%1, %%2}";
1817 snprintf (buf, sizeof (buf), ops, suffix);
1820 [(set_attr "isa" "noavx,avx")
1821 (set_attr "type" "sselog")
1822 (set_attr "prefix" "orig,vex")
1824 (cond [(match_test "TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL")
1825 (const_string "<ssePSmode>")
1826 (match_test "TARGET_AVX")
1827 (const_string "<MODE>")
1828 (match_test "optimize_function_for_size_p (cfun)")
1829 (const_string "V4SF")
1831 (const_string "<MODE>")))])
1833 (define_expand "copysign<mode>3"
1836 (not:VF (match_dup 3))
1837 (match_operand:VF 1 "nonimmediate_operand")))
1839 (and:VF (match_dup 3)
1840 (match_operand:VF 2 "nonimmediate_operand")))
1841 (set (match_operand:VF 0 "register_operand")
1842 (ior:VF (match_dup 4) (match_dup 5)))]
1845 operands[3] = ix86_build_signbit_mask (<MODE>mode, 1, 0);
1847 operands[4] = gen_reg_rtx (<MODE>mode);
1848 operands[5] = gen_reg_rtx (<MODE>mode);
1851 ;; Also define scalar versions. These are used for abs, neg, and
1852 ;; conditional move. Using subregs into vector modes causes register
1853 ;; allocation lossage. These patterns do not allow memory operands
1854 ;; because the native instructions read the full 128-bits.
1856 (define_insn "*andnot<mode>3"
1857 [(set (match_operand:MODEF 0 "register_operand" "=x,x")
1860 (match_operand:MODEF 1 "register_operand" "0,x"))
1861 (match_operand:MODEF 2 "register_operand" "x,x")))]
1862 "SSE_FLOAT_MODE_P (<MODE>mode)"
1864 static char buf[32];
1867 = (get_attr_mode (insn) == MODE_V4SF) ? "ps" : "<ssevecmodesuffix>";
1869 switch (which_alternative)
1872 ops = "andn%s\t{%%2, %%0|%%0, %%2}";
1875 ops = "vandn%s\t{%%2, %%1, %%0|%%0, %%1, %%2}";
1881 snprintf (buf, sizeof (buf), ops, suffix);
1884 [(set_attr "isa" "noavx,avx")
1885 (set_attr "type" "sselog")
1886 (set_attr "prefix" "orig,vex")
1888 (cond [(match_test "TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL")
1889 (const_string "V4SF")
1890 (match_test "TARGET_AVX")
1891 (const_string "<ssevecmode>")
1892 (match_test "optimize_function_for_size_p (cfun)")
1893 (const_string "V4SF")
1895 (const_string "<ssevecmode>")))])
1897 (define_insn "*andnottf3"
1898 [(set (match_operand:TF 0 "register_operand" "=x,x")
1900 (not:TF (match_operand:TF 1 "register_operand" "0,x"))
1901 (match_operand:TF 2 "nonimmediate_operand" "xm,xm")))]
1904 static char buf[32];
1907 = (get_attr_mode (insn) == MODE_V4SF) ? "andnps" : "pandn";
1909 switch (which_alternative)
1912 ops = "%s\t{%%2, %%0|%%0, %%2}";
1915 ops = "v%s\t{%%2, %%1, %%0|%%0, %%1, %%2}";
1921 snprintf (buf, sizeof (buf), ops, tmp);
1924 [(set_attr "isa" "noavx,avx")
1925 (set_attr "type" "sselog")
1926 (set (attr "prefix_data16")
1928 (and (eq_attr "alternative" "0")
1929 (eq_attr "mode" "TI"))
1931 (const_string "*")))
1932 (set_attr "prefix" "orig,vex")
1934 (cond [(match_test "TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL")
1935 (const_string "V4SF")
1936 (match_test "TARGET_AVX")
1938 (ior (not (match_test "TARGET_SSE2"))
1939 (match_test "optimize_function_for_size_p (cfun)"))
1940 (const_string "V4SF")
1942 (const_string "TI")))])
1944 (define_insn "*<code><mode>3"
1945 [(set (match_operand:MODEF 0 "register_operand" "=x,x")
1947 (match_operand:MODEF 1 "register_operand" "%0,x")
1948 (match_operand:MODEF 2 "register_operand" "x,x")))]
1949 "SSE_FLOAT_MODE_P (<MODE>mode)"
1951 static char buf[32];
1954 = (get_attr_mode (insn) == MODE_V4SF) ? "ps" : "<ssevecmodesuffix>";
1956 switch (which_alternative)
1959 ops = "<logic>%s\t{%%2, %%0|%%0, %%2}";
1962 ops = "v<logic>%s\t{%%2, %%1, %%0|%%0, %%1, %%2}";
1968 snprintf (buf, sizeof (buf), ops, suffix);
1971 [(set_attr "isa" "noavx,avx")
1972 (set_attr "type" "sselog")
1973 (set_attr "prefix" "orig,vex")
1975 (cond [(match_test "TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL")
1976 (const_string "V4SF")
1977 (match_test "TARGET_AVX")
1978 (const_string "<ssevecmode>")
1979 (match_test "optimize_function_for_size_p (cfun)")
1980 (const_string "V4SF")
1982 (const_string "<ssevecmode>")))])
1984 (define_expand "<code>tf3"
1985 [(set (match_operand:TF 0 "register_operand")
1987 (match_operand:TF 1 "nonimmediate_operand")
1988 (match_operand:TF 2 "nonimmediate_operand")))]
1990 "ix86_fixup_binary_operands_no_copy (<CODE>, TFmode, operands);")
1992 (define_insn "*<code>tf3"
1993 [(set (match_operand:TF 0 "register_operand" "=x,x")
1995 (match_operand:TF 1 "nonimmediate_operand" "%0,x")
1996 (match_operand:TF 2 "nonimmediate_operand" "xm,xm")))]
1998 && ix86_binary_operator_ok (<CODE>, TFmode, operands)"
2000 static char buf[32];
2003 = (get_attr_mode (insn) == MODE_V4SF) ? "<logic>ps" : "p<logic>";
2005 switch (which_alternative)
2008 ops = "%s\t{%%2, %%0|%%0, %%2}";
2011 ops = "v%s\t{%%2, %%1, %%0|%%0, %%1, %%2}";
2017 snprintf (buf, sizeof (buf), ops, tmp);
2020 [(set_attr "isa" "noavx,avx")
2021 (set_attr "type" "sselog")
2022 (set (attr "prefix_data16")
2024 (and (eq_attr "alternative" "0")
2025 (eq_attr "mode" "TI"))
2027 (const_string "*")))
2028 (set_attr "prefix" "orig,vex")
2030 (cond [(match_test "TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL")
2031 (const_string "V4SF")
2032 (match_test "TARGET_AVX")
2034 (ior (not (match_test "TARGET_SSE2"))
2035 (match_test "optimize_function_for_size_p (cfun)"))
2036 (const_string "V4SF")
2038 (const_string "TI")))])
2040 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
2042 ;; FMA floating point multiply/accumulate instructions. These include
2043 ;; scalar versions of the instructions as well as vector versions.
2045 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
2047 ;; The standard names for scalar FMA are only available with SSE math enabled.
2048 (define_mode_iterator FMAMODEM [(SF "TARGET_SSE_MATH")
2049 (DF "TARGET_SSE_MATH")
2050 V4SF V2DF V8SF V4DF])
2052 (define_expand "fma<mode>4"
2053 [(set (match_operand:FMAMODEM 0 "register_operand")
2055 (match_operand:FMAMODEM 1 "nonimmediate_operand")
2056 (match_operand:FMAMODEM 2 "nonimmediate_operand")
2057 (match_operand:FMAMODEM 3 "nonimmediate_operand")))]
2058 "TARGET_FMA || TARGET_FMA4")
2060 (define_expand "fms<mode>4"
2061 [(set (match_operand:FMAMODEM 0 "register_operand")
2063 (match_operand:FMAMODEM 1 "nonimmediate_operand")
2064 (match_operand:FMAMODEM 2 "nonimmediate_operand")
2065 (neg:FMAMODEM (match_operand:FMAMODEM 3 "nonimmediate_operand"))))]
2066 "TARGET_FMA || TARGET_FMA4")
2068 (define_expand "fnma<mode>4"
2069 [(set (match_operand:FMAMODEM 0 "register_operand")
2071 (neg:FMAMODEM (match_operand:FMAMODEM 1 "nonimmediate_operand"))
2072 (match_operand:FMAMODEM 2 "nonimmediate_operand")
2073 (match_operand:FMAMODEM 3 "nonimmediate_operand")))]
2074 "TARGET_FMA || TARGET_FMA4")
2076 (define_expand "fnms<mode>4"
2077 [(set (match_operand:FMAMODEM 0 "register_operand")
2079 (neg:FMAMODEM (match_operand:FMAMODEM 1 "nonimmediate_operand"))
2080 (match_operand:FMAMODEM 2 "nonimmediate_operand")
2081 (neg:FMAMODEM (match_operand:FMAMODEM 3 "nonimmediate_operand"))))]
2082 "TARGET_FMA || TARGET_FMA4")
2084 ;; The builtins for intrinsics are not constrained by SSE math enabled.
2085 (define_mode_iterator FMAMODE [SF DF V4SF V2DF V8SF V4DF])
2087 (define_expand "fma4i_fmadd_<mode>"
2088 [(set (match_operand:FMAMODE 0 "register_operand")
2090 (match_operand:FMAMODE 1 "nonimmediate_operand")
2091 (match_operand:FMAMODE 2 "nonimmediate_operand")
2092 (match_operand:FMAMODE 3 "nonimmediate_operand")))]
2093 "TARGET_FMA || TARGET_FMA4")
2095 (define_insn "*fma_fmadd_<mode>"
2096 [(set (match_operand:FMAMODE 0 "register_operand" "=x,x,x,x,x")
2098 (match_operand:FMAMODE 1 "nonimmediate_operand" "%0, 0,x, x,x")
2099 (match_operand:FMAMODE 2 "nonimmediate_operand" "xm, x,xm,x,m")
2100 (match_operand:FMAMODE 3 "nonimmediate_operand" " x,xm,0,xm,x")))]
2101 "TARGET_FMA || TARGET_FMA4"
2103 vfmadd132<ssemodesuffix>\t{%2, %3, %0|%0, %3, %2}
2104 vfmadd213<ssemodesuffix>\t{%3, %2, %0|%0, %2, %3}
2105 vfmadd231<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}
2106 vfmadd<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}
2107 vfmadd<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
2108 [(set_attr "isa" "fma,fma,fma,fma4,fma4")
2109 (set_attr "type" "ssemuladd")
2110 (set_attr "mode" "<MODE>")])
2112 (define_insn "*fma_fmsub_<mode>"
2113 [(set (match_operand:FMAMODE 0 "register_operand" "=x,x,x,x,x")
2115 (match_operand:FMAMODE 1 "nonimmediate_operand" "%0, 0,x, x,x")
2116 (match_operand:FMAMODE 2 "nonimmediate_operand" "xm, x,xm,x,m")
2118 (match_operand:FMAMODE 3 "nonimmediate_operand" " x,xm,0,xm,x"))))]
2119 "TARGET_FMA || TARGET_FMA4"
2121 vfmsub132<ssemodesuffix>\t{%2, %3, %0|%0, %3, %2}
2122 vfmsub213<ssemodesuffix>\t{%3, %2, %0|%0, %2, %3}
2123 vfmsub231<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}
2124 vfmsub<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}
2125 vfmsub<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
2126 [(set_attr "isa" "fma,fma,fma,fma4,fma4")
2127 (set_attr "type" "ssemuladd")
2128 (set_attr "mode" "<MODE>")])
2130 (define_insn "*fma_fnmadd_<mode>"
2131 [(set (match_operand:FMAMODE 0 "register_operand" "=x,x,x,x,x")
2134 (match_operand:FMAMODE 1 "nonimmediate_operand" "%0, 0,x, x,x"))
2135 (match_operand:FMAMODE 2 "nonimmediate_operand" "xm, x,xm,x,m")
2136 (match_operand:FMAMODE 3 "nonimmediate_operand" " x,xm,0,xm,x")))]
2137 "TARGET_FMA || TARGET_FMA4"
2139 vfnmadd132<ssemodesuffix>\t{%2, %3, %0|%0, %3, %2}
2140 vfnmadd213<ssemodesuffix>\t{%3, %2, %0|%0, %2, %3}
2141 vfnmadd231<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}
2142 vfnmadd<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}
2143 vfnmadd<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
2144 [(set_attr "isa" "fma,fma,fma,fma4,fma4")
2145 (set_attr "type" "ssemuladd")
2146 (set_attr "mode" "<MODE>")])
2148 (define_insn "*fma_fnmsub_<mode>"
2149 [(set (match_operand:FMAMODE 0 "register_operand" "=x,x,x,x,x")
2152 (match_operand:FMAMODE 1 "nonimmediate_operand" "%0, 0,x, x,x"))
2153 (match_operand:FMAMODE 2 "nonimmediate_operand" "xm, x,xm,x,m")
2155 (match_operand:FMAMODE 3 "nonimmediate_operand" " x,xm,0,xm,x"))))]
2156 "TARGET_FMA || TARGET_FMA4"
2158 vfnmsub132<ssemodesuffix>\t{%2, %3, %0|%0, %3, %2}
2159 vfnmsub213<ssemodesuffix>\t{%3, %2, %0|%0, %2, %3}
2160 vfnmsub231<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}
2161 vfnmsub<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}
2162 vfnmsub<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
2163 [(set_attr "isa" "fma,fma,fma,fma4,fma4")
2164 (set_attr "type" "ssemuladd")
2165 (set_attr "mode" "<MODE>")])
2167 ;; FMA parallel floating point multiply addsub and subadd operations.
2169 ;; It would be possible to represent these without the UNSPEC as
2172 ;; (fma op1 op2 op3)
2173 ;; (fma op1 op2 (neg op3))
2176 ;; But this doesn't seem useful in practice.
2178 (define_expand "fmaddsub_<mode>"
2179 [(set (match_operand:VF 0 "register_operand")
2181 [(match_operand:VF 1 "nonimmediate_operand")
2182 (match_operand:VF 2 "nonimmediate_operand")
2183 (match_operand:VF 3 "nonimmediate_operand")]
2185 "TARGET_FMA || TARGET_FMA4")
2187 (define_insn "*fma_fmaddsub_<mode>"
2188 [(set (match_operand:VF 0 "register_operand" "=x,x,x,x,x")
2190 [(match_operand:VF 1 "nonimmediate_operand" "%0, 0,x, x,x")
2191 (match_operand:VF 2 "nonimmediate_operand" "xm, x,xm,x,m")
2192 (match_operand:VF 3 "nonimmediate_operand" " x,xm,0,xm,x")]
2194 "TARGET_FMA || TARGET_FMA4"
2196 vfmaddsub132<ssemodesuffix>\t{%2, %3, %0|%0, %3, %2}
2197 vfmaddsub213<ssemodesuffix>\t{%3, %2, %0|%0, %2, %3}
2198 vfmaddsub231<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}
2199 vfmaddsub<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}
2200 vfmaddsub<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
2201 [(set_attr "isa" "fma,fma,fma,fma4,fma4")
2202 (set_attr "type" "ssemuladd")
2203 (set_attr "mode" "<MODE>")])
2205 (define_insn "*fma_fmsubadd_<mode>"
2206 [(set (match_operand:VF 0 "register_operand" "=x,x,x,x,x")
2208 [(match_operand:VF 1 "nonimmediate_operand" "%0, 0,x, x,x")
2209 (match_operand:VF 2 "nonimmediate_operand" "xm, x,xm,x,m")
2211 (match_operand:VF 3 "nonimmediate_operand" " x,xm,0,xm,x"))]
2213 "TARGET_FMA || TARGET_FMA4"
2215 vfmsubadd132<ssemodesuffix>\t{%2, %3, %0|%0, %3, %2}
2216 vfmsubadd213<ssemodesuffix>\t{%3, %2, %0|%0, %2, %3}
2217 vfmsubadd231<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}
2218 vfmsubadd<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}
2219 vfmsubadd<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
2220 [(set_attr "isa" "fma,fma,fma,fma4,fma4")
2221 (set_attr "type" "ssemuladd")
2222 (set_attr "mode" "<MODE>")])
2224 ;; FMA3 floating point scalar intrinsics. These merge result with
2225 ;; high-order elements from the destination register.
2227 (define_expand "fmai_vmfmadd_<mode>"
2228 [(set (match_operand:VF_128 0 "register_operand")
2231 (match_operand:VF_128 1 "nonimmediate_operand")
2232 (match_operand:VF_128 2 "nonimmediate_operand")
2233 (match_operand:VF_128 3 "nonimmediate_operand"))
2238 (define_insn "*fmai_fmadd_<mode>"
2239 [(set (match_operand:VF_128 0 "register_operand" "=x,x")
2242 (match_operand:VF_128 1 "nonimmediate_operand" " 0, 0")
2243 (match_operand:VF_128 2 "nonimmediate_operand" "xm, x")
2244 (match_operand:VF_128 3 "nonimmediate_operand" " x,xm"))
2249 vfmadd132<ssescalarmodesuffix>\t{%2, %3, %0|%0, %3, %2}
2250 vfmadd213<ssescalarmodesuffix>\t{%3, %2, %0|%0, %2, %3}"
2251 [(set_attr "type" "ssemuladd")
2252 (set_attr "mode" "<MODE>")])
2254 (define_insn "*fmai_fmsub_<mode>"
2255 [(set (match_operand:VF_128 0 "register_operand" "=x,x")
2258 (match_operand:VF_128 1 "nonimmediate_operand" " 0, 0")
2259 (match_operand:VF_128 2 "nonimmediate_operand" "xm, x")
2261 (match_operand:VF_128 3 "nonimmediate_operand" " x,xm")))
2266 vfmsub132<ssescalarmodesuffix>\t{%2, %3, %0|%0, %3, %2}
2267 vfmsub213<ssescalarmodesuffix>\t{%3, %2, %0|%0, %2, %3}"
2268 [(set_attr "type" "ssemuladd")
2269 (set_attr "mode" "<MODE>")])
2271 (define_insn "*fmai_fnmadd_<mode>"
2272 [(set (match_operand:VF_128 0 "register_operand" "=x,x")
2276 (match_operand:VF_128 2 "nonimmediate_operand" "xm, x"))
2277 (match_operand:VF_128 1 "nonimmediate_operand" " 0, 0")
2278 (match_operand:VF_128 3 "nonimmediate_operand" " x,xm"))
2283 vfnmadd132<ssescalarmodesuffix>\t{%2, %3, %0|%0, %3, %2}
2284 vfnmadd213<ssescalarmodesuffix>\t{%3, %2, %0|%0, %2, %3}"
2285 [(set_attr "type" "ssemuladd")
2286 (set_attr "mode" "<MODE>")])
2288 (define_insn "*fmai_fnmsub_<mode>"
2289 [(set (match_operand:VF_128 0 "register_operand" "=x,x")
2293 (match_operand:VF_128 2 "nonimmediate_operand" "xm, x"))
2294 (match_operand:VF_128 1 "nonimmediate_operand" " 0, 0")
2296 (match_operand:VF_128 3 "nonimmediate_operand" " x,xm")))
2301 vfnmsub132<ssescalarmodesuffix>\t{%2, %3, %0|%0, %3, %2}
2302 vfnmsub213<ssescalarmodesuffix>\t{%3, %2, %0|%0, %2, %3}"
2303 [(set_attr "type" "ssemuladd")
2304 (set_attr "mode" "<MODE>")])
2306 ;; FMA4 floating point scalar intrinsics. These write the
2307 ;; entire destination register, with the high-order elements zeroed.
2309 (define_expand "fma4i_vmfmadd_<mode>"
2310 [(set (match_operand:VF_128 0 "register_operand")
2313 (match_operand:VF_128 1 "nonimmediate_operand")
2314 (match_operand:VF_128 2 "nonimmediate_operand")
2315 (match_operand:VF_128 3 "nonimmediate_operand"))
2319 "operands[4] = CONST0_RTX (<MODE>mode);")
2321 (define_insn "*fma4i_vmfmadd_<mode>"
2322 [(set (match_operand:VF_128 0 "register_operand" "=x,x")
2325 (match_operand:VF_128 1 "nonimmediate_operand" "%x,x")
2326 (match_operand:VF_128 2 "nonimmediate_operand" " x,m")
2327 (match_operand:VF_128 3 "nonimmediate_operand" "xm,x"))
2328 (match_operand:VF_128 4 "const0_operand")
2331 "vfmadd<ssescalarmodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
2332 [(set_attr "type" "ssemuladd")
2333 (set_attr "mode" "<MODE>")])
2335 (define_insn "*fma4i_vmfmsub_<mode>"
2336 [(set (match_operand:VF_128 0 "register_operand" "=x,x")
2339 (match_operand:VF_128 1 "nonimmediate_operand" "%x,x")
2340 (match_operand:VF_128 2 "nonimmediate_operand" " x,m")
2342 (match_operand:VF_128 3 "nonimmediate_operand" "xm,x")))
2343 (match_operand:VF_128 4 "const0_operand")
2346 "vfmsub<ssescalarmodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
2347 [(set_attr "type" "ssemuladd")
2348 (set_attr "mode" "<MODE>")])
2350 (define_insn "*fma4i_vmfnmadd_<mode>"
2351 [(set (match_operand:VF_128 0 "register_operand" "=x,x")
2355 (match_operand:VF_128 1 "nonimmediate_operand" "%x,x"))
2356 (match_operand:VF_128 2 "nonimmediate_operand" " x,m")
2357 (match_operand:VF_128 3 "nonimmediate_operand" "xm,x"))
2358 (match_operand:VF_128 4 "const0_operand")
2361 "vfnmadd<ssescalarmodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
2362 [(set_attr "type" "ssemuladd")
2363 (set_attr "mode" "<MODE>")])
2365 (define_insn "*fma4i_vmfnmsub_<mode>"
2366 [(set (match_operand:VF_128 0 "register_operand" "=x,x")
2370 (match_operand:VF_128 1 "nonimmediate_operand" "%x,x"))
2371 (match_operand:VF_128 2 "nonimmediate_operand" " x,m")
2373 (match_operand:VF_128 3 "nonimmediate_operand" "xm,x")))
2374 (match_operand:VF_128 4 "const0_operand")
2377 "vfnmsub<ssescalarmodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
2378 [(set_attr "type" "ssemuladd")
2379 (set_attr "mode" "<MODE>")])
2381 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
2383 ;; Parallel single-precision floating point conversion operations
2385 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
2387 (define_insn "sse_cvtpi2ps"
2388 [(set (match_operand:V4SF 0 "register_operand" "=x")
2391 (float:V2SF (match_operand:V2SI 2 "nonimmediate_operand" "ym")))
2392 (match_operand:V4SF 1 "register_operand" "0")
2395 "cvtpi2ps\t{%2, %0|%0, %2}"
2396 [(set_attr "type" "ssecvt")
2397 (set_attr "mode" "V4SF")])
2399 (define_insn "sse_cvtps2pi"
2400 [(set (match_operand:V2SI 0 "register_operand" "=y")
2402 (unspec:V4SI [(match_operand:V4SF 1 "nonimmediate_operand" "xm")]
2404 (parallel [(const_int 0) (const_int 1)])))]
2406 "cvtps2pi\t{%1, %0|%0, %1}"
2407 [(set_attr "type" "ssecvt")
2408 (set_attr "unit" "mmx")
2409 (set_attr "mode" "DI")])
2411 (define_insn "sse_cvttps2pi"
2412 [(set (match_operand:V2SI 0 "register_operand" "=y")
2414 (fix:V4SI (match_operand:V4SF 1 "nonimmediate_operand" "xm"))
2415 (parallel [(const_int 0) (const_int 1)])))]
2417 "cvttps2pi\t{%1, %0|%0, %1}"
2418 [(set_attr "type" "ssecvt")
2419 (set_attr "unit" "mmx")
2420 (set_attr "prefix_rep" "0")
2421 (set_attr "mode" "SF")])
2423 (define_insn "sse_cvtsi2ss"
2424 [(set (match_operand:V4SF 0 "register_operand" "=x,x,x")
2427 (float:SF (match_operand:SI 2 "nonimmediate_operand" "r,m,rm")))
2428 (match_operand:V4SF 1 "register_operand" "0,0,x")
2432 cvtsi2ss\t{%2, %0|%0, %2}
2433 cvtsi2ss\t{%2, %0|%0, %2}
2434 vcvtsi2ss\t{%2, %1, %0|%0, %1, %2}"
2435 [(set_attr "isa" "noavx,noavx,avx")
2436 (set_attr "type" "sseicvt")
2437 (set_attr "athlon_decode" "vector,double,*")
2438 (set_attr "amdfam10_decode" "vector,double,*")
2439 (set_attr "bdver1_decode" "double,direct,*")
2440 (set_attr "btver2_decode" "double,double,double")
2441 (set_attr "prefix" "orig,orig,vex")
2442 (set_attr "mode" "SF")])
2444 (define_insn "sse_cvtsi2ssq"
2445 [(set (match_operand:V4SF 0 "register_operand" "=x,x,x")
2448 (float:SF (match_operand:DI 2 "nonimmediate_operand" "r,m,rm")))
2449 (match_operand:V4SF 1 "register_operand" "0,0,x")
2451 "TARGET_SSE && TARGET_64BIT"
2453 cvtsi2ssq\t{%2, %0|%0, %2}
2454 cvtsi2ssq\t{%2, %0|%0, %2}
2455 vcvtsi2ssq\t{%2, %1, %0|%0, %1, %2}"
2456 [(set_attr "isa" "noavx,noavx,avx")
2457 (set_attr "type" "sseicvt")
2458 (set_attr "athlon_decode" "vector,double,*")
2459 (set_attr "amdfam10_decode" "vector,double,*")
2460 (set_attr "bdver1_decode" "double,direct,*")
2461 (set_attr "btver2_decode" "double,double,double")
2462 (set_attr "length_vex" "*,*,4")
2463 (set_attr "prefix_rex" "1,1,*")
2464 (set_attr "prefix" "orig,orig,vex")
2465 (set_attr "mode" "SF")])
2467 (define_insn "sse_cvtss2si"
2468 [(set (match_operand:SI 0 "register_operand" "=r,r")
2471 (match_operand:V4SF 1 "nonimmediate_operand" "x,m")
2472 (parallel [(const_int 0)]))]
2473 UNSPEC_FIX_NOTRUNC))]
2475 "%vcvtss2si\t{%1, %0|%0, %1}"
2476 [(set_attr "type" "sseicvt")
2477 (set_attr "athlon_decode" "double,vector")
2478 (set_attr "bdver1_decode" "double,double")
2479 (set_attr "prefix_rep" "1")
2480 (set_attr "prefix" "maybe_vex")
2481 (set_attr "mode" "SI")])
2483 (define_insn "sse_cvtss2si_2"
2484 [(set (match_operand:SI 0 "register_operand" "=r,r")
2485 (unspec:SI [(match_operand:SF 1 "nonimmediate_operand" "x,m")]
2486 UNSPEC_FIX_NOTRUNC))]
2488 "%vcvtss2si\t{%1, %0|%0, %1}"
2489 [(set_attr "type" "sseicvt")
2490 (set_attr "athlon_decode" "double,vector")
2491 (set_attr "amdfam10_decode" "double,double")
2492 (set_attr "bdver1_decode" "double,double")
2493 (set_attr "prefix_rep" "1")
2494 (set_attr "prefix" "maybe_vex")
2495 (set_attr "mode" "SI")])
2497 (define_insn "sse_cvtss2siq"
2498 [(set (match_operand:DI 0 "register_operand" "=r,r")
2501 (match_operand:V4SF 1 "nonimmediate_operand" "x,m")
2502 (parallel [(const_int 0)]))]
2503 UNSPEC_FIX_NOTRUNC))]
2504 "TARGET_SSE && TARGET_64BIT"
2505 "%vcvtss2si{q}\t{%1, %0|%0, %1}"
2506 [(set_attr "type" "sseicvt")
2507 (set_attr "athlon_decode" "double,vector")
2508 (set_attr "bdver1_decode" "double,double")
2509 (set_attr "prefix_rep" "1")
2510 (set_attr "prefix" "maybe_vex")
2511 (set_attr "mode" "DI")])
2513 (define_insn "sse_cvtss2siq_2"
2514 [(set (match_operand:DI 0 "register_operand" "=r,r")
2515 (unspec:DI [(match_operand:SF 1 "nonimmediate_operand" "x,m")]
2516 UNSPEC_FIX_NOTRUNC))]
2517 "TARGET_SSE && TARGET_64BIT"
2518 "%vcvtss2si{q}\t{%1, %0|%0, %1}"
2519 [(set_attr "type" "sseicvt")
2520 (set_attr "athlon_decode" "double,vector")
2521 (set_attr "amdfam10_decode" "double,double")
2522 (set_attr "bdver1_decode" "double,double")
2523 (set_attr "prefix_rep" "1")
2524 (set_attr "prefix" "maybe_vex")
2525 (set_attr "mode" "DI")])
2527 (define_insn "sse_cvttss2si"
2528 [(set (match_operand:SI 0 "register_operand" "=r,r")
2531 (match_operand:V4SF 1 "nonimmediate_operand" "x,m")
2532 (parallel [(const_int 0)]))))]
2534 "%vcvttss2si\t{%1, %0|%0, %1}"
2535 [(set_attr "type" "sseicvt")
2536 (set_attr "athlon_decode" "double,vector")
2537 (set_attr "amdfam10_decode" "double,double")
2538 (set_attr "bdver1_decode" "double,double")
2539 (set_attr "prefix_rep" "1")
2540 (set_attr "prefix" "maybe_vex")
2541 (set_attr "mode" "SI")])
2543 (define_insn "sse_cvttss2siq"
2544 [(set (match_operand:DI 0 "register_operand" "=r,r")
2547 (match_operand:V4SF 1 "nonimmediate_operand" "x,m")
2548 (parallel [(const_int 0)]))))]
2549 "TARGET_SSE && TARGET_64BIT"
2550 "%vcvttss2si{q}\t{%1, %0|%0, %1}"
2551 [(set_attr "type" "sseicvt")
2552 (set_attr "athlon_decode" "double,vector")
2553 (set_attr "amdfam10_decode" "double,double")
2554 (set_attr "bdver1_decode" "double,double")
2555 (set_attr "prefix_rep" "1")
2556 (set_attr "prefix" "maybe_vex")
2557 (set_attr "mode" "DI")])
2559 (define_insn "float<sseintvecmodelower><mode>2"
2560 [(set (match_operand:VF1 0 "register_operand" "=x")
2562 (match_operand:<sseintvecmode> 1 "nonimmediate_operand" "xm")))]
2564 "%vcvtdq2ps\t{%1, %0|%0, %1}"
2565 [(set_attr "type" "ssecvt")
2566 (set_attr "prefix" "maybe_vex")
2567 (set_attr "mode" "<sseinsnmode>")])
2569 (define_expand "floatuns<sseintvecmodelower><mode>2"
2570 [(match_operand:VF1 0 "register_operand")
2571 (match_operand:<sseintvecmode> 1 "register_operand")]
2572 "TARGET_SSE2 && (<MODE>mode == V4SFmode || TARGET_AVX2)"
2574 ix86_expand_vector_convert_uns_vsivsf (operands[0], operands[1]);
2578 (define_insn "avx_cvtps2dq256"
2579 [(set (match_operand:V8SI 0 "register_operand" "=x")
2580 (unspec:V8SI [(match_operand:V8SF 1 "nonimmediate_operand" "xm")]
2581 UNSPEC_FIX_NOTRUNC))]
2583 "vcvtps2dq\t{%1, %0|%0, %1}"
2584 [(set_attr "type" "ssecvt")
2585 (set_attr "prefix" "vex")
2586 (set_attr "mode" "OI")])
2588 (define_insn "sse2_cvtps2dq"
2589 [(set (match_operand:V4SI 0 "register_operand" "=x")
2590 (unspec:V4SI [(match_operand:V4SF 1 "nonimmediate_operand" "xm")]
2591 UNSPEC_FIX_NOTRUNC))]
2593 "%vcvtps2dq\t{%1, %0|%0, %1}"
2594 [(set_attr "type" "ssecvt")
2595 (set (attr "prefix_data16")
2597 (match_test "TARGET_AVX")
2599 (const_string "1")))
2600 (set_attr "prefix" "maybe_vex")
2601 (set_attr "mode" "TI")])
2603 (define_insn "fix_truncv8sfv8si2"
2604 [(set (match_operand:V8SI 0 "register_operand" "=x")
2605 (fix:V8SI (match_operand:V8SF 1 "nonimmediate_operand" "xm")))]
2607 "vcvttps2dq\t{%1, %0|%0, %1}"
2608 [(set_attr "type" "ssecvt")
2609 (set_attr "prefix" "vex")
2610 (set_attr "mode" "OI")])
2612 (define_insn "fix_truncv4sfv4si2"
2613 [(set (match_operand:V4SI 0 "register_operand" "=x")
2614 (fix:V4SI (match_operand:V4SF 1 "nonimmediate_operand" "xm")))]
2616 "%vcvttps2dq\t{%1, %0|%0, %1}"
2617 [(set_attr "type" "ssecvt")
2618 (set (attr "prefix_rep")
2620 (match_test "TARGET_AVX")
2622 (const_string "1")))
2623 (set (attr "prefix_data16")
2625 (match_test "TARGET_AVX")
2627 (const_string "0")))
2628 (set_attr "prefix_data16" "0")
2629 (set_attr "prefix" "maybe_vex")
2630 (set_attr "mode" "TI")])
2632 (define_expand "fixuns_trunc<mode><sseintvecmodelower>2"
2633 [(match_operand:<sseintvecmode> 0 "register_operand")
2634 (match_operand:VF1 1 "register_operand")]
2638 tmp[0] = ix86_expand_adjust_ufix_to_sfix_si (operands[1], &tmp[2]);
2639 tmp[1] = gen_reg_rtx (<sseintvecmode>mode);
2640 emit_insn (gen_fix_trunc<mode><sseintvecmodelower>2 (tmp[1], tmp[0]));
2641 emit_insn (gen_xor<sseintvecmodelower>3 (operands[0], tmp[1], tmp[2]));
2645 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
2647 ;; Parallel double-precision floating point conversion operations
2649 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
2651 (define_insn "sse2_cvtpi2pd"
2652 [(set (match_operand:V2DF 0 "register_operand" "=x,x")
2653 (float:V2DF (match_operand:V2SI 1 "nonimmediate_operand" "y,m")))]
2655 "cvtpi2pd\t{%1, %0|%0, %1}"
2656 [(set_attr "type" "ssecvt")
2657 (set_attr "unit" "mmx,*")
2658 (set_attr "prefix_data16" "1,*")
2659 (set_attr "mode" "V2DF")])
2661 (define_insn "sse2_cvtpd2pi"
2662 [(set (match_operand:V2SI 0 "register_operand" "=y")
2663 (unspec:V2SI [(match_operand:V2DF 1 "nonimmediate_operand" "xm")]
2664 UNSPEC_FIX_NOTRUNC))]
2666 "cvtpd2pi\t{%1, %0|%0, %1}"
2667 [(set_attr "type" "ssecvt")
2668 (set_attr "unit" "mmx")
2669 (set_attr "bdver1_decode" "double")
2670 (set_attr "btver2_decode" "direct")
2671 (set_attr "prefix_data16" "1")
2672 (set_attr "mode" "DI")])
2674 (define_insn "sse2_cvttpd2pi"
2675 [(set (match_operand:V2SI 0 "register_operand" "=y")
2676 (fix:V2SI (match_operand:V2DF 1 "nonimmediate_operand" "xm")))]
2678 "cvttpd2pi\t{%1, %0|%0, %1}"
2679 [(set_attr "type" "ssecvt")
2680 (set_attr "unit" "mmx")
2681 (set_attr "bdver1_decode" "double")
2682 (set_attr "prefix_data16" "1")
2683 (set_attr "mode" "TI")])
2685 (define_insn "sse2_cvtsi2sd"
2686 [(set (match_operand:V2DF 0 "register_operand" "=x,x,x")
2689 (float:DF (match_operand:SI 2 "nonimmediate_operand" "r,m,rm")))
2690 (match_operand:V2DF 1 "register_operand" "0,0,x")
2694 cvtsi2sd\t{%2, %0|%0, %2}
2695 cvtsi2sd\t{%2, %0|%0, %2}
2696 vcvtsi2sd\t{%2, %1, %0|%0, %1, %2}"
2697 [(set_attr "isa" "noavx,noavx,avx")
2698 (set_attr "type" "sseicvt")
2699 (set_attr "athlon_decode" "double,direct,*")
2700 (set_attr "amdfam10_decode" "vector,double,*")
2701 (set_attr "bdver1_decode" "double,direct,*")
2702 (set_attr "btver2_decode" "double,double,double")
2703 (set_attr "prefix" "orig,orig,vex")
2704 (set_attr "mode" "DF")])
2706 (define_insn "sse2_cvtsi2sdq"
2707 [(set (match_operand:V2DF 0 "register_operand" "=x,x,x")
2710 (float:DF (match_operand:DI 2 "nonimmediate_operand" "r,m,rm")))
2711 (match_operand:V2DF 1 "register_operand" "0,0,x")
2713 "TARGET_SSE2 && TARGET_64BIT"
2715 cvtsi2sdq\t{%2, %0|%0, %2}
2716 cvtsi2sdq\t{%2, %0|%0, %2}
2717 vcvtsi2sdq\t{%2, %1, %0|%0, %1, %2}"
2718 [(set_attr "isa" "noavx,noavx,avx")
2719 (set_attr "type" "sseicvt")
2720 (set_attr "athlon_decode" "double,direct,*")
2721 (set_attr "amdfam10_decode" "vector,double,*")
2722 (set_attr "bdver1_decode" "double,direct,*")
2723 (set_attr "length_vex" "*,*,4")
2724 (set_attr "prefix_rex" "1,1,*")
2725 (set_attr "prefix" "orig,orig,vex")
2726 (set_attr "mode" "DF")])
2728 (define_insn "sse2_cvtsd2si"
2729 [(set (match_operand:SI 0 "register_operand" "=r,r")
2732 (match_operand:V2DF 1 "nonimmediate_operand" "x,m")
2733 (parallel [(const_int 0)]))]
2734 UNSPEC_FIX_NOTRUNC))]
2736 "%vcvtsd2si\t{%1, %0|%0, %1}"
2737 [(set_attr "type" "sseicvt")
2738 (set_attr "athlon_decode" "double,vector")
2739 (set_attr "bdver1_decode" "double,double")
2740 (set_attr "btver2_decode" "double,double")
2741 (set_attr "prefix_rep" "1")
2742 (set_attr "prefix" "maybe_vex")
2743 (set_attr "mode" "SI")])
2745 (define_insn "sse2_cvtsd2si_2"
2746 [(set (match_operand:SI 0 "register_operand" "=r,r")
2747 (unspec:SI [(match_operand:DF 1 "nonimmediate_operand" "x,m")]
2748 UNSPEC_FIX_NOTRUNC))]
2750 "%vcvtsd2si\t{%1, %0|%0, %1}"
2751 [(set_attr "type" "sseicvt")
2752 (set_attr "athlon_decode" "double,vector")
2753 (set_attr "amdfam10_decode" "double,double")
2754 (set_attr "bdver1_decode" "double,double")
2755 (set_attr "prefix_rep" "1")
2756 (set_attr "prefix" "maybe_vex")
2757 (set_attr "mode" "SI")])
2759 (define_insn "sse2_cvtsd2siq"
2760 [(set (match_operand:DI 0 "register_operand" "=r,r")
2763 (match_operand:V2DF 1 "nonimmediate_operand" "x,m")
2764 (parallel [(const_int 0)]))]
2765 UNSPEC_FIX_NOTRUNC))]
2766 "TARGET_SSE2 && TARGET_64BIT"
2767 "%vcvtsd2si{q}\t{%1, %0|%0, %1}"
2768 [(set_attr "type" "sseicvt")
2769 (set_attr "athlon_decode" "double,vector")
2770 (set_attr "bdver1_decode" "double,double")
2771 (set_attr "prefix_rep" "1")
2772 (set_attr "prefix" "maybe_vex")
2773 (set_attr "mode" "DI")])
2775 (define_insn "sse2_cvtsd2siq_2"
2776 [(set (match_operand:DI 0 "register_operand" "=r,r")
2777 (unspec:DI [(match_operand:DF 1 "nonimmediate_operand" "x,m")]
2778 UNSPEC_FIX_NOTRUNC))]
2779 "TARGET_SSE2 && TARGET_64BIT"
2780 "%vcvtsd2si{q}\t{%1, %0|%0, %1}"
2781 [(set_attr "type" "sseicvt")
2782 (set_attr "athlon_decode" "double,vector")
2783 (set_attr "amdfam10_decode" "double,double")
2784 (set_attr "bdver1_decode" "double,double")
2785 (set_attr "prefix_rep" "1")
2786 (set_attr "prefix" "maybe_vex")
2787 (set_attr "mode" "DI")])
2789 (define_insn "sse2_cvttsd2si"
2790 [(set (match_operand:SI 0 "register_operand" "=r,r")
2793 (match_operand:V2DF 1 "nonimmediate_operand" "x,m")
2794 (parallel [(const_int 0)]))))]
2796 "%vcvttsd2si\t{%1, %0|%0, %1}"
2797 [(set_attr "type" "sseicvt")
2798 (set_attr "athlon_decode" "double,vector")
2799 (set_attr "amdfam10_decode" "double,double")
2800 (set_attr "bdver1_decode" "double,double")
2801 (set_attr "btver2_decode" "double,double")
2802 (set_attr "prefix_rep" "1")
2803 (set_attr "prefix" "maybe_vex")
2804 (set_attr "mode" "SI")])
2806 (define_insn "sse2_cvttsd2siq"
2807 [(set (match_operand:DI 0 "register_operand" "=r,r")
2810 (match_operand:V2DF 1 "nonimmediate_operand" "x,m")
2811 (parallel [(const_int 0)]))))]
2812 "TARGET_SSE2 && TARGET_64BIT"
2813 "%vcvttsd2si{q}\t{%1, %0|%0, %1}"
2814 [(set_attr "type" "sseicvt")
2815 (set_attr "athlon_decode" "double,vector")
2816 (set_attr "amdfam10_decode" "double,double")
2817 (set_attr "bdver1_decode" "double,double")
2818 (set_attr "prefix_rep" "1")
2819 (set_attr "prefix" "maybe_vex")
2820 (set_attr "mode" "DI")])
2822 (define_insn "floatv4siv4df2"
2823 [(set (match_operand:V4DF 0 "register_operand" "=x")
2824 (float:V4DF (match_operand:V4SI 1 "nonimmediate_operand" "xm")))]
2826 "vcvtdq2pd\t{%1, %0|%0, %1}"
2827 [(set_attr "type" "ssecvt")
2828 (set_attr "prefix" "vex")
2829 (set_attr "mode" "V4DF")])
2831 (define_insn "avx_cvtdq2pd256_2"
2832 [(set (match_operand:V4DF 0 "register_operand" "=x")
2835 (match_operand:V8SI 1 "nonimmediate_operand" "xm")
2836 (parallel [(const_int 0) (const_int 1)
2837 (const_int 2) (const_int 3)]))))]
2839 "vcvtdq2pd\t{%x1, %0|%0, %x1}"
2840 [(set_attr "type" "ssecvt")
2841 (set_attr "prefix" "vex")
2842 (set_attr "mode" "V4DF")])
2844 (define_insn "sse2_cvtdq2pd"
2845 [(set (match_operand:V2DF 0 "register_operand" "=x")
2848 (match_operand:V4SI 1 "nonimmediate_operand" "xm")
2849 (parallel [(const_int 0) (const_int 1)]))))]
2851 "%vcvtdq2pd\t{%1, %0|%0, %q1}"
2852 [(set_attr "type" "ssecvt")
2853 (set_attr "prefix" "maybe_vex")
2854 (set_attr "mode" "V2DF")])
2856 (define_insn "avx_cvtpd2dq256"
2857 [(set (match_operand:V4SI 0 "register_operand" "=x")
2858 (unspec:V4SI [(match_operand:V4DF 1 "nonimmediate_operand" "xm")]
2859 UNSPEC_FIX_NOTRUNC))]
2861 "vcvtpd2dq{y}\t{%1, %0|%0, %1}"
2862 [(set_attr "type" "ssecvt")
2863 (set_attr "prefix" "vex")
2864 (set_attr "mode" "OI")])
2866 (define_expand "avx_cvtpd2dq256_2"
2867 [(set (match_operand:V8SI 0 "register_operand")
2869 (unspec:V4SI [(match_operand:V4DF 1 "nonimmediate_operand")]
2873 "operands[2] = CONST0_RTX (V4SImode);")
2875 (define_insn "*avx_cvtpd2dq256_2"
2876 [(set (match_operand:V8SI 0 "register_operand" "=x")
2878 (unspec:V4SI [(match_operand:V4DF 1 "nonimmediate_operand" "xm")]
2880 (match_operand:V4SI 2 "const0_operand")))]
2882 "vcvtpd2dq{y}\t{%1, %x0|%x0, %1}"
2883 [(set_attr "type" "ssecvt")
2884 (set_attr "prefix" "vex")
2885 (set_attr "btver2_decode" "vector")
2886 (set_attr "mode" "OI")])
2888 (define_expand "sse2_cvtpd2dq"
2889 [(set (match_operand:V4SI 0 "register_operand")
2891 (unspec:V2SI [(match_operand:V2DF 1 "nonimmediate_operand")]
2895 "operands[2] = CONST0_RTX (V2SImode);")
2897 (define_insn "*sse2_cvtpd2dq"
2898 [(set (match_operand:V4SI 0 "register_operand" "=x")
2900 (unspec:V2SI [(match_operand:V2DF 1 "nonimmediate_operand" "xm")]
2902 (match_operand:V2SI 2 "const0_operand")))]
2906 return "vcvtpd2dq{x}\t{%1, %0|%0, %1}";
2908 return "cvtpd2dq\t{%1, %0|%0, %1}";
2910 [(set_attr "type" "ssecvt")
2911 (set_attr "prefix_rep" "1")
2912 (set_attr "prefix_data16" "0")
2913 (set_attr "prefix" "maybe_vex")
2914 (set_attr "mode" "TI")
2915 (set_attr "amdfam10_decode" "double")
2916 (set_attr "athlon_decode" "vector")
2917 (set_attr "bdver1_decode" "double")])
2919 (define_insn "fix_truncv4dfv4si2"
2920 [(set (match_operand:V4SI 0 "register_operand" "=x")
2921 (fix:V4SI (match_operand:V4DF 1 "nonimmediate_operand" "xm")))]
2923 "vcvttpd2dq{y}\t{%1, %0|%0, %1}"
2924 [(set_attr "type" "ssecvt")
2925 (set_attr "prefix" "vex")
2926 (set_attr "mode" "OI")])
2928 (define_expand "avx_cvttpd2dq256_2"
2929 [(set (match_operand:V8SI 0 "register_operand")
2931 (fix:V4SI (match_operand:V4DF 1 "nonimmediate_operand"))
2934 "operands[2] = CONST0_RTX (V4SImode);")
2936 (define_insn "*avx_cvttpd2dq256_2"
2937 [(set (match_operand:V8SI 0 "register_operand" "=x")
2939 (fix:V4SI (match_operand:V4DF 1 "nonimmediate_operand" "xm"))
2940 (match_operand:V4SI 2 "const0_operand")))]
2942 "vcvttpd2dq{y}\t{%1, %x0|%x0, %1}"
2943 [(set_attr "type" "ssecvt")
2944 (set_attr "prefix" "vex")
2945 (set_attr "btver2_decode" "vector")
2946 (set_attr "mode" "OI")])
2948 (define_expand "sse2_cvttpd2dq"
2949 [(set (match_operand:V4SI 0 "register_operand")
2951 (fix:V2SI (match_operand:V2DF 1 "nonimmediate_operand"))
2954 "operands[2] = CONST0_RTX (V2SImode);")
2956 (define_insn "*sse2_cvttpd2dq"
2957 [(set (match_operand:V4SI 0 "register_operand" "=x")
2959 (fix:V2SI (match_operand:V2DF 1 "nonimmediate_operand" "xm"))
2960 (match_operand:V2SI 2 "const0_operand")))]
2964 return "vcvttpd2dq{x}\t{%1, %0|%0, %1}";
2966 return "cvttpd2dq\t{%1, %0|%0, %1}";
2968 [(set_attr "type" "ssecvt")
2969 (set_attr "amdfam10_decode" "double")
2970 (set_attr "athlon_decode" "vector")
2971 (set_attr "bdver1_decode" "double")
2972 (set_attr "prefix" "maybe_vex")
2973 (set_attr "mode" "TI")])
2975 (define_insn "sse2_cvtsd2ss"
2976 [(set (match_operand:V4SF 0 "register_operand" "=x,x,x")
2979 (float_truncate:V2SF
2980 (match_operand:V2DF 2 "nonimmediate_operand" "x,m,xm")))
2981 (match_operand:V4SF 1 "register_operand" "0,0,x")
2985 cvtsd2ss\t{%2, %0|%0, %2}
2986 cvtsd2ss\t{%2, %0|%0, %2}
2987 vcvtsd2ss\t{%2, %1, %0|%0, %1, %2}"
2988 [(set_attr "isa" "noavx,noavx,avx")
2989 (set_attr "type" "ssecvt")
2990 (set_attr "athlon_decode" "vector,double,*")
2991 (set_attr "amdfam10_decode" "vector,double,*")
2992 (set_attr "bdver1_decode" "direct,direct,*")
2993 (set_attr "btver2_decode" "double,double,double")
2994 (set_attr "prefix" "orig,orig,vex")
2995 (set_attr "mode" "SF")])
2997 (define_insn "sse2_cvtss2sd"
2998 [(set (match_operand:V2DF 0 "register_operand" "=x,x,x")
3002 (match_operand:V4SF 2 "nonimmediate_operand" "x,m,xm")
3003 (parallel [(const_int 0) (const_int 1)])))
3004 (match_operand:V2DF 1 "register_operand" "0,0,x")
3008 cvtss2sd\t{%2, %0|%0, %2}
3009 cvtss2sd\t{%2, %0|%0, %2}
3010 vcvtss2sd\t{%2, %1, %0|%0, %1, %2}"
3011 [(set_attr "isa" "noavx,noavx,avx")
3012 (set_attr "type" "ssecvt")
3013 (set_attr "amdfam10_decode" "vector,double,*")
3014 (set_attr "athlon_decode" "direct,direct,*")
3015 (set_attr "bdver1_decode" "direct,direct,*")
3016 (set_attr "btver2_decode" "double,double,double")
3017 (set_attr "prefix" "orig,orig,vex")
3018 (set_attr "mode" "DF")])
3020 (define_insn "avx_cvtpd2ps256"
3021 [(set (match_operand:V4SF 0 "register_operand" "=x")
3022 (float_truncate:V4SF
3023 (match_operand:V4DF 1 "nonimmediate_operand" "xm")))]
3025 "vcvtpd2ps{y}\t{%1, %0|%0, %1}"
3026 [(set_attr "type" "ssecvt")
3027 (set_attr "prefix" "vex")
3028 (set_attr "btver2_decode" "vector")
3029 (set_attr "mode" "V4SF")])
3031 (define_expand "sse2_cvtpd2ps"
3032 [(set (match_operand:V4SF 0 "register_operand")
3034 (float_truncate:V2SF
3035 (match_operand:V2DF 1 "nonimmediate_operand"))
3038 "operands[2] = CONST0_RTX (V2SFmode);")
3040 (define_insn "*sse2_cvtpd2ps"
3041 [(set (match_operand:V4SF 0 "register_operand" "=x")
3043 (float_truncate:V2SF
3044 (match_operand:V2DF 1 "nonimmediate_operand" "xm"))
3045 (match_operand:V2SF 2 "const0_operand")))]
3049 return "vcvtpd2ps{x}\t{%1, %0|%0, %1}";
3051 return "cvtpd2ps\t{%1, %0|%0, %1}";
3053 [(set_attr "type" "ssecvt")
3054 (set_attr "amdfam10_decode" "double")
3055 (set_attr "athlon_decode" "vector")
3056 (set_attr "bdver1_decode" "double")
3057 (set_attr "prefix_data16" "1")
3058 (set_attr "prefix" "maybe_vex")
3059 (set_attr "mode" "V4SF")])
3061 (define_insn "avx_cvtps2pd256"
3062 [(set (match_operand:V4DF 0 "register_operand" "=x")
3064 (match_operand:V4SF 1 "nonimmediate_operand" "xm")))]
3066 "vcvtps2pd\t{%1, %0|%0, %1}"
3067 [(set_attr "type" "ssecvt")
3068 (set_attr "prefix" "vex")
3069 (set_attr "mode" "V4DF")])
3071 (define_insn "*avx_cvtps2pd256_2"
3072 [(set (match_operand:V4DF 0 "register_operand" "=x")
3075 (match_operand:V8SF 1 "nonimmediate_operand" "xm")
3076 (parallel [(const_int 0) (const_int 1)
3077 (const_int 2) (const_int 3)]))))]
3079 "vcvtps2pd\t{%x1, %0|%0, %x1}"
3080 [(set_attr "type" "ssecvt")
3081 (set_attr "prefix" "vex")
3082 (set_attr "mode" "V4DF")])
3084 (define_insn "sse2_cvtps2pd"
3085 [(set (match_operand:V2DF 0 "register_operand" "=x")
3088 (match_operand:V4SF 1 "nonimmediate_operand" "xm")
3089 (parallel [(const_int 0) (const_int 1)]))))]
3091 "%vcvtps2pd\t{%1, %0|%0, %q1}"
3092 [(set_attr "type" "ssecvt")
3093 (set_attr "amdfam10_decode" "direct")
3094 (set_attr "athlon_decode" "double")
3095 (set_attr "bdver1_decode" "double")
3096 (set_attr "prefix_data16" "0")
3097 (set_attr "prefix" "maybe_vex")
3098 (set_attr "mode" "V2DF")])
3100 (define_expand "vec_unpacks_hi_v4sf"
3105 (match_operand:V4SF 1 "nonimmediate_operand"))
3106 (parallel [(const_int 6) (const_int 7)
3107 (const_int 2) (const_int 3)])))
3108 (set (match_operand:V2DF 0 "register_operand")
3112 (parallel [(const_int 0) (const_int 1)]))))]
3114 "operands[2] = gen_reg_rtx (V4SFmode);")
3116 (define_expand "vec_unpacks_hi_v8sf"
3119 (match_operand:V8SF 1 "nonimmediate_operand")
3120 (parallel [(const_int 4) (const_int 5)
3121 (const_int 6) (const_int 7)])))
3122 (set (match_operand:V4DF 0 "register_operand")
3126 "operands[2] = gen_reg_rtx (V4SFmode);")
3128 (define_expand "vec_unpacks_lo_v4sf"
3129 [(set (match_operand:V2DF 0 "register_operand")
3132 (match_operand:V4SF 1 "nonimmediate_operand")
3133 (parallel [(const_int 0) (const_int 1)]))))]
3136 (define_expand "vec_unpacks_lo_v8sf"
3137 [(set (match_operand:V4DF 0 "register_operand")
3140 (match_operand:V8SF 1 "nonimmediate_operand")
3141 (parallel [(const_int 0) (const_int 1)
3142 (const_int 2) (const_int 3)]))))]
3145 (define_mode_attr sseunpackfltmode
3146 [(V8HI "V4SF") (V4SI "V2DF") (V16HI "V8SF") (V8SI "V4DF")])
3148 (define_expand "vec_unpacks_float_hi_<mode>"
3149 [(match_operand:<sseunpackfltmode> 0 "register_operand")
3150 (match_operand:VI2_AVX2 1 "register_operand")]
3153 rtx tmp = gen_reg_rtx (<sseunpackmode>mode);
3155 emit_insn (gen_vec_unpacks_hi_<mode> (tmp, operands[1]));
3156 emit_insn (gen_rtx_SET (VOIDmode, operands[0],
3157 gen_rtx_FLOAT (<sseunpackfltmode>mode, tmp)));
3161 (define_expand "vec_unpacks_float_lo_<mode>"
3162 [(match_operand:<sseunpackfltmode> 0 "register_operand")
3163 (match_operand:VI2_AVX2 1 "register_operand")]
3166 rtx tmp = gen_reg_rtx (<sseunpackmode>mode);
3168 emit_insn (gen_vec_unpacks_lo_<mode> (tmp, operands[1]));
3169 emit_insn (gen_rtx_SET (VOIDmode, operands[0],
3170 gen_rtx_FLOAT (<sseunpackfltmode>mode, tmp)));
3174 (define_expand "vec_unpacku_float_hi_<mode>"
3175 [(match_operand:<sseunpackfltmode> 0 "register_operand")
3176 (match_operand:VI2_AVX2 1 "register_operand")]
3179 rtx tmp = gen_reg_rtx (<sseunpackmode>mode);
3181 emit_insn (gen_vec_unpacku_hi_<mode> (tmp, operands[1]));
3182 emit_insn (gen_rtx_SET (VOIDmode, operands[0],
3183 gen_rtx_FLOAT (<sseunpackfltmode>mode, tmp)));
3187 (define_expand "vec_unpacku_float_lo_<mode>"
3188 [(match_operand:<sseunpackfltmode> 0 "register_operand")
3189 (match_operand:VI2_AVX2 1 "register_operand")]
3192 rtx tmp = gen_reg_rtx (<sseunpackmode>mode);
3194 emit_insn (gen_vec_unpacku_lo_<mode> (tmp, operands[1]));
3195 emit_insn (gen_rtx_SET (VOIDmode, operands[0],
3196 gen_rtx_FLOAT (<sseunpackfltmode>mode, tmp)));
3200 (define_expand "vec_unpacks_float_hi_v4si"
3203 (match_operand:V4SI 1 "nonimmediate_operand")
3204 (parallel [(const_int 2) (const_int 3)
3205 (const_int 2) (const_int 3)])))
3206 (set (match_operand:V2DF 0 "register_operand")
3210 (parallel [(const_int 0) (const_int 1)]))))]
3212 "operands[2] = gen_reg_rtx (V4SImode);")
3214 (define_expand "vec_unpacks_float_lo_v4si"
3215 [(set (match_operand:V2DF 0 "register_operand")
3218 (match_operand:V4SI 1 "nonimmediate_operand")
3219 (parallel [(const_int 0) (const_int 1)]))))]
3222 (define_expand "vec_unpacks_float_hi_v8si"
3225 (match_operand:V8SI 1 "nonimmediate_operand")
3226 (parallel [(const_int 4) (const_int 5)
3227 (const_int 6) (const_int 7)])))
3228 (set (match_operand:V4DF 0 "register_operand")
3232 "operands[2] = gen_reg_rtx (V4SImode);")
3234 (define_expand "vec_unpacks_float_lo_v8si"
3235 [(set (match_operand:V4DF 0 "register_operand")
3238 (match_operand:V8SI 1 "nonimmediate_operand")
3239 (parallel [(const_int 0) (const_int 1)
3240 (const_int 2) (const_int 3)]))))]
3243 (define_expand "vec_unpacku_float_hi_v4si"
3246 (match_operand:V4SI 1 "nonimmediate_operand")
3247 (parallel [(const_int 2) (const_int 3)
3248 (const_int 2) (const_int 3)])))
3253 (parallel [(const_int 0) (const_int 1)]))))
3255 (lt:V2DF (match_dup 6) (match_dup 3)))
3257 (and:V2DF (match_dup 7) (match_dup 4)))
3258 (set (match_operand:V2DF 0 "register_operand")
3259 (plus:V2DF (match_dup 6) (match_dup 8)))]
3262 REAL_VALUE_TYPE TWO32r;
3266 real_ldexp (&TWO32r, &dconst1, 32);
3267 x = const_double_from_real_value (TWO32r, DFmode);
3269 operands[3] = force_reg (V2DFmode, CONST0_RTX (V2DFmode));
3270 operands[4] = force_reg (V2DFmode,
3271 ix86_build_const_vector (V2DFmode, 1, x));
3273 operands[5] = gen_reg_rtx (V4SImode);
3275 for (i = 6; i < 9; i++)
3276 operands[i] = gen_reg_rtx (V2DFmode);
3279 (define_expand "vec_unpacku_float_lo_v4si"
3283 (match_operand:V4SI 1 "nonimmediate_operand")
3284 (parallel [(const_int 0) (const_int 1)]))))
3286 (lt:V2DF (match_dup 5) (match_dup 3)))
3288 (and:V2DF (match_dup 6) (match_dup 4)))
3289 (set (match_operand:V2DF 0 "register_operand")
3290 (plus:V2DF (match_dup 5) (match_dup 7)))]
3293 REAL_VALUE_TYPE TWO32r;
3297 real_ldexp (&TWO32r, &dconst1, 32);
3298 x = const_double_from_real_value (TWO32r, DFmode);
3300 operands[3] = force_reg (V2DFmode, CONST0_RTX (V2DFmode));
3301 operands[4] = force_reg (V2DFmode,
3302 ix86_build_const_vector (V2DFmode, 1, x));
3304 for (i = 5; i < 8; i++)
3305 operands[i] = gen_reg_rtx (V2DFmode);
3308 (define_expand "vec_unpacku_float_hi_v8si"
3309 [(match_operand:V4DF 0 "register_operand")
3310 (match_operand:V8SI 1 "register_operand")]
3313 REAL_VALUE_TYPE TWO32r;
3317 real_ldexp (&TWO32r, &dconst1, 32);
3318 x = const_double_from_real_value (TWO32r, DFmode);
3320 tmp[0] = force_reg (V4DFmode, CONST0_RTX (V4DFmode));
3321 tmp[1] = force_reg (V4DFmode, ix86_build_const_vector (V4DFmode, 1, x));
3322 tmp[5] = gen_reg_rtx (V4SImode);
3324 for (i = 2; i < 5; i++)
3325 tmp[i] = gen_reg_rtx (V4DFmode);
3326 emit_insn (gen_vec_extract_hi_v8si (tmp[5], operands[1]));
3327 emit_insn (gen_floatv4siv4df2 (tmp[2], tmp[5]));
3328 emit_insn (gen_rtx_SET (VOIDmode, tmp[3],
3329 gen_rtx_LT (V4DFmode, tmp[2], tmp[0])));
3330 emit_insn (gen_andv4df3 (tmp[4], tmp[3], tmp[1]));
3331 emit_insn (gen_addv4df3 (operands[0], tmp[2], tmp[4]));
3335 (define_expand "vec_unpacku_float_lo_v8si"
3336 [(match_operand:V4DF 0 "register_operand")
3337 (match_operand:V8SI 1 "nonimmediate_operand")]
3340 REAL_VALUE_TYPE TWO32r;
3344 real_ldexp (&TWO32r, &dconst1, 32);
3345 x = const_double_from_real_value (TWO32r, DFmode);
3347 tmp[0] = force_reg (V4DFmode, CONST0_RTX (V4DFmode));
3348 tmp[1] = force_reg (V4DFmode, ix86_build_const_vector (V4DFmode, 1, x));
3350 for (i = 2; i < 5; i++)
3351 tmp[i] = gen_reg_rtx (V4DFmode);
3352 emit_insn (gen_avx_cvtdq2pd256_2 (tmp[2], operands[1]));
3353 emit_insn (gen_rtx_SET (VOIDmode, tmp[3],
3354 gen_rtx_LT (V4DFmode, tmp[2], tmp[0])));
3355 emit_insn (gen_andv4df3 (tmp[4], tmp[3], tmp[1]));
3356 emit_insn (gen_addv4df3 (operands[0], tmp[2], tmp[4]));
3360 (define_expand "vec_pack_trunc_v4df"
3362 (float_truncate:V4SF
3363 (match_operand:V4DF 1 "nonimmediate_operand")))
3365 (float_truncate:V4SF
3366 (match_operand:V4DF 2 "nonimmediate_operand")))
3367 (set (match_operand:V8SF 0 "register_operand")
3373 operands[3] = gen_reg_rtx (V4SFmode);
3374 operands[4] = gen_reg_rtx (V4SFmode);
3377 (define_expand "vec_pack_trunc_v2df"
3378 [(match_operand:V4SF 0 "register_operand")
3379 (match_operand:V2DF 1 "nonimmediate_operand")
3380 (match_operand:V2DF 2 "nonimmediate_operand")]
3385 if (TARGET_AVX && !TARGET_PREFER_AVX128)
3387 tmp0 = gen_reg_rtx (V4DFmode);
3388 tmp1 = force_reg (V2DFmode, operands[1]);
3390 emit_insn (gen_avx_vec_concatv4df (tmp0, tmp1, operands[2]));
3391 emit_insn (gen_avx_cvtpd2ps256 (operands[0], tmp0));
3395 tmp0 = gen_reg_rtx (V4SFmode);
3396 tmp1 = gen_reg_rtx (V4SFmode);
3398 emit_insn (gen_sse2_cvtpd2ps (tmp0, operands[1]));
3399 emit_insn (gen_sse2_cvtpd2ps (tmp1, operands[2]));
3400 emit_insn (gen_sse_movlhps (operands[0], tmp0, tmp1));
3405 (define_expand "vec_pack_sfix_trunc_v4df"
3406 [(match_operand:V8SI 0 "register_operand")
3407 (match_operand:V4DF 1 "nonimmediate_operand")
3408 (match_operand:V4DF 2 "nonimmediate_operand")]
3413 r1 = gen_reg_rtx (V4SImode);
3414 r2 = gen_reg_rtx (V4SImode);
3416 emit_insn (gen_fix_truncv4dfv4si2 (r1, operands[1]));
3417 emit_insn (gen_fix_truncv4dfv4si2 (r2, operands[2]));
3418 emit_insn (gen_avx_vec_concatv8si (operands[0], r1, r2));
3422 (define_expand "vec_pack_sfix_trunc_v2df"
3423 [(match_operand:V4SI 0 "register_operand")
3424 (match_operand:V2DF 1 "nonimmediate_operand")
3425 (match_operand:V2DF 2 "nonimmediate_operand")]
3430 if (TARGET_AVX && !TARGET_PREFER_AVX128)
3432 tmp0 = gen_reg_rtx (V4DFmode);
3433 tmp1 = force_reg (V2DFmode, operands[1]);
3435 emit_insn (gen_avx_vec_concatv4df (tmp0, tmp1, operands[2]));
3436 emit_insn (gen_fix_truncv4dfv4si2 (operands[0], tmp0));
3440 tmp0 = gen_reg_rtx (V4SImode);
3441 tmp1 = gen_reg_rtx (V4SImode);
3443 emit_insn (gen_sse2_cvttpd2dq (tmp0, operands[1]));
3444 emit_insn (gen_sse2_cvttpd2dq (tmp1, operands[2]));
3446 (gen_vec_interleave_lowv2di (gen_lowpart (V2DImode, operands[0]),
3447 gen_lowpart (V2DImode, tmp0),
3448 gen_lowpart (V2DImode, tmp1)));
3453 (define_mode_attr ssepackfltmode
3454 [(V4DF "V8SI") (V2DF "V4SI")])
3456 (define_expand "vec_pack_ufix_trunc_<mode>"
3457 [(match_operand:<ssepackfltmode> 0 "register_operand")
3458 (match_operand:VF2 1 "register_operand")
3459 (match_operand:VF2 2 "register_operand")]
3463 tmp[0] = ix86_expand_adjust_ufix_to_sfix_si (operands[1], &tmp[2]);
3464 tmp[1] = ix86_expand_adjust_ufix_to_sfix_si (operands[2], &tmp[3]);
3465 tmp[4] = gen_reg_rtx (<ssepackfltmode>mode);
3466 emit_insn (gen_vec_pack_sfix_trunc_<mode> (tmp[4], tmp[0], tmp[1]));
3467 if (<ssepackfltmode>mode == V4SImode || TARGET_AVX2)
3469 tmp[5] = gen_reg_rtx (<ssepackfltmode>mode);
3470 ix86_expand_vec_extract_even_odd (tmp[5], tmp[2], tmp[3], 0);
3474 tmp[5] = gen_reg_rtx (V8SFmode);
3475 ix86_expand_vec_extract_even_odd (tmp[5], gen_lowpart (V8SFmode, tmp[2]),
3476 gen_lowpart (V8SFmode, tmp[3]), 0);
3477 tmp[5] = gen_lowpart (V8SImode, tmp[5]);
3479 tmp[6] = expand_simple_binop (<ssepackfltmode>mode, XOR, tmp[4], tmp[5],
3480 operands[0], 0, OPTAB_DIRECT);
3481 if (tmp[6] != operands[0])
3482 emit_move_insn (operands[0], tmp[6]);
3486 (define_expand "vec_pack_sfix_v4df"
3487 [(match_operand:V8SI 0 "register_operand")
3488 (match_operand:V4DF 1 "nonimmediate_operand")
3489 (match_operand:V4DF 2 "nonimmediate_operand")]
3494 r1 = gen_reg_rtx (V4SImode);
3495 r2 = gen_reg_rtx (V4SImode);
3497 emit_insn (gen_avx_cvtpd2dq256 (r1, operands[1]));
3498 emit_insn (gen_avx_cvtpd2dq256 (r2, operands[2]));
3499 emit_insn (gen_avx_vec_concatv8si (operands[0], r1, r2));
3503 (define_expand "vec_pack_sfix_v2df"
3504 [(match_operand:V4SI 0 "register_operand")
3505 (match_operand:V2DF 1 "nonimmediate_operand")
3506 (match_operand:V2DF 2 "nonimmediate_operand")]
3511 if (TARGET_AVX && !TARGET_PREFER_AVX128)
3513 tmp0 = gen_reg_rtx (V4DFmode);
3514 tmp1 = force_reg (V2DFmode, operands[1]);
3516 emit_insn (gen_avx_vec_concatv4df (tmp0, tmp1, operands[2]));
3517 emit_insn (gen_avx_cvtpd2dq256 (operands[0], tmp0));
3521 tmp0 = gen_reg_rtx (V4SImode);
3522 tmp1 = gen_reg_rtx (V4SImode);
3524 emit_insn (gen_sse2_cvtpd2dq (tmp0, operands[1]));
3525 emit_insn (gen_sse2_cvtpd2dq (tmp1, operands[2]));
3527 (gen_vec_interleave_lowv2di (gen_lowpart (V2DImode, operands[0]),
3528 gen_lowpart (V2DImode, tmp0),
3529 gen_lowpart (V2DImode, tmp1)));
3534 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
3536 ;; Parallel single-precision floating point element swizzling
3538 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
3540 (define_expand "sse_movhlps_exp"
3541 [(set (match_operand:V4SF 0 "nonimmediate_operand")
3544 (match_operand:V4SF 1 "nonimmediate_operand")
3545 (match_operand:V4SF 2 "nonimmediate_operand"))
3546 (parallel [(const_int 6)
3552 rtx dst = ix86_fixup_binary_operands (UNKNOWN, V4SFmode, operands);
3554 emit_insn (gen_sse_movhlps (dst, operands[1], operands[2]));
3556 /* Fix up the destination if needed. */
3557 if (dst != operands[0])
3558 emit_move_insn (operands[0], dst);
3563 (define_insn "sse_movhlps"
3564 [(set (match_operand:V4SF 0 "nonimmediate_operand" "=x,x,x,x,m")
3567 (match_operand:V4SF 1 "nonimmediate_operand" " 0,x,0,x,0")
3568 (match_operand:V4SF 2 "nonimmediate_operand" " x,x,o,o,x"))
3569 (parallel [(const_int 6)
3573 "TARGET_SSE && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
3575 movhlps\t{%2, %0|%0, %2}
3576 vmovhlps\t{%2, %1, %0|%0, %1, %2}
3577 movlps\t{%H2, %0|%0, %H2}
3578 vmovlps\t{%H2, %1, %0|%0, %1, %H2}
3579 %vmovhps\t{%2, %0|%0, %2}"
3580 [(set_attr "isa" "noavx,avx,noavx,avx,*")
3581 (set_attr "type" "ssemov")
3582 (set_attr "prefix" "orig,vex,orig,vex,maybe_vex")
3583 (set_attr "mode" "V4SF,V4SF,V2SF,V2SF,V2SF")])
3585 (define_expand "sse_movlhps_exp"
3586 [(set (match_operand:V4SF 0 "nonimmediate_operand")
3589 (match_operand:V4SF 1 "nonimmediate_operand")
3590 (match_operand:V4SF 2 "nonimmediate_operand"))
3591 (parallel [(const_int 0)
3597 rtx dst = ix86_fixup_binary_operands (UNKNOWN, V4SFmode, operands);
3599 emit_insn (gen_sse_movlhps (dst, operands[1], operands[2]));
3601 /* Fix up the destination if needed. */
3602 if (dst != operands[0])
3603 emit_move_insn (operands[0], dst);
3608 (define_insn "sse_movlhps"
3609 [(set (match_operand:V4SF 0 "nonimmediate_operand" "=x,x,x,x,o")
3612 (match_operand:V4SF 1 "nonimmediate_operand" " 0,x,0,x,0")
3613 (match_operand:V4SF 2 "nonimmediate_operand" " x,x,m,x,x"))
3614 (parallel [(const_int 0)
3618 "TARGET_SSE && ix86_binary_operator_ok (UNKNOWN, V4SFmode, operands)"
3620 movlhps\t{%2, %0|%0, %2}
3621 vmovlhps\t{%2, %1, %0|%0, %1, %2}
3622 movhps\t{%2, %0|%0, %2}
3623 vmovhps\t{%2, %1, %0|%0, %1, %2}
3624 %vmovlps\t{%2, %H0|%H0, %2}"
3625 [(set_attr "isa" "noavx,avx,noavx,avx,*")
3626 (set_attr "type" "ssemov")
3627 (set_attr "prefix" "orig,vex,orig,vex,maybe_vex")
3628 (set_attr "mode" "V4SF,V4SF,V2SF,V2SF,V2SF")])
3630 ;; Recall that the 256-bit unpck insns only shuffle within their lanes.
3631 (define_insn "avx_unpckhps256"
3632 [(set (match_operand:V8SF 0 "register_operand" "=x")
3635 (match_operand:V8SF 1 "register_operand" "x")
3636 (match_operand:V8SF 2 "nonimmediate_operand" "xm"))
3637 (parallel [(const_int 2) (const_int 10)
3638 (const_int 3) (const_int 11)
3639 (const_int 6) (const_int 14)
3640 (const_int 7) (const_int 15)])))]
3642 "vunpckhps\t{%2, %1, %0|%0, %1, %2}"
3643 [(set_attr "type" "sselog")
3644 (set_attr "prefix" "vex")
3645 (set_attr "mode" "V8SF")])
3647 (define_expand "vec_interleave_highv8sf"
3651 (match_operand:V8SF 1 "register_operand" "x")
3652 (match_operand:V8SF 2 "nonimmediate_operand" "xm"))
3653 (parallel [(const_int 0) (const_int 8)
3654 (const_int 1) (const_int 9)
3655 (const_int 4) (const_int 12)
3656 (const_int 5) (const_int 13)])))
3662 (parallel [(const_int 2) (const_int 10)
3663 (const_int 3) (const_int 11)
3664 (const_int 6) (const_int 14)
3665 (const_int 7) (const_int 15)])))
3666 (set (match_operand:V8SF 0 "register_operand")
3671 (parallel [(const_int 4) (const_int 5)
3672 (const_int 6) (const_int 7)
3673 (const_int 12) (const_int 13)
3674 (const_int 14) (const_int 15)])))]
3677 operands[3] = gen_reg_rtx (V8SFmode);
3678 operands[4] = gen_reg_rtx (V8SFmode);
3681 (define_insn "vec_interleave_highv4sf"
3682 [(set (match_operand:V4SF 0 "register_operand" "=x,x")
3685 (match_operand:V4SF 1 "register_operand" "0,x")
3686 (match_operand:V4SF 2 "nonimmediate_operand" "xm,xm"))
3687 (parallel [(const_int 2) (const_int 6)
3688 (const_int 3) (const_int 7)])))]
3691 unpckhps\t{%2, %0|%0, %2}
3692 vunpckhps\t{%2, %1, %0|%0, %1, %2}"
3693 [(set_attr "isa" "noavx,avx")
3694 (set_attr "type" "sselog")
3695 (set_attr "prefix" "orig,vex")
3696 (set_attr "mode" "V4SF")])
3698 ;; Recall that the 256-bit unpck insns only shuffle within their lanes.
3699 (define_insn "avx_unpcklps256"
3700 [(set (match_operand:V8SF 0 "register_operand" "=x")
3703 (match_operand:V8SF 1 "register_operand" "x")
3704 (match_operand:V8SF 2 "nonimmediate_operand" "xm"))
3705 (parallel [(const_int 0) (const_int 8)
3706 (const_int 1) (const_int 9)
3707 (const_int 4) (const_int 12)
3708 (const_int 5) (const_int 13)])))]
3710 "vunpcklps\t{%2, %1, %0|%0, %1, %2}"
3711 [(set_attr "type" "sselog")
3712 (set_attr "prefix" "vex")
3713 (set_attr "mode" "V8SF")])
3715 (define_expand "vec_interleave_lowv8sf"
3719 (match_operand:V8SF 1 "register_operand" "x")
3720 (match_operand:V8SF 2 "nonimmediate_operand" "xm"))
3721 (parallel [(const_int 0) (const_int 8)
3722 (const_int 1) (const_int 9)
3723 (const_int 4) (const_int 12)
3724 (const_int 5) (const_int 13)])))
3730 (parallel [(const_int 2) (const_int 10)
3731 (const_int 3) (const_int 11)
3732 (const_int 6) (const_int 14)
3733 (const_int 7) (const_int 15)])))
3734 (set (match_operand:V8SF 0 "register_operand")
3739 (parallel [(const_int 0) (const_int 1)
3740 (const_int 2) (const_int 3)
3741 (const_int 8) (const_int 9)
3742 (const_int 10) (const_int 11)])))]
3745 operands[3] = gen_reg_rtx (V8SFmode);
3746 operands[4] = gen_reg_rtx (V8SFmode);
3749 (define_insn "vec_interleave_lowv4sf"
3750 [(set (match_operand:V4SF 0 "register_operand" "=x,x")
3753 (match_operand:V4SF 1 "register_operand" "0,x")
3754 (match_operand:V4SF 2 "nonimmediate_operand" "xm,xm"))
3755 (parallel [(const_int 0) (const_int 4)
3756 (const_int 1) (const_int 5)])))]
3759 unpcklps\t{%2, %0|%0, %2}
3760 vunpcklps\t{%2, %1, %0|%0, %1, %2}"
3761 [(set_attr "isa" "noavx,avx")
3762 (set_attr "type" "sselog")
3763 (set_attr "prefix" "orig,vex")
3764 (set_attr "mode" "V4SF")])
3766 ;; These are modeled with the same vec_concat as the others so that we
3767 ;; capture users of shufps that can use the new instructions
3768 (define_insn "avx_movshdup256"
3769 [(set (match_operand:V8SF 0 "register_operand" "=x")
3772 (match_operand:V8SF 1 "nonimmediate_operand" "xm")
3774 (parallel [(const_int 1) (const_int 1)
3775 (const_int 3) (const_int 3)
3776 (const_int 5) (const_int 5)
3777 (const_int 7) (const_int 7)])))]
3779 "vmovshdup\t{%1, %0|%0, %1}"
3780 [(set_attr "type" "sse")
3781 (set_attr "prefix" "vex")
3782 (set_attr "mode" "V8SF")])
3784 (define_insn "sse3_movshdup"
3785 [(set (match_operand:V4SF 0 "register_operand" "=x")
3788 (match_operand:V4SF 1 "nonimmediate_operand" "xm")
3790 (parallel [(const_int 1)
3795 "%vmovshdup\t{%1, %0|%0, %1}"
3796 [(set_attr "type" "sse")
3797 (set_attr "prefix_rep" "1")
3798 (set_attr "prefix" "maybe_vex")
3799 (set_attr "mode" "V4SF")])
3801 (define_insn "avx_movsldup256"
3802 [(set (match_operand:V8SF 0 "register_operand" "=x")
3805 (match_operand:V8SF 1 "nonimmediate_operand" "xm")
3807 (parallel [(const_int 0) (const_int 0)
3808 (const_int 2) (const_int 2)
3809 (const_int 4) (const_int 4)
3810 (const_int 6) (const_int 6)])))]
3812 "vmovsldup\t{%1, %0|%0, %1}"
3813 [(set_attr "type" "sse")
3814 (set_attr "prefix" "vex")
3815 (set_attr "mode" "V8SF")])
3817 (define_insn "sse3_movsldup"
3818 [(set (match_operand:V4SF 0 "register_operand" "=x")
3821 (match_operand:V4SF 1 "nonimmediate_operand" "xm")
3823 (parallel [(const_int 0)
3828 "%vmovsldup\t{%1, %0|%0, %1}"
3829 [(set_attr "type" "sse")
3830 (set_attr "prefix_rep" "1")
3831 (set_attr "prefix" "maybe_vex")
3832 (set_attr "mode" "V4SF")])
3834 (define_expand "avx_shufps256"
3835 [(match_operand:V8SF 0 "register_operand")
3836 (match_operand:V8SF 1 "register_operand")
3837 (match_operand:V8SF 2 "nonimmediate_operand")
3838 (match_operand:SI 3 "const_int_operand")]
3841 int mask = INTVAL (operands[3]);
3842 emit_insn (gen_avx_shufps256_1 (operands[0], operands[1], operands[2],
3843 GEN_INT ((mask >> 0) & 3),
3844 GEN_INT ((mask >> 2) & 3),
3845 GEN_INT (((mask >> 4) & 3) + 8),
3846 GEN_INT (((mask >> 6) & 3) + 8),
3847 GEN_INT (((mask >> 0) & 3) + 4),
3848 GEN_INT (((mask >> 2) & 3) + 4),
3849 GEN_INT (((mask >> 4) & 3) + 12),
3850 GEN_INT (((mask >> 6) & 3) + 12)));
3854 ;; One bit in mask selects 2 elements.
3855 (define_insn "avx_shufps256_1"
3856 [(set (match_operand:V8SF 0 "register_operand" "=x")
3859 (match_operand:V8SF 1 "register_operand" "x")
3860 (match_operand:V8SF 2 "nonimmediate_operand" "xm"))
3861 (parallel [(match_operand 3 "const_0_to_3_operand" )
3862 (match_operand 4 "const_0_to_3_operand" )
3863 (match_operand 5 "const_8_to_11_operand" )
3864 (match_operand 6 "const_8_to_11_operand" )
3865 (match_operand 7 "const_4_to_7_operand" )
3866 (match_operand 8 "const_4_to_7_operand" )
3867 (match_operand 9 "const_12_to_15_operand")
3868 (match_operand 10 "const_12_to_15_operand")])))]
3870 && (INTVAL (operands[3]) == (INTVAL (operands[7]) - 4)
3871 && INTVAL (operands[4]) == (INTVAL (operands[8]) - 4)
3872 && INTVAL (operands[5]) == (INTVAL (operands[9]) - 4)
3873 && INTVAL (operands[6]) == (INTVAL (operands[10]) - 4))"
3876 mask = INTVAL (operands[3]);
3877 mask |= INTVAL (operands[4]) << 2;
3878 mask |= (INTVAL (operands[5]) - 8) << 4;
3879 mask |= (INTVAL (operands[6]) - 8) << 6;
3880 operands[3] = GEN_INT (mask);
3882 return "vshufps\t{%3, %2, %1, %0|%0, %1, %2, %3}";
3884 [(set_attr "type" "sseshuf")
3885 (set_attr "length_immediate" "1")
3886 (set_attr "prefix" "vex")
3887 (set_attr "mode" "V8SF")])
3889 (define_expand "sse_shufps"
3890 [(match_operand:V4SF 0 "register_operand")
3891 (match_operand:V4SF 1 "register_operand")
3892 (match_operand:V4SF 2 "nonimmediate_operand")
3893 (match_operand:SI 3 "const_int_operand")]
3896 int mask = INTVAL (operands[3]);
3897 emit_insn (gen_sse_shufps_v4sf (operands[0], operands[1], operands[2],
3898 GEN_INT ((mask >> 0) & 3),
3899 GEN_INT ((mask >> 2) & 3),
3900 GEN_INT (((mask >> 4) & 3) + 4),
3901 GEN_INT (((mask >> 6) & 3) + 4)));
3905 (define_insn "sse_shufps_<mode>"
3906 [(set (match_operand:VI4F_128 0 "register_operand" "=x,x")
3907 (vec_select:VI4F_128
3908 (vec_concat:<ssedoublevecmode>
3909 (match_operand:VI4F_128 1 "register_operand" "0,x")
3910 (match_operand:VI4F_128 2 "nonimmediate_operand" "xm,xm"))
3911 (parallel [(match_operand 3 "const_0_to_3_operand")
3912 (match_operand 4 "const_0_to_3_operand")
3913 (match_operand 5 "const_4_to_7_operand")
3914 (match_operand 6 "const_4_to_7_operand")])))]
3918 mask |= INTVAL (operands[3]) << 0;
3919 mask |= INTVAL (operands[4]) << 2;
3920 mask |= (INTVAL (operands[5]) - 4) << 4;
3921 mask |= (INTVAL (operands[6]) - 4) << 6;
3922 operands[3] = GEN_INT (mask);
3924 switch (which_alternative)
3927 return "shufps\t{%3, %2, %0|%0, %2, %3}";
3929 return "vshufps\t{%3, %2, %1, %0|%0, %1, %2, %3}";
3934 [(set_attr "isa" "noavx,avx")
3935 (set_attr "type" "sseshuf")
3936 (set_attr "length_immediate" "1")
3937 (set_attr "prefix" "orig,vex")
3938 (set_attr "mode" "V4SF")])
3940 (define_insn "sse_storehps"
3941 [(set (match_operand:V2SF 0 "nonimmediate_operand" "=m,x,x")
3943 (match_operand:V4SF 1 "nonimmediate_operand" "x,x,o")
3944 (parallel [(const_int 2) (const_int 3)])))]
3947 %vmovhps\t{%1, %0|%0, %1}
3948 %vmovhlps\t{%1, %d0|%d0, %1}
3949 %vmovlps\t{%H1, %d0|%d0, %H1}"
3950 [(set_attr "type" "ssemov")
3951 (set_attr "prefix" "maybe_vex")
3952 (set_attr "mode" "V2SF,V4SF,V2SF")])
3954 (define_expand "sse_loadhps_exp"
3955 [(set (match_operand:V4SF 0 "nonimmediate_operand")
3958 (match_operand:V4SF 1 "nonimmediate_operand")
3959 (parallel [(const_int 0) (const_int 1)]))
3960 (match_operand:V2SF 2 "nonimmediate_operand")))]
3963 rtx dst = ix86_fixup_binary_operands (UNKNOWN, V4SFmode, operands);
3965 emit_insn (gen_sse_loadhps (dst, operands[1], operands[2]));
3967 /* Fix up the destination if needed. */
3968 if (dst != operands[0])
3969 emit_move_insn (operands[0], dst);
3974 (define_insn "sse_loadhps"
3975 [(set (match_operand:V4SF 0 "nonimmediate_operand" "=x,x,x,x,o")
3978 (match_operand:V4SF 1 "nonimmediate_operand" " 0,x,0,x,0")
3979 (parallel [(const_int 0) (const_int 1)]))
3980 (match_operand:V2SF 2 "nonimmediate_operand" " m,m,x,x,x")))]
3983 movhps\t{%2, %0|%0, %2}
3984 vmovhps\t{%2, %1, %0|%0, %1, %2}
3985 movlhps\t{%2, %0|%0, %2}
3986 vmovlhps\t{%2, %1, %0|%0, %1, %2}
3987 %vmovlps\t{%2, %H0|%H0, %2}"
3988 [(set_attr "isa" "noavx,avx,noavx,avx,*")
3989 (set_attr "type" "ssemov")
3990 (set_attr "prefix" "orig,vex,orig,vex,maybe_vex")
3991 (set_attr "mode" "V2SF,V2SF,V4SF,V4SF,V2SF")])
3993 (define_insn "sse_storelps"
3994 [(set (match_operand:V2SF 0 "nonimmediate_operand" "=m,x,x")
3996 (match_operand:V4SF 1 "nonimmediate_operand" " x,x,m")
3997 (parallel [(const_int 0) (const_int 1)])))]
4000 %vmovlps\t{%1, %0|%0, %1}
4001 %vmovaps\t{%1, %0|%0, %1}
4002 %vmovlps\t{%1, %d0|%d0, %1}"
4003 [(set_attr "type" "ssemov")
4004 (set_attr "prefix" "maybe_vex")
4005 (set_attr "mode" "V2SF,V4SF,V2SF")])
4007 (define_expand "sse_loadlps_exp"
4008 [(set (match_operand:V4SF 0 "nonimmediate_operand")
4010 (match_operand:V2SF 2 "nonimmediate_operand")
4012 (match_operand:V4SF 1 "nonimmediate_operand")
4013 (parallel [(const_int 2) (const_int 3)]))))]
4016 rtx dst = ix86_fixup_binary_operands (UNKNOWN, V4SFmode, operands);
4018 emit_insn (gen_sse_loadlps (dst, operands[1], operands[2]));
4020 /* Fix up the destination if needed. */
4021 if (dst != operands[0])
4022 emit_move_insn (operands[0], dst);
4027 (define_insn "sse_loadlps"
4028 [(set (match_operand:V4SF 0 "nonimmediate_operand" "=x,x,x,x,m")
4030 (match_operand:V2SF 2 "nonimmediate_operand" " 0,x,m,m,x")
4032 (match_operand:V4SF 1 "nonimmediate_operand" " x,x,0,x,0")
4033 (parallel [(const_int 2) (const_int 3)]))))]
4036 shufps\t{$0xe4, %1, %0|%0, %1, 0xe4}
4037 vshufps\t{$0xe4, %1, %2, %0|%0, %2, %1, 0xe4}
4038 movlps\t{%2, %0|%0, %2}
4039 vmovlps\t{%2, %1, %0|%0, %1, %2}
4040 %vmovlps\t{%2, %0|%0, %2}"
4041 [(set_attr "isa" "noavx,avx,noavx,avx,*")
4042 (set_attr "type" "sseshuf,sseshuf,ssemov,ssemov,ssemov")
4043 (set_attr "length_immediate" "1,1,*,*,*")
4044 (set_attr "prefix" "orig,vex,orig,vex,maybe_vex")
4045 (set_attr "mode" "V4SF,V4SF,V2SF,V2SF,V2SF")])
4047 (define_insn "sse_movss"
4048 [(set (match_operand:V4SF 0 "register_operand" "=x,x")
4050 (match_operand:V4SF 2 "register_operand" " x,x")
4051 (match_operand:V4SF 1 "register_operand" " 0,x")
4055 movss\t{%2, %0|%0, %2}
4056 vmovss\t{%2, %1, %0|%0, %1, %2}"
4057 [(set_attr "isa" "noavx,avx")
4058 (set_attr "type" "ssemov")
4059 (set_attr "prefix" "orig,vex")
4060 (set_attr "mode" "SF")])
4062 (define_insn "avx2_vec_dup<mode>"
4063 [(set (match_operand:VF1 0 "register_operand" "=x")
4066 (match_operand:V4SF 1 "register_operand" "x")
4067 (parallel [(const_int 0)]))))]
4069 "vbroadcastss\t{%1, %0|%0, %1}"
4070 [(set_attr "type" "sselog1")
4071 (set_attr "prefix" "vex")
4072 (set_attr "mode" "<MODE>")])
4074 (define_insn "avx2_vec_dupv8sf_1"
4075 [(set (match_operand:V8SF 0 "register_operand" "=x")
4078 (match_operand:V8SF 1 "register_operand" "x")
4079 (parallel [(const_int 0)]))))]
4081 "vbroadcastss\t{%x1, %0|%0, %x1}"
4082 [(set_attr "type" "sselog1")
4083 (set_attr "prefix" "vex")
4084 (set_attr "mode" "V8SF")])
4086 (define_insn "vec_dupv4sf"
4087 [(set (match_operand:V4SF 0 "register_operand" "=x,x,x")
4089 (match_operand:SF 1 "nonimmediate_operand" "x,m,0")))]
4092 vshufps\t{$0, %1, %1, %0|%0, %1, %1, 0}
4093 vbroadcastss\t{%1, %0|%0, %1}
4094 shufps\t{$0, %0, %0|%0, %0, 0}"
4095 [(set_attr "isa" "avx,avx,noavx")
4096 (set_attr "type" "sseshuf1,ssemov,sseshuf1")
4097 (set_attr "length_immediate" "1,0,1")
4098 (set_attr "prefix_extra" "0,1,*")
4099 (set_attr "prefix" "vex,vex,orig")
4100 (set_attr "mode" "V4SF")])
4102 ;; Although insertps takes register source, we prefer
4103 ;; unpcklps with register source since it is shorter.
4104 (define_insn "*vec_concatv2sf_sse4_1"
4105 [(set (match_operand:V2SF 0 "register_operand" "=x,x,x,x,x,*y ,*y")
4107 (match_operand:SF 1 "nonimmediate_operand" " 0,x,0,x,m, 0 , m")
4108 (match_operand:SF 2 "vector_move_operand" " x,x,m,m,C,*ym, C")))]
4111 unpcklps\t{%2, %0|%0, %2}
4112 vunpcklps\t{%2, %1, %0|%0, %1, %2}
4113 insertps\t{$0x10, %2, %0|%0, %2, 0x10}
4114 vinsertps\t{$0x10, %2, %1, %0|%0, %1, %2, 0x10}
4115 %vmovss\t{%1, %0|%0, %1}
4116 punpckldq\t{%2, %0|%0, %2}
4117 movd\t{%1, %0|%0, %1}"
4118 [(set_attr "isa" "noavx,avx,noavx,avx,*,*,*")
4119 (set_attr "type" "sselog,sselog,sselog,sselog,ssemov,mmxcvt,mmxmov")
4120 (set_attr "prefix_data16" "*,*,1,*,*,*,*")
4121 (set_attr "prefix_extra" "*,*,1,1,*,*,*")
4122 (set_attr "length_immediate" "*,*,1,1,*,*,*")
4123 (set_attr "prefix" "orig,vex,orig,vex,maybe_vex,orig,orig")
4124 (set_attr "mode" "V4SF,V4SF,V4SF,V4SF,SF,DI,DI")])
4126 ;; ??? In theory we can match memory for the MMX alternative, but allowing
4127 ;; nonimmediate_operand for operand 2 and *not* allowing memory for the SSE
4128 ;; alternatives pretty much forces the MMX alternative to be chosen.
4129 (define_insn "*vec_concatv2sf_sse"
4130 [(set (match_operand:V2SF 0 "register_operand" "=x,x,*y,*y")
4132 (match_operand:SF 1 "nonimmediate_operand" " 0,m, 0, m")
4133 (match_operand:SF 2 "reg_or_0_operand" " x,C,*y, C")))]
4136 unpcklps\t{%2, %0|%0, %2}
4137 movss\t{%1, %0|%0, %1}
4138 punpckldq\t{%2, %0|%0, %2}
4139 movd\t{%1, %0|%0, %1}"
4140 [(set_attr "type" "sselog,ssemov,mmxcvt,mmxmov")
4141 (set_attr "mode" "V4SF,SF,DI,DI")])
4143 (define_insn "*vec_concatv4sf"
4144 [(set (match_operand:V4SF 0 "register_operand" "=x,x,x,x")
4146 (match_operand:V2SF 1 "register_operand" " 0,x,0,x")
4147 (match_operand:V2SF 2 "nonimmediate_operand" " x,x,m,m")))]
4150 movlhps\t{%2, %0|%0, %2}
4151 vmovlhps\t{%2, %1, %0|%0, %1, %2}
4152 movhps\t{%2, %0|%0, %2}
4153 vmovhps\t{%2, %1, %0|%0, %1, %2}"
4154 [(set_attr "isa" "noavx,avx,noavx,avx")
4155 (set_attr "type" "ssemov")
4156 (set_attr "prefix" "orig,vex,orig,vex")
4157 (set_attr "mode" "V4SF,V4SF,V2SF,V2SF")])
4159 (define_expand "vec_init<mode>"
4160 [(match_operand:V_128 0 "register_operand")
4164 ix86_expand_vector_init (false, operands[0], operands[1]);
4168 ;; Avoid combining registers from different units in a single alternative,
4169 ;; see comment above inline_secondary_memory_needed function in i386.c
4170 (define_insn "vec_set<mode>_0"
4171 [(set (match_operand:VI4F_128 0 "nonimmediate_operand"
4172 "=x,x,x ,x,x,x,x ,x ,m ,m ,m")
4174 (vec_duplicate:VI4F_128
4175 (match_operand:<ssescalarmode> 2 "general_operand"
4176 " x,m,*r,m,x,x,*rm,*rm,!x,!*re,!*fF"))
4177 (match_operand:VI4F_128 1 "vector_move_operand"
4178 " C,C,C ,C,0,x,0 ,x ,0 ,0 ,0")
4182 %vinsertps\t{$0xe, %d2, %0|%0, %d2, 0xe}
4183 %vmov<ssescalarmodesuffix>\t{%2, %0|%0, %2}
4184 %vmovd\t{%2, %0|%0, %2}
4185 movss\t{%2, %0|%0, %2}
4186 movss\t{%2, %0|%0, %2}
4187 vmovss\t{%2, %1, %0|%0, %1, %2}
4188 pinsrd\t{$0, %2, %0|%0, %2, 0}
4189 vpinsrd\t{$0, %2, %1, %0|%0, %1, %2, 0}
4193 [(set_attr "isa" "sse4,sse2,sse2,noavx,noavx,avx,sse4_noavx,avx,*,*,*")
4195 (cond [(eq_attr "alternative" "0,6,7")
4196 (const_string "sselog")
4197 (eq_attr "alternative" "9")
4198 (const_string "imov")
4199 (eq_attr "alternative" "10")
4200 (const_string "fmov")
4202 (const_string "ssemov")))
4203 (set_attr "prefix_extra" "*,*,*,*,*,*,1,1,*,*,*")
4204 (set_attr "length_immediate" "*,*,*,*,*,*,1,1,*,*,*")
4205 (set_attr "prefix" "maybe_vex,maybe_vex,maybe_vex,orig,orig,vex,orig,vex,*,*,*")
4206 (set_attr "mode" "SF,<ssescalarmode>,SI,SF,SF,SF,TI,TI,*,*,*")])
4208 ;; A subset is vec_setv4sf.
4209 (define_insn "*vec_setv4sf_sse4_1"
4210 [(set (match_operand:V4SF 0 "register_operand" "=x,x")
4213 (match_operand:SF 2 "nonimmediate_operand" "xm,xm"))
4214 (match_operand:V4SF 1 "register_operand" "0,x")
4215 (match_operand:SI 3 "const_int_operand")))]
4217 && ((unsigned) exact_log2 (INTVAL (operands[3]))
4218 < GET_MODE_NUNITS (V4SFmode))"
4220 operands[3] = GEN_INT (exact_log2 (INTVAL (operands[3])) << 4);
4221 switch (which_alternative)
4224 return "insertps\t{%3, %2, %0|%0, %2, %3}";
4226 return "vinsertps\t{%3, %2, %1, %0|%0, %1, %2, %3}";
4231 [(set_attr "isa" "noavx,avx")
4232 (set_attr "type" "sselog")
4233 (set_attr "prefix_data16" "1,*")
4234 (set_attr "prefix_extra" "1")
4235 (set_attr "length_immediate" "1")
4236 (set_attr "prefix" "orig,vex")
4237 (set_attr "mode" "V4SF")])
4239 (define_insn "sse4_1_insertps"
4240 [(set (match_operand:V4SF 0 "register_operand" "=x,x")
4241 (unspec:V4SF [(match_operand:V4SF 2 "nonimmediate_operand" "xm,xm")
4242 (match_operand:V4SF 1 "register_operand" "0,x")
4243 (match_operand:SI 3 "const_0_to_255_operand" "n,n")]
4247 if (MEM_P (operands[2]))
4249 unsigned count_s = INTVAL (operands[3]) >> 6;
4251 operands[3] = GEN_INT (INTVAL (operands[3]) & 0x3f);
4252 operands[2] = adjust_address_nv (operands[2], SFmode, count_s * 4);
4254 switch (which_alternative)
4257 return "insertps\t{%3, %2, %0|%0, %2, %3}";
4259 return "vinsertps\t{%3, %2, %1, %0|%0, %1, %2, %3}";
4264 [(set_attr "isa" "noavx,avx")
4265 (set_attr "type" "sselog")
4266 (set_attr "prefix_data16" "1,*")
4267 (set_attr "prefix_extra" "1")
4268 (set_attr "length_immediate" "1")
4269 (set_attr "prefix" "orig,vex")
4270 (set_attr "mode" "V4SF")])
4273 [(set (match_operand:VI4F_128 0 "memory_operand")
4275 (vec_duplicate:VI4F_128
4276 (match_operand:<ssescalarmode> 1 "nonmemory_operand"))
4279 "TARGET_SSE && reload_completed"
4282 emit_move_insn (adjust_address (operands[0], <ssescalarmode>mode, 0),
4287 (define_expand "vec_set<mode>"
4288 [(match_operand:V 0 "register_operand")
4289 (match_operand:<ssescalarmode> 1 "register_operand")
4290 (match_operand 2 "const_int_operand")]
4293 ix86_expand_vector_set (false, operands[0], operands[1],
4294 INTVAL (operands[2]));
4298 (define_insn_and_split "*vec_extractv4sf_0"
4299 [(set (match_operand:SF 0 "nonimmediate_operand" "=x,m,f,r")
4301 (match_operand:V4SF 1 "nonimmediate_operand" "xm,x,m,m")
4302 (parallel [(const_int 0)])))]
4303 "TARGET_SSE && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
4305 "&& reload_completed"
4308 rtx op1 = operands[1];
4310 op1 = gen_rtx_REG (SFmode, REGNO (op1));
4312 op1 = gen_lowpart (SFmode, op1);
4313 emit_move_insn (operands[0], op1);
4317 (define_insn_and_split "*sse4_1_extractps"
4318 [(set (match_operand:SF 0 "nonimmediate_operand" "=rm,x,x")
4320 (match_operand:V4SF 1 "register_operand" "x,0,x")
4321 (parallel [(match_operand:SI 2 "const_0_to_3_operand" "n,n,n")])))]
4324 %vextractps\t{%2, %1, %0|%0, %1, %2}
4327 "&& reload_completed && SSE_REG_P (operands[0])"
4330 rtx dest = gen_rtx_REG (V4SFmode, REGNO (operands[0]));
4331 switch (INTVAL (operands[2]))
4335 emit_insn (gen_sse_shufps_v4sf (dest, operands[1], operands[1],
4336 operands[2], operands[2],
4337 GEN_INT (INTVAL (operands[2]) + 4),
4338 GEN_INT (INTVAL (operands[2]) + 4)));
4341 emit_insn (gen_vec_interleave_highv4sf (dest, operands[1], operands[1]));
4344 /* 0 should be handled by the *vec_extractv4sf_0 pattern above. */
4349 [(set_attr "isa" "*,noavx,avx")
4350 (set_attr "type" "sselog,*,*")
4351 (set_attr "prefix_data16" "1,*,*")
4352 (set_attr "prefix_extra" "1,*,*")
4353 (set_attr "length_immediate" "1,*,*")
4354 (set_attr "prefix" "maybe_vex,*,*")
4355 (set_attr "mode" "V4SF,*,*")])
4357 (define_insn_and_split "*vec_extractv4sf_mem"
4358 [(set (match_operand:SF 0 "register_operand" "=x,*r,f")
4360 (match_operand:V4SF 1 "memory_operand" "o,o,o")
4361 (parallel [(match_operand 2 "const_0_to_3_operand" "n,n,n")])))]
4364 "&& reload_completed"
4367 int i = INTVAL (operands[2]);
4369 emit_move_insn (operands[0], adjust_address (operands[1], SFmode, i*4));
4373 (define_expand "avx_vextractf128<mode>"
4374 [(match_operand:<ssehalfvecmode> 0 "nonimmediate_operand")
4375 (match_operand:V_256 1 "register_operand")
4376 (match_operand:SI 2 "const_0_to_1_operand")]
4379 rtx (*insn)(rtx, rtx);
4381 switch (INTVAL (operands[2]))
4384 insn = gen_vec_extract_lo_<mode>;
4387 insn = gen_vec_extract_hi_<mode>;
4393 emit_insn (insn (operands[0], operands[1]));
4397 (define_insn_and_split "vec_extract_lo_<mode>"
4398 [(set (match_operand:<ssehalfvecmode> 0 "nonimmediate_operand" "=x,m")
4399 (vec_select:<ssehalfvecmode>
4400 (match_operand:VI8F_256 1 "nonimmediate_operand" "xm,x")
4401 (parallel [(const_int 0) (const_int 1)])))]
4402 "TARGET_AVX && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
4404 "&& reload_completed"
4407 rtx op1 = operands[1];
4409 op1 = gen_rtx_REG (<ssehalfvecmode>mode, REGNO (op1));
4411 op1 = gen_lowpart (<ssehalfvecmode>mode, op1);
4412 emit_move_insn (operands[0], op1);
4416 (define_insn "vec_extract_hi_<mode>"
4417 [(set (match_operand:<ssehalfvecmode> 0 "nonimmediate_operand" "=x,m")
4418 (vec_select:<ssehalfvecmode>
4419 (match_operand:VI8F_256 1 "register_operand" "x,x")
4420 (parallel [(const_int 2) (const_int 3)])))]
4422 "vextract<i128>\t{$0x1, %1, %0|%0, %1, 0x1}"
4423 [(set_attr "type" "sselog")
4424 (set_attr "prefix_extra" "1")
4425 (set_attr "length_immediate" "1")
4426 (set_attr "memory" "none,store")
4427 (set_attr "prefix" "vex")
4428 (set_attr "mode" "<sseinsnmode>")])
4430 (define_insn_and_split "vec_extract_lo_<mode>"
4431 [(set (match_operand:<ssehalfvecmode> 0 "nonimmediate_operand" "=x,m")
4432 (vec_select:<ssehalfvecmode>
4433 (match_operand:VI4F_256 1 "nonimmediate_operand" "xm,x")
4434 (parallel [(const_int 0) (const_int 1)
4435 (const_int 2) (const_int 3)])))]
4436 "TARGET_AVX && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
4438 "&& reload_completed"
4441 rtx op1 = operands[1];
4443 op1 = gen_rtx_REG (<ssehalfvecmode>mode, REGNO (op1));
4445 op1 = gen_lowpart (<ssehalfvecmode>mode, op1);
4446 emit_move_insn (operands[0], op1);
4450 (define_insn "vec_extract_hi_<mode>"
4451 [(set (match_operand:<ssehalfvecmode> 0 "nonimmediate_operand" "=x,m")
4452 (vec_select:<ssehalfvecmode>
4453 (match_operand:VI4F_256 1 "register_operand" "x,x")
4454 (parallel [(const_int 4) (const_int 5)
4455 (const_int 6) (const_int 7)])))]
4457 "vextract<i128>\t{$0x1, %1, %0|%0, %1, 0x1}"
4458 [(set_attr "type" "sselog")
4459 (set_attr "prefix_extra" "1")
4460 (set_attr "length_immediate" "1")
4461 (set_attr "memory" "none,store")
4462 (set_attr "prefix" "vex")
4463 (set_attr "mode" "<sseinsnmode>")])
4465 (define_insn_and_split "vec_extract_lo_v16hi"
4466 [(set (match_operand:V8HI 0 "nonimmediate_operand" "=x,m")
4468 (match_operand:V16HI 1 "nonimmediate_operand" "xm,x")
4469 (parallel [(const_int 0) (const_int 1)
4470 (const_int 2) (const_int 3)
4471 (const_int 4) (const_int 5)
4472 (const_int 6) (const_int 7)])))]
4473 "TARGET_AVX && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
4475 "&& reload_completed"
4478 rtx op1 = operands[1];
4480 op1 = gen_rtx_REG (V8HImode, REGNO (op1));
4482 op1 = gen_lowpart (V8HImode, op1);
4483 emit_move_insn (operands[0], op1);
4487 (define_insn "vec_extract_hi_v16hi"
4488 [(set (match_operand:V8HI 0 "nonimmediate_operand" "=x,m")
4490 (match_operand:V16HI 1 "register_operand" "x,x")
4491 (parallel [(const_int 8) (const_int 9)
4492 (const_int 10) (const_int 11)
4493 (const_int 12) (const_int 13)
4494 (const_int 14) (const_int 15)])))]
4496 "vextract%~128\t{$0x1, %1, %0|%0, %1, 0x1}"
4497 [(set_attr "type" "sselog")
4498 (set_attr "prefix_extra" "1")
4499 (set_attr "length_immediate" "1")
4500 (set_attr "memory" "none,store")
4501 (set_attr "prefix" "vex")
4502 (set_attr "mode" "OI")])
4504 (define_insn_and_split "vec_extract_lo_v32qi"
4505 [(set (match_operand:V16QI 0 "nonimmediate_operand" "=x,m")
4507 (match_operand:V32QI 1 "nonimmediate_operand" "xm,x")
4508 (parallel [(const_int 0) (const_int 1)
4509 (const_int 2) (const_int 3)
4510 (const_int 4) (const_int 5)
4511 (const_int 6) (const_int 7)
4512 (const_int 8) (const_int 9)
4513 (const_int 10) (const_int 11)
4514 (const_int 12) (const_int 13)
4515 (const_int 14) (const_int 15)])))]
4516 "TARGET_AVX && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
4518 "&& reload_completed"
4521 rtx op1 = operands[1];
4523 op1 = gen_rtx_REG (V16QImode, REGNO (op1));
4525 op1 = gen_lowpart (V16QImode, op1);
4526 emit_move_insn (operands[0], op1);
4530 (define_insn "vec_extract_hi_v32qi"
4531 [(set (match_operand:V16QI 0 "nonimmediate_operand" "=x,m")
4533 (match_operand:V32QI 1 "register_operand" "x,x")
4534 (parallel [(const_int 16) (const_int 17)
4535 (const_int 18) (const_int 19)
4536 (const_int 20) (const_int 21)
4537 (const_int 22) (const_int 23)
4538 (const_int 24) (const_int 25)
4539 (const_int 26) (const_int 27)
4540 (const_int 28) (const_int 29)
4541 (const_int 30) (const_int 31)])))]
4543 "vextract%~128\t{$0x1, %1, %0|%0, %1, 0x1}"
4544 [(set_attr "type" "sselog")
4545 (set_attr "prefix_extra" "1")
4546 (set_attr "length_immediate" "1")
4547 (set_attr "memory" "none,store")
4548 (set_attr "prefix" "vex")
4549 (set_attr "mode" "OI")])
4551 ;; Modes handled by vec_extract patterns.
4552 (define_mode_iterator VEC_EXTRACT_MODE
4553 [(V32QI "TARGET_AVX") V16QI
4554 (V16HI "TARGET_AVX") V8HI
4555 (V8SI "TARGET_AVX") V4SI
4556 (V4DI "TARGET_AVX") V2DI
4557 (V8SF "TARGET_AVX") V4SF
4558 (V4DF "TARGET_AVX") V2DF])
4560 (define_expand "vec_extract<mode>"
4561 [(match_operand:<ssescalarmode> 0 "register_operand")
4562 (match_operand:VEC_EXTRACT_MODE 1 "register_operand")
4563 (match_operand 2 "const_int_operand")]
4566 ix86_expand_vector_extract (false, operands[0], operands[1],
4567 INTVAL (operands[2]));
4571 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
4573 ;; Parallel double-precision floating point element swizzling
4575 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
4577 ;; Recall that the 256-bit unpck insns only shuffle within their lanes.
4578 (define_insn "avx_unpckhpd256"
4579 [(set (match_operand:V4DF 0 "register_operand" "=x")
4582 (match_operand:V4DF 1 "register_operand" "x")
4583 (match_operand:V4DF 2 "nonimmediate_operand" "xm"))
4584 (parallel [(const_int 1) (const_int 5)
4585 (const_int 3) (const_int 7)])))]
4587 "vunpckhpd\t{%2, %1, %0|%0, %1, %2}"
4588 [(set_attr "type" "sselog")
4589 (set_attr "prefix" "vex")
4590 (set_attr "mode" "V4DF")])
4592 (define_expand "vec_interleave_highv4df"
4596 (match_operand:V4DF 1 "register_operand" "x")
4597 (match_operand:V4DF 2 "nonimmediate_operand" "xm"))
4598 (parallel [(const_int 0) (const_int 4)
4599 (const_int 2) (const_int 6)])))
4605 (parallel [(const_int 1) (const_int 5)
4606 (const_int 3) (const_int 7)])))
4607 (set (match_operand:V4DF 0 "register_operand")
4612 (parallel [(const_int 2) (const_int 3)
4613 (const_int 6) (const_int 7)])))]
4616 operands[3] = gen_reg_rtx (V4DFmode);
4617 operands[4] = gen_reg_rtx (V4DFmode);
4621 (define_expand "vec_interleave_highv2df"
4622 [(set (match_operand:V2DF 0 "register_operand")
4625 (match_operand:V2DF 1 "nonimmediate_operand")
4626 (match_operand:V2DF 2 "nonimmediate_operand"))
4627 (parallel [(const_int 1)
4631 if (!ix86_vec_interleave_v2df_operator_ok (operands, 1))
4632 operands[2] = force_reg (V2DFmode, operands[2]);
4635 (define_insn "*vec_interleave_highv2df"
4636 [(set (match_operand:V2DF 0 "nonimmediate_operand" "=x,x,x,x,x,m")
4639 (match_operand:V2DF 1 "nonimmediate_operand" " 0,x,o,o,o,x")
4640 (match_operand:V2DF 2 "nonimmediate_operand" " x,x,1,0,x,0"))
4641 (parallel [(const_int 1)
4643 "TARGET_SSE2 && ix86_vec_interleave_v2df_operator_ok (operands, 1)"
4645 unpckhpd\t{%2, %0|%0, %2}
4646 vunpckhpd\t{%2, %1, %0|%0, %1, %2}
4647 %vmovddup\t{%H1, %0|%0, %H1}
4648 movlpd\t{%H1, %0|%0, %H1}
4649 vmovlpd\t{%H1, %2, %0|%0, %2, %H1}
4650 %vmovhpd\t{%1, %0|%0, %1}"
4651 [(set_attr "isa" "noavx,avx,sse3,noavx,avx,*")
4652 (set_attr "type" "sselog,sselog,sselog,ssemov,ssemov,ssemov")
4653 (set_attr "prefix_data16" "*,*,*,1,*,1")
4654 (set_attr "prefix" "orig,vex,maybe_vex,orig,vex,maybe_vex")
4655 (set_attr "mode" "V2DF,V2DF,DF,V1DF,V1DF,V1DF")])
4657 ;; Recall that the 256-bit unpck insns only shuffle within their lanes.
4658 (define_expand "avx_movddup256"
4659 [(set (match_operand:V4DF 0 "register_operand")
4662 (match_operand:V4DF 1 "nonimmediate_operand")
4664 (parallel [(const_int 0) (const_int 4)
4665 (const_int 2) (const_int 6)])))]
4668 (define_expand "avx_unpcklpd256"
4669 [(set (match_operand:V4DF 0 "register_operand")
4672 (match_operand:V4DF 1 "register_operand")
4673 (match_operand:V4DF 2 "nonimmediate_operand"))
4674 (parallel [(const_int 0) (const_int 4)
4675 (const_int 2) (const_int 6)])))]
4678 (define_insn "*avx_unpcklpd256"
4679 [(set (match_operand:V4DF 0 "register_operand" "=x,x")
4682 (match_operand:V4DF 1 "nonimmediate_operand" " x,m")
4683 (match_operand:V4DF 2 "nonimmediate_operand" "xm,1"))
4684 (parallel [(const_int 0) (const_int 4)
4685 (const_int 2) (const_int 6)])))]
4688 vunpcklpd\t{%2, %1, %0|%0, %1, %2}
4689 vmovddup\t{%1, %0|%0, %1}"
4690 [(set_attr "type" "sselog")
4691 (set_attr "prefix" "vex")
4692 (set_attr "mode" "V4DF")])
4694 (define_expand "vec_interleave_lowv4df"
4698 (match_operand:V4DF 1 "register_operand" "x")
4699 (match_operand:V4DF 2 "nonimmediate_operand" "xm"))
4700 (parallel [(const_int 0) (const_int 4)
4701 (const_int 2) (const_int 6)])))
4707 (parallel [(const_int 1) (const_int 5)
4708 (const_int 3) (const_int 7)])))
4709 (set (match_operand:V4DF 0 "register_operand")
4714 (parallel [(const_int 0) (const_int 1)
4715 (const_int 4) (const_int 5)])))]
4718 operands[3] = gen_reg_rtx (V4DFmode);
4719 operands[4] = gen_reg_rtx (V4DFmode);
4722 (define_expand "vec_interleave_lowv2df"
4723 [(set (match_operand:V2DF 0 "register_operand")
4726 (match_operand:V2DF 1 "nonimmediate_operand")
4727 (match_operand:V2DF 2 "nonimmediate_operand"))
4728 (parallel [(const_int 0)
4732 if (!ix86_vec_interleave_v2df_operator_ok (operands, 0))
4733 operands[1] = force_reg (V2DFmode, operands[1]);
4736 (define_insn "*vec_interleave_lowv2df"
4737 [(set (match_operand:V2DF 0 "nonimmediate_operand" "=x,x,x,x,x,o")
4740 (match_operand:V2DF 1 "nonimmediate_operand" " 0,x,m,0,x,0")
4741 (match_operand:V2DF 2 "nonimmediate_operand" " x,x,1,m,m,x"))
4742 (parallel [(const_int 0)
4744 "TARGET_SSE2 && ix86_vec_interleave_v2df_operator_ok (operands, 0)"
4746 unpcklpd\t{%2, %0|%0, %2}
4747 vunpcklpd\t{%2, %1, %0|%0, %1, %2}
4748 %vmovddup\t{%1, %0|%0, %1}
4749 movhpd\t{%2, %0|%0, %2}
4750 vmovhpd\t{%2, %1, %0|%0, %1, %2}
4751 %vmovlpd\t{%2, %H0|%H0, %2}"
4752 [(set_attr "isa" "noavx,avx,sse3,noavx,avx,*")
4753 (set_attr "type" "sselog,sselog,sselog,ssemov,ssemov,ssemov")
4754 (set_attr "prefix_data16" "*,*,*,1,*,1")
4755 (set_attr "prefix" "orig,vex,maybe_vex,orig,vex,maybe_vex")
4756 (set_attr "mode" "V2DF,V2DF,DF,V1DF,V1DF,V1DF")])
4759 [(set (match_operand:V2DF 0 "memory_operand")
4762 (match_operand:V2DF 1 "register_operand")
4764 (parallel [(const_int 0)
4766 "TARGET_SSE3 && reload_completed"
4769 rtx low = gen_rtx_REG (DFmode, REGNO (operands[1]));
4770 emit_move_insn (adjust_address (operands[0], DFmode, 0), low);
4771 emit_move_insn (adjust_address (operands[0], DFmode, 8), low);
4776 [(set (match_operand:V2DF 0 "register_operand")
4779 (match_operand:V2DF 1 "memory_operand")
4781 (parallel [(match_operand:SI 2 "const_0_to_1_operand")
4782 (match_operand:SI 3 "const_int_operand")])))]
4783 "TARGET_SSE3 && INTVAL (operands[2]) + 2 == INTVAL (operands[3])"
4784 [(set (match_dup 0) (vec_duplicate:V2DF (match_dup 1)))]
4786 operands[1] = adjust_address (operands[1], DFmode, INTVAL (operands[2]) * 8);
4789 (define_expand "avx_shufpd256"
4790 [(match_operand:V4DF 0 "register_operand")
4791 (match_operand:V4DF 1 "register_operand")
4792 (match_operand:V4DF 2 "nonimmediate_operand")
4793 (match_operand:SI 3 "const_int_operand")]
4796 int mask = INTVAL (operands[3]);
4797 emit_insn (gen_avx_shufpd256_1 (operands[0], operands[1], operands[2],
4799 GEN_INT (mask & 2 ? 5 : 4),
4800 GEN_INT (mask & 4 ? 3 : 2),
4801 GEN_INT (mask & 8 ? 7 : 6)));
4805 (define_insn "avx_shufpd256_1"
4806 [(set (match_operand:V4DF 0 "register_operand" "=x")
4809 (match_operand:V4DF 1 "register_operand" "x")
4810 (match_operand:V4DF 2 "nonimmediate_operand" "xm"))
4811 (parallel [(match_operand 3 "const_0_to_1_operand")
4812 (match_operand 4 "const_4_to_5_operand")
4813 (match_operand 5 "const_2_to_3_operand")
4814 (match_operand 6 "const_6_to_7_operand")])))]
4818 mask = INTVAL (operands[3]);
4819 mask |= (INTVAL (operands[4]) - 4) << 1;
4820 mask |= (INTVAL (operands[5]) - 2) << 2;
4821 mask |= (INTVAL (operands[6]) - 6) << 3;
4822 operands[3] = GEN_INT (mask);
4824 return "vshufpd\t{%3, %2, %1, %0|%0, %1, %2, %3}";
4826 [(set_attr "type" "sseshuf")
4827 (set_attr "length_immediate" "1")
4828 (set_attr "prefix" "vex")
4829 (set_attr "mode" "V4DF")])
4831 (define_expand "sse2_shufpd"
4832 [(match_operand:V2DF 0 "register_operand")
4833 (match_operand:V2DF 1 "register_operand")
4834 (match_operand:V2DF 2 "nonimmediate_operand")
4835 (match_operand:SI 3 "const_int_operand")]
4838 int mask = INTVAL (operands[3]);
4839 emit_insn (gen_sse2_shufpd_v2df (operands[0], operands[1], operands[2],
4841 GEN_INT (mask & 2 ? 3 : 2)));
4845 ;; punpcklqdq and punpckhqdq are shorter than shufpd.
4846 (define_insn "avx2_interleave_highv4di"
4847 [(set (match_operand:V4DI 0 "register_operand" "=x")
4850 (match_operand:V4DI 1 "register_operand" "x")
4851 (match_operand:V4DI 2 "nonimmediate_operand" "xm"))
4852 (parallel [(const_int 1)
4857 "vpunpckhqdq\t{%2, %1, %0|%0, %1, %2}"
4858 [(set_attr "type" "sselog")
4859 (set_attr "prefix" "vex")
4860 (set_attr "mode" "OI")])
4862 (define_insn "vec_interleave_highv2di"
4863 [(set (match_operand:V2DI 0 "register_operand" "=x,x")
4866 (match_operand:V2DI 1 "register_operand" "0,x")
4867 (match_operand:V2DI 2 "nonimmediate_operand" "xm,xm"))
4868 (parallel [(const_int 1)
4872 punpckhqdq\t{%2, %0|%0, %2}
4873 vpunpckhqdq\t{%2, %1, %0|%0, %1, %2}"
4874 [(set_attr "isa" "noavx,avx")
4875 (set_attr "type" "sselog")
4876 (set_attr "prefix_data16" "1,*")
4877 (set_attr "prefix" "orig,vex")
4878 (set_attr "mode" "TI")])
4880 (define_insn "avx2_interleave_lowv4di"
4881 [(set (match_operand:V4DI 0 "register_operand" "=x")
4884 (match_operand:V4DI 1 "register_operand" "x")
4885 (match_operand:V4DI 2 "nonimmediate_operand" "xm"))
4886 (parallel [(const_int 0)
4891 "vpunpcklqdq\t{%2, %1, %0|%0, %1, %2}"
4892 [(set_attr "type" "sselog")
4893 (set_attr "prefix" "vex")
4894 (set_attr "mode" "OI")])
4896 (define_insn "vec_interleave_lowv2di"
4897 [(set (match_operand:V2DI 0 "register_operand" "=x,x")
4900 (match_operand:V2DI 1 "register_operand" "0,x")
4901 (match_operand:V2DI 2 "nonimmediate_operand" "xm,xm"))
4902 (parallel [(const_int 0)
4906 punpcklqdq\t{%2, %0|%0, %2}
4907 vpunpcklqdq\t{%2, %1, %0|%0, %1, %2}"
4908 [(set_attr "isa" "noavx,avx")
4909 (set_attr "type" "sselog")
4910 (set_attr "prefix_data16" "1,*")
4911 (set_attr "prefix" "orig,vex")
4912 (set_attr "mode" "TI")])
4914 (define_insn "sse2_shufpd_<mode>"
4915 [(set (match_operand:VI8F_128 0 "register_operand" "=x,x")
4916 (vec_select:VI8F_128
4917 (vec_concat:<ssedoublevecmode>
4918 (match_operand:VI8F_128 1 "register_operand" "0,x")
4919 (match_operand:VI8F_128 2 "nonimmediate_operand" "xm,xm"))
4920 (parallel [(match_operand 3 "const_0_to_1_operand")
4921 (match_operand 4 "const_2_to_3_operand")])))]
4925 mask = INTVAL (operands[3]);
4926 mask |= (INTVAL (operands[4]) - 2) << 1;
4927 operands[3] = GEN_INT (mask);
4929 switch (which_alternative)
4932 return "shufpd\t{%3, %2, %0|%0, %2, %3}";
4934 return "vshufpd\t{%3, %2, %1, %0|%0, %1, %2, %3}";
4939 [(set_attr "isa" "noavx,avx")
4940 (set_attr "type" "sseshuf")
4941 (set_attr "length_immediate" "1")
4942 (set_attr "prefix" "orig,vex")
4943 (set_attr "mode" "V2DF")])
4945 ;; Avoid combining registers from different units in a single alternative,
4946 ;; see comment above inline_secondary_memory_needed function in i386.c
4947 (define_insn "sse2_storehpd"
4948 [(set (match_operand:DF 0 "nonimmediate_operand" "=m,x,x,x,*f,r")
4950 (match_operand:V2DF 1 "nonimmediate_operand" " x,0,x,o,o,o")
4951 (parallel [(const_int 1)])))]
4952 "TARGET_SSE2 && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
4954 %vmovhpd\t{%1, %0|%0, %1}
4956 vunpckhpd\t{%d1, %0|%0, %d1}
4960 [(set_attr "isa" "*,noavx,avx,*,*,*")
4961 (set_attr "type" "ssemov,sselog1,sselog1,ssemov,fmov,imov")
4962 (set (attr "prefix_data16")
4964 (and (eq_attr "alternative" "0")
4965 (not (match_test "TARGET_AVX")))
4967 (const_string "*")))
4968 (set_attr "prefix" "maybe_vex,orig,vex,*,*,*")
4969 (set_attr "mode" "V1DF,V1DF,V2DF,DF,DF,DF")])
4972 [(set (match_operand:DF 0 "register_operand")
4974 (match_operand:V2DF 1 "memory_operand")
4975 (parallel [(const_int 1)])))]
4976 "TARGET_SSE2 && reload_completed"
4977 [(set (match_dup 0) (match_dup 1))]
4978 "operands[1] = adjust_address (operands[1], DFmode, 8);")
4980 (define_insn "*vec_extractv2df_1_sse"
4981 [(set (match_operand:DF 0 "nonimmediate_operand" "=m,x,x")
4983 (match_operand:V2DF 1 "nonimmediate_operand" "x,x,o")
4984 (parallel [(const_int 1)])))]
4985 "!TARGET_SSE2 && TARGET_SSE
4986 && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
4988 movhps\t{%1, %0|%0, %1}
4989 movhlps\t{%1, %0|%0, %1}
4990 movlps\t{%H1, %0|%0, %H1}"
4991 [(set_attr "type" "ssemov")
4992 (set_attr "mode" "V2SF,V4SF,V2SF")])
4994 ;; Avoid combining registers from different units in a single alternative,
4995 ;; see comment above inline_secondary_memory_needed function in i386.c
4996 (define_insn "sse2_storelpd"
4997 [(set (match_operand:DF 0 "nonimmediate_operand" "=m,x,x,*f,r")
4999 (match_operand:V2DF 1 "nonimmediate_operand" " x,x,m,m,m")
5000 (parallel [(const_int 0)])))]
5001 "TARGET_SSE2 && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
5003 %vmovlpd\t{%1, %0|%0, %1}
5008 [(set_attr "type" "ssemov,ssemov,ssemov,fmov,imov")
5009 (set_attr "prefix_data16" "1,*,*,*,*")
5010 (set_attr "prefix" "maybe_vex")
5011 (set_attr "mode" "V1DF,DF,DF,DF,DF")])
5014 [(set (match_operand:DF 0 "register_operand")
5016 (match_operand:V2DF 1 "nonimmediate_operand")
5017 (parallel [(const_int 0)])))]
5018 "TARGET_SSE2 && reload_completed"
5021 rtx op1 = operands[1];
5023 op1 = gen_rtx_REG (DFmode, REGNO (op1));
5025 op1 = gen_lowpart (DFmode, op1);
5026 emit_move_insn (operands[0], op1);
5030 (define_insn "*vec_extractv2df_0_sse"
5031 [(set (match_operand:DF 0 "nonimmediate_operand" "=m,x,x")
5033 (match_operand:V2DF 1 "nonimmediate_operand" "x,x,m")
5034 (parallel [(const_int 0)])))]
5035 "!TARGET_SSE2 && TARGET_SSE
5036 && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
5038 movlps\t{%1, %0|%0, %1}
5039 movaps\t{%1, %0|%0, %1}
5040 movlps\t{%1, %0|%0, %1}"
5041 [(set_attr "type" "ssemov")
5042 (set_attr "mode" "V2SF,V4SF,V2SF")])
5044 (define_expand "sse2_loadhpd_exp"
5045 [(set (match_operand:V2DF 0 "nonimmediate_operand")
5048 (match_operand:V2DF 1 "nonimmediate_operand")
5049 (parallel [(const_int 0)]))
5050 (match_operand:DF 2 "nonimmediate_operand")))]
5053 rtx dst = ix86_fixup_binary_operands (UNKNOWN, V2DFmode, operands);
5055 emit_insn (gen_sse2_loadhpd (dst, operands[1], operands[2]));
5057 /* Fix up the destination if needed. */
5058 if (dst != operands[0])
5059 emit_move_insn (operands[0], dst);
5064 ;; Avoid combining registers from different units in a single alternative,
5065 ;; see comment above inline_secondary_memory_needed function in i386.c
5066 (define_insn "sse2_loadhpd"
5067 [(set (match_operand:V2DF 0 "nonimmediate_operand"
5071 (match_operand:V2DF 1 "nonimmediate_operand"
5073 (parallel [(const_int 0)]))
5074 (match_operand:DF 2 "nonimmediate_operand"
5075 " m,m,x,x,x,*f,r")))]
5076 "TARGET_SSE2 && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
5078 movhpd\t{%2, %0|%0, %2}
5079 vmovhpd\t{%2, %1, %0|%0, %1, %2}
5080 unpcklpd\t{%2, %0|%0, %2}
5081 vunpcklpd\t{%2, %1, %0|%0, %1, %2}
5085 [(set_attr "isa" "noavx,avx,noavx,avx,*,*,*")
5086 (set_attr "type" "ssemov,ssemov,sselog,sselog,ssemov,fmov,imov")
5087 (set_attr "prefix_data16" "1,*,*,*,*,*,*")
5088 (set_attr "prefix" "orig,vex,orig,vex,*,*,*")
5089 (set_attr "mode" "V1DF,V1DF,V2DF,V2DF,DF,DF,DF")])
5092 [(set (match_operand:V2DF 0 "memory_operand")
5094 (vec_select:DF (match_dup 0) (parallel [(const_int 0)]))
5095 (match_operand:DF 1 "register_operand")))]
5096 "TARGET_SSE2 && reload_completed"
5097 [(set (match_dup 0) (match_dup 1))]
5098 "operands[0] = adjust_address (operands[0], DFmode, 8);")
5100 (define_expand "sse2_loadlpd_exp"
5101 [(set (match_operand:V2DF 0 "nonimmediate_operand")
5103 (match_operand:DF 2 "nonimmediate_operand")
5105 (match_operand:V2DF 1 "nonimmediate_operand")
5106 (parallel [(const_int 1)]))))]
5109 rtx dst = ix86_fixup_binary_operands (UNKNOWN, V2DFmode, operands);
5111 emit_insn (gen_sse2_loadlpd (dst, operands[1], operands[2]));
5113 /* Fix up the destination if needed. */
5114 if (dst != operands[0])
5115 emit_move_insn (operands[0], dst);
5120 ;; Avoid combining registers from different units in a single alternative,
5121 ;; see comment above inline_secondary_memory_needed function in i386.c
5122 (define_insn "sse2_loadlpd"
5123 [(set (match_operand:V2DF 0 "nonimmediate_operand"
5124 "=x,x,x,x,x,x,x,x,m,m ,m")
5126 (match_operand:DF 2 "nonimmediate_operand"
5127 " m,m,m,x,x,0,0,x,x,*f,r")
5129 (match_operand:V2DF 1 "vector_move_operand"
5130 " C,0,x,0,x,x,o,o,0,0 ,0")
5131 (parallel [(const_int 1)]))))]
5132 "TARGET_SSE2 && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
5134 %vmovsd\t{%2, %0|%0, %2}
5135 movlpd\t{%2, %0|%0, %2}
5136 vmovlpd\t{%2, %1, %0|%0, %1, %2}
5137 movsd\t{%2, %0|%0, %2}
5138 vmovsd\t{%2, %1, %0|%0, %1, %2}
5139 shufpd\t{$2, %1, %0|%0, %1, 2}
5140 movhpd\t{%H1, %0|%0, %H1}
5141 vmovhpd\t{%H1, %2, %0|%0, %2, %H1}
5145 [(set_attr "isa" "*,noavx,avx,noavx,avx,noavx,noavx,avx,*,*,*")
5147 (cond [(eq_attr "alternative" "5")
5148 (const_string "sselog")
5149 (eq_attr "alternative" "9")
5150 (const_string "fmov")
5151 (eq_attr "alternative" "10")
5152 (const_string "imov")
5154 (const_string "ssemov")))
5155 (set_attr "prefix_data16" "*,1,*,*,*,*,1,*,*,*,*")
5156 (set_attr "length_immediate" "*,*,*,*,*,1,*,*,*,*,*")
5157 (set_attr "prefix" "maybe_vex,orig,vex,orig,vex,orig,orig,vex,*,*,*")
5158 (set_attr "mode" "DF,V1DF,V1DF,V1DF,V1DF,V2DF,V1DF,V1DF,DF,DF,DF")])
5161 [(set (match_operand:V2DF 0 "memory_operand")
5163 (match_operand:DF 1 "register_operand")
5164 (vec_select:DF (match_dup 0) (parallel [(const_int 1)]))))]
5165 "TARGET_SSE2 && reload_completed"
5166 [(set (match_dup 0) (match_dup 1))]
5167 "operands[0] = adjust_address (operands[0], DFmode, 0);")
5169 (define_insn "sse2_movsd"
5170 [(set (match_operand:V2DF 0 "nonimmediate_operand" "=x,x,x,x,m,x,x,x,o")
5172 (match_operand:V2DF 2 "nonimmediate_operand" " x,x,m,m,x,0,0,x,0")
5173 (match_operand:V2DF 1 "nonimmediate_operand" " 0,x,0,x,0,x,o,o,x")
5177 movsd\t{%2, %0|%0, %2}
5178 vmovsd\t{%2, %1, %0|%0, %1, %2}
5179 movlpd\t{%2, %0|%0, %2}
5180 vmovlpd\t{%2, %1, %0|%0, %1, %2}
5181 %vmovlpd\t{%2, %0|%0, %2}
5182 shufpd\t{$2, %1, %0|%0, %1, 2}
5183 movhps\t{%H1, %0|%0, %H1}
5184 vmovhps\t{%H1, %2, %0|%0, %2, %H1}
5185 %vmovhps\t{%1, %H0|%H0, %1}"
5186 [(set_attr "isa" "noavx,avx,noavx,avx,*,noavx,noavx,avx,*")
5189 (eq_attr "alternative" "5")
5190 (const_string "sselog")
5191 (const_string "ssemov")))
5192 (set (attr "prefix_data16")
5194 (and (eq_attr "alternative" "2,4")
5195 (not (match_test "TARGET_AVX")))
5197 (const_string "*")))
5198 (set_attr "length_immediate" "*,*,*,*,*,1,*,*,*")
5199 (set_attr "prefix" "orig,vex,orig,vex,maybe_vex,orig,orig,vex,maybe_vex")
5200 (set_attr "mode" "DF,DF,V1DF,V1DF,V1DF,V2DF,V1DF,V1DF,V1DF")])
5202 (define_insn "vec_dupv2df"
5203 [(set (match_operand:V2DF 0 "register_operand" "=x,x")
5205 (match_operand:DF 1 "nonimmediate_operand" " 0,xm")))]
5209 %vmovddup\t{%1, %0|%0, %1}"
5210 [(set_attr "isa" "noavx,sse3")
5211 (set_attr "type" "sselog1")
5212 (set_attr "prefix" "orig,maybe_vex")
5213 (set_attr "mode" "V2DF,DF")])
5215 (define_insn "*vec_concatv2df"
5216 [(set (match_operand:V2DF 0 "register_operand" "=x,x,x,x,x,x,x,x")
5218 (match_operand:DF 1 "nonimmediate_operand" " 0,x,m,0,x,m,0,0")
5219 (match_operand:DF 2 "vector_move_operand" " x,x,1,m,m,C,x,m")))]
5222 unpcklpd\t{%2, %0|%0, %2}
5223 vunpcklpd\t{%2, %1, %0|%0, %1, %2}
5224 %vmovddup\t{%1, %0|%0, %1}
5225 movhpd\t{%2, %0|%0, %2}
5226 vmovhpd\t{%2, %1, %0|%0, %1, %2}
5227 %vmovsd\t{%1, %0|%0, %1}
5228 movlhps\t{%2, %0|%0, %2}
5229 movhps\t{%2, %0|%0, %2}"
5230 [(set_attr "isa" "sse2_noavx,avx,sse3,sse2_noavx,avx,sse2,noavx,noavx")
5233 (eq_attr "alternative" "0,1,2")
5234 (const_string "sselog")
5235 (const_string "ssemov")))
5236 (set_attr "prefix_data16" "*,*,*,1,*,*,*,*")
5237 (set_attr "prefix" "orig,vex,maybe_vex,orig,vex,maybe_vex,orig,orig")
5238 (set_attr "mode" "V2DF,V2DF,DF,V1DF,V1DF,DF,V4SF,V2SF")])
5240 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
5242 ;; Parallel integral arithmetic
5244 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
5246 (define_expand "neg<mode>2"
5247 [(set (match_operand:VI_AVX2 0 "register_operand")
5250 (match_operand:VI_AVX2 1 "nonimmediate_operand")))]
5252 "operands[2] = force_reg (<MODE>mode, CONST0_RTX (<MODE>mode));")
5254 (define_expand "<plusminus_insn><mode>3"
5255 [(set (match_operand:VI_AVX2 0 "register_operand")
5257 (match_operand:VI_AVX2 1 "nonimmediate_operand")
5258 (match_operand:VI_AVX2 2 "nonimmediate_operand")))]
5260 "ix86_fixup_binary_operands_no_copy (<CODE>, <MODE>mode, operands);")
5262 (define_insn "*<plusminus_insn><mode>3"
5263 [(set (match_operand:VI_AVX2 0 "register_operand" "=x,x")
5265 (match_operand:VI_AVX2 1 "nonimmediate_operand" "<comm>0,x")
5266 (match_operand:VI_AVX2 2 "nonimmediate_operand" "xm,xm")))]
5267 "TARGET_SSE2 && ix86_binary_operator_ok (<CODE>, <MODE>mode, operands)"
5269 p<plusminus_mnemonic><ssemodesuffix>\t{%2, %0|%0, %2}
5270 vp<plusminus_mnemonic><ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
5271 [(set_attr "isa" "noavx,avx")
5272 (set_attr "type" "sseiadd")
5273 (set_attr "prefix_data16" "1,*")
5274 (set_attr "prefix" "orig,vex")
5275 (set_attr "mode" "<sseinsnmode>")])
5277 (define_expand "<sse2_avx2>_<plusminus_insn><mode>3"
5278 [(set (match_operand:VI12_AVX2 0 "register_operand")
5279 (sat_plusminus:VI12_AVX2
5280 (match_operand:VI12_AVX2 1 "nonimmediate_operand")
5281 (match_operand:VI12_AVX2 2 "nonimmediate_operand")))]
5283 "ix86_fixup_binary_operands_no_copy (<CODE>, <MODE>mode, operands);")
5285 (define_insn "*<sse2_avx2>_<plusminus_insn><mode>3"
5286 [(set (match_operand:VI12_AVX2 0 "register_operand" "=x,x")
5287 (sat_plusminus:VI12_AVX2
5288 (match_operand:VI12_AVX2 1 "nonimmediate_operand" "<comm>0,x")
5289 (match_operand:VI12_AVX2 2 "nonimmediate_operand" "xm,xm")))]
5290 "TARGET_SSE2 && ix86_binary_operator_ok (<CODE>, <MODE>mode, operands)"
5292 p<plusminus_mnemonic><ssemodesuffix>\t{%2, %0|%0, %2}
5293 vp<plusminus_mnemonic><ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
5294 [(set_attr "isa" "noavx,avx")
5295 (set_attr "type" "sseiadd")
5296 (set_attr "prefix_data16" "1,*")
5297 (set_attr "prefix" "orig,vex")
5298 (set_attr "mode" "TI")])
5300 (define_expand "mul<mode>3"
5301 [(set (match_operand:VI1_AVX2 0 "register_operand")
5302 (mult:VI1_AVX2 (match_operand:VI1_AVX2 1 "register_operand")
5303 (match_operand:VI1_AVX2 2 "register_operand")))]
5306 ix86_expand_vecop_qihi (MULT, operands[0], operands[1], operands[2]);
5310 (define_expand "mul<mode>3"
5311 [(set (match_operand:VI2_AVX2 0 "register_operand")
5312 (mult:VI2_AVX2 (match_operand:VI2_AVX2 1 "nonimmediate_operand")
5313 (match_operand:VI2_AVX2 2 "nonimmediate_operand")))]
5315 "ix86_fixup_binary_operands_no_copy (MULT, <MODE>mode, operands);")
5317 (define_insn "*mul<mode>3"
5318 [(set (match_operand:VI2_AVX2 0 "register_operand" "=x,x")
5319 (mult:VI2_AVX2 (match_operand:VI2_AVX2 1 "nonimmediate_operand" "%0,x")
5320 (match_operand:VI2_AVX2 2 "nonimmediate_operand" "xm,xm")))]
5321 "TARGET_SSE2 && ix86_binary_operator_ok (MULT, <MODE>mode, operands)"
5323 pmullw\t{%2, %0|%0, %2}
5324 vpmullw\t{%2, %1, %0|%0, %1, %2}"
5325 [(set_attr "isa" "noavx,avx")
5326 (set_attr "type" "sseimul")
5327 (set_attr "prefix_data16" "1,*")
5328 (set_attr "prefix" "orig,vex")
5329 (set_attr "mode" "<sseinsnmode>")])
5331 (define_expand "<s>mul<mode>3_highpart"
5332 [(set (match_operand:VI2_AVX2 0 "register_operand")
5334 (lshiftrt:<ssedoublemode>
5335 (mult:<ssedoublemode>
5336 (any_extend:<ssedoublemode>
5337 (match_operand:VI2_AVX2 1 "nonimmediate_operand"))
5338 (any_extend:<ssedoublemode>
5339 (match_operand:VI2_AVX2 2 "nonimmediate_operand")))
5342 "ix86_fixup_binary_operands_no_copy (MULT, <MODE>mode, operands);")
5344 (define_insn "*<s>mul<mode>3_highpart"
5345 [(set (match_operand:VI2_AVX2 0 "register_operand" "=x,x")
5347 (lshiftrt:<ssedoublemode>
5348 (mult:<ssedoublemode>
5349 (any_extend:<ssedoublemode>
5350 (match_operand:VI2_AVX2 1 "nonimmediate_operand" "%0,x"))
5351 (any_extend:<ssedoublemode>
5352 (match_operand:VI2_AVX2 2 "nonimmediate_operand" "xm,xm")))
5354 "TARGET_SSE2 && ix86_binary_operator_ok (MULT, <MODE>mode, operands)"
5356 pmulh<u>w\t{%2, %0|%0, %2}
5357 vpmulh<u>w\t{%2, %1, %0|%0, %1, %2}"
5358 [(set_attr "isa" "noavx,avx")
5359 (set_attr "type" "sseimul")
5360 (set_attr "prefix_data16" "1,*")
5361 (set_attr "prefix" "orig,vex")
5362 (set_attr "mode" "<sseinsnmode>")])
5364 (define_expand "vec_widen_umult_even_v8si"
5365 [(set (match_operand:V4DI 0 "register_operand")
5369 (match_operand:V8SI 1 "nonimmediate_operand")
5370 (parallel [(const_int 0) (const_int 2)
5371 (const_int 4) (const_int 6)])))
5374 (match_operand:V8SI 2 "nonimmediate_operand")
5375 (parallel [(const_int 0) (const_int 2)
5376 (const_int 4) (const_int 6)])))))]
5378 "ix86_fixup_binary_operands_no_copy (MULT, V8SImode, operands);")
5380 (define_insn "*vec_widen_umult_even_v8si"
5381 [(set (match_operand:V4DI 0 "register_operand" "=x")
5385 (match_operand:V8SI 1 "nonimmediate_operand" "%x")
5386 (parallel [(const_int 0) (const_int 2)
5387 (const_int 4) (const_int 6)])))
5390 (match_operand:V8SI 2 "nonimmediate_operand" "xm")
5391 (parallel [(const_int 0) (const_int 2)
5392 (const_int 4) (const_int 6)])))))]
5393 "TARGET_AVX2 && ix86_binary_operator_ok (MULT, V8SImode, operands)"
5394 "vpmuludq\t{%2, %1, %0|%0, %1, %2}"
5395 [(set_attr "type" "sseimul")
5396 (set_attr "prefix" "vex")
5397 (set_attr "mode" "OI")])
5399 (define_expand "vec_widen_umult_even_v4si"
5400 [(set (match_operand:V2DI 0 "register_operand")
5404 (match_operand:V4SI 1 "nonimmediate_operand")
5405 (parallel [(const_int 0) (const_int 2)])))
5408 (match_operand:V4SI 2 "nonimmediate_operand")
5409 (parallel [(const_int 0) (const_int 2)])))))]
5411 "ix86_fixup_binary_operands_no_copy (MULT, V4SImode, operands);")
5413 (define_insn "*vec_widen_umult_even_v4si"
5414 [(set (match_operand:V2DI 0 "register_operand" "=x,x")
5418 (match_operand:V4SI 1 "nonimmediate_operand" "%0,x")
5419 (parallel [(const_int 0) (const_int 2)])))
5422 (match_operand:V4SI 2 "nonimmediate_operand" "xm,xm")
5423 (parallel [(const_int 0) (const_int 2)])))))]
5424 "TARGET_SSE2 && ix86_binary_operator_ok (MULT, V4SImode, operands)"
5426 pmuludq\t{%2, %0|%0, %2}
5427 vpmuludq\t{%2, %1, %0|%0, %1, %2}"
5428 [(set_attr "isa" "noavx,avx")
5429 (set_attr "type" "sseimul")
5430 (set_attr "prefix_data16" "1,*")
5431 (set_attr "prefix" "orig,vex")
5432 (set_attr "mode" "TI")])
5434 (define_expand "vec_widen_smult_even_v8si"
5435 [(set (match_operand:V4DI 0 "register_operand")
5439 (match_operand:V8SI 1 "nonimmediate_operand")
5440 (parallel [(const_int 0) (const_int 2)
5441 (const_int 4) (const_int 6)])))
5444 (match_operand:V8SI 2 "nonimmediate_operand")
5445 (parallel [(const_int 0) (const_int 2)
5446 (const_int 4) (const_int 6)])))))]
5448 "ix86_fixup_binary_operands_no_copy (MULT, V8SImode, operands);")
5450 (define_insn "*vec_widen_smult_even_v8si"
5451 [(set (match_operand:V4DI 0 "register_operand" "=x")
5455 (match_operand:V8SI 1 "nonimmediate_operand" "x")
5456 (parallel [(const_int 0) (const_int 2)
5457 (const_int 4) (const_int 6)])))
5460 (match_operand:V8SI 2 "nonimmediate_operand" "xm")
5461 (parallel [(const_int 0) (const_int 2)
5462 (const_int 4) (const_int 6)])))))]
5463 "TARGET_AVX2 && ix86_binary_operator_ok (MULT, V8SImode, operands)"
5464 "vpmuldq\t{%2, %1, %0|%0, %1, %2}"
5465 [(set_attr "isa" "avx")
5466 (set_attr "type" "sseimul")
5467 (set_attr "prefix_extra" "1")
5468 (set_attr "prefix" "vex")
5469 (set_attr "mode" "OI")])
5471 (define_expand "sse4_1_mulv2siv2di3"
5472 [(set (match_operand:V2DI 0 "register_operand")
5476 (match_operand:V4SI 1 "nonimmediate_operand")
5477 (parallel [(const_int 0) (const_int 2)])))
5480 (match_operand:V4SI 2 "nonimmediate_operand")
5481 (parallel [(const_int 0) (const_int 2)])))))]
5483 "ix86_fixup_binary_operands_no_copy (MULT, V4SImode, operands);")
5485 (define_insn "*sse4_1_mulv2siv2di3"
5486 [(set (match_operand:V2DI 0 "register_operand" "=x,x")
5490 (match_operand:V4SI 1 "nonimmediate_operand" "%0,x")
5491 (parallel [(const_int 0) (const_int 2)])))
5494 (match_operand:V4SI 2 "nonimmediate_operand" "xm,xm")
5495 (parallel [(const_int 0) (const_int 2)])))))]
5496 "TARGET_SSE4_1 && ix86_binary_operator_ok (MULT, V4SImode, operands)"
5498 pmuldq\t{%2, %0|%0, %2}
5499 vpmuldq\t{%2, %1, %0|%0, %1, %2}"
5500 [(set_attr "isa" "noavx,avx")
5501 (set_attr "type" "sseimul")
5502 (set_attr "prefix_data16" "1,*")
5503 (set_attr "prefix_extra" "1")
5504 (set_attr "prefix" "orig,vex")
5505 (set_attr "mode" "TI")])
5507 (define_expand "avx2_pmaddwd"
5508 [(set (match_operand:V8SI 0 "register_operand")
5513 (match_operand:V16HI 1 "nonimmediate_operand")
5514 (parallel [(const_int 0) (const_int 2)
5515 (const_int 4) (const_int 6)
5516 (const_int 8) (const_int 10)
5517 (const_int 12) (const_int 14)])))
5520 (match_operand:V16HI 2 "nonimmediate_operand")
5521 (parallel [(const_int 0) (const_int 2)
5522 (const_int 4) (const_int 6)
5523 (const_int 8) (const_int 10)
5524 (const_int 12) (const_int 14)]))))
5527 (vec_select:V8HI (match_dup 1)
5528 (parallel [(const_int 1) (const_int 3)
5529 (const_int 5) (const_int 7)
5530 (const_int 9) (const_int 11)
5531 (const_int 13) (const_int 15)])))
5533 (vec_select:V8HI (match_dup 2)
5534 (parallel [(const_int 1) (const_int 3)
5535 (const_int 5) (const_int 7)
5536 (const_int 9) (const_int 11)
5537 (const_int 13) (const_int 15)]))))))]
5539 "ix86_fixup_binary_operands_no_copy (MULT, V16HImode, operands);")
5541 (define_insn "*avx2_pmaddwd"
5542 [(set (match_operand:V8SI 0 "register_operand" "=x")
5547 (match_operand:V16HI 1 "nonimmediate_operand" "%x")
5548 (parallel [(const_int 0) (const_int 2)
5549 (const_int 4) (const_int 6)
5550 (const_int 8) (const_int 10)
5551 (const_int 12) (const_int 14)])))
5554 (match_operand:V16HI 2 "nonimmediate_operand" "xm")
5555 (parallel [(const_int 0) (const_int 2)
5556 (const_int 4) (const_int 6)
5557 (const_int 8) (const_int 10)
5558 (const_int 12) (const_int 14)]))))
5561 (vec_select:V8HI (match_dup 1)
5562 (parallel [(const_int 1) (const_int 3)
5563 (const_int 5) (const_int 7)
5564 (const_int 9) (const_int 11)
5565 (const_int 13) (const_int 15)])))
5567 (vec_select:V8HI (match_dup 2)
5568 (parallel [(const_int 1) (const_int 3)
5569 (const_int 5) (const_int 7)
5570 (const_int 9) (const_int 11)
5571 (const_int 13) (const_int 15)]))))))]
5572 "TARGET_AVX2 && ix86_binary_operator_ok (MULT, V16HImode, operands)"
5573 "vpmaddwd\t{%2, %1, %0|%0, %1, %2}"
5574 [(set_attr "type" "sseiadd")
5575 (set_attr "prefix" "vex")
5576 (set_attr "mode" "OI")])
5578 (define_expand "sse2_pmaddwd"
5579 [(set (match_operand:V4SI 0 "register_operand")
5584 (match_operand:V8HI 1 "nonimmediate_operand")
5585 (parallel [(const_int 0) (const_int 2)
5586 (const_int 4) (const_int 6)])))
5589 (match_operand:V8HI 2 "nonimmediate_operand")
5590 (parallel [(const_int 0) (const_int 2)
5591 (const_int 4) (const_int 6)]))))
5594 (vec_select:V4HI (match_dup 1)
5595 (parallel [(const_int 1) (const_int 3)
5596 (const_int 5) (const_int 7)])))
5598 (vec_select:V4HI (match_dup 2)
5599 (parallel [(const_int 1) (const_int 3)
5600 (const_int 5) (const_int 7)]))))))]
5602 "ix86_fixup_binary_operands_no_copy (MULT, V8HImode, operands);")
5604 (define_insn "*sse2_pmaddwd"
5605 [(set (match_operand:V4SI 0 "register_operand" "=x,x")
5610 (match_operand:V8HI 1 "nonimmediate_operand" "%0,x")
5611 (parallel [(const_int 0) (const_int 2)
5612 (const_int 4) (const_int 6)])))
5615 (match_operand:V8HI 2 "nonimmediate_operand" "xm,xm")
5616 (parallel [(const_int 0) (const_int 2)
5617 (const_int 4) (const_int 6)]))))
5620 (vec_select:V4HI (match_dup 1)
5621 (parallel [(const_int 1) (const_int 3)
5622 (const_int 5) (const_int 7)])))
5624 (vec_select:V4HI (match_dup 2)
5625 (parallel [(const_int 1) (const_int 3)
5626 (const_int 5) (const_int 7)]))))))]
5627 "TARGET_SSE2 && ix86_binary_operator_ok (MULT, V8HImode, operands)"
5629 pmaddwd\t{%2, %0|%0, %2}
5630 vpmaddwd\t{%2, %1, %0|%0, %1, %2}"
5631 [(set_attr "isa" "noavx,avx")
5632 (set_attr "type" "sseiadd")
5633 (set_attr "atom_unit" "simul")
5634 (set_attr "prefix_data16" "1,*")
5635 (set_attr "prefix" "orig,vex")
5636 (set_attr "mode" "TI")])
5638 (define_expand "mul<mode>3"
5639 [(set (match_operand:VI4_AVX2 0 "register_operand")
5641 (match_operand:VI4_AVX2 1 "general_vector_operand")
5642 (match_operand:VI4_AVX2 2 "general_vector_operand")))]
5647 if (!nonimmediate_operand (operands[1], <MODE>mode))
5648 operands[1] = force_reg (<MODE>mode, operands[1]);
5649 if (!nonimmediate_operand (operands[2], <MODE>mode))
5650 operands[2] = force_reg (<MODE>mode, operands[2]);
5651 ix86_fixup_binary_operands_no_copy (MULT, <MODE>mode, operands);
5655 ix86_expand_sse2_mulv4si3 (operands[0], operands[1], operands[2]);
5660 (define_insn "*<sse4_1_avx2>_mul<mode>3"
5661 [(set (match_operand:VI4_AVX2 0 "register_operand" "=x,x")
5663 (match_operand:VI4_AVX2 1 "nonimmediate_operand" "%0,x")
5664 (match_operand:VI4_AVX2 2 "nonimmediate_operand" "xm,xm")))]
5665 "TARGET_SSE4_1 && ix86_binary_operator_ok (MULT, <MODE>mode, operands)"
5667 pmulld\t{%2, %0|%0, %2}
5668 vpmulld\t{%2, %1, %0|%0, %1, %2}"
5669 [(set_attr "isa" "noavx,avx")
5670 (set_attr "type" "sseimul")
5671 (set_attr "prefix_extra" "1")
5672 (set_attr "prefix" "orig,vex")
5673 (set_attr "btver2_decode" "vector,vector")
5674 (set_attr "mode" "<sseinsnmode>")])
5676 (define_expand "mul<mode>3"
5677 [(set (match_operand:VI8_AVX2 0 "register_operand")
5678 (mult:VI8_AVX2 (match_operand:VI8_AVX2 1 "register_operand")
5679 (match_operand:VI8_AVX2 2 "register_operand")))]
5682 ix86_expand_sse2_mulvxdi3 (operands[0], operands[1], operands[2]);
5686 (define_expand "vec_widen_<s>mult_hi_<mode>"
5687 [(match_operand:<sseunpackmode> 0 "register_operand")
5688 (any_extend:<sseunpackmode>
5689 (match_operand:VI124_AVX2 1 "register_operand"))
5690 (match_operand:VI124_AVX2 2 "register_operand")]
5693 ix86_expand_mul_widen_hilo (operands[0], operands[1], operands[2],
5698 (define_expand "vec_widen_<s>mult_lo_<mode>"
5699 [(match_operand:<sseunpackmode> 0 "register_operand")
5700 (any_extend:<sseunpackmode>
5701 (match_operand:VI124_AVX2 1 "register_operand"))
5702 (match_operand:VI124_AVX2 2 "register_operand")]
5705 ix86_expand_mul_widen_hilo (operands[0], operands[1], operands[2],
5710 ;; Most widen_<s>mult_even_<mode> can be handled directly from other
5711 ;; named patterns, but signed V4SI needs special help for plain SSE2.
5712 (define_expand "vec_widen_smult_even_v4si"
5713 [(match_operand:V2DI 0 "register_operand")
5714 (match_operand:V4SI 1 "nonimmediate_operand")
5715 (match_operand:V4SI 2 "nonimmediate_operand")]
5718 ix86_expand_mul_widen_evenodd (operands[0], operands[1], operands[2],
5723 (define_expand "vec_widen_<s>mult_odd_<mode>"
5724 [(match_operand:<sseunpackmode> 0 "register_operand")
5725 (any_extend:<sseunpackmode>
5726 (match_operand:VI4_AVX2 1 "general_vector_operand"))
5727 (match_operand:VI4_AVX2 2 "general_vector_operand")]
5730 ix86_expand_mul_widen_evenodd (operands[0], operands[1], operands[2],
5735 (define_expand "sdot_prod<mode>"
5736 [(match_operand:<sseunpackmode> 0 "register_operand")
5737 (match_operand:VI2_AVX2 1 "register_operand")
5738 (match_operand:VI2_AVX2 2 "register_operand")
5739 (match_operand:<sseunpackmode> 3 "register_operand")]
5742 rtx t = gen_reg_rtx (<sseunpackmode>mode);
5743 emit_insn (gen_<sse2_avx2>_pmaddwd (t, operands[1], operands[2]));
5744 emit_insn (gen_rtx_SET (VOIDmode, operands[0],
5745 gen_rtx_PLUS (<sseunpackmode>mode,
5750 ;; Normally we use widen_mul_even/odd, but combine can't quite get it all
5751 ;; back together when madd is available.
5752 (define_expand "sdot_prodv4si"
5753 [(match_operand:V2DI 0 "register_operand")
5754 (match_operand:V4SI 1 "register_operand")
5755 (match_operand:V4SI 2 "register_operand")
5756 (match_operand:V2DI 3 "register_operand")]
5759 rtx t = gen_reg_rtx (V2DImode);
5760 emit_insn (gen_xop_pmacsdqh (t, operands[1], operands[2], operands[3]));
5761 emit_insn (gen_xop_pmacsdql (operands[0], operands[1], operands[2], t));
5765 (define_insn "ashr<mode>3"
5766 [(set (match_operand:VI24_AVX2 0 "register_operand" "=x,x")
5768 (match_operand:VI24_AVX2 1 "register_operand" "0,x")
5769 (match_operand:SI 2 "nonmemory_operand" "xN,xN")))]
5772 psra<ssemodesuffix>\t{%2, %0|%0, %2}
5773 vpsra<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
5774 [(set_attr "isa" "noavx,avx")
5775 (set_attr "type" "sseishft")
5776 (set (attr "length_immediate")
5777 (if_then_else (match_operand 2 "const_int_operand")
5779 (const_string "0")))
5780 (set_attr "prefix_data16" "1,*")
5781 (set_attr "prefix" "orig,vex")
5782 (set_attr "mode" "<sseinsnmode>")])
5784 (define_insn "<shift_insn><mode>3"
5785 [(set (match_operand:VI248_AVX2 0 "register_operand" "=x,x")
5786 (any_lshift:VI248_AVX2
5787 (match_operand:VI248_AVX2 1 "register_operand" "0,x")
5788 (match_operand:SI 2 "nonmemory_operand" "xN,xN")))]
5791 p<vshift><ssemodesuffix>\t{%2, %0|%0, %2}
5792 vp<vshift><ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
5793 [(set_attr "isa" "noavx,avx")
5794 (set_attr "type" "sseishft")
5795 (set (attr "length_immediate")
5796 (if_then_else (match_operand 2 "const_int_operand")
5798 (const_string "0")))
5799 (set_attr "prefix_data16" "1,*")
5800 (set_attr "prefix" "orig,vex")
5801 (set_attr "mode" "<sseinsnmode>")])
5803 (define_expand "vec_shl_<mode>"
5804 [(set (match_operand:VI_128 0 "register_operand")
5806 (match_operand:VI_128 1 "register_operand")
5807 (match_operand:SI 2 "const_0_to_255_mul_8_operand")))]
5810 operands[0] = gen_lowpart (V1TImode, operands[0]);
5811 operands[1] = gen_lowpart (V1TImode, operands[1]);
5814 (define_insn "<sse2_avx2>_ashl<mode>3"
5815 [(set (match_operand:VIMAX_AVX2 0 "register_operand" "=x,x")
5817 (match_operand:VIMAX_AVX2 1 "register_operand" "0,x")
5818 (match_operand:SI 2 "const_0_to_255_mul_8_operand" "n,n")))]
5821 operands[2] = GEN_INT (INTVAL (operands[2]) / 8);
5823 switch (which_alternative)
5826 return "pslldq\t{%2, %0|%0, %2}";
5828 return "vpslldq\t{%2, %1, %0|%0, %1, %2}";
5833 [(set_attr "isa" "noavx,avx")
5834 (set_attr "type" "sseishft")
5835 (set_attr "length_immediate" "1")
5836 (set_attr "prefix_data16" "1,*")
5837 (set_attr "prefix" "orig,vex")
5838 (set_attr "mode" "<sseinsnmode>")])
5840 (define_expand "vec_shr_<mode>"
5841 [(set (match_operand:VI_128 0 "register_operand")
5843 (match_operand:VI_128 1 "register_operand")
5844 (match_operand:SI 2 "const_0_to_255_mul_8_operand")))]
5847 operands[0] = gen_lowpart (V1TImode, operands[0]);
5848 operands[1] = gen_lowpart (V1TImode, operands[1]);
5851 (define_insn "<sse2_avx2>_lshr<mode>3"
5852 [(set (match_operand:VIMAX_AVX2 0 "register_operand" "=x,x")
5853 (lshiftrt:VIMAX_AVX2
5854 (match_operand:VIMAX_AVX2 1 "register_operand" "0,x")
5855 (match_operand:SI 2 "const_0_to_255_mul_8_operand" "n,n")))]
5858 operands[2] = GEN_INT (INTVAL (operands[2]) / 8);
5860 switch (which_alternative)
5863 return "psrldq\t{%2, %0|%0, %2}";
5865 return "vpsrldq\t{%2, %1, %0|%0, %1, %2}";
5870 [(set_attr "isa" "noavx,avx")
5871 (set_attr "type" "sseishft")
5872 (set_attr "length_immediate" "1")
5873 (set_attr "atom_unit" "sishuf")
5874 (set_attr "prefix_data16" "1,*")
5875 (set_attr "prefix" "orig,vex")
5876 (set_attr "mode" "<sseinsnmode>")])
5879 (define_expand "<code><mode>3"
5880 [(set (match_operand:VI124_256 0 "register_operand")
5882 (match_operand:VI124_256 1 "nonimmediate_operand")
5883 (match_operand:VI124_256 2 "nonimmediate_operand")))]
5885 "ix86_fixup_binary_operands_no_copy (<CODE>, <MODE>mode, operands);")
5887 (define_insn "*avx2_<code><mode>3"
5888 [(set (match_operand:VI124_256 0 "register_operand" "=x")
5890 (match_operand:VI124_256 1 "nonimmediate_operand" "%x")
5891 (match_operand:VI124_256 2 "nonimmediate_operand" "xm")))]
5892 "TARGET_AVX2 && ix86_binary_operator_ok (<CODE>, <MODE>mode, operands)"
5893 "vp<maxmin_int><ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
5894 [(set_attr "type" "sseiadd")
5895 (set_attr "prefix_extra" "1")
5896 (set_attr "prefix" "vex")
5897 (set_attr "mode" "OI")])
5899 (define_expand "<code><mode>3"
5900 [(set (match_operand:VI8_AVX2 0 "register_operand")
5902 (match_operand:VI8_AVX2 1 "register_operand")
5903 (match_operand:VI8_AVX2 2 "register_operand")))]
5910 xops[0] = operands[0];
5912 if (<CODE> == SMAX || <CODE> == UMAX)
5914 xops[1] = operands[1];
5915 xops[2] = operands[2];
5919 xops[1] = operands[2];
5920 xops[2] = operands[1];
5923 code = (<CODE> == UMAX || <CODE> == UMIN) ? GTU : GT;
5925 xops[3] = gen_rtx_fmt_ee (code, VOIDmode, operands[1], operands[2]);
5926 xops[4] = operands[1];
5927 xops[5] = operands[2];
5929 ok = ix86_expand_int_vcond (xops);
5934 (define_expand "<code><mode>3"
5935 [(set (match_operand:VI124_128 0 "register_operand")
5937 (match_operand:VI124_128 1 "nonimmediate_operand")
5938 (match_operand:VI124_128 2 "nonimmediate_operand")))]
5941 if (TARGET_SSE4_1 || <MODE>mode == V8HImode)
5942 ix86_fixup_binary_operands_no_copy (<CODE>, <MODE>mode, operands);
5948 xops[0] = operands[0];
5949 operands[1] = force_reg (<MODE>mode, operands[1]);
5950 operands[2] = force_reg (<MODE>mode, operands[2]);
5954 xops[1] = operands[1];
5955 xops[2] = operands[2];
5959 xops[1] = operands[2];
5960 xops[2] = operands[1];
5963 xops[3] = gen_rtx_GT (VOIDmode, operands[1], operands[2]);
5964 xops[4] = operands[1];
5965 xops[5] = operands[2];
5967 ok = ix86_expand_int_vcond (xops);
5973 (define_insn "*sse4_1_<code><mode>3"
5974 [(set (match_operand:VI14_128 0 "register_operand" "=x,x")
5976 (match_operand:VI14_128 1 "nonimmediate_operand" "%0,x")
5977 (match_operand:VI14_128 2 "nonimmediate_operand" "xm,xm")))]
5978 "TARGET_SSE4_1 && ix86_binary_operator_ok (<CODE>, <MODE>mode, operands)"
5980 p<maxmin_int><ssemodesuffix>\t{%2, %0|%0, %2}
5981 vp<maxmin_int><ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
5982 [(set_attr "isa" "noavx,avx")
5983 (set_attr "type" "sseiadd")
5984 (set_attr "prefix_extra" "1,*")
5985 (set_attr "prefix" "orig,vex")
5986 (set_attr "mode" "TI")])
5988 (define_insn "*<code>v8hi3"
5989 [(set (match_operand:V8HI 0 "register_operand" "=x,x")
5991 (match_operand:V8HI 1 "nonimmediate_operand" "%0,x")
5992 (match_operand:V8HI 2 "nonimmediate_operand" "xm,xm")))]
5993 "TARGET_SSE2 && ix86_binary_operator_ok (<CODE>, V8HImode, operands)"
5995 p<maxmin_int>w\t{%2, %0|%0, %2}
5996 vp<maxmin_int>w\t{%2, %1, %0|%0, %1, %2}"
5997 [(set_attr "isa" "noavx,avx")
5998 (set_attr "type" "sseiadd")
5999 (set_attr "prefix_data16" "1,*")
6000 (set_attr "prefix_extra" "*,1")
6001 (set_attr "prefix" "orig,vex")
6002 (set_attr "mode" "TI")])
6004 (define_expand "<code><mode>3"
6005 [(set (match_operand:VI124_128 0 "register_operand")
6007 (match_operand:VI124_128 1 "nonimmediate_operand")
6008 (match_operand:VI124_128 2 "nonimmediate_operand")))]
6011 if (TARGET_SSE4_1 || <MODE>mode == V16QImode)
6012 ix86_fixup_binary_operands_no_copy (<CODE>, <MODE>mode, operands);
6013 else if (<CODE> == UMAX && <MODE>mode == V8HImode)
6015 rtx op0 = operands[0], op2 = operands[2], op3 = op0;
6016 operands[1] = force_reg (<MODE>mode, operands[1]);
6017 if (rtx_equal_p (op3, op2))
6018 op3 = gen_reg_rtx (V8HImode);
6019 emit_insn (gen_sse2_ussubv8hi3 (op3, operands[1], op2));
6020 emit_insn (gen_addv8hi3 (op0, op3, op2));
6028 operands[1] = force_reg (<MODE>mode, operands[1]);
6029 operands[2] = force_reg (<MODE>mode, operands[2]);
6031 xops[0] = operands[0];
6035 xops[1] = operands[1];
6036 xops[2] = operands[2];
6040 xops[1] = operands[2];
6041 xops[2] = operands[1];
6044 xops[3] = gen_rtx_GTU (VOIDmode, operands[1], operands[2]);
6045 xops[4] = operands[1];
6046 xops[5] = operands[2];
6048 ok = ix86_expand_int_vcond (xops);
6054 (define_insn "*sse4_1_<code><mode>3"
6055 [(set (match_operand:VI24_128 0 "register_operand" "=x,x")
6057 (match_operand:VI24_128 1 "nonimmediate_operand" "%0,x")
6058 (match_operand:VI24_128 2 "nonimmediate_operand" "xm,xm")))]
6059 "TARGET_SSE4_1 && ix86_binary_operator_ok (<CODE>, <MODE>mode, operands)"
6061 p<maxmin_int><ssemodesuffix>\t{%2, %0|%0, %2}
6062 vp<maxmin_int><ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
6063 [(set_attr "isa" "noavx,avx")
6064 (set_attr "type" "sseiadd")
6065 (set_attr "prefix_extra" "1,*")
6066 (set_attr "prefix" "orig,vex")
6067 (set_attr "mode" "TI")])
6069 (define_insn "*<code>v16qi3"
6070 [(set (match_operand:V16QI 0 "register_operand" "=x,x")
6072 (match_operand:V16QI 1 "nonimmediate_operand" "%0,x")
6073 (match_operand:V16QI 2 "nonimmediate_operand" "xm,xm")))]
6074 "TARGET_SSE2 && ix86_binary_operator_ok (<CODE>, V16QImode, operands)"
6076 p<maxmin_int>b\t{%2, %0|%0, %2}
6077 vp<maxmin_int>b\t{%2, %1, %0|%0, %1, %2}"
6078 [(set_attr "isa" "noavx,avx")
6079 (set_attr "type" "sseiadd")
6080 (set_attr "prefix_data16" "1,*")
6081 (set_attr "prefix_extra" "*,1")
6082 (set_attr "prefix" "orig,vex")
6083 (set_attr "mode" "TI")])
6085 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
6087 ;; Parallel integral comparisons
6089 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
6091 (define_expand "avx2_eq<mode>3"
6092 [(set (match_operand:VI_256 0 "register_operand")
6094 (match_operand:VI_256 1 "nonimmediate_operand")
6095 (match_operand:VI_256 2 "nonimmediate_operand")))]
6097 "ix86_fixup_binary_operands_no_copy (EQ, <MODE>mode, operands);")
6099 (define_insn "*avx2_eq<mode>3"
6100 [(set (match_operand:VI_256 0 "register_operand" "=x")
6102 (match_operand:VI_256 1 "nonimmediate_operand" "%x")
6103 (match_operand:VI_256 2 "nonimmediate_operand" "xm")))]
6104 "TARGET_AVX2 && ix86_binary_operator_ok (EQ, <MODE>mode, operands)"
6105 "vpcmpeq<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
6106 [(set_attr "type" "ssecmp")
6107 (set_attr "prefix_extra" "1")
6108 (set_attr "prefix" "vex")
6109 (set_attr "mode" "OI")])
6111 (define_insn "*sse4_1_eqv2di3"
6112 [(set (match_operand:V2DI 0 "register_operand" "=x,x")
6114 (match_operand:V2DI 1 "nonimmediate_operand" "%0,x")
6115 (match_operand:V2DI 2 "nonimmediate_operand" "xm,xm")))]
6116 "TARGET_SSE4_1 && ix86_binary_operator_ok (EQ, V2DImode, operands)"
6118 pcmpeqq\t{%2, %0|%0, %2}
6119 vpcmpeqq\t{%2, %1, %0|%0, %1, %2}"
6120 [(set_attr "isa" "noavx,avx")
6121 (set_attr "type" "ssecmp")
6122 (set_attr "prefix_extra" "1")
6123 (set_attr "prefix" "orig,vex")
6124 (set_attr "mode" "TI")])
6126 (define_insn "*sse2_eq<mode>3"
6127 [(set (match_operand:VI124_128 0 "register_operand" "=x,x")
6129 (match_operand:VI124_128 1 "nonimmediate_operand" "%0,x")
6130 (match_operand:VI124_128 2 "nonimmediate_operand" "xm,xm")))]
6131 "TARGET_SSE2 && !TARGET_XOP
6132 && ix86_binary_operator_ok (EQ, <MODE>mode, operands)"
6134 pcmpeq<ssemodesuffix>\t{%2, %0|%0, %2}
6135 vpcmpeq<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
6136 [(set_attr "isa" "noavx,avx")
6137 (set_attr "type" "ssecmp")
6138 (set_attr "prefix_data16" "1,*")
6139 (set_attr "prefix" "orig,vex")
6140 (set_attr "mode" "TI")])
6142 (define_expand "sse2_eq<mode>3"
6143 [(set (match_operand:VI124_128 0 "register_operand")
6145 (match_operand:VI124_128 1 "nonimmediate_operand")
6146 (match_operand:VI124_128 2 "nonimmediate_operand")))]
6147 "TARGET_SSE2 && !TARGET_XOP "
6148 "ix86_fixup_binary_operands_no_copy (EQ, <MODE>mode, operands);")
6150 (define_expand "sse4_1_eqv2di3"
6151 [(set (match_operand:V2DI 0 "register_operand")
6153 (match_operand:V2DI 1 "nonimmediate_operand")
6154 (match_operand:V2DI 2 "nonimmediate_operand")))]
6156 "ix86_fixup_binary_operands_no_copy (EQ, V2DImode, operands);")
6158 (define_insn "sse4_2_gtv2di3"
6159 [(set (match_operand:V2DI 0 "register_operand" "=x,x")
6161 (match_operand:V2DI 1 "register_operand" "0,x")
6162 (match_operand:V2DI 2 "nonimmediate_operand" "xm,xm")))]
6165 pcmpgtq\t{%2, %0|%0, %2}
6166 vpcmpgtq\t{%2, %1, %0|%0, %1, %2}"
6167 [(set_attr "isa" "noavx,avx")
6168 (set_attr "type" "ssecmp")
6169 (set_attr "prefix_extra" "1")
6170 (set_attr "prefix" "orig,vex")
6171 (set_attr "mode" "TI")])
6173 (define_insn "avx2_gt<mode>3"
6174 [(set (match_operand:VI_256 0 "register_operand" "=x")
6176 (match_operand:VI_256 1 "register_operand" "x")
6177 (match_operand:VI_256 2 "nonimmediate_operand" "xm")))]
6179 "vpcmpgt<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
6180 [(set_attr "type" "ssecmp")
6181 (set_attr "prefix_extra" "1")
6182 (set_attr "prefix" "vex")
6183 (set_attr "mode" "OI")])
6185 (define_insn "sse2_gt<mode>3"
6186 [(set (match_operand:VI124_128 0 "register_operand" "=x,x")
6188 (match_operand:VI124_128 1 "register_operand" "0,x")
6189 (match_operand:VI124_128 2 "nonimmediate_operand" "xm,xm")))]
6190 "TARGET_SSE2 && !TARGET_XOP"
6192 pcmpgt<ssemodesuffix>\t{%2, %0|%0, %2}
6193 vpcmpgt<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
6194 [(set_attr "isa" "noavx,avx")
6195 (set_attr "type" "ssecmp")
6196 (set_attr "prefix_data16" "1,*")
6197 (set_attr "prefix" "orig,vex")
6198 (set_attr "mode" "TI")])
6200 (define_expand "vcond<V_256:mode><VI_256:mode>"
6201 [(set (match_operand:V_256 0 "register_operand")
6203 (match_operator 3 ""
6204 [(match_operand:VI_256 4 "nonimmediate_operand")
6205 (match_operand:VI_256 5 "general_operand")])
6206 (match_operand:V_256 1)
6207 (match_operand:V_256 2)))]
6209 && (GET_MODE_NUNITS (<V_256:MODE>mode)
6210 == GET_MODE_NUNITS (<VI_256:MODE>mode))"
6212 bool ok = ix86_expand_int_vcond (operands);
6217 (define_expand "vcond<V_128:mode><VI124_128:mode>"
6218 [(set (match_operand:V_128 0 "register_operand")
6220 (match_operator 3 ""
6221 [(match_operand:VI124_128 4 "nonimmediate_operand")
6222 (match_operand:VI124_128 5 "general_operand")])
6223 (match_operand:V_128 1)
6224 (match_operand:V_128 2)))]
6226 && (GET_MODE_NUNITS (<V_128:MODE>mode)
6227 == GET_MODE_NUNITS (<VI124_128:MODE>mode))"
6229 bool ok = ix86_expand_int_vcond (operands);
6234 (define_expand "vcond<VI8F_128:mode>v2di"
6235 [(set (match_operand:VI8F_128 0 "register_operand")
6236 (if_then_else:VI8F_128
6237 (match_operator 3 ""
6238 [(match_operand:V2DI 4 "nonimmediate_operand")
6239 (match_operand:V2DI 5 "general_operand")])
6240 (match_operand:VI8F_128 1)
6241 (match_operand:VI8F_128 2)))]
6244 bool ok = ix86_expand_int_vcond (operands);
6249 (define_expand "vcondu<V_256:mode><VI_256:mode>"
6250 [(set (match_operand:V_256 0 "register_operand")
6252 (match_operator 3 ""
6253 [(match_operand:VI_256 4 "nonimmediate_operand")
6254 (match_operand:VI_256 5 "nonimmediate_operand")])
6255 (match_operand:V_256 1 "general_operand")
6256 (match_operand:V_256 2 "general_operand")))]
6258 && (GET_MODE_NUNITS (<V_256:MODE>mode)
6259 == GET_MODE_NUNITS (<VI_256:MODE>mode))"
6261 bool ok = ix86_expand_int_vcond (operands);
6266 (define_expand "vcondu<V_128:mode><VI124_128:mode>"
6267 [(set (match_operand:V_128 0 "register_operand")
6269 (match_operator 3 ""
6270 [(match_operand:VI124_128 4 "nonimmediate_operand")
6271 (match_operand:VI124_128 5 "nonimmediate_operand")])
6272 (match_operand:V_128 1 "general_operand")
6273 (match_operand:V_128 2 "general_operand")))]
6275 && (GET_MODE_NUNITS (<V_128:MODE>mode)
6276 == GET_MODE_NUNITS (<VI124_128:MODE>mode))"
6278 bool ok = ix86_expand_int_vcond (operands);
6283 (define_expand "vcondu<VI8F_128:mode>v2di"
6284 [(set (match_operand:VI8F_128 0 "register_operand")
6285 (if_then_else:VI8F_128
6286 (match_operator 3 ""
6287 [(match_operand:V2DI 4 "nonimmediate_operand")
6288 (match_operand:V2DI 5 "nonimmediate_operand")])
6289 (match_operand:VI8F_128 1 "general_operand")
6290 (match_operand:VI8F_128 2 "general_operand")))]
6293 bool ok = ix86_expand_int_vcond (operands);
6298 (define_mode_iterator VEC_PERM_AVX2
6299 [V16QI V8HI V4SI V2DI V4SF V2DF
6300 (V32QI "TARGET_AVX2") (V16HI "TARGET_AVX2")
6301 (V8SI "TARGET_AVX2") (V4DI "TARGET_AVX2")
6302 (V8SF "TARGET_AVX2") (V4DF "TARGET_AVX2")])
6304 (define_expand "vec_perm<mode>"
6305 [(match_operand:VEC_PERM_AVX2 0 "register_operand")
6306 (match_operand:VEC_PERM_AVX2 1 "register_operand")
6307 (match_operand:VEC_PERM_AVX2 2 "register_operand")
6308 (match_operand:<sseintvecmode> 3 "register_operand")]
6309 "TARGET_SSSE3 || TARGET_AVX || TARGET_XOP"
6311 ix86_expand_vec_perm (operands);
6315 (define_mode_iterator VEC_PERM_CONST
6316 [(V4SF "TARGET_SSE") (V4SI "TARGET_SSE")
6317 (V2DF "TARGET_SSE") (V2DI "TARGET_SSE")
6318 (V16QI "TARGET_SSE2") (V8HI "TARGET_SSE2")
6319 (V8SF "TARGET_AVX") (V4DF "TARGET_AVX")
6320 (V8SI "TARGET_AVX") (V4DI "TARGET_AVX")
6321 (V32QI "TARGET_AVX2") (V16HI "TARGET_AVX2")])
6323 (define_expand "vec_perm_const<mode>"
6324 [(match_operand:VEC_PERM_CONST 0 "register_operand")
6325 (match_operand:VEC_PERM_CONST 1 "register_operand")
6326 (match_operand:VEC_PERM_CONST 2 "register_operand")
6327 (match_operand:<sseintvecmode> 3)]
6330 if (ix86_expand_vec_perm_const (operands))
6336 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
6338 ;; Parallel bitwise logical operations
6340 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
6342 (define_expand "one_cmpl<mode>2"
6343 [(set (match_operand:VI 0 "register_operand")
6344 (xor:VI (match_operand:VI 1 "nonimmediate_operand")
6348 int i, n = GET_MODE_NUNITS (<MODE>mode);
6349 rtvec v = rtvec_alloc (n);
6351 for (i = 0; i < n; ++i)
6352 RTVEC_ELT (v, i) = constm1_rtx;
6354 operands[2] = force_reg (<MODE>mode, gen_rtx_CONST_VECTOR (<MODE>mode, v));
6357 (define_expand "<sse2_avx2>_andnot<mode>3"
6358 [(set (match_operand:VI_AVX2 0 "register_operand")
6360 (not:VI_AVX2 (match_operand:VI_AVX2 1 "register_operand"))
6361 (match_operand:VI_AVX2 2 "nonimmediate_operand")))]
6364 (define_insn "*andnot<mode>3"
6365 [(set (match_operand:VI 0 "register_operand" "=x,x")
6367 (not:VI (match_operand:VI 1 "register_operand" "0,x"))
6368 (match_operand:VI 2 "nonimmediate_operand" "xm,xm")))]
6371 static char buf[32];
6375 switch (get_attr_mode (insn))
6378 gcc_assert (TARGET_AVX2);
6380 gcc_assert (TARGET_SSE2);
6386 gcc_assert (TARGET_AVX);
6388 gcc_assert (TARGET_SSE);
6397 switch (which_alternative)
6400 ops = "%s\t{%%2, %%0|%%0, %%2}";
6403 ops = "v%s\t{%%2, %%1, %%0|%%0, %%1, %%2}";
6409 snprintf (buf, sizeof (buf), ops, tmp);
6412 [(set_attr "isa" "noavx,avx")
6413 (set_attr "type" "sselog")
6414 (set (attr "prefix_data16")
6416 (and (eq_attr "alternative" "0")
6417 (eq_attr "mode" "TI"))
6419 (const_string "*")))
6420 (set_attr "prefix" "orig,vex")
6422 (cond [(match_test "TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL")
6423 (const_string "<ssePSmode>")
6424 (match_test "TARGET_AVX2")
6425 (const_string "<sseinsnmode>")
6426 (match_test "TARGET_AVX")
6428 (match_test "GET_MODE_SIZE (<MODE>mode) > 16")
6429 (const_string "V8SF")
6430 (const_string "<sseinsnmode>"))
6431 (ior (not (match_test "TARGET_SSE2"))
6432 (match_test "optimize_function_for_size_p (cfun)"))
6433 (const_string "V4SF")
6435 (const_string "<sseinsnmode>")))])
6437 (define_expand "<code><mode>3"
6438 [(set (match_operand:VI 0 "register_operand")
6440 (match_operand:VI 1 "nonimmediate_or_const_vector_operand")
6441 (match_operand:VI 2 "nonimmediate_or_const_vector_operand")))]
6444 ix86_expand_vector_logical_operator (<CODE>, <MODE>mode, operands);
6448 (define_insn "*<code><mode>3"
6449 [(set (match_operand:VI 0 "register_operand" "=x,x")
6451 (match_operand:VI 1 "nonimmediate_operand" "%0,x")
6452 (match_operand:VI 2 "nonimmediate_operand" "xm,xm")))]
6454 && ix86_binary_operator_ok (<CODE>, <MODE>mode, operands)"
6456 static char buf[32];
6460 switch (get_attr_mode (insn))
6463 gcc_assert (TARGET_AVX2);
6465 gcc_assert (TARGET_SSE2);
6471 gcc_assert (TARGET_AVX);
6473 gcc_assert (TARGET_SSE);
6482 switch (which_alternative)
6485 ops = "%s\t{%%2, %%0|%%0, %%2}";
6488 ops = "v%s\t{%%2, %%1, %%0|%%0, %%1, %%2}";
6494 snprintf (buf, sizeof (buf), ops, tmp);
6497 [(set_attr "isa" "noavx,avx")
6498 (set_attr "type" "sselog")
6499 (set (attr "prefix_data16")
6501 (and (eq_attr "alternative" "0")
6502 (eq_attr "mode" "TI"))
6504 (const_string "*")))
6505 (set_attr "prefix" "orig,vex")
6507 (cond [(match_test "TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL")
6508 (const_string "<ssePSmode>")
6509 (match_test "TARGET_AVX2")
6510 (const_string "<sseinsnmode>")
6511 (match_test "TARGET_AVX")
6513 (match_test "GET_MODE_SIZE (<MODE>mode) > 16")
6514 (const_string "V8SF")
6515 (const_string "<sseinsnmode>"))
6516 (ior (not (match_test "TARGET_SSE2"))
6517 (match_test "optimize_function_for_size_p (cfun)"))
6518 (const_string "V4SF")
6520 (const_string "<sseinsnmode>")))])
6522 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
6524 ;; Parallel integral element swizzling
6526 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
6528 (define_expand "vec_pack_trunc_<mode>"
6529 [(match_operand:<ssepackmode> 0 "register_operand")
6530 (match_operand:VI248_AVX2 1 "register_operand")
6531 (match_operand:VI248_AVX2 2 "register_operand")]
6534 rtx op1 = gen_lowpart (<ssepackmode>mode, operands[1]);
6535 rtx op2 = gen_lowpart (<ssepackmode>mode, operands[2]);
6536 ix86_expand_vec_extract_even_odd (operands[0], op1, op2, 0);
6540 (define_insn "<sse2_avx2>_packsswb"
6541 [(set (match_operand:VI1_AVX2 0 "register_operand" "=x,x")
6542 (vec_concat:VI1_AVX2
6543 (ss_truncate:<ssehalfvecmode>
6544 (match_operand:<sseunpackmode> 1 "register_operand" "0,x"))
6545 (ss_truncate:<ssehalfvecmode>
6546 (match_operand:<sseunpackmode> 2 "nonimmediate_operand" "xm,xm"))))]
6549 packsswb\t{%2, %0|%0, %2}
6550 vpacksswb\t{%2, %1, %0|%0, %1, %2}"
6551 [(set_attr "isa" "noavx,avx")
6552 (set_attr "type" "sselog")
6553 (set_attr "prefix_data16" "1,*")
6554 (set_attr "prefix" "orig,vex")
6555 (set_attr "mode" "<sseinsnmode>")])
6557 (define_insn "<sse2_avx2>_packssdw"
6558 [(set (match_operand:VI2_AVX2 0 "register_operand" "=x,x")
6559 (vec_concat:VI2_AVX2
6560 (ss_truncate:<ssehalfvecmode>
6561 (match_operand:<sseunpackmode> 1 "register_operand" "0,x"))
6562 (ss_truncate:<ssehalfvecmode>
6563 (match_operand:<sseunpackmode> 2 "nonimmediate_operand" "xm,xm"))))]
6566 packssdw\t{%2, %0|%0, %2}
6567 vpackssdw\t{%2, %1, %0|%0, %1, %2}"
6568 [(set_attr "isa" "noavx,avx")
6569 (set_attr "type" "sselog")
6570 (set_attr "prefix_data16" "1,*")
6571 (set_attr "prefix" "orig,vex")
6572 (set_attr "mode" "<sseinsnmode>")])
6574 (define_insn "<sse2_avx2>_packuswb"
6575 [(set (match_operand:VI1_AVX2 0 "register_operand" "=x,x")
6576 (vec_concat:VI1_AVX2
6577 (us_truncate:<ssehalfvecmode>
6578 (match_operand:<sseunpackmode> 1 "register_operand" "0,x"))
6579 (us_truncate:<ssehalfvecmode>
6580 (match_operand:<sseunpackmode> 2 "nonimmediate_operand" "xm,xm"))))]
6583 packuswb\t{%2, %0|%0, %2}
6584 vpackuswb\t{%2, %1, %0|%0, %1, %2}"
6585 [(set_attr "isa" "noavx,avx")
6586 (set_attr "type" "sselog")
6587 (set_attr "prefix_data16" "1,*")
6588 (set_attr "prefix" "orig,vex")
6589 (set_attr "mode" "<sseinsnmode>")])
6591 (define_insn "avx2_interleave_highv32qi"
6592 [(set (match_operand:V32QI 0 "register_operand" "=x")
6595 (match_operand:V32QI 1 "register_operand" "x")
6596 (match_operand:V32QI 2 "nonimmediate_operand" "xm"))
6597 (parallel [(const_int 8) (const_int 40)
6598 (const_int 9) (const_int 41)
6599 (const_int 10) (const_int 42)
6600 (const_int 11) (const_int 43)
6601 (const_int 12) (const_int 44)
6602 (const_int 13) (const_int 45)
6603 (const_int 14) (const_int 46)
6604 (const_int 15) (const_int 47)
6605 (const_int 24) (const_int 56)
6606 (const_int 25) (const_int 57)
6607 (const_int 26) (const_int 58)
6608 (const_int 27) (const_int 59)
6609 (const_int 28) (const_int 60)
6610 (const_int 29) (const_int 61)
6611 (const_int 30) (const_int 62)
6612 (const_int 31) (const_int 63)])))]
6614 "vpunpckhbw\t{%2, %1, %0|%0, %1, %2}"
6615 [(set_attr "type" "sselog")
6616 (set_attr "prefix" "vex")
6617 (set_attr "mode" "OI")])
6619 (define_insn "vec_interleave_highv16qi"
6620 [(set (match_operand:V16QI 0 "register_operand" "=x,x")
6623 (match_operand:V16QI 1 "register_operand" "0,x")
6624 (match_operand:V16QI 2 "nonimmediate_operand" "xm,xm"))
6625 (parallel [(const_int 8) (const_int 24)
6626 (const_int 9) (const_int 25)
6627 (const_int 10) (const_int 26)
6628 (const_int 11) (const_int 27)
6629 (const_int 12) (const_int 28)
6630 (const_int 13) (const_int 29)
6631 (const_int 14) (const_int 30)
6632 (const_int 15) (const_int 31)])))]
6635 punpckhbw\t{%2, %0|%0, %2}
6636 vpunpckhbw\t{%2, %1, %0|%0, %1, %2}"
6637 [(set_attr "isa" "noavx,avx")
6638 (set_attr "type" "sselog")
6639 (set_attr "prefix_data16" "1,*")
6640 (set_attr "prefix" "orig,vex")
6641 (set_attr "mode" "TI")])
6643 (define_insn "avx2_interleave_lowv32qi"
6644 [(set (match_operand:V32QI 0 "register_operand" "=x")
6647 (match_operand:V32QI 1 "register_operand" "x")
6648 (match_operand:V32QI 2 "nonimmediate_operand" "xm"))
6649 (parallel [(const_int 0) (const_int 32)
6650 (const_int 1) (const_int 33)
6651 (const_int 2) (const_int 34)
6652 (const_int 3) (const_int 35)
6653 (const_int 4) (const_int 36)
6654 (const_int 5) (const_int 37)
6655 (const_int 6) (const_int 38)
6656 (const_int 7) (const_int 39)
6657 (const_int 16) (const_int 48)
6658 (const_int 17) (const_int 49)
6659 (const_int 18) (const_int 50)
6660 (const_int 19) (const_int 51)
6661 (const_int 20) (const_int 52)
6662 (const_int 21) (const_int 53)
6663 (const_int 22) (const_int 54)
6664 (const_int 23) (const_int 55)])))]
6666 "vpunpcklbw\t{%2, %1, %0|%0, %1, %2}"
6667 [(set_attr "type" "sselog")
6668 (set_attr "prefix" "vex")
6669 (set_attr "mode" "OI")])
6671 (define_insn "vec_interleave_lowv16qi"
6672 [(set (match_operand:V16QI 0 "register_operand" "=x,x")
6675 (match_operand:V16QI 1 "register_operand" "0,x")
6676 (match_operand:V16QI 2 "nonimmediate_operand" "xm,xm"))
6677 (parallel [(const_int 0) (const_int 16)
6678 (const_int 1) (const_int 17)
6679 (const_int 2) (const_int 18)
6680 (const_int 3) (const_int 19)
6681 (const_int 4) (const_int 20)
6682 (const_int 5) (const_int 21)
6683 (const_int 6) (const_int 22)
6684 (const_int 7) (const_int 23)])))]
6687 punpcklbw\t{%2, %0|%0, %2}
6688 vpunpcklbw\t{%2, %1, %0|%0, %1, %2}"
6689 [(set_attr "isa" "noavx,avx")
6690 (set_attr "type" "sselog")
6691 (set_attr "prefix_data16" "1,*")
6692 (set_attr "prefix" "orig,vex")
6693 (set_attr "mode" "TI")])
6695 (define_insn "avx2_interleave_highv16hi"
6696 [(set (match_operand:V16HI 0 "register_operand" "=x")
6699 (match_operand:V16HI 1 "register_operand" "x")
6700 (match_operand:V16HI 2 "nonimmediate_operand" "xm"))
6701 (parallel [(const_int 4) (const_int 20)
6702 (const_int 5) (const_int 21)
6703 (const_int 6) (const_int 22)
6704 (const_int 7) (const_int 23)
6705 (const_int 12) (const_int 28)
6706 (const_int 13) (const_int 29)
6707 (const_int 14) (const_int 30)
6708 (const_int 15) (const_int 31)])))]
6710 "vpunpckhwd\t{%2, %1, %0|%0, %1, %2}"
6711 [(set_attr "type" "sselog")
6712 (set_attr "prefix" "vex")
6713 (set_attr "mode" "OI")])
6715 (define_insn "vec_interleave_highv8hi"
6716 [(set (match_operand:V8HI 0 "register_operand" "=x,x")
6719 (match_operand:V8HI 1 "register_operand" "0,x")
6720 (match_operand:V8HI 2 "nonimmediate_operand" "xm,xm"))
6721 (parallel [(const_int 4) (const_int 12)
6722 (const_int 5) (const_int 13)
6723 (const_int 6) (const_int 14)
6724 (const_int 7) (const_int 15)])))]
6727 punpckhwd\t{%2, %0|%0, %2}
6728 vpunpckhwd\t{%2, %1, %0|%0, %1, %2}"
6729 [(set_attr "isa" "noavx,avx")
6730 (set_attr "type" "sselog")
6731 (set_attr "prefix_data16" "1,*")
6732 (set_attr "prefix" "orig,vex")
6733 (set_attr "mode" "TI")])
6735 (define_insn "avx2_interleave_lowv16hi"
6736 [(set (match_operand:V16HI 0 "register_operand" "=x")
6739 (match_operand:V16HI 1 "register_operand" "x")
6740 (match_operand:V16HI 2 "nonimmediate_operand" "xm"))
6741 (parallel [(const_int 0) (const_int 16)
6742 (const_int 1) (const_int 17)
6743 (const_int 2) (const_int 18)
6744 (const_int 3) (const_int 19)
6745 (const_int 8) (const_int 24)
6746 (const_int 9) (const_int 25)
6747 (const_int 10) (const_int 26)
6748 (const_int 11) (const_int 27)])))]
6750 "vpunpcklwd\t{%2, %1, %0|%0, %1, %2}"
6751 [(set_attr "type" "sselog")
6752 (set_attr "prefix" "vex")
6753 (set_attr "mode" "OI")])
6755 (define_insn "vec_interleave_lowv8hi"
6756 [(set (match_operand:V8HI 0 "register_operand" "=x,x")
6759 (match_operand:V8HI 1 "register_operand" "0,x")
6760 (match_operand:V8HI 2 "nonimmediate_operand" "xm,xm"))
6761 (parallel [(const_int 0) (const_int 8)
6762 (const_int 1) (const_int 9)
6763 (const_int 2) (const_int 10)
6764 (const_int 3) (const_int 11)])))]
6767 punpcklwd\t{%2, %0|%0, %2}
6768 vpunpcklwd\t{%2, %1, %0|%0, %1, %2}"
6769 [(set_attr "isa" "noavx,avx")
6770 (set_attr "type" "sselog")
6771 (set_attr "prefix_data16" "1,*")
6772 (set_attr "prefix" "orig,vex")
6773 (set_attr "mode" "TI")])
6775 (define_insn "avx2_interleave_highv8si"
6776 [(set (match_operand:V8SI 0 "register_operand" "=x")
6779 (match_operand:V8SI 1 "register_operand" "x")
6780 (match_operand:V8SI 2 "nonimmediate_operand" "xm"))
6781 (parallel [(const_int 2) (const_int 10)
6782 (const_int 3) (const_int 11)
6783 (const_int 6) (const_int 14)
6784 (const_int 7) (const_int 15)])))]
6786 "vpunpckhdq\t{%2, %1, %0|%0, %1, %2}"
6787 [(set_attr "type" "sselog")
6788 (set_attr "prefix" "vex")
6789 (set_attr "mode" "OI")])
6791 (define_insn "vec_interleave_highv4si"
6792 [(set (match_operand:V4SI 0 "register_operand" "=x,x")
6795 (match_operand:V4SI 1 "register_operand" "0,x")
6796 (match_operand:V4SI 2 "nonimmediate_operand" "xm,xm"))
6797 (parallel [(const_int 2) (const_int 6)
6798 (const_int 3) (const_int 7)])))]
6801 punpckhdq\t{%2, %0|%0, %2}
6802 vpunpckhdq\t{%2, %1, %0|%0, %1, %2}"
6803 [(set_attr "isa" "noavx,avx")
6804 (set_attr "type" "sselog")
6805 (set_attr "prefix_data16" "1,*")
6806 (set_attr "prefix" "orig,vex")
6807 (set_attr "mode" "TI")])
6809 (define_insn "avx2_interleave_lowv8si"
6810 [(set (match_operand:V8SI 0 "register_operand" "=x")
6813 (match_operand:V8SI 1 "register_operand" "x")
6814 (match_operand:V8SI 2 "nonimmediate_operand" "xm"))
6815 (parallel [(const_int 0) (const_int 8)
6816 (const_int 1) (const_int 9)
6817 (const_int 4) (const_int 12)
6818 (const_int 5) (const_int 13)])))]
6820 "vpunpckldq\t{%2, %1, %0|%0, %1, %2}"
6821 [(set_attr "type" "sselog")
6822 (set_attr "prefix" "vex")
6823 (set_attr "mode" "OI")])
6825 (define_insn "vec_interleave_lowv4si"
6826 [(set (match_operand:V4SI 0 "register_operand" "=x,x")
6829 (match_operand:V4SI 1 "register_operand" "0,x")
6830 (match_operand:V4SI 2 "nonimmediate_operand" "xm,xm"))
6831 (parallel [(const_int 0) (const_int 4)
6832 (const_int 1) (const_int 5)])))]
6835 punpckldq\t{%2, %0|%0, %2}
6836 vpunpckldq\t{%2, %1, %0|%0, %1, %2}"
6837 [(set_attr "isa" "noavx,avx")
6838 (set_attr "type" "sselog")
6839 (set_attr "prefix_data16" "1,*")
6840 (set_attr "prefix" "orig,vex")
6841 (set_attr "mode" "TI")])
6843 (define_expand "vec_interleave_high<mode>"
6844 [(match_operand:VI_256 0 "register_operand" "=x")
6845 (match_operand:VI_256 1 "register_operand" "x")
6846 (match_operand:VI_256 2 "nonimmediate_operand" "xm")]
6849 rtx t1 = gen_reg_rtx (<MODE>mode);
6850 rtx t2 = gen_reg_rtx (<MODE>mode);
6851 emit_insn (gen_avx2_interleave_low<mode> (t1, operands[1], operands[2]));
6852 emit_insn (gen_avx2_interleave_high<mode> (t2, operands[1], operands[2]));
6853 emit_insn (gen_avx2_permv2ti
6854 (gen_lowpart (V4DImode, operands[0]),
6855 gen_lowpart (V4DImode, t1),
6856 gen_lowpart (V4DImode, t2), GEN_INT (1 + (3 << 4))));
6860 (define_expand "vec_interleave_low<mode>"
6861 [(match_operand:VI_256 0 "register_operand" "=x")
6862 (match_operand:VI_256 1 "register_operand" "x")
6863 (match_operand:VI_256 2 "nonimmediate_operand" "xm")]
6866 rtx t1 = gen_reg_rtx (<MODE>mode);
6867 rtx t2 = gen_reg_rtx (<MODE>mode);
6868 emit_insn (gen_avx2_interleave_low<mode> (t1, operands[1], operands[2]));
6869 emit_insn (gen_avx2_interleave_high<mode> (t2, operands[1], operands[2]));
6870 emit_insn (gen_avx2_permv2ti
6871 (gen_lowpart (V4DImode, operands[0]),
6872 gen_lowpart (V4DImode, t1),
6873 gen_lowpart (V4DImode, t2), GEN_INT (0 + (2 << 4))));
6877 ;; Modes handled by pinsr patterns.
6878 (define_mode_iterator PINSR_MODE
6879 [(V16QI "TARGET_SSE4_1") V8HI
6880 (V4SI "TARGET_SSE4_1")
6881 (V2DI "TARGET_SSE4_1 && TARGET_64BIT")])
6883 (define_mode_attr sse2p4_1
6884 [(V16QI "sse4_1") (V8HI "sse2")
6885 (V4SI "sse4_1") (V2DI "sse4_1")])
6887 ;; sse4_1_pinsrd must come before sse2_loadld since it is preferred.
6888 (define_insn "<sse2p4_1>_pinsr<ssemodesuffix>"
6889 [(set (match_operand:PINSR_MODE 0 "register_operand" "=x,x,x,x")
6890 (vec_merge:PINSR_MODE
6891 (vec_duplicate:PINSR_MODE
6892 (match_operand:<ssescalarmode> 2 "nonimmediate_operand" "r,m,r,m"))
6893 (match_operand:PINSR_MODE 1 "register_operand" "0,0,x,x")
6894 (match_operand:SI 3 "const_int_operand")))]
6896 && ((unsigned) exact_log2 (INTVAL (operands[3]))
6897 < GET_MODE_NUNITS (<MODE>mode))"
6899 operands[3] = GEN_INT (exact_log2 (INTVAL (operands[3])));
6901 switch (which_alternative)
6904 if (GET_MODE_SIZE (<ssescalarmode>mode) < GET_MODE_SIZE (SImode))
6905 return "pinsr<ssemodesuffix>\t{%3, %k2, %0|%0, %k2, %3}";
6908 return "pinsr<ssemodesuffix>\t{%3, %2, %0|%0, %2, %3}";
6910 if (GET_MODE_SIZE (<ssescalarmode>mode) < GET_MODE_SIZE (SImode))
6911 return "vpinsr<ssemodesuffix>\t{%3, %k2, %1, %0|%0, %1, %k2, %3}";
6914 return "vpinsr<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}";
6919 [(set_attr "isa" "noavx,noavx,avx,avx")
6920 (set_attr "type" "sselog")
6921 (set (attr "prefix_rex")
6923 (and (not (match_test "TARGET_AVX"))
6924 (eq (const_string "<MODE>mode") (const_string "V2DImode")))
6926 (const_string "*")))
6927 (set (attr "prefix_data16")
6929 (and (not (match_test "TARGET_AVX"))
6930 (eq (const_string "<MODE>mode") (const_string "V8HImode")))
6932 (const_string "*")))
6933 (set (attr "prefix_extra")
6935 (and (not (match_test "TARGET_AVX"))
6936 (eq (const_string "<MODE>mode") (const_string "V8HImode")))
6938 (const_string "1")))
6939 (set_attr "length_immediate" "1")
6940 (set_attr "prefix" "orig,orig,vex,vex")
6941 (set_attr "mode" "TI")])
6943 (define_expand "avx2_pshufdv3"
6944 [(match_operand:V8SI 0 "register_operand")
6945 (match_operand:V8SI 1 "nonimmediate_operand")
6946 (match_operand:SI 2 "const_0_to_255_operand")]
6949 int mask = INTVAL (operands[2]);
6950 emit_insn (gen_avx2_pshufd_1 (operands[0], operands[1],
6951 GEN_INT ((mask >> 0) & 3),
6952 GEN_INT ((mask >> 2) & 3),
6953 GEN_INT ((mask >> 4) & 3),
6954 GEN_INT ((mask >> 6) & 3),
6955 GEN_INT (((mask >> 0) & 3) + 4),
6956 GEN_INT (((mask >> 2) & 3) + 4),
6957 GEN_INT (((mask >> 4) & 3) + 4),
6958 GEN_INT (((mask >> 6) & 3) + 4)));
6962 (define_insn "avx2_pshufd_1"
6963 [(set (match_operand:V8SI 0 "register_operand" "=x")
6965 (match_operand:V8SI 1 "nonimmediate_operand" "xm")
6966 (parallel [(match_operand 2 "const_0_to_3_operand")
6967 (match_operand 3 "const_0_to_3_operand")
6968 (match_operand 4 "const_0_to_3_operand")
6969 (match_operand 5 "const_0_to_3_operand")
6970 (match_operand 6 "const_4_to_7_operand")
6971 (match_operand 7 "const_4_to_7_operand")
6972 (match_operand 8 "const_4_to_7_operand")
6973 (match_operand 9 "const_4_to_7_operand")])))]
6975 && INTVAL (operands[2]) + 4 == INTVAL (operands[6])
6976 && INTVAL (operands[3]) + 4 == INTVAL (operands[7])
6977 && INTVAL (operands[4]) + 4 == INTVAL (operands[8])
6978 && INTVAL (operands[5]) + 4 == INTVAL (operands[9])"
6981 mask |= INTVAL (operands[2]) << 0;
6982 mask |= INTVAL (operands[3]) << 2;
6983 mask |= INTVAL (operands[4]) << 4;
6984 mask |= INTVAL (operands[5]) << 6;
6985 operands[2] = GEN_INT (mask);
6987 return "vpshufd\t{%2, %1, %0|%0, %1, %2}";
6989 [(set_attr "type" "sselog1")
6990 (set_attr "prefix" "vex")
6991 (set_attr "length_immediate" "1")
6992 (set_attr "mode" "OI")])
6994 (define_expand "sse2_pshufd"
6995 [(match_operand:V4SI 0 "register_operand")
6996 (match_operand:V4SI 1 "nonimmediate_operand")
6997 (match_operand:SI 2 "const_int_operand")]
7000 int mask = INTVAL (operands[2]);
7001 emit_insn (gen_sse2_pshufd_1 (operands[0], operands[1],
7002 GEN_INT ((mask >> 0) & 3),
7003 GEN_INT ((mask >> 2) & 3),
7004 GEN_INT ((mask >> 4) & 3),
7005 GEN_INT ((mask >> 6) & 3)));
7009 (define_insn "sse2_pshufd_1"
7010 [(set (match_operand:V4SI 0 "register_operand" "=x")
7012 (match_operand:V4SI 1 "nonimmediate_operand" "xm")
7013 (parallel [(match_operand 2 "const_0_to_3_operand")
7014 (match_operand 3 "const_0_to_3_operand")
7015 (match_operand 4 "const_0_to_3_operand")
7016 (match_operand 5 "const_0_to_3_operand")])))]
7020 mask |= INTVAL (operands[2]) << 0;
7021 mask |= INTVAL (operands[3]) << 2;
7022 mask |= INTVAL (operands[4]) << 4;
7023 mask |= INTVAL (operands[5]) << 6;
7024 operands[2] = GEN_INT (mask);
7026 return "%vpshufd\t{%2, %1, %0|%0, %1, %2}";
7028 [(set_attr "type" "sselog1")
7029 (set_attr "prefix_data16" "1")
7030 (set_attr "prefix" "maybe_vex")
7031 (set_attr "length_immediate" "1")
7032 (set_attr "mode" "TI")])
7034 (define_expand "avx2_pshuflwv3"
7035 [(match_operand:V16HI 0 "register_operand")
7036 (match_operand:V16HI 1 "nonimmediate_operand")
7037 (match_operand:SI 2 "const_0_to_255_operand")]
7040 int mask = INTVAL (operands[2]);
7041 emit_insn (gen_avx2_pshuflw_1 (operands[0], operands[1],
7042 GEN_INT ((mask >> 0) & 3),
7043 GEN_INT ((mask >> 2) & 3),
7044 GEN_INT ((mask >> 4) & 3),
7045 GEN_INT ((mask >> 6) & 3),
7046 GEN_INT (((mask >> 0) & 3) + 8),
7047 GEN_INT (((mask >> 2) & 3) + 8),
7048 GEN_INT (((mask >> 4) & 3) + 8),
7049 GEN_INT (((mask >> 6) & 3) + 8)));
7053 (define_insn "avx2_pshuflw_1"
7054 [(set (match_operand:V16HI 0 "register_operand" "=x")
7056 (match_operand:V16HI 1 "nonimmediate_operand" "xm")
7057 (parallel [(match_operand 2 "const_0_to_3_operand")
7058 (match_operand 3 "const_0_to_3_operand")
7059 (match_operand 4 "const_0_to_3_operand")
7060 (match_operand 5 "const_0_to_3_operand")
7065 (match_operand 6 "const_8_to_11_operand")
7066 (match_operand 7 "const_8_to_11_operand")
7067 (match_operand 8 "const_8_to_11_operand")
7068 (match_operand 9 "const_8_to_11_operand")
7074 && INTVAL (operands[2]) + 8 == INTVAL (operands[6])
7075 && INTVAL (operands[3]) + 8 == INTVAL (operands[7])
7076 && INTVAL (operands[4]) + 8 == INTVAL (operands[8])
7077 && INTVAL (operands[5]) + 8 == INTVAL (operands[9])"
7080 mask |= INTVAL (operands[2]) << 0;
7081 mask |= INTVAL (operands[3]) << 2;
7082 mask |= INTVAL (operands[4]) << 4;
7083 mask |= INTVAL (operands[5]) << 6;
7084 operands[2] = GEN_INT (mask);
7086 return "vpshuflw\t{%2, %1, %0|%0, %1, %2}";
7088 [(set_attr "type" "sselog")
7089 (set_attr "prefix" "vex")
7090 (set_attr "length_immediate" "1")
7091 (set_attr "mode" "OI")])
7093 (define_expand "sse2_pshuflw"
7094 [(match_operand:V8HI 0 "register_operand")
7095 (match_operand:V8HI 1 "nonimmediate_operand")
7096 (match_operand:SI 2 "const_int_operand")]
7099 int mask = INTVAL (operands[2]);
7100 emit_insn (gen_sse2_pshuflw_1 (operands[0], operands[1],
7101 GEN_INT ((mask >> 0) & 3),
7102 GEN_INT ((mask >> 2) & 3),
7103 GEN_INT ((mask >> 4) & 3),
7104 GEN_INT ((mask >> 6) & 3)));
7108 (define_insn "sse2_pshuflw_1"
7109 [(set (match_operand:V8HI 0 "register_operand" "=x")
7111 (match_operand:V8HI 1 "nonimmediate_operand" "xm")
7112 (parallel [(match_operand 2 "const_0_to_3_operand")
7113 (match_operand 3 "const_0_to_3_operand")
7114 (match_operand 4 "const_0_to_3_operand")
7115 (match_operand 5 "const_0_to_3_operand")
7123 mask |= INTVAL (operands[2]) << 0;
7124 mask |= INTVAL (operands[3]) << 2;
7125 mask |= INTVAL (operands[4]) << 4;
7126 mask |= INTVAL (operands[5]) << 6;
7127 operands[2] = GEN_INT (mask);
7129 return "%vpshuflw\t{%2, %1, %0|%0, %1, %2}";
7131 [(set_attr "type" "sselog")
7132 (set_attr "prefix_data16" "0")
7133 (set_attr "prefix_rep" "1")
7134 (set_attr "prefix" "maybe_vex")
7135 (set_attr "length_immediate" "1")
7136 (set_attr "mode" "TI")])
7138 (define_expand "avx2_pshufhwv3"
7139 [(match_operand:V16HI 0 "register_operand")
7140 (match_operand:V16HI 1 "nonimmediate_operand")
7141 (match_operand:SI 2 "const_0_to_255_operand")]
7144 int mask = INTVAL (operands[2]);
7145 emit_insn (gen_avx2_pshufhw_1 (operands[0], operands[1],
7146 GEN_INT (((mask >> 0) & 3) + 4),
7147 GEN_INT (((mask >> 2) & 3) + 4),
7148 GEN_INT (((mask >> 4) & 3) + 4),
7149 GEN_INT (((mask >> 6) & 3) + 4),
7150 GEN_INT (((mask >> 0) & 3) + 12),
7151 GEN_INT (((mask >> 2) & 3) + 12),
7152 GEN_INT (((mask >> 4) & 3) + 12),
7153 GEN_INT (((mask >> 6) & 3) + 12)));
7157 (define_insn "avx2_pshufhw_1"
7158 [(set (match_operand:V16HI 0 "register_operand" "=x")
7160 (match_operand:V16HI 1 "nonimmediate_operand" "xm")
7161 (parallel [(const_int 0)
7165 (match_operand 2 "const_4_to_7_operand")
7166 (match_operand 3 "const_4_to_7_operand")
7167 (match_operand 4 "const_4_to_7_operand")
7168 (match_operand 5 "const_4_to_7_operand")
7173 (match_operand 6 "const_12_to_15_operand")
7174 (match_operand 7 "const_12_to_15_operand")
7175 (match_operand 8 "const_12_to_15_operand")
7176 (match_operand 9 "const_12_to_15_operand")])))]
7178 && INTVAL (operands[2]) + 8 == INTVAL (operands[6])
7179 && INTVAL (operands[3]) + 8 == INTVAL (operands[7])
7180 && INTVAL (operands[4]) + 8 == INTVAL (operands[8])
7181 && INTVAL (operands[5]) + 8 == INTVAL (operands[9])"
7184 mask |= (INTVAL (operands[2]) - 4) << 0;
7185 mask |= (INTVAL (operands[3]) - 4) << 2;
7186 mask |= (INTVAL (operands[4]) - 4) << 4;
7187 mask |= (INTVAL (operands[5]) - 4) << 6;
7188 operands[2] = GEN_INT (mask);
7190 return "vpshufhw\t{%2, %1, %0|%0, %1, %2}";
7192 [(set_attr "type" "sselog")
7193 (set_attr "prefix" "vex")
7194 (set_attr "length_immediate" "1")
7195 (set_attr "mode" "OI")])
7197 (define_expand "sse2_pshufhw"
7198 [(match_operand:V8HI 0 "register_operand")
7199 (match_operand:V8HI 1 "nonimmediate_operand")
7200 (match_operand:SI 2 "const_int_operand")]
7203 int mask = INTVAL (operands[2]);
7204 emit_insn (gen_sse2_pshufhw_1 (operands[0], operands[1],
7205 GEN_INT (((mask >> 0) & 3) + 4),
7206 GEN_INT (((mask >> 2) & 3) + 4),
7207 GEN_INT (((mask >> 4) & 3) + 4),
7208 GEN_INT (((mask >> 6) & 3) + 4)));
7212 (define_insn "sse2_pshufhw_1"
7213 [(set (match_operand:V8HI 0 "register_operand" "=x")
7215 (match_operand:V8HI 1 "nonimmediate_operand" "xm")
7216 (parallel [(const_int 0)
7220 (match_operand 2 "const_4_to_7_operand")
7221 (match_operand 3 "const_4_to_7_operand")
7222 (match_operand 4 "const_4_to_7_operand")
7223 (match_operand 5 "const_4_to_7_operand")])))]
7227 mask |= (INTVAL (operands[2]) - 4) << 0;
7228 mask |= (INTVAL (operands[3]) - 4) << 2;
7229 mask |= (INTVAL (operands[4]) - 4) << 4;
7230 mask |= (INTVAL (operands[5]) - 4) << 6;
7231 operands[2] = GEN_INT (mask);
7233 return "%vpshufhw\t{%2, %1, %0|%0, %1, %2}";
7235 [(set_attr "type" "sselog")
7236 (set_attr "prefix_rep" "1")
7237 (set_attr "prefix_data16" "0")
7238 (set_attr "prefix" "maybe_vex")
7239 (set_attr "length_immediate" "1")
7240 (set_attr "mode" "TI")])
7242 (define_expand "sse2_loadd"
7243 [(set (match_operand:V4SI 0 "register_operand")
7246 (match_operand:SI 1 "nonimmediate_operand"))
7250 "operands[2] = CONST0_RTX (V4SImode);")
7252 (define_insn "sse2_loadld"
7253 [(set (match_operand:V4SI 0 "register_operand" "=x,Yi,x,x,x")
7256 (match_operand:SI 2 "nonimmediate_operand" "m ,r ,m,x,x"))
7257 (match_operand:V4SI 1 "reg_or_0_operand" "C ,C ,C,0,x")
7261 %vmovd\t{%2, %0|%0, %2}
7262 %vmovd\t{%2, %0|%0, %2}
7263 movss\t{%2, %0|%0, %2}
7264 movss\t{%2, %0|%0, %2}
7265 vmovss\t{%2, %1, %0|%0, %1, %2}"
7266 [(set_attr "isa" "sse2,*,noavx,noavx,avx")
7267 (set_attr "type" "ssemov")
7268 (set_attr "prefix" "maybe_vex,maybe_vex,orig,orig,vex")
7269 (set_attr "mode" "TI,TI,V4SF,SF,SF")])
7271 (define_insn "*vec_extract<mode>"
7272 [(set (match_operand:<ssescalarmode> 0 "nonimmediate_operand" "=r,m")
7273 (vec_select:<ssescalarmode>
7274 (match_operand:VI12_128 1 "register_operand" "x,x")
7276 [(match_operand:SI 2 "const_0_to_<ssescalarnummask>_operand")])))]
7279 %vpextr<ssemodesuffix>\t{%2, %1, %k0|%k0, %1, %2}
7280 %vpextr<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
7281 [(set_attr "type" "sselog1")
7282 (set (attr "prefix_data16")
7284 (and (eq_attr "alternative" "0")
7285 (eq (const_string "<MODE>mode") (const_string "V8HImode")))
7287 (const_string "*")))
7288 (set (attr "prefix_extra")
7290 (and (eq_attr "alternative" "0")
7291 (eq (const_string "<MODE>mode") (const_string "V8HImode")))
7293 (const_string "1")))
7294 (set_attr "length_immediate" "1")
7295 (set_attr "prefix" "maybe_vex")
7296 (set_attr "mode" "TI")])
7298 (define_insn "*vec_extractv8hi_sse2"
7299 [(set (match_operand:HI 0 "register_operand" "=r")
7301 (match_operand:V8HI 1 "register_operand" "x")
7303 [(match_operand:SI 2 "const_0_to_7_operand")])))]
7304 "TARGET_SSE2 && !TARGET_SSE4_1"
7305 "pextrw\t{%2, %1, %k0|%k0, %1, %2}"
7306 [(set_attr "type" "sselog1")
7307 (set_attr "prefix_data16" "1")
7308 (set_attr "length_immediate" "1")
7309 (set_attr "mode" "TI")])
7311 (define_insn "*vec_extractv16qi_zext"
7312 [(set (match_operand:SWI48 0 "register_operand" "=r")
7315 (match_operand:V16QI 1 "register_operand" "x")
7317 [(match_operand:SI 2 "const_0_to_15_operand")]))))]
7319 "%vpextrb\t{%2, %1, %k0|%k0, %1, %2}"
7320 [(set_attr "type" "sselog1")
7321 (set_attr "prefix_extra" "1")
7322 (set_attr "length_immediate" "1")
7323 (set_attr "prefix" "maybe_vex")
7324 (set_attr "mode" "TI")])
7326 (define_insn "*vec_extractv8hi_zext"
7327 [(set (match_operand:SWI48 0 "register_operand" "=r")
7330 (match_operand:V8HI 1 "register_operand" "x")
7332 [(match_operand:SI 2 "const_0_to_7_operand")]))))]
7334 "%vpextrw\t{%2, %1, %k0|%k0, %1, %2}"
7335 [(set_attr "type" "sselog1")
7336 (set_attr "prefix_data16" "1")
7337 (set_attr "length_immediate" "1")
7338 (set_attr "prefix" "maybe_vex")
7339 (set_attr "mode" "TI")])
7341 (define_insn "*vec_extract<mode>_mem"
7342 [(set (match_operand:<ssescalarmode> 0 "register_operand" "=r")
7343 (vec_select:<ssescalarmode>
7344 (match_operand:VI12_128 1 "memory_operand" "o")
7346 [(match_operand 2 "const_0_to_<ssescalarnummask>_operand")])))]
7350 (define_insn "*vec_extract<ssevecmodelower>_0"
7351 [(set (match_operand:SWI48 0 "nonimmediate_operand" "=r ,r,x ,m")
7353 (match_operand:<ssevecmode> 1 "nonimmediate_operand" "mYj,x,xm,x")
7354 (parallel [(const_int 0)])))]
7355 "TARGET_SSE && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
7357 [(set_attr "isa" "*,sse4,*,*")])
7359 (define_insn "*vec_extractv2di_0_sse"
7360 [(set (match_operand:DI 0 "nonimmediate_operand" "=x,m")
7362 (match_operand:V2DI 1 "nonimmediate_operand" "xm,x")
7363 (parallel [(const_int 0)])))]
7364 "TARGET_SSE && !TARGET_64BIT
7365 && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
7369 [(set (match_operand:SWI48x 0 "nonimmediate_operand")
7371 (match_operand:<ssevecmode> 1 "register_operand")
7372 (parallel [(const_int 0)])))]
7373 "TARGET_SSE && reload_completed"
7374 [(set (match_dup 0) (match_dup 1))]
7375 "operands[1] = gen_rtx_REG (<MODE>mode, REGNO (operands[1]));")
7377 (define_insn "*vec_extractv4si"
7378 [(set (match_operand:SI 0 "nonimmediate_operand" "=rm")
7380 (match_operand:V4SI 1 "register_operand" "x")
7381 (parallel [(match_operand:SI 2 "const_0_to_3_operand")])))]
7383 "%vpextrd\t{%2, %1, %0|%0, %1, %2}"
7384 [(set_attr "type" "sselog1")
7385 (set_attr "prefix_extra" "1")
7386 (set_attr "length_immediate" "1")
7387 (set_attr "prefix" "maybe_vex")
7388 (set_attr "mode" "TI")])
7390 (define_insn "*vec_extractv4si_zext"
7391 [(set (match_operand:DI 0 "register_operand" "=r")
7394 (match_operand:V4SI 1 "register_operand" "x")
7395 (parallel [(match_operand:SI 2 "const_0_to_3_operand")]))))]
7396 "TARGET_64BIT && TARGET_SSE4_1"
7397 "%vpextrd\t{%2, %1, %k0|%k0, %1, %2}"
7398 [(set_attr "type" "sselog1")
7399 (set_attr "prefix_extra" "1")
7400 (set_attr "length_immediate" "1")
7401 (set_attr "prefix" "maybe_vex")
7402 (set_attr "mode" "TI")])
7404 (define_insn "*vec_extractv4si_mem"
7405 [(set (match_operand:SI 0 "register_operand" "=x,r")
7407 (match_operand:V4SI 1 "memory_operand" "o,o")
7408 (parallel [(match_operand 2 "const_0_to_3_operand")])))]
7412 (define_insn "*vec_extractv2di_1"
7413 [(set (match_operand:DI 0 "nonimmediate_operand" "=rm,m,x,x,x,x,r")
7415 (match_operand:V2DI 1 "nonimmediate_operand" "x ,x,0,x,x,o,o")
7416 (parallel [(const_int 1)])))]
7417 "TARGET_SSE && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
7419 %vpextrq\t{$1, %1, %0|%0, %1, 1}
7420 %vmovhps\t{%1, %0|%0, %1}
7421 psrldq\t{$8, %0|%0, 8}
7422 vpsrldq\t{$8, %1, %0|%0, %1, 8}
7423 movhlps\t{%1, %0|%0, %1}
7426 [(set_attr "isa" "x64_sse4,*,sse2_noavx,avx,noavx,*,x64")
7427 (set_attr "type" "sselog1,ssemov,sseishft1,sseishft1,ssemov,ssemov,imov")
7428 (set_attr "length_immediate" "1,*,1,1,*,*,*")
7429 (set_attr "memory" "*,*,none,none,*,*,*")
7430 (set_attr "prefix_rex" "1,*,*,*,*,*,*")
7431 (set_attr "prefix_extra" "1,*,*,*,*,*,*")
7432 (set_attr "prefix" "maybe_vex,maybe_vex,orig,vex,orig,*,*")
7433 (set_attr "mode" "TI,V2SF,TI,TI,V4SF,DI,DI")])
7436 [(set (match_operand:<ssescalarmode> 0 "register_operand")
7437 (vec_select:<ssescalarmode>
7438 (match_operand:VI_128 1 "memory_operand")
7440 [(match_operand 2 "const_0_to_<ssescalarnummask>_operand")])))]
7441 "TARGET_SSE && reload_completed"
7442 [(set (match_dup 0) (match_dup 1))]
7444 int offs = INTVAL (operands[2]) * GET_MODE_SIZE (<ssescalarmode>mode);
7446 operands[1] = adjust_address (operands[1], <ssescalarmode>mode, offs);
7449 (define_insn "*vec_dupv4si"
7450 [(set (match_operand:V4SI 0 "register_operand" "=x,x,x")
7452 (match_operand:SI 1 "nonimmediate_operand" " x,m,0")))]
7455 %vpshufd\t{$0, %1, %0|%0, %1, 0}
7456 vbroadcastss\t{%1, %0|%0, %1}
7457 shufps\t{$0, %0, %0|%0, %0, 0}"
7458 [(set_attr "isa" "sse2,avx,noavx")
7459 (set_attr "type" "sselog1,ssemov,sselog1")
7460 (set_attr "length_immediate" "1,0,1")
7461 (set_attr "prefix_extra" "0,1,*")
7462 (set_attr "prefix" "maybe_vex,vex,orig")
7463 (set_attr "mode" "TI,V4SF,V4SF")])
7465 (define_insn "*vec_dupv2di"
7466 [(set (match_operand:V2DI 0 "register_operand" "=x,x,x,x")
7468 (match_operand:DI 1 "nonimmediate_operand" " 0,x,m,0")))]
7472 vpunpcklqdq\t{%d1, %0|%0, %d1}
7473 %vmovddup\t{%1, %0|%0, %1}
7475 [(set_attr "isa" "sse2_noavx,avx,sse3,noavx")
7476 (set_attr "type" "sselog1,sselog1,sselog1,ssemov")
7477 (set_attr "prefix" "orig,vex,maybe_vex,orig")
7478 (set_attr "mode" "TI,TI,DF,V4SF")])
7480 (define_insn "*vec_concatv2si_sse4_1"
7481 [(set (match_operand:V2SI 0 "register_operand" "=x, x,x,x, x, *y,*y")
7483 (match_operand:SI 1 "nonimmediate_operand" " 0, x,0,x,rm, 0,rm")
7484 (match_operand:SI 2 "vector_move_operand" "rm,rm,x,x, C,*ym, C")))]
7487 pinsrd\t{$1, %2, %0|%0, %2, 1}
7488 vpinsrd\t{$1, %2, %1, %0|%0, %1, %2, 1}
7489 punpckldq\t{%2, %0|%0, %2}
7490 vpunpckldq\t{%2, %1, %0|%0, %1, %2}
7491 %vmovd\t{%1, %0|%0, %1}
7492 punpckldq\t{%2, %0|%0, %2}
7493 movd\t{%1, %0|%0, %1}"
7494 [(set_attr "isa" "noavx,avx,noavx,avx,*,*,*")
7495 (set_attr "type" "sselog,sselog,sselog,sselog,ssemov,mmxcvt,mmxmov")
7496 (set_attr "prefix_extra" "1,1,*,*,*,*,*")
7497 (set_attr "length_immediate" "1,1,*,*,*,*,*")
7498 (set_attr "prefix" "orig,vex,orig,vex,maybe_vex,orig,orig")
7499 (set_attr "mode" "TI,TI,TI,TI,TI,DI,DI")])
7501 ;; ??? In theory we can match memory for the MMX alternative, but allowing
7502 ;; nonimmediate_operand for operand 2 and *not* allowing memory for the SSE
7503 ;; alternatives pretty much forces the MMX alternative to be chosen.
7504 (define_insn "*vec_concatv2si"
7505 [(set (match_operand:V2SI 0 "register_operand" "=x,x ,*y,x,x,*y,*y")
7507 (match_operand:SI 1 "nonimmediate_operand" " 0,rm,rm,0,m, 0,*rm")
7508 (match_operand:SI 2 "reg_or_0_operand" " x,C ,C, x,C,*y,C")))]
7509 "TARGET_SSE && !TARGET_SSE4_1"
7511 punpckldq\t{%2, %0|%0, %2}
7512 movd\t{%1, %0|%0, %1}
7513 movd\t{%1, %0|%0, %1}
7514 unpcklps\t{%2, %0|%0, %2}
7515 movss\t{%1, %0|%0, %1}
7516 punpckldq\t{%2, %0|%0, %2}
7517 movd\t{%1, %0|%0, %1}"
7518 [(set_attr "isa" "sse2,sse2,sse2,*,*,*,*")
7519 (set_attr "type" "sselog,ssemov,mmxmov,sselog,ssemov,mmxcvt,mmxmov")
7520 (set_attr "mode" "TI,TI,DI,V4SF,SF,DI,DI")])
7522 (define_insn "*vec_concatv4si"
7523 [(set (match_operand:V4SI 0 "register_operand" "=x,x,x,x,x")
7525 (match_operand:V2SI 1 "register_operand" " 0,x,0,0,x")
7526 (match_operand:V2SI 2 "nonimmediate_operand" " x,x,x,m,m")))]
7529 punpcklqdq\t{%2, %0|%0, %2}
7530 vpunpcklqdq\t{%2, %1, %0|%0, %1, %2}
7531 movlhps\t{%2, %0|%0, %2}
7532 movhps\t{%2, %0|%0, %2}
7533 vmovhps\t{%2, %1, %0|%0, %1, %2}"
7534 [(set_attr "isa" "sse2_noavx,avx,noavx,noavx,avx")
7535 (set_attr "type" "sselog,sselog,ssemov,ssemov,ssemov")
7536 (set_attr "prefix" "orig,vex,orig,orig,vex")
7537 (set_attr "mode" "TI,TI,V4SF,V2SF,V2SF")])
7539 ;; movd instead of movq is required to handle broken assemblers.
7540 (define_insn "vec_concatv2di"
7541 [(set (match_operand:V2DI 0 "register_operand"
7542 "=x,x ,Yi,x ,!x,x,x,x,x,x")
7544 (match_operand:DI 1 "nonimmediate_operand"
7545 " 0,x ,r ,xm,*y,0,x,0,0,x")
7546 (match_operand:DI 2 "vector_move_operand"
7547 "rm,rm,C ,C ,C ,x,x,x,m,m")))]
7550 pinsrq\t{$1, %2, %0|%0, %2, 1}
7551 vpinsrq\t{$1, %2, %1, %0|%0, %1, %2, 1}
7552 %vmovd\t{%1, %0|%0, %1}
7553 %vmovq\t{%1, %0|%0, %1}
7554 movq2dq\t{%1, %0|%0, %1}
7555 punpcklqdq\t{%2, %0|%0, %2}
7556 vpunpcklqdq\t{%2, %1, %0|%0, %1, %2}
7557 movlhps\t{%2, %0|%0, %2}
7558 movhps\t{%2, %0|%0, %2}
7559 vmovhps\t{%2, %1, %0|%0, %1, %2}"
7560 [(set_attr "isa" "x64_sse4_noavx,x64_avx,x64,sse2,sse2,sse2_noavx,avx,noavx,noavx,avx")
7563 (eq_attr "alternative" "0,1,5,6")
7564 (const_string "sselog")
7565 (const_string "ssemov")))
7566 (set_attr "prefix_rex" "1,1,1,*,*,*,*,*,*,*")
7567 (set_attr "prefix_extra" "1,1,*,*,*,*,*,*,*,*")
7568 (set_attr "length_immediate" "1,1,*,*,*,*,*,*,*,*")
7569 (set_attr "prefix" "orig,vex,maybe_vex,maybe_vex,orig,orig,vex,orig,orig,vex")
7570 (set_attr "mode" "TI,TI,TI,TI,TI,TI,TI,V4SF,V2SF,V2SF")])
7572 (define_expand "vec_unpacks_lo_<mode>"
7573 [(match_operand:<sseunpackmode> 0 "register_operand")
7574 (match_operand:VI124_AVX2 1 "register_operand")]
7576 "ix86_expand_sse_unpack (operands[0], operands[1], false, false); DONE;")
7578 (define_expand "vec_unpacks_hi_<mode>"
7579 [(match_operand:<sseunpackmode> 0 "register_operand")
7580 (match_operand:VI124_AVX2 1 "register_operand")]
7582 "ix86_expand_sse_unpack (operands[0], operands[1], false, true); DONE;")
7584 (define_expand "vec_unpacku_lo_<mode>"
7585 [(match_operand:<sseunpackmode> 0 "register_operand")
7586 (match_operand:VI124_AVX2 1 "register_operand")]
7588 "ix86_expand_sse_unpack (operands[0], operands[1], true, false); DONE;")
7590 (define_expand "vec_unpacku_hi_<mode>"
7591 [(match_operand:<sseunpackmode> 0 "register_operand")
7592 (match_operand:VI124_AVX2 1 "register_operand")]
7594 "ix86_expand_sse_unpack (operands[0], operands[1], true, true); DONE;")
7596 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
7600 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
7602 (define_expand "<sse2_avx2>_uavg<mode>3"
7603 [(set (match_operand:VI12_AVX2 0 "register_operand")
7605 (lshiftrt:<ssedoublemode>
7606 (plus:<ssedoublemode>
7607 (plus:<ssedoublemode>
7608 (zero_extend:<ssedoublemode>
7609 (match_operand:VI12_AVX2 1 "nonimmediate_operand"))
7610 (zero_extend:<ssedoublemode>
7611 (match_operand:VI12_AVX2 2 "nonimmediate_operand")))
7616 operands[3] = CONST1_RTX(<MODE>mode);
7617 ix86_fixup_binary_operands_no_copy (PLUS, <MODE>mode, operands);
7620 (define_insn "*<sse2_avx2>_uavg<mode>3"
7621 [(set (match_operand:VI12_AVX2 0 "register_operand" "=x,x")
7623 (lshiftrt:<ssedoublemode>
7624 (plus:<ssedoublemode>
7625 (plus:<ssedoublemode>
7626 (zero_extend:<ssedoublemode>
7627 (match_operand:VI12_AVX2 1 "nonimmediate_operand" "%0,x"))
7628 (zero_extend:<ssedoublemode>
7629 (match_operand:VI12_AVX2 2 "nonimmediate_operand" "xm,xm")))
7630 (match_operand:VI12_AVX2 3 "const1_operand"))
7632 "TARGET_SSE2 && ix86_binary_operator_ok (PLUS, <MODE>mode, operands)"
7634 pavg<ssemodesuffix>\t{%2, %0|%0, %2}
7635 vpavg<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
7636 [(set_attr "isa" "noavx,avx")
7637 (set_attr "type" "sseiadd")
7638 (set_attr "prefix_data16" "1,*")
7639 (set_attr "prefix" "orig,vex")
7640 (set_attr "mode" "<sseinsnmode>")])
7642 ;; The correct representation for this is absolutely enormous, and
7643 ;; surely not generally useful.
7644 (define_insn "<sse2_avx2>_psadbw"
7645 [(set (match_operand:VI8_AVX2 0 "register_operand" "=x,x")
7647 [(match_operand:<ssebytemode> 1 "register_operand" "0,x")
7648 (match_operand:<ssebytemode> 2 "nonimmediate_operand" "xm,xm")]
7652 psadbw\t{%2, %0|%0, %2}
7653 vpsadbw\t{%2, %1, %0|%0, %1, %2}"
7654 [(set_attr "isa" "noavx,avx")
7655 (set_attr "type" "sseiadd")
7656 (set_attr "atom_unit" "simul")
7657 (set_attr "prefix_data16" "1,*")
7658 (set_attr "prefix" "orig,vex")
7659 (set_attr "mode" "<sseinsnmode>")])
7661 (define_insn "<sse>_movmsk<ssemodesuffix><avxsizesuffix>"
7662 [(set (match_operand:SI 0 "register_operand" "=r")
7664 [(match_operand:VF 1 "register_operand" "x")]
7667 "%vmovmsk<ssemodesuffix>\t{%1, %0|%0, %1}"
7668 [(set_attr "type" "ssemov")
7669 (set_attr "prefix" "maybe_vex")
7670 (set_attr "mode" "<MODE>")])
7672 (define_insn "avx2_pmovmskb"
7673 [(set (match_operand:SI 0 "register_operand" "=r")
7674 (unspec:SI [(match_operand:V32QI 1 "register_operand" "x")]
7677 "vpmovmskb\t{%1, %0|%0, %1}"
7678 [(set_attr "type" "ssemov")
7679 (set_attr "prefix" "vex")
7680 (set_attr "mode" "DI")])
7682 (define_insn "sse2_pmovmskb"
7683 [(set (match_operand:SI 0 "register_operand" "=r")
7684 (unspec:SI [(match_operand:V16QI 1 "register_operand" "x")]
7687 "%vpmovmskb\t{%1, %0|%0, %1}"
7688 [(set_attr "type" "ssemov")
7689 (set_attr "prefix_data16" "1")
7690 (set_attr "prefix" "maybe_vex")
7691 (set_attr "mode" "SI")])
7693 (define_expand "sse2_maskmovdqu"
7694 [(set (match_operand:V16QI 0 "memory_operand")
7695 (unspec:V16QI [(match_operand:V16QI 1 "register_operand")
7696 (match_operand:V16QI 2 "register_operand")
7701 (define_insn "*sse2_maskmovdqu"
7702 [(set (mem:V16QI (match_operand:P 0 "register_operand" "D"))
7703 (unspec:V16QI [(match_operand:V16QI 1 "register_operand" "x")
7704 (match_operand:V16QI 2 "register_operand" "x")
7705 (mem:V16QI (match_dup 0))]
7708 "%vmaskmovdqu\t{%2, %1|%1, %2}"
7709 [(set_attr "type" "ssemov")
7710 (set_attr "prefix_data16" "1")
7711 ;; The implicit %rdi operand confuses default length_vex computation.
7712 (set (attr "length_vex")
7713 (symbol_ref ("3 + REX_SSE_REGNO_P (REGNO (operands[2]))")))
7714 (set_attr "prefix" "maybe_vex")
7715 (set_attr "mode" "TI")])
7717 (define_insn "sse_ldmxcsr"
7718 [(unspec_volatile [(match_operand:SI 0 "memory_operand" "m")]
7722 [(set_attr "type" "sse")
7723 (set_attr "atom_sse_attr" "mxcsr")
7724 (set_attr "prefix" "maybe_vex")
7725 (set_attr "memory" "load")])
7727 (define_insn "sse_stmxcsr"
7728 [(set (match_operand:SI 0 "memory_operand" "=m")
7729 (unspec_volatile:SI [(const_int 0)] UNSPECV_STMXCSR))]
7732 [(set_attr "type" "sse")
7733 (set_attr "atom_sse_attr" "mxcsr")
7734 (set_attr "prefix" "maybe_vex")
7735 (set_attr "memory" "store")])
7737 (define_insn "sse2_clflush"
7738 [(unspec_volatile [(match_operand 0 "address_operand" "p")]
7742 [(set_attr "type" "sse")
7743 (set_attr "atom_sse_attr" "fence")
7744 (set_attr "memory" "unknown")])
7747 (define_insn "sse3_mwait"
7748 [(unspec_volatile [(match_operand:SI 0 "register_operand" "a")
7749 (match_operand:SI 1 "register_operand" "c")]
7752 ;; 64bit version is "mwait %rax,%rcx". But only lower 32bits are used.
7753 ;; Since 32bit register operands are implicitly zero extended to 64bit,
7754 ;; we only need to set up 32bit registers.
7756 [(set_attr "length" "3")])
7758 (define_insn "sse3_monitor"
7759 [(unspec_volatile [(match_operand:SI 0 "register_operand" "a")
7760 (match_operand:SI 1 "register_operand" "c")
7761 (match_operand:SI 2 "register_operand" "d")]
7763 "TARGET_SSE3 && !TARGET_64BIT"
7764 "monitor\t%0, %1, %2"
7765 [(set_attr "length" "3")])
7767 (define_insn "sse3_monitor64_<mode>"
7768 [(unspec_volatile [(match_operand:P 0 "register_operand" "a")
7769 (match_operand:SI 1 "register_operand" "c")
7770 (match_operand:SI 2 "register_operand" "d")]
7772 "TARGET_SSE3 && TARGET_64BIT"
7773 ;; 64bit version is "monitor %rax,%rcx,%rdx". But only lower 32bits in
7774 ;; RCX and RDX are used. Since 32bit register operands are implicitly
7775 ;; zero extended to 64bit, we only need to set up 32bit registers.
7777 [(set_attr "length" "3")])
7779 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
7781 ;; SSSE3 instructions
7783 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
7785 (define_code_iterator ssse3_plusminus [plus ss_plus minus ss_minus])
7787 (define_insn "avx2_ph<plusminus_mnemonic>wv16hi3"
7788 [(set (match_operand:V16HI 0 "register_operand" "=x")
7795 (match_operand:V16HI 1 "register_operand" "x")
7796 (parallel [(const_int 0)]))
7797 (vec_select:HI (match_dup 1) (parallel [(const_int 1)])))
7799 (vec_select:HI (match_dup 1) (parallel [(const_int 2)]))
7800 (vec_select:HI (match_dup 1) (parallel [(const_int 3)]))))
7803 (vec_select:HI (match_dup 1) (parallel [(const_int 4)]))
7804 (vec_select:HI (match_dup 1) (parallel [(const_int 5)])))
7806 (vec_select:HI (match_dup 1) (parallel [(const_int 6)]))
7807 (vec_select:HI (match_dup 1) (parallel [(const_int 7)])))))
7811 (vec_select:HI (match_dup 1) (parallel [(const_int 8)]))
7812 (vec_select:HI (match_dup 1) (parallel [(const_int 9)])))
7814 (vec_select:HI (match_dup 1) (parallel [(const_int 10)]))
7815 (vec_select:HI (match_dup 1) (parallel [(const_int 11)]))))
7818 (vec_select:HI (match_dup 1) (parallel [(const_int 12)]))
7819 (vec_select:HI (match_dup 1) (parallel [(const_int 13)])))
7821 (vec_select:HI (match_dup 1) (parallel [(const_int 14)]))
7822 (vec_select:HI (match_dup 1) (parallel [(const_int 15)]))))))
7828 (match_operand:V16HI 2 "nonimmediate_operand" "xm")
7829 (parallel [(const_int 0)]))
7830 (vec_select:HI (match_dup 2) (parallel [(const_int 1)])))
7832 (vec_select:HI (match_dup 2) (parallel [(const_int 2)]))
7833 (vec_select:HI (match_dup 2) (parallel [(const_int 3)]))))
7836 (vec_select:HI (match_dup 2) (parallel [(const_int 4)]))
7837 (vec_select:HI (match_dup 2) (parallel [(const_int 5)])))
7839 (vec_select:HI (match_dup 2) (parallel [(const_int 6)]))
7840 (vec_select:HI (match_dup 2) (parallel [(const_int 7)])))))
7844 (vec_select:HI (match_dup 2) (parallel [(const_int 8)]))
7845 (vec_select:HI (match_dup 2) (parallel [(const_int 9)])))
7847 (vec_select:HI (match_dup 2) (parallel [(const_int 10)]))
7848 (vec_select:HI (match_dup 2) (parallel [(const_int 11)]))))
7851 (vec_select:HI (match_dup 2) (parallel [(const_int 12)]))
7852 (vec_select:HI (match_dup 2) (parallel [(const_int 13)])))
7854 (vec_select:HI (match_dup 2) (parallel [(const_int 14)]))
7855 (vec_select:HI (match_dup 2) (parallel [(const_int 15)]))))))))]
7857 "vph<plusminus_mnemonic>w\t{%2, %1, %0|%0, %1, %2}"
7858 [(set_attr "type" "sseiadd")
7859 (set_attr "prefix_extra" "1")
7860 (set_attr "prefix" "vex")
7861 (set_attr "mode" "OI")])
7863 (define_insn "ssse3_ph<plusminus_mnemonic>wv8hi3"
7864 [(set (match_operand:V8HI 0 "register_operand" "=x,x")
7870 (match_operand:V8HI 1 "register_operand" "0,x")
7871 (parallel [(const_int 0)]))
7872 (vec_select:HI (match_dup 1) (parallel [(const_int 1)])))
7874 (vec_select:HI (match_dup 1) (parallel [(const_int 2)]))
7875 (vec_select:HI (match_dup 1) (parallel [(const_int 3)]))))
7878 (vec_select:HI (match_dup 1) (parallel [(const_int 4)]))
7879 (vec_select:HI (match_dup 1) (parallel [(const_int 5)])))
7881 (vec_select:HI (match_dup 1) (parallel [(const_int 6)]))
7882 (vec_select:HI (match_dup 1) (parallel [(const_int 7)])))))
7887 (match_operand:V8HI 2 "nonimmediate_operand" "xm,xm")
7888 (parallel [(const_int 0)]))
7889 (vec_select:HI (match_dup 2) (parallel [(const_int 1)])))
7891 (vec_select:HI (match_dup 2) (parallel [(const_int 2)]))
7892 (vec_select:HI (match_dup 2) (parallel [(const_int 3)]))))
7895 (vec_select:HI (match_dup 2) (parallel [(const_int 4)]))
7896 (vec_select:HI (match_dup 2) (parallel [(const_int 5)])))
7898 (vec_select:HI (match_dup 2) (parallel [(const_int 6)]))
7899 (vec_select:HI (match_dup 2) (parallel [(const_int 7)])))))))]
7902 ph<plusminus_mnemonic>w\t{%2, %0|%0, %2}
7903 vph<plusminus_mnemonic>w\t{%2, %1, %0|%0, %1, %2}"
7904 [(set_attr "isa" "noavx,avx")
7905 (set_attr "type" "sseiadd")
7906 (set_attr "atom_unit" "complex")
7907 (set_attr "prefix_data16" "1,*")
7908 (set_attr "prefix_extra" "1")
7909 (set_attr "prefix" "orig,vex")
7910 (set_attr "mode" "TI")])
7912 (define_insn "ssse3_ph<plusminus_mnemonic>wv4hi3"
7913 [(set (match_operand:V4HI 0 "register_operand" "=y")
7918 (match_operand:V4HI 1 "register_operand" "0")
7919 (parallel [(const_int 0)]))
7920 (vec_select:HI (match_dup 1) (parallel [(const_int 1)])))
7922 (vec_select:HI (match_dup 1) (parallel [(const_int 2)]))
7923 (vec_select:HI (match_dup 1) (parallel [(const_int 3)]))))
7927 (match_operand:V4HI 2 "nonimmediate_operand" "ym")
7928 (parallel [(const_int 0)]))
7929 (vec_select:HI (match_dup 2) (parallel [(const_int 1)])))
7931 (vec_select:HI (match_dup 2) (parallel [(const_int 2)]))
7932 (vec_select:HI (match_dup 2) (parallel [(const_int 3)]))))))]
7934 "ph<plusminus_mnemonic>w\t{%2, %0|%0, %2}"
7935 [(set_attr "type" "sseiadd")
7936 (set_attr "atom_unit" "complex")
7937 (set_attr "prefix_extra" "1")
7938 (set (attr "prefix_rex") (symbol_ref "x86_extended_reg_mentioned_p (insn)"))
7939 (set_attr "mode" "DI")])
7941 (define_insn "avx2_ph<plusminus_mnemonic>dv8si3"
7942 [(set (match_operand:V8SI 0 "register_operand" "=x")
7948 (match_operand:V8SI 1 "register_operand" "x")
7949 (parallel [(const_int 0)]))
7950 (vec_select:SI (match_dup 1) (parallel [(const_int 1)])))
7952 (vec_select:SI (match_dup 1) (parallel [(const_int 2)]))
7953 (vec_select:SI (match_dup 1) (parallel [(const_int 3)]))))
7956 (vec_select:SI (match_dup 1) (parallel [(const_int 4)]))
7957 (vec_select:SI (match_dup 1) (parallel [(const_int 5)])))
7959 (vec_select:SI (match_dup 1) (parallel [(const_int 6)]))
7960 (vec_select:SI (match_dup 1) (parallel [(const_int 7)])))))
7965 (match_operand:V8SI 2 "nonimmediate_operand" "xm")
7966 (parallel [(const_int 0)]))
7967 (vec_select:SI (match_dup 2) (parallel [(const_int 1)])))
7969 (vec_select:SI (match_dup 2) (parallel [(const_int 2)]))
7970 (vec_select:SI (match_dup 2) (parallel [(const_int 3)]))))
7973 (vec_select:SI (match_dup 2) (parallel [(const_int 4)]))
7974 (vec_select:SI (match_dup 2) (parallel [(const_int 5)])))
7976 (vec_select:SI (match_dup 2) (parallel [(const_int 6)]))
7977 (vec_select:SI (match_dup 2) (parallel [(const_int 7)])))))))]
7979 "vph<plusminus_mnemonic>d\t{%2, %1, %0|%0, %1, %2}"
7980 [(set_attr "type" "sseiadd")
7981 (set_attr "prefix_extra" "1")
7982 (set_attr "prefix" "vex")
7983 (set_attr "mode" "OI")])
7985 (define_insn "ssse3_ph<plusminus_mnemonic>dv4si3"
7986 [(set (match_operand:V4SI 0 "register_operand" "=x,x")
7991 (match_operand:V4SI 1 "register_operand" "0,x")
7992 (parallel [(const_int 0)]))
7993 (vec_select:SI (match_dup 1) (parallel [(const_int 1)])))
7995 (vec_select:SI (match_dup 1) (parallel [(const_int 2)]))
7996 (vec_select:SI (match_dup 1) (parallel [(const_int 3)]))))
8000 (match_operand:V4SI 2 "nonimmediate_operand" "xm,xm")
8001 (parallel [(const_int 0)]))
8002 (vec_select:SI (match_dup 2) (parallel [(const_int 1)])))
8004 (vec_select:SI (match_dup 2) (parallel [(const_int 2)]))
8005 (vec_select:SI (match_dup 2) (parallel [(const_int 3)]))))))]
8008 ph<plusminus_mnemonic>d\t{%2, %0|%0, %2}
8009 vph<plusminus_mnemonic>d\t{%2, %1, %0|%0, %1, %2}"
8010 [(set_attr "isa" "noavx,avx")
8011 (set_attr "type" "sseiadd")
8012 (set_attr "atom_unit" "complex")
8013 (set_attr "prefix_data16" "1,*")
8014 (set_attr "prefix_extra" "1")
8015 (set_attr "prefix" "orig,vex")
8016 (set_attr "mode" "TI")])
8018 (define_insn "ssse3_ph<plusminus_mnemonic>dv2si3"
8019 [(set (match_operand:V2SI 0 "register_operand" "=y")
8023 (match_operand:V2SI 1 "register_operand" "0")
8024 (parallel [(const_int 0)]))
8025 (vec_select:SI (match_dup 1) (parallel [(const_int 1)])))
8028 (match_operand:V2SI 2 "nonimmediate_operand" "ym")
8029 (parallel [(const_int 0)]))
8030 (vec_select:SI (match_dup 2) (parallel [(const_int 1)])))))]
8032 "ph<plusminus_mnemonic>d\t{%2, %0|%0, %2}"
8033 [(set_attr "type" "sseiadd")
8034 (set_attr "atom_unit" "complex")
8035 (set_attr "prefix_extra" "1")
8036 (set (attr "prefix_rex") (symbol_ref "x86_extended_reg_mentioned_p (insn)"))
8037 (set_attr "mode" "DI")])
8039 (define_insn "avx2_pmaddubsw256"
8040 [(set (match_operand:V16HI 0 "register_operand" "=x")
8045 (match_operand:V32QI 1 "register_operand" "x")
8046 (parallel [(const_int 0) (const_int 2)
8047 (const_int 4) (const_int 6)
8048 (const_int 8) (const_int 10)
8049 (const_int 12) (const_int 14)
8050 (const_int 16) (const_int 18)
8051 (const_int 20) (const_int 22)
8052 (const_int 24) (const_int 26)
8053 (const_int 28) (const_int 30)])))
8056 (match_operand:V32QI 2 "nonimmediate_operand" "xm")
8057 (parallel [(const_int 0) (const_int 2)
8058 (const_int 4) (const_int 6)
8059 (const_int 8) (const_int 10)
8060 (const_int 12) (const_int 14)
8061 (const_int 16) (const_int 18)
8062 (const_int 20) (const_int 22)
8063 (const_int 24) (const_int 26)
8064 (const_int 28) (const_int 30)]))))
8067 (vec_select:V16QI (match_dup 1)
8068 (parallel [(const_int 1) (const_int 3)
8069 (const_int 5) (const_int 7)
8070 (const_int 9) (const_int 11)
8071 (const_int 13) (const_int 15)
8072 (const_int 17) (const_int 19)
8073 (const_int 21) (const_int 23)
8074 (const_int 25) (const_int 27)
8075 (const_int 29) (const_int 31)])))
8077 (vec_select:V16QI (match_dup 2)
8078 (parallel [(const_int 1) (const_int 3)
8079 (const_int 5) (const_int 7)
8080 (const_int 9) (const_int 11)
8081 (const_int 13) (const_int 15)
8082 (const_int 17) (const_int 19)
8083 (const_int 21) (const_int 23)
8084 (const_int 25) (const_int 27)
8085 (const_int 29) (const_int 31)]))))))]
8087 "vpmaddubsw\t{%2, %1, %0|%0, %1, %2}"
8088 [(set_attr "type" "sseiadd")
8089 (set_attr "prefix_extra" "1")
8090 (set_attr "prefix" "vex")
8091 (set_attr "mode" "OI")])
8093 (define_insn "ssse3_pmaddubsw128"
8094 [(set (match_operand:V8HI 0 "register_operand" "=x,x")
8099 (match_operand:V16QI 1 "register_operand" "0,x")
8100 (parallel [(const_int 0) (const_int 2)
8101 (const_int 4) (const_int 6)
8102 (const_int 8) (const_int 10)
8103 (const_int 12) (const_int 14)])))
8106 (match_operand:V16QI 2 "nonimmediate_operand" "xm,xm")
8107 (parallel [(const_int 0) (const_int 2)
8108 (const_int 4) (const_int 6)
8109 (const_int 8) (const_int 10)
8110 (const_int 12) (const_int 14)]))))
8113 (vec_select:V8QI (match_dup 1)
8114 (parallel [(const_int 1) (const_int 3)
8115 (const_int 5) (const_int 7)
8116 (const_int 9) (const_int 11)
8117 (const_int 13) (const_int 15)])))
8119 (vec_select:V8QI (match_dup 2)
8120 (parallel [(const_int 1) (const_int 3)
8121 (const_int 5) (const_int 7)
8122 (const_int 9) (const_int 11)
8123 (const_int 13) (const_int 15)]))))))]
8126 pmaddubsw\t{%2, %0|%0, %2}
8127 vpmaddubsw\t{%2, %1, %0|%0, %1, %2}"
8128 [(set_attr "isa" "noavx,avx")
8129 (set_attr "type" "sseiadd")
8130 (set_attr "atom_unit" "simul")
8131 (set_attr "prefix_data16" "1,*")
8132 (set_attr "prefix_extra" "1")
8133 (set_attr "prefix" "orig,vex")
8134 (set_attr "mode" "TI")])
8136 (define_insn "ssse3_pmaddubsw"
8137 [(set (match_operand:V4HI 0 "register_operand" "=y")
8142 (match_operand:V8QI 1 "register_operand" "0")
8143 (parallel [(const_int 0) (const_int 2)
8144 (const_int 4) (const_int 6)])))
8147 (match_operand:V8QI 2 "nonimmediate_operand" "ym")
8148 (parallel [(const_int 0) (const_int 2)
8149 (const_int 4) (const_int 6)]))))
8152 (vec_select:V4QI (match_dup 1)
8153 (parallel [(const_int 1) (const_int 3)
8154 (const_int 5) (const_int 7)])))
8156 (vec_select:V4QI (match_dup 2)
8157 (parallel [(const_int 1) (const_int 3)
8158 (const_int 5) (const_int 7)]))))))]
8160 "pmaddubsw\t{%2, %0|%0, %2}"
8161 [(set_attr "type" "sseiadd")
8162 (set_attr "atom_unit" "simul")
8163 (set_attr "prefix_extra" "1")
8164 (set (attr "prefix_rex") (symbol_ref "x86_extended_reg_mentioned_p (insn)"))
8165 (set_attr "mode" "DI")])
8167 (define_mode_iterator PMULHRSW
8168 [V4HI V8HI (V16HI "TARGET_AVX2")])
8170 (define_expand "<ssse3_avx2>_pmulhrsw<mode>3"
8171 [(set (match_operand:PMULHRSW 0 "register_operand")
8173 (lshiftrt:<ssedoublemode>
8174 (plus:<ssedoublemode>
8175 (lshiftrt:<ssedoublemode>
8176 (mult:<ssedoublemode>
8177 (sign_extend:<ssedoublemode>
8178 (match_operand:PMULHRSW 1 "nonimmediate_operand"))
8179 (sign_extend:<ssedoublemode>
8180 (match_operand:PMULHRSW 2 "nonimmediate_operand")))
8186 operands[3] = CONST1_RTX(<MODE>mode);
8187 ix86_fixup_binary_operands_no_copy (MULT, <MODE>mode, operands);
8190 (define_insn "*<ssse3_avx2>_pmulhrsw<mode>3"
8191 [(set (match_operand:VI2_AVX2 0 "register_operand" "=x,x")
8193 (lshiftrt:<ssedoublemode>
8194 (plus:<ssedoublemode>
8195 (lshiftrt:<ssedoublemode>
8196 (mult:<ssedoublemode>
8197 (sign_extend:<ssedoublemode>
8198 (match_operand:VI2_AVX2 1 "nonimmediate_operand" "%0,x"))
8199 (sign_extend:<ssedoublemode>
8200 (match_operand:VI2_AVX2 2 "nonimmediate_operand" "xm,xm")))
8202 (match_operand:VI2_AVX2 3 "const1_operand"))
8204 "TARGET_SSSE3 && ix86_binary_operator_ok (MULT, <MODE>mode, operands)"
8206 pmulhrsw\t{%2, %0|%0, %2}
8207 vpmulhrsw\t{%2, %1, %0|%0, %1, %2}"
8208 [(set_attr "isa" "noavx,avx")
8209 (set_attr "type" "sseimul")
8210 (set_attr "prefix_data16" "1,*")
8211 (set_attr "prefix_extra" "1")
8212 (set_attr "prefix" "orig,vex")
8213 (set_attr "mode" "<sseinsnmode>")])
8215 (define_insn "*ssse3_pmulhrswv4hi3"
8216 [(set (match_operand:V4HI 0 "register_operand" "=y")
8223 (match_operand:V4HI 1 "nonimmediate_operand" "%0"))
8225 (match_operand:V4HI 2 "nonimmediate_operand" "ym")))
8227 (match_operand:V4HI 3 "const1_operand"))
8229 "TARGET_SSSE3 && ix86_binary_operator_ok (MULT, V4HImode, operands)"
8230 "pmulhrsw\t{%2, %0|%0, %2}"
8231 [(set_attr "type" "sseimul")
8232 (set_attr "prefix_extra" "1")
8233 (set (attr "prefix_rex") (symbol_ref "x86_extended_reg_mentioned_p (insn)"))
8234 (set_attr "mode" "DI")])
8236 (define_insn "<ssse3_avx2>_pshufb<mode>3"
8237 [(set (match_operand:VI1_AVX2 0 "register_operand" "=x,x")
8239 [(match_operand:VI1_AVX2 1 "register_operand" "0,x")
8240 (match_operand:VI1_AVX2 2 "nonimmediate_operand" "xm,xm")]
8244 pshufb\t{%2, %0|%0, %2}
8245 vpshufb\t{%2, %1, %0|%0, %1, %2}"
8246 [(set_attr "isa" "noavx,avx")
8247 (set_attr "type" "sselog1")
8248 (set_attr "prefix_data16" "1,*")
8249 (set_attr "prefix_extra" "1")
8250 (set_attr "prefix" "orig,vex")
8251 (set_attr "btver2_decode" "vector,vector")
8252 (set_attr "mode" "<sseinsnmode>")])
8254 (define_insn "ssse3_pshufbv8qi3"
8255 [(set (match_operand:V8QI 0 "register_operand" "=y")
8256 (unspec:V8QI [(match_operand:V8QI 1 "register_operand" "0")
8257 (match_operand:V8QI 2 "nonimmediate_operand" "ym")]
8260 "pshufb\t{%2, %0|%0, %2}";
8261 [(set_attr "type" "sselog1")
8262 (set_attr "prefix_extra" "1")
8263 (set (attr "prefix_rex") (symbol_ref "x86_extended_reg_mentioned_p (insn)"))
8264 (set_attr "mode" "DI")])
8266 (define_insn "<ssse3_avx2>_psign<mode>3"
8267 [(set (match_operand:VI124_AVX2 0 "register_operand" "=x,x")
8269 [(match_operand:VI124_AVX2 1 "register_operand" "0,x")
8270 (match_operand:VI124_AVX2 2 "nonimmediate_operand" "xm,xm")]
8274 psign<ssemodesuffix>\t{%2, %0|%0, %2}
8275 vpsign<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
8276 [(set_attr "isa" "noavx,avx")
8277 (set_attr "type" "sselog1")
8278 (set_attr "prefix_data16" "1,*")
8279 (set_attr "prefix_extra" "1")
8280 (set_attr "prefix" "orig,vex")
8281 (set_attr "mode" "<sseinsnmode>")])
8283 (define_insn "ssse3_psign<mode>3"
8284 [(set (match_operand:MMXMODEI 0 "register_operand" "=y")
8286 [(match_operand:MMXMODEI 1 "register_operand" "0")
8287 (match_operand:MMXMODEI 2 "nonimmediate_operand" "ym")]
8290 "psign<mmxvecsize>\t{%2, %0|%0, %2}";
8291 [(set_attr "type" "sselog1")
8292 (set_attr "prefix_extra" "1")
8293 (set (attr "prefix_rex") (symbol_ref "x86_extended_reg_mentioned_p (insn)"))
8294 (set_attr "mode" "DI")])
8296 (define_insn "<ssse3_avx2>_palignr<mode>"
8297 [(set (match_operand:SSESCALARMODE 0 "register_operand" "=x,x")
8298 (unspec:SSESCALARMODE
8299 [(match_operand:SSESCALARMODE 1 "register_operand" "0,x")
8300 (match_operand:SSESCALARMODE 2 "nonimmediate_operand" "xm,xm")
8301 (match_operand:SI 3 "const_0_to_255_mul_8_operand" "n,n")]
8305 operands[3] = GEN_INT (INTVAL (operands[3]) / 8);
8307 switch (which_alternative)
8310 return "palignr\t{%3, %2, %0|%0, %2, %3}";
8312 return "vpalignr\t{%3, %2, %1, %0|%0, %1, %2, %3}";
8317 [(set_attr "isa" "noavx,avx")
8318 (set_attr "type" "sseishft")
8319 (set_attr "atom_unit" "sishuf")
8320 (set_attr "prefix_data16" "1,*")
8321 (set_attr "prefix_extra" "1")
8322 (set_attr "length_immediate" "1")
8323 (set_attr "prefix" "orig,vex")
8324 (set_attr "mode" "<sseinsnmode>")])
8326 (define_insn "ssse3_palignrdi"
8327 [(set (match_operand:DI 0 "register_operand" "=y")
8328 (unspec:DI [(match_operand:DI 1 "register_operand" "0")
8329 (match_operand:DI 2 "nonimmediate_operand" "ym")
8330 (match_operand:SI 3 "const_0_to_255_mul_8_operand" "n")]
8334 operands[3] = GEN_INT (INTVAL (operands[3]) / 8);
8335 return "palignr\t{%3, %2, %0|%0, %2, %3}";
8337 [(set_attr "type" "sseishft")
8338 (set_attr "atom_unit" "sishuf")
8339 (set_attr "prefix_extra" "1")
8340 (set_attr "length_immediate" "1")
8341 (set (attr "prefix_rex") (symbol_ref "x86_extended_reg_mentioned_p (insn)"))
8342 (set_attr "mode" "DI")])
8344 (define_insn "abs<mode>2"
8345 [(set (match_operand:VI124_AVX2 0 "register_operand" "=x")
8347 (match_operand:VI124_AVX2 1 "nonimmediate_operand" "xm")))]
8349 "%vpabs<ssemodesuffix>\t{%1, %0|%0, %1}"
8350 [(set_attr "type" "sselog1")
8351 (set_attr "prefix_data16" "1")
8352 (set_attr "prefix_extra" "1")
8353 (set_attr "prefix" "maybe_vex")
8354 (set_attr "mode" "<sseinsnmode>")])
8356 (define_insn "abs<mode>2"
8357 [(set (match_operand:MMXMODEI 0 "register_operand" "=y")
8359 (match_operand:MMXMODEI 1 "nonimmediate_operand" "ym")))]
8361 "pabs<mmxvecsize>\t{%1, %0|%0, %1}";
8362 [(set_attr "type" "sselog1")
8363 (set_attr "prefix_rep" "0")
8364 (set_attr "prefix_extra" "1")
8365 (set (attr "prefix_rex") (symbol_ref "x86_extended_reg_mentioned_p (insn)"))
8366 (set_attr "mode" "DI")])
8368 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
8370 ;; AMD SSE4A instructions
8372 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
8374 (define_insn "sse4a_movnt<mode>"
8375 [(set (match_operand:MODEF 0 "memory_operand" "=m")
8377 [(match_operand:MODEF 1 "register_operand" "x")]
8380 "movnt<ssemodesuffix>\t{%1, %0|%0, %1}"
8381 [(set_attr "type" "ssemov")
8382 (set_attr "mode" "<MODE>")])
8384 (define_insn "sse4a_vmmovnt<mode>"
8385 [(set (match_operand:<ssescalarmode> 0 "memory_operand" "=m")
8386 (unspec:<ssescalarmode>
8387 [(vec_select:<ssescalarmode>
8388 (match_operand:VF_128 1 "register_operand" "x")
8389 (parallel [(const_int 0)]))]
8392 "movnt<ssescalarmodesuffix>\t{%1, %0|%0, %1}"
8393 [(set_attr "type" "ssemov")
8394 (set_attr "mode" "<ssescalarmode>")])
8396 (define_insn "sse4a_extrqi"
8397 [(set (match_operand:V2DI 0 "register_operand" "=x")
8398 (unspec:V2DI [(match_operand:V2DI 1 "register_operand" "0")
8399 (match_operand 2 "const_0_to_255_operand")
8400 (match_operand 3 "const_0_to_255_operand")]
8403 "extrq\t{%3, %2, %0|%0, %2, %3}"
8404 [(set_attr "type" "sse")
8405 (set_attr "prefix_data16" "1")
8406 (set_attr "length_immediate" "2")
8407 (set_attr "mode" "TI")])
8409 (define_insn "sse4a_extrq"
8410 [(set (match_operand:V2DI 0 "register_operand" "=x")
8411 (unspec:V2DI [(match_operand:V2DI 1 "register_operand" "0")
8412 (match_operand:V16QI 2 "register_operand" "x")]
8415 "extrq\t{%2, %0|%0, %2}"
8416 [(set_attr "type" "sse")
8417 (set_attr "prefix_data16" "1")
8418 (set_attr "mode" "TI")])
8420 (define_insn "sse4a_insertqi"
8421 [(set (match_operand:V2DI 0 "register_operand" "=x")
8422 (unspec:V2DI [(match_operand:V2DI 1 "register_operand" "0")
8423 (match_operand:V2DI 2 "register_operand" "x")
8424 (match_operand 3 "const_0_to_255_operand")
8425 (match_operand 4 "const_0_to_255_operand")]
8428 "insertq\t{%4, %3, %2, %0|%0, %2, %3, %4}"
8429 [(set_attr "type" "sseins")
8430 (set_attr "prefix_data16" "0")
8431 (set_attr "prefix_rep" "1")
8432 (set_attr "length_immediate" "2")
8433 (set_attr "mode" "TI")])
8435 (define_insn "sse4a_insertq"
8436 [(set (match_operand:V2DI 0 "register_operand" "=x")
8437 (unspec:V2DI [(match_operand:V2DI 1 "register_operand" "0")
8438 (match_operand:V2DI 2 "register_operand" "x")]
8441 "insertq\t{%2, %0|%0, %2}"
8442 [(set_attr "type" "sseins")
8443 (set_attr "prefix_data16" "0")
8444 (set_attr "prefix_rep" "1")
8445 (set_attr "mode" "TI")])
8447 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
8449 ;; Intel SSE4.1 instructions
8451 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
8453 (define_insn "<sse4_1>_blend<ssemodesuffix><avxsizesuffix>"
8454 [(set (match_operand:VF 0 "register_operand" "=x,x")
8456 (match_operand:VF 2 "nonimmediate_operand" "xm,xm")
8457 (match_operand:VF 1 "register_operand" "0,x")
8458 (match_operand:SI 3 "const_0_to_<blendbits>_operand")))]
8461 blend<ssemodesuffix>\t{%3, %2, %0|%0, %2, %3}
8462 vblend<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
8463 [(set_attr "isa" "noavx,avx")
8464 (set_attr "type" "ssemov")
8465 (set_attr "length_immediate" "1")
8466 (set_attr "prefix_data16" "1,*")
8467 (set_attr "prefix_extra" "1")
8468 (set_attr "prefix" "orig,vex")
8469 (set_attr "mode" "<MODE>")])
8471 (define_insn "<sse4_1>_blendv<ssemodesuffix><avxsizesuffix>"
8472 [(set (match_operand:VF 0 "register_operand" "=x,x")
8474 [(match_operand:VF 1 "register_operand" "0,x")
8475 (match_operand:VF 2 "nonimmediate_operand" "xm,xm")
8476 (match_operand:VF 3 "register_operand" "Yz,x")]
8480 blendv<ssemodesuffix>\t{%3, %2, %0|%0, %2, %3}
8481 vblendv<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
8482 [(set_attr "isa" "noavx,avx")
8483 (set_attr "type" "ssemov")
8484 (set_attr "length_immediate" "1")
8485 (set_attr "prefix_data16" "1,*")
8486 (set_attr "prefix_extra" "1")
8487 (set_attr "prefix" "orig,vex")
8488 (set_attr "btver2_decode" "vector,vector")
8489 (set_attr "mode" "<MODE>")])
8491 (define_insn "<sse4_1>_dp<ssemodesuffix><avxsizesuffix>"
8492 [(set (match_operand:VF 0 "register_operand" "=x,x")
8494 [(match_operand:VF 1 "nonimmediate_operand" "%0,x")
8495 (match_operand:VF 2 "nonimmediate_operand" "xm,xm")
8496 (match_operand:SI 3 "const_0_to_255_operand" "n,n")]
8500 dp<ssemodesuffix>\t{%3, %2, %0|%0, %2, %3}
8501 vdp<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
8502 [(set_attr "isa" "noavx,avx")
8503 (set_attr "type" "ssemul")
8504 (set_attr "length_immediate" "1")
8505 (set_attr "prefix_data16" "1,*")
8506 (set_attr "prefix_extra" "1")
8507 (set_attr "prefix" "orig,vex")
8508 (set_attr "btver2_decode" "vector,vector")
8509 (set_attr "mode" "<MODE>")])
8511 (define_insn "<sse4_1_avx2>_movntdqa"
8512 [(set (match_operand:VI8_AVX2 0 "register_operand" "=x")
8513 (unspec:VI8_AVX2 [(match_operand:VI8_AVX2 1 "memory_operand" "m")]
8516 "%vmovntdqa\t{%1, %0|%0, %1}"
8517 [(set_attr "type" "ssemov")
8518 (set_attr "prefix_extra" "1")
8519 (set_attr "prefix" "maybe_vex")
8520 (set_attr "mode" "<sseinsnmode>")])
8522 (define_insn "<sse4_1_avx2>_mpsadbw"
8523 [(set (match_operand:VI1_AVX2 0 "register_operand" "=x,x")
8525 [(match_operand:VI1_AVX2 1 "register_operand" "0,x")
8526 (match_operand:VI1_AVX2 2 "nonimmediate_operand" "xm,xm")
8527 (match_operand:SI 3 "const_0_to_255_operand" "n,n")]
8531 mpsadbw\t{%3, %2, %0|%0, %2, %3}
8532 vmpsadbw\t{%3, %2, %1, %0|%0, %1, %2, %3}"
8533 [(set_attr "isa" "noavx,avx")
8534 (set_attr "type" "sselog1")
8535 (set_attr "length_immediate" "1")
8536 (set_attr "prefix_extra" "1")
8537 (set_attr "prefix" "orig,vex")
8538 (set_attr "btver2_decode" "vector,vector")
8539 (set_attr "mode" "<sseinsnmode>")])
8541 (define_insn "avx2_packusdw"
8542 [(set (match_operand:V16HI 0 "register_operand" "=x")
8545 (match_operand:V8SI 1 "register_operand" "x"))
8547 (match_operand:V8SI 2 "nonimmediate_operand" "xm"))))]
8549 "vpackusdw\t{%2, %1, %0|%0, %1, %2}"
8550 [(set_attr "type" "sselog")
8551 (set_attr "prefix_extra" "1")
8552 (set_attr "prefix" "vex")
8553 (set_attr "mode" "OI")])
8555 (define_insn "sse4_1_packusdw"
8556 [(set (match_operand:V8HI 0 "register_operand" "=x,x")
8559 (match_operand:V4SI 1 "register_operand" "0,x"))
8561 (match_operand:V4SI 2 "nonimmediate_operand" "xm,xm"))))]
8564 packusdw\t{%2, %0|%0, %2}
8565 vpackusdw\t{%2, %1, %0|%0, %1, %2}"
8566 [(set_attr "isa" "noavx,avx")
8567 (set_attr "type" "sselog")
8568 (set_attr "prefix_extra" "1")
8569 (set_attr "prefix" "orig,vex")
8570 (set_attr "mode" "TI")])
8572 (define_insn "<sse4_1_avx2>_pblendvb"
8573 [(set (match_operand:VI1_AVX2 0 "register_operand" "=x,x")
8575 [(match_operand:VI1_AVX2 1 "register_operand" "0,x")
8576 (match_operand:VI1_AVX2 2 "nonimmediate_operand" "xm,xm")
8577 (match_operand:VI1_AVX2 3 "register_operand" "Yz,x")]
8581 pblendvb\t{%3, %2, %0|%0, %2, %3}
8582 vpblendvb\t{%3, %2, %1, %0|%0, %1, %2, %3}"
8583 [(set_attr "isa" "noavx,avx")
8584 (set_attr "type" "ssemov")
8585 (set_attr "prefix_extra" "1")
8586 (set_attr "length_immediate" "*,1")
8587 (set_attr "prefix" "orig,vex")
8588 (set_attr "btver2_decode" "vector,vector")
8589 (set_attr "mode" "<sseinsnmode>")])
8591 (define_insn "sse4_1_pblendw"
8592 [(set (match_operand:V8HI 0 "register_operand" "=x,x")
8594 (match_operand:V8HI 2 "nonimmediate_operand" "xm,xm")
8595 (match_operand:V8HI 1 "register_operand" "0,x")
8596 (match_operand:SI 3 "const_0_to_255_operand" "n,n")))]
8599 pblendw\t{%3, %2, %0|%0, %2, %3}
8600 vpblendw\t{%3, %2, %1, %0|%0, %1, %2, %3}"
8601 [(set_attr "isa" "noavx,avx")
8602 (set_attr "type" "ssemov")
8603 (set_attr "prefix_extra" "1")
8604 (set_attr "length_immediate" "1")
8605 (set_attr "prefix" "orig,vex")
8606 (set_attr "mode" "TI")])
8608 ;; The builtin uses an 8-bit immediate. Expand that.
8609 (define_expand "avx2_pblendw"
8610 [(set (match_operand:V16HI 0 "register_operand")
8612 (match_operand:V16HI 2 "nonimmediate_operand")
8613 (match_operand:V16HI 1 "register_operand")
8614 (match_operand:SI 3 "const_0_to_255_operand")))]
8617 HOST_WIDE_INT val = INTVAL (operands[3]) & 0xff;
8618 operands[3] = GEN_INT (val << 8 | val);
8621 (define_insn "*avx2_pblendw"
8622 [(set (match_operand:V16HI 0 "register_operand" "=x")
8624 (match_operand:V16HI 2 "nonimmediate_operand" "xm")
8625 (match_operand:V16HI 1 "register_operand" "x")
8626 (match_operand:SI 3 "avx2_pblendw_operand" "n")))]
8629 operands[3] = GEN_INT (INTVAL (operands[3]) & 0xff);
8630 return "vpblendw\t{%3, %2, %1, %0|%0, %1, %2, %3}";
8632 [(set_attr "type" "ssemov")
8633 (set_attr "prefix_extra" "1")
8634 (set_attr "length_immediate" "1")
8635 (set_attr "prefix" "vex")
8636 (set_attr "mode" "OI")])
8638 (define_insn "avx2_pblendd<mode>"
8639 [(set (match_operand:VI4_AVX2 0 "register_operand" "=x")
8641 (match_operand:VI4_AVX2 2 "nonimmediate_operand" "xm")
8642 (match_operand:VI4_AVX2 1 "register_operand" "x")
8643 (match_operand:SI 3 "const_0_to_255_operand" "n")))]
8645 "vpblendd\t{%3, %2, %1, %0|%0, %1, %2, %3}"
8646 [(set_attr "type" "ssemov")
8647 (set_attr "prefix_extra" "1")
8648 (set_attr "length_immediate" "1")
8649 (set_attr "prefix" "vex")
8650 (set_attr "mode" "<sseinsnmode>")])
8652 (define_insn "sse4_1_phminposuw"
8653 [(set (match_operand:V8HI 0 "register_operand" "=x")
8654 (unspec:V8HI [(match_operand:V8HI 1 "nonimmediate_operand" "xm")]
8655 UNSPEC_PHMINPOSUW))]
8657 "%vphminposuw\t{%1, %0|%0, %1}"
8658 [(set_attr "type" "sselog1")
8659 (set_attr "prefix_extra" "1")
8660 (set_attr "prefix" "maybe_vex")
8661 (set_attr "mode" "TI")])
8663 (define_insn "avx2_<code>v16qiv16hi2"
8664 [(set (match_operand:V16HI 0 "register_operand" "=x")
8666 (match_operand:V16QI 1 "nonimmediate_operand" "xm")))]
8668 "vpmov<extsuffix>bw\t{%1, %0|%0, %1}"
8669 [(set_attr "type" "ssemov")
8670 (set_attr "prefix_extra" "1")
8671 (set_attr "prefix" "vex")
8672 (set_attr "mode" "OI")])
8674 (define_insn "sse4_1_<code>v8qiv8hi2"
8675 [(set (match_operand:V8HI 0 "register_operand" "=x")
8678 (match_operand:V16QI 1 "nonimmediate_operand" "xm")
8679 (parallel [(const_int 0) (const_int 1)
8680 (const_int 2) (const_int 3)
8681 (const_int 4) (const_int 5)
8682 (const_int 6) (const_int 7)]))))]
8684 "%vpmov<extsuffix>bw\t{%1, %0|%0, %q1}"
8685 [(set_attr "type" "ssemov")
8686 (set_attr "prefix_extra" "1")
8687 (set_attr "prefix" "maybe_vex")
8688 (set_attr "mode" "TI")])
8690 (define_insn "avx2_<code>v8qiv8si2"
8691 [(set (match_operand:V8SI 0 "register_operand" "=x")
8694 (match_operand:V16QI 1 "nonimmediate_operand" "xm")
8695 (parallel [(const_int 0) (const_int 1)
8696 (const_int 2) (const_int 3)
8697 (const_int 4) (const_int 5)
8698 (const_int 6) (const_int 7)]))))]
8700 "vpmov<extsuffix>bd\t{%1, %0|%0, %q1}"
8701 [(set_attr "type" "ssemov")
8702 (set_attr "prefix_extra" "1")
8703 (set_attr "prefix" "vex")
8704 (set_attr "mode" "OI")])
8706 (define_insn "sse4_1_<code>v4qiv4si2"
8707 [(set (match_operand:V4SI 0 "register_operand" "=x")
8710 (match_operand:V16QI 1 "nonimmediate_operand" "xm")
8711 (parallel [(const_int 0) (const_int 1)
8712 (const_int 2) (const_int 3)]))))]
8714 "%vpmov<extsuffix>bd\t{%1, %0|%0, %k1}"
8715 [(set_attr "type" "ssemov")
8716 (set_attr "prefix_extra" "1")
8717 (set_attr "prefix" "maybe_vex")
8718 (set_attr "mode" "TI")])
8720 (define_insn "avx2_<code>v8hiv8si2"
8721 [(set (match_operand:V8SI 0 "register_operand" "=x")
8723 (match_operand:V8HI 1 "nonimmediate_operand" "xm")))]
8725 "vpmov<extsuffix>wd\t{%1, %0|%0, %1}"
8726 [(set_attr "type" "ssemov")
8727 (set_attr "prefix_extra" "1")
8728 (set_attr "prefix" "vex")
8729 (set_attr "mode" "OI")])
8731 (define_insn "sse4_1_<code>v4hiv4si2"
8732 [(set (match_operand:V4SI 0 "register_operand" "=x")
8735 (match_operand:V8HI 1 "nonimmediate_operand" "xm")
8736 (parallel [(const_int 0) (const_int 1)
8737 (const_int 2) (const_int 3)]))))]
8739 "%vpmov<extsuffix>wd\t{%1, %0|%0, %q1}"
8740 [(set_attr "type" "ssemov")
8741 (set_attr "prefix_extra" "1")
8742 (set_attr "prefix" "maybe_vex")
8743 (set_attr "mode" "TI")])
8745 (define_insn "avx2_<code>v4qiv4di2"
8746 [(set (match_operand:V4DI 0 "register_operand" "=x")
8749 (match_operand:V16QI 1 "nonimmediate_operand" "xm")
8750 (parallel [(const_int 0) (const_int 1)
8751 (const_int 2) (const_int 3)]))))]
8753 "vpmov<extsuffix>bq\t{%1, %0|%0, %k1}"
8754 [(set_attr "type" "ssemov")
8755 (set_attr "prefix_extra" "1")
8756 (set_attr "prefix" "vex")
8757 (set_attr "mode" "OI")])
8759 (define_insn "sse4_1_<code>v2qiv2di2"
8760 [(set (match_operand:V2DI 0 "register_operand" "=x")
8763 (match_operand:V16QI 1 "nonimmediate_operand" "xm")
8764 (parallel [(const_int 0) (const_int 1)]))))]
8766 "%vpmov<extsuffix>bq\t{%1, %0|%0, %w1}"
8767 [(set_attr "type" "ssemov")
8768 (set_attr "prefix_extra" "1")
8769 (set_attr "prefix" "maybe_vex")
8770 (set_attr "mode" "TI")])
8772 (define_insn "avx2_<code>v4hiv4di2"
8773 [(set (match_operand:V4DI 0 "register_operand" "=x")
8776 (match_operand:V8HI 1 "nonimmediate_operand" "xm")
8777 (parallel [(const_int 0) (const_int 1)
8778 (const_int 2) (const_int 3)]))))]
8780 "vpmov<extsuffix>wq\t{%1, %0|%0, %q1}"
8781 [(set_attr "type" "ssemov")
8782 (set_attr "prefix_extra" "1")
8783 (set_attr "prefix" "vex")
8784 (set_attr "mode" "OI")])
8786 (define_insn "sse4_1_<code>v2hiv2di2"
8787 [(set (match_operand:V2DI 0 "register_operand" "=x")
8790 (match_operand:V8HI 1 "nonimmediate_operand" "xm")
8791 (parallel [(const_int 0) (const_int 1)]))))]
8793 "%vpmov<extsuffix>wq\t{%1, %0|%0, %k1}"
8794 [(set_attr "type" "ssemov")
8795 (set_attr "prefix_extra" "1")
8796 (set_attr "prefix" "maybe_vex")
8797 (set_attr "mode" "TI")])
8799 (define_insn "avx2_<code>v4siv4di2"
8800 [(set (match_operand:V4DI 0 "register_operand" "=x")
8802 (match_operand:V4SI 1 "nonimmediate_operand" "xm")))]
8804 "vpmov<extsuffix>dq\t{%1, %0|%0, %1}"
8805 [(set_attr "type" "ssemov")
8806 (set_attr "prefix_extra" "1")
8807 (set_attr "mode" "OI")])
8809 (define_insn "sse4_1_<code>v2siv2di2"
8810 [(set (match_operand:V2DI 0 "register_operand" "=x")
8813 (match_operand:V4SI 1 "nonimmediate_operand" "xm")
8814 (parallel [(const_int 0) (const_int 1)]))))]
8816 "%vpmov<extsuffix>dq\t{%1, %0|%0, %q1}"
8817 [(set_attr "type" "ssemov")
8818 (set_attr "prefix_extra" "1")
8819 (set_attr "prefix" "maybe_vex")
8820 (set_attr "mode" "TI")])
8822 ;; ptestps/ptestpd are very similar to comiss and ucomiss when
8823 ;; setting FLAGS_REG. But it is not a really compare instruction.
8824 (define_insn "avx_vtest<ssemodesuffix><avxsizesuffix>"
8825 [(set (reg:CC FLAGS_REG)
8826 (unspec:CC [(match_operand:VF 0 "register_operand" "x")
8827 (match_operand:VF 1 "nonimmediate_operand" "xm")]
8830 "vtest<ssemodesuffix>\t{%1, %0|%0, %1}"
8831 [(set_attr "type" "ssecomi")
8832 (set_attr "prefix_extra" "1")
8833 (set_attr "prefix" "vex")
8834 (set_attr "mode" "<MODE>")])
8836 ;; ptest is very similar to comiss and ucomiss when setting FLAGS_REG.
8837 ;; But it is not a really compare instruction.
8838 (define_insn "avx_ptest256"
8839 [(set (reg:CC FLAGS_REG)
8840 (unspec:CC [(match_operand:V4DI 0 "register_operand" "x")
8841 (match_operand:V4DI 1 "nonimmediate_operand" "xm")]
8844 "vptest\t{%1, %0|%0, %1}"
8845 [(set_attr "type" "ssecomi")
8846 (set_attr "prefix_extra" "1")
8847 (set_attr "prefix" "vex")
8848 (set_attr "btver2_decode" "vector")
8849 (set_attr "mode" "OI")])
8851 (define_insn "sse4_1_ptest"
8852 [(set (reg:CC FLAGS_REG)
8853 (unspec:CC [(match_operand:V2DI 0 "register_operand" "x")
8854 (match_operand:V2DI 1 "nonimmediate_operand" "xm")]
8857 "%vptest\t{%1, %0|%0, %1}"
8858 [(set_attr "type" "ssecomi")
8859 (set_attr "prefix_extra" "1")
8860 (set_attr "prefix" "maybe_vex")
8861 (set_attr "mode" "TI")])
8863 (define_insn "<sse4_1>_round<ssemodesuffix><avxsizesuffix>"
8864 [(set (match_operand:VF 0 "register_operand" "=x")
8866 [(match_operand:VF 1 "nonimmediate_operand" "xm")
8867 (match_operand:SI 2 "const_0_to_15_operand" "n")]
8870 "%vround<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
8871 [(set_attr "type" "ssecvt")
8872 (set (attr "prefix_data16")
8874 (match_test "TARGET_AVX")
8876 (const_string "1")))
8877 (set_attr "prefix_extra" "1")
8878 (set_attr "length_immediate" "1")
8879 (set_attr "prefix" "maybe_vex")
8880 (set_attr "mode" "<MODE>")])
8882 (define_expand "<sse4_1>_round<ssemodesuffix>_sfix<avxsizesuffix>"
8883 [(match_operand:<sseintvecmode> 0 "register_operand")
8884 (match_operand:VF1 1 "nonimmediate_operand")
8885 (match_operand:SI 2 "const_0_to_15_operand")]
8888 rtx tmp = gen_reg_rtx (<MODE>mode);
8891 (gen_<sse4_1>_round<ssemodesuffix><avxsizesuffix> (tmp, operands[1],
8894 (gen_fix_trunc<mode><sseintvecmodelower>2 (operands[0], tmp));
8898 (define_expand "<sse4_1>_round<ssemodesuffix>_vec_pack_sfix<avxsizesuffix>"
8899 [(match_operand:<ssepackfltmode> 0 "register_operand")
8900 (match_operand:VF2 1 "nonimmediate_operand")
8901 (match_operand:VF2 2 "nonimmediate_operand")
8902 (match_operand:SI 3 "const_0_to_15_operand")]
8907 if (<MODE>mode == V2DFmode
8908 && TARGET_AVX && !TARGET_PREFER_AVX128)
8910 rtx tmp2 = gen_reg_rtx (V4DFmode);
8912 tmp0 = gen_reg_rtx (V4DFmode);
8913 tmp1 = force_reg (V2DFmode, operands[1]);
8915 emit_insn (gen_avx_vec_concatv4df (tmp0, tmp1, operands[2]));
8916 emit_insn (gen_avx_roundpd256 (tmp2, tmp0, operands[3]));
8917 emit_insn (gen_fix_truncv4dfv4si2 (operands[0], tmp2));
8921 tmp0 = gen_reg_rtx (<MODE>mode);
8922 tmp1 = gen_reg_rtx (<MODE>mode);
8925 (gen_<sse4_1>_round<ssemodesuffix><avxsizesuffix> (tmp0, operands[1],
8928 (gen_<sse4_1>_round<ssemodesuffix><avxsizesuffix> (tmp1, operands[2],
8931 (gen_vec_pack_sfix_trunc_<mode> (operands[0], tmp0, tmp1));
8936 (define_insn "sse4_1_round<ssescalarmodesuffix>"
8937 [(set (match_operand:VF_128 0 "register_operand" "=x,x")
8940 [(match_operand:VF_128 2 "register_operand" "x,x")
8941 (match_operand:SI 3 "const_0_to_15_operand" "n,n")]
8943 (match_operand:VF_128 1 "register_operand" "0,x")
8947 round<ssescalarmodesuffix>\t{%3, %2, %0|%0, %2, %3}
8948 vround<ssescalarmodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
8949 [(set_attr "isa" "noavx,avx")
8950 (set_attr "type" "ssecvt")
8951 (set_attr "length_immediate" "1")
8952 (set_attr "prefix_data16" "1,*")
8953 (set_attr "prefix_extra" "1")
8954 (set_attr "prefix" "orig,vex")
8955 (set_attr "mode" "<MODE>")])
8957 (define_expand "round<mode>2"
8960 (match_operand:VF 1 "register_operand")
8962 (set (match_operand:VF 0 "register_operand")
8964 [(match_dup 4) (match_dup 5)]
8966 "TARGET_ROUND && !flag_trapping_math"
8968 enum machine_mode scalar_mode;
8969 const struct real_format *fmt;
8970 REAL_VALUE_TYPE pred_half, half_minus_pred_half;
8973 scalar_mode = GET_MODE_INNER (<MODE>mode);
8975 /* load nextafter (0.5, 0.0) */
8976 fmt = REAL_MODE_FORMAT (scalar_mode);
8977 real_2expN (&half_minus_pred_half, -(fmt->p) - 1, scalar_mode);
8978 REAL_ARITHMETIC (pred_half, MINUS_EXPR, dconsthalf, half_minus_pred_half);
8979 half = const_double_from_real_value (pred_half, scalar_mode);
8981 vec_half = ix86_build_const_vector (<MODE>mode, true, half);
8982 vec_half = force_reg (<MODE>mode, vec_half);
8984 operands[3] = gen_reg_rtx (<MODE>mode);
8985 emit_insn (gen_copysign<mode>3 (operands[3], vec_half, operands[1]));
8987 operands[4] = gen_reg_rtx (<MODE>mode);
8988 operands[5] = GEN_INT (ROUND_TRUNC);
8991 (define_expand "round<mode>2_sfix"
8992 [(match_operand:<sseintvecmode> 0 "register_operand")
8993 (match_operand:VF1 1 "register_operand")]
8994 "TARGET_ROUND && !flag_trapping_math"
8996 rtx tmp = gen_reg_rtx (<MODE>mode);
8998 emit_insn (gen_round<mode>2 (tmp, operands[1]));
9001 (gen_fix_trunc<mode><sseintvecmodelower>2 (operands[0], tmp));
9005 (define_expand "round<mode>2_vec_pack_sfix"
9006 [(match_operand:<ssepackfltmode> 0 "register_operand")
9007 (match_operand:VF2 1 "register_operand")
9008 (match_operand:VF2 2 "register_operand")]
9009 "TARGET_ROUND && !flag_trapping_math"
9013 if (<MODE>mode == V2DFmode
9014 && TARGET_AVX && !TARGET_PREFER_AVX128)
9016 rtx tmp2 = gen_reg_rtx (V4DFmode);
9018 tmp0 = gen_reg_rtx (V4DFmode);
9019 tmp1 = force_reg (V2DFmode, operands[1]);
9021 emit_insn (gen_avx_vec_concatv4df (tmp0, tmp1, operands[2]));
9022 emit_insn (gen_roundv4df2 (tmp2, tmp0));
9023 emit_insn (gen_fix_truncv4dfv4si2 (operands[0], tmp2));
9027 tmp0 = gen_reg_rtx (<MODE>mode);
9028 tmp1 = gen_reg_rtx (<MODE>mode);
9030 emit_insn (gen_round<mode>2 (tmp0, operands[1]));
9031 emit_insn (gen_round<mode>2 (tmp1, operands[2]));
9034 (gen_vec_pack_sfix_trunc_<mode> (operands[0], tmp0, tmp1));
9039 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
9041 ;; Intel SSE4.2 string/text processing instructions
9043 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
9045 (define_insn_and_split "sse4_2_pcmpestr"
9046 [(set (match_operand:SI 0 "register_operand" "=c,c")
9048 [(match_operand:V16QI 2 "register_operand" "x,x")
9049 (match_operand:SI 3 "register_operand" "a,a")
9050 (match_operand:V16QI 4 "nonimmediate_operand" "x,m")
9051 (match_operand:SI 5 "register_operand" "d,d")
9052 (match_operand:SI 6 "const_0_to_255_operand" "n,n")]
9054 (set (match_operand:V16QI 1 "register_operand" "=Yz,Yz")
9062 (set (reg:CC FLAGS_REG)
9071 && can_create_pseudo_p ()"
9076 int ecx = !find_regno_note (curr_insn, REG_UNUSED, REGNO (operands[0]));
9077 int xmm0 = !find_regno_note (curr_insn, REG_UNUSED, REGNO (operands[1]));
9078 int flags = !find_regno_note (curr_insn, REG_UNUSED, FLAGS_REG);
9081 emit_insn (gen_sse4_2_pcmpestri (operands[0], operands[2],
9082 operands[3], operands[4],
9083 operands[5], operands[6]));
9085 emit_insn (gen_sse4_2_pcmpestrm (operands[1], operands[2],
9086 operands[3], operands[4],
9087 operands[5], operands[6]));
9088 if (flags && !(ecx || xmm0))
9089 emit_insn (gen_sse4_2_pcmpestr_cconly (NULL, NULL,
9090 operands[2], operands[3],
9091 operands[4], operands[5],
9093 if (!(flags || ecx || xmm0))
9094 emit_note (NOTE_INSN_DELETED);
9098 [(set_attr "type" "sselog")
9099 (set_attr "prefix_data16" "1")
9100 (set_attr "prefix_extra" "1")
9101 (set_attr "length_immediate" "1")
9102 (set_attr "memory" "none,load")
9103 (set_attr "mode" "TI")])
9105 (define_insn_and_split "*sse4_2_pcmpestr_unaligned"
9106 [(set (match_operand:SI 0 "register_operand" "=c")
9108 [(match_operand:V16QI 2 "register_operand" "x")
9109 (match_operand:SI 3 "register_operand" "a")
9111 [(match_operand:V16QI 4 "memory_operand" "m")]
9113 (match_operand:SI 5 "register_operand" "d")
9114 (match_operand:SI 6 "const_0_to_255_operand" "n")]
9116 (set (match_operand:V16QI 1 "register_operand" "=Yz")
9120 (unspec:V16QI [(match_dup 4)] UNSPEC_LOADU)
9124 (set (reg:CC FLAGS_REG)
9128 (unspec:V16QI [(match_dup 4)] UNSPEC_LOADU)
9133 && can_create_pseudo_p ()"
9138 int ecx = !find_regno_note (curr_insn, REG_UNUSED, REGNO (operands[0]));
9139 int xmm0 = !find_regno_note (curr_insn, REG_UNUSED, REGNO (operands[1]));
9140 int flags = !find_regno_note (curr_insn, REG_UNUSED, FLAGS_REG);
9143 emit_insn (gen_sse4_2_pcmpestri (operands[0], operands[2],
9144 operands[3], operands[4],
9145 operands[5], operands[6]));
9147 emit_insn (gen_sse4_2_pcmpestrm (operands[1], operands[2],
9148 operands[3], operands[4],
9149 operands[5], operands[6]));
9150 if (flags && !(ecx || xmm0))
9151 emit_insn (gen_sse4_2_pcmpestr_cconly (NULL, NULL,
9152 operands[2], operands[3],
9153 operands[4], operands[5],
9155 if (!(flags || ecx || xmm0))
9156 emit_note (NOTE_INSN_DELETED);
9160 [(set_attr "type" "sselog")
9161 (set_attr "prefix_data16" "1")
9162 (set_attr "prefix_extra" "1")
9163 (set_attr "length_immediate" "1")
9164 (set_attr "memory" "load")
9165 (set_attr "mode" "TI")])
9167 (define_insn "sse4_2_pcmpestri"
9168 [(set (match_operand:SI 0 "register_operand" "=c,c")
9170 [(match_operand:V16QI 1 "register_operand" "x,x")
9171 (match_operand:SI 2 "register_operand" "a,a")
9172 (match_operand:V16QI 3 "nonimmediate_operand" "x,m")
9173 (match_operand:SI 4 "register_operand" "d,d")
9174 (match_operand:SI 5 "const_0_to_255_operand" "n,n")]
9176 (set (reg:CC FLAGS_REG)
9185 "%vpcmpestri\t{%5, %3, %1|%1, %3, %5}"
9186 [(set_attr "type" "sselog")
9187 (set_attr "prefix_data16" "1")
9188 (set_attr "prefix_extra" "1")
9189 (set_attr "prefix" "maybe_vex")
9190 (set_attr "length_immediate" "1")
9191 (set_attr "btver2_decode" "vector")
9192 (set_attr "memory" "none,load")
9193 (set_attr "mode" "TI")])
9195 (define_insn "sse4_2_pcmpestrm"
9196 [(set (match_operand:V16QI 0 "register_operand" "=Yz,Yz")
9198 [(match_operand:V16QI 1 "register_operand" "x,x")
9199 (match_operand:SI 2 "register_operand" "a,a")
9200 (match_operand:V16QI 3 "nonimmediate_operand" "x,m")
9201 (match_operand:SI 4 "register_operand" "d,d")
9202 (match_operand:SI 5 "const_0_to_255_operand" "n,n")]
9204 (set (reg:CC FLAGS_REG)
9213 "%vpcmpestrm\t{%5, %3, %1|%1, %3, %5}"
9214 [(set_attr "type" "sselog")
9215 (set_attr "prefix_data16" "1")
9216 (set_attr "prefix_extra" "1")
9217 (set_attr "length_immediate" "1")
9218 (set_attr "prefix" "maybe_vex")
9219 (set_attr "btver2_decode" "vector")
9220 (set_attr "memory" "none,load")
9221 (set_attr "mode" "TI")])
9223 (define_insn "sse4_2_pcmpestr_cconly"
9224 [(set (reg:CC FLAGS_REG)
9226 [(match_operand:V16QI 2 "register_operand" "x,x,x,x")
9227 (match_operand:SI 3 "register_operand" "a,a,a,a")
9228 (match_operand:V16QI 4 "nonimmediate_operand" "x,m,x,m")
9229 (match_operand:SI 5 "register_operand" "d,d,d,d")
9230 (match_operand:SI 6 "const_0_to_255_operand" "n,n,n,n")]
9232 (clobber (match_scratch:V16QI 0 "=Yz,Yz,X,X"))
9233 (clobber (match_scratch:SI 1 "= X, X,c,c"))]
9236 %vpcmpestrm\t{%6, %4, %2|%2, %4, %6}
9237 %vpcmpestrm\t{%6, %4, %2|%2, %4, %6}
9238 %vpcmpestri\t{%6, %4, %2|%2, %4, %6}
9239 %vpcmpestri\t{%6, %4, %2|%2, %4, %6}"
9240 [(set_attr "type" "sselog")
9241 (set_attr "prefix_data16" "1")
9242 (set_attr "prefix_extra" "1")
9243 (set_attr "length_immediate" "1")
9244 (set_attr "memory" "none,load,none,load")
9245 (set_attr "btver2_decode" "vector,vector,vector,vector")
9246 (set_attr "prefix" "maybe_vex")
9247 (set_attr "mode" "TI")])
9249 (define_insn_and_split "sse4_2_pcmpistr"
9250 [(set (match_operand:SI 0 "register_operand" "=c,c")
9252 [(match_operand:V16QI 2 "register_operand" "x,x")
9253 (match_operand:V16QI 3 "nonimmediate_operand" "x,m")
9254 (match_operand:SI 4 "const_0_to_255_operand" "n,n")]
9256 (set (match_operand:V16QI 1 "register_operand" "=Yz,Yz")
9262 (set (reg:CC FLAGS_REG)
9269 && can_create_pseudo_p ()"
9274 int ecx = !find_regno_note (curr_insn, REG_UNUSED, REGNO (operands[0]));
9275 int xmm0 = !find_regno_note (curr_insn, REG_UNUSED, REGNO (operands[1]));
9276 int flags = !find_regno_note (curr_insn, REG_UNUSED, FLAGS_REG);
9279 emit_insn (gen_sse4_2_pcmpistri (operands[0], operands[2],
9280 operands[3], operands[4]));
9282 emit_insn (gen_sse4_2_pcmpistrm (operands[1], operands[2],
9283 operands[3], operands[4]));
9284 if (flags && !(ecx || xmm0))
9285 emit_insn (gen_sse4_2_pcmpistr_cconly (NULL, NULL,
9286 operands[2], operands[3],
9288 if (!(flags || ecx || xmm0))
9289 emit_note (NOTE_INSN_DELETED);
9293 [(set_attr "type" "sselog")
9294 (set_attr "prefix_data16" "1")
9295 (set_attr "prefix_extra" "1")
9296 (set_attr "length_immediate" "1")
9297 (set_attr "memory" "none,load")
9298 (set_attr "mode" "TI")])
9300 (define_insn_and_split "*sse4_2_pcmpistr_unaligned"
9301 [(set (match_operand:SI 0 "register_operand" "=c")
9303 [(match_operand:V16QI 2 "register_operand" "x")
9305 [(match_operand:V16QI 3 "memory_operand" "m")]
9307 (match_operand:SI 4 "const_0_to_255_operand" "n")]
9309 (set (match_operand:V16QI 1 "register_operand" "=Yz")
9312 (unspec:V16QI [(match_dup 3)] UNSPEC_LOADU)
9315 (set (reg:CC FLAGS_REG)
9318 (unspec:V16QI [(match_dup 3)] UNSPEC_LOADU)
9322 && can_create_pseudo_p ()"
9327 int ecx = !find_regno_note (curr_insn, REG_UNUSED, REGNO (operands[0]));
9328 int xmm0 = !find_regno_note (curr_insn, REG_UNUSED, REGNO (operands[1]));
9329 int flags = !find_regno_note (curr_insn, REG_UNUSED, FLAGS_REG);
9332 emit_insn (gen_sse4_2_pcmpistri (operands[0], operands[2],
9333 operands[3], operands[4]));
9335 emit_insn (gen_sse4_2_pcmpistrm (operands[1], operands[2],
9336 operands[3], operands[4]));
9337 if (flags && !(ecx || xmm0))
9338 emit_insn (gen_sse4_2_pcmpistr_cconly (NULL, NULL,
9339 operands[2], operands[3],
9341 if (!(flags || ecx || xmm0))
9342 emit_note (NOTE_INSN_DELETED);
9346 [(set_attr "type" "sselog")
9347 (set_attr "prefix_data16" "1")
9348 (set_attr "prefix_extra" "1")
9349 (set_attr "length_immediate" "1")
9350 (set_attr "memory" "load")
9351 (set_attr "mode" "TI")])
9353 (define_insn "sse4_2_pcmpistri"
9354 [(set (match_operand:SI 0 "register_operand" "=c,c")
9356 [(match_operand:V16QI 1 "register_operand" "x,x")
9357 (match_operand:V16QI 2 "nonimmediate_operand" "x,m")
9358 (match_operand:SI 3 "const_0_to_255_operand" "n,n")]
9360 (set (reg:CC FLAGS_REG)
9367 "%vpcmpistri\t{%3, %2, %1|%1, %2, %3}"
9368 [(set_attr "type" "sselog")
9369 (set_attr "prefix_data16" "1")
9370 (set_attr "prefix_extra" "1")
9371 (set_attr "length_immediate" "1")
9372 (set_attr "prefix" "maybe_vex")
9373 (set_attr "memory" "none,load")
9374 (set_attr "btver2_decode" "vector")
9375 (set_attr "mode" "TI")])
9377 (define_insn "sse4_2_pcmpistrm"
9378 [(set (match_operand:V16QI 0 "register_operand" "=Yz,Yz")
9380 [(match_operand:V16QI 1 "register_operand" "x,x")
9381 (match_operand:V16QI 2 "nonimmediate_operand" "x,m")
9382 (match_operand:SI 3 "const_0_to_255_operand" "n,n")]
9384 (set (reg:CC FLAGS_REG)
9391 "%vpcmpistrm\t{%3, %2, %1|%1, %2, %3}"
9392 [(set_attr "type" "sselog")
9393 (set_attr "prefix_data16" "1")
9394 (set_attr "prefix_extra" "1")
9395 (set_attr "length_immediate" "1")
9396 (set_attr "prefix" "maybe_vex")
9397 (set_attr "memory" "none,load")
9398 (set_attr "btver2_decode" "vector")
9399 (set_attr "mode" "TI")])
9401 (define_insn "sse4_2_pcmpistr_cconly"
9402 [(set (reg:CC FLAGS_REG)
9404 [(match_operand:V16QI 2 "register_operand" "x,x,x,x")
9405 (match_operand:V16QI 3 "nonimmediate_operand" "x,m,x,m")
9406 (match_operand:SI 4 "const_0_to_255_operand" "n,n,n,n")]
9408 (clobber (match_scratch:V16QI 0 "=Yz,Yz,X,X"))
9409 (clobber (match_scratch:SI 1 "= X, X,c,c"))]
9412 %vpcmpistrm\t{%4, %3, %2|%2, %3, %4}
9413 %vpcmpistrm\t{%4, %3, %2|%2, %3, %4}
9414 %vpcmpistri\t{%4, %3, %2|%2, %3, %4}
9415 %vpcmpistri\t{%4, %3, %2|%2, %3, %4}"
9416 [(set_attr "type" "sselog")
9417 (set_attr "prefix_data16" "1")
9418 (set_attr "prefix_extra" "1")
9419 (set_attr "length_immediate" "1")
9420 (set_attr "memory" "none,load,none,load")
9421 (set_attr "prefix" "maybe_vex")
9422 (set_attr "btver2_decode" "vector,vector,vector,vector")
9423 (set_attr "mode" "TI")])
9425 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
9429 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
9431 (define_code_iterator xop_plus [plus ss_plus])
9433 (define_code_attr macs [(plus "macs") (ss_plus "macss")])
9434 (define_code_attr madcs [(plus "madcs") (ss_plus "madcss")])
9436 ;; XOP parallel integer multiply/add instructions.
9438 (define_insn "xop_p<macs><ssemodesuffix><ssemodesuffix>"
9439 [(set (match_operand:VI24_128 0 "register_operand" "=x")
9442 (match_operand:VI24_128 1 "nonimmediate_operand" "%x")
9443 (match_operand:VI24_128 2 "nonimmediate_operand" "xm"))
9444 (match_operand:VI24_128 3 "register_operand" "x")))]
9446 "vp<macs><ssemodesuffix><ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
9447 [(set_attr "type" "ssemuladd")
9448 (set_attr "mode" "TI")])
9450 (define_insn "xop_p<macs>dql"
9451 [(set (match_operand:V2DI 0 "register_operand" "=x")
9456 (match_operand:V4SI 1 "nonimmediate_operand" "%x")
9457 (parallel [(const_int 0) (const_int 2)])))
9460 (match_operand:V4SI 2 "nonimmediate_operand" "xm")
9461 (parallel [(const_int 0) (const_int 2)]))))
9462 (match_operand:V2DI 3 "register_operand" "x")))]
9464 "vp<macs>dql\t{%3, %2, %1, %0|%0, %1, %2, %3}"
9465 [(set_attr "type" "ssemuladd")
9466 (set_attr "mode" "TI")])
9468 (define_insn "xop_p<macs>dqh"
9469 [(set (match_operand:V2DI 0 "register_operand" "=x")
9474 (match_operand:V4SI 1 "nonimmediate_operand" "%x")
9475 (parallel [(const_int 1) (const_int 3)])))
9478 (match_operand:V4SI 2 "nonimmediate_operand" "xm")
9479 (parallel [(const_int 1) (const_int 3)]))))
9480 (match_operand:V2DI 3 "register_operand" "x")))]
9482 "vp<macs>dqh\t{%3, %2, %1, %0|%0, %1, %2, %3}"
9483 [(set_attr "type" "ssemuladd")
9484 (set_attr "mode" "TI")])
9486 ;; XOP parallel integer multiply/add instructions for the intrinisics
9487 (define_insn "xop_p<macs>wd"
9488 [(set (match_operand:V4SI 0 "register_operand" "=x")
9493 (match_operand:V8HI 1 "nonimmediate_operand" "%x")
9494 (parallel [(const_int 1) (const_int 3)
9495 (const_int 5) (const_int 7)])))
9498 (match_operand:V8HI 2 "nonimmediate_operand" "xm")
9499 (parallel [(const_int 1) (const_int 3)
9500 (const_int 5) (const_int 7)]))))
9501 (match_operand:V4SI 3 "register_operand" "x")))]
9503 "vp<macs>wd\t{%3, %2, %1, %0|%0, %1, %2, %3}"
9504 [(set_attr "type" "ssemuladd")
9505 (set_attr "mode" "TI")])
9507 (define_insn "xop_p<madcs>wd"
9508 [(set (match_operand:V4SI 0 "register_operand" "=x")
9514 (match_operand:V8HI 1 "nonimmediate_operand" "%x")
9515 (parallel [(const_int 0) (const_int 2)
9516 (const_int 4) (const_int 6)])))
9519 (match_operand:V8HI 2 "nonimmediate_operand" "xm")
9520 (parallel [(const_int 0) (const_int 2)
9521 (const_int 4) (const_int 6)]))))
9526 (parallel [(const_int 1) (const_int 3)
9527 (const_int 5) (const_int 7)])))
9531 (parallel [(const_int 1) (const_int 3)
9532 (const_int 5) (const_int 7)])))))
9533 (match_operand:V4SI 3 "register_operand" "x")))]
9535 "vp<madcs>wd\t{%3, %2, %1, %0|%0, %1, %2, %3}"
9536 [(set_attr "type" "ssemuladd")
9537 (set_attr "mode" "TI")])
9539 ;; XOP parallel XMM conditional moves
9540 (define_insn "xop_pcmov_<mode><avxsizesuffix>"
9541 [(set (match_operand:V 0 "register_operand" "=x,x")
9543 (match_operand:V 3 "nonimmediate_operand" "x,m")
9544 (match_operand:V 1 "register_operand" "x,x")
9545 (match_operand:V 2 "nonimmediate_operand" "xm,x")))]
9547 "vpcmov\t{%3, %2, %1, %0|%0, %1, %2, %3}"
9548 [(set_attr "type" "sse4arg")])
9550 ;; XOP horizontal add/subtract instructions
9551 (define_insn "xop_phadd<u>bw"
9552 [(set (match_operand:V8HI 0 "register_operand" "=x")
9556 (match_operand:V16QI 1 "nonimmediate_operand" "xm")
9557 (parallel [(const_int 0) (const_int 2)
9558 (const_int 4) (const_int 6)
9559 (const_int 8) (const_int 10)
9560 (const_int 12) (const_int 14)])))
9564 (parallel [(const_int 1) (const_int 3)
9565 (const_int 5) (const_int 7)
9566 (const_int 9) (const_int 11)
9567 (const_int 13) (const_int 15)])))))]
9569 "vphadd<u>bw\t{%1, %0|%0, %1}"
9570 [(set_attr "type" "sseiadd1")])
9572 (define_insn "xop_phadd<u>bd"
9573 [(set (match_operand:V4SI 0 "register_operand" "=x")
9578 (match_operand:V16QI 1 "nonimmediate_operand" "xm")
9579 (parallel [(const_int 0) (const_int 4)
9580 (const_int 8) (const_int 12)])))
9584 (parallel [(const_int 1) (const_int 5)
9585 (const_int 9) (const_int 13)]))))
9590 (parallel [(const_int 2) (const_int 6)
9591 (const_int 10) (const_int 14)])))
9595 (parallel [(const_int 3) (const_int 7)
9596 (const_int 11) (const_int 15)]))))))]
9598 "vphadd<u>bd\t{%1, %0|%0, %1}"
9599 [(set_attr "type" "sseiadd1")])
9601 (define_insn "xop_phadd<u>bq"
9602 [(set (match_operand:V2DI 0 "register_operand" "=x")
9608 (match_operand:V16QI 1 "nonimmediate_operand" "xm")
9609 (parallel [(const_int 0) (const_int 8)])))
9613 (parallel [(const_int 1) (const_int 9)]))))
9618 (parallel [(const_int 2) (const_int 10)])))
9622 (parallel [(const_int 3) (const_int 11)])))))
9628 (parallel [(const_int 4) (const_int 12)])))
9632 (parallel [(const_int 5) (const_int 13)]))))
9637 (parallel [(const_int 6) (const_int 14)])))
9641 (parallel [(const_int 7) (const_int 15)])))))))]
9643 "vphadd<u>bq\t{%1, %0|%0, %1}"
9644 [(set_attr "type" "sseiadd1")])
9646 (define_insn "xop_phadd<u>wd"
9647 [(set (match_operand:V4SI 0 "register_operand" "=x")
9651 (match_operand:V8HI 1 "nonimmediate_operand" "xm")
9652 (parallel [(const_int 0) (const_int 2)
9653 (const_int 4) (const_int 6)])))
9657 (parallel [(const_int 1) (const_int 3)
9658 (const_int 5) (const_int 7)])))))]
9660 "vphadd<u>wd\t{%1, %0|%0, %1}"
9661 [(set_attr "type" "sseiadd1")])
9663 (define_insn "xop_phadd<u>wq"
9664 [(set (match_operand:V2DI 0 "register_operand" "=x")
9669 (match_operand:V8HI 1 "nonimmediate_operand" "xm")
9670 (parallel [(const_int 0) (const_int 4)])))
9674 (parallel [(const_int 1) (const_int 5)]))))
9679 (parallel [(const_int 2) (const_int 6)])))
9683 (parallel [(const_int 3) (const_int 7)]))))))]
9685 "vphadd<u>wq\t{%1, %0|%0, %1}"
9686 [(set_attr "type" "sseiadd1")])
9688 (define_insn "xop_phadd<u>dq"
9689 [(set (match_operand:V2DI 0 "register_operand" "=x")
9693 (match_operand:V4SI 1 "nonimmediate_operand" "xm")
9694 (parallel [(const_int 0) (const_int 2)])))
9698 (parallel [(const_int 1) (const_int 3)])))))]
9700 "vphadd<u>dq\t{%1, %0|%0, %1}"
9701 [(set_attr "type" "sseiadd1")])
9703 (define_insn "xop_phsubbw"
9704 [(set (match_operand:V8HI 0 "register_operand" "=x")
9708 (match_operand:V16QI 1 "nonimmediate_operand" "xm")
9709 (parallel [(const_int 0) (const_int 2)
9710 (const_int 4) (const_int 6)
9711 (const_int 8) (const_int 10)
9712 (const_int 12) (const_int 14)])))
9716 (parallel [(const_int 1) (const_int 3)
9717 (const_int 5) (const_int 7)
9718 (const_int 9) (const_int 11)
9719 (const_int 13) (const_int 15)])))))]
9721 "vphsubbw\t{%1, %0|%0, %1}"
9722 [(set_attr "type" "sseiadd1")])
9724 (define_insn "xop_phsubwd"
9725 [(set (match_operand:V4SI 0 "register_operand" "=x")
9729 (match_operand:V8HI 1 "nonimmediate_operand" "xm")
9730 (parallel [(const_int 0) (const_int 2)
9731 (const_int 4) (const_int 6)])))
9735 (parallel [(const_int 1) (const_int 3)
9736 (const_int 5) (const_int 7)])))))]
9738 "vphsubwd\t{%1, %0|%0, %1}"
9739 [(set_attr "type" "sseiadd1")])
9741 (define_insn "xop_phsubdq"
9742 [(set (match_operand:V2DI 0 "register_operand" "=x")
9746 (match_operand:V4SI 1 "nonimmediate_operand" "xm")
9747 (parallel [(const_int 0) (const_int 2)])))
9751 (parallel [(const_int 1) (const_int 3)])))))]
9753 "vphsubdq\t{%1, %0|%0, %1}"
9754 [(set_attr "type" "sseiadd1")])
9756 ;; XOP permute instructions
9757 (define_insn "xop_pperm"
9758 [(set (match_operand:V16QI 0 "register_operand" "=x,x")
9760 [(match_operand:V16QI 1 "register_operand" "x,x")
9761 (match_operand:V16QI 2 "nonimmediate_operand" "x,m")
9762 (match_operand:V16QI 3 "nonimmediate_operand" "xm,x")]
9763 UNSPEC_XOP_PERMUTE))]
9764 "TARGET_XOP && !(MEM_P (operands[2]) && MEM_P (operands[3]))"
9765 "vpperm\t{%3, %2, %1, %0|%0, %1, %2, %3}"
9766 [(set_attr "type" "sse4arg")
9767 (set_attr "mode" "TI")])
9769 ;; XOP pack instructions that combine two vectors into a smaller vector
9770 (define_insn "xop_pperm_pack_v2di_v4si"
9771 [(set (match_operand:V4SI 0 "register_operand" "=x,x")
9774 (match_operand:V2DI 1 "register_operand" "x,x"))
9776 (match_operand:V2DI 2 "nonimmediate_operand" "x,m"))))
9777 (use (match_operand:V16QI 3 "nonimmediate_operand" "xm,x"))]
9778 "TARGET_XOP && !(MEM_P (operands[2]) && MEM_P (operands[3]))"
9779 "vpperm\t{%3, %2, %1, %0|%0, %1, %2, %3}"
9780 [(set_attr "type" "sse4arg")
9781 (set_attr "mode" "TI")])
9783 (define_insn "xop_pperm_pack_v4si_v8hi"
9784 [(set (match_operand:V8HI 0 "register_operand" "=x,x")
9787 (match_operand:V4SI 1 "register_operand" "x,x"))
9789 (match_operand:V4SI 2 "nonimmediate_operand" "x,m"))))
9790 (use (match_operand:V16QI 3 "nonimmediate_operand" "xm,x"))]
9791 "TARGET_XOP && !(MEM_P (operands[2]) && MEM_P (operands[3]))"
9792 "vpperm\t{%3, %2, %1, %0|%0, %1, %2, %3}"
9793 [(set_attr "type" "sse4arg")
9794 (set_attr "mode" "TI")])
9796 (define_insn "xop_pperm_pack_v8hi_v16qi"
9797 [(set (match_operand:V16QI 0 "register_operand" "=x,x")
9800 (match_operand:V8HI 1 "register_operand" "x,x"))
9802 (match_operand:V8HI 2 "nonimmediate_operand" "x,m"))))
9803 (use (match_operand:V16QI 3 "nonimmediate_operand" "xm,x"))]
9804 "TARGET_XOP && !(MEM_P (operands[2]) && MEM_P (operands[3]))"
9805 "vpperm\t{%3, %2, %1, %0|%0, %1, %2, %3}"
9806 [(set_attr "type" "sse4arg")
9807 (set_attr "mode" "TI")])
9809 ;; XOP packed rotate instructions
9810 (define_expand "rotl<mode>3"
9811 [(set (match_operand:VI_128 0 "register_operand")
9813 (match_operand:VI_128 1 "nonimmediate_operand")
9814 (match_operand:SI 2 "general_operand")))]
9817 /* If we were given a scalar, convert it to parallel */
9818 if (! const_0_to_<sserotatemax>_operand (operands[2], SImode))
9820 rtvec vs = rtvec_alloc (<ssescalarnum>);
9821 rtx par = gen_rtx_PARALLEL (<MODE>mode, vs);
9822 rtx reg = gen_reg_rtx (<MODE>mode);
9823 rtx op2 = operands[2];
9826 if (GET_MODE (op2) != <ssescalarmode>mode)
9828 op2 = gen_reg_rtx (<ssescalarmode>mode);
9829 convert_move (op2, operands[2], false);
9832 for (i = 0; i < <ssescalarnum>; i++)
9833 RTVEC_ELT (vs, i) = op2;
9835 emit_insn (gen_vec_init<mode> (reg, par));
9836 emit_insn (gen_xop_vrotl<mode>3 (operands[0], operands[1], reg));
9841 (define_expand "rotr<mode>3"
9842 [(set (match_operand:VI_128 0 "register_operand")
9844 (match_operand:VI_128 1 "nonimmediate_operand")
9845 (match_operand:SI 2 "general_operand")))]
9848 /* If we were given a scalar, convert it to parallel */
9849 if (! const_0_to_<sserotatemax>_operand (operands[2], SImode))
9851 rtvec vs = rtvec_alloc (<ssescalarnum>);
9852 rtx par = gen_rtx_PARALLEL (<MODE>mode, vs);
9853 rtx neg = gen_reg_rtx (<MODE>mode);
9854 rtx reg = gen_reg_rtx (<MODE>mode);
9855 rtx op2 = operands[2];
9858 if (GET_MODE (op2) != <ssescalarmode>mode)
9860 op2 = gen_reg_rtx (<ssescalarmode>mode);
9861 convert_move (op2, operands[2], false);
9864 for (i = 0; i < <ssescalarnum>; i++)
9865 RTVEC_ELT (vs, i) = op2;
9867 emit_insn (gen_vec_init<mode> (reg, par));
9868 emit_insn (gen_neg<mode>2 (neg, reg));
9869 emit_insn (gen_xop_vrotl<mode>3 (operands[0], operands[1], neg));
9874 (define_insn "xop_rotl<mode>3"
9875 [(set (match_operand:VI_128 0 "register_operand" "=x")
9877 (match_operand:VI_128 1 "nonimmediate_operand" "xm")
9878 (match_operand:SI 2 "const_0_to_<sserotatemax>_operand" "n")))]
9880 "vprot<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
9881 [(set_attr "type" "sseishft")
9882 (set_attr "length_immediate" "1")
9883 (set_attr "mode" "TI")])
9885 (define_insn "xop_rotr<mode>3"
9886 [(set (match_operand:VI_128 0 "register_operand" "=x")
9888 (match_operand:VI_128 1 "nonimmediate_operand" "xm")
9889 (match_operand:SI 2 "const_0_to_<sserotatemax>_operand" "n")))]
9893 = GEN_INT (GET_MODE_BITSIZE (<ssescalarmode>mode) - INTVAL (operands[2]));
9894 return \"vprot<ssemodesuffix>\t{%3, %1, %0|%0, %1, %3}\";
9896 [(set_attr "type" "sseishft")
9897 (set_attr "length_immediate" "1")
9898 (set_attr "mode" "TI")])
9900 (define_expand "vrotr<mode>3"
9901 [(match_operand:VI_128 0 "register_operand")
9902 (match_operand:VI_128 1 "register_operand")
9903 (match_operand:VI_128 2 "register_operand")]
9906 rtx reg = gen_reg_rtx (<MODE>mode);
9907 emit_insn (gen_neg<mode>2 (reg, operands[2]));
9908 emit_insn (gen_xop_vrotl<mode>3 (operands[0], operands[1], reg));
9912 (define_expand "vrotl<mode>3"
9913 [(match_operand:VI_128 0 "register_operand")
9914 (match_operand:VI_128 1 "register_operand")
9915 (match_operand:VI_128 2 "register_operand")]
9918 emit_insn (gen_xop_vrotl<mode>3 (operands[0], operands[1], operands[2]));
9922 (define_insn "xop_vrotl<mode>3"
9923 [(set (match_operand:VI_128 0 "register_operand" "=x,x")
9924 (if_then_else:VI_128
9926 (match_operand:VI_128 2 "nonimmediate_operand" "x,m")
9929 (match_operand:VI_128 1 "nonimmediate_operand" "xm,x")
9933 (neg:VI_128 (match_dup 2)))))]
9934 "TARGET_XOP && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
9935 "vprot<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
9936 [(set_attr "type" "sseishft")
9937 (set_attr "prefix_data16" "0")
9938 (set_attr "prefix_extra" "2")
9939 (set_attr "mode" "TI")])
9941 ;; XOP packed shift instructions.
9942 (define_expand "vlshr<mode>3"
9943 [(set (match_operand:VI12_128 0 "register_operand")
9945 (match_operand:VI12_128 1 "register_operand")
9946 (match_operand:VI12_128 2 "nonimmediate_operand")))]
9949 rtx neg = gen_reg_rtx (<MODE>mode);
9950 emit_insn (gen_neg<mode>2 (neg, operands[2]));
9951 emit_insn (gen_xop_shl<mode>3 (operands[0], operands[1], neg));
9955 (define_expand "vlshr<mode>3"
9956 [(set (match_operand:VI48_128 0 "register_operand")
9958 (match_operand:VI48_128 1 "register_operand")
9959 (match_operand:VI48_128 2 "nonimmediate_operand")))]
9960 "TARGET_AVX2 || TARGET_XOP"
9964 rtx neg = gen_reg_rtx (<MODE>mode);
9965 emit_insn (gen_neg<mode>2 (neg, operands[2]));
9966 emit_insn (gen_xop_shl<mode>3 (operands[0], operands[1], neg));
9971 (define_expand "vlshr<mode>3"
9972 [(set (match_operand:VI48_256 0 "register_operand")
9974 (match_operand:VI48_256 1 "register_operand")
9975 (match_operand:VI48_256 2 "nonimmediate_operand")))]
9978 (define_expand "vashr<mode>3"
9979 [(set (match_operand:VI128_128 0 "register_operand")
9981 (match_operand:VI128_128 1 "register_operand")
9982 (match_operand:VI128_128 2 "nonimmediate_operand")))]
9985 rtx neg = gen_reg_rtx (<MODE>mode);
9986 emit_insn (gen_neg<mode>2 (neg, operands[2]));
9987 emit_insn (gen_xop_sha<mode>3 (operands[0], operands[1], neg));
9991 (define_expand "vashrv4si3"
9992 [(set (match_operand:V4SI 0 "register_operand")
9993 (ashiftrt:V4SI (match_operand:V4SI 1 "register_operand")
9994 (match_operand:V4SI 2 "nonimmediate_operand")))]
9995 "TARGET_AVX2 || TARGET_XOP"
9999 rtx neg = gen_reg_rtx (V4SImode);
10000 emit_insn (gen_negv4si2 (neg, operands[2]));
10001 emit_insn (gen_xop_shav4si3 (operands[0], operands[1], neg));
10006 (define_expand "vashrv8si3"
10007 [(set (match_operand:V8SI 0 "register_operand")
10008 (ashiftrt:V8SI (match_operand:V8SI 1 "register_operand")
10009 (match_operand:V8SI 2 "nonimmediate_operand")))]
10012 (define_expand "vashl<mode>3"
10013 [(set (match_operand:VI12_128 0 "register_operand")
10015 (match_operand:VI12_128 1 "register_operand")
10016 (match_operand:VI12_128 2 "nonimmediate_operand")))]
10019 emit_insn (gen_xop_sha<mode>3 (operands[0], operands[1], operands[2]));
10023 (define_expand "vashl<mode>3"
10024 [(set (match_operand:VI48_128 0 "register_operand")
10026 (match_operand:VI48_128 1 "register_operand")
10027 (match_operand:VI48_128 2 "nonimmediate_operand")))]
10028 "TARGET_AVX2 || TARGET_XOP"
10032 operands[2] = force_reg (<MODE>mode, operands[2]);
10033 emit_insn (gen_xop_sha<mode>3 (operands[0], operands[1], operands[2]));
10038 (define_expand "vashl<mode>3"
10039 [(set (match_operand:VI48_256 0 "register_operand")
10041 (match_operand:VI48_256 1 "register_operand")
10042 (match_operand:VI48_256 2 "nonimmediate_operand")))]
10045 (define_insn "xop_sha<mode>3"
10046 [(set (match_operand:VI_128 0 "register_operand" "=x,x")
10047 (if_then_else:VI_128
10049 (match_operand:VI_128 2 "nonimmediate_operand" "x,m")
10052 (match_operand:VI_128 1 "nonimmediate_operand" "xm,x")
10056 (neg:VI_128 (match_dup 2)))))]
10057 "TARGET_XOP && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
10058 "vpsha<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
10059 [(set_attr "type" "sseishft")
10060 (set_attr "prefix_data16" "0")
10061 (set_attr "prefix_extra" "2")
10062 (set_attr "mode" "TI")])
10064 (define_insn "xop_shl<mode>3"
10065 [(set (match_operand:VI_128 0 "register_operand" "=x,x")
10066 (if_then_else:VI_128
10068 (match_operand:VI_128 2 "nonimmediate_operand" "x,m")
10071 (match_operand:VI_128 1 "nonimmediate_operand" "xm,x")
10075 (neg:VI_128 (match_dup 2)))))]
10076 "TARGET_XOP && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
10077 "vpshl<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
10078 [(set_attr "type" "sseishft")
10079 (set_attr "prefix_data16" "0")
10080 (set_attr "prefix_extra" "2")
10081 (set_attr "mode" "TI")])
10083 (define_expand "<shift_insn><mode>3"
10084 [(set (match_operand:VI1_AVX2 0 "register_operand")
10085 (any_shift:VI1_AVX2
10086 (match_operand:VI1_AVX2 1 "register_operand")
10087 (match_operand:SI 2 "nonmemory_operand")))]
10090 if (TARGET_XOP && <MODE>mode == V16QImode)
10092 bool negate = false;
10093 rtx (*gen) (rtx, rtx, rtx);
10097 if (<CODE> != ASHIFT)
10099 if (CONST_INT_P (operands[2]))
10100 operands[2] = GEN_INT (-INTVAL (operands[2]));
10104 par = gen_rtx_PARALLEL (V16QImode, rtvec_alloc (16));
10105 for (i = 0; i < 16; i++)
10106 XVECEXP (par, 0, i) = operands[2];
10108 tmp = gen_reg_rtx (V16QImode);
10109 emit_insn (gen_vec_initv16qi (tmp, par));
10112 emit_insn (gen_negv16qi2 (tmp, tmp));
10114 gen = (<CODE> == LSHIFTRT ? gen_xop_shlv16qi3 : gen_xop_shav16qi3);
10115 emit_insn (gen (operands[0], operands[1], tmp));
10118 ix86_expand_vecop_qihi (<CODE>, operands[0], operands[1], operands[2]);
10122 (define_expand "ashrv2di3"
10123 [(set (match_operand:V2DI 0 "register_operand")
10125 (match_operand:V2DI 1 "register_operand")
10126 (match_operand:DI 2 "nonmemory_operand")))]
10129 rtx reg = gen_reg_rtx (V2DImode);
10131 bool negate = false;
10134 if (CONST_INT_P (operands[2]))
10135 operands[2] = GEN_INT (-INTVAL (operands[2]));
10139 par = gen_rtx_PARALLEL (V2DImode, rtvec_alloc (2));
10140 for (i = 0; i < 2; i++)
10141 XVECEXP (par, 0, i) = operands[2];
10143 emit_insn (gen_vec_initv2di (reg, par));
10146 emit_insn (gen_negv2di2 (reg, reg));
10148 emit_insn (gen_xop_shav2di3 (operands[0], operands[1], reg));
10152 ;; XOP FRCZ support
10153 (define_insn "xop_frcz<mode>2"
10154 [(set (match_operand:FMAMODE 0 "register_operand" "=x")
10156 [(match_operand:FMAMODE 1 "nonimmediate_operand" "xm")]
10159 "vfrcz<ssemodesuffix>\t{%1, %0|%0, %1}"
10160 [(set_attr "type" "ssecvt1")
10161 (set_attr "mode" "<MODE>")])
10164 (define_expand "xop_vmfrcz<mode>2"
10165 [(set (match_operand:VF_128 0 "register_operand")
10168 [(match_operand:VF_128 1 "nonimmediate_operand")]
10174 operands[3] = CONST0_RTX (<MODE>mode);
10177 (define_insn "*xop_vmfrcz_<mode>"
10178 [(set (match_operand:VF_128 0 "register_operand" "=x")
10181 [(match_operand:VF_128 1 "nonimmediate_operand" "xm")]
10183 (match_operand:VF_128 2 "const0_operand")
10186 "vfrcz<ssescalarmodesuffix>\t{%1, %0|%0, %1}"
10187 [(set_attr "type" "ssecvt1")
10188 (set_attr "mode" "<MODE>")])
10190 (define_insn "xop_maskcmp<mode>3"
10191 [(set (match_operand:VI_128 0 "register_operand" "=x")
10192 (match_operator:VI_128 1 "ix86_comparison_int_operator"
10193 [(match_operand:VI_128 2 "register_operand" "x")
10194 (match_operand:VI_128 3 "nonimmediate_operand" "xm")]))]
10196 "vpcom%Y1<ssemodesuffix>\t{%3, %2, %0|%0, %2, %3}"
10197 [(set_attr "type" "sse4arg")
10198 (set_attr "prefix_data16" "0")
10199 (set_attr "prefix_rep" "0")
10200 (set_attr "prefix_extra" "2")
10201 (set_attr "length_immediate" "1")
10202 (set_attr "mode" "TI")])
10204 (define_insn "xop_maskcmp_uns<mode>3"
10205 [(set (match_operand:VI_128 0 "register_operand" "=x")
10206 (match_operator:VI_128 1 "ix86_comparison_uns_operator"
10207 [(match_operand:VI_128 2 "register_operand" "x")
10208 (match_operand:VI_128 3 "nonimmediate_operand" "xm")]))]
10210 "vpcom%Y1u<ssemodesuffix>\t{%3, %2, %0|%0, %2, %3}"
10211 [(set_attr "type" "ssecmp")
10212 (set_attr "prefix_data16" "0")
10213 (set_attr "prefix_rep" "0")
10214 (set_attr "prefix_extra" "2")
10215 (set_attr "length_immediate" "1")
10216 (set_attr "mode" "TI")])
10218 ;; Version of pcom*u* that is called from the intrinsics that allows pcomequ*
10219 ;; and pcomneu* not to be converted to the signed ones in case somebody needs
10220 ;; the exact instruction generated for the intrinsic.
10221 (define_insn "xop_maskcmp_uns2<mode>3"
10222 [(set (match_operand:VI_128 0 "register_operand" "=x")
10224 [(match_operator:VI_128 1 "ix86_comparison_uns_operator"
10225 [(match_operand:VI_128 2 "register_operand" "x")
10226 (match_operand:VI_128 3 "nonimmediate_operand" "xm")])]
10227 UNSPEC_XOP_UNSIGNED_CMP))]
10229 "vpcom%Y1u<ssemodesuffix>\t{%3, %2, %0|%0, %2, %3}"
10230 [(set_attr "type" "ssecmp")
10231 (set_attr "prefix_data16" "0")
10232 (set_attr "prefix_extra" "2")
10233 (set_attr "length_immediate" "1")
10234 (set_attr "mode" "TI")])
10236 ;; Pcomtrue and pcomfalse support. These are useless instructions, but are
10237 ;; being added here to be complete.
10238 (define_insn "xop_pcom_tf<mode>3"
10239 [(set (match_operand:VI_128 0 "register_operand" "=x")
10241 [(match_operand:VI_128 1 "register_operand" "x")
10242 (match_operand:VI_128 2 "nonimmediate_operand" "xm")
10243 (match_operand:SI 3 "const_int_operand" "n")]
10244 UNSPEC_XOP_TRUEFALSE))]
10247 return ((INTVAL (operands[3]) != 0)
10248 ? "vpcomtrue<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
10249 : "vpcomfalse<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}");
10251 [(set_attr "type" "ssecmp")
10252 (set_attr "prefix_data16" "0")
10253 (set_attr "prefix_extra" "2")
10254 (set_attr "length_immediate" "1")
10255 (set_attr "mode" "TI")])
10257 (define_insn "xop_vpermil2<mode>3"
10258 [(set (match_operand:VF 0 "register_operand" "=x")
10260 [(match_operand:VF 1 "register_operand" "x")
10261 (match_operand:VF 2 "nonimmediate_operand" "%x")
10262 (match_operand:<sseintvecmode> 3 "nonimmediate_operand" "xm")
10263 (match_operand:SI 4 "const_0_to_3_operand" "n")]
10266 "vpermil2<ssemodesuffix>\t{%4, %3, %2, %1, %0|%0, %1, %2, %3, %4}"
10267 [(set_attr "type" "sse4arg")
10268 (set_attr "length_immediate" "1")
10269 (set_attr "mode" "<MODE>")])
10271 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
10273 (define_insn "aesenc"
10274 [(set (match_operand:V2DI 0 "register_operand" "=x,x")
10275 (unspec:V2DI [(match_operand:V2DI 1 "register_operand" "0,x")
10276 (match_operand:V2DI 2 "nonimmediate_operand" "xm,xm")]
10280 aesenc\t{%2, %0|%0, %2}
10281 vaesenc\t{%2, %1, %0|%0, %1, %2}"
10282 [(set_attr "isa" "noavx,avx")
10283 (set_attr "type" "sselog1")
10284 (set_attr "prefix_extra" "1")
10285 (set_attr "prefix" "orig,vex")
10286 (set_attr "btver2_decode" "double,double")
10287 (set_attr "mode" "TI")])
10289 (define_insn "aesenclast"
10290 [(set (match_operand:V2DI 0 "register_operand" "=x,x")
10291 (unspec:V2DI [(match_operand:V2DI 1 "register_operand" "0,x")
10292 (match_operand:V2DI 2 "nonimmediate_operand" "xm,xm")]
10293 UNSPEC_AESENCLAST))]
10296 aesenclast\t{%2, %0|%0, %2}
10297 vaesenclast\t{%2, %1, %0|%0, %1, %2}"
10298 [(set_attr "isa" "noavx,avx")
10299 (set_attr "type" "sselog1")
10300 (set_attr "prefix_extra" "1")
10301 (set_attr "prefix" "orig,vex")
10302 (set_attr "btver2_decode" "double,double")
10303 (set_attr "mode" "TI")])
10305 (define_insn "aesdec"
10306 [(set (match_operand:V2DI 0 "register_operand" "=x,x")
10307 (unspec:V2DI [(match_operand:V2DI 1 "register_operand" "0,x")
10308 (match_operand:V2DI 2 "nonimmediate_operand" "xm,xm")]
10312 aesdec\t{%2, %0|%0, %2}
10313 vaesdec\t{%2, %1, %0|%0, %1, %2}"
10314 [(set_attr "isa" "noavx,avx")
10315 (set_attr "type" "sselog1")
10316 (set_attr "prefix_extra" "1")
10317 (set_attr "prefix" "orig,vex")
10318 (set_attr "btver2_decode" "double,double")
10319 (set_attr "mode" "TI")])
10321 (define_insn "aesdeclast"
10322 [(set (match_operand:V2DI 0 "register_operand" "=x,x")
10323 (unspec:V2DI [(match_operand:V2DI 1 "register_operand" "0,x")
10324 (match_operand:V2DI 2 "nonimmediate_operand" "xm,xm")]
10325 UNSPEC_AESDECLAST))]
10328 aesdeclast\t{%2, %0|%0, %2}
10329 vaesdeclast\t{%2, %1, %0|%0, %1, %2}"
10330 [(set_attr "isa" "noavx,avx")
10331 (set_attr "type" "sselog1")
10332 (set_attr "prefix_extra" "1")
10333 (set_attr "prefix" "orig,vex")
10334 (set_attr "btver2_decode" "double,double")
10335 (set_attr "mode" "TI")])
10337 (define_insn "aesimc"
10338 [(set (match_operand:V2DI 0 "register_operand" "=x")
10339 (unspec:V2DI [(match_operand:V2DI 1 "nonimmediate_operand" "xm")]
10342 "%vaesimc\t{%1, %0|%0, %1}"
10343 [(set_attr "type" "sselog1")
10344 (set_attr "prefix_extra" "1")
10345 (set_attr "prefix" "maybe_vex")
10346 (set_attr "mode" "TI")])
10348 (define_insn "aeskeygenassist"
10349 [(set (match_operand:V2DI 0 "register_operand" "=x")
10350 (unspec:V2DI [(match_operand:V2DI 1 "nonimmediate_operand" "xm")
10351 (match_operand:SI 2 "const_0_to_255_operand" "n")]
10352 UNSPEC_AESKEYGENASSIST))]
10354 "%vaeskeygenassist\t{%2, %1, %0|%0, %1, %2}"
10355 [(set_attr "type" "sselog1")
10356 (set_attr "prefix_extra" "1")
10357 (set_attr "length_immediate" "1")
10358 (set_attr "prefix" "maybe_vex")
10359 (set_attr "mode" "TI")])
10361 (define_insn "pclmulqdq"
10362 [(set (match_operand:V2DI 0 "register_operand" "=x,x")
10363 (unspec:V2DI [(match_operand:V2DI 1 "register_operand" "0,x")
10364 (match_operand:V2DI 2 "nonimmediate_operand" "xm,xm")
10365 (match_operand:SI 3 "const_0_to_255_operand" "n,n")]
10369 pclmulqdq\t{%3, %2, %0|%0, %2, %3}
10370 vpclmulqdq\t{%3, %2, %1, %0|%0, %1, %2, %3}"
10371 [(set_attr "isa" "noavx,avx")
10372 (set_attr "type" "sselog1")
10373 (set_attr "prefix_extra" "1")
10374 (set_attr "length_immediate" "1")
10375 (set_attr "prefix" "orig,vex")
10376 (set_attr "mode" "TI")])
10378 (define_expand "avx_vzeroall"
10379 [(match_par_dup 0 [(const_int 0)])]
10382 int nregs = TARGET_64BIT ? 16 : 8;
10385 operands[0] = gen_rtx_PARALLEL (VOIDmode, rtvec_alloc (nregs + 1));
10387 XVECEXP (operands[0], 0, 0)
10388 = gen_rtx_UNSPEC_VOLATILE (VOIDmode, gen_rtvec (1, const0_rtx),
10391 for (regno = 0; regno < nregs; regno++)
10392 XVECEXP (operands[0], 0, regno + 1)
10393 = gen_rtx_SET (VOIDmode,
10394 gen_rtx_REG (V8SImode, SSE_REGNO (regno)),
10395 CONST0_RTX (V8SImode));
10398 (define_insn "*avx_vzeroall"
10399 [(match_parallel 0 "vzeroall_operation"
10400 [(unspec_volatile [(const_int 0)] UNSPECV_VZEROALL)])]
10403 [(set_attr "type" "sse")
10404 (set_attr "modrm" "0")
10405 (set_attr "memory" "none")
10406 (set_attr "prefix" "vex")
10407 (set_attr "btver2_decode" "vector")
10408 (set_attr "mode" "OI")])
10410 ;; Clear the upper 128bits of AVX registers, equivalent to a NOP
10411 ;; if the upper 128bits are unused.
10412 (define_insn "avx_vzeroupper"
10413 [(unspec_volatile [(const_int 0)] UNSPECV_VZEROUPPER)]
10416 [(set_attr "type" "sse")
10417 (set_attr "modrm" "0")
10418 (set_attr "memory" "none")
10419 (set_attr "prefix" "vex")
10420 (set_attr "btver2_decode" "vector")
10421 (set_attr "mode" "OI")])
10423 (define_mode_attr AVXTOSSEMODE
10424 [(V4DI "V2DI") (V2DI "V2DI")
10425 (V8SI "V4SI") (V4SI "V4SI")
10426 (V16HI "V8HI") (V8HI "V8HI")
10427 (V32QI "V16QI") (V16QI "V16QI")])
10429 (define_insn "avx2_pbroadcast<mode>"
10430 [(set (match_operand:VI 0 "register_operand" "=x")
10432 (vec_select:<ssescalarmode>
10433 (match_operand:<AVXTOSSEMODE> 1 "nonimmediate_operand" "xm")
10434 (parallel [(const_int 0)]))))]
10436 "vpbroadcast<ssemodesuffix>\t{%1, %0|%0, %1}"
10437 [(set_attr "type" "ssemov")
10438 (set_attr "prefix_extra" "1")
10439 (set_attr "prefix" "vex")
10440 (set_attr "mode" "<sseinsnmode>")])
10442 (define_insn "avx2_pbroadcast<mode>_1"
10443 [(set (match_operand:VI_256 0 "register_operand" "=x")
10444 (vec_duplicate:VI_256
10445 (vec_select:<ssescalarmode>
10446 (match_operand:VI_256 1 "nonimmediate_operand" "xm")
10447 (parallel [(const_int 0)]))))]
10449 "vpbroadcast<ssemodesuffix>\t{%x1, %0|%0, %x1}"
10450 [(set_attr "type" "ssemov")
10451 (set_attr "prefix_extra" "1")
10452 (set_attr "prefix" "vex")
10453 (set_attr "mode" "<sseinsnmode>")])
10455 (define_insn "avx2_permvar<mode>"
10456 [(set (match_operand:VI4F_256 0 "register_operand" "=x")
10458 [(match_operand:VI4F_256 1 "nonimmediate_operand" "xm")
10459 (match_operand:V8SI 2 "register_operand" "x")]
10462 "vperm<ssemodesuffix>\t{%1, %2, %0|%0, %2, %1}"
10463 [(set_attr "type" "sselog")
10464 (set_attr "prefix" "vex")
10465 (set_attr "mode" "OI")])
10467 (define_expand "avx2_perm<mode>"
10468 [(match_operand:VI8F_256 0 "register_operand")
10469 (match_operand:VI8F_256 1 "nonimmediate_operand")
10470 (match_operand:SI 2 "const_0_to_255_operand")]
10473 int mask = INTVAL (operands[2]);
10474 emit_insn (gen_avx2_perm<mode>_1 (operands[0], operands[1],
10475 GEN_INT ((mask >> 0) & 3),
10476 GEN_INT ((mask >> 2) & 3),
10477 GEN_INT ((mask >> 4) & 3),
10478 GEN_INT ((mask >> 6) & 3)));
10482 (define_insn "avx2_perm<mode>_1"
10483 [(set (match_operand:VI8F_256 0 "register_operand" "=x")
10484 (vec_select:VI8F_256
10485 (match_operand:VI8F_256 1 "nonimmediate_operand" "xm")
10486 (parallel [(match_operand 2 "const_0_to_3_operand")
10487 (match_operand 3 "const_0_to_3_operand")
10488 (match_operand 4 "const_0_to_3_operand")
10489 (match_operand 5 "const_0_to_3_operand")])))]
10493 mask |= INTVAL (operands[2]) << 0;
10494 mask |= INTVAL (operands[3]) << 2;
10495 mask |= INTVAL (operands[4]) << 4;
10496 mask |= INTVAL (operands[5]) << 6;
10497 operands[2] = GEN_INT (mask);
10498 return "vperm<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}";
10500 [(set_attr "type" "sselog")
10501 (set_attr "prefix" "vex")
10502 (set_attr "mode" "<sseinsnmode>")])
10504 (define_insn "avx2_permv2ti"
10505 [(set (match_operand:V4DI 0 "register_operand" "=x")
10507 [(match_operand:V4DI 1 "register_operand" "x")
10508 (match_operand:V4DI 2 "nonimmediate_operand" "xm")
10509 (match_operand:SI 3 "const_0_to_255_operand" "n")]
10512 "vperm2i128\t{%3, %2, %1, %0|%0, %1, %2, %3}"
10513 [(set_attr "type" "sselog")
10514 (set_attr "prefix" "vex")
10515 (set_attr "mode" "OI")])
10517 (define_insn "avx2_vec_dupv4df"
10518 [(set (match_operand:V4DF 0 "register_operand" "=x")
10519 (vec_duplicate:V4DF
10521 (match_operand:V2DF 1 "register_operand" "x")
10522 (parallel [(const_int 0)]))))]
10524 "vbroadcastsd\t{%1, %0|%0, %1}"
10525 [(set_attr "type" "sselog1")
10526 (set_attr "prefix" "vex")
10527 (set_attr "mode" "V4DF")])
10529 ;; Modes handled by AVX vec_dup patterns.
10530 (define_mode_iterator AVX_VEC_DUP_MODE
10531 [V8SI V8SF V4DI V4DF])
10533 (define_insn "vec_dup<mode>"
10534 [(set (match_operand:AVX_VEC_DUP_MODE 0 "register_operand" "=x,x,x")
10535 (vec_duplicate:AVX_VEC_DUP_MODE
10536 (match_operand:<ssescalarmode> 1 "nonimmediate_operand" "m,x,?x")))]
10539 vbroadcast<ssescalarmodesuffix>\t{%1, %0|%0, %1}
10540 vbroadcast<ssescalarmodesuffix>\t{%x1, %0|%0, %x1}
10542 [(set_attr "type" "ssemov")
10543 (set_attr "prefix_extra" "1")
10544 (set_attr "prefix" "vex")
10545 (set_attr "isa" "*,avx2,noavx2")
10546 (set_attr "mode" "V8SF")])
10548 (define_insn "avx2_vbroadcasti128_<mode>"
10549 [(set (match_operand:VI_256 0 "register_operand" "=x")
10551 (match_operand:<ssehalfvecmode> 1 "memory_operand" "m")
10554 "vbroadcasti128\t{%1, %0|%0, %1}"
10555 [(set_attr "type" "ssemov")
10556 (set_attr "prefix_extra" "1")
10557 (set_attr "prefix" "vex")
10558 (set_attr "mode" "OI")])
10561 [(set (match_operand:AVX_VEC_DUP_MODE 0 "register_operand")
10562 (vec_duplicate:AVX_VEC_DUP_MODE
10563 (match_operand:<ssescalarmode> 1 "register_operand")))]
10564 "TARGET_AVX && !TARGET_AVX2 && reload_completed"
10565 [(set (match_dup 2)
10566 (vec_duplicate:<ssehalfvecmode> (match_dup 1)))
10568 (vec_concat:AVX_VEC_DUP_MODE (match_dup 2) (match_dup 2)))]
10569 "operands[2] = gen_rtx_REG (<ssehalfvecmode>mode, REGNO (operands[0]));")
10571 (define_insn "avx_vbroadcastf128_<mode>"
10572 [(set (match_operand:V_256 0 "register_operand" "=x,x,x")
10574 (match_operand:<ssehalfvecmode> 1 "nonimmediate_operand" "m,0,?x")
10578 vbroadcast<i128>\t{%1, %0|%0, %1}
10579 vinsert<i128>\t{$1, %1, %0, %0|%0, %0, %1, 1}
10580 vperm2<i128>\t{$0, %t1, %t1, %0|%0, %t1, %t1, 0}"
10581 [(set_attr "type" "ssemov,sselog1,sselog1")
10582 (set_attr "prefix_extra" "1")
10583 (set_attr "length_immediate" "0,1,1")
10584 (set_attr "prefix" "vex")
10585 (set_attr "mode" "<sseinsnmode>")])
10587 ;; Recognize broadcast as a vec_select as produced by builtin_vec_perm.
10588 ;; If it so happens that the input is in memory, use vbroadcast.
10589 ;; Otherwise use vpermilp (and in the case of 256-bit modes, vperm2f128).
10590 (define_insn "*avx_vperm_broadcast_v4sf"
10591 [(set (match_operand:V4SF 0 "register_operand" "=x,x,x")
10593 (match_operand:V4SF 1 "nonimmediate_operand" "m,o,x")
10594 (match_parallel 2 "avx_vbroadcast_operand"
10595 [(match_operand 3 "const_int_operand" "C,n,n")])))]
10598 int elt = INTVAL (operands[3]);
10599 switch (which_alternative)
10603 operands[1] = adjust_address_nv (operands[1], SFmode, elt * 4);
10604 return "vbroadcastss\t{%1, %0|%0, %1}";
10606 operands[2] = GEN_INT (elt * 0x55);
10607 return "vpermilps\t{%2, %1, %0|%0, %1, %2}";
10609 gcc_unreachable ();
10612 [(set_attr "type" "ssemov,ssemov,sselog1")
10613 (set_attr "prefix_extra" "1")
10614 (set_attr "length_immediate" "0,0,1")
10615 (set_attr "prefix" "vex")
10616 (set_attr "mode" "SF,SF,V4SF")])
10618 (define_insn_and_split "*avx_vperm_broadcast_<mode>"
10619 [(set (match_operand:VF_256 0 "register_operand" "=x,x,x")
10621 (match_operand:VF_256 1 "nonimmediate_operand" "m,o,?x")
10622 (match_parallel 2 "avx_vbroadcast_operand"
10623 [(match_operand 3 "const_int_operand" "C,n,n")])))]
10626 "&& reload_completed && (<MODE>mode != V4DFmode || !TARGET_AVX2)"
10627 [(set (match_dup 0) (vec_duplicate:VF_256 (match_dup 1)))]
10629 rtx op0 = operands[0], op1 = operands[1];
10630 int elt = INTVAL (operands[3]);
10636 if (TARGET_AVX2 && elt == 0)
10638 emit_insn (gen_vec_dup<mode> (op0, gen_lowpart (<ssescalarmode>mode,
10643 /* Shuffle element we care about into all elements of the 128-bit lane.
10644 The other lane gets shuffled too, but we don't care. */
10645 if (<MODE>mode == V4DFmode)
10646 mask = (elt & 1 ? 15 : 0);
10648 mask = (elt & 3) * 0x55;
10649 emit_insn (gen_avx_vpermil<mode> (op0, op1, GEN_INT (mask)));
10651 /* Shuffle the lane we care about into both lanes of the dest. */
10652 mask = (elt / (<ssescalarnum> / 2)) * 0x11;
10653 emit_insn (gen_avx_vperm2f128<mode>3 (op0, op0, op0, GEN_INT (mask)));
10657 operands[1] = adjust_address_nv (op1, <ssescalarmode>mode,
10658 elt * GET_MODE_SIZE (<ssescalarmode>mode));
10661 (define_expand "avx_vpermil<mode>"
10662 [(set (match_operand:VF2 0 "register_operand")
10664 (match_operand:VF2 1 "nonimmediate_operand")
10665 (match_operand:SI 2 "const_0_to_255_operand")))]
10668 int mask = INTVAL (operands[2]);
10669 rtx perm[<ssescalarnum>];
10671 perm[0] = GEN_INT (mask & 1);
10672 perm[1] = GEN_INT ((mask >> 1) & 1);
10673 if (<MODE>mode == V4DFmode)
10675 perm[2] = GEN_INT (((mask >> 2) & 1) + 2);
10676 perm[3] = GEN_INT (((mask >> 3) & 1) + 2);
10680 = gen_rtx_PARALLEL (VOIDmode, gen_rtvec_v (<ssescalarnum>, perm));
10683 (define_expand "avx_vpermil<mode>"
10684 [(set (match_operand:VF1 0 "register_operand")
10686 (match_operand:VF1 1 "nonimmediate_operand")
10687 (match_operand:SI 2 "const_0_to_255_operand")))]
10690 int mask = INTVAL (operands[2]);
10691 rtx perm[<ssescalarnum>];
10693 perm[0] = GEN_INT (mask & 3);
10694 perm[1] = GEN_INT ((mask >> 2) & 3);
10695 perm[2] = GEN_INT ((mask >> 4) & 3);
10696 perm[3] = GEN_INT ((mask >> 6) & 3);
10697 if (<MODE>mode == V8SFmode)
10699 perm[4] = GEN_INT ((mask & 3) + 4);
10700 perm[5] = GEN_INT (((mask >> 2) & 3) + 4);
10701 perm[6] = GEN_INT (((mask >> 4) & 3) + 4);
10702 perm[7] = GEN_INT (((mask >> 6) & 3) + 4);
10706 = gen_rtx_PARALLEL (VOIDmode, gen_rtvec_v (<ssescalarnum>, perm));
10709 (define_insn "*avx_vpermilp<mode>"
10710 [(set (match_operand:VF 0 "register_operand" "=x")
10712 (match_operand:VF 1 "nonimmediate_operand" "xm")
10713 (match_parallel 2 ""
10714 [(match_operand 3 "const_int_operand")])))]
10716 && avx_vpermilp_parallel (operands[2], <MODE>mode)"
10718 int mask = avx_vpermilp_parallel (operands[2], <MODE>mode) - 1;
10719 operands[2] = GEN_INT (mask);
10720 return "vpermil<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}";
10722 [(set_attr "type" "sselog")
10723 (set_attr "prefix_extra" "1")
10724 (set_attr "length_immediate" "1")
10725 (set_attr "prefix" "vex")
10726 (set_attr "mode" "<MODE>")])
10728 (define_insn "avx_vpermilvar<mode>3"
10729 [(set (match_operand:VF 0 "register_operand" "=x")
10731 [(match_operand:VF 1 "register_operand" "x")
10732 (match_operand:<sseintvecmode> 2 "nonimmediate_operand" "xm")]
10735 "vpermil<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
10736 [(set_attr "type" "sselog")
10737 (set_attr "prefix_extra" "1")
10738 (set_attr "prefix" "vex")
10739 (set_attr "btver2_decode" "vector")
10740 (set_attr "mode" "<MODE>")])
10742 (define_expand "avx_vperm2f128<mode>3"
10743 [(set (match_operand:AVX256MODE2P 0 "register_operand")
10744 (unspec:AVX256MODE2P
10745 [(match_operand:AVX256MODE2P 1 "register_operand")
10746 (match_operand:AVX256MODE2P 2 "nonimmediate_operand")
10747 (match_operand:SI 3 "const_0_to_255_operand")]
10748 UNSPEC_VPERMIL2F128))]
10751 int mask = INTVAL (operands[3]);
10752 if ((mask & 0x88) == 0)
10754 rtx perm[<ssescalarnum>], t1, t2;
10755 int i, base, nelt = <ssescalarnum>, nelt2 = nelt / 2;
10757 base = (mask & 3) * nelt2;
10758 for (i = 0; i < nelt2; ++i)
10759 perm[i] = GEN_INT (base + i);
10761 base = ((mask >> 4) & 3) * nelt2;
10762 for (i = 0; i < nelt2; ++i)
10763 perm[i + nelt2] = GEN_INT (base + i);
10765 t2 = gen_rtx_VEC_CONCAT (<ssedoublevecmode>mode,
10766 operands[1], operands[2]);
10767 t1 = gen_rtx_PARALLEL (VOIDmode, gen_rtvec_v (nelt, perm));
10768 t2 = gen_rtx_VEC_SELECT (<MODE>mode, t2, t1);
10769 t2 = gen_rtx_SET (VOIDmode, operands[0], t2);
10775 ;; Note that bits 7 and 3 of the imm8 allow lanes to be zeroed, which
10776 ;; means that in order to represent this properly in rtl we'd have to
10777 ;; nest *another* vec_concat with a zero operand and do the select from
10778 ;; a 4x wide vector. That doesn't seem very nice.
10779 (define_insn "*avx_vperm2f128<mode>_full"
10780 [(set (match_operand:AVX256MODE2P 0 "register_operand" "=x")
10781 (unspec:AVX256MODE2P
10782 [(match_operand:AVX256MODE2P 1 "register_operand" "x")
10783 (match_operand:AVX256MODE2P 2 "nonimmediate_operand" "xm")
10784 (match_operand:SI 3 "const_0_to_255_operand" "n")]
10785 UNSPEC_VPERMIL2F128))]
10787 "vperm2<i128>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
10788 [(set_attr "type" "sselog")
10789 (set_attr "prefix_extra" "1")
10790 (set_attr "length_immediate" "1")
10791 (set_attr "prefix" "vex")
10792 (set_attr "mode" "<sseinsnmode>")])
10794 (define_insn "*avx_vperm2f128<mode>_nozero"
10795 [(set (match_operand:AVX256MODE2P 0 "register_operand" "=x")
10796 (vec_select:AVX256MODE2P
10797 (vec_concat:<ssedoublevecmode>
10798 (match_operand:AVX256MODE2P 1 "register_operand" "x")
10799 (match_operand:AVX256MODE2P 2 "nonimmediate_operand" "xm"))
10800 (match_parallel 3 ""
10801 [(match_operand 4 "const_int_operand")])))]
10803 && avx_vperm2f128_parallel (operands[3], <MODE>mode)"
10805 int mask = avx_vperm2f128_parallel (operands[3], <MODE>mode) - 1;
10807 return "vinsert<i128>\t{$0, %x2, %1, %0|%0, %1, %x2, 0}";
10809 return "vinsert<i128>\t{$1, %x2, %1, %0|%0, %1, %x2, 1}";
10810 operands[3] = GEN_INT (mask);
10811 return "vperm2<i128>\t{%3, %2, %1, %0|%0, %1, %2, %3}";
10813 [(set_attr "type" "sselog")
10814 (set_attr "prefix_extra" "1")
10815 (set_attr "length_immediate" "1")
10816 (set_attr "prefix" "vex")
10817 (set_attr "mode" "<sseinsnmode>")])
10819 (define_expand "avx_vinsertf128<mode>"
10820 [(match_operand:V_256 0 "register_operand")
10821 (match_operand:V_256 1 "register_operand")
10822 (match_operand:<ssehalfvecmode> 2 "nonimmediate_operand")
10823 (match_operand:SI 3 "const_0_to_1_operand")]
10826 rtx (*insn)(rtx, rtx, rtx);
10828 switch (INTVAL (operands[3]))
10831 insn = gen_vec_set_lo_<mode>;
10834 insn = gen_vec_set_hi_<mode>;
10837 gcc_unreachable ();
10840 emit_insn (insn (operands[0], operands[1], operands[2]));
10844 (define_insn "avx2_vec_set_lo_v4di"
10845 [(set (match_operand:V4DI 0 "register_operand" "=x")
10847 (match_operand:V2DI 2 "nonimmediate_operand" "xm")
10849 (match_operand:V4DI 1 "register_operand" "x")
10850 (parallel [(const_int 2) (const_int 3)]))))]
10852 "vinserti128\t{$0x0, %2, %1, %0|%0, %1, %2, 0x0}"
10853 [(set_attr "type" "sselog")
10854 (set_attr "prefix_extra" "1")
10855 (set_attr "length_immediate" "1")
10856 (set_attr "prefix" "vex")
10857 (set_attr "mode" "OI")])
10859 (define_insn "avx2_vec_set_hi_v4di"
10860 [(set (match_operand:V4DI 0 "register_operand" "=x")
10863 (match_operand:V4DI 1 "register_operand" "x")
10864 (parallel [(const_int 0) (const_int 1)]))
10865 (match_operand:V2DI 2 "nonimmediate_operand" "xm")))]
10867 "vinserti128\t{$0x1, %2, %1, %0|%0, %1, %2, 0x1}"
10868 [(set_attr "type" "sselog")
10869 (set_attr "prefix_extra" "1")
10870 (set_attr "length_immediate" "1")
10871 (set_attr "prefix" "vex")
10872 (set_attr "mode" "OI")])
10874 (define_insn "vec_set_lo_<mode>"
10875 [(set (match_operand:VI8F_256 0 "register_operand" "=x")
10876 (vec_concat:VI8F_256
10877 (match_operand:<ssehalfvecmode> 2 "nonimmediate_operand" "xm")
10878 (vec_select:<ssehalfvecmode>
10879 (match_operand:VI8F_256 1 "register_operand" "x")
10880 (parallel [(const_int 2) (const_int 3)]))))]
10882 "vinsert<i128>\t{$0x0, %2, %1, %0|%0, %1, %2, 0x0}"
10883 [(set_attr "type" "sselog")
10884 (set_attr "prefix_extra" "1")
10885 (set_attr "length_immediate" "1")
10886 (set_attr "prefix" "vex")
10887 (set_attr "mode" "<sseinsnmode>")])
10889 (define_insn "vec_set_hi_<mode>"
10890 [(set (match_operand:VI8F_256 0 "register_operand" "=x")
10891 (vec_concat:VI8F_256
10892 (vec_select:<ssehalfvecmode>
10893 (match_operand:VI8F_256 1 "register_operand" "x")
10894 (parallel [(const_int 0) (const_int 1)]))
10895 (match_operand:<ssehalfvecmode> 2 "nonimmediate_operand" "xm")))]
10897 "vinsert<i128>\t{$0x1, %2, %1, %0|%0, %1, %2, 0x1}"
10898 [(set_attr "type" "sselog")
10899 (set_attr "prefix_extra" "1")
10900 (set_attr "length_immediate" "1")
10901 (set_attr "prefix" "vex")
10902 (set_attr "mode" "<sseinsnmode>")])
10904 (define_insn "vec_set_lo_<mode>"
10905 [(set (match_operand:VI4F_256 0 "register_operand" "=x")
10906 (vec_concat:VI4F_256
10907 (match_operand:<ssehalfvecmode> 2 "nonimmediate_operand" "xm")
10908 (vec_select:<ssehalfvecmode>
10909 (match_operand:VI4F_256 1 "register_operand" "x")
10910 (parallel [(const_int 4) (const_int 5)
10911 (const_int 6) (const_int 7)]))))]
10913 "vinsert<i128>\t{$0x0, %2, %1, %0|%0, %1, %2, 0x0}"
10914 [(set_attr "type" "sselog")
10915 (set_attr "prefix_extra" "1")
10916 (set_attr "length_immediate" "1")
10917 (set_attr "prefix" "vex")
10918 (set_attr "mode" "<sseinsnmode>")])
10920 (define_insn "vec_set_hi_<mode>"
10921 [(set (match_operand:VI4F_256 0 "register_operand" "=x")
10922 (vec_concat:VI4F_256
10923 (vec_select:<ssehalfvecmode>
10924 (match_operand:VI4F_256 1 "register_operand" "x")
10925 (parallel [(const_int 0) (const_int 1)
10926 (const_int 2) (const_int 3)]))
10927 (match_operand:<ssehalfvecmode> 2 "nonimmediate_operand" "xm")))]
10929 "vinsert<i128>\t{$0x1, %2, %1, %0|%0, %1, %2, 0x1}"
10930 [(set_attr "type" "sselog")
10931 (set_attr "prefix_extra" "1")
10932 (set_attr "length_immediate" "1")
10933 (set_attr "prefix" "vex")
10934 (set_attr "mode" "<sseinsnmode>")])
10936 (define_insn "vec_set_lo_v16hi"
10937 [(set (match_operand:V16HI 0 "register_operand" "=x")
10939 (match_operand:V8HI 2 "nonimmediate_operand" "xm")
10941 (match_operand:V16HI 1 "register_operand" "x")
10942 (parallel [(const_int 8) (const_int 9)
10943 (const_int 10) (const_int 11)
10944 (const_int 12) (const_int 13)
10945 (const_int 14) (const_int 15)]))))]
10947 "vinsert%~128\t{$0x0, %2, %1, %0|%0, %1, %2, 0x0}"
10948 [(set_attr "type" "sselog")
10949 (set_attr "prefix_extra" "1")
10950 (set_attr "length_immediate" "1")
10951 (set_attr "prefix" "vex")
10952 (set_attr "mode" "OI")])
10954 (define_insn "vec_set_hi_v16hi"
10955 [(set (match_operand:V16HI 0 "register_operand" "=x")
10958 (match_operand:V16HI 1 "register_operand" "x")
10959 (parallel [(const_int 0) (const_int 1)
10960 (const_int 2) (const_int 3)
10961 (const_int 4) (const_int 5)
10962 (const_int 6) (const_int 7)]))
10963 (match_operand:V8HI 2 "nonimmediate_operand" "xm")))]
10965 "vinsert%~128\t{$0x1, %2, %1, %0|%0, %1, %2, 0x1}"
10966 [(set_attr "type" "sselog")
10967 (set_attr "prefix_extra" "1")
10968 (set_attr "length_immediate" "1")
10969 (set_attr "prefix" "vex")
10970 (set_attr "mode" "OI")])
10972 (define_insn "vec_set_lo_v32qi"
10973 [(set (match_operand:V32QI 0 "register_operand" "=x")
10975 (match_operand:V16QI 2 "nonimmediate_operand" "xm")
10977 (match_operand:V32QI 1 "register_operand" "x")
10978 (parallel [(const_int 16) (const_int 17)
10979 (const_int 18) (const_int 19)
10980 (const_int 20) (const_int 21)
10981 (const_int 22) (const_int 23)
10982 (const_int 24) (const_int 25)
10983 (const_int 26) (const_int 27)
10984 (const_int 28) (const_int 29)
10985 (const_int 30) (const_int 31)]))))]
10987 "vinsert%~128\t{$0x0, %2, %1, %0|%0, %1, %2, 0x0}"
10988 [(set_attr "type" "sselog")
10989 (set_attr "prefix_extra" "1")
10990 (set_attr "length_immediate" "1")
10991 (set_attr "prefix" "vex")
10992 (set_attr "mode" "OI")])
10994 (define_insn "vec_set_hi_v32qi"
10995 [(set (match_operand:V32QI 0 "register_operand" "=x")
10998 (match_operand:V32QI 1 "register_operand" "x")
10999 (parallel [(const_int 0) (const_int 1)
11000 (const_int 2) (const_int 3)
11001 (const_int 4) (const_int 5)
11002 (const_int 6) (const_int 7)
11003 (const_int 8) (const_int 9)
11004 (const_int 10) (const_int 11)
11005 (const_int 12) (const_int 13)
11006 (const_int 14) (const_int 15)]))
11007 (match_operand:V16QI 2 "nonimmediate_operand" "xm")))]
11009 "vinsert%~128\t{$0x1, %2, %1, %0|%0, %1, %2, 0x1}"
11010 [(set_attr "type" "sselog")
11011 (set_attr "prefix_extra" "1")
11012 (set_attr "length_immediate" "1")
11013 (set_attr "prefix" "vex")
11014 (set_attr "mode" "OI")])
11016 (define_insn "<avx_avx2>_maskload<ssemodesuffix><avxsizesuffix>"
11017 [(set (match_operand:V48_AVX2 0 "register_operand" "=x")
11019 [(match_operand:<sseintvecmode> 2 "register_operand" "x")
11020 (match_operand:V48_AVX2 1 "memory_operand" "m")]
11023 "v<sseintprefix>maskmov<ssemodesuffix>\t{%1, %2, %0|%0, %2, %1}"
11024 [(set_attr "type" "sselog1")
11025 (set_attr "prefix_extra" "1")
11026 (set_attr "prefix" "vex")
11027 (set_attr "btver2_decode" "vector")
11028 (set_attr "mode" "<sseinsnmode>")])
11030 (define_insn "<avx_avx2>_maskstore<ssemodesuffix><avxsizesuffix>"
11031 [(set (match_operand:V48_AVX2 0 "memory_operand" "+m")
11033 [(match_operand:<sseintvecmode> 1 "register_operand" "x")
11034 (match_operand:V48_AVX2 2 "register_operand" "x")
11038 "v<sseintprefix>maskmov<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
11039 [(set_attr "type" "sselog1")
11040 (set_attr "prefix_extra" "1")
11041 (set_attr "prefix" "vex")
11042 (set_attr "btver2_decode" "vector")
11043 (set_attr "mode" "<sseinsnmode>")])
11045 (define_insn_and_split "avx_<castmode><avxsizesuffix>_<castmode>"
11046 [(set (match_operand:AVX256MODE2P 0 "nonimmediate_operand" "=x,m")
11047 (unspec:AVX256MODE2P
11048 [(match_operand:<ssehalfvecmode> 1 "nonimmediate_operand" "xm,x")]
11052 "&& reload_completed"
11055 rtx op0 = operands[0];
11056 rtx op1 = operands[1];
11058 op0 = gen_rtx_REG (<ssehalfvecmode>mode, REGNO (op0));
11060 op1 = gen_rtx_REG (<MODE>mode, REGNO (op1));
11061 emit_move_insn (op0, op1);
11065 (define_expand "vec_init<mode>"
11066 [(match_operand:V_256 0 "register_operand")
11070 ix86_expand_vector_init (false, operands[0], operands[1]);
11074 (define_expand "avx2_extracti128"
11075 [(match_operand:V2DI 0 "nonimmediate_operand")
11076 (match_operand:V4DI 1 "register_operand")
11077 (match_operand:SI 2 "const_0_to_1_operand")]
11080 rtx (*insn)(rtx, rtx);
11082 switch (INTVAL (operands[2]))
11085 insn = gen_vec_extract_lo_v4di;
11088 insn = gen_vec_extract_hi_v4di;
11091 gcc_unreachable ();
11094 emit_insn (insn (operands[0], operands[1]));
11098 (define_expand "avx2_inserti128"
11099 [(match_operand:V4DI 0 "register_operand")
11100 (match_operand:V4DI 1 "register_operand")
11101 (match_operand:V2DI 2 "nonimmediate_operand")
11102 (match_operand:SI 3 "const_0_to_1_operand")]
11105 rtx (*insn)(rtx, rtx, rtx);
11107 switch (INTVAL (operands[3]))
11110 insn = gen_avx2_vec_set_lo_v4di;
11113 insn = gen_avx2_vec_set_hi_v4di;
11116 gcc_unreachable ();
11119 emit_insn (insn (operands[0], operands[1], operands[2]));
11123 (define_insn "avx2_ashrv<mode>"
11124 [(set (match_operand:VI4_AVX2 0 "register_operand" "=x")
11126 (match_operand:VI4_AVX2 1 "register_operand" "x")
11127 (match_operand:VI4_AVX2 2 "nonimmediate_operand" "xm")))]
11129 "vpsravd\t{%2, %1, %0|%0, %1, %2}"
11130 [(set_attr "type" "sseishft")
11131 (set_attr "prefix" "vex")
11132 (set_attr "mode" "<sseinsnmode>")])
11134 (define_insn "avx2_<shift_insn>v<mode>"
11135 [(set (match_operand:VI48_AVX2 0 "register_operand" "=x")
11136 (any_lshift:VI48_AVX2
11137 (match_operand:VI48_AVX2 1 "register_operand" "x")
11138 (match_operand:VI48_AVX2 2 "nonimmediate_operand" "xm")))]
11140 "vp<vshift>v<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
11141 [(set_attr "type" "sseishft")
11142 (set_attr "prefix" "vex")
11143 (set_attr "mode" "<sseinsnmode>")])
11145 (define_insn "avx_vec_concat<mode>"
11146 [(set (match_operand:V_256 0 "register_operand" "=x,x")
11148 (match_operand:<ssehalfvecmode> 1 "register_operand" "x,x")
11149 (match_operand:<ssehalfvecmode> 2 "vector_move_operand" "xm,C")))]
11152 switch (which_alternative)
11155 return "vinsert<i128>\t{$0x1, %2, %t1, %0|%0, %t1, %2, 0x1}";
11157 switch (get_attr_mode (insn))
11160 return "vmovaps\t{%1, %x0|%x0, %1}";
11162 return "vmovapd\t{%1, %x0|%x0, %1}";
11164 return "vmovdqa\t{%1, %x0|%x0, %1}";
11167 gcc_unreachable ();
11170 [(set_attr "type" "sselog,ssemov")
11171 (set_attr "prefix_extra" "1,*")
11172 (set_attr "length_immediate" "1,*")
11173 (set_attr "prefix" "vex")
11174 (set_attr "mode" "<sseinsnmode>")])
11176 (define_insn "vcvtph2ps"
11177 [(set (match_operand:V4SF 0 "register_operand" "=x")
11179 (unspec:V8SF [(match_operand:V8HI 1 "register_operand" "x")]
11181 (parallel [(const_int 0) (const_int 1)
11182 (const_int 2) (const_int 3)])))]
11184 "vcvtph2ps\t{%1, %0|%0, %1}"
11185 [(set_attr "type" "ssecvt")
11186 (set_attr "prefix" "vex")
11187 (set_attr "mode" "V4SF")])
11189 (define_insn "*vcvtph2ps_load"
11190 [(set (match_operand:V4SF 0 "register_operand" "=x")
11191 (unspec:V4SF [(match_operand:V4HI 1 "memory_operand" "m")]
11192 UNSPEC_VCVTPH2PS))]
11194 "vcvtph2ps\t{%1, %0|%0, %1}"
11195 [(set_attr "type" "ssecvt")
11196 (set_attr "prefix" "vex")
11197 (set_attr "mode" "V8SF")])
11199 (define_insn "vcvtph2ps256"
11200 [(set (match_operand:V8SF 0 "register_operand" "=x")
11201 (unspec:V8SF [(match_operand:V8HI 1 "nonimmediate_operand" "xm")]
11202 UNSPEC_VCVTPH2PS))]
11204 "vcvtph2ps\t{%1, %0|%0, %1}"
11205 [(set_attr "type" "ssecvt")
11206 (set_attr "prefix" "vex")
11207 (set_attr "btver2_decode" "double")
11208 (set_attr "mode" "V8SF")])
11210 (define_expand "vcvtps2ph"
11211 [(set (match_operand:V8HI 0 "register_operand")
11213 (unspec:V4HI [(match_operand:V4SF 1 "register_operand")
11214 (match_operand:SI 2 "const_0_to_255_operand")]
11218 "operands[3] = CONST0_RTX (V4HImode);")
11220 (define_insn "*vcvtps2ph"
11221 [(set (match_operand:V8HI 0 "register_operand" "=x")
11223 (unspec:V4HI [(match_operand:V4SF 1 "register_operand" "x")
11224 (match_operand:SI 2 "const_0_to_255_operand" "N")]
11226 (match_operand:V4HI 3 "const0_operand")))]
11228 "vcvtps2ph\t{%2, %1, %0|%0, %1, %2}"
11229 [(set_attr "type" "ssecvt")
11230 (set_attr "prefix" "vex")
11231 (set_attr "mode" "V4SF")])
11233 (define_insn "*vcvtps2ph_store"
11234 [(set (match_operand:V4HI 0 "memory_operand" "=m")
11235 (unspec:V4HI [(match_operand:V4SF 1 "register_operand" "x")
11236 (match_operand:SI 2 "const_0_to_255_operand" "N")]
11237 UNSPEC_VCVTPS2PH))]
11239 "vcvtps2ph\t{%2, %1, %0|%0, %1, %2}"
11240 [(set_attr "type" "ssecvt")
11241 (set_attr "prefix" "vex")
11242 (set_attr "mode" "V4SF")])
11244 (define_insn "vcvtps2ph256"
11245 [(set (match_operand:V8HI 0 "nonimmediate_operand" "=xm")
11246 (unspec:V8HI [(match_operand:V8SF 1 "register_operand" "x")
11247 (match_operand:SI 2 "const_0_to_255_operand" "N")]
11248 UNSPEC_VCVTPS2PH))]
11250 "vcvtps2ph\t{%2, %1, %0|%0, %1, %2}"
11251 [(set_attr "type" "ssecvt")
11252 (set_attr "prefix" "vex")
11253 (set_attr "btver2_decode" "vector")
11254 (set_attr "mode" "V8SF")])
11256 ;; For gather* insn patterns
11257 (define_mode_iterator VEC_GATHER_MODE
11258 [V2DI V2DF V4DI V4DF V4SI V4SF V8SI V8SF])
11259 (define_mode_attr VEC_GATHER_IDXSI
11260 [(V2DI "V4SI") (V2DF "V4SI")
11261 (V4DI "V4SI") (V4DF "V4SI")
11262 (V4SI "V4SI") (V4SF "V4SI")
11263 (V8SI "V8SI") (V8SF "V8SI")])
11264 (define_mode_attr VEC_GATHER_IDXDI
11265 [(V2DI "V2DI") (V2DF "V2DI")
11266 (V4DI "V4DI") (V4DF "V4DI")
11267 (V4SI "V2DI") (V4SF "V2DI")
11268 (V8SI "V4DI") (V8SF "V4DI")])
11269 (define_mode_attr VEC_GATHER_SRCDI
11270 [(V2DI "V2DI") (V2DF "V2DF")
11271 (V4DI "V4DI") (V4DF "V4DF")
11272 (V4SI "V4SI") (V4SF "V4SF")
11273 (V8SI "V4SI") (V8SF "V4SF")])
11275 (define_expand "avx2_gathersi<mode>"
11276 [(parallel [(set (match_operand:VEC_GATHER_MODE 0 "register_operand")
11277 (unspec:VEC_GATHER_MODE
11278 [(match_operand:VEC_GATHER_MODE 1 "register_operand")
11279 (mem:<ssescalarmode>
11281 [(match_operand 2 "vsib_address_operand")
11282 (match_operand:<VEC_GATHER_IDXSI>
11283 3 "register_operand")
11284 (match_operand:SI 5 "const1248_operand ")]))
11285 (mem:BLK (scratch))
11286 (match_operand:VEC_GATHER_MODE 4 "register_operand")]
11288 (clobber (match_scratch:VEC_GATHER_MODE 6))])]
11292 = gen_rtx_UNSPEC (Pmode, gen_rtvec (3, operands[2], operands[3],
11293 operands[5]), UNSPEC_VSIBADDR);
11296 (define_insn "*avx2_gathersi<mode>"
11297 [(set (match_operand:VEC_GATHER_MODE 0 "register_operand" "=&x")
11298 (unspec:VEC_GATHER_MODE
11299 [(match_operand:VEC_GATHER_MODE 2 "register_operand" "0")
11300 (match_operator:<ssescalarmode> 7 "vsib_mem_operator"
11302 [(match_operand:P 3 "vsib_address_operand" "p")
11303 (match_operand:<VEC_GATHER_IDXSI> 4 "register_operand" "x")
11304 (match_operand:SI 6 "const1248_operand" "n")]
11306 (mem:BLK (scratch))
11307 (match_operand:VEC_GATHER_MODE 5 "register_operand" "1")]
11309 (clobber (match_scratch:VEC_GATHER_MODE 1 "=&x"))]
11311 "v<sseintprefix>gatherd<ssemodesuffix>\t{%1, %7, %0|%0, %7, %1}"
11312 [(set_attr "type" "ssemov")
11313 (set_attr "prefix" "vex")
11314 (set_attr "mode" "<sseinsnmode>")])
11316 (define_insn "*avx2_gathersi<mode>_2"
11317 [(set (match_operand:VEC_GATHER_MODE 0 "register_operand" "=&x")
11318 (unspec:VEC_GATHER_MODE
11320 (match_operator:<ssescalarmode> 6 "vsib_mem_operator"
11322 [(match_operand:P 2 "vsib_address_operand" "p")
11323 (match_operand:<VEC_GATHER_IDXSI> 3 "register_operand" "x")
11324 (match_operand:SI 5 "const1248_operand" "n")]
11326 (mem:BLK (scratch))
11327 (match_operand:VEC_GATHER_MODE 4 "register_operand" "1")]
11329 (clobber (match_scratch:VEC_GATHER_MODE 1 "=&x"))]
11331 "v<sseintprefix>gatherd<ssemodesuffix>\t{%1, %6, %0|%0, %6, %1}"
11332 [(set_attr "type" "ssemov")
11333 (set_attr "prefix" "vex")
11334 (set_attr "mode" "<sseinsnmode>")])
11336 (define_expand "avx2_gatherdi<mode>"
11337 [(parallel [(set (match_operand:VEC_GATHER_MODE 0 "register_operand")
11338 (unspec:VEC_GATHER_MODE
11339 [(match_operand:<VEC_GATHER_SRCDI> 1 "register_operand")
11340 (mem:<ssescalarmode>
11342 [(match_operand 2 "vsib_address_operand")
11343 (match_operand:<VEC_GATHER_IDXDI>
11344 3 "register_operand")
11345 (match_operand:SI 5 "const1248_operand ")]))
11346 (mem:BLK (scratch))
11347 (match_operand:<VEC_GATHER_SRCDI>
11348 4 "register_operand")]
11350 (clobber (match_scratch:VEC_GATHER_MODE 6))])]
11354 = gen_rtx_UNSPEC (Pmode, gen_rtvec (3, operands[2], operands[3],
11355 operands[5]), UNSPEC_VSIBADDR);
11358 (define_insn "*avx2_gatherdi<mode>"
11359 [(set (match_operand:VEC_GATHER_MODE 0 "register_operand" "=&x")
11360 (unspec:VEC_GATHER_MODE
11361 [(match_operand:<VEC_GATHER_SRCDI> 2 "register_operand" "0")
11362 (match_operator:<ssescalarmode> 7 "vsib_mem_operator"
11364 [(match_operand:P 3 "vsib_address_operand" "p")
11365 (match_operand:<VEC_GATHER_IDXDI> 4 "register_operand" "x")
11366 (match_operand:SI 6 "const1248_operand" "n")]
11368 (mem:BLK (scratch))
11369 (match_operand:<VEC_GATHER_SRCDI> 5 "register_operand" "1")]
11371 (clobber (match_scratch:VEC_GATHER_MODE 1 "=&x"))]
11373 "v<sseintprefix>gatherq<ssemodesuffix>\t{%5, %7, %2|%2, %7, %5}"
11374 [(set_attr "type" "ssemov")
11375 (set_attr "prefix" "vex")
11376 (set_attr "mode" "<sseinsnmode>")])
11378 (define_insn "*avx2_gatherdi<mode>_2"
11379 [(set (match_operand:VEC_GATHER_MODE 0 "register_operand" "=&x")
11380 (unspec:VEC_GATHER_MODE
11382 (match_operator:<ssescalarmode> 6 "vsib_mem_operator"
11384 [(match_operand:P 2 "vsib_address_operand" "p")
11385 (match_operand:<VEC_GATHER_IDXDI> 3 "register_operand" "x")
11386 (match_operand:SI 5 "const1248_operand" "n")]
11388 (mem:BLK (scratch))
11389 (match_operand:<VEC_GATHER_SRCDI> 4 "register_operand" "1")]
11391 (clobber (match_scratch:VEC_GATHER_MODE 1 "=&x"))]
11394 if (<MODE>mode != <VEC_GATHER_SRCDI>mode)
11395 return "v<sseintprefix>gatherq<ssemodesuffix>\t{%4, %6, %x0|%x0, %6, %4}";
11396 return "v<sseintprefix>gatherq<ssemodesuffix>\t{%4, %6, %0|%0, %6, %4}";
11398 [(set_attr "type" "ssemov")
11399 (set_attr "prefix" "vex")
11400 (set_attr "mode" "<sseinsnmode>")])
11402 (define_insn "*avx2_gatherdi<mode>_3"
11403 [(set (match_operand:<VEC_GATHER_SRCDI> 0 "register_operand" "=&x")
11404 (vec_select:<VEC_GATHER_SRCDI>
11406 [(match_operand:<VEC_GATHER_SRCDI> 2 "register_operand" "0")
11407 (match_operator:<ssescalarmode> 7 "vsib_mem_operator"
11409 [(match_operand:P 3 "vsib_address_operand" "p")
11410 (match_operand:<VEC_GATHER_IDXDI> 4 "register_operand" "x")
11411 (match_operand:SI 6 "const1248_operand" "n")]
11413 (mem:BLK (scratch))
11414 (match_operand:<VEC_GATHER_SRCDI> 5 "register_operand" "1")]
11416 (parallel [(const_int 0) (const_int 1)
11417 (const_int 2) (const_int 3)])))
11418 (clobber (match_scratch:VI4F_256 1 "=&x"))]
11420 "v<sseintprefix>gatherq<ssemodesuffix>\t{%5, %7, %0|%0, %7, %5}"
11421 [(set_attr "type" "ssemov")
11422 (set_attr "prefix" "vex")
11423 (set_attr "mode" "<sseinsnmode>")])
11425 (define_insn "*avx2_gatherdi<mode>_4"
11426 [(set (match_operand:<VEC_GATHER_SRCDI> 0 "register_operand" "=&x")
11427 (vec_select:<VEC_GATHER_SRCDI>
11430 (match_operator:<ssescalarmode> 6 "vsib_mem_operator"
11432 [(match_operand:P 2 "vsib_address_operand" "p")
11433 (match_operand:<VEC_GATHER_IDXDI> 3 "register_operand" "x")
11434 (match_operand:SI 5 "const1248_operand" "n")]
11436 (mem:BLK (scratch))
11437 (match_operand:<VEC_GATHER_SRCDI> 4 "register_operand" "1")]
11439 (parallel [(const_int 0) (const_int 1)
11440 (const_int 2) (const_int 3)])))
11441 (clobber (match_scratch:VI4F_256 1 "=&x"))]
11443 "v<sseintprefix>gatherq<ssemodesuffix>\t{%4, %6, %0|%0, %6, %4}"
11444 [(set_attr "type" "ssemov")
11445 (set_attr "prefix" "vex")
11446 (set_attr "mode" "<sseinsnmode>")])