| blob | 3b4117eb3fff15c16bc1b55544f3a2b9bf954ef7 |
1 /* Subroutines used for code generation on IA-32.
2 Copyright (C) 1988, 1992, 1994, 1995, 1996, 1997, 1998, 1999, 2000,
3 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012
4 Free Software Foundation, Inc.
6 This file is part of GCC.
8 GCC is free software; you can redistribute it and/or modify
9 it under the terms of the GNU General Public License as published by
10 the Free Software Foundation; either version 3, or (at your option)
11 any later version.
13 GCC is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU General Public License for more details.
18 You should have received a copy of the GNU General Public License
19 along with GCC; see the file COPYING3. If not see
20 <http://www.gnu.org/licenses/>. */
66 enum upper_128bits_state
67 {
69 unused,
70 used
71 };
74 {
75 /* State of the upper 128bits of AVX registers at exit. */
77 /* TRUE if state of the upper 128bits of AVX registers is unchanged
78 in this block. */
80 /* TRUE if block has been processed. */
82 /* TRUE if block has been scanned. */
84 /* Previous state of the upper 128bits of AVX registers at entry. */
88 #define BLOCK_INFO(B) ((block_info) (B)->aux)
90 enum call_avx256_state
91 {
92 /* Callee returns 256bit AVX register. */
94 /* Callee returns and passes 256bit AVX register. */
95 callee_return_pass_avx256,
96 /* Callee passes 256bit AVX register. */
97 callee_pass_avx256,
98 /* Callee doesn't return nor passe 256bit AVX register, or no
99 256bit AVX register in function return. */
100 call_no_avx256,
101 /* vzeroupper intrinsic. */
102 vzeroupper_intrinsic
103 };
105 /* Check if a 256bit AVX register is referenced in stores. */
107 static void
109 {
115 {
119 }
120 }
122 /* Helper function for move_or_delete_vzeroupper_1. Look for vzeroupper
123 in basic block BB. Delete it if upper 128bit AVX registers are
124 unused. If it isn't deleted, move it to just before a jump insn.
126 STATE is state of the upper 128bits of AVX registers at entry. */
128 static void
131 {
134 rtx pat;
139 {
146 }
149 {
154 }
164 /* BB_END changes when it is deleted. */
168 {
174 /* Move vzeroupper before jump/call. */
176 {
181 {
183 {
188 }
191 }
194 }
198 /* Check insn for vzeroupper intrinsic. */
201 {
203 {
204 /* Found vzeroupper intrinsic. */
207 }
208 }
209 else
210 {
211 /* Check insn for vzeroall intrinsic. */
215 {
219 /* Delete pending vzeroupper insertion. */
221 {
224 }
225 }
227 {
231 }
233 }
235 /* Process vzeroupper intrinsic. */
239 {
240 /* Since the upper 128bits are cleared, callee must not pass
241 256bit AVX register. We only need to check if callee
242 returns 256bit AVX register. */
244 {
247 }
249 /* Remove unnecessary vzeroupper since upper 128bits are
250 cleared. */
252 {
255 }
257 }
258 else
259 {
260 /* Set state to UNUSED if callee doesn't return 256bit AVX
261 register. */
267 {
268 /* Must remove vzeroupper since callee passes in 256bit
269 AVX register. */
271 {
274 }
276 }
277 else
278 {
281 }
282 }
283 }
292 state);
293 }
295 /* Helper function for move_or_delete_vzeroupper. Process vzeroupper
296 in BLOCK and check its predecessor blocks. Treat UNKNOWN state
297 as USED if UNKNOWN_IS_UNUSED is true. Return TRUE if the exit
298 state is changed. */
300 static bool
302 {
303 edge e;
304 edge_iterator ei;
317 /* Check all predecessor edges of this block. */
320 {
324 {
333 }
334 }
339 done:
347 /* Need to rescan if the upper 128bits of AVX registers are changed
348 to USED at exit. */
350 {
354 }
355 else
357 }
359 /* Go through the instruction stream looking for vzeroupper. Delete
360 it if upper 128bit AVX registers are unused. If it isn't deleted,
361 move it to just before a jump insn. */
363 static void
365 {
366 edge e;
367 edge_iterator ei;
368 basic_block bb;
375 /* Set up block info for each basic block. */
378 /* Process outgoing edges of entry point. */
383 {
388 }
390 /* Compute reverse completion order of depth first search of the CFG
391 so that the data-flow runs faster. */
406 /* Don't check outgoing edges of entry point. */
411 else
412 {
415 }
421 {
434 {
439 {
440 edge_iterator ei;
446 {
452 {
454 {
455 /* Send E->DEST to next round. */
460 }
461 }
463 {
464 /* Add E->DEST to current round. */
468 }
469 }
470 }
471 }
475 }
491 }
495 #ifndef CHECK_STACK_LIMIT
496 #define CHECK_STACK_LIMIT (-1)
497 #endif
499 /* Return index of given mode in mult and division cost tables. */
500 #define MODE_INDEX(mode) \
501 ((mode) == QImode ? 0 \
502 : (mode) == HImode ? 1 \
503 : (mode) == SImode ? 2 \
504 : (mode) == DImode ? 3 \
505 : 4)
507 /* Processor costs (relative to an add) */
508 /* We assume COSTS_N_INSNS is defined as (N)*4 and an addition is 2 bytes. */
509 #define COSTS_N_BYTES(N) ((N) * 2)
511 #define DUMMY_STRINGOP_ALGS {libcall, {{-1, libcall}}}
513 const
536 in QImode, HImode and SImode.
537 Relative to reg-reg move (2). */
541 in SFmode, DFmode and XFmode */
543 in SFmode, DFmode and XFmode */
546 in SImode and DImode */
548 in SImode and DImode */
551 in SImode, DImode and TImode */
553 in SImode, DImode and TImode */
581 };
583 /* Processor costs (relative to an add) */
584 static const
607 in QImode, HImode and SImode.
608 Relative to reg-reg move (2). */
612 in SFmode, DFmode and XFmode */
614 in SFmode, DFmode and XFmode */
617 in SImode and DImode */
619 in SImode and DImode */
622 in SImode, DImode and TImode */
624 in SImode, DImode and TImode */
638 DUMMY_STRINGOP_ALGS},
640 DUMMY_STRINGOP_ALGS},
652 };
654 static const
677 in QImode, HImode and SImode.
678 Relative to reg-reg move (2). */
682 in SFmode, DFmode and XFmode */
684 in SFmode, DFmode and XFmode */
687 in SImode and DImode */
689 in SImode and DImode */
692 in SImode, DImode and TImode */
694 in SImode, DImode and TImode */
697 shared for code and data, so 4kB is
698 not really precise. */
710 DUMMY_STRINGOP_ALGS},
712 DUMMY_STRINGOP_ALGS},
724 };
726 static const
749 in QImode, HImode and SImode.
750 Relative to reg-reg move (2). */
754 in SFmode, DFmode and XFmode */
756 in SFmode, DFmode and XFmode */
759 in SImode and DImode */
761 in SImode and DImode */
764 in SImode, DImode and TImode */
766 in SImode, DImode and TImode */
780 DUMMY_STRINGOP_ALGS},
782 DUMMY_STRINGOP_ALGS},
794 };
796 static const
819 in QImode, HImode and SImode.
820 Relative to reg-reg move (2). */
824 in SFmode, DFmode and XFmode */
826 in SFmode, DFmode and XFmode */
829 in SImode and DImode */
831 in SImode and DImode */
834 in SImode, DImode and TImode */
836 in SImode, DImode and TImode */
849 /* PentiumPro has optimized rep instructions for blocks aligned by 8 bytes
850 (we ensure the alignment). For small blocks inline loop is still a
851 noticeable win, for bigger blocks either rep movsl or rep movsb is
852 way to go. Rep movsb has apparently more expensive startup time in CPU,
853 but after 4K the difference is down in the noise. */
856 DUMMY_STRINGOP_ALGS},
859 DUMMY_STRINGOP_ALGS},
871 };
873 static const
896 in QImode, HImode and SImode.
897 Relative to reg-reg move (2). */
901 in SFmode, DFmode and XFmode */
903 in SFmode, DFmode and XFmode */
907 in SImode and DImode */
909 in SImode and DImode */
912 in SImode, DImode and TImode */
914 in SImode, DImode and TImode */
928 DUMMY_STRINGOP_ALGS},
930 DUMMY_STRINGOP_ALGS},
942 };
944 static const
967 in QImode, HImode and SImode.
968 Relative to reg-reg move (2). */
972 in SFmode, DFmode and XFmode */
974 in SFmode, DFmode and XFmode */
977 in SImode and DImode */
979 in SImode and DImode */
982 in SImode, DImode and TImode */
984 in SImode, DImode and TImode */
988 have integrated l2 cache, but
989 optimizing for k6 is not important
990 enough to worry about that. */
1001 DUMMY_STRINGOP_ALGS},
1003 DUMMY_STRINGOP_ALGS},
1015 };
1017 static const
1040 in QImode, HImode and SImode.
1041 Relative to reg-reg move (2). */
1045 in SFmode, DFmode and XFmode */
1047 in SFmode, DFmode and XFmode */
1050 in SImode and DImode */
1052 in SImode and DImode */
1055 in SImode, DImode and TImode */
1057 in SImode, DImode and TImode */
1070 /* For some reason, Athlon deals better with REP prefix (relative to loops)
1071 compared to K8. Alignment becomes important after 8 bytes for memcpy and
1072 128 bytes for memset. */
1074 DUMMY_STRINGOP_ALGS},
1076 DUMMY_STRINGOP_ALGS},
1088 };
1090 static const
1113 in QImode, HImode and SImode.
1114 Relative to reg-reg move (2). */
1118 in SFmode, DFmode and XFmode */
1120 in SFmode, DFmode and XFmode */
1123 in SImode and DImode */
1125 in SImode and DImode */
1128 in SImode, DImode and TImode */
1130 in SImode, DImode and TImode */
1135 /* New AMD processors never drop prefetches; if they cannot be performed
1136 immediately, they are queued. We set number of simultaneous prefetches
1137 to a large constant to reflect this (it probably is not a good idea not
1138 to limit number of prefetches at all, as their execution also takes some
1139 time). */
1148 /* K8 has optimized REP instruction for medium sized blocks, but for very
1149 small blocks it is better to use loop. For large blocks, libcall can
1150 do nontemporary accesses and beat inline considerably. */
1167 };
1191 in QImode, HImode and SImode.
1192 Relative to reg-reg move (2). */
1196 in SFmode, DFmode and XFmode */
1198 in SFmode, DFmode and XFmode */
1201 in SImode and DImode */
1203 in SImode and DImode */
1206 in SImode, DImode and TImode */
1208 in SImode, DImode and TImode */
1210 /* On K8:
1211 MOVD reg64, xmmreg Double FSTORE 4
1212 MOVD reg32, xmmreg Double FSTORE 4
1213 On AMDFAM10:
1214 MOVD reg64, xmmreg Double FADD 3
1215 1/1 1/1
1216 MOVD reg32, xmmreg Double FADD 3
1217 1/1 1/1 */
1221 /* New AMD processors never drop prefetches; if they cannot be performed
1222 immediately, they are queued. We set number of simultaneous prefetches
1223 to a large constant to reflect this (it probably is not a good idea not
1224 to limit number of prefetches at all, as their execution also takes some
1225 time). */
1235 /* AMDFAM10 has optimized REP instruction for medium sized blocks, but for
1236 very small blocks it is better to use loop. For large blocks, libcall can
1237 do nontemporary accesses and beat inline considerably. */
1254 };
1278 in QImode, HImode and SImode.
1279 Relative to reg-reg move (2). */
1283 in SFmode, DFmode and XFmode */
1285 in SFmode, DFmode and XFmode */
1288 in SImode and DImode */
1290 in SImode and DImode */
1293 in SImode, DImode and TImode */
1295 in SImode, DImode and TImode */
1297 /* On K8:
1298 MOVD reg64, xmmreg Double FSTORE 4
1299 MOVD reg32, xmmreg Double FSTORE 4
1300 On AMDFAM10:
1301 MOVD reg64, xmmreg Double FADD 3
1302 1/1 1/1
1303 MOVD reg32, xmmreg Double FADD 3
1304 1/1 1/1 */
1308 /* New AMD processors never drop prefetches; if they cannot be performed
1309 immediately, they are queued. We set number of simultaneous prefetches
1310 to a large constant to reflect this (it probably is not a good idea not
1311 to limit number of prefetches at all, as their execution also takes some
1312 time). */
1322 /* BDVER1 has optimized REP instruction for medium sized blocks, but for
1323 very small blocks it is better to use loop. For large blocks, libcall
1324 can do nontemporary accesses and beat inline considerably. */
1341 };
1365 in QImode, HImode and SImode.
1366 Relative to reg-reg move (2). */
1370 in SFmode, DFmode and XFmode */
1372 in SFmode, DFmode and XFmode */
1375 in SImode and DImode */
1377 in SImode and DImode */
1380 in SImode, DImode and TImode */
1382 in SImode, DImode and TImode */
1384 /* On K8:
1385 MOVD reg64, xmmreg Double FSTORE 4
1386 MOVD reg32, xmmreg Double FSTORE 4
1387 On AMDFAM10:
1388 MOVD reg64, xmmreg Double FADD 3
1389 1/1 1/1
1390 MOVD reg32, xmmreg Double FADD 3
1391 1/1 1/1 */
1395 /* New AMD processors never drop prefetches; if they cannot be performed
1396 immediately, they are queued. We set number of simultaneous prefetches
1397 to a large constant to reflect this (it probably is not a good idea not
1398 to limit number of prefetches at all, as their execution also takes some
1399 time). */
1409 /* BDVER2 has optimized REP instruction for medium sized blocks, but for
1410 very small blocks it is better to use loop. For large blocks, libcall
1411 can do nontemporary accesses and beat inline considerably. */
1428 };
1452 in QImode, HImode and SImode.
1453 Relative to reg-reg move (2). */
1457 in SFmode, DFmode and XFmode */
1459 in SFmode, DFmode and XFmode */
1462 in SImode and DImode */
1464 in SImode and DImode */
1467 in SImode, DImode and TImode */
1469 in SImode, DImode and TImode */
1471 /* On K8:
1472 MOVD reg64, xmmreg Double FSTORE 4
1473 MOVD reg32, xmmreg Double FSTORE 4
1474 On AMDFAM10:
1475 MOVD reg64, xmmreg Double FADD 3
1476 1/1 1/1
1477 MOVD reg32, xmmreg Double FADD 3
1478 1/1 1/1 */
1491 /* BTVER1 has optimized REP instruction for medium sized blocks, but for
1492 very small blocks it is better to use loop. For large blocks, libcall can
1493 do nontemporary accesses and beat inline considerably. */
1510 };
1534 in QImode, HImode and SImode.
1535 Relative to reg-reg move (2). */
1539 in SFmode, DFmode and XFmode */
1541 in SFmode, DFmode and XFmode */
1544 in SImode and DImode */
1546 in SImode and DImode */
1549 in SImode, DImode and TImode */
1551 in SImode, DImode and TImode */
1553 /* On K8:
1554 MOVD reg64, xmmreg Double FSTORE 4
1555 MOVD reg32, xmmreg Double FSTORE 4
1556 On AMDFAM10:
1557 MOVD reg64, xmmreg Double FADD 3
1558 1/1 1/1
1559 MOVD reg32, xmmreg Double FADD 3
1560 1/1 1/1 */
1589 };
1591 static const
1614 in QImode, HImode and SImode.
1615 Relative to reg-reg move (2). */
1619 in SFmode, DFmode and XFmode */
1621 in SFmode, DFmode and XFmode */
1624 in SImode and DImode */
1626 in SImode and DImode */
1629 in SImode, DImode and TImode */
1631 in SImode, DImode and TImode */
1645 DUMMY_STRINGOP_ALGS},
1648 DUMMY_STRINGOP_ALGS},
1660 };
1662 static const
1685 in QImode, HImode and SImode.
1686 Relative to reg-reg move (2). */
1690 in SFmode, DFmode and XFmode */
1692 in SFmode, DFmode and XFmode */
1695 in SImode and DImode */
1697 in SImode and DImode */
1700 in SImode, DImode and TImode */
1702 in SImode, DImode and TImode */
1733 };
1735 static const
1758 in QImode, HImode and SImode.
1759 Relative to reg-reg move (2). */
1763 in SFmode, DFmode and XFmode */
1765 in SFmode, DFmode and XFmode */
1768 in SImode and DImode */
1770 in SImode and DImode */
1773 in SImode, DImode and TImode */
1775 in SImode, DImode and TImode */
1806 };
1808 /* Generic64 should produce code tuned for Nocona and K8. */
1809 static const
1812 /* On all chips taken into consideration lea is 2 cycles and more. With
1813 this cost however our current implementation of synth_mult results in
1814 use of unnecessary temporary registers causing regression on several
1815 SPECfp benchmarks. */
1836 in QImode, HImode and SImode.
1837 Relative to reg-reg move (2). */
1841 in SFmode, DFmode and XFmode */
1843 in SFmode, DFmode and XFmode */
1846 in SImode and DImode */
1848 in SImode and DImode */
1851 in SImode, DImode and TImode */
1853 in SImode, DImode and TImode */
1859 /* Benchmarks shows large regressions on K8 sixtrack benchmark when this
1860 value is increased to perhaps more appropriate value of 5. */
1883 };
1885 /* Generic32 should produce code tuned for PPro, Pentium4, Nocona,
1886 Athlon and K8. */
1887 static const
1910 in QImode, HImode and SImode.
1911 Relative to reg-reg move (2). */
1915 in SFmode, DFmode and XFmode */
1917 in SFmode, DFmode and XFmode */
1920 in SImode and DImode */
1922 in SImode and DImode */
1925 in SImode, DImode and TImode */
1927 in SImode, DImode and TImode */
1941 DUMMY_STRINGOP_ALGS},
1943 DUMMY_STRINGOP_ALGS},
1955 };
1957 /* Set by -mtune. */
1960 /* Set by -mtune or -Os. */
1963 /* Processor feature/optimization bitmasks. */
1964 #define m_386 (1<<PROCESSOR_I386)
1965 #define m_486 (1<<PROCESSOR_I486)
1966 #define m_PENT (1<<PROCESSOR_PENTIUM)
1967 #define m_PPRO (1<<PROCESSOR_PENTIUMPRO)
1968 #define m_PENT4 (1<<PROCESSOR_PENTIUM4)
1969 #define m_NOCONA (1<<PROCESSOR_NOCONA)
1970 #define m_P4_NOCONA (m_PENT4 | m_NOCONA)
1971 #define m_CORE2_32 (1<<PROCESSOR_CORE2_32)
1972 #define m_CORE2_64 (1<<PROCESSOR_CORE2_64)
1973 #define m_COREI7_32 (1<<PROCESSOR_COREI7_32)
1974 #define m_COREI7_64 (1<<PROCESSOR_COREI7_64)
1975 #define m_COREI7 (m_COREI7_32 | m_COREI7_64)
1976 #define m_CORE2I7_32 (m_CORE2_32 | m_COREI7_32)
1977 #define m_CORE2I7_64 (m_CORE2_64 | m_COREI7_64)
1978 #define m_CORE2I7 (m_CORE2I7_32 | m_CORE2I7_64)
1979 #define m_ATOM (1<<PROCESSOR_ATOM)
1981 #define m_GEODE (1<<PROCESSOR_GEODE)
1982 #define m_K6 (1<<PROCESSOR_K6)
1983 #define m_K6_GEODE (m_K6 | m_GEODE)
1984 #define m_K8 (1<<PROCESSOR_K8)
1985 #define m_ATHLON (1<<PROCESSOR_ATHLON)
1986 #define m_ATHLON_K8 (m_K8 | m_ATHLON)
1987 #define m_AMDFAM10 (1<<PROCESSOR_AMDFAM10)
1988 #define m_BDVER1 (1<<PROCESSOR_BDVER1)
1989 #define m_BDVER2 (1<<PROCESSOR_BDVER2)
1990 #define m_BDVER (m_BDVER1 | m_BDVER2)
1991 #define m_BTVER (m_BTVER1 | m_BTVER2)
1992 #define m_BTVER1 (1<<PROCESSOR_BTVER1)
1993 #define m_BTVER2 (1<<PROCESSOR_BTVER2)
1994 #define m_AMD_MULTIPLE (m_ATHLON_K8 | m_AMDFAM10 | m_BDVER | m_BTVER)
1996 #define m_GENERIC32 (1<<PROCESSOR_GENERIC32)
1997 #define m_GENERIC64 (1<<PROCESSOR_GENERIC64)
1999 /* Generic instruction choice should be common subset of supported CPUs
2000 (PPro/PENT4/NOCONA/CORE2/Athlon/K8). */
2001 #define m_GENERIC (m_GENERIC32 | m_GENERIC64)
2003 /* Feature tests against the various tunings. */
2006 /* Feature tests against the various tunings used to create ix86_tune_features
2007 based on the processor mask. */
2009 /* X86_TUNE_USE_LEAVE: Leave does not affect Nocona SPEC2000 results
2010 negatively, so enabling for Generic64 seems like good code size
2011 tradeoff. We can't enable it for 32bit generic because it does not
2012 work well with PPro base chips. */
2015 /* X86_TUNE_PUSH_MEMORY */
2018 /* X86_TUNE_ZERO_EXTEND_WITH_AND */
2021 /* X86_TUNE_UNROLL_STRLEN */
2024 /* X86_TUNE_BRANCH_PREDICTION_HINTS: Branch hints were put in P4 based
2025 on simulation result. But after P4 was made, no performance benefit
2026 was observed with branch hints. It also increases the code size.
2027 As a result, icc never generates branch hints. */
2030 /* X86_TUNE_DOUBLE_WITH_ADD */
2033 /* X86_TUNE_USE_SAHF */
2034 m_PPRO | m_P4_NOCONA | m_CORE2I7 | m_ATOM | m_K6_GEODE | m_K8 | m_AMDFAM10 | m_BDVER | m_BTVER | m_GENERIC,
2036 /* X86_TUNE_MOVX: Enable to zero extend integer registers to avoid
2037 partial dependencies. */
2040 /* X86_TUNE_PARTIAL_REG_STALL: We probably ought to watch for partial
2041 register stalls on Generic32 compilation setting as well. However
2042 in current implementation the partial register stalls are not eliminated
2043 very well - they can be introduced via subregs synthesized by combine
2044 and can happen in caller/callee saving sequences. Because this option
2045 pays back little on PPro based chips and is in conflict with partial reg
2046 dependencies used by Athlon/P4 based chips, it is better to leave it off
2047 for generic32 for now. */
2048 m_PPRO,
2050 /* X86_TUNE_PARTIAL_FLAG_REG_STALL */
2053 /* X86_TUNE_LCP_STALL: Avoid an expensive length-changing prefix stall
2054 * on 16-bit immediate moves into memory on Core2 and Corei7. */
2057 /* X86_TUNE_USE_HIMODE_FIOP */
2060 /* X86_TUNE_USE_SIMODE_FIOP */
2063 /* X86_TUNE_USE_MOV0 */
2064 m_K6,
2066 /* X86_TUNE_USE_CLTD */
2069 /* X86_TUNE_USE_XCHGB: Use xchgb %rh,%rl instead of rolw/rorw $8,rx. */
2070 m_PENT4,
2072 /* X86_TUNE_SPLIT_LONG_MOVES */
2073 m_PPRO,
2075 /* X86_TUNE_READ_MODIFY_WRITE */
2078 /* X86_TUNE_READ_MODIFY */
2081 /* X86_TUNE_PROMOTE_QIMODE */
2084 /* X86_TUNE_FAST_PREFIX */
2087 /* X86_TUNE_SINGLE_STRINGOP */
2090 /* X86_TUNE_QIMODE_MATH */
2093 /* X86_TUNE_HIMODE_MATH: On PPro this flag is meant to avoid partial
2094 register stalls. Just like X86_TUNE_PARTIAL_REG_STALL this option
2095 might be considered for Generic32 if our scheme for avoiding partial
2096 stalls was more effective. */
2099 /* X86_TUNE_PROMOTE_QI_REGS */
2102 /* X86_TUNE_PROMOTE_HI_REGS */
2103 m_PPRO,
2105 /* X86_TUNE_SINGLE_POP: Enable if single pop insn is preferred
2106 over esp addition. */
2109 /* X86_TUNE_DOUBLE_POP: Enable if double pop insn is preferred
2110 over esp addition. */
2111 m_PENT,
2113 /* X86_TUNE_SINGLE_PUSH: Enable if single push insn is preferred
2114 over esp subtraction. */
2117 /* X86_TUNE_DOUBLE_PUSH. Enable if double push insn is preferred
2118 over esp subtraction. */
2121 /* X86_TUNE_INTEGER_DFMODE_MOVES: Enable if integer moves are preferred
2122 for DFmode copies */
2125 /* X86_TUNE_PARTIAL_REG_DEPENDENCY */
2128 /* X86_TUNE_SSE_PARTIAL_REG_DEPENDENCY: In the Generic model we have a
2129 conflict here in between PPro/Pentium4 based chips that thread 128bit
2130 SSE registers as single units versus K8 based chips that divide SSE
2131 registers to two 64bit halves. This knob promotes all store destinations
2132 to be 128bit to allow register renaming on 128bit SSE units, but usually
2133 results in one extra microop on 64bit SSE units. Experimental results
2134 shows that disabling this option on P4 brings over 20% SPECfp regression,
2135 while enabling it on K8 brings roughly 2.4% regression that can be partly
2136 masked by careful scheduling of moves. */
2139 /* X86_TUNE_SSE_UNALIGNED_LOAD_OPTIMAL */
2142 /* X86_TUNE_SSE_UNALIGNED_STORE_OPTIMAL */
2145 /* X86_TUNE_SSE_PACKED_SINGLE_INSN_OPTIMAL */
2146 m_BDVER ,
2148 /* X86_TUNE_SSE_SPLIT_REGS: Set for machines where the type and dependencies
2149 are resolved on SSE register parts instead of whole registers, so we may
2150 maintain just lower part of scalar values in proper format leaving the
2151 upper part undefined. */
2152 m_ATHLON_K8,
2154 /* X86_TUNE_SSE_TYPELESS_STORES */
2155 m_AMD_MULTIPLE,
2157 /* X86_TUNE_SSE_LOAD0_BY_PXOR */
2160 /* X86_TUNE_MEMORY_MISMATCH_STALL */
2163 /* X86_TUNE_PROLOGUE_USING_MOVE */
2166 /* X86_TUNE_EPILOGUE_USING_MOVE */
2169 /* X86_TUNE_SHIFT1 */
2172 /* X86_TUNE_USE_FFREEP */
2173 m_AMD_MULTIPLE,
2175 /* X86_TUNE_INTER_UNIT_MOVES */
2178 /* X86_TUNE_INTER_UNIT_CONVERSIONS */
2181 /* X86_TUNE_FOUR_JUMP_LIMIT: Some CPU cores are not able to predict more
2182 than 4 branch instructions in the 16 byte window. */
2185 /* X86_TUNE_SCHEDULE */
2188 /* X86_TUNE_USE_BT */
2191 /* X86_TUNE_USE_INCDEC */
2194 /* X86_TUNE_PAD_RETURNS */
2197 /* X86_TUNE_PAD_SHORT_FUNCTION: Pad short funtion. */
2198 m_ATOM,
2200 /* X86_TUNE_EXT_80387_CONSTANTS */
2203 /* X86_TUNE_SHORTEN_X87_SSE */
2206 /* X86_TUNE_AVOID_VECTOR_DECODE */
2209 /* X86_TUNE_PROMOTE_HIMODE_IMUL: Modern CPUs have same latency for HImode
2210 and SImode multiply, but 386 and 486 do HImode multiply faster. */
2213 /* X86_TUNE_SLOW_IMUL_IMM32_MEM: Imul of 32-bit constant and memory is
2214 vector path on AMD machines. */
2217 /* X86_TUNE_SLOW_IMUL_IMM8: Imul of 8-bit constant is vector path on AMD
2218 machines. */
2221 /* X86_TUNE_MOVE_M1_VIA_OR: On pentiums, it is faster to load -1 via OR
2222 than a MOV. */
2223 m_PENT,
2225 /* X86_TUNE_NOT_UNPAIRABLE: NOT is not pairable on Pentium, while XOR is,
2226 but one byte longer. */
2227 m_PENT,
2229 /* X86_TUNE_NOT_VECTORMODE: On AMD K6, NOT is vector decoded with memory
2230 operand that cannot be represented using a modRM byte. The XOR
2231 replacement is long decoded, so this split helps here as well. */
2232 m_K6,
2234 /* X86_TUNE_USE_VECTOR_FP_CONVERTS: Prefer vector packed SSE conversion
2235 from FP to FP. */
2238 /* X86_TUNE_USE_VECTOR_CONVERTS: Prefer vector packed SSE conversion
2239 from integer to FP. */
2240 m_AMDFAM10,
2242 /* X86_TUNE_FUSE_CMP_AND_BRANCH: Fuse a compare or test instruction
2243 with a subsequent conditional jump instruction into a single
2244 compare-and-branch uop. */
2245 m_BDVER,
2247 /* X86_TUNE_OPT_AGU: Optimize for Address Generation Unit. This flag
2248 will impact LEA instruction selection. */
2249 m_ATOM,
2251 /* X86_TUNE_VECTORIZE_DOUBLE: Enable double precision vector
2252 instructions. */
2255 /* X86_SOFTARE_PREFETCHING_BENEFICIAL: Enable software prefetching
2256 at -O3. For the moment, the prefetching seems badly tuned for Intel
2257 chips. */
2260 /* X86_TUNE_AVX128_OPTIMAL: Enable 128-bit AVX instruction generation for
2261 the auto-vectorizer. */
2262 m_BDVER,
2264 /* X86_TUNE_REASSOC_INT_TO_PARALLEL: Try to produce parallel computations
2265 during reassociation of integer computation. */
2266 m_ATOM,
2268 /* X86_TUNE_REASSOC_FP_TO_PARALLEL: Try to produce parallel computations
2269 during reassociation of fp computation. */
2270 m_ATOM
2271 };
2273 /* Feature tests against the various architecture variations. */
2276 /* Feature tests against the various architecture variations, used to create
2277 ix86_arch_features based on the processor mask. */
2279 /* X86_ARCH_CMOV: Conditional move was added for pentiumpro. */
2282 /* X86_ARCH_CMPXCHG: Compare and exchange was added for 80486. */
2285 /* X86_ARCH_CMPXCHG8B: Compare and exchange 8 bytes was added for pentium. */
2288 /* X86_ARCH_XADD: Exchange and add was added for 80486. */
2291 /* X86_ARCH_BSWAP: Byteswap was added for 80486. */
2293 };
2295 static const unsigned int x86_accumulate_outgoing_args
2298 static const unsigned int x86_arch_always_fancy_math_387
2301 static const unsigned int x86_avx256_split_unaligned_load
2304 static const unsigned int x86_avx256_split_unaligned_store
2307 /* In case the average insn count for single function invocation is
2308 lower than this constant, emit fast (but longer) prologue and
2309 epilogue code. */
2310 #define FAST_PROLOGUE_INSN_COUNT 20
2312 /* Names for 8 (low), 8 (high), and 16-bit registers, respectively. */
2317 /* Array of the smallest class containing reg number REGNO, indexed by
2318 REGNO. Used by REGNO_REG_CLASS in i386.h. */
2321 {
2322 /* ax, dx, cx, bx */
2324 /* si, di, bp, sp */
2326 /* FP registers */
2329 /* arg pointer */
2330 NON_Q_REGS,
2331 /* flags, fpsr, fpcr, frame */
2333 /* SSE registers */
2336 /* MMX registers */
2339 /* REX registers */
2342 /* SSE REX registers */
2345 };
2347 /* The "default" register map used in 32bit mode. */
2350 {
2358 };
2360 /* The "default" register map used in 64bit mode. */
2363 {
2371 };
2373 /* Define the register numbers to be used in Dwarf debugging information.
2374 The SVR4 reference port C compiler uses the following register numbers
2375 in its Dwarf output code:
2376 0 for %eax (gcc regno = 0)
2377 1 for %ecx (gcc regno = 2)
2378 2 for %edx (gcc regno = 1)
2379 3 for %ebx (gcc regno = 3)
2380 4 for %esp (gcc regno = 7)
2381 5 for %ebp (gcc regno = 6)
2382 6 for %esi (gcc regno = 4)
2383 7 for %edi (gcc regno = 5)
2384 The following three DWARF register numbers are never generated by
2385 the SVR4 C compiler or by the GNU compilers, but SDB on x86/svr4
2386 believes these numbers have these meanings.
2387 8 for %eip (no gcc equivalent)
2388 9 for %eflags (gcc regno = 17)
2389 10 for %trapno (no gcc equivalent)
2390 It is not at all clear how we should number the FP stack registers
2391 for the x86 architecture. If the version of SDB on x86/svr4 were
2392 a bit less brain dead with respect to floating-point then we would
2393 have a precedent to follow with respect to DWARF register numbers
2394 for x86 FP registers, but the SDB on x86/svr4 is so completely
2395 broken with respect to FP registers that it is hardly worth thinking
2396 of it as something to strive for compatibility with.
2397 The version of x86/svr4 SDB I have at the moment does (partially)
2398 seem to believe that DWARF register number 11 is associated with
2399 the x86 register %st(0), but that's about all. Higher DWARF
2400 register numbers don't seem to be associated with anything in
2401 particular, and even for DWARF regno 11, SDB only seems to under-
2402 stand that it should say that a variable lives in %st(0) (when
2403 asked via an `=' command) if we said it was in DWARF regno 11,
2404 but SDB still prints garbage when asked for the value of the
2405 variable in question (via a `/' command).
2406 (Also note that the labels SDB prints for various FP stack regs
2407 when doing an `x' command are all wrong.)
2408 Note that these problems generally don't affect the native SVR4
2409 C compiler because it doesn't allow the use of -O with -g and
2410 because when it is *not* optimizing, it allocates a memory
2411 location for each floating-point variable, and the memory
2412 location is what gets described in the DWARF AT_location
2413 attribute for the variable in question.
2414 Regardless of the severe mental illness of the x86/svr4 SDB, we
2415 do something sensible here and we use the following DWARF
2416 register numbers. Note that these are all stack-top-relative
2417 numbers.
2418 11 for %st(0) (gcc regno = 8)
2419 12 for %st(1) (gcc regno = 9)
2420 13 for %st(2) (gcc regno = 10)
2421 14 for %st(3) (gcc regno = 11)
2422 15 for %st(4) (gcc regno = 12)
2423 16 for %st(5) (gcc regno = 13)
2424 17 for %st(6) (gcc regno = 14)
2425 18 for %st(7) (gcc regno = 15)
2426 */
2428 {
2436 };
2438 /* Define parameter passing and return registers. */
2441 {
2443 };
2446 {
2448 };
2451 {
2453 };
2455 /* Define the structure for the machine field in struct function. */
2460 rtx rtl;
2462 };
2464 /* Structure describing stack frame layout.
2465 Stack grows downward:
2467 [arguments]
2468 <- ARG_POINTER
2469 saved pc
2471 saved static chain if ix86_static_chain_on_stack
2473 saved frame pointer if frame_pointer_needed
2474 <- HARD_FRAME_POINTER
2475 [saved regs]
2476 <- regs_save_offset
2477 [padding0]
2479 [saved SSE regs]
2480 <- sse_regs_save_offset
2481 [padding1] |
2482 | <- FRAME_POINTER
2483 [va_arg registers] |
2484 |
2485 [frame] |
2486 |
2487 [padding2] | = to_allocate
2488 <- STACK_POINTER
2489 */
2490 struct ix86_frame
2491 {
2498 /* The offsets relative to ARG_POINTER. */
2499 HOST_WIDE_INT frame_pointer_offset;
2500 HOST_WIDE_INT hard_frame_pointer_offset;
2501 HOST_WIDE_INT stack_pointer_offset;
2502 HOST_WIDE_INT hfp_save_offset;
2503 HOST_WIDE_INT reg_save_offset;
2504 HOST_WIDE_INT sse_reg_save_offset;
2506 /* When save_regs_using_mov is set, emit prologue using
2507 move instead of push instructions. */
2509 };
2511 /* Which cpu are we scheduling for. */
2514 /* Which cpu are we optimizing for. */
2517 /* Which instruction set architecture to use. */
2520 /* true if sse prefetch instruction is not NOOP. */
2523 /* -mstackrealign option */
2540 /* Preferred alignment for stack boundary in bits. */
2543 /* Alignment for incoming stack boundary in bits specified at
2544 command line. */
2547 /* Default alignment for incoming stack boundary in bits. */
2550 /* Alignment for incoming stack boundary in bits. */
2553 /* Calling abi specific va_list type nodes. */
2557 /* Prefix built by ASM_GENERATE_INTERNAL_LABEL. */
2561 /* Fence to use after loop using movnt. */
2562 tree x86_mfence;
2564 /* Register class used for passing given 64bit part of the argument.
2565 These represent classes as documented by the PS ABI, with the exception
2566 of SSESF, SSEDF classes, that are basically SSE class, just gcc will
2567 use SF or DFmode move instead of DImode to avoid reformatting penalties.
2569 Similarly we play games with INTEGERSI_CLASS to use cheaper SImode moves
2570 whenever possible (upper half does contain padding). */
2571 enum x86_64_reg_class
2572 {
2573 X86_64_NO_CLASS,
2574 X86_64_INTEGER_CLASS,
2575 X86_64_INTEGERSI_CLASS,
2576 X86_64_SSE_CLASS,
2577 X86_64_SSESF_CLASS,
2578 X86_64_SSEDF_CLASS,
2579 X86_64_SSEUP_CLASS,
2580 X86_64_X87_CLASS,
2581 X86_64_X87UP_CLASS,
2582 X86_64_COMPLEX_X87_CLASS,
2583 X86_64_MEMORY_CLASS
2584 };
2586 #define MAX_CLASSES 4
2588 /* Table of constants used by fldpi, fldln2, etc.... */
2592 \f
2597 const_tree);
2609 enum ix86_function_specific_strings
2610 {
2611 IX86_FUNCTION_SPECIFIC_ARCH,
2612 IX86_FUNCTION_SPECIFIC_TUNE,
2613 IX86_FUNCTION_SPECIFIC_MAX
2614 };
2632 \f
2633 #ifndef SUBTARGET32_DEFAULT_CPU
2635 #endif
2637 /* The svr4 ABI for the i386 says that records and unions are returned
2638 in memory. */
2639 #ifndef DEFAULT_PCC_STRUCT_RETURN
2640 #define DEFAULT_PCC_STRUCT_RETURN 1
2641 #endif
2643 /* Whether -mtune= or -march= were specified */
2647 /* Vectorization library interface and handlers. */
2653 /* Processor target table, indexed by processor number */
2654 struct ptt
2655 {
2662 };
2665 {
2676 /* Core 2 32-bit. */
2678 /* Core 2 64-bit. */
2680 /* Core i7 32-bit. */
2682 /* Core i7 64-bit. */
2692 };
2695 {
2722 "btver2"
2723 };
2724 \f
2725 /* Return true if a red-zone is in use. */
2729 {
2731 }
2732 \f
2733 /* Return a string that documents the current -m options. The caller is
2734 responsible for freeing the string. */
2740 {
2741 struct ix86_target_opts
2742 {
2745 };
2747 /* This table is ordered so that options like -msse4.2 that imply
2748 preceding options while match those first. */
2750 {
2781 };
2783 /* Flag options. */
2785 {
2812 };
2829 /* Add -march= option. */
2831 {
2834 }
2836 /* Add -mtune= option. */
2838 {
2841 }
2843 /* Add -m32/-m64/-mx32. */
2845 {
2848 else
2850 isa &= ~ (OPTION_MASK_ISA_64BIT
2851 | OPTION_MASK_ABI_64
2853 }
2854 else
2858 /* Pick out the options in isa options. */
2860 {
2862 {
2865 }
2866 }
2869 {
2872 isa);
2873 }
2875 /* Add flag options. */
2877 {
2879 {
2882 }
2883 }
2886 {
2889 }
2891 /* Add -fpmath= option. */
2893 {
2896 {
2911 }
2912 }
2914 /* Any options? */
2920 /* Size the string. */
2924 {
2929 }
2931 /* Build the string. */
2936 {
2943 {
2945 line_len++;
2948 {
2952 }
2953 }
2957 {
2961 }
2962 }
2968 }
2970 /* Return true, if profiling code should be emitted before
2971 prologue. Otherwise it returns false.
2972 Note: For x86 with "hotfix" it is sorried. */
2973 static bool
2975 {
2977 }
2979 /* Function that is callable from the debugger to print the current
2980 options. */
2981 void
2983 {
2989 {
2992 }
2993 else
2997 }
2998 \f
2999 /* Override various settings based on options. If MAIN_ARGS_P, the
3000 options are from the command line, otherwise they are from
3001 attributes. */
3003 static void
3005 {
3013 #define PTA_3DNOW (HOST_WIDE_INT_1 << 0)
3014 #define PTA_3DNOW_A (HOST_WIDE_INT_1 << 1)
3015 #define PTA_64BIT (HOST_WIDE_INT_1 << 2)
3016 #define PTA_ABM (HOST_WIDE_INT_1 << 3)
3017 #define PTA_AES (HOST_WIDE_INT_1 << 4)
3018 #define PTA_AVX (HOST_WIDE_INT_1 << 5)
3019 #define PTA_BMI (HOST_WIDE_INT_1 << 6)
3020 #define PTA_CX16 (HOST_WIDE_INT_1 << 7)
3021 #define PTA_F16C (HOST_WIDE_INT_1 << 8)
3022 #define PTA_FMA (HOST_WIDE_INT_1 << 9)
3023 #define PTA_FMA4 (HOST_WIDE_INT_1 << 10)
3024 #define PTA_FSGSBASE (HOST_WIDE_INT_1 << 11)
3025 #define PTA_LWP (HOST_WIDE_INT_1 << 12)
3026 #define PTA_LZCNT (HOST_WIDE_INT_1 << 13)
3027 #define PTA_MMX (HOST_WIDE_INT_1 << 14)
3028 #define PTA_MOVBE (HOST_WIDE_INT_1 << 15)
3029 #define PTA_NO_SAHF (HOST_WIDE_INT_1 << 16)
3030 #define PTA_PCLMUL (HOST_WIDE_INT_1 << 17)
3031 #define PTA_POPCNT (HOST_WIDE_INT_1 << 18)
3032 #define PTA_PREFETCH_SSE (HOST_WIDE_INT_1 << 19)
3033 #define PTA_RDRND (HOST_WIDE_INT_1 << 20)
3034 #define PTA_SSE (HOST_WIDE_INT_1 << 21)
3035 #define PTA_SSE2 (HOST_WIDE_INT_1 << 22)
3036 #define PTA_SSE3 (HOST_WIDE_INT_1 << 23)
3037 #define PTA_SSE4_1 (HOST_WIDE_INT_1 << 24)
3038 #define PTA_SSE4_2 (HOST_WIDE_INT_1 << 25)
3039 #define PTA_SSE4A (HOST_WIDE_INT_1 << 26)
3040 #define PTA_SSSE3 (HOST_WIDE_INT_1 << 27)
3041 #define PTA_TBM (HOST_WIDE_INT_1 << 28)
3042 #define PTA_XOP (HOST_WIDE_INT_1 << 29)
3043 #define PTA_AVX2 (HOST_WIDE_INT_1 << 30)
3044 #define PTA_BMI2 (HOST_WIDE_INT_1 << 31)
3045 #define PTA_RTM (HOST_WIDE_INT_1 << 32)
3046 #define PTA_HLE (HOST_WIDE_INT_1 << 33)
3047 #define PTA_PRFCHW (HOST_WIDE_INT_1 << 34)
3048 /* if this reaches 64, need to widen struct pta flags below */
3050 static struct pta
3051 {
3056 }
3058 {
3176 PTA_64BIT
3178 };
3180 /* -mrecip options. */
3181 static struct
3182 {
3185 }
3187 {
3194 };
3198 /* Set up prefix/suffix so the error messages refer to either the command
3199 line argument, or the attribute(target). */
3201 {
3205 }
3206 else
3207 {
3211 }
3213 /* Turn off both OPTION_MASK_ABI_64 and OPTION_MASK_ABI_X32 if
3214 TARGET_64BIT_DEFAULT is true and TARGET_64BIT is false. */
3217 #ifdef TARGET_BI_ARCH
3218 else
3219 {
3220 #if TARGET_BI_ARCH == 1
3221 /* When TARGET_BI_ARCH == 1, by default, OPTION_MASK_ABI_64
3222 is on and OPTION_MASK_ABI_X32 is off. We turn off
3223 OPTION_MASK_ABI_64 if OPTION_MASK_ABI_X32 is turned on by
3224 -mx32. */
3227 #else
3228 /* When TARGET_BI_ARCH == 2, by default, OPTION_MASK_ABI_X32 is
3229 on and OPTION_MASK_ABI_64 is off. We turn off
3230 OPTION_MASK_ABI_X32 if OPTION_MASK_ABI_64 is turned on by
3231 -m64. */
3234 #endif
3235 }
3236 #endif
3239 {
3240 /* Always turn on OPTION_MASK_ISA_64BIT and turn off
3241 OPTION_MASK_ABI_64 for TARGET_X32. */
3244 }
3246 {
3247 /* Always turn on OPTION_MASK_ISA_64BIT and turn off
3248 OPTION_MASK_ABI_X32 for TARGET_LP64. */
3251 }
3253 #ifdef SUBTARGET_OVERRIDE_OPTIONS
3254 SUBTARGET_OVERRIDE_OPTIONS;
3255 #endif
3257 #ifdef SUBSUBTARGET_OVERRIDE_OPTIONS
3258 SUBSUBTARGET_OVERRIDE_OPTIONS;
3259 #endif
3261 /* -fPIC is the default for x86_64. */
3265 /* Need to check -mtune=generic first. */
3267 {
3270 /* As special support for cross compilers we read -mtune=native
3271 as -mtune=generic. With native compilers we won't see the
3272 -mtune=native, as it was changed by the driver. */
3274 {
3277 else
3279 }
3280 /* If this call is for setting the option attribute, allow the
3281 generic32/generic64 that was previously set. */
3285 ;
3293 }
3294 else
3295 {
3299 {
3302 }
3304 /* ix86_tune_string is set to ix86_arch_string or defaulted. We
3305 need to use a sensible tune option. */
3309 {
3312 else
3314 }
3315 }
3318 {
3319 /* rep; movq isn't available in 32-bit code. */
3322 }
3326 else
3330 {
3336 }
3337 else
3344 {
3346 {
3390 {
3393 }
3401 }
3402 }
3403 else
3404 {
3405 /* For TARGET_64BIT and MS_ABI, force pic on, in order to enable the
3406 use of rip-relative addressing. This eliminates fixups that
3407 would otherwise be needed if this object is to be placed in a
3408 DLL, and is essentially just as efficient as direct addressing. */
3413 else
3415 }
3417 {
3420 }
3427 {
3430 /* Default cpu tuning to the architecture. */
3540 }
3555 {
3559 {
3561 {
3563 {
3571 }
3572 else
3575 }
3576 }
3577 else
3578 {
3579 /* Adjust tuning when compiling for 32-bit ABI. */
3581 {
3597 }
3598 }
3599 /* Intel CPUs have always interpreted SSE prefetch instructions as
3600 NOPs; so, we can enable SSE prefetch instructions even when
3601 -mtune (rather than -march) points us to a processor that has them.
3602 However, the VIA C3 gives a SIGILL, so we only do that for i686 and
3603 higher processors. */
3608 }
3618 #ifndef USE_IX86_FRAME_POINTER
3619 #define USE_IX86_FRAME_POINTER 0
3620 #endif
3622 #ifndef USE_X86_64_FRAME_POINTER
3623 #define USE_X86_64_FRAME_POINTER 0
3624 #endif
3626 /* Set the default values for switches whose default depends on TARGET_64BIT
3627 in case they weren't overwritten by command line options. */
3629 {
3636 }
3637 else
3638 {
3645 }
3650 else
3653 /* Arrange to set up i386_stack_locals for all functions. */
3656 /* Validate -mregparm= value. */
3658 {
3662 {
3666 }
3667 }
3671 /* Default align_* from the processor table. */
3673 {
3676 }
3678 {
3681 }
3683 {
3685 }
3687 /* Provide default for -mbranch-cost= value. */
3692 {
3695 /* Enable by default the SSE and MMX builtins. Do allow the user to
3696 explicitly disable any of these. In particular, disabling SSE and
3697 MMX for kernel code is extremely useful. */
3699 ix86_isa_flags
3705 }
3706 else
3707 {
3711 ix86_isa_flags
3714 /* i386 ABI does not specify red zone. It still makes sense to use it
3715 when programmer takes care to stack from being destroyed. */
3718 }
3720 /* Keep nonleaf frame pointers. */
3726 /* If we're doing fast math, we don't care about comparison order
3727 wrt NaNs. This lets us use a shorter comparison sequence. */
3731 /* If the architecture always has an FPU, turn off NO_FANCY_MATH_387,
3732 since the insns won't need emulation. */
3736 /* Likewise, if the target doesn't have a 387, or we've specified
3737 software floating point, don't use 387 inline intrinsics. */
3741 /* Turn on MMX builtins for -msse. */
3745 /* Enable SSE prefetch. */
3749 /* Turn on popcnt instruction for -msse4.2 or -mabm. */
3753 /* Turn on lzcnt instruction for -mabm. */
3757 /* Validate -mpreferred-stack-boundary= value or default it to
3758 PREFERRED_STACK_BOUNDARY_DEFAULT. */
3761 {
3767 {
3771 else
3774 }
3775 else
3776 ix86_preferred_stack_boundary
3778 }
3780 /* Set the default value for -mstackrealign. */
3786 /* Validate -mincoming-stack-boundary= value or default it to
3787 MIN_STACK_BOUNDARY/PREFERRED_STACK_BOUNDARY. */
3790 {
3795 else
3796 {
3797 ix86_user_incoming_stack_boundary
3799 ix86_incoming_stack_boundary
3801 }
3802 }
3804 /* Accept -msseregparm only if at least SSE support is enabled. */
3810 {
3812 {
3814 {
3817 }
3819 {
3822 }
3823 }
3824 }
3825 else
3828 /* If the i387 is disabled, then do not return values in it. */
3832 /* Use external vectorized library in vectorizing intrinsics. */
3835 {
3846 }
3854 /* ??? Unwind info is not correct around the CFG unless either a frame
3855 pointer is present or M_A_O_A is set. Fixing this requires rewriting
3856 unwind info generation to be aware of the CFG and propagating states
3857 around edges. */
3860 && flag_omit_frame_pointer
3862 {
3868 }
3870 /* If stack probes are required, the space used for large function
3871 arguments on the stack must also be probed, so enable
3872 -maccumulate-outgoing-args so this happens in the prologue. */
3875 {
3880 }
3882 /* Figure out what ASM_GENERATE_INTERNAL_LABEL builds as a prefix. */
3883 {
3889 }
3891 /* When scheduling description is not available, disable scheduler pass
3892 so it won't slow down the compilation and make x87 code slower. */
3913 /* Enable sw prefetching at -O3 for CPUS that prefetching is helpful. */
3915 && HAVE_prefetch
3920 /* If using typedef char *va_list, signal that __builtin_va_start (&ap, 0)
3921 can be optimized to ap = __builtin_next_arg (0). */
3926 {
3929 {
3932 ix86_gen_tls_local_dynamic_base_64
3934 }
3935 else
3936 {
3939 ix86_gen_tls_local_dynamic_base_64
3941 }
3942 }
3943 else
3944 {
3947 }
3950 {
3959 }
3960 else
3961 {
3970 }
3972 #ifdef USE_IX86_CLD
3973 /* Use -mcld by default for 32-bit code if configured with --enable-cld. */
3976 #endif
3979 {
3984 }
3986 {
3990 }
3992 {
3993 #if defined(PROFILE_BEFORE_PROLOGUE)
3995 #else
3997 #endif
3998 }
4001 {
4002 /* When not optimize for size, enable vzeroupper optimization for
4003 TARGET_AVX with -fexpensive-optimizations and split 32-byte
4004 AVX unaligned load/store. */
4006 {
4016 /* Enable 128-bit AVX instruction generation for the auto-vectorizer. */
4019 }
4020 }
4021 else
4022 {
4023 /* Disable vzeroupper pass if TARGET_AVX is disabled. */
4025 }
4028 {
4035 {
4038 {
4040 q++;
4041 }
4042 else
4047 else
4048 {
4051 {
4054 }
4057 {
4061 }
4062 }
4067 else
4069 }
4070 }
4077 /* Save the initial options in case the user does function specific
4078 options. */
4080 target_option_default_node = target_option_current_node
4082 }
4084 /* Return TRUE if VAL is passed in register with 256bit AVX modes. */
4086 static bool
4088 {
4096 {
4098 rtx r;
4101 {
4109 }
4110 }
4113 }
4115 /* Implement the TARGET_OPTION_OVERRIDE hook. */
4117 static void
4119 {
4121 }
4123 /* Update register usage after having seen the compiler flags. */
4125 static void
4127 {
4132 {
4137 }
4139 /* The PIC register, if it exists, is fixed. */
4144 /* The 64-bit MS_ABI changes the set of call-used registers. */
4146 {
4153 }
4155 /* The default setting of CLOBBERED_REGS is for 32-bit; add in the
4156 other call-clobbered regs for 64-bit. */
4158 {
4165 }
4167 /* If MMX is disabled, squash the registers. */
4173 /* If SSE is disabled, squash the registers. */
4179 /* If the FPU is disabled, squash the registers. */
4185 /* If 32-bit, squash the 64-bit registers. */
4187 {
4192 }
4193 }
4195 \f
4196 /* Save the current options */
4198 static void
4200 {
4211 /* The fields are char but the variables are not; make sure the
4212 values fit in the fields. */
4217 }
4219 /* Restore the current options */
4221 static void
4223 {
4239 /* Recreate the arch feature tests if the arch changed */
4241 {
4246 }
4248 /* Recreate the tune optimization tests */
4250 {
4255 }
4256 }
4258 /* Print the current options */
4260 static void
4263 {
4285 {
4288 }
4289 }
4291 \f
4292 /* Inner function to process the attribute((target(...))), take an argument and
4293 set the current options from the argument. If we have a list, recursively go
4294 over the list. */
4296 static bool
4299 {
4303 #define IX86_ATTR_ISA(S,O) { S, sizeof (S)-1, ix86_opt_isa, O, 0 }
4304 #define IX86_ATTR_STR(S,O) { S, sizeof (S)-1, ix86_opt_str, O, 0 }
4305 #define IX86_ATTR_ENUM(S,O) { S, sizeof (S)-1, ix86_opt_enum, O, 0 }
4306 #define IX86_ATTR_YES(S,O,M) { S, sizeof (S)-1, ix86_opt_yes, O, M }
4307 #define IX86_ATTR_NO(S,O,M) { S, sizeof (S)-1, ix86_opt_no, O, M }
4309 enum ix86_opt_type
4310 {
4311 ix86_opt_unknown,
4312 ix86_opt_yes,
4313 ix86_opt_no,
4314 ix86_opt_str,
4315 ix86_opt_enum,
4316 ix86_opt_isa
4317 };
4319 static const struct
4320 {
4327 /* isa options */
4359 /* enum options */
4362 /* string options */
4366 /* flag options */
4368 OPT_mcld,
4369 MASK_CLD),
4372 OPT_mfancy_math_387,
4373 MASK_NO_FANCY_MATH_387),
4376 OPT_mieee_fp,
4377 MASK_IEEE_FP),
4380 OPT_minline_all_stringops,
4381 MASK_INLINE_ALL_STRINGOPS),
4384 OPT_minline_stringops_dynamically,
4385 MASK_INLINE_STRINGOPS_DYNAMICALLY),
4388 OPT_mno_align_stringops,
4389 MASK_NO_ALIGN_STRINGOPS),
4392 OPT_mrecip,
4393 MASK_RECIP),
4395 };
4397 /* If this is a list, recurse to get the options. */
4399 {
4409 }
4414 /* Handle multiple arguments separated by commas. */
4418 {
4432 {
4436 }
4437 else
4438 {
4441 }
4443 /* Recognize no-xxx. */
4445 {
4449 }
4450 else
4453 /* Find the option. */
4457 {
4465 {
4470 }
4471 }
4473 /* Process the option. */
4475 {
4478 }
4481 {
4487 }
4490 {
4496 else
4498 }
4501 {
4503 {
4506 }
4507 else
4509 }
4512 {
4520 global_dc);
4521 else
4522 {
4525 }
4526 }
4528 else
4530 }
4533 }
4535 /* Return a TARGET_OPTION_NODE tree of the target options listed or NULL. */
4537 tree
4539 {
4554 /* Process each of the options on the chain. */
4559 /* If the changed options are different from the default, rerun
4560 ix86_option_override_internal, and then save the options away.
4561 The string options are are attribute options, and will be undone
4562 when we copy the save structure. */
4568 {
4569 /* If we are using the default tune= or arch=, undo the string assigned,
4570 and use the default. */
4581 /* If fpmath= is not set, and we now have sse2 on 32-bit, use it. */
4585 {
4588 }
4590 /* Do any overrides, such as arch=xxx, or tune=xxx support. */
4593 /* Add any builtin functions with the new isa if any. */
4596 /* Save the current options unless we are validating options for
4597 #pragma. */
4604 /* Free up memory allocated to hold the strings */
4607 }
4610 }
4612 /* Hook to validate attribute((target("string"))). */
4614 static bool
4617 tree args,
4619 {
4626 /* If the function changed the optimization levels as well as setting target
4627 options, start with the optimizations specified. */
4632 /* The target attributes may also change some optimization flags, so update
4633 the optimization options if necessary. */
4642 {
4647 }
4656 }
4658 \f
4659 /* Hook to determine if one function can safely inline another. */
4661 static bool
4663 {
4668 /* If callee has no option attributes, then it is ok to inline. */
4672 /* If caller has no option attributes, but callee does then it is not ok to
4673 inline. */
4677 else
4678 {
4682 /* Callee's isa options should a subset of the caller's, i.e. a SSE4 function
4683 can inline a SSE2 function but a SSE2 function can't inline a SSE4
4684 function. */
4689 /* See if we have the same non-isa options. */
4693 /* See if arch, tune, etc. are the same. */
4706 else
4708 }
4711 }
4713 \f
4714 /* Remember the last target of ix86_set_current_function. */
4717 /* Establish appropriate back-end context for processing the function
4718 FNDECL. The argument might be NULL to indicate processing at top
4719 level, outside of any function scope. */
4720 static void
4722 {
4723 /* Only change the context if the function changes. This hook is called
4724 several times in the course of compiling a function, and we don't want to
4725 slow things down too much or call target_reinit when it isn't safe. */
4727 {
4728 tree old_tree = (ix86_previous_fndecl
4732 tree new_tree = (fndecl
4738 ;
4741 {
4745 }
4748 {
4754 }
4755 }
4756 }
4758 \f
4759 /* Return true if this goes in large data/bss. */
4761 static bool
4763 {
4767 /* Functions are never large data. */
4772 {
4778 }
4779 else
4780 {
4783 /* If this is an incomplete type with size 0, then we can't put it
4784 in data because it might be too big when completed. */
4787 }
4790 }
4792 /* Switch to the appropriate section for output of DECL.
4793 DECL is either a `VAR_DECL' node or a constant of some sort.
4794 RELOC indicates whether forming the initial value of DECL requires
4795 link-time relocations. */
4798 ATTRIBUTE_UNUSED;
4803 {
4806 {
4810 {
4844 /* We don't split these for medium model. Place them into
4845 default sections and hope for best. */
4847 }
4849 {
4850 /* We might get called with string constants, but get_named_section
4851 doesn't like them as they are not DECLs. Also, we need to set
4852 flags in that case. */
4856 }
4857 }
4859 }
4861 /* Build up a unique section name, expressed as a
4862 STRING_CST node, and assign it to DECL_SECTION_NAME (decl).
4863 RELOC indicates whether the initial value of EXP requires
4864 link-time relocations. */
4866 static void ATTRIBUTE_UNUSED
4868 {
4871 {
4873 /* We only need to use .gnu.linkonce if we don't have COMDAT groups. */
4877 {
4901 /* We don't split these for medium model. Place them into
4902 default sections and hope for best. */
4904 }
4906 {
4913 /* If we're using one_only, then there needs to be a .gnu.linkonce
4914 prefix to the section name. */
4921 }
4922 }
4924 }
4926 #ifdef COMMON_ASM_OP
4927 /* This says how to output assembler code to declare an
4928 uninitialized external linkage data object.
4930 For medium model x86-64 we need to use .largecomm opcode for
4931 large objects. */
4932 void
4936 {
4940 else
4945 }
4946 #endif
4948 /* Utility function for targets to use in implementing
4949 ASM_OUTPUT_ALIGNED_BSS. */
4951 void
4955 {
4959 else
4962 #ifdef ASM_DECLARE_OBJECT_NAME
4965 #else
4966 /* Standard thing is just output label for the object. */
4970 }
4971 \f
4972 /* Decide whether we must probe the stack before any space allocation
4973 on this target. It's essentially TARGET_STACK_PROBE except when
4974 -fstack-check causes the stack to be already probed differently. */
4976 bool
4978 {
4979 /* Do not probe the stack twice if static stack checking is enabled. */
4984 }
4985 \f
4986 /* Decide whether we can make a sibling call to a function. DECL is the
4987 declaration of the function being targeted by the call and EXP is the
4988 CALL_EXPR representing the call. */
4990 static bool
4992 {
4996 /* If we are generating position-independent code, we cannot sibcall
4997 optimize any indirect call, or a direct call to a global function,
4998 as the PLT requires %ebx be live. (Darwin does not have a PLT.) */
5000 && !TARGET_64BIT
5001 && flag_pic
5005 /* If we need to align the outgoing stack, then sibcalling would
5006 unalign the stack, which may break the called function. */
5012 {
5015 }
5016 else
5017 {
5018 /* We're looking at the CALL_EXPR, we need the type of the function. */
5023 }
5025 /* Check that the return value locations are the same. Like
5026 if we are returning floats on the 80387 register stack, we cannot
5027 make a sibcall from a function that doesn't return a float to a
5028 function that does or, conversely, from a function that does return
5029 a float to a function that doesn't; the necessary stack adjustment
5030 would not be executed. This is also the place we notice
5031 differences in the return value ABI. Note that it is ok for one
5032 of the functions to have void return type as long as the return
5033 value of the other is passed in a register. */
5038 {
5041 }
5043 {
5044 /* Disable sibcall if we need to generate vzeroupper after
5045 callee returns. */
5050 }
5055 {
5056 /* The SYSV ABI has more call-clobbered registers;
5057 disallow sibcalls from MS to SYSV. */
5061 }
5062 else
5063 {
5064 /* If this call is indirect, we'll need to be able to use a
5065 call-clobbered register for the address of the target function.
5066 Make sure that all such registers are not used for passing
5067 parameters. Note that DLLIMPORT functions are indirect. */
5070 {
5072 {
5073 /* ??? Need to count the actual number of registers to be used,
5074 not the possible number of registers. Fix later. */
5076 }
5077 }
5078 }
5080 /* Otherwise okay. That also includes certain types of indirect calls. */
5082 }
5084 /* Handle "cdecl", "stdcall", "fastcall", "regparm", "thiscall",
5085 and "sseregparm" calling convention attributes;
5086 arguments as in struct attribute_spec.handler. */
5088 static tree
5090 tree args,
5093 {
5098 {
5100 name);
5103 }
5105 /* Can combine regparm with all attributes but fastcall, and thiscall. */
5107 {
5108 tree cst;
5111 {
5113 }
5116 {
5118 }
5122 {
5125 name);
5127 }
5129 {
5133 }
5136 }
5139 {
5140 /* Do not warn when emulating the MS ABI. */
5145 name);
5148 }
5150 /* Can combine fastcall with stdcall (redundant) and sseregparm. */
5152 {
5154 {
5156 }
5158 {
5160 }
5162 {
5164 }
5166 {
5168 }
5169 }
5171 /* Can combine stdcall with fastcall (redundant), regparm and
5172 sseregparm. */
5174 {
5176 {
5178 }
5180 {
5182 }
5184 {
5186 }
5187 }
5189 /* Can combine cdecl with regparm and sseregparm. */
5191 {
5193 {
5195 }
5197 {
5199 }
5201 {
5203 }
5204 }
5206 {
5209 name);
5211 {
5213 }
5215 {
5217 }
5219 {
5221 }
5222 }
5224 /* Can combine sseregparm with all attributes. */
5227 }
5229 /* The transactional memory builtins are implicitly regparm or fastcall
5230 depending on the ABI. Override the generic do-nothing attribute that
5231 these builtins were declared with, and replace it with one of the two
5232 attributes that we expect elsewhere. */
5234 static tree
5236 tree args ATTRIBUTE_UNUSED,
5239 {
5240 tree alt;
5242 /* In no case do we want to add the placeholder attribute. */
5245 /* The 64-bit ABI is unchanged for transactional memory. */
5249 /* ??? Is there a better way to validate 32-bit windows? We have
5250 cfun->machine->call_abi, but that seems to be set only for 64-bit. */
5253 else
5254 {
5257 }
5261 }
5263 /* This function determines from TYPE the calling-convention. */
5265 unsigned int
5267 {
5270 tree attrs;
5277 {
5287 /* Regparam isn't allowed for thiscall and fastcall. */
5289 {
5294 }
5298 }
5305 || is_stdarg
5311 }
5313 /* Return 0 if the attributes for two types are incompatible, 1 if they
5314 are compatible, and 2 if they are nearly compatible (which causes a
5315 warning to be generated). */
5317 static int
5319 {
5335 }
5336 \f
5337 /* Return the regparm value for a function with the indicated TYPE and DECL.
5338 DECL may be NULL when calling function indirectly
5339 or considering a libcall. */
5341 static int
5343 {
5344 tree attr;
5355 {
5358 {
5361 }
5362 }
5368 /* Use register calling convention for local functions when possible. */
5371 && optimize
5373 {
5374 /* FIXME: remove this CONST_CAST when cgraph.[ch] is constified. */
5377 {
5380 /* Make sure no regparm register is taken by a
5381 fixed register variable. */
5386 /* We don't want to use regparm(3) for nested functions as
5387 these use a static chain pointer in the third argument. */
5391 /* In 32-bit mode save a register for the split stack. */
5395 /* Each fixed register usage increases register pressure,
5396 so less registers should be used for argument passing.
5397 This functionality can be overriden by an explicit
5398 regparm value. */
5401 globals++;
5403 local_regparm
5408 }
5409 }
5412 }
5414 /* Return 1 or 2, if we can pass up to SSE_REGPARM_MAX SFmode (1) and
5415 DFmode (2) arguments in SSE registers for a function with the
5416 indicated TYPE and DECL. DECL may be NULL when calling function
5417 indirectly or considering a libcall. Otherwise return 0. */
5419 static int
5421 {
5424 /* Use SSE registers to pass SFmode and DFmode arguments if requested
5425 by the sseregparm attribute. */
5428 {
5430 {
5432 {
5436 else
5439 }
5441 }
5444 }
5446 /* For local functions, pass up to SSE_REGPARM_MAX SFmode
5447 (and DFmode for SSE2) arguments in SSE registers. */
5450 {
5451 /* FIXME: remove this CONST_CAST when cgraph.[ch] is constified. */
5455 }
5458 }
5460 /* Return true if EAX is live at the start of the function. Used by
5461 ix86_expand_prologue to determine if we need special help before
5462 calling allocate_stack_worker. */
5464 static bool
5466 {
5467 /* Cheat. Don't bother working forward from ix86_function_regparm
5468 to the function type to whether an actual argument is located in
5469 eax. Instead just look at cfg info, which is still close enough
5470 to correct at this point. This gives false positives for broken
5471 functions that might use uninitialized data that happens to be
5472 allocated in eax, but who cares? */
5474 }
5476 static bool
5478 {
5479 tree attr;
5482 {
5488 /* For 32-bit MS-ABI the default is to keep aggregate
5489 return pointer. */
5492 }
5494 }
5496 /* Value is the number of bytes of arguments automatically
5497 popped when returning from a subroutine call.
5498 FUNDECL is the declaration node of the function (as a tree),
5499 FUNTYPE is the data type of the function (as a tree),
5500 or for a library call it is an identifier node for the subroutine name.
5501 SIZE is the number of bytes of arguments passed on the stack.
5503 On the 80386, the RTD insn may be used to pop them if the number
5504 of args is fixed, but if the number is variable then the caller
5505 must pop them all. RTD can't be used for library calls now
5506 because the library is compiled with the Unix compiler.
5507 Use of RTD is a selectable option, since it is incompatible with
5508 standard Unix calling sequences. If the option is not selected,
5509 the caller must always pop the args.
5511 The attribute stdcall is equivalent to RTD on a per module basis. */
5513 static int
5515 {
5518 /* None of the 64-bit ABIs pop arguments. */
5529 /* Lose any fake structure return argument if it is passed on the stack. */
5532 {
5536 }
5539 }
5540 \f
5541 /* Argument support functions. */
5543 /* Return true when register may be used to pass function parameters. */
5544 bool
5546 {
5551 {
5555 else
5561 }
5564 {
5567 }
5568 else
5569 {
5573 }
5575 /* TODO: The function should depend on current function ABI but
5576 builtins.c would need updating then. Therefore we use the
5577 default ABI. */
5579 /* RAX is used as hidden argument to va_arg functions. */
5585 else
5592 }
5594 /* Return if we do not know how to pass TYPE solely in registers. */
5596 static bool
5598 {
5602 /* For 32-bit, we want TImode aggregates to go on the stack. But watch out!
5603 The layout_type routine is crafty and tries to trick us into passing
5604 currently unsupported vector types on the stack by using TImode. */
5607 }
5609 /* It returns the size, in bytes, of the area reserved for arguments passed
5610 in registers for the function represented by fndecl dependent to the used
5611 abi format. */
5612 int
5614 {
5618 else
5623 }
5625 /* Returns value SYSV_ABI, MS_ABI dependent on fntype, specifying the
5626 call abi used. */
5627 enum calling_abi
5629 {
5631 {
5634 {
5637 }
5641 }
5643 }
5645 static bool
5647 {
5649 {
5653 else
5655 }
5657 }
5659 static enum calling_abi
5661 {
5665 }
5667 /* Returns value SYSV_ABI, MS_ABI dependent on cfun, specifying the
5668 call abi used. */
5669 enum calling_abi
5671 {
5675 }
5677 /* Write the extra assembler code needed to declare a function properly. */
5679 void
5681 tree decl)
5682 {
5686 {
5692 }
5694 #ifdef SUBTARGET_ASM_UNWIND_INIT
5696 #endif
5700 /* Output magic byte marker, if hot-patch attribute is set. */
5702 {
5704 {
5705 /* leaq [%rsp + 0], %rsp */
5708 }
5709 else
5710 {
5711 /* movl.s %edi, %edi
5712 push %ebp
5713 movl.s %esp, %ebp */
5716 }
5717 }
5718 }
5720 /* regclass.c */
5723 /* Implementation of call abi switching target hook. Specific to FNDECL
5724 the specific call register sets are set. See also
5725 ix86_conditional_register_usage for more details. */
5726 void
5728 {
5731 else
5733 }
5735 /* 64-bit MS and SYSV ABI have different set of call used registers. Avoid
5736 expensive re-initialization of init_regs each time we switch function context
5737 since this is needed only during RTL expansion. */
5738 static void
5740 {
5744 }
5746 /* Initialize a variable CUM of type CUMULATIVE_ARGS
5747 for a call to a function whose data type is FNTYPE.
5748 For a library call, FNTYPE is 0. */
5750 void
5754 tree fndecl,
5756 {
5758 tree fnret_type;
5762 /* Initialize for the current callee. */
5764 {
5767 }
5770 {
5774 }
5775 else
5776 {
5781 else
5783 }
5786 {
5790 {
5791 /* The return value of this function uses 256bit AVX modes. */
5794 else
5796 }
5797 }
5801 /* Set up the number of registers to use for passing arguments. */
5808 {
5810 ? X86_64_REGPARM_MAX
5812 }
5814 {
5817 {
5819 ? X86_64_SSE_REGPARM_MAX
5821 }
5822 }
5829 /* Because type might mismatch in between caller and callee, we need to
5830 use actual type of function for local calls.
5831 FIXME: cgraph_analyze can be told to actually record if function uses
5832 va_start so for local functions maybe_vaarg can be made aggressive
5833 helping K&R code.
5834 FIXME: once typesytem is fixed, we won't need this code anymore. */
5842 {
5843 /* If there are variable arguments, then we won't pass anything
5844 in registers in 32-bit mode. */
5846 {
5854 }
5856 /* Use ecx and edx registers if function has fastcall attribute,
5857 else look for regparm information. */
5859 {
5862 {
5865 }
5867 {
5870 }
5871 else
5873 }
5875 /* Set up the number of SSE registers used for passing SFmode
5876 and DFmode arguments. Warn for mismatching ABI. */
5878 }
5879 }
5881 /* Return the "natural" mode for TYPE. In most cases, this is just TYPE_MODE.
5882 But in the case of vector types, it is some vector mode.
5884 When we have only some of our vector isa extensions enabled, then there
5885 are some modes for which vector_mode_supported_p is false. For these
5886 modes, the generic vector support in gcc will choose some non-vector mode
5887 in order to implement the type. By computing the natural mode, we'll
5888 select the proper ABI location for the operand and not depend on whatever
5889 the middle-end decides to do with these vector types.
5891 The midde-end can't deal with the vector types > 16 bytes. In this
5892 case, we return the original mode and warn ABI change if CUM isn't
5893 NULL. */
5895 static enum machine_mode
5897 {
5901 {
5904 /* ??? Generic code allows us to create width 1 vectors. Ignore. */
5906 {
5911 else
5914 /* Get the mode which has this inner mode and number of units. */
5918 {
5920 {
5924 && !warnedavx
5926 {
5930 }
5932 }
5934 {
5938 && !warnedsse
5940 {
5944 }
5946 }
5947 else
5949 }
5952 }
5953 }
5956 }
5958 /* We want to pass a value in REGNO whose "natural" mode is MODE. However,
5959 this may not agree with the mode that the type system has chosen for the
5960 register, which is ORIG_MODE. If ORIG_MODE is not BLKmode, then we can
5961 go ahead and use it. Otherwise we have to build a PARALLEL instead. */
5963 static rtx
5966 {
5967 rtx tmp;
5971 else
5972 {
5976 }
5979 }
5981 /* x86-64 register passing implementation. See x86-64 ABI for details. Goal
5982 of this code is to classify each 8bytes of incoming argument by the register
5983 class and assign registers accordingly. */
5985 /* Return the union class of CLASS1 and CLASS2.
5986 See the x86-64 PS ABI for details. */
5988 static enum x86_64_reg_class
5990 {
5991 /* Rule #1: If both classes are equal, this is the resulting class. */
5995 /* Rule #2: If one of the classes is NO_CLASS, the resulting class is
5996 the other class. */
6002 /* Rule #3: If one of the classes is MEMORY, the result is MEMORY. */
6006 /* Rule #4: If one of the classes is INTEGER, the result is INTEGER. */
6014 /* Rule #5: If one of the classes is X87, X87UP, or COMPLEX_X87 class,
6015 MEMORY is used. */
6024 /* Rule #6: Otherwise class SSE is used. */
6026 }
6028 /* Classify the argument of type TYPE and mode MODE.
6029 CLASSES will be filled by the register class used to pass each word
6030 of the operand. The number of words is returned. In case the parameter
6031 should be passed in memory, 0 is returned. As a special case for zero
6032 sized containers, classes[0] will be NO_CLASS and 1 is returned.
6034 BIT_OFFSET is used internally for handling records and specifies offset
6035 of the offset in bits modulo 256 to avoid overflow cases.
6037 See the x86-64 PS ABI for details.
6038 */
6040 static int
6043 {
6044 HOST_WIDE_INT bytes =
6046 int words
6049 /* Variable sized entities are always passed/returned in memory. */
6058 {
6060 tree field;
6063 /* On x86-64 we pass structures larger than 32 bytes on the stack. */
6070 /* Zero sized arrays or structures are NO_CLASS. We return 0 to
6071 signalize memory class, so handle it as special case. */
6073 {
6076 }
6078 /* Classify each field of record and merge classes. */
6080 {
6082 /* And now merge the fields of structure. */
6084 {
6086 {
6092 /* Bitfields are always classified as integer. Handle them
6093 early, since later code would consider them to be
6094 misaligned integers. */
6096 {
6105 }
6106 else
6107 {
6112 /* Flexible array member is ignored. */
6119 {
6123 {
6126 "the ABI of passing struct with"
6127 " a flexible array member has"
6129 }
6131 }
6133 subclasses,
6143 }
6144 }
6145 }
6149 /* Arrays are handled as small records. */
6150 {
6157 /* The partial classes are now full classes. */
6168 }
6171 /* Unions are similar to RECORD_TYPE but offset is always 0.
6172 */
6174 {
6176 {
6184 bit_offset);
6189 }
6190 }
6195 }
6198 {
6199 /* When size > 16 bytes, if the first one isn't
6200 X86_64_SSE_CLASS or any other ones aren't
6201 X86_64_SSEUP_CLASS, everything should be passed in
6202 memory. */
6209 }
6211 /* Final merger cleanup. */
6213 {
6214 /* If one class is MEMORY, everything should be passed in
6215 memory. */
6219 /* The X86_64_SSEUP_CLASS should be always preceded by
6220 X86_64_SSE_CLASS or X86_64_SSEUP_CLASS. */
6224 {
6225 /* The first one should never be X86_64_SSEUP_CLASS. */
6228 }
6230 /* If X86_64_X87UP_CLASS isn't preceded by X86_64_X87_CLASS,
6231 everything should be passed in memory. */
6234 {
6237 /* The first one should never be X86_64_X87UP_CLASS. */
6240 {
6243 "the ABI of passing union with long double"
6245 }
6247 }
6248 }
6250 }
6252 /* Compute alignment needed. We align all types to natural boundaries with
6253 exception of XFmode that is aligned to 64bits. */
6255 {
6264 /* Misaligned fields are always returned in memory. */
6267 }
6269 /* for V1xx modes, just use the base mode */
6274 /* Classification of atomic types. */
6276 {
6292 {
6296 {
6299 }
6301 {
6304 }
6306 {
6310 }
6312 {
6315 }
6316 else
6318 }
6325 /* OImode shouldn't be used directly. */
6332 else
6350 else
6351 {
6355 {
6358 "the ABI of passing structure with complex float"
6360 }
6363 }
6372 /* This modes is larger than 16 bytes. */
6415 else
6419 }
6420 }
6422 /* Examine the argument and return set number of register required in each
6423 class. Return 0 iff parameter should be passed in memory. */
6424 static int
6427 {
6437 {
6459 }
6461 }
6463 /* Construct container for the argument used by GCC interface. See
6464 FUNCTION_ARG for the detailed description. */
6466 static rtx
6470 {
6471 /* The following variables hold the static issued_error state. */
6485 rtx ret;
6496 /* We allowed the user to turn off SSE for kernel mode. Don't crash if
6497 some less clueful developer tries to use floating-point anyway. */
6499 {
6501 {
6503 {
6506 }
6507 }
6509 {
6512 }
6514 }
6516 /* Likewise, error if the ABI requires us to return values in the
6517 x87 registers and the user specified -mno-80387. */
6523 {
6525 {
6528 }
6530 }
6532 /* First construct simple cases. Avoid SCmode, since we want to use
6533 single register to pass this type. */
6536 {
6551 /* Zero sized array, struct or class. */
6555 }
6582 /* Otherwise figure out the entries of the PARALLEL. */
6584 {
6588 {
6593 /* Merge TImodes on aligned occasions here too. */
6595 tmpmode
6599 else
6601 /* We've requested 24 bytes we
6602 don't have mode for. Use DImode. */
6609 intreg++;
6617 sse_regno++;
6625 sse_regno++;
6630 {
6636 {
6638 i++;
6639 }
6640 else
6653 }
6659 sse_regno++;
6663 }
6664 }
6666 /* Empty aligned struct, union or class. */
6674 }
6676 /* Update the data in CUM to advance over an argument of mode MODE
6677 and data type TYPE. (TYPE is null for libcalls where that information
6678 may not be available.) */
6680 static void
6683 HOST_WIDE_INT words)
6684 {
6686 {
6693 /* FALLTHRU */
6704 {
6707 }
6711 /* OImode shouldn't be used directly. */
6720 /* FALLTHRU */
6736 {
6741 {
6744 }
6745 }
6755 {
6760 {
6763 }
6764 }
6766 }
6767 }
6769 static void
6772 {
6775 /* Unnamed 256bit vector mode parameters are passed on stack. */
6781 {
6786 }
6787 else
6788 {
6792 }
6793 }
6795 static void
6797 HOST_WIDE_INT words)
6798 {
6799 /* Otherwise, this should be passed indirect. */
6804 {
6807 }
6808 }
6810 /* Update the data in CUM to advance over an argument of mode MODE and
6811 data type TYPE. (TYPE is null for libcalls where that information
6812 may not be available.) */
6814 static void
6817 {
6823 else
6834 else
6836 }
6838 /* Define where to put the arguments to a function.
6839 Value is zero to push the argument on the stack,
6840 or a hard register in which to store the argument.
6842 MODE is the argument's machine mode.
6843 TYPE is the data type of the argument (as a tree).
6844 This is null for libcalls where that information may
6845 not be available.
6846 CUM is a variable of type CUMULATIVE_ARGS which gives info about
6847 the preceding args and about the function being called.
6848 NAMED is nonzero if this argument is a named parameter
6849 (otherwise it is an extra parameter matching an ellipsis). */
6851 static rtx
6855 {
6858 /* Avoid the AL settings for the Unix64 ABI. */
6863 {
6870 /* FALLTHRU */
6876 {
6879 /* Fastcall allocates the first two DWORD (SImode) or
6880 smaller arguments to ECX and EDX if it isn't an
6881 aggregate type . */
6883 {
6889 /* ECX not EAX is the first allocated register. */
6892 }
6894 }
6903 /* FALLTHRU */
6905 /* In 32bit, we pass TImode in xmm registers. */
6913 {
6915 {
6919 }
6923 }
6927 /* OImode shouldn't be used directly. */
6937 {
6941 }
6951 {
6953 {
6957 }
6961 }
6963 }
6966 }
6968 static rtx
6971 {
6972 /* Handle a hidden AL argument containing number of registers
6973 for varargs x86-64 functions. */
6977 ? X86_64_SSE_REGPARM_MAX
6982 {
6992 /* Unnamed 256bit vector mode parameters are passed on stack. */
6996 }
7002 }
7004 static rtx
7007 HOST_WIDE_INT bytes)
7008 {
7011 /* We need to add clobber for MS_ABI->SYSV ABI calls in expand_call.
7012 We use value of -2 to specify that current function call is MSABI. */
7016 /* If we've run out of registers, it goes on the stack. */
7022 /* Only floating point modes are passed in anything but integer regs. */
7024 {
7027 else
7028 {
7031 /* Unnamed floating parameters are passed in both the
7032 SSE and integer registers. */
7038 }
7039 }
7040 /* Handle aggregated types passed in register. */
7042 {
7047 }
7050 }
7052 /* Return where to put the arguments to a function.
7053 Return zero to push the argument on the stack, or a hard register in which to store the argument.
7055 MODE is the argument's machine mode. TYPE is the data type of the
7056 argument. It is null for libcalls where that information may not be
7057 available. CUM gives information about the preceding args and about
7058 the function being called. NAMED is nonzero if this argument is a
7059 named parameter (otherwise it is an extra parameter matching an
7060 ellipsis). */
7062 static rtx
7065 {
7069 rtx arg;
7073 else
7077 /* To simplify the code below, represent vector types with a vector mode
7078 even if MMX/SSE are not active. */
7086 else
7090 {
7091 /* This argument uses 256bit AVX modes. */
7094 else
7096 }
7099 }
7101 /* A C expression that indicates when an argument must be passed by
7102 reference. If nonzero for an argument, a copy of that argument is
7103 made in memory and a pointer to the argument is passed instead of
7104 the argument itself. The pointer is passed in whatever way is
7105 appropriate for passing a pointer to that type. */
7107 static bool
7111 {
7114 /* See Windows x64 Software Convention. */
7116 {
7119 {
7120 /* Arrays are passed by reference. */
7125 {
7126 /* Structs/unions of sizes other than 8, 16, 32, or 64 bits
7127 are passed by reference. */
7129 }
7130 }
7132 /* __m128 is passed by reference. */
7138 }
7139 }
7144 }
7146 /* Return true when TYPE should be 128bit aligned for 32bit argument
7147 passing ABI. XXX: This function is obsolete and is only used for
7148 checking psABI compatibility with previous versions of GCC. */
7150 static bool
7152 {
7164 {
7165 /* Walk the aggregates recursively. */
7167 {
7171 {
7172 tree field;
7174 /* Walk all the structure fields. */
7176 {
7180 }
7182 }
7185 /* Just for use if some languages passes arrays by value. */
7192 }
7193 }
7195 }
7197 /* Return the alignment boundary for MODE and TYPE with alignment ALIGN.
7198 XXX: This function is obsolete and is only used for checking psABI
7199 compatibility with previous versions of GCC. */
7201 static unsigned int
7204 {
7205 /* In 32bit, only _Decimal128 and __float128 are aligned to their
7206 natural boundaries. */
7208 {
7209 /* i386 ABI defines all arguments to be 4 byte aligned. We have to
7210 make an exception for SSE modes since these require 128bit
7211 alignment.
7213 The handling here differs from field_alignment. ICC aligns MMX
7214 arguments to 4 byte boundaries, while structure fields are aligned
7215 to 8 byte boundaries. */
7217 {
7220 }
7221 else
7222 {
7225 }
7226 }
7230 }
7232 /* Return true when TYPE should be 128bit aligned for 32bit argument
7233 passing ABI. */
7235 static bool
7237 {
7247 {
7248 /* Walk the aggregates recursively. */
7250 {
7254 {
7255 tree field;
7257 /* Walk all the structure fields. */
7259 field;
7261 {
7265 }
7267 }
7270 /* Just for use if some languages passes arrays by value. */
7277 }
7278 }
7279 else
7283 }
7285 /* Gives the alignment boundary, in bits, of an argument with the
7286 specified mode and type. */
7288 static unsigned int
7290 {
7293 {
7294 /* Since the main variant type is used for call, we convert it to
7295 the main variant type. */
7298 }
7299 else
7303 else
7304 {
7309 {
7310 /* i386 ABI defines XFmode arguments to be 4 byte aligned. */
7312 {
7315 }
7321 }
7324 && !warned
7326 saved_align))
7327 {
7330 "The ABI for passing parameters with %d-byte"
7333 }
7334 }
7337 }
7339 /* Return true if N is a possible register number of function value. */
7341 static bool
7343 {
7345 {
7350 /* TODO: The function should depend on current function ABI but
7351 builtins.c would need updating then. Therefore we use the
7352 default ABI. */
7364 }
7367 }
7369 /* Define how to find the value returned by a function.
7370 VALTYPE is the data type of the value (as a tree).
7371 If the precise function being called is known, FUNC is its FUNCTION_DECL;
7372 otherwise, FUNC is 0. */
7374 static rtx
7377 {
7380 /* 8-byte vector modes in %mm0. See ix86_return_in_memory for where
7381 we normally prevent this case when mmx is not available. However
7382 some ABIs may require the result to be returned like DImode. */
7386 /* 16-byte vector modes in %xmm0. See ix86_return_in_memory for where
7387 we prevent this case when sse is not available. However some ABIs
7388 may require the result to be returned like integer TImode. */
7393 /* 32-byte vector modes in %ymm0. */
7397 /* Floating point return values in %st(0) (unless -mno-fp-ret-in-387). */
7400 else
7401 /* Most things go in %eax. */
7404 /* Override FP return register with %xmm0 for local functions when
7405 SSE math is enabled or for functions with sseregparm attribute. */
7407 {
7412 }
7414 /* OImode shouldn't be used directly. */
7418 }
7420 static rtx
7422 const_tree valtype)
7423 {
7424 rtx ret;
7426 /* Handle libcalls, which don't provide a type node. */
7428 {
7432 {
7451 }
7454 }
7456 {
7457 /* Pointers are always returned in word_mode. */
7459 }
7465 /* For zero sized structures, construct_container returns NULL, but we
7466 need to keep rest of compiler happy by returning meaningful value. */
7471 }
7473 static rtx
7475 {
7479 {
7481 {
7494 }
7495 }
7497 }
7499 static rtx
7502 {
7514 else
7516 }
7518 static rtx
7521 {
7527 }
7529 /* Pointer function arguments and return values are promoted to
7530 word_mode. */
7532 static enum machine_mode
7536 {
7538 {
7541 }
7543 for_return);
7544 }
7546 rtx
7548 {
7550 }
7552 /* Return true iff type is returned in memory. */
7554 static bool ATTRIBUTE_UNUSED
7556 {
7557 HOST_WIDE_INT size;
7568 {
7569 /* User-created vectors small enough to fit in EAX. */
7573 /* MMX/3dNow values are returned in MM0,
7574 except when it doesn't exits or the ABI prescribes otherwise. */
7578 /* SSE values are returned in XMM0, except when it doesn't exist. */
7582 /* AVX values are returned in YMM0, except when it doesn't exist. */
7585 }
7593 /* OImode shouldn't be used directly. */
7597 }
7599 static bool ATTRIBUTE_UNUSED
7601 {
7604 }
7606 static bool ATTRIBUTE_UNUSED
7608 {
7611 /* __m128 is returned in xmm0. */
7616 /* Otherwise, the size must be exactly in [1248]. */
7618 }
7620 static bool
7622 {
7623 #ifdef SUBTARGET_RETURN_IN_MEMORY
7625 #else
7629 {
7632 else
7634 }
7635 else
7637 #endif
7638 }
7640 /* When returning SSE vector types, we have a choice of either
7641 (1) being abi incompatible with a -march switch, or
7642 (2) generating an error.
7643 Given no good solution, I think the safest thing is one warning.
7644 The user won't be able to use -Werror, but....
7646 Choose the STRUCT_VALUE_RTX hook because that's (at present) only
7647 called in response to actually generating a caller or callee that
7648 uses such a type. As opposed to TARGET_RETURN_IN_MEMORY, which is called
7649 via aggregate_value_p for general type probing from tree-ssa. */
7651 static rtx
7653 {
7657 {
7658 /* Look at the return type of the function, not the function type. */
7662 {
7665 {
7669 }
7670 }
7673 {
7675 {
7679 }
7680 }
7681 }
7684 }
7686 \f
7687 /* Create the va_list data type. */
7689 /* Returns the calling convention specific va_list date type.
7690 The argument ABI can be DEFAULT_ABI, MS_ABI, or SYSV_ABI. */
7692 static tree
7694 {
7697 /* For i386 we use plain pointer to argument area. */
7707 unsigned_type_node);
7710 unsigned_type_node);
7713 ptr_type_node);
7716 ptr_type_node);
7735 /* The correct type is an array type of one element. */
7737 }
7739 /* Setup the builtin va_list data type and for 64-bit the additional
7740 calling convention specific va_list data types. */
7742 static tree
7744 {
7747 /* Initialize abi specific va_list builtin types. */
7749 {
7750 tree t;
7752 {
7757 }
7758 else
7759 {
7764 }
7766 {
7771 }
7772 else
7773 {
7778 }
7779 }
7782 }
7784 /* Worker function for TARGET_SETUP_INCOMING_VARARGS. */
7786 static void
7788 {
7790 alias_set_type set;
7793 /* GPR size of varargs save area. */
7796 else
7799 /* FPR size of varargs save area. We don't need it if we don't pass
7800 anything in SSE registers. */
7803 else
7817 {
7825 }
7828 {
7832 /* Now emit code to save SSE registers. The AX parameter contains number
7833 of SSE parameter registers used to call this function, though all we
7834 actually check here is the zero/non-zero status. */
7839 label));
7841 /* ??? If !TARGET_SSE_TYPELESS_STORES, would we perform better if
7842 we used movdqa (i.e. TImode) instead? Perhaps even better would
7843 be if we could determine the real mode of the data, via a hook
7844 into pass_stdarg. Ignore all that for now. */
7854 {
7863 }
7866 }
7867 }
7869 static void
7871 {
7875 /* Reset to zero, as there might be a sysv vaarg used
7876 before. */
7881 {
7892 }
7893 }
7895 static void
7899 {
7901 CUMULATIVE_ARGS next_cum;
7902 tree fntype;
7904 /* This argument doesn't appear to be used anymore. Which is good,
7905 because the old code here didn't suppress rtl generation. */
7913 /* For varargs, we do not want to skip the dummy va_dcl argument.
7914 For stdargs, we do want to skip the last named argument. */
7922 else
7924 }
7926 /* Checks if TYPE is of kind va_list char *. */
7928 static bool
7930 {
7931 tree canonic;
7933 /* For 32-bit it is always true. */
7939 }
7941 /* Implement va_start. */
7943 static void
7945 {
7949 tree type;
7950 rtx ovf_rtx;
7954 {
7957 /* When we are splitting the stack, we can't refer to the stack
7958 arguments using internal_arg_pointer, because they may be on
7959 the old stack. The split stack prologue will arrange to
7960 leave a pointer to the old stack arguments in a scratch
7961 register, which we here copy to a pseudo-register. The split
7962 stack prologue can't set the pseudo-register directly because
7963 it (the prologue) runs before any registers have been saved. */
7967 {
7981 }
7982 }
7984 /* Only 64bit target needs something special. */
7986 {
7989 else
7990 {
7999 }
8001 }
8010 /* The following should be folded into the MEM_REF offset. */
8020 /* Count number of gp and fp argument registers used. */
8026 {
8032 }
8035 {
8041 }
8043 /* Find the overflow area. */
8047 else
8057 {
8058 /* Find the register save area.
8059 Prologue of the function save it right above stack frame. */
8067 }
8068 }
8070 /* Implement va_arg. */
8072 static tree
8075 {
8082 rtx container;
8084 tree ptrtype;
8088 /* Only 64bit target needs something special. */
8112 {
8119 /* Unnamed 256bit vector mode parameters are passed on stack. */
8121 {
8124 }
8132 }
8134 /* Pull the value out of the saved registers. */
8139 {
8153 /* In case we are passing structure, verify that it is consecutive block
8154 on the register save area. If not we need to do moves. */
8156 {
8157 /* Verify that all registers are strictly consecutive */
8159 {
8163 {
8168 }
8169 }
8170 else
8171 {
8175 {
8180 }
8181 }
8182 }
8184 {
8187 }
8188 else
8189 {
8192 }
8194 /* First ensure that we fit completely in registers. */
8196 {
8203 }
8205 {
8213 }
8215 /* Compute index to start of area used for integer regs. */
8217 {
8218 /* int_addr = gpr + sav; */
8221 }
8223 {
8224 /* sse_addr = fpr + sav; */
8227 }
8229 {
8233 /* addr = &temp; */
8238 {
8242 tree piece_type;
8243 tree addr_type;
8244 tree daddr_type;
8253 {
8258 }
8262 else
8269 {
8272 }
8273 else
8274 {
8277 }
8284 {
8289 }
8290 else
8291 {
8292 tree copy
8297 }
8299 }
8300 }
8303 {
8307 }
8310 {
8314 }
8319 }
8321 /* ... otherwise out of the overflow area. */
8323 /* When we align parameter on stack for caller, if the parameter
8324 alignment is beyond MAX_SUPPORTED_STACK_ALIGNMENT, it will be
8325 aligned at MAX_SUPPORTED_STACK_ALIGNMENT. We will match callee
8326 here with caller. */
8331 /* Care for on-stack alignment if needed. */
8334 else
8335 {
8340 }
8357 }
8358 \f
8359 /* Return true if OPNUM's MEM should be matched
8360 in movabs* patterns. */
8362 bool
8364 {
8376 }
8377 \f
8378 /* Initialize the table of extra 80387 mathematical constants. */
8380 static void
8382 {
8384 {
8390 };
8394 {
8396 /* Ensure each constant is rounded to XFmode precision. */
8399 }
8402 }
8404 /* Return non-zero if the constant is something that
8405 can be loaded with a special instruction. */
8407 int
8409 {
8412 REAL_VALUE_TYPE r;
8424 /* For XFmode constants, try to find a special 80387 instruction when
8425 optimizing for size or on those CPUs that benefit from them. */
8428 {
8437 }
8439 /* Load of the constant -0.0 or -1.0 will be split as
8440 fldz;fchs or fld1;fchs sequence. */
8447 }
8449 /* Return the opcode of the special instruction to be used to load
8450 the constant X. */
8454 {
8456 {
8476 }
8477 }
8479 /* Return the CONST_DOUBLE representing the 80387 constant that is
8480 loaded by the specified special instruction. The argument IDX
8481 matches the return value from standard_80387_constant_p. */
8483 rtx
8485 {
8492 {
8503 }
8506 XFmode);
8507 }
8509 /* Return 1 if X is all 0s and 2 if x is all 1s
8510 in supported SSE/AVX vector mode. */
8512 int
8514 {
8521 {
8536 }
8539 }
8541 /* Return the opcode of the special instruction to be used to load
8542 the constant X. */
8546 {
8548 {
8551 {
8568 }
8573 else
8578 }
8580 }
8582 /* Returns true if OP contains a symbol reference */
8584 bool
8586 {
8595 {
8597 {
8603 }
8607 }
8610 }
8612 /* Return true if it is appropriate to emit `ret' instructions in the
8613 body of a function. Do this only if the epilogue is simple, needing a
8614 couple of insns. Prior to reloading, we can't tell how many registers
8615 must be saved, so return false then. Return false if there is no frame
8616 marker to de-allocate. */
8618 bool
8620 {
8626 /* Don't allow more than 32k pop, since that's all we can do
8627 with one instruction. */
8634 }
8635 \f
8636 /* Value should be nonzero if functions must have frame pointers.
8637 Zero means the frame pointer need not be set up (and parms may
8638 be accessed via the stack pointer) in functions that seem suitable. */
8640 static bool
8642 {
8643 /* If we accessed previous frames, then the generated code expects
8644 to be able to access the saved ebp value in our frame. */
8648 /* Several x86 os'es need a frame pointer for other reasons,
8649 usually pertaining to setjmp. */
8653 /* For older 32-bit runtimes setjmp requires valid frame-pointer. */
8657 /* Win64 SEH, very large frames need a frame-pointer as maximum stack
8658 allocation is 4GB. */
8662 /* In ix86_option_override_internal, TARGET_OMIT_LEAF_FRAME_POINTER
8663 turns off the frame pointer by default. Turn it back on now if
8664 we've not got a leaf function. */
8674 }
8676 /* Record that the current function accesses previous call frames. */
8678 void
8680 {
8682 }
8683 \f
8684 #ifndef USE_HIDDEN_LINKONCE
8685 # if defined(HAVE_GAS_HIDDEN) && (SUPPORTS_ONE_ONLY - 0)
8686 # define USE_HIDDEN_LINKONCE 1
8687 # else
8688 # define USE_HIDDEN_LINKONCE 0
8689 # endif
8690 #endif
8694 /* Fills in the label name that should be used for a pc thunk for
8695 the given register. */
8697 static void
8699 {
8704 else
8706 }
8709 /* This function generates code for -fpic that loads %ebx with
8710 the return address of the caller and then returns. */
8712 static void
8714 {
8719 {
8721 tree decl;
8737 #if TARGET_MACHO
8739 {
8748 }
8749 else
8750 #endif
8752 {
8763 }
8764 else
8765 {
8768 }
8774 /* Make sure unwind info is emitted for the thunk if needed. */
8777 /* Pad stack IP move with 4 instructions (two NOPs count
8778 as one instruction). */
8780 {
8785 }
8796 }
8800 }
8802 /* Emit code for the SET_GOT patterns. */
8806 {
8812 {
8813 /* Load (*VXWORKS_GOTT_BASE) into the PIC register. */
8818 /* Load (*VXWORKS_GOTT_BASE)[VXWORKS_GOTT_INDEX] into the PIC register.
8819 Use %P and a local symbol in order to print VXWORKS_GOTT_INDEX as
8820 an unadorned address. */
8825 }
8830 {
8835 #if TARGET_MACHO
8836 /* Output the Mach-O "canonical" label name ("Lxx$pb") here too. This
8837 is what will be referenced by the Mach-O PIC subsystem. */
8840 #endif
8844 }
8845 else
8846 {
8854 /* Output the Mach-O "canonical" label name ("Lxx$pb") here too. This
8855 is what will be referenced by the Mach-O PIC subsystem. */
8856 #if TARGET_MACHO
8859 else
8862 #endif
8863 }
8869 }
8871 /* Generate an "push" pattern for input ARG. */
8873 static rtx
8875 {
8888 stack_pointer_rtx)),
8889 arg);
8890 }
8892 /* Generate an "pop" pattern for input ARG. */
8894 static rtx
8896 {
8901 arg,
8904 stack_pointer_rtx)));
8905 }
8907 /* Return >= 0 if there is an unused call-clobbered register available
8908 for the entire function. */
8910 static unsigned int
8912 {
8916 {
8918 /* Can't use the same register for both PIC and DRAP. */
8921 else
8926 }
8929 }
8931 /* Return TRUE if we need to save REGNO. */
8933 static bool
8935 {
8945 {
8948 {
8954 }
8955 }
8964 }
8966 /* Return number of saved general prupose registers. */
8968 static int
8970 {
8976 nregs ++;
8978 }
8980 /* Return number of saved SSE registrers. */
8982 static int
8984 {
8992 nregs ++;
8994 }
8996 /* Given FROM and TO register numbers, say whether this elimination is
8997 allowed. If stack alignment is needed, we can only replace argument
8998 pointer with hard frame pointer, or replace frame pointer with stack
8999 pointer. Otherwise, frame pointer elimination is automatically
9000 handled and all other eliminations are valid. */
9002 static bool
9004 {
9010 else
9012 }
9014 /* Return the offset between two registers, one to be eliminated, and the other
9015 its replacement, at the start of a routine. */
9017 HOST_WIDE_INT
9019 {
9028 else
9029 {
9037 }
9038 }
9040 /* In a dynamically-aligned function, we can't know the offset from
9041 stack pointer to frame pointer, so we must ensure that setjmp
9042 eliminates fp against the hard fp (%ebp) rather than trying to
9043 index from %esp up to the top of the frame across a gap that is
9044 of unknown (at compile-time) size. */
9045 static rtx
9047 {
9049 }
9051 /* When using -fsplit-stack, the allocation routines set a field in
9052 the TCB to the bottom of the stack plus this much space, measured
9053 in bytes. */
9055 #define SPLIT_STACK_AVAILABLE 256
9057 /* Fill structure ix86_frame about frame of currently computed function. */
9059 static void
9061 {
9063 HOST_WIDE_INT offset;
9066 HOST_WIDE_INT to_allocate;
9074 /* 64-bit MS ABI seem to require stack alignment to be always 16 except for
9075 function prologues and leaf. */
9079 {
9084 }
9090 /* For SEH we have to limit the amount of code movement into the prologue.
9091 At present we do this via a BLOCKAGE, at which point there's very little
9092 scheduling that can be done, which means that there's very little point
9093 in doing anything except PUSHs. */
9097 /* During reload iteration the amount of registers saved can change.
9098 Recompute the value as needed. Do not recompute when amount of registers
9099 didn't change as reload does multiple calls to the function and does not
9100 expect the decision to change within single iteration. */
9103 {
9109 /* The fast prologue uses move instead of push to save registers. This
9110 is significantly longer, but also executes faster as modern hardware
9111 can execute the moves in parallel, but can't do that for push/pop.
9113 Be careful about choosing what prologue to emit: When function takes
9114 many instructions to execute we may use slow version as well as in
9115 case function is known to be outside hot spot (this is known with
9116 feedback only). Weight the size of function by number of registers
9117 to save as it is cheap to use one or two push instructions but very
9118 slow to use many of them. */
9122 || (flag_branch_probabilities
9125 else
9128 }
9130 frame->save_regs_using_mov
9132 /* If static stack checking is enabled and done with probes,
9133 the registers need to be saved before allocating the frame. */
9136 /* Skip return address. */
9139 /* Skip pushed static chain. */
9143 /* Skip saved base pointer. */
9148 /* The traditional frame pointer location is at the top of the frame. */
9151 /* Register save area */
9155 /* On SEH target, registers are pushed just before the frame pointer
9156 location. */
9160 /* Align and set SSE register save area. */
9162 {
9163 /* The only ABI that has saved SSE registers (Win64) also has a
9164 16-byte aligned default stack, and thus we don't need to be
9165 within the re-aligned local stack frame to save them. */
9169 }
9172 /* The re-aligned stack starts here. Values before this point are not
9173 directly comparable with values below this point. In order to make
9174 sure that no value happens to be the same before and after, force
9175 the alignment computation below to add a non-zero value. */
9179 /* Va-arg area */
9183 /* Align start of frame for local function. */
9192 /* Frame pointer points here. */
9197 /* Add outgoing arguments area. Can be skipped if we eliminated
9198 all the function calls as dead code.
9199 Skipping is however impossible when function calls alloca. Alloca
9200 expander assumes that last crtl->outgoing_args_size
9201 of stack frame are unused. */
9205 {
9208 }
9209 else
9212 /* Align stack boundary. Only needed if we're calling another function
9213 or using alloca. */
9218 /* We've reached end of stack frame. */
9221 /* Size prologue needs to allocate. */
9232 {
9238 }
9239 else
9243 /* The SEH frame pointer location is near the bottom of the frame.
9244 This is enforced by the fact that the difference between the
9245 stack pointer and the frame pointer is limited to 240 bytes in
9246 the unwind data structure. */
9248 {
9249 HOST_WIDE_INT diff;
9251 /* If we can leave the frame pointer where it is, do so. Also, returns
9252 the establisher frame for __builtin_frame_address (0). */
9257 {
9258 /* Ideally we'd determine what portion of the local stack frame
9259 (within the constraint of the lowest 240) is most heavily used.
9260 But without that complication, simply bias the frame pointer
9261 by 128 bytes so as to maximize the amount of the local stack
9262 frame that is addressable with 8-bit offsets. */
9264 }
9265 }
9266 }
9268 /* This is semi-inlined memory_address_length, but simplified
9269 since we know that we're always dealing with reg+offset, and
9270 to avoid having to create and discard all that rtl. */
9274 {
9278 {
9279 /* EBP and R13 cannot be encoded without an offset. */
9281 }
9285 /* ESP and R12 must be encoded with a SIB byte. */
9287 len++;
9290 }
9292 /* Return an RTX that points to CFA_OFFSET within the stack frame.
9293 The valid base registers are taken from CFUN->MACHINE->FS. */
9295 static rtx
9297 {
9303 {
9304 /* Choose the base register most likely to allow the most scheduling
9305 opportunities. Generally FP is valid throughout the function,
9306 while DRAP must be reloaded within the epilogue. But choose either
9307 over the SP due to increased encoding size. */
9310 {
9313 }
9315 {
9318 }
9320 {
9323 }
9324 }
9325 else
9326 {
9327 HOST_WIDE_INT toffset;
9330 /* Choose the base register with the smallest address encoding.
9331 With a tie, choose FP > DRAP > SP. */
9333 {
9337 }
9339 {
9343 {
9347 }
9348 }
9350 {
9354 {
9358 }
9359 }
9360 }
9364 }
9366 /* Emit code to save registers in the prologue. */
9368 static void
9370 {
9372 rtx insn;
9376 {
9379 }
9380 }
9382 /* Emit a single register save at CFA - CFA_OFFSET. */
9384 static void
9386 HOST_WIDE_INT cfa_offset)
9387 {
9395 /* For SSE saves, we need to indicate the 128-bit alignment. */
9406 /* When saving registers into a re-aligned local stack frame, avoid
9407 any tricky guessing by dwarf2out. */
9409 {
9413 {
9414 /* A bit of a hack. We force the DRAP register to be saved in
9415 the re-aligned stack frame, which provides us with a copy
9416 of the CFA that will last past the prologue. Install it. */
9422 }
9423 else
9424 {
9425 /* The frame pointer is a stable reference within the
9426 aligned frame. Use it. */
9433 }
9434 }
9436 /* The memory may not be relative to the current CFA register,
9437 which means that we may need to generate a new pattern for
9438 use by the unwind info. */
9440 {
9445 }
9446 }
9448 /* Emit code to save registers using MOV insns.
9449 First register is stored at CFA - CFA_OFFSET. */
9450 static void
9452 {
9457 {
9460 }
9461 }
9463 /* Emit code to save SSE registers using MOV insns.
9464 First register is stored at CFA - CFA_OFFSET. */
9465 static void
9467 {
9472 {
9475 }
9476 }
9480 /* Add a REG_CFA_RESTORE REG note to INSN or queue them until next stack
9481 manipulation insn. The value is on the stack at CFA - CFA_OFFSET.
9482 Don't add the note if the previously saved value will be left untouched
9483 within stack red-zone till return, as unwinders can find the same value
9484 in the register and on the stack. */
9486 static void
9488 {
9494 {
9497 }
9498 else
9499 queued_cfa_restores
9501 }
9503 /* Add queued REG_CFA_RESTORE notes if any to INSN. */
9505 static void
9507 {
9508 rtx last;
9512 ;
9517 }
9519 /* Expand prologue or epilogue stack adjustment.
9520 The pattern exist to put a dependency on all ebp-based memory accesses.
9521 STYLE should be negative if instructions should be marked as frame related,
9522 zero if %r11 register is live and cannot be freely used and positive
9523 otherwise. */
9525 static void
9528 {
9530 rtx insn;
9537 else
9538 {
9539 rtx tmp;
9540 /* r11 is used by indirect sibcall return as well, set before the
9541 epilogue and used after the epilogue. */
9544 else
9545 {
9549 }
9555 }
9562 {
9563 rtx r;
9573 }
9575 {
9578 {
9582 }
9583 }
9586 {
9591 {
9594 }
9596 {
9599 }
9600 else
9601 {
9602 /* Else there are two possibilities: SP itself, which we set
9603 up as the default above. Or EH_RETURN_STACKADJ_RTX, which is
9604 taken care of this by hand along the eh_return path. */
9607 }
9611 }
9612 }
9614 /* Find an available register to be used as dynamic realign argument
9615 pointer regsiter. Such a register will be written in prologue and
9616 used in begin of body, so it must not be
9617 1. parameter passing register.
9618 2. GOT pointer.
9619 We reuse static-chain register if it is available. Otherwise, we
9620 use DI for i386 and R13 for x86-64. We chose R13 since it has
9621 shorter encoding.
9623 Return: the regno of chosen register. */
9625 static unsigned int
9627 {
9631 {
9632 /* Use R13 for nested function or function need static chain.
9633 Since function with tail call may use any caller-saved
9634 registers in epilogue, DRAP must not use caller-saved
9635 register in such case. */
9640 }
9641 else
9642 {
9643 /* Use DI for nested function or function need static chain.
9644 Since function with tail call may use any caller-saved
9645 registers in epilogue, DRAP must not use caller-saved
9646 register in such case. */
9650 /* Reuse static chain register if it isn't used for parameter
9651 passing. */
9653 {
9657 }
9659 }
9660 }
9662 /* Return minimum incoming stack alignment. */
9664 static unsigned int
9666 {
9669 /* Prefer the one specified at command line. */
9672 /* In 32bit, use MIN_STACK_BOUNDARY for incoming stack boundary
9673 if -mstackrealign is used, it isn't used for sibcall check and
9674 estimated stack alignment is 128bit. */
9676 && !TARGET_64BIT
9677 && ix86_force_align_arg_pointer
9680 else
9683 /* Incoming stack alignment can be changed on individual functions
9684 via force_align_arg_pointer attribute. We use the smallest
9685 incoming stack boundary. */
9691 /* The incoming stack frame has to be aligned at least at
9692 parm_stack_boundary. */
9696 /* Stack at entrance of main is aligned by runtime. We use the
9697 smallest incoming stack boundary. */
9705 }
9707 /* Update incoming stack boundary and estimated stack alignment. */
9709 static void
9711 {
9712 ix86_incoming_stack_boundary
9715 /* x86_64 vararg needs 16byte stack alignment for register save
9716 area. */
9721 }
9723 /* Handle the TARGET_GET_DRAP_RTX hook. Return NULL if no DRAP is
9724 needed or an rtx for DRAP otherwise. */
9726 static rtx
9728 {
9733 {
9734 /* Assign DRAP to vDRAP and returns vDRAP */
9736 rtx drap_vreg;
9737 rtx arg_ptr;
9750 {
9753 }
9755 }
9756 else
9758 }
9760 /* Handle the TARGET_INTERNAL_ARG_POINTER hook. */
9762 static rtx
9764 {
9766 }
9769 rtx reg;
9771 };
9773 /* Return a short-lived scratch register for use on function entry.
9774 In 32-bit mode, it is valid only after the registers are saved
9775 in the prologue. This register must be released by means of
9776 release_scratch_register_on_entry once it is dead. */
9778 static void
9780 {
9786 {
9787 /* We always use R11 in 64-bit mode. */
9789 }
9790 else
9791 {
9793 bool fastcall_p
9797 int drap_regno
9800 /* 'fastcall' sets regparm to 2, uses ecx/edx for arguments and eax
9801 for the static chain register. */
9807 /* ecx is the static chain register. */
9813 /* esi is the static chain register. */
9819 else
9820 {
9823 }
9824 }
9828 {
9831 }
9832 }
9834 /* Release a scratch register obtained from the preceding function. */
9836 static void
9838 {
9840 {
9843 /* The RTX_FRAME_RELATED_P mechanism doesn't know about pop. */
9848 }
9849 }
9851 #define PROBE_INTERVAL (1 << STACK_CHECK_PROBE_INTERVAL_EXP)
9853 /* Emit code to adjust the stack pointer by SIZE bytes while probing it. */
9855 static void
9857 {
9858 /* We skip the probe for the first interval + a small dope of 4 words and
9859 probe that many bytes past the specified size to maintain a protection
9860 area at the botton of the stack. */
9864 /* See if we have a constant small number of probes to generate. If so,
9865 that's the easy case. The run-time loop is made up of 11 insns in the
9866 generic case while the compile-time loop is made up of 3+2*(n-1) insns
9867 for n # of intervals. */
9869 {
9873 /* Adjust SP and probe at PROBE_INTERVAL + N * PROBE_INTERVAL for
9874 values of N from 1 until it exceeds SIZE. If only one probe is
9875 needed, this will not generate any code. Then adjust and probe
9876 to PROBE_INTERVAL + SIZE. */
9878 {
9880 {
9883 }
9884 else
9891 }
9895 else
9903 /* Adjust back to account for the additional first interval. */
9907 }
9909 /* Otherwise, do the same as above, but in a loop. Note that we must be
9910 extra careful with variables wrapping around because we might be at
9911 the very top (or the very bottom) of the address space and we have
9912 to be able to handle this case properly; in particular, we use an
9913 equality test for the loop condition. */
9914 else
9915 {
9916 HOST_WIDE_INT rounded_size;
9922 /* Step 1: round SIZE to the previous multiple of the interval. */
9927 /* Step 2: compute initial and final value of the loop counter. */
9929 /* SP = SP_0 + PROBE_INTERVAL. */
9934 /* LAST_ADDR = SP_0 + PROBE_INTERVAL + ROUNDED_SIZE. */
9938 stack_pointer_rtx)));
9941 /* Step 3: the loop
9943 while (SP != LAST_ADDR)
9944 {
9945 SP = SP + PROBE_INTERVAL
9946 probe at SP
9947 }
9949 adjusts SP and probes to PROBE_INTERVAL + N * PROBE_INTERVAL for
9950 values of N from 1 until it is equal to ROUNDED_SIZE. */
9955 /* Step 4: adjust SP and probe at PROBE_INTERVAL + SIZE if we cannot
9956 assert at compile-time that SIZE is equal to ROUNDED_SIZE. */
9959 {
9964 }
9966 /* Adjust back to account for the additional first interval. */
9972 }
9976 /* Even if the stack pointer isn't the CFA register, we need to correctly
9977 describe the adjustments made to it, in particular differentiate the
9978 frame-related ones from the frame-unrelated ones. */
9980 {
9993 }
9995 /* Make sure nothing is scheduled before we are done. */
9997 }
9999 /* Adjust the stack pointer up to REG while probing it. */
10003 {
10013 /* Jump to END_LAB if SP == LAST_ADDR. */
10021 /* SP = SP + PROBE_INTERVAL. */
10025 /* Probe at SP. */
10036 }
10038 /* Emit code to probe a range of stack addresses from FIRST to FIRST+SIZE,
10039 inclusive. These are offsets from the current stack pointer. */
10041 static void
10043 {
10044 /* See if we have a constant small number of probes to generate. If so,
10045 that's the easy case. The run-time loop is made up of 7 insns in the
10046 generic case while the compile-time loop is made up of n insns for n #
10047 of intervals. */
10049 {
10050 HOST_WIDE_INT i;
10052 /* Probe at FIRST + N * PROBE_INTERVAL for values of N from 1 until
10053 it exceeds SIZE. If only one probe is needed, this will not
10054 generate any code. Then probe at FIRST + SIZE. */
10061 }
10063 /* Otherwise, do the same as above, but in a loop. Note that we must be
10064 extra careful with variables wrapping around because we might be at
10065 the very top (or the very bottom) of the address space and we have
10066 to be able to handle this case properly; in particular, we use an
10067 equality test for the loop condition. */
10068 else
10069 {
10076 /* Step 1: round SIZE to the previous multiple of the interval. */
10081 /* Step 2: compute initial and final value of the loop counter. */
10083 /* TEST_OFFSET = FIRST. */
10086 /* LAST_OFFSET = FIRST + ROUNDED_SIZE. */
10090 /* Step 3: the loop
10092 while (TEST_ADDR != LAST_ADDR)
10093 {
10094 TEST_ADDR = TEST_ADDR + PROBE_INTERVAL
10095 probe at TEST_ADDR
10096 }
10098 probes at FIRST + N * PROBE_INTERVAL for values of N from 1
10099 until it is equal to ROUNDED_SIZE. */
10104 /* Step 4: probe at FIRST + SIZE if we cannot assert at compile-time
10105 that SIZE is equal to ROUNDED_SIZE. */
10110 stack_pointer_rtx,
10115 }
10117 /* Make sure nothing is scheduled before we are done. */
10119 }
10121 /* Probe a range of stack addresses from REG to END, inclusive. These are
10122 offsets from the current stack pointer. */
10126 {
10136 /* Jump to END_LAB if TEST_ADDR == LAST_ADDR. */
10144 /* TEST_ADDR = TEST_ADDR + PROBE_INTERVAL. */
10148 /* Probe at TEST_ADDR. */
10161 }
10163 /* Finalize stack_realign_needed flag, which will guide prologue/epilogue
10164 to be generated in correct form. */
10165 static void
10167 {
10168 /* Check if stack realign is really needed after reload, and
10169 stores result in cfun */
10170 unsigned int incoming_stack_boundary
10179 {
10180 /* After stack_realign_needed is finalized, we can't no longer
10181 change it. */
10184 }
10186 /* If the only reason for frame_pointer_needed is that we conservatively
10187 assumed stack realignment might be needed, but in the end nothing that
10188 needed the stack alignment had been spilled, clear frame_pointer_needed
10189 and say we don't need stack realignment. */
10192 && frame_pointer_needed
10194 && flag_omit_frame_pointer
10196 && !ix86_current_function_calls_tls_descriptor
10205 {
10207 basic_block bb;
10214 HARD_FRAME_POINTER_REGNUM);
10216 {
10217 rtx insn;
10221 set_up_by_prologue))
10222 {
10226 }
10227 }
10241 }
10245 }
10247 /* Expand the prologue into a bunch of separate insns. */
10249 void
10251 {
10256 HOST_WIDE_INT allocate;
10262 /* DRAP should not coexist with stack_realign_fp */
10267 /* Initialize CFA state for before the prologue. */
10271 /* Track SP offset to the CFA. We continue tracking this after we've
10272 swapped the CFA register away from SP. In the case of re-alignment
10273 this is fudged; we're interested to offsets within the local frame. */
10280 {
10281 /* We should have already generated an error for any use of
10282 ms_hook on a nested function. */
10285 /* Check if profiling is active and we shall use profiling before
10286 prologue variant. If so sorry. */
10291 /* In ix86_asm_output_function_label we emitted:
10292 8b ff movl.s %edi,%edi
10293 55 push %ebp
10294 8b ec movl.s %esp,%ebp
10296 This matches the hookable function prologue in Win32 API
10297 functions in Microsoft Windows XP Service Pack 2 and newer.
10298 Wine uses this to enable Windows apps to hook the Win32 API
10299 functions provided by Wine.
10301 What that means is that we've already set up the frame pointer. */
10305 {
10308 /* We've decided to use the frame pointer already set up.
10309 Describe this to the unwinder by pretending that both
10310 push and mov insns happen right here.
10312 Putting the unwind info here at the end of the ms_hook
10313 is done so that we can make absolutely certain we get
10314 the required byte sequence at the start of the function,
10315 rather than relying on an assembler that can produce
10316 the exact encoding required.
10318 However it does mean (in the unpatched case) that we have
10319 a 1 insn window where the asynchronous unwind info is
10320 incorrect. However, if we placed the unwind info at
10321 its correct location we would have incorrect unwind info
10322 in the patched case. Which is probably all moot since
10323 I don't expect Wine generates dwarf2 unwind info for the
10324 system libraries that use this feature. */
10330 stack_pointer_rtx);
10338 /* Note that gen_push incremented m->fs.cfa_offset, even
10339 though we didn't emit the push insn here. */
10343 }
10344 else
10345 {
10346 /* The frame pointer is not needed so pop %ebp again.
10347 This leaves us with a pristine state. */
10349 }
10350 }
10352 /* The first insn of a function that accepts its static chain on the
10353 stack is to push the register that would be filled in by a direct
10354 call. This insn will be skipped by the trampoline. */
10356 {
10360 /* We don't want to interpret this push insn as a register save,
10361 only as a stack adjustment. The real copy of the register as
10362 a save will be done later, if needed. */
10367 }
10369 /* Emit prologue code to adjust stack alignment and setup DRAP, in case
10370 of DRAP is needed and stack realignment is really needed after reload */
10372 {
10375 /* Only need to push parameter pointer reg if it is caller saved. */
10377 {
10378 /* Push arg pointer reg */
10381 }
10383 /* Grab the argument pointer. */
10390 /* Align the stack. */
10392 stack_pointer_rtx,
10396 /* Replicate the return address on the stack so that return
10397 address can be reached via (argp - 1) slot. This is needed
10398 to implement macro RETURN_ADDR_RTX and intrinsic function
10399 expand_builtin_return_addr etc. */
10405 /* For the purposes of frame and register save area addressing,
10406 we've started over with a new frame. */
10409 }
10415 {
10416 /* Note: AT&T enter does NOT have reversed args. Enter is probably
10417 slower on all targets. Also sdb doesn't like it. */
10421 /* Push registers now, before setting the frame pointer
10422 on SEH target. */
10424 && TARGET_SEH
10426 {
10430 }
10433 {
10441 }
10442 }
10445 {
10446 /* If saving registers via PUSH, do so now. */
10448 {
10452 }
10454 /* When using red zone we may start register saving before allocating
10455 the stack frame saving one cycle of the prologue. However, avoid
10456 doing this if we have to probe the stack; at least on x86_64 the
10457 stack probe can turn into a call that clobbers a red zone location. */
10459 && (! TARGET_STACK_PROBE
10461 {
10464 }
10465 }
10468 {
10472 /* The computation of the size of the re-aligned stack frame means
10473 that we must allocate the size of the register save area before
10474 performing the actual alignment. Otherwise we cannot guarantee
10475 that there's enough storage above the realignment point. */
10482 /* Align the stack. */
10484 stack_pointer_rtx,
10487 /* For the purposes of register save area addressing, the stack
10488 pointer is no longer valid. As for the value of sp_offset,
10489 see ix86_compute_frame_layout, which we need to match in order
10490 to pass verification of stack_pointer_offset at the end. */
10493 }
10498 {
10499 /* We start to count from ARG_POINTER. */
10502 /* If it was realigned, take into account the fake frame. */
10504 {
10511 /* This over-estimates by 1 minimal-stack-alignment-unit but
10512 mitigates that by counting in the new return address slot. */
10513 current_function_dynamic_stack_size
10515 }
10518 }
10520 /* On SEH target with very large frame size, allocate an area to save
10521 SSE registers (as the very large allocation won't be described). */
10525 {
10526 HOST_WIDE_INT sse_size =
10531 /* No need to do stack checking as the area will be immediately
10532 written. */
10539 }
10541 /* The stack has already been decremented by the instruction calling us
10542 so probe if the size is non-negative to preserve the protection area. */
10544 {
10545 /* We expect the registers to be saved when probes are used. */
10549 {
10552 }
10553 else
10554 {
10562 else
10564 }
10565 }
10568 ;
10571 {
10575 }
10576 else
10577 {
10591 {
10594 }
10596 {
10600 }
10605 /* Use the fact that AX still contains ALLOCATE. */
10607 ? gen_pro_epilogue_adjust_stack_di_sub
10613 /* Note that SEH directives need to continue tracking the stack
10614 pointer even after the frame pointer has been set up. */
10616 {
10625 }
10629 {
10636 }
10638 {
10643 }
10644 }
10647 /* If we havn't already set up the frame pointer, do so now. */
10649 {
10661 }
10672 {
10679 }
10682 {
10684 {
10686 {
10696 label));
10700 }
10701 else
10703 }
10704 else
10705 {
10709 }
10710 }
10712 /* In the pic_reg_used case, make sure that the got load isn't deleted
10713 when mcount needs it. Blockage to avoid call movement across mcount
10714 call is emitted in generic code after the NOTE_INSN_PROLOGUE_END
10715 note. */
10720 {
10721 /* vDRAP is setup but after reload it turns out stack realign
10722 isn't necessary, here we will emit prologue to setup DRAP
10723 without stack realign adjustment */
10726 }
10728 /* Prevent instructions from being scheduled into register save push
10729 sequence when access to the redzone area is done through frame pointer.
10730 The offset between the frame pointer and the stack pointer is calculated
10731 relative to the value of the stack pointer at the end of the function
10732 prologue, and moving instructions that access redzone area via frame
10733 pointer inside push sequence violates this assumption. */
10737 /* Emit cld instruction if stringops are used in the function. */
10741 /* SEH requires that the prologue end within 256 bytes of the start of
10742 the function. Prevent instruction schedules that would extend that.
10743 Further, prevent alloca modifications to the stack pointer from being
10744 combined with prologue modifications. */
10747 }
10749 /* Emit code to restore REG using a POP insn. */
10751 static void
10753 {
10762 {
10763 /* Previously we'd represented the CFA as an expression
10764 like *(%ebp - 8). We've just popped that value from
10765 the stack, which means we need to reset the CFA to
10766 the drap register. This will remain until we restore
10767 the stack pointer. */
10771 /* This means that the DRAP register is valid for addressing too. */
10774 }
10777 {
10784 }
10786 /* When the frame pointer is the CFA, and we pop it, we are
10787 swapping back to the stack pointer as the CFA. This happens
10788 for stack frames that don't allocate other data, so we assume
10789 the stack pointer is now pointing at the return address, i.e.
10790 the function entry state, which makes the offset be 1 word. */
10792 {
10795 {
10803 }
10804 }
10805 }
10807 /* Emit code to restore saved registers using POP insns. */
10809 static void
10811 {
10817 }
10819 /* Emit code and notes for the LEAVE instruction. */
10821 static void
10823 {
10835 {
10843 }
10846 }
10848 /* Emit code to restore saved registers using MOV insns.
10849 First register is restored from CFA - CFA_OFFSET. */
10850 static void
10853 {
10859 {
10868 {
10869 /* Previously we'd represented the CFA as an expression
10870 like *(%ebp - 8). We've just popped that value from
10871 the stack, which means we need to reset the CFA to
10872 the drap register. This will remain until we restore
10873 the stack pointer. */
10877 /* This means that the DRAP register is valid for addressing. */
10879 }
10880 else
10884 }
10885 }
10887 /* Emit code to restore saved registers using MOV insns.
10888 First register is restored from CFA - CFA_OFFSET. */
10889 static void
10892 {
10897 {
10899 rtx mem;
10909 }
10910 }
10912 /* Emit vzeroupper if needed. */
10914 void
10916 {
10921 }
10923 /* Restore function stack, frame, and registers. */
10925 void
10927 {
10943 /* The FP must be valid if the frame pointer is present. */
10948 /* We must have *some* valid pointer to the stack frame. */
10951 /* The DRAP is never valid at this point. */
10954 /* See the comment about red zone and frame
10955 pointer usage in ix86_expand_prologue. */
10962 /* Determine the CFA offset of the end of the red-zone. */
10965 {
10966 /* The red-zone begins below the return address. */
10969 /* When the register save area is in the aligned portion of
10970 the stack, determine the maximum runtime displacement that
10971 matches up with the aligned frame. */
10975 }
10977 /* Special care must be taken for the normal return case of a function
10978 using eh_return: the eax and edx registers are marked as saved, but
10979 not restored along this path. Adjust the save location to match. */
10983 /* EH_RETURN requires the use of moves to function properly. */
10986 /* SEH requires the use of pops to identify the epilogue. */
10989 /* If we're only restoring one register and sp is not valid then
10990 using a move instruction to restore the register since it's
10991 less work than reloading sp and popping the register. */
11004 && TARGET_USE_LEAVE
11008 else
11012 {
11013 /* Ensure that the entire register save area is addressable via
11014 the stack pointer, if we will restore via sp. */
11019 {
11023 style,
11025 }
11026 }
11028 /* If there are any SSE registers to restore, then we have to do it
11029 via moves, since there's obviously no pop for SSE regs. */
11035 {
11036 rtx t;
11041 /* eh_return epilogues need %ecx added to the stack pointer. */
11043 {
11046 /* Stack align doesn't work with eh_return. */
11048 /* Neither does regparm nested functions. */
11052 {
11060 /* Note that we use SA as a temporary CFA, as the return
11061 address is at the proper place relative to it. We
11062 pretend this happens at the FP restore insn because
11063 prior to this insn the FP would be stored at the wrong
11064 offset relative to SA, and after this insn we have no
11065 other reasonable register to use for the CFA. We don't
11066 bother resetting the CFA to the SP for the duration of
11067 the return insn. */
11080 }
11081 else
11082 {
11090 {
11094 UNITS_PER_WORD));
11096 }
11097 }
11100 }
11101 }
11102 else
11103 {
11104 /* SEH requires that the function end with (1) a stack adjustment
11105 if necessary, (2) a sequence of pops, and (3) a return or
11106 jump instruction. Prevent insns from the function body from
11107 being scheduled into this sequence. */
11109 {
11110 /* Prevent a catch region from being adjacent to the standard
11111 epilogue sequence. Unfortuantely crtl->uses_eh_lsda nor
11112 several other flags that would be interesting to test are
11113 not yet set up. */
11116 else
11118 }
11120 /* First step is to deallocate the stack frame so that we can
11121 pop the registers. Also do it on SEH target for very large
11122 frame as the emitted instructions aren't allowed by the ABI in
11123 epilogues. */
11125 || (TARGET_SEH
11128 {
11133 }
11135 {
11139 style,
11141 }
11144 }
11146 /* If we used a stack pointer and haven't already got rid of it,
11147 then do so now. */
11149 {
11150 /* If the stack pointer is valid and pointing at the frame
11151 pointer store address, then we only need a pop. */
11154 /* Leave results in shorter dependency chains on CPUs that are
11155 able to grok it fast. */
11160 else
11161 {
11163 hard_frame_pointer_rtx,
11166 }
11167 }
11170 {
11172 rtx insn;
11198 }
11200 /* At this point the stack pointer must be valid, and we must have
11201 restored all of the registers. We may not have deallocated the
11202 entire stack frame. We've delayed this until now because it may
11203 be possible to merge the local stack deallocation with the
11204 deallocation forced by ix86_static_chain_on_stack. */
11209 {
11213 }
11214 else
11217 /* Sibcall epilogues don't want a return instruction. */
11219 {
11222 }
11224 /* Emit vzeroupper if needed. */
11228 {
11231 /* i386 can only pop 64K bytes. If asked to pop more, pop return
11232 address, do explicit add, and jump indirectly to the caller. */
11235 {
11237 rtx insn;
11239 /* There is no "pascal" calling convention in any 64bit ABI. */
11255 }
11256 else
11258 }
11259 else
11262 /* Restore the state back to the state from the prologue,
11263 so that it's correct for the next epilogue. */
11265 }
11267 /* Reset from the function's potential modifications. */
11269 static void
11271 HOST_WIDE_INT size ATTRIBUTE_UNUSED)
11272 {
11275 #if TARGET_MACHO
11276 /* Mach-O doesn't support labels at the end of objects, so if
11277 it looks like we might want one, insert a NOP. */
11278 {
11284 {
11285 /* Don't insert a nop for NOTE_INSN_DELETED_DEBUG_LABEL
11286 notes only, instead set their CODE_LABEL_NUMBER to -1,
11287 otherwise there would be code generation differences
11288 in between -g and -g0. */
11292 }
11302 }
11303 #endif
11305 }
11307 /* Return a scratch register to use in the split stack prologue. The
11308 split stack prologue is used for -fsplit-stack. It is the first
11309 instructions in the function, even before the regular prologue.
11310 The scratch register can be any caller-saved register which is not
11311 used for parameters or for the static chain. */
11313 static unsigned int
11315 {
11318 else
11319 {
11329 {
11331 {
11335 }
11337 }
11339 {
11342 else
11343 {
11345 {
11349 }
11351 }
11352 }
11353 else
11354 {
11355 /* FIXME: We could make this work by pushing a register
11356 around the addition and comparison. */
11359 }
11360 }
11361 }
11363 /* A SYMBOL_REF for the function which allocates new stackspace for
11364 -fsplit-stack. */
11368 /* A SYMBOL_REF for the more stack function when using the large
11369 model. */
11373 /* Handle -fsplit-stack. These are the first instructions in the
11374 function, even before the regular prologue. */
11376 void
11378 {
11380 HOST_WIDE_INT allocate;
11385 rtx fn;
11393 /* This is the label we will branch to if we have enough stack
11394 space. We expect the basic block reordering pass to reverse this
11395 branch if optimizing, so that we branch in the unlikely case. */
11398 /* We need to compare the stack pointer minus the frame size with
11399 the stack boundary in the TCB. The stack boundary always gives
11400 us SPLIT_STACK_AVAILABLE bytes, so if we need less than that we
11401 can compare directly. Otherwise we need to do an addition. */
11404 UNSPEC_STACK_CHECK);
11409 else
11410 {
11412 rtx offset;
11414 /* We need a scratch register to hold the stack pointer minus
11415 the required frame size. Since this is the very start of the
11416 function, the scratch register can be any caller-saved
11417 register which is not used for parameters. */
11424 {
11425 /* We don't use ix86_gen_add3 in this case because it will
11426 want to split to lea, but when not optimizing the insn
11427 will not be split after this point. */
11430 offset)));
11431 }
11432 else
11433 {
11436 stack_pointer_rtx));
11437 }
11439 }
11445 /* Mark the jump as very likely to be taken. */
11453 /* Get more stack space. We pass in the desired stack space and the
11454 size of the arguments to copy to the new stack. In 32-bit mode
11455 we push the parameters; __morestack will return on a new stack
11456 anyhow. In 64-bit mode we pass the parameters in r10 and
11457 r11. */
11462 {
11468 /* If this function uses a static chain, it will be in %r10.
11469 Preserve it across the call to __morestack. */
11471 {
11472 rtx rax;
11477 }
11480 {
11481 HOST_WIDE_INT argval;
11484 /* When using the large model we need to load the address
11485 into a register, and we've run out of registers. So we
11486 switch to a different calling convention, and we call a
11487 different function: __morestack_large. We pass the
11488 argument size in the upper 32 bits of r10 and pass the
11489 frame size in the lower 32 bits. */
11494 split_stack_fn_large =
11498 {
11508 UNSPEC_GOT);
11514 }
11515 else
11522 }
11523 else
11524 {
11528 }
11531 }
11532 else
11533 {
11536 }
11542 /* In order to make call/return prediction work right, we now need
11543 to execute a return instruction. See
11544 libgcc/config/i386/morestack.S for the details on how this works.
11546 For flow purposes gcc must not see this as a return
11547 instruction--we need control flow to continue at the subsequent
11548 label. Therefore, we use an unspec. */
11552 /* If we are in 64-bit mode and this function uses a static chain,
11553 we saved %r10 in %rax before calling _morestack. */
11558 /* If this function calls va_start, we need to store a pointer to
11559 the arguments on the old stack, because they may not have been
11560 all copied to the new stack. At this point the old stack can be
11561 found at the frame pointer value used by __morestack, because
11562 __morestack has set that up before calling back to us. Here we
11563 store that pointer in a scratch register, and in
11564 ix86_expand_prologue we store the scratch register in a stack
11565 slot. */
11567 {
11569 rtx frame_reg;
11576 /* 64-bit:
11577 fp -> old fp value
11578 return address within this function
11579 return address of caller of this function
11580 stack arguments
11581 So we add three words to get to the stack arguments.
11583 32-bit:
11584 fp -> old fp value
11585 return address within this function
11586 first argument to __morestack
11587 second argument to __morestack
11588 return address of caller of this function
11589 stack arguments
11590 So we add five words to get to the stack arguments.
11591 */
11602 }
11607 /* If this function calls va_start, we now have to set the scratch
11608 register for the case where we do not call __morestack. In this
11609 case we need to set it based on the stack pointer. */
11611 {
11618 }
11619 }
11621 /* We may have to tell the dataflow pass that the split stack prologue
11622 is initializing a scratch register. */
11624 static void
11626 {
11628 {
11631 }
11632 }
11633 \f
11634 /* Determine if op is suitable SUBREG RTX for address. */
11636 static bool
11638 {
11649 /* Don't allow SUBREGs that span more than a word. It can lead to spill
11650 failures when the register is one word out of a two word structure. */
11654 /* Allow only SUBREGs of non-eliminable hard registers. */
11656 }
11658 /* Extract the parts of an RTL expression that is a valid memory address
11659 for an instruction. Return 0 if the structure of the address is
11660 grossly off. Return -1 if the address contains ASHIFT, so it is not
11661 strictly valid, but still used for computing length of lea instruction. */
11663 int
11665 {
11670 rtx tmp;
11674 /* Allow zero-extended SImode addresses,
11675 they will be emitted with addr32 prefix. */
11677 {
11680 {
11684 }
11687 {
11690 /* Adjust SUBREGs. */
11693 {
11697 }
11702 }
11703 }
11705 /* Allow SImode subregs of DImode addresses,
11706 they will be emitted with addr32 prefix. */
11708 {
11711 {
11715 }
11716 }
11721 {
11724 else
11726 }
11728 {
11733 do
11734 {
11739 }
11746 {
11749 {
11774 /* FALLTHRU */
11778 && TARGET_TLS_DIRECT_SEG_REFS
11781 else
11788 /* FALLTHRU */
11795 else
11810 }
11811 }
11812 }
11814 {
11817 }
11819 {
11820 /* We're called for lea too, which implements ashift on occasion. */
11830 }
11832 {
11836 /* Constant addresses are sign extended to 64bit, we have to
11837 prevent addresses from 0x80000000 to 0xffffffff in x32 mode. */
11843 }
11844 else
11848 {
11850 ;
11853 ;
11854 else
11856 }
11858 /* Address override works only on the (%reg) part of %fs:(%reg). */
11864 /* Extract the integral value of scale. */
11866 {
11870 }
11875 /* Avoid useless 0 displacement. */
11879 /* Allow arg pointer and stack pointer as index if there is not scaling. */
11884 {
11885 rtx tmp;
11888 }
11890 /* Special case: %ebp cannot be encoded as a base without a displacement.
11891 Similarly %r13. */
11893 && base_reg
11902 /* Special case: on K6, [%esi] makes the instruction vector decoded.
11903 Avoid this by transforming to [%esi+0].
11904 Reload calls address legitimization without cfun defined, so we need
11905 to test cfun for being non-NULL. */
11911 /* Special case: encode reg+reg instead of reg*2. */
11915 /* Special case: scaling cannot be encoded without base or displacement. */
11926 }
11927 \f
11928 /* Return cost of the memory address x.
11929 For i386, it is better to use a complex address than let gcc copy
11930 the address into a reg and make a new pseudo. But not if the address
11931 requires to two regs - that would mean more pseudos with longer
11932 lifetimes. */
11933 static int
11935 {
11947 /* Attempt to minimize number of registers in the address. */
11953 cost++;
11960 cost++;
11962 /* AMD-K6 don't like addresses with ModR/M set to 00_xxx_100b,
11963 since it's predecode logic can't detect the length of instructions
11964 and it degenerates to vector decoded. Increase cost of such
11965 addresses here. The penalty is minimally 2 cycles. It may be worthwhile
11966 to split such addresses or even refuse such addresses at all.
11968 Following addressing modes are affected:
11969 [base+scale*index]
11970 [scale*index+disp]
11971 [base+index]
11973 The first and last case may be avoidable by explicitly coding the zero in
11974 memory address, but I don't have AMD-K6 machine handy to check this
11975 theory. */
11984 }
11985 \f
11986 /* Allow {LABEL | SYMBOL}_REF - SYMBOL_REF-FOR-PICBASE for Mach-O as
11987 this is used for to form addresses to local data when -fPIC is in
11988 use. */
11990 static bool
11992 {
11995 }
11997 /* Determine if a given RTX is a valid constant. We already know this
11998 satisfies CONSTANT_P. */
12000 static bool
12002 {
12004 {
12009 {
12013 }
12018 /* Only some unspecs are valid as "constants". */
12021 {
12037 }
12039 /* We must have drilled down to a symbol. */
12044 /* FALLTHRU */
12047 /* TLS symbols are never valid. */
12051 /* DLLIMPORT symbols are never valid. */
12056 #if TARGET_MACHO
12057 /* mdynamic-no-pic */
12060 #endif
12076 }
12078 /* Otherwise we handle everything else in the move patterns. */
12080 }
12082 /* Determine if it's legal to put X into the constant pool. This
12083 is not possible for the address of thread-local symbols, which
12084 is checked above. */
12086 static bool
12088 {
12089 /* We can always put integral constants and vectors in memory. */
12091 {
12099 }
12101 }
12104 /* Nonzero if the constant value X is a legitimate general operand
12105 when generating PIC code. It is given that flag_pic is on and
12106 that X satisfies CONSTANT_P or is a CONST_DOUBLE. */
12108 bool
12110 {
12111 rtx inner;
12114 {
12121 /* Only some unspecs are valid as "constants". */
12124 {
12137 }
12138 /* FALLTHRU */
12146 }
12147 }
12149 /* Determine if a given CONST RTX is a valid memory displacement
12150 in PIC mode. */
12152 bool
12154 {
12157 /* In 64bit mode we can allow direct addresses of symbols and labels
12158 when they are not dynamic symbols. */
12160 {
12164 {
12188 /* FALLTHRU */
12191 /* TLS references should always be enclosed in UNSPEC. */
12201 }
12202 }
12208 {
12209 /* We are unsafe to allow PLUS expressions. This limit allowed distance
12210 of GOT tables. We should not need these anyway. */
12222 }
12226 {
12231 }
12240 {
12244 /* We need to check for both symbols and labels because VxWorks loads
12245 text labels with @GOT rather than @GOTOFF. See gotoff_operand for
12246 details. */
12250 /* Refuse GOTOFF in 64bit mode since it is always 64bit when used.
12251 While ABI specify also 32bit relocation but we don't produce it in
12252 small PIC model at all. */
12274 }
12277 }
12279 /* Our implementation of LEGITIMIZE_RELOAD_ADDRESS. Returns a value to
12280 replace the input X, or the original X if no replacement is called for.
12281 The output parameter *WIN is 1 if the calling macro should goto WIN,
12282 0 if it should not. */
12284 bool
12289 {
12290 /* Reload can generate:
12292 (plus:DI (plus:DI (unspec:DI [(const_int 0 [0])] UNSPEC_TP)
12293 (reg:DI 97))
12294 (reg:DI 2 cx))
12296 This RTX is rejected from ix86_legitimate_address_p due to
12297 non-strictness of base register 97. Following this rejection,
12298 reload pushes all three components into separate registers,
12299 creating invalid memory address RTX.
12301 Following code reloads only the invalid part of the
12302 memory address RTX. */
12308 {
12314 {
12319 }
12323 {
12328 }
12332 }
12335 }
12337 /* Recognizes RTL expressions that are valid memory addresses for an
12338 instruction. The MODE argument is the machine mode for the MEM
12339 expression that wants to use this address.
12341 It only recognizes address in canonical form. LEGITIMIZE_ADDRESS should
12342 convert common non-canonical forms to canonical form so that they will
12343 be recognized. */
12345 static bool
12348 {
12351 HOST_WIDE_INT scale;
12354 /* Decomposition failed. */
12362 /* Validate base register. */
12364 {
12365 rtx reg;
12371 else
12372 /* Base is not a register. */
12380 /* Base is not valid. */
12382 }
12384 /* Validate index register. */
12386 {
12387 rtx reg;
12393 else
12394 /* Index is not a register. */
12402 /* Index is not valid. */
12404 }
12406 /* Index and base should have the same mode. */
12411 /* Validate scale factor. */
12413 {
12415 /* Scale without index. */
12419 /* Scale is not a valid multiplier. */
12421 }
12423 /* Validate displacement. */
12425 {
12430 {
12431 /* Refuse GOTOFF and GOT in 64bit mode since it is always 64bit when
12432 used. While ABI specify also 32bit relocations, we don't produce
12433 them at all and use IP relative instead. */
12440 /* 64bit address unspec. */
12460 /* Invalid address unspec. */
12462 }
12465 && (flag_pic
12466 || (TARGET_MACHO
12467 #if TARGET_MACHO
12468 && MACHOPIC_INDIRECT
12470 #endif
12471 )))
12472 {
12474 is_legitimate_pic:
12476 {
12477 /* foo@dtpoff(%rX) is ok. */
12484 /* Non-constant pic memory reference. */
12486 }
12489 /* Displacement is an invalid pic construct. */
12491 #if TARGET_MACHO
12494 /* displacment must be referenced via non_lazy_pointer */
12496 #endif
12498 /* This code used to verify that a symbolic pic displacement
12499 includes the pic_offset_table_rtx register.
12501 While this is good idea, unfortunately these constructs may
12502 be created by "adds using lea" optimization for incorrect
12503 code like:
12505 int a;
12506 int foo(int i)
12507 {
12508 return *(&a+i);
12509 }
12511 This code is nonsensical, but results in addressing
12512 GOT table with pic_offset_table_rtx base. We can't
12513 just refuse it easily, since it gets matched by
12514 "addsi3" pattern, that later gets split to lea in the
12515 case output register differs from input. While this
12516 can be handled by separate addsi pattern for this case
12517 that never results in lea, this seems to be easier and
12518 correct fix for crash to disable this test. */
12519 }
12526 /* Displacement is not constant. */
12530 /* Displacement is out of range. */
12532 }
12534 /* Everything looks valid. */
12536 }
12538 /* Determine if a given RTX is a valid constant address. */
12540 bool
12542 {
12544 }
12545 \f
12546 /* Return a unique alias set for the GOT. */
12548 static alias_set_type
12550 {
12555 }
12557 /* Return a legitimate reference for ORIG (an address) using the
12558 register REG. If REG is 0, a new pseudo is generated.
12560 There are two types of references that must be handled:
12562 1. Global data references must load the address from the GOT, via
12563 the PIC reg. An insn is emitted to do this load, and the reg is
12564 returned.
12566 2. Static data references, constant pool addresses, and code labels
12567 compute the address as an offset from the GOT, whose base is in
12568 the PIC reg. Static data objects have SYMBOL_FLAG_LOCAL set to
12569 differentiate them from global data objects. The returned
12570 address is the PIC reg + an unspec constant.
12572 TARGET_LEGITIMATE_ADDRESS_P rejects symbolic references unless the PIC
12573 reg also appears in the address. */
12575 static rtx
12577 {
12580 rtx base;
12582 #if TARGET_MACHO
12584 {
12587 /* Use the generic Mach-O PIC machinery. */
12589 }
12590 #endif
12597 {
12598 rtx tmpreg;
12599 /* This symbol may be referenced via a displacement from the PIC
12600 base address (@GOTOFF). */
12607 {
12609 UNSPEC_GOTOFF);
12611 }
12612 else
12617 else
12622 {
12626 }
12628 }
12630 {
12631 /* This symbol may be referenced via a displacement from the PIC
12632 base address (@GOTOFF). */
12639 {
12641 UNSPEC_GOTOFF);
12643 }
12644 else
12650 {
12653 }
12654 }
12656 /* We can't use @GOTOFF for text labels on VxWorks;
12657 see gotoff_operand. */
12659 {
12661 {
12667 {
12670 }
12671 }
12673 /* For x64 PE-COFF there is no GOT table. So we use address
12674 directly. */
12676 {
12684 }
12686 {
12694 /* Use directly gen_movsi, otherwise the address is loaded
12695 into register for CSE. We don't want to CSE this addresses,
12696 instead we CSE addresses from the GOT table, so skip this. */
12699 }
12700 else
12701 {
12702 /* This symbol must be referenced via a load from the
12703 Global Offset Table (@GOT). */
12719 }
12720 }
12721 else
12722 {
12725 {
12727 {
12730 }
12731 else
12733 }
12735 {
12738 /* We must match stuff we generate before. Assume the only
12739 unspecs that can get here are ours. Not that we could do
12740 anything with them anyway.... */
12746 }
12748 {
12751 /* Check first to see if this is a constant offset from a @GOTOFF
12752 symbol reference. */
12755 {
12757 {
12761 UNSPEC_GOTOFF);
12767 {
12770 }
12771 }
12772 else
12773 {
12776 {
12780 }
12781 }
12782 }
12783 else
12784 {
12791 else
12792 {
12794 {
12797 }
12799 }
12800 }
12801 }
12802 }
12804 }
12805 \f
12806 /* Load the thread pointer. If TO_REG is true, force it into a register. */
12808 static rtx
12810 {
12814 {
12819 }
12825 }
12827 /* Construct the SYMBOL_REF for the tls_get_addr function. */
12831 static rtx
12833 {
12835 {
12841 }
12844 }
12846 /* Construct the SYMBOL_REF for the _TLS_MODULE_BASE_ symbol. */
12850 rtx
12852 {
12854 {
12855 ix86_tls_module_base_symbol
12860 }
12863 }
12865 /* A subroutine of ix86_legitimize_address and ix86_expand_move. FOR_MOV is
12866 false if we expect this to be used for a memory address and true if
12867 we expect to load the address into a register. */
12869 static rtx
12871 {
12878 {
12883 {
12886 else
12887 {
12890 }
12891 }
12894 {
12897 else
12904 }
12905 else
12906 {
12910 {
12915 caddr));
12921 }
12922 else
12924 }
12931 {
12934 else
12935 {
12938 }
12939 }
12942 {
12947 else
12953 }
12954 else
12955 {
12959 {
12964 caddr));
12968 /* Attach a unique REG_EQUAL, to allow the RTL optimizers to
12969 share the LD_BASE result with other LD model accesses. */
12971 UNSPEC_TLS_LD_BASE);
12975 }
12976 else
12978 }
12986 {
12990 }
12995 {
12997 {
12998 /* The Sun linker took the AMD64 TLS spec literally
12999 and can only handle %rax as destination of the
13000 initial executable code sequence. */
13005 }
13007 /* Generate DImode references to avoid %fs:(%reg32)
13008 problems and linker IE->LE relaxation bug. */
13012 }
13014 {
13019 }
13021 {
13025 }
13026 else
13027 {
13030 }
13040 {
13045 }
13046 else
13047 {
13051 }
13061 {
13065 }
13066 else
13067 {
13071 }
13076 }
13079 }
13081 /* Create or return the unique __imp_DECL dllimport symbol corresponding
13082 to symbol DECL. */
13085 htab_t dllimport_map;
13087 static tree
13089 {
13096 tree to;
13097 rtx rtl;
13141 }
13143 /* Expand SYMBOL into its corresponding dllimport symbol. WANT_REG is
13144 true if we require the result be a register. */
13146 static rtx
13148 {
13149 tree imp_decl;
13150 rtx x;
13159 }
13161 /* Try machine-dependent ways of modifying an illegitimate address
13162 to be legitimate. If we find one, return the new, valid address.
13163 This macro is used in only one place: `memory_address' in explow.c.
13165 OLDX is the address as it was before break_out_memory_refs was called.
13166 In some cases it is useful to look at this to decide what needs to be done.
13168 It is always safe for this macro to do nothing. It exists to recognize
13169 opportunities to optimize the output.
13171 For the 80386, we handle X+REG by loading X into a register R and
13172 using R+REG. R will go in a general reg and indexing will be used.
13173 However, if REG is a broken-out memory address or multiplication,
13174 nothing needs to be done because REG can certainly go in a general reg.
13176 When -fpic is used, special handling is needed for symbolic references.
13177 See comments by legitimize_pic_address in i386.c for details. */
13179 static rtx
13182 {
13193 {
13197 }
13200 {
13207 {
13210 }
13211 }
13216 #if TARGET_MACHO
13219 #endif
13221 /* Canonicalize shifts by 0, 1, 2, 3 into multiply */
13225 {
13230 }
13233 {
13234 /* Canonicalize shifts by 0, 1, 2, 3 into multiply. */
13239 {
13245 }
13250 {
13256 }
13258 /* Put multiply first if it isn't already. */
13260 {
13265 }
13267 /* Canonicalize (plus (mult (reg) (const)) (plus (reg) (const)))
13268 into (plus (plus (mult (reg) (const)) (reg)) (const)). This can be
13269 created by virtual register instantiation, register elimination, and
13270 similar optimizations. */
13272 {
13278 }
13280 /* Canonicalize
13281 (plus (plus (mult (reg) (const)) (plus (reg) (const))) const)
13282 into (plus (plus (mult (reg) (const)) (reg)) (const)). */
13287 {
13288 rtx constant;
13292 {
13295 }
13297 {
13300 }
13301 else
13305 {
13312 }
13313 }
13319 {
13322 }
13325 {
13328 }
13336 {
13339 }
13345 {
13349 {
13353 }
13357 }
13360 {
13364 {
13368 }
13372 }
13373 }
13376 }
13377 \f
13378 /* Print an integer constant expression in assembler syntax. Addition
13379 and subtraction are the only arithmetic that may appear in these
13380 expressions. FILE is the stdio stream to write to, X is the rtx, and
13381 CODE is the operand print code from the output string. */
13383 static void
13385 {
13389 {
13398 else
13399 {
13402 /* Mark the decl as referenced so that cgraph will
13403 output the function. */
13407 #if TARGET_MACHO
13411 #endif
13413 }
13421 /* FALLTHRU */
13432 /* This used to output parentheses around the expression,
13433 but that does not work on the 386 (either ATT or BSD assembler). */
13439 {
13440 /* We can use %d if the number is <32 bits and positive. */
13445 else
13447 }
13448 else
13449 /* We can't handle floating point constants;
13450 TARGET_PRINT_OPERAND must handle them. */
13455 /* Some assemblers need integer constants to appear first. */
13457 {
13461 }
13462 else
13463 {
13468 }
13483 {
13487 }
13492 {
13511 /* FIXME: This might be @TPOFF in Sun ld too. */
13520 else
13530 else
13536 #if TARGET_MACHO
13541 #endif
13545 }
13550 }
13551 }
13553 /* This is called from dwarf2out.c via TARGET_ASM_OUTPUT_DWARF_DTPREL.
13554 We need to emit DTP-relative relocations. */
13556 static void ATTRIBUTE_UNUSED
13558 {
13563 {
13571 }
13572 }
13574 /* Return true if X is a representation of the PIC register. This copes
13575 with calls from ix86_find_base_term, where the register might have
13576 been replaced by a cselib value. */
13578 static bool
13580 {
13584 else
13586 }
13588 /* Helper function for ix86_delegitimize_address.
13589 Attempt to delegitimize TLS local-exec accesses. */
13591 static rtx
13593 {
13618 {
13623 }
13629 }
13631 /* In the name of slightly smaller debug output, and to cater to
13632 general assembler lossage, recognize PIC+GOTOFF and turn it back
13633 into a direct symbol reference.
13635 On Darwin, this is necessary to avoid a crash, because Darwin
13636 has a different PIC label for each routine but the DWARF debugging
13637 information is not associated with any particular routine, so it's
13638 necessary to remove references to the PIC label from RTL stored by
13639 the DWARF output code. */
13641 static rtx
13643 {
13645 /* addend is NULL or some rtx if x is something+GOTOFF where
13646 something doesn't include the PIC register. */
13648 /* reg_addend is NULL or a multiple of some register. */
13650 /* const_addend is NULL or a const_int. */
13652 /* This is the result, or NULL. */
13661 {
13668 {
13674 }
13683 {
13688 }
13690 }
13697 /* %ebx + GOT/GOTOFF */
13698 ;
13700 {
13701 /* %ebx + %reg * scale + GOT/GOTOFF */
13707 else
13708 {
13711 }
13712 }
13713 else
13719 {
13722 }
13741 {
13742 /* If the rest of original X doesn't involve the PIC register, add
13743 addend and subtract pic_offset_table_rtx. This can happen e.g.
13744 for code like:
13745 leal (%ebx, %ecx, 4), %ecx
13746 ...
13747 movl foo@GOTOFF(%ecx), %edx
13748 in which case we return (%ecx - %ebx) + foo. */
13751 pic_offset_table_rtx),
13752 result);
13753 else
13755 }
13757 {
13761 }
13763 }
13765 /* If X is a machine specific address (i.e. a symbol or label being
13766 referenced as a displacement from the GOT implemented using an
13767 UNSPEC), then return the base term. Otherwise return X. */
13769 rtx
13771 {
13772 rtx term;
13775 {
13789 }
13792 }
13793 \f
13794 static void
13797 {
13801 {
13804 }
13809 {
13812 {
13831 }
13835 {
13854 }
13861 /* ??? Use "nbe" instead of "a" for fcmov lossage on some assemblers.
13862 Those same assemblers have the same but opposite lossage on cmov. */
13867 else
13872 {
13885 }
13893 {
13906 }
13909 /* ??? As above. */
13918 /* ??? As above. */
13923 else
13934 }
13936 }
13938 /* Print the name of register X to FILE based on its machine mode and number.
13939 If CODE is 'w', pretend the mode is HImode.
13940 If CODE is 'b', pretend the mode is QImode.
13941 If CODE is 'k', pretend the mode is SImode.
13942 If CODE is 'q', pretend the mode is DImode.
13943 If CODE is 'x', pretend the mode is V4SFmode.
13944 If CODE is 't', pretend the mode is V8SFmode.
13945 If CODE is 'h', pretend the reg is the 'high' byte register.
13946 If CODE is 'y', print "st(0)" instead of "st", if the reg is stack op.
13947 If CODE is 'd', duplicate the operand for AVX instruction.
13948 */
13950 void
13952 {
13967 {
13971 }
13989 else
13992 /* Irritatingly, AMD extended registers use different naming convention
13993 from the normal registers: "r%d[bwd]" */
13995 {
14000 {
14014 /* no suffix */
14019 }
14021 }
14025 {
14028 {
14031 }
14032 /* FALLTHRU */
14038 /* FALLTHRU */
14041 normal:
14056 {
14061 }
14065 }
14069 {
14072 else
14074 }
14075 }
14077 /* Locate some local-dynamic symbol still in use by this function
14078 so that we can print its name in some tls_local_dynamic_base
14079 pattern. */
14081 static int
14083 {
14088 {
14091 }
14094 }
14098 {
14099 rtx insn;
14110 }
14112 /* Meaning of CODE:
14113 L,W,B,Q,S,T -- print the opcode suffix for specified size of operand.
14114 C -- print opcode suffix for set/cmov insn.
14115 c -- like C, but print reversed condition
14116 F,f -- likewise, but for floating-point.
14117 O -- if HAVE_AS_IX86_CMOV_SUN_SYNTAX, expand to "w.", "l." or "q.",
14118 otherwise nothing
14119 R -- print the prefix for register names.
14120 z -- print the opcode suffix for the size of the current operand.
14121 Z -- likewise, with special suffixes for x87 instructions.
14122 * -- print a star (in certain assembler syntax)
14123 A -- print an absolute memory reference.
14124 E -- print address with DImode register names if TARGET_64BIT.
14125 w -- print the operand as if it's a "word" (HImode) even if it isn't.
14126 s -- print a shift double count, followed by the assemblers argument
14127 delimiter.
14128 b -- print the QImode name of the register for the indicated operand.
14129 %b0 would print %al if operands[0] is reg 0.
14130 w -- likewise, print the HImode name of the register.
14131 k -- likewise, print the SImode name of the register.
14132 q -- likewise, print the DImode name of the register.
14133 x -- likewise, print the V4SFmode name of the register.
14134 t -- likewise, print the V8SFmode name of the register.
14135 h -- print the QImode name for a "high" register, either ah, bh, ch or dh.
14136 y -- print "st(0)" instead of "st" as a register.
14137 d -- print duplicated register operand for AVX instruction.
14138 D -- print condition for SSE cmp instruction.
14139 P -- if PIC, print an @PLT suffix.
14140 p -- print raw symbol name.
14141 X -- don't print any sort of PIC '@' suffix for a symbol.
14142 & -- print some in-use local-dynamic symbol name.
14143 H -- print a memory address offset by 8; used for sse high-parts
14144 Y -- print condition for XOP pcom* instruction.
14145 + -- print a branch hint as 'cs' or 'ds' prefix
14146 ; -- print a semicolon (after prefixes due to bug in older gas).
14147 ~ -- print "i" if TARGET_AVX2, "f" otherwise.
14148 @ -- print a segment register of thread base pointer load
14149 ^ -- print addr32 prefix if TARGET_64BIT and Pmode != word_mode
14150 */
14152 void
14154 {
14156 {
14158 {
14161 {
14167 /* Intel syntax. For absolute addresses, registers should not
14168 be surrounded by braces. */
14170 {
14175 }
14180 }
14186 /* Wrap address in an UNSPEC to declare special handling. */
14224 #ifdef HAVE_AS_IX86_CMOV_SUN_SYNTAX
14229 {
14243 output_operand_lossage
14246 }
14249 #endif
14254 {
14255 /* Opcodes don't get size suffixes if using Intel opcodes. */
14260 {
14278 output_operand_lossage
14281 }
14282 }
14285 warning
14287 /* FALLTHRU */
14290 /* 387 opcodes don't get size suffixes if using Intel opcodes. */
14295 {
14297 {
14299 #ifdef HAVE_AS_IX86_FILDS
14301 #endif
14309 #ifdef HAVE_AS_IX86_FILDQ
14311 #else
14313 #endif
14318 }
14319 }
14321 {
14322 /* 387 opcodes don't get size suffixes
14323 if the operands are registers. */
14328 {
14344 }
14345 }
14346 else
14347 {
14348 output_operand_lossage
14351 }
14353 output_operand_lossage
14373 {
14376 }
14381 {
14432 }
14436 /* Little bit of braindamage here. The SSE compare instructions
14437 does use completely different names for the comparisons that the
14438 fp conditional moves. */
14440 {
14443 {
14446 }
14452 {
14455 }
14461 {
14464 }
14473 {
14476 }
14482 {
14485 }
14491 {
14494 }
14505 }
14510 #ifdef HAVE_AS_IX86_CMOV_SUN_SYNTAX
14513 #endif
14518 {
14522 }
14526 file);
14531 {
14535 }
14536 /* It doesn't actually matter what mode we use here, as we're
14537 only going to use this for printing. */
14545 #ifdef HAVE_AS_IX86_HLE
14547 #else
14549 #endif
14551 #ifdef HAVE_AS_IX86_HLE
14553 #else
14555 #endif
14556 /* We do not want to print value of the operand. */
14565 {
14570 else
14573 }
14576 {
14577 rtx x;
14586 {
14591 {
14593 bool cputaken
14596 /* Emit hints only in the case default branch prediction
14597 heuristics would fail. */
14599 {
14600 /* We use 3e (DS) prefix for taken branches and
14601 2e (CS) prefix for not taken branches. */
14604 else
14606 }
14607 }
14608 }
14610 }
14613 #ifndef HAVE_AS_IX86_REP_LOCK_PREFIX
14615 #endif
14622 /* The kernel uses a different segment register for performance
14623 reasons; a system call would not have to trash the userspace
14624 segment register, which would be expensive. */
14627 else
14642 }
14643 }
14649 {
14650 /* No `byte ptr' prefix for call instructions or BLKmode operands. */
14653 {
14656 {
14665 else
14671 }
14673 /* Check for explicit size override (codes 'b', 'w', 'k',
14674 'q' and 'x') */
14688 }
14691 /* Avoid (%rip) for call operands. */
14697 else
14699 }
14702 {
14703 REAL_VALUE_TYPE r;
14711 /* Sign extend 32bit SFmode immediate to 8 bytes. */
14714 else
14716 }
14719 {
14720 REAL_VALUE_TYPE r;
14729 }
14731 /* These float cases don't actually occur as immediate operands. */
14733 {
14738 }
14740 else
14741 {
14742 /* We have patterns that allow zero sets of memory, for instance.
14743 In 64-bit mode, we should probably support all 8-byte vectors,
14744 since we can in fact encode that into an immediate. */
14746 {
14749 }
14752 {
14754 {
14757 }
14760 {
14763 else
14765 }
14766 }
14771 else
14773 }
14774 }
14776 static bool
14778 {
14781 }
14782 \f
14783 /* Print a memory operand whose address is ADDR. */
14785 static void
14787 {
14796 {
14803 }
14805 {
14809 }
14810 else
14816 {
14820 }
14823 {
14827 }
14835 {
14846 }
14848 /* Use one byte shorter RIP relative addressing for 64bit mode. */
14850 {
14862 }
14864 {
14865 /* Displacement only requires special attention. */
14868 {
14872 }
14875 else
14877 }
14878 else
14879 {
14880 /* Print SImode register names to force addr32 prefix. */
14882 {
14888 }
14891 {
14896 }
14899 {
14901 {
14906 else
14908 }
14914 {
14919 }
14921 }
14922 else
14923 {
14927 {
14928 /* Pull out the offset of a symbol; print any symbol itself. */
14932 {
14936 }
14944 else
14946 }
14950 {
14953 {
14957 }
14958 }
14961 else
14965 {
14970 }
14972 }
14973 }
14974 }
14976 /* Implementation of TARGET_ASM_OUTPUT_ADDR_CONST_EXTRA. */
14978 static bool
14980 {
14981 rtx op;
14988 {
14991 /* FIXME: This might be @TPOFF in Sun ld. */
15002 else
15014 else
15021 #if TARGET_MACHO
15027 #endif
15030 {
15035 #ifdef TARGET_THREAD_SPLIT_STACK_OFFSET
15037 #else
15039 #endif
15042 }
15047 }
15050 }
15051 \f
15052 /* Split one or more double-mode RTL references into pairs of half-mode
15053 references. The RTL can be REG, offsettable MEM, integer constant, or
15054 CONST_DOUBLE. "operands" is a pointer to an array of double-mode RTLs to
15055 split and "num" is its length. lo_half and hi_half are output arrays
15056 that parallel "operands". */
15058 void
15061 {
15066 {
15075 }
15080 {
15083 /* simplify_subreg refuse to split volatile memory addresses,
15084 but we still have to handle it. */
15086 {
15089 }
15090 else
15091 {
15098 }
15099 }
15100 }
15101 \f
15102 /* Output code to perform a 387 binary operation in INSN, one of PLUS,
15103 MINUS, MULT or DIV. OPERANDS are the insn operands, where operands[3]
15104 is the expression of the binary operation. The output may either be
15105 emitted here, or returned to the caller, like all output_* functions.
15107 There is no guarantee that the operands are the same mode, as they
15108 might be within FLOAT or FLOAT_EXTEND expressions. */
15110 #ifndef SYSV386_COMPAT
15111 /* Set to 1 for compatibility with brain-damaged assemblers. No-one
15112 wants to fix the assemblers because that causes incompatibility
15113 with gcc. No-one wants to fix gcc because that causes
15114 incompatibility with assemblers... You can use the option of
15115 -DSYSV386_COMPAT=0 if you recompile both gcc and gas this way. */
15116 #define SYSV386_COMPAT 1
15117 #endif
15121 {
15127 #ifdef ENABLE_CHECKING
15128 /* Even if we do not want to check the inputs, this documents input
15129 constraints. Which helps in understanding the following code. */
15139 else
15141 #endif
15144 {
15149 else
15158 else
15167 else
15176 else
15183 }
15186 {
15188 {
15192 else
15194 }
15195 else
15196 {
15200 else
15202 }
15204 }
15208 {
15212 {
15216 }
15218 /* know operands[0] == operands[1]. */
15221 {
15224 }
15227 {
15229 /* How is it that we are storing to a dead operand[2]?
15230 Well, presumably operands[1] is dead too. We can't
15231 store the result to st(0) as st(0) gets popped on this
15232 instruction. Instead store to operands[2] (which I
15233 think has to be st(1)). st(1) will be popped later.
15234 gcc <= 2.8.1 didn't have this check and generated
15235 assembly code that the Unixware assembler rejected. */
15237 else
15240 }
15244 else
15251 {
15254 }
15257 {
15260 }
15263 {
15264 #if SYSV386_COMPAT
15265 /* The SystemV/386 SVR3.2 assembler, and probably all AT&T
15266 derived assemblers, confusingly reverse the direction of
15267 the operation for fsub{r} and fdiv{r} when the
15268 destination register is not st(0). The Intel assembler
15269 doesn't have this brain damage. Read !SYSV386_COMPAT to
15270 figure out what the hardware really does. */
15273 else
15275 #else
15277 /* As above for fmul/fadd, we can't store to st(0). */
15279 else
15281 #endif
15283 }
15286 {
15287 #if SYSV386_COMPAT
15290 else
15292 #else
15295 else
15297 #endif
15299 }
15302 {
15305 else
15308 }
15310 {
15311 #if SYSV386_COMPAT
15313 #else
15315 #endif
15316 }
15317 else
15318 {
15319 #if SYSV386_COMPAT
15321 #else
15323 #endif
15324 }
15329 }
15333 }
15335 /* Return needed mode for entity in optimize_mode_switching pass. */
15337 int
15339 {
15342 /* The mode UNINITIALIZED is used to store control word after a
15343 function call or ASM pattern. The mode ANY specify that function
15344 has no requirements on the control word and make no changes in the
15345 bits we are interested in. */
15359 {
15382 }
15385 }
15387 /* Output code to initialize control word copies used by trunc?f?i and
15388 rounding patterns. CURRENT_MODE is set to current control word,
15389 while NEW_MODE is set to new control word. */
15391 void
15393 {
15395 rtx new_mode;
15406 {
15408 {
15410 /* round toward zero (truncate) */
15416 /* round down toward -oo */
15423 /* round up toward +oo */
15430 /* mask precision exception for nearbyint() */
15437 }
15438 }
15439 else
15440 {
15442 {
15444 /* round toward zero (truncate) */
15450 /* round down toward -oo */
15456 /* round up toward +oo */
15462 /* mask precision exception for nearbyint() */
15469 }
15470 }
15476 }
15478 /* Output code for INSN to convert a float to a signed int. OPERANDS
15479 are the insn operands. The output may be [HSD]Imode and the input
15480 operand may be [SDX]Fmode. */
15484 {
15489 /* Jump through a hoop or two for DImode, since the hardware has no
15490 non-popping instruction. We used to do this a different way, but
15491 that was somewhat fragile and broke with post-reload splitters. */
15501 else
15502 {
15507 else
15511 }
15514 }
15516 /* Output code for x87 ffreep insn. The OPNO argument, which may only
15517 have the values zero or one, indicates the ffreep insn's operand
15518 from the OPERANDS array. */
15522 {
15524 #ifdef HAVE_AS_IX86_FFREEP
15526 #else
15527 {
15537 }
15538 #endif
15541 }
15544 /* Output code for INSN to compare OPERANDS. EFLAGS_P is 1 when fcomi
15545 should be used. UNORDERED_P is true when fucom should be used. */
15549 {
15555 {
15558 }
15559 else
15560 {
15563 }
15566 {
15570 else
15572 else
15575 else
15577 }
15584 {
15586 {
15589 }
15590 else
15592 }
15595 && stack_top_dies
15598 {
15599 /* If both the top of the 387 stack dies, and the other operand
15600 is also a stack register that dies, then this must be a
15601 `fcompp' float compare */
15604 {
15605 /* There is no double popping fcomi variant. Fortunately,
15606 eflags is immune from the fstp's cc clobbering. */
15609 else
15612 }
15613 else
15614 {
15617 else
15619 }
15620 }
15621 else
15622 {
15623 /* Encoded here as eflags_p | intmode | unordered_p | stack_top_dies. */
15626 {
15634 NULL,
15635 NULL,
15642 NULL,
15643 NULL,
15644 NULL,
15645 NULL
15646 };
15661 }
15662 }
15664 void
15666 {
15669 #ifdef ASM_QUAD
15672 #else
15674 #endif
15677 }
15679 void
15681 {
15684 #ifdef ASM_QUAD
15687 #else
15689 #endif
15690 /* We can't use @GOTOFF for text labels on VxWorks; see gotoff_operand. */
15696 #if TARGET_MACHO
15698 {
15702 }
15703 #endif
15704 else
15707 }
15708 \f
15709 /* Generate either "mov $0, reg" or "xor reg, reg", as appropriate
15710 for the target. */
15712 void
15714 {
15715 rtx tmp;
15717 /* We play register width games, which are only valid after reload. */
15720 /* Avoid HImode and its attendant prefix byte. */
15725 /* This predicate should match that for movsi_xor and movdi_xor_rex64. */
15727 {
15730 }
15733 }
15735 /* X is an unchanging MEM. If it is a constant pool reference, return
15736 the constant pool rtx, else NULL. */
15738 rtx
15740 {
15747 }
15749 void
15751 {
15759 {
15762 {
15769 }
15773 }
15777 {
15790 {
15798 }
15799 }
15803 {
15805 {
15806 #if TARGET_MACHO
15807 /* dynamic-no-pic */
15809 {
15810 rtx temp = ((reload_in_progress
15818 }
15820 {
15824 }
15827 else
15828 {
15836 }
15837 /* dynamic-no-pic */
15838 #endif
15839 }
15840 else
15841 {
15845 {
15852 }
15853 }
15854 }
15855 else
15856 {
15867 /* Force large constants in 64bit compilation into register
15868 to get them CSEed. */
15880 {
15881 /* If we are loading a floating point constant to a register,
15882 force the value to memory now, since we'll get better code
15883 out the back end. */
15887 {
15892 }
15893 }
15894 }
15897 }
15899 void
15901 {
15905 /* Force constants other than zero into memory. We do not know how
15906 the instructions used to build constants modify the upper 64 bits
15907 of the register, once we have that information we may be able
15908 to handle some of them more efficiently. */
15917 /* We need to check memory alignment for SSE mode since attribute
15918 can make operands unaligned. */
15923 {
15926 /* ix86_expand_vector_move_misalign() does not like constants ... */
15932 /* ... nor both arguments in memory. */
15940 }
15942 /* Make operand1 a register if it isn't already. */
15946 {
15949 }
15952 }
15954 /* Split 32-byte AVX unaligned load and store if needed. */
15956 static void
15958 {
15959 rtx m;
15965 {
15983 }
15986 {
15993 }
15995 {
16000 }
16001 else
16003 }
16005 /* Implement the movmisalign patterns for SSE. Non-SSE modes go
16006 straight to ix86_expand_vector_move. */
16007 /* Code generation for scalar reg-reg moves of single and double precision data:
16008 if (x86_sse_partial_reg_dependency == true | x86_sse_split_regs == true)
16009 movaps reg, reg
16010 else
16011 movss reg, reg
16012 if (x86_sse_partial_reg_dependency == true)
16013 movapd reg, reg
16014 else
16015 movsd reg, reg
16017 Code generation for scalar loads of double precision data:
16018 if (x86_sse_split_regs == true)
16019 movlpd mem, reg (gas syntax)
16020 else
16021 movsd mem, reg
16023 Code generation for unaligned packed loads of single precision data
16024 (x86_sse_unaligned_move_optimal overrides x86_sse_partial_reg_dependency):
16025 if (x86_sse_unaligned_move_optimal)
16026 movups mem, reg
16028 if (x86_sse_partial_reg_dependency == true)
16029 {
16030 xorps reg, reg
16031 movlps mem, reg
16032 movhps mem+8, reg
16033 }
16034 else
16035 {
16036 movlps mem, reg
16037 movhps mem+8, reg
16038 }
16040 Code generation for unaligned packed loads of double precision data
16041 (x86_sse_unaligned_move_optimal overrides x86_sse_split_regs):
16042 if (x86_sse_unaligned_move_optimal)
16043 movupd mem, reg
16045 if (x86_sse_split_regs == true)
16046 {
16047 movlpd mem, reg
16048 movhpd mem+8, reg
16049 }
16050 else
16051 {
16052 movsd mem, reg
16053 movhpd mem+8, reg
16054 }
16055 */
16057 void
16059 {
16067 {
16069 {
16074 /* FALLTHRU */
16082 }
16085 }
16088 {
16089 /* ??? If we have typed data, then it would appear that using
16090 movdqu is the only way to get unaligned data loaded with
16091 integer type. */
16093 {
16096 /* We will eventually emit movups based on insn attributes. */
16098 }
16100 {
16101 rtx zero;
16104 || TARGET_SSE_UNALIGNED_LOAD_OPTIMAL
16105 || TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL
16107 {
16108 /* We will eventually emit movups based on insn attributes. */
16111 }
16113 /* When SSE registers are split into halves, we can avoid
16114 writing to the top half twice. */
16116 {
16119 }
16120 else
16121 {
16122 /* ??? Not sure about the best option for the Intel chips.
16123 The following would seem to satisfy; the register is
16124 entirely cleared, breaking the dependency chain. We
16125 then store to the upper half, with a dependency depth
16126 of one. A rumor has it that Intel recommends two movsd
16127 followed by an unpacklpd, but this is unconfirmed. And
16128 given that the dependency depth of the unpacklpd would
16129 still be one, I'm not sure why this would be better. */
16131 }
16137 }
16138 else
16139 {
16141 || TARGET_SSE_UNALIGNED_LOAD_OPTIMAL
16142 || TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL
16144 {
16149 }
16153 else
16163 }
16164 }
16166 {
16168 {
16171 /* We will eventually emit movups based on insn attributes. */
16173 }
16175 {
16177 || TARGET_SSE_UNALIGNED_STORE_OPTIMAL
16178 || TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL
16180 /* We will eventually emit movups based on insn attributes. */
16182 else
16183 {
16188 }
16189 }
16190 else
16191 {
16196 || TARGET_SSE_UNALIGNED_STORE_OPTIMAL
16197 || TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL
16199 {
16202 }
16203 else
16204 {
16209 }
16210 }
16211 }
16212 else
16214 }
16216 /* Expand a push in MODE. This is some mode for which we do not support
16217 proper push instructions, at least from the registers that we expect
16218 the value to live in. */
16220 void
16222 {
16223 rtx tmp;
16233 /* When we push an operand onto stack, it has to be aligned at least
16234 at the function argument boundary. However since we don't have
16235 the argument type, we can't determine the actual argument
16236 boundary. */
16238 }
16240 /* Helper function of ix86_fixup_binary_operands to canonicalize
16241 operand order. Returns true if the operands should be swapped. */
16243 static bool
16245 rtx operands[])
16246 {
16251 /* If the operation is not commutative, we can't do anything. */
16255 /* Highest priority is that src1 should match dst. */
16261 /* Next highest priority is that immediate constants come second. */
16267 /* Lowest priority is that memory references should come second. */
16274 }
16277 /* Fix up OPERANDS to satisfy ix86_binary_operator_ok. Return the
16278 destination to use for the operation. If different from the true
16279 destination in operands[0], a copy operation will be required. */
16281 rtx
16283 rtx operands[])
16284 {
16289 /* Canonicalize operand order. */
16291 {
16292 rtx temp;
16294 /* It is invalid to swap operands of different modes. */
16300 }
16302 /* Both source operands cannot be in memory. */
16304 {
16305 /* Optimization: Only read from memory once. */
16307 {
16310 }
16311 else
16313 }
16315 /* If the destination is memory, and we do not have matching source
16316 operands, do things in registers. */
16320 /* Source 1 cannot be a constant. */
16324 /* Source 1 cannot be a non-matching memory. */
16328 /* Improve address combine. */
16337 }
16339 /* Similarly, but assume that the destination has already been
16340 set up properly. */
16342 void
16345 {
16348 }
16350 /* Attempt to expand a binary operator. Make the expansion closer to the
16351 actual machine, then just general_operand, which will allow 3 separate
16352 memory references (one output, two input) in a single insn. */
16354 void
16356 rtx operands[])
16357 {
16364 /* Emit the instruction. */
16368 {
16369 /* Reload doesn't know about the flags register, and doesn't know that
16370 it doesn't want to clobber it. We can only do this with PLUS. */
16373 }
16377 {
16378 /* This is going to be an LEA; avoid splitting it later. */
16380 }
16381 else
16382 {
16385 }
16387 /* Fix up the destination if needed. */
16390 }
16392 /* Return TRUE or FALSE depending on whether the binary operator meets the
16393 appropriate constraints. */
16395 bool
16398 {
16403 /* Both source operands cannot be in memory. */
16407 /* Canonicalize operand order for commutative operators. */
16409 {
16413 }
16415 /* If the destination is memory, we must have a matching source operand. */
16419 /* Source 1 cannot be a constant. */
16423 /* Source 1 cannot be a non-matching memory. */
16425 /* Support "andhi/andsi/anddi" as a zero-extending move. */
16433 }
16435 /* Attempt to expand a unary operator. Make the expansion closer to the
16436 actual machine, then just general_operand, which will allow 2 separate
16437 memory references (one output, one input) in a single insn. */
16439 void
16441 rtx operands[])
16442 {
16449 /* If the destination is memory, and we do not have matching source
16450 operands, do things in registers. */
16453 {
16456 else
16458 }
16460 /* When source operand is memory, destination must match. */
16464 /* Emit the instruction. */
16468 {
16469 /* Reload doesn't know about the flags register, and doesn't know that
16470 it doesn't want to clobber it. */
16473 }
16474 else
16475 {
16478 }
16480 /* Fix up the destination if needed. */
16483 }
16485 /* Split 32bit/64bit divmod with 8bit unsigned divmod if dividend and
16486 divisor are within the range [0-255]. */
16488 void
16491 {
16500 {
16513 }
16520 /* Use 8bit unsigned divimod if dividend and divisor are within
16521 the range [0-255]. */
16530 pc_rtx);
16535 /* Generate original signed/unsigned divimod. */
16540 /* Branch to the end. */
16544 /* Generate 8bit unsigned divide. */
16546 /* Don't use operands[0] for result of 8bit divide since not all
16547 registers support QImode ZERO_EXTRACT. */
16554 {
16557 }
16558 else
16559 {
16562 }
16564 /* Extract remainder from AH. */
16568 else
16569 {
16570 /* Need a new scratch register since the old one has result
16571 of 8bit divide. */
16575 }
16578 /* Zero extend quotient from AL. */
16584 }
16586 #define LEA_MAX_STALL (3)
16587 #define LEA_SEARCH_THRESHOLD (LEA_MAX_STALL << 1)
16589 /* Increase given DISTANCE in half-cycles according to
16590 dependencies between PREV and NEXT instructions.
16591 Add 1 half-cycle if there is no dependency and
16592 go to next cycle if there is some dependecy. */
16594 static unsigned int
16596 {
16613 }
16615 /* Function checks if instruction INSN defines register number
16616 REGNO1 or REGNO2. */
16618 static bool
16620 rtx insn)
16621 {
16629 {
16631 }
16634 }
16636 /* Function checks if instruction INSN uses register number
16637 REGNO as a part of address expression. */
16639 static bool
16641 {
16649 }
16651 /* Search backward for non-agu definition of register number REGNO1
16652 or register number REGNO2 in basic block starting from instruction
16653 START up to head of basic block or instruction INSN.
16655 Function puts true value into *FOUND var if definition was found
16656 and false otherwise.
16658 Distance in half-cycles between START and found instruction or head
16659 of BB is added to DISTANCE and returned. */
16661 static int
16665 {
16675 {
16677 {
16680 {
16683 {
16686 }
16687 }
16690 }
16695 }
16698 }
16700 /* Search backward for non-agu definition of register number REGNO1
16701 or register number REGNO2 in INSN's basic block until
16702 1. Pass LEA_SEARCH_THRESHOLD instructions, or
16703 2. Reach neighbour BBs boundary, or
16704 3. Reach agu definition.
16705 Returns the distance between the non-agu definition point and INSN.
16706 If no definition point, returns -1. */
16708 static int
16710 rtx insn)
16711 {
16722 {
16723 edge e;
16724 edge_iterator ei;
16729 {
16732 }
16738 else
16739 {
16744 {
16745 int bb_dist
16751 {
16758 }
16759 }
16762 }
16763 }
16765 /* get_attr_type may modify recog data. We want to make sure
16766 that recog data is valid for instruction INSN, on which
16767 distance_non_agu_define is called. INSN is unchanged here. */
16774 }
16776 /* Return the distance in half-cycles between INSN and the next
16777 insn that uses register number REGNO in memory address added
16778 to DISTANCE. Return -1 if REGNO0 is set.
16780 Put true value into *FOUND if register usage was found and
16781 false otherwise.
16782 Put true value into *REDEFINED if register redefinition was
16783 found and false otherwise. */
16785 static int
16789 {
16800 {
16802 {
16805 {
16806 /* Return DISTANCE if OP0 is used in memory
16807 address in NEXT. */
16810 }
16813 {
16814 /* Return -1 if OP0 is set in NEXT. */
16817 }
16820 }
16826 }
16829 }
16831 /* Return the distance between INSN and the next insn that uses
16832 register number REGNO0 in memory address. Return -1 if no such
16833 a use is found within LEA_SEARCH_THRESHOLD or REGNO0 is set. */
16835 static int
16837 {
16849 {
16850 edge e;
16851 edge_iterator ei;
16856 {
16859 }
16865 else
16866 {
16872 {
16873 int bb_dist
16878 {
16885 }
16886 }
16889 }
16890 }
16896 }
16898 /* Define this macro to tune LEA priority vs ADD, it take effect when
16899 there is a dilemma of choicing LEA or ADD
16900 Negative value: ADD is more preferred than LEA
16901 Zero: Netrual
16902 Positive value: LEA is more preferred than ADD*/
16903 #define IX86_LEA_PRIORITY 0
16905 /* Return true if usage of lea INSN has performance advantage
16906 over a sequence of instructions. Instructions sequence has
16907 SPLIT_COST cycles higher latency than lea latency. */
16909 static bool
16912 {
16919 {
16920 /* If there is no non AGU operand definition, no AGU
16921 operand usage and split cost is 0 then both lea
16922 and non lea variants have same priority. Currently
16923 we prefer lea for 64 bit code and non lea on 32 bit
16924 code. */
16927 else
16929 }
16931 /* With longer definitions distance lea is more preferable.
16932 Here we change it to take into account splitting cost and
16933 lea priority. */
16936 /* If there is no use in memory addess then we just check
16937 that split cost does not exceed AGU stall. */
16941 /* If this insn has both backward non-agu dependence and forward
16942 agu dependence, the one with short distance takes effect. */
16944 }
16946 /* Return true if it is legal to clobber flags by INSN and
16947 false otherwise. */
16949 static bool
16951 {
16954 bitmap live;
16957 {
16959 {
16966 }
16972 }
16976 }
16978 /* Return true if we need to split op0 = op1 + op2 into a sequence of
16979 move and add to avoid AGU stalls. */
16981 bool
16983 {
16986 /* Check if we need to optimize. */
16990 /* Check it is correct to split here. */
16998 /* We need to split only adds with non destructive
16999 destination operand. */
17002 else
17004 }
17006 /* Return true if we should emit lea instruction instead of mov
17007 instruction. */
17009 bool
17011 {
17014 /* Check if we need to optimize. */
17018 /* Use lea for reg to reg moves only. */
17026 }
17028 /* Return true if we need to split lea into a sequence of
17029 instructions to avoid AGU stalls. */
17031 bool
17033 {
17039 /* FIXME: Handle zero-extended addresses. */
17044 /* Check we need to optimize. */
17048 /* Check it is correct to split here. */
17055 /* We should not split into add if non legitimate pic
17056 operand is used as displacement. */
17071 /* Compute how many cycles we will add to execution time
17072 if split lea into a sequence of instructions. */
17074 {
17075 /* Have to use mov instruction if non desctructive
17076 destination form is used. */
17080 /* Have to add index to base if both exist. */
17084 /* Have to use shift and adds if scale is 2 or greater. */
17086 {
17091 else
17093 }
17095 /* Have to use add instruction with immediate if
17096 disp is non zero. */
17100 /* Subtract the price of lea. */
17102 }
17105 }
17107 /* Emit x86 binary operand CODE in mode MODE, where the first operand
17108 matches destination. RTX includes clobber of FLAGS_REG. */
17110 static void
17113 {
17120 }
17122 /* Split lea instructions into a sequence of instructions
17123 which are executed on ALU to avoid AGU stalls.
17124 It is assumed that it is allowed to clobber flags register
17125 at lea position. */
17129 {
17145 {
17149 }
17152 {
17156 }
17159 {
17160 /* Case r1 = r1 + ... */
17162 {
17163 /* If we have a case r1 = r1 + C * r1 then we
17164 should use multiplication which is very
17165 expensive. Assume cost model is wrong if we
17166 have such case here. */
17171 }
17172 else
17173 {
17174 /* r1 = r2 + r3 * C case. Need to move r3 into r1. */
17178 /* Use shift for scaling. */
17187 }
17188 }
17190 {
17193 }
17194 else
17195 {
17197 {
17200 }
17202 {
17205 }
17206 else
17207 {
17212 else
17213 {
17216 }
17219 }
17223 }
17224 }
17226 /* Return true if it is ok to optimize an ADD operation to LEA
17227 operation to avoid flag register consumation. For most processors,
17228 ADD is faster than LEA. For the processors like ATOM, if the
17229 destination register of LEA holds an actual address which will be
17230 used soon, LEA is better and otherwise ADD is better. */
17232 bool
17234 {
17239 /* If a = b + c, (a!=b && a!=c), must use lea form. */
17247 }
17249 /* Return true if destination reg of SET_BODY is shift count of
17250 USE_BODY. */
17252 static bool
17254 {
17255 rtx set_dest;
17256 rtx shift_rtx;
17259 /* Retrieve destination of SET_BODY. */
17261 {
17270 use_body))
17275 }
17277 /* Retrieve shift count of USE_BODY. */
17279 {
17291 }
17299 {
17302 /* Return true if shift count is dest of SET_BODY. */
17306 }
17309 }
17311 /* Return true if destination reg of SET_INSN is shift count of
17312 USE_INSN. */
17314 bool
17316 {
17319 }
17321 /* Return TRUE or FALSE depending on whether the unary operator meets the
17322 appropriate constraints. */
17324 bool
17328 {
17329 /* If one of operands is memory, source and destination must match. */
17335 }
17337 /* Return TRUE if the operands to a vec_interleave_{high,low}v2df
17338 are ok, keeping in mind the possible movddup alternative. */
17340 bool
17342 {
17348 }
17350 /* Post-reload splitter for converting an SF or DFmode value in an
17351 SSE register into an unsigned SImode. */
17353 void
17355 {
17366 /* Load up the value into the low element. We must ensure that the other
17367 elements are valid floats -- zero is the easiest such value. */
17369 {
17372 else
17374 }
17375 else
17376 {
17381 else
17383 }
17403 else
17408 }
17410 /* Convert an unsigned DImode value into a DFmode, using only SSE.
17411 Expects the 64-bit DImode to be supplied in a pair of integral
17412 registers. Requires SSE2; will use SSE3 if available. For x86_32,
17413 -mfpmath=sse, !optimize_size only. */
17415 void
17417 {
17421 rtx x;
17427 {
17430 }
17431 else
17432 {
17436 }
17444 /* int_xmm = {0x45300000UL, fp_xmm/hi, 0x43300000, fp_xmm/lo } */
17447 /* Concatenating (juxtaposing) (0x43300000UL ## fp_value_low_xmm)
17448 yields a valid DF value equal to (0x1.0p52 + double(fp_value_lo_xmm)).
17449 Similarly (0x45300000UL ## fp_value_hi_xmm) yields
17450 (0x1.0p84 + double(fp_value_hi_xmm)).
17451 Note these exponents differ by 32. */
17455 /* Subtract off those 0x1.0p52 and 0x1.0p84 biases, to produce values
17456 in [0,2**32-1] and [0]+[2**32,2**64-1] respectively. */
17465 /* Add the upper and lower DFmode values together. */
17468 else
17469 {
17473 }
17476 }
17478 /* Not used, but eases macroization of patterns. */
17479 void
17481 rtx input ATTRIBUTE_UNUSED)
17482 {
17484 }
17486 /* Convert an unsigned SImode value into a DFmode. Only currently used
17487 for SSE, but applicable anywhere. */
17489 void
17491 {
17492 REAL_VALUE_TYPE TWO31r;
17507 }
17509 /* Convert a signed DImode value into a DFmode. Only used for SSE in
17510 32-bit mode; otherwise we have a direct convert instruction. */
17512 void
17514 {
17515 REAL_VALUE_TYPE TWO32r;
17533 }
17535 /* Convert an unsigned SImode value into a SFmode, using only SSE.
17536 For x86_32, -mfpmath=sse, !optimize_size only. */
17537 void
17539 {
17540 REAL_VALUE_TYPE ONE16r;
17559 }
17561 /* floatunsv{4,8}siv{4,8}sf2 expander. Expand code to convert
17562 a vector of unsigned ints VAL to vector of floats TARGET. */
17564 void
17566 {
17568 REAL_VALUE_TYPE TWO16r;
17575 else
17580 OPTAB_DIRECT);
17591 OPTAB_DIRECT);
17593 OPTAB_DIRECT);
17596 }
17598 /* Adjust a V*SFmode/V*DFmode value VAL so that *sfix_trunc* resp. fix_trunc*
17599 pattern can be used on it instead of *ufix_trunc* resp. fixuns_trunc*.
17600 This is done by doing just signed conversion if < 0x1p31, and otherwise by
17601 subtracting 0x1p31 first and xoring in 0x80000000 from *XORP afterwards. */
17603 rtx
17605 {
17606 REAL_VALUE_TYPE TWO31r;
17621 {
17627 }
17636 OPTAB_DIRECT);
17637 else
17638 {
17644 OPTAB_DIRECT);
17645 }
17648 }
17650 /* A subroutine of ix86_build_signbit_mask. If VECT is true,
17651 then replicate the value for all elements of the vector
17652 register. */
17654 rtx
17656 {
17658 rtvec v;
17662 {
17689 }
17690 }
17692 /* A subroutine of ix86_expand_fp_absneg_operator, copysign expanders
17693 and ix86_expand_int_vcond. Create a mask for the sign bit in MODE
17694 for an SSE register. If VECT is true, then replicate the mask for
17695 all elements of the vector register. If INVERT is true, then create
17696 a mask excluding the sign bit. */
17698 rtx
17700 {
17704 rtx v;
17705 rtx mask;
17707 /* Find the sign bit, sign extended to 2*HWI. */
17709 {
17729 else
17737 {
17740 }
17741 else
17742 {
17743 rtvec vec;
17749 {
17752 }
17753 else
17763 }
17768 }
17773 /* Force this value into the low part of a fp vector constant. */
17782 }
17784 /* Generate code for floating point ABS or NEG. */
17786 void
17788 rtx operands[])
17789 {
17800 {
17806 }
17808 /* NEG and ABS performed with SSE use bitwise mask operations.
17809 Create the appropriate mask now. */
17812 else
17822 {
17824 rtvec par;
17829 else
17830 {
17833 }
17835 }
17836 else
17838 }
17840 /* Expand a copysign operation. Special case operand 0 being a constant. */
17842 void
17844 {
17858 else
17862 {
17869 {
17872 else
17873 {
17877 }
17878 }
17888 else
17892 }
17893 else
17894 {
17904 else
17908 }
17909 }
17911 /* Deconstruct a copysign operation into bit masks. Operand 0 is known to
17912 be a constant, and so has already been expanded into a vector constant. */
17914 void
17916 {
17932 {
17935 }
17936 }
17938 /* Deconstruct a copysign operation into bit masks. Operand 0 is variable,
17939 so we have to do two masks. */
17941 void
17943 {
17958 {
17959 /* Shouldn't happen often (it's useless, obviously), but when it does
17960 we'd generate incorrect code if we continue below. */
17963 }
17966 {
17977 }
17978 else
17979 {
17981 {
17983 }
17985 {
17989 }
17993 {
17996 }
17998 {
18003 }
18005 }
18009 }
18011 /* Return TRUE or FALSE depending on whether the first SET in INSN
18012 has source and destination with matching CC modes, and that the
18013 CC mode is at least as constrained as REQ_MODE. */
18015 bool
18017 {
18018 rtx set;
18029 {
18039 /* FALLTHRU */
18043 /* FALLTHRU */
18047 /* FALLTHRU */
18061 }
18064 }
18066 /* Generate insn patterns to do an integer compare of OPERANDS. */
18068 static rtx
18070 {
18077 /* This is very simple, but making the interface the same as in the
18078 FP case makes the rest of the code easier. */
18082 /* Return the test that should be put into the flags user, i.e.
18083 the bcc, scc, or cmov instruction. */
18085 }
18087 /* Figure out whether to use ordered or unordered fp comparisons.
18088 Return the appropriate mode to use. */
18090 enum machine_mode
18092 {
18093 /* ??? In order to make all comparisons reversible, we do all comparisons
18094 non-trapping when compiling for IEEE. Once gcc is able to distinguish
18095 all forms trapping and nontrapping comparisons, we can make inequality
18096 comparisons trapping again, since it results in better code when using
18097 FCOM based compares. */
18099 }
18101 enum machine_mode
18103 {
18107 {
18110 }
18113 {
18114 /* Only zero flag is needed. */
18118 /* Codes needing carry flag. */
18121 /* Detect overflow checks. They need just the carry flag. */
18125 else
18129 /* Detect overflow checks. They need just the carry flag. */
18133 else
18135 /* Codes possibly doable only with sign flag when
18136 comparing against zero. */
18141 else
18142 /* For other cases Carry flag is not required. */
18144 /* Codes doable only with sign flag when comparing
18145 against zero, but we miss jump instruction for it
18146 so we need to use relational tests against overflow
18147 that thus needs to be zero. */
18152 else
18154 /* strcmp pattern do (use flags) and combine may ask us for proper
18155 mode. */
18160 }
18161 }
18163 /* Return the fixed registers used for condition codes. */
18165 static bool
18167 {
18171 }
18173 /* If two condition code modes are compatible, return a condition code
18174 mode which is compatible with both. Otherwise, return
18175 VOIDmode. */
18177 static enum machine_mode
18179 {
18196 {
18210 {
18224 }
18228 /* These are only compatible with themselves, which we already
18229 checked above. */
18231 }
18232 }
18235 /* Return a comparison we can do and that it is equivalent to
18236 swap_condition (code) apart possibly from orderedness.
18237 But, never change orderedness if TARGET_IEEE_FP, returning
18238 UNKNOWN in that case if necessary. */
18240 static enum rtx_code
18242 {
18244 {
18255 }
18256 }
18258 /* Return cost of comparison CODE using the best strategy for performance.
18259 All following functions do use number of instructions as a cost metrics.
18260 In future this should be tweaked to compute bytes for optimize_size and
18261 take into account performance of various instructions on various CPUs. */
18263 static int
18265 {
18268 /* The cost of code using bit-twiddling on %ah. */
18270 {
18293 }
18296 {
18303 }
18304 }
18306 /* Return strategy to use for floating-point. We assume that fcomi is always
18307 preferrable where available, since that is also true when looking at size
18308 (2 bytes, vs. 3 for fnstsw+sahf and at least 5 for fnstsw+test). */
18310 enum ix86_fpcmp_strategy
18312 {
18313 /* Do fcomi/sahf based test when profitable. */
18322 }
18324 /* Swap, force into registers, or otherwise massage the two operands
18325 to a fp comparison. The operands are updated in place; the new
18326 comparison code is returned. */
18328 static enum rtx_code
18330 {
18336 /* All of the unordered compare instructions only work on registers.
18337 The same is true of the fcomi compare instructions. The XFmode
18338 compare instructions require registers except when comparing
18339 against zero or when converting operand 1 from fixed point to
18340 floating point. */
18349 {
18352 }
18353 else
18354 {
18355 /* %%% We only allow op1 in memory; op0 must be st(0). So swap
18356 things around if they appear profitable, otherwise force op0
18357 into a register. */
18363 {
18366 {
18367 rtx tmp;
18370 }
18371 }
18377 {
18382 {
18385 }
18386 else
18388 }
18389 }
18391 /* Try to rearrange the comparison to make it cheaper. */
18395 {
18396 rtx tmp;
18401 }
18406 }
18408 /* Convert comparison codes we use to represent FP comparison to integer
18409 code that will result in proper branch. Return UNKNOWN if no such code
18410 is available. */
18412 enum rtx_code
18414 {
18416 {
18439 }
18440 }
18442 /* Generate insn patterns to do a floating point compare of OPERANDS. */
18444 static rtx
18446 {
18453 /* Do fcomi/sahf based test when profitable. */
18455 {
18460 tmp);
18468 tmp);
18477 /* Sadness wrt reg-stack pops killing fpsr -- gotta get fnstsw first. */
18484 /* In the unordered case, we have to check C2 for NaN's, which
18485 doesn't happen to work out to anything nice combination-wise.
18486 So do some bit twiddling on the value we've got in AH to come
18487 up with an appropriate set of condition codes. */
18491 {
18495 {
18498 }
18499 else
18500 {
18506 }
18511 {
18516 }
18517 else
18518 {
18521 }
18526 {
18529 }
18530 else
18531 {
18535 }
18540 {
18546 }
18547 else
18548 {
18551 }
18556 {
18561 }
18562 else
18563 {
18566 }
18571 {
18576 }
18577 else
18578 {
18581 }
18595 }
18600 }
18602 /* Return the test that should be put into the flags user, i.e.
18603 the bcc, scc, or cmov instruction. */
18606 const0_rtx);
18607 }
18609 static rtx
18611 {
18612 rtx ret;
18618 {
18621 }
18622 else
18626 }
18628 void
18630 {
18632 rtx tmp;
18635 {
18642 simple:
18646 pc_rtx);
18654 /* Expand DImode branch into multiple compare+branch. */
18655 {
18661 {
18664 }
18671 /* When comparing for equality, we can use (hi0^hi1)|(lo0^lo1) to
18672 avoid two branches. This costs one extra insn, so disable when
18673 optimizing for size. */
18678 {
18696 }
18698 /* Otherwise, if we are doing less-than or greater-or-equal-than,
18699 op1 is a constant and the low word is zero, then we can just
18700 examine the high word. Similarly for low word -1 and
18701 less-or-equal-than or greater-than. */
18705 {
18708 {
18711 }
18715 {
18718 }
18722 }
18724 /* Otherwise, we need two or three jumps. */
18733 {
18747 }
18749 /*
18750 * a < b =>
18751 * if (hi(a) < hi(b)) goto true;
18752 * if (hi(a) > hi(b)) goto false;
18753 * if (lo(a) < lo(b)) goto true;
18754 * false:
18755 */
18767 }
18772 }
18773 }
18775 /* Split branch based on floating point condition. */
18776 void
18779 {
18780 rtx condition;
18781 rtx i;
18784 {
18789 }
18792 tmp);
18794 /* Remove pushed operand from stack. */
18804 }
18806 void
18808 {
18809 rtx ret;
18816 }
18818 /* Expand comparison setting or clearing carry flag. Return true when
18819 successful and set pop for the operation. */
18820 static bool
18822 {
18826 /* Do not handle double-mode compares that go through special path. */
18831 {
18836 /* Shortcut: following common codes never translate
18837 into carry flag compares. */
18842 /* These comparisons require zero flag; swap operands so they won't. */
18845 {
18850 }
18852 /* Try to expand the comparison and verify that we end up with
18853 carry flag based comparison. This fails to be true only when
18854 we decide to expand comparison using arithmetic that is not
18855 too common scenario. */
18864 else
18873 }
18879 {
18884 /* Convert a==0 into (unsigned)a<1. */
18893 /* Convert a>b into b<a or a>=b-1. */
18897 {
18899 /* Bail out on overflow. We still can swap operands but that
18900 would force loading of the constant into register. */
18905 }
18906 else
18907 {
18912 }
18915 /* Convert a>=0 into (unsigned)a<0x80000000. */
18933 }
18934 /* Swapping operands may cause constant to appear as first operand. */
18936 {
18940 }
18944 }
18946 bool
18948 {
18972 /* Don't attempt mode expansion here -- if we had to expand 5 or 6
18973 HImode insns, we'd be swallowed in word prefix ops. */
18979 {
18983 HOST_WIDE_INT diff;
18986 /* Sign bit compares are better done using shifts than we do by using
18987 sbb. */
18990 {
18991 /* Detect overlap between destination and compare sources. */
18995 {
18996 rtx flags;
19005 {
19007 compare_code
19009 }
19011 /* To simplify rest of code, restrict to the GEU case. */
19013 {
19019 }
19020 else
19021 {
19024 reverse_condition_maybe_unordered
19026 else
19029 }
19038 else
19041 }
19042 else
19043 {
19046 else
19047 {
19052 }
19054 }
19057 {
19058 /*
19059 * cmpl op0,op1
19060 * sbbl dest,dest
19061 * [addl dest, ct]
19062 *
19063 * Size 5 - 8.
19064 */
19069 }
19071 {
19072 /*
19073 * cmpl op0,op1
19074 * sbbl dest,dest
19075 * orl $ct, dest
19076 *
19077 * Size 8.
19078 */
19082 }
19084 {
19085 /*
19086 * cmpl op0,op1
19087 * sbbl dest,dest
19088 * notl dest
19089 * [addl dest, cf]
19090 *
19091 * Size 8 - 11.
19092 */
19098 }
19099 else
19100 {
19101 /*
19102 * cmpl op0,op1
19103 * sbbl dest,dest
19104 * [notl dest]
19105 * andl cf - ct, dest
19106 * [addl dest, ct]
19107 *
19108 * Size 8 - 11.
19109 */
19112 {
19116 }
19126 }
19132 }
19135 {
19138 HOST_WIDE_INT tmp;
19143 {
19146 /* We may be reversing unordered compare to normal compare, that
19147 is not valid in general (we may convert non-trapping condition
19148 to trapping one), however on i386 we currently emit all
19149 comparisons unordered. */
19152 }
19153 else
19154 {
19157 }
19158 }
19163 {
19168 {
19173 }
19174 }
19176 /* Optimize dest = (op0 < 0) ? -1 : cf. */
19180 {
19181 /* If lea code below could be used, only optimize
19182 if it results in a 2 insn sequence. */
19188 {
19189 /*
19190 * notl op1 (if necessary)
19191 * sarl $31, op1
19192 * orl cf, op1
19193 */
19195 {
19199 }
19210 }
19211 }
19219 {
19220 /*
19221 * xorl dest,dest
19222 * cmpl op1,op2
19223 * setcc dest
19224 * lea cf(dest*(ct-cf)),dest
19225 *
19226 * Size 14.
19227 *
19228 * This also catches the degenerate setcc-only case.
19229 */
19231 rtx tmp;
19237 /* On x86_64 the lea instruction operates on Pmode, so we need
19238 to get arithmetics done in proper mode to match. */
19241 else
19242 {
19243 rtx out1;
19246 nops++;
19248 {
19250 nops++;
19251 }
19252 }
19254 {
19256 nops++;
19257 }
19259 {
19262 else
19264 }
19269 }
19271 /*
19272 * General case: Jumpful:
19273 * xorl dest,dest cmpl op1, op2
19274 * cmpl op1, op2 movl ct, dest
19275 * setcc dest jcc 1f
19276 * decl dest movl cf, dest
19277 * andl (cf-ct),dest 1:
19278 * addl ct,dest
19279 *
19280 * Size 20. Size 14.
19281 *
19282 * This is reasonably steep, but branch mispredict costs are
19283 * high on modern cpus, so consider failing only if optimizing
19284 * for space.
19285 */
19290 {
19292 {
19299 {
19302 /* We may be reversing unordered compare to normal compare,
19303 that is not valid in general (we may convert non-trapping
19304 condition to trapping one), however on i386 we currently
19305 emit all comparisons unordered. */
19307 }
19308 else
19309 {
19313 }
19314 }
19317 {
19318 /* notl op1 (if needed)
19319 sarl $31, op1
19320 andl (cf-ct), op1
19321 addl ct, op1
19323 For x < 0 (resp. x <= -1) there will be no notl,
19324 so if possible swap the constants to get rid of the
19325 complement.
19326 True/false will be -1/0 while code below (store flag
19327 followed by decrement) is 0/-1, so the constants need
19328 to be exchanged once more. */
19331 {
19334 }
19335 else
19336 {
19340 }
19343 }
19344 else
19345 {
19349 constm1_rtx,
19351 }
19363 }
19364 }
19367 {
19368 /* Try a few things more with specific constants and a variable. */
19370 optab op;
19376 /* If one of the two operands is an interesting constant, load a
19377 constant with the above and mask it in with a logical operation. */
19380 {
19386 else
19388 }
19390 {
19396 else
19398 }
19399 else
19406 /* Recurse to get the constant loaded. */
19410 /* Mask in the interesting variable. */
19412 OPTAB_WIDEN);
19417 }
19419 /*
19420 * For comparison with above,
19421 *
19422 * movl cf,dest
19423 * movl ct,tmp
19424 * cmpl op1,op2
19425 * cmovcc tmp,dest
19426 *
19427 * Size 15.
19428 */
19450 }
19452 /* Swap, force into registers, or otherwise massage the two operands
19453 to an sse comparison with a mask result. Thus we differ a bit from
19454 ix86_prepare_fp_compare_args which expects to produce a flags result.
19456 The DEST operand exists to help determine whether to commute commutative
19457 operators. The POP0/POP1 operands are updated in place. The new
19458 comparison code is returned, or UNKNOWN if not implementable. */
19460 static enum rtx_code
19463 {
19464 rtx tmp;
19467 {
19470 /* AVX supports all the needed comparisons. */
19473 /* We have no LTGT as an operator. We could implement it with
19474 NE & ORDERED, but this requires an extra temporary. It's
19475 not clear that it's worth it. */
19482 /* These are supported directly. */
19489 /* AVX has 3 operand comparisons, no need to swap anything. */
19492 /* For commutative operators, try to canonicalize the destination
19493 operand to be first in the comparison - this helps reload to
19494 avoid extra moves. */
19497 /* FALLTHRU */
19503 /* These are not supported directly before AVX, and furthermore
19504 ix86_expand_sse_fp_minmax only optimizes LT/UNGE. Swap the
19505 comparison operands to transform into something that is
19506 supported. */
19515 }
19518 }
19520 /* Detect conditional moves that exactly match min/max operational
19521 semantics. Note that this is IEEE safe, as long as we don't
19522 interchange the operands.
19524 Returns FALSE if this conditional move doesn't match a MIN/MAX,
19525 and TRUE if the operation is successful and instructions are emitted. */
19527 static bool
19530 {
19533 rtx tmp;
19536 ;
19538 {
19542 }
19543 else
19550 else
19555 /* We want to check HONOR_NANS and HONOR_SIGNED_ZEROS here,
19556 but MODE may be a vector mode and thus not appropriate. */
19558 {
19560 rtvec v;
19565 }
19566 else
19567 {
19570 }
19574 }
19576 /* Expand an sse vector comparison. Return the register with the result. */
19578 static rtx
19581 {
19584 rtx x;
19597 {
19600 }
19601 else
19605 }
19607 /* Expand DEST = CMP ? OP_TRUE : OP_FALSE into a sequence of logical
19608 operations. This is used for both scalar and vector conditional moves. */
19610 static void
19612 {
19618 {
19620 }
19622 {
19626 }
19628 {
19633 }
19635 {
19639 }
19641 {
19649 op_true,
19650 op_false)));
19651 }
19652 else
19653 {
19662 {
19676 {
19682 }
19697 {
19703 }
19707 }
19711 else
19712 {
19718 else
19730 }
19731 }
19732 }
19734 /* Expand a floating-point conditional move. Return true if successful. */
19736 bool
19738 {
19746 {
19749 /* Since we've no cmove for sse registers, don't force bad register
19750 allocation just to gain access to it. Deny movcc when the
19751 comparison mode doesn't match the move mode. */
19770 }
19772 /* The floating point conditional move instructions don't directly
19773 support conditions resulting from a signed integer comparison. */
19777 {
19782 }
19789 }
19791 /* Expand a floating-point vector conditional move; a vcond operation
19792 rather than a movcc operation. */
19794 bool
19796 {
19798 rtx cmp;
19803 {
19804 rtx temp;
19806 {
19823 }
19825 OPTAB_DIRECT);
19828 }
19838 }
19840 /* Expand a signed/unsigned integral vector conditional move. */
19842 bool
19844 {
19854 /* Try to optimize x < 0 ? -1 : 0 into (signed) x >> 31
19855 and x < 0 ? 1 : 0 into (unsigned) x >> 31. */
19864 {
19868 {
19874 }
19877 {
19883 }
19884 }
19893 /* XOP supports all of the comparisons on all 128-bit vector int types. */
19897 ;
19898 else
19899 {
19900 /* Canonicalize the comparison to EQ, GT, GTU. */
19902 {
19919 /* FALLTHRU */
19929 }
19931 /* Only SSE4.1/SSE4.2 supports V2DImode. */
19933 {
19935 {
19937 /* SSE4.1 supports EQ. */
19944 /* SSE4.2 supports GT/GTU. */
19951 }
19952 }
19954 /* Unsigned parallel compare is not supported by the hardware.
19955 Play some tricks to turn this into a signed comparison
19956 against 0. */
19958 {
19962 {
19967 {
19972 {
19979 }
19980 /* Subtract (-(INT MAX) - 1) from both operands to make
19981 them signed. */
19992 }
19999 /* Perform a parallel unsigned saturating subtraction. */
20012 }
20013 }
20014 }
20016 /* Allow the comparison to be done in one mode, but the movcc to
20017 happen in another mode. */
20019 {
20022 }
20023 else
20024 {
20030 }
20035 }
20037 /* Expand a variable vector permutation. */
20039 void
20041 {
20052 /* Number of elements in the vector. */
20058 {
20060 {
20061 /* Unfortunately, the VPERMQ and VPERMPD instructions only support
20062 an constant shuffle operand. With a tiny bit of effort we can
20063 use VPERMD instead. A re-interpretation stall for V4DFmode is
20064 unfortunate but there's no avoiding it.
20065 Similarly for V16HImode we don't have instructions for variable
20066 shuffling, while for V32QImode we can use after preparing suitable
20067 masks vpshufb; vpshufb; vpermq; vpor. */
20070 {
20074 }
20075 else
20076 {
20080 }
20083 /* Replicate the low bits of the V4DImode mask into V8SImode:
20084 mask = { A B C D }
20085 t1 = { A A B B C C D D }. */
20093 else
20096 /* Multiply the shuffle indicies by two. */
20098 OPTAB_DIRECT);
20100 /* Add one to the odd shuffle indicies:
20101 t1 = { A*2, A*2+1, B*2, B*2+1, ... }. */
20103 {
20106 }
20110 OPTAB_DIRECT);
20112 /* Continue as if V8SImode (resp. V32QImode) was used initially. */
20117 }
20120 {
20122 /* The VPERMD and VPERMPS instructions already properly ignore
20123 the high bits of the shuffle elements. No need for us to
20124 perform an AND ourselves. */
20127 else
20128 {
20134 }
20141 else
20142 {
20148 }
20152 /* By combining the two 128-bit input vectors into one 256-bit
20153 input vector, we can use VPERMD and VPERMPS for the full
20154 two-operand shuffle. */
20186 /* From mask create two adjusted masks, which contain the same
20187 bits as mask in the low 7 bits of each vector element.
20188 The first mask will have the most significant bit clear
20189 if it requests element from the same 128-bit lane
20190 and MSB set if it requests element from the other 128-bit lane.
20191 The second mask will have the opposite values of the MSB,
20192 and additionally will have its 128-bit lanes swapped.
20193 E.g. { 07 12 1e 09 ... | 17 19 05 1f ... } mask vector will have
20194 t1 { 07 92 9e 09 ... | 17 19 85 1f ... } and
20195 t3 { 97 99 05 9f ... | 87 12 1e 89 ... } where each ...
20196 stands for other 12 bytes. */
20197 /* The bit whether element is from the same lane or the other
20198 lane is bit 4, so shift it up by 3 to the MSB position. */
20202 /* Clear MSB bits from the mask just in case it had them set. */
20204 /* After this t1 will have MSB set for elements from other lane. */
20206 /* Clear bits other than MSB. */
20208 /* Or in the lower bits from mask into t3. */
20210 /* And invert MSB bits in t1, so MSB is set for elements from the same
20211 lane. */
20213 /* Swap 128-bit lanes in t3. */
20218 /* And or in the lower bits from mask into t1. */
20221 {
20222 /* Each of these shuffles will put 0s in places where
20223 element from the other 128-bit lane is needed, otherwise
20224 will shuffle in the requested value. */
20227 /* For t3 the 128-bit lanes are swapped again. */
20232 /* And oring both together leads to the result. */
20235 }
20238 /* Similarly to the above one_operand_shuffle code,
20239 just for repeated twice for each operand. merge_two:
20240 code will merge the two results together. */
20262 }
20263 }
20266 {
20267 /* The XOP VPPERM insn supports three inputs. By ignoring the
20268 one_operand_shuffle special case, we avoid creating another
20269 set of constant vectors in memory. */
20272 /* mask = mask & {2*w-1, ...} */
20274 }
20275 else
20276 {
20277 /* mask = mask & {w-1, ...} */
20279 }
20287 /* For non-QImode operations, convert the word permutation control
20288 into a byte permutation control. */
20290 {
20295 /* Convert mask to vector of chars. */
20298 /* Replicate each of the input bytes into byte positions:
20299 (v2di) --> {0,0,0,0,0,0,0,0, 8,8,8,8,8,8,8,8}
20300 (v4si) --> {0,0,0,0, 4,4,4,4, 8,8,8,8, 12,12,12,12}
20301 (v8hi) --> {0,0, 2,2, 4,4, 6,6, ...}. */
20308 else
20311 /* Convert it into the byte positions by doing
20312 mask = mask + {0,1,..,16/w, 0,1,..,16/w, ...} */
20318 }
20320 /* The actual shuffle operations all operate on V16QImode. */
20326 {
20328 }
20330 {
20332 }
20333 else
20334 {
20338 /* Shuffle the two input vectors independently. */
20344 merge_two:
20345 /* Then merge them together. The key is whether any given control
20346 element contained a bit set that indicates the second word. */
20350 {
20351 /* Without SSE4.1, we don't have V2DImode EQ. Perform one
20352 more shuffle to convert the V2DI input mask into a V4SI
20353 input mask. At which point the masking that expand_int_vcond
20354 will work as desired. */
20362 }
20379 }
20380 }
20382 /* Unpack OP[1] into the next wider integer vector type. UNSIGNED_P is
20383 true if we should do zero extension, else sign extension. HIGH_P is
20384 true if we want the N/2 high elements, else the low elements. */
20386 void
20388 {
20390 rtx tmp;
20393 {
20399 {
20403 else
20406 extract
20412 else
20415 extract
20421 else
20424 extract
20430 else
20436 else
20442 else
20447 }
20450 {
20453 }
20455 {
20456 /* Shift higher 8 bytes to lower 8 bytes. */
20461 }
20462 else
20466 }
20467 else
20468 {
20472 {
20476 else
20482 else
20488 else
20493 }
20497 else
20502 }
20503 }
20505 /* Expand conditional increment or decrement using adb/sbb instructions.
20506 The default case using setcc followed by the conditional move can be
20507 done by generic code. */
20508 bool
20510 {
20512 rtx flags;
20514 rtx compare_op;
20532 {
20535 }
20538 {
20542 reverse_condition_maybe_unordered
20544 else
20546 }
20550 /* Construct either adc or sbb insn. */
20552 {
20554 {
20569 }
20570 }
20571 else
20572 {
20574 {
20589 }
20590 }
20594 }
20597 /* Split operands 0 and 1 into half-mode parts. Similar to split_double_mode,
20598 but works for floating pointer parameters and nonoffsetable memories.
20599 For pushes, it returns just stack offsets; the values will be saved
20600 in the right order. Maximally three parts are generated. */
20602 static int
20604 {
20609 else
20615 /* Optimize constant pool reference to immediates. This is used by fp
20616 moves, that force all constants to memory to allow combining. */
20618 {
20622 }
20625 {
20626 /* The only non-offsetable memories we handle are pushes. */
20635 }
20638 {
20640 /* Caution: if we looked through a constant pool memory above,
20641 the operand may actually have a different mode now. That's
20642 ok, since we want to pun this all the way back to an integer. */
20646 }
20649 {
20652 else
20653 {
20657 {
20661 }
20663 {
20668 }
20670 {
20671 REAL_VALUE_TYPE r;
20676 {
20691 }
20694 }
20695 else
20697 }
20698 }
20699 else
20700 {
20704 {
20707 {
20711 }
20713 {
20717 }
20719 {
20720 REAL_VALUE_TYPE r;
20726 /* Do not use shift by 32 to avoid warning on 32bit systems. */
20729 = gen_int_mode
20732 DImode);
20733 else
20740 = gen_int_mode
20743 DImode);
20744 else
20746 }
20747 else
20749 }
20750 }
20753 }
20755 /* Emit insns to perform a move or push of DI, DF, XF, and TF values.
20756 Return false when normal moves are needed; true when all required
20757 insns have been emitted. Operands 2-4 contain the input values
20758 int the correct order; operands 5-7 contain the output values. */
20760 void
20762 {
20770 /* The DFmode expanders may ask us to move double.
20771 For 64bit target this is single move. By hiding the fact
20772 here we simplify i386.md splitters. */
20774 {
20775 /* Optimize constant pool reference to immediates. This is used by
20776 fp moves, that force all constants to memory to allow combining. */
20783 {
20786 }
20787 else
20792 }
20794 /* The only non-offsettable memory we handle is push. */
20797 else
20804 /* When emitting push, take care for source operands on the stack. */
20807 {
20810 /* Compensate for the stack decrement by 4. */
20815 /* src_base refers to the stack pointer and is
20816 automatically decreased by emitted push. */
20820 }
20822 /* We need to do copy in the right order in case an address register
20823 of the source overlaps the destination. */
20825 {
20826 rtx tmp;
20829 {
20833 collisions++;
20834 }
20836 /* Collision in the middle part can be handled by reordering. */
20838 {
20841 }
20845 {
20847 {
20850 }
20851 else
20852 {
20855 }
20856 }
20858 /* If there are more collisions, we can't handle it by reordering.
20859 Do an lea to the last part and use only one colliding move. */
20861 {
20862 rtx base;
20868 /* Handle the case when the last part isn't valid for lea.
20869 Happens in 64-bit mode storing the 12-byte XFmode. */
20876 {
20879 }
20880 }
20881 }
20884 {
20886 {
20888 {
20893 }
20895 {
20898 }
20899 }
20900 else
20901 {
20902 /* In 64bit mode we don't have 32bit push available. In case this is
20903 register, it is OK - we will just use larger counterpart. We also
20904 retype memory - these comes from attempt to avoid REX prefix on
20905 moving of second half of TFmode value. */
20907 {
20909 {
20920 }
20924 }
20925 }
20929 }
20931 /* Choose correct order to not overwrite the source before it is copied. */
20941 {
20943 {
20946 }
20947 }
20948 else
20949 {
20951 {
20954 }
20955 }
20957 /* If optimizing for size, attempt to locally unCSE nonzero constants. */
20959 {
20968 }
20974 }
20976 /* Helper function of ix86_split_ashl used to generate an SImode/DImode
20977 left shift by a constant, either using a single shift or
20978 a sequence of add instructions. */
20980 static void
20982 {
20988 {
20992 }
20993 else
20994 {
20997 }
20998 }
21000 void
21002 {
21011 {
21016 {
21022 }
21023 else
21024 {
21032 }
21034 }
21041 {
21042 /* Assuming we've chosen a QImode capable registers, then 1 << N
21043 can be done with two 32/64-bit shifts, no branches, no cmoves. */
21045 {
21061 }
21063 /* Otherwise, we can get the same results by manually performing
21064 a bit extract operation on bit 5/6, and then performing the two
21065 shifts. The two methods of getting 0/1 into low/high are exactly
21066 the same size. Avoiding the shift in the bit extract case helps
21067 pentium4 a bit; no one else seems to care much either way. */
21068 else
21069 {
21074 HOST_WIDE_INT bits;
21075 rtx x;
21078 {
21084 }
21085 else
21086 {
21092 }
21096 else
21104 }
21109 }
21112 {
21113 /* For -1 << N, we can avoid the shld instruction, because we
21114 know that we're shifting 0...31/63 ones into a -1. */
21118 else
21120 }
21121 else
21122 {
21130 }
21135 {
21141 }
21142 else
21143 {
21148 }
21149 }
21151 void
21153 {
21163 {
21168 {
21174 }
21176 {
21185 }
21186 else
21187 {
21195 }
21196 }
21197 else
21198 {
21210 {
21218 scratch));
21219 }
21220 else
21221 {
21226 }
21227 }
21228 }
21230 void
21232 {
21242 {
21247 {
21254 }
21255 else
21256 {
21264 }
21265 }
21266 else
21267 {
21279 {
21285 scratch));
21286 }
21287 else
21288 {
21293 }
21294 }
21295 }
21297 /* Predict just emitted jump instruction to be taken with probability PROB. */
21298 static void
21300 {
21304 }
21306 /* Helper function for the string operations below. Dest VARIABLE whether
21307 it is aligned to VALUE bytes. If true, jump to the label. */
21308 static rtx
21310 {
21315 else
21321 else
21324 }
21326 /* Adjust COUNTER by the VALUE. */
21327 static void
21329 {
21334 }
21336 /* Zero extend possibly SImode EXP to Pmode register. */
21337 rtx
21339 {
21343 }
21345 /* Divide COUNTREG by SCALE. */
21346 static rtx
21348 {
21349 rtx sc;
21361 }
21363 /* Return mode for the memcpy/memset loop counter. Prefer SImode over
21364 DImode for constant loop counts. */
21366 static enum machine_mode
21368 {
21376 }
21378 /* When SRCPTR is non-NULL, output simple loop to move memory
21379 pointer to SRCPTR to DESTPTR via chunks of MODE unrolled UNROLL times,
21380 overall size is COUNT specified in bytes. When SRCPTR is NULL, output the
21381 equivalent loop to set memory by VALUE (supposed to be in MODE).
21383 The size is rounded down to whole number of chunk size moved at once.
21384 SRCMEM and DESTMEM provide MEMrtx to feed proper aliasing info. */
21387 static void
21392 {
21397 rtx size;
21398 rtx x_addr;
21399 rtx y_addr;
21408 /* Those two should combine. */
21410 {
21414 }
21424 {
21428 /* When unrolling for chips that reorder memory reads and writes,
21429 we can save registers by using single temporary.
21430 Also using 4 temporaries is overkill in 32bit mode. */
21432 {
21434 {
21436 {
21437 destmem =
21439 srcmem =
21441 }
21443 }
21444 }
21445 else
21446 {
21450 {
21453 {
21454 srcmem =
21456 }
21458 }
21460 {
21462 {
21463 destmem =
21465 }
21467 }
21468 }
21469 }
21470 else
21472 {
21474 destmem =
21477 }
21487 {
21493 else
21495 }
21496 else
21504 {
21509 }
21511 }
21513 /* Output "rep; mov" instruction.
21514 Arguments have same meaning as for previous function */
21515 static void
21518 rtx count,
21520 {
21521 rtx destexp;
21522 rtx srcexp;
21523 rtx countreg;
21524 HOST_WIDE_INT rounded_count;
21526 /* If the size is known, it is shorter to use rep movs. */
21537 {
21544 }
21545 else
21546 {
21549 }
21551 {
21558 }
21559 else
21560 {
21565 }
21568 }
21570 /* Output "rep; stos" instruction.
21571 Arguments have same meaning as for previous function */
21572 static void
21575 rtx orig_value)
21576 {
21577 rtx destexp;
21578 rtx countreg;
21579 HOST_WIDE_INT rounded_count;
21586 {
21590 }
21591 else
21594 {
21599 }
21603 }
21605 static void
21608 {
21612 }
21614 /* Output code to copy at most count & (max_size - 1) bytes from SRC to DEST. */
21615 static void
21618 {
21621 {
21626 {
21628 {
21631 }
21632 else
21635 }
21637 {
21640 else
21641 {
21644 }
21646 }
21648 {
21651 }
21653 {
21656 }
21658 {
21661 }
21663 }
21665 {
21671 }
21673 /* When there are stringops, we can cheaply increase dest and src pointers.
21674 Otherwise we save code size by maintaining offset (zero is readily
21675 available from preceding rep operation) and using x86 addressing modes.
21676 */
21678 {
21680 {
21687 }
21689 {
21696 }
21698 {
21705 }
21706 }
21707 else
21708 {
21710 rtx tmp;
21713 {
21724 }
21726 {
21739 }
21741 {
21750 }
21751 }
21752 }
21754 /* Output code to set at most count & (max_size - 1) bytes starting by DEST. */
21755 static void
21758 {
21759 count =
21765 }
21767 /* Output code to set at most count & (max_size - 1) bytes starting by DEST. */
21768 static void
21770 {
21771 rtx dest;
21774 {
21779 {
21781 {
21786 }
21787 else
21790 }
21792 {
21794 {
21797 }
21798 else
21799 {
21804 }
21806 }
21808 {
21812 }
21814 {
21818 }
21820 {
21824 }
21826 }
21828 {
21831 }
21833 {
21836 {
21840 }
21841 else
21842 {
21848 }
21851 }
21853 {
21856 {
21859 }
21860 else
21861 {
21865 }
21868 }
21870 {
21876 }
21878 {
21884 }
21886 {
21892 }
21893 }
21895 /* Copy enough from DEST to SRC to align DEST known to by aligned by ALIGN to
21896 DESIRED_ALIGNMENT. */
21897 static void
21901 {
21903 {
21911 }
21913 {
21921 }
21923 {
21931 }
21933 }
21935 /* Copy enough from DST to SRC to align DST known to DESIRED_ALIGN.
21936 ALIGN_BYTES is how many bytes need to be copied. */
21937 static rtx
21940 {
21949 {
21954 }
21956 {
21967 }
21969 {
21975 {
21983 }
21986 }
21992 {
22004 }
22011 }
22013 /* Set enough from DEST to align DEST known to by aligned by ALIGN to
22014 DESIRED_ALIGNMENT. */
22015 static void
22018 {
22020 {
22027 }
22029 {
22036 }
22038 {
22045 }
22047 }
22049 /* Set enough from DST to align DST known to by aligned by ALIGN to
22050 DESIRED_ALIGN. ALIGN_BYTES is how many bytes need to be stored. */
22051 static rtx
22054 {
22058 {
22063 }
22065 {
22072 }
22074 {
22081 }
22088 }
22090 /* Given COUNT and EXPECTED_SIZE, decide on codegen of string operation. */
22091 static enum stringop_alg
22094 {
22097 /* Algorithms using the rep prefix want at least edi and ecx;
22098 additionally, memset wants eax and memcpy wants esi. Don't
22099 consider such algorithms if the user has appropriated those
22100 registers for their own purposes. */
22102 || (memset
22105 #define ALG_USABLE_P(alg) (rep_prefix_usable \
22106 || (alg != rep_prefix_1_byte \
22107 && alg != rep_prefix_4_byte \
22108 && alg != rep_prefix_8_byte))
22111 /* Even if the string operation call is cold, we still might spend a lot
22112 of time processing large blocks. */
22117 else
22125 else
22129 /* rep; movq or rep; movl is the smallest variant. */
22131 {
22134 else
22136 }
22137 /* Very tiny blocks are best handled via the loop, REP is expensive to setup.
22138 */
22142 {
22146 {
22147 /* We get here if the algorithms that were not libcall-based
22148 were rep-prefix based and we are unable to use rep prefixes
22149 based on global register usage. Break out of the loop and
22150 use the heuristic below. */
22154 {
22159 /* Honor TARGET_INLINE_ALL_STRINGOPS by picking
22160 last non-libcall inline algorithm. */
22162 {
22163 /* When the current size is best to be copied by a libcall,
22164 but we are still forced to inline, run the heuristic below
22165 that will pick code for medium sized blocks. */
22169 }
22172 }
22173 }
22175 }
22176 /* When asked to inline the call anyway, try to pick meaningful choice.
22177 We look for maximal size of block that is faster to copy by hand and
22178 take blocks of at most of that size guessing that average size will
22179 be roughly half of the block.
22181 If this turns out to be bad, we might simply specify the preferred
22182 choice in ix86_costs. */
22185 {
22192 {
22199 }
22200 /* If there aren't any usable algorithms, then recursing on
22201 smaller sizes isn't going to find anything. Just return the
22202 simple byte-at-a-time copy loop. */
22204 {
22205 /* Pick something reasonable. */
22209 }
22218 }
22220 #undef ALG_USABLE_P
22221 }
22223 /* Decide on alignment. We know that the operand is already aligned to ALIGN
22224 (ALIGN can be based on profile feedback and thus it is not 100% guaranteed). */
22225 static int
22229 {
22232 {
22243 /* PentiumPro has special logic triggering for 8 byte aligned blocks.
22244 copying whole cacheline at once. */
22247 else
22251 /* PentiumPro has special logic triggering for 8 byte aligned blocks.
22252 copying whole cacheline at once. */
22255 else
22263 }
22272 }
22274 /* Return the smallest power of 2 greater than VAL. */
22275 static int
22277 {
22282 }
22284 /* Expand string move (memcpy) operation. Use i386 string operations
22285 when profitable. expand_setmem contains similar code. The code
22286 depends upon architecture, block size and alignment, but always has
22287 the same overall structure:
22289 1) Prologue guard: Conditional that jumps up to epilogues for small
22290 blocks that can be handled by epilogue alone. This is faster
22291 but also needed for correctness, since prologue assume the block
22292 is larger than the desired alignment.
22294 Optional dynamic check for size and libcall for large
22295 blocks is emitted here too, with -minline-stringops-dynamically.
22297 2) Prologue: copy first few bytes in order to get destination
22298 aligned to DESIRED_ALIGN. It is emitted only when ALIGN is less
22299 than DESIRED_ALIGN and up to DESIRED_ALIGN - ALIGN bytes can be
22300 copied. We emit either a jump tree on power of two sized
22301 blocks, or a byte loop.
22303 3) Main body: the copying loop itself, copying in SIZE_NEEDED chunks
22304 with specified algorithm.
22306 4) Epilogue: code copying tail of the block that is too small to be
22307 handled by main body (or up to size guarded by prologue guard). */
22309 bool
22312 {
22313 rtx destreg;
22314 rtx srcreg;
22316 rtx tmp;
22329 /* i386 can do misaligned access on reasonably increased cost. */
22333 /* ALIGN is the minimum of destination and source alignment, but we care here
22334 just about destination alignment. */
22343 /* Make sure we don't need to care about overflow later on. */
22347 /* Step 0: Decide on preferred algorithm, desired alignment and
22348 size of chunks to be copied by main loop. */
22364 {
22389 }
22393 /* Step 1: Prologue guard. */
22395 /* Alignment code needs count to be in register. */
22397 {
22400 {
22401 align_bytes
22405 else
22407 }
22410 }
22413 /* Ensure that alignment prologue won't copy past end of block. */
22415 {
22417 /* Epilogue always copies COUNT_EXP & EPILOGUE_SIZE_NEEDED bytes.
22418 Make sure it is power of 2. */
22422 {
22424 {
22425 /* If main algorithm works on QImode, no epilogue is needed.
22426 For small sizes just don't align anything. */
22429 else
22431 }
22432 }
22433 else
22434 {
22441 else
22443 }
22444 }
22446 /* Emit code to decide on runtime whether library call or inline should be
22447 used. */
22449 {
22451 {
22453 {
22457 }
22458 }
22459 else
22460 {
22469 }
22470 }
22472 /* Step 2: Alignment prologue. */
22475 {
22477 {
22478 /* Except for the first move in epilogue, we no longer know
22479 constant offset in aliasing info. It don't seems to worth
22480 the pain to maintain it for the first move, so throw away
22481 the info early. */
22485 desired_align);
22486 }
22487 else
22488 {
22489 /* If we know how many bytes need to be stored before dst is
22490 sufficiently aligned, maintain aliasing info accurately. */
22496 }
22502 {
22503 /* It is possible that we copied enough so the main loop will not
22504 execute. */
22514 else
22516 }
22517 }
22519 {
22524 }
22528 /* Step 3: Main loop. */
22531 {
22544 /* Unroll only by factor of 2 in 32bit mode, since we don't have enough
22545 registers for 4 temporaries anyway. */
22548 expected_size);
22552 DImode);
22556 SImode);
22560 QImode);
22562 }
22563 /* Adjust properly the offset of src and dest memory for aliasing. */
22565 {
22570 }
22571 else
22572 {
22575 }
22577 /* Step 4: Epilogue to copy the remaining bytes. */
22578 epilogue:
22580 {
22581 /* When the main loop is done, COUNT_EXP might hold original count,
22582 while we want to copy only COUNT_EXP & SIZE_NEEDED bytes.
22583 Epilogue code will actually copy COUNT_EXP & EPILOGUE_SIZE_NEEDED
22584 bytes. Compensate if needed. */
22587 {
22588 tmp =
22591 OPTAB_DIRECT);
22594 }
22597 }
22601 epilogue_size_needed);
22605 }
22607 /* Helper function for memcpy. For QImode value 0xXY produce
22608 0xXYXYXYXY of wide specified by MODE. This is essentially
22609 a * 0x10101010, but we can do slightly better than
22610 synth_mult by unwinding the sequence by hand on CPUs with
22611 slow multiply. */
22612 static rtx
22614 {
22616 rtx tmp;
22623 {
22631 }
22640 nops--;
22645 {
22649 OPTAB_DIRECT);
22650 }
22651 else
22652 {
22658 else
22660 else
22661 {
22664 reg =
22666 }
22676 }
22677 }
22679 /* Duplicate value VAL using promote_duplicated_reg into maximal size that will
22680 be needed by main loop copying SIZE_NEEDED chunks and prologue getting
22681 alignment from ALIGN to DESIRED_ALIGN. */
22682 static rtx
22684 {
22685 rtx promoted_val;
22694 else
22698 }
22700 /* Expand string clear operation (bzero). Use i386 string operations when
22701 profitable. See expand_movmem comment for explanation of individual
22702 steps performed. */
22703 bool
22706 {
22707 rtx destreg;
22709 rtx tmp;
22724 /* i386 can do misaligned access on reasonably increased cost. */
22733 /* Make sure we don't need to care about overflow later on. */
22737 /* Step 0: Decide on preferred algorithm, desired alignment and
22738 size of chunks to be copied by main loop. */
22753 {
22778 }
22781 /* Step 1: Prologue guard. */
22783 /* Alignment code needs count to be in register. */
22785 {
22788 {
22789 align_bytes
22793 else
22795 }
22797 {
22802 }
22803 }
22804 /* Do the cheap promotion to allow better CSE across the
22805 main loop and epilogue (ie one load of the big constant in the
22806 front of all code. */
22810 /* Ensure that alignment prologue won't copy past end of block. */
22812 {
22814 /* Epilogue always copies COUNT_EXP & (EPILOGUE_SIZE_NEEDED - 1) bytes.
22815 Make sure it is power of 2. */
22818 /* To improve performance of small blocks, we jump around the VAL
22819 promoting mode. This mean that if the promoted VAL is not constant,
22820 we might not use it in the epilogue and have to use byte
22821 loop variant. */
22825 {
22827 {
22828 /* If main algorithm works on QImode, no epilogue is needed.
22829 For small sizes just don't align anything. */
22832 else
22834 }
22835 }
22836 else
22837 {
22844 else
22846 }
22847 }
22849 {
22858 }
22860 /* Step 2: Alignment prologue. */
22862 /* Do the expensive promotion once we branched off the small blocks. */
22869 {
22871 {
22872 /* Except for the first move in epilogue, we no longer know
22873 constant offset in aliasing info. It don't seems to worth
22874 the pain to maintain it for the first move, so throw away
22875 the info early. */
22878 desired_align);
22879 }
22880 else
22881 {
22882 /* If we know how many bytes need to be stored before dst is
22883 sufficiently aligned, maintain aliasing info accurately. */
22889 }
22895 {
22896 /* It is possible that we copied enough so the main loop will not
22897 execute. */
22907 else
22909 }
22910 }
22912 {
22918 }
22922 /* Step 3: Main loop. */
22925 {
22953 }
22954 /* Adjust properly the offset of src and dest memory for aliasing. */
22958 else
22961 /* Step 4: Epilogue to copy the remaining bytes. */
22964 {
22965 /* When the main loop is done, COUNT_EXP might hold original count,
22966 while we want to copy only COUNT_EXP & SIZE_NEEDED bytes.
22967 Epilogue code will actually copy COUNT_EXP & EPILOGUE_SIZE_NEEDED
22968 bytes. Compensate if needed. */
22971 {
22972 tmp =
22975 OPTAB_DIRECT);
22978 }
22981 }
22982 epilogue:
22984 {
22987 epilogue_size_needed);
22988 else
22990 epilogue_size_needed);
22991 }
22995 }
22997 /* Expand the appropriate insns for doing strlen if not just doing
22998 repnz; scasb
23000 out = result, initialized with the start address
23001 align_rtx = alignment of the address.
23002 scratch = scratch register, initialized with the startaddress when
23003 not aligned, otherwise undefined
23005 This is just the body. It needs the initializations mentioned above and
23006 some address computing at the end. These things are done in i386.md. */
23008 static void
23010 {
23012 rtx tmp;
23017 rtx mem;
23020 rtx cmp;
23026 /* Loop to check 1..3 bytes for null to get an aligned pointer. */
23028 /* Is there a known alignment and is it less than 4? */
23030 {
23033 /* Is there a known alignment and is it not 2? */
23035 {
23039 /* Leave just the 3 lower bits. */
23049 }
23050 else
23051 {
23052 /* Since the alignment is 2, we have to check 2 or 0 bytes;
23053 check if is aligned to 4 - byte. */
23060 }
23064 /* Now compare the bytes. */
23066 /* Compare the first n unaligned byte on a byte per byte basis. */
23070 /* Increment the address. */
23073 /* Not needed with an alignment of 2 */
23075 {
23079 end_0_label);
23084 }
23087 end_0_label);
23090 }
23092 /* Generate loop to check 4 bytes at a time. It is not a good idea to
23093 align this loop. It gives only huge programs, but does not help to
23094 speed up. */
23101 /* This formula yields a nonzero result iff one of the bytes is zero.
23102 This saves three branches inside loop and many cycles. */
23110 align_4_label);
23113 {
23119 /* If zero is not in the first two bytes, move two bytes forward. */
23125 reg,
23126 tmpreg)));
23127 /* Emit lea manually to avoid clobbering of flags. */
23135 reg2,
23136 out)));
23137 }
23138 else
23139 {
23141 /* Is zero in the first two bytes? */
23148 pc_rtx);
23152 /* Not in the first two. Move two bytes forward. */
23158 }
23160 /* Avoid branch in fixing the byte. */
23168 }
23170 /* Expand strlen. */
23172 bool
23174 {
23177 /* The generic case of strlen expander is long. Avoid it's
23178 expanding unless TARGET_INLINE_ALL_STRINGOPS. */
23181 && !TARGET_INLINE_ALL_STRINGOPS
23191 {
23192 /* Well it seems that some optimizer does not combine a call like
23193 foo(strlen(bar), strlen(bar));
23194 when the move and the subtraction is done here. It does calculate
23195 the length just once when these instructions are done inside of
23196 output_strlen_unroll(). But I think since &bar[strlen(bar)] is
23197 often used and I use one fewer register for the lifetime of
23198 output_strlen_unroll() this is better. */
23204 /* strlensi_unroll_1 returns the address of the zero at the end of
23205 the string, like memchr(), so compute the length by subtracting
23206 the start address. */
23208 }
23209 else
23210 {
23211 rtx unspec;
23213 /* Can't use this if the user has appropriated eax, ecx, or edi. */
23226 /* If .md starts supporting :P, this can be done in .md. */
23232 }
23234 }
23236 /* For given symbol (function) construct code to compute address of it's PLT
23237 entry in large x86-64 PIC model. */
23238 static rtx
23240 {
23253 }
23255 rtx
23257 rtx callarg2,
23259 {
23260 /* We need to represent that SI and DI registers are clobbered
23261 by SYSV calls. */
23267 };
23277 {
23278 #if TARGET_MACHO
23281 #endif
23282 }
23283 else
23284 {
23285 /* Static functions and indirect calls don't need the pic register. */
23290 }
23293 {
23297 }
23307 {
23312 }
23321 {
23325 }
23329 {
23333 UNSPEC_MS_TO_SYSV_CALL);
23342 }
23344 /* Add UNSPEC_CALL_NEEDS_VZEROUPPER decoration. */
23346 {
23349 {
23352 else
23354 }
23357 else
23362 else
23365 UNSPEC_CALL_NEEDS_VZEROUPPER);
23366 }
23375 }
23377 void
23379 {
23384 /* Strip off the last entry of the parallel. */
23389 else
23394 }
23396 /* Output the assembly for a call instruction. */
23400 {
23406 {
23409 /* SEH epilogue detection requires the indirect branch case
23410 to include REX.W. */
23413 else
23418 }
23420 /* SEH unwinding can require an extra nop to be emitted in several
23421 circumstances. Determine if we have one of those. */
23423 {
23424 rtx i;
23427 {
23428 /* If we get to another real insn, we don't need the nop. */
23432 /* If we get to the epilogue note, prevent a catch region from
23433 being adjacent to the standard epilogue sequence. If non-
23434 call-exceptions, we'll have done this during epilogue emission. */
23436 && !flag_non_call_exceptions
23438 {
23441 }
23442 }
23444 /* If we didn't find a real insn following the call, prevent the
23445 unwinder from looking into the next function. */
23448 }
23452 else
23461 }
23462 \f
23463 /* Clear stack slot assignments remembered from previous functions.
23464 This is called from INIT_EXPANDERS once before RTL is emitted for each
23465 function. */
23469 {
23478 }
23480 /* Return a MEM corresponding to a stack slot with mode MODE.
23481 Allocate a new slot if necessary.
23483 The RTL for a function can have several slots available: N is
23484 which slot to use. */
23486 rtx
23488 {
23493 /* Virtual slot is valid only before vregs are instantiated. */
23508 }
23509 \f
23510 /* Calculate the length of the memory address in the instruction encoding.
23511 Includes addr32 prefix, does not include the one-byte modrm, opcode,
23512 or other prefixes. */
23514 int
23516 {
23540 /* Add length of addr32 prefix. */
23544 /* Rule of thumb:
23545 - esp as the base always wants an index,
23546 - ebp as the base always wants a displacement,
23547 - r12 as the base always wants an index,
23548 - r13 as the base always wants a displacement. */
23550 /* Register Indirect. */
23552 {
23553 /* esp (for its index) and ebp (for its displacement) need
23554 the two-byte modrm form. Similarly for r12 and r13 in 64-bit
23555 code. */
23564 }
23566 /* Direct Addressing. In 64-bit mode mod 00 r/m 5
23567 is not disp32, but disp32(%rip), so for disp32
23568 SIB byte is needed, unless print_operand_address
23569 optimizes it into disp32(%rip) or (%rip) is implied
23570 by UNSPEC. */
23572 {
23575 {
23592 }
23593 }
23595 else
23596 {
23597 /* Find the length of the displacement constant. */
23599 {
23602 else
23604 }
23605 /* ebp always wants a displacement. Similarly r13. */
23610 /* An index requires the two-byte modrm form.... */
23612 /* ...like esp (or r12), which always wants an index. */
23618 }
23621 {
23628 }
23631 }
23633 /* Compute default value for "length_immediate" attribute. When SHORTFORM
23634 is set, expect that insn have 8bit immediate alternative. */
23635 int
23637 {
23643 {
23648 {
23651 {
23663 }
23665 {
23668 }
23669 }
23671 {
23681 /* Immediates for DImode instructions are encoded as 32bit sign extended values. */
23687 }
23688 }
23690 }
23691 /* Compute default value for "length_address" attribute. */
23692 int
23694 {
23698 {
23708 {
23713 }
23716 }
23721 {
23724 {
23729 constraints++;
23732 ;
23733 /* Skip ignored operands. */
23736 }
23738 }
23740 }
23742 /* Compute default value for "length_vex" attribute. It includes
23743 2 or 3 byte VEX prefix and 1 opcode byte. */
23745 int
23747 {
23750 /* Only 0f opcode can use 2 byte VEX prefix and VEX W bit uses 3
23751 byte VEX prefix. */
23755 /* We can always use 2 byte VEX prefix in 32bit. */
23763 {
23764 /* REX.W bit uses 3 byte VEX prefix. */
23768 }
23769 else
23770 {
23771 /* REX.X or REX.B bits use 3 byte VEX prefix. */
23775 }
23778 }
23779 \f
23780 /* Return the maximum number of instructions a cpu can issue. */
23782 static int
23784 {
23786 {
23812 }
23813 }
23815 /* A subroutine of ix86_adjust_cost -- return TRUE iff INSN reads flags set
23816 by DEP_INSN and nothing set by DEP_INSN. */
23818 static bool
23820 {
23823 /* Simplify the test for uninteresting insns. */
23831 {
23834 }
23839 {
23842 }
23843 else
23849 /* This test is true if the dependent insn reads the flags but
23850 not any other potentially set register. */
23858 }
23860 /* Return true iff USE_INSN has a memory address with operands set by
23861 SET_INSN. */
23863 bool
23865 {
23870 {
23873 }
23875 }
23877 static int
23879 {
23885 /* Anti and output dependencies have zero cost on all CPUs. */
23891 /* If we can't recognize the insns, we can't really do anything. */
23899 {
23901 /* Address Generation Interlock adds a cycle of latency. */
23903 {
23914 }
23918 /* ??? Compares pair with jump/setcc. */
23922 /* Floating point stores require value to be ready one cycle earlier. */
23932 /* INT->FP conversion is expensive. */
23936 /* There is one cycle extra latency between an FP op and a store. */
23944 /* Show ability of reorder buffer to hide latency of load by executing
23945 in parallel with previous instruction in case
23946 previous instruction is not needed to compute the address. */
23949 {
23950 /* Claim moves to take one cycle, as core can issue one load
23951 at time and the next load can start cycle later. */
23956 cost--;
23957 }
23963 /* The esp dependency is resolved before the instruction is really
23964 finished. */
23969 /* INT->FP conversion is expensive. */
23973 /* Show ability of reorder buffer to hide latency of load by executing
23974 in parallel with previous instruction in case
23975 previous instruction is not needed to compute the address. */
23978 {
23979 /* Claim moves to take one cycle, as core can issue one load
23980 at time and the next load can start cycle later. */
23986 else
23988 }
24003 /* Show ability of reorder buffer to hide latency of load by executing
24004 in parallel with previous instruction in case
24005 previous instruction is not needed to compute the address. */
24008 {
24012 /* Because of the difference between the length of integer and
24013 floating unit pipeline preparation stages, the memory operands
24014 for floating point are cheaper.
24016 ??? For Athlon it the difference is most probably 2. */
24019 else
24024 else
24026 }
24030 }
24033 }
24035 /* How many alternative schedules to try. This should be as wide as the
24036 scheduling freedom in the DFA, but no wider. Making this value too
24037 large results extra work for the scheduler. */
24039 static int
24041 {
24043 {
24055 /* Generally, we want haifa-sched:max_issue() to look ahead as far
24056 as many instructions can be executed on a cycle, i.e.,
24057 issue_rate. I wonder why tuning for many CPUs does not do this. */
24062 }
24063 }
24065 /* Try to reorder ready list to take advantage of Atom pipelined IMUL
24066 execution. It is applied if
24067 (1) IMUL instruction is on the top of list;
24068 (2) There exists the only producer of independent IMUL instruction in
24069 ready list;
24070 (3) Put found producer on the top of ready list.
24071 Returns issue rate. */
24073 static int
24076 {
24081 sd_iterator_def sd_it;
24082 dep_t dep;
24085 /* Set up issue rate. */
24088 /* Do reodering for Atom only. */
24091 /* Nothing to do if ready list contains only 1 instruction. */
24095 /* Check that IMUL instruction is on the top of ready list. */
24108 /* Search for producer of independent IMUL instruction. */
24110 {
24114 /* Skip IMUL instruction. */
24124 {
24125 rtx con;
24136 {
24137 sd_iterator_def sd_it1;
24138 dep_t dep1;
24139 /* Check if there is no other dependee for IMUL. */
24142 {
24143 rtx pro;
24149 }
24152 }
24153 }
24156 }
24164 /* Put IMUL producer (ready[index]) at the top of ready list. */
24171 }
24173 \f
24175 /* Model decoder of Core 2/i7.
24176 Below hooks for multipass scheduling (see haifa-sched.c:max_issue)
24177 track the instruction fetch block boundaries and make sure that long
24178 (9+ bytes) instructions are assigned to D0. */
24180 /* Maximum length of an insn that can be handled by
24181 a secondary decoder unit. '8' for Core 2/i7. */
24184 /* Ifetch block size, i.e., number of bytes decoder reads per cycle.
24185 '16' for Core 2/i7. */
24188 /* Maximum number of instructions decoder can handle per cycle.
24189 '6' for Core 2/i7. */
24193 ix86_first_cycle_multipass_data_t;
24195 const_ix86_first_cycle_multipass_data_t;
24197 /* A variable to store target state across calls to max_issue within
24198 one cycle. */
24202 /* Initialize DATA. */
24203 static void
24205 {
24206 ix86_first_cycle_multipass_data_t data
24213 }
24215 /* Advancing the cycle; reset ifetch block counts. */
24216 static void
24218 {
24225 }
24229 /* Filter out insns from ready_try that the core will not be able to issue
24230 on current cycle due to decoder. */
24231 static void
24232 core2i7_first_cycle_multipass_filter_ready_try
24235 {
24237 {
24238 rtx insn;
24248 (!first_cycle_insn_p
24250 /* ... or it would not fit into the ifetch block ... */
24252 /* ... or the decoder is full already ... */
24254 /* ... mask the insn out. */
24255 {
24260 }
24261 }
24262 }
24264 /* Prepare for a new round of multipass lookahead scheduling. */
24265 static void
24268 {
24269 ix86_first_cycle_multipass_data_t data
24271 const_ix86_first_cycle_multipass_data_t prev_data
24274 /* Restore the state from the end of the previous round. */
24278 /* Filter instructions that cannot be issued on current cycle due to
24279 decoder restrictions. */
24281 first_cycle_insn_p);
24282 }
24284 /* INSN is being issued in current solution. Account for its impact on
24285 the decoder model. */
24286 static void
24289 {
24290 ix86_first_cycle_multipass_data_t data
24292 const_ix86_first_cycle_multipass_data_t prev_data
24302 /* Allocate or resize the bitmap for storing INSN's effect on ready_try. */
24304 {
24307 }
24309 {
24313 }
24316 /* Filter out insns from ready_try that the core will not be able to issue
24317 on current cycle due to decoder. */
24320 }
24322 /* Revert the effect on ready_try. */
24323 static void
24327 {
24328 const_ix86_first_cycle_multipass_data_t data
24331 sbitmap_iterator sbi;
24335 {
24337 }
24338 }
24340 /* Save the result of multipass lookahead scheduling for the next round. */
24341 static void
24343 {
24344 const_ix86_first_cycle_multipass_data_t data
24346 ix86_first_cycle_multipass_data_t next_data
24350 {
24353 }
24354 }
24356 /* Deallocate target data. */
24357 static void
24359 {
24360 ix86_first_cycle_multipass_data_t data
24364 {
24368 }
24369 }
24371 /* Prepare for scheduling pass. */
24372 static void
24376 {
24377 /* Install scheduling hooks for current CPU. Some of these hooks are used
24378 in time-critical parts of the scheduler, so we only set them up when
24379 they are actually used. */
24381 {
24401 /* Set decoder parameters. */
24416 }
24417 }
24419 \f
24420 /* Compute the alignment given to a constant that is being placed in memory.
24421 EXP is the constant and ALIGN is the alignment that the object would
24422 ordinarily have.
24423 The value of this function is used instead of that alignment to align
24424 the object. */
24426 int
24428 {
24431 {
24436 }
24442 }
24444 /* Compute the alignment for a static variable.
24445 TYPE is the data type, and ALIGN is the alignment that
24446 the object would ordinarily have. The value of this function is used
24447 instead of that alignment to align the object. */
24449 int
24451 {
24462 /* x86-64 ABI requires arrays greater than 16 bytes to be aligned
24463 to 16byte boundary. */
24465 {
24472 }
24475 {
24480 }
24482 {
24489 }
24494 {
24499 }
24502 {
24507 }
24510 }
24512 /* Compute the alignment for a local variable or a stack slot. EXP is
24513 the data type or decl itself, MODE is the widest mode available and
24514 ALIGN is the alignment that the object would ordinarily have. The
24515 value of this macro is used instead of that alignment to align the
24516 object. */
24518 unsigned int
24521 {
24525 {
24528 }
24529 else
24530 {
24533 }
24535 /* Don't do dynamic stack realignment for long long objects with
24536 -mpreferred-stack-boundary=2. */
24545 /* If TYPE is NULL, we are allocating a stack slot for caller-save
24546 register in MODE. We will return the largest alignment of XF
24547 and DF. */
24549 {
24553 }
24555 /* x86-64 ABI requires arrays greater than 16 bytes to be aligned
24556 to 16byte boundary. Exact wording is:
24558 An array uses the same alignment as its elements, except that a local or
24559 global array variable of length at least 16 bytes or
24560 a C99 variable-length array variable always has alignment of at least 16 bytes.
24562 This was added to allow use of aligned SSE instructions at arrays. This
24563 rule is meant for static storage (where compiler can not do the analysis
24564 by itself). We follow it for automatic variables only when convenient.
24565 We fully control everything in the function compiled and functions from
24566 other unit can not rely on the alignment.
24568 Exclude va_list type. It is the common case of local array where
24569 we can not benefit from the alignment. */
24572 {
24582 }
24584 {
24589 }
24591 {
24597 }
24602 {
24607 }
24610 {
24616 }
24618 }
24620 /* Compute the minimum required alignment for dynamic stack realignment
24621 purposes for a local variable, parameter or a stack slot. EXP is
24622 the data type or decl itself, MODE is its mode and ALIGN is the
24623 alignment that the object would ordinarily have. */
24625 unsigned int
24628 {
24632 {
24635 }
24636 else
24637 {
24640 }
24645 /* Don't do dynamic stack realignment for long long objects with
24646 -mpreferred-stack-boundary=2. */
24653 }
24654 \f
24655 /* Find a location for the static chain incoming to a nested function.
24656 This is a register, unless all free registers are used by arguments. */
24658 static rtx
24660 {
24667 {
24668 /* We always use R10 in 64-bit mode. */
24670 }
24671 else
24672 {
24673 tree fntype;
24676 /* By default in 32-bit mode we use ECX to pass the static chain. */
24682 {
24683 /* Fastcall functions use ecx/edx for arguments, which leaves
24684 us with EAX for the static chain.
24685 Thiscall functions use ecx for arguments, which also
24686 leaves us with EAX for the static chain. */
24688 }
24690 {
24691 /* For regparm 3, we have no free call-clobbered registers in
24692 which to store the static chain. In order to implement this,
24693 we have the trampoline push the static chain to the stack.
24694 However, we can't push a value below the return address when
24695 we call the nested function directly, so we have to use an
24696 alternate entry point. For this we use ESI, and have the
24697 alternate entry point push ESI, so that things appear the
24698 same once we're executing the nested function. */
24700 {
24706 }
24708 }
24709 }
24712 }
24714 /* Emit RTL insns to initialize the variable parts of a trampoline.
24715 FNDECL is the decl of the target address; M_TRAMP is a MEM for
24716 the trampoline, and CHAIN_VALUE is an RTX for the static chain
24717 to be passed to the target function. */
24719 static void
24721 {
24729 {
24732 /* Load the function address to r11. Try to load address using
24733 the shorter movl instead of movabs. We may want to support
24734 movq for kernel mode, but kernel does not use trampolines at
24735 the moment. FNADDR is a 32bit address and may not be in
24736 DImode when ptr_mode == SImode. Always use movl in this
24737 case. */
24740 {
24749 }
24750 else
24751 {
24758 }
24760 /* Load static chain using movabs to r10. Use the shorter movl
24761 instead of movabs when ptr_mode == SImode. */
24763 {
24766 }
24767 else
24768 {
24771 }
24780 /* Jump to r11; the last (unused) byte is a nop, only there to
24781 pad the write out to a single 32-bit store. */
24785 }
24786 else
24787 {
24790 /* Depending on the static chain location, either load a register
24791 with a constant, or push the constant to the stack. All of the
24792 instructions are the same size. */
24795 {
24797 {
24804 }
24805 }
24806 else
24821 /* Compute offset from the end of the jmp to the target function.
24822 In the case in which the trampoline stores the static chain on
24823 the stack, we need to skip the first insn which pushes the
24824 (call-saved) register static chain; this push is 1 byte. */
24831 }
24835 #ifdef HAVE_ENABLE_EXECUTE_STACK
24836 #ifdef CHECK_EXECUTE_STACK_ENABLED
24838 #endif
24841 #endif
24842 }
24843 \f
24844 /* The following file contains several enumerations and data structures
24845 built from the definitions in i386-builtin-types.def. */
24849 /* Table for the ix86 builtin non-function types. */
24852 /* Retrieve an element from the above table, building some of
24853 the types lazily. */
24855 static tree
24857 {
24869 {
24877 }
24878 else
24879 {
24885 else
24893 }
24897 }
24899 /* Table for the ix86 builtin function types. */
24902 /* Retrieve an element from the above table, building some of
24903 the types lazily. */
24905 static tree
24907 {
24908 tree type;
24917 {
24925 {
24928 }
24931 }
24932 else
24933 {
24939 }
24943 }
24946 /* Codes for all the SSE/MMX builtins. */
24947 enum ix86_builtins
24948 {
24949 IX86_BUILTIN_ADDPS,
24950 IX86_BUILTIN_ADDSS,
24951 IX86_BUILTIN_DIVPS,
24952 IX86_BUILTIN_DIVSS,
24953 IX86_BUILTIN_MULPS,
24954 IX86_BUILTIN_MULSS,
24955 IX86_BUILTIN_SUBPS,
24956 IX86_BUILTIN_SUBSS,
24958 IX86_BUILTIN_CMPEQPS,
24959 IX86_BUILTIN_CMPLTPS,
24960 IX86_BUILTIN_CMPLEPS,
24961 IX86_BUILTIN_CMPGTPS,
24962 IX86_BUILTIN_CMPGEPS,
24963 IX86_BUILTIN_CMPNEQPS,
24964 IX86_BUILTIN_CMPNLTPS,
24965 IX86_BUILTIN_CMPNLEPS,
24966 IX86_BUILTIN_CMPNGTPS,
24967 IX86_BUILTIN_CMPNGEPS,
24968 IX86_BUILTIN_CMPORDPS,
24969 IX86_BUILTIN_CMPUNORDPS,
24970 IX86_BUILTIN_CMPEQSS,
24971 IX86_BUILTIN_CMPLTSS,
24972 IX86_BUILTIN_CMPLESS,
24973 IX86_BUILTIN_CMPNEQSS,
24974 IX86_BUILTIN_CMPNLTSS,
24975 IX86_BUILTIN_CMPNLESS,
24976 IX86_BUILTIN_CMPNGTSS,
24977 IX86_BUILTIN_CMPNGESS,
24978 IX86_BUILTIN_CMPORDSS,
24979 IX86_BUILTIN_CMPUNORDSS,
24981 IX86_BUILTIN_COMIEQSS,
24982 IX86_BUILTIN_COMILTSS,
24983 IX86_BUILTIN_COMILESS,
24984 IX86_BUILTIN_COMIGTSS,
24985 IX86_BUILTIN_COMIGESS,
24986 IX86_BUILTIN_COMINEQSS,
24987 IX86_BUILTIN_UCOMIEQSS,
24988 IX86_BUILTIN_UCOMILTSS,
24989 IX86_BUILTIN_UCOMILESS,
24990 IX86_BUILTIN_UCOMIGTSS,
24991 IX86_BUILTIN_UCOMIGESS,
24992 IX86_BUILTIN_UCOMINEQSS,
24994 IX86_BUILTIN_CVTPI2PS,
24995 IX86_BUILTIN_CVTPS2PI,
24996 IX86_BUILTIN_CVTSI2SS,
24997 IX86_BUILTIN_CVTSI642SS,
24998 IX86_BUILTIN_CVTSS2SI,
24999 IX86_BUILTIN_CVTSS2SI64,
25000 IX86_BUILTIN_CVTTPS2PI,
25001 IX86_BUILTIN_CVTTSS2SI,
25002 IX86_BUILTIN_CVTTSS2SI64,
25004 IX86_BUILTIN_MAXPS,
25005 IX86_BUILTIN_MAXSS,
25006 IX86_BUILTIN_MINPS,
25007 IX86_BUILTIN_MINSS,
25009 IX86_BUILTIN_LOADUPS,
25010 IX86_BUILTIN_STOREUPS,
25011 IX86_BUILTIN_MOVSS,
25013 IX86_BUILTIN_MOVHLPS,
25014 IX86_BUILTIN_MOVLHPS,
25015 IX86_BUILTIN_LOADHPS,
25016 IX86_BUILTIN_LOADLPS,
25017 IX86_BUILTIN_STOREHPS,
25018 IX86_BUILTIN_STORELPS,
25020 IX86_BUILTIN_MASKMOVQ,
25021 IX86_BUILTIN_MOVMSKPS,
25022 IX86_BUILTIN_PMOVMSKB,
25024 IX86_BUILTIN_MOVNTPS,
25025 IX86_BUILTIN_MOVNTQ,
25027 IX86_BUILTIN_LOADDQU,
25028 IX86_BUILTIN_STOREDQU,
25030 IX86_BUILTIN_PACKSSWB,
25031 IX86_BUILTIN_PACKSSDW,
25032 IX86_BUILTIN_PACKUSWB,
25034 IX86_BUILTIN_PADDB,
25035 IX86_BUILTIN_PADDW,
25036 IX86_BUILTIN_PADDD,
25037 IX86_BUILTIN_PADDQ,
25038 IX86_BUILTIN_PADDSB,
25039 IX86_BUILTIN_PADDSW,
25040 IX86_BUILTIN_PADDUSB,
25041 IX86_BUILTIN_PADDUSW,
25042 IX86_BUILTIN_PSUBB,
25043 IX86_BUILTIN_PSUBW,
25044 IX86_BUILTIN_PSUBD,
25045 IX86_BUILTIN_PSUBQ,
25046 IX86_BUILTIN_PSUBSB,
25047 IX86_BUILTIN_PSUBSW,
25048 IX86_BUILTIN_PSUBUSB,
25049 IX86_BUILTIN_PSUBUSW,
25051 IX86_BUILTIN_PAND,
25052 IX86_BUILTIN_PANDN,
25053 IX86_BUILTIN_POR,
25054 IX86_BUILTIN_PXOR,
25056 IX86_BUILTIN_PAVGB,
25057 IX86_BUILTIN_PAVGW,
25059 IX86_BUILTIN_PCMPEQB,
25060 IX86_BUILTIN_PCMPEQW,
25061 IX86_BUILTIN_PCMPEQD,
25062 IX86_BUILTIN_PCMPGTB,
25063 IX86_BUILTIN_PCMPGTW,
25064 IX86_BUILTIN_PCMPGTD,
25066 IX86_BUILTIN_PMADDWD,
25068 IX86_BUILTIN_PMAXSW,
25069 IX86_BUILTIN_PMAXUB,
25070 IX86_BUILTIN_PMINSW,
25071 IX86_BUILTIN_PMINUB,
25073 IX86_BUILTIN_PMULHUW,
25074 IX86_BUILTIN_PMULHW,
25075 IX86_BUILTIN_PMULLW,
25077 IX86_BUILTIN_PSADBW,
25078 IX86_BUILTIN_PSHUFW,
25080 IX86_BUILTIN_PSLLW,
25081 IX86_BUILTIN_PSLLD,
25082 IX86_BUILTIN_PSLLQ,
25083 IX86_BUILTIN_PSRAW,
25084 IX86_BUILTIN_PSRAD,
25085 IX86_BUILTIN_PSRLW,
25086 IX86_BUILTIN_PSRLD,
25087 IX86_BUILTIN_PSRLQ,
25088 IX86_BUILTIN_PSLLWI,
25089 IX86_BUILTIN_PSLLDI,
25090 IX86_BUILTIN_PSLLQI,
25091 IX86_BUILTIN_PSRAWI,
25092 IX86_BUILTIN_PSRADI,
25093 IX86_BUILTIN_PSRLWI,
25094 IX86_BUILTIN_PSRLDI,
25095 IX86_BUILTIN_PSRLQI,
25097 IX86_BUILTIN_PUNPCKHBW,
25098 IX86_BUILTIN_PUNPCKHWD,
25099 IX86_BUILTIN_PUNPCKHDQ,
25100 IX86_BUILTIN_PUNPCKLBW,
25101 IX86_BUILTIN_PUNPCKLWD,
25102 IX86_BUILTIN_PUNPCKLDQ,
25104 IX86_BUILTIN_SHUFPS,
25106 IX86_BUILTIN_RCPPS,
25107 IX86_BUILTIN_RCPSS,
25108 IX86_BUILTIN_RSQRTPS,
25109 IX86_BUILTIN_RSQRTPS_NR,
25110 IX86_BUILTIN_RSQRTSS,
25111 IX86_BUILTIN_RSQRTF,
25112 IX86_BUILTIN_SQRTPS,
25113 IX86_BUILTIN_SQRTPS_NR,
25114 IX86_BUILTIN_SQRTSS,
25116 IX86_BUILTIN_UNPCKHPS,
25117 IX86_BUILTIN_UNPCKLPS,
25119 IX86_BUILTIN_ANDPS,
25120 IX86_BUILTIN_ANDNPS,
25121 IX86_BUILTIN_ORPS,
25122 IX86_BUILTIN_XORPS,
25124 IX86_BUILTIN_EMMS,
25125 IX86_BUILTIN_LDMXCSR,
25126 IX86_BUILTIN_STMXCSR,
25127 IX86_BUILTIN_SFENCE,
25129 /* 3DNow! Original */
25130 IX86_BUILTIN_FEMMS,
25131 IX86_BUILTIN_PAVGUSB,
25132 IX86_BUILTIN_PF2ID,
25133 IX86_BUILTIN_PFACC,
25134 IX86_BUILTIN_PFADD,
25135 IX86_BUILTIN_PFCMPEQ,
25136 IX86_BUILTIN_PFCMPGE,
25137 IX86_BUILTIN_PFCMPGT,
25138 IX86_BUILTIN_PFMAX,
25139 IX86_BUILTIN_PFMIN,
25140 IX86_BUILTIN_PFMUL,
25141 IX86_BUILTIN_PFRCP,
25142 IX86_BUILTIN_PFRCPIT1,
25143 IX86_BUILTIN_PFRCPIT2,
25144 IX86_BUILTIN_PFRSQIT1,
25145 IX86_BUILTIN_PFRSQRT,
25146 IX86_BUILTIN_PFSUB,
25147 IX86_BUILTIN_PFSUBR,
25148 IX86_BUILTIN_PI2FD,
25149 IX86_BUILTIN_PMULHRW,
25151 /* 3DNow! Athlon Extensions */
25152 IX86_BUILTIN_PF2IW,
25153 IX86_BUILTIN_PFNACC,
25154 IX86_BUILTIN_PFPNACC,
25155 IX86_BUILTIN_PI2FW,
25156 IX86_BUILTIN_PSWAPDSI,
25157 IX86_BUILTIN_PSWAPDSF,
25159 /* SSE2 */
25160 IX86_BUILTIN_ADDPD,
25161 IX86_BUILTIN_ADDSD,
25162 IX86_BUILTIN_DIVPD,
25163 IX86_BUILTIN_DIVSD,
25164 IX86_BUILTIN_MULPD,
25165 IX86_BUILTIN_MULSD,
25166 IX86_BUILTIN_SUBPD,
25167 IX86_BUILTIN_SUBSD,
25169 IX86_BUILTIN_CMPEQPD,
25170 IX86_BUILTIN_CMPLTPD,
25171 IX86_BUILTIN_CMPLEPD,
25172 IX86_BUILTIN_CMPGTPD,
25173 IX86_BUILTIN_CMPGEPD,
25174 IX86_BUILTIN_CMPNEQPD,
25175 IX86_BUILTIN_CMPNLTPD,
25176 IX86_BUILTIN_CMPNLEPD,
25177 IX86_BUILTIN_CMPNGTPD,
25178 IX86_BUILTIN_CMPNGEPD,
25179 IX86_BUILTIN_CMPORDPD,
25180 IX86_BUILTIN_CMPUNORDPD,
25181 IX86_BUILTIN_CMPEQSD,
25182 IX86_BUILTIN_CMPLTSD,
25183 IX86_BUILTIN_CMPLESD,
25184 IX86_BUILTIN_CMPNEQSD,
25185 IX86_BUILTIN_CMPNLTSD,
25186 IX86_BUILTIN_CMPNLESD,
25187 IX86_BUILTIN_CMPORDSD,
25188 IX86_BUILTIN_CMPUNORDSD,
25190 IX86_BUILTIN_COMIEQSD,
25191 IX86_BUILTIN_COMILTSD,
25192 IX86_BUILTIN_COMILESD,
25193 IX86_BUILTIN_COMIGTSD,
25194 IX86_BUILTIN_COMIGESD,
25195 IX86_BUILTIN_COMINEQSD,
25196 IX86_BUILTIN_UCOMIEQSD,
25197 IX86_BUILTIN_UCOMILTSD,
25198 IX86_BUILTIN_UCOMILESD,
25199 IX86_BUILTIN_UCOMIGTSD,
25200 IX86_BUILTIN_UCOMIGESD,
25201 IX86_BUILTIN_UCOMINEQSD,
25203 IX86_BUILTIN_MAXPD,
25204 IX86_BUILTIN_MAXSD,
25205 IX86_BUILTIN_MINPD,
25206 IX86_BUILTIN_MINSD,
25208 IX86_BUILTIN_ANDPD,
25209 IX86_BUILTIN_ANDNPD,
25210 IX86_BUILTIN_ORPD,
25211 IX86_BUILTIN_XORPD,
25213 IX86_BUILTIN_SQRTPD,
25214 IX86_BUILTIN_SQRTSD,
25216 IX86_BUILTIN_UNPCKHPD,
25217 IX86_BUILTIN_UNPCKLPD,
25219 IX86_BUILTIN_SHUFPD,
25221 IX86_BUILTIN_LOADUPD,
25222 IX86_BUILTIN_STOREUPD,
25223 IX86_BUILTIN_MOVSD,
25225 IX86_BUILTIN_LOADHPD,
25226 IX86_BUILTIN_LOADLPD,
25228 IX86_BUILTIN_CVTDQ2PD,
25229 IX86_BUILTIN_CVTDQ2PS,
25231 IX86_BUILTIN_CVTPD2DQ,
25232 IX86_BUILTIN_CVTPD2PI,
25233 IX86_BUILTIN_CVTPD2PS,
25234 IX86_BUILTIN_CVTTPD2DQ,
25235 IX86_BUILTIN_CVTTPD2PI,
25237 IX86_BUILTIN_CVTPI2PD,
25238 IX86_BUILTIN_CVTSI2SD,
25239 IX86_BUILTIN_CVTSI642SD,
25241 IX86_BUILTIN_CVTSD2SI,
25242 IX86_BUILTIN_CVTSD2SI64,
25243 IX86_BUILTIN_CVTSD2SS,
25244 IX86_BUILTIN_CVTSS2SD,
25245 IX86_BUILTIN_CVTTSD2SI,
25246 IX86_BUILTIN_CVTTSD2SI64,
25248 IX86_BUILTIN_CVTPS2DQ,
25249 IX86_BUILTIN_CVTPS2PD,
25250 IX86_BUILTIN_CVTTPS2DQ,
25252 IX86_BUILTIN_MOVNTI,
25253 IX86_BUILTIN_MOVNTI64,
25254 IX86_BUILTIN_MOVNTPD,
25255 IX86_BUILTIN_MOVNTDQ,
25257 IX86_BUILTIN_MOVQ128,
25259 /* SSE2 MMX */
25260 IX86_BUILTIN_MASKMOVDQU,
25261 IX86_BUILTIN_MOVMSKPD,
25262 IX86_BUILTIN_PMOVMSKB128,
25264 IX86_BUILTIN_PACKSSWB128,
25265 IX86_BUILTIN_PACKSSDW128,
25266 IX86_BUILTIN_PACKUSWB128,
25268 IX86_BUILTIN_PADDB128,
25269 IX86_BUILTIN_PADDW128,
25270 IX86_BUILTIN_PADDD128,
25271 IX86_BUILTIN_PADDQ128,
25272 IX86_BUILTIN_PADDSB128,
25273 IX86_BUILTIN_PADDSW128,
25274 IX86_BUILTIN_PADDUSB128,
25275 IX86_BUILTIN_PADDUSW128,
25276 IX86_BUILTIN_PSUBB128,
25277 IX86_BUILTIN_PSUBW128,
25278 IX86_BUILTIN_PSUBD128,
25279 IX86_BUILTIN_PSUBQ128,
25280 IX86_BUILTIN_PSUBSB128,
25281 IX86_BUILTIN_PSUBSW128,
25282 IX86_BUILTIN_PSUBUSB128,
25283 IX86_BUILTIN_PSUBUSW128,
25285 IX86_BUILTIN_PAND128,
25286 IX86_BUILTIN_PANDN128,
25287 IX86_BUILTIN_POR128,
25288 IX86_BUILTIN_PXOR128,
25290 IX86_BUILTIN_PAVGB128,
25291 IX86_BUILTIN_PAVGW128,
25293 IX86_BUILTIN_PCMPEQB128,
25294 IX86_BUILTIN_PCMPEQW128,
25295 IX86_BUILTIN_PCMPEQD128,
25296 IX86_BUILTIN_PCMPGTB128,
25297 IX86_BUILTIN_PCMPGTW128,
25298 IX86_BUILTIN_PCMPGTD128,
25300 IX86_BUILTIN_PMADDWD128,
25302 IX86_BUILTIN_PMAXSW128,
25303 IX86_BUILTIN_PMAXUB128,
25304 IX86_BUILTIN_PMINSW128,
25305 IX86_BUILTIN_PMINUB128,
25307 IX86_BUILTIN_PMULUDQ,
25308 IX86_BUILTIN_PMULUDQ128,
25309 IX86_BUILTIN_PMULHUW128,
25310 IX86_BUILTIN_PMULHW128,
25311 IX86_BUILTIN_PMULLW128,
25313 IX86_BUILTIN_PSADBW128,
25314 IX86_BUILTIN_PSHUFHW,
25315 IX86_BUILTIN_PSHUFLW,
25316 IX86_BUILTIN_PSHUFD,
25318 IX86_BUILTIN_PSLLDQI128,
25319 IX86_BUILTIN_PSLLWI128,
25320 IX86_BUILTIN_PSLLDI128,
25321 IX86_BUILTIN_PSLLQI128,
25322 IX86_BUILTIN_PSRAWI128,
25323 IX86_BUILTIN_PSRADI128,
25324 IX86_BUILTIN_PSRLDQI128,
25325 IX86_BUILTIN_PSRLWI128,
25326 IX86_BUILTIN_PSRLDI128,
25327 IX86_BUILTIN_PSRLQI128,
25329 IX86_BUILTIN_PSLLDQ128,
25330 IX86_BUILTIN_PSLLW128,
25331 IX86_BUILTIN_PSLLD128,
25332 IX86_BUILTIN_PSLLQ128,
25333 IX86_BUILTIN_PSRAW128,
25334 IX86_BUILTIN_PSRAD128,
25335 IX86_BUILTIN_PSRLW128,
25336 IX86_BUILTIN_PSRLD128,
25337 IX86_BUILTIN_PSRLQ128,
25339 IX86_BUILTIN_PUNPCKHBW128,
25340 IX86_BUILTIN_PUNPCKHWD128,
25341 IX86_BUILTIN_PUNPCKHDQ128,
25342 IX86_BUILTIN_PUNPCKHQDQ128,
25343 IX86_BUILTIN_PUNPCKLBW128,
25344 IX86_BUILTIN_PUNPCKLWD128,
25345 IX86_BUILTIN_PUNPCKLDQ128,
25346 IX86_BUILTIN_PUNPCKLQDQ128,
25348 IX86_BUILTIN_CLFLUSH,
25349 IX86_BUILTIN_MFENCE,
25350 IX86_BUILTIN_LFENCE,
25351 IX86_BUILTIN_PAUSE,
25353 IX86_BUILTIN_BSRSI,
25354 IX86_BUILTIN_BSRDI,
25355 IX86_BUILTIN_RDPMC,
25356 IX86_BUILTIN_RDTSC,
25357 IX86_BUILTIN_RDTSCP,
25358 IX86_BUILTIN_ROLQI,
25359 IX86_BUILTIN_ROLHI,
25360 IX86_BUILTIN_RORQI,
25361 IX86_BUILTIN_RORHI,
25363 /* SSE3. */
25364 IX86_BUILTIN_ADDSUBPS,
25365 IX86_BUILTIN_HADDPS,
25366 IX86_BUILTIN_HSUBPS,
25367 IX86_BUILTIN_MOVSHDUP,
25368 IX86_BUILTIN_MOVSLDUP,
25369 IX86_BUILTIN_ADDSUBPD,
25370 IX86_BUILTIN_HADDPD,
25371 IX86_BUILTIN_HSUBPD,
25372 IX86_BUILTIN_LDDQU,
25374 IX86_BUILTIN_MONITOR,
25375 IX86_BUILTIN_MWAIT,
25377 /* SSSE3. */
25378 IX86_BUILTIN_PHADDW,
25379 IX86_BUILTIN_PHADDD,
25380 IX86_BUILTIN_PHADDSW,
25381 IX86_BUILTIN_PHSUBW,
25382 IX86_BUILTIN_PHSUBD,
25383 IX86_BUILTIN_PHSUBSW,
25384 IX86_BUILTIN_PMADDUBSW,
25385 IX86_BUILTIN_PMULHRSW,
25386 IX86_BUILTIN_PSHUFB,
25387 IX86_BUILTIN_PSIGNB,
25388 IX86_BUILTIN_PSIGNW,
25389 IX86_BUILTIN_PSIGND,
25390 IX86_BUILTIN_PALIGNR,
25391 IX86_BUILTIN_PABSB,
25392 IX86_BUILTIN_PABSW,
25393 IX86_BUILTIN_PABSD,
25395 IX86_BUILTIN_PHADDW128,
25396 IX86_BUILTIN_PHADDD128,
25397 IX86_BUILTIN_PHADDSW128,
25398 IX86_BUILTIN_PHSUBW128,
25399 IX86_BUILTIN_PHSUBD128,
25400 IX86_BUILTIN_PHSUBSW128,
25401 IX86_BUILTIN_PMADDUBSW128,
25402 IX86_BUILTIN_PMULHRSW128,
25403 IX86_BUILTIN_PSHUFB128,
25404 IX86_BUILTIN_PSIGNB128,
25405 IX86_BUILTIN_PSIGNW128,
25406 IX86_BUILTIN_PSIGND128,
25407 IX86_BUILTIN_PALIGNR128,
25408 IX86_BUILTIN_PABSB128,
25409 IX86_BUILTIN_PABSW128,
25410 IX86_BUILTIN_PABSD128,
25412 /* AMDFAM10 - SSE4A New Instructions. */
25413 IX86_BUILTIN_MOVNTSD,
25414 IX86_BUILTIN_MOVNTSS,
25415 IX86_BUILTIN_EXTRQI,
25416 IX86_BUILTIN_EXTRQ,
25417 IX86_BUILTIN_INSERTQI,
25418 IX86_BUILTIN_INSERTQ,
25420 /* SSE4.1. */
25421 IX86_BUILTIN_BLENDPD,
25422 IX86_BUILTIN_BLENDPS,
25423 IX86_BUILTIN_BLENDVPD,
25424 IX86_BUILTIN_BLENDVPS,
25425 IX86_BUILTIN_PBLENDVB128,
25426 IX86_BUILTIN_PBLENDW128,
25428 IX86_BUILTIN_DPPD,
25429 IX86_BUILTIN_DPPS,
25431 IX86_BUILTIN_INSERTPS128,
25433 IX86_BUILTIN_MOVNTDQA,
25434 IX86_BUILTIN_MPSADBW128,
25435 IX86_BUILTIN_PACKUSDW128,
25436 IX86_BUILTIN_PCMPEQQ,
25437 IX86_BUILTIN_PHMINPOSUW128,
25439 IX86_BUILTIN_PMAXSB128,
25440 IX86_BUILTIN_PMAXSD128,
25441 IX86_BUILTIN_PMAXUD128,
25442 IX86_BUILTIN_PMAXUW128,
25444 IX86_BUILTIN_PMINSB128,
25445 IX86_BUILTIN_PMINSD128,
25446 IX86_BUILTIN_PMINUD128,
25447 IX86_BUILTIN_PMINUW128,
25449 IX86_BUILTIN_PMOVSXBW128,
25450 IX86_BUILTIN_PMOVSXBD128,
25451 IX86_BUILTIN_PMOVSXBQ128,
25452 IX86_BUILTIN_PMOVSXWD128,
25453 IX86_BUILTIN_PMOVSXWQ128,
25454 IX86_BUILTIN_PMOVSXDQ128,
25456 IX86_BUILTIN_PMOVZXBW128,
25457 IX86_BUILTIN_PMOVZXBD128,
25458 IX86_BUILTIN_PMOVZXBQ128,
25459 IX86_BUILTIN_PMOVZXWD128,
25460 IX86_BUILTIN_PMOVZXWQ128,
25461 IX86_BUILTIN_PMOVZXDQ128,
25463 IX86_BUILTIN_PMULDQ128,
25464 IX86_BUILTIN_PMULLD128,
25466 IX86_BUILTIN_ROUNDSD,
25467 IX86_BUILTIN_ROUNDSS,
25469 IX86_BUILTIN_ROUNDPD,
25470 IX86_BUILTIN_ROUNDPS,
25472 IX86_BUILTIN_FLOORPD,
25473 IX86_BUILTIN_CEILPD,
25474 IX86_BUILTIN_TRUNCPD,
25475 IX86_BUILTIN_RINTPD,
25476 IX86_BUILTIN_ROUNDPD_AZ,
25478 IX86_BUILTIN_FLOORPD_VEC_PACK_SFIX,
25479 IX86_BUILTIN_CEILPD_VEC_PACK_SFIX,
25480 IX86_BUILTIN_ROUNDPD_AZ_VEC_PACK_SFIX,
25482 IX86_BUILTIN_FLOORPS,
25483 IX86_BUILTIN_CEILPS,
25484 IX86_BUILTIN_TRUNCPS,
25485 IX86_BUILTIN_RINTPS,
25486 IX86_BUILTIN_ROUNDPS_AZ,
25488 IX86_BUILTIN_FLOORPS_SFIX,
25489 IX86_BUILTIN_CEILPS_SFIX,
25490 IX86_BUILTIN_ROUNDPS_AZ_SFIX,
25492 IX86_BUILTIN_PTESTZ,
25493 IX86_BUILTIN_PTESTC,
25494 IX86_BUILTIN_PTESTNZC,
25496 IX86_BUILTIN_VEC_INIT_V2SI,
25497 IX86_BUILTIN_VEC_INIT_V4HI,
25498 IX86_BUILTIN_VEC_INIT_V8QI,
25499 IX86_BUILTIN_VEC_EXT_V2DF,
25500 IX86_BUILTIN_VEC_EXT_V2DI,
25501 IX86_BUILTIN_VEC_EXT_V4SF,
25502 IX86_BUILTIN_VEC_EXT_V4SI,
25503 IX86_BUILTIN_VEC_EXT_V8HI,
25504 IX86_BUILTIN_VEC_EXT_V2SI,
25505 IX86_BUILTIN_VEC_EXT_V4HI,
25506 IX86_BUILTIN_VEC_EXT_V16QI,
25507 IX86_BUILTIN_VEC_SET_V2DI,
25508 IX86_BUILTIN_VEC_SET_V4SF,
25509 IX86_BUILTIN_VEC_SET_V4SI,
25510 IX86_BUILTIN_VEC_SET_V8HI,
25511 IX86_BUILTIN_VEC_SET_V4HI,
25512 IX86_BUILTIN_VEC_SET_V16QI,
25514 IX86_BUILTIN_VEC_PACK_SFIX,
25515 IX86_BUILTIN_VEC_PACK_SFIX256,
25517 /* SSE4.2. */
25518 IX86_BUILTIN_CRC32QI,
25519 IX86_BUILTIN_CRC32HI,
25520 IX86_BUILTIN_CRC32SI,
25521 IX86_BUILTIN_CRC32DI,
25523 IX86_BUILTIN_PCMPESTRI128,
25524 IX86_BUILTIN_PCMPESTRM128,
25525 IX86_BUILTIN_PCMPESTRA128,
25526 IX86_BUILTIN_PCMPESTRC128,
25527 IX86_BUILTIN_PCMPESTRO128,
25528 IX86_BUILTIN_PCMPESTRS128,
25529 IX86_BUILTIN_PCMPESTRZ128,
25530 IX86_BUILTIN_PCMPISTRI128,
25531 IX86_BUILTIN_PCMPISTRM128,
25532 IX86_BUILTIN_PCMPISTRA128,
25533 IX86_BUILTIN_PCMPISTRC128,
25534 IX86_BUILTIN_PCMPISTRO128,
25535 IX86_BUILTIN_PCMPISTRS128,
25536 IX86_BUILTIN_PCMPISTRZ128,
25538 IX86_BUILTIN_PCMPGTQ,
25540 /* AES instructions */
25541 IX86_BUILTIN_AESENC128,
25542 IX86_BUILTIN_AESENCLAST128,
25543 IX86_BUILTIN_AESDEC128,
25544 IX86_BUILTIN_AESDECLAST128,
25545 IX86_BUILTIN_AESIMC128,
25546 IX86_BUILTIN_AESKEYGENASSIST128,
25548 /* PCLMUL instruction */
25549 IX86_BUILTIN_PCLMULQDQ128,
25551 /* AVX */
25552 IX86_BUILTIN_ADDPD256,
25553 IX86_BUILTIN_ADDPS256,
25554 IX86_BUILTIN_ADDSUBPD256,
25555 IX86_BUILTIN_ADDSUBPS256,
25556 IX86_BUILTIN_ANDPD256,
25557 IX86_BUILTIN_ANDPS256,
25558 IX86_BUILTIN_ANDNPD256,
25559 IX86_BUILTIN_ANDNPS256,
25560 IX86_BUILTIN_BLENDPD256,
25561 IX86_BUILTIN_BLENDPS256,
25562 IX86_BUILTIN_BLENDVPD256,
25563 IX86_BUILTIN_BLENDVPS256,
25564 IX86_BUILTIN_DIVPD256,
25565 IX86_BUILTIN_DIVPS256,
25566 IX86_BUILTIN_DPPS256,
25567 IX86_BUILTIN_HADDPD256,
25568 IX86_BUILTIN_HADDPS256,
25569 IX86_BUILTIN_HSUBPD256,
25570 IX86_BUILTIN_HSUBPS256,
25571 IX86_BUILTIN_MAXPD256,
25572 IX86_BUILTIN_MAXPS256,
25573 IX86_BUILTIN_MINPD256,
25574 IX86_BUILTIN_MINPS256,
25575 IX86_BUILTIN_MULPD256,
25576 IX86_BUILTIN_MULPS256,
25577 IX86_BUILTIN_ORPD256,
25578 IX86_BUILTIN_ORPS256,
25579 IX86_BUILTIN_SHUFPD256,
25580 IX86_BUILTIN_SHUFPS256,
25581 IX86_BUILTIN_SUBPD256,
25582 IX86_BUILTIN_SUBPS256,
25583 IX86_BUILTIN_XORPD256,
25584 IX86_BUILTIN_XORPS256,
25585 IX86_BUILTIN_CMPSD,
25586 IX86_BUILTIN_CMPSS,
25587 IX86_BUILTIN_CMPPD,
25588 IX86_BUILTIN_CMPPS,
25589 IX86_BUILTIN_CMPPD256,
25590 IX86_BUILTIN_CMPPS256,
25591 IX86_BUILTIN_CVTDQ2PD256,
25592 IX86_BUILTIN_CVTDQ2PS256,
25593 IX86_BUILTIN_CVTPD2PS256,
25594 IX86_BUILTIN_CVTPS2DQ256,
25595 IX86_BUILTIN_CVTPS2PD256,
25596 IX86_BUILTIN_CVTTPD2DQ256,
25597 IX86_BUILTIN_CVTPD2DQ256,
25598 IX86_BUILTIN_CVTTPS2DQ256,
25599 IX86_BUILTIN_EXTRACTF128PD256,
25600 IX86_BUILTIN_EXTRACTF128PS256,
25601 IX86_BUILTIN_EXTRACTF128SI256,
25602 IX86_BUILTIN_VZEROALL,
25603 IX86_BUILTIN_VZEROUPPER,
25604 IX86_BUILTIN_VPERMILVARPD,
25605 IX86_BUILTIN_VPERMILVARPS,
25606 IX86_BUILTIN_VPERMILVARPD256,
25607 IX86_BUILTIN_VPERMILVARPS256,
25608 IX86_BUILTIN_VPERMILPD,
25609 IX86_BUILTIN_VPERMILPS,
25610 IX86_BUILTIN_VPERMILPD256,
25611 IX86_BUILTIN_VPERMILPS256,
25612 IX86_BUILTIN_VPERMIL2PD,
25613 IX86_BUILTIN_VPERMIL2PS,
25614 IX86_BUILTIN_VPERMIL2PD256,
25615 IX86_BUILTIN_VPERMIL2PS256,
25616 IX86_BUILTIN_VPERM2F128PD256,
25617 IX86_BUILTIN_VPERM2F128PS256,
25618 IX86_BUILTIN_VPERM2F128SI256,
25619 IX86_BUILTIN_VBROADCASTSS,
25620 IX86_BUILTIN_VBROADCASTSD256,
25621 IX86_BUILTIN_VBROADCASTSS256,
25622 IX86_BUILTIN_VBROADCASTPD256,
25623 IX86_BUILTIN_VBROADCASTPS256,
25624 IX86_BUILTIN_VINSERTF128PD256,
25625 IX86_BUILTIN_VINSERTF128PS256,
25626 IX86_BUILTIN_VINSERTF128SI256,
25627 IX86_BUILTIN_LOADUPD256,
25628 IX86_BUILTIN_LOADUPS256,
25629 IX86_BUILTIN_STOREUPD256,
25630 IX86_BUILTIN_STOREUPS256,
25631 IX86_BUILTIN_LDDQU256,
25632 IX86_BUILTIN_MOVNTDQ256,
25633 IX86_BUILTIN_MOVNTPD256,
25634 IX86_BUILTIN_MOVNTPS256,
25635 IX86_BUILTIN_LOADDQU256,
25636 IX86_BUILTIN_STOREDQU256,
25637 IX86_BUILTIN_MASKLOADPD,
25638 IX86_BUILTIN_MASKLOADPS,
25639 IX86_BUILTIN_MASKSTOREPD,
25640 IX86_BUILTIN_MASKSTOREPS,
25641 IX86_BUILTIN_MASKLOADPD256,
25642 IX86_BUILTIN_MASKLOADPS256,
25643 IX86_BUILTIN_MASKSTOREPD256,
25644 IX86_BUILTIN_MASKSTOREPS256,
25645 IX86_BUILTIN_MOVSHDUP256,
25646 IX86_BUILTIN_MOVSLDUP256,
25647 IX86_BUILTIN_MOVDDUP256,
25649 IX86_BUILTIN_SQRTPD256,
25650 IX86_BUILTIN_SQRTPS256,
25651 IX86_BUILTIN_SQRTPS_NR256,
25652 IX86_BUILTIN_RSQRTPS256,
25653 IX86_BUILTIN_RSQRTPS_NR256,
25655 IX86_BUILTIN_RCPPS256,
25657 IX86_BUILTIN_ROUNDPD256,
25658 IX86_BUILTIN_ROUNDPS256,
25660 IX86_BUILTIN_FLOORPD256,
25661 IX86_BUILTIN_CEILPD256,
25662 IX86_BUILTIN_TRUNCPD256,
25663 IX86_BUILTIN_RINTPD256,
25664 IX86_BUILTIN_ROUNDPD_AZ256,
25666 IX86_BUILTIN_FLOORPD_VEC_PACK_SFIX256,
25667 IX86_BUILTIN_CEILPD_VEC_PACK_SFIX256,
25668 IX86_BUILTIN_ROUNDPD_AZ_VEC_PACK_SFIX256,
25670 IX86_BUILTIN_FLOORPS256,
25671 IX86_BUILTIN_CEILPS256,
25672 IX86_BUILTIN_TRUNCPS256,
25673 IX86_BUILTIN_RINTPS256,
25674 IX86_BUILTIN_ROUNDPS_AZ256,
25676 IX86_BUILTIN_FLOORPS_SFIX256,
25677 IX86_BUILTIN_CEILPS_SFIX256,
25678 IX86_BUILTIN_ROUNDPS_AZ_SFIX256,
25680 IX86_BUILTIN_UNPCKHPD256,
25681 IX86_BUILTIN_UNPCKLPD256,
25682 IX86_BUILTIN_UNPCKHPS256,
25683 IX86_BUILTIN_UNPCKLPS256,
25685 IX86_BUILTIN_SI256_SI,
25686 IX86_BUILTIN_PS256_PS,
25687 IX86_BUILTIN_PD256_PD,
25688 IX86_BUILTIN_SI_SI256,
25689 IX86_BUILTIN_PS_PS256,
25690 IX86_BUILTIN_PD_PD256,
25692 IX86_BUILTIN_VTESTZPD,
25693 IX86_BUILTIN_VTESTCPD,
25694 IX86_BUILTIN_VTESTNZCPD,
25695 IX86_BUILTIN_VTESTZPS,
25696 IX86_BUILTIN_VTESTCPS,
25697 IX86_BUILTIN_VTESTNZCPS,
25698 IX86_BUILTIN_VTESTZPD256,
25699 IX86_BUILTIN_VTESTCPD256,
25700 IX86_BUILTIN_VTESTNZCPD256,
25701 IX86_BUILTIN_VTESTZPS256,
25702 IX86_BUILTIN_VTESTCPS256,
25703 IX86_BUILTIN_VTESTNZCPS256,
25704 IX86_BUILTIN_PTESTZ256,
25705 IX86_BUILTIN_PTESTC256,
25706 IX86_BUILTIN_PTESTNZC256,
25708 IX86_BUILTIN_MOVMSKPD256,
25709 IX86_BUILTIN_MOVMSKPS256,
25711 /* AVX2 */
25712 IX86_BUILTIN_MPSADBW256,
25713 IX86_BUILTIN_PABSB256,
25714 IX86_BUILTIN_PABSW256,
25715 IX86_BUILTIN_PABSD256,
25716 IX86_BUILTIN_PACKSSDW256,
25717 IX86_BUILTIN_PACKSSWB256,
25718 IX86_BUILTIN_PACKUSDW256,
25719 IX86_BUILTIN_PACKUSWB256,
25720 IX86_BUILTIN_PADDB256,
25721 IX86_BUILTIN_PADDW256,
25722 IX86_BUILTIN_PADDD256,
25723 IX86_BUILTIN_PADDQ256,
25724 IX86_BUILTIN_PADDSB256,
25725 IX86_BUILTIN_PADDSW256,
25726 IX86_BUILTIN_PADDUSB256,
25727 IX86_BUILTIN_PADDUSW256,
25728 IX86_BUILTIN_PALIGNR256,
25729 IX86_BUILTIN_AND256I,
25730 IX86_BUILTIN_ANDNOT256I,
25731 IX86_BUILTIN_PAVGB256,
25732 IX86_BUILTIN_PAVGW256,
25733 IX86_BUILTIN_PBLENDVB256,
25734 IX86_BUILTIN_PBLENDVW256,
25735 IX86_BUILTIN_PCMPEQB256,
25736 IX86_BUILTIN_PCMPEQW256,
25737 IX86_BUILTIN_PCMPEQD256,
25738 IX86_BUILTIN_PCMPEQQ256,
25739 IX86_BUILTIN_PCMPGTB256,
25740 IX86_BUILTIN_PCMPGTW256,
25741 IX86_BUILTIN_PCMPGTD256,
25742 IX86_BUILTIN_PCMPGTQ256,
25743 IX86_BUILTIN_PHADDW256,
25744 IX86_BUILTIN_PHADDD256,
25745 IX86_BUILTIN_PHADDSW256,
25746 IX86_BUILTIN_PHSUBW256,
25747 IX86_BUILTIN_PHSUBD256,
25748 IX86_BUILTIN_PHSUBSW256,
25749 IX86_BUILTIN_PMADDUBSW256,
25750 IX86_BUILTIN_PMADDWD256,
25751 IX86_BUILTIN_PMAXSB256,
25752 IX86_BUILTIN_PMAXSW256,
25753 IX86_BUILTIN_PMAXSD256,
25754 IX86_BUILTIN_PMAXUB256,
25755 IX86_BUILTIN_PMAXUW256,
25756 IX86_BUILTIN_PMAXUD256,
25757 IX86_BUILTIN_PMINSB256,
25758 IX86_BUILTIN_PMINSW256,
25759 IX86_BUILTIN_PMINSD256,
25760 IX86_BUILTIN_PMINUB256,
25761 IX86_BUILTIN_PMINUW256,
25762 IX86_BUILTIN_PMINUD256,
25763 IX86_BUILTIN_PMOVMSKB256,
25764 IX86_BUILTIN_PMOVSXBW256,
25765 IX86_BUILTIN_PMOVSXBD256,
25766 IX86_BUILTIN_PMOVSXBQ256,
25767 IX86_BUILTIN_PMOVSXWD256,
25768 IX86_BUILTIN_PMOVSXWQ256,
25769 IX86_BUILTIN_PMOVSXDQ256,
25770 IX86_BUILTIN_PMOVZXBW256,
25771 IX86_BUILTIN_PMOVZXBD256,
25772 IX86_BUILTIN_PMOVZXBQ256,
25773 IX86_BUILTIN_PMOVZXWD256,
25774 IX86_BUILTIN_PMOVZXWQ256,
25775 IX86_BUILTIN_PMOVZXDQ256,
25776 IX86_BUILTIN_PMULDQ256,
25777 IX86_BUILTIN_PMULHRSW256,
25778 IX86_BUILTIN_PMULHUW256,
25779 IX86_BUILTIN_PMULHW256,
25780 IX86_BUILTIN_PMULLW256,
25781 IX86_BUILTIN_PMULLD256,
25782 IX86_BUILTIN_PMULUDQ256,
25783 IX86_BUILTIN_POR256,
25784 IX86_BUILTIN_PSADBW256,
25785 IX86_BUILTIN_PSHUFB256,
25786 IX86_BUILTIN_PSHUFD256,
25787 IX86_BUILTIN_PSHUFHW256,
25788 IX86_BUILTIN_PSHUFLW256,
25789 IX86_BUILTIN_PSIGNB256,
25790 IX86_BUILTIN_PSIGNW256,
25791 IX86_BUILTIN_PSIGND256,
25792 IX86_BUILTIN_PSLLDQI256,
25793 IX86_BUILTIN_PSLLWI256,
25794 IX86_BUILTIN_PSLLW256,
25795 IX86_BUILTIN_PSLLDI256,
25796 IX86_BUILTIN_PSLLD256,
25797 IX86_BUILTIN_PSLLQI256,
25798 IX86_BUILTIN_PSLLQ256,
25799 IX86_BUILTIN_PSRAWI256,
25800 IX86_BUILTIN_PSRAW256,
25801 IX86_BUILTIN_PSRADI256,
25802 IX86_BUILTIN_PSRAD256,
25803 IX86_BUILTIN_PSRLDQI256,
25804 IX86_BUILTIN_PSRLWI256,
25805 IX86_BUILTIN_PSRLW256,
25806 IX86_BUILTIN_PSRLDI256,
25807 IX86_BUILTIN_PSRLD256,
25808 IX86_BUILTIN_PSRLQI256,
25809 IX86_BUILTIN_PSRLQ256,
25810 IX86_BUILTIN_PSUBB256,
25811 IX86_BUILTIN_PSUBW256,
25812 IX86_BUILTIN_PSUBD256,
25813 IX86_BUILTIN_PSUBQ256,
25814 IX86_BUILTIN_PSUBSB256,
25815 IX86_BUILTIN_PSUBSW256,
25816 IX86_BUILTIN_PSUBUSB256,
25817 IX86_BUILTIN_PSUBUSW256,
25818 IX86_BUILTIN_PUNPCKHBW256,
25819 IX86_BUILTIN_PUNPCKHWD256,
25820 IX86_BUILTIN_PUNPCKHDQ256,
25821 IX86_BUILTIN_PUNPCKHQDQ256,
25822 IX86_BUILTIN_PUNPCKLBW256,
25823 IX86_BUILTIN_PUNPCKLWD256,
25824 IX86_BUILTIN_PUNPCKLDQ256,
25825 IX86_BUILTIN_PUNPCKLQDQ256,
25826 IX86_BUILTIN_PXOR256,
25827 IX86_BUILTIN_MOVNTDQA256,
25828 IX86_BUILTIN_VBROADCASTSS_PS,
25829 IX86_BUILTIN_VBROADCASTSS_PS256,
25830 IX86_BUILTIN_VBROADCASTSD_PD256,
25831 IX86_BUILTIN_VBROADCASTSI256,
25832 IX86_BUILTIN_PBLENDD256,
25833 IX86_BUILTIN_PBLENDD128,
25834 IX86_BUILTIN_PBROADCASTB256,
25835 IX86_BUILTIN_PBROADCASTW256,
25836 IX86_BUILTIN_PBROADCASTD256,
25837 IX86_BUILTIN_PBROADCASTQ256,
25838 IX86_BUILTIN_PBROADCASTB128,
25839 IX86_BUILTIN_PBROADCASTW128,
25840 IX86_BUILTIN_PBROADCASTD128,
25841 IX86_BUILTIN_PBROADCASTQ128,
25842 IX86_BUILTIN_VPERMVARSI256,
25843 IX86_BUILTIN_VPERMDF256,
25844 IX86_BUILTIN_VPERMVARSF256,
25845 IX86_BUILTIN_VPERMDI256,
25846 IX86_BUILTIN_VPERMTI256,
25847 IX86_BUILTIN_VEXTRACT128I256,
25848 IX86_BUILTIN_VINSERT128I256,
25849 IX86_BUILTIN_MASKLOADD,
25850 IX86_BUILTIN_MASKLOADQ,
25851 IX86_BUILTIN_MASKLOADD256,
25852 IX86_BUILTIN_MASKLOADQ256,
25853 IX86_BUILTIN_MASKSTORED,
25854 IX86_BUILTIN_MASKSTOREQ,
25855 IX86_BUILTIN_MASKSTORED256,
25856 IX86_BUILTIN_MASKSTOREQ256,
25857 IX86_BUILTIN_PSLLVV4DI,
25858 IX86_BUILTIN_PSLLVV2DI,
25859 IX86_BUILTIN_PSLLVV8SI,
25860 IX86_BUILTIN_PSLLVV4SI,
25861 IX86_BUILTIN_PSRAVV8SI,
25862 IX86_BUILTIN_PSRAVV4SI,
25863 IX86_BUILTIN_PSRLVV4DI,
25864 IX86_BUILTIN_PSRLVV2DI,
25865 IX86_BUILTIN_PSRLVV8SI,
25866 IX86_BUILTIN_PSRLVV4SI,
25868 IX86_BUILTIN_GATHERSIV2DF,
25869 IX86_BUILTIN_GATHERSIV4DF,
25870 IX86_BUILTIN_GATHERDIV2DF,
25871 IX86_BUILTIN_GATHERDIV4DF,
25872 IX86_BUILTIN_GATHERSIV4SF,
25873 IX86_BUILTIN_GATHERSIV8SF,
25874 IX86_BUILTIN_GATHERDIV4SF,
25875 IX86_BUILTIN_GATHERDIV8SF,
25876 IX86_BUILTIN_GATHERSIV2DI,
25877 IX86_BUILTIN_GATHERSIV4DI,
25878 IX86_BUILTIN_GATHERDIV2DI,
25879 IX86_BUILTIN_GATHERDIV4DI,
25880 IX86_BUILTIN_GATHERSIV4SI,
25881 IX86_BUILTIN_GATHERSIV8SI,
25882 IX86_BUILTIN_GATHERDIV4SI,
25883 IX86_BUILTIN_GATHERDIV8SI,
25885 /* Alternate 4 element gather for the vectorizer where
25886 all operands are 32-byte wide. */
25887 IX86_BUILTIN_GATHERALTSIV4DF,
25888 IX86_BUILTIN_GATHERALTDIV8SF,
25889 IX86_BUILTIN_GATHERALTSIV4DI,
25890 IX86_BUILTIN_GATHERALTDIV8SI,
25892 /* TFmode support builtins. */
25893 IX86_BUILTIN_INFQ,
25894 IX86_BUILTIN_HUGE_VALQ,
25895 IX86_BUILTIN_FABSQ,
25896 IX86_BUILTIN_COPYSIGNQ,
25898 /* Vectorizer support builtins. */
25899 IX86_BUILTIN_CPYSGNPS,
25900 IX86_BUILTIN_CPYSGNPD,
25901 IX86_BUILTIN_CPYSGNPS256,
25902 IX86_BUILTIN_CPYSGNPD256,
25904 /* FMA4 instructions. */
25905 IX86_BUILTIN_VFMADDSS,
25906 IX86_BUILTIN_VFMADDSD,
25907 IX86_BUILTIN_VFMADDPS,
25908 IX86_BUILTIN_VFMADDPD,
25909 IX86_BUILTIN_VFMADDPS256,
25910 IX86_BUILTIN_VFMADDPD256,
25911 IX86_BUILTIN_VFMADDSUBPS,
25912 IX86_BUILTIN_VFMADDSUBPD,
25913 IX86_BUILTIN_VFMADDSUBPS256,
25914 IX86_BUILTIN_VFMADDSUBPD256,
25916 /* FMA3 instructions. */
25917 IX86_BUILTIN_VFMADDSS3,
25918 IX86_BUILTIN_VFMADDSD3,
25920 /* XOP instructions. */
25921 IX86_BUILTIN_VPCMOV,
25922 IX86_BUILTIN_VPCMOV_V2DI,
25923 IX86_BUILTIN_VPCMOV_V4SI,
25924 IX86_BUILTIN_VPCMOV_V8HI,
25925 IX86_BUILTIN_VPCMOV_V16QI,
25926 IX86_BUILTIN_VPCMOV_V4SF,
25927 IX86_BUILTIN_VPCMOV_V2DF,
25928 IX86_BUILTIN_VPCMOV256,
25929 IX86_BUILTIN_VPCMOV_V4DI256,
25930 IX86_BUILTIN_VPCMOV_V8SI256,
25931 IX86_BUILTIN_VPCMOV_V16HI256,
25932 IX86_BUILTIN_VPCMOV_V32QI256,
25933 IX86_BUILTIN_VPCMOV_V8SF256,
25934 IX86_BUILTIN_VPCMOV_V4DF256,
25936 IX86_BUILTIN_VPPERM,
25938 IX86_BUILTIN_VPMACSSWW,
25939 IX86_BUILTIN_VPMACSWW,
25940 IX86_BUILTIN_VPMACSSWD,
25941 IX86_BUILTIN_VPMACSWD,
25942 IX86_BUILTIN_VPMACSSDD,
25943 IX86_BUILTIN_VPMACSDD,
25944 IX86_BUILTIN_VPMACSSDQL,
25945 IX86_BUILTIN_VPMACSSDQH,
25946 IX86_BUILTIN_VPMACSDQL,
25947 IX86_BUILTIN_VPMACSDQH,
25948 IX86_BUILTIN_VPMADCSSWD,
25949 IX86_BUILTIN_VPMADCSWD,
25951 IX86_BUILTIN_VPHADDBW,
25952 IX86_BUILTIN_VPHADDBD,
25953 IX86_BUILTIN_VPHADDBQ,
25954 IX86_BUILTIN_VPHADDWD,
25955 IX86_BUILTIN_VPHADDWQ,
25956 IX86_BUILTIN_VPHADDDQ,
25957 IX86_BUILTIN_VPHADDUBW,
25958 IX86_BUILTIN_VPHADDUBD,
25959 IX86_BUILTIN_VPHADDUBQ,
25960 IX86_BUILTIN_VPHADDUWD,
25961 IX86_BUILTIN_VPHADDUWQ,
25962 IX86_BUILTIN_VPHADDUDQ,
25963 IX86_BUILTIN_VPHSUBBW,
25964 IX86_BUILTIN_VPHSUBWD,
25965 IX86_BUILTIN_VPHSUBDQ,
25967 IX86_BUILTIN_VPROTB,
25968 IX86_BUILTIN_VPROTW,
25969 IX86_BUILTIN_VPROTD,
25970 IX86_BUILTIN_VPROTQ,
25971 IX86_BUILTIN_VPROTB_IMM,
25972 IX86_BUILTIN_VPROTW_IMM,
25973 IX86_BUILTIN_VPROTD_IMM,
25974 IX86_BUILTIN_VPROTQ_IMM,
25976 IX86_BUILTIN_VPSHLB,
25977 IX86_BUILTIN_VPSHLW,
25978 IX86_BUILTIN_VPSHLD,
25979 IX86_BUILTIN_VPSHLQ,
25980 IX86_BUILTIN_VPSHAB,
25981 IX86_BUILTIN_VPSHAW,
25982 IX86_BUILTIN_VPSHAD,
25983 IX86_BUILTIN_VPSHAQ,
25985 IX86_BUILTIN_VFRCZSS,
25986 IX86_BUILTIN_VFRCZSD,
25987 IX86_BUILTIN_VFRCZPS,
25988 IX86_BUILTIN_VFRCZPD,
25989 IX86_BUILTIN_VFRCZPS256,
25990 IX86_BUILTIN_VFRCZPD256,
25992 IX86_BUILTIN_VPCOMEQUB,
25993 IX86_BUILTIN_VPCOMNEUB,
25994 IX86_BUILTIN_VPCOMLTUB,
25995 IX86_BUILTIN_VPCOMLEUB,
25996 IX86_BUILTIN_VPCOMGTUB,
25997 IX86_BUILTIN_VPCOMGEUB,
25998 IX86_BUILTIN_VPCOMFALSEUB,
25999 IX86_BUILTIN_VPCOMTRUEUB,
26001 IX86_BUILTIN_VPCOMEQUW,
26002 IX86_BUILTIN_VPCOMNEUW,
26003 IX86_BUILTIN_VPCOMLTUW,
26004 IX86_BUILTIN_VPCOMLEUW,
26005 IX86_BUILTIN_VPCOMGTUW,
26006 IX86_BUILTIN_VPCOMGEUW,
26007 IX86_BUILTIN_VPCOMFALSEUW,
26008 IX86_BUILTIN_VPCOMTRUEUW,
26010 IX86_BUILTIN_VPCOMEQUD,
26011 IX86_BUILTIN_VPCOMNEUD,
26012 IX86_BUILTIN_VPCOMLTUD,
26013 IX86_BUILTIN_VPCOMLEUD,
26014 IX86_BUILTIN_VPCOMGTUD,
26015 IX86_BUILTIN_VPCOMGEUD,
26016 IX86_BUILTIN_VPCOMFALSEUD,
26017 IX86_BUILTIN_VPCOMTRUEUD,
26019 IX86_BUILTIN_VPCOMEQUQ,
26020 IX86_BUILTIN_VPCOMNEUQ,
26021 IX86_BUILTIN_VPCOMLTUQ,
26022 IX86_BUILTIN_VPCOMLEUQ,
26023 IX86_BUILTIN_VPCOMGTUQ,
26024 IX86_BUILTIN_VPCOMGEUQ,
26025 IX86_BUILTIN_VPCOMFALSEUQ,
26026 IX86_BUILTIN_VPCOMTRUEUQ,
26028 IX86_BUILTIN_VPCOMEQB,
26029 IX86_BUILTIN_VPCOMNEB,
26030 IX86_BUILTIN_VPCOMLTB,
26031 IX86_BUILTIN_VPCOMLEB,
26032 IX86_BUILTIN_VPCOMGTB,
26033 IX86_BUILTIN_VPCOMGEB,
26034 IX86_BUILTIN_VPCOMFALSEB,
26035 IX86_BUILTIN_VPCOMTRUEB,
26037 IX86_BUILTIN_VPCOMEQW,
26038 IX86_BUILTIN_VPCOMNEW,
26039 IX86_BUILTIN_VPCOMLTW,
26040 IX86_BUILTIN_VPCOMLEW,
26041 IX86_BUILTIN_VPCOMGTW,
26042 IX86_BUILTIN_VPCOMGEW,
26043 IX86_BUILTIN_VPCOMFALSEW,
26044 IX86_BUILTIN_VPCOMTRUEW,
26046 IX86_BUILTIN_VPCOMEQD,
26047 IX86_BUILTIN_VPCOMNED,
26048 IX86_BUILTIN_VPCOMLTD,
26049 IX86_BUILTIN_VPCOMLED,
26050 IX86_BUILTIN_VPCOMGTD,
26051 IX86_BUILTIN_VPCOMGED,
26052 IX86_BUILTIN_VPCOMFALSED,
26053 IX86_BUILTIN_VPCOMTRUED,
26055 IX86_BUILTIN_VPCOMEQQ,
26056 IX86_BUILTIN_VPCOMNEQ,
26057 IX86_BUILTIN_VPCOMLTQ,
26058 IX86_BUILTIN_VPCOMLEQ,
26059 IX86_BUILTIN_VPCOMGTQ,
26060 IX86_BUILTIN_VPCOMGEQ,
26061 IX86_BUILTIN_VPCOMFALSEQ,
26062 IX86_BUILTIN_VPCOMTRUEQ,
26064 /* LWP instructions. */
26065 IX86_BUILTIN_LLWPCB,
26066 IX86_BUILTIN_SLWPCB,
26067 IX86_BUILTIN_LWPVAL32,
26068 IX86_BUILTIN_LWPVAL64,
26069 IX86_BUILTIN_LWPINS32,
26070 IX86_BUILTIN_LWPINS64,
26072 IX86_BUILTIN_CLZS,
26074 /* RTM */
26075 IX86_BUILTIN_XBEGIN,
26076 IX86_BUILTIN_XEND,
26077 IX86_BUILTIN_XABORT,
26078 IX86_BUILTIN_XTEST,
26080 /* BMI instructions. */
26081 IX86_BUILTIN_BEXTR32,
26082 IX86_BUILTIN_BEXTR64,
26083 IX86_BUILTIN_CTZS,
26085 /* TBM instructions. */
26086 IX86_BUILTIN_BEXTRI32,
26087 IX86_BUILTIN_BEXTRI64,
26089 /* BMI2 instructions. */
26090 IX86_BUILTIN_BZHI32,
26091 IX86_BUILTIN_BZHI64,
26092 IX86_BUILTIN_PDEP32,
26093 IX86_BUILTIN_PDEP64,
26094 IX86_BUILTIN_PEXT32,
26095 IX86_BUILTIN_PEXT64,
26097 /* FSGSBASE instructions. */
26098 IX86_BUILTIN_RDFSBASE32,
26099 IX86_BUILTIN_RDFSBASE64,
26100 IX86_BUILTIN_RDGSBASE32,
26101 IX86_BUILTIN_RDGSBASE64,
26102 IX86_BUILTIN_WRFSBASE32,
26103 IX86_BUILTIN_WRFSBASE64,
26104 IX86_BUILTIN_WRGSBASE32,
26105 IX86_BUILTIN_WRGSBASE64,
26107 /* RDRND instructions. */
26108 IX86_BUILTIN_RDRAND16_STEP,
26109 IX86_BUILTIN_RDRAND32_STEP,
26110 IX86_BUILTIN_RDRAND64_STEP,
26112 /* F16C instructions. */
26113 IX86_BUILTIN_CVTPH2PS,
26114 IX86_BUILTIN_CVTPH2PS256,
26115 IX86_BUILTIN_CVTPS2PH,
26116 IX86_BUILTIN_CVTPS2PH256,
26118 /* CFString built-in for darwin */
26119 IX86_BUILTIN_CFSTRING,
26121 /* Builtins to get CPU type and supported features. */
26122 IX86_BUILTIN_CPU_INIT,
26123 IX86_BUILTIN_CPU_IS,
26124 IX86_BUILTIN_CPU_SUPPORTS,
26126 IX86_BUILTIN_MAX
26127 };
26129 /* Table for the ix86 builtin decls. */
26132 /* Table of all of the builtin functions that are possible with different ISA's
26133 but are waiting to be built until a function is declared to use that
26134 ISA. */
26141 };
26146 /* Add an ix86 target builtin function with CODE, NAME and TYPE. Save the MASK
26147 of which isa_flags to use in the ix86_builtins_isa array. Stores the
26148 function decl in the ix86_builtins array. Returns the function decl or
26149 NULL_TREE, if the builtin was not added.
26151 If the front end has a special hook for builtin functions, delay adding
26152 builtin functions that aren't in the current ISA until the ISA is changed
26153 with function specific optimization. Doing so, can save about 300K for the
26154 default compiler. When the builtin is expanded, check at that time whether
26155 it is valid.
26157 If the front end doesn't have a special hook, record all builtins, even if
26158 it isn't an instruction set in the current ISA in case the user uses
26159 function specific options for a different ISA, so that we don't get scope
26160 errors if a builtin is added in the middle of a function scope. */
26166 {
26170 {
26179 {
26185 }
26186 else
26187 {
26193 }
26194 }
26197 }
26199 /* Like def_builtin, but also marks the function decl "const". */
26204 {
26208 else
26212 }
26214 /* Add any new builtin functions for a given ISA that may not have been
26215 declared. This saves a bit of space compared to adding all of the
26216 declarations to the tree, even if we didn't use them. */
26218 static void
26220 {
26224 {
26227 {
26230 /* Don't define the builtin again. */
26236 NULL_TREE);
26241 }
26242 }
26243 }
26245 /* Bits for builtin_description.flag. */
26247 /* Set when we don't support the comparison natively, and should
26248 swap_comparison in order to support it. */
26249 #define BUILTIN_DESC_SWAP_OPERANDS 1
26251 struct builtin_description
26252 {
26259 };
26262 {
26263 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_comi, "__builtin_ia32_comieq", IX86_BUILTIN_COMIEQSS, UNEQ, 0 },
26264 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_comi, "__builtin_ia32_comilt", IX86_BUILTIN_COMILTSS, UNLT, 0 },
26265 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_comi, "__builtin_ia32_comile", IX86_BUILTIN_COMILESS, UNLE, 0 },
26266 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_comi, "__builtin_ia32_comigt", IX86_BUILTIN_COMIGTSS, GT, 0 },
26267 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_comi, "__builtin_ia32_comige", IX86_BUILTIN_COMIGESS, GE, 0 },
26268 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_comi, "__builtin_ia32_comineq", IX86_BUILTIN_COMINEQSS, LTGT, 0 },
26269 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_ucomi, "__builtin_ia32_ucomieq", IX86_BUILTIN_UCOMIEQSS, UNEQ, 0 },
26270 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_ucomi, "__builtin_ia32_ucomilt", IX86_BUILTIN_UCOMILTSS, UNLT, 0 },
26271 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_ucomi, "__builtin_ia32_ucomile", IX86_BUILTIN_UCOMILESS, UNLE, 0 },
26272 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_ucomi, "__builtin_ia32_ucomigt", IX86_BUILTIN_UCOMIGTSS, GT, 0 },
26273 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_ucomi, "__builtin_ia32_ucomige", IX86_BUILTIN_UCOMIGESS, GE, 0 },
26274 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_ucomi, "__builtin_ia32_ucomineq", IX86_BUILTIN_UCOMINEQSS, LTGT, 0 },
26275 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_comi, "__builtin_ia32_comisdeq", IX86_BUILTIN_COMIEQSD, UNEQ, 0 },
26276 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_comi, "__builtin_ia32_comisdlt", IX86_BUILTIN_COMILTSD, UNLT, 0 },
26277 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_comi, "__builtin_ia32_comisdle", IX86_BUILTIN_COMILESD, UNLE, 0 },
26278 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_comi, "__builtin_ia32_comisdgt", IX86_BUILTIN_COMIGTSD, GT, 0 },
26279 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_comi, "__builtin_ia32_comisdge", IX86_BUILTIN_COMIGESD, GE, 0 },
26280 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_comi, "__builtin_ia32_comisdneq", IX86_BUILTIN_COMINEQSD, LTGT, 0 },
26281 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ucomi, "__builtin_ia32_ucomisdeq", IX86_BUILTIN_UCOMIEQSD, UNEQ, 0 },
26282 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ucomi, "__builtin_ia32_ucomisdlt", IX86_BUILTIN_UCOMILTSD, UNLT, 0 },
26283 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ucomi, "__builtin_ia32_ucomisdle", IX86_BUILTIN_UCOMILESD, UNLE, 0 },
26284 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ucomi, "__builtin_ia32_ucomisdgt", IX86_BUILTIN_UCOMIGTSD, GT, 0 },
26285 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ucomi, "__builtin_ia32_ucomisdge", IX86_BUILTIN_UCOMIGESD, GE, 0 },
26286 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ucomi, "__builtin_ia32_ucomisdneq", IX86_BUILTIN_UCOMINEQSD, LTGT, 0 },
26287 };
26290 {
26291 /* SSE4.2 */
26292 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpestr, "__builtin_ia32_pcmpestri128", IX86_BUILTIN_PCMPESTRI128, UNKNOWN, 0 },
26293 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpestr, "__builtin_ia32_pcmpestrm128", IX86_BUILTIN_PCMPESTRM128, UNKNOWN, 0 },
26294 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpestr, "__builtin_ia32_pcmpestria128", IX86_BUILTIN_PCMPESTRA128, UNKNOWN, (int) CCAmode },
26295 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpestr, "__builtin_ia32_pcmpestric128", IX86_BUILTIN_PCMPESTRC128, UNKNOWN, (int) CCCmode },
26296 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpestr, "__builtin_ia32_pcmpestrio128", IX86_BUILTIN_PCMPESTRO128, UNKNOWN, (int) CCOmode },
26297 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpestr, "__builtin_ia32_pcmpestris128", IX86_BUILTIN_PCMPESTRS128, UNKNOWN, (int) CCSmode },
26298 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpestr, "__builtin_ia32_pcmpestriz128", IX86_BUILTIN_PCMPESTRZ128, UNKNOWN, (int) CCZmode },
26299 };
26302 {
26303 /* SSE4.2 */
26304 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpistr, "__builtin_ia32_pcmpistri128", IX86_BUILTIN_PCMPISTRI128, UNKNOWN, 0 },
26305 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpistr, "__builtin_ia32_pcmpistrm128", IX86_BUILTIN_PCMPISTRM128, UNKNOWN, 0 },
26306 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpistr, "__builtin_ia32_pcmpistria128", IX86_BUILTIN_PCMPISTRA128, UNKNOWN, (int) CCAmode },
26307 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpistr, "__builtin_ia32_pcmpistric128", IX86_BUILTIN_PCMPISTRC128, UNKNOWN, (int) CCCmode },
26308 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpistr, "__builtin_ia32_pcmpistrio128", IX86_BUILTIN_PCMPISTRO128, UNKNOWN, (int) CCOmode },
26309 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpistr, "__builtin_ia32_pcmpistris128", IX86_BUILTIN_PCMPISTRS128, UNKNOWN, (int) CCSmode },
26310 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpistr, "__builtin_ia32_pcmpistriz128", IX86_BUILTIN_PCMPISTRZ128, UNKNOWN, (int) CCZmode },
26311 };
26313 /* Special builtins with variable number of arguments. */
26315 {
26316 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_rdtsc, "__builtin_ia32_rdtsc", IX86_BUILTIN_RDTSC, UNKNOWN, (int) UINT64_FTYPE_VOID },
26317 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_rdtscp, "__builtin_ia32_rdtscp", IX86_BUILTIN_RDTSCP, UNKNOWN, (int) UINT64_FTYPE_PUNSIGNED },
26318 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_pause, "__builtin_ia32_pause", IX86_BUILTIN_PAUSE, UNKNOWN, (int) VOID_FTYPE_VOID },
26320 /* MMX */
26321 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_emms, "__builtin_ia32_emms", IX86_BUILTIN_EMMS, UNKNOWN, (int) VOID_FTYPE_VOID },
26323 /* 3DNow! */
26324 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_femms, "__builtin_ia32_femms", IX86_BUILTIN_FEMMS, UNKNOWN, (int) VOID_FTYPE_VOID },
26326 /* SSE */
26327 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_movups, "__builtin_ia32_storeups", IX86_BUILTIN_STOREUPS, UNKNOWN, (int) VOID_FTYPE_PFLOAT_V4SF },
26328 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_movntv4sf, "__builtin_ia32_movntps", IX86_BUILTIN_MOVNTPS, UNKNOWN, (int) VOID_FTYPE_PFLOAT_V4SF },
26329 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_movups, "__builtin_ia32_loadups", IX86_BUILTIN_LOADUPS, UNKNOWN, (int) V4SF_FTYPE_PCFLOAT },
26331 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_loadhps_exp, "__builtin_ia32_loadhps", IX86_BUILTIN_LOADHPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_PCV2SF },
26332 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_loadlps_exp, "__builtin_ia32_loadlps", IX86_BUILTIN_LOADLPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_PCV2SF },
26333 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_storehps, "__builtin_ia32_storehps", IX86_BUILTIN_STOREHPS, UNKNOWN, (int) VOID_FTYPE_PV2SF_V4SF },
26334 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_storelps, "__builtin_ia32_storelps", IX86_BUILTIN_STORELPS, UNKNOWN, (int) VOID_FTYPE_PV2SF_V4SF },
26336 /* SSE or 3DNow!A */
26337 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_sse_sfence, "__builtin_ia32_sfence", IX86_BUILTIN_SFENCE, UNKNOWN, (int) VOID_FTYPE_VOID },
26338 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_sse_movntq, "__builtin_ia32_movntq", IX86_BUILTIN_MOVNTQ, UNKNOWN, (int) VOID_FTYPE_PULONGLONG_ULONGLONG },
26340 /* SSE2 */
26341 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_lfence, "__builtin_ia32_lfence", IX86_BUILTIN_LFENCE, UNKNOWN, (int) VOID_FTYPE_VOID },
26342 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_mfence, 0, IX86_BUILTIN_MFENCE, UNKNOWN, (int) VOID_FTYPE_VOID },
26343 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_movupd, "__builtin_ia32_storeupd", IX86_BUILTIN_STOREUPD, UNKNOWN, (int) VOID_FTYPE_PDOUBLE_V2DF },
26344 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_movdqu, "__builtin_ia32_storedqu", IX86_BUILTIN_STOREDQU, UNKNOWN, (int) VOID_FTYPE_PCHAR_V16QI },
26345 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_movntv2df, "__builtin_ia32_movntpd", IX86_BUILTIN_MOVNTPD, UNKNOWN, (int) VOID_FTYPE_PDOUBLE_V2DF },
26346 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_movntv2di, "__builtin_ia32_movntdq", IX86_BUILTIN_MOVNTDQ, UNKNOWN, (int) VOID_FTYPE_PV2DI_V2DI },
26347 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_movntisi, "__builtin_ia32_movnti", IX86_BUILTIN_MOVNTI, UNKNOWN, (int) VOID_FTYPE_PINT_INT },
26348 { OPTION_MASK_ISA_SSE2 | OPTION_MASK_ISA_64BIT, CODE_FOR_sse2_movntidi, "__builtin_ia32_movnti64", IX86_BUILTIN_MOVNTI64, UNKNOWN, (int) VOID_FTYPE_PLONGLONG_LONGLONG },
26349 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_movupd, "__builtin_ia32_loadupd", IX86_BUILTIN_LOADUPD, UNKNOWN, (int) V2DF_FTYPE_PCDOUBLE },
26350 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_movdqu, "__builtin_ia32_loaddqu", IX86_BUILTIN_LOADDQU, UNKNOWN, (int) V16QI_FTYPE_PCCHAR },
26352 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_loadhpd_exp, "__builtin_ia32_loadhpd", IX86_BUILTIN_LOADHPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_PCDOUBLE },
26353 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_loadlpd_exp, "__builtin_ia32_loadlpd", IX86_BUILTIN_LOADLPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_PCDOUBLE },
26355 /* SSE3 */
26356 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_lddqu, "__builtin_ia32_lddqu", IX86_BUILTIN_LDDQU, UNKNOWN, (int) V16QI_FTYPE_PCCHAR },
26358 /* SSE4.1 */
26359 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_movntdqa, "__builtin_ia32_movntdqa", IX86_BUILTIN_MOVNTDQA, UNKNOWN, (int) V2DI_FTYPE_PV2DI },
26361 /* SSE4A */
26362 { OPTION_MASK_ISA_SSE4A, CODE_FOR_sse4a_vmmovntv2df, "__builtin_ia32_movntsd", IX86_BUILTIN_MOVNTSD, UNKNOWN, (int) VOID_FTYPE_PDOUBLE_V2DF },
26363 { OPTION_MASK_ISA_SSE4A, CODE_FOR_sse4a_vmmovntv4sf, "__builtin_ia32_movntss", IX86_BUILTIN_MOVNTSS, UNKNOWN, (int) VOID_FTYPE_PFLOAT_V4SF },
26365 /* AVX */
26366 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vzeroall, "__builtin_ia32_vzeroall", IX86_BUILTIN_VZEROALL, UNKNOWN, (int) VOID_FTYPE_VOID },
26367 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vzeroupper, "__builtin_ia32_vzeroupper", IX86_BUILTIN_VZEROUPPER, UNKNOWN, (int) VOID_FTYPE_VOID },
26369 { OPTION_MASK_ISA_AVX, CODE_FOR_vec_dupv4sf, "__builtin_ia32_vbroadcastss", IX86_BUILTIN_VBROADCASTSS, UNKNOWN, (int) V4SF_FTYPE_PCFLOAT },
26370 { OPTION_MASK_ISA_AVX, CODE_FOR_vec_dupv4df, "__builtin_ia32_vbroadcastsd256", IX86_BUILTIN_VBROADCASTSD256, UNKNOWN, (int) V4DF_FTYPE_PCDOUBLE },
26371 { OPTION_MASK_ISA_AVX, CODE_FOR_vec_dupv8sf, "__builtin_ia32_vbroadcastss256", IX86_BUILTIN_VBROADCASTSS256, UNKNOWN, (int) V8SF_FTYPE_PCFLOAT },
26372 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vbroadcastf128_v4df, "__builtin_ia32_vbroadcastf128_pd256", IX86_BUILTIN_VBROADCASTPD256, UNKNOWN, (int) V4DF_FTYPE_PCV2DF },
26373 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vbroadcastf128_v8sf, "__builtin_ia32_vbroadcastf128_ps256", IX86_BUILTIN_VBROADCASTPS256, UNKNOWN, (int) V8SF_FTYPE_PCV4SF },
26375 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movupd256, "__builtin_ia32_loadupd256", IX86_BUILTIN_LOADUPD256, UNKNOWN, (int) V4DF_FTYPE_PCDOUBLE },
26376 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movups256, "__builtin_ia32_loadups256", IX86_BUILTIN_LOADUPS256, UNKNOWN, (int) V8SF_FTYPE_PCFLOAT },
26377 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movupd256, "__builtin_ia32_storeupd256", IX86_BUILTIN_STOREUPD256, UNKNOWN, (int) VOID_FTYPE_PDOUBLE_V4DF },
26378 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movups256, "__builtin_ia32_storeups256", IX86_BUILTIN_STOREUPS256, UNKNOWN, (int) VOID_FTYPE_PFLOAT_V8SF },
26379 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movdqu256, "__builtin_ia32_loaddqu256", IX86_BUILTIN_LOADDQU256, UNKNOWN, (int) V32QI_FTYPE_PCCHAR },
26380 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movdqu256, "__builtin_ia32_storedqu256", IX86_BUILTIN_STOREDQU256, UNKNOWN, (int) VOID_FTYPE_PCHAR_V32QI },
26381 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_lddqu256, "__builtin_ia32_lddqu256", IX86_BUILTIN_LDDQU256, UNKNOWN, (int) V32QI_FTYPE_PCCHAR },
26383 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movntv4di, "__builtin_ia32_movntdq256", IX86_BUILTIN_MOVNTDQ256, UNKNOWN, (int) VOID_FTYPE_PV4DI_V4DI },
26384 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movntv4df, "__builtin_ia32_movntpd256", IX86_BUILTIN_MOVNTPD256, UNKNOWN, (int) VOID_FTYPE_PDOUBLE_V4DF },
26385 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movntv8sf, "__builtin_ia32_movntps256", IX86_BUILTIN_MOVNTPS256, UNKNOWN, (int) VOID_FTYPE_PFLOAT_V8SF },
26387 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskloadpd, "__builtin_ia32_maskloadpd", IX86_BUILTIN_MASKLOADPD, UNKNOWN, (int) V2DF_FTYPE_PCV2DF_V2DI },
26388 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskloadps, "__builtin_ia32_maskloadps", IX86_BUILTIN_MASKLOADPS, UNKNOWN, (int) V4SF_FTYPE_PCV4SF_V4SI },
26389 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskloadpd256, "__builtin_ia32_maskloadpd256", IX86_BUILTIN_MASKLOADPD256, UNKNOWN, (int) V4DF_FTYPE_PCV4DF_V4DI },
26390 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskloadps256, "__builtin_ia32_maskloadps256", IX86_BUILTIN_MASKLOADPS256, UNKNOWN, (int) V8SF_FTYPE_PCV8SF_V8SI },
26391 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskstorepd, "__builtin_ia32_maskstorepd", IX86_BUILTIN_MASKSTOREPD, UNKNOWN, (int) VOID_FTYPE_PV2DF_V2DI_V2DF },
26392 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskstoreps, "__builtin_ia32_maskstoreps", IX86_BUILTIN_MASKSTOREPS, UNKNOWN, (int) VOID_FTYPE_PV4SF_V4SI_V4SF },
26393 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskstorepd256, "__builtin_ia32_maskstorepd256", IX86_BUILTIN_MASKSTOREPD256, UNKNOWN, (int) VOID_FTYPE_PV4DF_V4DI_V4DF },
26394 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskstoreps256, "__builtin_ia32_maskstoreps256", IX86_BUILTIN_MASKSTOREPS256, UNKNOWN, (int) VOID_FTYPE_PV8SF_V8SI_V8SF },
26396 /* AVX2 */
26397 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_movntdqa, "__builtin_ia32_movntdqa256", IX86_BUILTIN_MOVNTDQA256, UNKNOWN, (int) V4DI_FTYPE_PV4DI },
26398 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_maskloadd, "__builtin_ia32_maskloadd", IX86_BUILTIN_MASKLOADD, UNKNOWN, (int) V4SI_FTYPE_PCV4SI_V4SI },
26399 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_maskloadq, "__builtin_ia32_maskloadq", IX86_BUILTIN_MASKLOADQ, UNKNOWN, (int) V2DI_FTYPE_PCV2DI_V2DI },
26400 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_maskloadd256, "__builtin_ia32_maskloadd256", IX86_BUILTIN_MASKLOADD256, UNKNOWN, (int) V8SI_FTYPE_PCV8SI_V8SI },
26401 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_maskloadq256, "__builtin_ia32_maskloadq256", IX86_BUILTIN_MASKLOADQ256, UNKNOWN, (int) V4DI_FTYPE_PCV4DI_V4DI },
26402 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_maskstored, "__builtin_ia32_maskstored", IX86_BUILTIN_MASKSTORED, UNKNOWN, (int) VOID_FTYPE_PV4SI_V4SI_V4SI },
26403 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_maskstoreq, "__builtin_ia32_maskstoreq", IX86_BUILTIN_MASKSTOREQ, UNKNOWN, (int) VOID_FTYPE_PV2DI_V2DI_V2DI },
26404 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_maskstored256, "__builtin_ia32_maskstored256", IX86_BUILTIN_MASKSTORED256, UNKNOWN, (int) VOID_FTYPE_PV8SI_V8SI_V8SI },
26405 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_maskstoreq256, "__builtin_ia32_maskstoreq256", IX86_BUILTIN_MASKSTOREQ256, UNKNOWN, (int) VOID_FTYPE_PV4DI_V4DI_V4DI },
26407 { OPTION_MASK_ISA_LWP, CODE_FOR_lwp_llwpcb, "__builtin_ia32_llwpcb", IX86_BUILTIN_LLWPCB, UNKNOWN, (int) VOID_FTYPE_PVOID },
26408 { OPTION_MASK_ISA_LWP, CODE_FOR_lwp_slwpcb, "__builtin_ia32_slwpcb", IX86_BUILTIN_SLWPCB, UNKNOWN, (int) PVOID_FTYPE_VOID },
26409 { OPTION_MASK_ISA_LWP, CODE_FOR_lwp_lwpvalsi3, "__builtin_ia32_lwpval32", IX86_BUILTIN_LWPVAL32, UNKNOWN, (int) VOID_FTYPE_UINT_UINT_UINT },
26410 { OPTION_MASK_ISA_LWP, CODE_FOR_lwp_lwpvaldi3, "__builtin_ia32_lwpval64", IX86_BUILTIN_LWPVAL64, UNKNOWN, (int) VOID_FTYPE_UINT64_UINT_UINT },
26411 { OPTION_MASK_ISA_LWP, CODE_FOR_lwp_lwpinssi3, "__builtin_ia32_lwpins32", IX86_BUILTIN_LWPINS32, UNKNOWN, (int) UCHAR_FTYPE_UINT_UINT_UINT },
26412 { OPTION_MASK_ISA_LWP, CODE_FOR_lwp_lwpinsdi3, "__builtin_ia32_lwpins64", IX86_BUILTIN_LWPINS64, UNKNOWN, (int) UCHAR_FTYPE_UINT64_UINT_UINT },
26414 /* FSGSBASE */
26415 { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_rdfsbasesi, "__builtin_ia32_rdfsbase32", IX86_BUILTIN_RDFSBASE32, UNKNOWN, (int) UNSIGNED_FTYPE_VOID },
26416 { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_rdfsbasedi, "__builtin_ia32_rdfsbase64", IX86_BUILTIN_RDFSBASE64, UNKNOWN, (int) UINT64_FTYPE_VOID },
26417 { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_rdgsbasesi, "__builtin_ia32_rdgsbase32", IX86_BUILTIN_RDGSBASE32, UNKNOWN, (int) UNSIGNED_FTYPE_VOID },
26418 { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_rdgsbasedi, "__builtin_ia32_rdgsbase64", IX86_BUILTIN_RDGSBASE64, UNKNOWN, (int) UINT64_FTYPE_VOID },
26419 { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_wrfsbasesi, "__builtin_ia32_wrfsbase32", IX86_BUILTIN_WRFSBASE32, UNKNOWN, (int) VOID_FTYPE_UNSIGNED },
26420 { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_wrfsbasedi, "__builtin_ia32_wrfsbase64", IX86_BUILTIN_WRFSBASE64, UNKNOWN, (int) VOID_FTYPE_UINT64 },
26421 { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_wrgsbasesi, "__builtin_ia32_wrgsbase32", IX86_BUILTIN_WRGSBASE32, UNKNOWN, (int) VOID_FTYPE_UNSIGNED },
26422 { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_wrgsbasedi, "__builtin_ia32_wrgsbase64", IX86_BUILTIN_WRGSBASE64, UNKNOWN, (int) VOID_FTYPE_UINT64 },
26424 /* RTM */
26425 { OPTION_MASK_ISA_RTM, CODE_FOR_xbegin, "__builtin_ia32_xbegin", IX86_BUILTIN_XBEGIN, UNKNOWN, (int) UNSIGNED_FTYPE_VOID },
26426 { OPTION_MASK_ISA_RTM, CODE_FOR_xend, "__builtin_ia32_xend", IX86_BUILTIN_XEND, UNKNOWN, (int) VOID_FTYPE_VOID },
26427 { OPTION_MASK_ISA_RTM, CODE_FOR_xtest, "__builtin_ia32_xtest", IX86_BUILTIN_XTEST, UNKNOWN, (int) INT_FTYPE_VOID },
26428 };
26430 /* Builtins with variable number of arguments. */
26432 {
26433 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_bsr, "__builtin_ia32_bsrsi", IX86_BUILTIN_BSRSI, UNKNOWN, (int) INT_FTYPE_INT },
26434 { OPTION_MASK_ISA_64BIT, CODE_FOR_bsr_rex64, "__builtin_ia32_bsrdi", IX86_BUILTIN_BSRDI, UNKNOWN, (int) INT64_FTYPE_INT64 },
26435 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_rdpmc, "__builtin_ia32_rdpmc", IX86_BUILTIN_RDPMC, UNKNOWN, (int) UINT64_FTYPE_INT },
26436 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_rotlqi3, "__builtin_ia32_rolqi", IX86_BUILTIN_ROLQI, UNKNOWN, (int) UINT8_FTYPE_UINT8_INT },
26437 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_rotlhi3, "__builtin_ia32_rolhi", IX86_BUILTIN_ROLHI, UNKNOWN, (int) UINT16_FTYPE_UINT16_INT },
26438 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_rotrqi3, "__builtin_ia32_rorqi", IX86_BUILTIN_RORQI, UNKNOWN, (int) UINT8_FTYPE_UINT8_INT },
26439 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_rotrhi3, "__builtin_ia32_rorhi", IX86_BUILTIN_RORHI, UNKNOWN, (int) UINT16_FTYPE_UINT16_INT },
26441 /* MMX */
26442 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_addv8qi3, "__builtin_ia32_paddb", IX86_BUILTIN_PADDB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
26443 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_addv4hi3, "__builtin_ia32_paddw", IX86_BUILTIN_PADDW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
26444 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_addv2si3, "__builtin_ia32_paddd", IX86_BUILTIN_PADDD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
26445 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_subv8qi3, "__builtin_ia32_psubb", IX86_BUILTIN_PSUBB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
26446 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_subv4hi3, "__builtin_ia32_psubw", IX86_BUILTIN_PSUBW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
26447 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_subv2si3, "__builtin_ia32_psubd", IX86_BUILTIN_PSUBD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
26449 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ssaddv8qi3, "__builtin_ia32_paddsb", IX86_BUILTIN_PADDSB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
26450 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ssaddv4hi3, "__builtin_ia32_paddsw", IX86_BUILTIN_PADDSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
26451 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_sssubv8qi3, "__builtin_ia32_psubsb", IX86_BUILTIN_PSUBSB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
26452 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_sssubv4hi3, "__builtin_ia32_psubsw", IX86_BUILTIN_PSUBSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
26453 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_usaddv8qi3, "__builtin_ia32_paddusb", IX86_BUILTIN_PADDUSB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
26454 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_usaddv4hi3, "__builtin_ia32_paddusw", IX86_BUILTIN_PADDUSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
26455 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ussubv8qi3, "__builtin_ia32_psubusb", IX86_BUILTIN_PSUBUSB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
26456 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ussubv4hi3, "__builtin_ia32_psubusw", IX86_BUILTIN_PSUBUSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
26458 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_mulv4hi3, "__builtin_ia32_pmullw", IX86_BUILTIN_PMULLW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
26459 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_smulv4hi3_highpart, "__builtin_ia32_pmulhw", IX86_BUILTIN_PMULHW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
26461 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_andv2si3, "__builtin_ia32_pand", IX86_BUILTIN_PAND, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
26462 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_andnotv2si3, "__builtin_ia32_pandn", IX86_BUILTIN_PANDN, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
26463 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_iorv2si3, "__builtin_ia32_por", IX86_BUILTIN_POR, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
26464 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_xorv2si3, "__builtin_ia32_pxor", IX86_BUILTIN_PXOR, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
26466 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_eqv8qi3, "__builtin_ia32_pcmpeqb", IX86_BUILTIN_PCMPEQB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
26467 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_eqv4hi3, "__builtin_ia32_pcmpeqw", IX86_BUILTIN_PCMPEQW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
26468 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_eqv2si3, "__builtin_ia32_pcmpeqd", IX86_BUILTIN_PCMPEQD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
26469 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_gtv8qi3, "__builtin_ia32_pcmpgtb", IX86_BUILTIN_PCMPGTB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
26470 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_gtv4hi3, "__builtin_ia32_pcmpgtw", IX86_BUILTIN_PCMPGTW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
26471 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_gtv2si3, "__builtin_ia32_pcmpgtd", IX86_BUILTIN_PCMPGTD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
26473 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_punpckhbw, "__builtin_ia32_punpckhbw", IX86_BUILTIN_PUNPCKHBW, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
26474 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_punpckhwd, "__builtin_ia32_punpckhwd", IX86_BUILTIN_PUNPCKHWD, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
26475 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_punpckhdq, "__builtin_ia32_punpckhdq", IX86_BUILTIN_PUNPCKHDQ, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
26476 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_punpcklbw, "__builtin_ia32_punpcklbw", IX86_BUILTIN_PUNPCKLBW, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
26477 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_punpcklwd, "__builtin_ia32_punpcklwd", IX86_BUILTIN_PUNPCKLWD, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI},
26478 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_punpckldq, "__builtin_ia32_punpckldq", IX86_BUILTIN_PUNPCKLDQ, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI},
26480 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_packsswb, "__builtin_ia32_packsswb", IX86_BUILTIN_PACKSSWB, UNKNOWN, (int) V8QI_FTYPE_V4HI_V4HI },
26481 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_packssdw, "__builtin_ia32_packssdw", IX86_BUILTIN_PACKSSDW, UNKNOWN, (int) V4HI_FTYPE_V2SI_V2SI },
26482 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_packuswb, "__builtin_ia32_packuswb", IX86_BUILTIN_PACKUSWB, UNKNOWN, (int) V8QI_FTYPE_V4HI_V4HI },
26484 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_pmaddwd, "__builtin_ia32_pmaddwd", IX86_BUILTIN_PMADDWD, UNKNOWN, (int) V2SI_FTYPE_V4HI_V4HI },
26486 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashlv4hi3, "__builtin_ia32_psllwi", IX86_BUILTIN_PSLLWI, UNKNOWN, (int) V4HI_FTYPE_V4HI_SI_COUNT },
26487 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashlv2si3, "__builtin_ia32_pslldi", IX86_BUILTIN_PSLLDI, UNKNOWN, (int) V2SI_FTYPE_V2SI_SI_COUNT },
26488 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashlv1di3, "__builtin_ia32_psllqi", IX86_BUILTIN_PSLLQI, UNKNOWN, (int) V1DI_FTYPE_V1DI_SI_COUNT },
26489 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashlv4hi3, "__builtin_ia32_psllw", IX86_BUILTIN_PSLLW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI_COUNT },
26490 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashlv2si3, "__builtin_ia32_pslld", IX86_BUILTIN_PSLLD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI_COUNT },
26491 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashlv1di3, "__builtin_ia32_psllq", IX86_BUILTIN_PSLLQ, UNKNOWN, (int) V1DI_FTYPE_V1DI_V1DI_COUNT },
26493 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_lshrv4hi3, "__builtin_ia32_psrlwi", IX86_BUILTIN_PSRLWI, UNKNOWN, (int) V4HI_FTYPE_V4HI_SI_COUNT },
26494 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_lshrv2si3, "__builtin_ia32_psrldi", IX86_BUILTIN_PSRLDI, UNKNOWN, (int) V2SI_FTYPE_V2SI_SI_COUNT },
26495 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_lshrv1di3, "__builtin_ia32_psrlqi", IX86_BUILTIN_PSRLQI, UNKNOWN, (int) V1DI_FTYPE_V1DI_SI_COUNT },
26496 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_lshrv4hi3, "__builtin_ia32_psrlw", IX86_BUILTIN_PSRLW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI_COUNT },
26497 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_lshrv2si3, "__builtin_ia32_psrld", IX86_BUILTIN_PSRLD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI_COUNT },
26498 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_lshrv1di3, "__builtin_ia32_psrlq", IX86_BUILTIN_PSRLQ, UNKNOWN, (int) V1DI_FTYPE_V1DI_V1DI_COUNT },
26500 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashrv4hi3, "__builtin_ia32_psrawi", IX86_BUILTIN_PSRAWI, UNKNOWN, (int) V4HI_FTYPE_V4HI_SI_COUNT },
26501 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashrv2si3, "__builtin_ia32_psradi", IX86_BUILTIN_PSRADI, UNKNOWN, (int) V2SI_FTYPE_V2SI_SI_COUNT },
26502 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashrv4hi3, "__builtin_ia32_psraw", IX86_BUILTIN_PSRAW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI_COUNT },
26503 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashrv2si3, "__builtin_ia32_psrad", IX86_BUILTIN_PSRAD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI_COUNT },
26505 /* 3DNow! */
26506 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_pf2id, "__builtin_ia32_pf2id", IX86_BUILTIN_PF2ID, UNKNOWN, (int) V2SI_FTYPE_V2SF },
26507 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_floatv2si2, "__builtin_ia32_pi2fd", IX86_BUILTIN_PI2FD, UNKNOWN, (int) V2SF_FTYPE_V2SI },
26508 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_rcpv2sf2, "__builtin_ia32_pfrcp", IX86_BUILTIN_PFRCP, UNKNOWN, (int) V2SF_FTYPE_V2SF },
26509 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_rsqrtv2sf2, "__builtin_ia32_pfrsqrt", IX86_BUILTIN_PFRSQRT, UNKNOWN, (int) V2SF_FTYPE_V2SF },
26511 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_uavgv8qi3, "__builtin_ia32_pavgusb", IX86_BUILTIN_PAVGUSB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
26512 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_haddv2sf3, "__builtin_ia32_pfacc", IX86_BUILTIN_PFACC, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
26513 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_addv2sf3, "__builtin_ia32_pfadd", IX86_BUILTIN_PFADD, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
26514 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_eqv2sf3, "__builtin_ia32_pfcmpeq", IX86_BUILTIN_PFCMPEQ, UNKNOWN, (int) V2SI_FTYPE_V2SF_V2SF },
26515 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_gev2sf3, "__builtin_ia32_pfcmpge", IX86_BUILTIN_PFCMPGE, UNKNOWN, (int) V2SI_FTYPE_V2SF_V2SF },
26516 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_gtv2sf3, "__builtin_ia32_pfcmpgt", IX86_BUILTIN_PFCMPGT, UNKNOWN, (int) V2SI_FTYPE_V2SF_V2SF },
26517 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_smaxv2sf3, "__builtin_ia32_pfmax", IX86_BUILTIN_PFMAX, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
26518 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_sminv2sf3, "__builtin_ia32_pfmin", IX86_BUILTIN_PFMIN, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
26519 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_mulv2sf3, "__builtin_ia32_pfmul", IX86_BUILTIN_PFMUL, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
26520 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_rcpit1v2sf3, "__builtin_ia32_pfrcpit1", IX86_BUILTIN_PFRCPIT1, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
26521 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_rcpit2v2sf3, "__builtin_ia32_pfrcpit2", IX86_BUILTIN_PFRCPIT2, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
26522 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_rsqit1v2sf3, "__builtin_ia32_pfrsqit1", IX86_BUILTIN_PFRSQIT1, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
26523 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_subv2sf3, "__builtin_ia32_pfsub", IX86_BUILTIN_PFSUB, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
26524 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_subrv2sf3, "__builtin_ia32_pfsubr", IX86_BUILTIN_PFSUBR, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
26525 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_pmulhrwv4hi3, "__builtin_ia32_pmulhrw", IX86_BUILTIN_PMULHRW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
26527 /* 3DNow!A */
26528 { OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_pf2iw, "__builtin_ia32_pf2iw", IX86_BUILTIN_PF2IW, UNKNOWN, (int) V2SI_FTYPE_V2SF },
26529 { OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_pi2fw, "__builtin_ia32_pi2fw", IX86_BUILTIN_PI2FW, UNKNOWN, (int) V2SF_FTYPE_V2SI },
26530 { OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_pswapdv2si2, "__builtin_ia32_pswapdsi", IX86_BUILTIN_PSWAPDSI, UNKNOWN, (int) V2SI_FTYPE_V2SI },
26531 { OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_pswapdv2sf2, "__builtin_ia32_pswapdsf", IX86_BUILTIN_PSWAPDSF, UNKNOWN, (int) V2SF_FTYPE_V2SF },
26532 { OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_hsubv2sf3, "__builtin_ia32_pfnacc", IX86_BUILTIN_PFNACC, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
26533 { OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_addsubv2sf3, "__builtin_ia32_pfpnacc", IX86_BUILTIN_PFPNACC, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
26535 /* SSE */
26536 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_movmskps, "__builtin_ia32_movmskps", IX86_BUILTIN_MOVMSKPS, UNKNOWN, (int) INT_FTYPE_V4SF },
26537 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_sqrtv4sf2, "__builtin_ia32_sqrtps", IX86_BUILTIN_SQRTPS, UNKNOWN, (int) V4SF_FTYPE_V4SF },
26538 { OPTION_MASK_ISA_SSE, CODE_FOR_sqrtv4sf2, "__builtin_ia32_sqrtps_nr", IX86_BUILTIN_SQRTPS_NR, UNKNOWN, (int) V4SF_FTYPE_V4SF },
26539 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_rsqrtv4sf2, "__builtin_ia32_rsqrtps", IX86_BUILTIN_RSQRTPS, UNKNOWN, (int) V4SF_FTYPE_V4SF },
26540 { OPTION_MASK_ISA_SSE, CODE_FOR_rsqrtv4sf2, "__builtin_ia32_rsqrtps_nr", IX86_BUILTIN_RSQRTPS_NR, UNKNOWN, (int) V4SF_FTYPE_V4SF },
26541 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_rcpv4sf2, "__builtin_ia32_rcpps", IX86_BUILTIN_RCPPS, UNKNOWN, (int) V4SF_FTYPE_V4SF },
26542 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_cvtps2pi, "__builtin_ia32_cvtps2pi", IX86_BUILTIN_CVTPS2PI, UNKNOWN, (int) V2SI_FTYPE_V4SF },
26543 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_cvtss2si, "__builtin_ia32_cvtss2si", IX86_BUILTIN_CVTSS2SI, UNKNOWN, (int) INT_FTYPE_V4SF },
26544 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_64BIT, CODE_FOR_sse_cvtss2siq, "__builtin_ia32_cvtss2si64", IX86_BUILTIN_CVTSS2SI64, UNKNOWN, (int) INT64_FTYPE_V4SF },
26545 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_cvttps2pi, "__builtin_ia32_cvttps2pi", IX86_BUILTIN_CVTTPS2PI, UNKNOWN, (int) V2SI_FTYPE_V4SF },
26546 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_cvttss2si, "__builtin_ia32_cvttss2si", IX86_BUILTIN_CVTTSS2SI, UNKNOWN, (int) INT_FTYPE_V4SF },
26547 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_64BIT, CODE_FOR_sse_cvttss2siq, "__builtin_ia32_cvttss2si64", IX86_BUILTIN_CVTTSS2SI64, UNKNOWN, (int) INT64_FTYPE_V4SF },
26549 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_shufps, "__builtin_ia32_shufps", IX86_BUILTIN_SHUFPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
26551 { OPTION_MASK_ISA_SSE, CODE_FOR_addv4sf3, "__builtin_ia32_addps", IX86_BUILTIN_ADDPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
26552 { OPTION_MASK_ISA_SSE, CODE_FOR_subv4sf3, "__builtin_ia32_subps", IX86_BUILTIN_SUBPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
26553 { OPTION_MASK_ISA_SSE, CODE_FOR_mulv4sf3, "__builtin_ia32_mulps", IX86_BUILTIN_MULPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
26554 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_divv4sf3, "__builtin_ia32_divps", IX86_BUILTIN_DIVPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
26555 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmaddv4sf3, "__builtin_ia32_addss", IX86_BUILTIN_ADDSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
26556 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmsubv4sf3, "__builtin_ia32_subss", IX86_BUILTIN_SUBSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
26557 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmulv4sf3, "__builtin_ia32_mulss", IX86_BUILTIN_MULSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
26558 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmdivv4sf3, "__builtin_ia32_divss", IX86_BUILTIN_DIVSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
26560 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpeqps", IX86_BUILTIN_CMPEQPS, EQ, (int) V4SF_FTYPE_V4SF_V4SF },
26561 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpltps", IX86_BUILTIN_CMPLTPS, LT, (int) V4SF_FTYPE_V4SF_V4SF },
26562 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpleps", IX86_BUILTIN_CMPLEPS, LE, (int) V4SF_FTYPE_V4SF_V4SF },
26563 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpgtps", IX86_BUILTIN_CMPGTPS, LT, (int) V4SF_FTYPE_V4SF_V4SF_SWAP },
26564 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpgeps", IX86_BUILTIN_CMPGEPS, LE, (int) V4SF_FTYPE_V4SF_V4SF_SWAP },
26565 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpunordps", IX86_BUILTIN_CMPUNORDPS, UNORDERED, (int) V4SF_FTYPE_V4SF_V4SF },
26566 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpneqps", IX86_BUILTIN_CMPNEQPS, NE, (int) V4SF_FTYPE_V4SF_V4SF },
26567 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpnltps", IX86_BUILTIN_CMPNLTPS, UNGE, (int) V4SF_FTYPE_V4SF_V4SF },
26568 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpnleps", IX86_BUILTIN_CMPNLEPS, UNGT, (int) V4SF_FTYPE_V4SF_V4SF },
26569 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpngtps", IX86_BUILTIN_CMPNGTPS, UNGE, (int) V4SF_FTYPE_V4SF_V4SF_SWAP },
26570 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpngeps", IX86_BUILTIN_CMPNGEPS, UNGT, (int) V4SF_FTYPE_V4SF_V4SF_SWAP},
26571 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpordps", IX86_BUILTIN_CMPORDPS, ORDERED, (int) V4SF_FTYPE_V4SF_V4SF },
26572 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpeqss", IX86_BUILTIN_CMPEQSS, EQ, (int) V4SF_FTYPE_V4SF_V4SF },
26573 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpltss", IX86_BUILTIN_CMPLTSS, LT, (int) V4SF_FTYPE_V4SF_V4SF },
26574 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpless", IX86_BUILTIN_CMPLESS, LE, (int) V4SF_FTYPE_V4SF_V4SF },
26575 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpunordss", IX86_BUILTIN_CMPUNORDSS, UNORDERED, (int) V4SF_FTYPE_V4SF_V4SF },
26576 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpneqss", IX86_BUILTIN_CMPNEQSS, NE, (int) V4SF_FTYPE_V4SF_V4SF },
26577 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpnltss", IX86_BUILTIN_CMPNLTSS, UNGE, (int) V4SF_FTYPE_V4SF_V4SF },
26578 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpnless", IX86_BUILTIN_CMPNLESS, UNGT, (int) V4SF_FTYPE_V4SF_V4SF },
26579 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpngtss", IX86_BUILTIN_CMPNGTSS, UNGE, (int) V4SF_FTYPE_V4SF_V4SF_SWAP },
26580 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpngess", IX86_BUILTIN_CMPNGESS, UNGT, (int) V4SF_FTYPE_V4SF_V4SF_SWAP },
26581 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpordss", IX86_BUILTIN_CMPORDSS, ORDERED, (int) V4SF_FTYPE_V4SF_V4SF },
26583 { OPTION_MASK_ISA_SSE, CODE_FOR_sminv4sf3, "__builtin_ia32_minps", IX86_BUILTIN_MINPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
26584 { OPTION_MASK_ISA_SSE, CODE_FOR_smaxv4sf3, "__builtin_ia32_maxps", IX86_BUILTIN_MAXPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
26585 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmsminv4sf3, "__builtin_ia32_minss", IX86_BUILTIN_MINSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
26586 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmsmaxv4sf3, "__builtin_ia32_maxss", IX86_BUILTIN_MAXSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
26588 { OPTION_MASK_ISA_SSE, CODE_FOR_andv4sf3, "__builtin_ia32_andps", IX86_BUILTIN_ANDPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
26589 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_andnotv4sf3, "__builtin_ia32_andnps", IX86_BUILTIN_ANDNPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
26590 { OPTION_MASK_ISA_SSE, CODE_FOR_iorv4sf3, "__builtin_ia32_orps", IX86_BUILTIN_ORPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
26591 { OPTION_MASK_ISA_SSE, CODE_FOR_xorv4sf3, "__builtin_ia32_xorps", IX86_BUILTIN_XORPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
26593 { OPTION_MASK_ISA_SSE, CODE_FOR_copysignv4sf3, "__builtin_ia32_copysignps", IX86_BUILTIN_CPYSGNPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
26595 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_movss, "__builtin_ia32_movss", IX86_BUILTIN_MOVSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
26596 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_movhlps_exp, "__builtin_ia32_movhlps", IX86_BUILTIN_MOVHLPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
26597 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_movlhps_exp, "__builtin_ia32_movlhps", IX86_BUILTIN_MOVLHPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
26598 { OPTION_MASK_ISA_SSE, CODE_FOR_vec_interleave_highv4sf, "__builtin_ia32_unpckhps", IX86_BUILTIN_UNPCKHPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
26599 { OPTION_MASK_ISA_SSE, CODE_FOR_vec_interleave_lowv4sf, "__builtin_ia32_unpcklps", IX86_BUILTIN_UNPCKLPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
26601 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_cvtpi2ps, "__builtin_ia32_cvtpi2ps", IX86_BUILTIN_CVTPI2PS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V2SI },
26602 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_cvtsi2ss, "__builtin_ia32_cvtsi2ss", IX86_BUILTIN_CVTSI2SS, UNKNOWN, (int) V4SF_FTYPE_V4SF_SI },
26603 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_64BIT, CODE_FOR_sse_cvtsi2ssq, "__builtin_ia32_cvtsi642ss", IX86_BUILTIN_CVTSI642SS, UNKNOWN, V4SF_FTYPE_V4SF_DI },
26605 { OPTION_MASK_ISA_SSE, CODE_FOR_rsqrtsf2, "__builtin_ia32_rsqrtf", IX86_BUILTIN_RSQRTF, UNKNOWN, (int) FLOAT_FTYPE_FLOAT },
26607 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmsqrtv4sf2, "__builtin_ia32_sqrtss", IX86_BUILTIN_SQRTSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_VEC_MERGE },
26608 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmrsqrtv4sf2, "__builtin_ia32_rsqrtss", IX86_BUILTIN_RSQRTSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_VEC_MERGE },
26609 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmrcpv4sf2, "__builtin_ia32_rcpss", IX86_BUILTIN_RCPSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_VEC_MERGE },
26611 { OPTION_MASK_ISA_SSE, CODE_FOR_abstf2, 0, IX86_BUILTIN_FABSQ, UNKNOWN, (int) FLOAT128_FTYPE_FLOAT128 },
26612 { OPTION_MASK_ISA_SSE, CODE_FOR_copysigntf3, 0, IX86_BUILTIN_COPYSIGNQ, UNKNOWN, (int) FLOAT128_FTYPE_FLOAT128_FLOAT128 },
26614 /* SSE MMX or 3Dnow!A */
26615 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_uavgv8qi3, "__builtin_ia32_pavgb", IX86_BUILTIN_PAVGB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
26616 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_uavgv4hi3, "__builtin_ia32_pavgw", IX86_BUILTIN_PAVGW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
26617 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_umulv4hi3_highpart, "__builtin_ia32_pmulhuw", IX86_BUILTIN_PMULHUW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
26619 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_umaxv8qi3, "__builtin_ia32_pmaxub", IX86_BUILTIN_PMAXUB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
26620 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_smaxv4hi3, "__builtin_ia32_pmaxsw", IX86_BUILTIN_PMAXSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
26621 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_uminv8qi3, "__builtin_ia32_pminub", IX86_BUILTIN_PMINUB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
26622 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_sminv4hi3, "__builtin_ia32_pminsw", IX86_BUILTIN_PMINSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
26624 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_psadbw, "__builtin_ia32_psadbw", IX86_BUILTIN_PSADBW, UNKNOWN, (int) V1DI_FTYPE_V8QI_V8QI },
26625 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_pmovmskb, "__builtin_ia32_pmovmskb", IX86_BUILTIN_PMOVMSKB, UNKNOWN, (int) INT_FTYPE_V8QI },
26627 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_pshufw, "__builtin_ia32_pshufw", IX86_BUILTIN_PSHUFW, UNKNOWN, (int) V4HI_FTYPE_V4HI_INT },
26629 /* SSE2 */
26630 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_shufpd, "__builtin_ia32_shufpd", IX86_BUILTIN_SHUFPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
26632 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_movmskpd, "__builtin_ia32_movmskpd", IX86_BUILTIN_MOVMSKPD, UNKNOWN, (int) INT_FTYPE_V2DF },
26633 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_pmovmskb, "__builtin_ia32_pmovmskb128", IX86_BUILTIN_PMOVMSKB128, UNKNOWN, (int) INT_FTYPE_V16QI },
26634 { OPTION_MASK_ISA_SSE2, CODE_FOR_sqrtv2df2, "__builtin_ia32_sqrtpd", IX86_BUILTIN_SQRTPD, UNKNOWN, (int) V2DF_FTYPE_V2DF },
26635 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtdq2pd, "__builtin_ia32_cvtdq2pd", IX86_BUILTIN_CVTDQ2PD, UNKNOWN, (int) V2DF_FTYPE_V4SI },
26636 { OPTION_MASK_ISA_SSE2, CODE_FOR_floatv4siv4sf2, "__builtin_ia32_cvtdq2ps", IX86_BUILTIN_CVTDQ2PS, UNKNOWN, (int) V4SF_FTYPE_V4SI },
26638 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtpd2dq, "__builtin_ia32_cvtpd2dq", IX86_BUILTIN_CVTPD2DQ, UNKNOWN, (int) V4SI_FTYPE_V2DF },
26639 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtpd2pi, "__builtin_ia32_cvtpd2pi", IX86_BUILTIN_CVTPD2PI, UNKNOWN, (int) V2SI_FTYPE_V2DF },
26640 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtpd2ps, "__builtin_ia32_cvtpd2ps", IX86_BUILTIN_CVTPD2PS, UNKNOWN, (int) V4SF_FTYPE_V2DF },
26641 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvttpd2dq, "__builtin_ia32_cvttpd2dq", IX86_BUILTIN_CVTTPD2DQ, UNKNOWN, (int) V4SI_FTYPE_V2DF },
26642 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvttpd2pi, "__builtin_ia32_cvttpd2pi", IX86_BUILTIN_CVTTPD2PI, UNKNOWN, (int) V2SI_FTYPE_V2DF },
26644 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtpi2pd, "__builtin_ia32_cvtpi2pd", IX86_BUILTIN_CVTPI2PD, UNKNOWN, (int) V2DF_FTYPE_V2SI },
26646 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtsd2si, "__builtin_ia32_cvtsd2si", IX86_BUILTIN_CVTSD2SI, UNKNOWN, (int) INT_FTYPE_V2DF },
26647 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvttsd2si, "__builtin_ia32_cvttsd2si", IX86_BUILTIN_CVTTSD2SI, UNKNOWN, (int) INT_FTYPE_V2DF },
26648 { OPTION_MASK_ISA_SSE2 | OPTION_MASK_ISA_64BIT, CODE_FOR_sse2_cvtsd2siq, "__builtin_ia32_cvtsd2si64", IX86_BUILTIN_CVTSD2SI64, UNKNOWN, (int) INT64_FTYPE_V2DF },
26649 { OPTION_MASK_ISA_SSE2 | OPTION_MASK_ISA_64BIT, CODE_FOR_sse2_cvttsd2siq, "__builtin_ia32_cvttsd2si64", IX86_BUILTIN_CVTTSD2SI64, UNKNOWN, (int) INT64_FTYPE_V2DF },
26651 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtps2dq, "__builtin_ia32_cvtps2dq", IX86_BUILTIN_CVTPS2DQ, UNKNOWN, (int) V4SI_FTYPE_V4SF },
26652 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtps2pd, "__builtin_ia32_cvtps2pd", IX86_BUILTIN_CVTPS2PD, UNKNOWN, (int) V2DF_FTYPE_V4SF },
26653 { OPTION_MASK_ISA_SSE2, CODE_FOR_fix_truncv4sfv4si2, "__builtin_ia32_cvttps2dq", IX86_BUILTIN_CVTTPS2DQ, UNKNOWN, (int) V4SI_FTYPE_V4SF },
26655 { OPTION_MASK_ISA_SSE2, CODE_FOR_addv2df3, "__builtin_ia32_addpd", IX86_BUILTIN_ADDPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
26656 { OPTION_MASK_ISA_SSE2, CODE_FOR_subv2df3, "__builtin_ia32_subpd", IX86_BUILTIN_SUBPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
26657 { OPTION_MASK_ISA_SSE2, CODE_FOR_mulv2df3, "__builtin_ia32_mulpd", IX86_BUILTIN_MULPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
26658 { OPTION_MASK_ISA_SSE2, CODE_FOR_divv2df3, "__builtin_ia32_divpd", IX86_BUILTIN_DIVPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
26659 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmaddv2df3, "__builtin_ia32_addsd", IX86_BUILTIN_ADDSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
26660 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmsubv2df3, "__builtin_ia32_subsd", IX86_BUILTIN_SUBSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
26661 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmulv2df3, "__builtin_ia32_mulsd", IX86_BUILTIN_MULSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
26662 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmdivv2df3, "__builtin_ia32_divsd", IX86_BUILTIN_DIVSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
26664 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpeqpd", IX86_BUILTIN_CMPEQPD, EQ, (int) V2DF_FTYPE_V2DF_V2DF },
26665 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpltpd", IX86_BUILTIN_CMPLTPD, LT, (int) V2DF_FTYPE_V2DF_V2DF },
26666 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmplepd", IX86_BUILTIN_CMPLEPD, LE, (int) V2DF_FTYPE_V2DF_V2DF },
26667 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpgtpd", IX86_BUILTIN_CMPGTPD, LT, (int) V2DF_FTYPE_V2DF_V2DF_SWAP },
26668 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpgepd", IX86_BUILTIN_CMPGEPD, LE, (int) V2DF_FTYPE_V2DF_V2DF_SWAP},
26669 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpunordpd", IX86_BUILTIN_CMPUNORDPD, UNORDERED, (int) V2DF_FTYPE_V2DF_V2DF },
26670 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpneqpd", IX86_BUILTIN_CMPNEQPD, NE, (int) V2DF_FTYPE_V2DF_V2DF },
26671 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpnltpd", IX86_BUILTIN_CMPNLTPD, UNGE, (int) V2DF_FTYPE_V2DF_V2DF },
26672 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpnlepd", IX86_BUILTIN_CMPNLEPD, UNGT, (int) V2DF_FTYPE_V2DF_V2DF },
26673 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpngtpd", IX86_BUILTIN_CMPNGTPD, UNGE, (int) V2DF_FTYPE_V2DF_V2DF_SWAP },
26674 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpngepd", IX86_BUILTIN_CMPNGEPD, UNGT, (int) V2DF_FTYPE_V2DF_V2DF_SWAP },
26675 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpordpd", IX86_BUILTIN_CMPORDPD, ORDERED, (int) V2DF_FTYPE_V2DF_V2DF },
26676 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmpeqsd", IX86_BUILTIN_CMPEQSD, EQ, (int) V2DF_FTYPE_V2DF_V2DF },
26677 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmpltsd", IX86_BUILTIN_CMPLTSD, LT, (int) V2DF_FTYPE_V2DF_V2DF },
26678 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmplesd", IX86_BUILTIN_CMPLESD, LE, (int) V2DF_FTYPE_V2DF_V2DF },
26679 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmpunordsd", IX86_BUILTIN_CMPUNORDSD, UNORDERED, (int) V2DF_FTYPE_V2DF_V2DF },
26680 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmpneqsd", IX86_BUILTIN_CMPNEQSD, NE, (int) V2DF_FTYPE_V2DF_V2DF },
26681 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmpnltsd", IX86_BUILTIN_CMPNLTSD, UNGE, (int) V2DF_FTYPE_V2DF_V2DF },
26682 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmpnlesd", IX86_BUILTIN_CMPNLESD, UNGT, (int) V2DF_FTYPE_V2DF_V2DF },
26683 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmpordsd", IX86_BUILTIN_CMPORDSD, ORDERED, (int) V2DF_FTYPE_V2DF_V2DF },
26685 { OPTION_MASK_ISA_SSE2, CODE_FOR_sminv2df3, "__builtin_ia32_minpd", IX86_BUILTIN_MINPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
26686 { OPTION_MASK_ISA_SSE2, CODE_FOR_smaxv2df3, "__builtin_ia32_maxpd", IX86_BUILTIN_MAXPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
26687 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmsminv2df3, "__builtin_ia32_minsd", IX86_BUILTIN_MINSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
26688 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmsmaxv2df3, "__builtin_ia32_maxsd", IX86_BUILTIN_MAXSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
26690 { OPTION_MASK_ISA_SSE2, CODE_FOR_andv2df3, "__builtin_ia32_andpd", IX86_BUILTIN_ANDPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
26691 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_andnotv2df3, "__builtin_ia32_andnpd", IX86_BUILTIN_ANDNPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
26692 { OPTION_MASK_ISA_SSE2, CODE_FOR_iorv2df3, "__builtin_ia32_orpd", IX86_BUILTIN_ORPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
26693 { OPTION_MASK_ISA_SSE2, CODE_FOR_xorv2df3, "__builtin_ia32_xorpd", IX86_BUILTIN_XORPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
26695 { OPTION_MASK_ISA_SSE2, CODE_FOR_copysignv2df3, "__builtin_ia32_copysignpd", IX86_BUILTIN_CPYSGNPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
26697 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_movsd, "__builtin_ia32_movsd", IX86_BUILTIN_MOVSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
26698 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_highv2df, "__builtin_ia32_unpckhpd", IX86_BUILTIN_UNPCKHPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
26699 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_lowv2df, "__builtin_ia32_unpcklpd", IX86_BUILTIN_UNPCKLPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
26701 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_pack_sfix_v2df, "__builtin_ia32_vec_pack_sfix", IX86_BUILTIN_VEC_PACK_SFIX, UNKNOWN, (int) V4SI_FTYPE_V2DF_V2DF },
26703 { OPTION_MASK_ISA_SSE2, CODE_FOR_addv16qi3, "__builtin_ia32_paddb128", IX86_BUILTIN_PADDB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
26704 { OPTION_MASK_ISA_SSE2, CODE_FOR_addv8hi3, "__builtin_ia32_paddw128", IX86_BUILTIN_PADDW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
26705 { OPTION_MASK_ISA_SSE2, CODE_FOR_addv4si3, "__builtin_ia32_paddd128", IX86_BUILTIN_PADDD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
26706 { OPTION_MASK_ISA_SSE2, CODE_FOR_addv2di3, "__builtin_ia32_paddq128", IX86_BUILTIN_PADDQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
26707 { OPTION_MASK_ISA_SSE2, CODE_FOR_subv16qi3, "__builtin_ia32_psubb128", IX86_BUILTIN_PSUBB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
26708 { OPTION_MASK_ISA_SSE2, CODE_FOR_subv8hi3, "__builtin_ia32_psubw128", IX86_BUILTIN_PSUBW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
26709 { OPTION_MASK_ISA_SSE2, CODE_FOR_subv4si3, "__builtin_ia32_psubd128", IX86_BUILTIN_PSUBD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
26710 { OPTION_MASK_ISA_SSE2, CODE_FOR_subv2di3, "__builtin_ia32_psubq128", IX86_BUILTIN_PSUBQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
26712 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ssaddv16qi3, "__builtin_ia32_paddsb128", IX86_BUILTIN_PADDSB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
26713 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ssaddv8hi3, "__builtin_ia32_paddsw128", IX86_BUILTIN_PADDSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
26714 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_sssubv16qi3, "__builtin_ia32_psubsb128", IX86_BUILTIN_PSUBSB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
26715 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_sssubv8hi3, "__builtin_ia32_psubsw128", IX86_BUILTIN_PSUBSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
26716 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_usaddv16qi3, "__builtin_ia32_paddusb128", IX86_BUILTIN_PADDUSB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
26717 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_usaddv8hi3, "__builtin_ia32_paddusw128", IX86_BUILTIN_PADDUSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
26718 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ussubv16qi3, "__builtin_ia32_psubusb128", IX86_BUILTIN_PSUBUSB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
26719 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ussubv8hi3, "__builtin_ia32_psubusw128", IX86_BUILTIN_PSUBUSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
26721 { OPTION_MASK_ISA_SSE2, CODE_FOR_mulv8hi3, "__builtin_ia32_pmullw128", IX86_BUILTIN_PMULLW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
26722 { OPTION_MASK_ISA_SSE2, CODE_FOR_smulv8hi3_highpart, "__builtin_ia32_pmulhw128", IX86_BUILTIN_PMULHW128, UNKNOWN,(int) V8HI_FTYPE_V8HI_V8HI },
26724 { OPTION_MASK_ISA_SSE2, CODE_FOR_andv2di3, "__builtin_ia32_pand128", IX86_BUILTIN_PAND128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
26725 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_andnotv2di3, "__builtin_ia32_pandn128", IX86_BUILTIN_PANDN128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
26726 { OPTION_MASK_ISA_SSE2, CODE_FOR_iorv2di3, "__builtin_ia32_por128", IX86_BUILTIN_POR128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
26727 { OPTION_MASK_ISA_SSE2, CODE_FOR_xorv2di3, "__builtin_ia32_pxor128", IX86_BUILTIN_PXOR128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
26729 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_uavgv16qi3, "__builtin_ia32_pavgb128", IX86_BUILTIN_PAVGB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
26730 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_uavgv8hi3, "__builtin_ia32_pavgw128", IX86_BUILTIN_PAVGW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
26732 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_eqv16qi3, "__builtin_ia32_pcmpeqb128", IX86_BUILTIN_PCMPEQB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
26733 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_eqv8hi3, "__builtin_ia32_pcmpeqw128", IX86_BUILTIN_PCMPEQW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
26734 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_eqv4si3, "__builtin_ia32_pcmpeqd128", IX86_BUILTIN_PCMPEQD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
26735 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_gtv16qi3, "__builtin_ia32_pcmpgtb128", IX86_BUILTIN_PCMPGTB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
26736 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_gtv8hi3, "__builtin_ia32_pcmpgtw128", IX86_BUILTIN_PCMPGTW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
26737 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_gtv4si3, "__builtin_ia32_pcmpgtd128", IX86_BUILTIN_PCMPGTD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
26739 { OPTION_MASK_ISA_SSE2, CODE_FOR_umaxv16qi3, "__builtin_ia32_pmaxub128", IX86_BUILTIN_PMAXUB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
26740 { OPTION_MASK_ISA_SSE2, CODE_FOR_smaxv8hi3, "__builtin_ia32_pmaxsw128", IX86_BUILTIN_PMAXSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
26741 { OPTION_MASK_ISA_SSE2, CODE_FOR_uminv16qi3, "__builtin_ia32_pminub128", IX86_BUILTIN_PMINUB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
26742 { OPTION_MASK_ISA_SSE2, CODE_FOR_sminv8hi3, "__builtin_ia32_pminsw128", IX86_BUILTIN_PMINSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
26744 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_highv16qi, "__builtin_ia32_punpckhbw128", IX86_BUILTIN_PUNPCKHBW128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
26745 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_highv8hi, "__builtin_ia32_punpckhwd128", IX86_BUILTIN_PUNPCKHWD128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
26746 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_highv4si, "__builtin_ia32_punpckhdq128", IX86_BUILTIN_PUNPCKHDQ128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
26747 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_highv2di, "__builtin_ia32_punpckhqdq128", IX86_BUILTIN_PUNPCKHQDQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
26748 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_lowv16qi, "__builtin_ia32_punpcklbw128", IX86_BUILTIN_PUNPCKLBW128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
26749 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_lowv8hi, "__builtin_ia32_punpcklwd128", IX86_BUILTIN_PUNPCKLWD128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
26750 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_lowv4si, "__builtin_ia32_punpckldq128", IX86_BUILTIN_PUNPCKLDQ128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
26751 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_lowv2di, "__builtin_ia32_punpcklqdq128", IX86_BUILTIN_PUNPCKLQDQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
26753 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_packsswb, "__builtin_ia32_packsswb128", IX86_BUILTIN_PACKSSWB128, UNKNOWN, (int) V16QI_FTYPE_V8HI_V8HI },
26754 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_packssdw, "__builtin_ia32_packssdw128", IX86_BUILTIN_PACKSSDW128, UNKNOWN, (int) V8HI_FTYPE_V4SI_V4SI },
26755 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_packuswb, "__builtin_ia32_packuswb128", IX86_BUILTIN_PACKUSWB128, UNKNOWN, (int) V16QI_FTYPE_V8HI_V8HI },
26757 { OPTION_MASK_ISA_SSE2, CODE_FOR_umulv8hi3_highpart, "__builtin_ia32_pmulhuw128", IX86_BUILTIN_PMULHUW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
26758 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_psadbw, "__builtin_ia32_psadbw128", IX86_BUILTIN_PSADBW128, UNKNOWN, (int) V2DI_FTYPE_V16QI_V16QI },
26760 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_umulv1siv1di3, "__builtin_ia32_pmuludq", IX86_BUILTIN_PMULUDQ, UNKNOWN, (int) V1DI_FTYPE_V2SI_V2SI },
26761 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_widen_umult_even_v4si, "__builtin_ia32_pmuludq128", IX86_BUILTIN_PMULUDQ128, UNKNOWN, (int) V2DI_FTYPE_V4SI_V4SI },
26763 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_pmaddwd, "__builtin_ia32_pmaddwd128", IX86_BUILTIN_PMADDWD128, UNKNOWN, (int) V4SI_FTYPE_V8HI_V8HI },
26765 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtsi2sd, "__builtin_ia32_cvtsi2sd", IX86_BUILTIN_CVTSI2SD, UNKNOWN, (int) V2DF_FTYPE_V2DF_SI },
26766 { OPTION_MASK_ISA_SSE2 | OPTION_MASK_ISA_64BIT, CODE_FOR_sse2_cvtsi2sdq, "__builtin_ia32_cvtsi642sd", IX86_BUILTIN_CVTSI642SD, UNKNOWN, (int) V2DF_FTYPE_V2DF_DI },
26767 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtsd2ss, "__builtin_ia32_cvtsd2ss", IX86_BUILTIN_CVTSD2SS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V2DF },
26768 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtss2sd, "__builtin_ia32_cvtss2sd", IX86_BUILTIN_CVTSS2SD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V4SF },
26770 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ashlv1ti3, "__builtin_ia32_pslldqi128", IX86_BUILTIN_PSLLDQI128, UNKNOWN, (int) V2DI_FTYPE_V2DI_INT_CONVERT },
26771 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashlv8hi3, "__builtin_ia32_psllwi128", IX86_BUILTIN_PSLLWI128, UNKNOWN, (int) V8HI_FTYPE_V8HI_SI_COUNT },
26772 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashlv4si3, "__builtin_ia32_pslldi128", IX86_BUILTIN_PSLLDI128, UNKNOWN, (int) V4SI_FTYPE_V4SI_SI_COUNT },
26773 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashlv2di3, "__builtin_ia32_psllqi128", IX86_BUILTIN_PSLLQI128, UNKNOWN, (int) V2DI_FTYPE_V2DI_SI_COUNT },
26774 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashlv8hi3, "__builtin_ia32_psllw128", IX86_BUILTIN_PSLLW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_COUNT },
26775 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashlv4si3, "__builtin_ia32_pslld128", IX86_BUILTIN_PSLLD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_COUNT },
26776 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashlv2di3, "__builtin_ia32_psllq128", IX86_BUILTIN_PSLLQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_COUNT },
26778 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_lshrv1ti3, "__builtin_ia32_psrldqi128", IX86_BUILTIN_PSRLDQI128, UNKNOWN, (int) V2DI_FTYPE_V2DI_INT_CONVERT },
26779 { OPTION_MASK_ISA_SSE2, CODE_FOR_lshrv8hi3, "__builtin_ia32_psrlwi128", IX86_BUILTIN_PSRLWI128, UNKNOWN, (int) V8HI_FTYPE_V8HI_SI_COUNT },
26780 { OPTION_MASK_ISA_SSE2, CODE_FOR_lshrv4si3, "__builtin_ia32_psrldi128", IX86_BUILTIN_PSRLDI128, UNKNOWN, (int) V4SI_FTYPE_V4SI_SI_COUNT },
26781 { OPTION_MASK_ISA_SSE2, CODE_FOR_lshrv2di3, "__builtin_ia32_psrlqi128", IX86_BUILTIN_PSRLQI128, UNKNOWN, (int) V2DI_FTYPE_V2DI_SI_COUNT },
26782 { OPTION_MASK_ISA_SSE2, CODE_FOR_lshrv8hi3, "__builtin_ia32_psrlw128", IX86_BUILTIN_PSRLW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_COUNT },
26783 { OPTION_MASK_ISA_SSE2, CODE_FOR_lshrv4si3, "__builtin_ia32_psrld128", IX86_BUILTIN_PSRLD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_COUNT },
26784 { OPTION_MASK_ISA_SSE2, CODE_FOR_lshrv2di3, "__builtin_ia32_psrlq128", IX86_BUILTIN_PSRLQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_COUNT },
26786 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashrv8hi3, "__builtin_ia32_psrawi128", IX86_BUILTIN_PSRAWI128, UNKNOWN, (int) V8HI_FTYPE_V8HI_SI_COUNT },
26787 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashrv4si3, "__builtin_ia32_psradi128", IX86_BUILTIN_PSRADI128, UNKNOWN, (int) V4SI_FTYPE_V4SI_SI_COUNT },
26788 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashrv8hi3, "__builtin_ia32_psraw128", IX86_BUILTIN_PSRAW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_COUNT },
26789 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashrv4si3, "__builtin_ia32_psrad128", IX86_BUILTIN_PSRAD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_COUNT },
26791 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_pshufd, "__builtin_ia32_pshufd", IX86_BUILTIN_PSHUFD, UNKNOWN, (int) V4SI_FTYPE_V4SI_INT },
26792 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_pshuflw, "__builtin_ia32_pshuflw", IX86_BUILTIN_PSHUFLW, UNKNOWN, (int) V8HI_FTYPE_V8HI_INT },
26793 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_pshufhw, "__builtin_ia32_pshufhw", IX86_BUILTIN_PSHUFHW, UNKNOWN, (int) V8HI_FTYPE_V8HI_INT },
26795 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmsqrtv2df2, "__builtin_ia32_sqrtsd", IX86_BUILTIN_SQRTSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_VEC_MERGE },
26797 { OPTION_MASK_ISA_SSE, CODE_FOR_sse2_movq128, "__builtin_ia32_movq128", IX86_BUILTIN_MOVQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI },
26799 /* SSE2 MMX */
26800 { OPTION_MASK_ISA_SSE2, CODE_FOR_mmx_addv1di3, "__builtin_ia32_paddq", IX86_BUILTIN_PADDQ, UNKNOWN, (int) V1DI_FTYPE_V1DI_V1DI },
26801 { OPTION_MASK_ISA_SSE2, CODE_FOR_mmx_subv1di3, "__builtin_ia32_psubq", IX86_BUILTIN_PSUBQ, UNKNOWN, (int) V1DI_FTYPE_V1DI_V1DI },
26803 /* SSE3 */
26804 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_movshdup, "__builtin_ia32_movshdup", IX86_BUILTIN_MOVSHDUP, UNKNOWN, (int) V4SF_FTYPE_V4SF},
26805 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_movsldup, "__builtin_ia32_movsldup", IX86_BUILTIN_MOVSLDUP, UNKNOWN, (int) V4SF_FTYPE_V4SF },
26807 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_addsubv4sf3, "__builtin_ia32_addsubps", IX86_BUILTIN_ADDSUBPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
26808 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_addsubv2df3, "__builtin_ia32_addsubpd", IX86_BUILTIN_ADDSUBPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
26809 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_haddv4sf3, "__builtin_ia32_haddps", IX86_BUILTIN_HADDPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
26810 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_haddv2df3, "__builtin_ia32_haddpd", IX86_BUILTIN_HADDPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
26811 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_hsubv4sf3, "__builtin_ia32_hsubps", IX86_BUILTIN_HSUBPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
26812 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_hsubv2df3, "__builtin_ia32_hsubpd", IX86_BUILTIN_HSUBPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
26814 /* SSSE3 */
26815 { OPTION_MASK_ISA_SSSE3, CODE_FOR_absv16qi2, "__builtin_ia32_pabsb128", IX86_BUILTIN_PABSB128, UNKNOWN, (int) V16QI_FTYPE_V16QI },
26816 { OPTION_MASK_ISA_SSSE3, CODE_FOR_absv8qi2, "__builtin_ia32_pabsb", IX86_BUILTIN_PABSB, UNKNOWN, (int) V8QI_FTYPE_V8QI },
26817 { OPTION_MASK_ISA_SSSE3, CODE_FOR_absv8hi2, "__builtin_ia32_pabsw128", IX86_BUILTIN_PABSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI },
26818 { OPTION_MASK_ISA_SSSE3, CODE_FOR_absv4hi2, "__builtin_ia32_pabsw", IX86_BUILTIN_PABSW, UNKNOWN, (int) V4HI_FTYPE_V4HI },
26819 { OPTION_MASK_ISA_SSSE3, CODE_FOR_absv4si2, "__builtin_ia32_pabsd128", IX86_BUILTIN_PABSD128, UNKNOWN, (int) V4SI_FTYPE_V4SI },
26820 { OPTION_MASK_ISA_SSSE3, CODE_FOR_absv2si2, "__builtin_ia32_pabsd", IX86_BUILTIN_PABSD, UNKNOWN, (int) V2SI_FTYPE_V2SI },
26822 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phaddwv8hi3, "__builtin_ia32_phaddw128", IX86_BUILTIN_PHADDW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
26823 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phaddwv4hi3, "__builtin_ia32_phaddw", IX86_BUILTIN_PHADDW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
26824 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phadddv4si3, "__builtin_ia32_phaddd128", IX86_BUILTIN_PHADDD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
26825 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phadddv2si3, "__builtin_ia32_phaddd", IX86_BUILTIN_PHADDD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
26826 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phaddswv8hi3, "__builtin_ia32_phaddsw128", IX86_BUILTIN_PHADDSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
26827 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phaddswv4hi3, "__builtin_ia32_phaddsw", IX86_BUILTIN_PHADDSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
26828 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phsubwv8hi3, "__builtin_ia32_phsubw128", IX86_BUILTIN_PHSUBW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
26829 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phsubwv4hi3, "__builtin_ia32_phsubw", IX86_BUILTIN_PHSUBW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
26830 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phsubdv4si3, "__builtin_ia32_phsubd128", IX86_BUILTIN_PHSUBD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
26831 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phsubdv2si3, "__builtin_ia32_phsubd", IX86_BUILTIN_PHSUBD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
26832 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phsubswv8hi3, "__builtin_ia32_phsubsw128", IX86_BUILTIN_PHSUBSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
26833 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phsubswv4hi3, "__builtin_ia32_phsubsw", IX86_BUILTIN_PHSUBSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
26834 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_pmaddubsw128, "__builtin_ia32_pmaddubsw128", IX86_BUILTIN_PMADDUBSW128, UNKNOWN, (int) V8HI_FTYPE_V16QI_V16QI },
26835 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_pmaddubsw, "__builtin_ia32_pmaddubsw", IX86_BUILTIN_PMADDUBSW, UNKNOWN, (int) V4HI_FTYPE_V8QI_V8QI },
26836 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_pmulhrswv8hi3, "__builtin_ia32_pmulhrsw128", IX86_BUILTIN_PMULHRSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
26837 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_pmulhrswv4hi3, "__builtin_ia32_pmulhrsw", IX86_BUILTIN_PMULHRSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
26838 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_pshufbv16qi3, "__builtin_ia32_pshufb128", IX86_BUILTIN_PSHUFB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
26839 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_pshufbv8qi3, "__builtin_ia32_pshufb", IX86_BUILTIN_PSHUFB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
26840 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_psignv16qi3, "__builtin_ia32_psignb128", IX86_BUILTIN_PSIGNB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
26841 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_psignv8qi3, "__builtin_ia32_psignb", IX86_BUILTIN_PSIGNB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
26842 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_psignv8hi3, "__builtin_ia32_psignw128", IX86_BUILTIN_PSIGNW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
26843 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_psignv4hi3, "__builtin_ia32_psignw", IX86_BUILTIN_PSIGNW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
26844 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_psignv4si3, "__builtin_ia32_psignd128", IX86_BUILTIN_PSIGND128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
26845 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_psignv2si3, "__builtin_ia32_psignd", IX86_BUILTIN_PSIGND, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
26847 /* SSSE3. */
26848 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_palignrti, "__builtin_ia32_palignr128", IX86_BUILTIN_PALIGNR128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_INT_CONVERT },
26849 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_palignrdi, "__builtin_ia32_palignr", IX86_BUILTIN_PALIGNR, UNKNOWN, (int) V1DI_FTYPE_V1DI_V1DI_INT_CONVERT },
26851 /* SSE4.1 */
26852 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_blendpd, "__builtin_ia32_blendpd", IX86_BUILTIN_BLENDPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
26853 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_blendps, "__builtin_ia32_blendps", IX86_BUILTIN_BLENDPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
26854 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_blendvpd, "__builtin_ia32_blendvpd", IX86_BUILTIN_BLENDVPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_V2DF },
26855 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_blendvps, "__builtin_ia32_blendvps", IX86_BUILTIN_BLENDVPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_V4SF },
26856 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_dppd, "__builtin_ia32_dppd", IX86_BUILTIN_DPPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
26857 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_dpps, "__builtin_ia32_dpps", IX86_BUILTIN_DPPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
26858 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_insertps, "__builtin_ia32_insertps128", IX86_BUILTIN_INSERTPS128, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
26859 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_mpsadbw, "__builtin_ia32_mpsadbw128", IX86_BUILTIN_MPSADBW128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI_INT },
26860 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_pblendvb, "__builtin_ia32_pblendvb128", IX86_BUILTIN_PBLENDVB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI_V16QI },
26861 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_pblendw, "__builtin_ia32_pblendw128", IX86_BUILTIN_PBLENDW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_INT },
26863 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_sign_extendv8qiv8hi2, "__builtin_ia32_pmovsxbw128", IX86_BUILTIN_PMOVSXBW128, UNKNOWN, (int) V8HI_FTYPE_V16QI },
26864 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_sign_extendv4qiv4si2, "__builtin_ia32_pmovsxbd128", IX86_BUILTIN_PMOVSXBD128, UNKNOWN, (int) V4SI_FTYPE_V16QI },
26865 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_sign_extendv2qiv2di2, "__builtin_ia32_pmovsxbq128", IX86_BUILTIN_PMOVSXBQ128, UNKNOWN, (int) V2DI_FTYPE_V16QI },
26866 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_sign_extendv4hiv4si2, "__builtin_ia32_pmovsxwd128", IX86_BUILTIN_PMOVSXWD128, UNKNOWN, (int) V4SI_FTYPE_V8HI },
26867 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_sign_extendv2hiv2di2, "__builtin_ia32_pmovsxwq128", IX86_BUILTIN_PMOVSXWQ128, UNKNOWN, (int) V2DI_FTYPE_V8HI },
26868 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_sign_extendv2siv2di2, "__builtin_ia32_pmovsxdq128", IX86_BUILTIN_PMOVSXDQ128, UNKNOWN, (int) V2DI_FTYPE_V4SI },
26869 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_zero_extendv8qiv8hi2, "__builtin_ia32_pmovzxbw128", IX86_BUILTIN_PMOVZXBW128, UNKNOWN, (int) V8HI_FTYPE_V16QI },
26870 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_zero_extendv4qiv4si2, "__builtin_ia32_pmovzxbd128", IX86_BUILTIN_PMOVZXBD128, UNKNOWN, (int) V4SI_FTYPE_V16QI },
26871 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_zero_extendv2qiv2di2, "__builtin_ia32_pmovzxbq128", IX86_BUILTIN_PMOVZXBQ128, UNKNOWN, (int) V2DI_FTYPE_V16QI },
26872 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_zero_extendv4hiv4si2, "__builtin_ia32_pmovzxwd128", IX86_BUILTIN_PMOVZXWD128, UNKNOWN, (int) V4SI_FTYPE_V8HI },
26873 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_zero_extendv2hiv2di2, "__builtin_ia32_pmovzxwq128", IX86_BUILTIN_PMOVZXWQ128, UNKNOWN, (int) V2DI_FTYPE_V8HI },
26874 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_zero_extendv2siv2di2, "__builtin_ia32_pmovzxdq128", IX86_BUILTIN_PMOVZXDQ128, UNKNOWN, (int) V2DI_FTYPE_V4SI },
26875 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_phminposuw, "__builtin_ia32_phminposuw128", IX86_BUILTIN_PHMINPOSUW128, UNKNOWN, (int) V8HI_FTYPE_V8HI },
26877 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_packusdw, "__builtin_ia32_packusdw128", IX86_BUILTIN_PACKUSDW128, UNKNOWN, (int) V8HI_FTYPE_V4SI_V4SI },
26878 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_eqv2di3, "__builtin_ia32_pcmpeqq", IX86_BUILTIN_PCMPEQQ, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
26879 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_smaxv16qi3, "__builtin_ia32_pmaxsb128", IX86_BUILTIN_PMAXSB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
26880 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_smaxv4si3, "__builtin_ia32_pmaxsd128", IX86_BUILTIN_PMAXSD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
26881 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_umaxv4si3, "__builtin_ia32_pmaxud128", IX86_BUILTIN_PMAXUD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
26882 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_umaxv8hi3, "__builtin_ia32_pmaxuw128", IX86_BUILTIN_PMAXUW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
26883 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sminv16qi3, "__builtin_ia32_pminsb128", IX86_BUILTIN_PMINSB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
26884 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sminv4si3, "__builtin_ia32_pminsd128", IX86_BUILTIN_PMINSD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
26885 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_uminv4si3, "__builtin_ia32_pminud128", IX86_BUILTIN_PMINUD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
26886 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_uminv8hi3, "__builtin_ia32_pminuw128", IX86_BUILTIN_PMINUW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
26887 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_mulv2siv2di3, "__builtin_ia32_pmuldq128", IX86_BUILTIN_PMULDQ128, UNKNOWN, (int) V2DI_FTYPE_V4SI_V4SI },
26888 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_mulv4si3, "__builtin_ia32_pmulld128", IX86_BUILTIN_PMULLD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
26890 /* SSE4.1 */
26891 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundpd, "__builtin_ia32_roundpd", IX86_BUILTIN_ROUNDPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_INT },
26892 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundps, "__builtin_ia32_roundps", IX86_BUILTIN_ROUNDPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_INT },
26893 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundsd, "__builtin_ia32_roundsd", IX86_BUILTIN_ROUNDSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
26894 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundss, "__builtin_ia32_roundss", IX86_BUILTIN_ROUNDSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
26896 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundpd, "__builtin_ia32_floorpd", IX86_BUILTIN_FLOORPD, (enum rtx_code) ROUND_FLOOR, (int) V2DF_FTYPE_V2DF_ROUND },
26897 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundpd, "__builtin_ia32_ceilpd", IX86_BUILTIN_CEILPD, (enum rtx_code) ROUND_CEIL, (int) V2DF_FTYPE_V2DF_ROUND },
26898 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundpd, "__builtin_ia32_truncpd", IX86_BUILTIN_TRUNCPD, (enum rtx_code) ROUND_TRUNC, (int) V2DF_FTYPE_V2DF_ROUND },
26899 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundpd, "__builtin_ia32_rintpd", IX86_BUILTIN_RINTPD, (enum rtx_code) ROUND_MXCSR, (int) V2DF_FTYPE_V2DF_ROUND },
26901 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundpd_vec_pack_sfix, "__builtin_ia32_floorpd_vec_pack_sfix", IX86_BUILTIN_FLOORPD_VEC_PACK_SFIX, (enum rtx_code) ROUND_FLOOR, (int) V4SI_FTYPE_V2DF_V2DF_ROUND },
26902 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundpd_vec_pack_sfix, "__builtin_ia32_ceilpd_vec_pack_sfix", IX86_BUILTIN_CEILPD_VEC_PACK_SFIX, (enum rtx_code) ROUND_CEIL, (int) V4SI_FTYPE_V2DF_V2DF_ROUND },
26904 { OPTION_MASK_ISA_ROUND, CODE_FOR_roundv2df2, "__builtin_ia32_roundpd_az", IX86_BUILTIN_ROUNDPD_AZ, UNKNOWN, (int) V2DF_FTYPE_V2DF },
26905 { OPTION_MASK_ISA_ROUND, CODE_FOR_roundv2df2_vec_pack_sfix, "__builtin_ia32_roundpd_az_vec_pack_sfix", IX86_BUILTIN_ROUNDPD_AZ_VEC_PACK_SFIX, UNKNOWN, (int) V4SI_FTYPE_V2DF_V2DF },
26907 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundps, "__builtin_ia32_floorps", IX86_BUILTIN_FLOORPS, (enum rtx_code) ROUND_FLOOR, (int) V4SF_FTYPE_V4SF_ROUND },
26908 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundps, "__builtin_ia32_ceilps", IX86_BUILTIN_CEILPS, (enum rtx_code) ROUND_CEIL, (int) V4SF_FTYPE_V4SF_ROUND },
26909 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundps, "__builtin_ia32_truncps", IX86_BUILTIN_TRUNCPS, (enum rtx_code) ROUND_TRUNC, (int) V4SF_FTYPE_V4SF_ROUND },
26910 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundps, "__builtin_ia32_rintps", IX86_BUILTIN_RINTPS, (enum rtx_code) ROUND_MXCSR, (int) V4SF_FTYPE_V4SF_ROUND },
26912 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundps_sfix, "__builtin_ia32_floorps_sfix", IX86_BUILTIN_FLOORPS_SFIX, (enum rtx_code) ROUND_FLOOR, (int) V4SI_FTYPE_V4SF_ROUND },
26913 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundps_sfix, "__builtin_ia32_ceilps_sfix", IX86_BUILTIN_CEILPS_SFIX, (enum rtx_code) ROUND_CEIL, (int) V4SI_FTYPE_V4SF_ROUND },
26915 { OPTION_MASK_ISA_ROUND, CODE_FOR_roundv4sf2, "__builtin_ia32_roundps_az", IX86_BUILTIN_ROUNDPS_AZ, UNKNOWN, (int) V4SF_FTYPE_V4SF },
26916 { OPTION_MASK_ISA_ROUND, CODE_FOR_roundv4sf2_sfix, "__builtin_ia32_roundps_az_sfix", IX86_BUILTIN_ROUNDPS_AZ_SFIX, UNKNOWN, (int) V4SI_FTYPE_V4SF },
26918 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_ptest, "__builtin_ia32_ptestz128", IX86_BUILTIN_PTESTZ, EQ, (int) INT_FTYPE_V2DI_V2DI_PTEST },
26919 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_ptest, "__builtin_ia32_ptestc128", IX86_BUILTIN_PTESTC, LTU, (int) INT_FTYPE_V2DI_V2DI_PTEST },
26920 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_ptest, "__builtin_ia32_ptestnzc128", IX86_BUILTIN_PTESTNZC, GTU, (int) INT_FTYPE_V2DI_V2DI_PTEST },
26922 /* SSE4.2 */
26923 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_gtv2di3, "__builtin_ia32_pcmpgtq", IX86_BUILTIN_PCMPGTQ, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
26924 { OPTION_MASK_ISA_SSE4_2 | OPTION_MASK_ISA_CRC32, CODE_FOR_sse4_2_crc32qi, "__builtin_ia32_crc32qi", IX86_BUILTIN_CRC32QI, UNKNOWN, (int) UINT_FTYPE_UINT_UCHAR },
26925 { OPTION_MASK_ISA_SSE4_2 | OPTION_MASK_ISA_CRC32, CODE_FOR_sse4_2_crc32hi, "__builtin_ia32_crc32hi", IX86_BUILTIN_CRC32HI, UNKNOWN, (int) UINT_FTYPE_UINT_USHORT },
26926 { OPTION_MASK_ISA_SSE4_2 | OPTION_MASK_ISA_CRC32, CODE_FOR_sse4_2_crc32si, "__builtin_ia32_crc32si", IX86_BUILTIN_CRC32SI, UNKNOWN, (int) UINT_FTYPE_UINT_UINT },
26927 { OPTION_MASK_ISA_SSE4_2 | OPTION_MASK_ISA_CRC32 | OPTION_MASK_ISA_64BIT, CODE_FOR_sse4_2_crc32di, "__builtin_ia32_crc32di", IX86_BUILTIN_CRC32DI, UNKNOWN, (int) UINT64_FTYPE_UINT64_UINT64 },
26929 /* SSE4A */
26930 { OPTION_MASK_ISA_SSE4A, CODE_FOR_sse4a_extrqi, "__builtin_ia32_extrqi", IX86_BUILTIN_EXTRQI, UNKNOWN, (int) V2DI_FTYPE_V2DI_UINT_UINT },
26931 { OPTION_MASK_ISA_SSE4A, CODE_FOR_sse4a_extrq, "__builtin_ia32_extrq", IX86_BUILTIN_EXTRQ, UNKNOWN, (int) V2DI_FTYPE_V2DI_V16QI },
26932 { OPTION_MASK_ISA_SSE4A, CODE_FOR_sse4a_insertqi, "__builtin_ia32_insertqi", IX86_BUILTIN_INSERTQI, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_UINT_UINT },
26933 { OPTION_MASK_ISA_SSE4A, CODE_FOR_sse4a_insertq, "__builtin_ia32_insertq", IX86_BUILTIN_INSERTQ, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
26935 /* AES */
26936 { OPTION_MASK_ISA_SSE2, CODE_FOR_aeskeygenassist, 0, IX86_BUILTIN_AESKEYGENASSIST128, UNKNOWN, (int) V2DI_FTYPE_V2DI_INT },
26937 { OPTION_MASK_ISA_SSE2, CODE_FOR_aesimc, 0, IX86_BUILTIN_AESIMC128, UNKNOWN, (int) V2DI_FTYPE_V2DI },
26939 { OPTION_MASK_ISA_SSE2, CODE_FOR_aesenc, 0, IX86_BUILTIN_AESENC128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
26940 { OPTION_MASK_ISA_SSE2, CODE_FOR_aesenclast, 0, IX86_BUILTIN_AESENCLAST128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
26941 { OPTION_MASK_ISA_SSE2, CODE_FOR_aesdec, 0, IX86_BUILTIN_AESDEC128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
26942 { OPTION_MASK_ISA_SSE2, CODE_FOR_aesdeclast, 0, IX86_BUILTIN_AESDECLAST128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
26944 /* PCLMUL */
26945 { OPTION_MASK_ISA_SSE2, CODE_FOR_pclmulqdq, 0, IX86_BUILTIN_PCLMULQDQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_INT },
26947 /* AVX */
26948 { OPTION_MASK_ISA_AVX, CODE_FOR_addv4df3, "__builtin_ia32_addpd256", IX86_BUILTIN_ADDPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
26949 { OPTION_MASK_ISA_AVX, CODE_FOR_addv8sf3, "__builtin_ia32_addps256", IX86_BUILTIN_ADDPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
26950 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_addsubv4df3, "__builtin_ia32_addsubpd256", IX86_BUILTIN_ADDSUBPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
26951 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_addsubv8sf3, "__builtin_ia32_addsubps256", IX86_BUILTIN_ADDSUBPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
26952 { OPTION_MASK_ISA_AVX, CODE_FOR_andv4df3, "__builtin_ia32_andpd256", IX86_BUILTIN_ANDPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
26953 { OPTION_MASK_ISA_AVX, CODE_FOR_andv8sf3, "__builtin_ia32_andps256", IX86_BUILTIN_ANDPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
26954 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_andnotv4df3, "__builtin_ia32_andnpd256", IX86_BUILTIN_ANDNPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
26955 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_andnotv8sf3, "__builtin_ia32_andnps256", IX86_BUILTIN_ANDNPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
26956 { OPTION_MASK_ISA_AVX, CODE_FOR_divv4df3, "__builtin_ia32_divpd256", IX86_BUILTIN_DIVPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
26957 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_divv8sf3, "__builtin_ia32_divps256", IX86_BUILTIN_DIVPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
26958 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_haddv4df3, "__builtin_ia32_haddpd256", IX86_BUILTIN_HADDPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
26959 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_hsubv8sf3, "__builtin_ia32_hsubps256", IX86_BUILTIN_HSUBPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
26960 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_hsubv4df3, "__builtin_ia32_hsubpd256", IX86_BUILTIN_HSUBPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
26961 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_haddv8sf3, "__builtin_ia32_haddps256", IX86_BUILTIN_HADDPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
26962 { OPTION_MASK_ISA_AVX, CODE_FOR_smaxv4df3, "__builtin_ia32_maxpd256", IX86_BUILTIN_MAXPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
26963 { OPTION_MASK_ISA_AVX, CODE_FOR_smaxv8sf3, "__builtin_ia32_maxps256", IX86_BUILTIN_MAXPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
26964 { OPTION_MASK_ISA_AVX, CODE_FOR_sminv4df3, "__builtin_ia32_minpd256", IX86_BUILTIN_MINPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
26965 { OPTION_MASK_ISA_AVX, CODE_FOR_sminv8sf3, "__builtin_ia32_minps256", IX86_BUILTIN_MINPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
26966 { OPTION_MASK_ISA_AVX, CODE_FOR_mulv4df3, "__builtin_ia32_mulpd256", IX86_BUILTIN_MULPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
26967 { OPTION_MASK_ISA_AVX, CODE_FOR_mulv8sf3, "__builtin_ia32_mulps256", IX86_BUILTIN_MULPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
26968 { OPTION_MASK_ISA_AVX, CODE_FOR_iorv4df3, "__builtin_ia32_orpd256", IX86_BUILTIN_ORPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
26969 { OPTION_MASK_ISA_AVX, CODE_FOR_iorv8sf3, "__builtin_ia32_orps256", IX86_BUILTIN_ORPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
26970 { OPTION_MASK_ISA_AVX, CODE_FOR_subv4df3, "__builtin_ia32_subpd256", IX86_BUILTIN_SUBPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
26971 { OPTION_MASK_ISA_AVX, CODE_FOR_subv8sf3, "__builtin_ia32_subps256", IX86_BUILTIN_SUBPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
26972 { OPTION_MASK_ISA_AVX, CODE_FOR_xorv4df3, "__builtin_ia32_xorpd256", IX86_BUILTIN_XORPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
26973 { OPTION_MASK_ISA_AVX, CODE_FOR_xorv8sf3, "__builtin_ia32_xorps256", IX86_BUILTIN_XORPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
26975 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilvarv2df3, "__builtin_ia32_vpermilvarpd", IX86_BUILTIN_VPERMILVARPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DI },
26976 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilvarv4sf3, "__builtin_ia32_vpermilvarps", IX86_BUILTIN_VPERMILVARPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SI },
26977 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilvarv4df3, "__builtin_ia32_vpermilvarpd256", IX86_BUILTIN_VPERMILVARPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DI },
26978 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilvarv8sf3, "__builtin_ia32_vpermilvarps256", IX86_BUILTIN_VPERMILVARPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SI },
26980 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_blendpd256, "__builtin_ia32_blendpd256", IX86_BUILTIN_BLENDPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_INT },
26981 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_blendps256, "__builtin_ia32_blendps256", IX86_BUILTIN_BLENDPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_INT },
26982 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_blendvpd256, "__builtin_ia32_blendvpd256", IX86_BUILTIN_BLENDVPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_V4DF },
26983 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_blendvps256, "__builtin_ia32_blendvps256", IX86_BUILTIN_BLENDVPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_V8SF },
26984 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_dpps256, "__builtin_ia32_dpps256", IX86_BUILTIN_DPPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_INT },
26985 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_shufpd256, "__builtin_ia32_shufpd256", IX86_BUILTIN_SHUFPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_INT },
26986 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_shufps256, "__builtin_ia32_shufps256", IX86_BUILTIN_SHUFPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_INT },
26987 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vmcmpv2df3, "__builtin_ia32_cmpsd", IX86_BUILTIN_CMPSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
26988 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vmcmpv4sf3, "__builtin_ia32_cmpss", IX86_BUILTIN_CMPSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
26989 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cmpv2df3, "__builtin_ia32_cmppd", IX86_BUILTIN_CMPPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
26990 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cmpv4sf3, "__builtin_ia32_cmpps", IX86_BUILTIN_CMPPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
26991 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cmpv4df3, "__builtin_ia32_cmppd256", IX86_BUILTIN_CMPPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_INT },
26992 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cmpv8sf3, "__builtin_ia32_cmpps256", IX86_BUILTIN_CMPPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_INT },
26993 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vextractf128v4df, "__builtin_ia32_vextractf128_pd256", IX86_BUILTIN_EXTRACTF128PD256, UNKNOWN, (int) V2DF_FTYPE_V4DF_INT },
26994 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vextractf128v8sf, "__builtin_ia32_vextractf128_ps256", IX86_BUILTIN_EXTRACTF128PS256, UNKNOWN, (int) V4SF_FTYPE_V8SF_INT },
26995 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vextractf128v8si, "__builtin_ia32_vextractf128_si256", IX86_BUILTIN_EXTRACTF128SI256, UNKNOWN, (int) V4SI_FTYPE_V8SI_INT },
26996 { OPTION_MASK_ISA_AVX, CODE_FOR_floatv4siv4df2, "__builtin_ia32_cvtdq2pd256", IX86_BUILTIN_CVTDQ2PD256, UNKNOWN, (int) V4DF_FTYPE_V4SI },
26997 { OPTION_MASK_ISA_AVX, CODE_FOR_floatv8siv8sf2, "__builtin_ia32_cvtdq2ps256", IX86_BUILTIN_CVTDQ2PS256, UNKNOWN, (int) V8SF_FTYPE_V8SI },
26998 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cvtpd2ps256, "__builtin_ia32_cvtpd2ps256", IX86_BUILTIN_CVTPD2PS256, UNKNOWN, (int) V4SF_FTYPE_V4DF },
26999 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cvtps2dq256, "__builtin_ia32_cvtps2dq256", IX86_BUILTIN_CVTPS2DQ256, UNKNOWN, (int) V8SI_FTYPE_V8SF },
27000 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cvtps2pd256, "__builtin_ia32_cvtps2pd256", IX86_BUILTIN_CVTPS2PD256, UNKNOWN, (int) V4DF_FTYPE_V4SF },
27001 { OPTION_MASK_ISA_AVX, CODE_FOR_fix_truncv4dfv4si2, "__builtin_ia32_cvttpd2dq256", IX86_BUILTIN_CVTTPD2DQ256, UNKNOWN, (int) V4SI_FTYPE_V4DF },
27002 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cvtpd2dq256, "__builtin_ia32_cvtpd2dq256", IX86_BUILTIN_CVTPD2DQ256, UNKNOWN, (int) V4SI_FTYPE_V4DF },
27003 { OPTION_MASK_ISA_AVX, CODE_FOR_fix_truncv8sfv8si2, "__builtin_ia32_cvttps2dq256", IX86_BUILTIN_CVTTPS2DQ256, UNKNOWN, (int) V8SI_FTYPE_V8SF },
27004 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vperm2f128v4df3, "__builtin_ia32_vperm2f128_pd256", IX86_BUILTIN_VPERM2F128PD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_INT },
27005 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vperm2f128v8sf3, "__builtin_ia32_vperm2f128_ps256", IX86_BUILTIN_VPERM2F128PS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_INT },
27006 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vperm2f128v8si3, "__builtin_ia32_vperm2f128_si256", IX86_BUILTIN_VPERM2F128SI256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_INT },
27007 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilv2df, "__builtin_ia32_vpermilpd", IX86_BUILTIN_VPERMILPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_INT },
27008 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilv4sf, "__builtin_ia32_vpermilps", IX86_BUILTIN_VPERMILPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_INT },
27009 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilv4df, "__builtin_ia32_vpermilpd256", IX86_BUILTIN_VPERMILPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_INT },
27010 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilv8sf, "__builtin_ia32_vpermilps256", IX86_BUILTIN_VPERMILPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_INT },
27011 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vinsertf128v4df, "__builtin_ia32_vinsertf128_pd256", IX86_BUILTIN_VINSERTF128PD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V2DF_INT },
27012 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vinsertf128v8sf, "__builtin_ia32_vinsertf128_ps256", IX86_BUILTIN_VINSERTF128PS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V4SF_INT },
27013 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vinsertf128v8si, "__builtin_ia32_vinsertf128_si256", IX86_BUILTIN_VINSERTF128SI256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V4SI_INT },
27015 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movshdup256, "__builtin_ia32_movshdup256", IX86_BUILTIN_MOVSHDUP256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
27016 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movsldup256, "__builtin_ia32_movsldup256", IX86_BUILTIN_MOVSLDUP256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
27017 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movddup256, "__builtin_ia32_movddup256", IX86_BUILTIN_MOVDDUP256, UNKNOWN, (int) V4DF_FTYPE_V4DF },
27019 { OPTION_MASK_ISA_AVX, CODE_FOR_sqrtv4df2, "__builtin_ia32_sqrtpd256", IX86_BUILTIN_SQRTPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF },
27020 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_sqrtv8sf2, "__builtin_ia32_sqrtps256", IX86_BUILTIN_SQRTPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
27021 { OPTION_MASK_ISA_AVX, CODE_FOR_sqrtv8sf2, "__builtin_ia32_sqrtps_nr256", IX86_BUILTIN_SQRTPS_NR256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
27022 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_rsqrtv8sf2, "__builtin_ia32_rsqrtps256", IX86_BUILTIN_RSQRTPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
27023 { OPTION_MASK_ISA_AVX, CODE_FOR_rsqrtv8sf2, "__builtin_ia32_rsqrtps_nr256", IX86_BUILTIN_RSQRTPS_NR256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
27025 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_rcpv8sf2, "__builtin_ia32_rcpps256", IX86_BUILTIN_RCPPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
27027 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundpd256, "__builtin_ia32_roundpd256", IX86_BUILTIN_ROUNDPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_INT },
27028 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundps256, "__builtin_ia32_roundps256", IX86_BUILTIN_ROUNDPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_INT },
27030 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundpd256, "__builtin_ia32_floorpd256", IX86_BUILTIN_FLOORPD256, (enum rtx_code) ROUND_FLOOR, (int) V4DF_FTYPE_V4DF_ROUND },
27031 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundpd256, "__builtin_ia32_ceilpd256", IX86_BUILTIN_CEILPD256, (enum rtx_code) ROUND_CEIL, (int) V4DF_FTYPE_V4DF_ROUND },
27032 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundpd256, "__builtin_ia32_truncpd256", IX86_BUILTIN_TRUNCPD256, (enum rtx_code) ROUND_TRUNC, (int) V4DF_FTYPE_V4DF_ROUND },
27033 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundpd256, "__builtin_ia32_rintpd256", IX86_BUILTIN_RINTPD256, (enum rtx_code) ROUND_MXCSR, (int) V4DF_FTYPE_V4DF_ROUND },
27035 { OPTION_MASK_ISA_AVX, CODE_FOR_roundv4df2, "__builtin_ia32_roundpd_az256", IX86_BUILTIN_ROUNDPD_AZ256, UNKNOWN, (int) V4DF_FTYPE_V4DF },
27036 { OPTION_MASK_ISA_AVX, CODE_FOR_roundv4df2_vec_pack_sfix, "__builtin_ia32_roundpd_az_vec_pack_sfix256", IX86_BUILTIN_ROUNDPD_AZ_VEC_PACK_SFIX256, UNKNOWN, (int) V8SI_FTYPE_V4DF_V4DF },
27038 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundpd_vec_pack_sfix256, "__builtin_ia32_floorpd_vec_pack_sfix256", IX86_BUILTIN_FLOORPD_VEC_PACK_SFIX256, (enum rtx_code) ROUND_FLOOR, (int) V8SI_FTYPE_V4DF_V4DF_ROUND },
27039 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundpd_vec_pack_sfix256, "__builtin_ia32_ceilpd_vec_pack_sfix256", IX86_BUILTIN_CEILPD_VEC_PACK_SFIX256, (enum rtx_code) ROUND_CEIL, (int) V8SI_FTYPE_V4DF_V4DF_ROUND },
27041 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundps256, "__builtin_ia32_floorps256", IX86_BUILTIN_FLOORPS256, (enum rtx_code) ROUND_FLOOR, (int) V8SF_FTYPE_V8SF_ROUND },
27042 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundps256, "__builtin_ia32_ceilps256", IX86_BUILTIN_CEILPS256, (enum rtx_code) ROUND_CEIL, (int) V8SF_FTYPE_V8SF_ROUND },
27043 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundps256, "__builtin_ia32_truncps256", IX86_BUILTIN_TRUNCPS256, (enum rtx_code) ROUND_TRUNC, (int) V8SF_FTYPE_V8SF_ROUND },
27044 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundps256, "__builtin_ia32_rintps256", IX86_BUILTIN_RINTPS256, (enum rtx_code) ROUND_MXCSR, (int) V8SF_FTYPE_V8SF_ROUND },
27046 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundps_sfix256, "__builtin_ia32_floorps_sfix256", IX86_BUILTIN_FLOORPS_SFIX256, (enum rtx_code) ROUND_FLOOR, (int) V8SI_FTYPE_V8SF_ROUND },
27047 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundps_sfix256, "__builtin_ia32_ceilps_sfix256", IX86_BUILTIN_CEILPS_SFIX256, (enum rtx_code) ROUND_CEIL, (int) V8SI_FTYPE_V8SF_ROUND },
27049 { OPTION_MASK_ISA_AVX, CODE_FOR_roundv8sf2, "__builtin_ia32_roundps_az256", IX86_BUILTIN_ROUNDPS_AZ256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
27050 { OPTION_MASK_ISA_AVX, CODE_FOR_roundv8sf2_sfix, "__builtin_ia32_roundps_az_sfix256", IX86_BUILTIN_ROUNDPS_AZ_SFIX256, UNKNOWN, (int) V8SI_FTYPE_V8SF },
27052 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_unpckhpd256, "__builtin_ia32_unpckhpd256", IX86_BUILTIN_UNPCKHPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
27053 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_unpcklpd256, "__builtin_ia32_unpcklpd256", IX86_BUILTIN_UNPCKLPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
27054 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_unpckhps256, "__builtin_ia32_unpckhps256", IX86_BUILTIN_UNPCKHPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
27055 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_unpcklps256, "__builtin_ia32_unpcklps256", IX86_BUILTIN_UNPCKLPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
27057 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_si256_si, "__builtin_ia32_si256_si", IX86_BUILTIN_SI256_SI, UNKNOWN, (int) V8SI_FTYPE_V4SI },
27058 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_ps256_ps, "__builtin_ia32_ps256_ps", IX86_BUILTIN_PS256_PS, UNKNOWN, (int) V8SF_FTYPE_V4SF },
27059 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_pd256_pd, "__builtin_ia32_pd256_pd", IX86_BUILTIN_PD256_PD, UNKNOWN, (int) V4DF_FTYPE_V2DF },
27060 { OPTION_MASK_ISA_AVX, CODE_FOR_vec_extract_lo_v8si, "__builtin_ia32_si_si256", IX86_BUILTIN_SI_SI256, UNKNOWN, (int) V4SI_FTYPE_V8SI },
27061 { OPTION_MASK_ISA_AVX, CODE_FOR_vec_extract_lo_v8sf, "__builtin_ia32_ps_ps256", IX86_BUILTIN_PS_PS256, UNKNOWN, (int) V4SF_FTYPE_V8SF },
27062 { OPTION_MASK_ISA_AVX, CODE_FOR_vec_extract_lo_v4df, "__builtin_ia32_pd_pd256", IX86_BUILTIN_PD_PD256, UNKNOWN, (int) V2DF_FTYPE_V4DF },
27064 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestpd, "__builtin_ia32_vtestzpd", IX86_BUILTIN_VTESTZPD, EQ, (int) INT_FTYPE_V2DF_V2DF_PTEST },
27065 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestpd, "__builtin_ia32_vtestcpd", IX86_BUILTIN_VTESTCPD, LTU, (int) INT_FTYPE_V2DF_V2DF_PTEST },
27066 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestpd, "__builtin_ia32_vtestnzcpd", IX86_BUILTIN_VTESTNZCPD, GTU, (int) INT_FTYPE_V2DF_V2DF_PTEST },
27067 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestps, "__builtin_ia32_vtestzps", IX86_BUILTIN_VTESTZPS, EQ, (int) INT_FTYPE_V4SF_V4SF_PTEST },
27068 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestps, "__builtin_ia32_vtestcps", IX86_BUILTIN_VTESTCPS, LTU, (int) INT_FTYPE_V4SF_V4SF_PTEST },
27069 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestps, "__builtin_ia32_vtestnzcps", IX86_BUILTIN_VTESTNZCPS, GTU, (int) INT_FTYPE_V4SF_V4SF_PTEST },
27070 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestpd256, "__builtin_ia32_vtestzpd256", IX86_BUILTIN_VTESTZPD256, EQ, (int) INT_FTYPE_V4DF_V4DF_PTEST },
27071 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestpd256, "__builtin_ia32_vtestcpd256", IX86_BUILTIN_VTESTCPD256, LTU, (int) INT_FTYPE_V4DF_V4DF_PTEST },
27072 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestpd256, "__builtin_ia32_vtestnzcpd256", IX86_BUILTIN_VTESTNZCPD256, GTU, (int) INT_FTYPE_V4DF_V4DF_PTEST },
27073 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestps256, "__builtin_ia32_vtestzps256", IX86_BUILTIN_VTESTZPS256, EQ, (int) INT_FTYPE_V8SF_V8SF_PTEST },
27074 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestps256, "__builtin_ia32_vtestcps256", IX86_BUILTIN_VTESTCPS256, LTU, (int) INT_FTYPE_V8SF_V8SF_PTEST },
27075 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestps256, "__builtin_ia32_vtestnzcps256", IX86_BUILTIN_VTESTNZCPS256, GTU, (int) INT_FTYPE_V8SF_V8SF_PTEST },
27076 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_ptest256, "__builtin_ia32_ptestz256", IX86_BUILTIN_PTESTZ256, EQ, (int) INT_FTYPE_V4DI_V4DI_PTEST },
27077 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_ptest256, "__builtin_ia32_ptestc256", IX86_BUILTIN_PTESTC256, LTU, (int) INT_FTYPE_V4DI_V4DI_PTEST },
27078 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_ptest256, "__builtin_ia32_ptestnzc256", IX86_BUILTIN_PTESTNZC256, GTU, (int) INT_FTYPE_V4DI_V4DI_PTEST },
27080 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movmskpd256, "__builtin_ia32_movmskpd256", IX86_BUILTIN_MOVMSKPD256, UNKNOWN, (int) INT_FTYPE_V4DF },
27081 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movmskps256, "__builtin_ia32_movmskps256", IX86_BUILTIN_MOVMSKPS256, UNKNOWN, (int) INT_FTYPE_V8SF },
27083 { OPTION_MASK_ISA_AVX, CODE_FOR_copysignv8sf3, "__builtin_ia32_copysignps256", IX86_BUILTIN_CPYSGNPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
27084 { OPTION_MASK_ISA_AVX, CODE_FOR_copysignv4df3, "__builtin_ia32_copysignpd256", IX86_BUILTIN_CPYSGNPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
27086 { OPTION_MASK_ISA_AVX, CODE_FOR_vec_pack_sfix_v4df, "__builtin_ia32_vec_pack_sfix256 ", IX86_BUILTIN_VEC_PACK_SFIX256, UNKNOWN, (int) V8SI_FTYPE_V4DF_V4DF },
27088 /* AVX2 */
27089 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_mpsadbw, "__builtin_ia32_mpsadbw256", IX86_BUILTIN_MPSADBW256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI_INT },
27090 { OPTION_MASK_ISA_AVX2, CODE_FOR_absv32qi2, "__builtin_ia32_pabsb256", IX86_BUILTIN_PABSB256, UNKNOWN, (int) V32QI_FTYPE_V32QI },
27091 { OPTION_MASK_ISA_AVX2, CODE_FOR_absv16hi2, "__builtin_ia32_pabsw256", IX86_BUILTIN_PABSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI },
27092 { OPTION_MASK_ISA_AVX2, CODE_FOR_absv8si2, "__builtin_ia32_pabsd256", IX86_BUILTIN_PABSD256, UNKNOWN, (int) V8SI_FTYPE_V8SI },
27093 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_packssdw, "__builtin_ia32_packssdw256", IX86_BUILTIN_PACKSSDW256, UNKNOWN, (int) V16HI_FTYPE_V8SI_V8SI },
27094 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_packsswb, "__builtin_ia32_packsswb256", IX86_BUILTIN_PACKSSWB256, UNKNOWN, (int) V32QI_FTYPE_V16HI_V16HI },
27095 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_packusdw, "__builtin_ia32_packusdw256", IX86_BUILTIN_PACKUSDW256, UNKNOWN, (int) V16HI_FTYPE_V8SI_V8SI },
27096 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_packuswb, "__builtin_ia32_packuswb256", IX86_BUILTIN_PACKUSWB256, UNKNOWN, (int) V32QI_FTYPE_V16HI_V16HI },
27097 { OPTION_MASK_ISA_AVX2, CODE_FOR_addv32qi3, "__builtin_ia32_paddb256", IX86_BUILTIN_PADDB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
27098 { OPTION_MASK_ISA_AVX2, CODE_FOR_addv16hi3, "__builtin_ia32_paddw256", IX86_BUILTIN_PADDW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
27099 { OPTION_MASK_ISA_AVX2, CODE_FOR_addv8si3, "__builtin_ia32_paddd256", IX86_BUILTIN_PADDD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
27100 { OPTION_MASK_ISA_AVX2, CODE_FOR_addv4di3, "__builtin_ia32_paddq256", IX86_BUILTIN_PADDQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
27101 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ssaddv32qi3, "__builtin_ia32_paddsb256", IX86_BUILTIN_PADDSB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
27102 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ssaddv16hi3, "__builtin_ia32_paddsw256", IX86_BUILTIN_PADDSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
27103 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_usaddv32qi3, "__builtin_ia32_paddusb256", IX86_BUILTIN_PADDUSB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
27104 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_usaddv16hi3, "__builtin_ia32_paddusw256", IX86_BUILTIN_PADDUSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
27105 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_palignrv2ti, "__builtin_ia32_palignr256", IX86_BUILTIN_PALIGNR256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_INT_CONVERT },
27106 { OPTION_MASK_ISA_AVX2, CODE_FOR_andv4di3, "__builtin_ia32_andsi256", IX86_BUILTIN_AND256I, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
27107 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_andnotv4di3, "__builtin_ia32_andnotsi256", IX86_BUILTIN_ANDNOT256I, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
27108 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_uavgv32qi3, "__builtin_ia32_pavgb256", IX86_BUILTIN_PAVGB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
27109 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_uavgv16hi3, "__builtin_ia32_pavgw256", IX86_BUILTIN_PAVGW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
27110 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pblendvb, "__builtin_ia32_pblendvb256", IX86_BUILTIN_PBLENDVB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI_V32QI },
27111 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pblendw, "__builtin_ia32_pblendw256", IX86_BUILTIN_PBLENDVW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI_INT },
27112 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_eqv32qi3, "__builtin_ia32_pcmpeqb256", IX86_BUILTIN_PCMPEQB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
27113 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_eqv16hi3, "__builtin_ia32_pcmpeqw256", IX86_BUILTIN_PCMPEQW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
27114 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_eqv8si3, "__builtin_ia32_pcmpeqd256", IX86_BUILTIN_PCMPEQD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
27115 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_eqv4di3, "__builtin_ia32_pcmpeqq256", IX86_BUILTIN_PCMPEQQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
27116 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_gtv32qi3, "__builtin_ia32_pcmpgtb256", IX86_BUILTIN_PCMPGTB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
27117 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_gtv16hi3, "__builtin_ia32_pcmpgtw256", IX86_BUILTIN_PCMPGTW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
27118 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_gtv8si3, "__builtin_ia32_pcmpgtd256", IX86_BUILTIN_PCMPGTD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
27119 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_gtv4di3, "__builtin_ia32_pcmpgtq256", IX86_BUILTIN_PCMPGTQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
27120 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_phaddwv16hi3, "__builtin_ia32_phaddw256", IX86_BUILTIN_PHADDW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
27121 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_phadddv8si3, "__builtin_ia32_phaddd256", IX86_BUILTIN_PHADDD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
27122 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_phaddswv16hi3, "__builtin_ia32_phaddsw256", IX86_BUILTIN_PHADDSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
27123 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_phsubwv16hi3, "__builtin_ia32_phsubw256", IX86_BUILTIN_PHSUBW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
27124 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_phsubdv8si3, "__builtin_ia32_phsubd256", IX86_BUILTIN_PHSUBD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
27125 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_phsubswv16hi3, "__builtin_ia32_phsubsw256", IX86_BUILTIN_PHSUBSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
27126 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pmaddubsw256, "__builtin_ia32_pmaddubsw256", IX86_BUILTIN_PMADDUBSW256, UNKNOWN, (int) V16HI_FTYPE_V32QI_V32QI },
27127 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pmaddwd, "__builtin_ia32_pmaddwd256", IX86_BUILTIN_PMADDWD256, UNKNOWN, (int) V8SI_FTYPE_V16HI_V16HI },
27128 { OPTION_MASK_ISA_AVX2, CODE_FOR_smaxv32qi3, "__builtin_ia32_pmaxsb256", IX86_BUILTIN_PMAXSB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
27129 { OPTION_MASK_ISA_AVX2, CODE_FOR_smaxv16hi3, "__builtin_ia32_pmaxsw256", IX86_BUILTIN_PMAXSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
27130 { OPTION_MASK_ISA_AVX2, CODE_FOR_smaxv8si3 , "__builtin_ia32_pmaxsd256", IX86_BUILTIN_PMAXSD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
27131 { OPTION_MASK_ISA_AVX2, CODE_FOR_umaxv32qi3, "__builtin_ia32_pmaxub256", IX86_BUILTIN_PMAXUB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
27132 { OPTION_MASK_ISA_AVX2, CODE_FOR_umaxv16hi3, "__builtin_ia32_pmaxuw256", IX86_BUILTIN_PMAXUW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
27133 { OPTION_MASK_ISA_AVX2, CODE_FOR_umaxv8si3 , "__builtin_ia32_pmaxud256", IX86_BUILTIN_PMAXUD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
27134 { OPTION_MASK_ISA_AVX2, CODE_FOR_sminv32qi3, "__builtin_ia32_pminsb256", IX86_BUILTIN_PMINSB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
27135 { OPTION_MASK_ISA_AVX2, CODE_FOR_sminv16hi3, "__builtin_ia32_pminsw256", IX86_BUILTIN_PMINSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
27136 { OPTION_MASK_ISA_AVX2, CODE_FOR_sminv8si3 , "__builtin_ia32_pminsd256", IX86_BUILTIN_PMINSD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
27137 { OPTION_MASK_ISA_AVX2, CODE_FOR_uminv32qi3, "__builtin_ia32_pminub256", IX86_BUILTIN_PMINUB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
27138 { OPTION_MASK_ISA_AVX2, CODE_FOR_uminv16hi3, "__builtin_ia32_pminuw256", IX86_BUILTIN_PMINUW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
27139 { OPTION_MASK_ISA_AVX2, CODE_FOR_uminv8si3 , "__builtin_ia32_pminud256", IX86_BUILTIN_PMINUD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
27140 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pmovmskb, "__builtin_ia32_pmovmskb256", IX86_BUILTIN_PMOVMSKB256, UNKNOWN, (int) INT_FTYPE_V32QI },
27141 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_sign_extendv16qiv16hi2, "__builtin_ia32_pmovsxbw256", IX86_BUILTIN_PMOVSXBW256, UNKNOWN, (int) V16HI_FTYPE_V16QI },
27142 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_sign_extendv8qiv8si2 , "__builtin_ia32_pmovsxbd256", IX86_BUILTIN_PMOVSXBD256, UNKNOWN, (int) V8SI_FTYPE_V16QI },
27143 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_sign_extendv4qiv4di2 , "__builtin_ia32_pmovsxbq256", IX86_BUILTIN_PMOVSXBQ256, UNKNOWN, (int) V4DI_FTYPE_V16QI },
27144 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_sign_extendv8hiv8si2 , "__builtin_ia32_pmovsxwd256", IX86_BUILTIN_PMOVSXWD256, UNKNOWN, (int) V8SI_FTYPE_V8HI },
27145 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_sign_extendv4hiv4di2 , "__builtin_ia32_pmovsxwq256", IX86_BUILTIN_PMOVSXWQ256, UNKNOWN, (int) V4DI_FTYPE_V8HI },
27146 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_sign_extendv4siv4di2 , "__builtin_ia32_pmovsxdq256", IX86_BUILTIN_PMOVSXDQ256, UNKNOWN, (int) V4DI_FTYPE_V4SI },
27147 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_zero_extendv16qiv16hi2, "__builtin_ia32_pmovzxbw256", IX86_BUILTIN_PMOVZXBW256, UNKNOWN, (int) V16HI_FTYPE_V16QI },
27148 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_zero_extendv8qiv8si2 , "__builtin_ia32_pmovzxbd256", IX86_BUILTIN_PMOVZXBD256, UNKNOWN, (int) V8SI_FTYPE_V16QI },
27149 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_zero_extendv4qiv4di2 , "__builtin_ia32_pmovzxbq256", IX86_BUILTIN_PMOVZXBQ256, UNKNOWN, (int) V4DI_FTYPE_V16QI },
27150 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_zero_extendv8hiv8si2 , "__builtin_ia32_pmovzxwd256", IX86_BUILTIN_PMOVZXWD256, UNKNOWN, (int) V8SI_FTYPE_V8HI },
27151 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_zero_extendv4hiv4di2 , "__builtin_ia32_pmovzxwq256", IX86_BUILTIN_PMOVZXWQ256, UNKNOWN, (int) V4DI_FTYPE_V8HI },
27152 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_zero_extendv4siv4di2 , "__builtin_ia32_pmovzxdq256", IX86_BUILTIN_PMOVZXDQ256, UNKNOWN, (int) V4DI_FTYPE_V4SI },
27153 { OPTION_MASK_ISA_AVX2, CODE_FOR_vec_widen_smult_even_v8si, "__builtin_ia32_pmuldq256", IX86_BUILTIN_PMULDQ256, UNKNOWN, (int) V4DI_FTYPE_V8SI_V8SI },
27154 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_umulhrswv16hi3 , "__builtin_ia32_pmulhrsw256", IX86_BUILTIN_PMULHRSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
27155 { OPTION_MASK_ISA_AVX2, CODE_FOR_umulv16hi3_highpart, "__builtin_ia32_pmulhuw256" , IX86_BUILTIN_PMULHUW256 , UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
27156 { OPTION_MASK_ISA_AVX2, CODE_FOR_smulv16hi3_highpart, "__builtin_ia32_pmulhw256" , IX86_BUILTIN_PMULHW256 , UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
27157 { OPTION_MASK_ISA_AVX2, CODE_FOR_mulv16hi3, "__builtin_ia32_pmullw256" , IX86_BUILTIN_PMULLW256 , UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
27158 { OPTION_MASK_ISA_AVX2, CODE_FOR_mulv8si3, "__builtin_ia32_pmulld256" , IX86_BUILTIN_PMULLD256 , UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
27159 { OPTION_MASK_ISA_AVX2, CODE_FOR_vec_widen_umult_even_v8si, "__builtin_ia32_pmuludq256", IX86_BUILTIN_PMULUDQ256, UNKNOWN, (int) V4DI_FTYPE_V8SI_V8SI },
27160 { OPTION_MASK_ISA_AVX2, CODE_FOR_iorv4di3, "__builtin_ia32_por256", IX86_BUILTIN_POR256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
27161 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_psadbw, "__builtin_ia32_psadbw256", IX86_BUILTIN_PSADBW256, UNKNOWN, (int) V16HI_FTYPE_V32QI_V32QI },
27162 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pshufbv32qi3, "__builtin_ia32_pshufb256", IX86_BUILTIN_PSHUFB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
27163 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pshufdv3, "__builtin_ia32_pshufd256", IX86_BUILTIN_PSHUFD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_INT },
27164 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pshufhwv3, "__builtin_ia32_pshufhw256", IX86_BUILTIN_PSHUFHW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_INT },
27165 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pshuflwv3, "__builtin_ia32_pshuflw256", IX86_BUILTIN_PSHUFLW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_INT },
27166 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_psignv32qi3, "__builtin_ia32_psignb256", IX86_BUILTIN_PSIGNB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
27167 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_psignv16hi3, "__builtin_ia32_psignw256", IX86_BUILTIN_PSIGNW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
27168 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_psignv8si3 , "__builtin_ia32_psignd256", IX86_BUILTIN_PSIGND256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
27169 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ashlv2ti3, "__builtin_ia32_pslldqi256", IX86_BUILTIN_PSLLDQI256, UNKNOWN, (int) V4DI_FTYPE_V4DI_INT_CONVERT },
27170 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashlv16hi3, "__builtin_ia32_psllwi256", IX86_BUILTIN_PSLLWI256 , UNKNOWN, (int) V16HI_FTYPE_V16HI_SI_COUNT },
27171 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashlv16hi3, "__builtin_ia32_psllw256", IX86_BUILTIN_PSLLW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V8HI_COUNT },
27172 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashlv8si3, "__builtin_ia32_pslldi256", IX86_BUILTIN_PSLLDI256, UNKNOWN, (int) V8SI_FTYPE_V8SI_SI_COUNT },
27173 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashlv8si3, "__builtin_ia32_pslld256", IX86_BUILTIN_PSLLD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V4SI_COUNT },
27174 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashlv4di3, "__builtin_ia32_psllqi256", IX86_BUILTIN_PSLLQI256, UNKNOWN, (int) V4DI_FTYPE_V4DI_INT_COUNT },
27175 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashlv4di3, "__builtin_ia32_psllq256", IX86_BUILTIN_PSLLQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V2DI_COUNT },
27176 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashrv16hi3, "__builtin_ia32_psrawi256", IX86_BUILTIN_PSRAWI256, UNKNOWN, (int) V16HI_FTYPE_V16HI_SI_COUNT },
27177 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashrv16hi3, "__builtin_ia32_psraw256", IX86_BUILTIN_PSRAW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V8HI_COUNT },
27178 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashrv8si3, "__builtin_ia32_psradi256", IX86_BUILTIN_PSRADI256, UNKNOWN, (int) V8SI_FTYPE_V8SI_SI_COUNT },
27179 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashrv8si3, "__builtin_ia32_psrad256", IX86_BUILTIN_PSRAD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V4SI_COUNT },
27180 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_lshrv2ti3, "__builtin_ia32_psrldqi256", IX86_BUILTIN_PSRLDQI256, UNKNOWN, (int) V4DI_FTYPE_V4DI_INT_CONVERT },
27181 { OPTION_MASK_ISA_AVX2, CODE_FOR_lshrv16hi3, "__builtin_ia32_psrlwi256", IX86_BUILTIN_PSRLWI256 , UNKNOWN, (int) V16HI_FTYPE_V16HI_SI_COUNT },
27182 { OPTION_MASK_ISA_AVX2, CODE_FOR_lshrv16hi3, "__builtin_ia32_psrlw256", IX86_BUILTIN_PSRLW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V8HI_COUNT },
27183 { OPTION_MASK_ISA_AVX2, CODE_FOR_lshrv8si3, "__builtin_ia32_psrldi256", IX86_BUILTIN_PSRLDI256, UNKNOWN, (int) V8SI_FTYPE_V8SI_SI_COUNT },
27184 { OPTION_MASK_ISA_AVX2, CODE_FOR_lshrv8si3, "__builtin_ia32_psrld256", IX86_BUILTIN_PSRLD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V4SI_COUNT },
27185 { OPTION_MASK_ISA_AVX2, CODE_FOR_lshrv4di3, "__builtin_ia32_psrlqi256", IX86_BUILTIN_PSRLQI256, UNKNOWN, (int) V4DI_FTYPE_V4DI_INT_COUNT },
27186 { OPTION_MASK_ISA_AVX2, CODE_FOR_lshrv4di3, "__builtin_ia32_psrlq256", IX86_BUILTIN_PSRLQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V2DI_COUNT },
27187 { OPTION_MASK_ISA_AVX2, CODE_FOR_subv32qi3, "__builtin_ia32_psubb256", IX86_BUILTIN_PSUBB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
27188 { OPTION_MASK_ISA_AVX2, CODE_FOR_subv16hi3, "__builtin_ia32_psubw256", IX86_BUILTIN_PSUBW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
27189 { OPTION_MASK_ISA_AVX2, CODE_FOR_subv8si3, "__builtin_ia32_psubd256", IX86_BUILTIN_PSUBD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
27190 { OPTION_MASK_ISA_AVX2, CODE_FOR_subv4di3, "__builtin_ia32_psubq256", IX86_BUILTIN_PSUBQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
27191 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_sssubv32qi3, "__builtin_ia32_psubsb256", IX86_BUILTIN_PSUBSB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
27192 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_sssubv16hi3, "__builtin_ia32_psubsw256", IX86_BUILTIN_PSUBSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
27193 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ussubv32qi3, "__builtin_ia32_psubusb256", IX86_BUILTIN_PSUBUSB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
27194 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ussubv16hi3, "__builtin_ia32_psubusw256", IX86_BUILTIN_PSUBUSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
27195 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_interleave_highv32qi, "__builtin_ia32_punpckhbw256", IX86_BUILTIN_PUNPCKHBW256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
27196 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_interleave_highv16hi, "__builtin_ia32_punpckhwd256", IX86_BUILTIN_PUNPCKHWD256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
27197 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_interleave_highv8si, "__builtin_ia32_punpckhdq256", IX86_BUILTIN_PUNPCKHDQ256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
27198 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_interleave_highv4di, "__builtin_ia32_punpckhqdq256", IX86_BUILTIN_PUNPCKHQDQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
27199 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_interleave_lowv32qi, "__builtin_ia32_punpcklbw256", IX86_BUILTIN_PUNPCKLBW256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
27200 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_interleave_lowv16hi, "__builtin_ia32_punpcklwd256", IX86_BUILTIN_PUNPCKLWD256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
27201 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_interleave_lowv8si, "__builtin_ia32_punpckldq256", IX86_BUILTIN_PUNPCKLDQ256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
27202 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_interleave_lowv4di, "__builtin_ia32_punpcklqdq256", IX86_BUILTIN_PUNPCKLQDQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
27203 { OPTION_MASK_ISA_AVX2, CODE_FOR_xorv4di3, "__builtin_ia32_pxor256", IX86_BUILTIN_PXOR256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
27204 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_vec_dupv4sf, "__builtin_ia32_vbroadcastss_ps", IX86_BUILTIN_VBROADCASTSS_PS, UNKNOWN, (int) V4SF_FTYPE_V4SF },
27205 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_vec_dupv8sf, "__builtin_ia32_vbroadcastss_ps256", IX86_BUILTIN_VBROADCASTSS_PS256, UNKNOWN, (int) V8SF_FTYPE_V4SF },
27206 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_vec_dupv4df, "__builtin_ia32_vbroadcastsd_pd256", IX86_BUILTIN_VBROADCASTSD_PD256, UNKNOWN, (int) V4DF_FTYPE_V2DF },
27207 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_vbroadcasti128_v4di, "__builtin_ia32_vbroadcastsi256", IX86_BUILTIN_VBROADCASTSI256, UNKNOWN, (int) V4DI_FTYPE_V2DI },
27208 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pblenddv4si, "__builtin_ia32_pblendd128", IX86_BUILTIN_PBLENDD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_INT },
27209 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pblenddv8si, "__builtin_ia32_pblendd256", IX86_BUILTIN_PBLENDD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_INT },
27210 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pbroadcastv32qi, "__builtin_ia32_pbroadcastb256", IX86_BUILTIN_PBROADCASTB256, UNKNOWN, (int) V32QI_FTYPE_V16QI },
27211 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pbroadcastv16hi, "__builtin_ia32_pbroadcastw256", IX86_BUILTIN_PBROADCASTW256, UNKNOWN, (int) V16HI_FTYPE_V8HI },
27212 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pbroadcastv8si, "__builtin_ia32_pbroadcastd256", IX86_BUILTIN_PBROADCASTD256, UNKNOWN, (int) V8SI_FTYPE_V4SI },
27213 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pbroadcastv4di, "__builtin_ia32_pbroadcastq256", IX86_BUILTIN_PBROADCASTQ256, UNKNOWN, (int) V4DI_FTYPE_V2DI },
27214 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pbroadcastv16qi, "__builtin_ia32_pbroadcastb128", IX86_BUILTIN_PBROADCASTB128, UNKNOWN, (int) V16QI_FTYPE_V16QI },
27215 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pbroadcastv8hi, "__builtin_ia32_pbroadcastw128", IX86_BUILTIN_PBROADCASTW128, UNKNOWN, (int) V8HI_FTYPE_V8HI },
27216 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pbroadcastv4si, "__builtin_ia32_pbroadcastd128", IX86_BUILTIN_PBROADCASTD128, UNKNOWN, (int) V4SI_FTYPE_V4SI },
27217 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pbroadcastv2di, "__builtin_ia32_pbroadcastq128", IX86_BUILTIN_PBROADCASTQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI },
27218 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_permvarv8si, "__builtin_ia32_permvarsi256", IX86_BUILTIN_VPERMVARSI256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
27219 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_permvarv8sf, "__builtin_ia32_permvarsf256", IX86_BUILTIN_VPERMVARSF256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SI },
27220 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_permv4df, "__builtin_ia32_permdf256", IX86_BUILTIN_VPERMDF256, UNKNOWN, (int) V4DF_FTYPE_V4DF_INT },
27221 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_permv4di, "__builtin_ia32_permdi256", IX86_BUILTIN_VPERMDI256, UNKNOWN, (int) V4DI_FTYPE_V4DI_INT },
27222 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_permv2ti, "__builtin_ia32_permti256", IX86_BUILTIN_VPERMTI256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_INT },
27223 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_extracti128, "__builtin_ia32_extract128i256", IX86_BUILTIN_VEXTRACT128I256, UNKNOWN, (int) V2DI_FTYPE_V4DI_INT },
27224 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_inserti128, "__builtin_ia32_insert128i256", IX86_BUILTIN_VINSERT128I256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V2DI_INT },
27225 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ashlvv4di, "__builtin_ia32_psllv4di", IX86_BUILTIN_PSLLVV4DI, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
27226 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ashlvv2di, "__builtin_ia32_psllv2di", IX86_BUILTIN_PSLLVV2DI, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
27227 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ashlvv8si, "__builtin_ia32_psllv8si", IX86_BUILTIN_PSLLVV8SI, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
27228 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ashlvv4si, "__builtin_ia32_psllv4si", IX86_BUILTIN_PSLLVV4SI, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
27229 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ashrvv8si, "__builtin_ia32_psrav8si", IX86_BUILTIN_PSRAVV8SI, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
27230 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ashrvv4si, "__builtin_ia32_psrav4si", IX86_BUILTIN_PSRAVV4SI, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
27231 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_lshrvv4di, "__builtin_ia32_psrlv4di", IX86_BUILTIN_PSRLVV4DI, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
27232 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_lshrvv2di, "__builtin_ia32_psrlv2di", IX86_BUILTIN_PSRLVV2DI, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
27233 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_lshrvv8si, "__builtin_ia32_psrlv8si", IX86_BUILTIN_PSRLVV8SI, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
27234 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_lshrvv4si, "__builtin_ia32_psrlv4si", IX86_BUILTIN_PSRLVV4SI, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
27236 { OPTION_MASK_ISA_LZCNT, CODE_FOR_clzhi2_lzcnt, "__builtin_clzs", IX86_BUILTIN_CLZS, UNKNOWN, (int) UINT16_FTYPE_UINT16 },
27238 /* BMI */
27239 { OPTION_MASK_ISA_BMI, CODE_FOR_bmi_bextr_si, "__builtin_ia32_bextr_u32", IX86_BUILTIN_BEXTR32, UNKNOWN, (int) UINT_FTYPE_UINT_UINT },
27240 { OPTION_MASK_ISA_BMI, CODE_FOR_bmi_bextr_di, "__builtin_ia32_bextr_u64", IX86_BUILTIN_BEXTR64, UNKNOWN, (int) UINT64_FTYPE_UINT64_UINT64 },
27241 { OPTION_MASK_ISA_BMI, CODE_FOR_ctzhi2, "__builtin_ctzs", IX86_BUILTIN_CTZS, UNKNOWN, (int) UINT16_FTYPE_UINT16 },
27243 /* TBM */
27244 { OPTION_MASK_ISA_TBM, CODE_FOR_tbm_bextri_si, "__builtin_ia32_bextri_u32", IX86_BUILTIN_BEXTRI32, UNKNOWN, (int) UINT_FTYPE_UINT_UINT },
27245 { OPTION_MASK_ISA_TBM, CODE_FOR_tbm_bextri_di, "__builtin_ia32_bextri_u64", IX86_BUILTIN_BEXTRI64, UNKNOWN, (int) UINT64_FTYPE_UINT64_UINT64 },
27247 /* F16C */
27248 { OPTION_MASK_ISA_F16C, CODE_FOR_vcvtph2ps, "__builtin_ia32_vcvtph2ps", IX86_BUILTIN_CVTPH2PS, UNKNOWN, (int) V4SF_FTYPE_V8HI },
27249 { OPTION_MASK_ISA_F16C, CODE_FOR_vcvtph2ps256, "__builtin_ia32_vcvtph2ps256", IX86_BUILTIN_CVTPH2PS256, UNKNOWN, (int) V8SF_FTYPE_V8HI },
27250 { OPTION_MASK_ISA_F16C, CODE_FOR_vcvtps2ph, "__builtin_ia32_vcvtps2ph", IX86_BUILTIN_CVTPS2PH, UNKNOWN, (int) V8HI_FTYPE_V4SF_INT },
27251 { OPTION_MASK_ISA_F16C, CODE_FOR_vcvtps2ph256, "__builtin_ia32_vcvtps2ph256", IX86_BUILTIN_CVTPS2PH256, UNKNOWN, (int) V8HI_FTYPE_V8SF_INT },
27253 /* BMI2 */
27254 { OPTION_MASK_ISA_BMI2, CODE_FOR_bmi2_bzhi_si3, "__builtin_ia32_bzhi_si", IX86_BUILTIN_BZHI32, UNKNOWN, (int) UINT_FTYPE_UINT_UINT },
27255 { OPTION_MASK_ISA_BMI2, CODE_FOR_bmi2_bzhi_di3, "__builtin_ia32_bzhi_di", IX86_BUILTIN_BZHI64, UNKNOWN, (int) UINT64_FTYPE_UINT64_UINT64 },
27256 { OPTION_MASK_ISA_BMI2, CODE_FOR_bmi2_pdep_si3, "__builtin_ia32_pdep_si", IX86_BUILTIN_PDEP32, UNKNOWN, (int) UINT_FTYPE_UINT_UINT },
27257 { OPTION_MASK_ISA_BMI2, CODE_FOR_bmi2_pdep_di3, "__builtin_ia32_pdep_di", IX86_BUILTIN_PDEP64, UNKNOWN, (int) UINT64_FTYPE_UINT64_UINT64 },
27258 { OPTION_MASK_ISA_BMI2, CODE_FOR_bmi2_pext_si3, "__builtin_ia32_pext_si", IX86_BUILTIN_PEXT32, UNKNOWN, (int) UINT_FTYPE_UINT_UINT },
27259 { OPTION_MASK_ISA_BMI2, CODE_FOR_bmi2_pext_di3, "__builtin_ia32_pext_di", IX86_BUILTIN_PEXT64, UNKNOWN, (int) UINT64_FTYPE_UINT64_UINT64 },
27260 };
27262 /* FMA4 and XOP. */
27263 #define MULTI_ARG_4_DF2_DI_I V2DF_FTYPE_V2DF_V2DF_V2DI_INT
27264 #define MULTI_ARG_4_DF2_DI_I1 V4DF_FTYPE_V4DF_V4DF_V4DI_INT
27265 #define MULTI_ARG_4_SF2_SI_I V4SF_FTYPE_V4SF_V4SF_V4SI_INT
27266 #define MULTI_ARG_4_SF2_SI_I1 V8SF_FTYPE_V8SF_V8SF_V8SI_INT
27267 #define MULTI_ARG_3_SF V4SF_FTYPE_V4SF_V4SF_V4SF
27268 #define MULTI_ARG_3_DF V2DF_FTYPE_V2DF_V2DF_V2DF
27269 #define MULTI_ARG_3_SF2 V8SF_FTYPE_V8SF_V8SF_V8SF
27270 #define MULTI_ARG_3_DF2 V4DF_FTYPE_V4DF_V4DF_V4DF
27271 #define MULTI_ARG_3_DI V2DI_FTYPE_V2DI_V2DI_V2DI
27272 #define MULTI_ARG_3_SI V4SI_FTYPE_V4SI_V4SI_V4SI
27273 #define MULTI_ARG_3_SI_DI V4SI_FTYPE_V4SI_V4SI_V2DI
27274 #define MULTI_ARG_3_HI V8HI_FTYPE_V8HI_V8HI_V8HI
27275 #define MULTI_ARG_3_HI_SI V8HI_FTYPE_V8HI_V8HI_V4SI
27276 #define MULTI_ARG_3_QI V16QI_FTYPE_V16QI_V16QI_V16QI
27277 #define MULTI_ARG_3_DI2 V4DI_FTYPE_V4DI_V4DI_V4DI
27278 #define MULTI_ARG_3_SI2 V8SI_FTYPE_V8SI_V8SI_V8SI
27279 #define MULTI_ARG_3_HI2 V16HI_FTYPE_V16HI_V16HI_V16HI
27280 #define MULTI_ARG_3_QI2 V32QI_FTYPE_V32QI_V32QI_V32QI
27281 #define MULTI_ARG_2_SF V4SF_FTYPE_V4SF_V4SF
27282 #define MULTI_ARG_2_DF V2DF_FTYPE_V2DF_V2DF
27283 #define MULTI_ARG_2_DI V2DI_FTYPE_V2DI_V2DI
27284 #define MULTI_ARG_2_SI V4SI_FTYPE_V4SI_V4SI
27285 #define MULTI_ARG_2_HI V8HI_FTYPE_V8HI_V8HI
27286 #define MULTI_ARG_2_QI V16QI_FTYPE_V16QI_V16QI
27287 #define MULTI_ARG_2_DI_IMM V2DI_FTYPE_V2DI_SI
27288 #define MULTI_ARG_2_SI_IMM V4SI_FTYPE_V4SI_SI
27289 #define MULTI_ARG_2_HI_IMM V8HI_FTYPE_V8HI_SI
27290 #define MULTI_ARG_2_QI_IMM V16QI_FTYPE_V16QI_SI
27291 #define MULTI_ARG_2_DI_CMP V2DI_FTYPE_V2DI_V2DI_CMP
27292 #define MULTI_ARG_2_SI_CMP V4SI_FTYPE_V4SI_V4SI_CMP
27293 #define MULTI_ARG_2_HI_CMP V8HI_FTYPE_V8HI_V8HI_CMP
27294 #define MULTI_ARG_2_QI_CMP V16QI_FTYPE_V16QI_V16QI_CMP
27295 #define MULTI_ARG_2_SF_TF V4SF_FTYPE_V4SF_V4SF_TF
27296 #define MULTI_ARG_2_DF_TF V2DF_FTYPE_V2DF_V2DF_TF
27297 #define MULTI_ARG_2_DI_TF V2DI_FTYPE_V2DI_V2DI_TF
27298 #define MULTI_ARG_2_SI_TF V4SI_FTYPE_V4SI_V4SI_TF
27299 #define MULTI_ARG_2_HI_TF V8HI_FTYPE_V8HI_V8HI_TF
27300 #define MULTI_ARG_2_QI_TF V16QI_FTYPE_V16QI_V16QI_TF
27301 #define MULTI_ARG_1_SF V4SF_FTYPE_V4SF
27302 #define MULTI_ARG_1_DF V2DF_FTYPE_V2DF
27303 #define MULTI_ARG_1_SF2 V8SF_FTYPE_V8SF
27304 #define MULTI_ARG_1_DF2 V4DF_FTYPE_V4DF
27305 #define MULTI_ARG_1_DI V2DI_FTYPE_V2DI
27306 #define MULTI_ARG_1_SI V4SI_FTYPE_V4SI
27307 #define MULTI_ARG_1_HI V8HI_FTYPE_V8HI
27308 #define MULTI_ARG_1_QI V16QI_FTYPE_V16QI
27309 #define MULTI_ARG_1_SI_DI V2DI_FTYPE_V4SI
27310 #define MULTI_ARG_1_HI_DI V2DI_FTYPE_V8HI
27311 #define MULTI_ARG_1_HI_SI V4SI_FTYPE_V8HI
27312 #define MULTI_ARG_1_QI_DI V2DI_FTYPE_V16QI
27313 #define MULTI_ARG_1_QI_SI V4SI_FTYPE_V16QI
27314 #define MULTI_ARG_1_QI_HI V8HI_FTYPE_V16QI
27317 {
27358 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v2di, "__builtin_ia32_vpcmov", IX86_BUILTIN_VPCMOV, UNKNOWN, (int)MULTI_ARG_3_DI },
27359 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v2di, "__builtin_ia32_vpcmov_v2di", IX86_BUILTIN_VPCMOV_V2DI, UNKNOWN, (int)MULTI_ARG_3_DI },
27360 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v4si, "__builtin_ia32_vpcmov_v4si", IX86_BUILTIN_VPCMOV_V4SI, UNKNOWN, (int)MULTI_ARG_3_SI },
27361 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v8hi, "__builtin_ia32_vpcmov_v8hi", IX86_BUILTIN_VPCMOV_V8HI, UNKNOWN, (int)MULTI_ARG_3_HI },
27362 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v16qi, "__builtin_ia32_vpcmov_v16qi",IX86_BUILTIN_VPCMOV_V16QI,UNKNOWN, (int)MULTI_ARG_3_QI },
27363 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v2df, "__builtin_ia32_vpcmov_v2df", IX86_BUILTIN_VPCMOV_V2DF, UNKNOWN, (int)MULTI_ARG_3_DF },
27364 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v4sf, "__builtin_ia32_vpcmov_v4sf", IX86_BUILTIN_VPCMOV_V4SF, UNKNOWN, (int)MULTI_ARG_3_SF },
27366 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v4di256, "__builtin_ia32_vpcmov256", IX86_BUILTIN_VPCMOV256, UNKNOWN, (int)MULTI_ARG_3_DI2 },
27367 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v4di256, "__builtin_ia32_vpcmov_v4di256", IX86_BUILTIN_VPCMOV_V4DI256, UNKNOWN, (int)MULTI_ARG_3_DI2 },
27368 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v8si256, "__builtin_ia32_vpcmov_v8si256", IX86_BUILTIN_VPCMOV_V8SI256, UNKNOWN, (int)MULTI_ARG_3_SI2 },
27369 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v16hi256, "__builtin_ia32_vpcmov_v16hi256", IX86_BUILTIN_VPCMOV_V16HI256, UNKNOWN, (int)MULTI_ARG_3_HI2 },
27370 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v32qi256, "__builtin_ia32_vpcmov_v32qi256", IX86_BUILTIN_VPCMOV_V32QI256, UNKNOWN, (int)MULTI_ARG_3_QI2 },
27371 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v4df256, "__builtin_ia32_vpcmov_v4df256", IX86_BUILTIN_VPCMOV_V4DF256, UNKNOWN, (int)MULTI_ARG_3_DF2 },
27372 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v8sf256, "__builtin_ia32_vpcmov_v8sf256", IX86_BUILTIN_VPCMOV_V8SF256, UNKNOWN, (int)MULTI_ARG_3_SF2 },
27374 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pperm, "__builtin_ia32_vpperm", IX86_BUILTIN_VPPERM, UNKNOWN, (int)MULTI_ARG_3_QI },
27376 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacssww, "__builtin_ia32_vpmacssww", IX86_BUILTIN_VPMACSSWW, UNKNOWN, (int)MULTI_ARG_3_HI },
27377 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacsww, "__builtin_ia32_vpmacsww", IX86_BUILTIN_VPMACSWW, UNKNOWN, (int)MULTI_ARG_3_HI },
27378 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacsswd, "__builtin_ia32_vpmacsswd", IX86_BUILTIN_VPMACSSWD, UNKNOWN, (int)MULTI_ARG_3_HI_SI },
27379 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacswd, "__builtin_ia32_vpmacswd", IX86_BUILTIN_VPMACSWD, UNKNOWN, (int)MULTI_ARG_3_HI_SI },
27380 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacssdd, "__builtin_ia32_vpmacssdd", IX86_BUILTIN_VPMACSSDD, UNKNOWN, (int)MULTI_ARG_3_SI },
27381 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacsdd, "__builtin_ia32_vpmacsdd", IX86_BUILTIN_VPMACSDD, UNKNOWN, (int)MULTI_ARG_3_SI },
27382 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacssdql, "__builtin_ia32_vpmacssdql", IX86_BUILTIN_VPMACSSDQL, UNKNOWN, (int)MULTI_ARG_3_SI_DI },
27383 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacssdqh, "__builtin_ia32_vpmacssdqh", IX86_BUILTIN_VPMACSSDQH, UNKNOWN, (int)MULTI_ARG_3_SI_DI },
27384 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacsdql, "__builtin_ia32_vpmacsdql", IX86_BUILTIN_VPMACSDQL, UNKNOWN, (int)MULTI_ARG_3_SI_DI },
27385 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacsdqh, "__builtin_ia32_vpmacsdqh", IX86_BUILTIN_VPMACSDQH, UNKNOWN, (int)MULTI_ARG_3_SI_DI },
27386 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmadcsswd, "__builtin_ia32_vpmadcsswd", IX86_BUILTIN_VPMADCSSWD, UNKNOWN, (int)MULTI_ARG_3_HI_SI },
27387 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmadcswd, "__builtin_ia32_vpmadcswd", IX86_BUILTIN_VPMADCSWD, UNKNOWN, (int)MULTI_ARG_3_HI_SI },
27389 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vrotlv2di3, "__builtin_ia32_vprotq", IX86_BUILTIN_VPROTQ, UNKNOWN, (int)MULTI_ARG_2_DI },
27390 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vrotlv4si3, "__builtin_ia32_vprotd", IX86_BUILTIN_VPROTD, UNKNOWN, (int)MULTI_ARG_2_SI },
27391 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vrotlv8hi3, "__builtin_ia32_vprotw", IX86_BUILTIN_VPROTW, UNKNOWN, (int)MULTI_ARG_2_HI },
27392 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vrotlv16qi3, "__builtin_ia32_vprotb", IX86_BUILTIN_VPROTB, UNKNOWN, (int)MULTI_ARG_2_QI },
27393 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_rotlv2di3, "__builtin_ia32_vprotqi", IX86_BUILTIN_VPROTQ_IMM, UNKNOWN, (int)MULTI_ARG_2_DI_IMM },
27394 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_rotlv4si3, "__builtin_ia32_vprotdi", IX86_BUILTIN_VPROTD_IMM, UNKNOWN, (int)MULTI_ARG_2_SI_IMM },
27395 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_rotlv8hi3, "__builtin_ia32_vprotwi", IX86_BUILTIN_VPROTW_IMM, UNKNOWN, (int)MULTI_ARG_2_HI_IMM },
27396 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_rotlv16qi3, "__builtin_ia32_vprotbi", IX86_BUILTIN_VPROTB_IMM, UNKNOWN, (int)MULTI_ARG_2_QI_IMM },
27397 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shav2di3, "__builtin_ia32_vpshaq", IX86_BUILTIN_VPSHAQ, UNKNOWN, (int)MULTI_ARG_2_DI },
27398 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shav4si3, "__builtin_ia32_vpshad", IX86_BUILTIN_VPSHAD, UNKNOWN, (int)MULTI_ARG_2_SI },
27399 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shav8hi3, "__builtin_ia32_vpshaw", IX86_BUILTIN_VPSHAW, UNKNOWN, (int)MULTI_ARG_2_HI },
27400 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shav16qi3, "__builtin_ia32_vpshab", IX86_BUILTIN_VPSHAB, UNKNOWN, (int)MULTI_ARG_2_QI },
27401 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shlv2di3, "__builtin_ia32_vpshlq", IX86_BUILTIN_VPSHLQ, UNKNOWN, (int)MULTI_ARG_2_DI },
27402 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shlv4si3, "__builtin_ia32_vpshld", IX86_BUILTIN_VPSHLD, UNKNOWN, (int)MULTI_ARG_2_SI },
27403 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shlv8hi3, "__builtin_ia32_vpshlw", IX86_BUILTIN_VPSHLW, UNKNOWN, (int)MULTI_ARG_2_HI },
27404 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shlv16qi3, "__builtin_ia32_vpshlb", IX86_BUILTIN_VPSHLB, UNKNOWN, (int)MULTI_ARG_2_QI },
27406 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vmfrczv4sf2, "__builtin_ia32_vfrczss", IX86_BUILTIN_VFRCZSS, UNKNOWN, (int)MULTI_ARG_2_SF },
27407 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vmfrczv2df2, "__builtin_ia32_vfrczsd", IX86_BUILTIN_VFRCZSD, UNKNOWN, (int)MULTI_ARG_2_DF },
27408 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_frczv4sf2, "__builtin_ia32_vfrczps", IX86_BUILTIN_VFRCZPS, UNKNOWN, (int)MULTI_ARG_1_SF },
27409 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_frczv2df2, "__builtin_ia32_vfrczpd", IX86_BUILTIN_VFRCZPD, UNKNOWN, (int)MULTI_ARG_1_DF },
27410 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_frczv8sf2, "__builtin_ia32_vfrczps256", IX86_BUILTIN_VFRCZPS256, UNKNOWN, (int)MULTI_ARG_1_SF2 },
27411 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_frczv4df2, "__builtin_ia32_vfrczpd256", IX86_BUILTIN_VFRCZPD256, UNKNOWN, (int)MULTI_ARG_1_DF2 },
27413 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddbw, "__builtin_ia32_vphaddbw", IX86_BUILTIN_VPHADDBW, UNKNOWN, (int)MULTI_ARG_1_QI_HI },
27414 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddbd, "__builtin_ia32_vphaddbd", IX86_BUILTIN_VPHADDBD, UNKNOWN, (int)MULTI_ARG_1_QI_SI },
27415 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddbq, "__builtin_ia32_vphaddbq", IX86_BUILTIN_VPHADDBQ, UNKNOWN, (int)MULTI_ARG_1_QI_DI },
27416 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddwd, "__builtin_ia32_vphaddwd", IX86_BUILTIN_VPHADDWD, UNKNOWN, (int)MULTI_ARG_1_HI_SI },
27417 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddwq, "__builtin_ia32_vphaddwq", IX86_BUILTIN_VPHADDWQ, UNKNOWN, (int)MULTI_ARG_1_HI_DI },
27418 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phadddq, "__builtin_ia32_vphadddq", IX86_BUILTIN_VPHADDDQ, UNKNOWN, (int)MULTI_ARG_1_SI_DI },
27419 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddubw, "__builtin_ia32_vphaddubw", IX86_BUILTIN_VPHADDUBW, UNKNOWN, (int)MULTI_ARG_1_QI_HI },
27420 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddubd, "__builtin_ia32_vphaddubd", IX86_BUILTIN_VPHADDUBD, UNKNOWN, (int)MULTI_ARG_1_QI_SI },
27421 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddubq, "__builtin_ia32_vphaddubq", IX86_BUILTIN_VPHADDUBQ, UNKNOWN, (int)MULTI_ARG_1_QI_DI },
27422 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phadduwd, "__builtin_ia32_vphadduwd", IX86_BUILTIN_VPHADDUWD, UNKNOWN, (int)MULTI_ARG_1_HI_SI },
27423 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phadduwq, "__builtin_ia32_vphadduwq", IX86_BUILTIN_VPHADDUWQ, UNKNOWN, (int)MULTI_ARG_1_HI_DI },
27424 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddudq, "__builtin_ia32_vphaddudq", IX86_BUILTIN_VPHADDUDQ, UNKNOWN, (int)MULTI_ARG_1_SI_DI },
27425 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phsubbw, "__builtin_ia32_vphsubbw", IX86_BUILTIN_VPHSUBBW, UNKNOWN, (int)MULTI_ARG_1_QI_HI },
27426 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phsubwd, "__builtin_ia32_vphsubwd", IX86_BUILTIN_VPHSUBWD, UNKNOWN, (int)MULTI_ARG_1_HI_SI },
27427 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phsubdq, "__builtin_ia32_vphsubdq", IX86_BUILTIN_VPHSUBDQ, UNKNOWN, (int)MULTI_ARG_1_SI_DI },
27429 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv16qi3, "__builtin_ia32_vpcomeqb", IX86_BUILTIN_VPCOMEQB, EQ, (int)MULTI_ARG_2_QI_CMP },
27430 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv16qi3, "__builtin_ia32_vpcomneb", IX86_BUILTIN_VPCOMNEB, NE, (int)MULTI_ARG_2_QI_CMP },
27431 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv16qi3, "__builtin_ia32_vpcomneqb", IX86_BUILTIN_VPCOMNEB, NE, (int)MULTI_ARG_2_QI_CMP },
27432 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv16qi3, "__builtin_ia32_vpcomltb", IX86_BUILTIN_VPCOMLTB, LT, (int)MULTI_ARG_2_QI_CMP },
27433 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv16qi3, "__builtin_ia32_vpcomleb", IX86_BUILTIN_VPCOMLEB, LE, (int)MULTI_ARG_2_QI_CMP },
27434 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv16qi3, "__builtin_ia32_vpcomgtb", IX86_BUILTIN_VPCOMGTB, GT, (int)MULTI_ARG_2_QI_CMP },
27435 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv16qi3, "__builtin_ia32_vpcomgeb", IX86_BUILTIN_VPCOMGEB, GE, (int)MULTI_ARG_2_QI_CMP },
27437 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv8hi3, "__builtin_ia32_vpcomeqw", IX86_BUILTIN_VPCOMEQW, EQ, (int)MULTI_ARG_2_HI_CMP },
27438 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv8hi3, "__builtin_ia32_vpcomnew", IX86_BUILTIN_VPCOMNEW, NE, (int)MULTI_ARG_2_HI_CMP },
27439 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv8hi3, "__builtin_ia32_vpcomneqw", IX86_BUILTIN_VPCOMNEW, NE, (int)MULTI_ARG_2_HI_CMP },
27440 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv8hi3, "__builtin_ia32_vpcomltw", IX86_BUILTIN_VPCOMLTW, LT, (int)MULTI_ARG_2_HI_CMP },
27441 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv8hi3, "__builtin_ia32_vpcomlew", IX86_BUILTIN_VPCOMLEW, LE, (int)MULTI_ARG_2_HI_CMP },
27442 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv8hi3, "__builtin_ia32_vpcomgtw", IX86_BUILTIN_VPCOMGTW, GT, (int)MULTI_ARG_2_HI_CMP },
27443 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv8hi3, "__builtin_ia32_vpcomgew", IX86_BUILTIN_VPCOMGEW, GE, (int)MULTI_ARG_2_HI_CMP },
27445 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv4si3, "__builtin_ia32_vpcomeqd", IX86_BUILTIN_VPCOMEQD, EQ, (int)MULTI_ARG_2_SI_CMP },
27446 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv4si3, "__builtin_ia32_vpcomned", IX86_BUILTIN_VPCOMNED, NE, (int)MULTI_ARG_2_SI_CMP },
27447 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv4si3, "__builtin_ia32_vpcomneqd", IX86_BUILTIN_VPCOMNED, NE, (int)MULTI_ARG_2_SI_CMP },
27448 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv4si3, "__builtin_ia32_vpcomltd", IX86_BUILTIN_VPCOMLTD, LT, (int)MULTI_ARG_2_SI_CMP },
27449 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv4si3, "__builtin_ia32_vpcomled", IX86_BUILTIN_VPCOMLED, LE, (int)MULTI_ARG_2_SI_CMP },
27450 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv4si3, "__builtin_ia32_vpcomgtd", IX86_BUILTIN_VPCOMGTD, GT, (int)MULTI_ARG_2_SI_CMP },
27451 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv4si3, "__builtin_ia32_vpcomged", IX86_BUILTIN_VPCOMGED, GE, (int)MULTI_ARG_2_SI_CMP },
27453 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv2di3, "__builtin_ia32_vpcomeqq", IX86_BUILTIN_VPCOMEQQ, EQ, (int)MULTI_ARG_2_DI_CMP },
27454 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv2di3, "__builtin_ia32_vpcomneq", IX86_BUILTIN_VPCOMNEQ, NE, (int)MULTI_ARG_2_DI_CMP },
27455 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv2di3, "__builtin_ia32_vpcomneqq", IX86_BUILTIN_VPCOMNEQ, NE, (int)MULTI_ARG_2_DI_CMP },
27456 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv2di3, "__builtin_ia32_vpcomltq", IX86_BUILTIN_VPCOMLTQ, LT, (int)MULTI_ARG_2_DI_CMP },
27457 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv2di3, "__builtin_ia32_vpcomleq", IX86_BUILTIN_VPCOMLEQ, LE, (int)MULTI_ARG_2_DI_CMP },
27458 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv2di3, "__builtin_ia32_vpcomgtq", IX86_BUILTIN_VPCOMGTQ, GT, (int)MULTI_ARG_2_DI_CMP },
27459 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv2di3, "__builtin_ia32_vpcomgeq", IX86_BUILTIN_VPCOMGEQ, GE, (int)MULTI_ARG_2_DI_CMP },
27461 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v16qi3,"__builtin_ia32_vpcomequb", IX86_BUILTIN_VPCOMEQUB, EQ, (int)MULTI_ARG_2_QI_CMP },
27462 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v16qi3,"__builtin_ia32_vpcomneub", IX86_BUILTIN_VPCOMNEUB, NE, (int)MULTI_ARG_2_QI_CMP },
27463 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v16qi3,"__builtin_ia32_vpcomnequb", IX86_BUILTIN_VPCOMNEUB, NE, (int)MULTI_ARG_2_QI_CMP },
27464 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv16qi3, "__builtin_ia32_vpcomltub", IX86_BUILTIN_VPCOMLTUB, LTU, (int)MULTI_ARG_2_QI_CMP },
27465 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv16qi3, "__builtin_ia32_vpcomleub", IX86_BUILTIN_VPCOMLEUB, LEU, (int)MULTI_ARG_2_QI_CMP },
27466 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv16qi3, "__builtin_ia32_vpcomgtub", IX86_BUILTIN_VPCOMGTUB, GTU, (int)MULTI_ARG_2_QI_CMP },
27467 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv16qi3, "__builtin_ia32_vpcomgeub", IX86_BUILTIN_VPCOMGEUB, GEU, (int)MULTI_ARG_2_QI_CMP },
27469 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v8hi3, "__builtin_ia32_vpcomequw", IX86_BUILTIN_VPCOMEQUW, EQ, (int)MULTI_ARG_2_HI_CMP },
27470 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v8hi3, "__builtin_ia32_vpcomneuw", IX86_BUILTIN_VPCOMNEUW, NE, (int)MULTI_ARG_2_HI_CMP },
27471 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v8hi3, "__builtin_ia32_vpcomnequw", IX86_BUILTIN_VPCOMNEUW, NE, (int)MULTI_ARG_2_HI_CMP },
27472 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv8hi3, "__builtin_ia32_vpcomltuw", IX86_BUILTIN_VPCOMLTUW, LTU, (int)MULTI_ARG_2_HI_CMP },
27473 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv8hi3, "__builtin_ia32_vpcomleuw", IX86_BUILTIN_VPCOMLEUW, LEU, (int)MULTI_ARG_2_HI_CMP },
27474 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv8hi3, "__builtin_ia32_vpcomgtuw", IX86_BUILTIN_VPCOMGTUW, GTU, (int)MULTI_ARG_2_HI_CMP },
27475 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv8hi3, "__builtin_ia32_vpcomgeuw", IX86_BUILTIN_VPCOMGEUW, GEU, (int)MULTI_ARG_2_HI_CMP },
27477 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v4si3, "__builtin_ia32_vpcomequd", IX86_BUILTIN_VPCOMEQUD, EQ, (int)MULTI_ARG_2_SI_CMP },
27478 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v4si3, "__builtin_ia32_vpcomneud", IX86_BUILTIN_VPCOMNEUD, NE, (int)MULTI_ARG_2_SI_CMP },
27479 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v4si3, "__builtin_ia32_vpcomnequd", IX86_BUILTIN_VPCOMNEUD, NE, (int)MULTI_ARG_2_SI_CMP },
27480 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv4si3, "__builtin_ia32_vpcomltud", IX86_BUILTIN_VPCOMLTUD, LTU, (int)MULTI_ARG_2_SI_CMP },
27481 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv4si3, "__builtin_ia32_vpcomleud", IX86_BUILTIN_VPCOMLEUD, LEU, (int)MULTI_ARG_2_SI_CMP },
27482 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv4si3, "__builtin_ia32_vpcomgtud", IX86_BUILTIN_VPCOMGTUD, GTU, (int)MULTI_ARG_2_SI_CMP },
27483 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv4si3, "__builtin_ia32_vpcomgeud", IX86_BUILTIN_VPCOMGEUD, GEU, (int)MULTI_ARG_2_SI_CMP },
27485 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v2di3, "__builtin_ia32_vpcomequq", IX86_BUILTIN_VPCOMEQUQ, EQ, (int)MULTI_ARG_2_DI_CMP },
27486 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v2di3, "__builtin_ia32_vpcomneuq", IX86_BUILTIN_VPCOMNEUQ, NE, (int)MULTI_ARG_2_DI_CMP },
27487 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v2di3, "__builtin_ia32_vpcomnequq", IX86_BUILTIN_VPCOMNEUQ, NE, (int)MULTI_ARG_2_DI_CMP },
27488 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv2di3, "__builtin_ia32_vpcomltuq", IX86_BUILTIN_VPCOMLTUQ, LTU, (int)MULTI_ARG_2_DI_CMP },
27489 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv2di3, "__builtin_ia32_vpcomleuq", IX86_BUILTIN_VPCOMLEUQ, LEU, (int)MULTI_ARG_2_DI_CMP },
27490 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv2di3, "__builtin_ia32_vpcomgtuq", IX86_BUILTIN_VPCOMGTUQ, GTU, (int)MULTI_ARG_2_DI_CMP },
27491 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv2di3, "__builtin_ia32_vpcomgeuq", IX86_BUILTIN_VPCOMGEUQ, GEU, (int)MULTI_ARG_2_DI_CMP },
27493 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv16qi3, "__builtin_ia32_vpcomfalseb", IX86_BUILTIN_VPCOMFALSEB, (enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_QI_TF },
27494 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv8hi3, "__builtin_ia32_vpcomfalsew", IX86_BUILTIN_VPCOMFALSEW, (enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_HI_TF },
27495 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv4si3, "__builtin_ia32_vpcomfalsed", IX86_BUILTIN_VPCOMFALSED, (enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_SI_TF },
27496 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv2di3, "__builtin_ia32_vpcomfalseq", IX86_BUILTIN_VPCOMFALSEQ, (enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_DI_TF },
27497 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv16qi3, "__builtin_ia32_vpcomfalseub",IX86_BUILTIN_VPCOMFALSEUB,(enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_QI_TF },
27498 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv8hi3, "__builtin_ia32_vpcomfalseuw",IX86_BUILTIN_VPCOMFALSEUW,(enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_HI_TF },
27499 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv4si3, "__builtin_ia32_vpcomfalseud",IX86_BUILTIN_VPCOMFALSEUD,(enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_SI_TF },
27500 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv2di3, "__builtin_ia32_vpcomfalseuq",IX86_BUILTIN_VPCOMFALSEUQ,(enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_DI_TF },
27502 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv16qi3, "__builtin_ia32_vpcomtrueb", IX86_BUILTIN_VPCOMTRUEB, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_QI_TF },
27503 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv8hi3, "__builtin_ia32_vpcomtruew", IX86_BUILTIN_VPCOMTRUEW, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_HI_TF },
27504 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv4si3, "__builtin_ia32_vpcomtrued", IX86_BUILTIN_VPCOMTRUED, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_SI_TF },
27505 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv2di3, "__builtin_ia32_vpcomtrueq", IX86_BUILTIN_VPCOMTRUEQ, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_DI_TF },
27506 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv16qi3, "__builtin_ia32_vpcomtrueub", IX86_BUILTIN_VPCOMTRUEUB, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_QI_TF },
27507 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv8hi3, "__builtin_ia32_vpcomtrueuw", IX86_BUILTIN_VPCOMTRUEUW, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_HI_TF },
27508 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv4si3, "__builtin_ia32_vpcomtrueud", IX86_BUILTIN_VPCOMTRUEUD, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_SI_TF },
27509 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv2di3, "__builtin_ia32_vpcomtrueuq", IX86_BUILTIN_VPCOMTRUEUQ, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_DI_TF },
27511 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vpermil2v2df3, "__builtin_ia32_vpermil2pd", IX86_BUILTIN_VPERMIL2PD, UNKNOWN, (int)MULTI_ARG_4_DF2_DI_I },
27512 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vpermil2v4sf3, "__builtin_ia32_vpermil2ps", IX86_BUILTIN_VPERMIL2PS, UNKNOWN, (int)MULTI_ARG_4_SF2_SI_I },
27513 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vpermil2v4df3, "__builtin_ia32_vpermil2pd256", IX86_BUILTIN_VPERMIL2PD256, UNKNOWN, (int)MULTI_ARG_4_DF2_DI_I1 },
27514 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vpermil2v8sf3, "__builtin_ia32_vpermil2ps256", IX86_BUILTIN_VPERMIL2PS256, UNKNOWN, (int)MULTI_ARG_4_SF2_SI_I1 },
27516 };
27517 \f
27518 /* TM vector builtins. */
27520 /* Reuse the existing x86-specific `struct builtin_description' cause
27521 we're lazy. Add casts to make them fit. */
27523 {
27524 { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_WM64", (enum ix86_builtins) BUILT_IN_TM_STORE_M64, UNKNOWN, VOID_FTYPE_PV2SI_V2SI },
27525 { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_WaRM64", (enum ix86_builtins) BUILT_IN_TM_STORE_WAR_M64, UNKNOWN, VOID_FTYPE_PV2SI_V2SI },
27526 { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_WaWM64", (enum ix86_builtins) BUILT_IN_TM_STORE_WAW_M64, UNKNOWN, VOID_FTYPE_PV2SI_V2SI },
27527 { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_RM64", (enum ix86_builtins) BUILT_IN_TM_LOAD_M64, UNKNOWN, V2SI_FTYPE_PCV2SI },
27528 { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_RaRM64", (enum ix86_builtins) BUILT_IN_TM_LOAD_RAR_M64, UNKNOWN, V2SI_FTYPE_PCV2SI },
27529 { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_RaWM64", (enum ix86_builtins) BUILT_IN_TM_LOAD_RAW_M64, UNKNOWN, V2SI_FTYPE_PCV2SI },
27530 { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_RfWM64", (enum ix86_builtins) BUILT_IN_TM_LOAD_RFW_M64, UNKNOWN, V2SI_FTYPE_PCV2SI },
27532 { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_WM128", (enum ix86_builtins) BUILT_IN_TM_STORE_M128, UNKNOWN, VOID_FTYPE_PV4SF_V4SF },
27533 { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_WaRM128", (enum ix86_builtins) BUILT_IN_TM_STORE_WAR_M128, UNKNOWN, VOID_FTYPE_PV4SF_V4SF },
27534 { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_WaWM128", (enum ix86_builtins) BUILT_IN_TM_STORE_WAW_M128, UNKNOWN, VOID_FTYPE_PV4SF_V4SF },
27535 { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_RM128", (enum ix86_builtins) BUILT_IN_TM_LOAD_M128, UNKNOWN, V4SF_FTYPE_PCV4SF },
27536 { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_RaRM128", (enum ix86_builtins) BUILT_IN_TM_LOAD_RAR_M128, UNKNOWN, V4SF_FTYPE_PCV4SF },
27537 { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_RaWM128", (enum ix86_builtins) BUILT_IN_TM_LOAD_RAW_M128, UNKNOWN, V4SF_FTYPE_PCV4SF },
27538 { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_RfWM128", (enum ix86_builtins) BUILT_IN_TM_LOAD_RFW_M128, UNKNOWN, V4SF_FTYPE_PCV4SF },
27540 { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_WM256", (enum ix86_builtins) BUILT_IN_TM_STORE_M256, UNKNOWN, VOID_FTYPE_PV8SF_V8SF },
27541 { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_WaRM256", (enum ix86_builtins) BUILT_IN_TM_STORE_WAR_M256, UNKNOWN, VOID_FTYPE_PV8SF_V8SF },
27542 { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_WaWM256", (enum ix86_builtins) BUILT_IN_TM_STORE_WAW_M256, UNKNOWN, VOID_FTYPE_PV8SF_V8SF },
27543 { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_RM256", (enum ix86_builtins) BUILT_IN_TM_LOAD_M256, UNKNOWN, V8SF_FTYPE_PCV8SF },
27544 { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_RaRM256", (enum ix86_builtins) BUILT_IN_TM_LOAD_RAR_M256, UNKNOWN, V8SF_FTYPE_PCV8SF },
27545 { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_RaWM256", (enum ix86_builtins) BUILT_IN_TM_LOAD_RAW_M256, UNKNOWN, V8SF_FTYPE_PCV8SF },
27546 { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_RfWM256", (enum ix86_builtins) BUILT_IN_TM_LOAD_RFW_M256, UNKNOWN, V8SF_FTYPE_PCV8SF },
27548 { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_LM64", (enum ix86_builtins) BUILT_IN_TM_LOG_M64, UNKNOWN, VOID_FTYPE_PCVOID },
27549 { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_LM128", (enum ix86_builtins) BUILT_IN_TM_LOG_M128, UNKNOWN, VOID_FTYPE_PCVOID },
27550 { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_LM256", (enum ix86_builtins) BUILT_IN_TM_LOG_M256, UNKNOWN, VOID_FTYPE_PCVOID },
27551 };
27553 /* TM callbacks. */
27555 /* Return the builtin decl needed to load a vector of TYPE. */
27557 static tree
27559 {
27561 {
27563 {
27570 }
27571 }
27573 }
27575 /* Return the builtin decl needed to store a vector of TYPE. */
27577 static tree
27579 {
27581 {
27583 {
27590 }
27591 }
27593 }
27594 \f
27595 /* Initialize the transactional memory vector load/store builtins. */
27597 static void
27599 {
27603 tree decl;
27610 /* If there are no builtins defined, we must be compiling in a
27611 language without trans-mem support. */
27615 /* Use whatever attributes a normal TM load has. */
27619 /* Use whatever attributes a normal TM store has. */
27623 /* Use whatever attributes a normal TM log has. */
27631 {
27635 {
27643 {
27646 }
27648 {
27651 }
27652 else
27653 {
27656 }
27658 /* The builtin without the prefix for
27659 calling it directly. */
27661 attrs);
27662 /* add_builtin_function() will set the DECL_ATTRIBUTES, now
27663 set the TYPE_ATTRIBUTES. */
27667 }
27668 }
27669 }
27671 /* Set up all the MMX/SSE builtins, even builtins for instructions that are not
27672 in the current target ISA to allow the user to compile particular modules
27673 with different target specific options that differ from the command line
27674 options. */
27675 static void
27677 {
27682 /* Add all special builtins with variable number of operands. */
27686 {
27692 }
27694 /* Add all builtins with variable number of operands. */
27698 {
27704 }
27706 /* pcmpestr[im] insns. */
27710 {
27713 else
27716 }
27718 /* pcmpistr[im] insns. */
27722 {
27725 else
27728 }
27730 /* comi/ucomi insns. */
27732 {
27735 else
27738 }
27740 /* SSE */
27746 /* SSE or 3DNow!A */
27749 IX86_BUILTIN_MASKMOVQ);
27751 /* SSE2 */
27760 /* SSE3. */
27766 /* AES */
27780 /* PCLMUL */
27784 /* RDRND */
27791 IX86_BUILTIN_RDRAND64_STEP);
27793 /* AVX2 */
27795 V2DF_FTYPE_V2DF_PCDOUBLE_V4SI_V2DF_INT,
27796 IX86_BUILTIN_GATHERSIV2DF);
27799 V4DF_FTYPE_V4DF_PCDOUBLE_V4SI_V4DF_INT,
27800 IX86_BUILTIN_GATHERSIV4DF);
27803 V2DF_FTYPE_V2DF_PCDOUBLE_V2DI_V2DF_INT,
27804 IX86_BUILTIN_GATHERDIV2DF);
27807 V4DF_FTYPE_V4DF_PCDOUBLE_V4DI_V4DF_INT,
27808 IX86_BUILTIN_GATHERDIV4DF);
27811 V4SF_FTYPE_V4SF_PCFLOAT_V4SI_V4SF_INT,
27812 IX86_BUILTIN_GATHERSIV4SF);
27815 V8SF_FTYPE_V8SF_PCFLOAT_V8SI_V8SF_INT,
27816 IX86_BUILTIN_GATHERSIV8SF);
27819 V4SF_FTYPE_V4SF_PCFLOAT_V2DI_V4SF_INT,
27820 IX86_BUILTIN_GATHERDIV4SF);
27823 V4SF_FTYPE_V4SF_PCFLOAT_V4DI_V4SF_INT,
27824 IX86_BUILTIN_GATHERDIV8SF);
27827 V2DI_FTYPE_V2DI_PCINT64_V4SI_V2DI_INT,
27828 IX86_BUILTIN_GATHERSIV2DI);
27831 V4DI_FTYPE_V4DI_PCINT64_V4SI_V4DI_INT,
27832 IX86_BUILTIN_GATHERSIV4DI);
27835 V2DI_FTYPE_V2DI_PCINT64_V2DI_V2DI_INT,
27836 IX86_BUILTIN_GATHERDIV2DI);
27839 V4DI_FTYPE_V4DI_PCINT64_V4DI_V4DI_INT,
27840 IX86_BUILTIN_GATHERDIV4DI);
27843 V4SI_FTYPE_V4SI_PCINT_V4SI_V4SI_INT,
27844 IX86_BUILTIN_GATHERSIV4SI);
27847 V8SI_FTYPE_V8SI_PCINT_V8SI_V8SI_INT,
27848 IX86_BUILTIN_GATHERSIV8SI);
27851 V4SI_FTYPE_V4SI_PCINT_V2DI_V4SI_INT,
27852 IX86_BUILTIN_GATHERDIV4SI);
27855 V4SI_FTYPE_V4SI_PCINT_V4DI_V4SI_INT,
27856 IX86_BUILTIN_GATHERDIV8SI);
27859 V4DF_FTYPE_V4DF_PCDOUBLE_V8SI_V4DF_INT,
27860 IX86_BUILTIN_GATHERALTSIV4DF);
27863 V8SF_FTYPE_V8SF_PCFLOAT_V4DI_V8SF_INT,
27864 IX86_BUILTIN_GATHERALTDIV8SF);
27867 V4DI_FTYPE_V4DI_PCINT64_V8SI_V4DI_INT,
27868 IX86_BUILTIN_GATHERALTSIV4DI);
27871 V8SI_FTYPE_V8SI_PCINT_V4DI_V8SI_INT,
27872 IX86_BUILTIN_GATHERALTDIV8SI);
27874 /* RTM. */
27878 /* MMX access to the vec_init patterns. */
27883 V4HI_FTYPE_HI_HI_HI_HI,
27884 IX86_BUILTIN_VEC_INIT_V4HI);
27887 V8QI_FTYPE_QI_QI_QI_QI_QI_QI_QI_QI,
27888 IX86_BUILTIN_VEC_INIT_V8QI);
27890 /* Access to the vec_extract patterns. */
27912 /* Access to the vec_set patterns. */
27933 /* Add FMA4 multi-arg argument instructions */
27935 {
27941 }
27942 }
27944 /* This builds the processor_model struct type defined in
27945 libgcc/config/i386/cpuinfo.c */
27947 static tree
27949 {
27956 /* The first 3 fields are unsigned int. */
27958 {
27964 }
27966 /* The last field is an array of unsigned integers of size one. */
27977 }
27979 /* Returns a extern, comdat VAR_DECL of type TYPE and name NAME. */
27981 static tree
27983 {
27984 tree new_decl;
27987 VAR_DECL,
27989 type);
28002 }
28004 /* FNDECL is a __builtin_cpu_is or a __builtin_cpu_supports call that is folded
28005 into an integer defined in libgcc/config/i386/cpuinfo.c */
28007 static tree
28009 {
28015 /* This is the order of bit-fields in __processor_features in cpuinfo.c */
28016 enum processor_features
28017 {
28019 F_MMX,
28020 F_POPCNT,
28021 F_SSE,
28022 F_SSE2,
28023 F_SSE3,
28024 F_SSSE3,
28025 F_SSE4_1,
28026 F_SSE4_2,
28027 F_AVX,
28028 F_AVX2,
28029 F_MAX
28030 };
28032 /* These are the values for vendor types and cpu types and subtypes
28033 in cpuinfo.c. Cpu types and subtypes should be subtracted by
28034 the corresponding start value. */
28035 enum processor_model
28036 {
28038 M_AMD,
28039 M_CPU_TYPE_START,
28040 M_INTEL_ATOM,
28041 M_INTEL_CORE2,
28042 M_INTEL_COREI7,
28043 M_AMDFAM10H,
28044 M_AMDFAM15H,
28045 M_CPU_SUBTYPE_START,
28046 M_INTEL_COREI7_NEHALEM,
28047 M_INTEL_COREI7_WESTMERE,
28048 M_INTEL_COREI7_SANDYBRIDGE,
28049 M_AMDFAM10H_BARCELONA,
28050 M_AMDFAM10H_SHANGHAI,
28051 M_AMDFAM10H_ISTANBUL,
28052 M_AMDFAM15H_BDVER1,
28053 M_AMDFAM15H_BDVER2
28054 };
28056 static struct _arch_names_table
28057 {
28060 }
28062 {
28078 };
28080 static struct _isa_names_table
28081 {
28084 }
28086 {
28098 };
28115 {
28116 /* *args must be a expr that can contain other EXPRS leading to a
28117 STRING_CST. */
28119 {
28122 }
28124 }
28129 {
28130 tree ref;
28131 tree field;
28133 unsigned int NUM_ARCH_NAMES
28142 {
28146 }
28151 /* CPU types are stored in the next field. */
28154 {
28157 }
28159 /* CPU subtypes are stored in the next field. */
28161 {
28164 }
28166 /* Get the appropriate field in __cpu_model. */
28170 /* Check the value. */
28173 }
28175 {
28176 tree ref;
28177 tree array_elt;
28178 tree field;
28180 unsigned int NUM_ISA_NAMES
28189 {
28193 }
28196 /* Get the last field, which is __cpu_features. */
28200 /* Get the appropriate field: __cpu_model.__cpu_features */
28204 /* Access the 0th element of __cpu_features array. */
28209 /* Return __cpu_model.__cpu_features[0] & field_val */
28212 }
28214 }
28216 static tree
28219 {
28221 {
28226 {
28229 }
28230 }
28232 #ifdef SUBTARGET_FOLD_BUILTIN
28234 #endif
28237 }
28239 /* Make builtins to detect cpu type and features supported. NAME is
28240 the builtin name, CODE is the builtin code, and FTYPE is the function
28241 type of the builtin. */
28243 static void
28246 {
28247 tree decl;
28248 tree type;
28256 }
28258 /* Make builtins to get CPU type and features supported. The created
28259 builtins are :
28261 __builtin_cpu_init (), to detect cpu type and features,
28262 __builtin_cpu_is ("<CPUNAME>"), to check if cpu is of type <CPUNAME>,
28263 __builtin_cpu_supports ("<FEATURE>"), to check if cpu supports <FEATURE>
28264 */
28266 static void
28268 {
28275 }
28277 /* Internal method for ix86_init_builtins. */
28279 static void
28281 {
28293 sysv_va_ref =
28296 fnvoid_va_end_ms =
28298 fnvoid_va_start_ms =
28300 fnvoid_va_end_sysv =
28302 fnvoid_va_start_sysv =
28304 NULL_TREE);
28305 fnvoid_va_copy_ms =
28307 NULL_TREE);
28308 fnvoid_va_copy_sysv =
28324 }
28326 static void
28328 {
28331 /* The __float80 type. */
28334 {
28335 /* The __float80 type. */
28340 }
28343 /* The __float128 type. */
28349 /* This macro is built by i386-builtin-types.awk. */
28350 DEFINE_BUILTIN_PRIMITIVE_TYPES;
28351 }
28353 static void
28355 {
28356 tree t;
28360 /* Builtins to get CPU type and features. */
28363 /* TFmode support builtins. */
28369 /* We will expand them to normal call if SSE isn't available since
28370 they are used by libgcc. */
28389 #ifdef SUBTARGET_INIT_BUILTINS
28390 SUBTARGET_INIT_BUILTINS;
28391 #endif
28392 }
28394 /* Return the ix86 builtin for CODE. */
28396 static tree
28398 {
28403 }
28405 /* Errors in the source file can cause expand_expr to return const0_rtx
28406 where we expect a vector. To avoid crashing, use one of the vector
28407 clear instructions. */
28408 static rtx
28410 {
28414 }
28416 /* Subroutine of ix86_expand_builtin to take care of binop insns. */
28418 static rtx
28420 {
28421 rtx pat;
28441 {
28445 }
28459 }
28461 /* Subroutine of ix86_expand_builtin to take care of 2-4 argument insns. */
28463 static rtx
28467 {
28468 rtx pat;
28476 rtx op;
28483 {
28563 }
28573 {
28580 {
28582 {
28585 {
28603 xop_rotl:
28605 {
28608 gcc_checking_assert
28610 }
28611 else
28612 {
28613 gcc_checking_assert
28624 }
28628 }
28629 }
28630 }
28631 else
28632 {
28633 non_constant:
28637 /* If we aren't optimizing, only allow one memory operand to be
28638 generated. */
28640 num_memory++;
28648 }
28652 }
28655 {
28666 else
28667 {
28673 }
28686 }
28693 }
28695 /* Subroutine of ix86_expand_args_builtin to take care of scalar unop
28696 insns with vec_merge. */
28698 static rtx
28700 rtx target)
28701 {
28702 rtx pat;
28729 }
28731 /* Subroutine of ix86_expand_builtin to take care of comparison insns. */
28733 static rtx
28736 {
28737 rtx pat;
28742 rtx op2;
28753 /* Swap operands if we have a comparison that isn't available in
28754 hardware. */
28756 {
28761 }
28781 }
28783 /* Subroutine of ix86_expand_builtin to take care of comi insns. */
28785 static rtx
28787 rtx target)
28788 {
28789 rtx pat;
28803 /* Swap operands if we have a comparison that isn't available in
28804 hardware. */
28806 {
28810 }
28831 const0_rtx)));
28834 }
28836 /* Subroutines of ix86_expand_args_builtin to take care of round insns. */
28838 static rtx
28840 rtx target)
28841 {
28842 rtx pat;
28867 }
28869 static rtx
28872 {
28873 rtx pat;
28878 rtx op2;
28905 }
28907 /* Subroutine of ix86_expand_builtin to take care of ptest insns. */
28909 static rtx
28911 rtx target)
28912 {
28913 rtx pat;
28946 const0_rtx)));
28949 }
28951 /* Subroutine of ix86_expand_builtin to take care of pcmpestr[im] insns. */
28953 static rtx
28956 {
28957 rtx pat;
28995 {
28998 }
29001 {
29010 }
29012 {
29021 }
29022 else
29023 {
29030 }
29038 {
29043 emit_insn
29047 FLAGS_REG),
29048 const0_rtx)));
29050 }
29051 else
29053 }
29056 /* Subroutine of ix86_expand_builtin to take care of pcmpistr[im] insns. */
29058 static rtx
29061 {
29062 rtx pat;
29090 {
29093 }
29096 {
29105 }
29107 {
29116 }
29117 else
29118 {
29125 }
29133 {
29138 emit_insn
29142 FLAGS_REG),
29143 const0_rtx)));
29145 }
29146 else
29148 }
29150 /* Subroutine of ix86_expand_builtin to take care of insns with
29151 variable number of operands. */
29153 static rtx
29156 {
29161 struct
29162 {
29163 rtx op;
29175 {
29450 }
29455 {
29458 }
29461 {
29468 }
29469 else
29470 {
29473 }
29476 {
29483 {
29484 /* SIMD shift insns take either an 8-bit immediate or
29485 register as count. But builtin functions take int as
29486 count. If count doesn't match, we put it in register. */
29488 {
29492 }
29493 }
29495 {
29498 {
29552 {
29555 {
29558 }
29564 }
29566 }
29567 }
29568 else
29569 {
29573 /* If we aren't optimizing, only allow one memory operand to
29574 be generated. */
29576 num_memory++;
29579 {
29582 }
29583 else
29584 {
29587 }
29588 }
29592 }
29595 {
29612 }
29619 }
29621 /* Subroutine of ix86_expand_builtin to take care of special insns
29622 with variable number of operands. */
29624 static rtx
29627 {
29628 tree arg;
29631 struct
29632 {
29633 rtx op;
29643 {
29693 /* Reserve memory operand for target. */
29724 /* Reserve memory operand for target. */
29738 }
29743 {
29748 {
29752 }
29753 else
29756 }
29757 else
29758 {
29765 }
29768 {
29777 {
29779 {
29785 else
29788 }
29789 }
29790 else
29791 {
29793 {
29794 /* This must be the memory operand. */
29800 }
29801 else
29802 {
29803 /* This must be register. */
29810 }
29811 }
29815 }
29818 {
29833 }
29839 }
29841 /* Return the integer constant in ARG. Constrain it to be in the range
29842 of the subparts of VEC_TYPE; issue an error if not. */
29844 static int
29846 {
29851 {
29854 }
29857 }
29859 /* A subroutine of ix86_expand_builtin. These builtins are a wrapper around
29860 ix86_expand_vector_init. We DO have language-level syntax for this, in
29861 the form of (type){ init-list }. Except that since we can't place emms
29862 instructions from inside the compiler, we can't allow the use of MMX
29863 registers unless the user explicitly asks for it. So we do *not* define
29864 vec_set/vec_extract/vec_init patterns for MMX modes in mmx.md. Instead
29865 we have builtins invoked by mmintrin.h that gives us license to emit
29866 these sorts of instructions. */
29868 static rtx
29870 {
29880 {
29883 }
29890 }
29892 /* A subroutine of ix86_expand_builtin. These builtins are a wrapper around
29893 ix86_expand_vector_extract. They would be redundant (for non-MMX) if we
29894 had a language-level syntax for referencing vector elements. */
29896 static rtx
29898 {
29902 rtx op0;
29922 }
29924 /* A subroutine of ix86_expand_builtin. These builtins are a wrapper around
29925 ix86_expand_vector_set. They would be redundant (for non-MMX) if we had
29926 a language-level syntax for referencing vector elements. */
29928 static rtx
29930 {
29954 /* OP0 is the source of these builtin functions and shouldn't be
29955 modified. Create a copy, use it and return it as target. */
29961 }
29963 /* Expand an expression EXP that calls a built-in function,
29964 with result going to TARGET if that's convenient
29965 (and in mode MODE if that's convenient).
29966 SUBTARGET may be used as the target for computing one of EXP's operands.
29967 IGNORE is nonzero if the value is to be ignored. */
29969 static rtx
29973 {
29983 /* For CPU builtins that can be folded, fold first and expand the fold. */
29985 {
29987 {
29988 /* Make it call __cpu_indicator_init in libgcc. */
29994 }
29997 {
30002 }
30003 }
30005 /* Determine whether the builtin function is available under the current ISA.
30006 Originally the builtin was not created if it wasn't applicable to the
30007 current ISA based on the command line switches. With function specific
30008 options, we need to check in the context of the function making the call
30009 whether it is supported. */
30012 {
30018 else
30019 {
30023 }
30025 }
30028 {
30032 ? CODE_FOR_mmx_maskmovq
30034 /* Note the arg order is different from the operand order. */
30078 {
30082 }
30095 {
30099 }
30144 {
30145 REAL_VALUE_TYPE inf;
30146 rtx tmp;
30158 }
30165 {
30169 }
30188 ? CODE_FOR_tbm_bextri_si
30191 {
30194 }
30195 else
30196 {
30205 }
30221 rdrand_step:
30228 {
30231 }
30237 /* Emit SImode conditional move. */
30239 {
30242 }
30245 else
30252 const0_rtx);
30318 gather_gen:
30329 /* Note the arg order is different from the operand order. */
30338 else
30343 {
30349 }
30352 {
30357 else
30367 }
30369 /* Force memory operand only with base register here. But we
30370 don't want to do it on memory operand for other builtin
30371 functions. */
30385 {
30388 }
30390 /* Optimize. If mask is known to have all high bits set,
30391 replace op0 with pc_rtx to signal that the instruction
30392 overwrites the whole destination and doesn't use its
30393 previous contents. */
30395 {
30397 {
30400 {
30404 negative++;
30407 negative++;
30408 }
30411 }
30413 {
30414 /* Recognize also when mask is like:
30415 __v2df src = _mm_setzero_pd ();
30416 __v2df mask = _mm_cmpeq_pd (src, src);
30417 or
30418 __v8sf src = _mm256_setzero_ps ();
30419 __v8sf mask = _mm256_cmp_ps (src, src, _CMP_EQ_OQ);
30420 as that is a cheaper way to load all ones into
30421 a register than having to load a constant from
30422 memory. */
30425 {
30430 {
30437 /* FALLTHRU */
30447 }
30448 }
30449 }
30450 }
30459 {
30466 else
30468 }
30469 else
30480 {
30483 }
30489 }
30502 {
30506 /* Emit a normal call if SSE isn't available. */
30510 }
30535 }
30537 /* Returns a function decl for a vectorized version of the builtin function
30538 with builtin function code FN and the result vector type TYPE, or NULL_TREE
30539 if it is not available. */
30541 static tree
30543 tree type_in)
30544 {
30560 {
30563 {
30568 }
30573 {
30578 }
30584 /* The round insn does not trap on denormals. */
30589 {
30594 }
30600 /* The round insn does not trap on denormals. */
30605 {
30610 }
30616 /* The round insn does not trap on denormals. */
30621 {
30626 }
30632 /* The round insn does not trap on denormals. */
30637 {
30642 }
30649 {
30654 }
30661 {
30666 }
30672 /* The round insn does not trap on denormals. */
30677 {
30682 }
30688 /* The round insn does not trap on denormals. */
30693 {
30698 }
30703 {
30708 }
30713 {
30718 }
30722 /* The round insn does not trap on denormals. */
30727 {
30732 }
30736 /* The round insn does not trap on denormals. */
30741 {
30746 }
30750 /* The round insn does not trap on denormals. */
30755 {
30760 }
30764 /* The round insn does not trap on denormals. */
30769 {
30774 }
30778 /* The round insn does not trap on denormals. */
30783 {
30788 }
30792 /* The round insn does not trap on denormals. */
30797 {
30802 }
30806 /* The round insn does not trap on denormals. */
30811 {
30816 }
30820 /* The round insn does not trap on denormals. */
30825 {
30830 }
30834 /* The round insn does not trap on denormals. */
30839 {
30844 }
30848 /* The round insn does not trap on denormals. */
30853 {
30858 }
30863 {
30868 }
30873 {
30878 }
30883 }
30885 /* Dispatch to a handler for a vectorization library. */
30888 type_in);
30891 }
30893 /* Handler for an SVML-style interface to
30894 a library with vectorized intrinsics. */
30896 static tree
30898 {
30906 /* The SVML is suitable for unsafe math only. */
30919 {
30966 }
30975 {
30978 }
30979 else
30982 /* Convert to uppercase. */
30987 args;
30989 arity++;
30993 else
30996 /* Build a function declaration for the vectorized function. */
31005 }
31007 /* Handler for an ACML-style interface to
31008 a library with vectorized intrinsics. */
31010 static tree
31012 {
31020 /* The ACML is 64bits only and suitable for unsafe math only as
31021 it does not correctly support parts of IEEE with the required
31022 precision such as denormals. */
31036 {
31066 }
31073 args;
31075 arity++;
31079 else
31082 /* Build a function declaration for the vectorized function. */
31091 }
31093 /* Returns a decl of a function that implements gather load with
31094 memory type MEM_VECTYPE and index type INDEX_VECTYPE and SCALE.
31095 Return NULL_TREE if it is not available. */
31097 static tree
31100 {
31116 /* v*gather* insn sign extends index to pointer mode. */
31128 {
31155 }
31158 }
31160 /* Returns a code for a target-specific builtin that implements
31161 reciprocal of the function, or NULL_TREE if not available. */
31163 static tree
31166 {
31173 /* Machine dependent builtins. */
31175 {
31176 /* Vectorized version of sqrt to rsqrt conversion. */
31185 }
31186 else
31187 /* Normal builtins. */
31189 {
31190 /* Sqrt to rsqrt conversion. */
31196 }
31197 }
31198 \f
31199 /* Helper for avx_vpermilps256_operand et al. This is also used by
31200 the expansion functions to turn the parallel back into a mask.
31201 The return value is 0 for no match and the imm8+1 for a match. */
31203 int
31205 {
31213 /* Validate that all of the elements are constants, and not totally
31214 out of range. Copy the data into an integral array to make the
31215 subsequent checks easier. */
31217 {
31227 }
31230 {
31232 /* In the 256-bit DFmode case, we can only move elements within
31233 a 128-bit lane. */
31235 {
31239 }
31241 {
31245 }
31249 /* In the 256-bit SFmode case, we have full freedom of movement
31250 within the low 128-bit lane, but the high 128-bit lane must
31251 mirror the exact same pattern. */
31256 /* FALLTHRU */
31260 /* In the 128-bit case, we've full freedom in the placement of
31261 the elements from the source operand. */
31268 }
31270 /* Make sure success has a non-zero value by adding one. */
31272 }
31274 /* Helper for avx_vperm2f128_v4df_operand et al. This is also used by
31275 the expansion functions to turn the parallel back into a mask.
31276 The return value is 0 for no match and the imm8+1 for a match. */
31278 int
31280 {
31288 /* Validate that all of the elements are constants, and not totally
31289 out of range. Copy the data into an integral array to make the
31290 subsequent checks easier. */
31292 {
31302 }
31304 /* Validate that the halves of the permute are halves. */
31312 /* Reconstruct the mask. */
31314 {
31320 }
31322 /* Make sure success has a non-zero value by adding one. */
31324 }
31325 \f
31326 /* Store OPERAND to the memory after reload is completed. This means
31327 that we can't easily use assign_stack_local. */
31328 rtx
31330 {
31331 rtx result;
31335 {
31338 stack_pointer_rtx,
31341 }
31343 {
31345 {
31349 /* FALLTHRU */
31355 stack_pointer_rtx)),
31356 operand));
31360 }
31362 }
31363 else
31364 {
31366 {
31368 {
31375 stack_pointer_rtx)),
31381 stack_pointer_rtx)),
31383 }
31386 /* Store HImodes as SImodes. */
31388 /* FALLTHRU */
31394 stack_pointer_rtx)),
31395 operand));
31399 }
31401 }
31403 }
31405 /* Free operand from the memory. */
31406 void
31408 {
31410 {
31415 else
31417 /* Use LEA to deallocate stack space. In peephole2 it will be converted
31418 to pop or add instruction if registers are available. */
31422 }
31423 }
31425 /* Implement TARGET_PREFERRED_RELOAD_CLASS.
31427 Put float CONST_DOUBLE in the constant pool instead of fp regs.
31428 QImode must go into class Q_REGS.
31429 Narrow ALL_REGS to GENERAL_REGS. This supports allowing movsf and
31430 movdf to do mem-to-mem moves through integer regs. */
31432 static reg_class_t
31434 {
31437 /* We're only allowed to return a subclass of CLASS. Many of the
31438 following checks fail for NO_REGS, so eliminate that early. */
31442 /* All classes can load zeros. */
31446 /* Force constants into memory if we are loading a (nonzero) constant into
31447 an MMX or SSE register. This is because there are no MMX/SSE instructions
31448 to load from a constant. */
31453 /* Prefer SSE regs only, if we can use them for math. */
31457 /* Floating-point constants need more complex checks. */
31459 {
31460 /* General regs can load everything. */
31464 /* Floats can load 0 and 1 plus some others. Note that we eliminated
31465 zero above. We only want to wind up preferring 80387 registers if
31466 we plan on doing computation with them. */
31469 {
31470 /* Limit class to non-sse. */
31479 }
31482 }
31484 /* Generally when we see PLUS here, it's the function invariant
31485 (plus soft-fp const_int). Which can only be computed into general
31486 regs. */
31490 /* QImode constants are easy to load, but non-constant QImode data
31491 must go into Q_REGS. */
31493 {
31499 }
31502 }
31504 /* Discourage putting floating-point values in SSE registers unless
31505 SSE math is being used, and likewise for the 387 registers. */
31506 static reg_class_t
31508 {
31511 /* Restrict the output reload class to the register bank that we are doing
31512 math on. If we would like not to return a subset of CLASS, reject this
31513 alternative: if reload cannot do this, it will still use its choice. */
31519 {
31524 else
31526 }
31529 }
31531 static reg_class_t
31534 {
31535 /* Double-word spills from general registers to non-offsettable memory
31536 references (zero-extended addresses) require special handling. */
31542 {
31544 ? CODE_FOR_reload_noff_load
31546 /* Add the cost of moving address to a temporary. */
31550 }
31552 /* QImode spills from non-QI registers require
31553 intermediate register on 32bit targets. */
31559 {
31564 else
31570 /* Return Q_REGS if the operand is in memory. */
31573 }
31575 /* This condition handles corner case where an expression involving
31576 pointers gets vectorized. We're trying to use the address of a
31577 stack slot as a vector initializer.
31579 (set (reg:V2DI 74 [ vect_cst_.2 ])
31580 (vec_duplicate:V2DI (reg/f:DI 20 frame)))
31582 Eventually frame gets turned into sp+offset like this:
31584 (set (reg:V2DI 21 xmm0 [orig:74 vect_cst_.2 ] [74])
31585 (vec_duplicate:V2DI (plus:DI (reg/f:DI 7 sp)
31586 (const_int 392 [0x188]))))
31588 That later gets turned into:
31590 (set (reg:V2DI 21 xmm0 [orig:74 vect_cst_.2 ] [74])
31591 (vec_duplicate:V2DI (plus:DI (reg/f:DI 7 sp)
31592 (mem/u/c/i:DI (symbol_ref/u:DI ("*.LC0") [flags 0x2]) [0 S8 A64]))))
31594 We'll have the following reload recorded:
31596 Reload 0: reload_in (DI) =
31597 (plus:DI (reg/f:DI 7 sp)
31598 (mem/u/c/i:DI (symbol_ref/u:DI ("*.LC0") [flags 0x2]) [0 S8 A64]))
31599 reload_out (V2DI) = (reg:V2DI 21 xmm0 [orig:74 vect_cst_.2 ] [74])
31600 SSE_REGS, RELOAD_OTHER (opnum = 0), can't combine
31601 reload_in_reg: (plus:DI (reg/f:DI 7 sp) (const_int 392 [0x188]))
31602 reload_out_reg: (reg:V2DI 21 xmm0 [orig:74 vect_cst_.2 ] [74])
31603 reload_reg_rtx: (reg:V2DI 22 xmm1)
31605 Which isn't going to work since SSE instructions can't handle scalar
31606 additions. Returning GENERAL_REGS forces the addition into integer
31607 register and reload can handle subsequent reloads without problems. */
31615 }
31617 /* Implement TARGET_CLASS_LIKELY_SPILLED_P. */
31619 static bool
31621 {
31623 {
31638 }
31641 }
31643 /* If we are copying between general and FP registers, we need a memory
31644 location. The same is true for SSE and MMX registers.
31646 To optimize register_move_cost performance, allow inline variant.
31648 The macro can't work reliably when one of the CLASSES is class containing
31649 registers from multiple units (SSE, MMX, integer). We avoid this by never
31650 combining those units in single alternative in the machine description.
31651 Ensure that this constraint holds to avoid unexpected surprises.
31653 When STRICT is false, we are being called from REGISTER_MOVE_COST, so do not
31654 enforce these sanity checks. */
31659 {
31666 {
31669 }
31674 /* ??? This is a lie. We do have moves between mmx/general, and for
31675 mmx/sse2. But by saying we need secondary memory we discourage the
31676 register allocator from using the mmx registers unless needed. */
31681 {
31682 /* SSE1 doesn't have any direct moves from other classes. */
31686 /* If the target says that inter-unit moves are more expensive
31687 than moving through memory, then don't generate them. */
31691 /* Between SSE and general, we have moves no larger than word size. */
31694 }
31697 }
31699 bool
31702 {
31704 }
31706 /* Implement the TARGET_CLASS_MAX_NREGS hook.
31708 On the 80386, this is the size of MODE in words,
31709 except in the FP regs, where a single reg is always enough. */
31711 static unsigned char
31713 {
31715 {
31720 else
31722 }
31723 else
31724 {
31727 else
31729 }
31730 }
31732 /* Return true if the registers in CLASS cannot represent the change from
31733 modes FROM to TO. */
31735 bool
31738 {
31742 /* x87 registers can't do subreg at all, as all values are reformatted
31743 to extended precision. */
31748 {
31749 /* Vector registers do not support QI or HImode loads. If we don't
31750 disallow a change to these modes, reload will assume it's ok to
31751 drop the subreg from (subreg:SI (reg:HI 100) 0). This affects
31752 the vec_dupv4hi pattern. */
31756 /* Vector registers do not support subreg with nonzero offsets, which
31757 are otherwise valid for integer registers. Since we can't see
31758 whether we have a nonzero offset from here, prohibit all
31759 nonparadoxical subregs changing size. */
31762 }
31765 }
31767 /* Return the cost of moving data of mode M between a
31768 register and memory. A value of 2 is the default; this cost is
31769 relative to those in `REGISTER_MOVE_COST'.
31771 This function is used extensively by register_move_cost that is used to
31772 build tables at startup. Make it inline in this case.
31773 When IN is 2, return maximum of in and out move cost.
31775 If moving between registers and memory is more expensive than
31776 between two registers, you should define this macro to express the
31777 relative cost.
31779 Model also increased moving costs of QImode registers in non
31780 Q_REGS classes.
31781 */
31785 {
31788 {
31791 {
31803 }
31807 }
31809 {
31812 {
31824 }
31828 }
31830 {
31833 {
31842 }
31846 }
31848 {
31851 {
31857 else
31862 }
31863 else
31864 {
31869 else
31871 }
31878 /* Compute number of 32bit moves needed. TFmode is moved as XFmode. */
31885 else
31889 }
31890 }
31892 static int
31895 {
31897 }
31900 /* Return the cost of moving data from a register in class CLASS1 to
31901 one in class CLASS2.
31903 It is not required that the cost always equal 2 when FROM is the same as TO;
31904 on some machines it is expensive to move between registers if they are not
31905 general registers. */
31907 static int
31909 reg_class_t class2_i)
31910 {
31914 /* In case we require secondary memory, compute cost of the store followed
31915 by load. In order to avoid bad register allocation choices, we need
31916 for this to be *at least* as high as the symmetric MEMORY_MOVE_COST. */
31919 {
31925 /* In case of copying from general_purpose_register we may emit multiple
31926 stores followed by single load causing memory size mismatch stall.
31927 Count this as arbitrarily high cost of 20. */
31932 /* In the case of FP/MMX moves, the registers actually overlap, and we
31933 have to switch modes in order to treat them differently. */
31939 }
31941 /* Moves between SSE/MMX and integer unit are expensive. */
31945 /* ??? By keeping returned value relatively high, we limit the number
31946 of moves between integer and MMX/SSE registers for all targets.
31947 Additionally, high value prevents problem with x86_modes_tieable_p(),
31948 where integer modes in MMX/SSE registers are not tieable
31949 because of missing QImode and HImode moves to, from or between
31950 MMX/SSE registers. */
31960 }
31962 /* Return TRUE if hard register REGNO can hold a value of machine-mode
31963 MODE. */
31965 bool
31967 {
31968 /* Flags and only flags can only hold CCmode values. */
31978 {
31979 /* We implement the move patterns for all vector modes into and
31980 out of SSE registers, even when no operation instructions
31981 are available. OImode move is available only when AVX is
31982 enabled. */
31989 }
31991 {
31992 /* We implement the move patterns for 3DNOW modes even in MMX mode,
31993 so if the register is available at all, then we can move data of
31994 the given mode into or out of it. */
31997 }
32000 {
32001 /* Take care for QImode values - they can be in non-QI regs,
32002 but then they do cause partial register stalls. */
32008 }
32009 /* We handle both integer and floats in the general purpose registers. */
32016 /* Lots of MMX code casts 8 byte vector modes to DImode. If we then go
32017 on to use that value in smaller contexts, this can easily force a
32018 pseudo to be allocated to GENERAL_REGS. Since this is no worse than
32019 supporting DImode, allow it. */
32024 }
32026 /* A subroutine of ix86_modes_tieable_p. Return true if MODE is a
32027 tieable integer mode. */
32029 static bool
32031 {
32033 {
32046 }
32047 }
32049 /* Return true if MODE1 is accessible in a register that can hold MODE2
32050 without copying. That is, all register classes that can hold MODE2
32051 can also hold MODE1. */
32053 bool
32055 {
32063 /* MODE2 being XFmode implies fp stack or general regs, which means we
32064 can tie any smaller floating point modes to it. Note that we do not
32065 tie this with TFmode. */
32069 /* MODE2 being DFmode implies fp stack, general or sse regs, which means
32070 that we can tie it with SFmode. */
32074 /* If MODE2 is only appropriate for an SSE register, then tie with
32075 any other mode acceptable to SSE registers. */
32085 /* If MODE2 is appropriate for an MMX register, then tie
32086 with any other mode acceptable to MMX registers. */
32093 }
32095 /* Return the cost of moving between two registers of mode MODE. */
32097 static int
32099 {
32103 {
32134 }
32136 /* Return the cost of moving between two registers of mode MODE,
32137 assuming that the move will be in pieces of at most UNITS bytes. */
32139 }
32141 /* Compute a (partial) cost for rtx X. Return true if the complete
32142 cost has been computed, and false if subexpressions should be
32143 scanned. In either case, *TOTAL contains the cost result. */
32145 static bool
32148 {
32155 {
32159 {
32162 }
32174 && (!TARGET_64BIT
32179 else
32185 {
32188 }
32190 {
32200 }
32202 {
32205 {
32214 }
32215 }
32216 /* Fall back to (MEM (SYMBOL_REF)), since that's where
32217 it'll probably end up. Add a penalty for size. */
32224 /* The zero extensions is often completely free on x86_64, so make
32225 it as cheap as possible. */
32231 else
32243 {
32246 {
32249 }
32252 {
32255 }
32256 }
32257 /* FALLTHRU */
32264 {
32265 /* ??? Should be SSE vector operation cost. */
32266 /* At least for published AMD latencies, this really is the same
32267 as the latency for a simple fpu operation like fabs. */
32268 /* V*QImode is emulated with 1-11 insns. */
32270 {
32273 {
32274 /* For XOP we use vpshab, which requires a broadcast of the
32275 value to the variable shift insn. For constants this
32276 means a V16Q const in mem; even when we can perform the
32277 shift with one insn set the cost to prefer paddb. */
32279 {
32284 }
32286 }
32290 }
32291 else
32293 }
32295 {
32297 {
32300 else
32302 }
32303 else
32304 {
32307 else
32309 }
32310 }
32311 else
32312 {
32315 else
32317 }
32321 {
32322 rtx sub;
32327 /* ??? SSE scalar/vector cost should be used here. */
32328 /* ??? Bald assumption that fma has the same cost as fmul. */
32332 /* Negate in op0 or op2 is free: FMS, FNMA, FNMS. */
32343 }
32347 {
32348 /* ??? SSE scalar cost should be used here. */
32351 }
32353 {
32356 }
32358 {
32359 /* ??? SSE vector cost should be used here. */
32362 }
32364 {
32365 /* V*QImode is emulated with 7-13 insns. */
32367 {
32374 }
32375 /* V*DImode is emulated with 5-8 insns. */
32377 {
32380 else
32382 }
32383 /* Without sse4.1, we don't have PMULLD; it's emulated with 7
32384 insns, including two PMULUDQ. */
32387 else
32390 }
32391 else
32392 {
32397 {
32400 nbits++;
32401 }
32402 else
32403 /* This is arbitrary. */
32406 /* Compute costs correctly for widening multiplication. */
32410 {
32417 {
32421 else
32423 }
32427 }
32435 }
32442 /* ??? SSE cost should be used here. */
32447 /* ??? SSE vector cost should be used here. */
32449 else
32456 {
32461 {
32464 {
32472 }
32473 }
32476 {
32479 {
32485 }
32486 }
32488 {
32496 }
32497 }
32498 /* FALLTHRU */
32502 {
32503 /* ??? SSE cost should be used here. */
32506 }
32508 {
32511 }
32513 {
32514 /* ??? SSE vector cost should be used here. */
32517 }
32518 /* FALLTHRU */
32524 {
32531 }
32532 /* FALLTHRU */
32536 {
32537 /* ??? SSE cost should be used here. */
32540 }
32542 {
32545 }
32547 {
32548 /* ??? SSE vector cost should be used here. */
32551 }
32552 /* FALLTHRU */
32556 {
32557 /* ??? Should be SSE vector operation cost. */
32558 /* At least for published AMD latencies, this really is the same
32559 as the latency for a simple fpu operation like fabs. */
32561 }
32564 else
32573 {
32574 /* This kind of construct is implemented using test[bwl].
32575 Treat it as if we had an AND. */
32580 }
32590 /* ??? SSE cost should be used here. */
32595 /* ??? SSE vector cost should be used here. */
32601 /* ??? SSE cost should be used here. */
32606 /* ??? SSE vector cost should be used here. */
32619 /* ??? Assume all of these vector manipulation patterns are
32620 recognizable. In which case they all pretty much have the
32621 same cost. */
32627 }
32628 }
32630 #if TARGET_MACHO
32634 /* Given a symbol name and its associated stub, write out the
32635 definition of the stub. */
32637 void
32639 {
32644 /* For 64-bit we shouldn't get here. */
32647 /* Lose our funky encoding stuff so it doesn't contaminate the stub. */
32664 else
32671 {
32673 }
32675 {
32676 /* PIC stub. */
32677 /* 25-byte PIC stub using "CALL get_pc_thunk". */
32683 }
32684 else
32687 /* The AT&T-style ("self-modifying") stub is not lazily bound, thus
32688 it needs no stub-binding-helper. */
32695 {
32698 }
32699 else
32704 /* N.B. Keep the correspondence of these
32705 'symbol_ptr/symbol_ptr2/symbol_ptr3' sections consistent with the
32706 old-pic/new-pic/non-pic stubs; altering this will break
32707 compatibility with existing dylibs. */
32709 {
32710 /* 25-byte PIC stub using "CALL get_pc_thunk". */
32712 }
32713 else
32714 /* 16-byte -mdynamic-no-pic stub. */
32720 }
32723 /* Order the registers for register allocator. */
32725 void
32727 {
32731 /* First allocate the local general purpose registers. */
32736 /* Global general purpose registers. */
32741 /* x87 registers come first in case we are doing FP math
32742 using them. */
32747 /* SSE registers. */
32753 /* x87 registers. */
32761 /* Initialize the rest of array as we do not allocate some registers
32762 at all. */
32765 }
32767 /* Handle a "callee_pop_aggregate_return" attribute; arguments as
32768 in struct attribute_spec handler. */
32769 static tree
32771 tree args,
32774 {
32779 {
32781 name);
32784 }
32786 {
32788 name);
32791 }
32793 {
32794 tree cst;
32798 {
32801 name);
32803 }
32806 {
32809 name);
32811 }
32814 }
32817 }
32819 /* Handle a "ms_abi" or "sysv" attribute; arguments as in
32820 struct attribute_spec.handler. */
32821 static tree
32823 tree args ATTRIBUTE_UNUSED,
32825 {
32830 {
32832 name);
32835 }
32837 /* Can combine regparm with all attributes but fastcall. */
32839 {
32841 {
32843 }
32846 }
32848 {
32850 {
32852 }
32855 }
32858 }
32860 /* Handle a "ms_struct" or "gcc_struct" attribute; arguments as in
32861 struct attribute_spec.handler. */
32862 static tree
32864 tree args ATTRIBUTE_UNUSED,
32866 {
32869 {
32872 }
32873 else
32877 {
32879 name);
32881 }
32887 {
32889 name);
32891 }
32894 }
32896 static tree
32898 tree args ATTRIBUTE_UNUSED,
32900 {
32902 {
32904 name);
32906 }
32908 }
32910 static bool
32912 {
32916 }
32918 /* Returns an expression indicating where the this parameter is
32919 located on entry to the FUNCTION. */
32921 static rtx
32923 {
32929 {
32934 else
32937 }
32942 {
32949 {
32954 }
32955 else
32956 {
32959 {
32965 }
32966 }
32968 }
32972 }
32974 /* Determine whether x86_output_mi_thunk can succeed. */
32976 static bool
32978 HOST_WIDE_INT delta ATTRIBUTE_UNUSED,
32980 {
32981 /* 64-bit can handle anything. */
32985 /* For 32-bit, everything's fine if we have one free register. */
32989 /* Need a free register for vcall_offset. */
32993 /* Need a free register for GOT references. */
32997 /* Otherwise ok. */
32999 }
33001 /* Output the assembler code for a thunk function. THUNK_DECL is the
33002 declaration for the thunk function itself, FUNCTION is the decl for
33003 the target function. DELTA is an immediate constant offset to be
33004 added to THIS. If VCALL_OFFSET is nonzero, the word at
33005 *(*this + vcall_offset) should be added to THIS. */
33007 static void
33011 {
33018 else
33019 {
33023 else
33025 }
33029 /* If VCALL_OFFSET, we'll need THIS in a register. Might as well
33030 pull it in now and let DELTA benefit. */
33034 {
33035 /* Put the this parameter into %eax. */
33038 }
33039 else
33042 /* Adjust the this parameter by a fixed constant. */
33044 {
33049 {
33051 {
33055 }
33056 }
33059 }
33061 /* Adjust the this parameter by a value stored in the vtable. */
33063 {
33073 /* Adjust the this parameter. */
33077 {
33081 }
33088 vcall_mem));
33089 else
33091 }
33093 /* If necessary, drop THIS back to its stack slot. */
33099 {
33102 ;
33103 else
33104 {
33108 }
33109 }
33110 else
33111 {
33113 ;
33114 #if TARGET_MACHO
33116 {
33119 }
33121 else
33122 {
33129 }
33130 }
33132 /* Our sibling call patterns do not allow memories, because we have no
33133 predicate that can distinguish between frame and non-frame memory.
33134 For our purposes here, we can get away with (ab)using a jump pattern,
33135 because we're going to do no optimization. */
33138 else
33139 {
33145 {
33151 }
33157 }
33160 /* Emit just enough of rest_of_compilation to get the insns emitted.
33161 Note that use_thunk calls assemble_start_function et al. */
33168 }
33170 static void
33172 {
33174 #if TARGET_MACHO
33176 #endif
33183 }
33185 int
33187 {
33199 }
33201 /* Output assembler code to FILE to increment profiler label # LABELNO
33202 for profiling a function entry. */
33203 void
33205 {
33210 {
33211 #ifndef NO_PROFILE_COUNTERS
33213 #endif
33217 else
33219 }
33221 {
33222 #ifndef NO_PROFILE_COUNTERS
33225 #endif
33227 }
33228 else
33229 {
33230 #ifndef NO_PROFILE_COUNTERS
33233 #endif
33235 }
33236 }
33238 /* We don't have exact information about the insn sizes, but we may assume
33239 quite safely that we are informed about all 1 byte insns and memory
33240 address sizes. This is enough to eliminate unnecessary padding in
33241 99% of cases. */
33243 static int
33245 {
33251 /* Discard alignments we've emit and jump instructions. */
33258 /* Important case - calls are always 5 bytes.
33259 It is common to have many calls in the row. */
33268 /* For normal instructions we rely on get_attr_length being exact,
33269 with a few exceptions. */
33271 {
33275 {
33285 /* Otherwise trust get_attr_length. */
33287 }
33292 }
33295 else
33297 }
33299 #ifdef ASM_OUTPUT_MAX_SKIP_PAD
33301 /* AMD K8 core mispredicts jumps when there are more than 3 jumps in 16 byte
33302 window. */
33304 static void
33306 {
33311 /* Look for all minimal intervals of instructions containing 4 jumps.
33312 The intervals are bounded by START and INSN. NBYTES is the total
33313 size of instructions in the interval including INSN and not including
33314 START. When the NBYTES is smaller than 16 bytes, it is possible
33315 that the end of START and INSN ends up in the same 16byte page.
33317 The smallest offset in the page INSN can start is the case where START
33318 ends on the offset 0. Offset of INSN is then NBYTES - sizeof (INSN).
33319 We add p2align to 16byte window with maxskip 15 - NBYTES + sizeof (INSN).
33320 */
33322 {
33326 {
33332 /* If align > 3, only up to 16 - max_skip - 1 bytes can be
33333 already in the current 16 byte page, because otherwise
33334 ASM_OUTPUT_MAX_SKIP_ALIGN could skip max_skip or fewer
33335 bytes to reach 16 byte boundary. */
33343 {
33345 {
33352 else
33355 }
33356 }
33358 }
33369 njumps++;
33370 else
33374 {
33381 else
33384 }
33391 {
33398 }
33399 }
33400 }
33401 #endif
33403 /* AMD Athlon works faster
33404 when RET is not destination of conditional jump or directly preceded
33405 by other jump instruction. We avoid the penalty by inserting NOP just
33406 before the RET instructions in such cases. */
33407 static void
33409 {
33410 edge e;
33411 edge_iterator ei;
33414 {
33417 rtx prev;
33427 {
33428 edge e;
33429 edge_iterator ei;
33435 }
33437 {
33443 /* Empty functions get branch mispredict even when
33444 the jump destination is not visible to us. */
33447 }
33449 {
33452 }
33453 }
33454 }
33456 /* Count the minimum number of instructions in BB. Return 4 if the
33457 number of instructions >= 4. */
33459 static int
33461 {
33462 rtx insn;
33465 /* Count number of instructions in this block. Return 4 if the number
33466 of instructions >= 4. */
33468 {
33469 /* Only happen in exit blocks. */
33477 {
33478 insn_count++;
33481 }
33482 }
33485 }
33488 /* Count the minimum number of instructions in code path in BB.
33489 Return 4 if the number of instructions >= 4. */
33491 static int
33493 {
33494 edge e;
33495 edge_iterator ei;
33498 /* Only bother counting instructions along paths with no
33499 more than 2 basic blocks between entry and exit. Given
33500 that BB has an edge to exit, determine if a predecessor
33501 of BB has an edge from entry. If so, compute the number
33502 of instructions in the predecessor block. If there
33503 happen to be multiple such blocks, compute the minimum. */
33506 {
33507 edge prev_e;
33508 edge_iterator prev_ei;
33511 {
33514 }
33516 {
33518 {
33523 }
33524 }
33525 }
33531 }
33533 /* Pad short function to 4 instructions. */
33535 static void
33537 {
33538 edge e;
33539 edge_iterator ei;
33542 {
33545 {
33548 /* Pad short function. */
33550 {
33553 /* Find epilogue. */
33562 /* Two NOPs count as one instruction. */
33565 }
33566 }
33567 }
33568 }
33570 /* Implement machine specific optimizations. We implement padding of returns
33571 for K8 CPUs and pass to avoid 4 jumps in the single 16 byte window. */
33572 static void
33574 {
33575 /* We are freeing block_for_insn in the toplev to keep compatibility
33576 with old MDEP_REORGS that are not CFG based. Recompute it now. */
33579 /* Run the vzeroupper optimization if needed. */
33584 {
33589 #ifdef ASM_OUTPUT_MAX_SKIP_PAD
33592 #endif
33593 }
33594 }
33596 /* Return nonzero when QImode register that must be represented via REX prefix
33597 is used. */
33598 bool
33600 {
33608 }
33610 /* Return nonzero when P points to register encoded via REX prefix.
33611 Called via for_each_rtx. */
33612 static int
33614 {
33620 }
33622 /* Return true when INSN mentions register that must be encoded using REX
33623 prefix. */
33624 bool
33626 {
33629 }
33631 /* If profitable, negate (without causing overflow) integer constant
33632 of mode MODE at location LOC. Return true in this case. */
33633 bool
33635 {
33636 HOST_WIDE_INT val;
33642 {
33644 /* DImode x86_64 constants must fit in 32 bits. */
33657 }
33659 /* Avoid overflows. */
33665 /* Make things pretty and `subl $4,%eax' rather than `addl $-4,%eax'.
33666 Exceptions: -128 encodes smaller than 128, so swap sign and op. */
33669 {
33672 }
33675 }
33677 /* Generate an unsigned DImode/SImode to FP conversion. This is the same code
33678 optabs would emit if we didn't have TFmode patterns. */
33680 void
33682 {
33716 }
33717 \f
33718 /* AVX2 does support 32-byte integer vector operations,
33719 thus the longest vector we are faced with is V32QImode. */
33720 #define MAX_VECT_LEN 32
33722 struct expand_vec_perm_d
33723 {
33730 };
33736 /* Get a vector mode of the same size as the original but with elements
33737 twice as wide. This is only guaranteed to apply to integral vectors. */
33741 {
33742 /* ??? Rely on the ordering that genmodes.c gives to vectors. */
33747 }
33749 /* A subroutine of ix86_expand_vector_init. Store into TARGET a vector
33750 with all elements equal to VAR. Return true if successful. */
33752 static bool
33755 {
33759 {
33764 /* FALLTHRU */
33774 {
33777 /* First attempt to recognize VAL as-is. */
33781 {
33782 rtx seq;
33783 /* If that fails, force VAL into a register. */
33794 }
33795 }
33802 {
33803 rtx x;
33810 }
33820 {
33824 permute:
33832 /* Extend to SImode using a paradoxical SUBREG. */
33836 /* Insert the SImode value as low element of a V4SImode vector. */
33844 }
33852 widen:
33853 /* Replicate the value once into the next wider mode and recurse. */
33854 {
33856 rtx x;
33872 }
33876 {
33885 }
33890 }
33891 }
33893 /* A subroutine of ix86_expand_vector_init. Store into TARGET a vector
33894 whose ONE_VAR element is VAR, and other elements are zero. Return true
33895 if successful. */
33897 static bool
33900 {
33902 rtx new_target;
33907 {
33909 /* For SSE4.1, we normally use vector set. But if the second
33910 element is zero and inter-unit moves are OK, we use movq
33911 instead. */
33912 use_vector_set = (TARGET_64BIT
33913 && TARGET_SSE4_1
33914 && !(TARGET_INTER_UNIT_MOVES
33936 /* Use ix86_expand_vector_set in 64bit mode only. */
33941 }
33944 {
33949 }
33952 {
33957 /* FALLTHRU */
33972 else
33979 {
33980 /* We need to shuffle the value to the correct position, so
33981 create a new pseudo to store the intermediate result. */
33983 /* With SSE2, we can use the integer shuffle insns. */
33985 {
33987 const1_rtx,
33994 }
33996 /* Otherwise convert the intermediate result to V4SFmode and
33997 use the SSE1 shuffle instructions. */
33999 {
34002 }
34003 else
34007 const1_rtx,
34016 }
34031 widen:
34035 /* Zero extend the variable element to SImode and recurse. */
34048 }
34049 }
34051 /* A subroutine of ix86_expand_vector_init. Store into TARGET a vector
34052 consisting of the values in VALS. It is known that all elements
34053 except ONE_VAR are constants. Return true if successful. */
34055 static bool
34058 {
34068 {
34073 /* For the two element vectors, it's just as easy to use
34074 the general case. */
34078 /* Use ix86_expand_vector_set in 64bit mode only. */
34100 widen:
34101 /* There's no way to set one QImode entry easily. Combine
34102 the variable value with its adjacent constant value, and
34103 promote to an HImode set. */
34106 {
34111 }
34112 else
34113 {
34116 }
34130 }
34135 }
34137 /* A subroutine of ix86_expand_vector_init_general. Use vector
34138 concatenate to handle the most general case: all values variable,
34139 and none identical. */
34141 static void
34144 {
34147 rtvec v;
34151 {
34154 {
34187 }
34200 {
34215 }
34220 {
34231 }
34234 half:
34235 /* FIXME: We process inputs backward to help RA. PR 36222. */
34239 {
34244 }
34248 {
34251 {
34255 }
34258 }
34259 else
34265 }
34266 }
34268 /* A subroutine of ix86_expand_vector_init_general. Use vector
34269 interleave to handle the most general case: all values variable,
34270 and none identical. */
34272 static void
34275 {
34284 {
34305 }
34308 {
34309 /* Extend the odd elment to SImode using a paradoxical SUBREG. */
34313 /* Insert the SImode value as low element of V4SImode vector. */
34317 op0),
34319 const1_rtx);
34322 /* Cast the V4SImode vector back to a vector in orignal mode. */
34326 /* Load even elements into the second positon. */
34330 const1_rtx));
34332 /* Cast vector to FIRST_IMODE vector. */
34335 }
34337 /* Interleave low FIRST_IMODE vectors. */
34339 {
34343 /* Cast FIRST_IMODE vector to SECOND_IMODE vector. */
34346 }
34348 /* Interleave low SECOND_IMODE vectors. */
34350 {
34353 {
34358 /* Cast the SECOND_IMODE vector to the THIRD_IMODE
34359 vector. */
34362 }
34365 /* FALLTHRU */
34372 /* Cast the SECOND_IMODE vector back to a vector on original
34373 mode. */
34380 }
34381 }
34383 /* A subroutine of ix86_expand_vector_init. Handle the most general case:
34384 all values variable, and none identical. */
34386 static void
34389 {
34395 {
34400 /* FALLTHRU */
34424 half:
34441 /* FALLTHRU */
34447 /* Don't use ix86_expand_vector_init_interleave if we can't
34448 move from GPR to SSE register directly. */
34464 }
34466 {
34478 {
34482 {
34488 else
34489 {
34494 }
34495 }
34498 }
34503 {
34509 }
34511 {
34517 }
34518 else
34520 }
34521 }
34523 /* Initialize vector TARGET via VALS. Suppress the use of MMX
34524 instructions unless MMX_OK is true. */
34526 void
34528 {
34535 rtx x;
34538 {
34548 }
34550 /* Constants are best loaded from the constant pool. */
34552 {
34555 }
34557 /* If all values are identical, broadcast the value. */
34563 /* Values where only one field is non-constant are best loaded from
34564 the pool and overwritten via move later. */
34566 {
34570 one_var))
34575 }
34578 }
34580 void
34582 {
34587 rtx tmp;
34589 = {
34596 };
34598 = {
34605 };
34609 {
34613 {
34618 else
34622 }
34634 else
34640 {
34643 /* For the two element vectors, we implement a VEC_CONCAT with
34644 the extraction of the other element. */
34651 else
34656 }
34665 {
34671 /* tmp = target = A B C D */
34673 /* target = A A B B */
34675 /* target = X A B B */
34677 /* target = A X C D */
34684 /* tmp = target = A B C D */
34686 /* tmp = X B C D */
34688 /* target = A B X D */
34695 /* tmp = target = A B C D */
34697 /* tmp = X B C D */
34699 /* target = A B X D */
34707 }
34715 /* Element 0 handled by vec_merge below. */
34717 {
34720 }
34723 {
34724 /* With SSE2, use integer shuffles to swap element 0 and ELT,
34725 store into element 0, then shuffle them back. */
34742 }
34743 else
34744 {
34745 /* For SSE1, we have to reuse the V4SF code. */
34748 }
34801 half:
34802 /* Compute offset. */
34808 /* Extract the half. */
34812 /* Put val in tmp at elt. */
34815 /* Put it back. */
34821 }
34824 {
34828 }
34829 else
34830 {
34839 }
34840 }
34842 void
34844 {
34848 rtx tmp;
34851 {
34856 /* FALLTHRU */
34869 {
34889 }
34901 {
34903 {
34923 }
34927 }
34928 else
34929 {
34930 /* For SSE1, we have to reuse the V4SF code. */
34934 }
34950 {
34954 else
34958 }
34963 {
34967 else
34971 }
34976 {
34980 else
34984 }
34989 {
34993 else
34997 }
35002 {
35006 else
35010 }
35015 {
35019 else
35023 }
35027 /* ??? Could extract the appropriate HImode element and shift. */
35030 }
35033 {
35037 /* Let the rtl optimizers know about the zero extension performed. */
35039 {
35042 }
35045 }
35046 else
35047 {
35054 }
35055 }
35057 /* Generate code to copy vector bits i / 2 ... i - 1 from vector SRC
35058 to bits 0 ... i / 2 - 1 of vector DEST, which has the same mode.
35059 The upper bits of DEST are undefined, though they shouldn't cause
35060 exceptions (some bits from src or all zeros are ok). */
35062 static void
35064 {
35065 rtx tem;
35067 {
35071 else
35089 else
35096 else
35107 const1_rtx);
35108 else
35115 }
35117 }
35119 /* Expand a vector reduction. FN is the binary pattern to reduce;
35120 DEST is the destination; IN is the input vector. */
35122 void
35124 {
35129 /* SSE4 has a special instruction for V8HImode UMIN reduction. */
35133 {
35136 }
35141 {
35146 else
35150 }
35151 }
35152 \f
35153 /* Target hook for scalar_mode_supported_p. */
35154 static bool
35156 {
35161 else
35163 }
35165 /* Implements target hook vector_mode_supported_p. */
35166 static bool
35168 {
35180 }
35182 /* Target hook for c_mode_for_suffix. */
35183 static enum machine_mode
35185 {
35192 }
35194 /* Worker function for TARGET_MD_ASM_CLOBBERS.
35196 We do this in the new i386 backend to maintain source compatibility
35197 with the old cc0-based compiler. */
35199 static tree
35201 tree inputs ATTRIBUTE_UNUSED,
35202 tree clobbers)
35203 {
35205 clobbers);
35207 clobbers);
35209 }
35211 /* Implements target vector targetm.asm.encode_section_info. */
35213 static void ATTRIBUTE_UNUSED
35215 {
35222 }
35224 /* Worker function for REVERSE_CONDITION. */
35226 enum rtx_code
35228 {
35232 }
35234 /* Output code to perform an x87 FP register move, from OPERANDS[1]
35235 to OPERANDS[0]. */
35239 {
35241 {
35244 {
35248 }
35252 }
35254 {
35258 else
35259 {
35260 /* There is no non-popping store to memory for XFmode.
35261 So if we need one, follow the store with a load. */
35264 else
35266 }
35267 }
35268 else
35270 }
35272 /* Output code to perform a conditional jump to LABEL, if C2 flag in
35273 FP status register is set. */
35275 void
35277 {
35279 rtx temp;
35284 {
35289 }
35290 else
35291 {
35296 }
35300 pc_rtx);
35305 }
35307 /* Output code to perform a log1p XFmode calculation. */
35310 {
35316 rtx test;
35322 XFmode));
35336 }
35338 /* Emit code for round calculation. */
35340 {
35347 rtx insn;
35352 {
35364 }
35367 {
35388 }
35396 /* round(a) = sgn(a) * floor(fabs(a) + 0.5) */
35398 /* scratch = fxam(op1) */
35401 UNSPEC_FXAM)));
35402 /* e1 = fabs(op1) */
35405 /* e2 = e1 + 0.5 */
35410 /* res = floor(e2) */
35412 {
35417 }
35418 else
35422 {
35425 {
35432 UNSPEC_TRUNC_NOOP)));
35433 }
35449 }
35451 /* flags = signbit(a) */
35454 /* if (flags) then res = -res */
35458 pc_rtx);
35469 }
35471 /* Output code to perform a Newton-Rhapson approximation of a single precision
35472 floating point divide [http://en.wikipedia.org/wiki/N-th_root_algorithm]. */
35475 {
35483 /* a / b = a * ((rcp(b) + rcp(b)) - (b * rcp(b) * rcp (b))) */
35487 /* x0 = rcp(b) estimate */
35490 UNSPEC_RCP)));
35491 /* e0 = x0 * b */
35495 /* e0 = x0 * e0 */
35499 /* e1 = x0 + x0 */
35503 /* x1 = e1 - e0 */
35507 /* res = a * x1 */
35510 }
35512 /* Output code to perform a Newton-Rhapson approximation of a
35513 single precision floating point [reciprocal] square root. */
35517 {
35519 REAL_VALUE_TYPE r;
35534 {
35537 }
35539 /* sqrt(a) = -0.5 * a * rsqrtss(a) * (a * rsqrtss(a) * rsqrtss(a) - 3.0)
35540 rsqrt(a) = -0.5 * rsqrtss(a) * (a * rsqrtss(a) * rsqrtss(a) - 3.0) */
35544 /* x0 = rsqrt(a) estimate */
35547 UNSPEC_RSQRT)));
35549 /* If (a == 0.0) Filter out infinity to prevent NaN for sqrt(0.0). */
35551 {
35563 }
35565 /* e0 = x0 * a */
35568 /* e1 = e0 * x0 */
35572 /* e2 = e1 - 3. */
35579 /* e3 = -.5 * x0 */
35582 else
35583 /* e3 = -.5 * e0 */
35586 /* ret = e2 * e3 */
35589 }
35591 #ifdef TARGET_SOLARIS
35592 /* Solaris implementation of TARGET_ASM_NAMED_SECTION. */
35594 static void
35596 tree decl)
35597 {
35598 /* With Binutils 2.15, the "@unwind" marker must be specified on
35599 every occurrence of the ".eh_frame" section, not just the first
35600 one. */
35603 {
35607 }
35609 #ifndef USE_GAS
35611 {
35614 }
35615 #endif
35618 }
35621 /* Return the mangling of TYPE if it is an extended fundamental type. */
35625 {
35633 {
35635 /* __float128 is "g". */
35638 /* "long double" or __float80 is "e". */
35642 }
35643 }
35645 /* For 32-bit code we can save PIC register setup by using
35646 __stack_chk_fail_local hidden function instead of calling
35647 __stack_chk_fail directly. 64-bit code doesn't need to setup any PIC
35648 register, so it is better to call __stack_chk_fail directly. */
35650 static tree ATTRIBUTE_UNUSED
35652 {
35653 return TARGET_64BIT
35656 }
35658 /* Select a format to encode pointers in exception handling data. CODE
35659 is 0 for data, 1 for code labels, 2 for function pointers. GLOBAL is
35660 true if the symbol may be affected by dynamic relocations.
35662 ??? All x86 object file formats are capable of representing this.
35663 After all, the relocation needed is the same as for the call insn.
35664 Whether or not a particular assembler allows us to enter such, I
35665 guess we'll have to see. */
35666 int
35668 {
35670 {
35677 }
35682 }
35683 \f
35684 /* Expand copysign from SIGN to the positive value ABS_VALUE
35685 storing in RESULT. If MASK is non-null, it shall be a mask to mask out
35686 the sign-bit. */
35687 static void
35689 {
35693 {
35700 else
35705 {
35706 /* We need to generate a scalar mode mask in this case. */
35711 }
35712 }
35713 else
35719 }
35721 /* Expand fabs (OP0) and return a new rtx that holds the result. The
35722 mask for masking out the sign-bit is stored in *SMASK, if that is
35723 non-null. */
35724 static rtx
35726 {
35735 else
35739 {
35740 /* We need to generate a scalar mode mask in this case. */
35745 }
35753 }
35755 /* Expands a comparison of OP0 with OP1 using comparison code CODE,
35756 swapping the operands if SWAP_OPERANDS is true. The expanded
35757 code is a forward jump to a newly created label in case the
35758 comparison is true. The generated label rtx is returned. */
35759 static rtx
35762 {
35766 {
35770 }
35783 }
35785 /* Expand a mask generating SSE comparison instruction comparing OP0 with OP1
35786 using comparison code CODE. Operands are swapped for the comparison if
35787 SWAP_OPERANDS is true. Returns a rtx for the generated mask. */
35788 static rtx
35791 {
35797 {
35801 }
35808 }
35810 /* Generate and return a rtx of mode MODE for 2**n where n is the number
35811 of bits of the mantissa of MODE, which must be one of DFmode or SFmode. */
35812 static rtx
35814 {
35815 REAL_VALUE_TYPE TWO52r;
35816 rtx TWO52;
35823 }
35825 /* Expand SSE sequence for computing lround from OP1 storing
35826 into OP0. */
35827 void
35829 {
35830 /* C code for the stuff we're doing below:
35831 tmp = op1 + copysign (nextafter (0.5, 0.0), op1)
35832 return (long)tmp;
35833 */
35837 rtx adj;
35839 /* load nextafter (0.5, 0.0) */
35844 /* adj = copysign (0.5, op1) */
35848 /* adj = op1 + adj */
35851 /* op0 = (imode)adj */
35853 }
35855 /* Expand SSE2 sequence for computing lround from OPERAND1 storing
35856 into OPERAND0. */
35857 void
35859 {
35860 /* C code for the stuff we're doing below (for do_floor):
35861 xi = (long)op1;
35862 xi -= (double)xi > op1 ? 1 : 0;
35863 return xi;
35864 */
35869 /* reg = (long)op1 */
35873 /* freg = (double)reg */
35877 /* ireg = (freg > op1) ? ireg - 1 : ireg */
35888 }
35890 /* Expand rint (IEEE round to nearest) rounding OPERAND1 and storing the
35891 result in OPERAND0. */
35892 void
35894 {
35895 /* C code for the stuff we're doing below:
35896 xa = fabs (operand1);
35897 if (!isless (xa, 2**52))
35898 return operand1;
35899 xa = xa + 2**52 - 2**52;
35900 return copysign (xa, operand1);
35901 */
35908 /* xa = abs (operand1) */
35911 /* if (!isless (xa, TWO52)) goto label; */
35924 }
35926 /* Expand SSE2 sequence for computing floor or ceil from OPERAND1 storing
35927 into OPERAND0. */
35928 void
35930 {
35931 /* C code for the stuff we expand below.
35932 double xa = fabs (x), x2;
35933 if (!isless (xa, TWO52))
35934 return x;
35935 xa = xa + TWO52 - TWO52;
35936 x2 = copysign (xa, x);
35937 Compensate. Floor:
35938 if (x2 > x)
35939 x2 -= 1;
35940 Compensate. Ceil:
35941 if (x2 < x)
35942 x2 -= -1;
35943 return x2;
35944 */
35950 /* Temporary for holding the result, initialized to the input
35951 operand to ease control flow. */
35955 /* xa = abs (operand1) */
35958 /* if (!isless (xa, TWO52)) goto label; */
35961 /* xa = xa + TWO52 - TWO52; */
35965 /* xa = copysign (xa, operand1) */
35968 /* generate 1.0 or -1.0 */
35973 /* Compensate: xa = xa - (xa > operand1 ? 1 : 0) */
35977 /* We always need to subtract here to preserve signed zero. */
35986 }
35988 /* Expand SSE2 sequence for computing floor or ceil from OPERAND1 storing
35989 into OPERAND0. */
35990 void
35992 {
35993 /* C code for the stuff we expand below.
35994 double xa = fabs (x), x2;
35995 if (!isless (xa, TWO52))
35996 return x;
35997 x2 = (double)(long)x;
35998 Compensate. Floor:
35999 if (x2 > x)
36000 x2 -= 1;
36001 Compensate. Ceil:
36002 if (x2 < x)
36003 x2 += 1;
36004 if (HONOR_SIGNED_ZEROS (mode))
36005 return copysign (x2, x);
36006 return x2;
36007 */
36013 /* Temporary for holding the result, initialized to the input
36014 operand to ease control flow. */
36018 /* xa = abs (operand1) */
36021 /* if (!isless (xa, TWO52)) goto label; */
36024 /* xa = (double)(long)x */
36029 /* generate 1.0 */
36032 /* Compensate: xa = xa - (xa > operand1 ? 1 : 0) */
36047 }
36049 /* Expand SSE sequence for computing round from OPERAND1 storing
36050 into OPERAND0. Sequence that works without relying on DImode truncation
36051 via cvttsd2siq that is only available on 64bit targets. */
36052 void
36054 {
36055 /* C code for the stuff we expand below.
36056 double xa = fabs (x), xa2, x2;
36057 if (!isless (xa, TWO52))
36058 return x;
36059 Using the absolute value and copying back sign makes
36060 -0.0 -> -0.0 correct.
36061 xa2 = xa + TWO52 - TWO52;
36062 Compensate.
36063 dxa = xa2 - xa;
36064 if (dxa <= -0.5)
36065 xa2 += 1;
36066 else if (dxa > 0.5)
36067 xa2 -= 1;
36068 x2 = copysign (xa2, x);
36069 return x2;
36070 */
36076 /* Temporary for holding the result, initialized to the input
36077 operand to ease control flow. */
36081 /* xa = abs (operand1) */
36084 /* if (!isless (xa, TWO52)) goto label; */
36087 /* xa2 = xa + TWO52 - TWO52; */
36091 /* dxa = xa2 - xa; */
36094 /* generate 0.5, 1.0 and -0.5 */
36100 /* Compensate. */
36102 /* xa2 = xa2 - (dxa > 0.5 ? 1 : 0) */
36107 /* xa2 = xa2 + (dxa <= -0.5 ? 1 : 0) */
36113 /* res = copysign (xa2, operand1) */
36120 }
36122 /* Expand SSE sequence for computing trunc from OPERAND1 storing
36123 into OPERAND0. */
36124 void
36126 {
36127 /* C code for SSE variant we expand below.
36128 double xa = fabs (x), x2;
36129 if (!isless (xa, TWO52))
36130 return x;
36131 x2 = (double)(long)x;
36132 if (HONOR_SIGNED_ZEROS (mode))
36133 return copysign (x2, x);
36134 return x2;
36135 */
36141 /* Temporary for holding the result, initialized to the input
36142 operand to ease control flow. */
36146 /* xa = abs (operand1) */
36149 /* if (!isless (xa, TWO52)) goto label; */
36152 /* x = (double)(long)x */
36164 }
36166 /* Expand SSE sequence for computing trunc from OPERAND1 storing
36167 into OPERAND0. */
36168 void
36170 {
36174 /* C code for SSE variant we expand below.
36175 double xa = fabs (x), x2;
36176 if (!isless (xa, TWO52))
36177 return x;
36178 xa2 = xa + TWO52 - TWO52;
36179 Compensate:
36180 if (xa2 > xa)
36181 xa2 -= 1.0;
36182 x2 = copysign (xa2, x);
36183 return x2;
36184 */
36188 /* Temporary for holding the result, initialized to the input
36189 operand to ease control flow. */
36193 /* xa = abs (operand1) */
36196 /* if (!isless (xa, TWO52)) goto label; */
36199 /* res = xa + TWO52 - TWO52; */
36204 /* generate 1.0 */
36207 /* Compensate: res = xa2 - (res > xa ? 1 : 0) */
36215 /* res = copysign (res, operand1) */
36222 }
36224 /* Expand SSE sequence for computing round from OPERAND1 storing
36225 into OPERAND0. */
36226 void
36228 {
36229 /* C code for the stuff we're doing below:
36230 double xa = fabs (x);
36231 if (!isless (xa, TWO52))
36232 return x;
36233 xa = (double)(long)(xa + nextafter (0.5, 0.0));
36234 return copysign (xa, x);
36235 */
36241 /* Temporary for holding the result, initialized to the input
36242 operand to ease control flow. */
36250 /* load nextafter (0.5, 0.0) */
36255 /* xa = xa + 0.5 */
36259 /* xa = (double)(int64_t)xa */
36264 /* res = copysign (xa, operand1) */
36271 }
36273 /* Expand SSE sequence for computing round
36274 from OP1 storing into OP0 using sse4 round insn. */
36275 void
36277 {
36286 {
36297 }
36299 /* round (a) = trunc (a + copysign (0.5, a)) */
36301 /* load nextafter (0.5, 0.0) */
36307 /* e1 = copysign (0.5, op1) */
36311 /* e2 = op1 + e1 */
36314 /* res = trunc (e2) */
36319 }
36320 \f
36322 /* Table of valid machine attributes. */
36324 {
36325 /* { name, min_len, max_len, decl_req, type_req, fn_type_req, handler,
36326 affects_type_identity } */
36327 /* Stdcall attribute says callee is responsible for popping arguments
36328 if they are not variable. */
36331 /* Fastcall attribute says callee is responsible for popping arguments
36332 if they are not variable. */
36335 /* Thiscall attribute says callee is responsible for popping arguments
36336 if they are not variable. */
36339 /* Cdecl attribute says the callee is a normal C declaration */
36342 /* Regparm attribute specifies how many integer arguments are to be
36343 passed in registers. */
36346 /* Sseregparm attribute says we are using x86_64 calling conventions
36347 for FP arguments. */
36350 /* The transactional memory builtins are implicitly regparm or fastcall
36351 depending on the ABI. Override the generic do-nothing attribute that
36352 these builtins were declared with. */
36355 /* force_align_arg_pointer says this function realigns the stack at entry. */
36358 #if TARGET_DLLIMPORT_DECL_ATTRIBUTES
36363 #endif
36368 #ifdef SUBTARGET_ATTRIBUTE_TABLE
36369 SUBTARGET_ATTRIBUTE_TABLE,
36370 #endif
36371 /* ms_abi and sysv_abi calling convention function attributes. */
36378 /* End element. */
36380 };
36382 /* Implement targetm.vectorize.builtin_vectorization_cost. */
36383 static int
36385 tree vectype,
36387 {
36391 {
36436 }
36437 }
36439 /* A cached (set (nil) (vselect (vconcat (nil) (nil)) (parallel [])))
36440 insn, so that expand_vselect{,_vconcat} doesn't have to create a fresh
36441 insn every time. */
36445 /* Initialize vselect_insn. */
36447 static void
36449 {
36451 rtx x;
36462 }
36464 /* Construct (set target (vec_select op0 (parallel perm))) and
36465 return true if that's a valid instruction in the active ISA. */
36467 static bool
36470 {
36496 }
36498 /* Similar, but generate a vec_concat from op0 and op1 as well. */
36500 static bool
36504 {
36506 rtx x;
36521 }
36523 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to implement D
36524 in terms of blendp[sd] / pblendw / pblendvb / vpblendd. */
36526 static bool
36528 {
36537 ;
36539 ;
36541 ;
36542 else
36545 /* This is a blend, not a permute. Elements must stay in their
36546 respective lanes. */
36548 {
36552 }
36557 /* ??? Without SSE4.1, we could implement this with and/andn/or. This
36558 decision should be extracted elsewhere, so that we only try that
36559 sequence once all budget==3 options have been tried. */
36566 {
36590 /* See if bytes move in pairs so we can use pblendw with
36591 an immediate argument, rather than pblendvb with a vector
36592 argument. */
36595 {
36596 use_pblendvb:
36600 finish_pblendvb:
36606 else
36609 }
36614 /* FALLTHRU */
36616 do_subreg:
36623 /* See if bytes move in pairs. If not, vpblendvb must be used. */
36627 /* See if bytes move in quadruplets. If yes, vpblendd
36628 with immediate can be used. */
36633 {
36634 /* See if bytes move the same in both lanes. If yes,
36635 vpblendw with immediate can be used. */
36640 /* Use vpblendw. */
36645 }
36647 /* Use vpblendd. */
36654 /* See if words move in pairs. If yes, vpblendd can be used. */
36659 {
36660 /* See if words move the same in both lanes. If not,
36661 vpblendvb must be used. */
36664 {
36665 /* Use vpblendvb. */
36675 }
36677 /* Use vpblendw. */
36681 }
36683 /* Use vpblendd. */
36690 /* Use vpblendd. */
36698 }
36700 /* This matches five different patterns with the different modes. */
36706 }
36708 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to implement D
36709 in terms of the variable form of vpermilps.
36711 Note that we will have already failed the immediate input vpermilps,
36712 which requires that the high and low part shuffle be identical; the
36713 variable form doesn't require that. */
36715 static bool
36717 {
36724 /* We can only permute within the 128-bit lane. */
36726 {
36730 }
36736 {
36739 /* Within each 128-bit lane, the elements of op0 are numbered
36740 from 0 and the elements of op1 are numbered from 4. */
36747 }
36754 }
36756 /* Return true if permutation D can be performed as VMODE permutation
36757 instead. */
36759 static bool
36761 {
36776 else
36782 }
36784 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to implement D
36785 in terms of pshufb, vpperm, vpermq, vpermd, vpermps or vperm2i128. */
36787 static bool
36789 {
36798 {
36800 {
36803 {
36807 /* Use vperm2i128 insn. The pattern uses
36808 V4DImode instead of V2TImode. */
36817 }
36819 }
36820 }
36821 else
36822 {
36824 {
36827 }
36829 {
36833 /* V4DImode should be already handled through
36834 expand_vselect by vpermq instruction. */
36841 {
36842 /* First see if vpermq can be used for
36843 V8SImode/V16HImode/V32QImode. */
36845 {
36853 }
36855 /* Next see if vpermd can be used. */
36858 }
36859 /* Or if vpermps can be used. */
36864 {
36865 /* vpshufb only works intra lanes, it is not
36866 possible to shuffle bytes in between the lanes. */
36870 }
36871 }
36872 else
36874 }
36882 else
36883 {
36889 else
36893 {
36897 }
36898 }
36907 {
36914 else
36916 }
36917 else
36918 {
36921 }
36924 }
36926 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to instantiate D
36927 in a single instruction. */
36929 static bool
36931 {
36935 /* Check plain VEC_SELECT first, because AVX has instructions that could
36936 match both SEL and SEL+CONCAT, but the plain SEL will allow a memory
36937 input where SEL+CONCAT may not. */
36939 {
36945 {
36951 }
36954 {
36958 }
36960 {
36961 /* Use vpbroadcast{b,w,d}. */
36964 {
36983 /* For other modes prefer other shuffles this function creates. */
36985 }
36987 {
36991 }
36992 }
36997 /* There are plenty of patterns in sse.md that are written for
36998 SEL+CONCAT and are not replicated for a single op. Perhaps
36999 that should be changed, to avoid the nastiness here. */
37001 /* Recognize interleave style patterns, which means incrementing
37002 every other permutation operand. */
37004 {
37007 }
37012 /* Recognize shufps, which means adding {0, 0, nelt, nelt}. */
37014 {
37016 {
37021 }
37026 }
37027 }
37029 /* Finally, try the fully general two operand permute. */
37034 /* Recognize interleave style patterns with reversed operands. */
37036 {
37038 {
37042 else
37045 }
37050 }
37052 /* Try the SSE4.1 blend variable merge instructions. */
37056 /* Try one of the AVX vpermil variable permutations. */
37060 /* Try the SSSE3 pshufb or XOP vpperm or AVX2 vperm2i128,
37061 vpshufb, vpermd, vpermps or vpermq variable permutation. */
37066 }
37068 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to implement D
37069 in terms of a pair of pshuflw + pshufhw instructions. */
37071 static bool
37073 {
37081 /* The two permutations only operate in 64-bit lanes. */
37092 /* Emit the pshuflw. */
37099 /* Emit the pshufhw. */
37107 }
37109 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to simplify
37110 the permutation using the SSSE3 palignr instruction. This succeeds
37111 when all of the elements in PERM fit within one vector and we merely
37112 need to shift them down so that a single vector permutation has a
37113 chance to succeed. */
37115 static bool
37117 {
37121 rtx shift;
37123 /* Even with AVX, palignr only operates on 128-bit vectors. */
37129 {
37135 }
37139 /* Given that we have SSSE3, we know we'll be able to implement the
37140 single operand permutation after the palignr with pshufb. */
37154 {
37159 }
37161 /* Test for the degenerate case where the alignment by itself
37162 produces the desired permutation. */
37170 }
37174 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to simplify
37175 a two vector permutation into a single vector permutation by using
37176 an interleave operation to merge the vectors. */
37178 static bool
37180 {
37185 rtx seq;
37189 {
37192 }
37194 {
37197 /* For 32-byte modes allow even d->one_operand_p.
37198 The lack of cross-lane shuffling in some instructions
37199 might prevent a single insn shuffle. */
37202 /* If expand_vec_perm_interleave3 can expand this into
37203 a 3 insn sequence, give up and let it be expanded as
37204 3 insn sequence. While that is one insn longer,
37205 it doesn't need a memory operand and in the common
37206 case that both interleave low and high permutations
37207 with the same operands are adjacent needs 4 insns
37208 for both after CSE. */
37211 }
37212 else
37215 /* Examine from whence the elements come. */
37224 {
37227 /* Split the two input vectors into 4 halves. */
37233 /* If the elements from the low halves use interleave low, and similarly
37234 for interleave high. If the elements are from mis-matched halves, we
37235 can use shufps for V4SF/V4SI or do a DImode shuffle. */
37237 {
37238 /* punpckl* */
37240 {
37245 }
37248 }
37250 {
37251 /* punpckh* */
37253 {
37258 }
37261 }
37263 {
37264 /* shufps */
37266 {
37271 }
37273 {
37274 /* shufpd */
37279 }
37280 }
37282 {
37283 /* shufps */
37285 {
37290 }
37292 {
37293 /* shufpd */
37298 }
37299 }
37300 else
37302 }
37303 else
37304 {
37309 /* Split the two input vectors into 8 quarters. */
37315 {
37318 }
37321 {
37325 }
37327 {
37330 }
37333 {
37335 {
37336 /* Attempt to increase the likelihood that dfinal
37337 shuffle will be intra-lane. */
37341 }
37343 /* vperm2f128 or vperm2i128. */
37345 {
37350 }
37355 {
37359 {
37362 }
37363 }
37364 }
37369 {
37370 /* vpunpckl* */
37372 {
37381 }
37382 }
37385 {
37386 /* vpunpckh* */
37388 {
37397 }
37398 }
37399 else
37401 }
37403 /* Use the remapping array set up above to move the elements from their
37404 swizzled locations into their final destinations. */
37407 {
37410 /* If same_halves is true, both halves of the remapped vector are the
37411 same. Avoid cross-lane accesses if possible. */
37413 {
37416 }
37417 else
37419 }
37425 /* Test if the final remap can be done with a single insn. For V4SFmode or
37426 V4SImode this *will* succeed. For V8HImode or V16QImode it may not. */
37439 {
37443 }
37450 }
37452 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to simplify
37453 a single vector cross-lane permutation into vpermq followed
37454 by any of the single insn permutations. */
37456 static bool
37458 {
37472 {
37475 }
37478 {
37483 }
37496 {
37503 }
37510 {
37515 ;
37518 else
37520 }
37529 }
37531 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to expand
37532 a vector permutation using two instructions, vperm2f128 resp.
37533 vperm2i128 followed by any single in-lane permutation. */
37535 static bool
37537 {
37551 /* ((perm << 2)|perm) & 0x33 is the vperm2[fi]128
37552 immediate. For perm < 16 the second permutation uses
37553 d->op0 as first operand, for perm >= 16 it uses d->op1
37554 as first operand. The second operand is the result of
37555 vperm2[fi]128. */
37557 {
37558 /* Ignore permutations which do not move anything cross-lane. */
37560 {
37561 /* The second shuffle for e.g. V4DFmode has
37562 0123 and ABCD operands.
37563 Ignore AB23, as 23 is already in the second lane
37564 of the first operand. */
37566 /* And 01CD, as 01 is in the first lane of the first
37567 operand. */
37569 /* And 4567, as then the vperm2[fi]128 doesn't change
37570 anything on the original 4567 second operand. */
37572 }
37573 else
37574 {
37575 /* The second shuffle for e.g. V4DFmode has
37576 4567 and ABCD operands.
37577 Ignore AB67, as 67 is already in the second lane
37578 of the first operand. */
37580 /* And 45CD, as 45 is in the first lane of the first
37581 operand. */
37583 /* And 0123, as then the vperm2[fi]128 doesn't change
37584 anything on the original 0123 first operand. */
37586 }
37589 {
37595 else
37597 }
37600 {
37604 }
37605 else
37609 {
37613 /* Found a usable second shuffle. dfirst will be
37614 vperm2f128 on d->op0 and d->op1. */
37625 /* And dsecond is some single insn shuffle, taking
37626 d->op0 and result of vperm2f128 (if perm < 16) or
37627 d->op1 and result of vperm2f128 (otherwise). */
37636 }
37638 /* For one operand, the only useful vperm2f128 permutation is 0x10. */
37641 }
37644 }
37646 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to simplify
37647 a two vector permutation using 2 intra-lane interleave insns
37648 and cross-lane shuffle for 32-byte vectors. */
37650 static bool
37652 {
37659 ;
37661 ;
37662 else
37677 {
37681 else
37687 else
37693 else
37699 else
37705 else
37711 else
37716 }
37720 }
37722 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to implement
37723 a single vector permutation using a single intra-lane vector
37724 permutation, vperm2f128 swapping the lanes and vblend* insn blending
37725 the non-swapped and swapped vectors together. */
37727 static bool
37729 {
37732 rtx seq;
37737 || TARGET_AVX2
37746 {
37753 }
37788 }
37790 /* A subroutine of ix86_expand_vec_perm_builtin_1. Implement a V4DF
37791 permutation using two vperm2f128, followed by a vshufpd insn blending
37792 the two vectors together. */
37794 static bool
37796 {
37839 }
37841 /* A subroutine of expand_vec_perm_even_odd_1. Implement the double-word
37842 permutation with two pshufb insns and an ior. We should have already
37843 failed all two instruction sequences. */
37845 static bool
37847 {
37858 /* Generate two permutation masks. If the required element is within
37859 the given vector it is shuffled into the proper lane. If the required
37860 element is in the other vector, force a zero into the lane by setting
37861 bit 7 in the permutation mask. */
37864 {
37871 {
37874 }
37875 }
37895 }
37897 /* Implement arbitrary permutation of one V32QImode and V16QImode operand
37898 with two vpshufb insns, vpermq and vpor. We should have already failed
37899 all two or three instruction sequences. */
37901 static bool
37903 {
37918 /* Generate two permutation masks. If the required element is within
37919 the same lane, it is shuffled in. If the required element from the
37920 other lane, force a zero by setting bit 7 in the permutation mask.
37921 In the other mask the mask has non-negative elements if element
37922 is requested from the other lane, but also moved to the other lane,
37923 so that the result of vpshufb can have the two V2TImode halves
37924 swapped. */
37927 {
37932 {
37935 }
37936 }
37945 /* Swap the 128-byte lanes of h into hp. */
37949 const1_rtx));
37962 }
37964 /* A subroutine of expand_vec_perm_even_odd_1. Implement extract-even
37965 and extract-odd permutations of two V32QImode and V16QImode operand
37966 with two vpshufb insns, vpor and vpermq. We should have already
37967 failed all two or three instruction sequences. */
37969 static bool
37971 {
37990 /* Generate two permutation masks. In the first permutation mask
37991 the first quarter will contain indexes for the first half
37992 of the op0, the second quarter will contain bit 7 set, third quarter
37993 will contain indexes for the second half of the op0 and the
37994 last quarter bit 7 set. In the second permutation mask
37995 the first quarter will contain bit 7 set, the second quarter
37996 indexes for the first half of the op1, the third quarter bit 7 set
37997 and last quarter indexes for the second half of the op1.
37998 I.e. the first mask e.g. for V32QImode extract even will be:
37999 0, 2, ..., 0xe, -128, ..., -128, 0, 2, ..., 0xe, -128, ..., -128
38000 (all values masked with 0xf except for -128) and second mask
38001 for extract even will be
38002 -128, ..., -128, 0, 2, ..., 0xe, -128, ..., -128, 0, 2, ..., 0xe. */
38005 {
38011 {
38014 }
38015 }
38034 /* Permute the V4DImode quarters using { 0, 2, 1, 3 } permutation. */
38041 }
38043 /* A subroutine of ix86_expand_vec_perm_builtin_1. Implement extract-even
38044 and extract-odd permutations. */
38046 static bool
38048 {
38052 {
38057 /* Shuffle the lanes around into { 0 1 4 5 } and { 2 3 6 7 }. */
38061 /* Now an unpck[lh]pd will produce the result required. */
38064 else
38070 {
38077 /* Shuffle within the 128-bit lanes to produce:
38078 { 0 2 8 a 4 6 c e } | { 1 3 9 b 5 7 d f }. */
38082 /* Shuffle the lanes around to produce:
38083 { 4 6 c e 0 2 8 a } and { 5 7 d f 1 3 9 b }. */
38087 /* Shuffle within the 128-bit lanes to produce:
38088 { 0 2 4 6 4 6 0 2 } | { 1 3 5 7 5 7 1 3 }. */
38091 /* Shuffle within the 128-bit lanes to produce:
38092 { 8 a c e c e 8 a } | { 9 b d f d f 9 b }. */
38095 /* Shuffle the lanes around to produce:
38096 { 0 2 4 6 8 a c e } | { 1 3 5 7 9 b d f }. */
38099 }
38106 /* These are always directly implementable by expand_vec_perm_1. */
38112 else
38113 {
38114 /* We need 2*log2(N)-1 operations to achieve odd/even
38115 with interleave. */
38124 else
38127 }
38133 else
38134 {
38146 else
38149 }
38158 {
38165 }
38170 /* Shuffle the lanes around into { 0 1 4 5 } and { 2 3 6 7 }. */
38174 /* Now an vpunpck[lh]qdq will produce the result required. */
38177 else
38184 {
38191 }
38196 /* Shuffle the lanes around into
38197 { 0 1 2 3 8 9 a b } and { 4 5 6 7 c d e f }. */
38207 /* Swap the 2nd and 3rd position in each lane into
38208 { 0 2 1 3 8 a 9 b } and { 4 6 5 7 c e d f }. */
38214 /* Now an vpunpck[lh]qdq will produce
38215 { 0 2 4 6 8 a c e } resp. { 1 3 5 7 9 b d f }. */
38220 else
38229 }
38232 }
38234 /* A subroutine of ix86_expand_vec_perm_builtin_1. Pattern match
38235 extract-even and extract-odd permutations. */
38237 static bool
38239 {
38251 }
38253 /* A subroutine of ix86_expand_vec_perm_builtin_1. Implement broadcast
38254 permutations. We assume that expand_vec_perm_1 has already failed. */
38256 static bool
38258 {
38266 {
38269 /* These are special-cased in sse.md so that we can optionally
38270 use the vbroadcast instruction. They expand to two insns
38271 if the input happens to be in a register. */
38278 /* These are always implementable using standard shuffle patterns. */
38283 /* These can be implemented via interleave. We save one insn by
38284 stopping once we have promoted to V4SImode and then use pshufd. */
38285 do
38286 {
38287 rtx dest;
38293 {
38297 }
38304 }
38317 /* For AVX2 broadcasts of the first element vpbroadcast* or
38318 vpermq should be used by expand_vec_perm_1. */
38324 }
38325 }
38327 /* A subroutine of ix86_expand_vec_perm_builtin_1. Pattern match
38328 broadcast permutations. */
38330 static bool
38332 {
38344 }
38346 /* Implement arbitrary permutation of two V32QImode and V16QImode operands
38347 with 4 vpshufb insns, 2 vpermq and 3 vpor. We should have already failed
38348 all the shorter instruction sequences. */
38350 static bool
38352 {
38368 /* Generate 4 permutation masks. If the required element is within
38369 the same lane, it is shuffled in. If the required element from the
38370 other lane, force a zero by setting bit 7 in the permutation mask.
38371 In the other mask the mask has non-negative elements if element
38372 is requested from the other lane, but also moved to the other lane,
38373 so that the result of vpshufb can have the two V2TImode halves
38374 swapped. */
38377 {
38382 }
38388 {
38396 }
38399 {
38401 {
38404 }
38411 }
38413 /* Swap the 128-byte lanes of h[X]. */
38415 {
38421 const1_rtx));
38423 }
38426 {
38428 {
38431 }
38437 }
38440 {
38442 {
38446 }
38449 }
38455 }
38457 /* The guts of ix86_expand_vec_perm_const, also used by the ok hook.
38458 With all of the interface bits taken care of, perform the expansion
38459 in D and return true on success. */
38461 static bool
38463 {
38464 /* Try a single instruction expansion. */
38468 /* Try sequences of two instructions. */
38488 /* Try sequences of three instructions. */
38502 /* Try sequences of four instructions. */
38510 /* ??? Look for narrow permutations whose element orderings would
38511 allow the promotion to a wider mode. */
38513 /* ??? Look for sequences of interleave or a wider permute that place
38514 the data into the correct lanes for a half-vector shuffle like
38515 pshuf[lh]w or vpermilps. */
38517 /* ??? Look for sequences of interleave that produce the desired results.
38518 The combinatorics of punpck[lh] get pretty ugly... */
38523 /* Even longer sequences. */
38528 }
38530 /* If a permutation only uses one operand, make it clear. Returns true
38531 if the permutation references both operands. */
38533 static bool
38535 {
38543 {
38549 {
38552 }
38553 /* The elements of PERM do not suggest that only the first operand
38554 is used, but both operands are identical. Allow easier matching
38555 of the permutation by folding the permutation into the single
38556 input vector. */
38557 /* FALLTHRU */
38568 }
38571 }
38573 bool
38575 {
38580 rtx sel;
38597 {
38602 }
38609 /* If the selector says both arguments are needed, but the operands are the
38610 same, the above tried to expand with one_operand_p and flattened selector.
38611 If that didn't work, retry without one_operand_p; we succeeded with that
38612 during testing. */
38614 {
38618 }
38621 }
38623 /* Implement targetm.vectorize.vec_perm_const_ok. */
38625 static bool
38628 {
38637 /* Given sufficient ISA support we can just return true here
38638 for selected vector modes. */
38640 {
38641 /* All implementable with a single vpperm insn. */
38644 /* All implementable with 2 pshufb + 1 ior. */
38647 /* All implementable with shufpd or unpck[lh]pd. */
38650 }
38652 /* Extract the values from the vector CST into the permutation
38653 array in D. */
38656 {
38660 }
38662 /* For all elements from second vector, fold the elements to first. */
38667 /* Check whether the mask can be applied to the vector type. */
38670 /* Implementable with shufps or pshufd. */
38674 /* Otherwise we have to go through the motions and see if we can
38675 figure out how to generate the requested permutation. */
38686 }
38688 void
38690 {
38705 /* We'll either be able to implement the permutation directly... */
38709 /* ... or we use the special-case patterns. */
38711 }
38713 static void
38715 {
38730 {
38733 }
38735 /* Note that for AVX this isn't one instruction. */
38738 }
38741 /* Expand a vector operation CODE for a V*QImode in terms of the
38742 same operation on V*HImode. */
38744 void
38746 {
38758 {
38771 }
38775 {
38777 /* Unpack data such that we've got a source byte in each low byte of
38778 each word. We don't care what goes into the high byte of each word.
38779 Rather than trying to get zero in there, most convenient is to let
38780 it be a copy of the low byte. */
38785 /* FALLTHRU */
38797 /* FALLTHRU */
38807 }
38809 /* Perform the operation. */
38816 /* Merge the data back into the right place. */
38826 {
38827 /* For SSE2, we used an full interleave, so the desired
38828 results are in the even elements. */
38831 }
38832 else
38833 {
38834 /* For AVX, the interleave used above was not cross-lane. So the
38835 extraction is evens but with the second and third quarter swapped.
38836 Happily, that is even one insn shorter than even extraction. */
38839 }
38846 }
38848 void
38851 {
38854 rtx x;
38856 /* We only play even/odd games with vectors of SImode. */
38859 /* If we're looking for the odd results, shift those members down to
38860 the even slots. For some cpus this is faster than a PSHUFD. */
38862 {
38864 {
38868 }
38877 }
38880 {
38883 else
38885 }
38890 else
38891 {
38894 /* The easiest way to implement this without PMULDQ is to go through
38895 the motions as if we are performing a full 64-bit multiply. With
38896 the exception that we need to do less shuffling of the elements. */
38898 /* Compute the sign-extension, aka highparts, of the two operands. */
38904 /* Multiply LO(A) * HI(B), and vice-versa. */
38910 /* Multiply LO(A) * LO(B). */
38914 /* Combine and shift the highparts into place. */
38919 /* Combine high and low parts. */
38922 }
38924 }
38926 void
38929 {
38935 {
38940 {
38941 /* With XOP, we have pmacsdqh, aka mul_widen_odd. In this case,
38942 shuffle the elements once so that all elements are in the right
38943 place for immediate use: { A C B D }. */
38948 }
38949 else
38950 {
38951 /* Put the elements into place for the multiply. */
38955 }
38960 /* Shuffle the elements between the lanes. After this we
38961 have { A B E F | C D G H } for each operand. */
38971 /* Shuffle the elements within the lanes. After this we
38972 have { A A B B | C C D D } or { E E F F | G G H H }. */
39008 }
39009 }
39011 void
39013 {
39023 /* Move the results in element 2 down to element 1; we don't care
39024 what goes in elements 2 and 3. Then we can merge the parts
39025 back together with an interleave.
39027 Note that two other sequences were tried:
39028 (1) Use interleaves at the start instead of psrldq, which allows
39029 us to use a single shufps to merge things back at the end.
39030 (2) Use shufps here to combine the two vectors, then pshufd to
39031 put the elements in the correct order.
39032 In both cases the cost of the reformatting stall was too high
39033 and the overall sequence slower. */
39042 }
39044 void
39046 {
39051 {
39052 /* op1: A,B,C,D, op2: E,F,G,H */
39061 /* t1: B,A,D,C */
39068 /* t2: (B*E),(A*F),(D*G),(C*H) */
39071 /* t3: (B*E)+(A*F), (D*G)+(C*H) */
39074 /* t4: ((B*E)+(A*F))<<32, ((D*G)+(C*H))<<32 */
39077 /* op0: (((B*E)+(A*F))<<32)+(B*F), (((D*G)+(C*H))<<32)+(D*H) */
39079 }
39080 else
39081 {
39086 {
39089 }
39091 {
39094 }
39095 else
39099 /* Multiply low parts. */
39103 /* Shift input vectors right 32 bits so we can multiply high parts. */
39108 /* Multiply high parts by low parts. */
39114 /* Combine and shift the highparts back. */
39118 /* Combine high and low parts. */
39120 }
39124 }
39126 /* Expand an insert into a vector register through pinsr insn.
39127 Return true if successful. */
39129 bool
39131 {
39139 {
39142 }
39148 {
39153 {
39160 {
39192 }
39201 }
39205 }
39206 }
39207 \f
39208 /* This function returns the calling abi specific va_list type node.
39209 It returns the FNDECL specific va_list type. */
39211 static tree
39213 {
39220 else
39222 }
39224 /* Returns the canonical va_list type specified by TYPE. If there
39225 is no valid TYPE provided, it return NULL_TREE. */
39227 static tree
39229 {
39232 /* Resolve references and pointers to va_list type. */
39241 {
39246 {
39247 /* If va_list is an array type, the argument may have decayed
39248 to a pointer type, e.g. by being passed to another function.
39249 In that case, unwrap both types so that we can compare the
39250 underlying records. */
39253 {
39256 }
39257 }
39264 {
39265 /* If va_list is an array type, the argument may have decayed
39266 to a pointer type, e.g. by being passed to another function.
39267 In that case, unwrap both types so that we can compare the
39268 underlying records. */
39271 {
39274 }
39275 }
39282 {
39283 /* If va_list is an array type, the argument may have decayed
39284 to a pointer type, e.g. by being passed to another function.
39285 In that case, unwrap both types so that we can compare the
39286 underlying records. */
39289 {
39292 }
39293 }
39297 }
39299 }
39301 /* Iterate through the target-specific builtin types for va_list.
39302 IDX denotes the iterator, *PTREE is set to the result type of
39303 the va_list builtin, and *PNAME to its internal type.
39304 Returns zero if there is no element for this index, otherwise
39305 IDX should be increased upon the next call.
39306 Note, do not iterate a base builtin's name like __builtin_va_list.
39307 Used from c_common_nodes_and_builtins. */
39309 static int
39311 {
39313 {
39315 {
39328 }
39329 }
39332 }
39334 #undef TARGET_SCHED_DISPATCH
39335 #define TARGET_SCHED_DISPATCH has_dispatch
39336 #undef TARGET_SCHED_DISPATCH_DO
39337 #define TARGET_SCHED_DISPATCH_DO do_dispatch
39338 #undef TARGET_SCHED_REASSOCIATION_WIDTH
39339 #define TARGET_SCHED_REASSOCIATION_WIDTH ix86_reassociation_width
39340 #undef TARGET_SCHED_REORDER
39341 #define TARGET_SCHED_REORDER ix86_sched_reorder
39343 /* The size of the dispatch window is the total number of bytes of
39344 object code allowed in a window. */
39345 #define DISPATCH_WINDOW_SIZE 16
39347 /* Number of dispatch windows considered for scheduling. */
39348 #define MAX_DISPATCH_WINDOWS 3
39350 /* Maximum number of instructions in a window. */
39351 #define MAX_INSN 4
39353 /* Maximum number of immediate operands in a window. */
39354 #define MAX_IMM 4
39356 /* Maximum number of immediate bits allowed in a window. */
39357 #define MAX_IMM_SIZE 128
39359 /* Maximum number of 32 bit immediates allowed in a window. */
39360 #define MAX_IMM_32 4
39362 /* Maximum number of 64 bit immediates allowed in a window. */
39363 #define MAX_IMM_64 2
39365 /* Maximum total of loads or prefetches allowed in a window. */
39366 #define MAX_LOAD 2
39368 /* Maximum total of stores allowed in a window. */
39369 #define MAX_STORE 1
39371 #undef BIG
39372 #define BIG 100
39375 /* Dispatch groups. Istructions that affect the mix in a dispatch window. */
39378 disp_load,
39379 disp_store,
39380 disp_load_store,
39381 disp_prefetch,
39382 disp_imm,
39383 disp_imm_32,
39384 disp_imm_64,
39385 disp_branch,
39386 disp_cmp,
39387 disp_jcc,
39388 disp_last
39389 };
39391 /* Number of allowable groups in a dispatch window. It is an array
39392 indexed by dispatch_group enum. 100 is used as a big number,
39393 because the number of these kind of operations does not have any
39394 effect in dispatch window, but we need them for other reasons in
39395 the table. */
39398 };
39404 };
39406 /* Instruction path. */
39412 last_path
39413 };
39415 /* sched_insn_info defines a window to the instructions scheduled in
39416 the basic block. It contains a pointer to the insn_info table and
39417 the instruction scheduled.
39419 Windows are allocated for each basic block and are linked
39420 together. */
39422 rtx insn;
39429 /* Linked list of dispatch windows. This is a two way list of
39430 dispatch windows of a basic block. It contains information about
39431 the number of uops in the window and the total number of
39432 instructions and of bytes in the object code for this dispatch
39433 window. */
39451 /* Immediate valuse used in an insn. */
39453 {
39462 /* Get dispatch group of insn. */
39464 static enum dispatch_group
39466 {
39482 }
39484 /* Return true if insn is a compare instruction. */
39486 static bool
39488 {
39496 }
39498 /* Return true if a dispatch violation encountered. */
39500 static bool
39502 {
39506 }
39508 /* Return true if insn is a branch instruction. */
39510 static bool
39512 {
39514 }
39516 /* Return true if insn is a prefetch instruction. */
39518 static bool
39520 {
39522 }
39524 /* This function initializes a dispatch window and the list container holding a
39525 pointer to the window. */
39527 static void
39529 {
39535 else
39553 {
39559 }
39561 }
39563 /* This function allocates and initializes a dispatch window and the
39564 list container holding a pointer to the window. */
39568 {
39573 }
39575 /* This routine initializes the dispatch scheduling information. It
39576 initiates building dispatch scheduler tables and constructs the
39577 first dispatch window. */
39579 static void
39581 {
39582 /* Allocate a dispatch list and a window. */
39587 }
39589 /* This function returns true if a branch is detected. End of a basic block
39590 does not have to be a branch, but here we assume only branches end a
39591 window. */
39593 static bool
39595 {
39597 }
39599 /* This function is called when the end of a window processing is reached. */
39601 static void
39603 {
39606 {
39611 }
39613 }
39615 /* Allocates a new dispatch window and adds it to WINDOW_LIST.
39616 WINDOW_NUM is either 0 or 1. A maximum of two windows are generated
39617 for 48 bytes of instructions. Note that these windows are not dispatch
39618 windows that their sizes are DISPATCH_WINDOW_SIZE. */
39622 {
39624 {
39629 }
39635 }
39637 /* Increment the number of immediate operands of an instruction. */
39639 static int
39641 {
39646 {
39653 else
39664 {
39667 }
39672 }
39675 }
39677 /* Compute number of immediate operands of an instruction. */
39679 static void
39681 {
39684 }
39686 /* Return total size of immediate operands of an instruction along with number
39687 of corresponding immediate-operands. It initializes its parameters to zero
39688 befor calling FIND_CONSTANT.
39689 INSN is the input instruction. IMM is the total of immediates.
39690 IMM32 is the number of 32 bit immediates. IMM64 is the number of 64
39691 bit immediates. */
39693 static int
39695 {
39703 }
39705 /* This function indicates if an operand of an instruction is an
39706 immediate. */
39708 static bool
39710 {
39719 }
39721 /* Return single or double path for instructions. */
39723 static enum insn_path
39725 {
39735 }
39737 /* Return insn dispatch group. */
39739 static enum dispatch_group
39741 {
39759 }
39761 /* Count number of GROUP restricted instructions in a dispatch
39762 window WINDOW_LIST. */
39764 static int
39766 {
39777 {
39796 }
39809 }
39811 /* This function returns true if insn satisfies dispatch rules on the
39812 last window scheduled. */
39814 static bool
39816 {
39824 /* Make disp_cmp and disp_jcc get scheduled at the latest. These
39825 instructions should be given the lowest priority in the
39826 scheduling process in Haifa scheduler to make sure they will be
39827 scheduled in the same dispatch window as the reference to them. */
39831 /* Check nonrestricted. */
39835 /* Get last dispatch window. */
39840 {
39845 /* Window 1 is full. Go for next window. */
39847 }
39854 /* See if it fits in the first window. */
39856 {
39857 /* The first widow should have only single and double path
39858 uops. */
39864 }
39866 }
39868 /* Add an instruction INSN with NUM_UOPS micro-operations to the
39869 dispatch window WINDOW_LIST. */
39871 static void
39873 {
39891 /* Initialize window with new instruction. */
39912 {
39915 }
39916 }
39918 /* Adds a scheduled instruction, INSN, to the current dispatch window.
39919 If the total bytes of instructions or the number of instructions in
39920 the window exceed allowable, it allocates a new window. */
39922 static void
39924 {
39947 /* Get the last dispatch window. */
39955 else
39958 /* If current window is full, get a new window.
39959 Window number zero is full, if MAX_INSN uops are scheduled in it.
39960 Window number one is full, if window zero's bytes plus window
39961 one's bytes is 32, or if the bytes of the new instruction added
39962 to the total makes it greater than 48, or it has already MAX_INSN
39963 instructions in it. */
39972 {
39975 }
39978 {
39982 {
39985 }
39986 }
39988 {
39993 {
39996 }
39999 }
40000 else
40004 {
40005 /* End of basic block is reached do end-basic-block process. */
40008 }
40009 }
40011 /* Print the dispatch window, WINDOW_NUM, to FILE. */
40013 DEBUG_FUNCTION static void
40015 {
40021 else
40035 {
40038 fprintf (file, " group[%d] = %s, insn[%d] = %p, path[%d] = %d byte_len[%d] = %d, imm_bytes[%d] = %d\n",
40044 }
40045 }
40047 /* Print to stdout a dispatch window. */
40049 DEBUG_FUNCTION void
40051 {
40053 }
40055 /* Print INSN dispatch information to FILE. */
40057 DEBUG_FUNCTION static void
40059 {
40082 }
40084 /* Print to STDERR the status of the ready list with respect to
40085 dispatch windows. */
40087 DEBUG_FUNCTION void
40089 {
40097 }
40099 /* This routine is the driver of the dispatch scheduler. */
40101 static void
40103 {
40108 }
40110 /* Return TRUE if Dispatch Scheduling is supported. */
40112 static bool
40114 {
40118 {
40134 }
40137 }
40139 /* Implementation of reassociation_width target hook used by
40140 reassoc phase to identify parallelism level in reassociated
40141 tree. Statements tree_code is passed in OPC. Arguments type
40142 is passed in MODE.
40144 Currently parallel reassociation is enabled for Atom
40145 processors only and we set reassociation width to be 2
40146 because Atom may issue up to 2 instructions per cycle.
40148 Return value should be fixed if parallel reassociation is
40149 enabled for other processors. */
40151 static int
40154 {
40163 }
40165 /* ??? No autovectorization into MMX or 3DNOW until we can reliably
40166 place emms and femms instructions. */
40168 static enum machine_mode
40170 {
40175 {
40188 else
40198 /* FALLTHRU */
40202 }
40203 }
40205 /* If AVX is enabled then try vectorizing with both 256bit and 128bit
40206 vectors. */
40208 static unsigned int
40210 {
40212 }
40214 /* Implement targetm.vectorize.init_cost. */
40218 {
40222 }
40224 /* Implement targetm.vectorize.add_stmt_cost. */
40226 static unsigned
40230 {
40235 {
40239 /* Statements in an inner loop relative to the loop being
40240 vectorized are weighted more heavily. The value here is
40241 arbitrary and could potentially be improved with analysis. */
40247 }
40250 }
40252 /* Implement targetm.vectorize.finish_cost. */
40254 static void
40257 {
40262 }
40264 /* Implement targetm.vectorize.destroy_cost_data. */
40266 static void
40268 {
40270 }
40272 /* Validate target specific memory model bits in VAL. */
40274 static unsigned HOST_WIDE_INT
40276 {
40283 {
40287 }
40290 {
40294 }
40296 {
40300 }
40302 }
40304 /* Initialize the GCC target structure. */
40305 #undef TARGET_RETURN_IN_MEMORY
40306 #define TARGET_RETURN_IN_MEMORY ix86_return_in_memory
40308 #undef TARGET_LEGITIMIZE_ADDRESS
40309 #define TARGET_LEGITIMIZE_ADDRESS ix86_legitimize_address
40311 #undef TARGET_ATTRIBUTE_TABLE
40312 #define TARGET_ATTRIBUTE_TABLE ix86_attribute_table
40313 #if TARGET_DLLIMPORT_DECL_ATTRIBUTES
40314 # undef TARGET_MERGE_DECL_ATTRIBUTES
40315 # define TARGET_MERGE_DECL_ATTRIBUTES merge_dllimport_decl_attributes
40316 #endif
40318 #undef TARGET_COMP_TYPE_ATTRIBUTES
40319 #define TARGET_COMP_TYPE_ATTRIBUTES ix86_comp_type_attributes
40321 #undef TARGET_INIT_BUILTINS
40322 #define TARGET_INIT_BUILTINS ix86_init_builtins
40323 #undef TARGET_BUILTIN_DECL
40324 #define TARGET_BUILTIN_DECL ix86_builtin_decl
40325 #undef TARGET_EXPAND_BUILTIN
40326 #define TARGET_EXPAND_BUILTIN ix86_expand_builtin
40328 #undef TARGET_VECTORIZE_BUILTIN_VECTORIZED_FUNCTION
40329 #define TARGET_VECTORIZE_BUILTIN_VECTORIZED_FUNCTION \
40330 ix86_builtin_vectorized_function
40332 #undef TARGET_VECTORIZE_BUILTIN_TM_LOAD
40333 #define TARGET_VECTORIZE_BUILTIN_TM_LOAD ix86_builtin_tm_load
40335 #undef TARGET_VECTORIZE_BUILTIN_TM_STORE
40336 #define TARGET_VECTORIZE_BUILTIN_TM_STORE ix86_builtin_tm_store
40338 #undef TARGET_VECTORIZE_BUILTIN_GATHER
40339 #define TARGET_VECTORIZE_BUILTIN_GATHER ix86_vectorize_builtin_gather
40341 #undef TARGET_BUILTIN_RECIPROCAL
40342 #define TARGET_BUILTIN_RECIPROCAL ix86_builtin_reciprocal
40344 #undef TARGET_ASM_FUNCTION_EPILOGUE
40345 #define TARGET_ASM_FUNCTION_EPILOGUE ix86_output_function_epilogue
40347 #undef TARGET_ENCODE_SECTION_INFO
40348 #ifndef SUBTARGET_ENCODE_SECTION_INFO
40349 #define TARGET_ENCODE_SECTION_INFO ix86_encode_section_info
40350 #else
40351 #define TARGET_ENCODE_SECTION_INFO SUBTARGET_ENCODE_SECTION_INFO
40352 #endif
40354 #undef TARGET_ASM_OPEN_PAREN
40356 #undef TARGET_ASM_CLOSE_PAREN
40359 #undef TARGET_ASM_BYTE_OP
40360 #define TARGET_ASM_BYTE_OP ASM_BYTE
40362 #undef TARGET_ASM_ALIGNED_HI_OP
40363 #define TARGET_ASM_ALIGNED_HI_OP ASM_SHORT
40364 #undef TARGET_ASM_ALIGNED_SI_OP
40365 #define TARGET_ASM_ALIGNED_SI_OP ASM_LONG
40366 #ifdef ASM_QUAD
40367 #undef TARGET_ASM_ALIGNED_DI_OP
40368 #define TARGET_ASM_ALIGNED_DI_OP ASM_QUAD
40369 #endif
40371 #undef TARGET_PROFILE_BEFORE_PROLOGUE
40372 #define TARGET_PROFILE_BEFORE_PROLOGUE ix86_profile_before_prologue
40374 #undef TARGET_ASM_UNALIGNED_HI_OP
40375 #define TARGET_ASM_UNALIGNED_HI_OP TARGET_ASM_ALIGNED_HI_OP
40376 #undef TARGET_ASM_UNALIGNED_SI_OP
40377 #define TARGET_ASM_UNALIGNED_SI_OP TARGET_ASM_ALIGNED_SI_OP
40378 #undef TARGET_ASM_UNALIGNED_DI_OP
40379 #define TARGET_ASM_UNALIGNED_DI_OP TARGET_ASM_ALIGNED_DI_OP
40381 #undef TARGET_PRINT_OPERAND
40382 #define TARGET_PRINT_OPERAND ix86_print_operand
40383 #undef TARGET_PRINT_OPERAND_ADDRESS
40384 #define TARGET_PRINT_OPERAND_ADDRESS ix86_print_operand_address
40385 #undef TARGET_PRINT_OPERAND_PUNCT_VALID_P
40386 #define TARGET_PRINT_OPERAND_PUNCT_VALID_P ix86_print_operand_punct_valid_p
40387 #undef TARGET_ASM_OUTPUT_ADDR_CONST_EXTRA
40388 #define TARGET_ASM_OUTPUT_ADDR_CONST_EXTRA i386_asm_output_addr_const_extra
40390 #undef TARGET_SCHED_INIT_GLOBAL
40391 #define TARGET_SCHED_INIT_GLOBAL ix86_sched_init_global
40392 #undef TARGET_SCHED_ADJUST_COST
40393 #define TARGET_SCHED_ADJUST_COST ix86_adjust_cost
40394 #undef TARGET_SCHED_ISSUE_RATE
40395 #define TARGET_SCHED_ISSUE_RATE ix86_issue_rate
40396 #undef TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD
40397 #define TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD \
40398 ia32_multipass_dfa_lookahead
40400 #undef TARGET_FUNCTION_OK_FOR_SIBCALL
40401 #define TARGET_FUNCTION_OK_FOR_SIBCALL ix86_function_ok_for_sibcall
40403 #undef TARGET_MEMMODEL_CHECK
40404 #define TARGET_MEMMODEL_CHECK ix86_memmodel_check
40406 #ifdef HAVE_AS_TLS
40407 #undef TARGET_HAVE_TLS
40408 #define TARGET_HAVE_TLS true
40409 #endif
40410 #undef TARGET_CANNOT_FORCE_CONST_MEM
40411 #define TARGET_CANNOT_FORCE_CONST_MEM ix86_cannot_force_const_mem
40412 #undef TARGET_USE_BLOCKS_FOR_CONSTANT_P
40413 #define TARGET_USE_BLOCKS_FOR_CONSTANT_P hook_bool_mode_const_rtx_true
40415 #undef TARGET_DELEGITIMIZE_ADDRESS
40416 #define TARGET_DELEGITIMIZE_ADDRESS ix86_delegitimize_address
40418 #undef TARGET_MS_BITFIELD_LAYOUT_P
40419 #define TARGET_MS_BITFIELD_LAYOUT_P ix86_ms_bitfield_layout_p
40421 #if TARGET_MACHO
40422 #undef TARGET_BINDS_LOCAL_P
40423 #define TARGET_BINDS_LOCAL_P darwin_binds_local_p
40424 #endif
40425 #if TARGET_DLLIMPORT_DECL_ATTRIBUTES
40426 #undef TARGET_BINDS_LOCAL_P
40427 #define TARGET_BINDS_LOCAL_P i386_pe_binds_local_p
40428 #endif
40430 #undef TARGET_ASM_OUTPUT_MI_THUNK
40431 #define TARGET_ASM_OUTPUT_MI_THUNK x86_output_mi_thunk
40432 #undef TARGET_ASM_CAN_OUTPUT_MI_THUNK
40433 #define TARGET_ASM_CAN_OUTPUT_MI_THUNK x86_can_output_mi_thunk
40435 #undef TARGET_ASM_FILE_START
40436 #define TARGET_ASM_FILE_START x86_file_start
40438 #undef TARGET_OPTION_OVERRIDE
40439 #define TARGET_OPTION_OVERRIDE ix86_option_override
40441 #undef TARGET_REGISTER_MOVE_COST
40442 #define TARGET_REGISTER_MOVE_COST ix86_register_move_cost
40443 #undef TARGET_MEMORY_MOVE_COST
40444 #define TARGET_MEMORY_MOVE_COST ix86_memory_move_cost
40445 #undef TARGET_RTX_COSTS
40446 #define TARGET_RTX_COSTS ix86_rtx_costs
40447 #undef TARGET_ADDRESS_COST
40448 #define TARGET_ADDRESS_COST ix86_address_cost
40450 #undef TARGET_FIXED_CONDITION_CODE_REGS
40451 #define TARGET_FIXED_CONDITION_CODE_REGS ix86_fixed_condition_code_regs
40452 #undef TARGET_CC_MODES_COMPATIBLE
40453 #define TARGET_CC_MODES_COMPATIBLE ix86_cc_modes_compatible
40455 #undef TARGET_MACHINE_DEPENDENT_REORG
40456 #define TARGET_MACHINE_DEPENDENT_REORG ix86_reorg
40458 #undef TARGET_BUILTIN_SETJMP_FRAME_VALUE
40459 #define TARGET_BUILTIN_SETJMP_FRAME_VALUE ix86_builtin_setjmp_frame_value
40461 #undef TARGET_BUILD_BUILTIN_VA_LIST
40462 #define TARGET_BUILD_BUILTIN_VA_LIST ix86_build_builtin_va_list
40464 #undef TARGET_FOLD_BUILTIN
40465 #define TARGET_FOLD_BUILTIN ix86_fold_builtin
40467 #undef TARGET_ENUM_VA_LIST_P
40468 #define TARGET_ENUM_VA_LIST_P ix86_enum_va_list
40470 #undef TARGET_FN_ABI_VA_LIST
40471 #define TARGET_FN_ABI_VA_LIST ix86_fn_abi_va_list
40473 #undef TARGET_CANONICAL_VA_LIST_TYPE
40474 #define TARGET_CANONICAL_VA_LIST_TYPE ix86_canonical_va_list_type
40476 #undef TARGET_EXPAND_BUILTIN_VA_START
40477 #define TARGET_EXPAND_BUILTIN_VA_START ix86_va_start
40479 #undef TARGET_MD_ASM_CLOBBERS
40480 #define TARGET_MD_ASM_CLOBBERS ix86_md_asm_clobbers
40482 #undef TARGET_PROMOTE_PROTOTYPES
40483 #define TARGET_PROMOTE_PROTOTYPES hook_bool_const_tree_true
40484 #undef TARGET_STRUCT_VALUE_RTX
40485 #define TARGET_STRUCT_VALUE_RTX ix86_struct_value_rtx
40486 #undef TARGET_SETUP_INCOMING_VARARGS
40487 #define TARGET_SETUP_INCOMING_VARARGS ix86_setup_incoming_varargs
40488 #undef TARGET_MUST_PASS_IN_STACK
40489 #define TARGET_MUST_PASS_IN_STACK ix86_must_pass_in_stack
40490 #undef TARGET_FUNCTION_ARG_ADVANCE
40491 #define TARGET_FUNCTION_ARG_ADVANCE ix86_function_arg_advance
40492 #undef TARGET_FUNCTION_ARG
40493 #define TARGET_FUNCTION_ARG ix86_function_arg
40494 #undef TARGET_FUNCTION_ARG_BOUNDARY
40495 #define TARGET_FUNCTION_ARG_BOUNDARY ix86_function_arg_boundary
40496 #undef TARGET_PASS_BY_REFERENCE
40497 #define TARGET_PASS_BY_REFERENCE ix86_pass_by_reference
40498 #undef TARGET_INTERNAL_ARG_POINTER
40499 #define TARGET_INTERNAL_ARG_POINTER ix86_internal_arg_pointer
40500 #undef TARGET_UPDATE_STACK_BOUNDARY
40501 #define TARGET_UPDATE_STACK_BOUNDARY ix86_update_stack_boundary
40502 #undef TARGET_GET_DRAP_RTX
40503 #define TARGET_GET_DRAP_RTX ix86_get_drap_rtx
40504 #undef TARGET_STRICT_ARGUMENT_NAMING
40505 #define TARGET_STRICT_ARGUMENT_NAMING hook_bool_CUMULATIVE_ARGS_true
40506 #undef TARGET_STATIC_CHAIN
40507 #define TARGET_STATIC_CHAIN ix86_static_chain
40508 #undef TARGET_TRAMPOLINE_INIT
40509 #define TARGET_TRAMPOLINE_INIT ix86_trampoline_init
40510 #undef TARGET_RETURN_POPS_ARGS
40511 #define TARGET_RETURN_POPS_ARGS ix86_return_pops_args
40513 #undef TARGET_GIMPLIFY_VA_ARG_EXPR
40514 #define TARGET_GIMPLIFY_VA_ARG_EXPR ix86_gimplify_va_arg
40516 #undef TARGET_SCALAR_MODE_SUPPORTED_P
40517 #define TARGET_SCALAR_MODE_SUPPORTED_P ix86_scalar_mode_supported_p
40519 #undef TARGET_VECTOR_MODE_SUPPORTED_P
40520 #define TARGET_VECTOR_MODE_SUPPORTED_P ix86_vector_mode_supported_p
40522 #undef TARGET_C_MODE_FOR_SUFFIX
40523 #define TARGET_C_MODE_FOR_SUFFIX ix86_c_mode_for_suffix
40525 #ifdef HAVE_AS_TLS
40526 #undef TARGET_ASM_OUTPUT_DWARF_DTPREL
40527 #define TARGET_ASM_OUTPUT_DWARF_DTPREL i386_output_dwarf_dtprel
40528 #endif
40530 #ifdef SUBTARGET_INSERT_ATTRIBUTES
40531 #undef TARGET_INSERT_ATTRIBUTES
40532 #define TARGET_INSERT_ATTRIBUTES SUBTARGET_INSERT_ATTRIBUTES
40533 #endif
40535 #undef TARGET_MANGLE_TYPE
40536 #define TARGET_MANGLE_TYPE ix86_mangle_type
40538 #if !TARGET_MACHO
40539 #undef TARGET_STACK_PROTECT_FAIL
40540 #define TARGET_STACK_PROTECT_FAIL ix86_stack_protect_fail
40541 #endif
40543 #undef TARGET_FUNCTION_VALUE
40544 #define TARGET_FUNCTION_VALUE ix86_function_value
40546 #undef TARGET_FUNCTION_VALUE_REGNO_P
40547 #define TARGET_FUNCTION_VALUE_REGNO_P ix86_function_value_regno_p
40549 #undef TARGET_PROMOTE_FUNCTION_MODE
40550 #define TARGET_PROMOTE_FUNCTION_MODE ix86_promote_function_mode
40552 #undef TARGET_SECONDARY_RELOAD
40553 #define TARGET_SECONDARY_RELOAD ix86_secondary_reload
40555 #undef TARGET_CLASS_MAX_NREGS
40556 #define TARGET_CLASS_MAX_NREGS ix86_class_max_nregs
40558 #undef TARGET_PREFERRED_RELOAD_CLASS
40559 #define TARGET_PREFERRED_RELOAD_CLASS ix86_preferred_reload_class
40560 #undef TARGET_PREFERRED_OUTPUT_RELOAD_CLASS
40561 #define TARGET_PREFERRED_OUTPUT_RELOAD_CLASS ix86_preferred_output_reload_class
40562 #undef TARGET_CLASS_LIKELY_SPILLED_P
40563 #define TARGET_CLASS_LIKELY_SPILLED_P ix86_class_likely_spilled_p
40565 #undef TARGET_VECTORIZE_BUILTIN_VECTORIZATION_COST
40566 #define TARGET_VECTORIZE_BUILTIN_VECTORIZATION_COST \
40567 ix86_builtin_vectorization_cost
40568 #undef TARGET_VECTORIZE_VEC_PERM_CONST_OK
40569 #define TARGET_VECTORIZE_VEC_PERM_CONST_OK \
40570 ix86_vectorize_vec_perm_const_ok
40571 #undef TARGET_VECTORIZE_PREFERRED_SIMD_MODE
40572 #define TARGET_VECTORIZE_PREFERRED_SIMD_MODE \
40573 ix86_preferred_simd_mode
40574 #undef TARGET_VECTORIZE_AUTOVECTORIZE_VECTOR_SIZES
40575 #define TARGET_VECTORIZE_AUTOVECTORIZE_VECTOR_SIZES \
40576 ix86_autovectorize_vector_sizes
40577 #undef TARGET_VECTORIZE_INIT_COST
40578 #define TARGET_VECTORIZE_INIT_COST ix86_init_cost
40579 #undef TARGET_VECTORIZE_ADD_STMT_COST
40580 #define TARGET_VECTORIZE_ADD_STMT_COST ix86_add_stmt_cost
40581 #undef TARGET_VECTORIZE_FINISH_COST
40582 #define TARGET_VECTORIZE_FINISH_COST ix86_finish_cost
40583 #undef TARGET_VECTORIZE_DESTROY_COST_DATA
40584 #define TARGET_VECTORIZE_DESTROY_COST_DATA ix86_destroy_cost_data
40586 #undef TARGET_SET_CURRENT_FUNCTION
40587 #define TARGET_SET_CURRENT_FUNCTION ix86_set_current_function
40589 #undef TARGET_OPTION_VALID_ATTRIBUTE_P
40590 #define TARGET_OPTION_VALID_ATTRIBUTE_P ix86_valid_target_attribute_p
40592 #undef TARGET_OPTION_SAVE
40593 #define TARGET_OPTION_SAVE ix86_function_specific_save
40595 #undef TARGET_OPTION_RESTORE
40596 #define TARGET_OPTION_RESTORE ix86_function_specific_restore
40598 #undef TARGET_OPTION_PRINT
40599 #define TARGET_OPTION_PRINT ix86_function_specific_print
40601 #undef TARGET_CAN_INLINE_P
40602 #define TARGET_CAN_INLINE_P ix86_can_inline_p
40604 #undef TARGET_EXPAND_TO_RTL_HOOK
40605 #define TARGET_EXPAND_TO_RTL_HOOK ix86_maybe_switch_abi
40607 #undef TARGET_LEGITIMATE_ADDRESS_P
40608 #define TARGET_LEGITIMATE_ADDRESS_P ix86_legitimate_address_p
40610 #undef TARGET_LEGITIMATE_CONSTANT_P
40611 #define TARGET_LEGITIMATE_CONSTANT_P ix86_legitimate_constant_p
40613 #undef TARGET_FRAME_POINTER_REQUIRED
40614 #define TARGET_FRAME_POINTER_REQUIRED ix86_frame_pointer_required
40616 #undef TARGET_CAN_ELIMINATE
40617 #define TARGET_CAN_ELIMINATE ix86_can_eliminate
40619 #undef TARGET_EXTRA_LIVE_ON_ENTRY
40620 #define TARGET_EXTRA_LIVE_ON_ENTRY ix86_live_on_entry
40622 #undef TARGET_ASM_CODE_END
40623 #define TARGET_ASM_CODE_END ix86_code_end
40625 #undef TARGET_CONDITIONAL_REGISTER_USAGE
40626 #define TARGET_CONDITIONAL_REGISTER_USAGE ix86_conditional_register_usage
40628 #if TARGET_MACHO
40629 #undef TARGET_INIT_LIBFUNCS
40630 #define TARGET_INIT_LIBFUNCS darwin_rename_builtins
40631 #endif
40634 \f