| blob | 2410236b93bc6b5225bfbc0faa8bfed5269d7ac8 |
1 /* Subroutines used for code generation on IA-32.
2 Copyright (C) 1988, 1992, 1994, 1995, 1996, 1997, 1998, 1999, 2000,
3 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012
4 Free Software Foundation, Inc.
6 This file is part of GCC.
8 GCC is free software; you can redistribute it and/or modify
9 it under the terms of the GNU General Public License as published by
10 the Free Software Foundation; either version 3, or (at your option)
11 any later version.
13 GCC is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU General Public License for more details.
18 You should have received a copy of the GNU General Public License
19 along with GCC; see the file COPYING3. If not see
20 <http://www.gnu.org/licenses/>. */
64 enum upper_128bits_state
65 {
67 unused,
68 used
69 };
72 {
73 /* State of the upper 128bits of AVX registers at exit. */
75 /* TRUE if state of the upper 128bits of AVX registers is unchanged
76 in this block. */
78 /* TRUE if block has been processed. */
80 /* TRUE if block has been scanned. */
82 /* Previous state of the upper 128bits of AVX registers at entry. */
86 #define BLOCK_INFO(B) ((block_info) (B)->aux)
88 enum call_avx256_state
89 {
90 /* Callee returns 256bit AVX register. */
92 /* Callee returns and passes 256bit AVX register. */
93 callee_return_pass_avx256,
94 /* Callee passes 256bit AVX register. */
95 callee_pass_avx256,
96 /* Callee doesn't return nor passe 256bit AVX register, or no
97 256bit AVX register in function return. */
98 call_no_avx256,
99 /* vzeroupper intrinsic. */
100 vzeroupper_intrinsic
101 };
103 /* Check if a 256bit AVX register is referenced in stores. */
105 static void
107 {
113 {
117 }
118 }
120 /* Helper function for move_or_delete_vzeroupper_1. Look for vzeroupper
121 in basic block BB. Delete it if upper 128bit AVX registers are
122 unused. If it isn't deleted, move it to just before a jump insn.
124 STATE is state of the upper 128bits of AVX registers at entry. */
126 static void
129 {
132 rtx pat;
137 {
144 }
147 {
152 }
162 /* BB_END changes when it is deleted. */
166 {
172 /* Move vzeroupper before jump/call. */
174 {
179 {
181 {
186 }
189 }
192 }
196 /* Check insn for vzeroupper intrinsic. */
199 {
201 {
202 /* Found vzeroupper intrinsic. */
205 }
206 }
207 else
208 {
209 /* Check insn for vzeroall intrinsic. */
213 {
217 /* Delete pending vzeroupper insertion. */
219 {
222 }
223 }
225 {
229 }
231 }
233 /* Process vzeroupper intrinsic. */
237 {
238 /* Since the upper 128bits are cleared, callee must not pass
239 256bit AVX register. We only need to check if callee
240 returns 256bit AVX register. */
242 {
245 }
247 /* Remove unnecessary vzeroupper since upper 128bits are
248 cleared. */
250 {
253 }
255 }
256 else
257 {
258 /* Set state to UNUSED if callee doesn't return 256bit AVX
259 register. */
265 {
266 /* Must remove vzeroupper since callee passes in 256bit
267 AVX register. */
269 {
272 }
274 }
275 else
276 {
279 }
280 }
281 }
290 state);
291 }
293 /* Helper function for move_or_delete_vzeroupper. Process vzeroupper
294 in BLOCK and check its predecessor blocks. Treat UNKNOWN state
295 as USED if UNKNOWN_IS_UNUSED is true. Return TRUE if the exit
296 state is changed. */
298 static bool
300 {
301 edge e;
302 edge_iterator ei;
315 /* Check all predecessor edges of this block. */
318 {
322 {
331 }
332 }
337 done:
345 /* Need to rescan if the upper 128bits of AVX registers are changed
346 to USED at exit. */
348 {
352 }
353 else
355 }
357 /* Go through the instruction stream looking for vzeroupper. Delete
358 it if upper 128bit AVX registers are unused. If it isn't deleted,
359 move it to just before a jump insn. */
361 static void
363 {
364 edge e;
365 edge_iterator ei;
366 basic_block bb;
373 /* Set up block info for each basic block. */
376 /* Process outgoing edges of entry point. */
381 {
386 }
388 /* Compute reverse completion order of depth first search of the CFG
389 so that the data-flow runs faster. */
404 /* Don't check outgoing edges of entry point. */
409 else
410 {
413 }
419 {
432 {
437 {
438 edge_iterator ei;
444 {
450 {
452 {
453 /* Send E->DEST to next round. */
458 }
459 }
461 {
462 /* Add E->DEST to current round. */
466 }
467 }
468 }
469 }
473 }
489 }
493 #ifndef CHECK_STACK_LIMIT
494 #define CHECK_STACK_LIMIT (-1)
495 #endif
497 /* Return index of given mode in mult and division cost tables. */
498 #define MODE_INDEX(mode) \
499 ((mode) == QImode ? 0 \
500 : (mode) == HImode ? 1 \
501 : (mode) == SImode ? 2 \
502 : (mode) == DImode ? 3 \
503 : 4)
505 /* Processor costs (relative to an add) */
506 /* We assume COSTS_N_INSNS is defined as (N)*4 and an addition is 2 bytes. */
507 #define COSTS_N_BYTES(N) ((N) * 2)
509 #define DUMMY_STRINGOP_ALGS {libcall, {{-1, libcall}}}
511 const
534 in QImode, HImode and SImode.
535 Relative to reg-reg move (2). */
539 in SFmode, DFmode and XFmode */
541 in SFmode, DFmode and XFmode */
544 in SImode and DImode */
546 in SImode and DImode */
549 in SImode, DImode and TImode */
551 in SImode, DImode and TImode */
579 };
581 /* Processor costs (relative to an add) */
582 static const
605 in QImode, HImode and SImode.
606 Relative to reg-reg move (2). */
610 in SFmode, DFmode and XFmode */
612 in SFmode, DFmode and XFmode */
615 in SImode and DImode */
617 in SImode and DImode */
620 in SImode, DImode and TImode */
622 in SImode, DImode and TImode */
636 DUMMY_STRINGOP_ALGS},
638 DUMMY_STRINGOP_ALGS},
650 };
652 static const
675 in QImode, HImode and SImode.
676 Relative to reg-reg move (2). */
680 in SFmode, DFmode and XFmode */
682 in SFmode, DFmode and XFmode */
685 in SImode and DImode */
687 in SImode and DImode */
690 in SImode, DImode and TImode */
692 in SImode, DImode and TImode */
695 shared for code and data, so 4kB is
696 not really precise. */
708 DUMMY_STRINGOP_ALGS},
710 DUMMY_STRINGOP_ALGS},
722 };
724 static const
747 in QImode, HImode and SImode.
748 Relative to reg-reg move (2). */
752 in SFmode, DFmode and XFmode */
754 in SFmode, DFmode and XFmode */
757 in SImode and DImode */
759 in SImode and DImode */
762 in SImode, DImode and TImode */
764 in SImode, DImode and TImode */
778 DUMMY_STRINGOP_ALGS},
780 DUMMY_STRINGOP_ALGS},
792 };
794 static const
817 in QImode, HImode and SImode.
818 Relative to reg-reg move (2). */
822 in SFmode, DFmode and XFmode */
824 in SFmode, DFmode and XFmode */
827 in SImode and DImode */
829 in SImode and DImode */
832 in SImode, DImode and TImode */
834 in SImode, DImode and TImode */
847 /* PentiumPro has optimized rep instructions for blocks aligned by 8 bytes
848 (we ensure the alignment). For small blocks inline loop is still a
849 noticeable win, for bigger blocks either rep movsl or rep movsb is
850 way to go. Rep movsb has apparently more expensive startup time in CPU,
851 but after 4K the difference is down in the noise. */
854 DUMMY_STRINGOP_ALGS},
857 DUMMY_STRINGOP_ALGS},
869 };
871 static const
894 in QImode, HImode and SImode.
895 Relative to reg-reg move (2). */
899 in SFmode, DFmode and XFmode */
901 in SFmode, DFmode and XFmode */
905 in SImode and DImode */
907 in SImode and DImode */
910 in SImode, DImode and TImode */
912 in SImode, DImode and TImode */
926 DUMMY_STRINGOP_ALGS},
928 DUMMY_STRINGOP_ALGS},
940 };
942 static const
965 in QImode, HImode and SImode.
966 Relative to reg-reg move (2). */
970 in SFmode, DFmode and XFmode */
972 in SFmode, DFmode and XFmode */
975 in SImode and DImode */
977 in SImode and DImode */
980 in SImode, DImode and TImode */
982 in SImode, DImode and TImode */
986 have integrated l2 cache, but
987 optimizing for k6 is not important
988 enough to worry about that. */
999 DUMMY_STRINGOP_ALGS},
1001 DUMMY_STRINGOP_ALGS},
1013 };
1015 static const
1038 in QImode, HImode and SImode.
1039 Relative to reg-reg move (2). */
1043 in SFmode, DFmode and XFmode */
1045 in SFmode, DFmode and XFmode */
1048 in SImode and DImode */
1050 in SImode and DImode */
1053 in SImode, DImode and TImode */
1055 in SImode, DImode and TImode */
1068 /* For some reason, Athlon deals better with REP prefix (relative to loops)
1069 compared to K8. Alignment becomes important after 8 bytes for memcpy and
1070 128 bytes for memset. */
1072 DUMMY_STRINGOP_ALGS},
1074 DUMMY_STRINGOP_ALGS},
1086 };
1088 static const
1111 in QImode, HImode and SImode.
1112 Relative to reg-reg move (2). */
1116 in SFmode, DFmode and XFmode */
1118 in SFmode, DFmode and XFmode */
1121 in SImode and DImode */
1123 in SImode and DImode */
1126 in SImode, DImode and TImode */
1128 in SImode, DImode and TImode */
1133 /* New AMD processors never drop prefetches; if they cannot be performed
1134 immediately, they are queued. We set number of simultaneous prefetches
1135 to a large constant to reflect this (it probably is not a good idea not
1136 to limit number of prefetches at all, as their execution also takes some
1137 time). */
1146 /* K8 has optimized REP instruction for medium sized blocks, but for very
1147 small blocks it is better to use loop. For large blocks, libcall can
1148 do nontemporary accesses and beat inline considerably. */
1165 };
1189 in QImode, HImode and SImode.
1190 Relative to reg-reg move (2). */
1194 in SFmode, DFmode and XFmode */
1196 in SFmode, DFmode and XFmode */
1199 in SImode and DImode */
1201 in SImode and DImode */
1204 in SImode, DImode and TImode */
1206 in SImode, DImode and TImode */
1208 /* On K8:
1209 MOVD reg64, xmmreg Double FSTORE 4
1210 MOVD reg32, xmmreg Double FSTORE 4
1211 On AMDFAM10:
1212 MOVD reg64, xmmreg Double FADD 3
1213 1/1 1/1
1214 MOVD reg32, xmmreg Double FADD 3
1215 1/1 1/1 */
1219 /* New AMD processors never drop prefetches; if they cannot be performed
1220 immediately, they are queued. We set number of simultaneous prefetches
1221 to a large constant to reflect this (it probably is not a good idea not
1222 to limit number of prefetches at all, as their execution also takes some
1223 time). */
1233 /* AMDFAM10 has optimized REP instruction for medium sized blocks, but for
1234 very small blocks it is better to use loop. For large blocks, libcall can
1235 do nontemporary accesses and beat inline considerably. */
1252 };
1276 in QImode, HImode and SImode.
1277 Relative to reg-reg move (2). */
1281 in SFmode, DFmode and XFmode */
1283 in SFmode, DFmode and XFmode */
1286 in SImode and DImode */
1288 in SImode and DImode */
1291 in SImode, DImode and TImode */
1293 in SImode, DImode and TImode */
1295 /* On K8:
1296 MOVD reg64, xmmreg Double FSTORE 4
1297 MOVD reg32, xmmreg Double FSTORE 4
1298 On AMDFAM10:
1299 MOVD reg64, xmmreg Double FADD 3
1300 1/1 1/1
1301 MOVD reg32, xmmreg Double FADD 3
1302 1/1 1/1 */
1306 /* New AMD processors never drop prefetches; if they cannot be performed
1307 immediately, they are queued. We set number of simultaneous prefetches
1308 to a large constant to reflect this (it probably is not a good idea not
1309 to limit number of prefetches at all, as their execution also takes some
1310 time). */
1320 /* BDVER1 has optimized REP instruction for medium sized blocks, but for
1321 very small blocks it is better to use loop. For large blocks, libcall
1322 can do nontemporary accesses and beat inline considerably. */
1339 };
1363 in QImode, HImode and SImode.
1364 Relative to reg-reg move (2). */
1368 in SFmode, DFmode and XFmode */
1370 in SFmode, DFmode and XFmode */
1373 in SImode and DImode */
1375 in SImode and DImode */
1378 in SImode, DImode and TImode */
1380 in SImode, DImode and TImode */
1382 /* On K8:
1383 MOVD reg64, xmmreg Double FSTORE 4
1384 MOVD reg32, xmmreg Double FSTORE 4
1385 On AMDFAM10:
1386 MOVD reg64, xmmreg Double FADD 3
1387 1/1 1/1
1388 MOVD reg32, xmmreg Double FADD 3
1389 1/1 1/1 */
1393 /* New AMD processors never drop prefetches; if they cannot be performed
1394 immediately, they are queued. We set number of simultaneous prefetches
1395 to a large constant to reflect this (it probably is not a good idea not
1396 to limit number of prefetches at all, as their execution also takes some
1397 time). */
1407 /* BDVER2 has optimized REP instruction for medium sized blocks, but for
1408 very small blocks it is better to use loop. For large blocks, libcall
1409 can do nontemporary accesses and beat inline considerably. */
1426 };
1450 in QImode, HImode and SImode.
1451 Relative to reg-reg move (2). */
1455 in SFmode, DFmode and XFmode */
1457 in SFmode, DFmode and XFmode */
1460 in SImode and DImode */
1462 in SImode and DImode */
1465 in SImode, DImode and TImode */
1467 in SImode, DImode and TImode */
1469 /* On K8:
1470 MOVD reg64, xmmreg Double FSTORE 4
1471 MOVD reg32, xmmreg Double FSTORE 4
1472 On AMDFAM10:
1473 MOVD reg64, xmmreg Double FADD 3
1474 1/1 1/1
1475 MOVD reg32, xmmreg Double FADD 3
1476 1/1 1/1 */
1489 /* BTVER1 has optimized REP instruction for medium sized blocks, but for
1490 very small blocks it is better to use loop. For large blocks, libcall can
1491 do nontemporary accesses and beat inline considerably. */
1508 };
1510 static const
1533 in QImode, HImode and SImode.
1534 Relative to reg-reg move (2). */
1538 in SFmode, DFmode and XFmode */
1540 in SFmode, DFmode and XFmode */
1543 in SImode and DImode */
1545 in SImode and DImode */
1548 in SImode, DImode and TImode */
1550 in SImode, DImode and TImode */
1564 DUMMY_STRINGOP_ALGS},
1567 DUMMY_STRINGOP_ALGS},
1579 };
1581 static const
1604 in QImode, HImode and SImode.
1605 Relative to reg-reg move (2). */
1609 in SFmode, DFmode and XFmode */
1611 in SFmode, DFmode and XFmode */
1614 in SImode and DImode */
1616 in SImode and DImode */
1619 in SImode, DImode and TImode */
1621 in SImode, DImode and TImode */
1652 };
1654 static const
1677 in QImode, HImode and SImode.
1678 Relative to reg-reg move (2). */
1682 in SFmode, DFmode and XFmode */
1684 in SFmode, DFmode and XFmode */
1687 in SImode and DImode */
1689 in SImode and DImode */
1692 in SImode, DImode and TImode */
1694 in SImode, DImode and TImode */
1725 };
1727 /* Generic64 should produce code tuned for Nocona and K8. */
1728 static const
1731 /* On all chips taken into consideration lea is 2 cycles and more. With
1732 this cost however our current implementation of synth_mult results in
1733 use of unnecessary temporary registers causing regression on several
1734 SPECfp benchmarks. */
1755 in QImode, HImode and SImode.
1756 Relative to reg-reg move (2). */
1760 in SFmode, DFmode and XFmode */
1762 in SFmode, DFmode and XFmode */
1765 in SImode and DImode */
1767 in SImode and DImode */
1770 in SImode, DImode and TImode */
1772 in SImode, DImode and TImode */
1778 /* Benchmarks shows large regressions on K8 sixtrack benchmark when this
1779 value is increased to perhaps more appropriate value of 5. */
1802 };
1804 /* Generic32 should produce code tuned for PPro, Pentium4, Nocona,
1805 Athlon and K8. */
1806 static const
1829 in QImode, HImode and SImode.
1830 Relative to reg-reg move (2). */
1834 in SFmode, DFmode and XFmode */
1836 in SFmode, DFmode and XFmode */
1839 in SImode and DImode */
1841 in SImode and DImode */
1844 in SImode, DImode and TImode */
1846 in SImode, DImode and TImode */
1860 DUMMY_STRINGOP_ALGS},
1862 DUMMY_STRINGOP_ALGS},
1874 };
1878 /* Processor feature/optimization bitmasks. */
1879 #define m_386 (1<<PROCESSOR_I386)
1880 #define m_486 (1<<PROCESSOR_I486)
1881 #define m_PENT (1<<PROCESSOR_PENTIUM)
1882 #define m_PPRO (1<<PROCESSOR_PENTIUMPRO)
1883 #define m_PENT4 (1<<PROCESSOR_PENTIUM4)
1884 #define m_NOCONA (1<<PROCESSOR_NOCONA)
1885 #define m_P4_NOCONA (m_PENT4 | m_NOCONA)
1886 #define m_CORE2_32 (1<<PROCESSOR_CORE2_32)
1887 #define m_CORE2_64 (1<<PROCESSOR_CORE2_64)
1888 #define m_COREI7_32 (1<<PROCESSOR_COREI7_32)
1889 #define m_COREI7_64 (1<<PROCESSOR_COREI7_64)
1890 #define m_COREI7 (m_COREI7_32 | m_COREI7_64)
1891 #define m_CORE2I7_32 (m_CORE2_32 | m_COREI7_32)
1892 #define m_CORE2I7_64 (m_CORE2_64 | m_COREI7_64)
1893 #define m_CORE2I7 (m_CORE2I7_32 | m_CORE2I7_64)
1894 #define m_ATOM (1<<PROCESSOR_ATOM)
1896 #define m_GEODE (1<<PROCESSOR_GEODE)
1897 #define m_K6 (1<<PROCESSOR_K6)
1898 #define m_K6_GEODE (m_K6 | m_GEODE)
1899 #define m_K8 (1<<PROCESSOR_K8)
1900 #define m_ATHLON (1<<PROCESSOR_ATHLON)
1901 #define m_ATHLON_K8 (m_K8 | m_ATHLON)
1902 #define m_AMDFAM10 (1<<PROCESSOR_AMDFAM10)
1903 #define m_BDVER1 (1<<PROCESSOR_BDVER1)
1904 #define m_BDVER2 (1<<PROCESSOR_BDVER2)
1905 #define m_BDVER (m_BDVER1 | m_BDVER2)
1906 #define m_BTVER1 (1<<PROCESSOR_BTVER1)
1907 #define m_AMD_MULTIPLE (m_ATHLON_K8 | m_AMDFAM10 | m_BDVER | m_BTVER1)
1909 #define m_GENERIC32 (1<<PROCESSOR_GENERIC32)
1910 #define m_GENERIC64 (1<<PROCESSOR_GENERIC64)
1912 /* Generic instruction choice should be common subset of supported CPUs
1913 (PPro/PENT4/NOCONA/CORE2/Athlon/K8). */
1914 #define m_GENERIC (m_GENERIC32 | m_GENERIC64)
1916 /* Feature tests against the various tunings. */
1919 /* Feature tests against the various tunings used to create ix86_tune_features
1920 based on the processor mask. */
1922 /* X86_TUNE_USE_LEAVE: Leave does not affect Nocona SPEC2000 results
1923 negatively, so enabling for Generic64 seems like good code size
1924 tradeoff. We can't enable it for 32bit generic because it does not
1925 work well with PPro base chips. */
1928 /* X86_TUNE_PUSH_MEMORY */
1931 /* X86_TUNE_ZERO_EXTEND_WITH_AND */
1934 /* X86_TUNE_UNROLL_STRLEN */
1937 /* X86_TUNE_BRANCH_PREDICTION_HINTS: Branch hints were put in P4 based
1938 on simulation result. But after P4 was made, no performance benefit
1939 was observed with branch hints. It also increases the code size.
1940 As a result, icc never generates branch hints. */
1943 /* X86_TUNE_DOUBLE_WITH_ADD */
1946 /* X86_TUNE_USE_SAHF */
1947 m_PPRO | m_P4_NOCONA | m_CORE2I7 | m_ATOM | m_K6_GEODE | m_K8 | m_AMDFAM10 | m_BDVER | m_BTVER1 | m_GENERIC,
1949 /* X86_TUNE_MOVX: Enable to zero extend integer registers to avoid
1950 partial dependencies. */
1953 /* X86_TUNE_PARTIAL_REG_STALL: We probably ought to watch for partial
1954 register stalls on Generic32 compilation setting as well. However
1955 in current implementation the partial register stalls are not eliminated
1956 very well - they can be introduced via subregs synthesized by combine
1957 and can happen in caller/callee saving sequences. Because this option
1958 pays back little on PPro based chips and is in conflict with partial reg
1959 dependencies used by Athlon/P4 based chips, it is better to leave it off
1960 for generic32 for now. */
1961 m_PPRO,
1963 /* X86_TUNE_PARTIAL_FLAG_REG_STALL */
1966 /* X86_TUNE_USE_HIMODE_FIOP */
1969 /* X86_TUNE_USE_SIMODE_FIOP */
1972 /* X86_TUNE_USE_MOV0 */
1973 m_K6,
1975 /* X86_TUNE_USE_CLTD */
1978 /* X86_TUNE_USE_XCHGB: Use xchgb %rh,%rl instead of rolw/rorw $8,rx. */
1979 m_PENT4,
1981 /* X86_TUNE_SPLIT_LONG_MOVES */
1982 m_PPRO,
1984 /* X86_TUNE_READ_MODIFY_WRITE */
1987 /* X86_TUNE_READ_MODIFY */
1990 /* X86_TUNE_PROMOTE_QIMODE */
1993 /* X86_TUNE_FAST_PREFIX */
1996 /* X86_TUNE_SINGLE_STRINGOP */
1999 /* X86_TUNE_QIMODE_MATH */
2002 /* X86_TUNE_HIMODE_MATH: On PPro this flag is meant to avoid partial
2003 register stalls. Just like X86_TUNE_PARTIAL_REG_STALL this option
2004 might be considered for Generic32 if our scheme for avoiding partial
2005 stalls was more effective. */
2008 /* X86_TUNE_PROMOTE_QI_REGS */
2011 /* X86_TUNE_PROMOTE_HI_REGS */
2012 m_PPRO,
2014 /* X86_TUNE_SINGLE_POP: Enable if single pop insn is preferred
2015 over esp addition. */
2018 /* X86_TUNE_DOUBLE_POP: Enable if double pop insn is preferred
2019 over esp addition. */
2020 m_PENT,
2022 /* X86_TUNE_SINGLE_PUSH: Enable if single push insn is preferred
2023 over esp subtraction. */
2026 /* X86_TUNE_DOUBLE_PUSH. Enable if double push insn is preferred
2027 over esp subtraction. */
2030 /* X86_TUNE_INTEGER_DFMODE_MOVES: Enable if integer moves are preferred
2031 for DFmode copies */
2034 /* X86_TUNE_PARTIAL_REG_DEPENDENCY */
2037 /* X86_TUNE_SSE_PARTIAL_REG_DEPENDENCY: In the Generic model we have a
2038 conflict here in between PPro/Pentium4 based chips that thread 128bit
2039 SSE registers as single units versus K8 based chips that divide SSE
2040 registers to two 64bit halves. This knob promotes all store destinations
2041 to be 128bit to allow register renaming on 128bit SSE units, but usually
2042 results in one extra microop on 64bit SSE units. Experimental results
2043 shows that disabling this option on P4 brings over 20% SPECfp regression,
2044 while enabling it on K8 brings roughly 2.4% regression that can be partly
2045 masked by careful scheduling of moves. */
2048 /* X86_TUNE_SSE_UNALIGNED_LOAD_OPTIMAL */
2051 /* X86_TUNE_SSE_UNALIGNED_STORE_OPTIMAL */
2054 /* X86_TUNE_SSE_PACKED_SINGLE_INSN_OPTIMAL */
2055 m_BDVER ,
2057 /* X86_TUNE_SSE_SPLIT_REGS: Set for machines where the type and dependencies
2058 are resolved on SSE register parts instead of whole registers, so we may
2059 maintain just lower part of scalar values in proper format leaving the
2060 upper part undefined. */
2061 m_ATHLON_K8,
2063 /* X86_TUNE_SSE_TYPELESS_STORES */
2064 m_AMD_MULTIPLE,
2066 /* X86_TUNE_SSE_LOAD0_BY_PXOR */
2069 /* X86_TUNE_MEMORY_MISMATCH_STALL */
2072 /* X86_TUNE_PROLOGUE_USING_MOVE */
2075 /* X86_TUNE_EPILOGUE_USING_MOVE */
2078 /* X86_TUNE_SHIFT1 */
2081 /* X86_TUNE_USE_FFREEP */
2082 m_AMD_MULTIPLE,
2084 /* X86_TUNE_INTER_UNIT_MOVES */
2087 /* X86_TUNE_INTER_UNIT_CONVERSIONS */
2090 /* X86_TUNE_FOUR_JUMP_LIMIT: Some CPU cores are not able to predict more
2091 than 4 branch instructions in the 16 byte window. */
2094 /* X86_TUNE_SCHEDULE */
2097 /* X86_TUNE_USE_BT */
2100 /* X86_TUNE_USE_INCDEC */
2103 /* X86_TUNE_PAD_RETURNS */
2106 /* X86_TUNE_PAD_SHORT_FUNCTION: Pad short funtion. */
2107 m_ATOM,
2109 /* X86_TUNE_EXT_80387_CONSTANTS */
2112 /* X86_TUNE_SHORTEN_X87_SSE */
2115 /* X86_TUNE_AVOID_VECTOR_DECODE */
2118 /* X86_TUNE_PROMOTE_HIMODE_IMUL: Modern CPUs have same latency for HImode
2119 and SImode multiply, but 386 and 486 do HImode multiply faster. */
2122 /* X86_TUNE_SLOW_IMUL_IMM32_MEM: Imul of 32-bit constant and memory is
2123 vector path on AMD machines. */
2126 /* X86_TUNE_SLOW_IMUL_IMM8: Imul of 8-bit constant is vector path on AMD
2127 machines. */
2130 /* X86_TUNE_MOVE_M1_VIA_OR: On pentiums, it is faster to load -1 via OR
2131 than a MOV. */
2132 m_PENT,
2134 /* X86_TUNE_NOT_UNPAIRABLE: NOT is not pairable on Pentium, while XOR is,
2135 but one byte longer. */
2136 m_PENT,
2138 /* X86_TUNE_NOT_VECTORMODE: On AMD K6, NOT is vector decoded with memory
2139 operand that cannot be represented using a modRM byte. The XOR
2140 replacement is long decoded, so this split helps here as well. */
2141 m_K6,
2143 /* X86_TUNE_USE_VECTOR_FP_CONVERTS: Prefer vector packed SSE conversion
2144 from FP to FP. */
2147 /* X86_TUNE_USE_VECTOR_CONVERTS: Prefer vector packed SSE conversion
2148 from integer to FP. */
2149 m_AMDFAM10,
2151 /* X86_TUNE_FUSE_CMP_AND_BRANCH: Fuse a compare or test instruction
2152 with a subsequent conditional jump instruction into a single
2153 compare-and-branch uop. */
2154 m_BDVER,
2156 /* X86_TUNE_OPT_AGU: Optimize for Address Generation Unit. This flag
2157 will impact LEA instruction selection. */
2158 m_ATOM,
2160 /* X86_TUNE_VECTORIZE_DOUBLE: Enable double precision vector
2161 instructions. */
2164 /* X86_SOFTARE_PREFETCHING_BENEFICIAL: Enable software prefetching
2165 at -O3. For the moment, the prefetching seems badly tuned for Intel
2166 chips. */
2169 /* X86_TUNE_AVX128_OPTIMAL: Enable 128-bit AVX instruction generation for
2170 the auto-vectorizer. */
2171 m_BDVER,
2173 /* X86_TUNE_REASSOC_INT_TO_PARALLEL: Try to produce parallel computations
2174 during reassociation of integer computation. */
2175 m_ATOM,
2177 /* X86_TUNE_REASSOC_FP_TO_PARALLEL: Try to produce parallel computations
2178 during reassociation of fp computation. */
2179 m_ATOM
2180 };
2182 /* Feature tests against the various architecture variations. */
2185 /* Feature tests against the various architecture variations, used to create
2186 ix86_arch_features based on the processor mask. */
2188 /* X86_ARCH_CMOVE: Conditional move was added for pentiumpro. */
2191 /* X86_ARCH_CMPXCHG: Compare and exchange was added for 80486. */
2194 /* X86_ARCH_CMPXCHG8B: Compare and exchange 8 bytes was added for pentium. */
2197 /* X86_ARCH_XADD: Exchange and add was added for 80486. */
2200 /* X86_ARCH_BSWAP: Byteswap was added for 80486. */
2202 };
2204 static const unsigned int x86_accumulate_outgoing_args
2207 static const unsigned int x86_arch_always_fancy_math_387
2210 static const unsigned int x86_avx256_split_unaligned_load
2213 static const unsigned int x86_avx256_split_unaligned_store
2216 /* In case the average insn count for single function invocation is
2217 lower than this constant, emit fast (but longer) prologue and
2218 epilogue code. */
2219 #define FAST_PROLOGUE_INSN_COUNT 20
2221 /* Names for 8 (low), 8 (high), and 16-bit registers, respectively. */
2226 /* Array of the smallest class containing reg number REGNO, indexed by
2227 REGNO. Used by REGNO_REG_CLASS in i386.h. */
2230 {
2231 /* ax, dx, cx, bx */
2233 /* si, di, bp, sp */
2235 /* FP registers */
2238 /* arg pointer */
2239 NON_Q_REGS,
2240 /* flags, fpsr, fpcr, frame */
2242 /* SSE registers */
2245 /* MMX registers */
2248 /* REX registers */
2251 /* SSE REX registers */
2254 };
2256 /* The "default" register map used in 32bit mode. */
2259 {
2267 };
2269 /* The "default" register map used in 64bit mode. */
2272 {
2280 };
2282 /* Define the register numbers to be used in Dwarf debugging information.
2283 The SVR4 reference port C compiler uses the following register numbers
2284 in its Dwarf output code:
2285 0 for %eax (gcc regno = 0)
2286 1 for %ecx (gcc regno = 2)
2287 2 for %edx (gcc regno = 1)
2288 3 for %ebx (gcc regno = 3)
2289 4 for %esp (gcc regno = 7)
2290 5 for %ebp (gcc regno = 6)
2291 6 for %esi (gcc regno = 4)
2292 7 for %edi (gcc regno = 5)
2293 The following three DWARF register numbers are never generated by
2294 the SVR4 C compiler or by the GNU compilers, but SDB on x86/svr4
2295 believes these numbers have these meanings.
2296 8 for %eip (no gcc equivalent)
2297 9 for %eflags (gcc regno = 17)
2298 10 for %trapno (no gcc equivalent)
2299 It is not at all clear how we should number the FP stack registers
2300 for the x86 architecture. If the version of SDB on x86/svr4 were
2301 a bit less brain dead with respect to floating-point then we would
2302 have a precedent to follow with respect to DWARF register numbers
2303 for x86 FP registers, but the SDB on x86/svr4 is so completely
2304 broken with respect to FP registers that it is hardly worth thinking
2305 of it as something to strive for compatibility with.
2306 The version of x86/svr4 SDB I have at the moment does (partially)
2307 seem to believe that DWARF register number 11 is associated with
2308 the x86 register %st(0), but that's about all. Higher DWARF
2309 register numbers don't seem to be associated with anything in
2310 particular, and even for DWARF regno 11, SDB only seems to under-
2311 stand that it should say that a variable lives in %st(0) (when
2312 asked via an `=' command) if we said it was in DWARF regno 11,
2313 but SDB still prints garbage when asked for the value of the
2314 variable in question (via a `/' command).
2315 (Also note that the labels SDB prints for various FP stack regs
2316 when doing an `x' command are all wrong.)
2317 Note that these problems generally don't affect the native SVR4
2318 C compiler because it doesn't allow the use of -O with -g and
2319 because when it is *not* optimizing, it allocates a memory
2320 location for each floating-point variable, and the memory
2321 location is what gets described in the DWARF AT_location
2322 attribute for the variable in question.
2323 Regardless of the severe mental illness of the x86/svr4 SDB, we
2324 do something sensible here and we use the following DWARF
2325 register numbers. Note that these are all stack-top-relative
2326 numbers.
2327 11 for %st(0) (gcc regno = 8)
2328 12 for %st(1) (gcc regno = 9)
2329 13 for %st(2) (gcc regno = 10)
2330 14 for %st(3) (gcc regno = 11)
2331 15 for %st(4) (gcc regno = 12)
2332 16 for %st(5) (gcc regno = 13)
2333 17 for %st(6) (gcc regno = 14)
2334 18 for %st(7) (gcc regno = 15)
2335 */
2337 {
2345 };
2347 /* Define parameter passing and return registers. */
2350 {
2352 };
2355 {
2357 };
2360 {
2362 };
2364 /* Define the structure for the machine field in struct function. */
2369 rtx rtl;
2371 };
2373 /* Structure describing stack frame layout.
2374 Stack grows downward:
2376 [arguments]
2377 <- ARG_POINTER
2378 saved pc
2380 saved static chain if ix86_static_chain_on_stack
2382 saved frame pointer if frame_pointer_needed
2383 <- HARD_FRAME_POINTER
2384 [saved regs]
2385 <- regs_save_offset
2386 [padding0]
2388 [saved SSE regs]
2389 <- sse_regs_save_offset
2390 [padding1] |
2391 | <- FRAME_POINTER
2392 [va_arg registers] |
2393 |
2394 [frame] |
2395 |
2396 [padding2] | = to_allocate
2397 <- STACK_POINTER
2398 */
2399 struct ix86_frame
2400 {
2406 HOST_WIDE_INT frame;
2408 /* The offsets relative to ARG_POINTER. */
2409 HOST_WIDE_INT frame_pointer_offset;
2410 HOST_WIDE_INT hard_frame_pointer_offset;
2411 HOST_WIDE_INT stack_pointer_offset;
2412 HOST_WIDE_INT hfp_save_offset;
2413 HOST_WIDE_INT reg_save_offset;
2414 HOST_WIDE_INT sse_reg_save_offset;
2416 /* When save_regs_using_mov is set, emit prologue using
2417 move instead of push instructions. */
2419 };
2421 /* Which cpu are we scheduling for. */
2424 /* Which cpu are we optimizing for. */
2427 /* Which instruction set architecture to use. */
2430 /* true if sse prefetch instruction is not NOOP. */
2433 /* -mstackrealign option */
2450 /* Preferred alignment for stack boundary in bits. */
2453 /* Alignment for incoming stack boundary in bits specified at
2454 command line. */
2457 /* Default alignment for incoming stack boundary in bits. */
2460 /* Alignment for incoming stack boundary in bits. */
2463 /* Calling abi specific va_list type nodes. */
2467 /* Prefix built by ASM_GENERATE_INTERNAL_LABEL. */
2471 /* Fence to use after loop using movnt. */
2472 tree x86_mfence;
2474 /* Register class used for passing given 64bit part of the argument.
2475 These represent classes as documented by the PS ABI, with the exception
2476 of SSESF, SSEDF classes, that are basically SSE class, just gcc will
2477 use SF or DFmode move instead of DImode to avoid reformatting penalties.
2479 Similarly we play games with INTEGERSI_CLASS to use cheaper SImode moves
2480 whenever possible (upper half does contain padding). */
2481 enum x86_64_reg_class
2482 {
2483 X86_64_NO_CLASS,
2484 X86_64_INTEGER_CLASS,
2485 X86_64_INTEGERSI_CLASS,
2486 X86_64_SSE_CLASS,
2487 X86_64_SSESF_CLASS,
2488 X86_64_SSEDF_CLASS,
2489 X86_64_SSEUP_CLASS,
2490 X86_64_X87_CLASS,
2491 X86_64_X87UP_CLASS,
2492 X86_64_COMPLEX_X87_CLASS,
2493 X86_64_MEMORY_CLASS
2494 };
2496 #define MAX_CLASSES 4
2498 /* Table of constants used by fldpi, fldln2, etc.... */
2502 \f
2507 const_tree);
2519 enum ix86_function_specific_strings
2520 {
2521 IX86_FUNCTION_SPECIFIC_ARCH,
2522 IX86_FUNCTION_SPECIFIC_TUNE,
2523 IX86_FUNCTION_SPECIFIC_MAX
2524 };
2542 \f
2543 #ifndef SUBTARGET32_DEFAULT_CPU
2545 #endif
2547 /* The svr4 ABI for the i386 says that records and unions are returned
2548 in memory. */
2549 #ifndef DEFAULT_PCC_STRUCT_RETURN
2550 #define DEFAULT_PCC_STRUCT_RETURN 1
2551 #endif
2553 /* Whether -mtune= or -march= were specified */
2557 /* Vectorization library interface and handlers. */
2563 /* Processor target table, indexed by processor number */
2564 struct ptt
2565 {
2572 };
2575 {
2586 /* Core 2 32-bit. */
2588 /* Core 2 64-bit. */
2590 /* Core i7 32-bit. */
2592 /* Core i7 64-bit. */
2601 };
2604 {
2630 "btver1"
2631 };
2632 \f
2633 /* Return true if a red-zone is in use. */
2637 {
2639 }
2640 \f
2641 /* Return a string that documents the current -m options. The caller is
2642 responsible for freeing the string. */
2648 {
2649 struct ix86_target_opts
2650 {
2653 };
2655 /* This table is ordered so that options like -msse4.2 that imply
2656 preceding options while match those first. */
2658 {
2688 };
2690 /* Flag options. */
2692 {
2719 };
2735 /* Add -march= option. */
2737 {
2740 }
2742 /* Add -mtune= option. */
2744 {
2747 }
2749 /* Pick out the options in isa options. */
2751 {
2753 {
2756 }
2757 }
2760 {
2763 isa);
2764 }
2766 /* Add flag options. */
2768 {
2770 {
2773 }
2774 }
2777 {
2780 }
2782 /* Add -fpmath= option. */
2784 {
2787 {
2802 }
2803 }
2805 /* Any options? */
2811 /* Size the string. */
2815 {
2820 }
2822 /* Build the string. */
2827 {
2834 {
2836 line_len++;
2839 {
2843 }
2844 }
2848 {
2852 }
2853 }
2859 }
2861 /* Return true, if profiling code should be emitted before
2862 prologue. Otherwise it returns false.
2863 Note: For x86 with "hotfix" it is sorried. */
2864 static bool
2866 {
2868 }
2870 /* Function that is callable from the debugger to print the current
2871 options. */
2872 void
2874 {
2880 {
2883 }
2884 else
2888 }
2889 \f
2890 /* Override various settings based on options. If MAIN_ARGS_P, the
2891 options are from the command line, otherwise they are from
2892 attributes. */
2894 static void
2896 {
2904 #define PTA_3DNOW (HOST_WIDE_INT_1 << 0)
2905 #define PTA_3DNOW_A (HOST_WIDE_INT_1 << 1)
2906 #define PTA_64BIT (HOST_WIDE_INT_1 << 2)
2907 #define PTA_ABM (HOST_WIDE_INT_1 << 3)
2908 #define PTA_AES (HOST_WIDE_INT_1 << 4)
2909 #define PTA_AVX (HOST_WIDE_INT_1 << 5)
2910 #define PTA_BMI (HOST_WIDE_INT_1 << 6)
2911 #define PTA_CX16 (HOST_WIDE_INT_1 << 7)
2912 #define PTA_F16C (HOST_WIDE_INT_1 << 8)
2913 #define PTA_FMA (HOST_WIDE_INT_1 << 9)
2914 #define PTA_FMA4 (HOST_WIDE_INT_1 << 10)
2915 #define PTA_FSGSBASE (HOST_WIDE_INT_1 << 11)
2916 #define PTA_LWP (HOST_WIDE_INT_1 << 12)
2917 #define PTA_LZCNT (HOST_WIDE_INT_1 << 13)
2918 #define PTA_MMX (HOST_WIDE_INT_1 << 14)
2919 #define PTA_MOVBE (HOST_WIDE_INT_1 << 15)
2920 #define PTA_NO_SAHF (HOST_WIDE_INT_1 << 16)
2921 #define PTA_PCLMUL (HOST_WIDE_INT_1 << 17)
2922 #define PTA_POPCNT (HOST_WIDE_INT_1 << 18)
2923 #define PTA_PREFETCH_SSE (HOST_WIDE_INT_1 << 19)
2924 #define PTA_RDRND (HOST_WIDE_INT_1 << 20)
2925 #define PTA_SSE (HOST_WIDE_INT_1 << 21)
2926 #define PTA_SSE2 (HOST_WIDE_INT_1 << 22)
2927 #define PTA_SSE3 (HOST_WIDE_INT_1 << 23)
2928 #define PTA_SSE4_1 (HOST_WIDE_INT_1 << 24)
2929 #define PTA_SSE4_2 (HOST_WIDE_INT_1 << 25)
2930 #define PTA_SSE4A (HOST_WIDE_INT_1 << 26)
2931 #define PTA_SSSE3 (HOST_WIDE_INT_1 << 27)
2932 #define PTA_TBM (HOST_WIDE_INT_1 << 28)
2933 #define PTA_XOP (HOST_WIDE_INT_1 << 29)
2934 #define PTA_AVX2 (HOST_WIDE_INT_1 << 30)
2935 #define PTA_BMI2 (HOST_WIDE_INT_1 << 31)
2936 #define PTA_RTM (HOST_WIDE_INT_1 << 32)
2937 /* if this reaches 64, need to widen struct pta flags below */
2939 static struct pta
2940 {
2945 }
2947 {
3061 };
3063 /* -mrecip options. */
3064 static struct
3065 {
3068 }
3070 {
3077 };
3081 /* Set up prefix/suffix so the error messages refer to either the command
3082 line argument, or the attribute(target). */
3084 {
3088 }
3089 else
3090 {
3094 }
3096 #ifdef SUBTARGET_OVERRIDE_OPTIONS
3097 SUBTARGET_OVERRIDE_OPTIONS;
3098 #endif
3100 #ifdef SUBSUBTARGET_OVERRIDE_OPTIONS
3101 SUBSUBTARGET_OVERRIDE_OPTIONS;
3102 #endif
3107 /* -fPIC is the default for x86_64. */
3111 /* Need to check -mtune=generic first. */
3113 {
3116 /* As special support for cross compilers we read -mtune=native
3117 as -mtune=generic. With native compilers we won't see the
3118 -mtune=native, as it was changed by the driver. */
3120 {
3123 else
3125 }
3126 /* If this call is for setting the option attribute, allow the
3127 generic32/generic64 that was previously set. */
3131 ;
3139 }
3140 else
3141 {
3145 {
3148 }
3150 /* ix86_tune_string is set to ix86_arch_string or defaulted. We
3151 need to use a sensible tune option. */
3155 {
3158 else
3160 }
3161 }
3164 {
3165 /* rep; movq isn't available in 32-bit code. */
3168 }
3172 else
3176 {
3182 }
3183 else
3190 {
3192 {
3236 {
3239 }
3247 }
3248 }
3249 else
3250 {
3251 /* For TARGET_64BIT and MS_ABI, force pic on, in order to enable the
3252 use of rip-relative addressing. This eliminates fixups that
3253 would otherwise be needed if this object is to be placed in a
3254 DLL, and is essentially just as efficient as direct addressing. */
3259 else
3261 }
3263 {
3266 }
3273 {
3276 /* Default cpu tuning to the architecture. */
3380 }
3395 {
3399 {
3401 {
3403 {
3411 }
3412 else
3415 }
3416 }
3417 else
3418 {
3419 /* Adjust tuning when compiling for 32-bit ABI. */
3421 {
3437 }
3438 }
3439 /* Intel CPUs have always interpreted SSE prefetch instructions as
3440 NOPs; so, we can enable SSE prefetch instructions even when
3441 -mtune (rather than -march) points us to a processor that has them.
3442 However, the VIA C3 gives a SIGILL, so we only do that for i686 and
3443 higher processors. */
3448 }
3458 #ifndef USE_IX86_FRAME_POINTER
3459 #define USE_IX86_FRAME_POINTER 0
3460 #endif
3462 #ifndef USE_X86_64_FRAME_POINTER
3463 #define USE_X86_64_FRAME_POINTER 0
3464 #endif
3466 /* Set the default values for switches whose default depends on TARGET_64BIT
3467 in case they weren't overwritten by command line options. */
3469 {
3476 }
3477 else
3478 {
3485 }
3489 else
3492 /* Arrange to set up i386_stack_locals for all functions. */
3495 /* Validate -mregparm= value. */
3497 {
3501 {
3505 }
3506 }
3510 /* Default align_* from the processor table. */
3512 {
3515 }
3517 {
3520 }
3522 {
3524 }
3526 /* Provide default for -mbranch-cost= value. */
3531 {
3534 /* Enable by default the SSE and MMX builtins. Do allow the user to
3535 explicitly disable any of these. In particular, disabling SSE and
3536 MMX for kernel code is extremely useful. */
3538 ix86_isa_flags
3544 }
3545 else
3546 {
3550 ix86_isa_flags
3553 /* i386 ABI does not specify red zone. It still makes sense to use it
3554 when programmer takes care to stack from being destroyed. */
3557 }
3559 /* Keep nonleaf frame pointers. */
3565 /* If we're doing fast math, we don't care about comparison order
3566 wrt NaNs. This lets us use a shorter comparison sequence. */
3570 /* If the architecture always has an FPU, turn off NO_FANCY_MATH_387,
3571 since the insns won't need emulation. */
3575 /* Likewise, if the target doesn't have a 387, or we've specified
3576 software floating point, don't use 387 inline intrinsics. */
3580 /* Turn on MMX builtins for -msse. */
3582 {
3585 }
3587 /* Turn on popcnt instruction for -msse4.2 or -mabm. */
3591 /* Turn on lzcnt instruction for -mabm. */
3595 /* Validate -mpreferred-stack-boundary= value or default it to
3596 PREFERRED_STACK_BOUNDARY_DEFAULT. */
3599 {
3605 {
3609 else
3612 }
3613 else
3614 ix86_preferred_stack_boundary
3616 }
3618 /* Set the default value for -mstackrealign. */
3624 /* Validate -mincoming-stack-boundary= value or default it to
3625 MIN_STACK_BOUNDARY/PREFERRED_STACK_BOUNDARY. */
3628 {
3633 else
3634 {
3635 ix86_user_incoming_stack_boundary
3637 ix86_incoming_stack_boundary
3639 }
3640 }
3642 /* Accept -msseregparm only if at least SSE support is enabled. */
3648 {
3650 {
3652 {
3655 }
3657 {
3660 }
3661 }
3662 }
3663 else
3666 /* If the i387 is disabled, then do not return values in it. */
3670 /* Use external vectorized library in vectorizing intrinsics. */
3673 {
3684 }
3692 /* ??? Unwind info is not correct around the CFG unless either a frame
3693 pointer is present or M_A_O_A is set. Fixing this requires rewriting
3694 unwind info generation to be aware of the CFG and propagating states
3695 around edges. */
3698 && flag_omit_frame_pointer
3700 {
3706 }
3708 /* If stack probes are required, the space used for large function
3709 arguments on the stack must also be probed, so enable
3710 -maccumulate-outgoing-args so this happens in the prologue. */
3713 {
3718 }
3720 /* For sane SSE instruction set generation we need fcomi instruction.
3721 It is safe to enable all CMOVE instructions. Also, RDRAND intrinsic
3722 expands to a sequence that includes conditional move. */
3726 /* Figure out what ASM_GENERATE_INTERNAL_LABEL builds as a prefix. */
3727 {
3733 }
3735 /* When scheduling description is not available, disable scheduler pass
3736 so it won't slow down the compilation and make x87 code slower. */
3754 /* Enable sw prefetching at -O3 for CPUS that prefetching is helpful. */
3756 && HAVE_prefetch
3761 /* If using typedef char *va_list, signal that __builtin_va_start (&ap, 0)
3762 can be optimized to ap = __builtin_next_arg (0). */
3767 {
3770 {
3773 ix86_gen_tls_local_dynamic_base_64
3775 }
3776 else
3777 {
3780 ix86_gen_tls_local_dynamic_base_64
3782 }
3783 }
3784 else
3785 {
3788 }
3791 {
3800 }
3801 else
3802 {
3811 }
3813 #ifdef USE_IX86_CLD
3814 /* Use -mcld by default for 32-bit code if configured with --enable-cld. */
3817 #endif
3820 {
3825 }
3827 {
3831 }
3833 {
3834 #if defined(PROFILE_BEFORE_PROLOGUE)
3836 #else
3838 #endif
3839 }
3842 {
3843 /* When not optimize for size, enable vzeroupper optimization for
3844 TARGET_AVX with -fexpensive-optimizations and split 32-byte
3845 AVX unaligned load/store. */
3847 {
3857 /* Enable 128-bit AVX instruction generation for the auto-vectorizer. */
3860 }
3861 }
3862 else
3863 {
3864 /* Disable vzeroupper pass if TARGET_AVX is disabled. */
3866 }
3869 {
3876 {
3879 {
3881 q++;
3882 }
3883 else
3888 else
3889 {
3892 {
3895 }
3898 {
3902 }
3903 }
3908 else
3910 }
3911 }
3918 /* Save the initial options in case the user does function specific
3919 options. */
3921 target_option_default_node = target_option_current_node
3923 }
3925 /* Return TRUE if VAL is passed in register with 256bit AVX modes. */
3927 static bool
3929 {
3937 {
3939 rtx r;
3942 {
3950 }
3951 }
3954 }
3956 /* Implement the TARGET_OPTION_OVERRIDE hook. */
3958 static void
3960 {
3962 }
3964 /* Update register usage after having seen the compiler flags. */
3966 static void
3968 {
3973 {
3978 }
3980 /* The PIC register, if it exists, is fixed. */
3985 /* The 64-bit MS_ABI changes the set of call-used registers. */
3987 {
3994 }
3996 /* The default setting of CLOBBERED_REGS is for 32-bit; add in the
3997 other call-clobbered regs for 64-bit. */
3999 {
4006 }
4008 /* If MMX is disabled, squash the registers. */
4014 /* If SSE is disabled, squash the registers. */
4020 /* If the FPU is disabled, squash the registers. */
4026 /* If 32-bit, squash the 64-bit registers. */
4028 {
4033 }
4034 }
4036 \f
4037 /* Save the current options */
4039 static void
4041 {
4052 /* The fields are char but the variables are not; make sure the
4053 values fit in the fields. */
4058 }
4060 /* Restore the current options */
4062 static void
4064 {
4080 /* Recreate the arch feature tests if the arch changed */
4082 {
4087 }
4089 /* Recreate the tune optimization tests */
4091 {
4096 }
4097 }
4099 /* Print the current options */
4101 static void
4104 {
4126 {
4129 }
4130 }
4132 \f
4133 /* Inner function to process the attribute((target(...))), take an argument and
4134 set the current options from the argument. If we have a list, recursively go
4135 over the list. */
4137 static bool
4140 {
4144 #define IX86_ATTR_ISA(S,O) { S, sizeof (S)-1, ix86_opt_isa, O, 0 }
4145 #define IX86_ATTR_STR(S,O) { S, sizeof (S)-1, ix86_opt_str, O, 0 }
4146 #define IX86_ATTR_ENUM(S,O) { S, sizeof (S)-1, ix86_opt_enum, O, 0 }
4147 #define IX86_ATTR_YES(S,O,M) { S, sizeof (S)-1, ix86_opt_yes, O, M }
4148 #define IX86_ATTR_NO(S,O,M) { S, sizeof (S)-1, ix86_opt_no, O, M }
4150 enum ix86_opt_type
4151 {
4152 ix86_opt_unknown,
4153 ix86_opt_yes,
4154 ix86_opt_no,
4155 ix86_opt_str,
4156 ix86_opt_enum,
4157 ix86_opt_isa
4158 };
4160 static const struct
4161 {
4168 /* isa options */
4198 /* enum options */
4201 /* string options */
4205 /* flag options */
4207 OPT_mcld,
4208 MASK_CLD),
4211 OPT_mfancy_math_387,
4212 MASK_NO_FANCY_MATH_387),
4215 OPT_mieee_fp,
4216 MASK_IEEE_FP),
4219 OPT_minline_all_stringops,
4220 MASK_INLINE_ALL_STRINGOPS),
4223 OPT_minline_stringops_dynamically,
4224 MASK_INLINE_STRINGOPS_DYNAMICALLY),
4227 OPT_mno_align_stringops,
4228 MASK_NO_ALIGN_STRINGOPS),
4231 OPT_mrecip,
4232 MASK_RECIP),
4234 };
4236 /* If this is a list, recurse to get the options. */
4238 {
4248 }
4253 /* Handle multiple arguments separated by commas. */
4257 {
4271 {
4275 }
4276 else
4277 {
4280 }
4282 /* Recognize no-xxx. */
4284 {
4288 }
4289 else
4292 /* Find the option. */
4296 {
4304 {
4309 }
4310 }
4312 /* Process the option. */
4314 {
4317 }
4320 {
4326 }
4329 {
4335 else
4337 }
4340 {
4342 {
4345 }
4346 else
4348 }
4351 {
4359 global_dc);
4360 else
4361 {
4364 }
4365 }
4367 else
4369 }
4372 }
4374 /* Return a TARGET_OPTION_NODE tree of the target options listed or NULL. */
4376 tree
4378 {
4393 /* Process each of the options on the chain. */
4398 /* If the changed options are different from the default, rerun
4399 ix86_option_override_internal, and then save the options away.
4400 The string options are are attribute options, and will be undone
4401 when we copy the save structure. */
4407 {
4408 /* If we are using the default tune= or arch=, undo the string assigned,
4409 and use the default. */
4420 /* If fpmath= is not set, and we now have sse2 on 32-bit, use it. */
4424 {
4427 }
4429 /* Do any overrides, such as arch=xxx, or tune=xxx support. */
4432 /* Add any builtin functions with the new isa if any. */
4435 /* Save the current options unless we are validating options for
4436 #pragma. */
4443 /* Free up memory allocated to hold the strings */
4446 }
4449 }
4451 /* Hook to validate attribute((target("string"))). */
4453 static bool
4456 tree args,
4458 {
4465 /* If the function changed the optimization levels as well as setting target
4466 options, start with the optimizations specified. */
4471 /* The target attributes may also change some optimization flags, so update
4472 the optimization options if necessary. */
4481 {
4486 }
4495 }
4497 \f
4498 /* Hook to determine if one function can safely inline another. */
4500 static bool
4502 {
4507 /* If callee has no option attributes, then it is ok to inline. */
4511 /* If caller has no option attributes, but callee does then it is not ok to
4512 inline. */
4516 else
4517 {
4521 /* Callee's isa options should a subset of the caller's, i.e. a SSE4 function
4522 can inline a SSE2 function but a SSE2 function can't inline a SSE4
4523 function. */
4528 /* See if we have the same non-isa options. */
4532 /* See if arch, tune, etc. are the same. */
4545 else
4547 }
4550 }
4552 \f
4553 /* Remember the last target of ix86_set_current_function. */
4556 /* Establish appropriate back-end context for processing the function
4557 FNDECL. The argument might be NULL to indicate processing at top
4558 level, outside of any function scope. */
4559 static void
4561 {
4562 /* Only change the context if the function changes. This hook is called
4563 several times in the course of compiling a function, and we don't want to
4564 slow things down too much or call target_reinit when it isn't safe. */
4566 {
4567 tree old_tree = (ix86_previous_fndecl
4571 tree new_tree = (fndecl
4577 ;
4580 {
4584 }
4587 {
4593 }
4594 }
4595 }
4597 \f
4598 /* Return true if this goes in large data/bss. */
4600 static bool
4602 {
4606 /* Functions are never large data. */
4611 {
4617 }
4618 else
4619 {
4622 /* If this is an incomplete type with size 0, then we can't put it
4623 in data because it might be too big when completed. */
4626 }
4629 }
4631 /* Switch to the appropriate section for output of DECL.
4632 DECL is either a `VAR_DECL' node or a constant of some sort.
4633 RELOC indicates whether forming the initial value of DECL requires
4634 link-time relocations. */
4637 ATTRIBUTE_UNUSED;
4642 {
4645 {
4649 {
4683 /* We don't split these for medium model. Place them into
4684 default sections and hope for best. */
4686 }
4688 {
4689 /* We might get called with string constants, but get_named_section
4690 doesn't like them as they are not DECLs. Also, we need to set
4691 flags in that case. */
4695 }
4696 }
4698 }
4700 /* Build up a unique section name, expressed as a
4701 STRING_CST node, and assign it to DECL_SECTION_NAME (decl).
4702 RELOC indicates whether the initial value of EXP requires
4703 link-time relocations. */
4705 static void ATTRIBUTE_UNUSED
4707 {
4710 {
4712 /* We only need to use .gnu.linkonce if we don't have COMDAT groups. */
4716 {
4740 /* We don't split these for medium model. Place them into
4741 default sections and hope for best. */
4743 }
4745 {
4752 /* If we're using one_only, then there needs to be a .gnu.linkonce
4753 prefix to the section name. */
4760 }
4761 }
4763 }
4765 #ifdef COMMON_ASM_OP
4766 /* This says how to output assembler code to declare an
4767 uninitialized external linkage data object.
4769 For medium model x86-64 we need to use .largecomm opcode for
4770 large objects. */
4771 void
4775 {
4779 else
4784 }
4785 #endif
4787 /* Utility function for targets to use in implementing
4788 ASM_OUTPUT_ALIGNED_BSS. */
4790 void
4794 {
4798 else
4801 #ifdef ASM_DECLARE_OBJECT_NAME
4804 #else
4805 /* Standard thing is just output label for the object. */
4809 }
4810 \f
4811 /* Decide whether we must probe the stack before any space allocation
4812 on this target. It's essentially TARGET_STACK_PROBE except when
4813 -fstack-check causes the stack to be already probed differently. */
4815 bool
4817 {
4818 /* Do not probe the stack twice if static stack checking is enabled. */
4823 }
4824 \f
4825 /* Decide whether we can make a sibling call to a function. DECL is the
4826 declaration of the function being targeted by the call and EXP is the
4827 CALL_EXPR representing the call. */
4829 static bool
4831 {
4835 /* If we are generating position-independent code, we cannot sibcall
4836 optimize any indirect call, or a direct call to a global function,
4837 as the PLT requires %ebx be live. (Darwin does not have a PLT.) */
4839 && !TARGET_64BIT
4840 && flag_pic
4844 /* If we need to align the outgoing stack, then sibcalling would
4845 unalign the stack, which may break the called function. */
4851 {
4854 }
4855 else
4856 {
4857 /* We're looking at the CALL_EXPR, we need the type of the function. */
4862 }
4864 /* Check that the return value locations are the same. Like
4865 if we are returning floats on the 80387 register stack, we cannot
4866 make a sibcall from a function that doesn't return a float to a
4867 function that does or, conversely, from a function that does return
4868 a float to a function that doesn't; the necessary stack adjustment
4869 would not be executed. This is also the place we notice
4870 differences in the return value ABI. Note that it is ok for one
4871 of the functions to have void return type as long as the return
4872 value of the other is passed in a register. */
4877 {
4880 }
4882 {
4883 /* Disable sibcall if we need to generate vzeroupper after
4884 callee returns. */
4889 }
4894 {
4895 /* The SYSV ABI has more call-clobbered registers;
4896 disallow sibcalls from MS to SYSV. */
4900 }
4901 else
4902 {
4903 /* If this call is indirect, we'll need to be able to use a
4904 call-clobbered register for the address of the target function.
4905 Make sure that all such registers are not used for passing
4906 parameters. Note that DLLIMPORT functions are indirect. */
4909 {
4911 {
4912 /* ??? Need to count the actual number of registers to be used,
4913 not the possible number of registers. Fix later. */
4915 }
4916 }
4917 }
4919 /* Otherwise okay. That also includes certain types of indirect calls. */
4921 }
4923 /* Handle "cdecl", "stdcall", "fastcall", "regparm", "thiscall",
4924 and "sseregparm" calling convention attributes;
4925 arguments as in struct attribute_spec.handler. */
4927 static tree
4929 tree args,
4932 {
4937 {
4939 name);
4942 }
4944 /* Can combine regparm with all attributes but fastcall, and thiscall. */
4946 {
4947 tree cst;
4950 {
4952 }
4955 {
4957 }
4961 {
4964 name);
4966 }
4968 {
4972 }
4975 }
4978 {
4979 /* Do not warn when emulating the MS ABI. */
4984 name);
4987 }
4989 /* Can combine fastcall with stdcall (redundant) and sseregparm. */
4991 {
4993 {
4995 }
4997 {
4999 }
5001 {
5003 }
5005 {
5007 }
5008 }
5010 /* Can combine stdcall with fastcall (redundant), regparm and
5011 sseregparm. */
5013 {
5015 {
5017 }
5019 {
5021 }
5023 {
5025 }
5026 }
5028 /* Can combine cdecl with regparm and sseregparm. */
5030 {
5032 {
5034 }
5036 {
5038 }
5040 {
5042 }
5043 }
5045 {
5048 name);
5050 {
5052 }
5054 {
5056 }
5058 {
5060 }
5061 }
5063 /* Can combine sseregparm with all attributes. */
5066 }
5068 /* The transactional memory builtins are implicitly regparm or fastcall
5069 depending on the ABI. Override the generic do-nothing attribute that
5070 these builtins were declared with, and replace it with one of the two
5071 attributes that we expect elsewhere. */
5073 static tree
5075 tree args ATTRIBUTE_UNUSED,
5078 {
5079 tree alt;
5081 /* In no case do we want to add the placeholder attribute. */
5084 /* The 64-bit ABI is unchanged for transactional memory. */
5088 /* ??? Is there a better way to validate 32-bit windows? We have
5089 cfun->machine->call_abi, but that seems to be set only for 64-bit. */
5092 else
5093 {
5096 }
5100 }
5102 /* This function determines from TYPE the calling-convention. */
5104 unsigned int
5106 {
5109 tree attrs;
5116 {
5126 /* Regparam isn't allowed for thiscall and fastcall. */
5128 {
5133 }
5137 }
5144 || is_stdarg
5150 }
5152 /* Return 0 if the attributes for two types are incompatible, 1 if they
5153 are compatible, and 2 if they are nearly compatible (which causes a
5154 warning to be generated). */
5156 static int
5158 {
5174 }
5175 \f
5176 /* Return the regparm value for a function with the indicated TYPE and DECL.
5177 DECL may be NULL when calling function indirectly
5178 or considering a libcall. */
5180 static int
5182 {
5183 tree attr;
5194 {
5197 {
5200 }
5201 }
5207 /* Use register calling convention for local functions when possible. */
5210 && optimize
5212 {
5213 /* FIXME: remove this CONST_CAST when cgraph.[ch] is constified. */
5216 {
5219 /* Make sure no regparm register is taken by a
5220 fixed register variable. */
5225 /* We don't want to use regparm(3) for nested functions as
5226 these use a static chain pointer in the third argument. */
5230 /* In 32-bit mode save a register for the split stack. */
5234 /* Each fixed register usage increases register pressure,
5235 so less registers should be used for argument passing.
5236 This functionality can be overriden by an explicit
5237 regparm value. */
5240 globals++;
5242 local_regparm
5247 }
5248 }
5251 }
5253 /* Return 1 or 2, if we can pass up to SSE_REGPARM_MAX SFmode (1) and
5254 DFmode (2) arguments in SSE registers for a function with the
5255 indicated TYPE and DECL. DECL may be NULL when calling function
5256 indirectly or considering a libcall. Otherwise return 0. */
5258 static int
5260 {
5263 /* Use SSE registers to pass SFmode and DFmode arguments if requested
5264 by the sseregparm attribute. */
5267 {
5269 {
5271 {
5275 else
5278 }
5280 }
5283 }
5285 /* For local functions, pass up to SSE_REGPARM_MAX SFmode
5286 (and DFmode for SSE2) arguments in SSE registers. */
5289 {
5290 /* FIXME: remove this CONST_CAST when cgraph.[ch] is constified. */
5294 }
5297 }
5299 /* Return true if EAX is live at the start of the function. Used by
5300 ix86_expand_prologue to determine if we need special help before
5301 calling allocate_stack_worker. */
5303 static bool
5305 {
5306 /* Cheat. Don't bother working forward from ix86_function_regparm
5307 to the function type to whether an actual argument is located in
5308 eax. Instead just look at cfg info, which is still close enough
5309 to correct at this point. This gives false positives for broken
5310 functions that might use uninitialized data that happens to be
5311 allocated in eax, but who cares? */
5313 }
5315 static bool
5317 {
5318 tree attr;
5321 {
5327 /* For 32-bit MS-ABI the default is to keep aggregate
5328 return pointer. */
5331 }
5333 }
5335 /* Value is the number of bytes of arguments automatically
5336 popped when returning from a subroutine call.
5337 FUNDECL is the declaration node of the function (as a tree),
5338 FUNTYPE is the data type of the function (as a tree),
5339 or for a library call it is an identifier node for the subroutine name.
5340 SIZE is the number of bytes of arguments passed on the stack.
5342 On the 80386, the RTD insn may be used to pop them if the number
5343 of args is fixed, but if the number is variable then the caller
5344 must pop them all. RTD can't be used for library calls now
5345 because the library is compiled with the Unix compiler.
5346 Use of RTD is a selectable option, since it is incompatible with
5347 standard Unix calling sequences. If the option is not selected,
5348 the caller must always pop the args.
5350 The attribute stdcall is equivalent to RTD on a per module basis. */
5352 static int
5354 {
5357 /* None of the 64-bit ABIs pop arguments. */
5368 /* Lose any fake structure return argument if it is passed on the stack. */
5371 {
5375 }
5378 }
5379 \f
5380 /* Argument support functions. */
5382 /* Return true when register may be used to pass function parameters. */
5383 bool
5385 {
5390 {
5394 else
5400 }
5403 {
5406 }
5407 else
5408 {
5412 }
5414 /* TODO: The function should depend on current function ABI but
5415 builtins.c would need updating then. Therefore we use the
5416 default ABI. */
5418 /* RAX is used as hidden argument to va_arg functions. */
5424 else
5431 }
5433 /* Return if we do not know how to pass TYPE solely in registers. */
5435 static bool
5437 {
5441 /* For 32-bit, we want TImode aggregates to go on the stack. But watch out!
5442 The layout_type routine is crafty and tries to trick us into passing
5443 currently unsupported vector types on the stack by using TImode. */
5446 }
5448 /* It returns the size, in bytes, of the area reserved for arguments passed
5449 in registers for the function represented by fndecl dependent to the used
5450 abi format. */
5451 int
5453 {
5457 else
5462 }
5464 /* Returns value SYSV_ABI, MS_ABI dependent on fntype, specifying the
5465 call abi used. */
5466 enum calling_abi
5468 {
5470 {
5473 {
5476 }
5480 }
5482 }
5484 static bool
5486 {
5488 {
5492 else
5494 }
5496 }
5498 static enum calling_abi
5500 {
5504 }
5506 /* Returns value SYSV_ABI, MS_ABI dependent on cfun, specifying the
5507 call abi used. */
5508 enum calling_abi
5510 {
5514 }
5516 /* Write the extra assembler code needed to declare a function properly. */
5518 void
5520 tree decl)
5521 {
5525 {
5531 }
5533 #ifdef SUBTARGET_ASM_UNWIND_INIT
5535 #endif
5539 /* Output magic byte marker, if hot-patch attribute is set. */
5541 {
5543 {
5544 /* leaq [%rsp + 0], %rsp */
5547 }
5548 else
5549 {
5550 /* movl.s %edi, %edi
5551 push %ebp
5552 movl.s %esp, %ebp */
5555 }
5556 }
5557 }
5559 /* regclass.c */
5562 /* Implementation of call abi switching target hook. Specific to FNDECL
5563 the specific call register sets are set. See also
5564 ix86_conditional_register_usage for more details. */
5565 void
5567 {
5570 else
5572 }
5574 /* 64-bit MS and SYSV ABI have different set of call used registers. Avoid
5575 expensive re-initialization of init_regs each time we switch function context
5576 since this is needed only during RTL expansion. */
5577 static void
5579 {
5583 }
5585 /* Initialize a variable CUM of type CUMULATIVE_ARGS
5586 for a call to a function whose data type is FNTYPE.
5587 For a library call, FNTYPE is 0. */
5589 void
5593 tree fndecl,
5595 {
5597 tree fnret_type;
5601 /* Initialize for the current callee. */
5603 {
5606 }
5609 {
5613 }
5614 else
5615 {
5620 else
5622 }
5625 {
5629 {
5630 /* The return value of this function uses 256bit AVX modes. */
5633 else
5635 }
5636 }
5640 /* Set up the number of registers to use for passing arguments. */
5647 {
5649 ? X86_64_REGPARM_MAX
5651 }
5653 {
5656 {
5658 ? X86_64_SSE_REGPARM_MAX
5660 }
5661 }
5668 /* Because type might mismatch in between caller and callee, we need to
5669 use actual type of function for local calls.
5670 FIXME: cgraph_analyze can be told to actually record if function uses
5671 va_start so for local functions maybe_vaarg can be made aggressive
5672 helping K&R code.
5673 FIXME: once typesytem is fixed, we won't need this code anymore. */
5681 {
5682 /* If there are variable arguments, then we won't pass anything
5683 in registers in 32-bit mode. */
5685 {
5693 }
5695 /* Use ecx and edx registers if function has fastcall attribute,
5696 else look for regparm information. */
5698 {
5701 {
5704 }
5706 {
5709 }
5710 else
5712 }
5714 /* Set up the number of SSE registers used for passing SFmode
5715 and DFmode arguments. Warn for mismatching ABI. */
5717 }
5718 }
5720 /* Return the "natural" mode for TYPE. In most cases, this is just TYPE_MODE.
5721 But in the case of vector types, it is some vector mode.
5723 When we have only some of our vector isa extensions enabled, then there
5724 are some modes for which vector_mode_supported_p is false. For these
5725 modes, the generic vector support in gcc will choose some non-vector mode
5726 in order to implement the type. By computing the natural mode, we'll
5727 select the proper ABI location for the operand and not depend on whatever
5728 the middle-end decides to do with these vector types.
5730 The midde-end can't deal with the vector types > 16 bytes. In this
5731 case, we return the original mode and warn ABI change if CUM isn't
5732 NULL. */
5734 static enum machine_mode
5736 {
5740 {
5743 /* ??? Generic code allows us to create width 1 vectors. Ignore. */
5745 {
5750 else
5753 /* Get the mode which has this inner mode and number of units. */
5757 {
5759 {
5763 && !warnedavx
5765 {
5769 }
5771 }
5772 else
5774 }
5777 }
5778 }
5781 }
5783 /* We want to pass a value in REGNO whose "natural" mode is MODE. However,
5784 this may not agree with the mode that the type system has chosen for the
5785 register, which is ORIG_MODE. If ORIG_MODE is not BLKmode, then we can
5786 go ahead and use it. Otherwise we have to build a PARALLEL instead. */
5788 static rtx
5791 {
5792 rtx tmp;
5796 else
5797 {
5801 }
5804 }
5806 /* x86-64 register passing implementation. See x86-64 ABI for details. Goal
5807 of this code is to classify each 8bytes of incoming argument by the register
5808 class and assign registers accordingly. */
5810 /* Return the union class of CLASS1 and CLASS2.
5811 See the x86-64 PS ABI for details. */
5813 static enum x86_64_reg_class
5815 {
5816 /* Rule #1: If both classes are equal, this is the resulting class. */
5820 /* Rule #2: If one of the classes is NO_CLASS, the resulting class is
5821 the other class. */
5827 /* Rule #3: If one of the classes is MEMORY, the result is MEMORY. */
5831 /* Rule #4: If one of the classes is INTEGER, the result is INTEGER. */
5839 /* Rule #5: If one of the classes is X87, X87UP, or COMPLEX_X87 class,
5840 MEMORY is used. */
5849 /* Rule #6: Otherwise class SSE is used. */
5851 }
5853 /* Classify the argument of type TYPE and mode MODE.
5854 CLASSES will be filled by the register class used to pass each word
5855 of the operand. The number of words is returned. In case the parameter
5856 should be passed in memory, 0 is returned. As a special case for zero
5857 sized containers, classes[0] will be NO_CLASS and 1 is returned.
5859 BIT_OFFSET is used internally for handling records and specifies offset
5860 of the offset in bits modulo 256 to avoid overflow cases.
5862 See the x86-64 PS ABI for details.
5863 */
5865 static int
5868 {
5869 HOST_WIDE_INT bytes =
5871 int words
5874 /* Variable sized entities are always passed/returned in memory. */
5883 {
5885 tree field;
5888 /* On x86-64 we pass structures larger than 32 bytes on the stack. */
5895 /* Zero sized arrays or structures are NO_CLASS. We return 0 to
5896 signalize memory class, so handle it as special case. */
5898 {
5901 }
5903 /* Classify each field of record and merge classes. */
5905 {
5907 /* And now merge the fields of structure. */
5909 {
5911 {
5917 /* Bitfields are always classified as integer. Handle them
5918 early, since later code would consider them to be
5919 misaligned integers. */
5921 {
5930 }
5931 else
5932 {
5937 /* Flexible array member is ignored. */
5944 {
5948 {
5951 "the ABI of passing struct with"
5952 " a flexible array member has"
5954 }
5956 }
5958 subclasses,
5968 }
5969 }
5970 }
5974 /* Arrays are handled as small records. */
5975 {
5982 /* The partial classes are now full classes. */
5993 }
5996 /* Unions are similar to RECORD_TYPE but offset is always 0.
5997 */
5999 {
6001 {
6009 bit_offset);
6014 }
6015 }
6020 }
6023 {
6024 /* When size > 16 bytes, if the first one isn't
6025 X86_64_SSE_CLASS or any other ones aren't
6026 X86_64_SSEUP_CLASS, everything should be passed in
6027 memory. */
6034 }
6036 /* Final merger cleanup. */
6038 {
6039 /* If one class is MEMORY, everything should be passed in
6040 memory. */
6044 /* The X86_64_SSEUP_CLASS should be always preceded by
6045 X86_64_SSE_CLASS or X86_64_SSEUP_CLASS. */
6049 {
6050 /* The first one should never be X86_64_SSEUP_CLASS. */
6053 }
6055 /* If X86_64_X87UP_CLASS isn't preceded by X86_64_X87_CLASS,
6056 everything should be passed in memory. */
6059 {
6062 /* The first one should never be X86_64_X87UP_CLASS. */
6065 {
6068 "the ABI of passing union with long double"
6070 }
6072 }
6073 }
6075 }
6077 /* Compute alignment needed. We align all types to natural boundaries with
6078 exception of XFmode that is aligned to 64bits. */
6080 {
6089 /* Misaligned fields are always returned in memory. */
6092 }
6094 /* for V1xx modes, just use the base mode */
6099 /* Classification of atomic types. */
6101 {
6117 {
6121 {
6124 }
6126 {
6129 }
6131 {
6135 }
6137 {
6140 }
6141 else
6143 }
6150 /* OImode shouldn't be used directly. */
6157 else
6175 else
6176 {
6180 {
6183 "the ABI of passing structure with complex float"
6185 }
6188 }
6197 /* This modes is larger than 16 bytes. */
6240 else
6244 }
6245 }
6247 /* Examine the argument and return set number of register required in each
6248 class. Return 0 iff parameter should be passed in memory. */
6249 static int
6252 {
6262 {
6284 }
6286 }
6288 /* Construct container for the argument used by GCC interface. See
6289 FUNCTION_ARG for the detailed description. */
6291 static rtx
6295 {
6296 /* The following variables hold the static issued_error state. */
6310 rtx ret;
6321 /* We allowed the user to turn off SSE for kernel mode. Don't crash if
6322 some less clueful developer tries to use floating-point anyway. */
6324 {
6326 {
6328 {
6331 }
6332 }
6334 {
6337 }
6339 }
6341 /* Likewise, error if the ABI requires us to return values in the
6342 x87 registers and the user specified -mno-80387. */
6348 {
6350 {
6353 }
6355 }
6357 /* First construct simple cases. Avoid SCmode, since we want to use
6358 single register to pass this type. */
6361 {
6376 /* Zero sized array, struct or class. */
6380 }
6407 /* Otherwise figure out the entries of the PARALLEL. */
6409 {
6413 {
6418 /* Merge TImodes on aligned occasions here too. */
6420 tmpmode
6424 else
6426 /* We've requested 24 bytes we
6427 don't have mode for. Use DImode. */
6434 intreg++;
6442 sse_regno++;
6450 sse_regno++;
6455 {
6461 {
6463 i++;
6464 }
6465 else
6478 }
6484 sse_regno++;
6488 }
6489 }
6491 /* Empty aligned struct, union or class. */
6499 }
6501 /* Update the data in CUM to advance over an argument of mode MODE
6502 and data type TYPE. (TYPE is null for libcalls where that information
6503 may not be available.) */
6505 static void
6508 HOST_WIDE_INT words)
6509 {
6511 {
6518 /* FALLTHRU */
6529 {
6532 }
6536 /* OImode shouldn't be used directly. */
6545 /* FALLTHRU */
6561 {
6566 {
6569 }
6570 }
6580 {
6585 {
6588 }
6589 }
6591 }
6592 }
6594 static void
6597 {
6600 /* Unnamed 256bit vector mode parameters are passed on stack. */
6606 {
6611 }
6612 else
6613 {
6617 }
6618 }
6620 static void
6622 HOST_WIDE_INT words)
6623 {
6624 /* Otherwise, this should be passed indirect. */
6629 {
6632 }
6633 }
6635 /* Update the data in CUM to advance over an argument of mode MODE and
6636 data type TYPE. (TYPE is null for libcalls where that information
6637 may not be available.) */
6639 static void
6642 {
6648 else
6659 else
6661 }
6663 /* Define where to put the arguments to a function.
6664 Value is zero to push the argument on the stack,
6665 or a hard register in which to store the argument.
6667 MODE is the argument's machine mode.
6668 TYPE is the data type of the argument (as a tree).
6669 This is null for libcalls where that information may
6670 not be available.
6671 CUM is a variable of type CUMULATIVE_ARGS which gives info about
6672 the preceding args and about the function being called.
6673 NAMED is nonzero if this argument is a named parameter
6674 (otherwise it is an extra parameter matching an ellipsis). */
6676 static rtx
6680 {
6683 /* Avoid the AL settings for the Unix64 ABI. */
6688 {
6695 /* FALLTHRU */
6701 {
6704 /* Fastcall allocates the first two DWORD (SImode) or
6705 smaller arguments to ECX and EDX if it isn't an
6706 aggregate type . */
6708 {
6714 /* ECX not EAX is the first allocated register. */
6717 }
6719 }
6728 /* FALLTHRU */
6730 /* In 32bit, we pass TImode in xmm registers. */
6738 {
6740 {
6744 }
6748 }
6752 /* OImode shouldn't be used directly. */
6762 {
6766 }
6776 {
6778 {
6782 }
6786 }
6788 }
6791 }
6793 static rtx
6796 {
6797 /* Handle a hidden AL argument containing number of registers
6798 for varargs x86-64 functions. */
6802 ? X86_64_SSE_REGPARM_MAX
6807 {
6817 /* Unnamed 256bit vector mode parameters are passed on stack. */
6821 }
6827 }
6829 static rtx
6832 HOST_WIDE_INT bytes)
6833 {
6836 /* We need to add clobber for MS_ABI->SYSV ABI calls in expand_call.
6837 We use value of -2 to specify that current function call is MSABI. */
6841 /* If we've run out of registers, it goes on the stack. */
6847 /* Only floating point modes are passed in anything but integer regs. */
6849 {
6852 else
6853 {
6856 /* Unnamed floating parameters are passed in both the
6857 SSE and integer registers. */
6863 }
6864 }
6865 /* Handle aggregated types passed in register. */
6867 {
6872 }
6875 }
6877 /* Return where to put the arguments to a function.
6878 Return zero to push the argument on the stack, or a hard register in which to store the argument.
6880 MODE is the argument's machine mode. TYPE is the data type of the
6881 argument. It is null for libcalls where that information may not be
6882 available. CUM gives information about the preceding args and about
6883 the function being called. NAMED is nonzero if this argument is a
6884 named parameter (otherwise it is an extra parameter matching an
6885 ellipsis). */
6887 static rtx
6890 {
6894 rtx arg;
6898 else
6902 /* To simplify the code below, represent vector types with a vector mode
6903 even if MMX/SSE are not active. */
6911 else
6915 {
6916 /* This argument uses 256bit AVX modes. */
6919 else
6921 }
6924 }
6926 /* A C expression that indicates when an argument must be passed by
6927 reference. If nonzero for an argument, a copy of that argument is
6928 made in memory and a pointer to the argument is passed instead of
6929 the argument itself. The pointer is passed in whatever way is
6930 appropriate for passing a pointer to that type. */
6932 static bool
6936 {
6939 /* See Windows x64 Software Convention. */
6941 {
6944 {
6945 /* Arrays are passed by reference. */
6950 {
6951 /* Structs/unions of sizes other than 8, 16, 32, or 64 bits
6952 are passed by reference. */
6954 }
6955 }
6957 /* __m128 is passed by reference. */
6963 }
6964 }
6969 }
6971 /* Return true when TYPE should be 128bit aligned for 32bit argument
6972 passing ABI. XXX: This function is obsolete and is only used for
6973 checking psABI compatibility with previous versions of GCC. */
6975 static bool
6977 {
6989 {
6990 /* Walk the aggregates recursively. */
6992 {
6996 {
6997 tree field;
6999 /* Walk all the structure fields. */
7001 {
7005 }
7007 }
7010 /* Just for use if some languages passes arrays by value. */
7017 }
7018 }
7020 }
7022 /* Return the alignment boundary for MODE and TYPE with alignment ALIGN.
7023 XXX: This function is obsolete and is only used for checking psABI
7024 compatibility with previous versions of GCC. */
7026 static unsigned int
7029 {
7030 /* In 32bit, only _Decimal128 and __float128 are aligned to their
7031 natural boundaries. */
7033 {
7034 /* i386 ABI defines all arguments to be 4 byte aligned. We have to
7035 make an exception for SSE modes since these require 128bit
7036 alignment.
7038 The handling here differs from field_alignment. ICC aligns MMX
7039 arguments to 4 byte boundaries, while structure fields are aligned
7040 to 8 byte boundaries. */
7042 {
7045 }
7046 else
7047 {
7050 }
7051 }
7055 }
7057 /* Return true when TYPE should be 128bit aligned for 32bit argument
7058 passing ABI. */
7060 static bool
7062 {
7072 {
7073 /* Walk the aggregates recursively. */
7075 {
7079 {
7080 tree field;
7082 /* Walk all the structure fields. */
7084 field;
7086 {
7090 }
7092 }
7095 /* Just for use if some languages passes arrays by value. */
7102 }
7103 }
7104 else
7108 }
7110 /* Gives the alignment boundary, in bits, of an argument with the
7111 specified mode and type. */
7113 static unsigned int
7115 {
7118 {
7119 /* Since the main variant type is used for call, we convert it to
7120 the main variant type. */
7123 }
7124 else
7128 else
7129 {
7134 {
7135 /* i386 ABI defines XFmode arguments to be 4 byte aligned. */
7137 {
7140 }
7146 }
7149 && !warned
7151 saved_align))
7152 {
7155 "The ABI for passing parameters with %d-byte"
7158 }
7159 }
7162 }
7164 /* Return true if N is a possible register number of function value. */
7166 static bool
7168 {
7170 {
7175 /* TODO: The function should depend on current function ABI but
7176 builtins.c would need updating then. Therefore we use the
7177 default ABI. */
7189 }
7192 }
7194 /* Define how to find the value returned by a function.
7195 VALTYPE is the data type of the value (as a tree).
7196 If the precise function being called is known, FUNC is its FUNCTION_DECL;
7197 otherwise, FUNC is 0. */
7199 static rtx
7202 {
7205 /* 8-byte vector modes in %mm0. See ix86_return_in_memory for where
7206 we normally prevent this case when mmx is not available. However
7207 some ABIs may require the result to be returned like DImode. */
7211 /* 16-byte vector modes in %xmm0. See ix86_return_in_memory for where
7212 we prevent this case when sse is not available. However some ABIs
7213 may require the result to be returned like integer TImode. */
7218 /* 32-byte vector modes in %ymm0. */
7222 /* Floating point return values in %st(0) (unless -mno-fp-ret-in-387). */
7225 else
7226 /* Most things go in %eax. */
7229 /* Override FP return register with %xmm0 for local functions when
7230 SSE math is enabled or for functions with sseregparm attribute. */
7232 {
7237 }
7239 /* OImode shouldn't be used directly. */
7243 }
7245 static rtx
7247 const_tree valtype)
7248 {
7249 rtx ret;
7251 /* Handle libcalls, which don't provide a type node. */
7253 {
7257 {
7276 }
7279 }
7281 {
7282 /* Pointers are always returned in word_mode. */
7284 }
7290 /* For zero sized structures, construct_container returns NULL, but we
7291 need to keep rest of compiler happy by returning meaningful value. */
7296 }
7298 static rtx
7300 {
7304 {
7306 {
7319 }
7320 }
7322 }
7324 static rtx
7327 {
7339 else
7341 }
7343 static rtx
7346 {
7352 }
7354 /* Pointer function arguments and return values are promoted to
7355 word_mode. */
7357 static enum machine_mode
7361 {
7363 {
7366 }
7368 for_return);
7369 }
7371 rtx
7373 {
7375 }
7377 /* Return true iff type is returned in memory. */
7379 static bool ATTRIBUTE_UNUSED
7381 {
7382 HOST_WIDE_INT size;
7393 {
7394 /* User-created vectors small enough to fit in EAX. */
7398 /* MMX/3dNow values are returned in MM0,
7399 except when it doesn't exits or the ABI prescribes otherwise. */
7403 /* SSE values are returned in XMM0, except when it doesn't exist. */
7407 /* AVX values are returned in YMM0, except when it doesn't exist. */
7410 }
7418 /* OImode shouldn't be used directly. */
7422 }
7424 static bool ATTRIBUTE_UNUSED
7426 {
7429 }
7431 static bool ATTRIBUTE_UNUSED
7433 {
7436 /* __m128 is returned in xmm0. */
7441 /* Otherwise, the size must be exactly in [1248]. */
7443 }
7445 static bool
7447 {
7448 #ifdef SUBTARGET_RETURN_IN_MEMORY
7450 #else
7454 {
7457 else
7459 }
7460 else
7462 #endif
7463 }
7465 /* When returning SSE vector types, we have a choice of either
7466 (1) being abi incompatible with a -march switch, or
7467 (2) generating an error.
7468 Given no good solution, I think the safest thing is one warning.
7469 The user won't be able to use -Werror, but....
7471 Choose the STRUCT_VALUE_RTX hook because that's (at present) only
7472 called in response to actually generating a caller or callee that
7473 uses such a type. As opposed to TARGET_RETURN_IN_MEMORY, which is called
7474 via aggregate_value_p for general type probing from tree-ssa. */
7476 static rtx
7478 {
7482 {
7483 /* Look at the return type of the function, not the function type. */
7487 {
7490 {
7494 }
7495 }
7498 {
7500 {
7504 }
7505 }
7506 }
7509 }
7511 \f
7512 /* Create the va_list data type. */
7514 /* Returns the calling convention specific va_list date type.
7515 The argument ABI can be DEFAULT_ABI, MS_ABI, or SYSV_ABI. */
7517 static tree
7519 {
7522 /* For i386 we use plain pointer to argument area. */
7532 unsigned_type_node);
7535 unsigned_type_node);
7538 ptr_type_node);
7541 ptr_type_node);
7560 /* The correct type is an array type of one element. */
7562 }
7564 /* Setup the builtin va_list data type and for 64-bit the additional
7565 calling convention specific va_list data types. */
7567 static tree
7569 {
7572 /* Initialize abi specific va_list builtin types. */
7574 {
7575 tree t;
7577 {
7582 }
7583 else
7584 {
7589 }
7591 {
7596 }
7597 else
7598 {
7603 }
7604 }
7607 }
7609 /* Worker function for TARGET_SETUP_INCOMING_VARARGS. */
7611 static void
7613 {
7615 alias_set_type set;
7618 /* GPR size of varargs save area. */
7621 else
7624 /* FPR size of varargs save area. We don't need it if we don't pass
7625 anything in SSE registers. */
7628 else
7642 {
7650 }
7653 {
7657 /* Now emit code to save SSE registers. The AX parameter contains number
7658 of SSE parameter registers used to call this function, though all we
7659 actually check here is the zero/non-zero status. */
7664 label));
7666 /* ??? If !TARGET_SSE_TYPELESS_STORES, would we perform better if
7667 we used movdqa (i.e. TImode) instead? Perhaps even better would
7668 be if we could determine the real mode of the data, via a hook
7669 into pass_stdarg. Ignore all that for now. */
7679 {
7687 }
7690 }
7691 }
7693 static void
7695 {
7699 /* Reset to zero, as there might be a sysv vaarg used
7700 before. */
7705 {
7716 }
7717 }
7719 static void
7723 {
7725 CUMULATIVE_ARGS next_cum;
7726 tree fntype;
7728 /* This argument doesn't appear to be used anymore. Which is good,
7729 because the old code here didn't suppress rtl generation. */
7737 /* For varargs, we do not want to skip the dummy va_dcl argument.
7738 For stdargs, we do want to skip the last named argument. */
7746 else
7748 }
7750 /* Checks if TYPE is of kind va_list char *. */
7752 static bool
7754 {
7755 tree canonic;
7757 /* For 32-bit it is always true. */
7763 }
7765 /* Implement va_start. */
7767 static void
7769 {
7773 tree type;
7774 rtx ovf_rtx;
7778 {
7781 /* When we are splitting the stack, we can't refer to the stack
7782 arguments using internal_arg_pointer, because they may be on
7783 the old stack. The split stack prologue will arrange to
7784 leave a pointer to the old stack arguments in a scratch
7785 register, which we here copy to a pseudo-register. The split
7786 stack prologue can't set the pseudo-register directly because
7787 it (the prologue) runs before any registers have been saved. */
7791 {
7805 }
7806 }
7808 /* Only 64bit target needs something special. */
7810 {
7813 else
7814 {
7823 }
7825 }
7834 /* The following should be folded into the MEM_REF offset. */
7844 /* Count number of gp and fp argument registers used. */
7850 {
7856 }
7859 {
7865 }
7867 /* Find the overflow area. */
7871 else
7881 {
7882 /* Find the register save area.
7883 Prologue of the function save it right above stack frame. */
7891 }
7892 }
7894 /* Implement va_arg. */
7896 static tree
7899 {
7906 rtx container;
7908 tree ptrtype;
7912 /* Only 64bit target needs something special. */
7936 {
7943 /* Unnamed 256bit vector mode parameters are passed on stack. */
7945 {
7948 }
7956 }
7958 /* Pull the value out of the saved registers. */
7963 {
7977 /* In case we are passing structure, verify that it is consecutive block
7978 on the register save area. If not we need to do moves. */
7980 {
7981 /* Verify that all registers are strictly consecutive */
7983 {
7987 {
7992 }
7993 }
7994 else
7995 {
7999 {
8004 }
8005 }
8006 }
8008 {
8011 }
8012 else
8013 {
8016 }
8018 /* First ensure that we fit completely in registers. */
8020 {
8027 }
8029 {
8037 }
8039 /* Compute index to start of area used for integer regs. */
8041 {
8042 /* int_addr = gpr + sav; */
8045 }
8047 {
8048 /* sse_addr = fpr + sav; */
8051 }
8053 {
8057 /* addr = &temp; */
8062 {
8066 tree piece_type;
8067 tree addr_type;
8068 tree daddr_type;
8077 {
8082 }
8086 else
8093 {
8096 }
8097 else
8098 {
8101 }
8108 {
8113 }
8114 else
8115 {
8116 tree copy
8121 }
8123 }
8124 }
8127 {
8131 }
8134 {
8138 }
8143 }
8145 /* ... otherwise out of the overflow area. */
8147 /* When we align parameter on stack for caller, if the parameter
8148 alignment is beyond MAX_SUPPORTED_STACK_ALIGNMENT, it will be
8149 aligned at MAX_SUPPORTED_STACK_ALIGNMENT. We will match callee
8150 here with caller. */
8155 /* Care for on-stack alignment if needed. */
8158 else
8159 {
8164 }
8181 }
8182 \f
8183 /* Return true if OPNUM's MEM should be matched
8184 in movabs* patterns. */
8186 bool
8188 {
8200 }
8201 \f
8202 /* Initialize the table of extra 80387 mathematical constants. */
8204 static void
8206 {
8208 {
8214 };
8218 {
8220 /* Ensure each constant is rounded to XFmode precision. */
8223 }
8226 }
8228 /* Return non-zero if the constant is something that
8229 can be loaded with a special instruction. */
8231 int
8233 {
8236 REAL_VALUE_TYPE r;
8248 /* For XFmode constants, try to find a special 80387 instruction when
8249 optimizing for size or on those CPUs that benefit from them. */
8252 {
8261 }
8263 /* Load of the constant -0.0 or -1.0 will be split as
8264 fldz;fchs or fld1;fchs sequence. */
8271 }
8273 /* Return the opcode of the special instruction to be used to load
8274 the constant X. */
8278 {
8280 {
8300 }
8301 }
8303 /* Return the CONST_DOUBLE representing the 80387 constant that is
8304 loaded by the specified special instruction. The argument IDX
8305 matches the return value from standard_80387_constant_p. */
8307 rtx
8309 {
8316 {
8327 }
8330 XFmode);
8331 }
8333 /* Return 1 if X is all 0s and 2 if x is all 1s
8334 in supported SSE/AVX vector mode. */
8336 int
8338 {
8345 {
8360 }
8363 }
8365 /* Return the opcode of the special instruction to be used to load
8366 the constant X. */
8370 {
8372 {
8375 {
8396 }
8401 else
8406 }
8408 }
8410 /* Returns true if OP contains a symbol reference */
8412 bool
8414 {
8423 {
8425 {
8431 }
8435 }
8438 }
8440 /* Return true if it is appropriate to emit `ret' instructions in the
8441 body of a function. Do this only if the epilogue is simple, needing a
8442 couple of insns. Prior to reloading, we can't tell how many registers
8443 must be saved, so return false then. Return false if there is no frame
8444 marker to de-allocate. */
8446 bool
8448 {
8454 /* Don't allow more than 32k pop, since that's all we can do
8455 with one instruction. */
8462 }
8463 \f
8464 /* Value should be nonzero if functions must have frame pointers.
8465 Zero means the frame pointer need not be set up (and parms may
8466 be accessed via the stack pointer) in functions that seem suitable. */
8468 static bool
8470 {
8471 /* If we accessed previous frames, then the generated code expects
8472 to be able to access the saved ebp value in our frame. */
8476 /* Several x86 os'es need a frame pointer for other reasons,
8477 usually pertaining to setjmp. */
8481 /* For older 32-bit runtimes setjmp requires valid frame-pointer. */
8485 /* In ix86_option_override_internal, TARGET_OMIT_LEAF_FRAME_POINTER
8486 turns off the frame pointer by default. Turn it back on now if
8487 we've not got a leaf function. */
8489 && (!current_function_is_leaf
8497 }
8499 /* Record that the current function accesses previous call frames. */
8501 void
8503 {
8505 }
8506 \f
8507 #ifndef USE_HIDDEN_LINKONCE
8508 # if defined(HAVE_GAS_HIDDEN) && (SUPPORTS_ONE_ONLY - 0)
8509 # define USE_HIDDEN_LINKONCE 1
8510 # else
8511 # define USE_HIDDEN_LINKONCE 0
8512 # endif
8513 #endif
8517 /* Fills in the label name that should be used for a pc thunk for
8518 the given register. */
8520 static void
8522 {
8527 else
8529 }
8532 /* This function generates code for -fpic that loads %ebx with
8533 the return address of the caller and then returns. */
8535 static void
8537 {
8542 {
8544 tree decl;
8559 #if TARGET_MACHO
8561 {
8570 }
8571 else
8572 #endif
8574 {
8585 }
8586 else
8587 {
8590 }
8596 /* Make sure unwind info is emitted for the thunk if needed. */
8599 /* Pad stack IP move with 4 instructions (two NOPs count
8600 as one instruction). */
8602 {
8607 }
8618 }
8622 }
8624 /* Emit code for the SET_GOT patterns. */
8628 {
8634 {
8635 /* Load (*VXWORKS_GOTT_BASE) into the PIC register. */
8640 /* Load (*VXWORKS_GOTT_BASE)[VXWORKS_GOTT_INDEX] into the PIC register.
8641 Use %P and a local symbol in order to print VXWORKS_GOTT_INDEX as
8642 an unadorned address. */
8647 }
8652 {
8657 #if TARGET_MACHO
8658 /* Output the Mach-O "canonical" label name ("Lxx$pb") here too. This
8659 is what will be referenced by the Mach-O PIC subsystem. */
8662 #endif
8666 }
8667 else
8668 {
8676 /* Output the Mach-O "canonical" label name ("Lxx$pb") here too. This
8677 is what will be referenced by the Mach-O PIC subsystem. */
8678 #if TARGET_MACHO
8681 else
8684 #endif
8685 }
8691 }
8693 /* Generate an "push" pattern for input ARG. */
8695 static rtx
8697 {
8710 stack_pointer_rtx)),
8711 arg);
8712 }
8714 /* Generate an "pop" pattern for input ARG. */
8716 static rtx
8718 {
8723 arg,
8726 stack_pointer_rtx)));
8727 }
8729 /* Return >= 0 if there is an unused call-clobbered register available
8730 for the entire function. */
8732 static unsigned int
8734 {
8738 {
8740 /* Can't use the same register for both PIC and DRAP. */
8743 else
8748 }
8751 }
8753 /* Return TRUE if we need to save REGNO. */
8755 static bool
8757 {
8767 {
8770 {
8776 }
8777 }
8786 }
8788 /* Return number of saved general prupose registers. */
8790 static int
8792 {
8798 nregs ++;
8800 }
8802 /* Return number of saved SSE registrers. */
8804 static int
8806 {
8814 nregs ++;
8816 }
8818 /* Given FROM and TO register numbers, say whether this elimination is
8819 allowed. If stack alignment is needed, we can only replace argument
8820 pointer with hard frame pointer, or replace frame pointer with stack
8821 pointer. Otherwise, frame pointer elimination is automatically
8822 handled and all other eliminations are valid. */
8824 static bool
8826 {
8832 else
8834 }
8836 /* Return the offset between two registers, one to be eliminated, and the other
8837 its replacement, at the start of a routine. */
8839 HOST_WIDE_INT
8841 {
8850 else
8851 {
8859 }
8860 }
8862 /* In a dynamically-aligned function, we can't know the offset from
8863 stack pointer to frame pointer, so we must ensure that setjmp
8864 eliminates fp against the hard fp (%ebp) rather than trying to
8865 index from %esp up to the top of the frame across a gap that is
8866 of unknown (at compile-time) size. */
8867 static rtx
8869 {
8871 }
8873 /* When using -fsplit-stack, the allocation routines set a field in
8874 the TCB to the bottom of the stack plus this much space, measured
8875 in bytes. */
8877 #define SPLIT_STACK_AVAILABLE 256
8879 /* Fill structure ix86_frame about frame of currently computed function. */
8881 static void
8883 {
8885 HOST_WIDE_INT offset;
8888 HOST_WIDE_INT to_allocate;
8896 /* 64-bit MS ABI seem to require stack alignment to be always 16 except for
8897 function prologues and leaf. */
8901 {
8906 }
8912 /* For SEH we have to limit the amount of code movement into the prologue.
8913 At present we do this via a BLOCKAGE, at which point there's very little
8914 scheduling that can be done, which means that there's very little point
8915 in doing anything except PUSHs. */
8919 /* During reload iteration the amount of registers saved can change.
8920 Recompute the value as needed. Do not recompute when amount of registers
8921 didn't change as reload does multiple calls to the function and does not
8922 expect the decision to change within single iteration. */
8925 {
8931 /* The fast prologue uses move instead of push to save registers. This
8932 is significantly longer, but also executes faster as modern hardware
8933 can execute the moves in parallel, but can't do that for push/pop.
8935 Be careful about choosing what prologue to emit: When function takes
8936 many instructions to execute we may use slow version as well as in
8937 case function is known to be outside hot spot (this is known with
8938 feedback only). Weight the size of function by number of registers
8939 to save as it is cheap to use one or two push instructions but very
8940 slow to use many of them. */
8944 || (flag_branch_probabilities
8947 else
8950 }
8952 frame->save_regs_using_mov
8954 /* If static stack checking is enabled and done with probes,
8955 the registers need to be saved before allocating the frame. */
8958 /* Skip return address. */
8961 /* Skip pushed static chain. */
8965 /* Skip saved base pointer. */
8970 /* The traditional frame pointer location is at the top of the frame. */
8973 /* Register save area */
8977 /* Align and set SSE register save area. */
8979 {
8980 /* The only ABI that has saved SSE registers (Win64) also has a
8981 16-byte aligned default stack, and thus we don't need to be
8982 within the re-aligned local stack frame to save them. */
8986 }
8989 /* The re-aligned stack starts here. Values before this point are not
8990 directly comparable with values below this point. In order to make
8991 sure that no value happens to be the same before and after, force
8992 the alignment computation below to add a non-zero value. */
8996 /* Va-arg area */
9000 /* Align start of frame for local function. */
9004 || !current_function_is_leaf
9009 /* Frame pointer points here. */
9014 /* Add outgoing arguments area. Can be skipped if we eliminated
9015 all the function calls as dead code.
9016 Skipping is however impossible when function calls alloca. Alloca
9017 expander assumes that last crtl->outgoing_args_size
9018 of stack frame are unused. */
9022 {
9025 }
9026 else
9029 /* Align stack boundary. Only needed if we're calling another function
9030 or using alloca. */
9035 /* We've reached end of stack frame. */
9038 /* Size prologue needs to allocate. */
9046 && current_function_sp_is_unchanging
9047 && current_function_is_leaf
9049 {
9055 }
9056 else
9060 /* The SEH frame pointer location is near the bottom of the frame.
9061 This is enforced by the fact that the difference between the
9062 stack pointer and the frame pointer is limited to 240 bytes in
9063 the unwind data structure. */
9065 {
9066 HOST_WIDE_INT diff;
9068 /* If we can leave the frame pointer where it is, do so. */
9071 {
9072 /* Ideally we'd determine what portion of the local stack frame
9073 (within the constraint of the lowest 240) is most heavily used.
9074 But without that complication, simply bias the frame pointer
9075 by 128 bytes so as to maximize the amount of the local stack
9076 frame that is addressable with 8-bit offsets. */
9078 }
9079 }
9080 }
9082 /* This is semi-inlined memory_address_length, but simplified
9083 since we know that we're always dealing with reg+offset, and
9084 to avoid having to create and discard all that rtl. */
9088 {
9092 {
9093 /* EBP and R13 cannot be encoded without an offset. */
9095 }
9099 /* ESP and R12 must be encoded with a SIB byte. */
9101 len++;
9104 }
9106 /* Return an RTX that points to CFA_OFFSET within the stack frame.
9107 The valid base registers are taken from CFUN->MACHINE->FS. */
9109 static rtx
9111 {
9117 {
9118 /* Choose the base register most likely to allow the most scheduling
9119 opportunities. Generally FP is valid througout the function,
9120 while DRAP must be reloaded within the epilogue. But choose either
9121 over the SP due to increased encoding size. */
9124 {
9127 }
9129 {
9132 }
9134 {
9137 }
9138 }
9139 else
9140 {
9141 HOST_WIDE_INT toffset;
9144 /* Choose the base register with the smallest address encoding.
9145 With a tie, choose FP > DRAP > SP. */
9147 {
9151 }
9153 {
9157 {
9161 }
9162 }
9164 {
9168 {
9172 }
9173 }
9174 }
9178 }
9180 /* Emit code to save registers in the prologue. */
9182 static void
9184 {
9186 rtx insn;
9190 {
9193 }
9194 }
9196 /* Emit a single register save at CFA - CFA_OFFSET. */
9198 static void
9200 HOST_WIDE_INT cfa_offset)
9201 {
9209 /* For SSE saves, we need to indicate the 128-bit alignment. */
9220 /* When saving registers into a re-aligned local stack frame, avoid
9221 any tricky guessing by dwarf2out. */
9223 {
9227 {
9228 /* A bit of a hack. We force the DRAP register to be saved in
9229 the re-aligned stack frame, which provides us with a copy
9230 of the CFA that will last past the prologue. Install it. */
9236 }
9237 else
9238 {
9239 /* The frame pointer is a stable reference within the
9240 aligned frame. Use it. */
9247 }
9248 }
9250 /* The memory may not be relative to the current CFA register,
9251 which means that we may need to generate a new pattern for
9252 use by the unwind info. */
9254 {
9258 }
9259 }
9261 /* Emit code to save registers using MOV insns.
9262 First register is stored at CFA - CFA_OFFSET. */
9263 static void
9265 {
9270 {
9273 }
9274 }
9276 /* Emit code to save SSE registers using MOV insns.
9277 First register is stored at CFA - CFA_OFFSET. */
9278 static void
9280 {
9285 {
9288 }
9289 }
9293 /* Add a REG_CFA_RESTORE REG note to INSN or queue them until next stack
9294 manipulation insn. The value is on the stack at CFA - CFA_OFFSET.
9295 Don't add the note if the previously saved value will be left untouched
9296 within stack red-zone till return, as unwinders can find the same value
9297 in the register and on the stack. */
9299 static void
9301 {
9307 {
9310 }
9311 else
9312 queued_cfa_restores
9314 }
9316 /* Add queued REG_CFA_RESTORE notes if any to INSN. */
9318 static void
9320 {
9321 rtx last;
9325 ;
9330 }
9332 /* Expand prologue or epilogue stack adjustment.
9333 The pattern exist to put a dependency on all ebp-based memory accesses.
9334 STYLE should be negative if instructions should be marked as frame related,
9335 zero if %r11 register is live and cannot be freely used and positive
9336 otherwise. */
9338 static void
9341 {
9343 rtx insn;
9350 else
9351 {
9352 rtx tmp;
9353 /* r11 is used by indirect sibcall return as well, set before the
9354 epilogue and used after the epilogue. */
9357 else
9358 {
9362 }
9368 }
9375 {
9376 rtx r;
9386 }
9388 {
9391 {
9395 }
9396 }
9399 {
9404 {
9407 }
9409 {
9412 }
9413 else
9414 {
9415 /* Else there are two possibilities: SP itself, which we set
9416 up as the default above. Or EH_RETURN_STACKADJ_RTX, which is
9417 taken care of this by hand along the eh_return path. */
9420 }
9424 }
9425 }
9427 /* Find an available register to be used as dynamic realign argument
9428 pointer regsiter. Such a register will be written in prologue and
9429 used in begin of body, so it must not be
9430 1. parameter passing register.
9431 2. GOT pointer.
9432 We reuse static-chain register if it is available. Otherwise, we
9433 use DI for i386 and R13 for x86-64. We chose R13 since it has
9434 shorter encoding.
9436 Return: the regno of chosen register. */
9438 static unsigned int
9440 {
9444 {
9445 /* Use R13 for nested function or function need static chain.
9446 Since function with tail call may use any caller-saved
9447 registers in epilogue, DRAP must not use caller-saved
9448 register in such case. */
9453 }
9454 else
9455 {
9456 /* Use DI for nested function or function need static chain.
9457 Since function with tail call may use any caller-saved
9458 registers in epilogue, DRAP must not use caller-saved
9459 register in such case. */
9463 /* Reuse static chain register if it isn't used for parameter
9464 passing. */
9466 {
9470 }
9472 }
9473 }
9475 /* Return minimum incoming stack alignment. */
9477 static unsigned int
9479 {
9482 /* Prefer the one specified at command line. */
9485 /* In 32bit, use MIN_STACK_BOUNDARY for incoming stack boundary
9486 if -mstackrealign is used, it isn't used for sibcall check and
9487 estimated stack alignment is 128bit. */
9489 && !TARGET_64BIT
9490 && ix86_force_align_arg_pointer
9493 else
9496 /* Incoming stack alignment can be changed on individual functions
9497 via force_align_arg_pointer attribute. We use the smallest
9498 incoming stack boundary. */
9504 /* The incoming stack frame has to be aligned at least at
9505 parm_stack_boundary. */
9509 /* Stack at entrance of main is aligned by runtime. We use the
9510 smallest incoming stack boundary. */
9518 }
9520 /* Update incoming stack boundary and estimated stack alignment. */
9522 static void
9524 {
9525 ix86_incoming_stack_boundary
9528 /* x86_64 vararg needs 16byte stack alignment for register save
9529 area. */
9534 }
9536 /* Handle the TARGET_GET_DRAP_RTX hook. Return NULL if no DRAP is
9537 needed or an rtx for DRAP otherwise. */
9539 static rtx
9541 {
9546 {
9547 /* Assign DRAP to vDRAP and returns vDRAP */
9549 rtx drap_vreg;
9550 rtx arg_ptr;
9563 {
9566 }
9568 }
9569 else
9571 }
9573 /* Handle the TARGET_INTERNAL_ARG_POINTER hook. */
9575 static rtx
9577 {
9579 }
9582 rtx reg;
9584 };
9586 /* Return a short-lived scratch register for use on function entry.
9587 In 32-bit mode, it is valid only after the registers are saved
9588 in the prologue. This register must be released by means of
9589 release_scratch_register_on_entry once it is dead. */
9591 static void
9593 {
9599 {
9600 /* We always use R11 in 64-bit mode. */
9602 }
9603 else
9604 {
9606 bool fastcall_p
9610 int drap_regno
9613 /* 'fastcall' sets regparm to 2, uses ecx/edx for arguments and eax
9614 for the static chain register. */
9620 /* ecx is the static chain register. */
9626 /* esi is the static chain register. */
9632 else
9633 {
9636 }
9637 }
9641 {
9644 }
9645 }
9647 /* Release a scratch register obtained from the preceding function. */
9649 static void
9651 {
9653 {
9656 /* The RTX_FRAME_RELATED_P mechanism doesn't know about pop. */
9661 }
9662 }
9664 #define PROBE_INTERVAL (1 << STACK_CHECK_PROBE_INTERVAL_EXP)
9666 /* Emit code to adjust the stack pointer by SIZE bytes while probing it. */
9668 static void
9670 {
9671 /* We skip the probe for the first interval + a small dope of 4 words and
9672 probe that many bytes past the specified size to maintain a protection
9673 area at the botton of the stack. */
9677 /* See if we have a constant small number of probes to generate. If so,
9678 that's the easy case. The run-time loop is made up of 11 insns in the
9679 generic case while the compile-time loop is made up of 3+2*(n-1) insns
9680 for n # of intervals. */
9682 {
9686 /* Adjust SP and probe at PROBE_INTERVAL + N * PROBE_INTERVAL for
9687 values of N from 1 until it exceeds SIZE. If only one probe is
9688 needed, this will not generate any code. Then adjust and probe
9689 to PROBE_INTERVAL + SIZE. */
9691 {
9693 {
9696 }
9697 else
9703 }
9707 else
9714 /* Adjust back to account for the additional first interval. */
9718 }
9720 /* Otherwise, do the same as above, but in a loop. Note that we must be
9721 extra careful with variables wrapping around because we might be at
9722 the very top (or the very bottom) of the address space and we have
9723 to be able to handle this case properly; in particular, we use an
9724 equality test for the loop condition. */
9725 else
9726 {
9727 HOST_WIDE_INT rounded_size;
9733 /* Step 1: round SIZE to the previous multiple of the interval. */
9738 /* Step 2: compute initial and final value of the loop counter. */
9740 /* SP = SP_0 + PROBE_INTERVAL. */
9745 /* LAST_ADDR = SP_0 + PROBE_INTERVAL + ROUNDED_SIZE. */
9749 stack_pointer_rtx)));
9752 /* Step 3: the loop
9754 while (SP != LAST_ADDR)
9755 {
9756 SP = SP + PROBE_INTERVAL
9757 probe at SP
9758 }
9760 adjusts SP and probes to PROBE_INTERVAL + N * PROBE_INTERVAL for
9761 values of N from 1 until it is equal to ROUNDED_SIZE. */
9766 /* Step 4: adjust SP and probe at PROBE_INTERVAL + SIZE if we cannot
9767 assert at compile-time that SIZE is equal to ROUNDED_SIZE. */
9770 {
9775 }
9777 /* Adjust back to account for the additional first interval. */
9783 }
9787 /* Even if the stack pointer isn't the CFA register, we need to correctly
9788 describe the adjustments made to it, in particular differentiate the
9789 frame-related ones from the frame-unrelated ones. */
9791 {
9804 }
9806 /* Make sure nothing is scheduled before we are done. */
9808 }
9810 /* Adjust the stack pointer up to REG while probing it. */
9814 {
9824 /* Jump to END_LAB if SP == LAST_ADDR. */
9832 /* SP = SP + PROBE_INTERVAL. */
9836 /* Probe at SP. */
9847 }
9849 /* Emit code to probe a range of stack addresses from FIRST to FIRST+SIZE,
9850 inclusive. These are offsets from the current stack pointer. */
9852 static void
9854 {
9855 /* See if we have a constant small number of probes to generate. If so,
9856 that's the easy case. The run-time loop is made up of 7 insns in the
9857 generic case while the compile-time loop is made up of n insns for n #
9858 of intervals. */
9860 {
9861 HOST_WIDE_INT i;
9863 /* Probe at FIRST + N * PROBE_INTERVAL for values of N from 1 until
9864 it exceeds SIZE. If only one probe is needed, this will not
9865 generate any code. Then probe at FIRST + SIZE. */
9870 }
9872 /* Otherwise, do the same as above, but in a loop. Note that we must be
9873 extra careful with variables wrapping around because we might be at
9874 the very top (or the very bottom) of the address space and we have
9875 to be able to handle this case properly; in particular, we use an
9876 equality test for the loop condition. */
9877 else
9878 {
9885 /* Step 1: round SIZE to the previous multiple of the interval. */
9890 /* Step 2: compute initial and final value of the loop counter. */
9892 /* TEST_OFFSET = FIRST. */
9895 /* LAST_OFFSET = FIRST + ROUNDED_SIZE. */
9899 /* Step 3: the loop
9901 while (TEST_ADDR != LAST_ADDR)
9902 {
9903 TEST_ADDR = TEST_ADDR + PROBE_INTERVAL
9904 probe at TEST_ADDR
9905 }
9907 probes at FIRST + N * PROBE_INTERVAL for values of N from 1
9908 until it is equal to ROUNDED_SIZE. */
9913 /* Step 4: probe at FIRST + SIZE if we cannot assert at compile-time
9914 that SIZE is equal to ROUNDED_SIZE. */
9918 stack_pointer_rtx,
9923 }
9925 /* Make sure nothing is scheduled before we are done. */
9927 }
9929 /* Probe a range of stack addresses from REG to END, inclusive. These are
9930 offsets from the current stack pointer. */
9934 {
9944 /* Jump to END_LAB if TEST_ADDR == LAST_ADDR. */
9952 /* TEST_ADDR = TEST_ADDR + PROBE_INTERVAL. */
9956 /* Probe at TEST_ADDR. */
9969 }
9971 /* Finalize stack_realign_needed flag, which will guide prologue/epilogue
9972 to be generated in correct form. */
9973 static void
9975 {
9976 /* Check if stack realign is really needed after reload, and
9977 stores result in cfun */
9978 unsigned int incoming_stack_boundary
9982 < (current_function_is_leaf
9987 {
9988 /* After stack_realign_needed is finalized, we can't no longer
9989 change it. */
9992 }
9994 /* If the only reason for frame_pointer_needed is that we conservatively
9995 assumed stack realignment might be needed, but in the end nothing that
9996 needed the stack alignment had been spilled, clear frame_pointer_needed
9997 and say we don't need stack realignment. */
10000 && frame_pointer_needed
10001 && current_function_is_leaf
10002 && flag_omit_frame_pointer
10003 && current_function_sp_is_unchanging
10004 && !ix86_current_function_calls_tls_descriptor
10013 {
10015 basic_block bb;
10022 HARD_FRAME_POINTER_REGNUM);
10024 {
10025 rtx insn;
10029 set_up_by_prologue))
10030 {
10034 }
10035 }
10049 }
10053 }
10055 /* Expand the prologue into a bunch of separate insns. */
10057 void
10059 {
10064 HOST_WIDE_INT allocate;
10069 /* DRAP should not coexist with stack_realign_fp */
10074 /* Initialize CFA state for before the prologue. */
10078 /* Track SP offset to the CFA. We continue tracking this after we've
10079 swapped the CFA register away from SP. In the case of re-alignment
10080 this is fudged; we're interested to offsets within the local frame. */
10087 {
10088 /* We should have already generated an error for any use of
10089 ms_hook on a nested function. */
10092 /* Check if profiling is active and we shall use profiling before
10093 prologue variant. If so sorry. */
10098 /* In ix86_asm_output_function_label we emitted:
10099 8b ff movl.s %edi,%edi
10100 55 push %ebp
10101 8b ec movl.s %esp,%ebp
10103 This matches the hookable function prologue in Win32 API
10104 functions in Microsoft Windows XP Service Pack 2 and newer.
10105 Wine uses this to enable Windows apps to hook the Win32 API
10106 functions provided by Wine.
10108 What that means is that we've already set up the frame pointer. */
10112 {
10115 /* We've decided to use the frame pointer already set up.
10116 Describe this to the unwinder by pretending that both
10117 push and mov insns happen right here.
10119 Putting the unwind info here at the end of the ms_hook
10120 is done so that we can make absolutely certain we get
10121 the required byte sequence at the start of the function,
10122 rather than relying on an assembler that can produce
10123 the exact encoding required.
10125 However it does mean (in the unpatched case) that we have
10126 a 1 insn window where the asynchronous unwind info is
10127 incorrect. However, if we placed the unwind info at
10128 its correct location we would have incorrect unwind info
10129 in the patched case. Which is probably all moot since
10130 I don't expect Wine generates dwarf2 unwind info for the
10131 system libraries that use this feature. */
10137 stack_pointer_rtx);
10145 /* Note that gen_push incremented m->fs.cfa_offset, even
10146 though we didn't emit the push insn here. */
10150 }
10151 else
10152 {
10153 /* The frame pointer is not needed so pop %ebp again.
10154 This leaves us with a pristine state. */
10156 }
10157 }
10159 /* The first insn of a function that accepts its static chain on the
10160 stack is to push the register that would be filled in by a direct
10161 call. This insn will be skipped by the trampoline. */
10163 {
10167 /* We don't want to interpret this push insn as a register save,
10168 only as a stack adjustment. The real copy of the register as
10169 a save will be done later, if needed. */
10174 }
10176 /* Emit prologue code to adjust stack alignment and setup DRAP, in case
10177 of DRAP is needed and stack realignment is really needed after reload */
10179 {
10182 /* Only need to push parameter pointer reg if it is caller saved. */
10184 {
10185 /* Push arg pointer reg */
10188 }
10190 /* Grab the argument pointer. */
10197 /* Align the stack. */
10199 stack_pointer_rtx,
10203 /* Replicate the return address on the stack so that return
10204 address can be reached via (argp - 1) slot. This is needed
10205 to implement macro RETURN_ADDR_RTX and intrinsic function
10206 expand_builtin_return_addr etc. */
10212 /* For the purposes of frame and register save area addressing,
10213 we've started over with a new frame. */
10216 }
10219 {
10220 /* Note: AT&T enter does NOT have reversed args. Enter is probably
10221 slower on all targets. Also sdb doesn't like it. */
10226 {
10234 }
10235 }
10240 {
10241 /* If saving registers via PUSH, do so now. */
10243 {
10247 }
10249 /* When using red zone we may start register saving before allocating
10250 the stack frame saving one cycle of the prologue. However, avoid
10251 doing this if we have to probe the stack; at least on x86_64 the
10252 stack probe can turn into a call that clobbers a red zone location. */
10254 && (! TARGET_STACK_PROBE
10256 {
10259 }
10260 }
10263 {
10267 /* The computation of the size of the re-aligned stack frame means
10268 that we must allocate the size of the register save area before
10269 performing the actual alignment. Otherwise we cannot guarantee
10270 that there's enough storage above the realignment point. */
10277 /* Align the stack. */
10279 stack_pointer_rtx,
10282 /* For the purposes of register save area addressing, the stack
10283 pointer is no longer valid. As for the value of sp_offset,
10284 see ix86_compute_frame_layout, which we need to match in order
10285 to pass verification of stack_pointer_offset at the end. */
10288 }
10293 {
10294 /* We start to count from ARG_POINTER. */
10297 /* If it was realigned, take into account the fake frame. */
10299 {
10306 /* This over-estimates by 1 minimal-stack-alignment-unit but
10307 mitigates that by counting in the new return address slot. */
10308 current_function_dynamic_stack_size
10310 }
10313 }
10315 /* The stack has already been decremented by the instruction calling us
10316 so probe if the size is non-negative to preserve the protection area. */
10318 {
10319 /* We expect the registers to be saved when probes are used. */
10323 {
10326 }
10327 else
10328 {
10336 else
10338 }
10339 }
10342 ;
10345 {
10349 }
10350 else
10351 {
10365 {
10368 }
10370 {
10374 }
10379 /* Use the fact that AX still contains ALLOCATE. */
10381 ? gen_pro_epilogue_adjust_stack_di_sub
10387 /* Note that SEH directives need to continue tracking the stack
10388 pointer even after the frame pointer has been set up. */
10390 {
10399 }
10403 {
10410 }
10412 {
10417 }
10418 }
10421 /* If we havn't already set up the frame pointer, do so now. */
10423 {
10435 }
10446 {
10453 }
10456 {
10458 {
10460 {
10470 label));
10474 }
10475 else
10477 }
10478 else
10479 {
10483 }
10484 }
10486 /* In the pic_reg_used case, make sure that the got load isn't deleted
10487 when mcount needs it. Blockage to avoid call movement across mcount
10488 call is emitted in generic code after the NOTE_INSN_PROLOGUE_END
10489 note. */
10494 {
10495 /* vDRAP is setup but after reload it turns out stack realign
10496 isn't necessary, here we will emit prologue to setup DRAP
10497 without stack realign adjustment */
10500 }
10502 /* Prevent instructions from being scheduled into register save push
10503 sequence when access to the redzone area is done through frame pointer.
10504 The offset between the frame pointer and the stack pointer is calculated
10505 relative to the value of the stack pointer at the end of the function
10506 prologue, and moving instructions that access redzone area via frame
10507 pointer inside push sequence violates this assumption. */
10511 /* Emit cld instruction if stringops are used in the function. */
10515 /* SEH requires that the prologue end within 256 bytes of the start of
10516 the function. Prevent instruction schedules that would extend that.
10517 Further, prevent alloca modifications to the stack pointer from being
10518 combined with prologue modifications. */
10521 }
10523 /* Emit code to restore REG using a POP insn. */
10525 static void
10527 {
10536 {
10537 /* Previously we'd represented the CFA as an expression
10538 like *(%ebp - 8). We've just popped that value from
10539 the stack, which means we need to reset the CFA to
10540 the drap register. This will remain until we restore
10541 the stack pointer. */
10545 /* This means that the DRAP register is valid for addressing too. */
10548 }
10551 {
10558 }
10560 /* When the frame pointer is the CFA, and we pop it, we are
10561 swapping back to the stack pointer as the CFA. This happens
10562 for stack frames that don't allocate other data, so we assume
10563 the stack pointer is now pointing at the return address, i.e.
10564 the function entry state, which makes the offset be 1 word. */
10566 {
10569 {
10577 }
10578 }
10579 }
10581 /* Emit code to restore saved registers using POP insns. */
10583 static void
10585 {
10591 }
10593 /* Emit code and notes for the LEAVE instruction. */
10595 static void
10597 {
10609 {
10616 }
10619 }
10621 /* Emit code to restore saved registers using MOV insns.
10622 First register is restored from CFA - CFA_OFFSET. */
10623 static void
10626 {
10632 {
10641 {
10642 /* Previously we'd represented the CFA as an expression
10643 like *(%ebp - 8). We've just popped that value from
10644 the stack, which means we need to reset the CFA to
10645 the drap register. This will remain until we restore
10646 the stack pointer. */
10650 /* This means that the DRAP register is valid for addressing. */
10652 }
10653 else
10657 }
10658 }
10660 /* Emit code to restore saved registers using MOV insns.
10661 First register is restored from CFA - CFA_OFFSET. */
10662 static void
10665 {
10670 {
10672 rtx mem;
10682 }
10683 }
10685 /* Emit vzeroupper if needed. */
10687 void
10689 {
10694 }
10696 /* Restore function stack, frame, and registers. */
10698 void
10700 {
10711 || (current_function_sp_is_unchanging
10716 /* The FP must be valid if the frame pointer is present. */
10721 /* We must have *some* valid pointer to the stack frame. */
10724 /* The DRAP is never valid at this point. */
10727 /* See the comment about red zone and frame
10728 pointer usage in ix86_expand_prologue. */
10735 /* Determine the CFA offset of the end of the red-zone. */
10738 {
10739 /* The red-zone begins below the return address. */
10742 /* When the register save area is in the aligned portion of
10743 the stack, determine the maximum runtime displacement that
10744 matches up with the aligned frame. */
10748 }
10750 /* Special care must be taken for the normal return case of a function
10751 using eh_return: the eax and edx registers are marked as saved, but
10752 not restored along this path. Adjust the save location to match. */
10756 /* EH_RETURN requires the use of moves to function properly. */
10759 /* SEH requires the use of pops to identify the epilogue. */
10762 /* If we're only restoring one register and sp is not valid then
10763 using a move instruction to restore the register since it's
10764 less work than reloading sp and popping the register. */
10777 && TARGET_USE_LEAVE
10781 else
10785 {
10786 /* Ensure that the entire register save area is addressable via
10787 the stack pointer, if we will restore via sp. */
10792 {
10796 style,
10798 }
10799 }
10801 /* If there are any SSE registers to restore, then we have to do it
10802 via moves, since there's obviously no pop for SSE regs. */
10808 {
10809 rtx t;
10814 /* eh_return epilogues need %ecx added to the stack pointer. */
10816 {
10819 /* Stack align doesn't work with eh_return. */
10821 /* Neither does regparm nested functions. */
10825 {
10833 /* Note that we use SA as a temporary CFA, as the return
10834 address is at the proper place relative to it. We
10835 pretend this happens at the FP restore insn because
10836 prior to this insn the FP would be stored at the wrong
10837 offset relative to SA, and after this insn we have no
10838 other reasonable register to use for the CFA. We don't
10839 bother resetting the CFA to the SP for the duration of
10840 the return insn. */
10853 }
10854 else
10855 {
10863 {
10867 UNITS_PER_WORD));
10869 }
10870 }
10873 }
10874 }
10875 else
10876 {
10877 /* SEH requires that the function end with (1) a stack adjustment
10878 if necessary, (2) a sequence of pops, and (3) a return or
10879 jump instruction. Prevent insns from the function body from
10880 being scheduled into this sequence. */
10882 {
10883 /* Prevent a catch region from being adjacent to the standard
10884 epilogue sequence. Unfortuantely crtl->uses_eh_lsda nor
10885 several other flags that would be interesting to test are
10886 not yet set up. */
10889 else
10891 }
10893 /* First step is to deallocate the stack frame so that we can
10894 pop the registers. */
10896 {
10901 }
10903 {
10907 style,
10909 }
10912 }
10914 /* If we used a stack pointer and haven't already got rid of it,
10915 then do so now. */
10917 {
10918 /* If the stack pointer is valid and pointing at the frame
10919 pointer store address, then we only need a pop. */
10922 /* Leave results in shorter dependency chains on CPUs that are
10923 able to grok it fast. */
10928 else
10929 {
10931 hard_frame_pointer_rtx,
10934 }
10935 }
10938 {
10940 rtx insn;
10966 }
10968 /* At this point the stack pointer must be valid, and we must have
10969 restored all of the registers. We may not have deallocated the
10970 entire stack frame. We've delayed this until now because it may
10971 be possible to merge the local stack deallocation with the
10972 deallocation forced by ix86_static_chain_on_stack. */
10977 {
10981 }
10982 else
10985 /* Sibcall epilogues don't want a return instruction. */
10987 {
10990 }
10992 /* Emit vzeroupper if needed. */
10996 {
10999 /* i386 can only pop 64K bytes. If asked to pop more, pop return
11000 address, do explicit add, and jump indirectly to the caller. */
11003 {
11005 rtx insn;
11007 /* There is no "pascal" calling convention in any 64bit ABI. */
11023 }
11024 else
11026 }
11027 else
11030 /* Restore the state back to the state from the prologue,
11031 so that it's correct for the next epilogue. */
11033 }
11035 /* Reset from the function's potential modifications. */
11037 static void
11039 HOST_WIDE_INT size ATTRIBUTE_UNUSED)
11040 {
11043 #if TARGET_MACHO
11044 /* Mach-O doesn't support labels at the end of objects, so if
11045 it looks like we might want one, insert a NOP. */
11046 {
11052 {
11053 /* Don't insert a nop for NOTE_INSN_DELETED_DEBUG_LABEL
11054 notes only, instead set their CODE_LABEL_NUMBER to -1,
11055 otherwise there would be code generation differences
11056 in between -g and -g0. */
11060 }
11070 }
11071 #endif
11073 }
11075 /* Return a scratch register to use in the split stack prologue. The
11076 split stack prologue is used for -fsplit-stack. It is the first
11077 instructions in the function, even before the regular prologue.
11078 The scratch register can be any caller-saved register which is not
11079 used for parameters or for the static chain. */
11081 static unsigned int
11083 {
11086 else
11087 {
11097 {
11099 {
11103 }
11105 }
11107 {
11110 else
11111 {
11113 {
11117 }
11119 }
11120 }
11121 else
11122 {
11123 /* FIXME: We could make this work by pushing a register
11124 around the addition and comparison. */
11127 }
11128 }
11129 }
11131 /* A SYMBOL_REF for the function which allocates new stackspace for
11132 -fsplit-stack. */
11136 /* A SYMBOL_REF for the more stack function when using the large
11137 model. */
11141 /* Handle -fsplit-stack. These are the first instructions in the
11142 function, even before the regular prologue. */
11144 void
11146 {
11148 HOST_WIDE_INT allocate;
11153 rtx fn;
11161 /* This is the label we will branch to if we have enough stack
11162 space. We expect the basic block reordering pass to reverse this
11163 branch if optimizing, so that we branch in the unlikely case. */
11166 /* We need to compare the stack pointer minus the frame size with
11167 the stack boundary in the TCB. The stack boundary always gives
11168 us SPLIT_STACK_AVAILABLE bytes, so if we need less than that we
11169 can compare directly. Otherwise we need to do an addition. */
11172 UNSPEC_STACK_CHECK);
11177 else
11178 {
11180 rtx offset;
11182 /* We need a scratch register to hold the stack pointer minus
11183 the required frame size. Since this is the very start of the
11184 function, the scratch register can be any caller-saved
11185 register which is not used for parameters. */
11192 {
11193 /* We don't use ix86_gen_add3 in this case because it will
11194 want to split to lea, but when not optimizing the insn
11195 will not be split after this point. */
11198 offset)));
11199 }
11200 else
11201 {
11204 stack_pointer_rtx));
11205 }
11207 }
11213 /* Mark the jump as very likely to be taken. */
11221 /* Get more stack space. We pass in the desired stack space and the
11222 size of the arguments to copy to the new stack. In 32-bit mode
11223 we push the parameters; __morestack will return on a new stack
11224 anyhow. In 64-bit mode we pass the parameters in r10 and
11225 r11. */
11230 {
11236 /* If this function uses a static chain, it will be in %r10.
11237 Preserve it across the call to __morestack. */
11239 {
11240 rtx rax;
11245 }
11248 {
11249 HOST_WIDE_INT argval;
11252 /* When using the large model we need to load the address
11253 into a register, and we've run out of registers. So we
11254 switch to a different calling convention, and we call a
11255 different function: __morestack_large. We pass the
11256 argument size in the upper 32 bits of r10 and pass the
11257 frame size in the lower 32 bits. */
11262 split_stack_fn_large =
11266 {
11276 UNSPEC_GOT);
11282 }
11283 else
11290 }
11291 else
11292 {
11296 }
11299 }
11300 else
11301 {
11304 }
11310 /* In order to make call/return prediction work right, we now need
11311 to execute a return instruction. See
11312 libgcc/config/i386/morestack.S for the details on how this works.
11314 For flow purposes gcc must not see this as a return
11315 instruction--we need control flow to continue at the subsequent
11316 label. Therefore, we use an unspec. */
11320 /* If we are in 64-bit mode and this function uses a static chain,
11321 we saved %r10 in %rax before calling _morestack. */
11326 /* If this function calls va_start, we need to store a pointer to
11327 the arguments on the old stack, because they may not have been
11328 all copied to the new stack. At this point the old stack can be
11329 found at the frame pointer value used by __morestack, because
11330 __morestack has set that up before calling back to us. Here we
11331 store that pointer in a scratch register, and in
11332 ix86_expand_prologue we store the scratch register in a stack
11333 slot. */
11335 {
11337 rtx frame_reg;
11344 /* 64-bit:
11345 fp -> old fp value
11346 return address within this function
11347 return address of caller of this function
11348 stack arguments
11349 So we add three words to get to the stack arguments.
11351 32-bit:
11352 fp -> old fp value
11353 return address within this function
11354 first argument to __morestack
11355 second argument to __morestack
11356 return address of caller of this function
11357 stack arguments
11358 So we add five words to get to the stack arguments.
11359 */
11370 }
11375 /* If this function calls va_start, we now have to set the scratch
11376 register for the case where we do not call __morestack. In this
11377 case we need to set it based on the stack pointer. */
11379 {
11386 }
11387 }
11389 /* We may have to tell the dataflow pass that the split stack prologue
11390 is initializing a scratch register. */
11392 static void
11394 {
11396 {
11399 }
11400 }
11401 \f
11402 /* Determine if op is suitable SUBREG RTX for address. */
11404 static bool
11406 {
11417 /* Don't allow SUBREGs that span more than a word. It can lead to spill
11418 failures when the register is one word out of a two word structure. */
11422 /* Allow only SUBREGs of non-eliminable hard registers. */
11424 }
11426 /* Extract the parts of an RTL expression that is a valid memory address
11427 for an instruction. Return 0 if the structure of the address is
11428 grossly off. Return -1 if the address contains ASHIFT, so it is not
11429 strictly valid, but still used for computing length of lea instruction. */
11431 int
11433 {
11438 rtx tmp;
11442 /* Allow zero-extended SImode addresses,
11443 they will be emitted with addr32 prefix. */
11445 {
11451 {
11454 /* Adjust SUBREGs. */
11460 else
11462 }
11463 }
11468 {
11471 else
11473 }
11475 {
11480 do
11481 {
11486 }
11493 {
11496 {
11521 /* FALLTHRU */
11525 && TARGET_TLS_DIRECT_SEG_REFS
11528 else
11535 /* FALLTHRU */
11542 else
11557 }
11558 }
11559 }
11561 {
11564 }
11566 {
11567 /* We're called for lea too, which implements ashift on occasion. */
11577 }
11578 else
11582 {
11584 ;
11587 ;
11588 else
11590 }
11592 /* Address override works only on the (%reg) part of %fs:(%reg). */
11598 /* Extract the integral value of scale. */
11600 {
11604 }
11609 /* Avoid useless 0 displacement. */
11613 /* Allow arg pointer and stack pointer as index if there is not scaling. */
11618 {
11619 rtx tmp;
11622 }
11624 /* Special case: %ebp cannot be encoded as a base without a displacement.
11625 Similarly %r13. */
11627 && base_reg
11636 /* Special case: on K6, [%esi] makes the instruction vector decoded.
11637 Avoid this by transforming to [%esi+0].
11638 Reload calls address legitimization without cfun defined, so we need
11639 to test cfun for being non-NULL. */
11645 /* Special case: encode reg+reg instead of reg*2. */
11649 /* Special case: scaling cannot be encoded without base or displacement. */
11660 }
11661 \f
11662 /* Return cost of the memory address x.
11663 For i386, it is better to use a complex address than let gcc copy
11664 the address into a reg and make a new pseudo. But not if the address
11665 requires to two regs - that would mean more pseudos with longer
11666 lifetimes. */
11667 static int
11669 {
11681 /* Attempt to minimize number of registers in the address. */
11687 cost++;
11694 cost++;
11696 /* AMD-K6 don't like addresses with ModR/M set to 00_xxx_100b,
11697 since it's predecode logic can't detect the length of instructions
11698 and it degenerates to vector decoded. Increase cost of such
11699 addresses here. The penalty is minimally 2 cycles. It may be worthwhile
11700 to split such addresses or even refuse such addresses at all.
11702 Following addressing modes are affected:
11703 [base+scale*index]
11704 [scale*index+disp]
11705 [base+index]
11707 The first and last case may be avoidable by explicitly coding the zero in
11708 memory address, but I don't have AMD-K6 machine handy to check this
11709 theory. */
11718 }
11719 \f
11720 /* Allow {LABEL | SYMBOL}_REF - SYMBOL_REF-FOR-PICBASE for Mach-O as
11721 this is used for to form addresses to local data when -fPIC is in
11722 use. */
11724 static bool
11726 {
11729 }
11731 /* Determine if a given RTX is a valid constant. We already know this
11732 satisfies CONSTANT_P. */
11734 static bool
11736 {
11738 {
11743 {
11747 }
11752 /* Only some unspecs are valid as "constants". */
11755 {
11771 }
11773 /* We must have drilled down to a symbol. */
11778 /* FALLTHRU */
11781 /* TLS symbols are never valid. */
11785 /* DLLIMPORT symbols are never valid. */
11790 #if TARGET_MACHO
11791 /* mdynamic-no-pic */
11794 #endif
11810 }
11812 /* Otherwise we handle everything else in the move patterns. */
11814 }
11816 /* Determine if it's legal to put X into the constant pool. This
11817 is not possible for the address of thread-local symbols, which
11818 is checked above. */
11820 static bool
11822 {
11823 /* We can always put integral constants and vectors in memory. */
11825 {
11833 }
11835 }
11838 /* Nonzero if the constant value X is a legitimate general operand
11839 when generating PIC code. It is given that flag_pic is on and
11840 that X satisfies CONSTANT_P or is a CONST_DOUBLE. */
11842 bool
11844 {
11845 rtx inner;
11848 {
11855 /* Only some unspecs are valid as "constants". */
11858 {
11871 }
11872 /* FALLTHRU */
11880 }
11881 }
11883 /* Determine if a given CONST RTX is a valid memory displacement
11884 in PIC mode. */
11886 bool
11888 {
11891 /* In 64bit mode we can allow direct addresses of symbols and labels
11892 when they are not dynamic symbols. */
11894 {
11898 {
11922 /* FALLTHRU */
11925 /* TLS references should always be enclosed in UNSPEC. */
11935 }
11936 }
11942 {
11943 /* We are unsafe to allow PLUS expressions. This limit allowed distance
11944 of GOT tables. We should not need these anyway. */
11956 }
11960 {
11965 }
11974 {
11978 /* We need to check for both symbols and labels because VxWorks loads
11979 text labels with @GOT rather than @GOTOFF. See gotoff_operand for
11980 details. */
11984 /* Refuse GOTOFF in 64bit mode since it is always 64bit when used.
11985 While ABI specify also 32bit relocation but we don't produce it in
11986 small PIC model at all. */
12008 }
12011 }
12013 /* Recognizes RTL expressions that are valid memory addresses for an
12014 instruction. The MODE argument is the machine mode for the MEM
12015 expression that wants to use this address.
12017 It only recognizes address in canonical form. LEGITIMIZE_ADDRESS should
12018 convert common non-canonical forms to canonical form so that they will
12019 be recognized. */
12021 static bool
12024 {
12027 HOST_WIDE_INT scale;
12029 /* Since constant address in x32 is signed extended to 64bit,
12030 we have to prevent addresses from 0x80000000 to 0xffffffff. */
12037 /* Decomposition failed. */
12045 /* Validate base register. */
12047 {
12048 rtx reg;
12054 else
12055 /* Base is not a register. */
12063 /* Base is not valid. */
12065 }
12067 /* Validate index register. */
12069 {
12070 rtx reg;
12076 else
12077 /* Index is not a register. */
12085 /* Index is not valid. */
12087 }
12089 /* Index and base should have the same mode. */
12094 /* Validate scale factor. */
12096 {
12098 /* Scale without index. */
12102 /* Scale is not a valid multiplier. */
12104 }
12106 /* Validate displacement. */
12108 {
12113 {
12114 /* Refuse GOTOFF and GOT in 64bit mode since it is always 64bit when
12115 used. While ABI specify also 32bit relocations, we don't produce
12116 them at all and use IP relative instead. */
12123 /* 64bit address unspec. */
12143 /* Invalid address unspec. */
12145 }
12148 && (flag_pic
12149 || (TARGET_MACHO
12150 #if TARGET_MACHO
12151 && MACHOPIC_INDIRECT
12153 #endif
12154 )))
12155 {
12157 is_legitimate_pic:
12159 {
12160 /* foo@dtpoff(%rX) is ok. */
12167 /* Non-constant pic memory reference. */
12169 }
12172 /* Displacement is an invalid pic construct. */
12174 #if TARGET_MACHO
12177 /* displacment must be referenced via non_lazy_pointer */
12179 #endif
12181 /* This code used to verify that a symbolic pic displacement
12182 includes the pic_offset_table_rtx register.
12184 While this is good idea, unfortunately these constructs may
12185 be created by "adds using lea" optimization for incorrect
12186 code like:
12188 int a;
12189 int foo(int i)
12190 {
12191 return *(&a+i);
12192 }
12194 This code is nonsensical, but results in addressing
12195 GOT table with pic_offset_table_rtx base. We can't
12196 just refuse it easily, since it gets matched by
12197 "addsi3" pattern, that later gets split to lea in the
12198 case output register differs from input. While this
12199 can be handled by separate addsi pattern for this case
12200 that never results in lea, this seems to be easier and
12201 correct fix for crash to disable this test. */
12202 }
12209 /* Displacement is not constant. */
12213 /* Displacement is out of range. */
12215 }
12217 /* Everything looks valid. */
12219 }
12221 /* Determine if a given RTX is a valid constant address. */
12223 bool
12225 {
12227 }
12228 \f
12229 /* Return a unique alias set for the GOT. */
12231 static alias_set_type
12233 {
12238 }
12240 /* Return a legitimate reference for ORIG (an address) using the
12241 register REG. If REG is 0, a new pseudo is generated.
12243 There are two types of references that must be handled:
12245 1. Global data references must load the address from the GOT, via
12246 the PIC reg. An insn is emitted to do this load, and the reg is
12247 returned.
12249 2. Static data references, constant pool addresses, and code labels
12250 compute the address as an offset from the GOT, whose base is in
12251 the PIC reg. Static data objects have SYMBOL_FLAG_LOCAL set to
12252 differentiate them from global data objects. The returned
12253 address is the PIC reg + an unspec constant.
12255 TARGET_LEGITIMATE_ADDRESS_P rejects symbolic references unless the PIC
12256 reg also appears in the address. */
12258 static rtx
12260 {
12263 rtx base;
12265 #if TARGET_MACHO
12267 {
12270 /* Use the generic Mach-O PIC machinery. */
12272 }
12273 #endif
12280 {
12281 rtx tmpreg;
12282 /* This symbol may be referenced via a displacement from the PIC
12283 base address (@GOTOFF). */
12290 {
12292 UNSPEC_GOTOFF);
12294 }
12295 else
12300 else
12305 {
12309 }
12311 }
12313 {
12314 /* This symbol may be referenced via a displacement from the PIC
12315 base address (@GOTOFF). */
12322 {
12324 UNSPEC_GOTOFF);
12326 }
12327 else
12333 {
12336 }
12337 }
12339 /* We can't use @GOTOFF for text labels on VxWorks;
12340 see gotoff_operand. */
12342 {
12344 {
12350 {
12353 }
12354 }
12356 /* For x64 PE-COFF there is no GOT table. So we use address
12357 directly. */
12359 {
12367 }
12369 {
12377 /* Use directly gen_movsi, otherwise the address is loaded
12378 into register for CSE. We don't want to CSE this addresses,
12379 instead we CSE addresses from the GOT table, so skip this. */
12382 }
12383 else
12384 {
12385 /* This symbol must be referenced via a load from the
12386 Global Offset Table (@GOT). */
12402 }
12403 }
12404 else
12405 {
12408 {
12410 {
12413 }
12414 else
12416 }
12418 {
12421 /* We must match stuff we generate before. Assume the only
12422 unspecs that can get here are ours. Not that we could do
12423 anything with them anyway.... */
12429 }
12431 {
12434 /* Check first to see if this is a constant offset from a @GOTOFF
12435 symbol reference. */
12438 {
12440 {
12444 UNSPEC_GOTOFF);
12450 {
12453 }
12454 }
12455 else
12456 {
12459 {
12463 }
12464 }
12465 }
12466 else
12467 {
12474 else
12475 {
12477 {
12480 }
12482 }
12483 }
12484 }
12485 }
12487 }
12488 \f
12489 /* Load the thread pointer. If TO_REG is true, force it into a register. */
12491 static rtx
12493 {
12497 {
12502 }
12508 }
12510 /* Construct the SYMBOL_REF for the tls_get_addr function. */
12514 static rtx
12516 {
12518 {
12524 }
12527 }
12529 /* Construct the SYMBOL_REF for the _TLS_MODULE_BASE_ symbol. */
12533 rtx
12535 {
12537 {
12538 ix86_tls_module_base_symbol
12543 }
12546 }
12548 /* A subroutine of ix86_legitimize_address and ix86_expand_move. FOR_MOV is
12549 false if we expect this to be used for a memory address and true if
12550 we expect to load the address into a register. */
12552 static rtx
12554 {
12561 {
12566 {
12569 else
12570 {
12573 }
12574 }
12577 {
12580 else
12587 }
12588 else
12589 {
12593 {
12598 caddr));
12604 }
12605 else
12607 }
12614 {
12617 else
12618 {
12621 }
12622 }
12625 {
12630 else
12636 }
12637 else
12638 {
12642 {
12647 caddr));
12651 /* Attach a unique REG_EQUAL, to allow the RTL optimizers to
12652 share the LD_BASE result with other LD model accesses. */
12654 UNSPEC_TLS_LD_BASE);
12658 }
12659 else
12661 }
12669 {
12673 }
12678 {
12680 {
12681 /* The Sun linker took the AMD64 TLS spec literally
12682 and can only handle %rax as destination of the
12683 initial executable code sequence. */
12688 }
12690 /* Generate DImode references to avoid %fs:(%reg32)
12691 problems and linker IE->LE relaxation bug. */
12695 }
12697 {
12702 }
12704 {
12708 }
12709 else
12710 {
12713 }
12723 {
12728 }
12729 else
12730 {
12734 }
12744 {
12748 }
12749 else
12750 {
12754 }
12759 }
12762 }
12764 /* Create or return the unique __imp_DECL dllimport symbol corresponding
12765 to symbol DECL. */
12768 htab_t dllimport_map;
12770 static tree
12772 {
12779 tree to;
12780 rtx rtl;
12824 }
12826 /* Expand SYMBOL into its corresponding dllimport symbol. WANT_REG is
12827 true if we require the result be a register. */
12829 static rtx
12831 {
12832 tree imp_decl;
12833 rtx x;
12842 }
12844 /* Try machine-dependent ways of modifying an illegitimate address
12845 to be legitimate. If we find one, return the new, valid address.
12846 This macro is used in only one place: `memory_address' in explow.c.
12848 OLDX is the address as it was before break_out_memory_refs was called.
12849 In some cases it is useful to look at this to decide what needs to be done.
12851 It is always safe for this macro to do nothing. It exists to recognize
12852 opportunities to optimize the output.
12854 For the 80386, we handle X+REG by loading X into a register R and
12855 using R+REG. R will go in a general reg and indexing will be used.
12856 However, if REG is a broken-out memory address or multiplication,
12857 nothing needs to be done because REG can certainly go in a general reg.
12859 When -fpic is used, special handling is needed for symbolic references.
12860 See comments by legitimize_pic_address in i386.c for details. */
12862 static rtx
12865 {
12876 {
12880 }
12883 {
12890 {
12893 }
12894 }
12899 #if TARGET_MACHO
12902 #endif
12904 /* Canonicalize shifts by 0, 1, 2, 3 into multiply */
12908 {
12913 }
12916 {
12917 /* Canonicalize shifts by 0, 1, 2, 3 into multiply. */
12922 {
12928 }
12933 {
12939 }
12941 /* Put multiply first if it isn't already. */
12943 {
12948 }
12950 /* Canonicalize (plus (mult (reg) (const)) (plus (reg) (const)))
12951 into (plus (plus (mult (reg) (const)) (reg)) (const)). This can be
12952 created by virtual register instantiation, register elimination, and
12953 similar optimizations. */
12955 {
12961 }
12963 /* Canonicalize
12964 (plus (plus (mult (reg) (const)) (plus (reg) (const))) const)
12965 into (plus (plus (mult (reg) (const)) (reg)) (const)). */
12970 {
12971 rtx constant;
12975 {
12978 }
12980 {
12983 }
12984 else
12988 {
12994 }
12995 }
13001 {
13004 }
13007 {
13010 }
13018 {
13021 }
13027 {
13031 {
13035 }
13039 }
13042 {
13046 {
13050 }
13054 }
13055 }
13058 }
13059 \f
13060 /* Print an integer constant expression in assembler syntax. Addition
13061 and subtraction are the only arithmetic that may appear in these
13062 expressions. FILE is the stdio stream to write to, X is the rtx, and
13063 CODE is the operand print code from the output string. */
13065 static void
13067 {
13071 {
13080 else
13081 {
13084 /* Mark the decl as referenced so that cgraph will
13085 output the function. */
13089 #if TARGET_MACHO
13093 #endif
13095 }
13103 /* FALLTHRU */
13114 /* This used to output parentheses around the expression,
13115 but that does not work on the 386 (either ATT or BSD assembler). */
13121 {
13122 /* We can use %d if the number is <32 bits and positive. */
13127 else
13129 }
13130 else
13131 /* We can't handle floating point constants;
13132 TARGET_PRINT_OPERAND must handle them. */
13137 /* Some assemblers need integer constants to appear first. */
13139 {
13143 }
13144 else
13145 {
13150 }
13165 {
13169 }
13174 {
13193 /* FIXME: This might be @TPOFF in Sun ld too. */
13202 else
13212 else
13218 #if TARGET_MACHO
13223 #endif
13227 }
13232 }
13233 }
13235 /* This is called from dwarf2out.c via TARGET_ASM_OUTPUT_DWARF_DTPREL.
13236 We need to emit DTP-relative relocations. */
13238 static void ATTRIBUTE_UNUSED
13240 {
13245 {
13253 }
13254 }
13256 /* Return true if X is a representation of the PIC register. This copes
13257 with calls from ix86_find_base_term, where the register might have
13258 been replaced by a cselib value. */
13260 static bool
13262 {
13266 else
13268 }
13270 /* Helper function for ix86_delegitimize_address.
13271 Attempt to delegitimize TLS local-exec accesses. */
13273 static rtx
13275 {
13300 {
13305 }
13311 }
13313 /* In the name of slightly smaller debug output, and to cater to
13314 general assembler lossage, recognize PIC+GOTOFF and turn it back
13315 into a direct symbol reference.
13317 On Darwin, this is necessary to avoid a crash, because Darwin
13318 has a different PIC label for each routine but the DWARF debugging
13319 information is not associated with any particular routine, so it's
13320 necessary to remove references to the PIC label from RTL stored by
13321 the DWARF output code. */
13323 static rtx
13325 {
13327 /* addend is NULL or some rtx if x is something+GOTOFF where
13328 something doesn't include the PIC register. */
13330 /* reg_addend is NULL or a multiple of some register. */
13332 /* const_addend is NULL or a const_int. */
13334 /* This is the result, or NULL. */
13343 {
13350 {
13356 }
13365 {
13370 }
13372 }
13379 /* %ebx + GOT/GOTOFF */
13380 ;
13382 {
13383 /* %ebx + %reg * scale + GOT/GOTOFF */
13389 else
13390 {
13393 }
13394 }
13395 else
13401 {
13404 }
13423 {
13424 /* If the rest of original X doesn't involve the PIC register, add
13425 addend and subtract pic_offset_table_rtx. This can happen e.g.
13426 for code like:
13427 leal (%ebx, %ecx, 4), %ecx
13428 ...
13429 movl foo@GOTOFF(%ecx), %edx
13430 in which case we return (%ecx - %ebx) + foo. */
13433 pic_offset_table_rtx),
13434 result);
13435 else
13437 }
13439 {
13443 }
13445 }
13447 /* If X is a machine specific address (i.e. a symbol or label being
13448 referenced as a displacement from the GOT implemented using an
13449 UNSPEC), then return the base term. Otherwise return X. */
13451 rtx
13453 {
13454 rtx term;
13457 {
13471 }
13474 }
13475 \f
13476 static void
13479 {
13483 {
13486 }
13491 {
13494 {
13513 }
13517 {
13536 }
13543 /* ??? Use "nbe" instead of "a" for fcmov lossage on some assemblers.
13544 Those same assemblers have the same but opposite lossage on cmov. */
13549 else
13554 {
13567 }
13575 {
13588 }
13591 /* ??? As above. */
13600 /* ??? As above. */
13605 else
13616 }
13618 }
13620 /* Print the name of register X to FILE based on its machine mode and number.
13621 If CODE is 'w', pretend the mode is HImode.
13622 If CODE is 'b', pretend the mode is QImode.
13623 If CODE is 'k', pretend the mode is SImode.
13624 If CODE is 'q', pretend the mode is DImode.
13625 If CODE is 'x', pretend the mode is V4SFmode.
13626 If CODE is 't', pretend the mode is V8SFmode.
13627 If CODE is 'h', pretend the reg is the 'high' byte register.
13628 If CODE is 'y', print "st(0)" instead of "st", if the reg is stack op.
13629 If CODE is 'd', duplicate the operand for AVX instruction.
13630 */
13632 void
13634 {
13649 {
13653 }
13671 else
13674 /* Irritatingly, AMD extended registers use different naming convention
13675 from the normal registers: "r%d[bwd]" */
13677 {
13682 {
13696 /* no suffix */
13701 }
13703 }
13707 {
13710 {
13713 }
13714 /* FALLTHRU */
13720 /* FALLTHRU */
13723 normal:
13738 {
13743 }
13747 }
13751 {
13754 else
13756 }
13757 }
13759 /* Locate some local-dynamic symbol still in use by this function
13760 so that we can print its name in some tls_local_dynamic_base
13761 pattern. */
13763 static int
13765 {
13770 {
13773 }
13776 }
13780 {
13781 rtx insn;
13792 }
13794 /* Meaning of CODE:
13795 L,W,B,Q,S,T -- print the opcode suffix for specified size of operand.
13796 C -- print opcode suffix for set/cmov insn.
13797 c -- like C, but print reversed condition
13798 F,f -- likewise, but for floating-point.
13799 O -- if HAVE_AS_IX86_CMOV_SUN_SYNTAX, expand to "w.", "l." or "q.",
13800 otherwise nothing
13801 R -- print the prefix for register names.
13802 z -- print the opcode suffix for the size of the current operand.
13803 Z -- likewise, with special suffixes for x87 instructions.
13804 * -- print a star (in certain assembler syntax)
13805 A -- print an absolute memory reference.
13806 E -- print address with DImode register names if TARGET_64BIT.
13807 w -- print the operand as if it's a "word" (HImode) even if it isn't.
13808 s -- print a shift double count, followed by the assemblers argument
13809 delimiter.
13810 b -- print the QImode name of the register for the indicated operand.
13811 %b0 would print %al if operands[0] is reg 0.
13812 w -- likewise, print the HImode name of the register.
13813 k -- likewise, print the SImode name of the register.
13814 q -- likewise, print the DImode name of the register.
13815 x -- likewise, print the V4SFmode name of the register.
13816 t -- likewise, print the V8SFmode name of the register.
13817 h -- print the QImode name for a "high" register, either ah, bh, ch or dh.
13818 y -- print "st(0)" instead of "st" as a register.
13819 d -- print duplicated register operand for AVX instruction.
13820 D -- print condition for SSE cmp instruction.
13821 P -- if PIC, print an @PLT suffix.
13822 p -- print raw symbol name.
13823 X -- don't print any sort of PIC '@' suffix for a symbol.
13824 & -- print some in-use local-dynamic symbol name.
13825 H -- print a memory address offset by 8; used for sse high-parts
13826 Y -- print condition for XOP pcom* instruction.
13827 + -- print a branch hint as 'cs' or 'ds' prefix
13828 ; -- print a semicolon (after prefixes due to bug in older gas).
13829 ~ -- print "i" if TARGET_AVX2, "f" otherwise.
13830 @ -- print a segment register of thread base pointer load
13831 ^ -- print addr32 prefix if TARGET_64BIT and Pmode != word_mode
13832 */
13834 void
13836 {
13838 {
13840 {
13847 {
13852 else
13855 }
13859 {
13865 /* Intel syntax. For absolute addresses, registers should not
13866 be surrounded by braces. */
13868 {
13873 }
13878 }
13884 /* Wrap address in an UNSPEC to declare special handling. */
13923 {
13924 /* Opcodes don't get size suffixes if using Intel opcodes. */
13929 {
13947 output_operand_lossage
13950 }
13951 }
13954 warning
13956 /* FALLTHRU */
13959 /* 387 opcodes don't get size suffixes if using Intel opcodes. */
13964 {
13966 {
13968 #ifdef HAVE_AS_IX86_FILDS
13970 #endif
13978 #ifdef HAVE_AS_IX86_FILDQ
13980 #else
13982 #endif
13987 }
13988 }
13990 {
13991 /* 387 opcodes don't get size suffixes
13992 if the operands are registers. */
13997 {
14013 }
14014 }
14015 else
14016 {
14017 output_operand_lossage
14020 }
14022 output_operand_lossage
14042 {
14045 }
14049 /* Little bit of braindamage here. The SSE compare instructions
14050 does use completely different names for the comparisons that the
14051 fp conditional moves. */
14053 {
14055 {
14102 }
14103 }
14104 else
14105 {
14107 {
14142 }
14143 }
14146 #ifdef HAVE_AS_IX86_CMOV_SUN_SYNTAX
14148 {
14150 {
14157 }
14159 }
14160 #endif
14164 {
14166 "condition code, invalid operand code "
14169 }
14174 {
14176 "condition code, invalid operand code "
14179 }
14180 #ifdef HAVE_AS_IX86_CMOV_SUN_SYNTAX
14183 #endif
14187 /* Like above, but reverse condition */
14189 /* Check to see if argument to %c is really a constant
14190 and not a condition code which needs to be reversed. */
14192 {
14194 "condition code, invalid operand "
14197 }
14202 {
14204 "condition code, invalid operand "
14207 }
14208 #ifdef HAVE_AS_IX86_CMOV_SUN_SYNTAX
14211 #endif
14217 {
14219 "reference, invalid operand "
14222 }
14223 /* It doesn't actually matter what mode we use here, as we're
14224 only going to use this for printing. */
14229 {
14230 rtx x;
14239 {
14244 {
14246 bool cputaken
14249 /* Emit hints only in the case default branch prediction
14250 heuristics would fail. */
14252 {
14253 /* We use 3e (DS) prefix for taken branches and
14254 2e (CS) prefix for not taken branches. */
14257 else
14259 }
14260 }
14261 }
14263 }
14267 {
14318 }
14322 #ifndef HAVE_AS_IX86_REP_LOCK_PREFIX
14324 #endif
14331 /* The kernel uses a different segment register for performance
14332 reasons; a system call would not have to trash the userspace
14333 segment register, which would be expensive. */
14336 else
14351 }
14352 }
14358 {
14359 /* No `byte ptr' prefix for call instructions or BLKmode operands. */
14362 {
14365 {
14374 else
14380 }
14382 /* Check for explicit size override (codes 'b', 'w', 'k',
14383 'q' and 'x') */
14397 }
14400 /* Avoid (%rip) for call operands. */
14406 else
14408 }
14411 {
14412 REAL_VALUE_TYPE r;
14420 /* Sign extend 32bit SFmode immediate to 8 bytes. */
14423 else
14425 }
14428 {
14429 REAL_VALUE_TYPE r;
14438 }
14440 /* These float cases don't actually occur as immediate operands. */
14442 {
14447 }
14449 else
14450 {
14451 /* We have patterns that allow zero sets of memory, for instance.
14452 In 64-bit mode, we should probably support all 8-byte vectors,
14453 since we can in fact encode that into an immediate. */
14455 {
14458 }
14461 {
14463 {
14466 }
14469 {
14472 else
14474 }
14475 }
14480 else
14482 }
14483 }
14485 static bool
14487 {
14490 }
14491 \f
14492 /* Print a memory operand whose address is ADDR. */
14494 static void
14496 {
14505 {
14512 }
14514 {
14518 }
14519 else
14525 {
14529 }
14532 {
14536 }
14544 {
14555 }
14557 /* Use one byte shorter RIP relative addressing for 64bit mode. */
14559 {
14571 }
14573 {
14574 /* Displacement only requires special attention. */
14577 {
14581 }
14584 else
14586 }
14587 else
14588 {
14589 /* Print SImode register names for zero-extended
14590 addresses to force addr32 prefix. */
14594 {
14597 }
14600 {
14602 {
14607 else
14609 }
14615 {
14620 }
14622 }
14623 else
14624 {
14628 {
14629 /* Pull out the offset of a symbol; print any symbol itself. */
14633 {
14637 }
14645 else
14647 }
14651 {
14654 {
14658 }
14659 }
14662 else
14666 {
14671 }
14673 }
14674 }
14675 }
14677 /* Implementation of TARGET_ASM_OUTPUT_ADDR_CONST_EXTRA. */
14679 static bool
14681 {
14682 rtx op;
14689 {
14692 /* FIXME: This might be @TPOFF in Sun ld. */
14703 else
14715 else
14722 #if TARGET_MACHO
14728 #endif
14731 {
14736 #ifdef TARGET_THREAD_SPLIT_STACK_OFFSET
14738 #else
14740 #endif
14743 }
14748 }
14751 }
14752 \f
14753 /* Split one or more double-mode RTL references into pairs of half-mode
14754 references. The RTL can be REG, offsettable MEM, integer constant, or
14755 CONST_DOUBLE. "operands" is a pointer to an array of double-mode RTLs to
14756 split and "num" is its length. lo_half and hi_half are output arrays
14757 that parallel "operands". */
14759 void
14762 {
14767 {
14776 }
14781 {
14784 /* simplify_subreg refuse to split volatile memory addresses,
14785 but we still have to handle it. */
14787 {
14790 }
14791 else
14792 {
14799 }
14800 }
14801 }
14802 \f
14803 /* Output code to perform a 387 binary operation in INSN, one of PLUS,
14804 MINUS, MULT or DIV. OPERANDS are the insn operands, where operands[3]
14805 is the expression of the binary operation. The output may either be
14806 emitted here, or returned to the caller, like all output_* functions.
14808 There is no guarantee that the operands are the same mode, as they
14809 might be within FLOAT or FLOAT_EXTEND expressions. */
14811 #ifndef SYSV386_COMPAT
14812 /* Set to 1 for compatibility with brain-damaged assemblers. No-one
14813 wants to fix the assemblers because that causes incompatibility
14814 with gcc. No-one wants to fix gcc because that causes
14815 incompatibility with assemblers... You can use the option of
14816 -DSYSV386_COMPAT=0 if you recompile both gcc and gas this way. */
14817 #define SYSV386_COMPAT 1
14818 #endif
14822 {
14828 #ifdef ENABLE_CHECKING
14829 /* Even if we do not want to check the inputs, this documents input
14830 constraints. Which helps in understanding the following code. */
14840 else
14842 #endif
14845 {
14850 else
14859 else
14868 else
14877 else
14884 }
14887 {
14889 {
14893 else
14895 }
14896 else
14897 {
14901 else
14903 }
14905 }
14909 {
14913 {
14917 }
14919 /* know operands[0] == operands[1]. */
14922 {
14925 }
14928 {
14930 /* How is it that we are storing to a dead operand[2]?
14931 Well, presumably operands[1] is dead too. We can't
14932 store the result to st(0) as st(0) gets popped on this
14933 instruction. Instead store to operands[2] (which I
14934 think has to be st(1)). st(1) will be popped later.
14935 gcc <= 2.8.1 didn't have this check and generated
14936 assembly code that the Unixware assembler rejected. */
14938 else
14941 }
14945 else
14952 {
14955 }
14958 {
14961 }
14964 {
14965 #if SYSV386_COMPAT
14966 /* The SystemV/386 SVR3.2 assembler, and probably all AT&T
14967 derived assemblers, confusingly reverse the direction of
14968 the operation for fsub{r} and fdiv{r} when the
14969 destination register is not st(0). The Intel assembler
14970 doesn't have this brain damage. Read !SYSV386_COMPAT to
14971 figure out what the hardware really does. */
14974 else
14976 #else
14978 /* As above for fmul/fadd, we can't store to st(0). */
14980 else
14982 #endif
14984 }
14987 {
14988 #if SYSV386_COMPAT
14991 else
14993 #else
14996 else
14998 #endif
15000 }
15003 {
15006 else
15009 }
15011 {
15012 #if SYSV386_COMPAT
15014 #else
15016 #endif
15017 }
15018 else
15019 {
15020 #if SYSV386_COMPAT
15022 #else
15024 #endif
15025 }
15030 }
15034 }
15036 /* Return needed mode for entity in optimize_mode_switching pass. */
15038 int
15040 {
15043 /* The mode UNINITIALIZED is used to store control word after a
15044 function call or ASM pattern. The mode ANY specify that function
15045 has no requirements on the control word and make no changes in the
15046 bits we are interested in. */
15060 {
15083 }
15086 }
15088 /* Output code to initialize control word copies used by trunc?f?i and
15089 rounding patterns. CURRENT_MODE is set to current control word,
15090 while NEW_MODE is set to new control word. */
15092 void
15094 {
15096 rtx new_mode;
15107 {
15109 {
15111 /* round toward zero (truncate) */
15117 /* round down toward -oo */
15124 /* round up toward +oo */
15131 /* mask precision exception for nearbyint() */
15138 }
15139 }
15140 else
15141 {
15143 {
15145 /* round toward zero (truncate) */
15151 /* round down toward -oo */
15157 /* round up toward +oo */
15163 /* mask precision exception for nearbyint() */
15170 }
15171 }
15177 }
15179 /* Output code for INSN to convert a float to a signed int. OPERANDS
15180 are the insn operands. The output may be [HSD]Imode and the input
15181 operand may be [SDX]Fmode. */
15185 {
15190 /* Jump through a hoop or two for DImode, since the hardware has no
15191 non-popping instruction. We used to do this a different way, but
15192 that was somewhat fragile and broke with post-reload splitters. */
15202 else
15203 {
15208 else
15212 }
15215 }
15217 /* Output code for x87 ffreep insn. The OPNO argument, which may only
15218 have the values zero or one, indicates the ffreep insn's operand
15219 from the OPERANDS array. */
15223 {
15225 #ifdef HAVE_AS_IX86_FFREEP
15227 #else
15228 {
15238 }
15239 #endif
15242 }
15245 /* Output code for INSN to compare OPERANDS. EFLAGS_P is 1 when fcomi
15246 should be used. UNORDERED_P is true when fucom should be used. */
15250 {
15256 {
15259 }
15260 else
15261 {
15264 }
15267 {
15271 else
15273 else
15276 else
15278 }
15285 {
15287 {
15290 }
15291 else
15293 }
15296 && stack_top_dies
15299 {
15300 /* If both the top of the 387 stack dies, and the other operand
15301 is also a stack register that dies, then this must be a
15302 `fcompp' float compare */
15305 {
15306 /* There is no double popping fcomi variant. Fortunately,
15307 eflags is immune from the fstp's cc clobbering. */
15310 else
15313 }
15314 else
15315 {
15318 else
15320 }
15321 }
15322 else
15323 {
15324 /* Encoded here as eflags_p | intmode | unordered_p | stack_top_dies. */
15327 {
15335 NULL,
15336 NULL,
15343 NULL,
15344 NULL,
15345 NULL,
15346 NULL
15347 };
15362 }
15363 }
15365 void
15367 {
15370 #ifdef ASM_QUAD
15373 #else
15375 #endif
15378 }
15380 void
15382 {
15385 #ifdef ASM_QUAD
15388 #else
15390 #endif
15391 /* We can't use @GOTOFF for text labels on VxWorks; see gotoff_operand. */
15397 #if TARGET_MACHO
15399 {
15403 }
15404 #endif
15405 else
15408 }
15409 \f
15410 /* Generate either "mov $0, reg" or "xor reg, reg", as appropriate
15411 for the target. */
15413 void
15415 {
15416 rtx tmp;
15418 /* We play register width games, which are only valid after reload. */
15421 /* Avoid HImode and its attendant prefix byte. */
15426 /* This predicate should match that for movsi_xor and movdi_xor_rex64. */
15428 {
15431 }
15434 }
15436 /* X is an unchanging MEM. If it is a constant pool reference, return
15437 the constant pool rtx, else NULL. */
15439 rtx
15441 {
15448 }
15450 void
15452 {
15460 {
15463 {
15470 }
15474 }
15478 {
15491 {
15499 }
15500 }
15504 {
15506 {
15507 #if TARGET_MACHO
15508 /* dynamic-no-pic */
15510 {
15511 rtx temp = ((reload_in_progress
15519 }
15521 {
15525 }
15528 else
15529 {
15537 }
15538 /* dynamic-no-pic */
15539 #endif
15540 }
15541 else
15542 {
15546 {
15553 }
15554 }
15555 }
15556 else
15557 {
15568 /* Force large constants in 64bit compilation into register
15569 to get them CSEed. */
15581 {
15582 /* If we are loading a floating point constant to a register,
15583 force the value to memory now, since we'll get better code
15584 out the back end. */
15588 {
15593 }
15594 }
15595 }
15598 }
15600 void
15602 {
15606 /* Force constants other than zero into memory. We do not know how
15607 the instructions used to build constants modify the upper 64 bits
15608 of the register, once we have that information we may be able
15609 to handle some of them more efficiently. */
15618 /* We need to check memory alignment for SSE mode since attribute
15619 can make operands unaligned. */
15624 {
15627 /* ix86_expand_vector_move_misalign() does not like constants ... */
15633 /* ... nor both arguments in memory. */
15641 }
15643 /* Make operand1 a register if it isn't already. */
15647 {
15650 }
15653 }
15655 /* Split 32-byte AVX unaligned load and store if needed. */
15657 static void
15659 {
15660 rtx m;
15666 {
15684 }
15687 {
15694 }
15696 {
15701 }
15702 else
15704 }
15706 /* Implement the movmisalign patterns for SSE. Non-SSE modes go
15707 straight to ix86_expand_vector_move. */
15708 /* Code generation for scalar reg-reg moves of single and double precision data:
15709 if (x86_sse_partial_reg_dependency == true | x86_sse_split_regs == true)
15710 movaps reg, reg
15711 else
15712 movss reg, reg
15713 if (x86_sse_partial_reg_dependency == true)
15714 movapd reg, reg
15715 else
15716 movsd reg, reg
15718 Code generation for scalar loads of double precision data:
15719 if (x86_sse_split_regs == true)
15720 movlpd mem, reg (gas syntax)
15721 else
15722 movsd mem, reg
15724 Code generation for unaligned packed loads of single precision data
15725 (x86_sse_unaligned_move_optimal overrides x86_sse_partial_reg_dependency):
15726 if (x86_sse_unaligned_move_optimal)
15727 movups mem, reg
15729 if (x86_sse_partial_reg_dependency == true)
15730 {
15731 xorps reg, reg
15732 movlps mem, reg
15733 movhps mem+8, reg
15734 }
15735 else
15736 {
15737 movlps mem, reg
15738 movhps mem+8, reg
15739 }
15741 Code generation for unaligned packed loads of double precision data
15742 (x86_sse_unaligned_move_optimal overrides x86_sse_split_regs):
15743 if (x86_sse_unaligned_move_optimal)
15744 movupd mem, reg
15746 if (x86_sse_split_regs == true)
15747 {
15748 movlpd mem, reg
15749 movhpd mem+8, reg
15750 }
15751 else
15752 {
15753 movsd mem, reg
15754 movhpd mem+8, reg
15755 }
15756 */
15758 void
15760 {
15767 {
15769 {
15773 {
15775 /* If we're optimizing for size, movups is the smallest. */
15777 {
15782 }
15794 }
15801 {
15810 {
15815 }
15823 }
15828 }
15831 }
15834 {
15835 /* If we're optimizing for size, movups is the smallest. */
15838 {
15843 }
15845 /* ??? If we have typed data, then it would appear that using
15846 movdqu is the only way to get unaligned data loaded with
15847 integer type. */
15849 {
15854 }
15857 {
15858 rtx zero;
15861 {
15866 }
15868 /* When SSE registers are split into halves, we can avoid
15869 writing to the top half twice. */
15871 {
15874 }
15875 else
15876 {
15877 /* ??? Not sure about the best option for the Intel chips.
15878 The following would seem to satisfy; the register is
15879 entirely cleared, breaking the dependency chain. We
15880 then store to the upper half, with a dependency depth
15881 of one. A rumor has it that Intel recommends two movsd
15882 followed by an unpacklpd, but this is unconfirmed. And
15883 given that the dependency depth of the unpacklpd would
15884 still be one, I'm not sure why this would be better. */
15886 }
15892 }
15893 else
15894 {
15896 {
15901 }
15905 else
15914 }
15915 }
15917 {
15918 /* If we're optimizing for size, movups is the smallest. */
15921 {
15926 }
15928 /* ??? Similar to above, only less clear because of quote
15929 typeless stores unquote. */
15932 {
15937 }
15940 {
15942 {
15946 }
15947 else
15948 {
15953 }
15954 }
15955 else
15956 {
15961 {
15964 }
15965 else
15966 {
15971 }
15972 }
15973 }
15974 else
15976 }
15978 /* Expand a push in MODE. This is some mode for which we do not support
15979 proper push instructions, at least from the registers that we expect
15980 the value to live in. */
15982 void
15984 {
15985 rtx tmp;
15995 /* When we push an operand onto stack, it has to be aligned at least
15996 at the function argument boundary. However since we don't have
15997 the argument type, we can't determine the actual argument
15998 boundary. */
16000 }
16002 /* Helper function of ix86_fixup_binary_operands to canonicalize
16003 operand order. Returns true if the operands should be swapped. */
16005 static bool
16007 rtx operands[])
16008 {
16013 /* If the operation is not commutative, we can't do anything. */
16017 /* Highest priority is that src1 should match dst. */
16023 /* Next highest priority is that immediate constants come second. */
16029 /* Lowest priority is that memory references should come second. */
16036 }
16039 /* Fix up OPERANDS to satisfy ix86_binary_operator_ok. Return the
16040 destination to use for the operation. If different from the true
16041 destination in operands[0], a copy operation will be required. */
16043 rtx
16045 rtx operands[])
16046 {
16051 /* Canonicalize operand order. */
16053 {
16054 rtx temp;
16056 /* It is invalid to swap operands of different modes. */
16062 }
16064 /* Both source operands cannot be in memory. */
16066 {
16067 /* Optimization: Only read from memory once. */
16069 {
16072 }
16073 else
16075 }
16077 /* If the destination is memory, and we do not have matching source
16078 operands, do things in registers. */
16082 /* Source 1 cannot be a constant. */
16086 /* Source 1 cannot be a non-matching memory. */
16090 /* Improve address combine. */
16099 }
16101 /* Similarly, but assume that the destination has already been
16102 set up properly. */
16104 void
16107 {
16110 }
16112 /* Attempt to expand a binary operator. Make the expansion closer to the
16113 actual machine, then just general_operand, which will allow 3 separate
16114 memory references (one output, two input) in a single insn. */
16116 void
16118 rtx operands[])
16119 {
16126 /* Emit the instruction. */
16130 {
16131 /* Reload doesn't know about the flags register, and doesn't know that
16132 it doesn't want to clobber it. We can only do this with PLUS. */
16135 }
16139 {
16140 /* This is going to be an LEA; avoid splitting it later. */
16142 }
16143 else
16144 {
16147 }
16149 /* Fix up the destination if needed. */
16152 }
16154 /* Return TRUE or FALSE depending on whether the binary operator meets the
16155 appropriate constraints. */
16157 bool
16160 {
16165 /* Both source operands cannot be in memory. */
16169 /* Canonicalize operand order for commutative operators. */
16171 {
16175 }
16177 /* If the destination is memory, we must have a matching source operand. */
16181 /* Source 1 cannot be a constant. */
16185 /* Source 1 cannot be a non-matching memory. */
16187 /* Support "andhi/andsi/anddi" as a zero-extending move. */
16195 }
16197 /* Attempt to expand a unary operator. Make the expansion closer to the
16198 actual machine, then just general_operand, which will allow 2 separate
16199 memory references (one output, one input) in a single insn. */
16201 void
16203 rtx operands[])
16204 {
16211 /* If the destination is memory, and we do not have matching source
16212 operands, do things in registers. */
16215 {
16218 else
16220 }
16222 /* When source operand is memory, destination must match. */
16226 /* Emit the instruction. */
16230 {
16231 /* Reload doesn't know about the flags register, and doesn't know that
16232 it doesn't want to clobber it. */
16235 }
16236 else
16237 {
16240 }
16242 /* Fix up the destination if needed. */
16245 }
16247 /* Split 32bit/64bit divmod with 8bit unsigned divmod if dividend and
16248 divisor are within the range [0-255]. */
16250 void
16253 {
16262 {
16275 }
16282 /* Use 8bit unsigned divimod if dividend and divisor are within
16283 the range [0-255]. */
16292 pc_rtx);
16297 /* Generate original signed/unsigned divimod. */
16302 /* Branch to the end. */
16306 /* Generate 8bit unsigned divide. */
16308 /* Don't use operands[0] for result of 8bit divide since not all
16309 registers support QImode ZERO_EXTRACT. */
16316 {
16319 }
16320 else
16321 {
16324 }
16326 /* Extract remainder from AH. */
16330 else
16331 {
16332 /* Need a new scratch register since the old one has result
16333 of 8bit divide. */
16337 }
16340 /* Zero extend quotient from AL. */
16346 }
16348 #define LEA_MAX_STALL (3)
16349 #define LEA_SEARCH_THRESHOLD (LEA_MAX_STALL << 1)
16351 /* Increase given DISTANCE in half-cycles according to
16352 dependencies between PREV and NEXT instructions.
16353 Add 1 half-cycle if there is no dependency and
16354 go to next cycle if there is some dependecy. */
16356 static unsigned int
16358 {
16375 }
16377 /* Function checks if instruction INSN defines register number
16378 REGNO1 or REGNO2. */
16380 static bool
16382 rtx insn)
16383 {
16391 {
16393 }
16396 }
16398 /* Function checks if instruction INSN uses register number
16399 REGNO as a part of address expression. */
16401 static bool
16403 {
16411 }
16413 /* Search backward for non-agu definition of register number REGNO1
16414 or register number REGNO2 in basic block starting from instruction
16415 START up to head of basic block or instruction INSN.
16417 Function puts true value into *FOUND var if definition was found
16418 and false otherwise.
16420 Distance in half-cycles between START and found instruction or head
16421 of BB is added to DISTANCE and returned. */
16423 static int
16427 {
16437 {
16439 {
16442 {
16445 {
16448 }
16449 }
16452 }
16457 }
16460 }
16462 /* Search backward for non-agu definition of register number REGNO1
16463 or register number REGNO2 in INSN's basic block until
16464 1. Pass LEA_SEARCH_THRESHOLD instructions, or
16465 2. Reach neighbour BBs boundary, or
16466 3. Reach agu definition.
16467 Returns the distance between the non-agu definition point and INSN.
16468 If no definition point, returns -1. */
16470 static int
16472 rtx insn)
16473 {
16484 {
16485 edge e;
16486 edge_iterator ei;
16491 {
16494 }
16500 else
16501 {
16506 {
16507 int bb_dist
16513 {
16520 }
16521 }
16524 }
16525 }
16527 /* get_attr_type may modify recog data. We want to make sure
16528 that recog data is valid for instruction INSN, on which
16529 distance_non_agu_define is called. INSN is unchanged here. */
16536 }
16538 /* Return the distance in half-cycles between INSN and the next
16539 insn that uses register number REGNO in memory address added
16540 to DISTANCE. Return -1 if REGNO0 is set.
16542 Put true value into *FOUND if register usage was found and
16543 false otherwise.
16544 Put true value into *REDEFINED if register redefinition was
16545 found and false otherwise. */
16547 static int
16551 {
16562 {
16564 {
16567 {
16568 /* Return DISTANCE if OP0 is used in memory
16569 address in NEXT. */
16572 }
16575 {
16576 /* Return -1 if OP0 is set in NEXT. */
16579 }
16582 }
16588 }
16591 }
16593 /* Return the distance between INSN and the next insn that uses
16594 register number REGNO0 in memory address. Return -1 if no such
16595 a use is found within LEA_SEARCH_THRESHOLD or REGNO0 is set. */
16597 static int
16599 {
16611 {
16612 edge e;
16613 edge_iterator ei;
16618 {
16621 }
16627 else
16628 {
16634 {
16635 int bb_dist
16640 {
16647 }
16648 }
16651 }
16652 }
16658 }
16660 /* Define this macro to tune LEA priority vs ADD, it take effect when
16661 there is a dilemma of choicing LEA or ADD
16662 Negative value: ADD is more preferred than LEA
16663 Zero: Netrual
16664 Positive value: LEA is more preferred than ADD*/
16665 #define IX86_LEA_PRIORITY 0
16667 /* Return true if usage of lea INSN has performance advantage
16668 over a sequence of instructions. Instructions sequence has
16669 SPLIT_COST cycles higher latency than lea latency. */
16671 bool
16674 {
16681 {
16682 /* If there is no non AGU operand definition, no AGU
16683 operand usage and split cost is 0 then both lea
16684 and non lea variants have same priority. Currently
16685 we prefer lea for 64 bit code and non lea on 32 bit
16686 code. */
16689 else
16691 }
16693 /* With longer definitions distance lea is more preferable.
16694 Here we change it to take into account splitting cost and
16695 lea priority. */
16698 /* If there is no use in memory addess then we just check
16699 that split cost does not exceed AGU stall. */
16703 /* If this insn has both backward non-agu dependence and forward
16704 agu dependence, the one with short distance takes effect. */
16706 }
16708 /* Return true if it is legal to clobber flags by INSN and
16709 false otherwise. */
16711 static bool
16713 {
16716 bitmap live;
16719 {
16721 {
16728 }
16734 }
16738 }
16740 /* Return true if we need to split op0 = op1 + op2 into a sequence of
16741 move and add to avoid AGU stalls. */
16743 bool
16745 {
16750 /* Check if we need to optimize. */
16754 /* Check it is correct to split here. */
16758 /* We need to split only adds with non destructive
16759 destination operand. */
16762 else
16764 }
16766 /* Return true if we should emit lea instruction instead of mov
16767 instruction. */
16769 bool
16771 {
16775 /* Check if we need to optimize. */
16779 /* Use lea for reg to reg moves only. */
16787 }
16789 /* Return true if we need to split lea into a sequence of
16790 instructions to avoid AGU stalls. */
16792 bool
16794 {
16802 /* Check we need to optimize. */
16806 /* Check it is correct to split here. */
16813 /* We should not split into add if non legitimate pic
16814 operand is used as displacement. */
16823 /* Compute how many cycles we will add to execution time
16824 if split lea into a sequence of instructions. */
16826 {
16827 /* Have to use mov instruction if non desctructive
16828 destination form is used. */
16832 /* Have to add index to base if both exist. */
16836 /* Have to use shift and adds if scale is 2 or greater. */
16838 {
16843 else
16845 }
16847 /* Have to use add instruction with immediate if
16848 disp is non zero. */
16852 /* Subtract the price of lea. */
16854 }
16857 }
16859 /* Emit x86 binary operand CODE in mode MODE, where the first operand
16860 matches destination. RTX includes clobber of FLAGS_REG. */
16862 static void
16865 {
16872 }
16874 /* Split lea instructions into a sequence of instructions
16875 which are executed on ALU to avoid AGU stalls.
16876 It is assumed that it is allowed to clobber flags register
16877 at lea position. */
16881 {
16886 rtx tmp;
16893 {
16897 }
16900 {
16904 }
16907 {
16908 /* Case r1 = r1 + ... */
16910 {
16911 /* If we have a case r1 = r1 + C * r1 then we
16912 should use multiplication which is very
16913 expensive. Assume cost model is wrong if we
16914 have such case here. */
16919 }
16920 else
16921 {
16922 /* r1 = r2 + r3 * C case. Need to move r3 into r1. */
16926 /* Use shift for scaling. */
16935 }
16936 }
16938 {
16941 }
16942 else
16943 {
16945 {
16948 }
16950 {
16953 }
16954 else
16955 {
16960 else
16961 {
16964 }
16967 }
16971 }
16972 }
16974 /* Return true if it is ok to optimize an ADD operation to LEA
16975 operation to avoid flag register consumation. For most processors,
16976 ADD is faster than LEA. For the processors like ATOM, if the
16977 destination register of LEA holds an actual address which will be
16978 used soon, LEA is better and otherwise ADD is better. */
16980 bool
16982 {
16987 /* If a = b + c, (a!=b && a!=c), must use lea form. */
16995 }
16997 /* Return true if destination reg of SET_BODY is shift count of
16998 USE_BODY. */
17000 static bool
17002 {
17003 rtx set_dest;
17004 rtx shift_rtx;
17007 /* Retrieve destination of SET_BODY. */
17009 {
17018 use_body))
17023 }
17025 /* Retrieve shift count of USE_BODY. */
17027 {
17039 }
17047 {
17050 /* Return true if shift count is dest of SET_BODY. */
17054 }
17057 }
17059 /* Return true if destination reg of SET_INSN is shift count of
17060 USE_INSN. */
17062 bool
17064 {
17067 }
17069 /* Return TRUE or FALSE depending on whether the unary operator meets the
17070 appropriate constraints. */
17072 bool
17076 {
17077 /* If one of operands is memory, source and destination must match. */
17083 }
17085 /* Return TRUE if the operands to a vec_interleave_{high,low}v2df
17086 are ok, keeping in mind the possible movddup alternative. */
17088 bool
17090 {
17096 }
17098 /* Post-reload splitter for converting an SF or DFmode value in an
17099 SSE register into an unsigned SImode. */
17101 void
17103 {
17114 /* Load up the value into the low element. We must ensure that the other
17115 elements are valid floats -- zero is the easiest such value. */
17117 {
17120 else
17122 }
17123 else
17124 {
17129 else
17131 }
17151 else
17156 }
17158 /* Convert an unsigned DImode value into a DFmode, using only SSE.
17159 Expects the 64-bit DImode to be supplied in a pair of integral
17160 registers. Requires SSE2; will use SSE3 if available. For x86_32,
17161 -mfpmath=sse, !optimize_size only. */
17163 void
17165 {
17169 rtx x;
17175 {
17178 }
17179 else
17180 {
17184 }
17192 /* int_xmm = {0x45300000UL, fp_xmm/hi, 0x43300000, fp_xmm/lo } */
17195 /* Concatenating (juxtaposing) (0x43300000UL ## fp_value_low_xmm)
17196 yields a valid DF value equal to (0x1.0p52 + double(fp_value_lo_xmm)).
17197 Similarly (0x45300000UL ## fp_value_hi_xmm) yields
17198 (0x1.0p84 + double(fp_value_hi_xmm)).
17199 Note these exponents differ by 32. */
17203 /* Subtract off those 0x1.0p52 and 0x1.0p84 biases, to produce values
17204 in [0,2**32-1] and [0]+[2**32,2**64-1] respectively. */
17213 /* Add the upper and lower DFmode values together. */
17216 else
17217 {
17221 }
17224 }
17226 /* Not used, but eases macroization of patterns. */
17227 void
17229 rtx input ATTRIBUTE_UNUSED)
17230 {
17232 }
17234 /* Convert an unsigned SImode value into a DFmode. Only currently used
17235 for SSE, but applicable anywhere. */
17237 void
17239 {
17240 REAL_VALUE_TYPE TWO31r;
17255 }
17257 /* Convert a signed DImode value into a DFmode. Only used for SSE in
17258 32-bit mode; otherwise we have a direct convert instruction. */
17260 void
17262 {
17263 REAL_VALUE_TYPE TWO32r;
17281 }
17283 /* Convert an unsigned SImode value into a SFmode, using only SSE.
17284 For x86_32, -mfpmath=sse, !optimize_size only. */
17285 void
17287 {
17288 REAL_VALUE_TYPE ONE16r;
17307 }
17309 /* floatunsv{4,8}siv{4,8}sf2 expander. Expand code to convert
17310 a vector of unsigned ints VAL to vector of floats TARGET. */
17312 void
17314 {
17316 REAL_VALUE_TYPE TWO16r;
17323 else
17328 OPTAB_DIRECT);
17339 OPTAB_DIRECT);
17341 OPTAB_DIRECT);
17344 }
17346 /* Adjust a V*SFmode/V*DFmode value VAL so that *sfix_trunc* resp. fix_trunc*
17347 pattern can be used on it instead of *ufix_trunc* resp. fixuns_trunc*.
17348 This is done by doing just signed conversion if < 0x1p31, and otherwise by
17349 subtracting 0x1p31 first and xoring in 0x80000000 from *XORP afterwards. */
17351 rtx
17353 {
17354 REAL_VALUE_TYPE TWO31r;
17369 {
17375 }
17384 OPTAB_DIRECT);
17385 else
17386 {
17392 OPTAB_DIRECT);
17393 }
17396 }
17398 /* A subroutine of ix86_build_signbit_mask. If VECT is true,
17399 then replicate the value for all elements of the vector
17400 register. */
17402 rtx
17404 {
17406 rtvec v;
17410 {
17437 }
17438 }
17440 /* A subroutine of ix86_expand_fp_absneg_operator, copysign expanders
17441 and ix86_expand_int_vcond. Create a mask for the sign bit in MODE
17442 for an SSE register. If VECT is true, then replicate the mask for
17443 all elements of the vector register. If INVERT is true, then create
17444 a mask excluding the sign bit. */
17446 rtx
17448 {
17452 rtx v;
17453 rtx mask;
17455 /* Find the sign bit, sign extended to 2*HWI. */
17457 {
17477 else
17485 {
17488 }
17489 else
17490 {
17491 rtvec vec;
17497 {
17500 }
17501 else
17511 }
17516 }
17521 /* Force this value into the low part of a fp vector constant. */
17530 }
17532 /* Generate code for floating point ABS or NEG. */
17534 void
17536 rtx operands[])
17537 {
17548 {
17554 }
17556 /* NEG and ABS performed with SSE use bitwise mask operations.
17557 Create the appropriate mask now. */
17560 else
17570 {
17572 rtvec par;
17577 else
17578 {
17581 }
17583 }
17584 else
17586 }
17588 /* Expand a copysign operation. Special case operand 0 being a constant. */
17590 void
17592 {
17606 else
17610 {
17617 {
17620 else
17621 {
17625 }
17626 }
17636 else
17640 }
17641 else
17642 {
17652 else
17656 }
17657 }
17659 /* Deconstruct a copysign operation into bit masks. Operand 0 is known to
17660 be a constant, and so has already been expanded into a vector constant. */
17662 void
17664 {
17680 {
17683 }
17684 }
17686 /* Deconstruct a copysign operation into bit masks. Operand 0 is variable,
17687 so we have to do two masks. */
17689 void
17691 {
17706 {
17707 /* Shouldn't happen often (it's useless, obviously), but when it does
17708 we'd generate incorrect code if we continue below. */
17711 }
17714 {
17725 }
17726 else
17727 {
17729 {
17731 }
17733 {
17737 }
17741 {
17744 }
17746 {
17751 }
17753 }
17757 }
17759 /* Return TRUE or FALSE depending on whether the first SET in INSN
17760 has source and destination with matching CC modes, and that the
17761 CC mode is at least as constrained as REQ_MODE. */
17763 bool
17765 {
17766 rtx set;
17777 {
17787 /* FALLTHRU */
17791 /* FALLTHRU */
17795 /* FALLTHRU */
17809 }
17812 }
17814 /* Generate insn patterns to do an integer compare of OPERANDS. */
17816 static rtx
17818 {
17825 /* This is very simple, but making the interface the same as in the
17826 FP case makes the rest of the code easier. */
17830 /* Return the test that should be put into the flags user, i.e.
17831 the bcc, scc, or cmov instruction. */
17833 }
17835 /* Figure out whether to use ordered or unordered fp comparisons.
17836 Return the appropriate mode to use. */
17838 enum machine_mode
17840 {
17841 /* ??? In order to make all comparisons reversible, we do all comparisons
17842 non-trapping when compiling for IEEE. Once gcc is able to distinguish
17843 all forms trapping and nontrapping comparisons, we can make inequality
17844 comparisons trapping again, since it results in better code when using
17845 FCOM based compares. */
17847 }
17849 enum machine_mode
17851 {
17855 {
17858 }
17861 {
17862 /* Only zero flag is needed. */
17866 /* Codes needing carry flag. */
17869 /* Detect overflow checks. They need just the carry flag. */
17873 else
17877 /* Detect overflow checks. They need just the carry flag. */
17881 else
17883 /* Codes possibly doable only with sign flag when
17884 comparing against zero. */
17889 else
17890 /* For other cases Carry flag is not required. */
17892 /* Codes doable only with sign flag when comparing
17893 against zero, but we miss jump instruction for it
17894 so we need to use relational tests against overflow
17895 that thus needs to be zero. */
17900 else
17902 /* strcmp pattern do (use flags) and combine may ask us for proper
17903 mode. */
17908 }
17909 }
17911 /* Return the fixed registers used for condition codes. */
17913 static bool
17915 {
17919 }
17921 /* If two condition code modes are compatible, return a condition code
17922 mode which is compatible with both. Otherwise, return
17923 VOIDmode. */
17925 static enum machine_mode
17927 {
17944 {
17958 {
17972 }
17976 /* These are only compatible with themselves, which we already
17977 checked above. */
17979 }
17980 }
17983 /* Return a comparison we can do and that it is equivalent to
17984 swap_condition (code) apart possibly from orderedness.
17985 But, never change orderedness if TARGET_IEEE_FP, returning
17986 UNKNOWN in that case if necessary. */
17988 static enum rtx_code
17990 {
17992 {
18003 }
18004 }
18006 /* Return cost of comparison CODE using the best strategy for performance.
18007 All following functions do use number of instructions as a cost metrics.
18008 In future this should be tweaked to compute bytes for optimize_size and
18009 take into account performance of various instructions on various CPUs. */
18011 static int
18013 {
18016 /* The cost of code using bit-twiddling on %ah. */
18018 {
18041 }
18044 {
18051 }
18052 }
18054 /* Return strategy to use for floating-point. We assume that fcomi is always
18055 preferrable where available, since that is also true when looking at size
18056 (2 bytes, vs. 3 for fnstsw+sahf and at least 5 for fnstsw+test). */
18058 enum ix86_fpcmp_strategy
18060 {
18061 /* Do fcomi/sahf based test when profitable. */
18070 }
18072 /* Swap, force into registers, or otherwise massage the two operands
18073 to a fp comparison. The operands are updated in place; the new
18074 comparison code is returned. */
18076 static enum rtx_code
18078 {
18084 /* All of the unordered compare instructions only work on registers.
18085 The same is true of the fcomi compare instructions. The XFmode
18086 compare instructions require registers except when comparing
18087 against zero or when converting operand 1 from fixed point to
18088 floating point. */
18097 {
18100 }
18101 else
18102 {
18103 /* %%% We only allow op1 in memory; op0 must be st(0). So swap
18104 things around if they appear profitable, otherwise force op0
18105 into a register. */
18111 {
18114 {
18115 rtx tmp;
18118 }
18119 }
18125 {
18130 {
18133 }
18134 else
18136 }
18137 }
18139 /* Try to rearrange the comparison to make it cheaper. */
18143 {
18144 rtx tmp;
18149 }
18154 }
18156 /* Convert comparison codes we use to represent FP comparison to integer
18157 code that will result in proper branch. Return UNKNOWN if no such code
18158 is available. */
18160 enum rtx_code
18162 {
18164 {
18187 }
18188 }
18190 /* Generate insn patterns to do a floating point compare of OPERANDS. */
18192 static rtx
18194 {
18201 /* Do fcomi/sahf based test when profitable. */
18203 {
18208 tmp);
18216 tmp);
18225 /* Sadness wrt reg-stack pops killing fpsr -- gotta get fnstsw first. */
18232 /* In the unordered case, we have to check C2 for NaN's, which
18233 doesn't happen to work out to anything nice combination-wise.
18234 So do some bit twiddling on the value we've got in AH to come
18235 up with an appropriate set of condition codes. */
18239 {
18243 {
18246 }
18247 else
18248 {
18254 }
18259 {
18264 }
18265 else
18266 {
18269 }
18274 {
18277 }
18278 else
18279 {
18283 }
18288 {
18294 }
18295 else
18296 {
18299 }
18304 {
18309 }
18310 else
18311 {
18314 }
18319 {
18324 }
18325 else
18326 {
18329 }
18343 }
18348 }
18350 /* Return the test that should be put into the flags user, i.e.
18351 the bcc, scc, or cmov instruction. */
18354 const0_rtx);
18355 }
18357 static rtx
18359 {
18360 rtx ret;
18366 {
18369 }
18370 else
18374 }
18376 void
18378 {
18380 rtx tmp;
18383 {
18390 simple:
18394 pc_rtx);
18402 /* Expand DImode branch into multiple compare+branch. */
18403 {
18409 {
18412 }
18419 /* When comparing for equality, we can use (hi0^hi1)|(lo0^lo1) to
18420 avoid two branches. This costs one extra insn, so disable when
18421 optimizing for size. */
18426 {
18444 }
18446 /* Otherwise, if we are doing less-than or greater-or-equal-than,
18447 op1 is a constant and the low word is zero, then we can just
18448 examine the high word. Similarly for low word -1 and
18449 less-or-equal-than or greater-than. */
18453 {
18456 {
18459 }
18463 {
18466 }
18470 }
18472 /* Otherwise, we need two or three jumps. */
18481 {
18495 }
18497 /*
18498 * a < b =>
18499 * if (hi(a) < hi(b)) goto true;
18500 * if (hi(a) > hi(b)) goto false;
18501 * if (lo(a) < lo(b)) goto true;
18502 * false:
18503 */
18515 }
18520 }
18521 }
18523 /* Split branch based on floating point condition. */
18524 void
18527 {
18528 rtx condition;
18529 rtx i;
18532 {
18537 }
18540 tmp);
18542 /* Remove pushed operand from stack. */
18552 }
18554 void
18556 {
18557 rtx ret;
18564 }
18566 /* Expand comparison setting or clearing carry flag. Return true when
18567 successful and set pop for the operation. */
18568 static bool
18570 {
18574 /* Do not handle double-mode compares that go through special path. */
18579 {
18584 /* Shortcut: following common codes never translate
18585 into carry flag compares. */
18590 /* These comparisons require zero flag; swap operands so they won't. */
18593 {
18598 }
18600 /* Try to expand the comparison and verify that we end up with
18601 carry flag based comparison. This fails to be true only when
18602 we decide to expand comparison using arithmetic that is not
18603 too common scenario. */
18612 else
18621 }
18627 {
18632 /* Convert a==0 into (unsigned)a<1. */
18641 /* Convert a>b into b<a or a>=b-1. */
18645 {
18647 /* Bail out on overflow. We still can swap operands but that
18648 would force loading of the constant into register. */
18653 }
18654 else
18655 {
18660 }
18663 /* Convert a>=0 into (unsigned)a<0x80000000. */
18681 }
18682 /* Swapping operands may cause constant to appear as first operand. */
18684 {
18688 }
18692 }
18694 bool
18696 {
18715 /* Don't attempt mode expansion here -- if we had to expand 5 or 6
18716 HImode insns, we'd be swallowed in word prefix ops. */
18722 {
18726 HOST_WIDE_INT diff;
18729 /* Sign bit compares are better done using shifts than we do by using
18730 sbb. */
18733 {
18734 /* Detect overlap between destination and compare sources. */
18738 {
18739 rtx flags;
18748 {
18750 compare_code
18752 }
18754 /* To simplify rest of code, restrict to the GEU case. */
18756 {
18762 }
18763 else
18764 {
18767 reverse_condition_maybe_unordered
18769 else
18772 }
18781 else
18784 }
18785 else
18786 {
18789 else
18790 {
18795 }
18797 }
18800 {
18801 /*
18802 * cmpl op0,op1
18803 * sbbl dest,dest
18804 * [addl dest, ct]
18805 *
18806 * Size 5 - 8.
18807 */
18812 }
18814 {
18815 /*
18816 * cmpl op0,op1
18817 * sbbl dest,dest
18818 * orl $ct, dest
18819 *
18820 * Size 8.
18821 */
18825 }
18827 {
18828 /*
18829 * cmpl op0,op1
18830 * sbbl dest,dest
18831 * notl dest
18832 * [addl dest, cf]
18833 *
18834 * Size 8 - 11.
18835 */
18841 }
18842 else
18843 {
18844 /*
18845 * cmpl op0,op1
18846 * sbbl dest,dest
18847 * [notl dest]
18848 * andl cf - ct, dest
18849 * [addl dest, ct]
18850 *
18851 * Size 8 - 11.
18852 */
18855 {
18859 }
18869 }
18875 }
18878 {
18881 HOST_WIDE_INT tmp;
18886 {
18889 /* We may be reversing unordered compare to normal compare, that
18890 is not valid in general (we may convert non-trapping condition
18891 to trapping one), however on i386 we currently emit all
18892 comparisons unordered. */
18895 }
18896 else
18897 {
18900 }
18901 }
18906 {
18911 {
18916 }
18917 }
18919 /* Optimize dest = (op0 < 0) ? -1 : cf. */
18923 {
18924 /* If lea code below could be used, only optimize
18925 if it results in a 2 insn sequence. */
18931 {
18932 /*
18933 * notl op1 (if necessary)
18934 * sarl $31, op1
18935 * orl cf, op1
18936 */
18938 {
18942 }
18953 }
18954 }
18962 {
18963 /*
18964 * xorl dest,dest
18965 * cmpl op1,op2
18966 * setcc dest
18967 * lea cf(dest*(ct-cf)),dest
18968 *
18969 * Size 14.
18970 *
18971 * This also catches the degenerate setcc-only case.
18972 */
18974 rtx tmp;
18980 /* On x86_64 the lea instruction operates on Pmode, so we need
18981 to get arithmetics done in proper mode to match. */
18984 else
18985 {
18986 rtx out1;
18989 nops++;
18991 {
18993 nops++;
18994 }
18995 }
18997 {
18999 nops++;
19000 }
19002 {
19005 else
19007 }
19012 }
19014 /*
19015 * General case: Jumpful:
19016 * xorl dest,dest cmpl op1, op2
19017 * cmpl op1, op2 movl ct, dest
19018 * setcc dest jcc 1f
19019 * decl dest movl cf, dest
19020 * andl (cf-ct),dest 1:
19021 * addl ct,dest
19022 *
19023 * Size 20. Size 14.
19024 *
19025 * This is reasonably steep, but branch mispredict costs are
19026 * high on modern cpus, so consider failing only if optimizing
19027 * for space.
19028 */
19033 {
19035 {
19042 {
19045 /* We may be reversing unordered compare to normal compare,
19046 that is not valid in general (we may convert non-trapping
19047 condition to trapping one), however on i386 we currently
19048 emit all comparisons unordered. */
19050 }
19051 else
19052 {
19056 }
19057 }
19060 {
19061 /* notl op1 (if needed)
19062 sarl $31, op1
19063 andl (cf-ct), op1
19064 addl ct, op1
19066 For x < 0 (resp. x <= -1) there will be no notl,
19067 so if possible swap the constants to get rid of the
19068 complement.
19069 True/false will be -1/0 while code below (store flag
19070 followed by decrement) is 0/-1, so the constants need
19071 to be exchanged once more. */
19074 {
19077 }
19078 else
19079 {
19083 }
19086 }
19087 else
19088 {
19092 constm1_rtx,
19094 }
19106 }
19107 }
19110 {
19111 /* Try a few things more with specific constants and a variable. */
19113 optab op;
19119 /* If one of the two operands is an interesting constant, load a
19120 constant with the above and mask it in with a logical operation. */
19123 {
19129 else
19131 }
19133 {
19139 else
19141 }
19142 else
19149 /* Recurse to get the constant loaded. */
19153 /* Mask in the interesting variable. */
19155 OPTAB_WIDEN);
19160 }
19162 /*
19163 * For comparison with above,
19164 *
19165 * movl cf,dest
19166 * movl ct,tmp
19167 * cmpl op1,op2
19168 * cmovcc tmp,dest
19169 *
19170 * Size 15.
19171 */
19193 }
19195 /* Swap, force into registers, or otherwise massage the two operands
19196 to an sse comparison with a mask result. Thus we differ a bit from
19197 ix86_prepare_fp_compare_args which expects to produce a flags result.
19199 The DEST operand exists to help determine whether to commute commutative
19200 operators. The POP0/POP1 operands are updated in place. The new
19201 comparison code is returned, or UNKNOWN if not implementable. */
19203 static enum rtx_code
19206 {
19207 rtx tmp;
19210 {
19213 /* AVX supports all the needed comparisons. */
19216 /* We have no LTGT as an operator. We could implement it with
19217 NE & ORDERED, but this requires an extra temporary. It's
19218 not clear that it's worth it. */
19225 /* These are supported directly. */
19232 /* AVX has 3 operand comparisons, no need to swap anything. */
19235 /* For commutative operators, try to canonicalize the destination
19236 operand to be first in the comparison - this helps reload to
19237 avoid extra moves. */
19240 /* FALLTHRU */
19246 /* These are not supported directly before AVX, and furthermore
19247 ix86_expand_sse_fp_minmax only optimizes LT/UNGE. Swap the
19248 comparison operands to transform into something that is
19249 supported. */
19258 }
19261 }
19263 /* Detect conditional moves that exactly match min/max operational
19264 semantics. Note that this is IEEE safe, as long as we don't
19265 interchange the operands.
19267 Returns FALSE if this conditional move doesn't match a MIN/MAX,
19268 and TRUE if the operation is successful and instructions are emitted. */
19270 static bool
19273 {
19276 rtx tmp;
19279 ;
19281 {
19285 }
19286 else
19293 else
19298 /* We want to check HONOR_NANS and HONOR_SIGNED_ZEROS here,
19299 but MODE may be a vector mode and thus not appropriate. */
19301 {
19303 rtvec v;
19308 }
19309 else
19310 {
19313 }
19317 }
19319 /* Expand an sse vector comparison. Return the register with the result. */
19321 static rtx
19324 {
19327 rtx x;
19340 {
19343 }
19344 else
19348 }
19350 /* Expand DEST = CMP ? OP_TRUE : OP_FALSE into a sequence of logical
19351 operations. This is used for both scalar and vector conditional moves. */
19353 static void
19355 {
19361 {
19363 }
19365 {
19369 }
19371 {
19376 }
19378 {
19382 }
19384 {
19392 op_true,
19393 op_false)));
19394 }
19395 else
19396 {
19405 {
19419 {
19425 }
19440 {
19446 }
19450 }
19454 else
19455 {
19461 else
19473 }
19474 }
19475 }
19477 /* Expand a floating-point conditional move. Return true if successful. */
19479 bool
19481 {
19489 {
19492 /* Since we've no cmove for sse registers, don't force bad register
19493 allocation just to gain access to it. Deny movcc when the
19494 comparison mode doesn't match the move mode. */
19513 }
19515 /* The floating point conditional move instructions don't directly
19516 support conditions resulting from a signed integer comparison. */
19520 {
19525 }
19532 }
19534 /* Expand a floating-point vector conditional move; a vcond operation
19535 rather than a movcc operation. */
19537 bool
19539 {
19541 rtx cmp;
19546 {
19547 rtx temp;
19549 {
19566 }
19568 OPTAB_DIRECT);
19571 }
19581 }
19583 /* Expand a signed/unsigned integral vector conditional move. */
19585 bool
19587 {
19597 /* Try to optimize x < 0 ? -1 : 0 into (signed) x >> 31
19598 and x < 0 ? 1 : 0 into (unsigned) x >> 31. */
19607 {
19611 {
19617 }
19620 {
19626 }
19627 }
19636 /* XOP supports all of the comparisons on all 128-bit vector int types. */
19640 ;
19641 else
19642 {
19643 /* Canonicalize the comparison to EQ, GT, GTU. */
19645 {
19662 /* FALLTHRU */
19672 }
19674 /* Only SSE4.1/SSE4.2 supports V2DImode. */
19676 {
19678 {
19680 /* SSE4.1 supports EQ. */
19687 /* SSE4.2 supports GT/GTU. */
19694 }
19695 }
19697 /* Unsigned parallel compare is not supported by the hardware.
19698 Play some tricks to turn this into a signed comparison
19699 against 0. */
19701 {
19705 {
19710 {
19715 {
19722 }
19723 /* Subtract (-(INT MAX) - 1) from both operands to make
19724 them signed. */
19735 }
19742 /* Perform a parallel unsigned saturating subtraction. */
19755 }
19756 }
19757 }
19759 /* Allow the comparison to be done in one mode, but the movcc to
19760 happen in another mode. */
19762 {
19765 }
19766 else
19767 {
19773 }
19778 }
19780 /* Expand a variable vector permutation. */
19782 void
19784 {
19795 /* Number of elements in the vector. */
19801 {
19803 {
19804 /* Unfortunately, the VPERMQ and VPERMPD instructions only support
19805 an constant shuffle operand. With a tiny bit of effort we can
19806 use VPERMD instead. A re-interpretation stall for V4DFmode is
19807 unfortunate but there's no avoiding it.
19808 Similarly for V16HImode we don't have instructions for variable
19809 shuffling, while for V32QImode we can use after preparing suitable
19810 masks vpshufb; vpshufb; vpermq; vpor. */
19813 {
19817 }
19818 else
19819 {
19823 }
19826 /* Replicate the low bits of the V4DImode mask into V8SImode:
19827 mask = { A B C D }
19828 t1 = { A A B B C C D D }. */
19836 else
19839 /* Multiply the shuffle indicies by two. */
19841 OPTAB_DIRECT);
19843 /* Add one to the odd shuffle indicies:
19844 t1 = { A*2, A*2+1, B*2, B*2+1, ... }. */
19846 {
19849 }
19853 OPTAB_DIRECT);
19855 /* Continue as if V8SImode (resp. V32QImode) was used initially. */
19860 }
19863 {
19865 /* The VPERMD and VPERMPS instructions already properly ignore
19866 the high bits of the shuffle elements. No need for us to
19867 perform an AND ourselves. */
19870 else
19871 {
19877 }
19884 else
19885 {
19891 }
19895 /* By combining the two 128-bit input vectors into one 256-bit
19896 input vector, we can use VPERMD and VPERMPS for the full
19897 two-operand shuffle. */
19929 /* From mask create two adjusted masks, which contain the same
19930 bits as mask in the low 7 bits of each vector element.
19931 The first mask will have the most significant bit clear
19932 if it requests element from the same 128-bit lane
19933 and MSB set if it requests element from the other 128-bit lane.
19934 The second mask will have the opposite values of the MSB,
19935 and additionally will have its 128-bit lanes swapped.
19936 E.g. { 07 12 1e 09 ... | 17 19 05 1f ... } mask vector will have
19937 t1 { 07 92 9e 09 ... | 17 19 85 1f ... } and
19938 t3 { 97 99 05 9f ... | 87 12 1e 89 ... } where each ...
19939 stands for other 12 bytes. */
19940 /* The bit whether element is from the same lane or the other
19941 lane is bit 4, so shift it up by 3 to the MSB position. */
19945 /* Clear MSB bits from the mask just in case it had them set. */
19947 /* After this t1 will have MSB set for elements from other lane. */
19949 /* Clear bits other than MSB. */
19951 /* Or in the lower bits from mask into t3. */
19953 /* And invert MSB bits in t1, so MSB is set for elements from the same
19954 lane. */
19956 /* Swap 128-bit lanes in t3. */
19961 /* And or in the lower bits from mask into t1. */
19964 {
19965 /* Each of these shuffles will put 0s in places where
19966 element from the other 128-bit lane is needed, otherwise
19967 will shuffle in the requested value. */
19970 /* For t3 the 128-bit lanes are swapped again. */
19975 /* And oring both together leads to the result. */
19978 }
19981 /* Similarly to the above one_operand_shuffle code,
19982 just for repeated twice for each operand. merge_two:
19983 code will merge the two results together. */
20005 }
20006 }
20009 {
20010 /* The XOP VPPERM insn supports three inputs. By ignoring the
20011 one_operand_shuffle special case, we avoid creating another
20012 set of constant vectors in memory. */
20015 /* mask = mask & {2*w-1, ...} */
20017 }
20018 else
20019 {
20020 /* mask = mask & {w-1, ...} */
20022 }
20030 /* For non-QImode operations, convert the word permutation control
20031 into a byte permutation control. */
20033 {
20038 /* Convert mask to vector of chars. */
20041 /* Replicate each of the input bytes into byte positions:
20042 (v2di) --> {0,0,0,0,0,0,0,0, 8,8,8,8,8,8,8,8}
20043 (v4si) --> {0,0,0,0, 4,4,4,4, 8,8,8,8, 12,12,12,12}
20044 (v8hi) --> {0,0, 2,2, 4,4, 6,6, ...}. */
20051 else
20054 /* Convert it into the byte positions by doing
20055 mask = mask + {0,1,..,16/w, 0,1,..,16/w, ...} */
20061 }
20063 /* The actual shuffle operations all operate on V16QImode. */
20069 {
20071 }
20073 {
20075 }
20076 else
20077 {
20081 /* Shuffle the two input vectors independently. */
20087 merge_two:
20088 /* Then merge them together. The key is whether any given control
20089 element contained a bit set that indicates the second word. */
20093 {
20094 /* Without SSE4.1, we don't have V2DImode EQ. Perform one
20095 more shuffle to convert the V2DI input mask into a V4SI
20096 input mask. At which point the masking that expand_int_vcond
20097 will work as desired. */
20105 }
20122 }
20123 }
20125 /* Unpack OP[1] into the next wider integer vector type. UNSIGNED_P is
20126 true if we should do zero extension, else sign extension. HIGH_P is
20127 true if we want the N/2 high elements, else the low elements. */
20129 void
20131 {
20136 {
20142 {
20146 else
20149 extract
20155 else
20158 extract
20164 else
20167 extract
20173 else
20179 else
20185 else
20190 }
20193 {
20196 }
20198 {
20199 /* Shift higher 8 bytes to lower 8 bytes. */
20204 }
20205 else
20209 }
20210 else
20211 {
20215 {
20219 else
20225 else
20231 else
20236 }
20242 else
20247 }
20248 }
20250 /* Expand conditional increment or decrement using adb/sbb instructions.
20251 The default case using setcc followed by the conditional move can be
20252 done by generic code. */
20253 bool
20255 {
20257 rtx flags;
20259 rtx compare_op;
20277 {
20280 }
20283 {
20287 reverse_condition_maybe_unordered
20289 else
20291 }
20295 /* Construct either adc or sbb insn. */
20297 {
20299 {
20314 }
20315 }
20316 else
20317 {
20319 {
20334 }
20335 }
20339 }
20342 /* Split operands 0 and 1 into half-mode parts. Similar to split_double_mode,
20343 but works for floating pointer parameters and nonoffsetable memories.
20344 For pushes, it returns just stack offsets; the values will be saved
20345 in the right order. Maximally three parts are generated. */
20347 static int
20349 {
20354 else
20360 /* Optimize constant pool reference to immediates. This is used by fp
20361 moves, that force all constants to memory to allow combining. */
20363 {
20367 }
20370 {
20371 /* The only non-offsetable memories we handle are pushes. */
20380 }
20383 {
20385 /* Caution: if we looked through a constant pool memory above,
20386 the operand may actually have a different mode now. That's
20387 ok, since we want to pun this all the way back to an integer. */
20391 }
20394 {
20397 else
20398 {
20402 {
20406 }
20408 {
20413 }
20415 {
20416 REAL_VALUE_TYPE r;
20421 {
20436 }
20439 }
20440 else
20442 }
20443 }
20444 else
20445 {
20449 {
20452 {
20456 }
20458 {
20462 }
20464 {
20465 REAL_VALUE_TYPE r;
20471 /* Do not use shift by 32 to avoid warning on 32bit systems. */
20474 = gen_int_mode
20477 DImode);
20478 else
20485 = gen_int_mode
20488 DImode);
20489 else
20491 }
20492 else
20494 }
20495 }
20498 }
20500 /* Emit insns to perform a move or push of DI, DF, XF, and TF values.
20501 Return false when normal moves are needed; true when all required
20502 insns have been emitted. Operands 2-4 contain the input values
20503 int the correct order; operands 5-7 contain the output values. */
20505 void
20507 {
20515 /* The DFmode expanders may ask us to move double.
20516 For 64bit target this is single move. By hiding the fact
20517 here we simplify i386.md splitters. */
20519 {
20520 /* Optimize constant pool reference to immediates. This is used by
20521 fp moves, that force all constants to memory to allow combining. */
20528 {
20531 }
20532 else
20537 }
20539 /* The only non-offsettable memory we handle is push. */
20542 else
20549 /* When emitting push, take care for source operands on the stack. */
20552 {
20555 /* Compensate for the stack decrement by 4. */
20560 /* src_base refers to the stack pointer and is
20561 automatically decreased by emitted push. */
20565 }
20567 /* We need to do copy in the right order in case an address register
20568 of the source overlaps the destination. */
20570 {
20571 rtx tmp;
20574 {
20578 collisions++;
20579 }
20581 /* Collision in the middle part can be handled by reordering. */
20583 {
20586 }
20590 {
20592 {
20595 }
20596 else
20597 {
20600 }
20601 }
20603 /* If there are more collisions, we can't handle it by reordering.
20604 Do an lea to the last part and use only one colliding move. */
20606 {
20607 rtx base;
20613 /* Handle the case when the last part isn't valid for lea.
20614 Happens in 64-bit mode storing the 12-byte XFmode. */
20621 {
20624 }
20625 }
20626 }
20629 {
20631 {
20633 {
20638 }
20640 {
20643 }
20644 }
20645 else
20646 {
20647 /* In 64bit mode we don't have 32bit push available. In case this is
20648 register, it is OK - we will just use larger counterpart. We also
20649 retype memory - these comes from attempt to avoid REX prefix on
20650 moving of second half of TFmode value. */
20652 {
20654 {
20665 }
20669 }
20670 }
20674 }
20676 /* Choose correct order to not overwrite the source before it is copied. */
20686 {
20688 {
20691 }
20692 }
20693 else
20694 {
20696 {
20699 }
20700 }
20702 /* If optimizing for size, attempt to locally unCSE nonzero constants. */
20704 {
20713 }
20719 }
20721 /* Helper function of ix86_split_ashl used to generate an SImode/DImode
20722 left shift by a constant, either using a single shift or
20723 a sequence of add instructions. */
20725 static void
20727 {
20733 {
20737 }
20738 else
20739 {
20742 }
20743 }
20745 void
20747 {
20756 {
20761 {
20767 }
20768 else
20769 {
20777 }
20779 }
20786 {
20787 /* Assuming we've chosen a QImode capable registers, then 1 << N
20788 can be done with two 32/64-bit shifts, no branches, no cmoves. */
20790 {
20806 }
20808 /* Otherwise, we can get the same results by manually performing
20809 a bit extract operation on bit 5/6, and then performing the two
20810 shifts. The two methods of getting 0/1 into low/high are exactly
20811 the same size. Avoiding the shift in the bit extract case helps
20812 pentium4 a bit; no one else seems to care much either way. */
20813 else
20814 {
20819 HOST_WIDE_INT bits;
20820 rtx x;
20823 {
20829 }
20830 else
20831 {
20837 }
20841 else
20849 }
20854 }
20857 {
20858 /* For -1 << N, we can avoid the shld instruction, because we
20859 know that we're shifting 0...31/63 ones into a -1. */
20863 else
20865 }
20866 else
20867 {
20875 }
20880 {
20886 }
20887 else
20888 {
20893 }
20894 }
20896 void
20898 {
20908 {
20913 {
20919 }
20921 {
20930 }
20931 else
20932 {
20940 }
20941 }
20942 else
20943 {
20955 {
20963 scratch));
20964 }
20965 else
20966 {
20971 }
20972 }
20973 }
20975 void
20977 {
20987 {
20992 {
20999 }
21000 else
21001 {
21009 }
21010 }
21011 else
21012 {
21024 {
21030 scratch));
21031 }
21032 else
21033 {
21038 }
21039 }
21040 }
21042 /* Predict just emitted jump instruction to be taken with probability PROB. */
21043 static void
21045 {
21049 }
21051 /* Helper function for the string operations below. Dest VARIABLE whether
21052 it is aligned to VALUE bytes. If true, jump to the label. */
21053 static rtx
21055 {
21060 else
21066 else
21069 }
21071 /* Adjust COUNTER by the VALUE. */
21072 static void
21074 {
21079 }
21081 /* Zero extend possibly SImode EXP to Pmode register. */
21082 rtx
21084 {
21088 }
21090 /* Divide COUNTREG by SCALE. */
21091 static rtx
21093 {
21094 rtx sc;
21106 }
21108 /* Return mode for the memcpy/memset loop counter. Prefer SImode over
21109 DImode for constant loop counts. */
21111 static enum machine_mode
21113 {
21121 }
21123 /* When SRCPTR is non-NULL, output simple loop to move memory
21124 pointer to SRCPTR to DESTPTR via chunks of MODE unrolled UNROLL times,
21125 overall size is COUNT specified in bytes. When SRCPTR is NULL, output the
21126 equivalent loop to set memory by VALUE (supposed to be in MODE).
21128 The size is rounded down to whole number of chunk size moved at once.
21129 SRCMEM and DESTMEM provide MEMrtx to feed proper aliasing info. */
21132 static void
21137 {
21142 rtx size;
21143 rtx x_addr;
21144 rtx y_addr;
21153 /* Those two should combine. */
21155 {
21159 }
21169 {
21173 /* When unrolling for chips that reorder memory reads and writes,
21174 we can save registers by using single temporary.
21175 Also using 4 temporaries is overkill in 32bit mode. */
21177 {
21179 {
21181 {
21182 destmem =
21184 srcmem =
21186 }
21188 }
21189 }
21190 else
21191 {
21195 {
21198 {
21199 srcmem =
21201 }
21203 }
21205 {
21207 {
21208 destmem =
21210 }
21212 }
21213 }
21214 }
21215 else
21217 {
21219 destmem =
21222 }
21232 {
21238 else
21240 }
21241 else
21249 {
21254 }
21256 }
21258 /* Output "rep; mov" instruction.
21259 Arguments have same meaning as for previous function */
21260 static void
21263 rtx count,
21265 {
21266 rtx destexp;
21267 rtx srcexp;
21268 rtx countreg;
21269 HOST_WIDE_INT rounded_count;
21271 /* If the size is known, it is shorter to use rep movs. */
21282 {
21289 }
21290 else
21291 {
21294 }
21296 {
21303 }
21304 else
21305 {
21310 }
21313 }
21315 /* Output "rep; stos" instruction.
21316 Arguments have same meaning as for previous function */
21317 static void
21320 rtx orig_value)
21321 {
21322 rtx destexp;
21323 rtx countreg;
21324 HOST_WIDE_INT rounded_count;
21331 {
21335 }
21336 else
21339 {
21344 }
21348 }
21350 static void
21353 {
21357 }
21359 /* Output code to copy at most count & (max_size - 1) bytes from SRC to DEST. */
21360 static void
21363 {
21366 {
21371 {
21373 {
21376 }
21377 else
21380 }
21382 {
21385 else
21386 {
21389 }
21391 }
21393 {
21396 }
21398 {
21401 }
21403 {
21406 }
21408 }
21410 {
21416 }
21418 /* When there are stringops, we can cheaply increase dest and src pointers.
21419 Otherwise we save code size by maintaining offset (zero is readily
21420 available from preceding rep operation) and using x86 addressing modes.
21421 */
21423 {
21425 {
21432 }
21434 {
21441 }
21443 {
21450 }
21451 }
21452 else
21453 {
21455 rtx tmp;
21458 {
21469 }
21471 {
21484 }
21486 {
21495 }
21496 }
21497 }
21499 /* Output code to set at most count & (max_size - 1) bytes starting by DEST. */
21500 static void
21503 {
21504 count =
21510 }
21512 /* Output code to set at most count & (max_size - 1) bytes starting by DEST. */
21513 static void
21515 {
21516 rtx dest;
21519 {
21524 {
21526 {
21531 }
21532 else
21535 }
21537 {
21539 {
21542 }
21543 else
21544 {
21549 }
21551 }
21553 {
21557 }
21559 {
21563 }
21565 {
21569 }
21571 }
21573 {
21576 }
21578 {
21581 {
21585 }
21586 else
21587 {
21593 }
21596 }
21598 {
21601 {
21604 }
21605 else
21606 {
21610 }
21613 }
21615 {
21621 }
21623 {
21629 }
21631 {
21637 }
21638 }
21640 /* Copy enough from DEST to SRC to align DEST known to by aligned by ALIGN to
21641 DESIRED_ALIGNMENT. */
21642 static void
21646 {
21648 {
21656 }
21658 {
21666 }
21668 {
21676 }
21678 }
21680 /* Copy enough from DST to SRC to align DST known to DESIRED_ALIGN.
21681 ALIGN_BYTES is how many bytes need to be copied. */
21682 static rtx
21685 {
21694 {
21699 }
21701 {
21712 }
21714 {
21720 {
21728 }
21731 }
21737 {
21749 }
21756 }
21758 /* Set enough from DEST to align DEST known to by aligned by ALIGN to
21759 DESIRED_ALIGNMENT. */
21760 static void
21763 {
21765 {
21772 }
21774 {
21781 }
21783 {
21790 }
21792 }
21794 /* Set enough from DST to align DST known to by aligned by ALIGN to
21795 DESIRED_ALIGN. ALIGN_BYTES is how many bytes need to be stored. */
21796 static rtx
21799 {
21803 {
21808 }
21810 {
21817 }
21819 {
21826 }
21833 }
21835 /* Given COUNT and EXPECTED_SIZE, decide on codegen of string operation. */
21836 static enum stringop_alg
21839 {
21842 /* Algorithms using the rep prefix want at least edi and ecx;
21843 additionally, memset wants eax and memcpy wants esi. Don't
21844 consider such algorithms if the user has appropriated those
21845 registers for their own purposes. */
21847 || (memset
21850 #define ALG_USABLE_P(alg) (rep_prefix_usable \
21851 || (alg != rep_prefix_1_byte \
21852 && alg != rep_prefix_4_byte \
21853 && alg != rep_prefix_8_byte))
21856 /* Even if the string operation call is cold, we still might spend a lot
21857 of time processing large blocks. */
21862 else
21870 else
21874 /* rep; movq or rep; movl is the smallest variant. */
21876 {
21879 else
21881 }
21882 /* Very tiny blocks are best handled via the loop, REP is expensive to setup.
21883 */
21887 {
21891 {
21892 /* We get here if the algorithms that were not libcall-based
21893 were rep-prefix based and we are unable to use rep prefixes
21894 based on global register usage. Break out of the loop and
21895 use the heuristic below. */
21899 {
21904 /* Honor TARGET_INLINE_ALL_STRINGOPS by picking
21905 last non-libcall inline algorithm. */
21907 {
21908 /* When the current size is best to be copied by a libcall,
21909 but we are still forced to inline, run the heuristic below
21910 that will pick code for medium sized blocks. */
21914 }
21917 }
21918 }
21920 }
21921 /* When asked to inline the call anyway, try to pick meaningful choice.
21922 We look for maximal size of block that is faster to copy by hand and
21923 take blocks of at most of that size guessing that average size will
21924 be roughly half of the block.
21926 If this turns out to be bad, we might simply specify the preferred
21927 choice in ix86_costs. */
21930 {
21937 {
21944 }
21945 /* If there aren't any usable algorithms, then recursing on
21946 smaller sizes isn't going to find anything. Just return the
21947 simple byte-at-a-time copy loop. */
21949 {
21950 /* Pick something reasonable. */
21954 }
21963 }
21965 #undef ALG_USABLE_P
21966 }
21968 /* Decide on alignment. We know that the operand is already aligned to ALIGN
21969 (ALIGN can be based on profile feedback and thus it is not 100% guaranteed). */
21970 static int
21974 {
21977 {
21988 /* PentiumPro has special logic triggering for 8 byte aligned blocks.
21989 copying whole cacheline at once. */
21992 else
21996 /* PentiumPro has special logic triggering for 8 byte aligned blocks.
21997 copying whole cacheline at once. */
22000 else
22008 }
22017 }
22019 /* Return the smallest power of 2 greater than VAL. */
22020 static int
22022 {
22027 }
22029 /* Expand string move (memcpy) operation. Use i386 string operations
22030 when profitable. expand_setmem contains similar code. The code
22031 depends upon architecture, block size and alignment, but always has
22032 the same overall structure:
22034 1) Prologue guard: Conditional that jumps up to epilogues for small
22035 blocks that can be handled by epilogue alone. This is faster
22036 but also needed for correctness, since prologue assume the block
22037 is larger than the desired alignment.
22039 Optional dynamic check for size and libcall for large
22040 blocks is emitted here too, with -minline-stringops-dynamically.
22042 2) Prologue: copy first few bytes in order to get destination
22043 aligned to DESIRED_ALIGN. It is emitted only when ALIGN is less
22044 than DESIRED_ALIGN and up to DESIRED_ALIGN - ALIGN bytes can be
22045 copied. We emit either a jump tree on power of two sized
22046 blocks, or a byte loop.
22048 3) Main body: the copying loop itself, copying in SIZE_NEEDED chunks
22049 with specified algorithm.
22051 4) Epilogue: code copying tail of the block that is too small to be
22052 handled by main body (or up to size guarded by prologue guard). */
22054 bool
22057 {
22058 rtx destreg;
22059 rtx srcreg;
22061 rtx tmp;
22074 /* i386 can do misaligned access on reasonably increased cost. */
22078 /* ALIGN is the minimum of destination and source alignment, but we care here
22079 just about destination alignment. */
22088 /* Make sure we don't need to care about overflow later on. */
22092 /* Step 0: Decide on preferred algorithm, desired alignment and
22093 size of chunks to be copied by main loop. */
22109 {
22134 }
22138 /* Step 1: Prologue guard. */
22140 /* Alignment code needs count to be in register. */
22142 {
22145 {
22146 align_bytes
22150 else
22152 }
22155 }
22158 /* Ensure that alignment prologue won't copy past end of block. */
22160 {
22162 /* Epilogue always copies COUNT_EXP & EPILOGUE_SIZE_NEEDED bytes.
22163 Make sure it is power of 2. */
22167 {
22169 {
22170 /* If main algorithm works on QImode, no epilogue is needed.
22171 For small sizes just don't align anything. */
22174 else
22176 }
22177 }
22178 else
22179 {
22186 else
22188 }
22189 }
22191 /* Emit code to decide on runtime whether library call or inline should be
22192 used. */
22194 {
22196 {
22198 {
22202 }
22203 }
22204 else
22205 {
22214 }
22215 }
22217 /* Step 2: Alignment prologue. */
22220 {
22222 {
22223 /* Except for the first move in epilogue, we no longer know
22224 constant offset in aliasing info. It don't seems to worth
22225 the pain to maintain it for the first move, so throw away
22226 the info early. */
22230 desired_align);
22231 }
22232 else
22233 {
22234 /* If we know how many bytes need to be stored before dst is
22235 sufficiently aligned, maintain aliasing info accurately. */
22240 }
22246 {
22247 /* It is possible that we copied enough so the main loop will not
22248 execute. */
22258 else
22260 }
22261 }
22263 {
22268 }
22272 /* Step 3: Main loop. */
22275 {
22288 /* Unroll only by factor of 2 in 32bit mode, since we don't have enough
22289 registers for 4 temporaries anyway. */
22292 expected_size);
22296 DImode);
22300 SImode);
22304 QImode);
22306 }
22307 /* Adjust properly the offset of src and dest memory for aliasing. */
22309 {
22314 }
22315 else
22316 {
22319 }
22321 /* Step 4: Epilogue to copy the remaining bytes. */
22322 epilogue:
22324 {
22325 /* When the main loop is done, COUNT_EXP might hold original count,
22326 while we want to copy only COUNT_EXP & SIZE_NEEDED bytes.
22327 Epilogue code will actually copy COUNT_EXP & EPILOGUE_SIZE_NEEDED
22328 bytes. Compensate if needed. */
22331 {
22332 tmp =
22335 OPTAB_DIRECT);
22338 }
22341 }
22345 epilogue_size_needed);
22349 }
22351 /* Helper function for memcpy. For QImode value 0xXY produce
22352 0xXYXYXYXY of wide specified by MODE. This is essentially
22353 a * 0x10101010, but we can do slightly better than
22354 synth_mult by unwinding the sequence by hand on CPUs with
22355 slow multiply. */
22356 static rtx
22358 {
22360 rtx tmp;
22367 {
22375 }
22384 nops--;
22389 {
22393 OPTAB_DIRECT);
22394 }
22395 else
22396 {
22402 else
22404 else
22405 {
22408 reg =
22410 }
22420 }
22421 }
22423 /* Duplicate value VAL using promote_duplicated_reg into maximal size that will
22424 be needed by main loop copying SIZE_NEEDED chunks and prologue getting
22425 alignment from ALIGN to DESIRED_ALIGN. */
22426 static rtx
22428 {
22429 rtx promoted_val;
22438 else
22442 }
22444 /* Expand string clear operation (bzero). Use i386 string operations when
22445 profitable. See expand_movmem comment for explanation of individual
22446 steps performed. */
22447 bool
22450 {
22451 rtx destreg;
22453 rtx tmp;
22468 /* i386 can do misaligned access on reasonably increased cost. */
22477 /* Make sure we don't need to care about overflow later on. */
22481 /* Step 0: Decide on preferred algorithm, desired alignment and
22482 size of chunks to be copied by main loop. */
22497 {
22522 }
22525 /* Step 1: Prologue guard. */
22527 /* Alignment code needs count to be in register. */
22529 {
22532 {
22533 align_bytes
22537 else
22539 }
22541 {
22546 }
22547 }
22548 /* Do the cheap promotion to allow better CSE across the
22549 main loop and epilogue (ie one load of the big constant in the
22550 front of all code. */
22554 /* Ensure that alignment prologue won't copy past end of block. */
22556 {
22558 /* Epilogue always copies COUNT_EXP & (EPILOGUE_SIZE_NEEDED - 1) bytes.
22559 Make sure it is power of 2. */
22562 /* To improve performance of small blocks, we jump around the VAL
22563 promoting mode. This mean that if the promoted VAL is not constant,
22564 we might not use it in the epilogue and have to use byte
22565 loop variant. */
22569 {
22571 {
22572 /* If main algorithm works on QImode, no epilogue is needed.
22573 For small sizes just don't align anything. */
22576 else
22578 }
22579 }
22580 else
22581 {
22588 else
22590 }
22591 }
22593 {
22602 }
22604 /* Step 2: Alignment prologue. */
22606 /* Do the expensive promotion once we branched off the small blocks. */
22613 {
22615 {
22616 /* Except for the first move in epilogue, we no longer know
22617 constant offset in aliasing info. It don't seems to worth
22618 the pain to maintain it for the first move, so throw away
22619 the info early. */
22622 desired_align);
22623 }
22624 else
22625 {
22626 /* If we know how many bytes need to be stored before dst is
22627 sufficiently aligned, maintain aliasing info accurately. */
22632 }
22638 {
22639 /* It is possible that we copied enough so the main loop will not
22640 execute. */
22650 else
22652 }
22653 }
22655 {
22661 }
22665 /* Step 3: Main loop. */
22668 {
22696 }
22697 /* Adjust properly the offset of src and dest memory for aliasing. */
22701 else
22704 /* Step 4: Epilogue to copy the remaining bytes. */
22707 {
22708 /* When the main loop is done, COUNT_EXP might hold original count,
22709 while we want to copy only COUNT_EXP & SIZE_NEEDED bytes.
22710 Epilogue code will actually copy COUNT_EXP & EPILOGUE_SIZE_NEEDED
22711 bytes. Compensate if needed. */
22714 {
22715 tmp =
22718 OPTAB_DIRECT);
22721 }
22724 }
22725 epilogue:
22727 {
22730 epilogue_size_needed);
22731 else
22733 epilogue_size_needed);
22734 }
22738 }
22740 /* Expand the appropriate insns for doing strlen if not just doing
22741 repnz; scasb
22743 out = result, initialized with the start address
22744 align_rtx = alignment of the address.
22745 scratch = scratch register, initialized with the startaddress when
22746 not aligned, otherwise undefined
22748 This is just the body. It needs the initializations mentioned above and
22749 some address computing at the end. These things are done in i386.md. */
22751 static void
22753 {
22755 rtx tmp;
22760 rtx mem;
22763 rtx cmp;
22769 /* Loop to check 1..3 bytes for null to get an aligned pointer. */
22771 /* Is there a known alignment and is it less than 4? */
22773 {
22776 /* Is there a known alignment and is it not 2? */
22778 {
22782 /* Leave just the 3 lower bits. */
22792 }
22793 else
22794 {
22795 /* Since the alignment is 2, we have to check 2 or 0 bytes;
22796 check if is aligned to 4 - byte. */
22803 }
22807 /* Now compare the bytes. */
22809 /* Compare the first n unaligned byte on a byte per byte basis. */
22813 /* Increment the address. */
22816 /* Not needed with an alignment of 2 */
22818 {
22822 end_0_label);
22827 }
22830 end_0_label);
22833 }
22835 /* Generate loop to check 4 bytes at a time. It is not a good idea to
22836 align this loop. It gives only huge programs, but does not help to
22837 speed up. */
22844 /* This formula yields a nonzero result iff one of the bytes is zero.
22845 This saves three branches inside loop and many cycles. */
22853 align_4_label);
22856 {
22862 /* If zero is not in the first two bytes, move two bytes forward. */
22868 reg,
22869 tmpreg)));
22870 /* Emit lea manually to avoid clobbering of flags. */
22878 reg2,
22879 out)));
22880 }
22881 else
22882 {
22884 /* Is zero in the first two bytes? */
22891 pc_rtx);
22895 /* Not in the first two. Move two bytes forward. */
22901 }
22903 /* Avoid branch in fixing the byte. */
22911 }
22913 /* Expand strlen. */
22915 bool
22917 {
22920 /* The generic case of strlen expander is long. Avoid it's
22921 expanding unless TARGET_INLINE_ALL_STRINGOPS. */
22924 && !TARGET_INLINE_ALL_STRINGOPS
22934 {
22935 /* Well it seems that some optimizer does not combine a call like
22936 foo(strlen(bar), strlen(bar));
22937 when the move and the subtraction is done here. It does calculate
22938 the length just once when these instructions are done inside of
22939 output_strlen_unroll(). But I think since &bar[strlen(bar)] is
22940 often used and I use one fewer register for the lifetime of
22941 output_strlen_unroll() this is better. */
22947 /* strlensi_unroll_1 returns the address of the zero at the end of
22948 the string, like memchr(), so compute the length by subtracting
22949 the start address. */
22951 }
22952 else
22953 {
22954 rtx unspec;
22956 /* Can't use this if the user has appropriated eax, ecx, or edi. */
22969 /* If .md starts supporting :P, this can be done in .md. */
22975 }
22977 }
22979 /* For given symbol (function) construct code to compute address of it's PLT
22980 entry in large x86-64 PIC model. */
22981 rtx
22983 {
22996 }
22998 rtx
23000 rtx callarg2,
23002 {
23003 /* We need to represent that SI and DI registers are clobbered
23004 by SYSV calls. */
23010 };
23020 {
23021 #if TARGET_MACHO
23024 #endif
23025 }
23026 else
23027 {
23028 /* Static functions and indirect calls don't need the pic register. */
23033 }
23036 {
23040 }
23050 {
23055 }
23064 {
23068 }
23072 {
23076 UNSPEC_MS_TO_SYSV_CALL);
23085 }
23087 /* Add UNSPEC_CALL_NEEDS_VZEROUPPER decoration. */
23089 {
23092 {
23095 else
23097 }
23100 else
23105 else
23108 UNSPEC_CALL_NEEDS_VZEROUPPER);
23109 }
23118 }
23120 void
23122 {
23127 /* Strip off the last entry of the parallel. */
23132 else
23137 }
23139 /* Output the assembly for a call instruction. */
23143 {
23149 {
23152 /* SEH epilogue detection requires the indirect branch case
23153 to include REX.W. */
23156 else
23161 }
23163 /* SEH unwinding can require an extra nop to be emitted in several
23164 circumstances. Determine if we have one of those. */
23166 {
23167 rtx i;
23170 {
23171 /* If we get to another real insn, we don't need the nop. */
23175 /* If we get to the epilogue note, prevent a catch region from
23176 being adjacent to the standard epilogue sequence. If non-
23177 call-exceptions, we'll have done this during epilogue emission. */
23179 && !flag_non_call_exceptions
23181 {
23184 }
23185 }
23187 /* If we didn't find a real insn following the call, prevent the
23188 unwinder from looking into the next function. */
23191 }
23195 else
23204 }
23205 \f
23206 /* Clear stack slot assignments remembered from previous functions.
23207 This is called from INIT_EXPANDERS once before RTL is emitted for each
23208 function. */
23212 {
23221 }
23223 /* Return a MEM corresponding to a stack slot with mode MODE.
23224 Allocate a new slot if necessary.
23226 The RTL for a function can have several slots available: N is
23227 which slot to use. */
23229 rtx
23231 {
23236 /* Virtual slot is valid only before vregs are instantiated. */
23251 }
23252 \f
23253 /* Calculate the length of the memory address in the instruction encoding.
23254 Includes addr32 prefix, does not include the one-byte modrm, opcode,
23255 or other prefixes. */
23257 int
23259 {
23283 /* Add length of addr32 prefix. */
23287 /* Rule of thumb:
23288 - esp as the base always wants an index,
23289 - ebp as the base always wants a displacement,
23290 - r12 as the base always wants an index,
23291 - r13 as the base always wants a displacement. */
23293 /* Register Indirect. */
23295 {
23296 /* esp (for its index) and ebp (for its displacement) need
23297 the two-byte modrm form. Similarly for r12 and r13 in 64-bit
23298 code. */
23307 }
23309 /* Direct Addressing. In 64-bit mode mod 00 r/m 5
23310 is not disp32, but disp32(%rip), so for disp32
23311 SIB byte is needed, unless print_operand_address
23312 optimizes it into disp32(%rip) or (%rip) is implied
23313 by UNSPEC. */
23315 {
23318 {
23335 }
23336 }
23338 else
23339 {
23340 /* Find the length of the displacement constant. */
23342 {
23345 else
23347 }
23348 /* ebp always wants a displacement. Similarly r13. */
23353 /* An index requires the two-byte modrm form.... */
23355 /* ...like esp (or r12), which always wants an index. */
23361 }
23364 {
23371 }
23374 }
23376 /* Compute default value for "length_immediate" attribute. When SHORTFORM
23377 is set, expect that insn have 8bit immediate alternative. */
23378 int
23380 {
23386 {
23391 {
23394 {
23406 }
23408 {
23411 }
23412 }
23414 {
23424 /* Immediates for DImode instructions are encoded as 32bit sign extended values. */
23430 }
23431 }
23433 }
23434 /* Compute default value for "length_address" attribute. */
23435 int
23437 {
23441 {
23451 {
23456 }
23459 }
23464 {
23467 {
23472 constraints++;
23475 ;
23476 /* Skip ignored operands. */
23479 }
23481 }
23483 }
23485 /* Compute default value for "length_vex" attribute. It includes
23486 2 or 3 byte VEX prefix and 1 opcode byte. */
23488 int
23490 {
23493 /* Only 0f opcode can use 2 byte VEX prefix and VEX W bit uses 3
23494 byte VEX prefix. */
23498 /* We can always use 2 byte VEX prefix in 32bit. */
23506 {
23507 /* REX.W bit uses 3 byte VEX prefix. */
23511 }
23512 else
23513 {
23514 /* REX.X or REX.B bits use 3 byte VEX prefix. */
23518 }
23521 }
23522 \f
23523 /* Return the maximum number of instructions a cpu can issue. */
23525 static int
23527 {
23529 {
23554 }
23555 }
23557 /* A subroutine of ix86_adjust_cost -- return TRUE iff INSN reads flags set
23558 by DEP_INSN and nothing set by DEP_INSN. */
23560 static bool
23562 {
23565 /* Simplify the test for uninteresting insns. */
23573 {
23576 }
23581 {
23584 }
23585 else
23591 /* This test is true if the dependent insn reads the flags but
23592 not any other potentially set register. */
23600 }
23602 /* Return true iff USE_INSN has a memory address with operands set by
23603 SET_INSN. */
23605 bool
23607 {
23612 {
23615 }
23617 }
23619 static int
23621 {
23627 /* Anti and output dependencies have zero cost on all CPUs. */
23633 /* If we can't recognize the insns, we can't really do anything. */
23641 {
23643 /* Address Generation Interlock adds a cycle of latency. */
23645 {
23656 }
23660 /* ??? Compares pair with jump/setcc. */
23664 /* Floating point stores require value to be ready one cycle earlier. */
23674 /* INT->FP conversion is expensive. */
23678 /* There is one cycle extra latency between an FP op and a store. */
23686 /* Show ability of reorder buffer to hide latency of load by executing
23687 in parallel with previous instruction in case
23688 previous instruction is not needed to compute the address. */
23691 {
23692 /* Claim moves to take one cycle, as core can issue one load
23693 at time and the next load can start cycle later. */
23698 cost--;
23699 }
23705 /* The esp dependency is resolved before the instruction is really
23706 finished. */
23711 /* INT->FP conversion is expensive. */
23715 /* Show ability of reorder buffer to hide latency of load by executing
23716 in parallel with previous instruction in case
23717 previous instruction is not needed to compute the address. */
23720 {
23721 /* Claim moves to take one cycle, as core can issue one load
23722 at time and the next load can start cycle later. */
23728 else
23730 }
23744 /* Show ability of reorder buffer to hide latency of load by executing
23745 in parallel with previous instruction in case
23746 previous instruction is not needed to compute the address. */
23749 {
23753 /* Because of the difference between the length of integer and
23754 floating unit pipeline preparation stages, the memory operands
23755 for floating point are cheaper.
23757 ??? For Athlon it the difference is most probably 2. */
23760 else
23765 else
23767 }
23771 }
23774 }
23776 /* How many alternative schedules to try. This should be as wide as the
23777 scheduling freedom in the DFA, but no wider. Making this value too
23778 large results extra work for the scheduler. */
23780 static int
23782 {
23784 {
23796 /* Generally, we want haifa-sched:max_issue() to look ahead as far
23797 as many instructions can be executed on a cycle, i.e.,
23798 issue_rate. I wonder why tuning for many CPUs does not do this. */
23803 }
23804 }
23806 \f
23808 /* Model decoder of Core 2/i7.
23809 Below hooks for multipass scheduling (see haifa-sched.c:max_issue)
23810 track the instruction fetch block boundaries and make sure that long
23811 (9+ bytes) instructions are assigned to D0. */
23813 /* Maximum length of an insn that can be handled by
23814 a secondary decoder unit. '8' for Core 2/i7. */
23817 /* Ifetch block size, i.e., number of bytes decoder reads per cycle.
23818 '16' for Core 2/i7. */
23821 /* Maximum number of instructions decoder can handle per cycle.
23822 '6' for Core 2/i7. */
23826 ix86_first_cycle_multipass_data_t;
23828 const_ix86_first_cycle_multipass_data_t;
23830 /* A variable to store target state across calls to max_issue within
23831 one cycle. */
23835 /* Initialize DATA. */
23836 static void
23838 {
23839 ix86_first_cycle_multipass_data_t data
23846 }
23848 /* Advancing the cycle; reset ifetch block counts. */
23849 static void
23851 {
23858 }
23862 /* Filter out insns from ready_try that the core will not be able to issue
23863 on current cycle due to decoder. */
23864 static void
23865 core2i7_first_cycle_multipass_filter_ready_try
23868 {
23870 {
23871 rtx insn;
23881 (!first_cycle_insn_p
23883 /* ... or it would not fit into the ifetch block ... */
23885 /* ... or the decoder is full already ... */
23887 /* ... mask the insn out. */
23888 {
23893 }
23894 }
23895 }
23897 /* Prepare for a new round of multipass lookahead scheduling. */
23898 static void
23901 {
23902 ix86_first_cycle_multipass_data_t data
23904 const_ix86_first_cycle_multipass_data_t prev_data
23907 /* Restore the state from the end of the previous round. */
23911 /* Filter instructions that cannot be issued on current cycle due to
23912 decoder restrictions. */
23914 first_cycle_insn_p);
23915 }
23917 /* INSN is being issued in current solution. Account for its impact on
23918 the decoder model. */
23919 static void
23922 {
23923 ix86_first_cycle_multipass_data_t data
23925 const_ix86_first_cycle_multipass_data_t prev_data
23935 /* Allocate or resize the bitmap for storing INSN's effect on ready_try. */
23937 {
23940 }
23942 {
23946 }
23949 /* Filter out insns from ready_try that the core will not be able to issue
23950 on current cycle due to decoder. */
23953 }
23955 /* Revert the effect on ready_try. */
23956 static void
23960 {
23961 const_ix86_first_cycle_multipass_data_t data
23964 sbitmap_iterator sbi;
23968 {
23970 }
23971 }
23973 /* Save the result of multipass lookahead scheduling for the next round. */
23974 static void
23976 {
23977 const_ix86_first_cycle_multipass_data_t data
23979 ix86_first_cycle_multipass_data_t next_data
23983 {
23986 }
23987 }
23989 /* Deallocate target data. */
23990 static void
23992 {
23993 ix86_first_cycle_multipass_data_t data
23997 {
24001 }
24002 }
24004 /* Prepare for scheduling pass. */
24005 static void
24009 {
24010 /* Install scheduling hooks for current CPU. Some of these hooks are used
24011 in time-critical parts of the scheduler, so we only set them up when
24012 they are actually used. */
24014 {
24034 /* Set decoder parameters. */
24049 }
24050 }
24052 \f
24053 /* Compute the alignment given to a constant that is being placed in memory.
24054 EXP is the constant and ALIGN is the alignment that the object would
24055 ordinarily have.
24056 The value of this function is used instead of that alignment to align
24057 the object. */
24059 int
24061 {
24064 {
24069 }
24075 }
24077 /* Compute the alignment for a static variable.
24078 TYPE is the data type, and ALIGN is the alignment that
24079 the object would ordinarily have. The value of this function is used
24080 instead of that alignment to align the object. */
24082 int
24084 {
24095 /* x86-64 ABI requires arrays greater than 16 bytes to be aligned
24096 to 16byte boundary. */
24098 {
24105 }
24108 {
24113 }
24115 {
24122 }
24127 {
24132 }
24135 {
24140 }
24143 }
24145 /* Compute the alignment for a local variable or a stack slot. EXP is
24146 the data type or decl itself, MODE is the widest mode available and
24147 ALIGN is the alignment that the object would ordinarily have. The
24148 value of this macro is used instead of that alignment to align the
24149 object. */
24151 unsigned int
24154 {
24158 {
24161 }
24162 else
24163 {
24166 }
24168 /* Don't do dynamic stack realignment for long long objects with
24169 -mpreferred-stack-boundary=2. */
24178 /* If TYPE is NULL, we are allocating a stack slot for caller-save
24179 register in MODE. We will return the largest alignment of XF
24180 and DF. */
24182 {
24186 }
24188 /* x86-64 ABI requires arrays greater than 16 bytes to be aligned
24189 to 16byte boundary. Exact wording is:
24191 An array uses the same alignment as its elements, except that a local or
24192 global array variable of length at least 16 bytes or
24193 a C99 variable-length array variable always has alignment of at least 16 bytes.
24195 This was added to allow use of aligned SSE instructions at arrays. This
24196 rule is meant for static storage (where compiler can not do the analysis
24197 by itself). We follow it for automatic variables only when convenient.
24198 We fully control everything in the function compiled and functions from
24199 other unit can not rely on the alignment.
24201 Exclude va_list type. It is the common case of local array where
24202 we can not benefit from the alignment. */
24205 {
24215 }
24217 {
24222 }
24224 {
24230 }
24235 {
24240 }
24243 {
24249 }
24251 }
24253 /* Compute the minimum required alignment for dynamic stack realignment
24254 purposes for a local variable, parameter or a stack slot. EXP is
24255 the data type or decl itself, MODE is its mode and ALIGN is the
24256 alignment that the object would ordinarily have. */
24258 unsigned int
24261 {
24265 {
24268 }
24269 else
24270 {
24273 }
24278 /* Don't do dynamic stack realignment for long long objects with
24279 -mpreferred-stack-boundary=2. */
24286 }
24287 \f
24288 /* Find a location for the static chain incoming to a nested function.
24289 This is a register, unless all free registers are used by arguments. */
24291 static rtx
24293 {
24300 {
24301 /* We always use R10 in 64-bit mode. */
24303 }
24304 else
24305 {
24306 tree fntype;
24309 /* By default in 32-bit mode we use ECX to pass the static chain. */
24315 {
24316 /* Fastcall functions use ecx/edx for arguments, which leaves
24317 us with EAX for the static chain.
24318 Thiscall functions use ecx for arguments, which also
24319 leaves us with EAX for the static chain. */
24321 }
24323 {
24324 /* For regparm 3, we have no free call-clobbered registers in
24325 which to store the static chain. In order to implement this,
24326 we have the trampoline push the static chain to the stack.
24327 However, we can't push a value below the return address when
24328 we call the nested function directly, so we have to use an
24329 alternate entry point. For this we use ESI, and have the
24330 alternate entry point push ESI, so that things appear the
24331 same once we're executing the nested function. */
24333 {
24338 }
24340 }
24341 }
24344 }
24346 /* Emit RTL insns to initialize the variable parts of a trampoline.
24347 FNDECL is the decl of the target address; M_TRAMP is a MEM for
24348 the trampoline, and CHAIN_VALUE is an RTX for the static chain
24349 to be passed to the target function. */
24351 static void
24353 {
24361 {
24364 /* Load the function address to r11. Try to load address using
24365 the shorter movl instead of movabs. We may want to support
24366 movq for kernel mode, but kernel does not use trampolines at
24367 the moment. FNADDR is a 32bit address and may not be in
24368 DImode when ptr_mode == SImode. Always use movl in this
24369 case. */
24372 {
24381 }
24382 else
24383 {
24390 }
24392 /* Load static chain using movabs to r10. Use the shorter movl
24393 instead of movabs when ptr_mode == SImode. */
24395 {
24398 }
24399 else
24400 {
24403 }
24412 /* Jump to r11; the last (unused) byte is a nop, only there to
24413 pad the write out to a single 32-bit store. */
24417 }
24418 else
24419 {
24422 /* Depending on the static chain location, either load a register
24423 with a constant, or push the constant to the stack. All of the
24424 instructions are the same size. */
24427 {
24429 {
24436 }
24437 }
24438 else
24453 /* Compute offset from the end of the jmp to the target function.
24454 In the case in which the trampoline stores the static chain on
24455 the stack, we need to skip the first insn which pushes the
24456 (call-saved) register static chain; this push is 1 byte. */
24463 }
24467 #ifdef HAVE_ENABLE_EXECUTE_STACK
24468 #ifdef CHECK_EXECUTE_STACK_ENABLED
24470 #endif
24473 #endif
24474 }
24475 \f
24476 /* The following file contains several enumerations and data structures
24477 built from the definitions in i386-builtin-types.def. */
24481 /* Table for the ix86 builtin non-function types. */
24484 /* Retrieve an element from the above table, building some of
24485 the types lazily. */
24487 static tree
24489 {
24501 {
24509 }
24510 else
24511 {
24517 else
24525 }
24529 }
24531 /* Table for the ix86 builtin function types. */
24534 /* Retrieve an element from the above table, building some of
24535 the types lazily. */
24537 static tree
24539 {
24540 tree type;
24549 {
24557 {
24560 }
24563 }
24564 else
24565 {
24571 }
24575 }
24578 /* Codes for all the SSE/MMX builtins. */
24579 enum ix86_builtins
24580 {
24581 IX86_BUILTIN_ADDPS,
24582 IX86_BUILTIN_ADDSS,
24583 IX86_BUILTIN_DIVPS,
24584 IX86_BUILTIN_DIVSS,
24585 IX86_BUILTIN_MULPS,
24586 IX86_BUILTIN_MULSS,
24587 IX86_BUILTIN_SUBPS,
24588 IX86_BUILTIN_SUBSS,
24590 IX86_BUILTIN_CMPEQPS,
24591 IX86_BUILTIN_CMPLTPS,
24592 IX86_BUILTIN_CMPLEPS,
24593 IX86_BUILTIN_CMPGTPS,
24594 IX86_BUILTIN_CMPGEPS,
24595 IX86_BUILTIN_CMPNEQPS,
24596 IX86_BUILTIN_CMPNLTPS,
24597 IX86_BUILTIN_CMPNLEPS,
24598 IX86_BUILTIN_CMPNGTPS,
24599 IX86_BUILTIN_CMPNGEPS,
24600 IX86_BUILTIN_CMPORDPS,
24601 IX86_BUILTIN_CMPUNORDPS,
24602 IX86_BUILTIN_CMPEQSS,
24603 IX86_BUILTIN_CMPLTSS,
24604 IX86_BUILTIN_CMPLESS,
24605 IX86_BUILTIN_CMPNEQSS,
24606 IX86_BUILTIN_CMPNLTSS,
24607 IX86_BUILTIN_CMPNLESS,
24608 IX86_BUILTIN_CMPNGTSS,
24609 IX86_BUILTIN_CMPNGESS,
24610 IX86_BUILTIN_CMPORDSS,
24611 IX86_BUILTIN_CMPUNORDSS,
24613 IX86_BUILTIN_COMIEQSS,
24614 IX86_BUILTIN_COMILTSS,
24615 IX86_BUILTIN_COMILESS,
24616 IX86_BUILTIN_COMIGTSS,
24617 IX86_BUILTIN_COMIGESS,
24618 IX86_BUILTIN_COMINEQSS,
24619 IX86_BUILTIN_UCOMIEQSS,
24620 IX86_BUILTIN_UCOMILTSS,
24621 IX86_BUILTIN_UCOMILESS,
24622 IX86_BUILTIN_UCOMIGTSS,
24623 IX86_BUILTIN_UCOMIGESS,
24624 IX86_BUILTIN_UCOMINEQSS,
24626 IX86_BUILTIN_CVTPI2PS,
24627 IX86_BUILTIN_CVTPS2PI,
24628 IX86_BUILTIN_CVTSI2SS,
24629 IX86_BUILTIN_CVTSI642SS,
24630 IX86_BUILTIN_CVTSS2SI,
24631 IX86_BUILTIN_CVTSS2SI64,
24632 IX86_BUILTIN_CVTTPS2PI,
24633 IX86_BUILTIN_CVTTSS2SI,
24634 IX86_BUILTIN_CVTTSS2SI64,
24636 IX86_BUILTIN_MAXPS,
24637 IX86_BUILTIN_MAXSS,
24638 IX86_BUILTIN_MINPS,
24639 IX86_BUILTIN_MINSS,
24641 IX86_BUILTIN_LOADUPS,
24642 IX86_BUILTIN_STOREUPS,
24643 IX86_BUILTIN_MOVSS,
24645 IX86_BUILTIN_MOVHLPS,
24646 IX86_BUILTIN_MOVLHPS,
24647 IX86_BUILTIN_LOADHPS,
24648 IX86_BUILTIN_LOADLPS,
24649 IX86_BUILTIN_STOREHPS,
24650 IX86_BUILTIN_STORELPS,
24652 IX86_BUILTIN_MASKMOVQ,
24653 IX86_BUILTIN_MOVMSKPS,
24654 IX86_BUILTIN_PMOVMSKB,
24656 IX86_BUILTIN_MOVNTPS,
24657 IX86_BUILTIN_MOVNTQ,
24659 IX86_BUILTIN_LOADDQU,
24660 IX86_BUILTIN_STOREDQU,
24662 IX86_BUILTIN_PACKSSWB,
24663 IX86_BUILTIN_PACKSSDW,
24664 IX86_BUILTIN_PACKUSWB,
24666 IX86_BUILTIN_PADDB,
24667 IX86_BUILTIN_PADDW,
24668 IX86_BUILTIN_PADDD,
24669 IX86_BUILTIN_PADDQ,
24670 IX86_BUILTIN_PADDSB,
24671 IX86_BUILTIN_PADDSW,
24672 IX86_BUILTIN_PADDUSB,
24673 IX86_BUILTIN_PADDUSW,
24674 IX86_BUILTIN_PSUBB,
24675 IX86_BUILTIN_PSUBW,
24676 IX86_BUILTIN_PSUBD,
24677 IX86_BUILTIN_PSUBQ,
24678 IX86_BUILTIN_PSUBSB,
24679 IX86_BUILTIN_PSUBSW,
24680 IX86_BUILTIN_PSUBUSB,
24681 IX86_BUILTIN_PSUBUSW,
24683 IX86_BUILTIN_PAND,
24684 IX86_BUILTIN_PANDN,
24685 IX86_BUILTIN_POR,
24686 IX86_BUILTIN_PXOR,
24688 IX86_BUILTIN_PAVGB,
24689 IX86_BUILTIN_PAVGW,
24691 IX86_BUILTIN_PCMPEQB,
24692 IX86_BUILTIN_PCMPEQW,
24693 IX86_BUILTIN_PCMPEQD,
24694 IX86_BUILTIN_PCMPGTB,
24695 IX86_BUILTIN_PCMPGTW,
24696 IX86_BUILTIN_PCMPGTD,
24698 IX86_BUILTIN_PMADDWD,
24700 IX86_BUILTIN_PMAXSW,
24701 IX86_BUILTIN_PMAXUB,
24702 IX86_BUILTIN_PMINSW,
24703 IX86_BUILTIN_PMINUB,
24705 IX86_BUILTIN_PMULHUW,
24706 IX86_BUILTIN_PMULHW,
24707 IX86_BUILTIN_PMULLW,
24709 IX86_BUILTIN_PSADBW,
24710 IX86_BUILTIN_PSHUFW,
24712 IX86_BUILTIN_PSLLW,
24713 IX86_BUILTIN_PSLLD,
24714 IX86_BUILTIN_PSLLQ,
24715 IX86_BUILTIN_PSRAW,
24716 IX86_BUILTIN_PSRAD,
24717 IX86_BUILTIN_PSRLW,
24718 IX86_BUILTIN_PSRLD,
24719 IX86_BUILTIN_PSRLQ,
24720 IX86_BUILTIN_PSLLWI,
24721 IX86_BUILTIN_PSLLDI,
24722 IX86_BUILTIN_PSLLQI,
24723 IX86_BUILTIN_PSRAWI,
24724 IX86_BUILTIN_PSRADI,
24725 IX86_BUILTIN_PSRLWI,
24726 IX86_BUILTIN_PSRLDI,
24727 IX86_BUILTIN_PSRLQI,
24729 IX86_BUILTIN_PUNPCKHBW,
24730 IX86_BUILTIN_PUNPCKHWD,
24731 IX86_BUILTIN_PUNPCKHDQ,
24732 IX86_BUILTIN_PUNPCKLBW,
24733 IX86_BUILTIN_PUNPCKLWD,
24734 IX86_BUILTIN_PUNPCKLDQ,
24736 IX86_BUILTIN_SHUFPS,
24738 IX86_BUILTIN_RCPPS,
24739 IX86_BUILTIN_RCPSS,
24740 IX86_BUILTIN_RSQRTPS,
24741 IX86_BUILTIN_RSQRTPS_NR,
24742 IX86_BUILTIN_RSQRTSS,
24743 IX86_BUILTIN_RSQRTF,
24744 IX86_BUILTIN_SQRTPS,
24745 IX86_BUILTIN_SQRTPS_NR,
24746 IX86_BUILTIN_SQRTSS,
24748 IX86_BUILTIN_UNPCKHPS,
24749 IX86_BUILTIN_UNPCKLPS,
24751 IX86_BUILTIN_ANDPS,
24752 IX86_BUILTIN_ANDNPS,
24753 IX86_BUILTIN_ORPS,
24754 IX86_BUILTIN_XORPS,
24756 IX86_BUILTIN_EMMS,
24757 IX86_BUILTIN_LDMXCSR,
24758 IX86_BUILTIN_STMXCSR,
24759 IX86_BUILTIN_SFENCE,
24761 /* 3DNow! Original */
24762 IX86_BUILTIN_FEMMS,
24763 IX86_BUILTIN_PAVGUSB,
24764 IX86_BUILTIN_PF2ID,
24765 IX86_BUILTIN_PFACC,
24766 IX86_BUILTIN_PFADD,
24767 IX86_BUILTIN_PFCMPEQ,
24768 IX86_BUILTIN_PFCMPGE,
24769 IX86_BUILTIN_PFCMPGT,
24770 IX86_BUILTIN_PFMAX,
24771 IX86_BUILTIN_PFMIN,
24772 IX86_BUILTIN_PFMUL,
24773 IX86_BUILTIN_PFRCP,
24774 IX86_BUILTIN_PFRCPIT1,
24775 IX86_BUILTIN_PFRCPIT2,
24776 IX86_BUILTIN_PFRSQIT1,
24777 IX86_BUILTIN_PFRSQRT,
24778 IX86_BUILTIN_PFSUB,
24779 IX86_BUILTIN_PFSUBR,
24780 IX86_BUILTIN_PI2FD,
24781 IX86_BUILTIN_PMULHRW,
24783 /* 3DNow! Athlon Extensions */
24784 IX86_BUILTIN_PF2IW,
24785 IX86_BUILTIN_PFNACC,
24786 IX86_BUILTIN_PFPNACC,
24787 IX86_BUILTIN_PI2FW,
24788 IX86_BUILTIN_PSWAPDSI,
24789 IX86_BUILTIN_PSWAPDSF,
24791 /* SSE2 */
24792 IX86_BUILTIN_ADDPD,
24793 IX86_BUILTIN_ADDSD,
24794 IX86_BUILTIN_DIVPD,
24795 IX86_BUILTIN_DIVSD,
24796 IX86_BUILTIN_MULPD,
24797 IX86_BUILTIN_MULSD,
24798 IX86_BUILTIN_SUBPD,
24799 IX86_BUILTIN_SUBSD,
24801 IX86_BUILTIN_CMPEQPD,
24802 IX86_BUILTIN_CMPLTPD,
24803 IX86_BUILTIN_CMPLEPD,
24804 IX86_BUILTIN_CMPGTPD,
24805 IX86_BUILTIN_CMPGEPD,
24806 IX86_BUILTIN_CMPNEQPD,
24807 IX86_BUILTIN_CMPNLTPD,
24808 IX86_BUILTIN_CMPNLEPD,
24809 IX86_BUILTIN_CMPNGTPD,
24810 IX86_BUILTIN_CMPNGEPD,
24811 IX86_BUILTIN_CMPORDPD,
24812 IX86_BUILTIN_CMPUNORDPD,
24813 IX86_BUILTIN_CMPEQSD,
24814 IX86_BUILTIN_CMPLTSD,
24815 IX86_BUILTIN_CMPLESD,
24816 IX86_BUILTIN_CMPNEQSD,
24817 IX86_BUILTIN_CMPNLTSD,
24818 IX86_BUILTIN_CMPNLESD,
24819 IX86_BUILTIN_CMPORDSD,
24820 IX86_BUILTIN_CMPUNORDSD,
24822 IX86_BUILTIN_COMIEQSD,
24823 IX86_BUILTIN_COMILTSD,
24824 IX86_BUILTIN_COMILESD,
24825 IX86_BUILTIN_COMIGTSD,
24826 IX86_BUILTIN_COMIGESD,
24827 IX86_BUILTIN_COMINEQSD,
24828 IX86_BUILTIN_UCOMIEQSD,
24829 IX86_BUILTIN_UCOMILTSD,
24830 IX86_BUILTIN_UCOMILESD,
24831 IX86_BUILTIN_UCOMIGTSD,
24832 IX86_BUILTIN_UCOMIGESD,
24833 IX86_BUILTIN_UCOMINEQSD,
24835 IX86_BUILTIN_MAXPD,
24836 IX86_BUILTIN_MAXSD,
24837 IX86_BUILTIN_MINPD,
24838 IX86_BUILTIN_MINSD,
24840 IX86_BUILTIN_ANDPD,
24841 IX86_BUILTIN_ANDNPD,
24842 IX86_BUILTIN_ORPD,
24843 IX86_BUILTIN_XORPD,
24845 IX86_BUILTIN_SQRTPD,
24846 IX86_BUILTIN_SQRTSD,
24848 IX86_BUILTIN_UNPCKHPD,
24849 IX86_BUILTIN_UNPCKLPD,
24851 IX86_BUILTIN_SHUFPD,
24853 IX86_BUILTIN_LOADUPD,
24854 IX86_BUILTIN_STOREUPD,
24855 IX86_BUILTIN_MOVSD,
24857 IX86_BUILTIN_LOADHPD,
24858 IX86_BUILTIN_LOADLPD,
24860 IX86_BUILTIN_CVTDQ2PD,
24861 IX86_BUILTIN_CVTDQ2PS,
24863 IX86_BUILTIN_CVTPD2DQ,
24864 IX86_BUILTIN_CVTPD2PI,
24865 IX86_BUILTIN_CVTPD2PS,
24866 IX86_BUILTIN_CVTTPD2DQ,
24867 IX86_BUILTIN_CVTTPD2PI,
24869 IX86_BUILTIN_CVTPI2PD,
24870 IX86_BUILTIN_CVTSI2SD,
24871 IX86_BUILTIN_CVTSI642SD,
24873 IX86_BUILTIN_CVTSD2SI,
24874 IX86_BUILTIN_CVTSD2SI64,
24875 IX86_BUILTIN_CVTSD2SS,
24876 IX86_BUILTIN_CVTSS2SD,
24877 IX86_BUILTIN_CVTTSD2SI,
24878 IX86_BUILTIN_CVTTSD2SI64,
24880 IX86_BUILTIN_CVTPS2DQ,
24881 IX86_BUILTIN_CVTPS2PD,
24882 IX86_BUILTIN_CVTTPS2DQ,
24884 IX86_BUILTIN_MOVNTI,
24885 IX86_BUILTIN_MOVNTI64,
24886 IX86_BUILTIN_MOVNTPD,
24887 IX86_BUILTIN_MOVNTDQ,
24889 IX86_BUILTIN_MOVQ128,
24891 /* SSE2 MMX */
24892 IX86_BUILTIN_MASKMOVDQU,
24893 IX86_BUILTIN_MOVMSKPD,
24894 IX86_BUILTIN_PMOVMSKB128,
24896 IX86_BUILTIN_PACKSSWB128,
24897 IX86_BUILTIN_PACKSSDW128,
24898 IX86_BUILTIN_PACKUSWB128,
24900 IX86_BUILTIN_PADDB128,
24901 IX86_BUILTIN_PADDW128,
24902 IX86_BUILTIN_PADDD128,
24903 IX86_BUILTIN_PADDQ128,
24904 IX86_BUILTIN_PADDSB128,
24905 IX86_BUILTIN_PADDSW128,
24906 IX86_BUILTIN_PADDUSB128,
24907 IX86_BUILTIN_PADDUSW128,
24908 IX86_BUILTIN_PSUBB128,
24909 IX86_BUILTIN_PSUBW128,
24910 IX86_BUILTIN_PSUBD128,
24911 IX86_BUILTIN_PSUBQ128,
24912 IX86_BUILTIN_PSUBSB128,
24913 IX86_BUILTIN_PSUBSW128,
24914 IX86_BUILTIN_PSUBUSB128,
24915 IX86_BUILTIN_PSUBUSW128,
24917 IX86_BUILTIN_PAND128,
24918 IX86_BUILTIN_PANDN128,
24919 IX86_BUILTIN_POR128,
24920 IX86_BUILTIN_PXOR128,
24922 IX86_BUILTIN_PAVGB128,
24923 IX86_BUILTIN_PAVGW128,
24925 IX86_BUILTIN_PCMPEQB128,
24926 IX86_BUILTIN_PCMPEQW128,
24927 IX86_BUILTIN_PCMPEQD128,
24928 IX86_BUILTIN_PCMPGTB128,
24929 IX86_BUILTIN_PCMPGTW128,
24930 IX86_BUILTIN_PCMPGTD128,
24932 IX86_BUILTIN_PMADDWD128,
24934 IX86_BUILTIN_PMAXSW128,
24935 IX86_BUILTIN_PMAXUB128,
24936 IX86_BUILTIN_PMINSW128,
24937 IX86_BUILTIN_PMINUB128,
24939 IX86_BUILTIN_PMULUDQ,
24940 IX86_BUILTIN_PMULUDQ128,
24941 IX86_BUILTIN_PMULHUW128,
24942 IX86_BUILTIN_PMULHW128,
24943 IX86_BUILTIN_PMULLW128,
24945 IX86_BUILTIN_PSADBW128,
24946 IX86_BUILTIN_PSHUFHW,
24947 IX86_BUILTIN_PSHUFLW,
24948 IX86_BUILTIN_PSHUFD,
24950 IX86_BUILTIN_PSLLDQI128,
24951 IX86_BUILTIN_PSLLWI128,
24952 IX86_BUILTIN_PSLLDI128,
24953 IX86_BUILTIN_PSLLQI128,
24954 IX86_BUILTIN_PSRAWI128,
24955 IX86_BUILTIN_PSRADI128,
24956 IX86_BUILTIN_PSRLDQI128,
24957 IX86_BUILTIN_PSRLWI128,
24958 IX86_BUILTIN_PSRLDI128,
24959 IX86_BUILTIN_PSRLQI128,
24961 IX86_BUILTIN_PSLLDQ128,
24962 IX86_BUILTIN_PSLLW128,
24963 IX86_BUILTIN_PSLLD128,
24964 IX86_BUILTIN_PSLLQ128,
24965 IX86_BUILTIN_PSRAW128,
24966 IX86_BUILTIN_PSRAD128,
24967 IX86_BUILTIN_PSRLW128,
24968 IX86_BUILTIN_PSRLD128,
24969 IX86_BUILTIN_PSRLQ128,
24971 IX86_BUILTIN_PUNPCKHBW128,
24972 IX86_BUILTIN_PUNPCKHWD128,
24973 IX86_BUILTIN_PUNPCKHDQ128,
24974 IX86_BUILTIN_PUNPCKHQDQ128,
24975 IX86_BUILTIN_PUNPCKLBW128,
24976 IX86_BUILTIN_PUNPCKLWD128,
24977 IX86_BUILTIN_PUNPCKLDQ128,
24978 IX86_BUILTIN_PUNPCKLQDQ128,
24980 IX86_BUILTIN_CLFLUSH,
24981 IX86_BUILTIN_MFENCE,
24982 IX86_BUILTIN_LFENCE,
24983 IX86_BUILTIN_PAUSE,
24985 IX86_BUILTIN_BSRSI,
24986 IX86_BUILTIN_BSRDI,
24987 IX86_BUILTIN_RDPMC,
24988 IX86_BUILTIN_RDTSC,
24989 IX86_BUILTIN_RDTSCP,
24990 IX86_BUILTIN_ROLQI,
24991 IX86_BUILTIN_ROLHI,
24992 IX86_BUILTIN_RORQI,
24993 IX86_BUILTIN_RORHI,
24995 /* SSE3. */
24996 IX86_BUILTIN_ADDSUBPS,
24997 IX86_BUILTIN_HADDPS,
24998 IX86_BUILTIN_HSUBPS,
24999 IX86_BUILTIN_MOVSHDUP,
25000 IX86_BUILTIN_MOVSLDUP,
25001 IX86_BUILTIN_ADDSUBPD,
25002 IX86_BUILTIN_HADDPD,
25003 IX86_BUILTIN_HSUBPD,
25004 IX86_BUILTIN_LDDQU,
25006 IX86_BUILTIN_MONITOR,
25007 IX86_BUILTIN_MWAIT,
25009 /* SSSE3. */
25010 IX86_BUILTIN_PHADDW,
25011 IX86_BUILTIN_PHADDD,
25012 IX86_BUILTIN_PHADDSW,
25013 IX86_BUILTIN_PHSUBW,
25014 IX86_BUILTIN_PHSUBD,
25015 IX86_BUILTIN_PHSUBSW,
25016 IX86_BUILTIN_PMADDUBSW,
25017 IX86_BUILTIN_PMULHRSW,
25018 IX86_BUILTIN_PSHUFB,
25019 IX86_BUILTIN_PSIGNB,
25020 IX86_BUILTIN_PSIGNW,
25021 IX86_BUILTIN_PSIGND,
25022 IX86_BUILTIN_PALIGNR,
25023 IX86_BUILTIN_PABSB,
25024 IX86_BUILTIN_PABSW,
25025 IX86_BUILTIN_PABSD,
25027 IX86_BUILTIN_PHADDW128,
25028 IX86_BUILTIN_PHADDD128,
25029 IX86_BUILTIN_PHADDSW128,
25030 IX86_BUILTIN_PHSUBW128,
25031 IX86_BUILTIN_PHSUBD128,
25032 IX86_BUILTIN_PHSUBSW128,
25033 IX86_BUILTIN_PMADDUBSW128,
25034 IX86_BUILTIN_PMULHRSW128,
25035 IX86_BUILTIN_PSHUFB128,
25036 IX86_BUILTIN_PSIGNB128,
25037 IX86_BUILTIN_PSIGNW128,
25038 IX86_BUILTIN_PSIGND128,
25039 IX86_BUILTIN_PALIGNR128,
25040 IX86_BUILTIN_PABSB128,
25041 IX86_BUILTIN_PABSW128,
25042 IX86_BUILTIN_PABSD128,
25044 /* AMDFAM10 - SSE4A New Instructions. */
25045 IX86_BUILTIN_MOVNTSD,
25046 IX86_BUILTIN_MOVNTSS,
25047 IX86_BUILTIN_EXTRQI,
25048 IX86_BUILTIN_EXTRQ,
25049 IX86_BUILTIN_INSERTQI,
25050 IX86_BUILTIN_INSERTQ,
25052 /* SSE4.1. */
25053 IX86_BUILTIN_BLENDPD,
25054 IX86_BUILTIN_BLENDPS,
25055 IX86_BUILTIN_BLENDVPD,
25056 IX86_BUILTIN_BLENDVPS,
25057 IX86_BUILTIN_PBLENDVB128,
25058 IX86_BUILTIN_PBLENDW128,
25060 IX86_BUILTIN_DPPD,
25061 IX86_BUILTIN_DPPS,
25063 IX86_BUILTIN_INSERTPS128,
25065 IX86_BUILTIN_MOVNTDQA,
25066 IX86_BUILTIN_MPSADBW128,
25067 IX86_BUILTIN_PACKUSDW128,
25068 IX86_BUILTIN_PCMPEQQ,
25069 IX86_BUILTIN_PHMINPOSUW128,
25071 IX86_BUILTIN_PMAXSB128,
25072 IX86_BUILTIN_PMAXSD128,
25073 IX86_BUILTIN_PMAXUD128,
25074 IX86_BUILTIN_PMAXUW128,
25076 IX86_BUILTIN_PMINSB128,
25077 IX86_BUILTIN_PMINSD128,
25078 IX86_BUILTIN_PMINUD128,
25079 IX86_BUILTIN_PMINUW128,
25081 IX86_BUILTIN_PMOVSXBW128,
25082 IX86_BUILTIN_PMOVSXBD128,
25083 IX86_BUILTIN_PMOVSXBQ128,
25084 IX86_BUILTIN_PMOVSXWD128,
25085 IX86_BUILTIN_PMOVSXWQ128,
25086 IX86_BUILTIN_PMOVSXDQ128,
25088 IX86_BUILTIN_PMOVZXBW128,
25089 IX86_BUILTIN_PMOVZXBD128,
25090 IX86_BUILTIN_PMOVZXBQ128,
25091 IX86_BUILTIN_PMOVZXWD128,
25092 IX86_BUILTIN_PMOVZXWQ128,
25093 IX86_BUILTIN_PMOVZXDQ128,
25095 IX86_BUILTIN_PMULDQ128,
25096 IX86_BUILTIN_PMULLD128,
25098 IX86_BUILTIN_ROUNDSD,
25099 IX86_BUILTIN_ROUNDSS,
25101 IX86_BUILTIN_ROUNDPD,
25102 IX86_BUILTIN_ROUNDPS,
25104 IX86_BUILTIN_FLOORPD,
25105 IX86_BUILTIN_CEILPD,
25106 IX86_BUILTIN_TRUNCPD,
25107 IX86_BUILTIN_RINTPD,
25108 IX86_BUILTIN_ROUNDPD_AZ,
25110 IX86_BUILTIN_FLOORPD_VEC_PACK_SFIX,
25111 IX86_BUILTIN_CEILPD_VEC_PACK_SFIX,
25112 IX86_BUILTIN_ROUNDPD_AZ_VEC_PACK_SFIX,
25114 IX86_BUILTIN_FLOORPS,
25115 IX86_BUILTIN_CEILPS,
25116 IX86_BUILTIN_TRUNCPS,
25117 IX86_BUILTIN_RINTPS,
25118 IX86_BUILTIN_ROUNDPS_AZ,
25120 IX86_BUILTIN_FLOORPS_SFIX,
25121 IX86_BUILTIN_CEILPS_SFIX,
25122 IX86_BUILTIN_ROUNDPS_AZ_SFIX,
25124 IX86_BUILTIN_PTESTZ,
25125 IX86_BUILTIN_PTESTC,
25126 IX86_BUILTIN_PTESTNZC,
25128 IX86_BUILTIN_VEC_INIT_V2SI,
25129 IX86_BUILTIN_VEC_INIT_V4HI,
25130 IX86_BUILTIN_VEC_INIT_V8QI,
25131 IX86_BUILTIN_VEC_EXT_V2DF,
25132 IX86_BUILTIN_VEC_EXT_V2DI,
25133 IX86_BUILTIN_VEC_EXT_V4SF,
25134 IX86_BUILTIN_VEC_EXT_V4SI,
25135 IX86_BUILTIN_VEC_EXT_V8HI,
25136 IX86_BUILTIN_VEC_EXT_V2SI,
25137 IX86_BUILTIN_VEC_EXT_V4HI,
25138 IX86_BUILTIN_VEC_EXT_V16QI,
25139 IX86_BUILTIN_VEC_SET_V2DI,
25140 IX86_BUILTIN_VEC_SET_V4SF,
25141 IX86_BUILTIN_VEC_SET_V4SI,
25142 IX86_BUILTIN_VEC_SET_V8HI,
25143 IX86_BUILTIN_VEC_SET_V4HI,
25144 IX86_BUILTIN_VEC_SET_V16QI,
25146 IX86_BUILTIN_VEC_PACK_SFIX,
25147 IX86_BUILTIN_VEC_PACK_SFIX256,
25149 /* SSE4.2. */
25150 IX86_BUILTIN_CRC32QI,
25151 IX86_BUILTIN_CRC32HI,
25152 IX86_BUILTIN_CRC32SI,
25153 IX86_BUILTIN_CRC32DI,
25155 IX86_BUILTIN_PCMPESTRI128,
25156 IX86_BUILTIN_PCMPESTRM128,
25157 IX86_BUILTIN_PCMPESTRA128,
25158 IX86_BUILTIN_PCMPESTRC128,
25159 IX86_BUILTIN_PCMPESTRO128,
25160 IX86_BUILTIN_PCMPESTRS128,
25161 IX86_BUILTIN_PCMPESTRZ128,
25162 IX86_BUILTIN_PCMPISTRI128,
25163 IX86_BUILTIN_PCMPISTRM128,
25164 IX86_BUILTIN_PCMPISTRA128,
25165 IX86_BUILTIN_PCMPISTRC128,
25166 IX86_BUILTIN_PCMPISTRO128,
25167 IX86_BUILTIN_PCMPISTRS128,
25168 IX86_BUILTIN_PCMPISTRZ128,
25170 IX86_BUILTIN_PCMPGTQ,
25172 /* AES instructions */
25173 IX86_BUILTIN_AESENC128,
25174 IX86_BUILTIN_AESENCLAST128,
25175 IX86_BUILTIN_AESDEC128,
25176 IX86_BUILTIN_AESDECLAST128,
25177 IX86_BUILTIN_AESIMC128,
25178 IX86_BUILTIN_AESKEYGENASSIST128,
25180 /* PCLMUL instruction */
25181 IX86_BUILTIN_PCLMULQDQ128,
25183 /* AVX */
25184 IX86_BUILTIN_ADDPD256,
25185 IX86_BUILTIN_ADDPS256,
25186 IX86_BUILTIN_ADDSUBPD256,
25187 IX86_BUILTIN_ADDSUBPS256,
25188 IX86_BUILTIN_ANDPD256,
25189 IX86_BUILTIN_ANDPS256,
25190 IX86_BUILTIN_ANDNPD256,
25191 IX86_BUILTIN_ANDNPS256,
25192 IX86_BUILTIN_BLENDPD256,
25193 IX86_BUILTIN_BLENDPS256,
25194 IX86_BUILTIN_BLENDVPD256,
25195 IX86_BUILTIN_BLENDVPS256,
25196 IX86_BUILTIN_DIVPD256,
25197 IX86_BUILTIN_DIVPS256,
25198 IX86_BUILTIN_DPPS256,
25199 IX86_BUILTIN_HADDPD256,
25200 IX86_BUILTIN_HADDPS256,
25201 IX86_BUILTIN_HSUBPD256,
25202 IX86_BUILTIN_HSUBPS256,
25203 IX86_BUILTIN_MAXPD256,
25204 IX86_BUILTIN_MAXPS256,
25205 IX86_BUILTIN_MINPD256,
25206 IX86_BUILTIN_MINPS256,
25207 IX86_BUILTIN_MULPD256,
25208 IX86_BUILTIN_MULPS256,
25209 IX86_BUILTIN_ORPD256,
25210 IX86_BUILTIN_ORPS256,
25211 IX86_BUILTIN_SHUFPD256,
25212 IX86_BUILTIN_SHUFPS256,
25213 IX86_BUILTIN_SUBPD256,
25214 IX86_BUILTIN_SUBPS256,
25215 IX86_BUILTIN_XORPD256,
25216 IX86_BUILTIN_XORPS256,
25217 IX86_BUILTIN_CMPSD,
25218 IX86_BUILTIN_CMPSS,
25219 IX86_BUILTIN_CMPPD,
25220 IX86_BUILTIN_CMPPS,
25221 IX86_BUILTIN_CMPPD256,
25222 IX86_BUILTIN_CMPPS256,
25223 IX86_BUILTIN_CVTDQ2PD256,
25224 IX86_BUILTIN_CVTDQ2PS256,
25225 IX86_BUILTIN_CVTPD2PS256,
25226 IX86_BUILTIN_CVTPS2DQ256,
25227 IX86_BUILTIN_CVTPS2PD256,
25228 IX86_BUILTIN_CVTTPD2DQ256,
25229 IX86_BUILTIN_CVTPD2DQ256,
25230 IX86_BUILTIN_CVTTPS2DQ256,
25231 IX86_BUILTIN_EXTRACTF128PD256,
25232 IX86_BUILTIN_EXTRACTF128PS256,
25233 IX86_BUILTIN_EXTRACTF128SI256,
25234 IX86_BUILTIN_VZEROALL,
25235 IX86_BUILTIN_VZEROUPPER,
25236 IX86_BUILTIN_VPERMILVARPD,
25237 IX86_BUILTIN_VPERMILVARPS,
25238 IX86_BUILTIN_VPERMILVARPD256,
25239 IX86_BUILTIN_VPERMILVARPS256,
25240 IX86_BUILTIN_VPERMILPD,
25241 IX86_BUILTIN_VPERMILPS,
25242 IX86_BUILTIN_VPERMILPD256,
25243 IX86_BUILTIN_VPERMILPS256,
25244 IX86_BUILTIN_VPERMIL2PD,
25245 IX86_BUILTIN_VPERMIL2PS,
25246 IX86_BUILTIN_VPERMIL2PD256,
25247 IX86_BUILTIN_VPERMIL2PS256,
25248 IX86_BUILTIN_VPERM2F128PD256,
25249 IX86_BUILTIN_VPERM2F128PS256,
25250 IX86_BUILTIN_VPERM2F128SI256,
25251 IX86_BUILTIN_VBROADCASTSS,
25252 IX86_BUILTIN_VBROADCASTSD256,
25253 IX86_BUILTIN_VBROADCASTSS256,
25254 IX86_BUILTIN_VBROADCASTPD256,
25255 IX86_BUILTIN_VBROADCASTPS256,
25256 IX86_BUILTIN_VINSERTF128PD256,
25257 IX86_BUILTIN_VINSERTF128PS256,
25258 IX86_BUILTIN_VINSERTF128SI256,
25259 IX86_BUILTIN_LOADUPD256,
25260 IX86_BUILTIN_LOADUPS256,
25261 IX86_BUILTIN_STOREUPD256,
25262 IX86_BUILTIN_STOREUPS256,
25263 IX86_BUILTIN_LDDQU256,
25264 IX86_BUILTIN_MOVNTDQ256,
25265 IX86_BUILTIN_MOVNTPD256,
25266 IX86_BUILTIN_MOVNTPS256,
25267 IX86_BUILTIN_LOADDQU256,
25268 IX86_BUILTIN_STOREDQU256,
25269 IX86_BUILTIN_MASKLOADPD,
25270 IX86_BUILTIN_MASKLOADPS,
25271 IX86_BUILTIN_MASKSTOREPD,
25272 IX86_BUILTIN_MASKSTOREPS,
25273 IX86_BUILTIN_MASKLOADPD256,
25274 IX86_BUILTIN_MASKLOADPS256,
25275 IX86_BUILTIN_MASKSTOREPD256,
25276 IX86_BUILTIN_MASKSTOREPS256,
25277 IX86_BUILTIN_MOVSHDUP256,
25278 IX86_BUILTIN_MOVSLDUP256,
25279 IX86_BUILTIN_MOVDDUP256,
25281 IX86_BUILTIN_SQRTPD256,
25282 IX86_BUILTIN_SQRTPS256,
25283 IX86_BUILTIN_SQRTPS_NR256,
25284 IX86_BUILTIN_RSQRTPS256,
25285 IX86_BUILTIN_RSQRTPS_NR256,
25287 IX86_BUILTIN_RCPPS256,
25289 IX86_BUILTIN_ROUNDPD256,
25290 IX86_BUILTIN_ROUNDPS256,
25292 IX86_BUILTIN_FLOORPD256,
25293 IX86_BUILTIN_CEILPD256,
25294 IX86_BUILTIN_TRUNCPD256,
25295 IX86_BUILTIN_RINTPD256,
25296 IX86_BUILTIN_ROUNDPD_AZ256,
25298 IX86_BUILTIN_FLOORPD_VEC_PACK_SFIX256,
25299 IX86_BUILTIN_CEILPD_VEC_PACK_SFIX256,
25300 IX86_BUILTIN_ROUNDPD_AZ_VEC_PACK_SFIX256,
25302 IX86_BUILTIN_FLOORPS256,
25303 IX86_BUILTIN_CEILPS256,
25304 IX86_BUILTIN_TRUNCPS256,
25305 IX86_BUILTIN_RINTPS256,
25306 IX86_BUILTIN_ROUNDPS_AZ256,
25308 IX86_BUILTIN_FLOORPS_SFIX256,
25309 IX86_BUILTIN_CEILPS_SFIX256,
25310 IX86_BUILTIN_ROUNDPS_AZ_SFIX256,
25312 IX86_BUILTIN_UNPCKHPD256,
25313 IX86_BUILTIN_UNPCKLPD256,
25314 IX86_BUILTIN_UNPCKHPS256,
25315 IX86_BUILTIN_UNPCKLPS256,
25317 IX86_BUILTIN_SI256_SI,
25318 IX86_BUILTIN_PS256_PS,
25319 IX86_BUILTIN_PD256_PD,
25320 IX86_BUILTIN_SI_SI256,
25321 IX86_BUILTIN_PS_PS256,
25322 IX86_BUILTIN_PD_PD256,
25324 IX86_BUILTIN_VTESTZPD,
25325 IX86_BUILTIN_VTESTCPD,
25326 IX86_BUILTIN_VTESTNZCPD,
25327 IX86_BUILTIN_VTESTZPS,
25328 IX86_BUILTIN_VTESTCPS,
25329 IX86_BUILTIN_VTESTNZCPS,
25330 IX86_BUILTIN_VTESTZPD256,
25331 IX86_BUILTIN_VTESTCPD256,
25332 IX86_BUILTIN_VTESTNZCPD256,
25333 IX86_BUILTIN_VTESTZPS256,
25334 IX86_BUILTIN_VTESTCPS256,
25335 IX86_BUILTIN_VTESTNZCPS256,
25336 IX86_BUILTIN_PTESTZ256,
25337 IX86_BUILTIN_PTESTC256,
25338 IX86_BUILTIN_PTESTNZC256,
25340 IX86_BUILTIN_MOVMSKPD256,
25341 IX86_BUILTIN_MOVMSKPS256,
25343 /* AVX2 */
25344 IX86_BUILTIN_MPSADBW256,
25345 IX86_BUILTIN_PABSB256,
25346 IX86_BUILTIN_PABSW256,
25347 IX86_BUILTIN_PABSD256,
25348 IX86_BUILTIN_PACKSSDW256,
25349 IX86_BUILTIN_PACKSSWB256,
25350 IX86_BUILTIN_PACKUSDW256,
25351 IX86_BUILTIN_PACKUSWB256,
25352 IX86_BUILTIN_PADDB256,
25353 IX86_BUILTIN_PADDW256,
25354 IX86_BUILTIN_PADDD256,
25355 IX86_BUILTIN_PADDQ256,
25356 IX86_BUILTIN_PADDSB256,
25357 IX86_BUILTIN_PADDSW256,
25358 IX86_BUILTIN_PADDUSB256,
25359 IX86_BUILTIN_PADDUSW256,
25360 IX86_BUILTIN_PALIGNR256,
25361 IX86_BUILTIN_AND256I,
25362 IX86_BUILTIN_ANDNOT256I,
25363 IX86_BUILTIN_PAVGB256,
25364 IX86_BUILTIN_PAVGW256,
25365 IX86_BUILTIN_PBLENDVB256,
25366 IX86_BUILTIN_PBLENDVW256,
25367 IX86_BUILTIN_PCMPEQB256,
25368 IX86_BUILTIN_PCMPEQW256,
25369 IX86_BUILTIN_PCMPEQD256,
25370 IX86_BUILTIN_PCMPEQQ256,
25371 IX86_BUILTIN_PCMPGTB256,
25372 IX86_BUILTIN_PCMPGTW256,
25373 IX86_BUILTIN_PCMPGTD256,
25374 IX86_BUILTIN_PCMPGTQ256,
25375 IX86_BUILTIN_PHADDW256,
25376 IX86_BUILTIN_PHADDD256,
25377 IX86_BUILTIN_PHADDSW256,
25378 IX86_BUILTIN_PHSUBW256,
25379 IX86_BUILTIN_PHSUBD256,
25380 IX86_BUILTIN_PHSUBSW256,
25381 IX86_BUILTIN_PMADDUBSW256,
25382 IX86_BUILTIN_PMADDWD256,
25383 IX86_BUILTIN_PMAXSB256,
25384 IX86_BUILTIN_PMAXSW256,
25385 IX86_BUILTIN_PMAXSD256,
25386 IX86_BUILTIN_PMAXUB256,
25387 IX86_BUILTIN_PMAXUW256,
25388 IX86_BUILTIN_PMAXUD256,
25389 IX86_BUILTIN_PMINSB256,
25390 IX86_BUILTIN_PMINSW256,
25391 IX86_BUILTIN_PMINSD256,
25392 IX86_BUILTIN_PMINUB256,
25393 IX86_BUILTIN_PMINUW256,
25394 IX86_BUILTIN_PMINUD256,
25395 IX86_BUILTIN_PMOVMSKB256,
25396 IX86_BUILTIN_PMOVSXBW256,
25397 IX86_BUILTIN_PMOVSXBD256,
25398 IX86_BUILTIN_PMOVSXBQ256,
25399 IX86_BUILTIN_PMOVSXWD256,
25400 IX86_BUILTIN_PMOVSXWQ256,
25401 IX86_BUILTIN_PMOVSXDQ256,
25402 IX86_BUILTIN_PMOVZXBW256,
25403 IX86_BUILTIN_PMOVZXBD256,
25404 IX86_BUILTIN_PMOVZXBQ256,
25405 IX86_BUILTIN_PMOVZXWD256,
25406 IX86_BUILTIN_PMOVZXWQ256,
25407 IX86_BUILTIN_PMOVZXDQ256,
25408 IX86_BUILTIN_PMULDQ256,
25409 IX86_BUILTIN_PMULHRSW256,
25410 IX86_BUILTIN_PMULHUW256,
25411 IX86_BUILTIN_PMULHW256,
25412 IX86_BUILTIN_PMULLW256,
25413 IX86_BUILTIN_PMULLD256,
25414 IX86_BUILTIN_PMULUDQ256,
25415 IX86_BUILTIN_POR256,
25416 IX86_BUILTIN_PSADBW256,
25417 IX86_BUILTIN_PSHUFB256,
25418 IX86_BUILTIN_PSHUFD256,
25419 IX86_BUILTIN_PSHUFHW256,
25420 IX86_BUILTIN_PSHUFLW256,
25421 IX86_BUILTIN_PSIGNB256,
25422 IX86_BUILTIN_PSIGNW256,
25423 IX86_BUILTIN_PSIGND256,
25424 IX86_BUILTIN_PSLLDQI256,
25425 IX86_BUILTIN_PSLLWI256,
25426 IX86_BUILTIN_PSLLW256,
25427 IX86_BUILTIN_PSLLDI256,
25428 IX86_BUILTIN_PSLLD256,
25429 IX86_BUILTIN_PSLLQI256,
25430 IX86_BUILTIN_PSLLQ256,
25431 IX86_BUILTIN_PSRAWI256,
25432 IX86_BUILTIN_PSRAW256,
25433 IX86_BUILTIN_PSRADI256,
25434 IX86_BUILTIN_PSRAD256,
25435 IX86_BUILTIN_PSRLDQI256,
25436 IX86_BUILTIN_PSRLWI256,
25437 IX86_BUILTIN_PSRLW256,
25438 IX86_BUILTIN_PSRLDI256,
25439 IX86_BUILTIN_PSRLD256,
25440 IX86_BUILTIN_PSRLQI256,
25441 IX86_BUILTIN_PSRLQ256,
25442 IX86_BUILTIN_PSUBB256,
25443 IX86_BUILTIN_PSUBW256,
25444 IX86_BUILTIN_PSUBD256,
25445 IX86_BUILTIN_PSUBQ256,
25446 IX86_BUILTIN_PSUBSB256,
25447 IX86_BUILTIN_PSUBSW256,
25448 IX86_BUILTIN_PSUBUSB256,
25449 IX86_BUILTIN_PSUBUSW256,
25450 IX86_BUILTIN_PUNPCKHBW256,
25451 IX86_BUILTIN_PUNPCKHWD256,
25452 IX86_BUILTIN_PUNPCKHDQ256,
25453 IX86_BUILTIN_PUNPCKHQDQ256,
25454 IX86_BUILTIN_PUNPCKLBW256,
25455 IX86_BUILTIN_PUNPCKLWD256,
25456 IX86_BUILTIN_PUNPCKLDQ256,
25457 IX86_BUILTIN_PUNPCKLQDQ256,
25458 IX86_BUILTIN_PXOR256,
25459 IX86_BUILTIN_MOVNTDQA256,
25460 IX86_BUILTIN_VBROADCASTSS_PS,
25461 IX86_BUILTIN_VBROADCASTSS_PS256,
25462 IX86_BUILTIN_VBROADCASTSD_PD256,
25463 IX86_BUILTIN_VBROADCASTSI256,
25464 IX86_BUILTIN_PBLENDD256,
25465 IX86_BUILTIN_PBLENDD128,
25466 IX86_BUILTIN_PBROADCASTB256,
25467 IX86_BUILTIN_PBROADCASTW256,
25468 IX86_BUILTIN_PBROADCASTD256,
25469 IX86_BUILTIN_PBROADCASTQ256,
25470 IX86_BUILTIN_PBROADCASTB128,
25471 IX86_BUILTIN_PBROADCASTW128,
25472 IX86_BUILTIN_PBROADCASTD128,
25473 IX86_BUILTIN_PBROADCASTQ128,
25474 IX86_BUILTIN_VPERMVARSI256,
25475 IX86_BUILTIN_VPERMDF256,
25476 IX86_BUILTIN_VPERMVARSF256,
25477 IX86_BUILTIN_VPERMDI256,
25478 IX86_BUILTIN_VPERMTI256,
25479 IX86_BUILTIN_VEXTRACT128I256,
25480 IX86_BUILTIN_VINSERT128I256,
25481 IX86_BUILTIN_MASKLOADD,
25482 IX86_BUILTIN_MASKLOADQ,
25483 IX86_BUILTIN_MASKLOADD256,
25484 IX86_BUILTIN_MASKLOADQ256,
25485 IX86_BUILTIN_MASKSTORED,
25486 IX86_BUILTIN_MASKSTOREQ,
25487 IX86_BUILTIN_MASKSTORED256,
25488 IX86_BUILTIN_MASKSTOREQ256,
25489 IX86_BUILTIN_PSLLVV4DI,
25490 IX86_BUILTIN_PSLLVV2DI,
25491 IX86_BUILTIN_PSLLVV8SI,
25492 IX86_BUILTIN_PSLLVV4SI,
25493 IX86_BUILTIN_PSRAVV8SI,
25494 IX86_BUILTIN_PSRAVV4SI,
25495 IX86_BUILTIN_PSRLVV4DI,
25496 IX86_BUILTIN_PSRLVV2DI,
25497 IX86_BUILTIN_PSRLVV8SI,
25498 IX86_BUILTIN_PSRLVV4SI,
25500 IX86_BUILTIN_GATHERSIV2DF,
25501 IX86_BUILTIN_GATHERSIV4DF,
25502 IX86_BUILTIN_GATHERDIV2DF,
25503 IX86_BUILTIN_GATHERDIV4DF,
25504 IX86_BUILTIN_GATHERSIV4SF,
25505 IX86_BUILTIN_GATHERSIV8SF,
25506 IX86_BUILTIN_GATHERDIV4SF,
25507 IX86_BUILTIN_GATHERDIV8SF,
25508 IX86_BUILTIN_GATHERSIV2DI,
25509 IX86_BUILTIN_GATHERSIV4DI,
25510 IX86_BUILTIN_GATHERDIV2DI,
25511 IX86_BUILTIN_GATHERDIV4DI,
25512 IX86_BUILTIN_GATHERSIV4SI,
25513 IX86_BUILTIN_GATHERSIV8SI,
25514 IX86_BUILTIN_GATHERDIV4SI,
25515 IX86_BUILTIN_GATHERDIV8SI,
25517 /* Alternate 4 element gather for the vectorizer where
25518 all operands are 32-byte wide. */
25519 IX86_BUILTIN_GATHERALTSIV4DF,
25520 IX86_BUILTIN_GATHERALTDIV8SF,
25521 IX86_BUILTIN_GATHERALTSIV4DI,
25522 IX86_BUILTIN_GATHERALTDIV8SI,
25524 /* TFmode support builtins. */
25525 IX86_BUILTIN_INFQ,
25526 IX86_BUILTIN_HUGE_VALQ,
25527 IX86_BUILTIN_FABSQ,
25528 IX86_BUILTIN_COPYSIGNQ,
25530 /* Vectorizer support builtins. */
25531 IX86_BUILTIN_CPYSGNPS,
25532 IX86_BUILTIN_CPYSGNPD,
25533 IX86_BUILTIN_CPYSGNPS256,
25534 IX86_BUILTIN_CPYSGNPD256,
25536 /* FMA4 instructions. */
25537 IX86_BUILTIN_VFMADDSS,
25538 IX86_BUILTIN_VFMADDSD,
25539 IX86_BUILTIN_VFMADDPS,
25540 IX86_BUILTIN_VFMADDPD,
25541 IX86_BUILTIN_VFMADDPS256,
25542 IX86_BUILTIN_VFMADDPD256,
25543 IX86_BUILTIN_VFMADDSUBPS,
25544 IX86_BUILTIN_VFMADDSUBPD,
25545 IX86_BUILTIN_VFMADDSUBPS256,
25546 IX86_BUILTIN_VFMADDSUBPD256,
25548 /* FMA3 instructions. */
25549 IX86_BUILTIN_VFMADDSS3,
25550 IX86_BUILTIN_VFMADDSD3,
25552 /* XOP instructions. */
25553 IX86_BUILTIN_VPCMOV,
25554 IX86_BUILTIN_VPCMOV_V2DI,
25555 IX86_BUILTIN_VPCMOV_V4SI,
25556 IX86_BUILTIN_VPCMOV_V8HI,
25557 IX86_BUILTIN_VPCMOV_V16QI,
25558 IX86_BUILTIN_VPCMOV_V4SF,
25559 IX86_BUILTIN_VPCMOV_V2DF,
25560 IX86_BUILTIN_VPCMOV256,
25561 IX86_BUILTIN_VPCMOV_V4DI256,
25562 IX86_BUILTIN_VPCMOV_V8SI256,
25563 IX86_BUILTIN_VPCMOV_V16HI256,
25564 IX86_BUILTIN_VPCMOV_V32QI256,
25565 IX86_BUILTIN_VPCMOV_V8SF256,
25566 IX86_BUILTIN_VPCMOV_V4DF256,
25568 IX86_BUILTIN_VPPERM,
25570 IX86_BUILTIN_VPMACSSWW,
25571 IX86_BUILTIN_VPMACSWW,
25572 IX86_BUILTIN_VPMACSSWD,
25573 IX86_BUILTIN_VPMACSWD,
25574 IX86_BUILTIN_VPMACSSDD,
25575 IX86_BUILTIN_VPMACSDD,
25576 IX86_BUILTIN_VPMACSSDQL,
25577 IX86_BUILTIN_VPMACSSDQH,
25578 IX86_BUILTIN_VPMACSDQL,
25579 IX86_BUILTIN_VPMACSDQH,
25580 IX86_BUILTIN_VPMADCSSWD,
25581 IX86_BUILTIN_VPMADCSWD,
25583 IX86_BUILTIN_VPHADDBW,
25584 IX86_BUILTIN_VPHADDBD,
25585 IX86_BUILTIN_VPHADDBQ,
25586 IX86_BUILTIN_VPHADDWD,
25587 IX86_BUILTIN_VPHADDWQ,
25588 IX86_BUILTIN_VPHADDDQ,
25589 IX86_BUILTIN_VPHADDUBW,
25590 IX86_BUILTIN_VPHADDUBD,
25591 IX86_BUILTIN_VPHADDUBQ,
25592 IX86_BUILTIN_VPHADDUWD,
25593 IX86_BUILTIN_VPHADDUWQ,
25594 IX86_BUILTIN_VPHADDUDQ,
25595 IX86_BUILTIN_VPHSUBBW,
25596 IX86_BUILTIN_VPHSUBWD,
25597 IX86_BUILTIN_VPHSUBDQ,
25599 IX86_BUILTIN_VPROTB,
25600 IX86_BUILTIN_VPROTW,
25601 IX86_BUILTIN_VPROTD,
25602 IX86_BUILTIN_VPROTQ,
25603 IX86_BUILTIN_VPROTB_IMM,
25604 IX86_BUILTIN_VPROTW_IMM,
25605 IX86_BUILTIN_VPROTD_IMM,
25606 IX86_BUILTIN_VPROTQ_IMM,
25608 IX86_BUILTIN_VPSHLB,
25609 IX86_BUILTIN_VPSHLW,
25610 IX86_BUILTIN_VPSHLD,
25611 IX86_BUILTIN_VPSHLQ,
25612 IX86_BUILTIN_VPSHAB,
25613 IX86_BUILTIN_VPSHAW,
25614 IX86_BUILTIN_VPSHAD,
25615 IX86_BUILTIN_VPSHAQ,
25617 IX86_BUILTIN_VFRCZSS,
25618 IX86_BUILTIN_VFRCZSD,
25619 IX86_BUILTIN_VFRCZPS,
25620 IX86_BUILTIN_VFRCZPD,
25621 IX86_BUILTIN_VFRCZPS256,
25622 IX86_BUILTIN_VFRCZPD256,
25624 IX86_BUILTIN_VPCOMEQUB,
25625 IX86_BUILTIN_VPCOMNEUB,
25626 IX86_BUILTIN_VPCOMLTUB,
25627 IX86_BUILTIN_VPCOMLEUB,
25628 IX86_BUILTIN_VPCOMGTUB,
25629 IX86_BUILTIN_VPCOMGEUB,
25630 IX86_BUILTIN_VPCOMFALSEUB,
25631 IX86_BUILTIN_VPCOMTRUEUB,
25633 IX86_BUILTIN_VPCOMEQUW,
25634 IX86_BUILTIN_VPCOMNEUW,
25635 IX86_BUILTIN_VPCOMLTUW,
25636 IX86_BUILTIN_VPCOMLEUW,
25637 IX86_BUILTIN_VPCOMGTUW,
25638 IX86_BUILTIN_VPCOMGEUW,
25639 IX86_BUILTIN_VPCOMFALSEUW,
25640 IX86_BUILTIN_VPCOMTRUEUW,
25642 IX86_BUILTIN_VPCOMEQUD,
25643 IX86_BUILTIN_VPCOMNEUD,
25644 IX86_BUILTIN_VPCOMLTUD,
25645 IX86_BUILTIN_VPCOMLEUD,
25646 IX86_BUILTIN_VPCOMGTUD,
25647 IX86_BUILTIN_VPCOMGEUD,
25648 IX86_BUILTIN_VPCOMFALSEUD,
25649 IX86_BUILTIN_VPCOMTRUEUD,
25651 IX86_BUILTIN_VPCOMEQUQ,
25652 IX86_BUILTIN_VPCOMNEUQ,
25653 IX86_BUILTIN_VPCOMLTUQ,
25654 IX86_BUILTIN_VPCOMLEUQ,
25655 IX86_BUILTIN_VPCOMGTUQ,
25656 IX86_BUILTIN_VPCOMGEUQ,
25657 IX86_BUILTIN_VPCOMFALSEUQ,
25658 IX86_BUILTIN_VPCOMTRUEUQ,
25660 IX86_BUILTIN_VPCOMEQB,
25661 IX86_BUILTIN_VPCOMNEB,
25662 IX86_BUILTIN_VPCOMLTB,
25663 IX86_BUILTIN_VPCOMLEB,
25664 IX86_BUILTIN_VPCOMGTB,
25665 IX86_BUILTIN_VPCOMGEB,
25666 IX86_BUILTIN_VPCOMFALSEB,
25667 IX86_BUILTIN_VPCOMTRUEB,
25669 IX86_BUILTIN_VPCOMEQW,
25670 IX86_BUILTIN_VPCOMNEW,
25671 IX86_BUILTIN_VPCOMLTW,
25672 IX86_BUILTIN_VPCOMLEW,
25673 IX86_BUILTIN_VPCOMGTW,
25674 IX86_BUILTIN_VPCOMGEW,
25675 IX86_BUILTIN_VPCOMFALSEW,
25676 IX86_BUILTIN_VPCOMTRUEW,
25678 IX86_BUILTIN_VPCOMEQD,
25679 IX86_BUILTIN_VPCOMNED,
25680 IX86_BUILTIN_VPCOMLTD,
25681 IX86_BUILTIN_VPCOMLED,
25682 IX86_BUILTIN_VPCOMGTD,
25683 IX86_BUILTIN_VPCOMGED,
25684 IX86_BUILTIN_VPCOMFALSED,
25685 IX86_BUILTIN_VPCOMTRUED,
25687 IX86_BUILTIN_VPCOMEQQ,
25688 IX86_BUILTIN_VPCOMNEQ,
25689 IX86_BUILTIN_VPCOMLTQ,
25690 IX86_BUILTIN_VPCOMLEQ,
25691 IX86_BUILTIN_VPCOMGTQ,
25692 IX86_BUILTIN_VPCOMGEQ,
25693 IX86_BUILTIN_VPCOMFALSEQ,
25694 IX86_BUILTIN_VPCOMTRUEQ,
25696 /* LWP instructions. */
25697 IX86_BUILTIN_LLWPCB,
25698 IX86_BUILTIN_SLWPCB,
25699 IX86_BUILTIN_LWPVAL32,
25700 IX86_BUILTIN_LWPVAL64,
25701 IX86_BUILTIN_LWPINS32,
25702 IX86_BUILTIN_LWPINS64,
25704 IX86_BUILTIN_CLZS,
25706 /* RTM */
25707 IX86_BUILTIN_XBEGIN,
25708 IX86_BUILTIN_XEND,
25709 IX86_BUILTIN_XABORT,
25710 IX86_BUILTIN_XTEST,
25712 /* BMI instructions. */
25713 IX86_BUILTIN_BEXTR32,
25714 IX86_BUILTIN_BEXTR64,
25715 IX86_BUILTIN_CTZS,
25717 /* TBM instructions. */
25718 IX86_BUILTIN_BEXTRI32,
25719 IX86_BUILTIN_BEXTRI64,
25721 /* BMI2 instructions. */
25722 IX86_BUILTIN_BZHI32,
25723 IX86_BUILTIN_BZHI64,
25724 IX86_BUILTIN_PDEP32,
25725 IX86_BUILTIN_PDEP64,
25726 IX86_BUILTIN_PEXT32,
25727 IX86_BUILTIN_PEXT64,
25729 /* FSGSBASE instructions. */
25730 IX86_BUILTIN_RDFSBASE32,
25731 IX86_BUILTIN_RDFSBASE64,
25732 IX86_BUILTIN_RDGSBASE32,
25733 IX86_BUILTIN_RDGSBASE64,
25734 IX86_BUILTIN_WRFSBASE32,
25735 IX86_BUILTIN_WRFSBASE64,
25736 IX86_BUILTIN_WRGSBASE32,
25737 IX86_BUILTIN_WRGSBASE64,
25739 /* RDRND instructions. */
25740 IX86_BUILTIN_RDRAND16_STEP,
25741 IX86_BUILTIN_RDRAND32_STEP,
25742 IX86_BUILTIN_RDRAND64_STEP,
25744 /* F16C instructions. */
25745 IX86_BUILTIN_CVTPH2PS,
25746 IX86_BUILTIN_CVTPH2PS256,
25747 IX86_BUILTIN_CVTPS2PH,
25748 IX86_BUILTIN_CVTPS2PH256,
25750 /* CFString built-in for darwin */
25751 IX86_BUILTIN_CFSTRING,
25753 IX86_BUILTIN_MAX
25754 };
25756 /* Table for the ix86 builtin decls. */
25759 /* Table of all of the builtin functions that are possible with different ISA's
25760 but are waiting to be built until a function is declared to use that
25761 ISA. */
25768 };
25773 /* Add an ix86 target builtin function with CODE, NAME and TYPE. Save the MASK
25774 of which isa_flags to use in the ix86_builtins_isa array. Stores the
25775 function decl in the ix86_builtins array. Returns the function decl or
25776 NULL_TREE, if the builtin was not added.
25778 If the front end has a special hook for builtin functions, delay adding
25779 builtin functions that aren't in the current ISA until the ISA is changed
25780 with function specific optimization. Doing so, can save about 300K for the
25781 default compiler. When the builtin is expanded, check at that time whether
25782 it is valid.
25784 If the front end doesn't have a special hook, record all builtins, even if
25785 it isn't an instruction set in the current ISA in case the user uses
25786 function specific options for a different ISA, so that we don't get scope
25787 errors if a builtin is added in the middle of a function scope. */
25793 {
25797 {
25806 {
25812 }
25813 else
25814 {
25820 }
25821 }
25824 }
25826 /* Like def_builtin, but also marks the function decl "const". */
25831 {
25835 else
25839 }
25841 /* Add any new builtin functions for a given ISA that may not have been
25842 declared. This saves a bit of space compared to adding all of the
25843 declarations to the tree, even if we didn't use them. */
25845 static void
25847 {
25851 {
25854 {
25857 /* Don't define the builtin again. */
25863 NULL_TREE);
25868 }
25869 }
25870 }
25872 /* Bits for builtin_description.flag. */
25874 /* Set when we don't support the comparison natively, and should
25875 swap_comparison in order to support it. */
25876 #define BUILTIN_DESC_SWAP_OPERANDS 1
25878 struct builtin_description
25879 {
25886 };
25889 {
25890 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_comi, "__builtin_ia32_comieq", IX86_BUILTIN_COMIEQSS, UNEQ, 0 },
25891 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_comi, "__builtin_ia32_comilt", IX86_BUILTIN_COMILTSS, UNLT, 0 },
25892 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_comi, "__builtin_ia32_comile", IX86_BUILTIN_COMILESS, UNLE, 0 },
25893 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_comi, "__builtin_ia32_comigt", IX86_BUILTIN_COMIGTSS, GT, 0 },
25894 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_comi, "__builtin_ia32_comige", IX86_BUILTIN_COMIGESS, GE, 0 },
25895 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_comi, "__builtin_ia32_comineq", IX86_BUILTIN_COMINEQSS, LTGT, 0 },
25896 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_ucomi, "__builtin_ia32_ucomieq", IX86_BUILTIN_UCOMIEQSS, UNEQ, 0 },
25897 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_ucomi, "__builtin_ia32_ucomilt", IX86_BUILTIN_UCOMILTSS, UNLT, 0 },
25898 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_ucomi, "__builtin_ia32_ucomile", IX86_BUILTIN_UCOMILESS, UNLE, 0 },
25899 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_ucomi, "__builtin_ia32_ucomigt", IX86_BUILTIN_UCOMIGTSS, GT, 0 },
25900 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_ucomi, "__builtin_ia32_ucomige", IX86_BUILTIN_UCOMIGESS, GE, 0 },
25901 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_ucomi, "__builtin_ia32_ucomineq", IX86_BUILTIN_UCOMINEQSS, LTGT, 0 },
25902 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_comi, "__builtin_ia32_comisdeq", IX86_BUILTIN_COMIEQSD, UNEQ, 0 },
25903 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_comi, "__builtin_ia32_comisdlt", IX86_BUILTIN_COMILTSD, UNLT, 0 },
25904 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_comi, "__builtin_ia32_comisdle", IX86_BUILTIN_COMILESD, UNLE, 0 },
25905 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_comi, "__builtin_ia32_comisdgt", IX86_BUILTIN_COMIGTSD, GT, 0 },
25906 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_comi, "__builtin_ia32_comisdge", IX86_BUILTIN_COMIGESD, GE, 0 },
25907 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_comi, "__builtin_ia32_comisdneq", IX86_BUILTIN_COMINEQSD, LTGT, 0 },
25908 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ucomi, "__builtin_ia32_ucomisdeq", IX86_BUILTIN_UCOMIEQSD, UNEQ, 0 },
25909 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ucomi, "__builtin_ia32_ucomisdlt", IX86_BUILTIN_UCOMILTSD, UNLT, 0 },
25910 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ucomi, "__builtin_ia32_ucomisdle", IX86_BUILTIN_UCOMILESD, UNLE, 0 },
25911 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ucomi, "__builtin_ia32_ucomisdgt", IX86_BUILTIN_UCOMIGTSD, GT, 0 },
25912 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ucomi, "__builtin_ia32_ucomisdge", IX86_BUILTIN_UCOMIGESD, GE, 0 },
25913 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ucomi, "__builtin_ia32_ucomisdneq", IX86_BUILTIN_UCOMINEQSD, LTGT, 0 },
25914 };
25917 {
25918 /* SSE4.2 */
25919 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpestr, "__builtin_ia32_pcmpestri128", IX86_BUILTIN_PCMPESTRI128, UNKNOWN, 0 },
25920 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpestr, "__builtin_ia32_pcmpestrm128", IX86_BUILTIN_PCMPESTRM128, UNKNOWN, 0 },
25921 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpestr, "__builtin_ia32_pcmpestria128", IX86_BUILTIN_PCMPESTRA128, UNKNOWN, (int) CCAmode },
25922 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpestr, "__builtin_ia32_pcmpestric128", IX86_BUILTIN_PCMPESTRC128, UNKNOWN, (int) CCCmode },
25923 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpestr, "__builtin_ia32_pcmpestrio128", IX86_BUILTIN_PCMPESTRO128, UNKNOWN, (int) CCOmode },
25924 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpestr, "__builtin_ia32_pcmpestris128", IX86_BUILTIN_PCMPESTRS128, UNKNOWN, (int) CCSmode },
25925 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpestr, "__builtin_ia32_pcmpestriz128", IX86_BUILTIN_PCMPESTRZ128, UNKNOWN, (int) CCZmode },
25926 };
25929 {
25930 /* SSE4.2 */
25931 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpistr, "__builtin_ia32_pcmpistri128", IX86_BUILTIN_PCMPISTRI128, UNKNOWN, 0 },
25932 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpistr, "__builtin_ia32_pcmpistrm128", IX86_BUILTIN_PCMPISTRM128, UNKNOWN, 0 },
25933 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpistr, "__builtin_ia32_pcmpistria128", IX86_BUILTIN_PCMPISTRA128, UNKNOWN, (int) CCAmode },
25934 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpistr, "__builtin_ia32_pcmpistric128", IX86_BUILTIN_PCMPISTRC128, UNKNOWN, (int) CCCmode },
25935 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpistr, "__builtin_ia32_pcmpistrio128", IX86_BUILTIN_PCMPISTRO128, UNKNOWN, (int) CCOmode },
25936 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpistr, "__builtin_ia32_pcmpistris128", IX86_BUILTIN_PCMPISTRS128, UNKNOWN, (int) CCSmode },
25937 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpistr, "__builtin_ia32_pcmpistriz128", IX86_BUILTIN_PCMPISTRZ128, UNKNOWN, (int) CCZmode },
25938 };
25940 /* Special builtins with variable number of arguments. */
25942 {
25943 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_rdtsc, "__builtin_ia32_rdtsc", IX86_BUILTIN_RDTSC, UNKNOWN, (int) UINT64_FTYPE_VOID },
25944 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_rdtscp, "__builtin_ia32_rdtscp", IX86_BUILTIN_RDTSCP, UNKNOWN, (int) UINT64_FTYPE_PUNSIGNED },
25945 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_pause, "__builtin_ia32_pause", IX86_BUILTIN_PAUSE, UNKNOWN, (int) VOID_FTYPE_VOID },
25947 /* MMX */
25948 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_emms, "__builtin_ia32_emms", IX86_BUILTIN_EMMS, UNKNOWN, (int) VOID_FTYPE_VOID },
25950 /* 3DNow! */
25951 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_femms, "__builtin_ia32_femms", IX86_BUILTIN_FEMMS, UNKNOWN, (int) VOID_FTYPE_VOID },
25953 /* SSE */
25954 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_movups, "__builtin_ia32_storeups", IX86_BUILTIN_STOREUPS, UNKNOWN, (int) VOID_FTYPE_PFLOAT_V4SF },
25955 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_movntv4sf, "__builtin_ia32_movntps", IX86_BUILTIN_MOVNTPS, UNKNOWN, (int) VOID_FTYPE_PFLOAT_V4SF },
25956 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_movups, "__builtin_ia32_loadups", IX86_BUILTIN_LOADUPS, UNKNOWN, (int) V4SF_FTYPE_PCFLOAT },
25958 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_loadhps_exp, "__builtin_ia32_loadhps", IX86_BUILTIN_LOADHPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_PCV2SF },
25959 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_loadlps_exp, "__builtin_ia32_loadlps", IX86_BUILTIN_LOADLPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_PCV2SF },
25960 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_storehps, "__builtin_ia32_storehps", IX86_BUILTIN_STOREHPS, UNKNOWN, (int) VOID_FTYPE_PV2SF_V4SF },
25961 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_storelps, "__builtin_ia32_storelps", IX86_BUILTIN_STORELPS, UNKNOWN, (int) VOID_FTYPE_PV2SF_V4SF },
25963 /* SSE or 3DNow!A */
25964 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_sse_sfence, "__builtin_ia32_sfence", IX86_BUILTIN_SFENCE, UNKNOWN, (int) VOID_FTYPE_VOID },
25965 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_sse_movntq, "__builtin_ia32_movntq", IX86_BUILTIN_MOVNTQ, UNKNOWN, (int) VOID_FTYPE_PULONGLONG_ULONGLONG },
25967 /* SSE2 */
25968 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_lfence, "__builtin_ia32_lfence", IX86_BUILTIN_LFENCE, UNKNOWN, (int) VOID_FTYPE_VOID },
25969 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_mfence, 0, IX86_BUILTIN_MFENCE, UNKNOWN, (int) VOID_FTYPE_VOID },
25970 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_movupd, "__builtin_ia32_storeupd", IX86_BUILTIN_STOREUPD, UNKNOWN, (int) VOID_FTYPE_PDOUBLE_V2DF },
25971 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_movdqu, "__builtin_ia32_storedqu", IX86_BUILTIN_STOREDQU, UNKNOWN, (int) VOID_FTYPE_PCHAR_V16QI },
25972 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_movntv2df, "__builtin_ia32_movntpd", IX86_BUILTIN_MOVNTPD, UNKNOWN, (int) VOID_FTYPE_PDOUBLE_V2DF },
25973 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_movntv2di, "__builtin_ia32_movntdq", IX86_BUILTIN_MOVNTDQ, UNKNOWN, (int) VOID_FTYPE_PV2DI_V2DI },
25974 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_movntisi, "__builtin_ia32_movnti", IX86_BUILTIN_MOVNTI, UNKNOWN, (int) VOID_FTYPE_PINT_INT },
25975 { OPTION_MASK_ISA_SSE2 | OPTION_MASK_ISA_64BIT, CODE_FOR_sse2_movntidi, "__builtin_ia32_movnti64", IX86_BUILTIN_MOVNTI64, UNKNOWN, (int) VOID_FTYPE_PLONGLONG_LONGLONG },
25976 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_movupd, "__builtin_ia32_loadupd", IX86_BUILTIN_LOADUPD, UNKNOWN, (int) V2DF_FTYPE_PCDOUBLE },
25977 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_movdqu, "__builtin_ia32_loaddqu", IX86_BUILTIN_LOADDQU, UNKNOWN, (int) V16QI_FTYPE_PCCHAR },
25979 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_loadhpd_exp, "__builtin_ia32_loadhpd", IX86_BUILTIN_LOADHPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_PCDOUBLE },
25980 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_loadlpd_exp, "__builtin_ia32_loadlpd", IX86_BUILTIN_LOADLPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_PCDOUBLE },
25982 /* SSE3 */
25983 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_lddqu, "__builtin_ia32_lddqu", IX86_BUILTIN_LDDQU, UNKNOWN, (int) V16QI_FTYPE_PCCHAR },
25985 /* SSE4.1 */
25986 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_movntdqa, "__builtin_ia32_movntdqa", IX86_BUILTIN_MOVNTDQA, UNKNOWN, (int) V2DI_FTYPE_PV2DI },
25988 /* SSE4A */
25989 { OPTION_MASK_ISA_SSE4A, CODE_FOR_sse4a_vmmovntv2df, "__builtin_ia32_movntsd", IX86_BUILTIN_MOVNTSD, UNKNOWN, (int) VOID_FTYPE_PDOUBLE_V2DF },
25990 { OPTION_MASK_ISA_SSE4A, CODE_FOR_sse4a_vmmovntv4sf, "__builtin_ia32_movntss", IX86_BUILTIN_MOVNTSS, UNKNOWN, (int) VOID_FTYPE_PFLOAT_V4SF },
25992 /* AVX */
25993 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vzeroall, "__builtin_ia32_vzeroall", IX86_BUILTIN_VZEROALL, UNKNOWN, (int) VOID_FTYPE_VOID },
25994 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vzeroupper, "__builtin_ia32_vzeroupper", IX86_BUILTIN_VZEROUPPER, UNKNOWN, (int) VOID_FTYPE_VOID },
25996 { OPTION_MASK_ISA_AVX, CODE_FOR_vec_dupv4sf, "__builtin_ia32_vbroadcastss", IX86_BUILTIN_VBROADCASTSS, UNKNOWN, (int) V4SF_FTYPE_PCFLOAT },
25997 { OPTION_MASK_ISA_AVX, CODE_FOR_vec_dupv4df, "__builtin_ia32_vbroadcastsd256", IX86_BUILTIN_VBROADCASTSD256, UNKNOWN, (int) V4DF_FTYPE_PCDOUBLE },
25998 { OPTION_MASK_ISA_AVX, CODE_FOR_vec_dupv8sf, "__builtin_ia32_vbroadcastss256", IX86_BUILTIN_VBROADCASTSS256, UNKNOWN, (int) V8SF_FTYPE_PCFLOAT },
25999 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vbroadcastf128_v4df, "__builtin_ia32_vbroadcastf128_pd256", IX86_BUILTIN_VBROADCASTPD256, UNKNOWN, (int) V4DF_FTYPE_PCV2DF },
26000 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vbroadcastf128_v8sf, "__builtin_ia32_vbroadcastf128_ps256", IX86_BUILTIN_VBROADCASTPS256, UNKNOWN, (int) V8SF_FTYPE_PCV4SF },
26002 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movupd256, "__builtin_ia32_loadupd256", IX86_BUILTIN_LOADUPD256, UNKNOWN, (int) V4DF_FTYPE_PCDOUBLE },
26003 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movups256, "__builtin_ia32_loadups256", IX86_BUILTIN_LOADUPS256, UNKNOWN, (int) V8SF_FTYPE_PCFLOAT },
26004 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movupd256, "__builtin_ia32_storeupd256", IX86_BUILTIN_STOREUPD256, UNKNOWN, (int) VOID_FTYPE_PDOUBLE_V4DF },
26005 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movups256, "__builtin_ia32_storeups256", IX86_BUILTIN_STOREUPS256, UNKNOWN, (int) VOID_FTYPE_PFLOAT_V8SF },
26006 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movdqu256, "__builtin_ia32_loaddqu256", IX86_BUILTIN_LOADDQU256, UNKNOWN, (int) V32QI_FTYPE_PCCHAR },
26007 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movdqu256, "__builtin_ia32_storedqu256", IX86_BUILTIN_STOREDQU256, UNKNOWN, (int) VOID_FTYPE_PCHAR_V32QI },
26008 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_lddqu256, "__builtin_ia32_lddqu256", IX86_BUILTIN_LDDQU256, UNKNOWN, (int) V32QI_FTYPE_PCCHAR },
26010 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movntv4di, "__builtin_ia32_movntdq256", IX86_BUILTIN_MOVNTDQ256, UNKNOWN, (int) VOID_FTYPE_PV4DI_V4DI },
26011 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movntv4df, "__builtin_ia32_movntpd256", IX86_BUILTIN_MOVNTPD256, UNKNOWN, (int) VOID_FTYPE_PDOUBLE_V4DF },
26012 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movntv8sf, "__builtin_ia32_movntps256", IX86_BUILTIN_MOVNTPS256, UNKNOWN, (int) VOID_FTYPE_PFLOAT_V8SF },
26014 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskloadpd, "__builtin_ia32_maskloadpd", IX86_BUILTIN_MASKLOADPD, UNKNOWN, (int) V2DF_FTYPE_PCV2DF_V2DI },
26015 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskloadps, "__builtin_ia32_maskloadps", IX86_BUILTIN_MASKLOADPS, UNKNOWN, (int) V4SF_FTYPE_PCV4SF_V4SI },
26016 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskloadpd256, "__builtin_ia32_maskloadpd256", IX86_BUILTIN_MASKLOADPD256, UNKNOWN, (int) V4DF_FTYPE_PCV4DF_V4DI },
26017 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskloadps256, "__builtin_ia32_maskloadps256", IX86_BUILTIN_MASKLOADPS256, UNKNOWN, (int) V8SF_FTYPE_PCV8SF_V8SI },
26018 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskstorepd, "__builtin_ia32_maskstorepd", IX86_BUILTIN_MASKSTOREPD, UNKNOWN, (int) VOID_FTYPE_PV2DF_V2DI_V2DF },
26019 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskstoreps, "__builtin_ia32_maskstoreps", IX86_BUILTIN_MASKSTOREPS, UNKNOWN, (int) VOID_FTYPE_PV4SF_V4SI_V4SF },
26020 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskstorepd256, "__builtin_ia32_maskstorepd256", IX86_BUILTIN_MASKSTOREPD256, UNKNOWN, (int) VOID_FTYPE_PV4DF_V4DI_V4DF },
26021 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskstoreps256, "__builtin_ia32_maskstoreps256", IX86_BUILTIN_MASKSTOREPS256, UNKNOWN, (int) VOID_FTYPE_PV8SF_V8SI_V8SF },
26023 /* AVX2 */
26024 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_movntdqa, "__builtin_ia32_movntdqa256", IX86_BUILTIN_MOVNTDQA256, UNKNOWN, (int) V4DI_FTYPE_PV4DI },
26025 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_maskloadd, "__builtin_ia32_maskloadd", IX86_BUILTIN_MASKLOADD, UNKNOWN, (int) V4SI_FTYPE_PCV4SI_V4SI },
26026 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_maskloadq, "__builtin_ia32_maskloadq", IX86_BUILTIN_MASKLOADQ, UNKNOWN, (int) V2DI_FTYPE_PCV2DI_V2DI },
26027 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_maskloadd256, "__builtin_ia32_maskloadd256", IX86_BUILTIN_MASKLOADD256, UNKNOWN, (int) V8SI_FTYPE_PCV8SI_V8SI },
26028 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_maskloadq256, "__builtin_ia32_maskloadq256", IX86_BUILTIN_MASKLOADQ256, UNKNOWN, (int) V4DI_FTYPE_PCV4DI_V4DI },
26029 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_maskstored, "__builtin_ia32_maskstored", IX86_BUILTIN_MASKSTORED, UNKNOWN, (int) VOID_FTYPE_PV4SI_V4SI_V4SI },
26030 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_maskstoreq, "__builtin_ia32_maskstoreq", IX86_BUILTIN_MASKSTOREQ, UNKNOWN, (int) VOID_FTYPE_PV2DI_V2DI_V2DI },
26031 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_maskstored256, "__builtin_ia32_maskstored256", IX86_BUILTIN_MASKSTORED256, UNKNOWN, (int) VOID_FTYPE_PV8SI_V8SI_V8SI },
26032 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_maskstoreq256, "__builtin_ia32_maskstoreq256", IX86_BUILTIN_MASKSTOREQ256, UNKNOWN, (int) VOID_FTYPE_PV4DI_V4DI_V4DI },
26034 { OPTION_MASK_ISA_LWP, CODE_FOR_lwp_llwpcb, "__builtin_ia32_llwpcb", IX86_BUILTIN_LLWPCB, UNKNOWN, (int) VOID_FTYPE_PVOID },
26035 { OPTION_MASK_ISA_LWP, CODE_FOR_lwp_slwpcb, "__builtin_ia32_slwpcb", IX86_BUILTIN_SLWPCB, UNKNOWN, (int) PVOID_FTYPE_VOID },
26036 { OPTION_MASK_ISA_LWP, CODE_FOR_lwp_lwpvalsi3, "__builtin_ia32_lwpval32", IX86_BUILTIN_LWPVAL32, UNKNOWN, (int) VOID_FTYPE_UINT_UINT_UINT },
26037 { OPTION_MASK_ISA_LWP, CODE_FOR_lwp_lwpvaldi3, "__builtin_ia32_lwpval64", IX86_BUILTIN_LWPVAL64, UNKNOWN, (int) VOID_FTYPE_UINT64_UINT_UINT },
26038 { OPTION_MASK_ISA_LWP, CODE_FOR_lwp_lwpinssi3, "__builtin_ia32_lwpins32", IX86_BUILTIN_LWPINS32, UNKNOWN, (int) UCHAR_FTYPE_UINT_UINT_UINT },
26039 { OPTION_MASK_ISA_LWP, CODE_FOR_lwp_lwpinsdi3, "__builtin_ia32_lwpins64", IX86_BUILTIN_LWPINS64, UNKNOWN, (int) UCHAR_FTYPE_UINT64_UINT_UINT },
26041 /* FSGSBASE */
26042 { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_rdfsbasesi, "__builtin_ia32_rdfsbase32", IX86_BUILTIN_RDFSBASE32, UNKNOWN, (int) UNSIGNED_FTYPE_VOID },
26043 { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_rdfsbasedi, "__builtin_ia32_rdfsbase64", IX86_BUILTIN_RDFSBASE64, UNKNOWN, (int) UINT64_FTYPE_VOID },
26044 { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_rdgsbasesi, "__builtin_ia32_rdgsbase32", IX86_BUILTIN_RDGSBASE32, UNKNOWN, (int) UNSIGNED_FTYPE_VOID },
26045 { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_rdgsbasedi, "__builtin_ia32_rdgsbase64", IX86_BUILTIN_RDGSBASE64, UNKNOWN, (int) UINT64_FTYPE_VOID },
26046 { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_wrfsbasesi, "__builtin_ia32_wrfsbase32", IX86_BUILTIN_WRFSBASE32, UNKNOWN, (int) VOID_FTYPE_UNSIGNED },
26047 { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_wrfsbasedi, "__builtin_ia32_wrfsbase64", IX86_BUILTIN_WRFSBASE64, UNKNOWN, (int) VOID_FTYPE_UINT64 },
26048 { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_wrgsbasesi, "__builtin_ia32_wrgsbase32", IX86_BUILTIN_WRGSBASE32, UNKNOWN, (int) VOID_FTYPE_UNSIGNED },
26049 { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_wrgsbasedi, "__builtin_ia32_wrgsbase64", IX86_BUILTIN_WRGSBASE64, UNKNOWN, (int) VOID_FTYPE_UINT64 },
26051 /* RTM */
26052 { OPTION_MASK_ISA_RTM, CODE_FOR_xbegin, "__builtin_ia32_xbegin", IX86_BUILTIN_XBEGIN, UNKNOWN, (int) UNSIGNED_FTYPE_VOID },
26053 { OPTION_MASK_ISA_RTM, CODE_FOR_xend, "__builtin_ia32_xend", IX86_BUILTIN_XEND, UNKNOWN, (int) VOID_FTYPE_VOID },
26054 { OPTION_MASK_ISA_RTM, CODE_FOR_xtest, "__builtin_ia32_xtest", IX86_BUILTIN_XTEST, UNKNOWN, (int) INT_FTYPE_VOID },
26055 };
26057 /* Builtins with variable number of arguments. */
26059 {
26060 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_bsr, "__builtin_ia32_bsrsi", IX86_BUILTIN_BSRSI, UNKNOWN, (int) INT_FTYPE_INT },
26061 { OPTION_MASK_ISA_64BIT, CODE_FOR_bsr_rex64, "__builtin_ia32_bsrdi", IX86_BUILTIN_BSRDI, UNKNOWN, (int) INT64_FTYPE_INT64 },
26062 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_rdpmc, "__builtin_ia32_rdpmc", IX86_BUILTIN_RDPMC, UNKNOWN, (int) UINT64_FTYPE_INT },
26063 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_rotlqi3, "__builtin_ia32_rolqi", IX86_BUILTIN_ROLQI, UNKNOWN, (int) UINT8_FTYPE_UINT8_INT },
26064 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_rotlhi3, "__builtin_ia32_rolhi", IX86_BUILTIN_ROLHI, UNKNOWN, (int) UINT16_FTYPE_UINT16_INT },
26065 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_rotrqi3, "__builtin_ia32_rorqi", IX86_BUILTIN_RORQI, UNKNOWN, (int) UINT8_FTYPE_UINT8_INT },
26066 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_rotrhi3, "__builtin_ia32_rorhi", IX86_BUILTIN_RORHI, UNKNOWN, (int) UINT16_FTYPE_UINT16_INT },
26068 /* MMX */
26069 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_addv8qi3, "__builtin_ia32_paddb", IX86_BUILTIN_PADDB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
26070 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_addv4hi3, "__builtin_ia32_paddw", IX86_BUILTIN_PADDW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
26071 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_addv2si3, "__builtin_ia32_paddd", IX86_BUILTIN_PADDD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
26072 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_subv8qi3, "__builtin_ia32_psubb", IX86_BUILTIN_PSUBB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
26073 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_subv4hi3, "__builtin_ia32_psubw", IX86_BUILTIN_PSUBW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
26074 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_subv2si3, "__builtin_ia32_psubd", IX86_BUILTIN_PSUBD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
26076 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ssaddv8qi3, "__builtin_ia32_paddsb", IX86_BUILTIN_PADDSB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
26077 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ssaddv4hi3, "__builtin_ia32_paddsw", IX86_BUILTIN_PADDSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
26078 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_sssubv8qi3, "__builtin_ia32_psubsb", IX86_BUILTIN_PSUBSB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
26079 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_sssubv4hi3, "__builtin_ia32_psubsw", IX86_BUILTIN_PSUBSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
26080 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_usaddv8qi3, "__builtin_ia32_paddusb", IX86_BUILTIN_PADDUSB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
26081 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_usaddv4hi3, "__builtin_ia32_paddusw", IX86_BUILTIN_PADDUSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
26082 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ussubv8qi3, "__builtin_ia32_psubusb", IX86_BUILTIN_PSUBUSB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
26083 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ussubv4hi3, "__builtin_ia32_psubusw", IX86_BUILTIN_PSUBUSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
26085 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_mulv4hi3, "__builtin_ia32_pmullw", IX86_BUILTIN_PMULLW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
26086 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_smulv4hi3_highpart, "__builtin_ia32_pmulhw", IX86_BUILTIN_PMULHW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
26088 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_andv2si3, "__builtin_ia32_pand", IX86_BUILTIN_PAND, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
26089 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_andnotv2si3, "__builtin_ia32_pandn", IX86_BUILTIN_PANDN, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
26090 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_iorv2si3, "__builtin_ia32_por", IX86_BUILTIN_POR, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
26091 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_xorv2si3, "__builtin_ia32_pxor", IX86_BUILTIN_PXOR, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
26093 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_eqv8qi3, "__builtin_ia32_pcmpeqb", IX86_BUILTIN_PCMPEQB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
26094 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_eqv4hi3, "__builtin_ia32_pcmpeqw", IX86_BUILTIN_PCMPEQW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
26095 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_eqv2si3, "__builtin_ia32_pcmpeqd", IX86_BUILTIN_PCMPEQD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
26096 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_gtv8qi3, "__builtin_ia32_pcmpgtb", IX86_BUILTIN_PCMPGTB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
26097 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_gtv4hi3, "__builtin_ia32_pcmpgtw", IX86_BUILTIN_PCMPGTW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
26098 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_gtv2si3, "__builtin_ia32_pcmpgtd", IX86_BUILTIN_PCMPGTD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
26100 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_punpckhbw, "__builtin_ia32_punpckhbw", IX86_BUILTIN_PUNPCKHBW, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
26101 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_punpckhwd, "__builtin_ia32_punpckhwd", IX86_BUILTIN_PUNPCKHWD, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
26102 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_punpckhdq, "__builtin_ia32_punpckhdq", IX86_BUILTIN_PUNPCKHDQ, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
26103 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_punpcklbw, "__builtin_ia32_punpcklbw", IX86_BUILTIN_PUNPCKLBW, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
26104 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_punpcklwd, "__builtin_ia32_punpcklwd", IX86_BUILTIN_PUNPCKLWD, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI},
26105 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_punpckldq, "__builtin_ia32_punpckldq", IX86_BUILTIN_PUNPCKLDQ, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI},
26107 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_packsswb, "__builtin_ia32_packsswb", IX86_BUILTIN_PACKSSWB, UNKNOWN, (int) V8QI_FTYPE_V4HI_V4HI },
26108 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_packssdw, "__builtin_ia32_packssdw", IX86_BUILTIN_PACKSSDW, UNKNOWN, (int) V4HI_FTYPE_V2SI_V2SI },
26109 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_packuswb, "__builtin_ia32_packuswb", IX86_BUILTIN_PACKUSWB, UNKNOWN, (int) V8QI_FTYPE_V4HI_V4HI },
26111 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_pmaddwd, "__builtin_ia32_pmaddwd", IX86_BUILTIN_PMADDWD, UNKNOWN, (int) V2SI_FTYPE_V4HI_V4HI },
26113 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashlv4hi3, "__builtin_ia32_psllwi", IX86_BUILTIN_PSLLWI, UNKNOWN, (int) V4HI_FTYPE_V4HI_SI_COUNT },
26114 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashlv2si3, "__builtin_ia32_pslldi", IX86_BUILTIN_PSLLDI, UNKNOWN, (int) V2SI_FTYPE_V2SI_SI_COUNT },
26115 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashlv1di3, "__builtin_ia32_psllqi", IX86_BUILTIN_PSLLQI, UNKNOWN, (int) V1DI_FTYPE_V1DI_SI_COUNT },
26116 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashlv4hi3, "__builtin_ia32_psllw", IX86_BUILTIN_PSLLW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI_COUNT },
26117 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashlv2si3, "__builtin_ia32_pslld", IX86_BUILTIN_PSLLD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI_COUNT },
26118 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashlv1di3, "__builtin_ia32_psllq", IX86_BUILTIN_PSLLQ, UNKNOWN, (int) V1DI_FTYPE_V1DI_V1DI_COUNT },
26120 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_lshrv4hi3, "__builtin_ia32_psrlwi", IX86_BUILTIN_PSRLWI, UNKNOWN, (int) V4HI_FTYPE_V4HI_SI_COUNT },
26121 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_lshrv2si3, "__builtin_ia32_psrldi", IX86_BUILTIN_PSRLDI, UNKNOWN, (int) V2SI_FTYPE_V2SI_SI_COUNT },
26122 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_lshrv1di3, "__builtin_ia32_psrlqi", IX86_BUILTIN_PSRLQI, UNKNOWN, (int) V1DI_FTYPE_V1DI_SI_COUNT },
26123 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_lshrv4hi3, "__builtin_ia32_psrlw", IX86_BUILTIN_PSRLW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI_COUNT },
26124 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_lshrv2si3, "__builtin_ia32_psrld", IX86_BUILTIN_PSRLD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI_COUNT },
26125 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_lshrv1di3, "__builtin_ia32_psrlq", IX86_BUILTIN_PSRLQ, UNKNOWN, (int) V1DI_FTYPE_V1DI_V1DI_COUNT },
26127 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashrv4hi3, "__builtin_ia32_psrawi", IX86_BUILTIN_PSRAWI, UNKNOWN, (int) V4HI_FTYPE_V4HI_SI_COUNT },
26128 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashrv2si3, "__builtin_ia32_psradi", IX86_BUILTIN_PSRADI, UNKNOWN, (int) V2SI_FTYPE_V2SI_SI_COUNT },
26129 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashrv4hi3, "__builtin_ia32_psraw", IX86_BUILTIN_PSRAW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI_COUNT },
26130 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashrv2si3, "__builtin_ia32_psrad", IX86_BUILTIN_PSRAD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI_COUNT },
26132 /* 3DNow! */
26133 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_pf2id, "__builtin_ia32_pf2id", IX86_BUILTIN_PF2ID, UNKNOWN, (int) V2SI_FTYPE_V2SF },
26134 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_floatv2si2, "__builtin_ia32_pi2fd", IX86_BUILTIN_PI2FD, UNKNOWN, (int) V2SF_FTYPE_V2SI },
26135 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_rcpv2sf2, "__builtin_ia32_pfrcp", IX86_BUILTIN_PFRCP, UNKNOWN, (int) V2SF_FTYPE_V2SF },
26136 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_rsqrtv2sf2, "__builtin_ia32_pfrsqrt", IX86_BUILTIN_PFRSQRT, UNKNOWN, (int) V2SF_FTYPE_V2SF },
26138 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_uavgv8qi3, "__builtin_ia32_pavgusb", IX86_BUILTIN_PAVGUSB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
26139 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_haddv2sf3, "__builtin_ia32_pfacc", IX86_BUILTIN_PFACC, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
26140 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_addv2sf3, "__builtin_ia32_pfadd", IX86_BUILTIN_PFADD, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
26141 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_eqv2sf3, "__builtin_ia32_pfcmpeq", IX86_BUILTIN_PFCMPEQ, UNKNOWN, (int) V2SI_FTYPE_V2SF_V2SF },
26142 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_gev2sf3, "__builtin_ia32_pfcmpge", IX86_BUILTIN_PFCMPGE, UNKNOWN, (int) V2SI_FTYPE_V2SF_V2SF },
26143 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_gtv2sf3, "__builtin_ia32_pfcmpgt", IX86_BUILTIN_PFCMPGT, UNKNOWN, (int) V2SI_FTYPE_V2SF_V2SF },
26144 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_smaxv2sf3, "__builtin_ia32_pfmax", IX86_BUILTIN_PFMAX, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
26145 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_sminv2sf3, "__builtin_ia32_pfmin", IX86_BUILTIN_PFMIN, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
26146 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_mulv2sf3, "__builtin_ia32_pfmul", IX86_BUILTIN_PFMUL, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
26147 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_rcpit1v2sf3, "__builtin_ia32_pfrcpit1", IX86_BUILTIN_PFRCPIT1, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
26148 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_rcpit2v2sf3, "__builtin_ia32_pfrcpit2", IX86_BUILTIN_PFRCPIT2, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
26149 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_rsqit1v2sf3, "__builtin_ia32_pfrsqit1", IX86_BUILTIN_PFRSQIT1, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
26150 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_subv2sf3, "__builtin_ia32_pfsub", IX86_BUILTIN_PFSUB, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
26151 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_subrv2sf3, "__builtin_ia32_pfsubr", IX86_BUILTIN_PFSUBR, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
26152 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_pmulhrwv4hi3, "__builtin_ia32_pmulhrw", IX86_BUILTIN_PMULHRW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
26154 /* 3DNow!A */
26155 { OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_pf2iw, "__builtin_ia32_pf2iw", IX86_BUILTIN_PF2IW, UNKNOWN, (int) V2SI_FTYPE_V2SF },
26156 { OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_pi2fw, "__builtin_ia32_pi2fw", IX86_BUILTIN_PI2FW, UNKNOWN, (int) V2SF_FTYPE_V2SI },
26157 { OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_pswapdv2si2, "__builtin_ia32_pswapdsi", IX86_BUILTIN_PSWAPDSI, UNKNOWN, (int) V2SI_FTYPE_V2SI },
26158 { OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_pswapdv2sf2, "__builtin_ia32_pswapdsf", IX86_BUILTIN_PSWAPDSF, UNKNOWN, (int) V2SF_FTYPE_V2SF },
26159 { OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_hsubv2sf3, "__builtin_ia32_pfnacc", IX86_BUILTIN_PFNACC, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
26160 { OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_addsubv2sf3, "__builtin_ia32_pfpnacc", IX86_BUILTIN_PFPNACC, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
26162 /* SSE */
26163 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_movmskps, "__builtin_ia32_movmskps", IX86_BUILTIN_MOVMSKPS, UNKNOWN, (int) INT_FTYPE_V4SF },
26164 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_sqrtv4sf2, "__builtin_ia32_sqrtps", IX86_BUILTIN_SQRTPS, UNKNOWN, (int) V4SF_FTYPE_V4SF },
26165 { OPTION_MASK_ISA_SSE, CODE_FOR_sqrtv4sf2, "__builtin_ia32_sqrtps_nr", IX86_BUILTIN_SQRTPS_NR, UNKNOWN, (int) V4SF_FTYPE_V4SF },
26166 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_rsqrtv4sf2, "__builtin_ia32_rsqrtps", IX86_BUILTIN_RSQRTPS, UNKNOWN, (int) V4SF_FTYPE_V4SF },
26167 { OPTION_MASK_ISA_SSE, CODE_FOR_rsqrtv4sf2, "__builtin_ia32_rsqrtps_nr", IX86_BUILTIN_RSQRTPS_NR, UNKNOWN, (int) V4SF_FTYPE_V4SF },
26168 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_rcpv4sf2, "__builtin_ia32_rcpps", IX86_BUILTIN_RCPPS, UNKNOWN, (int) V4SF_FTYPE_V4SF },
26169 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_cvtps2pi, "__builtin_ia32_cvtps2pi", IX86_BUILTIN_CVTPS2PI, UNKNOWN, (int) V2SI_FTYPE_V4SF },
26170 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_cvtss2si, "__builtin_ia32_cvtss2si", IX86_BUILTIN_CVTSS2SI, UNKNOWN, (int) INT_FTYPE_V4SF },
26171 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_64BIT, CODE_FOR_sse_cvtss2siq, "__builtin_ia32_cvtss2si64", IX86_BUILTIN_CVTSS2SI64, UNKNOWN, (int) INT64_FTYPE_V4SF },
26172 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_cvttps2pi, "__builtin_ia32_cvttps2pi", IX86_BUILTIN_CVTTPS2PI, UNKNOWN, (int) V2SI_FTYPE_V4SF },
26173 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_cvttss2si, "__builtin_ia32_cvttss2si", IX86_BUILTIN_CVTTSS2SI, UNKNOWN, (int) INT_FTYPE_V4SF },
26174 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_64BIT, CODE_FOR_sse_cvttss2siq, "__builtin_ia32_cvttss2si64", IX86_BUILTIN_CVTTSS2SI64, UNKNOWN, (int) INT64_FTYPE_V4SF },
26176 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_shufps, "__builtin_ia32_shufps", IX86_BUILTIN_SHUFPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
26178 { OPTION_MASK_ISA_SSE, CODE_FOR_addv4sf3, "__builtin_ia32_addps", IX86_BUILTIN_ADDPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
26179 { OPTION_MASK_ISA_SSE, CODE_FOR_subv4sf3, "__builtin_ia32_subps", IX86_BUILTIN_SUBPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
26180 { OPTION_MASK_ISA_SSE, CODE_FOR_mulv4sf3, "__builtin_ia32_mulps", IX86_BUILTIN_MULPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
26181 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_divv4sf3, "__builtin_ia32_divps", IX86_BUILTIN_DIVPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
26182 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmaddv4sf3, "__builtin_ia32_addss", IX86_BUILTIN_ADDSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
26183 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmsubv4sf3, "__builtin_ia32_subss", IX86_BUILTIN_SUBSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
26184 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmulv4sf3, "__builtin_ia32_mulss", IX86_BUILTIN_MULSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
26185 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmdivv4sf3, "__builtin_ia32_divss", IX86_BUILTIN_DIVSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
26187 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpeqps", IX86_BUILTIN_CMPEQPS, EQ, (int) V4SF_FTYPE_V4SF_V4SF },
26188 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpltps", IX86_BUILTIN_CMPLTPS, LT, (int) V4SF_FTYPE_V4SF_V4SF },
26189 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpleps", IX86_BUILTIN_CMPLEPS, LE, (int) V4SF_FTYPE_V4SF_V4SF },
26190 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpgtps", IX86_BUILTIN_CMPGTPS, LT, (int) V4SF_FTYPE_V4SF_V4SF_SWAP },
26191 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpgeps", IX86_BUILTIN_CMPGEPS, LE, (int) V4SF_FTYPE_V4SF_V4SF_SWAP },
26192 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpunordps", IX86_BUILTIN_CMPUNORDPS, UNORDERED, (int) V4SF_FTYPE_V4SF_V4SF },
26193 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpneqps", IX86_BUILTIN_CMPNEQPS, NE, (int) V4SF_FTYPE_V4SF_V4SF },
26194 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpnltps", IX86_BUILTIN_CMPNLTPS, UNGE, (int) V4SF_FTYPE_V4SF_V4SF },
26195 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpnleps", IX86_BUILTIN_CMPNLEPS, UNGT, (int) V4SF_FTYPE_V4SF_V4SF },
26196 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpngtps", IX86_BUILTIN_CMPNGTPS, UNGE, (int) V4SF_FTYPE_V4SF_V4SF_SWAP },
26197 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpngeps", IX86_BUILTIN_CMPNGEPS, UNGT, (int) V4SF_FTYPE_V4SF_V4SF_SWAP},
26198 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpordps", IX86_BUILTIN_CMPORDPS, ORDERED, (int) V4SF_FTYPE_V4SF_V4SF },
26199 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpeqss", IX86_BUILTIN_CMPEQSS, EQ, (int) V4SF_FTYPE_V4SF_V4SF },
26200 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpltss", IX86_BUILTIN_CMPLTSS, LT, (int) V4SF_FTYPE_V4SF_V4SF },
26201 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpless", IX86_BUILTIN_CMPLESS, LE, (int) V4SF_FTYPE_V4SF_V4SF },
26202 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpunordss", IX86_BUILTIN_CMPUNORDSS, UNORDERED, (int) V4SF_FTYPE_V4SF_V4SF },
26203 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpneqss", IX86_BUILTIN_CMPNEQSS, NE, (int) V4SF_FTYPE_V4SF_V4SF },
26204 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpnltss", IX86_BUILTIN_CMPNLTSS, UNGE, (int) V4SF_FTYPE_V4SF_V4SF },
26205 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpnless", IX86_BUILTIN_CMPNLESS, UNGT, (int) V4SF_FTYPE_V4SF_V4SF },
26206 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpngtss", IX86_BUILTIN_CMPNGTSS, UNGE, (int) V4SF_FTYPE_V4SF_V4SF_SWAP },
26207 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpngess", IX86_BUILTIN_CMPNGESS, UNGT, (int) V4SF_FTYPE_V4SF_V4SF_SWAP },
26208 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpordss", IX86_BUILTIN_CMPORDSS, ORDERED, (int) V4SF_FTYPE_V4SF_V4SF },
26210 { OPTION_MASK_ISA_SSE, CODE_FOR_sminv4sf3, "__builtin_ia32_minps", IX86_BUILTIN_MINPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
26211 { OPTION_MASK_ISA_SSE, CODE_FOR_smaxv4sf3, "__builtin_ia32_maxps", IX86_BUILTIN_MAXPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
26212 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmsminv4sf3, "__builtin_ia32_minss", IX86_BUILTIN_MINSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
26213 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmsmaxv4sf3, "__builtin_ia32_maxss", IX86_BUILTIN_MAXSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
26215 { OPTION_MASK_ISA_SSE, CODE_FOR_andv4sf3, "__builtin_ia32_andps", IX86_BUILTIN_ANDPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
26216 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_andnotv4sf3, "__builtin_ia32_andnps", IX86_BUILTIN_ANDNPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
26217 { OPTION_MASK_ISA_SSE, CODE_FOR_iorv4sf3, "__builtin_ia32_orps", IX86_BUILTIN_ORPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
26218 { OPTION_MASK_ISA_SSE, CODE_FOR_xorv4sf3, "__builtin_ia32_xorps", IX86_BUILTIN_XORPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
26220 { OPTION_MASK_ISA_SSE, CODE_FOR_copysignv4sf3, "__builtin_ia32_copysignps", IX86_BUILTIN_CPYSGNPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
26222 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_movss, "__builtin_ia32_movss", IX86_BUILTIN_MOVSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
26223 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_movhlps_exp, "__builtin_ia32_movhlps", IX86_BUILTIN_MOVHLPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
26224 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_movlhps_exp, "__builtin_ia32_movlhps", IX86_BUILTIN_MOVLHPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
26225 { OPTION_MASK_ISA_SSE, CODE_FOR_vec_interleave_highv4sf, "__builtin_ia32_unpckhps", IX86_BUILTIN_UNPCKHPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
26226 { OPTION_MASK_ISA_SSE, CODE_FOR_vec_interleave_lowv4sf, "__builtin_ia32_unpcklps", IX86_BUILTIN_UNPCKLPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
26228 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_cvtpi2ps, "__builtin_ia32_cvtpi2ps", IX86_BUILTIN_CVTPI2PS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V2SI },
26229 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_cvtsi2ss, "__builtin_ia32_cvtsi2ss", IX86_BUILTIN_CVTSI2SS, UNKNOWN, (int) V4SF_FTYPE_V4SF_SI },
26230 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_64BIT, CODE_FOR_sse_cvtsi2ssq, "__builtin_ia32_cvtsi642ss", IX86_BUILTIN_CVTSI642SS, UNKNOWN, V4SF_FTYPE_V4SF_DI },
26232 { OPTION_MASK_ISA_SSE, CODE_FOR_rsqrtsf2, "__builtin_ia32_rsqrtf", IX86_BUILTIN_RSQRTF, UNKNOWN, (int) FLOAT_FTYPE_FLOAT },
26234 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmsqrtv4sf2, "__builtin_ia32_sqrtss", IX86_BUILTIN_SQRTSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_VEC_MERGE },
26235 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmrsqrtv4sf2, "__builtin_ia32_rsqrtss", IX86_BUILTIN_RSQRTSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_VEC_MERGE },
26236 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmrcpv4sf2, "__builtin_ia32_rcpss", IX86_BUILTIN_RCPSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_VEC_MERGE },
26238 /* SSE MMX or 3Dnow!A */
26239 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_uavgv8qi3, "__builtin_ia32_pavgb", IX86_BUILTIN_PAVGB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
26240 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_uavgv4hi3, "__builtin_ia32_pavgw", IX86_BUILTIN_PAVGW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
26241 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_umulv4hi3_highpart, "__builtin_ia32_pmulhuw", IX86_BUILTIN_PMULHUW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
26243 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_umaxv8qi3, "__builtin_ia32_pmaxub", IX86_BUILTIN_PMAXUB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
26244 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_smaxv4hi3, "__builtin_ia32_pmaxsw", IX86_BUILTIN_PMAXSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
26245 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_uminv8qi3, "__builtin_ia32_pminub", IX86_BUILTIN_PMINUB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
26246 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_sminv4hi3, "__builtin_ia32_pminsw", IX86_BUILTIN_PMINSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
26248 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_psadbw, "__builtin_ia32_psadbw", IX86_BUILTIN_PSADBW, UNKNOWN, (int) V1DI_FTYPE_V8QI_V8QI },
26249 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_pmovmskb, "__builtin_ia32_pmovmskb", IX86_BUILTIN_PMOVMSKB, UNKNOWN, (int) INT_FTYPE_V8QI },
26251 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_pshufw, "__builtin_ia32_pshufw", IX86_BUILTIN_PSHUFW, UNKNOWN, (int) V4HI_FTYPE_V4HI_INT },
26253 /* SSE2 */
26254 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_shufpd, "__builtin_ia32_shufpd", IX86_BUILTIN_SHUFPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
26256 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_movmskpd, "__builtin_ia32_movmskpd", IX86_BUILTIN_MOVMSKPD, UNKNOWN, (int) INT_FTYPE_V2DF },
26257 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_pmovmskb, "__builtin_ia32_pmovmskb128", IX86_BUILTIN_PMOVMSKB128, UNKNOWN, (int) INT_FTYPE_V16QI },
26258 { OPTION_MASK_ISA_SSE2, CODE_FOR_sqrtv2df2, "__builtin_ia32_sqrtpd", IX86_BUILTIN_SQRTPD, UNKNOWN, (int) V2DF_FTYPE_V2DF },
26259 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtdq2pd, "__builtin_ia32_cvtdq2pd", IX86_BUILTIN_CVTDQ2PD, UNKNOWN, (int) V2DF_FTYPE_V4SI },
26260 { OPTION_MASK_ISA_SSE2, CODE_FOR_floatv4siv4sf2, "__builtin_ia32_cvtdq2ps", IX86_BUILTIN_CVTDQ2PS, UNKNOWN, (int) V4SF_FTYPE_V4SI },
26262 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtpd2dq, "__builtin_ia32_cvtpd2dq", IX86_BUILTIN_CVTPD2DQ, UNKNOWN, (int) V4SI_FTYPE_V2DF },
26263 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtpd2pi, "__builtin_ia32_cvtpd2pi", IX86_BUILTIN_CVTPD2PI, UNKNOWN, (int) V2SI_FTYPE_V2DF },
26264 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtpd2ps, "__builtin_ia32_cvtpd2ps", IX86_BUILTIN_CVTPD2PS, UNKNOWN, (int) V4SF_FTYPE_V2DF },
26265 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvttpd2dq, "__builtin_ia32_cvttpd2dq", IX86_BUILTIN_CVTTPD2DQ, UNKNOWN, (int) V4SI_FTYPE_V2DF },
26266 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvttpd2pi, "__builtin_ia32_cvttpd2pi", IX86_BUILTIN_CVTTPD2PI, UNKNOWN, (int) V2SI_FTYPE_V2DF },
26268 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtpi2pd, "__builtin_ia32_cvtpi2pd", IX86_BUILTIN_CVTPI2PD, UNKNOWN, (int) V2DF_FTYPE_V2SI },
26270 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtsd2si, "__builtin_ia32_cvtsd2si", IX86_BUILTIN_CVTSD2SI, UNKNOWN, (int) INT_FTYPE_V2DF },
26271 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvttsd2si, "__builtin_ia32_cvttsd2si", IX86_BUILTIN_CVTTSD2SI, UNKNOWN, (int) INT_FTYPE_V2DF },
26272 { OPTION_MASK_ISA_SSE2 | OPTION_MASK_ISA_64BIT, CODE_FOR_sse2_cvtsd2siq, "__builtin_ia32_cvtsd2si64", IX86_BUILTIN_CVTSD2SI64, UNKNOWN, (int) INT64_FTYPE_V2DF },
26273 { OPTION_MASK_ISA_SSE2 | OPTION_MASK_ISA_64BIT, CODE_FOR_sse2_cvttsd2siq, "__builtin_ia32_cvttsd2si64", IX86_BUILTIN_CVTTSD2SI64, UNKNOWN, (int) INT64_FTYPE_V2DF },
26275 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtps2dq, "__builtin_ia32_cvtps2dq", IX86_BUILTIN_CVTPS2DQ, UNKNOWN, (int) V4SI_FTYPE_V4SF },
26276 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtps2pd, "__builtin_ia32_cvtps2pd", IX86_BUILTIN_CVTPS2PD, UNKNOWN, (int) V2DF_FTYPE_V4SF },
26277 { OPTION_MASK_ISA_SSE2, CODE_FOR_fix_truncv4sfv4si2, "__builtin_ia32_cvttps2dq", IX86_BUILTIN_CVTTPS2DQ, UNKNOWN, (int) V4SI_FTYPE_V4SF },
26279 { OPTION_MASK_ISA_SSE2, CODE_FOR_addv2df3, "__builtin_ia32_addpd", IX86_BUILTIN_ADDPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
26280 { OPTION_MASK_ISA_SSE2, CODE_FOR_subv2df3, "__builtin_ia32_subpd", IX86_BUILTIN_SUBPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
26281 { OPTION_MASK_ISA_SSE2, CODE_FOR_mulv2df3, "__builtin_ia32_mulpd", IX86_BUILTIN_MULPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
26282 { OPTION_MASK_ISA_SSE2, CODE_FOR_divv2df3, "__builtin_ia32_divpd", IX86_BUILTIN_DIVPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
26283 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmaddv2df3, "__builtin_ia32_addsd", IX86_BUILTIN_ADDSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
26284 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmsubv2df3, "__builtin_ia32_subsd", IX86_BUILTIN_SUBSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
26285 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmulv2df3, "__builtin_ia32_mulsd", IX86_BUILTIN_MULSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
26286 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmdivv2df3, "__builtin_ia32_divsd", IX86_BUILTIN_DIVSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
26288 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpeqpd", IX86_BUILTIN_CMPEQPD, EQ, (int) V2DF_FTYPE_V2DF_V2DF },
26289 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpltpd", IX86_BUILTIN_CMPLTPD, LT, (int) V2DF_FTYPE_V2DF_V2DF },
26290 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmplepd", IX86_BUILTIN_CMPLEPD, LE, (int) V2DF_FTYPE_V2DF_V2DF },
26291 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpgtpd", IX86_BUILTIN_CMPGTPD, LT, (int) V2DF_FTYPE_V2DF_V2DF_SWAP },
26292 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpgepd", IX86_BUILTIN_CMPGEPD, LE, (int) V2DF_FTYPE_V2DF_V2DF_SWAP},
26293 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpunordpd", IX86_BUILTIN_CMPUNORDPD, UNORDERED, (int) V2DF_FTYPE_V2DF_V2DF },
26294 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpneqpd", IX86_BUILTIN_CMPNEQPD, NE, (int) V2DF_FTYPE_V2DF_V2DF },
26295 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpnltpd", IX86_BUILTIN_CMPNLTPD, UNGE, (int) V2DF_FTYPE_V2DF_V2DF },
26296 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpnlepd", IX86_BUILTIN_CMPNLEPD, UNGT, (int) V2DF_FTYPE_V2DF_V2DF },
26297 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpngtpd", IX86_BUILTIN_CMPNGTPD, UNGE, (int) V2DF_FTYPE_V2DF_V2DF_SWAP },
26298 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpngepd", IX86_BUILTIN_CMPNGEPD, UNGT, (int) V2DF_FTYPE_V2DF_V2DF_SWAP },
26299 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpordpd", IX86_BUILTIN_CMPORDPD, ORDERED, (int) V2DF_FTYPE_V2DF_V2DF },
26300 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmpeqsd", IX86_BUILTIN_CMPEQSD, EQ, (int) V2DF_FTYPE_V2DF_V2DF },
26301 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmpltsd", IX86_BUILTIN_CMPLTSD, LT, (int) V2DF_FTYPE_V2DF_V2DF },
26302 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmplesd", IX86_BUILTIN_CMPLESD, LE, (int) V2DF_FTYPE_V2DF_V2DF },
26303 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmpunordsd", IX86_BUILTIN_CMPUNORDSD, UNORDERED, (int) V2DF_FTYPE_V2DF_V2DF },
26304 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmpneqsd", IX86_BUILTIN_CMPNEQSD, NE, (int) V2DF_FTYPE_V2DF_V2DF },
26305 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmpnltsd", IX86_BUILTIN_CMPNLTSD, UNGE, (int) V2DF_FTYPE_V2DF_V2DF },
26306 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmpnlesd", IX86_BUILTIN_CMPNLESD, UNGT, (int) V2DF_FTYPE_V2DF_V2DF },
26307 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmpordsd", IX86_BUILTIN_CMPORDSD, ORDERED, (int) V2DF_FTYPE_V2DF_V2DF },
26309 { OPTION_MASK_ISA_SSE2, CODE_FOR_sminv2df3, "__builtin_ia32_minpd", IX86_BUILTIN_MINPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
26310 { OPTION_MASK_ISA_SSE2, CODE_FOR_smaxv2df3, "__builtin_ia32_maxpd", IX86_BUILTIN_MAXPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
26311 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmsminv2df3, "__builtin_ia32_minsd", IX86_BUILTIN_MINSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
26312 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmsmaxv2df3, "__builtin_ia32_maxsd", IX86_BUILTIN_MAXSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
26314 { OPTION_MASK_ISA_SSE2, CODE_FOR_andv2df3, "__builtin_ia32_andpd", IX86_BUILTIN_ANDPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
26315 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_andnotv2df3, "__builtin_ia32_andnpd", IX86_BUILTIN_ANDNPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
26316 { OPTION_MASK_ISA_SSE2, CODE_FOR_iorv2df3, "__builtin_ia32_orpd", IX86_BUILTIN_ORPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
26317 { OPTION_MASK_ISA_SSE2, CODE_FOR_xorv2df3, "__builtin_ia32_xorpd", IX86_BUILTIN_XORPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
26319 { OPTION_MASK_ISA_SSE2, CODE_FOR_copysignv2df3, "__builtin_ia32_copysignpd", IX86_BUILTIN_CPYSGNPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
26321 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_movsd, "__builtin_ia32_movsd", IX86_BUILTIN_MOVSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
26322 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_highv2df, "__builtin_ia32_unpckhpd", IX86_BUILTIN_UNPCKHPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
26323 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_lowv2df, "__builtin_ia32_unpcklpd", IX86_BUILTIN_UNPCKLPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
26325 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_pack_sfix_v2df, "__builtin_ia32_vec_pack_sfix", IX86_BUILTIN_VEC_PACK_SFIX, UNKNOWN, (int) V4SI_FTYPE_V2DF_V2DF },
26327 { OPTION_MASK_ISA_SSE2, CODE_FOR_addv16qi3, "__builtin_ia32_paddb128", IX86_BUILTIN_PADDB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
26328 { OPTION_MASK_ISA_SSE2, CODE_FOR_addv8hi3, "__builtin_ia32_paddw128", IX86_BUILTIN_PADDW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
26329 { OPTION_MASK_ISA_SSE2, CODE_FOR_addv4si3, "__builtin_ia32_paddd128", IX86_BUILTIN_PADDD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
26330 { OPTION_MASK_ISA_SSE2, CODE_FOR_addv2di3, "__builtin_ia32_paddq128", IX86_BUILTIN_PADDQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
26331 { OPTION_MASK_ISA_SSE2, CODE_FOR_subv16qi3, "__builtin_ia32_psubb128", IX86_BUILTIN_PSUBB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
26332 { OPTION_MASK_ISA_SSE2, CODE_FOR_subv8hi3, "__builtin_ia32_psubw128", IX86_BUILTIN_PSUBW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
26333 { OPTION_MASK_ISA_SSE2, CODE_FOR_subv4si3, "__builtin_ia32_psubd128", IX86_BUILTIN_PSUBD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
26334 { OPTION_MASK_ISA_SSE2, CODE_FOR_subv2di3, "__builtin_ia32_psubq128", IX86_BUILTIN_PSUBQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
26336 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ssaddv16qi3, "__builtin_ia32_paddsb128", IX86_BUILTIN_PADDSB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
26337 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ssaddv8hi3, "__builtin_ia32_paddsw128", IX86_BUILTIN_PADDSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
26338 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_sssubv16qi3, "__builtin_ia32_psubsb128", IX86_BUILTIN_PSUBSB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
26339 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_sssubv8hi3, "__builtin_ia32_psubsw128", IX86_BUILTIN_PSUBSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
26340 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_usaddv16qi3, "__builtin_ia32_paddusb128", IX86_BUILTIN_PADDUSB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
26341 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_usaddv8hi3, "__builtin_ia32_paddusw128", IX86_BUILTIN_PADDUSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
26342 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ussubv16qi3, "__builtin_ia32_psubusb128", IX86_BUILTIN_PSUBUSB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
26343 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ussubv8hi3, "__builtin_ia32_psubusw128", IX86_BUILTIN_PSUBUSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
26345 { OPTION_MASK_ISA_SSE2, CODE_FOR_mulv8hi3, "__builtin_ia32_pmullw128", IX86_BUILTIN_PMULLW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
26346 { OPTION_MASK_ISA_SSE2, CODE_FOR_smulv8hi3_highpart, "__builtin_ia32_pmulhw128", IX86_BUILTIN_PMULHW128, UNKNOWN,(int) V8HI_FTYPE_V8HI_V8HI },
26348 { OPTION_MASK_ISA_SSE2, CODE_FOR_andv2di3, "__builtin_ia32_pand128", IX86_BUILTIN_PAND128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
26349 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_andnotv2di3, "__builtin_ia32_pandn128", IX86_BUILTIN_PANDN128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
26350 { OPTION_MASK_ISA_SSE2, CODE_FOR_iorv2di3, "__builtin_ia32_por128", IX86_BUILTIN_POR128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
26351 { OPTION_MASK_ISA_SSE2, CODE_FOR_xorv2di3, "__builtin_ia32_pxor128", IX86_BUILTIN_PXOR128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
26353 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_uavgv16qi3, "__builtin_ia32_pavgb128", IX86_BUILTIN_PAVGB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
26354 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_uavgv8hi3, "__builtin_ia32_pavgw128", IX86_BUILTIN_PAVGW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
26356 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_eqv16qi3, "__builtin_ia32_pcmpeqb128", IX86_BUILTIN_PCMPEQB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
26357 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_eqv8hi3, "__builtin_ia32_pcmpeqw128", IX86_BUILTIN_PCMPEQW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
26358 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_eqv4si3, "__builtin_ia32_pcmpeqd128", IX86_BUILTIN_PCMPEQD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
26359 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_gtv16qi3, "__builtin_ia32_pcmpgtb128", IX86_BUILTIN_PCMPGTB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
26360 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_gtv8hi3, "__builtin_ia32_pcmpgtw128", IX86_BUILTIN_PCMPGTW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
26361 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_gtv4si3, "__builtin_ia32_pcmpgtd128", IX86_BUILTIN_PCMPGTD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
26363 { OPTION_MASK_ISA_SSE2, CODE_FOR_umaxv16qi3, "__builtin_ia32_pmaxub128", IX86_BUILTIN_PMAXUB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
26364 { OPTION_MASK_ISA_SSE2, CODE_FOR_smaxv8hi3, "__builtin_ia32_pmaxsw128", IX86_BUILTIN_PMAXSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
26365 { OPTION_MASK_ISA_SSE2, CODE_FOR_uminv16qi3, "__builtin_ia32_pminub128", IX86_BUILTIN_PMINUB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
26366 { OPTION_MASK_ISA_SSE2, CODE_FOR_sminv8hi3, "__builtin_ia32_pminsw128", IX86_BUILTIN_PMINSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
26368 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_highv16qi, "__builtin_ia32_punpckhbw128", IX86_BUILTIN_PUNPCKHBW128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
26369 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_highv8hi, "__builtin_ia32_punpckhwd128", IX86_BUILTIN_PUNPCKHWD128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
26370 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_highv4si, "__builtin_ia32_punpckhdq128", IX86_BUILTIN_PUNPCKHDQ128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
26371 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_highv2di, "__builtin_ia32_punpckhqdq128", IX86_BUILTIN_PUNPCKHQDQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
26372 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_lowv16qi, "__builtin_ia32_punpcklbw128", IX86_BUILTIN_PUNPCKLBW128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
26373 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_lowv8hi, "__builtin_ia32_punpcklwd128", IX86_BUILTIN_PUNPCKLWD128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
26374 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_lowv4si, "__builtin_ia32_punpckldq128", IX86_BUILTIN_PUNPCKLDQ128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
26375 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_lowv2di, "__builtin_ia32_punpcklqdq128", IX86_BUILTIN_PUNPCKLQDQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
26377 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_packsswb, "__builtin_ia32_packsswb128", IX86_BUILTIN_PACKSSWB128, UNKNOWN, (int) V16QI_FTYPE_V8HI_V8HI },
26378 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_packssdw, "__builtin_ia32_packssdw128", IX86_BUILTIN_PACKSSDW128, UNKNOWN, (int) V8HI_FTYPE_V4SI_V4SI },
26379 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_packuswb, "__builtin_ia32_packuswb128", IX86_BUILTIN_PACKUSWB128, UNKNOWN, (int) V16QI_FTYPE_V8HI_V8HI },
26381 { OPTION_MASK_ISA_SSE2, CODE_FOR_umulv8hi3_highpart, "__builtin_ia32_pmulhuw128", IX86_BUILTIN_PMULHUW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
26382 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_psadbw, "__builtin_ia32_psadbw128", IX86_BUILTIN_PSADBW128, UNKNOWN, (int) V2DI_FTYPE_V16QI_V16QI },
26384 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_umulv1siv1di3, "__builtin_ia32_pmuludq", IX86_BUILTIN_PMULUDQ, UNKNOWN, (int) V1DI_FTYPE_V2SI_V2SI },
26385 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_umulv2siv2di3, "__builtin_ia32_pmuludq128", IX86_BUILTIN_PMULUDQ128, UNKNOWN, (int) V2DI_FTYPE_V4SI_V4SI },
26387 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_pmaddwd, "__builtin_ia32_pmaddwd128", IX86_BUILTIN_PMADDWD128, UNKNOWN, (int) V4SI_FTYPE_V8HI_V8HI },
26389 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtsi2sd, "__builtin_ia32_cvtsi2sd", IX86_BUILTIN_CVTSI2SD, UNKNOWN, (int) V2DF_FTYPE_V2DF_SI },
26390 { OPTION_MASK_ISA_SSE2 | OPTION_MASK_ISA_64BIT, CODE_FOR_sse2_cvtsi2sdq, "__builtin_ia32_cvtsi642sd", IX86_BUILTIN_CVTSI642SD, UNKNOWN, (int) V2DF_FTYPE_V2DF_DI },
26391 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtsd2ss, "__builtin_ia32_cvtsd2ss", IX86_BUILTIN_CVTSD2SS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V2DF },
26392 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtss2sd, "__builtin_ia32_cvtss2sd", IX86_BUILTIN_CVTSS2SD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V4SF },
26394 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ashlv1ti3, "__builtin_ia32_pslldqi128", IX86_BUILTIN_PSLLDQI128, UNKNOWN, (int) V2DI_FTYPE_V2DI_INT_CONVERT },
26395 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashlv8hi3, "__builtin_ia32_psllwi128", IX86_BUILTIN_PSLLWI128, UNKNOWN, (int) V8HI_FTYPE_V8HI_SI_COUNT },
26396 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashlv4si3, "__builtin_ia32_pslldi128", IX86_BUILTIN_PSLLDI128, UNKNOWN, (int) V4SI_FTYPE_V4SI_SI_COUNT },
26397 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashlv2di3, "__builtin_ia32_psllqi128", IX86_BUILTIN_PSLLQI128, UNKNOWN, (int) V2DI_FTYPE_V2DI_SI_COUNT },
26398 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashlv8hi3, "__builtin_ia32_psllw128", IX86_BUILTIN_PSLLW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_COUNT },
26399 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashlv4si3, "__builtin_ia32_pslld128", IX86_BUILTIN_PSLLD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_COUNT },
26400 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashlv2di3, "__builtin_ia32_psllq128", IX86_BUILTIN_PSLLQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_COUNT },
26402 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_lshrv1ti3, "__builtin_ia32_psrldqi128", IX86_BUILTIN_PSRLDQI128, UNKNOWN, (int) V2DI_FTYPE_V2DI_INT_CONVERT },
26403 { OPTION_MASK_ISA_SSE2, CODE_FOR_lshrv8hi3, "__builtin_ia32_psrlwi128", IX86_BUILTIN_PSRLWI128, UNKNOWN, (int) V8HI_FTYPE_V8HI_SI_COUNT },
26404 { OPTION_MASK_ISA_SSE2, CODE_FOR_lshrv4si3, "__builtin_ia32_psrldi128", IX86_BUILTIN_PSRLDI128, UNKNOWN, (int) V4SI_FTYPE_V4SI_SI_COUNT },
26405 { OPTION_MASK_ISA_SSE2, CODE_FOR_lshrv2di3, "__builtin_ia32_psrlqi128", IX86_BUILTIN_PSRLQI128, UNKNOWN, (int) V2DI_FTYPE_V2DI_SI_COUNT },
26406 { OPTION_MASK_ISA_SSE2, CODE_FOR_lshrv8hi3, "__builtin_ia32_psrlw128", IX86_BUILTIN_PSRLW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_COUNT },
26407 { OPTION_MASK_ISA_SSE2, CODE_FOR_lshrv4si3, "__builtin_ia32_psrld128", IX86_BUILTIN_PSRLD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_COUNT },
26408 { OPTION_MASK_ISA_SSE2, CODE_FOR_lshrv2di3, "__builtin_ia32_psrlq128", IX86_BUILTIN_PSRLQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_COUNT },
26410 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashrv8hi3, "__builtin_ia32_psrawi128", IX86_BUILTIN_PSRAWI128, UNKNOWN, (int) V8HI_FTYPE_V8HI_SI_COUNT },
26411 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashrv4si3, "__builtin_ia32_psradi128", IX86_BUILTIN_PSRADI128, UNKNOWN, (int) V4SI_FTYPE_V4SI_SI_COUNT },
26412 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashrv8hi3, "__builtin_ia32_psraw128", IX86_BUILTIN_PSRAW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_COUNT },
26413 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashrv4si3, "__builtin_ia32_psrad128", IX86_BUILTIN_PSRAD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_COUNT },
26415 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_pshufd, "__builtin_ia32_pshufd", IX86_BUILTIN_PSHUFD, UNKNOWN, (int) V4SI_FTYPE_V4SI_INT },
26416 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_pshuflw, "__builtin_ia32_pshuflw", IX86_BUILTIN_PSHUFLW, UNKNOWN, (int) V8HI_FTYPE_V8HI_INT },
26417 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_pshufhw, "__builtin_ia32_pshufhw", IX86_BUILTIN_PSHUFHW, UNKNOWN, (int) V8HI_FTYPE_V8HI_INT },
26419 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmsqrtv2df2, "__builtin_ia32_sqrtsd", IX86_BUILTIN_SQRTSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_VEC_MERGE },
26421 { OPTION_MASK_ISA_SSE2, CODE_FOR_abstf2, 0, IX86_BUILTIN_FABSQ, UNKNOWN, (int) FLOAT128_FTYPE_FLOAT128 },
26422 { OPTION_MASK_ISA_SSE2, CODE_FOR_copysigntf3, 0, IX86_BUILTIN_COPYSIGNQ, UNKNOWN, (int) FLOAT128_FTYPE_FLOAT128_FLOAT128 },
26424 { OPTION_MASK_ISA_SSE, CODE_FOR_sse2_movq128, "__builtin_ia32_movq128", IX86_BUILTIN_MOVQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI },
26426 /* SSE2 MMX */
26427 { OPTION_MASK_ISA_SSE2, CODE_FOR_mmx_addv1di3, "__builtin_ia32_paddq", IX86_BUILTIN_PADDQ, UNKNOWN, (int) V1DI_FTYPE_V1DI_V1DI },
26428 { OPTION_MASK_ISA_SSE2, CODE_FOR_mmx_subv1di3, "__builtin_ia32_psubq", IX86_BUILTIN_PSUBQ, UNKNOWN, (int) V1DI_FTYPE_V1DI_V1DI },
26430 /* SSE3 */
26431 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_movshdup, "__builtin_ia32_movshdup", IX86_BUILTIN_MOVSHDUP, UNKNOWN, (int) V4SF_FTYPE_V4SF},
26432 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_movsldup, "__builtin_ia32_movsldup", IX86_BUILTIN_MOVSLDUP, UNKNOWN, (int) V4SF_FTYPE_V4SF },
26434 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_addsubv4sf3, "__builtin_ia32_addsubps", IX86_BUILTIN_ADDSUBPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
26435 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_addsubv2df3, "__builtin_ia32_addsubpd", IX86_BUILTIN_ADDSUBPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
26436 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_haddv4sf3, "__builtin_ia32_haddps", IX86_BUILTIN_HADDPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
26437 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_haddv2df3, "__builtin_ia32_haddpd", IX86_BUILTIN_HADDPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
26438 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_hsubv4sf3, "__builtin_ia32_hsubps", IX86_BUILTIN_HSUBPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
26439 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_hsubv2df3, "__builtin_ia32_hsubpd", IX86_BUILTIN_HSUBPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
26441 /* SSSE3 */
26442 { OPTION_MASK_ISA_SSSE3, CODE_FOR_absv16qi2, "__builtin_ia32_pabsb128", IX86_BUILTIN_PABSB128, UNKNOWN, (int) V16QI_FTYPE_V16QI },
26443 { OPTION_MASK_ISA_SSSE3, CODE_FOR_absv8qi2, "__builtin_ia32_pabsb", IX86_BUILTIN_PABSB, UNKNOWN, (int) V8QI_FTYPE_V8QI },
26444 { OPTION_MASK_ISA_SSSE3, CODE_FOR_absv8hi2, "__builtin_ia32_pabsw128", IX86_BUILTIN_PABSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI },
26445 { OPTION_MASK_ISA_SSSE3, CODE_FOR_absv4hi2, "__builtin_ia32_pabsw", IX86_BUILTIN_PABSW, UNKNOWN, (int) V4HI_FTYPE_V4HI },
26446 { OPTION_MASK_ISA_SSSE3, CODE_FOR_absv4si2, "__builtin_ia32_pabsd128", IX86_BUILTIN_PABSD128, UNKNOWN, (int) V4SI_FTYPE_V4SI },
26447 { OPTION_MASK_ISA_SSSE3, CODE_FOR_absv2si2, "__builtin_ia32_pabsd", IX86_BUILTIN_PABSD, UNKNOWN, (int) V2SI_FTYPE_V2SI },
26449 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phaddwv8hi3, "__builtin_ia32_phaddw128", IX86_BUILTIN_PHADDW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
26450 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phaddwv4hi3, "__builtin_ia32_phaddw", IX86_BUILTIN_PHADDW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
26451 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phadddv4si3, "__builtin_ia32_phaddd128", IX86_BUILTIN_PHADDD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
26452 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phadddv2si3, "__builtin_ia32_phaddd", IX86_BUILTIN_PHADDD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
26453 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phaddswv8hi3, "__builtin_ia32_phaddsw128", IX86_BUILTIN_PHADDSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
26454 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phaddswv4hi3, "__builtin_ia32_phaddsw", IX86_BUILTIN_PHADDSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
26455 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phsubwv8hi3, "__builtin_ia32_phsubw128", IX86_BUILTIN_PHSUBW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
26456 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phsubwv4hi3, "__builtin_ia32_phsubw", IX86_BUILTIN_PHSUBW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
26457 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phsubdv4si3, "__builtin_ia32_phsubd128", IX86_BUILTIN_PHSUBD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
26458 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phsubdv2si3, "__builtin_ia32_phsubd", IX86_BUILTIN_PHSUBD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
26459 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phsubswv8hi3, "__builtin_ia32_phsubsw128", IX86_BUILTIN_PHSUBSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
26460 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phsubswv4hi3, "__builtin_ia32_phsubsw", IX86_BUILTIN_PHSUBSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
26461 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_pmaddubsw128, "__builtin_ia32_pmaddubsw128", IX86_BUILTIN_PMADDUBSW128, UNKNOWN, (int) V8HI_FTYPE_V16QI_V16QI },
26462 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_pmaddubsw, "__builtin_ia32_pmaddubsw", IX86_BUILTIN_PMADDUBSW, UNKNOWN, (int) V4HI_FTYPE_V8QI_V8QI },
26463 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_pmulhrswv8hi3, "__builtin_ia32_pmulhrsw128", IX86_BUILTIN_PMULHRSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
26464 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_pmulhrswv4hi3, "__builtin_ia32_pmulhrsw", IX86_BUILTIN_PMULHRSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
26465 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_pshufbv16qi3, "__builtin_ia32_pshufb128", IX86_BUILTIN_PSHUFB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
26466 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_pshufbv8qi3, "__builtin_ia32_pshufb", IX86_BUILTIN_PSHUFB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
26467 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_psignv16qi3, "__builtin_ia32_psignb128", IX86_BUILTIN_PSIGNB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
26468 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_psignv8qi3, "__builtin_ia32_psignb", IX86_BUILTIN_PSIGNB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
26469 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_psignv8hi3, "__builtin_ia32_psignw128", IX86_BUILTIN_PSIGNW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
26470 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_psignv4hi3, "__builtin_ia32_psignw", IX86_BUILTIN_PSIGNW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
26471 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_psignv4si3, "__builtin_ia32_psignd128", IX86_BUILTIN_PSIGND128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
26472 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_psignv2si3, "__builtin_ia32_psignd", IX86_BUILTIN_PSIGND, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
26474 /* SSSE3. */
26475 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_palignrti, "__builtin_ia32_palignr128", IX86_BUILTIN_PALIGNR128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_INT_CONVERT },
26476 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_palignrdi, "__builtin_ia32_palignr", IX86_BUILTIN_PALIGNR, UNKNOWN, (int) V1DI_FTYPE_V1DI_V1DI_INT_CONVERT },
26478 /* SSE4.1 */
26479 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_blendpd, "__builtin_ia32_blendpd", IX86_BUILTIN_BLENDPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
26480 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_blendps, "__builtin_ia32_blendps", IX86_BUILTIN_BLENDPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
26481 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_blendvpd, "__builtin_ia32_blendvpd", IX86_BUILTIN_BLENDVPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_V2DF },
26482 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_blendvps, "__builtin_ia32_blendvps", IX86_BUILTIN_BLENDVPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_V4SF },
26483 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_dppd, "__builtin_ia32_dppd", IX86_BUILTIN_DPPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
26484 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_dpps, "__builtin_ia32_dpps", IX86_BUILTIN_DPPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
26485 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_insertps, "__builtin_ia32_insertps128", IX86_BUILTIN_INSERTPS128, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
26486 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_mpsadbw, "__builtin_ia32_mpsadbw128", IX86_BUILTIN_MPSADBW128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI_INT },
26487 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_pblendvb, "__builtin_ia32_pblendvb128", IX86_BUILTIN_PBLENDVB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI_V16QI },
26488 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_pblendw, "__builtin_ia32_pblendw128", IX86_BUILTIN_PBLENDW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_INT },
26490 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_sign_extendv8qiv8hi2, "__builtin_ia32_pmovsxbw128", IX86_BUILTIN_PMOVSXBW128, UNKNOWN, (int) V8HI_FTYPE_V16QI },
26491 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_sign_extendv4qiv4si2, "__builtin_ia32_pmovsxbd128", IX86_BUILTIN_PMOVSXBD128, UNKNOWN, (int) V4SI_FTYPE_V16QI },
26492 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_sign_extendv2qiv2di2, "__builtin_ia32_pmovsxbq128", IX86_BUILTIN_PMOVSXBQ128, UNKNOWN, (int) V2DI_FTYPE_V16QI },
26493 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_sign_extendv4hiv4si2, "__builtin_ia32_pmovsxwd128", IX86_BUILTIN_PMOVSXWD128, UNKNOWN, (int) V4SI_FTYPE_V8HI },
26494 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_sign_extendv2hiv2di2, "__builtin_ia32_pmovsxwq128", IX86_BUILTIN_PMOVSXWQ128, UNKNOWN, (int) V2DI_FTYPE_V8HI },
26495 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_sign_extendv2siv2di2, "__builtin_ia32_pmovsxdq128", IX86_BUILTIN_PMOVSXDQ128, UNKNOWN, (int) V2DI_FTYPE_V4SI },
26496 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_zero_extendv8qiv8hi2, "__builtin_ia32_pmovzxbw128", IX86_BUILTIN_PMOVZXBW128, UNKNOWN, (int) V8HI_FTYPE_V16QI },
26497 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_zero_extendv4qiv4si2, "__builtin_ia32_pmovzxbd128", IX86_BUILTIN_PMOVZXBD128, UNKNOWN, (int) V4SI_FTYPE_V16QI },
26498 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_zero_extendv2qiv2di2, "__builtin_ia32_pmovzxbq128", IX86_BUILTIN_PMOVZXBQ128, UNKNOWN, (int) V2DI_FTYPE_V16QI },
26499 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_zero_extendv4hiv4si2, "__builtin_ia32_pmovzxwd128", IX86_BUILTIN_PMOVZXWD128, UNKNOWN, (int) V4SI_FTYPE_V8HI },
26500 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_zero_extendv2hiv2di2, "__builtin_ia32_pmovzxwq128", IX86_BUILTIN_PMOVZXWQ128, UNKNOWN, (int) V2DI_FTYPE_V8HI },
26501 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_zero_extendv2siv2di2, "__builtin_ia32_pmovzxdq128", IX86_BUILTIN_PMOVZXDQ128, UNKNOWN, (int) V2DI_FTYPE_V4SI },
26502 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_phminposuw, "__builtin_ia32_phminposuw128", IX86_BUILTIN_PHMINPOSUW128, UNKNOWN, (int) V8HI_FTYPE_V8HI },
26504 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_packusdw, "__builtin_ia32_packusdw128", IX86_BUILTIN_PACKUSDW128, UNKNOWN, (int) V8HI_FTYPE_V4SI_V4SI },
26505 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_eqv2di3, "__builtin_ia32_pcmpeqq", IX86_BUILTIN_PCMPEQQ, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
26506 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_smaxv16qi3, "__builtin_ia32_pmaxsb128", IX86_BUILTIN_PMAXSB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
26507 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_smaxv4si3, "__builtin_ia32_pmaxsd128", IX86_BUILTIN_PMAXSD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
26508 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_umaxv4si3, "__builtin_ia32_pmaxud128", IX86_BUILTIN_PMAXUD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
26509 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_umaxv8hi3, "__builtin_ia32_pmaxuw128", IX86_BUILTIN_PMAXUW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
26510 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sminv16qi3, "__builtin_ia32_pminsb128", IX86_BUILTIN_PMINSB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
26511 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sminv4si3, "__builtin_ia32_pminsd128", IX86_BUILTIN_PMINSD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
26512 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_uminv4si3, "__builtin_ia32_pminud128", IX86_BUILTIN_PMINUD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
26513 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_uminv8hi3, "__builtin_ia32_pminuw128", IX86_BUILTIN_PMINUW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
26514 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_mulv2siv2di3, "__builtin_ia32_pmuldq128", IX86_BUILTIN_PMULDQ128, UNKNOWN, (int) V2DI_FTYPE_V4SI_V4SI },
26515 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_mulv4si3, "__builtin_ia32_pmulld128", IX86_BUILTIN_PMULLD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
26517 /* SSE4.1 */
26518 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundpd, "__builtin_ia32_roundpd", IX86_BUILTIN_ROUNDPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_INT },
26519 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundps, "__builtin_ia32_roundps", IX86_BUILTIN_ROUNDPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_INT },
26520 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundsd, "__builtin_ia32_roundsd", IX86_BUILTIN_ROUNDSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
26521 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundss, "__builtin_ia32_roundss", IX86_BUILTIN_ROUNDSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
26523 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundpd, "__builtin_ia32_floorpd", IX86_BUILTIN_FLOORPD, (enum rtx_code) ROUND_FLOOR, (int) V2DF_FTYPE_V2DF_ROUND },
26524 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundpd, "__builtin_ia32_ceilpd", IX86_BUILTIN_CEILPD, (enum rtx_code) ROUND_CEIL, (int) V2DF_FTYPE_V2DF_ROUND },
26525 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundpd, "__builtin_ia32_truncpd", IX86_BUILTIN_TRUNCPD, (enum rtx_code) ROUND_TRUNC, (int) V2DF_FTYPE_V2DF_ROUND },
26526 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundpd, "__builtin_ia32_rintpd", IX86_BUILTIN_RINTPD, (enum rtx_code) ROUND_MXCSR, (int) V2DF_FTYPE_V2DF_ROUND },
26528 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundpd_vec_pack_sfix, "__builtin_ia32_floorpd_vec_pack_sfix", IX86_BUILTIN_FLOORPD_VEC_PACK_SFIX, (enum rtx_code) ROUND_FLOOR, (int) V4SI_FTYPE_V2DF_V2DF_ROUND },
26529 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundpd_vec_pack_sfix, "__builtin_ia32_ceilpd_vec_pack_sfix", IX86_BUILTIN_CEILPD_VEC_PACK_SFIX, (enum rtx_code) ROUND_CEIL, (int) V4SI_FTYPE_V2DF_V2DF_ROUND },
26531 { OPTION_MASK_ISA_ROUND, CODE_FOR_roundv2df2, "__builtin_ia32_roundpd_az", IX86_BUILTIN_ROUNDPD_AZ, UNKNOWN, (int) V2DF_FTYPE_V2DF },
26532 { OPTION_MASK_ISA_ROUND, CODE_FOR_roundv2df2_vec_pack_sfix, "__builtin_ia32_roundpd_az_vec_pack_sfix", IX86_BUILTIN_ROUNDPD_AZ_VEC_PACK_SFIX, UNKNOWN, (int) V4SI_FTYPE_V2DF_V2DF },
26534 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundps, "__builtin_ia32_floorps", IX86_BUILTIN_FLOORPS, (enum rtx_code) ROUND_FLOOR, (int) V4SF_FTYPE_V4SF_ROUND },
26535 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundps, "__builtin_ia32_ceilps", IX86_BUILTIN_CEILPS, (enum rtx_code) ROUND_CEIL, (int) V4SF_FTYPE_V4SF_ROUND },
26536 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundps, "__builtin_ia32_truncps", IX86_BUILTIN_TRUNCPS, (enum rtx_code) ROUND_TRUNC, (int) V4SF_FTYPE_V4SF_ROUND },
26537 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundps, "__builtin_ia32_rintps", IX86_BUILTIN_RINTPS, (enum rtx_code) ROUND_MXCSR, (int) V4SF_FTYPE_V4SF_ROUND },
26539 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundps_sfix, "__builtin_ia32_floorps_sfix", IX86_BUILTIN_FLOORPS_SFIX, (enum rtx_code) ROUND_FLOOR, (int) V4SI_FTYPE_V4SF_ROUND },
26540 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundps_sfix, "__builtin_ia32_ceilps_sfix", IX86_BUILTIN_CEILPS_SFIX, (enum rtx_code) ROUND_CEIL, (int) V4SI_FTYPE_V4SF_ROUND },
26542 { OPTION_MASK_ISA_ROUND, CODE_FOR_roundv4sf2, "__builtin_ia32_roundps_az", IX86_BUILTIN_ROUNDPS_AZ, UNKNOWN, (int) V4SF_FTYPE_V4SF },
26543 { OPTION_MASK_ISA_ROUND, CODE_FOR_roundv4sf2_sfix, "__builtin_ia32_roundps_az_sfix", IX86_BUILTIN_ROUNDPS_AZ_SFIX, UNKNOWN, (int) V4SI_FTYPE_V4SF },
26545 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_ptest, "__builtin_ia32_ptestz128", IX86_BUILTIN_PTESTZ, EQ, (int) INT_FTYPE_V2DI_V2DI_PTEST },
26546 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_ptest, "__builtin_ia32_ptestc128", IX86_BUILTIN_PTESTC, LTU, (int) INT_FTYPE_V2DI_V2DI_PTEST },
26547 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_ptest, "__builtin_ia32_ptestnzc128", IX86_BUILTIN_PTESTNZC, GTU, (int) INT_FTYPE_V2DI_V2DI_PTEST },
26549 /* SSE4.2 */
26550 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_gtv2di3, "__builtin_ia32_pcmpgtq", IX86_BUILTIN_PCMPGTQ, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
26551 { OPTION_MASK_ISA_SSE4_2 | OPTION_MASK_ISA_CRC32, CODE_FOR_sse4_2_crc32qi, "__builtin_ia32_crc32qi", IX86_BUILTIN_CRC32QI, UNKNOWN, (int) UINT_FTYPE_UINT_UCHAR },
26552 { OPTION_MASK_ISA_SSE4_2 | OPTION_MASK_ISA_CRC32, CODE_FOR_sse4_2_crc32hi, "__builtin_ia32_crc32hi", IX86_BUILTIN_CRC32HI, UNKNOWN, (int) UINT_FTYPE_UINT_USHORT },
26553 { OPTION_MASK_ISA_SSE4_2 | OPTION_MASK_ISA_CRC32, CODE_FOR_sse4_2_crc32si, "__builtin_ia32_crc32si", IX86_BUILTIN_CRC32SI, UNKNOWN, (int) UINT_FTYPE_UINT_UINT },
26554 { OPTION_MASK_ISA_SSE4_2 | OPTION_MASK_ISA_CRC32 | OPTION_MASK_ISA_64BIT, CODE_FOR_sse4_2_crc32di, "__builtin_ia32_crc32di", IX86_BUILTIN_CRC32DI, UNKNOWN, (int) UINT64_FTYPE_UINT64_UINT64 },
26556 /* SSE4A */
26557 { OPTION_MASK_ISA_SSE4A, CODE_FOR_sse4a_extrqi, "__builtin_ia32_extrqi", IX86_BUILTIN_EXTRQI, UNKNOWN, (int) V2DI_FTYPE_V2DI_UINT_UINT },
26558 { OPTION_MASK_ISA_SSE4A, CODE_FOR_sse4a_extrq, "__builtin_ia32_extrq", IX86_BUILTIN_EXTRQ, UNKNOWN, (int) V2DI_FTYPE_V2DI_V16QI },
26559 { OPTION_MASK_ISA_SSE4A, CODE_FOR_sse4a_insertqi, "__builtin_ia32_insertqi", IX86_BUILTIN_INSERTQI, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_UINT_UINT },
26560 { OPTION_MASK_ISA_SSE4A, CODE_FOR_sse4a_insertq, "__builtin_ia32_insertq", IX86_BUILTIN_INSERTQ, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
26562 /* AES */
26563 { OPTION_MASK_ISA_SSE2, CODE_FOR_aeskeygenassist, 0, IX86_BUILTIN_AESKEYGENASSIST128, UNKNOWN, (int) V2DI_FTYPE_V2DI_INT },
26564 { OPTION_MASK_ISA_SSE2, CODE_FOR_aesimc, 0, IX86_BUILTIN_AESIMC128, UNKNOWN, (int) V2DI_FTYPE_V2DI },
26566 { OPTION_MASK_ISA_SSE2, CODE_FOR_aesenc, 0, IX86_BUILTIN_AESENC128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
26567 { OPTION_MASK_ISA_SSE2, CODE_FOR_aesenclast, 0, IX86_BUILTIN_AESENCLAST128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
26568 { OPTION_MASK_ISA_SSE2, CODE_FOR_aesdec, 0, IX86_BUILTIN_AESDEC128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
26569 { OPTION_MASK_ISA_SSE2, CODE_FOR_aesdeclast, 0, IX86_BUILTIN_AESDECLAST128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
26571 /* PCLMUL */
26572 { OPTION_MASK_ISA_SSE2, CODE_FOR_pclmulqdq, 0, IX86_BUILTIN_PCLMULQDQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_INT },
26574 /* AVX */
26575 { OPTION_MASK_ISA_AVX, CODE_FOR_addv4df3, "__builtin_ia32_addpd256", IX86_BUILTIN_ADDPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
26576 { OPTION_MASK_ISA_AVX, CODE_FOR_addv8sf3, "__builtin_ia32_addps256", IX86_BUILTIN_ADDPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
26577 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_addsubv4df3, "__builtin_ia32_addsubpd256", IX86_BUILTIN_ADDSUBPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
26578 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_addsubv8sf3, "__builtin_ia32_addsubps256", IX86_BUILTIN_ADDSUBPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
26579 { OPTION_MASK_ISA_AVX, CODE_FOR_andv4df3, "__builtin_ia32_andpd256", IX86_BUILTIN_ANDPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
26580 { OPTION_MASK_ISA_AVX, CODE_FOR_andv8sf3, "__builtin_ia32_andps256", IX86_BUILTIN_ANDPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
26581 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_andnotv4df3, "__builtin_ia32_andnpd256", IX86_BUILTIN_ANDNPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
26582 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_andnotv8sf3, "__builtin_ia32_andnps256", IX86_BUILTIN_ANDNPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
26583 { OPTION_MASK_ISA_AVX, CODE_FOR_divv4df3, "__builtin_ia32_divpd256", IX86_BUILTIN_DIVPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
26584 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_divv8sf3, "__builtin_ia32_divps256", IX86_BUILTIN_DIVPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
26585 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_haddv4df3, "__builtin_ia32_haddpd256", IX86_BUILTIN_HADDPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
26586 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_hsubv8sf3, "__builtin_ia32_hsubps256", IX86_BUILTIN_HSUBPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
26587 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_hsubv4df3, "__builtin_ia32_hsubpd256", IX86_BUILTIN_HSUBPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
26588 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_haddv8sf3, "__builtin_ia32_haddps256", IX86_BUILTIN_HADDPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
26589 { OPTION_MASK_ISA_AVX, CODE_FOR_smaxv4df3, "__builtin_ia32_maxpd256", IX86_BUILTIN_MAXPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
26590 { OPTION_MASK_ISA_AVX, CODE_FOR_smaxv8sf3, "__builtin_ia32_maxps256", IX86_BUILTIN_MAXPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
26591 { OPTION_MASK_ISA_AVX, CODE_FOR_sminv4df3, "__builtin_ia32_minpd256", IX86_BUILTIN_MINPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
26592 { OPTION_MASK_ISA_AVX, CODE_FOR_sminv8sf3, "__builtin_ia32_minps256", IX86_BUILTIN_MINPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
26593 { OPTION_MASK_ISA_AVX, CODE_FOR_mulv4df3, "__builtin_ia32_mulpd256", IX86_BUILTIN_MULPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
26594 { OPTION_MASK_ISA_AVX, CODE_FOR_mulv8sf3, "__builtin_ia32_mulps256", IX86_BUILTIN_MULPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
26595 { OPTION_MASK_ISA_AVX, CODE_FOR_iorv4df3, "__builtin_ia32_orpd256", IX86_BUILTIN_ORPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
26596 { OPTION_MASK_ISA_AVX, CODE_FOR_iorv8sf3, "__builtin_ia32_orps256", IX86_BUILTIN_ORPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
26597 { OPTION_MASK_ISA_AVX, CODE_FOR_subv4df3, "__builtin_ia32_subpd256", IX86_BUILTIN_SUBPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
26598 { OPTION_MASK_ISA_AVX, CODE_FOR_subv8sf3, "__builtin_ia32_subps256", IX86_BUILTIN_SUBPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
26599 { OPTION_MASK_ISA_AVX, CODE_FOR_xorv4df3, "__builtin_ia32_xorpd256", IX86_BUILTIN_XORPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
26600 { OPTION_MASK_ISA_AVX, CODE_FOR_xorv8sf3, "__builtin_ia32_xorps256", IX86_BUILTIN_XORPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
26602 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilvarv2df3, "__builtin_ia32_vpermilvarpd", IX86_BUILTIN_VPERMILVARPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DI },
26603 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilvarv4sf3, "__builtin_ia32_vpermilvarps", IX86_BUILTIN_VPERMILVARPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SI },
26604 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilvarv4df3, "__builtin_ia32_vpermilvarpd256", IX86_BUILTIN_VPERMILVARPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DI },
26605 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilvarv8sf3, "__builtin_ia32_vpermilvarps256", IX86_BUILTIN_VPERMILVARPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SI },
26607 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_blendpd256, "__builtin_ia32_blendpd256", IX86_BUILTIN_BLENDPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_INT },
26608 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_blendps256, "__builtin_ia32_blendps256", IX86_BUILTIN_BLENDPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_INT },
26609 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_blendvpd256, "__builtin_ia32_blendvpd256", IX86_BUILTIN_BLENDVPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_V4DF },
26610 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_blendvps256, "__builtin_ia32_blendvps256", IX86_BUILTIN_BLENDVPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_V8SF },
26611 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_dpps256, "__builtin_ia32_dpps256", IX86_BUILTIN_DPPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_INT },
26612 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_shufpd256, "__builtin_ia32_shufpd256", IX86_BUILTIN_SHUFPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_INT },
26613 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_shufps256, "__builtin_ia32_shufps256", IX86_BUILTIN_SHUFPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_INT },
26614 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vmcmpv2df3, "__builtin_ia32_cmpsd", IX86_BUILTIN_CMPSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
26615 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vmcmpv4sf3, "__builtin_ia32_cmpss", IX86_BUILTIN_CMPSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
26616 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cmpv2df3, "__builtin_ia32_cmppd", IX86_BUILTIN_CMPPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
26617 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cmpv4sf3, "__builtin_ia32_cmpps", IX86_BUILTIN_CMPPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
26618 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cmpv4df3, "__builtin_ia32_cmppd256", IX86_BUILTIN_CMPPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_INT },
26619 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cmpv8sf3, "__builtin_ia32_cmpps256", IX86_BUILTIN_CMPPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_INT },
26620 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vextractf128v4df, "__builtin_ia32_vextractf128_pd256", IX86_BUILTIN_EXTRACTF128PD256, UNKNOWN, (int) V2DF_FTYPE_V4DF_INT },
26621 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vextractf128v8sf, "__builtin_ia32_vextractf128_ps256", IX86_BUILTIN_EXTRACTF128PS256, UNKNOWN, (int) V4SF_FTYPE_V8SF_INT },
26622 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vextractf128v8si, "__builtin_ia32_vextractf128_si256", IX86_BUILTIN_EXTRACTF128SI256, UNKNOWN, (int) V4SI_FTYPE_V8SI_INT },
26623 { OPTION_MASK_ISA_AVX, CODE_FOR_floatv4siv4df2, "__builtin_ia32_cvtdq2pd256", IX86_BUILTIN_CVTDQ2PD256, UNKNOWN, (int) V4DF_FTYPE_V4SI },
26624 { OPTION_MASK_ISA_AVX, CODE_FOR_floatv8siv8sf2, "__builtin_ia32_cvtdq2ps256", IX86_BUILTIN_CVTDQ2PS256, UNKNOWN, (int) V8SF_FTYPE_V8SI },
26625 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cvtpd2ps256, "__builtin_ia32_cvtpd2ps256", IX86_BUILTIN_CVTPD2PS256, UNKNOWN, (int) V4SF_FTYPE_V4DF },
26626 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cvtps2dq256, "__builtin_ia32_cvtps2dq256", IX86_BUILTIN_CVTPS2DQ256, UNKNOWN, (int) V8SI_FTYPE_V8SF },
26627 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cvtps2pd256, "__builtin_ia32_cvtps2pd256", IX86_BUILTIN_CVTPS2PD256, UNKNOWN, (int) V4DF_FTYPE_V4SF },
26628 { OPTION_MASK_ISA_AVX, CODE_FOR_fix_truncv4dfv4si2, "__builtin_ia32_cvttpd2dq256", IX86_BUILTIN_CVTTPD2DQ256, UNKNOWN, (int) V4SI_FTYPE_V4DF },
26629 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cvtpd2dq256, "__builtin_ia32_cvtpd2dq256", IX86_BUILTIN_CVTPD2DQ256, UNKNOWN, (int) V4SI_FTYPE_V4DF },
26630 { OPTION_MASK_ISA_AVX, CODE_FOR_fix_truncv8sfv8si2, "__builtin_ia32_cvttps2dq256", IX86_BUILTIN_CVTTPS2DQ256, UNKNOWN, (int) V8SI_FTYPE_V8SF },
26631 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vperm2f128v4df3, "__builtin_ia32_vperm2f128_pd256", IX86_BUILTIN_VPERM2F128PD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_INT },
26632 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vperm2f128v8sf3, "__builtin_ia32_vperm2f128_ps256", IX86_BUILTIN_VPERM2F128PS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_INT },
26633 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vperm2f128v8si3, "__builtin_ia32_vperm2f128_si256", IX86_BUILTIN_VPERM2F128SI256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_INT },
26634 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilv2df, "__builtin_ia32_vpermilpd", IX86_BUILTIN_VPERMILPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_INT },
26635 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilv4sf, "__builtin_ia32_vpermilps", IX86_BUILTIN_VPERMILPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_INT },
26636 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilv4df, "__builtin_ia32_vpermilpd256", IX86_BUILTIN_VPERMILPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_INT },
26637 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilv8sf, "__builtin_ia32_vpermilps256", IX86_BUILTIN_VPERMILPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_INT },
26638 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vinsertf128v4df, "__builtin_ia32_vinsertf128_pd256", IX86_BUILTIN_VINSERTF128PD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V2DF_INT },
26639 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vinsertf128v8sf, "__builtin_ia32_vinsertf128_ps256", IX86_BUILTIN_VINSERTF128PS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V4SF_INT },
26640 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vinsertf128v8si, "__builtin_ia32_vinsertf128_si256", IX86_BUILTIN_VINSERTF128SI256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V4SI_INT },
26642 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movshdup256, "__builtin_ia32_movshdup256", IX86_BUILTIN_MOVSHDUP256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
26643 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movsldup256, "__builtin_ia32_movsldup256", IX86_BUILTIN_MOVSLDUP256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
26644 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movddup256, "__builtin_ia32_movddup256", IX86_BUILTIN_MOVDDUP256, UNKNOWN, (int) V4DF_FTYPE_V4DF },
26646 { OPTION_MASK_ISA_AVX, CODE_FOR_sqrtv4df2, "__builtin_ia32_sqrtpd256", IX86_BUILTIN_SQRTPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF },
26647 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_sqrtv8sf2, "__builtin_ia32_sqrtps256", IX86_BUILTIN_SQRTPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
26648 { OPTION_MASK_ISA_AVX, CODE_FOR_sqrtv8sf2, "__builtin_ia32_sqrtps_nr256", IX86_BUILTIN_SQRTPS_NR256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
26649 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_rsqrtv8sf2, "__builtin_ia32_rsqrtps256", IX86_BUILTIN_RSQRTPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
26650 { OPTION_MASK_ISA_AVX, CODE_FOR_rsqrtv8sf2, "__builtin_ia32_rsqrtps_nr256", IX86_BUILTIN_RSQRTPS_NR256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
26652 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_rcpv8sf2, "__builtin_ia32_rcpps256", IX86_BUILTIN_RCPPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
26654 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundpd256, "__builtin_ia32_roundpd256", IX86_BUILTIN_ROUNDPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_INT },
26655 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundps256, "__builtin_ia32_roundps256", IX86_BUILTIN_ROUNDPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_INT },
26657 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundpd256, "__builtin_ia32_floorpd256", IX86_BUILTIN_FLOORPD256, (enum rtx_code) ROUND_FLOOR, (int) V4DF_FTYPE_V4DF_ROUND },
26658 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundpd256, "__builtin_ia32_ceilpd256", IX86_BUILTIN_CEILPD256, (enum rtx_code) ROUND_CEIL, (int) V4DF_FTYPE_V4DF_ROUND },
26659 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundpd256, "__builtin_ia32_truncpd256", IX86_BUILTIN_TRUNCPD256, (enum rtx_code) ROUND_TRUNC, (int) V4DF_FTYPE_V4DF_ROUND },
26660 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundpd256, "__builtin_ia32_rintpd256", IX86_BUILTIN_RINTPD256, (enum rtx_code) ROUND_MXCSR, (int) V4DF_FTYPE_V4DF_ROUND },
26662 { OPTION_MASK_ISA_AVX, CODE_FOR_roundv4df2, "__builtin_ia32_roundpd_az256", IX86_BUILTIN_ROUNDPD_AZ256, UNKNOWN, (int) V4DF_FTYPE_V4DF },
26663 { OPTION_MASK_ISA_AVX, CODE_FOR_roundv4df2_vec_pack_sfix, "__builtin_ia32_roundpd_az_vec_pack_sfix256", IX86_BUILTIN_ROUNDPD_AZ_VEC_PACK_SFIX256, UNKNOWN, (int) V8SI_FTYPE_V4DF_V4DF },
26665 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundpd_vec_pack_sfix256, "__builtin_ia32_floorpd_vec_pack_sfix256", IX86_BUILTIN_FLOORPD_VEC_PACK_SFIX256, (enum rtx_code) ROUND_FLOOR, (int) V8SI_FTYPE_V4DF_V4DF_ROUND },
26666 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundpd_vec_pack_sfix256, "__builtin_ia32_ceilpd_vec_pack_sfix256", IX86_BUILTIN_CEILPD_VEC_PACK_SFIX256, (enum rtx_code) ROUND_CEIL, (int) V8SI_FTYPE_V4DF_V4DF_ROUND },
26668 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundps256, "__builtin_ia32_floorps256", IX86_BUILTIN_FLOORPS256, (enum rtx_code) ROUND_FLOOR, (int) V8SF_FTYPE_V8SF_ROUND },
26669 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundps256, "__builtin_ia32_ceilps256", IX86_BUILTIN_CEILPS256, (enum rtx_code) ROUND_CEIL, (int) V8SF_FTYPE_V8SF_ROUND },
26670 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundps256, "__builtin_ia32_truncps256", IX86_BUILTIN_TRUNCPS256, (enum rtx_code) ROUND_TRUNC, (int) V8SF_FTYPE_V8SF_ROUND },
26671 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundps256, "__builtin_ia32_rintps256", IX86_BUILTIN_RINTPS256, (enum rtx_code) ROUND_MXCSR, (int) V8SF_FTYPE_V8SF_ROUND },
26673 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundps_sfix256, "__builtin_ia32_floorps_sfix256", IX86_BUILTIN_FLOORPS_SFIX256, (enum rtx_code) ROUND_FLOOR, (int) V8SI_FTYPE_V8SF_ROUND },
26674 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundps_sfix256, "__builtin_ia32_ceilps_sfix256", IX86_BUILTIN_CEILPS_SFIX256, (enum rtx_code) ROUND_CEIL, (int) V8SI_FTYPE_V8SF_ROUND },
26676 { OPTION_MASK_ISA_AVX, CODE_FOR_roundv8sf2, "__builtin_ia32_roundps_az256", IX86_BUILTIN_ROUNDPS_AZ256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
26677 { OPTION_MASK_ISA_AVX, CODE_FOR_roundv8sf2_sfix, "__builtin_ia32_roundps_az_sfix256", IX86_BUILTIN_ROUNDPS_AZ_SFIX256, UNKNOWN, (int) V8SI_FTYPE_V8SF },
26679 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_unpckhpd256, "__builtin_ia32_unpckhpd256", IX86_BUILTIN_UNPCKHPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
26680 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_unpcklpd256, "__builtin_ia32_unpcklpd256", IX86_BUILTIN_UNPCKLPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
26681 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_unpckhps256, "__builtin_ia32_unpckhps256", IX86_BUILTIN_UNPCKHPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
26682 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_unpcklps256, "__builtin_ia32_unpcklps256", IX86_BUILTIN_UNPCKLPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
26684 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_si256_si, "__builtin_ia32_si256_si", IX86_BUILTIN_SI256_SI, UNKNOWN, (int) V8SI_FTYPE_V4SI },
26685 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_ps256_ps, "__builtin_ia32_ps256_ps", IX86_BUILTIN_PS256_PS, UNKNOWN, (int) V8SF_FTYPE_V4SF },
26686 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_pd256_pd, "__builtin_ia32_pd256_pd", IX86_BUILTIN_PD256_PD, UNKNOWN, (int) V4DF_FTYPE_V2DF },
26687 { OPTION_MASK_ISA_AVX, CODE_FOR_vec_extract_lo_v8si, "__builtin_ia32_si_si256", IX86_BUILTIN_SI_SI256, UNKNOWN, (int) V4SI_FTYPE_V8SI },
26688 { OPTION_MASK_ISA_AVX, CODE_FOR_vec_extract_lo_v8sf, "__builtin_ia32_ps_ps256", IX86_BUILTIN_PS_PS256, UNKNOWN, (int) V4SF_FTYPE_V8SF },
26689 { OPTION_MASK_ISA_AVX, CODE_FOR_vec_extract_lo_v4df, "__builtin_ia32_pd_pd256", IX86_BUILTIN_PD_PD256, UNKNOWN, (int) V2DF_FTYPE_V4DF },
26691 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestpd, "__builtin_ia32_vtestzpd", IX86_BUILTIN_VTESTZPD, EQ, (int) INT_FTYPE_V2DF_V2DF_PTEST },
26692 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestpd, "__builtin_ia32_vtestcpd", IX86_BUILTIN_VTESTCPD, LTU, (int) INT_FTYPE_V2DF_V2DF_PTEST },
26693 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestpd, "__builtin_ia32_vtestnzcpd", IX86_BUILTIN_VTESTNZCPD, GTU, (int) INT_FTYPE_V2DF_V2DF_PTEST },
26694 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestps, "__builtin_ia32_vtestzps", IX86_BUILTIN_VTESTZPS, EQ, (int) INT_FTYPE_V4SF_V4SF_PTEST },
26695 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestps, "__builtin_ia32_vtestcps", IX86_BUILTIN_VTESTCPS, LTU, (int) INT_FTYPE_V4SF_V4SF_PTEST },
26696 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestps, "__builtin_ia32_vtestnzcps", IX86_BUILTIN_VTESTNZCPS, GTU, (int) INT_FTYPE_V4SF_V4SF_PTEST },
26697 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestpd256, "__builtin_ia32_vtestzpd256", IX86_BUILTIN_VTESTZPD256, EQ, (int) INT_FTYPE_V4DF_V4DF_PTEST },
26698 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestpd256, "__builtin_ia32_vtestcpd256", IX86_BUILTIN_VTESTCPD256, LTU, (int) INT_FTYPE_V4DF_V4DF_PTEST },
26699 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestpd256, "__builtin_ia32_vtestnzcpd256", IX86_BUILTIN_VTESTNZCPD256, GTU, (int) INT_FTYPE_V4DF_V4DF_PTEST },
26700 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestps256, "__builtin_ia32_vtestzps256", IX86_BUILTIN_VTESTZPS256, EQ, (int) INT_FTYPE_V8SF_V8SF_PTEST },
26701 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestps256, "__builtin_ia32_vtestcps256", IX86_BUILTIN_VTESTCPS256, LTU, (int) INT_FTYPE_V8SF_V8SF_PTEST },
26702 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestps256, "__builtin_ia32_vtestnzcps256", IX86_BUILTIN_VTESTNZCPS256, GTU, (int) INT_FTYPE_V8SF_V8SF_PTEST },
26703 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_ptest256, "__builtin_ia32_ptestz256", IX86_BUILTIN_PTESTZ256, EQ, (int) INT_FTYPE_V4DI_V4DI_PTEST },
26704 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_ptest256, "__builtin_ia32_ptestc256", IX86_BUILTIN_PTESTC256, LTU, (int) INT_FTYPE_V4DI_V4DI_PTEST },
26705 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_ptest256, "__builtin_ia32_ptestnzc256", IX86_BUILTIN_PTESTNZC256, GTU, (int) INT_FTYPE_V4DI_V4DI_PTEST },
26707 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movmskpd256, "__builtin_ia32_movmskpd256", IX86_BUILTIN_MOVMSKPD256, UNKNOWN, (int) INT_FTYPE_V4DF },
26708 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movmskps256, "__builtin_ia32_movmskps256", IX86_BUILTIN_MOVMSKPS256, UNKNOWN, (int) INT_FTYPE_V8SF },
26710 { OPTION_MASK_ISA_AVX, CODE_FOR_copysignv8sf3, "__builtin_ia32_copysignps256", IX86_BUILTIN_CPYSGNPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
26711 { OPTION_MASK_ISA_AVX, CODE_FOR_copysignv4df3, "__builtin_ia32_copysignpd256", IX86_BUILTIN_CPYSGNPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
26713 { OPTION_MASK_ISA_AVX, CODE_FOR_vec_pack_sfix_v4df, "__builtin_ia32_vec_pack_sfix256 ", IX86_BUILTIN_VEC_PACK_SFIX256, UNKNOWN, (int) V8SI_FTYPE_V4DF_V4DF },
26715 /* AVX2 */
26716 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_mpsadbw, "__builtin_ia32_mpsadbw256", IX86_BUILTIN_MPSADBW256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI_INT },
26717 { OPTION_MASK_ISA_AVX2, CODE_FOR_absv32qi2, "__builtin_ia32_pabsb256", IX86_BUILTIN_PABSB256, UNKNOWN, (int) V32QI_FTYPE_V32QI },
26718 { OPTION_MASK_ISA_AVX2, CODE_FOR_absv16hi2, "__builtin_ia32_pabsw256", IX86_BUILTIN_PABSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI },
26719 { OPTION_MASK_ISA_AVX2, CODE_FOR_absv8si2, "__builtin_ia32_pabsd256", IX86_BUILTIN_PABSD256, UNKNOWN, (int) V8SI_FTYPE_V8SI },
26720 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_packssdw, "__builtin_ia32_packssdw256", IX86_BUILTIN_PACKSSDW256, UNKNOWN, (int) V16HI_FTYPE_V8SI_V8SI },
26721 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_packsswb, "__builtin_ia32_packsswb256", IX86_BUILTIN_PACKSSWB256, UNKNOWN, (int) V32QI_FTYPE_V16HI_V16HI },
26722 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_packusdw, "__builtin_ia32_packusdw256", IX86_BUILTIN_PACKUSDW256, UNKNOWN, (int) V16HI_FTYPE_V8SI_V8SI },
26723 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_packuswb, "__builtin_ia32_packuswb256", IX86_BUILTIN_PACKUSWB256, UNKNOWN, (int) V32QI_FTYPE_V16HI_V16HI },
26724 { OPTION_MASK_ISA_AVX2, CODE_FOR_addv32qi3, "__builtin_ia32_paddb256", IX86_BUILTIN_PADDB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
26725 { OPTION_MASK_ISA_AVX2, CODE_FOR_addv16hi3, "__builtin_ia32_paddw256", IX86_BUILTIN_PADDW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
26726 { OPTION_MASK_ISA_AVX2, CODE_FOR_addv8si3, "__builtin_ia32_paddd256", IX86_BUILTIN_PADDD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
26727 { OPTION_MASK_ISA_AVX2, CODE_FOR_addv4di3, "__builtin_ia32_paddq256", IX86_BUILTIN_PADDQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
26728 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ssaddv32qi3, "__builtin_ia32_paddsb256", IX86_BUILTIN_PADDSB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
26729 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ssaddv16hi3, "__builtin_ia32_paddsw256", IX86_BUILTIN_PADDSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
26730 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_usaddv32qi3, "__builtin_ia32_paddusb256", IX86_BUILTIN_PADDUSB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
26731 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_usaddv16hi3, "__builtin_ia32_paddusw256", IX86_BUILTIN_PADDUSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
26732 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_palignrv2ti, "__builtin_ia32_palignr256", IX86_BUILTIN_PALIGNR256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_INT_CONVERT },
26733 { OPTION_MASK_ISA_AVX2, CODE_FOR_andv4di3, "__builtin_ia32_andsi256", IX86_BUILTIN_AND256I, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
26734 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_andnotv4di3, "__builtin_ia32_andnotsi256", IX86_BUILTIN_ANDNOT256I, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
26735 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_uavgv32qi3, "__builtin_ia32_pavgb256", IX86_BUILTIN_PAVGB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
26736 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_uavgv16hi3, "__builtin_ia32_pavgw256", IX86_BUILTIN_PAVGW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
26737 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pblendvb, "__builtin_ia32_pblendvb256", IX86_BUILTIN_PBLENDVB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI_V32QI },
26738 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pblendw, "__builtin_ia32_pblendw256", IX86_BUILTIN_PBLENDVW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI_INT },
26739 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_eqv32qi3, "__builtin_ia32_pcmpeqb256", IX86_BUILTIN_PCMPEQB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
26740 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_eqv16hi3, "__builtin_ia32_pcmpeqw256", IX86_BUILTIN_PCMPEQW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
26741 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_eqv8si3, "__builtin_ia32_pcmpeqd256", IX86_BUILTIN_PCMPEQD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
26742 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_eqv4di3, "__builtin_ia32_pcmpeqq256", IX86_BUILTIN_PCMPEQQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
26743 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_gtv32qi3, "__builtin_ia32_pcmpgtb256", IX86_BUILTIN_PCMPGTB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
26744 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_gtv16hi3, "__builtin_ia32_pcmpgtw256", IX86_BUILTIN_PCMPGTW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
26745 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_gtv8si3, "__builtin_ia32_pcmpgtd256", IX86_BUILTIN_PCMPGTD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
26746 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_gtv4di3, "__builtin_ia32_pcmpgtq256", IX86_BUILTIN_PCMPGTQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
26747 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_phaddwv16hi3, "__builtin_ia32_phaddw256", IX86_BUILTIN_PHADDW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
26748 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_phadddv8si3, "__builtin_ia32_phaddd256", IX86_BUILTIN_PHADDD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
26749 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_phaddswv16hi3, "__builtin_ia32_phaddsw256", IX86_BUILTIN_PHADDSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
26750 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_phsubwv16hi3, "__builtin_ia32_phsubw256", IX86_BUILTIN_PHSUBW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
26751 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_phsubdv8si3, "__builtin_ia32_phsubd256", IX86_BUILTIN_PHSUBD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
26752 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_phsubswv16hi3, "__builtin_ia32_phsubsw256", IX86_BUILTIN_PHSUBSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
26753 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pmaddubsw256, "__builtin_ia32_pmaddubsw256", IX86_BUILTIN_PMADDUBSW256, UNKNOWN, (int) V16HI_FTYPE_V32QI_V32QI },
26754 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pmaddwd, "__builtin_ia32_pmaddwd256", IX86_BUILTIN_PMADDWD256, UNKNOWN, (int) V8SI_FTYPE_V16HI_V16HI },
26755 { OPTION_MASK_ISA_AVX2, CODE_FOR_smaxv32qi3, "__builtin_ia32_pmaxsb256", IX86_BUILTIN_PMAXSB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
26756 { OPTION_MASK_ISA_AVX2, CODE_FOR_smaxv16hi3, "__builtin_ia32_pmaxsw256", IX86_BUILTIN_PMAXSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
26757 { OPTION_MASK_ISA_AVX2, CODE_FOR_smaxv8si3 , "__builtin_ia32_pmaxsd256", IX86_BUILTIN_PMAXSD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
26758 { OPTION_MASK_ISA_AVX2, CODE_FOR_umaxv32qi3, "__builtin_ia32_pmaxub256", IX86_BUILTIN_PMAXUB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
26759 { OPTION_MASK_ISA_AVX2, CODE_FOR_umaxv16hi3, "__builtin_ia32_pmaxuw256", IX86_BUILTIN_PMAXUW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
26760 { OPTION_MASK_ISA_AVX2, CODE_FOR_umaxv8si3 , "__builtin_ia32_pmaxud256", IX86_BUILTIN_PMAXUD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
26761 { OPTION_MASK_ISA_AVX2, CODE_FOR_sminv32qi3, "__builtin_ia32_pminsb256", IX86_BUILTIN_PMINSB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
26762 { OPTION_MASK_ISA_AVX2, CODE_FOR_sminv16hi3, "__builtin_ia32_pminsw256", IX86_BUILTIN_PMINSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
26763 { OPTION_MASK_ISA_AVX2, CODE_FOR_sminv8si3 , "__builtin_ia32_pminsd256", IX86_BUILTIN_PMINSD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
26764 { OPTION_MASK_ISA_AVX2, CODE_FOR_uminv32qi3, "__builtin_ia32_pminub256", IX86_BUILTIN_PMINUB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
26765 { OPTION_MASK_ISA_AVX2, CODE_FOR_uminv16hi3, "__builtin_ia32_pminuw256", IX86_BUILTIN_PMINUW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
26766 { OPTION_MASK_ISA_AVX2, CODE_FOR_uminv8si3 , "__builtin_ia32_pminud256", IX86_BUILTIN_PMINUD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
26767 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pmovmskb, "__builtin_ia32_pmovmskb256", IX86_BUILTIN_PMOVMSKB256, UNKNOWN, (int) INT_FTYPE_V32QI },
26768 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_sign_extendv16qiv16hi2, "__builtin_ia32_pmovsxbw256", IX86_BUILTIN_PMOVSXBW256, UNKNOWN, (int) V16HI_FTYPE_V16QI },
26769 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_sign_extendv8qiv8si2 , "__builtin_ia32_pmovsxbd256", IX86_BUILTIN_PMOVSXBD256, UNKNOWN, (int) V8SI_FTYPE_V16QI },
26770 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_sign_extendv4qiv4di2 , "__builtin_ia32_pmovsxbq256", IX86_BUILTIN_PMOVSXBQ256, UNKNOWN, (int) V4DI_FTYPE_V16QI },
26771 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_sign_extendv8hiv8si2 , "__builtin_ia32_pmovsxwd256", IX86_BUILTIN_PMOVSXWD256, UNKNOWN, (int) V8SI_FTYPE_V8HI },
26772 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_sign_extendv4hiv4di2 , "__builtin_ia32_pmovsxwq256", IX86_BUILTIN_PMOVSXWQ256, UNKNOWN, (int) V4DI_FTYPE_V8HI },
26773 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_sign_extendv4siv4di2 , "__builtin_ia32_pmovsxdq256", IX86_BUILTIN_PMOVSXDQ256, UNKNOWN, (int) V4DI_FTYPE_V4SI },
26774 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_zero_extendv16qiv16hi2, "__builtin_ia32_pmovzxbw256", IX86_BUILTIN_PMOVZXBW256, UNKNOWN, (int) V16HI_FTYPE_V16QI },
26775 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_zero_extendv8qiv8si2 , "__builtin_ia32_pmovzxbd256", IX86_BUILTIN_PMOVZXBD256, UNKNOWN, (int) V8SI_FTYPE_V16QI },
26776 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_zero_extendv4qiv4di2 , "__builtin_ia32_pmovzxbq256", IX86_BUILTIN_PMOVZXBQ256, UNKNOWN, (int) V4DI_FTYPE_V16QI },
26777 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_zero_extendv8hiv8si2 , "__builtin_ia32_pmovzxwd256", IX86_BUILTIN_PMOVZXWD256, UNKNOWN, (int) V8SI_FTYPE_V8HI },
26778 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_zero_extendv4hiv4di2 , "__builtin_ia32_pmovzxwq256", IX86_BUILTIN_PMOVZXWQ256, UNKNOWN, (int) V4DI_FTYPE_V8HI },
26779 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_zero_extendv4siv4di2 , "__builtin_ia32_pmovzxdq256", IX86_BUILTIN_PMOVZXDQ256, UNKNOWN, (int) V4DI_FTYPE_V4SI },
26780 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_mulv4siv4di3 , "__builtin_ia32_pmuldq256" , IX86_BUILTIN_PMULDQ256 , UNKNOWN, (int) V4DI_FTYPE_V8SI_V8SI },
26781 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_umulhrswv16hi3 , "__builtin_ia32_pmulhrsw256", IX86_BUILTIN_PMULHRSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
26782 { OPTION_MASK_ISA_AVX2, CODE_FOR_umulv16hi3_highpart, "__builtin_ia32_pmulhuw256" , IX86_BUILTIN_PMULHUW256 , UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
26783 { OPTION_MASK_ISA_AVX2, CODE_FOR_smulv16hi3_highpart, "__builtin_ia32_pmulhw256" , IX86_BUILTIN_PMULHW256 , UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
26784 { OPTION_MASK_ISA_AVX2, CODE_FOR_mulv16hi3, "__builtin_ia32_pmullw256" , IX86_BUILTIN_PMULLW256 , UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
26785 { OPTION_MASK_ISA_AVX2, CODE_FOR_mulv8si3, "__builtin_ia32_pmulld256" , IX86_BUILTIN_PMULLD256 , UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
26786 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_umulv4siv4di3 , "__builtin_ia32_pmuludq256" , IX86_BUILTIN_PMULUDQ256 , UNKNOWN, (int) V4DI_FTYPE_V8SI_V8SI },
26787 { OPTION_MASK_ISA_AVX2, CODE_FOR_iorv4di3, "__builtin_ia32_por256", IX86_BUILTIN_POR256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
26788 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_psadbw, "__builtin_ia32_psadbw256", IX86_BUILTIN_PSADBW256, UNKNOWN, (int) V16HI_FTYPE_V32QI_V32QI },
26789 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pshufbv32qi3, "__builtin_ia32_pshufb256", IX86_BUILTIN_PSHUFB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
26790 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pshufdv3, "__builtin_ia32_pshufd256", IX86_BUILTIN_PSHUFD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_INT },
26791 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pshufhwv3, "__builtin_ia32_pshufhw256", IX86_BUILTIN_PSHUFHW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_INT },
26792 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pshuflwv3, "__builtin_ia32_pshuflw256", IX86_BUILTIN_PSHUFLW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_INT },
26793 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_psignv32qi3, "__builtin_ia32_psignb256", IX86_BUILTIN_PSIGNB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
26794 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_psignv16hi3, "__builtin_ia32_psignw256", IX86_BUILTIN_PSIGNW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
26795 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_psignv8si3 , "__builtin_ia32_psignd256", IX86_BUILTIN_PSIGND256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
26796 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ashlv2ti3, "__builtin_ia32_pslldqi256", IX86_BUILTIN_PSLLDQI256, UNKNOWN, (int) V4DI_FTYPE_V4DI_INT_CONVERT },
26797 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashlv16hi3, "__builtin_ia32_psllwi256", IX86_BUILTIN_PSLLWI256 , UNKNOWN, (int) V16HI_FTYPE_V16HI_SI_COUNT },
26798 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashlv16hi3, "__builtin_ia32_psllw256", IX86_BUILTIN_PSLLW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V8HI_COUNT },
26799 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashlv8si3, "__builtin_ia32_pslldi256", IX86_BUILTIN_PSLLDI256, UNKNOWN, (int) V8SI_FTYPE_V8SI_SI_COUNT },
26800 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashlv8si3, "__builtin_ia32_pslld256", IX86_BUILTIN_PSLLD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V4SI_COUNT },
26801 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashlv4di3, "__builtin_ia32_psllqi256", IX86_BUILTIN_PSLLQI256, UNKNOWN, (int) V4DI_FTYPE_V4DI_INT_COUNT },
26802 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashlv4di3, "__builtin_ia32_psllq256", IX86_BUILTIN_PSLLQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V2DI_COUNT },
26803 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashrv16hi3, "__builtin_ia32_psrawi256", IX86_BUILTIN_PSRAWI256, UNKNOWN, (int) V16HI_FTYPE_V16HI_SI_COUNT },
26804 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashrv16hi3, "__builtin_ia32_psraw256", IX86_BUILTIN_PSRAW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V8HI_COUNT },
26805 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashrv8si3, "__builtin_ia32_psradi256", IX86_BUILTIN_PSRADI256, UNKNOWN, (int) V8SI_FTYPE_V8SI_SI_COUNT },
26806 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashrv8si3, "__builtin_ia32_psrad256", IX86_BUILTIN_PSRAD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V4SI_COUNT },
26807 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_lshrv2ti3, "__builtin_ia32_psrldqi256", IX86_BUILTIN_PSRLDQI256, UNKNOWN, (int) V4DI_FTYPE_V4DI_INT_CONVERT },
26808 { OPTION_MASK_ISA_AVX2, CODE_FOR_lshrv16hi3, "__builtin_ia32_psrlwi256", IX86_BUILTIN_PSRLWI256 , UNKNOWN, (int) V16HI_FTYPE_V16HI_SI_COUNT },
26809 { OPTION_MASK_ISA_AVX2, CODE_FOR_lshrv16hi3, "__builtin_ia32_psrlw256", IX86_BUILTIN_PSRLW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V8HI_COUNT },
26810 { OPTION_MASK_ISA_AVX2, CODE_FOR_lshrv8si3, "__builtin_ia32_psrldi256", IX86_BUILTIN_PSRLDI256, UNKNOWN, (int) V8SI_FTYPE_V8SI_SI_COUNT },
26811 { OPTION_MASK_ISA_AVX2, CODE_FOR_lshrv8si3, "__builtin_ia32_psrld256", IX86_BUILTIN_PSRLD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V4SI_COUNT },
26812 { OPTION_MASK_ISA_AVX2, CODE_FOR_lshrv4di3, "__builtin_ia32_psrlqi256", IX86_BUILTIN_PSRLQI256, UNKNOWN, (int) V4DI_FTYPE_V4DI_INT_COUNT },
26813 { OPTION_MASK_ISA_AVX2, CODE_FOR_lshrv4di3, "__builtin_ia32_psrlq256", IX86_BUILTIN_PSRLQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V2DI_COUNT },
26814 { OPTION_MASK_ISA_AVX2, CODE_FOR_subv32qi3, "__builtin_ia32_psubb256", IX86_BUILTIN_PSUBB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
26815 { OPTION_MASK_ISA_AVX2, CODE_FOR_subv16hi3, "__builtin_ia32_psubw256", IX86_BUILTIN_PSUBW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
26816 { OPTION_MASK_ISA_AVX2, CODE_FOR_subv8si3, "__builtin_ia32_psubd256", IX86_BUILTIN_PSUBD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
26817 { OPTION_MASK_ISA_AVX2, CODE_FOR_subv4di3, "__builtin_ia32_psubq256", IX86_BUILTIN_PSUBQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
26818 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_sssubv32qi3, "__builtin_ia32_psubsb256", IX86_BUILTIN_PSUBSB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
26819 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_sssubv16hi3, "__builtin_ia32_psubsw256", IX86_BUILTIN_PSUBSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
26820 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ussubv32qi3, "__builtin_ia32_psubusb256", IX86_BUILTIN_PSUBUSB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
26821 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ussubv16hi3, "__builtin_ia32_psubusw256", IX86_BUILTIN_PSUBUSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
26822 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_interleave_highv32qi, "__builtin_ia32_punpckhbw256", IX86_BUILTIN_PUNPCKHBW256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
26823 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_interleave_highv16hi, "__builtin_ia32_punpckhwd256", IX86_BUILTIN_PUNPCKHWD256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
26824 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_interleave_highv8si, "__builtin_ia32_punpckhdq256", IX86_BUILTIN_PUNPCKHDQ256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
26825 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_interleave_highv4di, "__builtin_ia32_punpckhqdq256", IX86_BUILTIN_PUNPCKHQDQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
26826 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_interleave_lowv32qi, "__builtin_ia32_punpcklbw256", IX86_BUILTIN_PUNPCKLBW256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
26827 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_interleave_lowv16hi, "__builtin_ia32_punpcklwd256", IX86_BUILTIN_PUNPCKLWD256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
26828 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_interleave_lowv8si, "__builtin_ia32_punpckldq256", IX86_BUILTIN_PUNPCKLDQ256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
26829 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_interleave_lowv4di, "__builtin_ia32_punpcklqdq256", IX86_BUILTIN_PUNPCKLQDQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
26830 { OPTION_MASK_ISA_AVX2, CODE_FOR_xorv4di3, "__builtin_ia32_pxor256", IX86_BUILTIN_PXOR256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
26831 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_vec_dupv4sf, "__builtin_ia32_vbroadcastss_ps", IX86_BUILTIN_VBROADCASTSS_PS, UNKNOWN, (int) V4SF_FTYPE_V4SF },
26832 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_vec_dupv8sf, "__builtin_ia32_vbroadcastss_ps256", IX86_BUILTIN_VBROADCASTSS_PS256, UNKNOWN, (int) V8SF_FTYPE_V4SF },
26833 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_vec_dupv4df, "__builtin_ia32_vbroadcastsd_pd256", IX86_BUILTIN_VBROADCASTSD_PD256, UNKNOWN, (int) V4DF_FTYPE_V2DF },
26834 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_vbroadcasti128_v4di, "__builtin_ia32_vbroadcastsi256", IX86_BUILTIN_VBROADCASTSI256, UNKNOWN, (int) V4DI_FTYPE_V2DI },
26835 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pblenddv4si, "__builtin_ia32_pblendd128", IX86_BUILTIN_PBLENDD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_INT },
26836 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pblenddv8si, "__builtin_ia32_pblendd256", IX86_BUILTIN_PBLENDD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_INT },
26837 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pbroadcastv32qi, "__builtin_ia32_pbroadcastb256", IX86_BUILTIN_PBROADCASTB256, UNKNOWN, (int) V32QI_FTYPE_V16QI },
26838 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pbroadcastv16hi, "__builtin_ia32_pbroadcastw256", IX86_BUILTIN_PBROADCASTW256, UNKNOWN, (int) V16HI_FTYPE_V8HI },
26839 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pbroadcastv8si, "__builtin_ia32_pbroadcastd256", IX86_BUILTIN_PBROADCASTD256, UNKNOWN, (int) V8SI_FTYPE_V4SI },
26840 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pbroadcastv4di, "__builtin_ia32_pbroadcastq256", IX86_BUILTIN_PBROADCASTQ256, UNKNOWN, (int) V4DI_FTYPE_V2DI },
26841 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pbroadcastv16qi, "__builtin_ia32_pbroadcastb128", IX86_BUILTIN_PBROADCASTB128, UNKNOWN, (int) V16QI_FTYPE_V16QI },
26842 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pbroadcastv8hi, "__builtin_ia32_pbroadcastw128", IX86_BUILTIN_PBROADCASTW128, UNKNOWN, (int) V8HI_FTYPE_V8HI },
26843 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pbroadcastv4si, "__builtin_ia32_pbroadcastd128", IX86_BUILTIN_PBROADCASTD128, UNKNOWN, (int) V4SI_FTYPE_V4SI },
26844 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pbroadcastv2di, "__builtin_ia32_pbroadcastq128", IX86_BUILTIN_PBROADCASTQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI },
26845 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_permvarv8si, "__builtin_ia32_permvarsi256", IX86_BUILTIN_VPERMVARSI256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
26846 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_permv4df, "__builtin_ia32_permdf256", IX86_BUILTIN_VPERMDF256, UNKNOWN, (int) V4DF_FTYPE_V4DF_INT },
26847 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_permvarv8sf, "__builtin_ia32_permvarsf256", IX86_BUILTIN_VPERMVARSF256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
26848 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_permv4di, "__builtin_ia32_permdi256", IX86_BUILTIN_VPERMDI256, UNKNOWN, (int) V4DI_FTYPE_V4DI_INT },
26849 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_permv2ti, "__builtin_ia32_permti256", IX86_BUILTIN_VPERMTI256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_INT },
26850 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_extracti128, "__builtin_ia32_extract128i256", IX86_BUILTIN_VEXTRACT128I256, UNKNOWN, (int) V2DI_FTYPE_V4DI_INT },
26851 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_inserti128, "__builtin_ia32_insert128i256", IX86_BUILTIN_VINSERT128I256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V2DI_INT },
26852 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ashlvv4di, "__builtin_ia32_psllv4di", IX86_BUILTIN_PSLLVV4DI, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
26853 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ashlvv2di, "__builtin_ia32_psllv2di", IX86_BUILTIN_PSLLVV2DI, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
26854 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ashlvv8si, "__builtin_ia32_psllv8si", IX86_BUILTIN_PSLLVV8SI, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
26855 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ashlvv4si, "__builtin_ia32_psllv4si", IX86_BUILTIN_PSLLVV4SI, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
26856 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ashrvv8si, "__builtin_ia32_psrav8si", IX86_BUILTIN_PSRAVV8SI, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
26857 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ashrvv4si, "__builtin_ia32_psrav4si", IX86_BUILTIN_PSRAVV4SI, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
26858 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_lshrvv4di, "__builtin_ia32_psrlv4di", IX86_BUILTIN_PSRLVV4DI, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
26859 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_lshrvv2di, "__builtin_ia32_psrlv2di", IX86_BUILTIN_PSRLVV2DI, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
26860 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_lshrvv8si, "__builtin_ia32_psrlv8si", IX86_BUILTIN_PSRLVV8SI, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
26861 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_lshrvv4si, "__builtin_ia32_psrlv4si", IX86_BUILTIN_PSRLVV4SI, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
26863 { OPTION_MASK_ISA_LZCNT, CODE_FOR_clzhi2_lzcnt, "__builtin_clzs", IX86_BUILTIN_CLZS, UNKNOWN, (int) UINT16_FTYPE_UINT16 },
26865 /* BMI */
26866 { OPTION_MASK_ISA_BMI, CODE_FOR_bmi_bextr_si, "__builtin_ia32_bextr_u32", IX86_BUILTIN_BEXTR32, UNKNOWN, (int) UINT_FTYPE_UINT_UINT },
26867 { OPTION_MASK_ISA_BMI, CODE_FOR_bmi_bextr_di, "__builtin_ia32_bextr_u64", IX86_BUILTIN_BEXTR64, UNKNOWN, (int) UINT64_FTYPE_UINT64_UINT64 },
26868 { OPTION_MASK_ISA_BMI, CODE_FOR_ctzhi2, "__builtin_ctzs", IX86_BUILTIN_CTZS, UNKNOWN, (int) UINT16_FTYPE_UINT16 },
26870 /* TBM */
26871 { OPTION_MASK_ISA_TBM, CODE_FOR_tbm_bextri_si, "__builtin_ia32_bextri_u32", IX86_BUILTIN_BEXTRI32, UNKNOWN, (int) UINT_FTYPE_UINT_UINT },
26872 { OPTION_MASK_ISA_TBM, CODE_FOR_tbm_bextri_di, "__builtin_ia32_bextri_u64", IX86_BUILTIN_BEXTRI64, UNKNOWN, (int) UINT64_FTYPE_UINT64_UINT64 },
26874 /* F16C */
26875 { OPTION_MASK_ISA_F16C, CODE_FOR_vcvtph2ps, "__builtin_ia32_vcvtph2ps", IX86_BUILTIN_CVTPH2PS, UNKNOWN, (int) V4SF_FTYPE_V8HI },
26876 { OPTION_MASK_ISA_F16C, CODE_FOR_vcvtph2ps256, "__builtin_ia32_vcvtph2ps256", IX86_BUILTIN_CVTPH2PS256, UNKNOWN, (int) V8SF_FTYPE_V8HI },
26877 { OPTION_MASK_ISA_F16C, CODE_FOR_vcvtps2ph, "__builtin_ia32_vcvtps2ph", IX86_BUILTIN_CVTPS2PH, UNKNOWN, (int) V8HI_FTYPE_V4SF_INT },
26878 { OPTION_MASK_ISA_F16C, CODE_FOR_vcvtps2ph256, "__builtin_ia32_vcvtps2ph256", IX86_BUILTIN_CVTPS2PH256, UNKNOWN, (int) V8HI_FTYPE_V8SF_INT },
26880 /* BMI2 */
26881 { OPTION_MASK_ISA_BMI2, CODE_FOR_bmi2_bzhi_si3, "__builtin_ia32_bzhi_si", IX86_BUILTIN_BZHI32, UNKNOWN, (int) UINT_FTYPE_UINT_UINT },
26882 { OPTION_MASK_ISA_BMI2, CODE_FOR_bmi2_bzhi_di3, "__builtin_ia32_bzhi_di", IX86_BUILTIN_BZHI64, UNKNOWN, (int) UINT64_FTYPE_UINT64_UINT64 },
26883 { OPTION_MASK_ISA_BMI2, CODE_FOR_bmi2_pdep_si3, "__builtin_ia32_pdep_si", IX86_BUILTIN_PDEP32, UNKNOWN, (int) UINT_FTYPE_UINT_UINT },
26884 { OPTION_MASK_ISA_BMI2, CODE_FOR_bmi2_pdep_di3, "__builtin_ia32_pdep_di", IX86_BUILTIN_PDEP64, UNKNOWN, (int) UINT64_FTYPE_UINT64_UINT64 },
26885 { OPTION_MASK_ISA_BMI2, CODE_FOR_bmi2_pext_si3, "__builtin_ia32_pext_si", IX86_BUILTIN_PEXT32, UNKNOWN, (int) UINT_FTYPE_UINT_UINT },
26886 { OPTION_MASK_ISA_BMI2, CODE_FOR_bmi2_pext_di3, "__builtin_ia32_pext_di", IX86_BUILTIN_PEXT64, UNKNOWN, (int) UINT64_FTYPE_UINT64_UINT64 },
26887 };
26889 /* FMA4 and XOP. */
26890 #define MULTI_ARG_4_DF2_DI_I V2DF_FTYPE_V2DF_V2DF_V2DI_INT
26891 #define MULTI_ARG_4_DF2_DI_I1 V4DF_FTYPE_V4DF_V4DF_V4DI_INT
26892 #define MULTI_ARG_4_SF2_SI_I V4SF_FTYPE_V4SF_V4SF_V4SI_INT
26893 #define MULTI_ARG_4_SF2_SI_I1 V8SF_FTYPE_V8SF_V8SF_V8SI_INT
26894 #define MULTI_ARG_3_SF V4SF_FTYPE_V4SF_V4SF_V4SF
26895 #define MULTI_ARG_3_DF V2DF_FTYPE_V2DF_V2DF_V2DF
26896 #define MULTI_ARG_3_SF2 V8SF_FTYPE_V8SF_V8SF_V8SF
26897 #define MULTI_ARG_3_DF2 V4DF_FTYPE_V4DF_V4DF_V4DF
26898 #define MULTI_ARG_3_DI V2DI_FTYPE_V2DI_V2DI_V2DI
26899 #define MULTI_ARG_3_SI V4SI_FTYPE_V4SI_V4SI_V4SI
26900 #define MULTI_ARG_3_SI_DI V4SI_FTYPE_V4SI_V4SI_V2DI
26901 #define MULTI_ARG_3_HI V8HI_FTYPE_V8HI_V8HI_V8HI
26902 #define MULTI_ARG_3_HI_SI V8HI_FTYPE_V8HI_V8HI_V4SI
26903 #define MULTI_ARG_3_QI V16QI_FTYPE_V16QI_V16QI_V16QI
26904 #define MULTI_ARG_3_DI2 V4DI_FTYPE_V4DI_V4DI_V4DI
26905 #define MULTI_ARG_3_SI2 V8SI_FTYPE_V8SI_V8SI_V8SI
26906 #define MULTI_ARG_3_HI2 V16HI_FTYPE_V16HI_V16HI_V16HI
26907 #define MULTI_ARG_3_QI2 V32QI_FTYPE_V32QI_V32QI_V32QI
26908 #define MULTI_ARG_2_SF V4SF_FTYPE_V4SF_V4SF
26909 #define MULTI_ARG_2_DF V2DF_FTYPE_V2DF_V2DF
26910 #define MULTI_ARG_2_DI V2DI_FTYPE_V2DI_V2DI
26911 #define MULTI_ARG_2_SI V4SI_FTYPE_V4SI_V4SI
26912 #define MULTI_ARG_2_HI V8HI_FTYPE_V8HI_V8HI
26913 #define MULTI_ARG_2_QI V16QI_FTYPE_V16QI_V16QI
26914 #define MULTI_ARG_2_DI_IMM V2DI_FTYPE_V2DI_SI
26915 #define MULTI_ARG_2_SI_IMM V4SI_FTYPE_V4SI_SI
26916 #define MULTI_ARG_2_HI_IMM V8HI_FTYPE_V8HI_SI
26917 #define MULTI_ARG_2_QI_IMM V16QI_FTYPE_V16QI_SI
26918 #define MULTI_ARG_2_DI_CMP V2DI_FTYPE_V2DI_V2DI_CMP
26919 #define MULTI_ARG_2_SI_CMP V4SI_FTYPE_V4SI_V4SI_CMP
26920 #define MULTI_ARG_2_HI_CMP V8HI_FTYPE_V8HI_V8HI_CMP
26921 #define MULTI_ARG_2_QI_CMP V16QI_FTYPE_V16QI_V16QI_CMP
26922 #define MULTI_ARG_2_SF_TF V4SF_FTYPE_V4SF_V4SF_TF
26923 #define MULTI_ARG_2_DF_TF V2DF_FTYPE_V2DF_V2DF_TF
26924 #define MULTI_ARG_2_DI_TF V2DI_FTYPE_V2DI_V2DI_TF
26925 #define MULTI_ARG_2_SI_TF V4SI_FTYPE_V4SI_V4SI_TF
26926 #define MULTI_ARG_2_HI_TF V8HI_FTYPE_V8HI_V8HI_TF
26927 #define MULTI_ARG_2_QI_TF V16QI_FTYPE_V16QI_V16QI_TF
26928 #define MULTI_ARG_1_SF V4SF_FTYPE_V4SF
26929 #define MULTI_ARG_1_DF V2DF_FTYPE_V2DF
26930 #define MULTI_ARG_1_SF2 V8SF_FTYPE_V8SF
26931 #define MULTI_ARG_1_DF2 V4DF_FTYPE_V4DF
26932 #define MULTI_ARG_1_DI V2DI_FTYPE_V2DI
26933 #define MULTI_ARG_1_SI V4SI_FTYPE_V4SI
26934 #define MULTI_ARG_1_HI V8HI_FTYPE_V8HI
26935 #define MULTI_ARG_1_QI V16QI_FTYPE_V16QI
26936 #define MULTI_ARG_1_SI_DI V2DI_FTYPE_V4SI
26937 #define MULTI_ARG_1_HI_DI V2DI_FTYPE_V8HI
26938 #define MULTI_ARG_1_HI_SI V4SI_FTYPE_V8HI
26939 #define MULTI_ARG_1_QI_DI V2DI_FTYPE_V16QI
26940 #define MULTI_ARG_1_QI_SI V4SI_FTYPE_V16QI
26941 #define MULTI_ARG_1_QI_HI V8HI_FTYPE_V16QI
26944 {
26985 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v2di, "__builtin_ia32_vpcmov", IX86_BUILTIN_VPCMOV, UNKNOWN, (int)MULTI_ARG_3_DI },
26986 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v2di, "__builtin_ia32_vpcmov_v2di", IX86_BUILTIN_VPCMOV_V2DI, UNKNOWN, (int)MULTI_ARG_3_DI },
26987 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v4si, "__builtin_ia32_vpcmov_v4si", IX86_BUILTIN_VPCMOV_V4SI, UNKNOWN, (int)MULTI_ARG_3_SI },
26988 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v8hi, "__builtin_ia32_vpcmov_v8hi", IX86_BUILTIN_VPCMOV_V8HI, UNKNOWN, (int)MULTI_ARG_3_HI },
26989 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v16qi, "__builtin_ia32_vpcmov_v16qi",IX86_BUILTIN_VPCMOV_V16QI,UNKNOWN, (int)MULTI_ARG_3_QI },
26990 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v2df, "__builtin_ia32_vpcmov_v2df", IX86_BUILTIN_VPCMOV_V2DF, UNKNOWN, (int)MULTI_ARG_3_DF },
26991 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v4sf, "__builtin_ia32_vpcmov_v4sf", IX86_BUILTIN_VPCMOV_V4SF, UNKNOWN, (int)MULTI_ARG_3_SF },
26993 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v4di256, "__builtin_ia32_vpcmov256", IX86_BUILTIN_VPCMOV256, UNKNOWN, (int)MULTI_ARG_3_DI2 },
26994 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v4di256, "__builtin_ia32_vpcmov_v4di256", IX86_BUILTIN_VPCMOV_V4DI256, UNKNOWN, (int)MULTI_ARG_3_DI2 },
26995 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v8si256, "__builtin_ia32_vpcmov_v8si256", IX86_BUILTIN_VPCMOV_V8SI256, UNKNOWN, (int)MULTI_ARG_3_SI2 },
26996 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v16hi256, "__builtin_ia32_vpcmov_v16hi256", IX86_BUILTIN_VPCMOV_V16HI256, UNKNOWN, (int)MULTI_ARG_3_HI2 },
26997 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v32qi256, "__builtin_ia32_vpcmov_v32qi256", IX86_BUILTIN_VPCMOV_V32QI256, UNKNOWN, (int)MULTI_ARG_3_QI2 },
26998 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v4df256, "__builtin_ia32_vpcmov_v4df256", IX86_BUILTIN_VPCMOV_V4DF256, UNKNOWN, (int)MULTI_ARG_3_DF2 },
26999 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v8sf256, "__builtin_ia32_vpcmov_v8sf256", IX86_BUILTIN_VPCMOV_V8SF256, UNKNOWN, (int)MULTI_ARG_3_SF2 },
27001 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pperm, "__builtin_ia32_vpperm", IX86_BUILTIN_VPPERM, UNKNOWN, (int)MULTI_ARG_3_QI },
27003 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacssww, "__builtin_ia32_vpmacssww", IX86_BUILTIN_VPMACSSWW, UNKNOWN, (int)MULTI_ARG_3_HI },
27004 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacsww, "__builtin_ia32_vpmacsww", IX86_BUILTIN_VPMACSWW, UNKNOWN, (int)MULTI_ARG_3_HI },
27005 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacsswd, "__builtin_ia32_vpmacsswd", IX86_BUILTIN_VPMACSSWD, UNKNOWN, (int)MULTI_ARG_3_HI_SI },
27006 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacswd, "__builtin_ia32_vpmacswd", IX86_BUILTIN_VPMACSWD, UNKNOWN, (int)MULTI_ARG_3_HI_SI },
27007 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacssdd, "__builtin_ia32_vpmacssdd", IX86_BUILTIN_VPMACSSDD, UNKNOWN, (int)MULTI_ARG_3_SI },
27008 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacsdd, "__builtin_ia32_vpmacsdd", IX86_BUILTIN_VPMACSDD, UNKNOWN, (int)MULTI_ARG_3_SI },
27009 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacssdql, "__builtin_ia32_vpmacssdql", IX86_BUILTIN_VPMACSSDQL, UNKNOWN, (int)MULTI_ARG_3_SI_DI },
27010 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacssdqh, "__builtin_ia32_vpmacssdqh", IX86_BUILTIN_VPMACSSDQH, UNKNOWN, (int)MULTI_ARG_3_SI_DI },
27011 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacsdql, "__builtin_ia32_vpmacsdql", IX86_BUILTIN_VPMACSDQL, UNKNOWN, (int)MULTI_ARG_3_SI_DI },
27012 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacsdqh, "__builtin_ia32_vpmacsdqh", IX86_BUILTIN_VPMACSDQH, UNKNOWN, (int)MULTI_ARG_3_SI_DI },
27013 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmadcsswd, "__builtin_ia32_vpmadcsswd", IX86_BUILTIN_VPMADCSSWD, UNKNOWN, (int)MULTI_ARG_3_HI_SI },
27014 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmadcswd, "__builtin_ia32_vpmadcswd", IX86_BUILTIN_VPMADCSWD, UNKNOWN, (int)MULTI_ARG_3_HI_SI },
27016 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vrotlv2di3, "__builtin_ia32_vprotq", IX86_BUILTIN_VPROTQ, UNKNOWN, (int)MULTI_ARG_2_DI },
27017 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vrotlv4si3, "__builtin_ia32_vprotd", IX86_BUILTIN_VPROTD, UNKNOWN, (int)MULTI_ARG_2_SI },
27018 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vrotlv8hi3, "__builtin_ia32_vprotw", IX86_BUILTIN_VPROTW, UNKNOWN, (int)MULTI_ARG_2_HI },
27019 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vrotlv16qi3, "__builtin_ia32_vprotb", IX86_BUILTIN_VPROTB, UNKNOWN, (int)MULTI_ARG_2_QI },
27020 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_rotlv2di3, "__builtin_ia32_vprotqi", IX86_BUILTIN_VPROTQ_IMM, UNKNOWN, (int)MULTI_ARG_2_DI_IMM },
27021 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_rotlv4si3, "__builtin_ia32_vprotdi", IX86_BUILTIN_VPROTD_IMM, UNKNOWN, (int)MULTI_ARG_2_SI_IMM },
27022 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_rotlv8hi3, "__builtin_ia32_vprotwi", IX86_BUILTIN_VPROTW_IMM, UNKNOWN, (int)MULTI_ARG_2_HI_IMM },
27023 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_rotlv16qi3, "__builtin_ia32_vprotbi", IX86_BUILTIN_VPROTB_IMM, UNKNOWN, (int)MULTI_ARG_2_QI_IMM },
27024 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shav2di3, "__builtin_ia32_vpshaq", IX86_BUILTIN_VPSHAQ, UNKNOWN, (int)MULTI_ARG_2_DI },
27025 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shav4si3, "__builtin_ia32_vpshad", IX86_BUILTIN_VPSHAD, UNKNOWN, (int)MULTI_ARG_2_SI },
27026 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shav8hi3, "__builtin_ia32_vpshaw", IX86_BUILTIN_VPSHAW, UNKNOWN, (int)MULTI_ARG_2_HI },
27027 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shav16qi3, "__builtin_ia32_vpshab", IX86_BUILTIN_VPSHAB, UNKNOWN, (int)MULTI_ARG_2_QI },
27028 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shlv2di3, "__builtin_ia32_vpshlq", IX86_BUILTIN_VPSHLQ, UNKNOWN, (int)MULTI_ARG_2_DI },
27029 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shlv4si3, "__builtin_ia32_vpshld", IX86_BUILTIN_VPSHLD, UNKNOWN, (int)MULTI_ARG_2_SI },
27030 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shlv8hi3, "__builtin_ia32_vpshlw", IX86_BUILTIN_VPSHLW, UNKNOWN, (int)MULTI_ARG_2_HI },
27031 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shlv16qi3, "__builtin_ia32_vpshlb", IX86_BUILTIN_VPSHLB, UNKNOWN, (int)MULTI_ARG_2_QI },
27033 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vmfrczv4sf2, "__builtin_ia32_vfrczss", IX86_BUILTIN_VFRCZSS, UNKNOWN, (int)MULTI_ARG_2_SF },
27034 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vmfrczv2df2, "__builtin_ia32_vfrczsd", IX86_BUILTIN_VFRCZSD, UNKNOWN, (int)MULTI_ARG_2_DF },
27035 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_frczv4sf2, "__builtin_ia32_vfrczps", IX86_BUILTIN_VFRCZPS, UNKNOWN, (int)MULTI_ARG_1_SF },
27036 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_frczv2df2, "__builtin_ia32_vfrczpd", IX86_BUILTIN_VFRCZPD, UNKNOWN, (int)MULTI_ARG_1_DF },
27037 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_frczv8sf2, "__builtin_ia32_vfrczps256", IX86_BUILTIN_VFRCZPS256, UNKNOWN, (int)MULTI_ARG_1_SF2 },
27038 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_frczv4df2, "__builtin_ia32_vfrczpd256", IX86_BUILTIN_VFRCZPD256, UNKNOWN, (int)MULTI_ARG_1_DF2 },
27040 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddbw, "__builtin_ia32_vphaddbw", IX86_BUILTIN_VPHADDBW, UNKNOWN, (int)MULTI_ARG_1_QI_HI },
27041 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddbd, "__builtin_ia32_vphaddbd", IX86_BUILTIN_VPHADDBD, UNKNOWN, (int)MULTI_ARG_1_QI_SI },
27042 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddbq, "__builtin_ia32_vphaddbq", IX86_BUILTIN_VPHADDBQ, UNKNOWN, (int)MULTI_ARG_1_QI_DI },
27043 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddwd, "__builtin_ia32_vphaddwd", IX86_BUILTIN_VPHADDWD, UNKNOWN, (int)MULTI_ARG_1_HI_SI },
27044 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddwq, "__builtin_ia32_vphaddwq", IX86_BUILTIN_VPHADDWQ, UNKNOWN, (int)MULTI_ARG_1_HI_DI },
27045 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phadddq, "__builtin_ia32_vphadddq", IX86_BUILTIN_VPHADDDQ, UNKNOWN, (int)MULTI_ARG_1_SI_DI },
27046 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddubw, "__builtin_ia32_vphaddubw", IX86_BUILTIN_VPHADDUBW, UNKNOWN, (int)MULTI_ARG_1_QI_HI },
27047 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddubd, "__builtin_ia32_vphaddubd", IX86_BUILTIN_VPHADDUBD, UNKNOWN, (int)MULTI_ARG_1_QI_SI },
27048 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddubq, "__builtin_ia32_vphaddubq", IX86_BUILTIN_VPHADDUBQ, UNKNOWN, (int)MULTI_ARG_1_QI_DI },
27049 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phadduwd, "__builtin_ia32_vphadduwd", IX86_BUILTIN_VPHADDUWD, UNKNOWN, (int)MULTI_ARG_1_HI_SI },
27050 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phadduwq, "__builtin_ia32_vphadduwq", IX86_BUILTIN_VPHADDUWQ, UNKNOWN, (int)MULTI_ARG_1_HI_DI },
27051 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddudq, "__builtin_ia32_vphaddudq", IX86_BUILTIN_VPHADDUDQ, UNKNOWN, (int)MULTI_ARG_1_SI_DI },
27052 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phsubbw, "__builtin_ia32_vphsubbw", IX86_BUILTIN_VPHSUBBW, UNKNOWN, (int)MULTI_ARG_1_QI_HI },
27053 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phsubwd, "__builtin_ia32_vphsubwd", IX86_BUILTIN_VPHSUBWD, UNKNOWN, (int)MULTI_ARG_1_HI_SI },
27054 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phsubdq, "__builtin_ia32_vphsubdq", IX86_BUILTIN_VPHSUBDQ, UNKNOWN, (int)MULTI_ARG_1_SI_DI },
27056 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv16qi3, "__builtin_ia32_vpcomeqb", IX86_BUILTIN_VPCOMEQB, EQ, (int)MULTI_ARG_2_QI_CMP },
27057 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv16qi3, "__builtin_ia32_vpcomneb", IX86_BUILTIN_VPCOMNEB, NE, (int)MULTI_ARG_2_QI_CMP },
27058 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv16qi3, "__builtin_ia32_vpcomneqb", IX86_BUILTIN_VPCOMNEB, NE, (int)MULTI_ARG_2_QI_CMP },
27059 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv16qi3, "__builtin_ia32_vpcomltb", IX86_BUILTIN_VPCOMLTB, LT, (int)MULTI_ARG_2_QI_CMP },
27060 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv16qi3, "__builtin_ia32_vpcomleb", IX86_BUILTIN_VPCOMLEB, LE, (int)MULTI_ARG_2_QI_CMP },
27061 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv16qi3, "__builtin_ia32_vpcomgtb", IX86_BUILTIN_VPCOMGTB, GT, (int)MULTI_ARG_2_QI_CMP },
27062 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv16qi3, "__builtin_ia32_vpcomgeb", IX86_BUILTIN_VPCOMGEB, GE, (int)MULTI_ARG_2_QI_CMP },
27064 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv8hi3, "__builtin_ia32_vpcomeqw", IX86_BUILTIN_VPCOMEQW, EQ, (int)MULTI_ARG_2_HI_CMP },
27065 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv8hi3, "__builtin_ia32_vpcomnew", IX86_BUILTIN_VPCOMNEW, NE, (int)MULTI_ARG_2_HI_CMP },
27066 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv8hi3, "__builtin_ia32_vpcomneqw", IX86_BUILTIN_VPCOMNEW, NE, (int)MULTI_ARG_2_HI_CMP },
27067 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv8hi3, "__builtin_ia32_vpcomltw", IX86_BUILTIN_VPCOMLTW, LT, (int)MULTI_ARG_2_HI_CMP },
27068 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv8hi3, "__builtin_ia32_vpcomlew", IX86_BUILTIN_VPCOMLEW, LE, (int)MULTI_ARG_2_HI_CMP },
27069 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv8hi3, "__builtin_ia32_vpcomgtw", IX86_BUILTIN_VPCOMGTW, GT, (int)MULTI_ARG_2_HI_CMP },
27070 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv8hi3, "__builtin_ia32_vpcomgew", IX86_BUILTIN_VPCOMGEW, GE, (int)MULTI_ARG_2_HI_CMP },
27072 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv4si3, "__builtin_ia32_vpcomeqd", IX86_BUILTIN_VPCOMEQD, EQ, (int)MULTI_ARG_2_SI_CMP },
27073 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv4si3, "__builtin_ia32_vpcomned", IX86_BUILTIN_VPCOMNED, NE, (int)MULTI_ARG_2_SI_CMP },
27074 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv4si3, "__builtin_ia32_vpcomneqd", IX86_BUILTIN_VPCOMNED, NE, (int)MULTI_ARG_2_SI_CMP },
27075 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv4si3, "__builtin_ia32_vpcomltd", IX86_BUILTIN_VPCOMLTD, LT, (int)MULTI_ARG_2_SI_CMP },
27076 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv4si3, "__builtin_ia32_vpcomled", IX86_BUILTIN_VPCOMLED, LE, (int)MULTI_ARG_2_SI_CMP },
27077 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv4si3, "__builtin_ia32_vpcomgtd", IX86_BUILTIN_VPCOMGTD, GT, (int)MULTI_ARG_2_SI_CMP },
27078 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv4si3, "__builtin_ia32_vpcomged", IX86_BUILTIN_VPCOMGED, GE, (int)MULTI_ARG_2_SI_CMP },
27080 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv2di3, "__builtin_ia32_vpcomeqq", IX86_BUILTIN_VPCOMEQQ, EQ, (int)MULTI_ARG_2_DI_CMP },
27081 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv2di3, "__builtin_ia32_vpcomneq", IX86_BUILTIN_VPCOMNEQ, NE, (int)MULTI_ARG_2_DI_CMP },
27082 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv2di3, "__builtin_ia32_vpcomneqq", IX86_BUILTIN_VPCOMNEQ, NE, (int)MULTI_ARG_2_DI_CMP },
27083 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv2di3, "__builtin_ia32_vpcomltq", IX86_BUILTIN_VPCOMLTQ, LT, (int)MULTI_ARG_2_DI_CMP },
27084 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv2di3, "__builtin_ia32_vpcomleq", IX86_BUILTIN_VPCOMLEQ, LE, (int)MULTI_ARG_2_DI_CMP },
27085 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv2di3, "__builtin_ia32_vpcomgtq", IX86_BUILTIN_VPCOMGTQ, GT, (int)MULTI_ARG_2_DI_CMP },
27086 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv2di3, "__builtin_ia32_vpcomgeq", IX86_BUILTIN_VPCOMGEQ, GE, (int)MULTI_ARG_2_DI_CMP },
27088 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v16qi3,"__builtin_ia32_vpcomequb", IX86_BUILTIN_VPCOMEQUB, EQ, (int)MULTI_ARG_2_QI_CMP },
27089 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v16qi3,"__builtin_ia32_vpcomneub", IX86_BUILTIN_VPCOMNEUB, NE, (int)MULTI_ARG_2_QI_CMP },
27090 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v16qi3,"__builtin_ia32_vpcomnequb", IX86_BUILTIN_VPCOMNEUB, NE, (int)MULTI_ARG_2_QI_CMP },
27091 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv16qi3, "__builtin_ia32_vpcomltub", IX86_BUILTIN_VPCOMLTUB, LTU, (int)MULTI_ARG_2_QI_CMP },
27092 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv16qi3, "__builtin_ia32_vpcomleub", IX86_BUILTIN_VPCOMLEUB, LEU, (int)MULTI_ARG_2_QI_CMP },
27093 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv16qi3, "__builtin_ia32_vpcomgtub", IX86_BUILTIN_VPCOMGTUB, GTU, (int)MULTI_ARG_2_QI_CMP },
27094 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv16qi3, "__builtin_ia32_vpcomgeub", IX86_BUILTIN_VPCOMGEUB, GEU, (int)MULTI_ARG_2_QI_CMP },
27096 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v8hi3, "__builtin_ia32_vpcomequw", IX86_BUILTIN_VPCOMEQUW, EQ, (int)MULTI_ARG_2_HI_CMP },
27097 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v8hi3, "__builtin_ia32_vpcomneuw", IX86_BUILTIN_VPCOMNEUW, NE, (int)MULTI_ARG_2_HI_CMP },
27098 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v8hi3, "__builtin_ia32_vpcomnequw", IX86_BUILTIN_VPCOMNEUW, NE, (int)MULTI_ARG_2_HI_CMP },
27099 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv8hi3, "__builtin_ia32_vpcomltuw", IX86_BUILTIN_VPCOMLTUW, LTU, (int)MULTI_ARG_2_HI_CMP },
27100 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv8hi3, "__builtin_ia32_vpcomleuw", IX86_BUILTIN_VPCOMLEUW, LEU, (int)MULTI_ARG_2_HI_CMP },
27101 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv8hi3, "__builtin_ia32_vpcomgtuw", IX86_BUILTIN_VPCOMGTUW, GTU, (int)MULTI_ARG_2_HI_CMP },
27102 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv8hi3, "__builtin_ia32_vpcomgeuw", IX86_BUILTIN_VPCOMGEUW, GEU, (int)MULTI_ARG_2_HI_CMP },
27104 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v4si3, "__builtin_ia32_vpcomequd", IX86_BUILTIN_VPCOMEQUD, EQ, (int)MULTI_ARG_2_SI_CMP },
27105 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v4si3, "__builtin_ia32_vpcomneud", IX86_BUILTIN_VPCOMNEUD, NE, (int)MULTI_ARG_2_SI_CMP },
27106 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v4si3, "__builtin_ia32_vpcomnequd", IX86_BUILTIN_VPCOMNEUD, NE, (int)MULTI_ARG_2_SI_CMP },
27107 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv4si3, "__builtin_ia32_vpcomltud", IX86_BUILTIN_VPCOMLTUD, LTU, (int)MULTI_ARG_2_SI_CMP },
27108 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv4si3, "__builtin_ia32_vpcomleud", IX86_BUILTIN_VPCOMLEUD, LEU, (int)MULTI_ARG_2_SI_CMP },
27109 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv4si3, "__builtin_ia32_vpcomgtud", IX86_BUILTIN_VPCOMGTUD, GTU, (int)MULTI_ARG_2_SI_CMP },
27110 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv4si3, "__builtin_ia32_vpcomgeud", IX86_BUILTIN_VPCOMGEUD, GEU, (int)MULTI_ARG_2_SI_CMP },
27112 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v2di3, "__builtin_ia32_vpcomequq", IX86_BUILTIN_VPCOMEQUQ, EQ, (int)MULTI_ARG_2_DI_CMP },
27113 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v2di3, "__builtin_ia32_vpcomneuq", IX86_BUILTIN_VPCOMNEUQ, NE, (int)MULTI_ARG_2_DI_CMP },
27114 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v2di3, "__builtin_ia32_vpcomnequq", IX86_BUILTIN_VPCOMNEUQ, NE, (int)MULTI_ARG_2_DI_CMP },
27115 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv2di3, "__builtin_ia32_vpcomltuq", IX86_BUILTIN_VPCOMLTUQ, LTU, (int)MULTI_ARG_2_DI_CMP },
27116 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv2di3, "__builtin_ia32_vpcomleuq", IX86_BUILTIN_VPCOMLEUQ, LEU, (int)MULTI_ARG_2_DI_CMP },
27117 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv2di3, "__builtin_ia32_vpcomgtuq", IX86_BUILTIN_VPCOMGTUQ, GTU, (int)MULTI_ARG_2_DI_CMP },
27118 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv2di3, "__builtin_ia32_vpcomgeuq", IX86_BUILTIN_VPCOMGEUQ, GEU, (int)MULTI_ARG_2_DI_CMP },
27120 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv16qi3, "__builtin_ia32_vpcomfalseb", IX86_BUILTIN_VPCOMFALSEB, (enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_QI_TF },
27121 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv8hi3, "__builtin_ia32_vpcomfalsew", IX86_BUILTIN_VPCOMFALSEW, (enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_HI_TF },
27122 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv4si3, "__builtin_ia32_vpcomfalsed", IX86_BUILTIN_VPCOMFALSED, (enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_SI_TF },
27123 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv2di3, "__builtin_ia32_vpcomfalseq", IX86_BUILTIN_VPCOMFALSEQ, (enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_DI_TF },
27124 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv16qi3, "__builtin_ia32_vpcomfalseub",IX86_BUILTIN_VPCOMFALSEUB,(enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_QI_TF },
27125 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv8hi3, "__builtin_ia32_vpcomfalseuw",IX86_BUILTIN_VPCOMFALSEUW,(enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_HI_TF },
27126 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv4si3, "__builtin_ia32_vpcomfalseud",IX86_BUILTIN_VPCOMFALSEUD,(enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_SI_TF },
27127 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv2di3, "__builtin_ia32_vpcomfalseuq",IX86_BUILTIN_VPCOMFALSEUQ,(enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_DI_TF },
27129 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv16qi3, "__builtin_ia32_vpcomtrueb", IX86_BUILTIN_VPCOMTRUEB, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_QI_TF },
27130 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv8hi3, "__builtin_ia32_vpcomtruew", IX86_BUILTIN_VPCOMTRUEW, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_HI_TF },
27131 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv4si3, "__builtin_ia32_vpcomtrued", IX86_BUILTIN_VPCOMTRUED, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_SI_TF },
27132 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv2di3, "__builtin_ia32_vpcomtrueq", IX86_BUILTIN_VPCOMTRUEQ, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_DI_TF },
27133 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv16qi3, "__builtin_ia32_vpcomtrueub", IX86_BUILTIN_VPCOMTRUEUB, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_QI_TF },
27134 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv8hi3, "__builtin_ia32_vpcomtrueuw", IX86_BUILTIN_VPCOMTRUEUW, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_HI_TF },
27135 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv4si3, "__builtin_ia32_vpcomtrueud", IX86_BUILTIN_VPCOMTRUEUD, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_SI_TF },
27136 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv2di3, "__builtin_ia32_vpcomtrueuq", IX86_BUILTIN_VPCOMTRUEUQ, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_DI_TF },
27138 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vpermil2v2df3, "__builtin_ia32_vpermil2pd", IX86_BUILTIN_VPERMIL2PD, UNKNOWN, (int)MULTI_ARG_4_DF2_DI_I },
27139 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vpermil2v4sf3, "__builtin_ia32_vpermil2ps", IX86_BUILTIN_VPERMIL2PS, UNKNOWN, (int)MULTI_ARG_4_SF2_SI_I },
27140 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vpermil2v4df3, "__builtin_ia32_vpermil2pd256", IX86_BUILTIN_VPERMIL2PD256, UNKNOWN, (int)MULTI_ARG_4_DF2_DI_I1 },
27141 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vpermil2v8sf3, "__builtin_ia32_vpermil2ps256", IX86_BUILTIN_VPERMIL2PS256, UNKNOWN, (int)MULTI_ARG_4_SF2_SI_I1 },
27143 };
27144 \f
27145 /* TM vector builtins. */
27147 /* Reuse the existing x86-specific `struct builtin_description' cause
27148 we're lazy. Add casts to make them fit. */
27150 {
27151 { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_WM64", (enum ix86_builtins) BUILT_IN_TM_STORE_M64, UNKNOWN, VOID_FTYPE_PV2SI_V2SI },
27152 { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_WaRM64", (enum ix86_builtins) BUILT_IN_TM_STORE_WAR_M64, UNKNOWN, VOID_FTYPE_PV2SI_V2SI },
27153 { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_WaWM64", (enum ix86_builtins) BUILT_IN_TM_STORE_WAW_M64, UNKNOWN, VOID_FTYPE_PV2SI_V2SI },
27154 { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_RM64", (enum ix86_builtins) BUILT_IN_TM_LOAD_M64, UNKNOWN, V2SI_FTYPE_PCV2SI },
27155 { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_RaRM64", (enum ix86_builtins) BUILT_IN_TM_LOAD_RAR_M64, UNKNOWN, V2SI_FTYPE_PCV2SI },
27156 { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_RaWM64", (enum ix86_builtins) BUILT_IN_TM_LOAD_RAW_M64, UNKNOWN, V2SI_FTYPE_PCV2SI },
27157 { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_RfWM64", (enum ix86_builtins) BUILT_IN_TM_LOAD_RFW_M64, UNKNOWN, V2SI_FTYPE_PCV2SI },
27159 { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_WM128", (enum ix86_builtins) BUILT_IN_TM_STORE_M128, UNKNOWN, VOID_FTYPE_PV4SF_V4SF },
27160 { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_WaRM128", (enum ix86_builtins) BUILT_IN_TM_STORE_WAR_M128, UNKNOWN, VOID_FTYPE_PV4SF_V4SF },
27161 { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_WaWM128", (enum ix86_builtins) BUILT_IN_TM_STORE_WAW_M128, UNKNOWN, VOID_FTYPE_PV4SF_V4SF },
27162 { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_RM128", (enum ix86_builtins) BUILT_IN_TM_LOAD_M128, UNKNOWN, V4SF_FTYPE_PCV4SF },
27163 { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_RaRM128", (enum ix86_builtins) BUILT_IN_TM_LOAD_RAR_M128, UNKNOWN, V4SF_FTYPE_PCV4SF },
27164 { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_RaWM128", (enum ix86_builtins) BUILT_IN_TM_LOAD_RAW_M128, UNKNOWN, V4SF_FTYPE_PCV4SF },
27165 { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_RfWM128", (enum ix86_builtins) BUILT_IN_TM_LOAD_RFW_M128, UNKNOWN, V4SF_FTYPE_PCV4SF },
27167 { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_WM256", (enum ix86_builtins) BUILT_IN_TM_STORE_M256, UNKNOWN, VOID_FTYPE_PV8SF_V8SF },
27168 { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_WaRM256", (enum ix86_builtins) BUILT_IN_TM_STORE_WAR_M256, UNKNOWN, VOID_FTYPE_PV8SF_V8SF },
27169 { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_WaWM256", (enum ix86_builtins) BUILT_IN_TM_STORE_WAW_M256, UNKNOWN, VOID_FTYPE_PV8SF_V8SF },
27170 { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_RM256", (enum ix86_builtins) BUILT_IN_TM_LOAD_M256, UNKNOWN, V8SF_FTYPE_PCV8SF },
27171 { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_RaRM256", (enum ix86_builtins) BUILT_IN_TM_LOAD_RAR_M256, UNKNOWN, V8SF_FTYPE_PCV8SF },
27172 { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_RaWM256", (enum ix86_builtins) BUILT_IN_TM_LOAD_RAW_M256, UNKNOWN, V8SF_FTYPE_PCV8SF },
27173 { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_RfWM256", (enum ix86_builtins) BUILT_IN_TM_LOAD_RFW_M256, UNKNOWN, V8SF_FTYPE_PCV8SF },
27175 { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_LM64", (enum ix86_builtins) BUILT_IN_TM_LOG_M64, UNKNOWN, VOID_FTYPE_PCVOID },
27176 { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_LM128", (enum ix86_builtins) BUILT_IN_TM_LOG_M128, UNKNOWN, VOID_FTYPE_PCVOID },
27177 { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_LM256", (enum ix86_builtins) BUILT_IN_TM_LOG_M256, UNKNOWN, VOID_FTYPE_PCVOID },
27178 };
27180 /* TM callbacks. */
27182 /* Return the builtin decl needed to load a vector of TYPE. */
27184 static tree
27186 {
27188 {
27190 {
27197 }
27198 }
27200 }
27202 /* Return the builtin decl needed to store a vector of TYPE. */
27204 static tree
27206 {
27208 {
27210 {
27217 }
27218 }
27220 }
27221 \f
27222 /* Initialize the transactional memory vector load/store builtins. */
27224 static void
27226 {
27230 tree decl;
27237 /* If there are no builtins defined, we must be compiling in a
27238 language without trans-mem support. */
27242 /* Use whatever attributes a normal TM load has. */
27246 /* Use whatever attributes a normal TM store has. */
27250 /* Use whatever attributes a normal TM log has. */
27258 {
27262 {
27270 {
27273 }
27275 {
27278 }
27279 else
27280 {
27283 }
27285 /* The builtin without the prefix for
27286 calling it directly. */
27288 attrs);
27289 /* add_builtin_function() will set the DECL_ATTRIBUTES, now
27290 set the TYPE_ATTRIBUTES. */
27294 }
27295 }
27296 }
27298 /* Set up all the MMX/SSE builtins, even builtins for instructions that are not
27299 in the current target ISA to allow the user to compile particular modules
27300 with different target specific options that differ from the command line
27301 options. */
27302 static void
27304 {
27309 /* Add all special builtins with variable number of operands. */
27313 {
27319 }
27321 /* Add all builtins with variable number of operands. */
27325 {
27331 }
27333 /* pcmpestr[im] insns. */
27337 {
27340 else
27343 }
27345 /* pcmpistr[im] insns. */
27349 {
27352 else
27355 }
27357 /* comi/ucomi insns. */
27359 {
27362 else
27365 }
27367 /* SSE */
27373 /* SSE or 3DNow!A */
27376 IX86_BUILTIN_MASKMOVQ);
27378 /* SSE2 */
27387 /* SSE3. */
27393 /* AES */
27407 /* PCLMUL */
27411 /* RDRND */
27418 IX86_BUILTIN_RDRAND64_STEP);
27420 /* AVX2 */
27422 V2DF_FTYPE_V2DF_PCDOUBLE_V4SI_V2DF_INT,
27423 IX86_BUILTIN_GATHERSIV2DF);
27426 V4DF_FTYPE_V4DF_PCDOUBLE_V4SI_V4DF_INT,
27427 IX86_BUILTIN_GATHERSIV4DF);
27430 V2DF_FTYPE_V2DF_PCDOUBLE_V2DI_V2DF_INT,
27431 IX86_BUILTIN_GATHERDIV2DF);
27434 V4DF_FTYPE_V4DF_PCDOUBLE_V4DI_V4DF_INT,
27435 IX86_BUILTIN_GATHERDIV4DF);
27438 V4SF_FTYPE_V4SF_PCFLOAT_V4SI_V4SF_INT,
27439 IX86_BUILTIN_GATHERSIV4SF);
27442 V8SF_FTYPE_V8SF_PCFLOAT_V8SI_V8SF_INT,
27443 IX86_BUILTIN_GATHERSIV8SF);
27446 V4SF_FTYPE_V4SF_PCFLOAT_V2DI_V4SF_INT,
27447 IX86_BUILTIN_GATHERDIV4SF);
27450 V4SF_FTYPE_V4SF_PCFLOAT_V4DI_V4SF_INT,
27451 IX86_BUILTIN_GATHERDIV8SF);
27454 V2DI_FTYPE_V2DI_PCINT64_V4SI_V2DI_INT,
27455 IX86_BUILTIN_GATHERSIV2DI);
27458 V4DI_FTYPE_V4DI_PCINT64_V4SI_V4DI_INT,
27459 IX86_BUILTIN_GATHERSIV4DI);
27462 V2DI_FTYPE_V2DI_PCINT64_V2DI_V2DI_INT,
27463 IX86_BUILTIN_GATHERDIV2DI);
27466 V4DI_FTYPE_V4DI_PCINT64_V4DI_V4DI_INT,
27467 IX86_BUILTIN_GATHERDIV4DI);
27470 V4SI_FTYPE_V4SI_PCINT_V4SI_V4SI_INT,
27471 IX86_BUILTIN_GATHERSIV4SI);
27474 V8SI_FTYPE_V8SI_PCINT_V8SI_V8SI_INT,
27475 IX86_BUILTIN_GATHERSIV8SI);
27478 V4SI_FTYPE_V4SI_PCINT_V2DI_V4SI_INT,
27479 IX86_BUILTIN_GATHERDIV4SI);
27482 V4SI_FTYPE_V4SI_PCINT_V4DI_V4SI_INT,
27483 IX86_BUILTIN_GATHERDIV8SI);
27486 V4DF_FTYPE_V4DF_PCDOUBLE_V8SI_V4DF_INT,
27487 IX86_BUILTIN_GATHERALTSIV4DF);
27490 V8SF_FTYPE_V8SF_PCFLOAT_V4DI_V8SF_INT,
27491 IX86_BUILTIN_GATHERALTDIV8SF);
27494 V4DI_FTYPE_V4DI_PCINT64_V8SI_V4DI_INT,
27495 IX86_BUILTIN_GATHERALTSIV4DI);
27498 V8SI_FTYPE_V8SI_PCINT_V4DI_V8SI_INT,
27499 IX86_BUILTIN_GATHERALTDIV8SI);
27501 /* RTM. */
27505 /* MMX access to the vec_init patterns. */
27510 V4HI_FTYPE_HI_HI_HI_HI,
27511 IX86_BUILTIN_VEC_INIT_V4HI);
27514 V8QI_FTYPE_QI_QI_QI_QI_QI_QI_QI_QI,
27515 IX86_BUILTIN_VEC_INIT_V8QI);
27517 /* Access to the vec_extract patterns. */
27539 /* Access to the vec_set patterns. */
27560 /* Add FMA4 multi-arg argument instructions */
27562 {
27568 }
27569 }
27571 /* Internal method for ix86_init_builtins. */
27573 static void
27575 {
27587 sysv_va_ref =
27590 fnvoid_va_end_ms =
27592 fnvoid_va_start_ms =
27594 fnvoid_va_end_sysv =
27596 fnvoid_va_start_sysv =
27598 NULL_TREE);
27599 fnvoid_va_copy_ms =
27601 NULL_TREE);
27602 fnvoid_va_copy_sysv =
27618 }
27620 static void
27622 {
27625 /* The __float80 type. */
27628 {
27629 /* The __float80 type. */
27634 }
27637 /* The __float128 type. */
27643 /* This macro is built by i386-builtin-types.awk. */
27644 DEFINE_BUILTIN_PRIMITIVE_TYPES;
27645 }
27647 static void
27649 {
27650 tree t;
27654 /* TFmode support builtins. */
27660 /* We will expand them to normal call if SSE2 isn't available since
27661 they are used by libgcc. */
27680 #ifdef SUBTARGET_INIT_BUILTINS
27681 SUBTARGET_INIT_BUILTINS;
27682 #endif
27683 }
27685 /* Return the ix86 builtin for CODE. */
27687 static tree
27689 {
27694 }
27696 /* Errors in the source file can cause expand_expr to return const0_rtx
27697 where we expect a vector. To avoid crashing, use one of the vector
27698 clear instructions. */
27699 static rtx
27701 {
27705 }
27707 /* Subroutine of ix86_expand_builtin to take care of binop insns. */
27709 static rtx
27711 {
27712 rtx pat;
27732 {
27736 }
27750 }
27752 /* Subroutine of ix86_expand_builtin to take care of 2-4 argument insns. */
27754 static rtx
27758 {
27759 rtx pat;
27767 rtx op;
27774 {
27854 }
27864 {
27871 {
27873 {
27876 {
27894 xop_rotl:
27896 {
27899 gcc_checking_assert
27901 }
27902 else
27903 {
27904 gcc_checking_assert
27915 }
27919 }
27920 }
27921 }
27922 else
27923 {
27924 non_constant:
27928 /* If we aren't optimizing, only allow one memory operand to be
27929 generated. */
27931 num_memory++;
27939 }
27943 }
27946 {
27957 else
27958 {
27964 }
27977 }
27984 }
27986 /* Subroutine of ix86_expand_args_builtin to take care of scalar unop
27987 insns with vec_merge. */
27989 static rtx
27991 rtx target)
27992 {
27993 rtx pat;
28020 }
28022 /* Subroutine of ix86_expand_builtin to take care of comparison insns. */
28024 static rtx
28027 {
28028 rtx pat;
28033 rtx op2;
28044 /* Swap operands if we have a comparison that isn't available in
28045 hardware. */
28047 {
28052 }
28072 }
28074 /* Subroutine of ix86_expand_builtin to take care of comi insns. */
28076 static rtx
28078 rtx target)
28079 {
28080 rtx pat;
28094 /* Swap operands if we have a comparison that isn't available in
28095 hardware. */
28097 {
28101 }
28122 const0_rtx)));
28125 }
28127 /* Subroutines of ix86_expand_args_builtin to take care of round insns. */
28129 static rtx
28131 rtx target)
28132 {
28133 rtx pat;
28158 }
28160 static rtx
28163 {
28164 rtx pat;
28169 rtx op2;
28196 }
28198 /* Subroutine of ix86_expand_builtin to take care of ptest insns. */
28200 static rtx
28202 rtx target)
28203 {
28204 rtx pat;
28237 const0_rtx)));
28240 }
28242 /* Subroutine of ix86_expand_builtin to take care of pcmpestr[im] insns. */
28244 static rtx
28247 {
28248 rtx pat;
28286 {
28289 }
28292 {
28301 }
28303 {
28312 }
28313 else
28314 {
28321 }
28329 {
28334 emit_insn
28338 FLAGS_REG),
28339 const0_rtx)));
28341 }
28342 else
28344 }
28347 /* Subroutine of ix86_expand_builtin to take care of pcmpistr[im] insns. */
28349 static rtx
28352 {
28353 rtx pat;
28381 {
28384 }
28387 {
28396 }
28398 {
28407 }
28408 else
28409 {
28416 }
28424 {
28429 emit_insn
28433 FLAGS_REG),
28434 const0_rtx)));
28436 }
28437 else
28439 }
28441 /* Subroutine of ix86_expand_builtin to take care of insns with
28442 variable number of operands. */
28444 static rtx
28447 {
28452 struct
28453 {
28454 rtx op;
28466 {
28739 }
28744 {
28747 }
28750 {
28757 }
28758 else
28759 {
28762 }
28765 {
28772 {
28773 /* SIMD shift insns take either an 8-bit immediate or
28774 register as count. But builtin functions take int as
28775 count. If count doesn't match, we put it in register. */
28777 {
28781 }
28782 }
28784 {
28787 {
28841 {
28844 {
28847 }
28853 }
28855 }
28856 }
28857 else
28858 {
28862 /* If we aren't optimizing, only allow one memory operand to
28863 be generated. */
28865 num_memory++;
28868 {
28871 }
28872 else
28873 {
28876 }
28877 }
28881 }
28884 {
28901 }
28908 }
28910 /* Subroutine of ix86_expand_builtin to take care of special insns
28911 with variable number of operands. */
28913 static rtx
28916 {
28917 tree arg;
28920 struct
28921 {
28922 rtx op;
28932 {
28982 /* Reserve memory operand for target. */
29013 /* Reserve memory operand for target. */
29027 }
29032 {
29037 {
29041 }
29042 else
29045 }
29046 else
29047 {
29054 }
29057 {
29066 {
29068 {
29074 else
29077 }
29078 }
29079 else
29080 {
29082 {
29083 /* This must be the memory operand. */
29089 }
29090 else
29091 {
29092 /* This must be register. */
29099 }
29100 }
29104 }
29107 {
29122 }
29128 }
29130 /* Return the integer constant in ARG. Constrain it to be in the range
29131 of the subparts of VEC_TYPE; issue an error if not. */
29133 static int
29135 {
29140 {
29143 }
29146 }
29148 /* A subroutine of ix86_expand_builtin. These builtins are a wrapper around
29149 ix86_expand_vector_init. We DO have language-level syntax for this, in
29150 the form of (type){ init-list }. Except that since we can't place emms
29151 instructions from inside the compiler, we can't allow the use of MMX
29152 registers unless the user explicitly asks for it. So we do *not* define
29153 vec_set/vec_extract/vec_init patterns for MMX modes in mmx.md. Instead
29154 we have builtins invoked by mmintrin.h that gives us license to emit
29155 these sorts of instructions. */
29157 static rtx
29159 {
29169 {
29172 }
29179 }
29181 /* A subroutine of ix86_expand_builtin. These builtins are a wrapper around
29182 ix86_expand_vector_extract. They would be redundant (for non-MMX) if we
29183 had a language-level syntax for referencing vector elements. */
29185 static rtx
29187 {
29191 rtx op0;
29211 }
29213 /* A subroutine of ix86_expand_builtin. These builtins are a wrapper around
29214 ix86_expand_vector_set. They would be redundant (for non-MMX) if we had
29215 a language-level syntax for referencing vector elements. */
29217 static rtx
29219 {
29243 /* OP0 is the source of these builtin functions and shouldn't be
29244 modified. Create a copy, use it and return it as target. */
29250 }
29252 /* Expand an expression EXP that calls a built-in function,
29253 with result going to TARGET if that's convenient
29254 (and in mode MODE if that's convenient).
29255 SUBTARGET may be used as the target for computing one of EXP's operands.
29256 IGNORE is nonzero if the value is to be ignored. */
29258 static rtx
29262 {
29272 /* Determine whether the builtin function is available under the current ISA.
29273 Originally the builtin was not created if it wasn't applicable to the
29274 current ISA based on the command line switches. With function specific
29275 options, we need to check in the context of the function making the call
29276 whether it is supported. */
29279 {
29285 else
29286 {
29290 }
29292 }
29295 {
29299 ? CODE_FOR_mmx_maskmovq
29301 /* Note the arg order is different from the operand order. */
29345 {
29349 }
29362 {
29366 }
29411 {
29412 REAL_VALUE_TYPE inf;
29413 rtx tmp;
29425 }
29432 {
29436 }
29455 ? CODE_FOR_tbm_bextri_si
29458 {
29461 }
29462 else
29463 {
29472 }
29488 rdrand_step:
29495 {
29498 }
29504 /* Emit SImode conditional move. */
29506 {
29509 }
29512 else
29519 const0_rtx);
29585 gather_gen:
29596 /* Note the arg order is different from the operand order. */
29605 else
29610 {
29616 }
29619 {
29624 else
29634 }
29636 /* Force memory operand only with base register here. But we
29637 don't want to do it on memory operand for other builtin
29638 functions. */
29652 {
29655 }
29657 /* Optimize. If mask is known to have all high bits set,
29658 replace op0 with pc_rtx to signal that the instruction
29659 overwrites the whole destination and doesn't use its
29660 previous contents. */
29662 {
29664 {
29667 {
29671 negative++;
29674 negative++;
29675 }
29678 }
29680 {
29681 /* Recognize also when mask is like:
29682 __v2df src = _mm_setzero_pd ();
29683 __v2df mask = _mm_cmpeq_pd (src, src);
29684 or
29685 __v8sf src = _mm256_setzero_ps ();
29686 __v8sf mask = _mm256_cmp_ps (src, src, _CMP_EQ_OQ);
29687 as that is a cheaper way to load all ones into
29688 a register than having to load a constant from
29689 memory. */
29692 {
29697 {
29704 /* FALLTHRU */
29714 }
29715 }
29716 }
29717 }
29726 {
29733 else
29735 }
29736 else
29747 {
29750 }
29756 }
29769 {
29773 /* Emit a normal call if SSE2 isn't available. */
29777 }
29802 }
29804 /* Returns a function decl for a vectorized version of the builtin function
29805 with builtin function code FN and the result vector type TYPE, or NULL_TREE
29806 if it is not available. */
29808 static tree
29810 tree type_in)
29811 {
29827 {
29830 {
29835 }
29840 {
29845 }
29851 /* The round insn does not trap on denormals. */
29856 {
29861 }
29867 /* The round insn does not trap on denormals. */
29872 {
29877 }
29883 /* The round insn does not trap on denormals. */
29888 {
29893 }
29899 /* The round insn does not trap on denormals. */
29904 {
29909 }
29916 {
29921 }
29928 {
29933 }
29939 /* The round insn does not trap on denormals. */
29944 {
29949 }
29955 /* The round insn does not trap on denormals. */
29960 {
29965 }
29970 {
29975 }
29980 {
29985 }
29989 /* The round insn does not trap on denormals. */
29994 {
29999 }
30003 /* The round insn does not trap on denormals. */
30008 {
30013 }
30017 /* The round insn does not trap on denormals. */
30022 {
30027 }
30031 /* The round insn does not trap on denormals. */
30036 {
30041 }
30045 /* The round insn does not trap on denormals. */
30050 {
30055 }
30059 /* The round insn does not trap on denormals. */
30064 {
30069 }
30073 /* The round insn does not trap on denormals. */
30078 {
30083 }
30087 /* The round insn does not trap on denormals. */
30092 {
30097 }
30101 /* The round insn does not trap on denormals. */
30106 {
30111 }
30115 /* The round insn does not trap on denormals. */
30120 {
30125 }
30130 {
30135 }
30140 {
30145 }
30150 }
30152 /* Dispatch to a handler for a vectorization library. */
30155 type_in);
30158 }
30160 /* Handler for an SVML-style interface to
30161 a library with vectorized intrinsics. */
30163 static tree
30165 {
30173 /* The SVML is suitable for unsafe math only. */
30186 {
30233 }
30242 {
30245 }
30246 else
30249 /* Convert to uppercase. */
30254 args;
30256 arity++;
30260 else
30263 /* Build a function declaration for the vectorized function. */
30272 }
30274 /* Handler for an ACML-style interface to
30275 a library with vectorized intrinsics. */
30277 static tree
30279 {
30287 /* The ACML is 64bits only and suitable for unsafe math only as
30288 it does not correctly support parts of IEEE with the required
30289 precision such as denormals. */
30303 {
30333 }
30340 args;
30342 arity++;
30346 else
30349 /* Build a function declaration for the vectorized function. */
30358 }
30360 /* Returns a decl of a function that implements gather load with
30361 memory type MEM_VECTYPE and index type INDEX_VECTYPE and SCALE.
30362 Return NULL_TREE if it is not available. */
30364 static tree
30367 {
30383 /* v*gather* insn sign extends index to pointer mode. */
30395 {
30422 }
30425 }
30427 /* Returns a code for a target-specific builtin that implements
30428 reciprocal of the function, or NULL_TREE if not available. */
30430 static tree
30433 {
30440 /* Machine dependent builtins. */
30442 {
30443 /* Vectorized version of sqrt to rsqrt conversion. */
30452 }
30453 else
30454 /* Normal builtins. */
30456 {
30457 /* Sqrt to rsqrt conversion. */
30463 }
30464 }
30465 \f
30466 /* Helper for avx_vpermilps256_operand et al. This is also used by
30467 the expansion functions to turn the parallel back into a mask.
30468 The return value is 0 for no match and the imm8+1 for a match. */
30470 int
30472 {
30480 /* Validate that all of the elements are constants, and not totally
30481 out of range. Copy the data into an integral array to make the
30482 subsequent checks easier. */
30484 {
30494 }
30497 {
30499 /* In the 256-bit DFmode case, we can only move elements within
30500 a 128-bit lane. */
30502 {
30506 }
30508 {
30512 }
30516 /* In the 256-bit SFmode case, we have full freedom of movement
30517 within the low 128-bit lane, but the high 128-bit lane must
30518 mirror the exact same pattern. */
30523 /* FALLTHRU */
30527 /* In the 128-bit case, we've full freedom in the placement of
30528 the elements from the source operand. */
30535 }
30537 /* Make sure success has a non-zero value by adding one. */
30539 }
30541 /* Helper for avx_vperm2f128_v4df_operand et al. This is also used by
30542 the expansion functions to turn the parallel back into a mask.
30543 The return value is 0 for no match and the imm8+1 for a match. */
30545 int
30547 {
30555 /* Validate that all of the elements are constants, and not totally
30556 out of range. Copy the data into an integral array to make the
30557 subsequent checks easier. */
30559 {
30569 }
30571 /* Validate that the halves of the permute are halves. */
30579 /* Reconstruct the mask. */
30581 {
30587 }
30589 /* Make sure success has a non-zero value by adding one. */
30591 }
30592 \f
30593 /* Store OPERAND to the memory after reload is completed. This means
30594 that we can't easily use assign_stack_local. */
30595 rtx
30597 {
30598 rtx result;
30602 {
30605 stack_pointer_rtx,
30608 }
30610 {
30612 {
30616 /* FALLTHRU */
30622 stack_pointer_rtx)),
30623 operand));
30627 }
30629 }
30630 else
30631 {
30633 {
30635 {
30642 stack_pointer_rtx)),
30648 stack_pointer_rtx)),
30650 }
30653 /* Store HImodes as SImodes. */
30655 /* FALLTHRU */
30661 stack_pointer_rtx)),
30662 operand));
30666 }
30668 }
30670 }
30672 /* Free operand from the memory. */
30673 void
30675 {
30677 {
30682 else
30684 /* Use LEA to deallocate stack space. In peephole2 it will be converted
30685 to pop or add instruction if registers are available. */
30689 }
30690 }
30692 /* Implement TARGET_PREFERRED_RELOAD_CLASS.
30694 Put float CONST_DOUBLE in the constant pool instead of fp regs.
30695 QImode must go into class Q_REGS.
30696 Narrow ALL_REGS to GENERAL_REGS. This supports allowing movsf and
30697 movdf to do mem-to-mem moves through integer regs. */
30699 static reg_class_t
30701 {
30704 /* We're only allowed to return a subclass of CLASS. Many of the
30705 following checks fail for NO_REGS, so eliminate that early. */
30709 /* All classes can load zeros. */
30713 /* Force constants into memory if we are loading a (nonzero) constant into
30714 an MMX or SSE register. This is because there are no MMX/SSE instructions
30715 to load from a constant. */
30720 /* Prefer SSE regs only, if we can use them for math. */
30724 /* Floating-point constants need more complex checks. */
30726 {
30727 /* General regs can load everything. */
30731 /* Floats can load 0 and 1 plus some others. Note that we eliminated
30732 zero above. We only want to wind up preferring 80387 registers if
30733 we plan on doing computation with them. */
30736 {
30737 /* Limit class to non-sse. */
30746 }
30749 }
30751 /* Generally when we see PLUS here, it's the function invariant
30752 (plus soft-fp const_int). Which can only be computed into general
30753 regs. */
30757 /* QImode constants are easy to load, but non-constant QImode data
30758 must go into Q_REGS. */
30760 {
30766 }
30769 }
30771 /* Discourage putting floating-point values in SSE registers unless
30772 SSE math is being used, and likewise for the 387 registers. */
30773 static reg_class_t
30775 {
30778 /* Restrict the output reload class to the register bank that we are doing
30779 math on. If we would like not to return a subset of CLASS, reject this
30780 alternative: if reload cannot do this, it will still use its choice. */
30786 {
30791 else
30793 }
30796 }
30798 static reg_class_t
30801 {
30802 /* Double-word spills from general registers to non-offsettable memory
30803 references (zero-extended addresses) require special handling. */
30809 {
30811 ? CODE_FOR_reload_noff_load
30813 /* Add the cost of moving address to a temporary. */
30817 }
30819 /* QImode spills from non-QI registers require
30820 intermediate register on 32bit targets. */
30826 {
30831 else
30837 /* Return Q_REGS if the operand is in memory. */
30840 }
30842 /* This condition handles corner case where an expression involving
30843 pointers gets vectorized. We're trying to use the address of a
30844 stack slot as a vector initializer.
30846 (set (reg:V2DI 74 [ vect_cst_.2 ])
30847 (vec_duplicate:V2DI (reg/f:DI 20 frame)))
30849 Eventually frame gets turned into sp+offset like this:
30851 (set (reg:V2DI 21 xmm0 [orig:74 vect_cst_.2 ] [74])
30852 (vec_duplicate:V2DI (plus:DI (reg/f:DI 7 sp)
30853 (const_int 392 [0x188]))))
30855 That later gets turned into:
30857 (set (reg:V2DI 21 xmm0 [orig:74 vect_cst_.2 ] [74])
30858 (vec_duplicate:V2DI (plus:DI (reg/f:DI 7 sp)
30859 (mem/u/c/i:DI (symbol_ref/u:DI ("*.LC0") [flags 0x2]) [0 S8 A64]))))
30861 We'll have the following reload recorded:
30863 Reload 0: reload_in (DI) =
30864 (plus:DI (reg/f:DI 7 sp)
30865 (mem/u/c/i:DI (symbol_ref/u:DI ("*.LC0") [flags 0x2]) [0 S8 A64]))
30866 reload_out (V2DI) = (reg:V2DI 21 xmm0 [orig:74 vect_cst_.2 ] [74])
30867 SSE_REGS, RELOAD_OTHER (opnum = 0), can't combine
30868 reload_in_reg: (plus:DI (reg/f:DI 7 sp) (const_int 392 [0x188]))
30869 reload_out_reg: (reg:V2DI 21 xmm0 [orig:74 vect_cst_.2 ] [74])
30870 reload_reg_rtx: (reg:V2DI 22 xmm1)
30872 Which isn't going to work since SSE instructions can't handle scalar
30873 additions. Returning GENERAL_REGS forces the addition into integer
30874 register and reload can handle subsequent reloads without problems. */
30882 }
30884 /* Implement TARGET_CLASS_LIKELY_SPILLED_P. */
30886 static bool
30888 {
30890 {
30905 }
30908 }
30910 /* If we are copying between general and FP registers, we need a memory
30911 location. The same is true for SSE and MMX registers.
30913 To optimize register_move_cost performance, allow inline variant.
30915 The macro can't work reliably when one of the CLASSES is class containing
30916 registers from multiple units (SSE, MMX, integer). We avoid this by never
30917 combining those units in single alternative in the machine description.
30918 Ensure that this constraint holds to avoid unexpected surprises.
30920 When STRICT is false, we are being called from REGISTER_MOVE_COST, so do not
30921 enforce these sanity checks. */
30926 {
30933 {
30936 }
30941 /* ??? This is a lie. We do have moves between mmx/general, and for
30942 mmx/sse2. But by saying we need secondary memory we discourage the
30943 register allocator from using the mmx registers unless needed. */
30948 {
30949 /* SSE1 doesn't have any direct moves from other classes. */
30953 /* If the target says that inter-unit moves are more expensive
30954 than moving through memory, then don't generate them. */
30958 /* Between SSE and general, we have moves no larger than word size. */
30961 }
30964 }
30966 bool
30969 {
30971 }
30973 /* Implement the TARGET_CLASS_MAX_NREGS hook.
30975 On the 80386, this is the size of MODE in words,
30976 except in the FP regs, where a single reg is always enough. */
30978 static unsigned char
30980 {
30982 {
30987 else
30989 }
30990 else
30991 {
30994 else
30996 }
30997 }
30999 /* Return true if the registers in CLASS cannot represent the change from
31000 modes FROM to TO. */
31002 bool
31005 {
31009 /* x87 registers can't do subreg at all, as all values are reformatted
31010 to extended precision. */
31015 {
31016 /* Vector registers do not support QI or HImode loads. If we don't
31017 disallow a change to these modes, reload will assume it's ok to
31018 drop the subreg from (subreg:SI (reg:HI 100) 0). This affects
31019 the vec_dupv4hi pattern. */
31023 /* Vector registers do not support subreg with nonzero offsets, which
31024 are otherwise valid for integer registers. Since we can't see
31025 whether we have a nonzero offset from here, prohibit all
31026 nonparadoxical subregs changing size. */
31029 }
31032 }
31034 /* Return the cost of moving data of mode M between a
31035 register and memory. A value of 2 is the default; this cost is
31036 relative to those in `REGISTER_MOVE_COST'.
31038 This function is used extensively by register_move_cost that is used to
31039 build tables at startup. Make it inline in this case.
31040 When IN is 2, return maximum of in and out move cost.
31042 If moving between registers and memory is more expensive than
31043 between two registers, you should define this macro to express the
31044 relative cost.
31046 Model also increased moving costs of QImode registers in non
31047 Q_REGS classes.
31048 */
31052 {
31055 {
31058 {
31070 }
31074 }
31076 {
31079 {
31091 }
31095 }
31097 {
31100 {
31109 }
31113 }
31115 {
31118 {
31124 else
31129 }
31130 else
31131 {
31136 else
31138 }
31145 /* Compute number of 32bit moves needed. TFmode is moved as XFmode. */
31152 else
31156 }
31157 }
31159 static int
31162 {
31164 }
31167 /* Return the cost of moving data from a register in class CLASS1 to
31168 one in class CLASS2.
31170 It is not required that the cost always equal 2 when FROM is the same as TO;
31171 on some machines it is expensive to move between registers if they are not
31172 general registers. */
31174 static int
31176 reg_class_t class2_i)
31177 {
31181 /* In case we require secondary memory, compute cost of the store followed
31182 by load. In order to avoid bad register allocation choices, we need
31183 for this to be *at least* as high as the symmetric MEMORY_MOVE_COST. */
31186 {
31192 /* In case of copying from general_purpose_register we may emit multiple
31193 stores followed by single load causing memory size mismatch stall.
31194 Count this as arbitrarily high cost of 20. */
31199 /* In the case of FP/MMX moves, the registers actually overlap, and we
31200 have to switch modes in order to treat them differently. */
31206 }
31208 /* Moves between SSE/MMX and integer unit are expensive. */
31212 /* ??? By keeping returned value relatively high, we limit the number
31213 of moves between integer and MMX/SSE registers for all targets.
31214 Additionally, high value prevents problem with x86_modes_tieable_p(),
31215 where integer modes in MMX/SSE registers are not tieable
31216 because of missing QImode and HImode moves to, from or between
31217 MMX/SSE registers. */
31227 }
31229 /* Return TRUE if hard register REGNO can hold a value of machine-mode
31230 MODE. */
31232 bool
31234 {
31235 /* Flags and only flags can only hold CCmode values. */
31245 {
31246 /* We implement the move patterns for all vector modes into and
31247 out of SSE registers, even when no operation instructions
31248 are available. OImode move is available only when AVX is
31249 enabled. */
31256 }
31258 {
31259 /* We implement the move patterns for 3DNOW modes even in MMX mode,
31260 so if the register is available at all, then we can move data of
31261 the given mode into or out of it. */
31264 }
31267 {
31268 /* Take care for QImode values - they can be in non-QI regs,
31269 but then they do cause partial register stalls. */
31275 }
31276 /* We handle both integer and floats in the general purpose registers. */
31283 /* Lots of MMX code casts 8 byte vector modes to DImode. If we then go
31284 on to use that value in smaller contexts, this can easily force a
31285 pseudo to be allocated to GENERAL_REGS. Since this is no worse than
31286 supporting DImode, allow it. */
31291 }
31293 /* A subroutine of ix86_modes_tieable_p. Return true if MODE is a
31294 tieable integer mode. */
31296 static bool
31298 {
31300 {
31313 }
31314 }
31316 /* Return true if MODE1 is accessible in a register that can hold MODE2
31317 without copying. That is, all register classes that can hold MODE2
31318 can also hold MODE1. */
31320 bool
31322 {
31330 /* MODE2 being XFmode implies fp stack or general regs, which means we
31331 can tie any smaller floating point modes to it. Note that we do not
31332 tie this with TFmode. */
31336 /* MODE2 being DFmode implies fp stack, general or sse regs, which means
31337 that we can tie it with SFmode. */
31341 /* If MODE2 is only appropriate for an SSE register, then tie with
31342 any other mode acceptable to SSE registers. */
31348 /* If MODE2 is appropriate for an MMX register, then tie
31349 with any other mode acceptable to MMX registers. */
31356 }
31358 /* Compute a (partial) cost for rtx X. Return true if the complete
31359 cost has been computed, and false if subexpressions should be
31360 scanned. In either case, *TOTAL contains the cost result. */
31362 static bool
31365 {
31371 {
31381 && (!TARGET_64BIT
31386 else
31393 else
31395 {
31404 /* Start with (MEM (SYMBOL_REF)), since that's where
31405 it'll probably end up. Add a penalty for size. */
31410 }
31414 /* The zero extensions is often completely free on x86_64, so make
31415 it as cheap as possible. */
31421 else
31432 {
31435 {
31438 }
31441 {
31444 }
31445 }
31446 /* FALLTHRU */
31453 {
31455 {
31458 else
31460 }
31461 else
31462 {
31465 else
31467 }
31468 }
31469 else
31470 {
31473 else
31475 }
31479 {
31480 rtx sub;
31485 /* ??? SSE scalar/vector cost should be used here. */
31486 /* ??? Bald assumption that fma has the same cost as fmul. */
31490 /* Negate in op0 or op2 is free: FMS, FNMA, FNMS. */
31501 }
31505 {
31506 /* ??? SSE scalar cost should be used here. */
31509 }
31511 {
31514 }
31516 {
31517 /* ??? SSE vector cost should be used here. */
31520 }
31521 else
31522 {
31527 {
31530 nbits++;
31531 }
31532 else
31533 /* This is arbitrary. */
31536 /* Compute costs correctly for widening multiplication. */
31540 {
31547 {
31551 else
31553 }
31557 }
31565 }
31572 /* ??? SSE cost should be used here. */
31577 /* ??? SSE vector cost should be used here. */
31579 else
31586 {
31591 {
31594 {
31602 }
31603 }
31606 {
31609 {
31615 }
31616 }
31618 {
31626 }
31627 }
31628 /* FALLTHRU */
31632 {
31633 /* ??? SSE cost should be used here. */
31636 }
31638 {
31641 }
31643 {
31644 /* ??? SSE vector cost should be used here. */
31647 }
31648 /* FALLTHRU */
31654 {
31661 }
31662 /* FALLTHRU */
31666 {
31667 /* ??? SSE cost should be used here. */
31670 }
31672 {
31675 }
31677 {
31678 /* ??? SSE vector cost should be used here. */
31681 }
31682 /* FALLTHRU */
31687 else
31696 {
31697 /* This kind of construct is implemented using test[bwl].
31698 Treat it as if we had an AND. */
31703 }
31713 /* ??? SSE cost should be used here. */
31718 /* ??? SSE vector cost should be used here. */
31724 /* ??? SSE cost should be used here. */
31729 /* ??? SSE vector cost should be used here. */
31742 /* ??? Assume all of these vector manipulation patterns are
31743 recognizable. In which case they all pretty much have the
31744 same cost. */
31750 }
31751 }
31753 #if TARGET_MACHO
31757 /* Given a symbol name and its associated stub, write out the
31758 definition of the stub. */
31760 void
31762 {
31767 /* For 64-bit we shouldn't get here. */
31770 /* Lose our funky encoding stuff so it doesn't contaminate the stub. */
31787 else
31794 {
31796 }
31798 {
31799 /* PIC stub. */
31800 /* 25-byte PIC stub using "CALL get_pc_thunk". */
31806 }
31807 else
31810 /* The AT&T-style ("self-modifying") stub is not lazily bound, thus
31811 it needs no stub-binding-helper. */
31818 {
31821 }
31822 else
31827 /* N.B. Keep the correspondence of these
31828 'symbol_ptr/symbol_ptr2/symbol_ptr3' sections consistent with the
31829 old-pic/new-pic/non-pic stubs; altering this will break
31830 compatibility with existing dylibs. */
31832 {
31833 /* 25-byte PIC stub using "CALL get_pc_thunk". */
31835 }
31836 else
31837 /* 16-byte -mdynamic-no-pic stub. */
31843 }
31846 /* Order the registers for register allocator. */
31848 void
31850 {
31854 /* First allocate the local general purpose registers. */
31859 /* Global general purpose registers. */
31864 /* x87 registers come first in case we are doing FP math
31865 using them. */
31870 /* SSE registers. */
31876 /* x87 registers. */
31884 /* Initialize the rest of array as we do not allocate some registers
31885 at all. */
31888 }
31890 /* Handle a "callee_pop_aggregate_return" attribute; arguments as
31891 in struct attribute_spec handler. */
31892 static tree
31894 tree args,
31897 {
31902 {
31904 name);
31907 }
31909 {
31911 name);
31914 }
31916 {
31917 tree cst;
31921 {
31924 name);
31926 }
31929 {
31932 name);
31934 }
31937 }
31940 }
31942 /* Handle a "ms_abi" or "sysv" attribute; arguments as in
31943 struct attribute_spec.handler. */
31944 static tree
31946 tree args ATTRIBUTE_UNUSED,
31948 {
31953 {
31955 name);
31958 }
31960 /* Can combine regparm with all attributes but fastcall. */
31962 {
31964 {
31966 }
31969 }
31971 {
31973 {
31975 }
31978 }
31981 }
31983 /* Handle a "ms_struct" or "gcc_struct" attribute; arguments as in
31984 struct attribute_spec.handler. */
31985 static tree
31987 tree args ATTRIBUTE_UNUSED,
31989 {
31992 {
31995 }
31996 else
32001 {
32003 name);
32005 }
32011 {
32013 name);
32015 }
32018 }
32020 static tree
32022 tree args ATTRIBUTE_UNUSED,
32024 {
32026 {
32028 name);
32030 }
32032 }
32034 static bool
32036 {
32040 }
32042 /* Returns an expression indicating where the this parameter is
32043 located on entry to the FUNCTION. */
32045 static rtx
32047 {
32053 {
32058 else
32061 }
32066 {
32073 {
32078 }
32079 else
32080 {
32083 {
32088 }
32089 }
32091 }
32094 }
32096 /* Determine whether x86_output_mi_thunk can succeed. */
32098 static bool
32100 HOST_WIDE_INT delta ATTRIBUTE_UNUSED,
32102 {
32103 /* 64-bit can handle anything. */
32107 /* For 32-bit, everything's fine if we have one free register. */
32111 /* Need a free register for vcall_offset. */
32115 /* Need a free register for GOT references. */
32119 /* Otherwise ok. */
32121 }
32123 /* Output the assembler code for a thunk function. THUNK_DECL is the
32124 declaration for the thunk function itself, FUNCTION is the decl for
32125 the target function. DELTA is an immediate constant offset to be
32126 added to THIS. If VCALL_OFFSET is nonzero, the word at
32127 *(*this + vcall_offset) should be added to THIS. */
32129 static void
32133 {
32139 /* If VCALL_OFFSET, we'll need THIS in a register. Might as well
32140 pull it in now and let DELTA benefit. */
32144 {
32145 /* Put the this parameter into %eax. */
32148 }
32149 else
32152 /* Adjust the this parameter by a fixed constant. */
32154 {
32159 {
32161 {
32165 }
32166 }
32169 }
32171 /* Adjust the this parameter by a value stored in the vtable. */
32173 {
32179 else
32180 {
32184 else
32186 }
32194 /* Adjust the this parameter. */
32198 {
32202 }
32209 vcall_mem));
32210 else
32212 }
32214 /* If necessary, drop THIS back to its stack slot. */
32220 {
32223 ;
32224 else
32225 {
32229 }
32230 }
32231 else
32232 {
32234 ;
32235 #if TARGET_MACHO
32237 {
32240 }
32242 else
32243 {
32250 }
32251 }
32253 /* Our sibling call patterns do not allow memories, because we have no
32254 predicate that can distinguish between frame and non-frame memory.
32255 For our purposes here, we can get away with (ab)using a jump pattern,
32256 because we're going to do no optimization. */
32259 else
32260 {
32265 }
32268 /* Emit just enough of rest_of_compilation to get the insns emitted.
32269 Note that use_thunk calls assemble_start_function et al. */
32276 }
32278 static void
32280 {
32282 #if TARGET_MACHO
32284 #endif
32291 }
32293 int
32295 {
32307 }
32309 /* Output assembler code to FILE to increment profiler label # LABELNO
32310 for profiling a function entry. */
32311 void
32313 {
32318 {
32319 #ifndef NO_PROFILE_COUNTERS
32321 #endif
32325 else
32327 }
32329 {
32330 #ifndef NO_PROFILE_COUNTERS
32333 #endif
32335 }
32336 else
32337 {
32338 #ifndef NO_PROFILE_COUNTERS
32341 #endif
32343 }
32344 }
32346 /* We don't have exact information about the insn sizes, but we may assume
32347 quite safely that we are informed about all 1 byte insns and memory
32348 address sizes. This is enough to eliminate unnecessary padding in
32349 99% of cases. */
32351 static int
32353 {
32359 /* Discard alignments we've emit and jump instructions. */
32366 /* Important case - calls are always 5 bytes.
32367 It is common to have many calls in the row. */
32376 /* For normal instructions we rely on get_attr_length being exact,
32377 with a few exceptions. */
32379 {
32383 {
32393 /* Otherwise trust get_attr_length. */
32395 }
32400 }
32403 else
32405 }
32407 #ifdef ASM_OUTPUT_MAX_SKIP_PAD
32409 /* AMD K8 core mispredicts jumps when there are more than 3 jumps in 16 byte
32410 window. */
32412 static void
32414 {
32419 /* Look for all minimal intervals of instructions containing 4 jumps.
32420 The intervals are bounded by START and INSN. NBYTES is the total
32421 size of instructions in the interval including INSN and not including
32422 START. When the NBYTES is smaller than 16 bytes, it is possible
32423 that the end of START and INSN ends up in the same 16byte page.
32425 The smallest offset in the page INSN can start is the case where START
32426 ends on the offset 0. Offset of INSN is then NBYTES - sizeof (INSN).
32427 We add p2align to 16byte window with maxskip 15 - NBYTES + sizeof (INSN).
32428 */
32430 {
32434 {
32440 /* If align > 3, only up to 16 - max_skip - 1 bytes can be
32441 already in the current 16 byte page, because otherwise
32442 ASM_OUTPUT_MAX_SKIP_ALIGN could skip max_skip or fewer
32443 bytes to reach 16 byte boundary. */
32451 {
32453 {
32460 else
32463 }
32464 }
32466 }
32477 njumps++;
32478 else
32482 {
32489 else
32492 }
32499 {
32506 }
32507 }
32508 }
32509 #endif
32511 /* AMD Athlon works faster
32512 when RET is not destination of conditional jump or directly preceded
32513 by other jump instruction. We avoid the penalty by inserting NOP just
32514 before the RET instructions in such cases. */
32515 static void
32517 {
32518 edge e;
32519 edge_iterator ei;
32522 {
32525 rtx prev;
32535 {
32536 edge e;
32537 edge_iterator ei;
32543 }
32545 {
32551 /* Empty functions get branch mispredict even when
32552 the jump destination is not visible to us. */
32555 }
32557 {
32560 }
32561 }
32562 }
32564 /* Count the minimum number of instructions in BB. Return 4 if the
32565 number of instructions >= 4. */
32567 static int
32569 {
32570 rtx insn;
32573 /* Count number of instructions in this block. Return 4 if the number
32574 of instructions >= 4. */
32576 {
32577 /* Only happen in exit blocks. */
32585 {
32586 insn_count++;
32589 }
32590 }
32593 }
32596 /* Count the minimum number of instructions in code path in BB.
32597 Return 4 if the number of instructions >= 4. */
32599 static int
32601 {
32602 edge e;
32603 edge_iterator ei;
32606 /* Only bother counting instructions along paths with no
32607 more than 2 basic blocks between entry and exit. Given
32608 that BB has an edge to exit, determine if a predecessor
32609 of BB has an edge from entry. If so, compute the number
32610 of instructions in the predecessor block. If there
32611 happen to be multiple such blocks, compute the minimum. */
32614 {
32615 edge prev_e;
32616 edge_iterator prev_ei;
32619 {
32622 }
32624 {
32626 {
32631 }
32632 }
32633 }
32639 }
32641 /* Pad short funtion to 4 instructions. */
32643 static void
32645 {
32646 edge e;
32647 edge_iterator ei;
32650 {
32653 {
32656 /* Pad short function. */
32658 {
32661 /* Find epilogue. */
32670 /* Two NOPs count as one instruction. */
32673 }
32674 }
32675 }
32676 }
32678 /* Implement machine specific optimizations. We implement padding of returns
32679 for K8 CPUs and pass to avoid 4 jumps in the single 16 byte window. */
32680 static void
32682 {
32683 /* We are freeing block_for_insn in the toplev to keep compatibility
32684 with old MDEP_REORGS that are not CFG based. Recompute it now. */
32687 /* Run the vzeroupper optimization if needed. */
32692 {
32697 #ifdef ASM_OUTPUT_MAX_SKIP_PAD
32700 #endif
32701 }
32702 }
32704 /* Return nonzero when QImode register that must be represented via REX prefix
32705 is used. */
32706 bool
32708 {
32716 }
32718 /* Return nonzero when P points to register encoded via REX prefix.
32719 Called via for_each_rtx. */
32720 static int
32722 {
32728 }
32730 /* Return true when INSN mentions register that must be encoded using REX
32731 prefix. */
32732 bool
32734 {
32737 }
32739 /* If profitable, negate (without causing overflow) integer constant
32740 of mode MODE at location LOC. Return true in this case. */
32741 bool
32743 {
32744 HOST_WIDE_INT val;
32750 {
32752 /* DImode x86_64 constants must fit in 32 bits. */
32765 }
32767 /* Avoid overflows. */
32773 /* Make things pretty and `subl $4,%eax' rather than `addl $-4,%eax'.
32774 Exceptions: -128 encodes smaller than 128, so swap sign and op. */
32777 {
32780 }
32783 }
32785 /* Generate an unsigned DImode/SImode to FP conversion. This is the same code
32786 optabs would emit if we didn't have TFmode patterns. */
32788 void
32790 {
32824 }
32825 \f
32826 /* AVX2 does support 32-byte integer vector operations,
32827 thus the longest vector we are faced with is V32QImode. */
32828 #define MAX_VECT_LEN 32
32830 struct expand_vec_perm_d
32831 {
32837 };
32842 /* Get a vector mode of the same size as the original but with elements
32843 twice as wide. This is only guaranteed to apply to integral vectors. */
32847 {
32848 /* ??? Rely on the ordering that genmodes.c gives to vectors. */
32853 }
32855 /* A subroutine of ix86_expand_vector_init. Store into TARGET a vector
32856 with all elements equal to VAR. Return true if successful. */
32858 static bool
32861 {
32865 {
32870 /* FALLTHRU */
32880 {
32883 /* First attempt to recognize VAL as-is. */
32887 {
32888 rtx seq;
32889 /* If that fails, force VAL into a register. */
32900 }
32901 }
32908 {
32909 rtx x;
32916 }
32926 {
32930 permute:
32937 /* Extend to SImode using a paradoxical SUBREG. */
32941 /* Insert the SImode value as low element of a V4SImode vector. */
32949 }
32957 widen:
32958 /* Replicate the value once into the next wider mode and recurse. */
32959 {
32961 rtx x;
32977 }
32981 {
32990 }
32995 }
32996 }
32998 /* A subroutine of ix86_expand_vector_init. Store into TARGET a vector
32999 whose ONE_VAR element is VAR, and other elements are zero. Return true
33000 if successful. */
33002 static bool
33005 {
33007 rtx new_target;
33012 {
33014 /* For SSE4.1, we normally use vector set. But if the second
33015 element is zero and inter-unit moves are OK, we use movq
33016 instead. */
33017 use_vector_set = (TARGET_64BIT
33018 && TARGET_SSE4_1
33019 && !(TARGET_INTER_UNIT_MOVES
33041 /* Use ix86_expand_vector_set in 64bit mode only. */
33046 }
33049 {
33054 }
33057 {
33062 /* FALLTHRU */
33077 else
33084 {
33085 /* We need to shuffle the value to the correct position, so
33086 create a new pseudo to store the intermediate result. */
33088 /* With SSE2, we can use the integer shuffle insns. */
33090 {
33092 const1_rtx,
33099 }
33101 /* Otherwise convert the intermediate result to V4SFmode and
33102 use the SSE1 shuffle instructions. */
33104 {
33107 }
33108 else
33112 const1_rtx,
33121 }
33136 widen:
33140 /* Zero extend the variable element to SImode and recurse. */
33153 }
33154 }
33156 /* A subroutine of ix86_expand_vector_init. Store into TARGET a vector
33157 consisting of the values in VALS. It is known that all elements
33158 except ONE_VAR are constants. Return true if successful. */
33160 static bool
33163 {
33173 {
33178 /* For the two element vectors, it's just as easy to use
33179 the general case. */
33183 /* Use ix86_expand_vector_set in 64bit mode only. */
33205 widen:
33206 /* There's no way to set one QImode entry easily. Combine
33207 the variable value with its adjacent constant value, and
33208 promote to an HImode set. */
33211 {
33216 }
33217 else
33218 {
33221 }
33235 }
33240 }
33242 /* A subroutine of ix86_expand_vector_init_general. Use vector
33243 concatenate to handle the most general case: all values variable,
33244 and none identical. */
33246 static void
33249 {
33252 rtvec v;
33256 {
33259 {
33292 }
33305 {
33320 }
33325 {
33336 }
33339 half:
33340 /* FIXME: We process inputs backward to help RA. PR 36222. */
33344 {
33349 }
33353 {
33356 {
33360 }
33363 }
33364 else
33370 }
33371 }
33373 /* A subroutine of ix86_expand_vector_init_general. Use vector
33374 interleave to handle the most general case: all values variable,
33375 and none identical. */
33377 static void
33380 {
33389 {
33410 }
33413 {
33414 /* Extend the odd elment to SImode using a paradoxical SUBREG. */
33418 /* Insert the SImode value as low element of V4SImode vector. */
33422 op0),
33424 const1_rtx);
33427 /* Cast the V4SImode vector back to a vector in orignal mode. */
33431 /* Load even elements into the second positon. */
33435 const1_rtx));
33437 /* Cast vector to FIRST_IMODE vector. */
33440 }
33442 /* Interleave low FIRST_IMODE vectors. */
33444 {
33448 /* Cast FIRST_IMODE vector to SECOND_IMODE vector. */
33451 }
33453 /* Interleave low SECOND_IMODE vectors. */
33455 {
33458 {
33463 /* Cast the SECOND_IMODE vector to the THIRD_IMODE
33464 vector. */
33467 }
33470 /* FALLTHRU */
33477 /* Cast the SECOND_IMODE vector back to a vector on original
33478 mode. */
33485 }
33486 }
33488 /* A subroutine of ix86_expand_vector_init. Handle the most general case:
33489 all values variable, and none identical. */
33491 static void
33494 {
33500 {
33505 /* FALLTHRU */
33529 half:
33546 /* FALLTHRU */
33552 /* Don't use ix86_expand_vector_init_interleave if we can't
33553 move from GPR to SSE register directly. */
33569 }
33571 {
33583 {
33587 {
33593 else
33594 {
33599 }
33600 }
33603 }
33608 {
33614 }
33616 {
33622 }
33623 else
33625 }
33626 }
33628 /* Initialize vector TARGET via VALS. Suppress the use of MMX
33629 instructions unless MMX_OK is true. */
33631 void
33633 {
33640 rtx x;
33643 {
33653 }
33655 /* Constants are best loaded from the constant pool. */
33657 {
33660 }
33662 /* If all values are identical, broadcast the value. */
33668 /* Values where only one field is non-constant are best loaded from
33669 the pool and overwritten via move later. */
33671 {
33675 one_var))
33680 }
33683 }
33685 void
33687 {
33692 rtx tmp;
33694 = {
33701 };
33703 = {
33710 };
33714 {
33718 {
33723 else
33727 }
33739 else
33745 {
33748 /* For the two element vectors, we implement a VEC_CONCAT with
33749 the extraction of the other element. */
33756 else
33761 }
33770 {
33776 /* tmp = target = A B C D */
33778 /* target = A A B B */
33780 /* target = X A B B */
33782 /* target = A X C D */
33789 /* tmp = target = A B C D */
33791 /* tmp = X B C D */
33793 /* target = A B X D */
33800 /* tmp = target = A B C D */
33802 /* tmp = X B C D */
33804 /* target = A B X D */
33812 }
33820 /* Element 0 handled by vec_merge below. */
33822 {
33825 }
33828 {
33829 /* With SSE2, use integer shuffles to swap element 0 and ELT,
33830 store into element 0, then shuffle them back. */
33847 }
33848 else
33849 {
33850 /* For SSE1, we have to reuse the V4SF code. */
33853 }
33906 half:
33907 /* Compute offset. */
33913 /* Extract the half. */
33917 /* Put val in tmp at elt. */
33920 /* Put it back. */
33926 }
33929 {
33933 }
33934 else
33935 {
33944 }
33945 }
33947 void
33949 {
33953 rtx tmp;
33956 {
33961 /* FALLTHRU */
33974 {
33994 }
34006 {
34008 {
34028 }
34032 }
34033 else
34034 {
34035 /* For SSE1, we have to reuse the V4SF code. */
34039 }
34055 {
34059 else
34063 }
34068 {
34072 else
34076 }
34081 {
34085 else
34089 }
34094 {
34098 else
34102 }
34107 {
34111 else
34115 }
34120 {
34124 else
34128 }
34132 /* ??? Could extract the appropriate HImode element and shift. */
34135 }
34138 {
34142 /* Let the rtl optimizers know about the zero extension performed. */
34144 {
34147 }
34150 }
34151 else
34152 {
34159 }
34160 }
34162 /* Generate code to copy vector bits i / 2 ... i - 1 from vector SRC
34163 to bits 0 ... i / 2 - 1 of vector DEST, which has the same mode.
34164 The upper bits of DEST are undefined, though they shouldn't cause
34165 exceptions (some bits from src or all zeros are ok). */
34167 static void
34169 {
34170 rtx tem;
34172 {
34176 else
34194 else
34201 else
34212 const1_rtx);
34213 else
34220 }
34222 }
34224 /* Expand a vector reduction. FN is the binary pattern to reduce;
34225 DEST is the destination; IN is the input vector. */
34227 void
34229 {
34234 /* SSE4 has a special instruction for V8HImode UMIN reduction. */
34238 {
34241 }
34246 {
34251 else
34255 }
34256 }
34257 \f
34258 /* Target hook for scalar_mode_supported_p. */
34259 static bool
34261 {
34266 else
34268 }
34270 /* Implements target hook vector_mode_supported_p. */
34271 static bool
34273 {
34285 }
34287 /* Target hook for c_mode_for_suffix. */
34288 static enum machine_mode
34290 {
34297 }
34299 /* Worker function for TARGET_MD_ASM_CLOBBERS.
34301 We do this in the new i386 backend to maintain source compatibility
34302 with the old cc0-based compiler. */
34304 static tree
34306 tree inputs ATTRIBUTE_UNUSED,
34307 tree clobbers)
34308 {
34310 clobbers);
34312 clobbers);
34314 }
34316 /* Implements target vector targetm.asm.encode_section_info. */
34318 static void ATTRIBUTE_UNUSED
34320 {
34327 }
34329 /* Worker function for REVERSE_CONDITION. */
34331 enum rtx_code
34333 {
34337 }
34339 /* Output code to perform an x87 FP register move, from OPERANDS[1]
34340 to OPERANDS[0]. */
34344 {
34346 {
34349 {
34353 }
34357 }
34359 {
34363 else
34364 {
34365 /* There is no non-popping store to memory for XFmode.
34366 So if we need one, follow the store with a load. */
34369 else
34371 }
34372 }
34373 else
34375 }
34377 /* Output code to perform a conditional jump to LABEL, if C2 flag in
34378 FP status register is set. */
34380 void
34382 {
34384 rtx temp;
34389 {
34394 }
34395 else
34396 {
34401 }
34405 pc_rtx);
34410 }
34412 /* Output code to perform a log1p XFmode calculation. */
34415 {
34421 rtx test;
34427 XFmode));
34441 }
34443 /* Emit code for round calculation. */
34445 {
34452 rtx insn;
34457 {
34469 }
34472 {
34493 }
34501 /* round(a) = sgn(a) * floor(fabs(a) + 0.5) */
34503 /* scratch = fxam(op1) */
34506 UNSPEC_FXAM)));
34507 /* e1 = fabs(op1) */
34510 /* e2 = e1 + 0.5 */
34515 /* res = floor(e2) */
34517 {
34522 }
34523 else
34527 {
34530 {
34537 UNSPEC_TRUNC_NOOP)));
34538 }
34554 }
34556 /* flags = signbit(a) */
34559 /* if (flags) then res = -res */
34563 pc_rtx);
34574 }
34576 /* Output code to perform a Newton-Rhapson approximation of a single precision
34577 floating point divide [http://en.wikipedia.org/wiki/N-th_root_algorithm]. */
34580 {
34588 /* a / b = a * ((rcp(b) + rcp(b)) - (b * rcp(b) * rcp (b))) */
34592 /* x0 = rcp(b) estimate */
34595 UNSPEC_RCP)));
34596 /* e0 = x0 * b */
34600 /* e0 = x0 * e0 */
34604 /* e1 = x0 + x0 */
34608 /* x1 = e1 - e0 */
34612 /* res = a * x1 */
34615 }
34617 /* Output code to perform a Newton-Rhapson approximation of a
34618 single precision floating point [reciprocal] square root. */
34622 {
34624 REAL_VALUE_TYPE r;
34639 {
34642 }
34644 /* sqrt(a) = -0.5 * a * rsqrtss(a) * (a * rsqrtss(a) * rsqrtss(a) - 3.0)
34645 rsqrt(a) = -0.5 * rsqrtss(a) * (a * rsqrtss(a) * rsqrtss(a) - 3.0) */
34649 /* x0 = rsqrt(a) estimate */
34652 UNSPEC_RSQRT)));
34654 /* If (a == 0.0) Filter out infinity to prevent NaN for sqrt(0.0). */
34656 {
34668 }
34670 /* e0 = x0 * a */
34673 /* e1 = e0 * x0 */
34677 /* e2 = e1 - 3. */
34684 /* e3 = -.5 * x0 */
34687 else
34688 /* e3 = -.5 * e0 */
34691 /* ret = e2 * e3 */
34694 }
34696 #ifdef TARGET_SOLARIS
34697 /* Solaris implementation of TARGET_ASM_NAMED_SECTION. */
34699 static void
34701 tree decl)
34702 {
34703 /* With Binutils 2.15, the "@unwind" marker must be specified on
34704 every occurrence of the ".eh_frame" section, not just the first
34705 one. */
34708 {
34712 }
34714 #ifndef USE_GAS
34716 {
34719 }
34720 #endif
34723 }
34726 /* Return the mangling of TYPE if it is an extended fundamental type. */
34730 {
34738 {
34740 /* __float128 is "g". */
34743 /* "long double" or __float80 is "e". */
34747 }
34748 }
34750 /* For 32-bit code we can save PIC register setup by using
34751 __stack_chk_fail_local hidden function instead of calling
34752 __stack_chk_fail directly. 64-bit code doesn't need to setup any PIC
34753 register, so it is better to call __stack_chk_fail directly. */
34755 static tree ATTRIBUTE_UNUSED
34757 {
34758 return TARGET_64BIT
34761 }
34763 /* Select a format to encode pointers in exception handling data. CODE
34764 is 0 for data, 1 for code labels, 2 for function pointers. GLOBAL is
34765 true if the symbol may be affected by dynamic relocations.
34767 ??? All x86 object file formats are capable of representing this.
34768 After all, the relocation needed is the same as for the call insn.
34769 Whether or not a particular assembler allows us to enter such, I
34770 guess we'll have to see. */
34771 int
34773 {
34775 {
34782 }
34787 }
34788 \f
34789 /* Expand copysign from SIGN to the positive value ABS_VALUE
34790 storing in RESULT. If MASK is non-null, it shall be a mask to mask out
34791 the sign-bit. */
34792 static void
34794 {
34798 {
34805 else
34810 {
34811 /* We need to generate a scalar mode mask in this case. */
34816 }
34817 }
34818 else
34824 }
34826 /* Expand fabs (OP0) and return a new rtx that holds the result. The
34827 mask for masking out the sign-bit is stored in *SMASK, if that is
34828 non-null. */
34829 static rtx
34831 {
34840 else
34844 {
34845 /* We need to generate a scalar mode mask in this case. */
34850 }
34858 }
34860 /* Expands a comparison of OP0 with OP1 using comparison code CODE,
34861 swapping the operands if SWAP_OPERANDS is true. The expanded
34862 code is a forward jump to a newly created label in case the
34863 comparison is true. The generated label rtx is returned. */
34864 static rtx
34867 {
34871 {
34875 }
34888 }
34890 /* Expand a mask generating SSE comparison instruction comparing OP0 with OP1
34891 using comparison code CODE. Operands are swapped for the comparison if
34892 SWAP_OPERANDS is true. Returns a rtx for the generated mask. */
34893 static rtx
34896 {
34902 {
34906 }
34913 }
34915 /* Generate and return a rtx of mode MODE for 2**n where n is the number
34916 of bits of the mantissa of MODE, which must be one of DFmode or SFmode. */
34917 static rtx
34919 {
34920 REAL_VALUE_TYPE TWO52r;
34921 rtx TWO52;
34928 }
34930 /* Expand SSE sequence for computing lround from OP1 storing
34931 into OP0. */
34932 void
34934 {
34935 /* C code for the stuff we're doing below:
34936 tmp = op1 + copysign (nextafter (0.5, 0.0), op1)
34937 return (long)tmp;
34938 */
34942 rtx adj;
34944 /* load nextafter (0.5, 0.0) */
34949 /* adj = copysign (0.5, op1) */
34953 /* adj = op1 + adj */
34956 /* op0 = (imode)adj */
34958 }
34960 /* Expand SSE2 sequence for computing lround from OPERAND1 storing
34961 into OPERAND0. */
34962 void
34964 {
34965 /* C code for the stuff we're doing below (for do_floor):
34966 xi = (long)op1;
34967 xi -= (double)xi > op1 ? 1 : 0;
34968 return xi;
34969 */
34974 /* reg = (long)op1 */
34978 /* freg = (double)reg */
34982 /* ireg = (freg > op1) ? ireg - 1 : ireg */
34993 }
34995 /* Expand rint (IEEE round to nearest) rounding OPERAND1 and storing the
34996 result in OPERAND0. */
34997 void
34999 {
35000 /* C code for the stuff we're doing below:
35001 xa = fabs (operand1);
35002 if (!isless (xa, 2**52))
35003 return operand1;
35004 xa = xa + 2**52 - 2**52;
35005 return copysign (xa, operand1);
35006 */
35013 /* xa = abs (operand1) */
35016 /* if (!isless (xa, TWO52)) goto label; */
35029 }
35031 /* Expand SSE2 sequence for computing floor or ceil from OPERAND1 storing
35032 into OPERAND0. */
35033 void
35035 {
35036 /* C code for the stuff we expand below.
35037 double xa = fabs (x), x2;
35038 if (!isless (xa, TWO52))
35039 return x;
35040 xa = xa + TWO52 - TWO52;
35041 x2 = copysign (xa, x);
35042 Compensate. Floor:
35043 if (x2 > x)
35044 x2 -= 1;
35045 Compensate. Ceil:
35046 if (x2 < x)
35047 x2 -= -1;
35048 return x2;
35049 */
35055 /* Temporary for holding the result, initialized to the input
35056 operand to ease control flow. */
35060 /* xa = abs (operand1) */
35063 /* if (!isless (xa, TWO52)) goto label; */
35066 /* xa = xa + TWO52 - TWO52; */
35070 /* xa = copysign (xa, operand1) */
35073 /* generate 1.0 or -1.0 */
35078 /* Compensate: xa = xa - (xa > operand1 ? 1 : 0) */
35082 /* We always need to subtract here to preserve signed zero. */
35091 }
35093 /* Expand SSE2 sequence for computing floor or ceil from OPERAND1 storing
35094 into OPERAND0. */
35095 void
35097 {
35098 /* C code for the stuff we expand below.
35099 double xa = fabs (x), x2;
35100 if (!isless (xa, TWO52))
35101 return x;
35102 x2 = (double)(long)x;
35103 Compensate. Floor:
35104 if (x2 > x)
35105 x2 -= 1;
35106 Compensate. Ceil:
35107 if (x2 < x)
35108 x2 += 1;
35109 if (HONOR_SIGNED_ZEROS (mode))
35110 return copysign (x2, x);
35111 return x2;
35112 */
35118 /* Temporary for holding the result, initialized to the input
35119 operand to ease control flow. */
35123 /* xa = abs (operand1) */
35126 /* if (!isless (xa, TWO52)) goto label; */
35129 /* xa = (double)(long)x */
35134 /* generate 1.0 */
35137 /* Compensate: xa = xa - (xa > operand1 ? 1 : 0) */
35152 }
35154 /* Expand SSE sequence for computing round from OPERAND1 storing
35155 into OPERAND0. Sequence that works without relying on DImode truncation
35156 via cvttsd2siq that is only available on 64bit targets. */
35157 void
35159 {
35160 /* C code for the stuff we expand below.
35161 double xa = fabs (x), xa2, x2;
35162 if (!isless (xa, TWO52))
35163 return x;
35164 Using the absolute value and copying back sign makes
35165 -0.0 -> -0.0 correct.
35166 xa2 = xa + TWO52 - TWO52;
35167 Compensate.
35168 dxa = xa2 - xa;
35169 if (dxa <= -0.5)
35170 xa2 += 1;
35171 else if (dxa > 0.5)
35172 xa2 -= 1;
35173 x2 = copysign (xa2, x);
35174 return x2;
35175 */
35181 /* Temporary for holding the result, initialized to the input
35182 operand to ease control flow. */
35186 /* xa = abs (operand1) */
35189 /* if (!isless (xa, TWO52)) goto label; */
35192 /* xa2 = xa + TWO52 - TWO52; */
35196 /* dxa = xa2 - xa; */
35199 /* generate 0.5, 1.0 and -0.5 */
35205 /* Compensate. */
35207 /* xa2 = xa2 - (dxa > 0.5 ? 1 : 0) */
35212 /* xa2 = xa2 + (dxa <= -0.5 ? 1 : 0) */
35218 /* res = copysign (xa2, operand1) */
35225 }
35227 /* Expand SSE sequence for computing trunc from OPERAND1 storing
35228 into OPERAND0. */
35229 void
35231 {
35232 /* C code for SSE variant we expand below.
35233 double xa = fabs (x), x2;
35234 if (!isless (xa, TWO52))
35235 return x;
35236 x2 = (double)(long)x;
35237 if (HONOR_SIGNED_ZEROS (mode))
35238 return copysign (x2, x);
35239 return x2;
35240 */
35246 /* Temporary for holding the result, initialized to the input
35247 operand to ease control flow. */
35251 /* xa = abs (operand1) */
35254 /* if (!isless (xa, TWO52)) goto label; */
35257 /* x = (double)(long)x */
35269 }
35271 /* Expand SSE sequence for computing trunc from OPERAND1 storing
35272 into OPERAND0. */
35273 void
35275 {
35279 /* C code for SSE variant we expand below.
35280 double xa = fabs (x), x2;
35281 if (!isless (xa, TWO52))
35282 return x;
35283 xa2 = xa + TWO52 - TWO52;
35284 Compensate:
35285 if (xa2 > xa)
35286 xa2 -= 1.0;
35287 x2 = copysign (xa2, x);
35288 return x2;
35289 */
35293 /* Temporary for holding the result, initialized to the input
35294 operand to ease control flow. */
35298 /* xa = abs (operand1) */
35301 /* if (!isless (xa, TWO52)) goto label; */
35304 /* res = xa + TWO52 - TWO52; */
35309 /* generate 1.0 */
35312 /* Compensate: res = xa2 - (res > xa ? 1 : 0) */
35320 /* res = copysign (res, operand1) */
35327 }
35329 /* Expand SSE sequence for computing round from OPERAND1 storing
35330 into OPERAND0. */
35331 void
35333 {
35334 /* C code for the stuff we're doing below:
35335 double xa = fabs (x);
35336 if (!isless (xa, TWO52))
35337 return x;
35338 xa = (double)(long)(xa + nextafter (0.5, 0.0));
35339 return copysign (xa, x);
35340 */
35346 /* Temporary for holding the result, initialized to the input
35347 operand to ease control flow. */
35355 /* load nextafter (0.5, 0.0) */
35360 /* xa = xa + 0.5 */
35364 /* xa = (double)(int64_t)xa */
35369 /* res = copysign (xa, operand1) */
35376 }
35378 /* Expand SSE sequence for computing round
35379 from OP1 storing into OP0 using sse4 round insn. */
35380 void
35382 {
35391 {
35402 }
35404 /* round (a) = trunc (a + copysign (0.5, a)) */
35406 /* load nextafter (0.5, 0.0) */
35412 /* e1 = copysign (0.5, op1) */
35416 /* e2 = op1 + e1 */
35419 /* res = trunc (e2) */
35424 }
35425 \f
35427 /* Table of valid machine attributes. */
35429 {
35430 /* { name, min_len, max_len, decl_req, type_req, fn_type_req, handler,
35431 affects_type_identity } */
35432 /* Stdcall attribute says callee is responsible for popping arguments
35433 if they are not variable. */
35436 /* Fastcall attribute says callee is responsible for popping arguments
35437 if they are not variable. */
35440 /* Thiscall attribute says callee is responsible for popping arguments
35441 if they are not variable. */
35444 /* Cdecl attribute says the callee is a normal C declaration */
35447 /* Regparm attribute specifies how many integer arguments are to be
35448 passed in registers. */
35451 /* Sseregparm attribute says we are using x86_64 calling conventions
35452 for FP arguments. */
35455 /* The transactional memory builtins are implicitly regparm or fastcall
35456 depending on the ABI. Override the generic do-nothing attribute that
35457 these builtins were declared with. */
35460 /* force_align_arg_pointer says this function realigns the stack at entry. */
35463 #if TARGET_DLLIMPORT_DECL_ATTRIBUTES
35468 #endif
35473 #ifdef SUBTARGET_ATTRIBUTE_TABLE
35474 SUBTARGET_ATTRIBUTE_TABLE,
35475 #endif
35476 /* ms_abi and sysv_abi calling convention function attributes. */
35483 /* End element. */
35485 };
35487 /* Implement targetm.vectorize.builtin_vectorization_cost. */
35488 static int
35490 tree vectype ATTRIBUTE_UNUSED,
35492 {
35494 {
35535 }
35536 }
35538 /* Construct (set target (vec_select op0 (parallel perm))) and
35539 return true if that's a valid instruction in the active ISA. */
35541 static bool
35543 {
35556 {
35559 }
35561 }
35563 /* Similar, but generate a vec_concat from op0 and op1 as well. */
35565 static bool
35568 {
35570 rtx x;
35575 }
35577 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to implement D
35578 in terms of blendp[sd] / pblendw / pblendvb / vpblendd. */
35580 static bool
35582 {
35591 ;
35593 ;
35595 ;
35596 else
35599 /* This is a blend, not a permute. Elements must stay in their
35600 respective lanes. */
35602 {
35606 }
35611 /* ??? Without SSE4.1, we could implement this with and/andn/or. This
35612 decision should be extracted elsewhere, so that we only try that
35613 sequence once all budget==3 options have been tried. */
35620 {
35644 /* See if bytes move in pairs so we can use pblendw with
35645 an immediate argument, rather than pblendvb with a vector
35646 argument. */
35649 {
35650 use_pblendvb:
35654 finish_pblendvb:
35660 else
35663 }
35668 /* FALLTHRU */
35670 do_subreg:
35677 /* See if bytes move in pairs. If not, vpblendvb must be used. */
35681 /* See if bytes move in quadruplets. If yes, vpblendd
35682 with immediate can be used. */
35687 {
35688 /* See if bytes move the same in both lanes. If yes,
35689 vpblendw with immediate can be used. */
35694 /* Use vpblendw. */
35699 }
35701 /* Use vpblendd. */
35708 /* See if words move in pairs. If yes, vpblendd can be used. */
35713 {
35714 /* See if words move the same in both lanes. If not,
35715 vpblendvb must be used. */
35718 {
35719 /* Use vpblendvb. */
35729 }
35731 /* Use vpblendw. */
35735 }
35737 /* Use vpblendd. */
35744 /* Use vpblendd. */
35752 }
35754 /* This matches five different patterns with the different modes. */
35760 }
35762 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to implement D
35763 in terms of the variable form of vpermilps.
35765 Note that we will have already failed the immediate input vpermilps,
35766 which requires that the high and low part shuffle be identical; the
35767 variable form doesn't require that. */
35769 static bool
35771 {
35778 /* We can only permute within the 128-bit lane. */
35780 {
35784 }
35790 {
35793 /* Within each 128-bit lane, the elements of op0 are numbered
35794 from 0 and the elements of op1 are numbered from 4. */
35801 }
35808 }
35810 /* Return true if permutation D can be performed as VMODE permutation
35811 instead. */
35813 static bool
35815 {
35830 else
35836 }
35838 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to implement D
35839 in terms of pshufb, vpperm, vpermq, vpermd or vperm2i128. */
35841 static bool
35843 {
35852 {
35854 {
35857 {
35861 /* Use vperm2i128 insn. The pattern uses
35862 V4DImode instead of V2TImode. */
35871 }
35873 }
35874 }
35875 else
35876 {
35878 {
35881 }
35883 {
35887 /* V4DImode should be already handled through
35888 expand_vselect by vpermq instruction. */
35895 {
35896 /* First see if vpermq can be used for
35897 V8SImode/V16HImode/V32QImode. */
35899 {
35907 }
35909 /* Next see if vpermd can be used. */
35912 }
35915 {
35916 /* vpshufb only works intra lanes, it is not
35917 possible to shuffle bytes in between the lanes. */
35921 }
35922 }
35923 else
35925 }
35933 else
35934 {
35940 else
35944 {
35948 }
35949 }
35958 {
35963 else
35965 }
35966 else
35967 {
35970 }
35973 }
35975 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to instantiate D
35976 in a single instruction. */
35978 static bool
35980 {
35984 /* Check plain VEC_SELECT first, because AVX has instructions that could
35985 match both SEL and SEL+CONCAT, but the plain SEL will allow a memory
35986 input where SEL+CONCAT may not. */
35988 {
35994 {
36000 }
36003 {
36007 }
36009 {
36010 /* Use vpbroadcast{b,w,d}. */
36013 {
36032 /* For other modes prefer other shuffles this function creates. */
36034 }
36036 {
36040 }
36041 }
36046 /* There are plenty of patterns in sse.md that are written for
36047 SEL+CONCAT and are not replicated for a single op. Perhaps
36048 that should be changed, to avoid the nastiness here. */
36050 /* Recognize interleave style patterns, which means incrementing
36051 every other permutation operand. */
36053 {
36056 }
36060 /* Recognize shufps, which means adding {0, 0, nelt, nelt}. */
36062 {
36064 {
36069 }
36073 }
36074 }
36076 /* Finally, try the fully general two operand permute. */
36080 /* Recognize interleave style patterns with reversed operands. */
36082 {
36084 {
36088 else
36091 }
36095 }
36097 /* Try the SSE4.1 blend variable merge instructions. */
36101 /* Try one of the AVX vpermil variable permutations. */
36105 /* Try the SSSE3 pshufb or XOP vpperm or AVX2 vperm2i128,
36106 vpshufb, vpermd or vpermq variable permutation. */
36111 }
36113 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to implement D
36114 in terms of a pair of pshuflw + pshufhw instructions. */
36116 static bool
36118 {
36126 /* The two permutations only operate in 64-bit lanes. */
36137 /* Emit the pshuflw. */
36144 /* Emit the pshufhw. */
36152 }
36154 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to simplify
36155 the permutation using the SSSE3 palignr instruction. This succeeds
36156 when all of the elements in PERM fit within one vector and we merely
36157 need to shift them down so that a single vector permutation has a
36158 chance to succeed. */
36160 static bool
36162 {
36166 rtx shift;
36168 /* Even with AVX, palignr only operates on 128-bit vectors. */
36174 {
36180 }
36184 /* Given that we have SSSE3, we know we'll be able to implement the
36185 single operand permutation after the palignr with pshufb. */
36198 {
36203 }
36205 /* Test for the degenerate case where the alignment by itself
36206 produces the desired permutation. */
36214 }
36218 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to simplify
36219 a two vector permutation into a single vector permutation by using
36220 an interleave operation to merge the vectors. */
36222 static bool
36224 {
36229 rtx seq;
36233 {
36236 }
36238 {
36241 /* For 32-byte modes allow even d->op0 == d->op1.
36242 The lack of cross-lane shuffling in some instructions
36243 might prevent a single insn shuffle. */
36246 /* If expand_vec_perm_interleave3 can expand this into
36247 a 3 insn sequence, give up and let it be expanded as
36248 3 insn sequence. While that is one insn longer,
36249 it doesn't need a memory operand and in the common
36250 case that both interleave low and high permutations
36251 with the same operands are adjacent needs 4 insns
36252 for both after CSE. */
36255 }
36256 else
36259 /* Examine from whence the elements come. */
36268 {
36271 /* Split the two input vectors into 4 halves. */
36277 /* If the elements from the low halves use interleave low, and similarly
36278 for interleave high. If the elements are from mis-matched halves, we
36279 can use shufps for V4SF/V4SI or do a DImode shuffle. */
36281 {
36282 /* punpckl* */
36284 {
36289 }
36292 }
36294 {
36295 /* punpckh* */
36297 {
36302 }
36305 }
36307 {
36308 /* shufps */
36310 {
36315 }
36317 {
36318 /* shufpd */
36323 }
36324 }
36326 {
36327 /* shufps */
36329 {
36334 }
36336 {
36337 /* shufpd */
36342 }
36343 }
36344 else
36346 }
36347 else
36348 {
36353 /* Split the two input vectors into 8 quarters. */
36359 {
36362 }
36365 {
36369 }
36371 {
36374 }
36377 {
36379 {
36380 /* Attempt to increase the likelyhood that dfinal
36381 shuffle will be intra-lane. */
36385 }
36387 /* vperm2f128 or vperm2i128. */
36389 {
36394 }
36399 {
36403 {
36406 }
36407 }
36408 }
36413 {
36414 /* vpunpckl* */
36416 {
36425 }
36426 }
36429 {
36430 /* vpunpckh* */
36432 {
36441 }
36442 }
36443 else
36445 }
36447 /* Use the remapping array set up above to move the elements from their
36448 swizzled locations into their final destinations. */
36451 {
36454 /* If same_halves is true, both halves of the remapped vector are the
36455 same. Avoid cross-lane accesses if possible. */
36457 {
36460 }
36461 else
36463 }
36468 /* Test if the final remap can be done with a single insn. For V4SFmode or
36469 V4SImode this *will* succeed. For V8HImode or V16QImode it may not. */
36482 {
36486 }
36493 }
36495 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to simplify
36496 a single vector cross-lane permutation into vpermq followed
36497 by any of the single insn permutations. */
36499 static bool
36501 {
36515 {
36518 }
36521 {
36526 }
36538 {
36545 }
36551 {
36556 ;
36559 else
36561 }
36570 }
36572 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to simplify
36573 a two vector permutation using 2 intra-lane interleave insns
36574 and cross-lane shuffle for 32-byte vectors. */
36576 static bool
36578 {
36585 ;
36587 ;
36588 else
36603 {
36607 else
36613 else
36619 else
36625 else
36631 else
36637 else
36642 }
36646 }
36648 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to implement
36649 a single vector permutation using a single intra-lane vector
36650 permutation, vperm2f128 swapping the lanes and vblend* insn blending
36651 the non-swapped and swapped vectors together. */
36653 static bool
36655 {
36658 rtx seq;
36663 || TARGET_AVX2
36672 {
36679 }
36713 }
36715 /* A subroutine of expand_vec_perm_even_odd_1. Implement the double-word
36716 permutation with two pshufb insns and an ior. We should have already
36717 failed all two instruction sequences. */
36719 static bool
36721 {
36732 /* Generate two permutation masks. If the required element is within
36733 the given vector it is shuffled into the proper lane. If the required
36734 element is in the other vector, force a zero into the lane by setting
36735 bit 7 in the permutation mask. */
36738 {
36745 {
36748 }
36749 }
36769 }
36771 /* Implement arbitrary permutation of one V32QImode and V16QImode operand
36772 with two vpshufb insns, vpermq and vpor. We should have already failed
36773 all two or three instruction sequences. */
36775 static bool
36777 {
36792 /* Generate two permutation masks. If the required element is within
36793 the same lane, it is shuffled in. If the required element from the
36794 other lane, force a zero by setting bit 7 in the permutation mask.
36795 In the other mask the mask has non-negative elements if element
36796 is requested from the other lane, but also moved to the other lane,
36797 so that the result of vpshufb can have the two V2TImode halves
36798 swapped. */
36801 {
36806 {
36809 }
36810 }
36819 /* Swap the 128-byte lanes of h into hp. */
36823 const1_rtx));
36836 }
36838 /* A subroutine of expand_vec_perm_even_odd_1. Implement extract-even
36839 and extract-odd permutations of two V32QImode and V16QImode operand
36840 with two vpshufb insns, vpor and vpermq. We should have already
36841 failed all two or three instruction sequences. */
36843 static bool
36845 {
36864 /* Generate two permutation masks. In the first permutation mask
36865 the first quarter will contain indexes for the first half
36866 of the op0, the second quarter will contain bit 7 set, third quarter
36867 will contain indexes for the second half of the op0 and the
36868 last quarter bit 7 set. In the second permutation mask
36869 the first quarter will contain bit 7 set, the second quarter
36870 indexes for the first half of the op1, the third quarter bit 7 set
36871 and last quarter indexes for the second half of the op1.
36872 I.e. the first mask e.g. for V32QImode extract even will be:
36873 0, 2, ..., 0xe, -128, ..., -128, 0, 2, ..., 0xe, -128, ..., -128
36874 (all values masked with 0xf except for -128) and second mask
36875 for extract even will be
36876 -128, ..., -128, 0, 2, ..., 0xe, -128, ..., -128, 0, 2, ..., 0xe. */
36879 {
36885 {
36888 }
36889 }
36908 /* Permute the V4DImode quarters using { 0, 2, 1, 3 } permutation. */
36915 }
36917 /* A subroutine of ix86_expand_vec_perm_builtin_1. Implement extract-even
36918 and extract-odd permutations. */
36920 static bool
36922 {
36926 {
36931 /* Shuffle the lanes around into { 0 1 4 5 } and { 2 3 6 7 }. */
36935 /* Now an unpck[lh]pd will produce the result required. */
36938 else
36944 {
36951 /* Shuffle within the 128-bit lanes to produce:
36952 { 0 2 8 a 4 6 c e } | { 1 3 9 b 5 7 d f }. */
36956 /* Shuffle the lanes around to produce:
36957 { 4 6 c e 0 2 8 a } and { 5 7 d f 1 3 9 b }. */
36961 /* Shuffle within the 128-bit lanes to produce:
36962 { 0 2 4 6 4 6 0 2 } | { 1 3 5 7 5 7 1 3 }. */
36965 /* Shuffle within the 128-bit lanes to produce:
36966 { 8 a c e c e 8 a } | { 9 b d f d f 9 b }. */
36969 /* Shuffle the lanes around to produce:
36970 { 0 2 4 6 8 a c e } | { 1 3 5 7 9 b d f }. */
36973 }
36980 /* These are always directly implementable by expand_vec_perm_1. */
36986 else
36987 {
36988 /* We need 2*log2(N)-1 operations to achieve odd/even
36989 with interleave. */
36998 else
37001 }
37007 else
37008 {
37020 else
37023 }
37032 {
37039 }
37044 /* Shuffle the lanes around into { 0 1 4 5 } and { 2 3 6 7 }. */
37048 /* Now an vpunpck[lh]qdq will produce the result required. */
37051 else
37058 {
37065 }
37070 /* Shuffle the lanes around into
37071 { 0 1 2 3 8 9 a b } and { 4 5 6 7 c d e f }. */
37081 /* Swap the 2nd and 3rd position in each lane into
37082 { 0 2 1 3 8 a 9 b } and { 4 6 5 7 c e d f }. */
37088 /* Now an vpunpck[lh]qdq will produce
37089 { 0 2 4 6 8 a c e } resp. { 1 3 5 7 9 b d f }. */
37094 else
37103 }
37106 }
37108 /* A subroutine of ix86_expand_vec_perm_builtin_1. Pattern match
37109 extract-even and extract-odd permutations. */
37111 static bool
37113 {
37125 }
37127 /* A subroutine of ix86_expand_vec_perm_builtin_1. Implement broadcast
37128 permutations. We assume that expand_vec_perm_1 has already failed. */
37130 static bool
37132 {
37140 {
37143 /* These are special-cased in sse.md so that we can optionally
37144 use the vbroadcast instruction. They expand to two insns
37145 if the input happens to be in a register. */
37152 /* These are always implementable using standard shuffle patterns. */
37157 /* These can be implemented via interleave. We save one insn by
37158 stopping once we have promoted to V4SImode and then use pshufd. */
37159 do
37160 {
37161 rtx dest;
37167 {
37171 }
37178 }
37190 /* For AVX2 broadcasts of the first element vpbroadcast* or
37191 vpermq should be used by expand_vec_perm_1. */
37197 }
37198 }
37200 /* A subroutine of ix86_expand_vec_perm_builtin_1. Pattern match
37201 broadcast permutations. */
37203 static bool
37205 {
37217 }
37219 /* Implement arbitrary permutation of two V32QImode and V16QImode operands
37220 with 4 vpshufb insns, 2 vpermq and 3 vpor. We should have already failed
37221 all the shorter instruction sequences. */
37223 static bool
37225 {
37241 /* Generate 4 permutation masks. If the required element is within
37242 the same lane, it is shuffled in. If the required element from the
37243 other lane, force a zero by setting bit 7 in the permutation mask.
37244 In the other mask the mask has non-negative elements if element
37245 is requested from the other lane, but also moved to the other lane,
37246 so that the result of vpshufb can have the two V2TImode halves
37247 swapped. */
37250 {
37255 }
37261 {
37269 }
37272 {
37274 {
37277 }
37284 }
37286 /* Swap the 128-byte lanes of h[X]. */
37288 {
37294 const1_rtx));
37296 }
37299 {
37301 {
37304 }
37310 }
37313 {
37315 {
37319 }
37322 }
37328 }
37330 /* The guts of ix86_expand_vec_perm_const, also used by the ok hook.
37331 With all of the interface bits taken care of, perform the expansion
37332 in D and return true on success. */
37334 static bool
37336 {
37337 /* Try a single instruction expansion. */
37341 /* Try sequences of two instructions. */
37358 /* Try sequences of three instructions. */
37369 /* Try sequences of four instructions. */
37377 /* ??? Look for narrow permutations whose element orderings would
37378 allow the promotion to a wider mode. */
37380 /* ??? Look for sequences of interleave or a wider permute that place
37381 the data into the correct lanes for a half-vector shuffle like
37382 pshuf[lh]w or vpermilps. */
37384 /* ??? Look for sequences of interleave that produce the desired results.
37385 The combinatorics of punpck[lh] get pretty ugly... */
37390 /* Even longer sequences. */
37395 }
37397 bool
37399 {
37403 rtx sel;
37420 {
37427 }
37430 {
37438 /* The elements of PERM do not suggest that only the first operand
37439 is used, but both operands are identical. Allow easier matching
37440 of the permutation by folding the permutation into the single
37441 input vector. */
37445 /* FALLTHRU */
37456 }
37461 /* If the mask says both arguments are needed, but they are the same,
37462 the above tried to expand with d.op0 == d.op1. If that didn't work,
37463 retry with d.op0 != d.op1 as that is what testing has been done with. */
37465 {
37466 rtx seq;
37476 {
37480 }
37481 }
37484 }
37486 /* Implement targetm.vectorize.vec_perm_const_ok. */
37488 static bool
37491 {
37500 /* Given sufficient ISA support we can just return true here
37501 for selected vector modes. */
37503 {
37504 /* All implementable with a single vpperm insn. */
37507 /* All implementable with 2 pshufb + 1 ior. */
37510 /* All implementable with shufpd or unpck[lh]pd. */
37513 }
37515 /* Extract the values from the vector CST into the permutation
37516 array in D. */
37519 {
37523 }
37525 /* For all elements from second vector, fold the elements to first. */
37530 /* Check whether the mask can be applied to the vector type. */
37533 /* Implementable with shufps or pshufd. */
37537 /* Otherwise we have to go through the motions and see if we can
37538 figure out how to generate the requested permutation. */
37549 }
37551 void
37553 {
37567 /* We'll either be able to implement the permutation directly... */
37571 /* ... or we use the special-case patterns. */
37573 }
37575 /* Expand an insert into a vector register through pinsr insn.
37576 Return true if successful. */
37578 bool
37580 {
37588 {
37591 }
37597 {
37602 {
37609 {
37641 }
37650 }
37654 }
37655 }
37656 \f
37657 /* This function returns the calling abi specific va_list type node.
37658 It returns the FNDECL specific va_list type. */
37660 static tree
37662 {
37669 else
37671 }
37673 /* Returns the canonical va_list type specified by TYPE. If there
37674 is no valid TYPE provided, it return NULL_TREE. */
37676 static tree
37678 {
37681 /* Resolve references and pointers to va_list type. */
37690 {
37695 {
37696 /* If va_list is an array type, the argument may have decayed
37697 to a pointer type, e.g. by being passed to another function.
37698 In that case, unwrap both types so that we can compare the
37699 underlying records. */
37702 {
37705 }
37706 }
37713 {
37714 /* If va_list is an array type, the argument may have decayed
37715 to a pointer type, e.g. by being passed to another function.
37716 In that case, unwrap both types so that we can compare the
37717 underlying records. */
37720 {
37723 }
37724 }
37731 {
37732 /* If va_list is an array type, the argument may have decayed
37733 to a pointer type, e.g. by being passed to another function.
37734 In that case, unwrap both types so that we can compare the
37735 underlying records. */
37738 {
37741 }
37742 }
37746 }
37748 }
37750 /* Iterate through the target-specific builtin types for va_list.
37751 IDX denotes the iterator, *PTREE is set to the result type of
37752 the va_list builtin, and *PNAME to its internal type.
37753 Returns zero if there is no element for this index, otherwise
37754 IDX should be increased upon the next call.
37755 Note, do not iterate a base builtin's name like __builtin_va_list.
37756 Used from c_common_nodes_and_builtins. */
37758 static int
37760 {
37762 {
37764 {
37777 }
37778 }
37781 }
37783 #undef TARGET_SCHED_DISPATCH
37784 #define TARGET_SCHED_DISPATCH has_dispatch
37785 #undef TARGET_SCHED_DISPATCH_DO
37786 #define TARGET_SCHED_DISPATCH_DO do_dispatch
37787 #undef TARGET_SCHED_REASSOCIATION_WIDTH
37788 #define TARGET_SCHED_REASSOCIATION_WIDTH ix86_reassociation_width
37790 /* The size of the dispatch window is the total number of bytes of
37791 object code allowed in a window. */
37792 #define DISPATCH_WINDOW_SIZE 16
37794 /* Number of dispatch windows considered for scheduling. */
37795 #define MAX_DISPATCH_WINDOWS 3
37797 /* Maximum number of instructions in a window. */
37798 #define MAX_INSN 4
37800 /* Maximum number of immediate operands in a window. */
37801 #define MAX_IMM 4
37803 /* Maximum number of immediate bits allowed in a window. */
37804 #define MAX_IMM_SIZE 128
37806 /* Maximum number of 32 bit immediates allowed in a window. */
37807 #define MAX_IMM_32 4
37809 /* Maximum number of 64 bit immediates allowed in a window. */
37810 #define MAX_IMM_64 2
37812 /* Maximum total of loads or prefetches allowed in a window. */
37813 #define MAX_LOAD 2
37815 /* Maximum total of stores allowed in a window. */
37816 #define MAX_STORE 1
37818 #undef BIG
37819 #define BIG 100
37822 /* Dispatch groups. Istructions that affect the mix in a dispatch window. */
37825 disp_load,
37826 disp_store,
37827 disp_load_store,
37828 disp_prefetch,
37829 disp_imm,
37830 disp_imm_32,
37831 disp_imm_64,
37832 disp_branch,
37833 disp_cmp,
37834 disp_jcc,
37835 disp_last
37836 };
37838 /* Number of allowable groups in a dispatch window. It is an array
37839 indexed by dispatch_group enum. 100 is used as a big number,
37840 because the number of these kind of operations does not have any
37841 effect in dispatch window, but we need them for other reasons in
37842 the table. */
37845 };
37851 };
37853 /* Instruction path. */
37859 last_path
37860 };
37862 /* sched_insn_info defines a window to the instructions scheduled in
37863 the basic block. It contains a pointer to the insn_info table and
37864 the instruction scheduled.
37866 Windows are allocated for each basic block and are linked
37867 together. */
37869 rtx insn;
37876 /* Linked list of dispatch windows. This is a two way list of
37877 dispatch windows of a basic block. It contains information about
37878 the number of uops in the window and the total number of
37879 instructions and of bytes in the object code for this dispatch
37880 window. */
37898 /* Immediate valuse used in an insn. */
37900 {
37909 /* Get dispatch group of insn. */
37911 static enum dispatch_group
37913 {
37929 }
37931 /* Return true if insn is a compare instruction. */
37933 static bool
37935 {
37943 }
37945 /* Return true if a dispatch violation encountered. */
37947 static bool
37949 {
37953 }
37955 /* Return true if insn is a branch instruction. */
37957 static bool
37959 {
37961 }
37963 /* Return true if insn is a prefetch instruction. */
37965 static bool
37967 {
37969 }
37971 /* This function initializes a dispatch window and the list container holding a
37972 pointer to the window. */
37974 static void
37976 {
37982 else
38000 {
38006 }
38008 }
38010 /* This function allocates and initializes a dispatch window and the
38011 list container holding a pointer to the window. */
38015 {
38020 }
38022 /* This routine initializes the dispatch scheduling information. It
38023 initiates building dispatch scheduler tables and constructs the
38024 first dispatch window. */
38026 static void
38028 {
38029 /* Allocate a dispatch list and a window. */
38034 }
38036 /* This function returns true if a branch is detected. End of a basic block
38037 does not have to be a branch, but here we assume only branches end a
38038 window. */
38040 static bool
38042 {
38044 }
38046 /* This function is called when the end of a window processing is reached. */
38048 static void
38050 {
38053 {
38058 }
38060 }
38062 /* Allocates a new dispatch window and adds it to WINDOW_LIST.
38063 WINDOW_NUM is either 0 or 1. A maximum of two windows are generated
38064 for 48 bytes of instructions. Note that these windows are not dispatch
38065 windows that their sizes are DISPATCH_WINDOW_SIZE. */
38069 {
38071 {
38076 }
38082 }
38084 /* Increment the number of immediate operands of an instruction. */
38086 static int
38088 {
38093 {
38100 else
38111 {
38114 }
38119 }
38122 }
38124 /* Compute number of immediate operands of an instruction. */
38126 static void
38128 {
38131 }
38133 /* Return total size of immediate operands of an instruction along with number
38134 of corresponding immediate-operands. It initializes its parameters to zero
38135 befor calling FIND_CONSTANT.
38136 INSN is the input instruction. IMM is the total of immediates.
38137 IMM32 is the number of 32 bit immediates. IMM64 is the number of 64
38138 bit immediates. */
38140 static int
38142 {
38150 }
38152 /* This function indicates if an operand of an instruction is an
38153 immediate. */
38155 static bool
38157 {
38166 }
38168 /* Return single or double path for instructions. */
38170 static enum insn_path
38172 {
38182 }
38184 /* Return insn dispatch group. */
38186 static enum dispatch_group
38188 {
38206 }
38208 /* Count number of GROUP restricted instructions in a dispatch
38209 window WINDOW_LIST. */
38211 static int
38213 {
38224 {
38243 }
38256 }
38258 /* This function returns true if insn satisfies dispatch rules on the
38259 last window scheduled. */
38261 static bool
38263 {
38271 /* Make disp_cmp and disp_jcc get scheduled at the latest. These
38272 instructions should be given the lowest priority in the
38273 scheduling process in Haifa scheduler to make sure they will be
38274 scheduled in the same dispatch window as the refrence to them. */
38278 /* Check nonrestricted. */
38282 /* Get last dispatch window. */
38287 {
38292 /* Window 1 is full. Go for next window. */
38294 }
38301 /* See if it fits in the first window. */
38303 {
38304 /* The first widow should have only single and double path
38305 uops. */
38311 }
38313 }
38315 /* Add an instruction INSN with NUM_UOPS micro-operations to the
38316 dispatch window WINDOW_LIST. */
38318 static void
38320 {
38338 /* Initialize window with new instruction. */
38359 {
38362 }
38363 }
38365 /* Adds a scheduled instruction, INSN, to the current dispatch window.
38366 If the total bytes of instructions or the number of instructions in
38367 the window exceed allowable, it allocates a new window. */
38369 static void
38371 {
38394 /* Get the last dispatch window. */
38402 else
38405 /* If current window is full, get a new window.
38406 Window number zero is full, if MAX_INSN uops are scheduled in it.
38407 Window number one is full, if window zero's bytes plus window
38408 one's bytes is 32, or if the bytes of the new instruction added
38409 to the total makes it greater than 48, or it has already MAX_INSN
38410 instructions in it. */
38419 {
38422 }
38425 {
38429 {
38432 }
38433 }
38435 {
38440 {
38443 }
38446 }
38447 else
38451 {
38452 /* End of basic block is reached do end-basic-block process. */
38455 }
38456 }
38458 /* Print the dispatch window, WINDOW_NUM, to FILE. */
38460 DEBUG_FUNCTION static void
38462 {
38468 else
38482 {
38485 fprintf (file, " group[%d] = %s, insn[%d] = %p, path[%d] = %d byte_len[%d] = %d, imm_bytes[%d] = %d\n",
38491 }
38492 }
38494 /* Print to stdout a dispatch window. */
38496 DEBUG_FUNCTION void
38498 {
38500 }
38502 /* Print INSN dispatch information to FILE. */
38504 DEBUG_FUNCTION static void
38506 {
38529 }
38531 /* Print to STDERR the status of the ready list with respect to
38532 dispatch windows. */
38534 DEBUG_FUNCTION void
38536 {
38544 }
38546 /* This routine is the driver of the dispatch scheduler. */
38548 static void
38550 {
38555 }
38557 /* Return TRUE if Dispatch Scheduling is supported. */
38559 static bool
38561 {
38565 {
38581 }
38584 }
38586 /* Implementation of reassociation_width target hook used by
38587 reassoc phase to identify parallelism level in reassociated
38588 tree. Statements tree_code is passed in OPC. Arguments type
38589 is passed in MODE.
38591 Currently parallel reassociation is enabled for Atom
38592 processors only and we set reassociation width to be 2
38593 because Atom may issue up to 2 instructions per cycle.
38595 Return value should be fixed if parallel reassociation is
38596 enabled for other processors. */
38598 static int
38601 {
38610 }
38612 /* ??? No autovectorization into MMX or 3DNOW until we can reliably
38613 place emms and femms instructions. */
38615 static enum machine_mode
38617 {
38622 {
38635 else
38645 /* FALLTHRU */
38649 }
38650 }
38652 /* If AVX is enabled then try vectorizing with both 256bit and 128bit
38653 vectors. */
38655 static unsigned int
38657 {
38659 }
38661 /* Initialize the GCC target structure. */
38662 #undef TARGET_RETURN_IN_MEMORY
38663 #define TARGET_RETURN_IN_MEMORY ix86_return_in_memory
38665 #undef TARGET_LEGITIMIZE_ADDRESS
38666 #define TARGET_LEGITIMIZE_ADDRESS ix86_legitimize_address
38668 #undef TARGET_ATTRIBUTE_TABLE
38669 #define TARGET_ATTRIBUTE_TABLE ix86_attribute_table
38670 #if TARGET_DLLIMPORT_DECL_ATTRIBUTES
38671 # undef TARGET_MERGE_DECL_ATTRIBUTES
38672 # define TARGET_MERGE_DECL_ATTRIBUTES merge_dllimport_decl_attributes
38673 #endif
38675 #undef TARGET_COMP_TYPE_ATTRIBUTES
38676 #define TARGET_COMP_TYPE_ATTRIBUTES ix86_comp_type_attributes
38678 #undef TARGET_INIT_BUILTINS
38679 #define TARGET_INIT_BUILTINS ix86_init_builtins
38680 #undef TARGET_BUILTIN_DECL
38681 #define TARGET_BUILTIN_DECL ix86_builtin_decl
38682 #undef TARGET_EXPAND_BUILTIN
38683 #define TARGET_EXPAND_BUILTIN ix86_expand_builtin
38685 #undef TARGET_VECTORIZE_BUILTIN_VECTORIZED_FUNCTION
38686 #define TARGET_VECTORIZE_BUILTIN_VECTORIZED_FUNCTION \
38687 ix86_builtin_vectorized_function
38689 #undef TARGET_VECTORIZE_BUILTIN_TM_LOAD
38690 #define TARGET_VECTORIZE_BUILTIN_TM_LOAD ix86_builtin_tm_load
38692 #undef TARGET_VECTORIZE_BUILTIN_TM_STORE
38693 #define TARGET_VECTORIZE_BUILTIN_TM_STORE ix86_builtin_tm_store
38695 #undef TARGET_VECTORIZE_BUILTIN_GATHER
38696 #define TARGET_VECTORIZE_BUILTIN_GATHER ix86_vectorize_builtin_gather
38698 #undef TARGET_BUILTIN_RECIPROCAL
38699 #define TARGET_BUILTIN_RECIPROCAL ix86_builtin_reciprocal
38701 #undef TARGET_ASM_FUNCTION_EPILOGUE
38702 #define TARGET_ASM_FUNCTION_EPILOGUE ix86_output_function_epilogue
38704 #undef TARGET_ENCODE_SECTION_INFO
38705 #ifndef SUBTARGET_ENCODE_SECTION_INFO
38706 #define TARGET_ENCODE_SECTION_INFO ix86_encode_section_info
38707 #else
38708 #define TARGET_ENCODE_SECTION_INFO SUBTARGET_ENCODE_SECTION_INFO
38709 #endif
38711 #undef TARGET_ASM_OPEN_PAREN
38713 #undef TARGET_ASM_CLOSE_PAREN
38716 #undef TARGET_ASM_BYTE_OP
38717 #define TARGET_ASM_BYTE_OP ASM_BYTE
38719 #undef TARGET_ASM_ALIGNED_HI_OP
38720 #define TARGET_ASM_ALIGNED_HI_OP ASM_SHORT
38721 #undef TARGET_ASM_ALIGNED_SI_OP
38722 #define TARGET_ASM_ALIGNED_SI_OP ASM_LONG
38723 #ifdef ASM_QUAD
38724 #undef TARGET_ASM_ALIGNED_DI_OP
38725 #define TARGET_ASM_ALIGNED_DI_OP ASM_QUAD
38726 #endif
38728 #undef TARGET_PROFILE_BEFORE_PROLOGUE
38729 #define TARGET_PROFILE_BEFORE_PROLOGUE ix86_profile_before_prologue
38731 #undef TARGET_ASM_UNALIGNED_HI_OP
38732 #define TARGET_ASM_UNALIGNED_HI_OP TARGET_ASM_ALIGNED_HI_OP
38733 #undef TARGET_ASM_UNALIGNED_SI_OP
38734 #define TARGET_ASM_UNALIGNED_SI_OP TARGET_ASM_ALIGNED_SI_OP
38735 #undef TARGET_ASM_UNALIGNED_DI_OP
38736 #define TARGET_ASM_UNALIGNED_DI_OP TARGET_ASM_ALIGNED_DI_OP
38738 #undef TARGET_PRINT_OPERAND
38739 #define TARGET_PRINT_OPERAND ix86_print_operand
38740 #undef TARGET_PRINT_OPERAND_ADDRESS
38741 #define TARGET_PRINT_OPERAND_ADDRESS ix86_print_operand_address
38742 #undef TARGET_PRINT_OPERAND_PUNCT_VALID_P
38743 #define TARGET_PRINT_OPERAND_PUNCT_VALID_P ix86_print_operand_punct_valid_p
38744 #undef TARGET_ASM_OUTPUT_ADDR_CONST_EXTRA
38745 #define TARGET_ASM_OUTPUT_ADDR_CONST_EXTRA i386_asm_output_addr_const_extra
38747 #undef TARGET_SCHED_INIT_GLOBAL
38748 #define TARGET_SCHED_INIT_GLOBAL ix86_sched_init_global
38749 #undef TARGET_SCHED_ADJUST_COST
38750 #define TARGET_SCHED_ADJUST_COST ix86_adjust_cost
38751 #undef TARGET_SCHED_ISSUE_RATE
38752 #define TARGET_SCHED_ISSUE_RATE ix86_issue_rate
38753 #undef TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD
38754 #define TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD \
38755 ia32_multipass_dfa_lookahead
38757 #undef TARGET_FUNCTION_OK_FOR_SIBCALL
38758 #define TARGET_FUNCTION_OK_FOR_SIBCALL ix86_function_ok_for_sibcall
38760 #ifdef HAVE_AS_TLS
38761 #undef TARGET_HAVE_TLS
38762 #define TARGET_HAVE_TLS true
38763 #endif
38764 #undef TARGET_CANNOT_FORCE_CONST_MEM
38765 #define TARGET_CANNOT_FORCE_CONST_MEM ix86_cannot_force_const_mem
38766 #undef TARGET_USE_BLOCKS_FOR_CONSTANT_P
38767 #define TARGET_USE_BLOCKS_FOR_CONSTANT_P hook_bool_mode_const_rtx_true
38769 #undef TARGET_DELEGITIMIZE_ADDRESS
38770 #define TARGET_DELEGITIMIZE_ADDRESS ix86_delegitimize_address
38772 #undef TARGET_MS_BITFIELD_LAYOUT_P
38773 #define TARGET_MS_BITFIELD_LAYOUT_P ix86_ms_bitfield_layout_p
38775 #if TARGET_MACHO
38776 #undef TARGET_BINDS_LOCAL_P
38777 #define TARGET_BINDS_LOCAL_P darwin_binds_local_p
38778 #endif
38779 #if TARGET_DLLIMPORT_DECL_ATTRIBUTES
38780 #undef TARGET_BINDS_LOCAL_P
38781 #define TARGET_BINDS_LOCAL_P i386_pe_binds_local_p
38782 #endif
38784 #undef TARGET_ASM_OUTPUT_MI_THUNK
38785 #define TARGET_ASM_OUTPUT_MI_THUNK x86_output_mi_thunk
38786 #undef TARGET_ASM_CAN_OUTPUT_MI_THUNK
38787 #define TARGET_ASM_CAN_OUTPUT_MI_THUNK x86_can_output_mi_thunk
38789 #undef TARGET_ASM_FILE_START
38790 #define TARGET_ASM_FILE_START x86_file_start
38792 #undef TARGET_OPTION_OVERRIDE
38793 #define TARGET_OPTION_OVERRIDE ix86_option_override
38795 #undef TARGET_REGISTER_MOVE_COST
38796 #define TARGET_REGISTER_MOVE_COST ix86_register_move_cost
38797 #undef TARGET_MEMORY_MOVE_COST
38798 #define TARGET_MEMORY_MOVE_COST ix86_memory_move_cost
38799 #undef TARGET_RTX_COSTS
38800 #define TARGET_RTX_COSTS ix86_rtx_costs
38801 #undef TARGET_ADDRESS_COST
38802 #define TARGET_ADDRESS_COST ix86_address_cost
38804 #undef TARGET_FIXED_CONDITION_CODE_REGS
38805 #define TARGET_FIXED_CONDITION_CODE_REGS ix86_fixed_condition_code_regs
38806 #undef TARGET_CC_MODES_COMPATIBLE
38807 #define TARGET_CC_MODES_COMPATIBLE ix86_cc_modes_compatible
38809 #undef TARGET_MACHINE_DEPENDENT_REORG
38810 #define TARGET_MACHINE_DEPENDENT_REORG ix86_reorg
38812 #undef TARGET_BUILTIN_SETJMP_FRAME_VALUE
38813 #define TARGET_BUILTIN_SETJMP_FRAME_VALUE ix86_builtin_setjmp_frame_value
38815 #undef TARGET_BUILD_BUILTIN_VA_LIST
38816 #define TARGET_BUILD_BUILTIN_VA_LIST ix86_build_builtin_va_list
38818 #undef TARGET_ENUM_VA_LIST_P
38819 #define TARGET_ENUM_VA_LIST_P ix86_enum_va_list
38821 #undef TARGET_FN_ABI_VA_LIST
38822 #define TARGET_FN_ABI_VA_LIST ix86_fn_abi_va_list
38824 #undef TARGET_CANONICAL_VA_LIST_TYPE
38825 #define TARGET_CANONICAL_VA_LIST_TYPE ix86_canonical_va_list_type
38827 #undef TARGET_EXPAND_BUILTIN_VA_START
38828 #define TARGET_EXPAND_BUILTIN_VA_START ix86_va_start
38830 #undef TARGET_MD_ASM_CLOBBERS
38831 #define TARGET_MD_ASM_CLOBBERS ix86_md_asm_clobbers
38833 #undef TARGET_PROMOTE_PROTOTYPES
38834 #define TARGET_PROMOTE_PROTOTYPES hook_bool_const_tree_true
38835 #undef TARGET_STRUCT_VALUE_RTX
38836 #define TARGET_STRUCT_VALUE_RTX ix86_struct_value_rtx
38837 #undef TARGET_SETUP_INCOMING_VARARGS
38838 #define TARGET_SETUP_INCOMING_VARARGS ix86_setup_incoming_varargs
38839 #undef TARGET_MUST_PASS_IN_STACK
38840 #define TARGET_MUST_PASS_IN_STACK ix86_must_pass_in_stack
38841 #undef TARGET_FUNCTION_ARG_ADVANCE
38842 #define TARGET_FUNCTION_ARG_ADVANCE ix86_function_arg_advance
38843 #undef TARGET_FUNCTION_ARG
38844 #define TARGET_FUNCTION_ARG ix86_function_arg
38845 #undef TARGET_FUNCTION_ARG_BOUNDARY
38846 #define TARGET_FUNCTION_ARG_BOUNDARY ix86_function_arg_boundary
38847 #undef TARGET_PASS_BY_REFERENCE
38848 #define TARGET_PASS_BY_REFERENCE ix86_pass_by_reference
38849 #undef TARGET_INTERNAL_ARG_POINTER
38850 #define TARGET_INTERNAL_ARG_POINTER ix86_internal_arg_pointer
38851 #undef TARGET_UPDATE_STACK_BOUNDARY
38852 #define TARGET_UPDATE_STACK_BOUNDARY ix86_update_stack_boundary
38853 #undef TARGET_GET_DRAP_RTX
38854 #define TARGET_GET_DRAP_RTX ix86_get_drap_rtx
38855 #undef TARGET_STRICT_ARGUMENT_NAMING
38856 #define TARGET_STRICT_ARGUMENT_NAMING hook_bool_CUMULATIVE_ARGS_true
38857 #undef TARGET_STATIC_CHAIN
38858 #define TARGET_STATIC_CHAIN ix86_static_chain
38859 #undef TARGET_TRAMPOLINE_INIT
38860 #define TARGET_TRAMPOLINE_INIT ix86_trampoline_init
38861 #undef TARGET_RETURN_POPS_ARGS
38862 #define TARGET_RETURN_POPS_ARGS ix86_return_pops_args
38864 #undef TARGET_GIMPLIFY_VA_ARG_EXPR
38865 #define TARGET_GIMPLIFY_VA_ARG_EXPR ix86_gimplify_va_arg
38867 #undef TARGET_SCALAR_MODE_SUPPORTED_P
38868 #define TARGET_SCALAR_MODE_SUPPORTED_P ix86_scalar_mode_supported_p
38870 #undef TARGET_VECTOR_MODE_SUPPORTED_P
38871 #define TARGET_VECTOR_MODE_SUPPORTED_P ix86_vector_mode_supported_p
38873 #undef TARGET_C_MODE_FOR_SUFFIX
38874 #define TARGET_C_MODE_FOR_SUFFIX ix86_c_mode_for_suffix
38876 #ifdef HAVE_AS_TLS
38877 #undef TARGET_ASM_OUTPUT_DWARF_DTPREL
38878 #define TARGET_ASM_OUTPUT_DWARF_DTPREL i386_output_dwarf_dtprel
38879 #endif
38881 #ifdef SUBTARGET_INSERT_ATTRIBUTES
38882 #undef TARGET_INSERT_ATTRIBUTES
38883 #define TARGET_INSERT_ATTRIBUTES SUBTARGET_INSERT_ATTRIBUTES
38884 #endif
38886 #undef TARGET_MANGLE_TYPE
38887 #define TARGET_MANGLE_TYPE ix86_mangle_type
38889 #if !TARGET_MACHO
38890 #undef TARGET_STACK_PROTECT_FAIL
38891 #define TARGET_STACK_PROTECT_FAIL ix86_stack_protect_fail
38892 #endif
38894 #undef TARGET_FUNCTION_VALUE
38895 #define TARGET_FUNCTION_VALUE ix86_function_value
38897 #undef TARGET_FUNCTION_VALUE_REGNO_P
38898 #define TARGET_FUNCTION_VALUE_REGNO_P ix86_function_value_regno_p
38900 #undef TARGET_PROMOTE_FUNCTION_MODE
38901 #define TARGET_PROMOTE_FUNCTION_MODE ix86_promote_function_mode
38903 #undef TARGET_SECONDARY_RELOAD
38904 #define TARGET_SECONDARY_RELOAD ix86_secondary_reload
38906 #undef TARGET_CLASS_MAX_NREGS
38907 #define TARGET_CLASS_MAX_NREGS ix86_class_max_nregs
38909 #undef TARGET_PREFERRED_RELOAD_CLASS
38910 #define TARGET_PREFERRED_RELOAD_CLASS ix86_preferred_reload_class
38911 #undef TARGET_PREFERRED_OUTPUT_RELOAD_CLASS
38912 #define TARGET_PREFERRED_OUTPUT_RELOAD_CLASS ix86_preferred_output_reload_class
38913 #undef TARGET_CLASS_LIKELY_SPILLED_P
38914 #define TARGET_CLASS_LIKELY_SPILLED_P ix86_class_likely_spilled_p
38916 #undef TARGET_VECTORIZE_BUILTIN_VECTORIZATION_COST
38917 #define TARGET_VECTORIZE_BUILTIN_VECTORIZATION_COST \
38918 ix86_builtin_vectorization_cost
38919 #undef TARGET_VECTORIZE_VEC_PERM_CONST_OK
38920 #define TARGET_VECTORIZE_VEC_PERM_CONST_OK \
38921 ix86_vectorize_vec_perm_const_ok
38922 #undef TARGET_VECTORIZE_PREFERRED_SIMD_MODE
38923 #define TARGET_VECTORIZE_PREFERRED_SIMD_MODE \
38924 ix86_preferred_simd_mode
38925 #undef TARGET_VECTORIZE_AUTOVECTORIZE_VECTOR_SIZES
38926 #define TARGET_VECTORIZE_AUTOVECTORIZE_VECTOR_SIZES \
38927 ix86_autovectorize_vector_sizes
38929 #undef TARGET_SET_CURRENT_FUNCTION
38930 #define TARGET_SET_CURRENT_FUNCTION ix86_set_current_function
38932 #undef TARGET_OPTION_VALID_ATTRIBUTE_P
38933 #define TARGET_OPTION_VALID_ATTRIBUTE_P ix86_valid_target_attribute_p
38935 #undef TARGET_OPTION_SAVE
38936 #define TARGET_OPTION_SAVE ix86_function_specific_save
38938 #undef TARGET_OPTION_RESTORE
38939 #define TARGET_OPTION_RESTORE ix86_function_specific_restore
38941 #undef TARGET_OPTION_PRINT
38942 #define TARGET_OPTION_PRINT ix86_function_specific_print
38944 #undef TARGET_CAN_INLINE_P
38945 #define TARGET_CAN_INLINE_P ix86_can_inline_p
38947 #undef TARGET_EXPAND_TO_RTL_HOOK
38948 #define TARGET_EXPAND_TO_RTL_HOOK ix86_maybe_switch_abi
38950 #undef TARGET_LEGITIMATE_ADDRESS_P
38951 #define TARGET_LEGITIMATE_ADDRESS_P ix86_legitimate_address_p
38953 #undef TARGET_LEGITIMATE_CONSTANT_P
38954 #define TARGET_LEGITIMATE_CONSTANT_P ix86_legitimate_constant_p
38956 #undef TARGET_FRAME_POINTER_REQUIRED
38957 #define TARGET_FRAME_POINTER_REQUIRED ix86_frame_pointer_required
38959 #undef TARGET_CAN_ELIMINATE
38960 #define TARGET_CAN_ELIMINATE ix86_can_eliminate
38962 #undef TARGET_EXTRA_LIVE_ON_ENTRY
38963 #define TARGET_EXTRA_LIVE_ON_ENTRY ix86_live_on_entry
38965 #undef TARGET_ASM_CODE_END
38966 #define TARGET_ASM_CODE_END ix86_code_end
38968 #undef TARGET_CONDITIONAL_REGISTER_USAGE
38969 #define TARGET_CONDITIONAL_REGISTER_USAGE ix86_conditional_register_usage
38971 #if TARGET_MACHO
38972 #undef TARGET_INIT_LIBFUNCS
38973 #define TARGET_INIT_LIBFUNCS darwin_rename_builtins
38974 #endif
38977 \f