| blob | 0be671da3902918e645a898692e6b691c95ca385 |
1 /* Subroutines used for code generation on IA-32.
2 Copyright (C) 1988-2013 Free Software Foundation, Inc.
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 3, or (at your option)
9 any later version.
11 GCC is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
87 #ifndef CHECK_STACK_LIMIT
88 #define CHECK_STACK_LIMIT (-1)
89 #endif
91 /* Return index of given mode in mult and division cost tables. */
92 #define MODE_INDEX(mode) \
93 ((mode) == QImode ? 0 \
94 : (mode) == HImode ? 1 \
95 : (mode) == SImode ? 2 \
96 : (mode) == DImode ? 3 \
97 : 4)
99 /* Processor costs (relative to an add) */
100 /* We assume COSTS_N_INSNS is defined as (N)*4 and an addition is 2 bytes. */
101 #define COSTS_N_BYTES(N) ((N) * 2)
103 #define DUMMY_STRINGOP_ALGS {libcall, {{-1, libcall, false}}}
112 const
135 in QImode, HImode and SImode.
136 Relative to reg-reg move (2). */
140 in SFmode, DFmode and XFmode */
142 in SFmode, DFmode and XFmode */
145 in SImode and DImode */
147 in SImode and DImode */
150 in SImode, DImode and TImode */
152 in SImode, DImode and TImode */
165 ix86_size_memcpy,
166 ix86_size_memset,
178 };
180 /* Processor costs (relative to an add) */
183 DUMMY_STRINGOP_ALGS};
186 DUMMY_STRINGOP_ALGS};
188 static const
211 in QImode, HImode and SImode.
212 Relative to reg-reg move (2). */
216 in SFmode, DFmode and XFmode */
218 in SFmode, DFmode and XFmode */
221 in SImode and DImode */
223 in SImode and DImode */
226 in SImode, DImode and TImode */
228 in SImode, DImode and TImode */
241 i386_memcpy,
242 i386_memset,
254 };
258 DUMMY_STRINGOP_ALGS};
261 DUMMY_STRINGOP_ALGS};
263 static const
286 in QImode, HImode and SImode.
287 Relative to reg-reg move (2). */
291 in SFmode, DFmode and XFmode */
293 in SFmode, DFmode and XFmode */
296 in SImode and DImode */
298 in SImode and DImode */
301 in SImode, DImode and TImode */
303 in SImode, DImode and TImode */
306 shared for code and data, so 4kB is
307 not really precise. */
318 i486_memcpy,
319 i486_memset,
331 };
335 DUMMY_STRINGOP_ALGS};
338 DUMMY_STRINGOP_ALGS};
340 static const
363 in QImode, HImode and SImode.
364 Relative to reg-reg move (2). */
368 in SFmode, DFmode and XFmode */
370 in SFmode, DFmode and XFmode */
373 in SImode and DImode */
375 in SImode and DImode */
378 in SImode, DImode and TImode */
380 in SImode, DImode and TImode */
393 pentium_memcpy,
394 pentium_memset,
406 };
408 /* PentiumPro has optimized rep instructions for blocks aligned by 8 bytes
409 (we ensure the alignment). For small blocks inline loop is still a
410 noticeable win, for bigger blocks either rep movsl or rep movsb is
411 way to go. Rep movsb has apparently more expensive startup time in CPU,
412 but after 4K the difference is down in the noise. */
417 DUMMY_STRINGOP_ALGS};
422 DUMMY_STRINGOP_ALGS};
423 static const
446 in QImode, HImode and SImode.
447 Relative to reg-reg move (2). */
451 in SFmode, DFmode and XFmode */
453 in SFmode, DFmode and XFmode */
456 in SImode and DImode */
458 in SImode and DImode */
461 in SImode, DImode and TImode */
463 in SImode, DImode and TImode */
476 pentiumpro_memcpy,
477 pentiumpro_memset,
489 };
493 DUMMY_STRINGOP_ALGS};
496 DUMMY_STRINGOP_ALGS};
497 static const
520 in QImode, HImode and SImode.
521 Relative to reg-reg move (2). */
525 in SFmode, DFmode and XFmode */
527 in SFmode, DFmode and XFmode */
531 in SImode and DImode */
533 in SImode and DImode */
536 in SImode, DImode and TImode */
538 in SImode, DImode and TImode */
551 geode_memcpy,
552 geode_memset,
564 };
568 DUMMY_STRINGOP_ALGS};
571 DUMMY_STRINGOP_ALGS};
572 static const
595 in QImode, HImode and SImode.
596 Relative to reg-reg move (2). */
600 in SFmode, DFmode and XFmode */
602 in SFmode, DFmode and XFmode */
605 in SImode and DImode */
607 in SImode and DImode */
610 in SImode, DImode and TImode */
612 in SImode, DImode and TImode */
616 have integrated l2 cache, but
617 optimizing for k6 is not important
618 enough to worry about that. */
628 k6_memcpy,
629 k6_memset,
641 };
643 /* For some reason, Athlon deals better with REP prefix (relative to loops)
644 compared to K8. Alignment becomes important after 8 bytes for memcpy and
645 128 bytes for memset. */
648 DUMMY_STRINGOP_ALGS};
651 DUMMY_STRINGOP_ALGS};
652 static const
675 in QImode, HImode and SImode.
676 Relative to reg-reg move (2). */
680 in SFmode, DFmode and XFmode */
682 in SFmode, DFmode and XFmode */
685 in SImode and DImode */
687 in SImode and DImode */
690 in SImode, DImode and TImode */
692 in SImode, DImode and TImode */
705 athlon_memcpy,
706 athlon_memset,
718 };
720 /* K8 has optimized REP instruction for medium sized blocks, but for very
721 small blocks it is better to use loop. For large blocks, libcall can
722 do nontemporary accesses and beat inline considerably. */
733 static const
756 in QImode, HImode and SImode.
757 Relative to reg-reg move (2). */
761 in SFmode, DFmode and XFmode */
763 in SFmode, DFmode and XFmode */
766 in SImode and DImode */
768 in SImode and DImode */
771 in SImode, DImode and TImode */
773 in SImode, DImode and TImode */
778 /* New AMD processors never drop prefetches; if they cannot be performed
779 immediately, they are queued. We set number of simultaneous prefetches
780 to a large constant to reflect this (it probably is not a good idea not
781 to limit number of prefetches at all, as their execution also takes some
782 time). */
792 k8_memcpy,
793 k8_memset,
805 };
807 /* AMDFAM10 has optimized REP instruction for medium sized blocks, but for
808 very small blocks it is better to use loop. For large blocks, libcall can
809 do nontemporary accesses and beat inline considerably. */
842 in QImode, HImode and SImode.
843 Relative to reg-reg move (2). */
847 in SFmode, DFmode and XFmode */
849 in SFmode, DFmode and XFmode */
852 in SImode and DImode */
854 in SImode and DImode */
857 in SImode, DImode and TImode */
859 in SImode, DImode and TImode */
861 /* On K8:
862 MOVD reg64, xmmreg Double FSTORE 4
863 MOVD reg32, xmmreg Double FSTORE 4
864 On AMDFAM10:
865 MOVD reg64, xmmreg Double FADD 3
866 1/1 1/1
867 MOVD reg32, xmmreg Double FADD 3
868 1/1 1/1 */
872 /* New AMD processors never drop prefetches; if they cannot be performed
873 immediately, they are queued. We set number of simultaneous prefetches
874 to a large constant to reflect this (it probably is not a good idea not
875 to limit number of prefetches at all, as their execution also takes some
876 time). */
886 amdfam10_memcpy,
887 amdfam10_memset,
899 };
901 /* BDVER1 has optimized REP instruction for medium sized blocks, but for
902 very small blocks it is better to use loop. For large blocks, libcall
903 can do nontemporary accesses and beat inline considerably. */
937 in QImode, HImode and SImode.
938 Relative to reg-reg move (2). */
942 in SFmode, DFmode and XFmode */
944 in SFmode, DFmode and XFmode */
947 in SImode and DImode */
949 in SImode and DImode */
952 in SImode, DImode and TImode */
954 in SImode, DImode and TImode */
956 /* On K8:
957 MOVD reg64, xmmreg Double FSTORE 4
958 MOVD reg32, xmmreg Double FSTORE 4
959 On AMDFAM10:
960 MOVD reg64, xmmreg Double FADD 3
961 1/1 1/1
962 MOVD reg32, xmmreg Double FADD 3
963 1/1 1/1 */
967 /* New AMD processors never drop prefetches; if they cannot be performed
968 immediately, they are queued. We set number of simultaneous prefetches
969 to a large constant to reflect this (it probably is not a good idea not
970 to limit number of prefetches at all, as their execution also takes some
971 time). */
981 bdver1_memcpy,
982 bdver1_memset,
994 };
996 /* BDVER2 has optimized REP instruction for medium sized blocks, but for
997 very small blocks it is better to use loop. For large blocks, libcall
998 can do nontemporary accesses and beat inline considerably. */
1033 in QImode, HImode and SImode.
1034 Relative to reg-reg move (2). */
1038 in SFmode, DFmode and XFmode */
1040 in SFmode, DFmode and XFmode */
1043 in SImode and DImode */
1045 in SImode and DImode */
1048 in SImode, DImode and TImode */
1050 in SImode, DImode and TImode */
1052 /* On K8:
1053 MOVD reg64, xmmreg Double FSTORE 4
1054 MOVD reg32, xmmreg Double FSTORE 4
1055 On AMDFAM10:
1056 MOVD reg64, xmmreg Double FADD 3
1057 1/1 1/1
1058 MOVD reg32, xmmreg Double FADD 3
1059 1/1 1/1 */
1063 /* New AMD processors never drop prefetches; if they cannot be performed
1064 immediately, they are queued. We set number of simultaneous prefetches
1065 to a large constant to reflect this (it probably is not a good idea not
1066 to limit number of prefetches at all, as their execution also takes some
1067 time). */
1077 bdver2_memcpy,
1078 bdver2_memset,
1090 };
1093 /* BDVER3 has optimized REP instruction for medium sized blocks, but for
1094 very small blocks it is better to use loop. For large blocks, libcall
1095 can do nontemporary accesses and beat inline considerably. */
1128 in QImode, HImode and SImode.
1129 Relative to reg-reg move (2). */
1133 in SFmode, DFmode and XFmode */
1135 in SFmode, DFmode and XFmode */
1138 in SImode and DImode */
1140 in SImode and DImode */
1143 in SImode, DImode and TImode */
1145 in SImode, DImode and TImode */
1150 /* New AMD processors never drop prefetches; if they cannot be performed
1151 immediately, they are queued. We set number of simultaneous prefetches
1152 to a large constant to reflect this (it probably is not a good idea not
1153 to limit number of prefetches at all, as their execution also takes some
1154 time). */
1164 bdver3_memcpy,
1165 bdver3_memset,
1177 };
1179 /* BDVER4 has optimized REP instruction for medium sized blocks, but for
1180 very small blocks it is better to use loop. For large blocks, libcall
1181 can do nontemporary accesses and beat inline considerably. */
1214 in QImode, HImode and SImode.
1215 Relative to reg-reg move (2). */
1219 in SFmode, DFmode and XFmode */
1221 in SFmode, DFmode and XFmode */
1224 in SImode and DImode */
1226 in SImode and DImode */
1229 in SImode, DImode and TImode */
1231 in SImode, DImode and TImode */
1236 /* New AMD processors never drop prefetches; if they cannot be performed
1237 immediately, they are queued. We set number of simultaneous prefetches
1238 to a large constant to reflect this (it probably is not a good idea not
1239 to limit number of prefetches at all, as their execution also takes some
1240 time). */
1250 bdver4_memcpy,
1251 bdver4_memset,
1263 };
1265 /* BTVER1 has optimized REP instruction for medium sized blocks, but for
1266 very small blocks it is better to use loop. For large blocks, libcall can
1267 do nontemporary accesses and beat inline considerably. */
1300 in QImode, HImode and SImode.
1301 Relative to reg-reg move (2). */
1305 in SFmode, DFmode and XFmode */
1307 in SFmode, DFmode and XFmode */
1310 in SImode and DImode */
1312 in SImode and DImode */
1315 in SImode, DImode and TImode */
1317 in SImode, DImode and TImode */
1319 /* On K8:
1320 MOVD reg64, xmmreg Double FSTORE 4
1321 MOVD reg32, xmmreg Double FSTORE 4
1322 On AMDFAM10:
1323 MOVD reg64, xmmreg Double FADD 3
1324 1/1 1/1
1325 MOVD reg32, xmmreg Double FADD 3
1326 1/1 1/1 */
1339 btver1_memcpy,
1340 btver1_memset,
1352 };
1386 in QImode, HImode and SImode.
1387 Relative to reg-reg move (2). */
1391 in SFmode, DFmode and XFmode */
1393 in SFmode, DFmode and XFmode */
1396 in SImode and DImode */
1398 in SImode and DImode */
1401 in SImode, DImode and TImode */
1403 in SImode, DImode and TImode */
1405 /* On K8:
1406 MOVD reg64, xmmreg Double FSTORE 4
1407 MOVD reg32, xmmreg Double FSTORE 4
1408 On AMDFAM10:
1409 MOVD reg64, xmmreg Double FADD 3
1410 1/1 1/1
1411 MOVD reg32, xmmreg Double FADD 3
1412 1/1 1/1 */
1424 btver2_memcpy,
1425 btver2_memset,
1437 };
1441 DUMMY_STRINGOP_ALGS};
1445 DUMMY_STRINGOP_ALGS};
1447 static const
1470 in QImode, HImode and SImode.
1471 Relative to reg-reg move (2). */
1475 in SFmode, DFmode and XFmode */
1477 in SFmode, DFmode and XFmode */
1480 in SImode and DImode */
1482 in SImode and DImode */
1485 in SImode, DImode and TImode */
1487 in SImode, DImode and TImode */
1500 pentium4_memcpy,
1501 pentium4_memset,
1513 };
1526 static const
1549 in QImode, HImode and SImode.
1550 Relative to reg-reg move (2). */
1554 in SFmode, DFmode and XFmode */
1556 in SFmode, DFmode and XFmode */
1559 in SImode and DImode */
1561 in SImode and DImode */
1564 in SImode, DImode and TImode */
1566 in SImode, DImode and TImode */
1579 nocona_memcpy,
1580 nocona_memset,
1592 };
1603 static const
1626 in QImode, HImode and SImode.
1627 Relative to reg-reg move (2). */
1631 in SFmode, DFmode and XFmode */
1633 in SFmode, DFmode and XFmode */
1636 in SImode and DImode */
1638 in SImode and DImode */
1641 in SImode, DImode and TImode */
1643 in SImode, DImode and TImode */
1656 atom_memcpy,
1657 atom_memset,
1669 };
1680 static const
1703 in QImode, HImode and SImode.
1704 Relative to reg-reg move (2). */
1708 in SFmode, DFmode and XFmode */
1710 in SFmode, DFmode and XFmode */
1713 in SImode and DImode */
1715 in SImode and DImode */
1718 in SImode, DImode and TImode */
1720 in SImode, DImode and TImode */
1733 slm_memcpy,
1734 slm_memset,
1746 };
1748 /* Generic should produce code tuned for Core-i7 (and newer chips)
1749 and btver1 (and newer chips). */
1761 static const
1764 /* On all chips taken into consideration lea is 2 cycles and more. With
1765 this cost however our current implementation of synth_mult results in
1766 use of unnecessary temporary registers causing regression on several
1767 SPECfp benchmarks. */
1788 in QImode, HImode and SImode.
1789 Relative to reg-reg move (2). */
1793 in SFmode, DFmode and XFmode */
1795 in SFmode, DFmode and XFmode */
1798 in SImode and DImode */
1800 in SImode and DImode */
1803 in SImode, DImode and TImode */
1805 in SImode, DImode and TImode */
1811 /* Benchmarks shows large regressions on K8 sixtrack benchmark when this
1812 value is increased to perhaps more appropriate value of 5. */
1820 generic_memcpy,
1821 generic_memset,
1833 };
1835 /* core_cost should produce code tuned for Core familly of CPUs. */
1848 static const
1851 /* On all chips taken into consideration lea is 2 cycles and more. With
1852 this cost however our current implementation of synth_mult results in
1853 use of unnecessary temporary registers causing regression on several
1854 SPECfp benchmarks. */
1875 in QImode, HImode and SImode.
1876 Relative to reg-reg move (2). */
1880 in SFmode, DFmode and XFmode */
1882 in SFmode, DFmode and XFmode */
1885 in SImode and DImode */
1887 in SImode and DImode */
1890 in SImode, DImode and TImode */
1892 in SImode, DImode and TImode */
1898 /* FIXME perhaps more appropriate value is 5. */
1906 core_memcpy,
1907 core_memset,
1919 };
1922 /* Set by -mtune. */
1925 /* Set by -mtune or -Os. */
1928 /* Processor feature/optimization bitmasks. */
1929 #define m_386 (1<<PROCESSOR_I386)
1930 #define m_486 (1<<PROCESSOR_I486)
1931 #define m_PENT (1<<PROCESSOR_PENTIUM)
1932 #define m_PPRO (1<<PROCESSOR_PENTIUMPRO)
1933 #define m_PENT4 (1<<PROCESSOR_PENTIUM4)
1934 #define m_NOCONA (1<<PROCESSOR_NOCONA)
1935 #define m_P4_NOCONA (m_PENT4 | m_NOCONA)
1936 #define m_CORE2 (1<<PROCESSOR_CORE2)
1937 #define m_COREI7 (1<<PROCESSOR_COREI7)
1938 #define m_COREI7_AVX (1<<PROCESSOR_COREI7_AVX)
1939 #define m_HASWELL (1<<PROCESSOR_HASWELL)
1940 #define m_CORE_ALL (m_CORE2 | m_COREI7 | m_COREI7_AVX | m_HASWELL)
1941 #define m_ATOM (1<<PROCESSOR_ATOM)
1942 #define m_SLM (1<<PROCESSOR_SLM)
1944 #define m_GEODE (1<<PROCESSOR_GEODE)
1945 #define m_K6 (1<<PROCESSOR_K6)
1946 #define m_K6_GEODE (m_K6 | m_GEODE)
1947 #define m_K8 (1<<PROCESSOR_K8)
1948 #define m_ATHLON (1<<PROCESSOR_ATHLON)
1949 #define m_ATHLON_K8 (m_K8 | m_ATHLON)
1950 #define m_AMDFAM10 (1<<PROCESSOR_AMDFAM10)
1951 #define m_BDVER1 (1<<PROCESSOR_BDVER1)
1952 #define m_BDVER2 (1<<PROCESSOR_BDVER2)
1953 #define m_BDVER3 (1<<PROCESSOR_BDVER3)
1954 #define m_BDVER4 (1<<PROCESSOR_BDVER4)
1955 #define m_BTVER1 (1<<PROCESSOR_BTVER1)
1956 #define m_BTVER2 (1<<PROCESSOR_BTVER2)
1957 #define m_BDVER (m_BDVER1 | m_BDVER2 | m_BDVER3 | m_BDVER4)
1958 #define m_BTVER (m_BTVER1 | m_BTVER2)
1959 #define m_AMD_MULTIPLE (m_ATHLON_K8 | m_AMDFAM10 | m_BDVER | m_BTVER)
1961 #define m_GENERIC (1<<PROCESSOR_GENERIC)
1964 #undef DEF_TUNE
1965 #define DEF_TUNE(tune, name, selector) name,
1967 #undef DEF_TUNE
1968 };
1970 /* Feature tests against the various tunings. */
1973 /* Feature tests against the various tunings used to create ix86_tune_features
1974 based on the processor mask. */
1976 #undef DEF_TUNE
1977 #define DEF_TUNE(tune, name, selector) selector,
1979 #undef DEF_TUNE
1980 };
1982 /* Feature tests against the various architecture variations. */
1985 /* Feature tests against the various architecture variations, used to create
1986 ix86_arch_features based on the processor mask. */
1988 /* X86_ARCH_CMOV: Conditional move was added for pentiumpro. */
1991 /* X86_ARCH_CMPXCHG: Compare and exchange was added for 80486. */
1994 /* X86_ARCH_CMPXCHG8B: Compare and exchange 8 bytes was added for pentium. */
1997 /* X86_ARCH_XADD: Exchange and add was added for 80486. */
2000 /* X86_ARCH_BSWAP: Byteswap was added for 80486. */
2002 };
2004 /* In case the average insn count for single function invocation is
2005 lower than this constant, emit fast (but longer) prologue and
2006 epilogue code. */
2007 #define FAST_PROLOGUE_INSN_COUNT 20
2009 /* Names for 8 (low), 8 (high), and 16-bit registers, respectively. */
2014 /* Array of the smallest class containing reg number REGNO, indexed by
2015 REGNO. Used by REGNO_REG_CLASS in i386.h. */
2018 {
2019 /* ax, dx, cx, bx */
2021 /* si, di, bp, sp */
2023 /* FP registers */
2026 /* arg pointer */
2027 NON_Q_REGS,
2028 /* flags, fpsr, fpcr, frame */
2030 /* SSE registers */
2033 /* MMX registers */
2036 /* REX registers */
2039 /* SSE REX registers */
2042 /* AVX-512 SSE registers */
2047 /* Mask registers. */
2050 };
2052 /* The "default" register map used in 32bit mode. */
2055 {
2066 };
2068 /* The "default" register map used in 64bit mode. */
2071 {
2082 };
2084 /* Define the register numbers to be used in Dwarf debugging information.
2085 The SVR4 reference port C compiler uses the following register numbers
2086 in its Dwarf output code:
2087 0 for %eax (gcc regno = 0)
2088 1 for %ecx (gcc regno = 2)
2089 2 for %edx (gcc regno = 1)
2090 3 for %ebx (gcc regno = 3)
2091 4 for %esp (gcc regno = 7)
2092 5 for %ebp (gcc regno = 6)
2093 6 for %esi (gcc regno = 4)
2094 7 for %edi (gcc regno = 5)
2095 The following three DWARF register numbers are never generated by
2096 the SVR4 C compiler or by the GNU compilers, but SDB on x86/svr4
2097 believes these numbers have these meanings.
2098 8 for %eip (no gcc equivalent)
2099 9 for %eflags (gcc regno = 17)
2100 10 for %trapno (no gcc equivalent)
2101 It is not at all clear how we should number the FP stack registers
2102 for the x86 architecture. If the version of SDB on x86/svr4 were
2103 a bit less brain dead with respect to floating-point then we would
2104 have a precedent to follow with respect to DWARF register numbers
2105 for x86 FP registers, but the SDB on x86/svr4 is so completely
2106 broken with respect to FP registers that it is hardly worth thinking
2107 of it as something to strive for compatibility with.
2108 The version of x86/svr4 SDB I have at the moment does (partially)
2109 seem to believe that DWARF register number 11 is associated with
2110 the x86 register %st(0), but that's about all. Higher DWARF
2111 register numbers don't seem to be associated with anything in
2112 particular, and even for DWARF regno 11, SDB only seems to under-
2113 stand that it should say that a variable lives in %st(0) (when
2114 asked via an `=' command) if we said it was in DWARF regno 11,
2115 but SDB still prints garbage when asked for the value of the
2116 variable in question (via a `/' command).
2117 (Also note that the labels SDB prints for various FP stack regs
2118 when doing an `x' command are all wrong.)
2119 Note that these problems generally don't affect the native SVR4
2120 C compiler because it doesn't allow the use of -O with -g and
2121 because when it is *not* optimizing, it allocates a memory
2122 location for each floating-point variable, and the memory
2123 location is what gets described in the DWARF AT_location
2124 attribute for the variable in question.
2125 Regardless of the severe mental illness of the x86/svr4 SDB, we
2126 do something sensible here and we use the following DWARF
2127 register numbers. Note that these are all stack-top-relative
2128 numbers.
2129 11 for %st(0) (gcc regno = 8)
2130 12 for %st(1) (gcc regno = 9)
2131 13 for %st(2) (gcc regno = 10)
2132 14 for %st(3) (gcc regno = 11)
2133 15 for %st(4) (gcc regno = 12)
2134 16 for %st(5) (gcc regno = 13)
2135 17 for %st(6) (gcc regno = 14)
2136 18 for %st(7) (gcc regno = 15)
2137 */
2139 {
2150 };
2152 /* Define parameter passing and return registers. */
2155 {
2157 };
2160 {
2162 };
2165 {
2167 };
2169 /* Additional registers that are clobbered by SYSV calls. */
2172 {
2177 };
2179 /* Define the structure for the machine field in struct function. */
2184 rtx rtl;
2186 };
2188 /* Structure describing stack frame layout.
2189 Stack grows downward:
2191 [arguments]
2192 <- ARG_POINTER
2193 saved pc
2195 saved static chain if ix86_static_chain_on_stack
2197 saved frame pointer if frame_pointer_needed
2198 <- HARD_FRAME_POINTER
2199 [saved regs]
2200 <- regs_save_offset
2201 [padding0]
2203 [saved SSE regs]
2204 <- sse_regs_save_offset
2205 [padding1] |
2206 | <- FRAME_POINTER
2207 [va_arg registers] |
2208 |
2209 [frame] |
2210 |
2211 [padding2] | = to_allocate
2212 <- STACK_POINTER
2213 */
2214 struct ix86_frame
2215 {
2222 /* The offsets relative to ARG_POINTER. */
2223 HOST_WIDE_INT frame_pointer_offset;
2224 HOST_WIDE_INT hard_frame_pointer_offset;
2225 HOST_WIDE_INT stack_pointer_offset;
2226 HOST_WIDE_INT hfp_save_offset;
2227 HOST_WIDE_INT reg_save_offset;
2228 HOST_WIDE_INT sse_reg_save_offset;
2230 /* When save_regs_using_mov is set, emit prologue using
2231 move instead of push instructions. */
2233 };
2235 /* Which cpu are we scheduling for. */
2238 /* Which cpu are we optimizing for. */
2241 /* Which instruction set architecture to use. */
2244 /* True if processor has SSE prefetch instruction. */
2247 /* -mstackrealign option */
2264 /* Preferred alignment for stack boundary in bits. */
2267 /* Alignment for incoming stack boundary in bits specified at
2268 command line. */
2271 /* Default alignment for incoming stack boundary in bits. */
2274 /* Alignment for incoming stack boundary in bits. */
2277 /* Calling abi specific va_list type nodes. */
2281 /* Prefix built by ASM_GENERATE_INTERNAL_LABEL. */
2285 /* Fence to use after loop using movnt. */
2286 tree x86_mfence;
2288 /* Register class used for passing given 64bit part of the argument.
2289 These represent classes as documented by the PS ABI, with the exception
2290 of SSESF, SSEDF classes, that are basically SSE class, just gcc will
2291 use SF or DFmode move instead of DImode to avoid reformatting penalties.
2293 Similarly we play games with INTEGERSI_CLASS to use cheaper SImode moves
2294 whenever possible (upper half does contain padding). */
2295 enum x86_64_reg_class
2296 {
2297 X86_64_NO_CLASS,
2298 X86_64_INTEGER_CLASS,
2299 X86_64_INTEGERSI_CLASS,
2300 X86_64_SSE_CLASS,
2301 X86_64_SSESF_CLASS,
2302 X86_64_SSEDF_CLASS,
2303 X86_64_SSEUP_CLASS,
2304 X86_64_X87_CLASS,
2305 X86_64_X87UP_CLASS,
2306 X86_64_COMPLEX_X87_CLASS,
2307 X86_64_MEMORY_CLASS
2308 };
2310 #define MAX_CLASSES 4
2312 /* Table of constants used by fldpi, fldln2, etc.... */
2316 \f
2321 const_tree);
2333 enum ix86_function_specific_strings
2334 {
2335 IX86_FUNCTION_SPECIFIC_ARCH,
2336 IX86_FUNCTION_SPECIFIC_TUNE,
2337 IX86_FUNCTION_SPECIFIC_MAX
2338 };
2359 \f
2360 #ifndef SUBTARGET32_DEFAULT_CPU
2362 #endif
2364 /* Whether -mtune= or -march= were specified */
2368 /* Vectorization library interface and handlers. */
2374 /* Processor target table, indexed by processor number */
2375 struct ptt
2376 {
2383 };
2386 {
2397 /* Core 2 */
2399 /* Core i7 */
2401 /* Core i7 avx */
2403 /* Core avx2 */
2415 };
2418 {
2451 "btver2"
2452 };
2453 \f
2454 static bool
2456 {
2458 }
2460 static unsigned int
2462 {
2465 /* vzeroupper instructions are inserted immediately after reload to
2466 account for possible spills from 256bit registers. The pass
2467 reuses mode switching infrastructure by re-running mode insertion
2468 pass, so disable entities that have already been processed. */
2474 /* Call optimize_mode_switching. */
2477 }
2482 {
2494 };
2497 {
2501 {}
2503 /* opt_pass methods: */
2511 rtl_opt_pass *
2513 {
2515 }
2517 /* Return true if a red-zone is in use. */
2521 {
2523 }
2524 \f
2525 /* Return a string that documents the current -m options. The caller is
2526 responsible for freeing the string. */
2532 {
2533 struct ix86_target_opts
2534 {
2537 };
2539 /* This table is ordered so that options like -msse4.2 that imply
2540 preceding options while match those first. */
2542 {
2582 };
2584 /* Flag options. */
2586 {
2614 };
2631 /* Add -march= option. */
2633 {
2636 }
2638 /* Add -mtune= option. */
2640 {
2643 }
2645 /* Add -m32/-m64/-mx32. */
2647 {
2650 else
2652 isa &= ~ (OPTION_MASK_ISA_64BIT
2653 | OPTION_MASK_ABI_64
2655 }
2656 else
2660 /* Pick out the options in isa options. */
2662 {
2664 {
2667 }
2668 }
2671 {
2674 isa);
2675 }
2677 /* Add flag options. */
2679 {
2681 {
2684 }
2685 }
2688 {
2691 }
2693 /* Add -fpmath= option. */
2695 {
2698 {
2713 }
2714 }
2716 /* Any options? */
2722 /* Size the string. */
2726 {
2731 }
2733 /* Build the string. */
2738 {
2745 {
2747 line_len++;
2750 {
2754 }
2755 }
2759 {
2763 }
2764 }
2770 }
2772 /* Return true, if profiling code should be emitted before
2773 prologue. Otherwise it returns false.
2774 Note: For x86 with "hotfix" it is sorried. */
2775 static bool
2777 {
2779 }
2781 /* Function that is callable from the debugger to print the current
2782 options. */
2783 void ATTRIBUTE_UNUSED
2785 {
2791 {
2794 }
2795 else
2799 }
2802 #define DEF_ENUM
2803 #define DEF_ALG(alg, name) #name,
2805 #undef DEF_ENUM
2806 #undef DEF_ALG
2807 };
2809 /* Parse parameter string passed to -mmemcpy-strategy= or -mmemset-strategy=.
2810 The string is of the following form (or comma separated list of it):
2812 strategy_alg:max_size:[align|noalign]
2814 where the full size range for the strategy is either [0, max_size] or
2815 [min_size, max_size], in which min_size is the max_size + 1 of the
2816 preceding range. The last size range must have max_size == -1.
2818 Examples:
2820 1.
2821 -mmemcpy-strategy=libcall:-1:noalign
2823 this is equivalent to (for known size memcpy) -mstringop-strategy=libcall
2826 2.
2827 -mmemset-strategy=rep_8byte:16:noalign,vector_loop:2048:align,libcall:-1:noalign
2829 This is to tell the compiler to use the following strategy for memset
2830 1) when the expected size is between [1, 16], use rep_8byte strategy;
2831 2) when the size is between [17, 2048], use vector_loop;
2832 3) when the size is > 2048, use libcall. */
2834 struct stringop_size_range
2835 {
2837 stringop_alg alg;
2839 };
2841 static void
2843 {
2851 else
2856 do
2857 {
2859 stringop_alg alg;
2868 {
2872 }
2875 {
2879 }
2882 {
2884 {
2887 }
2888 }
2891 {
2893 alg_name,
2896 }
2904 else
2905 {
2909 }
2910 n++;
2912 }
2916 {
2921 }
2924 {
2928 }
2930 /* Now override the default algs array. */
2932 {
2938 }
2939 }
2941 \f
2942 /* parse -mtune-ctrl= option. When DUMP is true,
2943 print the features that are explicitly set. */
2945 static void
2947 {
2955 do
2956 {
2963 {
2964 curr_feature_string++;
2966 }
2968 {
2970 {
2976 }
2977 }
2982 }
2985 }
2987 /* Helper function to set ix86_tune_features. IX86_TUNE is the
2988 processor type. */
2990 static void
2992 {
2997 {
3000 else
3002 }
3005 {
3010 }
3013 }
3016 /* Override various settings based on options. If MAIN_ARGS_P, the
3017 options are from the command line, otherwise they are from
3018 attributes. */
3020 static void
3024 {
3032 #define PTA_3DNOW (HOST_WIDE_INT_1 << 0)
3033 #define PTA_3DNOW_A (HOST_WIDE_INT_1 << 1)
3034 #define PTA_64BIT (HOST_WIDE_INT_1 << 2)
3035 #define PTA_ABM (HOST_WIDE_INT_1 << 3)
3036 #define PTA_AES (HOST_WIDE_INT_1 << 4)
3037 #define PTA_AVX (HOST_WIDE_INT_1 << 5)
3038 #define PTA_BMI (HOST_WIDE_INT_1 << 6)
3039 #define PTA_CX16 (HOST_WIDE_INT_1 << 7)
3040 #define PTA_F16C (HOST_WIDE_INT_1 << 8)
3041 #define PTA_FMA (HOST_WIDE_INT_1 << 9)
3042 #define PTA_FMA4 (HOST_WIDE_INT_1 << 10)
3043 #define PTA_FSGSBASE (HOST_WIDE_INT_1 << 11)
3044 #define PTA_LWP (HOST_WIDE_INT_1 << 12)
3045 #define PTA_LZCNT (HOST_WIDE_INT_1 << 13)
3046 #define PTA_MMX (HOST_WIDE_INT_1 << 14)
3047 #define PTA_MOVBE (HOST_WIDE_INT_1 << 15)
3048 #define PTA_NO_SAHF (HOST_WIDE_INT_1 << 16)
3049 #define PTA_PCLMUL (HOST_WIDE_INT_1 << 17)
3050 #define PTA_POPCNT (HOST_WIDE_INT_1 << 18)
3051 #define PTA_PREFETCH_SSE (HOST_WIDE_INT_1 << 19)
3052 #define PTA_RDRND (HOST_WIDE_INT_1 << 20)
3053 #define PTA_SSE (HOST_WIDE_INT_1 << 21)
3054 #define PTA_SSE2 (HOST_WIDE_INT_1 << 22)
3055 #define PTA_SSE3 (HOST_WIDE_INT_1 << 23)
3056 #define PTA_SSE4_1 (HOST_WIDE_INT_1 << 24)
3057 #define PTA_SSE4_2 (HOST_WIDE_INT_1 << 25)
3058 #define PTA_SSE4A (HOST_WIDE_INT_1 << 26)
3059 #define PTA_SSSE3 (HOST_WIDE_INT_1 << 27)
3060 #define PTA_TBM (HOST_WIDE_INT_1 << 28)
3061 #define PTA_XOP (HOST_WIDE_INT_1 << 29)
3062 #define PTA_AVX2 (HOST_WIDE_INT_1 << 30)
3063 #define PTA_BMI2 (HOST_WIDE_INT_1 << 31)
3064 #define PTA_RTM (HOST_WIDE_INT_1 << 32)
3065 #define PTA_HLE (HOST_WIDE_INT_1 << 33)
3066 #define PTA_PRFCHW (HOST_WIDE_INT_1 << 34)
3067 #define PTA_RDSEED (HOST_WIDE_INT_1 << 35)
3068 #define PTA_ADX (HOST_WIDE_INT_1 << 36)
3069 #define PTA_FXSR (HOST_WIDE_INT_1 << 37)
3070 #define PTA_XSAVE (HOST_WIDE_INT_1 << 38)
3071 #define PTA_XSAVEOPT (HOST_WIDE_INT_1 << 39)
3072 #define PTA_AVX512F (HOST_WIDE_INT_1 << 40)
3073 #define PTA_AVX512ER (HOST_WIDE_INT_1 << 41)
3074 #define PTA_AVX512PF (HOST_WIDE_INT_1 << 42)
3075 #define PTA_AVX512CD (HOST_WIDE_INT_1 << 43)
3077 /* if this reaches 64, need to widen struct pta flags below */
3079 static struct pta
3080 {
3085 }
3087 {
3230 PTA_64BIT
3232 };
3234 /* -mrecip options. */
3235 static struct
3236 {
3239 }
3241 {
3248 };
3252 /* Set up prefix/suffix so the error messages refer to either the command
3253 line argument, or the attribute(target). */
3255 {
3259 }
3260 else
3261 {
3265 }
3267 /* Turn off both OPTION_MASK_ABI_64 and OPTION_MASK_ABI_X32 if
3268 TARGET_64BIT_DEFAULT is true and TARGET_64BIT is false. */
3271 #ifdef TARGET_BI_ARCH
3272 else
3273 {
3274 #if TARGET_BI_ARCH == 1
3275 /* When TARGET_BI_ARCH == 1, by default, OPTION_MASK_ABI_64
3276 is on and OPTION_MASK_ABI_X32 is off. We turn off
3277 OPTION_MASK_ABI_64 if OPTION_MASK_ABI_X32 is turned on by
3278 -mx32. */
3281 #else
3282 /* When TARGET_BI_ARCH == 2, by default, OPTION_MASK_ABI_X32 is
3283 on and OPTION_MASK_ABI_64 is off. We turn off
3284 OPTION_MASK_ABI_X32 if OPTION_MASK_ABI_64 is turned on by
3285 -m64. */
3288 #endif
3289 }
3290 #endif
3293 {
3294 /* Always turn on OPTION_MASK_ISA_64BIT and turn off
3295 OPTION_MASK_ABI_64 for TARGET_X32. */
3298 }
3300 {
3301 /* Always turn on OPTION_MASK_ISA_64BIT and turn off
3302 OPTION_MASK_ABI_X32 for TARGET_LP64. */
3305 }
3307 #ifdef SUBTARGET_OVERRIDE_OPTIONS
3308 SUBTARGET_OVERRIDE_OPTIONS;
3309 #endif
3311 #ifdef SUBSUBTARGET_OVERRIDE_OPTIONS
3312 SUBSUBTARGET_OVERRIDE_OPTIONS;
3313 #endif
3315 /* -fPIC is the default for x86_64. */
3319 /* Need to check -mtune=generic first. */
3321 {
3324 /* As special support for cross compilers we read -mtune=native
3325 as -mtune=generic. With native compilers we won't see the
3326 -mtune=native, as it was changed by the driver. */
3328 {
3330 }
3331 /* If this call is for setting the option attribute, allow the
3332 generic that was previously set. */
3335 ;
3343 }
3344 else
3345 {
3349 {
3352 }
3354 /* opts->x_ix86_tune_string is set to opts->x_ix86_arch_string
3355 or defaulted. We need to use a sensible tune option. */
3359 {
3361 }
3362 }
3366 {
3367 /* rep; movq isn't available in 32-bit code. */
3370 }
3373 opts->x_ix86_arch_string
3376 else
3380 {
3388 }
3389 else
3396 /* For targets using ms ABI enable ms-extensions, if not
3397 explicit turned off. For non-ms ABI we turn off this
3398 option. */
3403 {
3405 {
3449 {
3452 }
3460 }
3461 }
3462 else
3463 {
3464 /* For TARGET_64BIT and MS_ABI, force pic on, in order to enable the
3465 use of rip-relative addressing. This eliminates fixups that
3466 would otherwise be needed if this object is to be placed in a
3467 DLL, and is essentially just as efficient as direct addressing. */
3473 else
3475 }
3477 {
3480 }
3488 {
3491 /* Default cpu tuning to the architecture. */
3630 }
3648 {
3652 {
3654 {
3656 {
3664 }
3665 else
3668 }
3669 }
3670 /* Intel CPUs have always interpreted SSE prefetch instructions as
3671 NOPs; so, we can enable SSE prefetch instructions even when
3672 -mtune (rather than -march) points us to a processor that has them.
3673 However, the VIA C3 gives a SIGILL, so we only do that for i686 and
3674 higher processors. */
3679 }
3687 #ifndef USE_IX86_FRAME_POINTER
3688 #define USE_IX86_FRAME_POINTER 0
3689 #endif
3691 #ifndef USE_X86_64_FRAME_POINTER
3692 #define USE_X86_64_FRAME_POINTER 0
3693 #endif
3695 /* Set the default values for switches whose default depends on TARGET_64BIT
3696 in case they weren't overwritten by command line options. */
3698 {
3706 opts->x_flag_unwind_tables
3710 }
3711 else
3712 {
3714 opts->x_flag_omit_frame_pointer
3720 }
3725 else
3728 /* Arrange to set up i386_stack_locals for all functions. */
3731 /* Validate -mregparm= value. */
3733 {
3737 {
3741 }
3742 }
3746 /* Default align_* from the processor table. */
3748 {
3751 }
3753 {
3756 }
3758 {
3760 }
3762 /* Provide default for -mbranch-cost= value. */
3767 {
3768 opts->x_target_flags
3771 /* Enable by default the SSE and MMX builtins. Do allow the user to
3772 explicitly disable any of these. In particular, disabling SSE and
3773 MMX for kernel code is extremely useful. */
3775 opts->x_ix86_isa_flags
3782 }
3783 else
3784 {
3785 opts->x_target_flags
3789 opts->x_ix86_isa_flags
3792 /* i386 ABI does not specify red zone. It still makes sense to use it
3793 when programmer takes care to stack from being destroyed. */
3796 }
3798 /* Keep nonleaf frame pointers. */
3804 /* If we're doing fast math, we don't care about comparison order
3805 wrt NaNs. This lets us use a shorter comparison sequence. */
3809 /* If the architecture always has an FPU, turn off NO_FANCY_MATH_387,
3810 since the insns won't need emulation. */
3814 /* Likewise, if the target doesn't have a 387, or we've specified
3815 software floating point, don't use 387 inline intrinsics. */
3819 /* Turn on MMX builtins for -msse. */
3821 opts->x_ix86_isa_flags
3824 /* Enable SSE prefetch. */
3829 /* Enable prefetch{,w} instructions for -m3dnow. */
3831 opts->x_ix86_isa_flags
3834 /* Enable popcnt instruction for -msse4.2 or -mabm. */
3837 opts->x_ix86_isa_flags
3840 /* Enable lzcnt instruction for -mabm. */
3842 opts->x_ix86_isa_flags
3845 /* Validate -mpreferred-stack-boundary= value or default it to
3846 PREFERRED_STACK_BOUNDARY_DEFAULT. */
3849 {
3856 {
3860 else
3863 }
3864 else
3865 ix86_preferred_stack_boundary
3867 }
3869 /* Set the default value for -mstackrealign. */
3875 /* Validate -mincoming-stack-boundary= value or default it to
3876 MIN_STACK_BOUNDARY/PREFERRED_STACK_BOUNDARY. */
3879 {
3886 else
3887 {
3888 ix86_user_incoming_stack_boundary
3890 ix86_incoming_stack_boundary
3892 }
3893 }
3895 /* Accept -msseregparm only if at least SSE support is enabled. */
3901 {
3903 {
3905 {
3908 }
3911 {
3914 }
3915 }
3916 }
3917 /* For all chips supporting SSE2, -mfpmath=sse performs better than
3918 fpmath=387. The second is however default at many targets since the
3919 extra 80bit precision of temporaries is considered to be part of ABI.
3920 Overwrite the default at least for -ffast-math.
3921 TODO: -mfpmath=both seems to produce same performing code with bit
3922 smaller binaries. It is however not clear if register allocation is
3923 ready for this setting.
3924 Also -mfpmath=387 is overall a lot more compact (bout 4-5%) than SSE
3925 codegen. We may switch to 387 with -ffast-math for size optimized
3926 functions. */
3930 else
3933 /* If the i387 is disabled, then do not return values in it. */
3937 /* Use external vectorized library in vectorizing intrinsics. */
3940 {
3951 }
3958 /* If stack probes are required, the space used for large function
3959 arguments on the stack must also be probed, so enable
3960 -maccumulate-outgoing-args so this happens in the prologue. */
3963 {
3968 }
3970 /* Figure out what ASM_GENERATE_INTERNAL_LABEL builds as a prefix. */
3971 {
3977 }
3979 /* When scheduling description is not available, disable scheduler pass
3980 so it won't slow down the compilation and make x87 code slower. */
4001 /* Enable sw prefetching at -O3 for CPUS that prefetching is helpful. */
4003 && HAVE_prefetch
4008 /* If using typedef char *va_list, signal that __builtin_va_start (&ap, 0)
4009 can be opts->x_optimized to ap = __builtin_next_arg (0). */
4014 {
4017 {
4019 ix86_gen_tls_local_dynamic_base_64
4021 }
4022 else
4023 {
4025 ix86_gen_tls_local_dynamic_base_64
4027 }
4028 }
4029 else
4033 {
4043 }
4044 else
4045 {
4055 }
4057 #ifdef USE_IX86_CLD
4058 /* Use -mcld by default for 32-bit code if configured with --enable-cld. */
4061 #endif
4064 {
4069 }
4071 {
4075 }
4077 {
4078 #if defined(PROFILE_BEFORE_PROLOGUE)
4080 #else
4082 #endif
4083 }
4085 /* When not opts->x_optimize for size, enable vzeroupper optimization for
4086 TARGET_AVX with -fexpensive-optimizations and split 32-byte
4087 AVX unaligned load/store. */
4089 {
4099 /* Enable 128-bit AVX instruction generation
4100 for the auto-vectorizer. */
4104 }
4107 {
4114 {
4117 {
4119 q++;
4120 }
4121 else
4126 else
4127 {
4130 {
4133 }
4136 {
4140 }
4141 }
4146 else
4148 }
4149 }
4156 /* Default long double to 64-bit for Bionic. */
4161 /* Save the initial options in case the user does function specific
4162 options. */
4164 target_option_default_node = target_option_current_node
4167 /* Handle stack protector */
4169 opts->x_ix86_stack_protector_guard
4172 /* Handle -mmemcpy-strategy= and -mmemset-strategy= */
4174 {
4178 }
4181 {
4185 }
4186 }
4188 /* Implement the TARGET_OPTION_OVERRIDE hook. */
4190 static void
4192 {
4194 static struct register_pass_info insert_vzeroupper_info
4197 };
4202 /* This needs to be done at start up. It's convenient to do it here. */
4204 }
4206 /* Update register usage after having seen the compiler flags. */
4208 static void
4210 {
4214 /* The PIC register, if it exists, is fixed. */
4219 /* For 32-bit targets, squash the REX registers. */
4221 {
4228 }
4230 /* See the definition of CALL_USED_REGISTERS in i386.h. */
4238 {
4239 /* Set/reset conditionally defined registers from
4240 CALL_USED_REGISTERS initializer. */
4244 /* Calculate registers of CLOBBERED_REGS register set
4245 as call used registers from GENERAL_REGS register set. */
4249 }
4251 /* If MMX is disabled, squash the registers. */
4257 /* If SSE is disabled, squash the registers. */
4263 /* If the FPU is disabled, squash the registers. */
4269 /* If AVX512F is disabled, squash the registers. */
4271 {
4277 }
4278 }
4280 \f
4281 /* Save the current options */
4283 static void
4286 {
4321 /* The fields are char but the variables are not; make sure the
4322 values fit in the fields. */
4327 }
4329 /* Restore the current options */
4331 static void
4334 {
4340 /* We don't change -fPIC. */
4377 /* Recreate the arch feature tests if the arch changed */
4379 {
4384 }
4386 /* Recreate the tune optimization tests */
4389 }
4391 /* Print the current options */
4393 static void
4396 {
4418 {
4421 }
4422 }
4424 \f
4425 /* Inner function to process the attribute((target(...))), take an argument and
4426 set the current options from the argument. If we have a list, recursively go
4427 over the list. */
4429 static bool
4434 {
4438 #define IX86_ATTR_ISA(S,O) { S, sizeof (S)-1, ix86_opt_isa, O, 0 }
4439 #define IX86_ATTR_STR(S,O) { S, sizeof (S)-1, ix86_opt_str, O, 0 }
4440 #define IX86_ATTR_ENUM(S,O) { S, sizeof (S)-1, ix86_opt_enum, O, 0 }
4441 #define IX86_ATTR_YES(S,O,M) { S, sizeof (S)-1, ix86_opt_yes, O, M }
4442 #define IX86_ATTR_NO(S,O,M) { S, sizeof (S)-1, ix86_opt_no, O, M }
4444 enum ix86_opt_type
4445 {
4446 ix86_opt_unknown,
4447 ix86_opt_yes,
4448 ix86_opt_no,
4449 ix86_opt_str,
4450 ix86_opt_enum,
4451 ix86_opt_isa
4452 };
4454 static const struct
4455 {
4462 /* isa options */
4503 /* enum options */
4506 /* string options */
4510 /* flag options */
4512 OPT_mcld,
4513 MASK_CLD),
4516 OPT_mfancy_math_387,
4517 MASK_NO_FANCY_MATH_387),
4520 OPT_mieee_fp,
4521 MASK_IEEE_FP),
4524 OPT_minline_all_stringops,
4525 MASK_INLINE_ALL_STRINGOPS),
4528 OPT_minline_stringops_dynamically,
4529 MASK_INLINE_STRINGOPS_DYNAMICALLY),
4532 OPT_mno_align_stringops,
4533 MASK_NO_ALIGN_STRINGOPS),
4536 OPT_mrecip,
4537 MASK_RECIP),
4539 };
4541 /* If this is a list, recurse to get the options. */
4543 {
4550 enum_opts_set))
4554 }
4557 {
4560 }
4562 /* Handle multiple arguments separated by commas. */
4566 {
4580 {
4584 }
4585 else
4586 {
4589 }
4591 /* Recognize no-xxx. */
4593 {
4597 }
4598 else
4601 /* Find the option. */
4605 {
4613 {
4618 }
4619 }
4621 /* Process the option. */
4623 {
4626 }
4629 {
4635 }
4638 {
4644 else
4646 }
4649 {
4651 {
4654 }
4655 else
4657 }
4660 {
4668 global_dc);
4669 else
4670 {
4673 }
4674 }
4676 else
4678 }
4681 }
4683 /* Return a TARGET_OPTION_NODE tree of the target options listed or NULL. */
4685 tree
4689 {
4704 /* Process each of the options on the chain. */
4709 /* If the changed options are different from the default, rerun
4710 ix86_option_override_internal, and then save the options away.
4711 The string options are are attribute options, and will be undone
4712 when we copy the save structure. */
4718 {
4719 /* If we are using the default tune= or arch=, undo the string assigned,
4720 and use the default. */
4731 /* If fpmath= is not set, and we now have sse2 on 32-bit, use it. */
4736 {
4739 }
4741 /* Do any overrides, such as arch=xxx, or tune=xxx support. */
4744 /* Add any builtin functions with the new isa if any. */
4747 /* Save the current options unless we are validating options for
4748 #pragma. */
4755 /* Free up memory allocated to hold the strings */
4758 }
4761 }
4763 /* Hook to validate attribute((target("string"))). */
4765 static bool
4768 tree args,
4770 {
4775 /* attribute((target("default"))) does nothing, beyond
4776 affecting multi-versioning. */
4785 /* Get the optimization options of the current function. */
4791 /* Init func_options. */
4799 /* Initialize func_options to the default before its target options can
4800 be set. */
4813 {
4818 }
4821 }
4823 \f
4824 /* Hook to determine if one function can safely inline another. */
4826 static bool
4828 {
4833 /* If callee has no option attributes, then it is ok to inline. */
4837 /* If caller has no option attributes, but callee does then it is not ok to
4838 inline. */
4842 else
4843 {
4847 /* Callee's isa options should a subset of the caller's, i.e. a SSE4 function
4848 can inline a SSE2 function but a SSE2 function can't inline a SSE4
4849 function. */
4854 /* See if we have the same non-isa options. */
4858 /* See if arch, tune, etc. are the same. */
4871 else
4873 }
4876 }
4878 \f
4879 /* Remember the last target of ix86_set_current_function. */
4882 /* Invalidate ix86_previous_fndecl cache. */
4883 void
4885 {
4887 }
4889 /* Establish appropriate back-end context for processing the function
4890 FNDECL. The argument might be NULL to indicate processing at top
4891 level, outside of any function scope. */
4892 static void
4894 {
4895 /* Only change the context if the function changes. This hook is called
4896 several times in the course of compiling a function, and we don't want to
4897 slow things down too much or call target_reinit when it isn't safe. */
4899 {
4900 tree old_tree = (ix86_previous_fndecl
4904 tree new_tree = (fndecl
4910 ;
4913 {
4917 }
4920 {
4926 }
4927 }
4928 }
4930 \f
4931 /* Return true if this goes in large data/bss. */
4933 static bool
4935 {
4939 /* Functions are never large data. */
4944 {
4950 }
4951 else
4952 {
4955 /* If this is an incomplete type with size 0, then we can't put it
4956 in data because it might be too big when completed. */
4959 }
4962 }
4964 /* Switch to the appropriate section for output of DECL.
4965 DECL is either a `VAR_DECL' node or a constant of some sort.
4966 RELOC indicates whether forming the initial value of DECL requires
4967 link-time relocations. */
4972 {
4975 {
4979 {
5013 /* We don't split these for medium model. Place them into
5014 default sections and hope for best. */
5016 }
5018 {
5019 /* We might get called with string constants, but get_named_section
5020 doesn't like them as they are not DECLs. Also, we need to set
5021 flags in that case. */
5025 }
5026 }
5028 }
5030 /* Select a set of attributes for section NAME based on the properties
5031 of DECL and whether or not RELOC indicates that DECL's initializer
5032 might contain runtime relocations. */
5034 static unsigned int ATTRIBUTE_UNUSED
5036 {
5050 }
5052 /* Build up a unique section name, expressed as a
5053 STRING_CST node, and assign it to DECL_SECTION_NAME (decl).
5054 RELOC indicates whether the initial value of EXP requires
5055 link-time relocations. */
5057 static void ATTRIBUTE_UNUSED
5059 {
5062 {
5064 /* We only need to use .gnu.linkonce if we don't have COMDAT groups. */
5068 {
5092 /* We don't split these for medium model. Place them into
5093 default sections and hope for best. */
5095 }
5097 {
5104 /* If we're using one_only, then there needs to be a .gnu.linkonce
5105 prefix to the section name. */
5112 }
5113 }
5115 }
5117 #ifdef COMMON_ASM_OP
5118 /* This says how to output assembler code to declare an
5119 uninitialized external linkage data object.
5121 For medium model x86-64 we need to use .largecomm opcode for
5122 large objects. */
5123 void
5127 {
5131 else
5136 }
5137 #endif
5139 /* Utility function for targets to use in implementing
5140 ASM_OUTPUT_ALIGNED_BSS. */
5142 void
5146 {
5150 else
5153 #ifdef ASM_DECLARE_OBJECT_NAME
5156 #else
5157 /* Standard thing is just output label for the object. */
5161 }
5162 \f
5163 /* Decide whether we must probe the stack before any space allocation
5164 on this target. It's essentially TARGET_STACK_PROBE except when
5165 -fstack-check causes the stack to be already probed differently. */
5167 bool
5169 {
5170 /* Do not probe the stack twice if static stack checking is enabled. */
5175 }
5176 \f
5177 /* Decide whether we can make a sibling call to a function. DECL is the
5178 declaration of the function being targeted by the call and EXP is the
5179 CALL_EXPR representing the call. */
5181 static bool
5183 {
5187 /* If we are generating position-independent code, we cannot sibcall
5188 optimize any indirect call, or a direct call to a global function,
5189 as the PLT requires %ebx be live. (Darwin does not have a PLT.) */
5191 && !TARGET_64BIT
5192 && flag_pic
5196 /* If we need to align the outgoing stack, then sibcalling would
5197 unalign the stack, which may break the called function. */
5203 {
5206 }
5207 else
5208 {
5209 /* We're looking at the CALL_EXPR, we need the type of the function. */
5214 }
5216 /* Check that the return value locations are the same. Like
5217 if we are returning floats on the 80387 register stack, we cannot
5218 make a sibcall from a function that doesn't return a float to a
5219 function that does or, conversely, from a function that does return
5220 a float to a function that doesn't; the necessary stack adjustment
5221 would not be executed. This is also the place we notice
5222 differences in the return value ABI. Note that it is ok for one
5223 of the functions to have void return type as long as the return
5224 value of the other is passed in a register. */
5229 {
5232 }
5234 ;
5239 {
5240 /* The SYSV ABI has more call-clobbered registers;
5241 disallow sibcalls from MS to SYSV. */
5245 }
5246 else
5247 {
5248 /* If this call is indirect, we'll need to be able to use a
5249 call-clobbered register for the address of the target function.
5250 Make sure that all such registers are not used for passing
5251 parameters. Note that DLLIMPORT functions are indirect. */
5254 {
5256 {
5257 /* ??? Need to count the actual number of registers to be used,
5258 not the possible number of registers. Fix later. */
5260 }
5261 }
5262 }
5264 /* Otherwise okay. That also includes certain types of indirect calls. */
5266 }
5268 /* Handle "cdecl", "stdcall", "fastcall", "regparm", "thiscall",
5269 and "sseregparm" calling convention attributes;
5270 arguments as in struct attribute_spec.handler. */
5272 static tree
5274 tree args,
5277 {
5282 {
5284 name);
5287 }
5289 /* Can combine regparm with all attributes but fastcall, and thiscall. */
5291 {
5292 tree cst;
5295 {
5297 }
5300 {
5302 }
5306 {
5309 name);
5311 }
5313 {
5317 }
5320 }
5323 {
5324 /* Do not warn when emulating the MS ABI. */
5329 name);
5332 }
5334 /* Can combine fastcall with stdcall (redundant) and sseregparm. */
5336 {
5338 {
5340 }
5342 {
5344 }
5346 {
5348 }
5350 {
5352 }
5353 }
5355 /* Can combine stdcall with fastcall (redundant), regparm and
5356 sseregparm. */
5358 {
5360 {
5362 }
5364 {
5366 }
5368 {
5370 }
5371 }
5373 /* Can combine cdecl with regparm and sseregparm. */
5375 {
5377 {
5379 }
5381 {
5383 }
5385 {
5387 }
5388 }
5390 {
5393 name);
5395 {
5397 }
5399 {
5401 }
5403 {
5405 }
5406 }
5408 /* Can combine sseregparm with all attributes. */
5411 }
5413 /* The transactional memory builtins are implicitly regparm or fastcall
5414 depending on the ABI. Override the generic do-nothing attribute that
5415 these builtins were declared with, and replace it with one of the two
5416 attributes that we expect elsewhere. */
5418 static tree
5420 tree args ATTRIBUTE_UNUSED,
5422 {
5423 tree alt;
5425 /* In no case do we want to add the placeholder attribute. */
5428 /* The 64-bit ABI is unchanged for transactional memory. */
5432 /* ??? Is there a better way to validate 32-bit windows? We have
5433 cfun->machine->call_abi, but that seems to be set only for 64-bit. */
5436 else
5437 {
5440 }
5444 }
5446 /* This function determines from TYPE the calling-convention. */
5448 unsigned int
5450 {
5453 tree attrs;
5460 {
5470 /* Regparam isn't allowed for thiscall and fastcall. */
5472 {
5477 }
5481 }
5488 || is_stdarg
5494 }
5496 /* Return 0 if the attributes for two types are incompatible, 1 if they
5497 are compatible, and 2 if they are nearly compatible (which causes a
5498 warning to be generated). */
5500 static int
5502 {
5518 }
5519 \f
5520 /* Return the regparm value for a function with the indicated TYPE and DECL.
5521 DECL may be NULL when calling function indirectly
5522 or considering a libcall. */
5524 static int
5526 {
5527 tree attr;
5538 {
5541 {
5544 }
5545 }
5551 /* Use register calling convention for local functions when possible. */
5554 && optimize
5556 {
5557 /* FIXME: remove this CONST_CAST when cgraph.[ch] is constified. */
5560 {
5563 /* Make sure no regparm register is taken by a
5564 fixed register variable. */
5569 /* We don't want to use regparm(3) for nested functions as
5570 these use a static chain pointer in the third argument. */
5574 /* In 32-bit mode save a register for the split stack. */
5578 /* Each fixed register usage increases register pressure,
5579 so less registers should be used for argument passing.
5580 This functionality can be overriden by an explicit
5581 regparm value. */
5584 globals++;
5586 local_regparm
5591 }
5592 }
5595 }
5597 /* Return 1 or 2, if we can pass up to SSE_REGPARM_MAX SFmode (1) and
5598 DFmode (2) arguments in SSE registers for a function with the
5599 indicated TYPE and DECL. DECL may be NULL when calling function
5600 indirectly or considering a libcall. Otherwise return 0. */
5602 static int
5604 {
5607 /* Use SSE registers to pass SFmode and DFmode arguments if requested
5608 by the sseregparm attribute. */
5611 {
5613 {
5615 {
5619 else
5622 }
5624 }
5627 }
5629 /* For local functions, pass up to SSE_REGPARM_MAX SFmode
5630 (and DFmode for SSE2) arguments in SSE registers. */
5633 {
5634 /* FIXME: remove this CONST_CAST when cgraph.[ch] is constified. */
5638 }
5641 }
5643 /* Return true if EAX is live at the start of the function. Used by
5644 ix86_expand_prologue to determine if we need special help before
5645 calling allocate_stack_worker. */
5647 static bool
5649 {
5650 /* Cheat. Don't bother working forward from ix86_function_regparm
5651 to the function type to whether an actual argument is located in
5652 eax. Instead just look at cfg info, which is still close enough
5653 to correct at this point. This gives false positives for broken
5654 functions that might use uninitialized data that happens to be
5655 allocated in eax, but who cares? */
5657 }
5659 static bool
5661 {
5662 tree attr;
5665 {
5671 /* For 32-bit MS-ABI the default is to keep aggregate
5672 return pointer. */
5675 }
5677 }
5679 /* Value is the number of bytes of arguments automatically
5680 popped when returning from a subroutine call.
5681 FUNDECL is the declaration node of the function (as a tree),
5682 FUNTYPE is the data type of the function (as a tree),
5683 or for a library call it is an identifier node for the subroutine name.
5684 SIZE is the number of bytes of arguments passed on the stack.
5686 On the 80386, the RTD insn may be used to pop them if the number
5687 of args is fixed, but if the number is variable then the caller
5688 must pop them all. RTD can't be used for library calls now
5689 because the library is compiled with the Unix compiler.
5690 Use of RTD is a selectable option, since it is incompatible with
5691 standard Unix calling sequences. If the option is not selected,
5692 the caller must always pop the args.
5694 The attribute stdcall is equivalent to RTD on a per module basis. */
5696 static int
5698 {
5701 /* None of the 64-bit ABIs pop arguments. */
5712 /* Lose any fake structure return argument if it is passed on the stack. */
5715 {
5719 }
5722 }
5724 /* Implement the TARGET_LEGITIMATE_COMBINED_INSN hook. */
5726 static bool
5728 {
5729 /* Check operand constraints in case hard registers were propagated
5730 into insn pattern. This check prevents combine pass from
5731 generating insn patterns with invalid hard register operands.
5732 These invalid insns can eventually confuse reload to error out
5733 with a spill failure. See also PRs 46829 and 46843. */
5735 {
5742 {
5750 /* For pre-AVX disallow unaligned loads/stores where the
5751 instructions don't support it. */
5755 {
5759 }
5761 /* A unary operator may be accepted by the predicate, but it
5762 is irrelevant for matching constraints. */
5767 {
5775 }
5782 /* Operand has no constraints, anything is OK. */
5786 {
5789 && operands_match_p
5793 {
5796 }
5797 }
5801 }
5802 }
5805 }
5806 \f
5807 /* Implement the TARGET_ASAN_SHADOW_OFFSET hook. */
5809 static unsigned HOST_WIDE_INT
5811 {
5815 }
5816 \f
5817 /* Argument support functions. */
5819 /* Return true when register may be used to pass function parameters. */
5820 bool
5822 {
5827 {
5831 else
5837 }
5843 /* TODO: The function should depend on current function ABI but
5844 builtins.c would need updating then. Therefore we use the
5845 default ABI. */
5847 /* RAX is used as hidden argument to va_arg functions. */
5853 else
5860 }
5862 /* Return if we do not know how to pass TYPE solely in registers. */
5864 static bool
5866 {
5870 /* For 32-bit, we want TImode aggregates to go on the stack. But watch out!
5871 The layout_type routine is crafty and tries to trick us into passing
5872 currently unsupported vector types on the stack by using TImode. */
5875 }
5877 /* It returns the size, in bytes, of the area reserved for arguments passed
5878 in registers for the function represented by fndecl dependent to the used
5879 abi format. */
5880 int
5882 {
5886 else
5891 }
5893 /* Returns value SYSV_ABI, MS_ABI dependent on fntype, specifying the
5894 call abi used. */
5895 enum calling_abi
5897 {
5899 {
5902 {
5905 }
5909 }
5911 }
5913 /* We add this as a workaround in order to use libc_has_function
5914 hook in i386.md. */
5915 bool
5917 {
5919 }
5921 static bool
5923 {
5925 {
5929 else
5931 }
5933 }
5935 static enum calling_abi
5937 {
5941 }
5943 /* Returns value SYSV_ABI, MS_ABI dependent on cfun, specifying the
5944 call abi used. */
5945 enum calling_abi
5947 {
5951 }
5953 /* Write the extra assembler code needed to declare a function properly. */
5955 void
5957 tree decl)
5958 {
5962 {
5968 }
5970 #ifdef SUBTARGET_ASM_UNWIND_INIT
5972 #endif
5976 /* Output magic byte marker, if hot-patch attribute is set. */
5978 {
5980 {
5981 /* leaq [%rsp + 0], %rsp */
5984 }
5985 else
5986 {
5987 /* movl.s %edi, %edi
5988 push %ebp
5989 movl.s %esp, %ebp */
5992 }
5993 }
5994 }
5996 /* regclass.c */
5999 /* Implementation of call abi switching target hook. Specific to FNDECL
6000 the specific call register sets are set. See also
6001 ix86_conditional_register_usage for more details. */
6002 void
6004 {
6007 else
6009 }
6011 /* 64-bit MS and SYSV ABI have different set of call used registers. Avoid
6012 expensive re-initialization of init_regs each time we switch function context
6013 since this is needed only during RTL expansion. */
6014 static void
6016 {
6020 }
6022 /* Initialize a variable CUM of type CUMULATIVE_ARGS
6023 for a call to a function whose data type is FNTYPE.
6024 For a library call, FNTYPE is 0. */
6026 void
6030 tree fndecl,
6032 {
6038 {
6041 }
6042 else
6043 {
6046 }
6050 /* Set up the number of registers to use for passing arguments. */
6057 {
6059 ? X86_64_REGPARM_MAX
6061 }
6063 {
6066 {
6068 ? X86_64_SSE_REGPARM_MAX
6070 }
6071 }
6078 /* Because type might mismatch in between caller and callee, we need to
6079 use actual type of function for local calls.
6080 FIXME: cgraph_analyze can be told to actually record if function uses
6081 va_start so for local functions maybe_vaarg can be made aggressive
6082 helping K&R code.
6083 FIXME: once typesytem is fixed, we won't need this code anymore. */
6091 {
6092 /* If there are variable arguments, then we won't pass anything
6093 in registers in 32-bit mode. */
6095 {
6103 }
6105 /* Use ecx and edx registers if function has fastcall attribute,
6106 else look for regparm information. */
6108 {
6111 {
6114 }
6116 {
6119 }
6120 else
6122 }
6124 /* Set up the number of SSE registers used for passing SFmode
6125 and DFmode arguments. Warn for mismatching ABI. */
6127 }
6128 }
6130 /* Return the "natural" mode for TYPE. In most cases, this is just TYPE_MODE.
6131 But in the case of vector types, it is some vector mode.
6133 When we have only some of our vector isa extensions enabled, then there
6134 are some modes for which vector_mode_supported_p is false. For these
6135 modes, the generic vector support in gcc will choose some non-vector mode
6136 in order to implement the type. By computing the natural mode, we'll
6137 select the proper ABI location for the operand and not depend on whatever
6138 the middle-end decides to do with these vector types.
6140 The midde-end can't deal with the vector types > 16 bytes. In this
6141 case, we return the original mode and warn ABI change if CUM isn't
6142 NULL. */
6144 static enum machine_mode
6146 {
6150 {
6153 /* ??? Generic code allows us to create width 1 vectors. Ignore. */
6155 {
6160 else
6163 /* Get the mode which has this inner mode and number of units. */
6167 {
6169 {
6173 && !warnedavx
6175 {
6179 }
6181 }
6184 {
6188 && !warnedsse
6190 {
6194 }
6195 }
6197 {
6201 && !warnedmmx
6203 {
6207 }
6208 }
6210 }
6213 }
6214 }
6217 }
6219 /* We want to pass a value in REGNO whose "natural" mode is MODE. However,
6220 this may not agree with the mode that the type system has chosen for the
6221 register, which is ORIG_MODE. If ORIG_MODE is not BLKmode, then we can
6222 go ahead and use it. Otherwise we have to build a PARALLEL instead. */
6224 static rtx
6227 {
6228 rtx tmp;
6232 else
6233 {
6237 }
6240 }
6242 /* x86-64 register passing implementation. See x86-64 ABI for details. Goal
6243 of this code is to classify each 8bytes of incoming argument by the register
6244 class and assign registers accordingly. */
6246 /* Return the union class of CLASS1 and CLASS2.
6247 See the x86-64 PS ABI for details. */
6249 static enum x86_64_reg_class
6251 {
6252 /* Rule #1: If both classes are equal, this is the resulting class. */
6256 /* Rule #2: If one of the classes is NO_CLASS, the resulting class is
6257 the other class. */
6263 /* Rule #3: If one of the classes is MEMORY, the result is MEMORY. */
6267 /* Rule #4: If one of the classes is INTEGER, the result is INTEGER. */
6275 /* Rule #5: If one of the classes is X87, X87UP, or COMPLEX_X87 class,
6276 MEMORY is used. */
6285 /* Rule #6: Otherwise class SSE is used. */
6287 }
6289 /* Classify the argument of type TYPE and mode MODE.
6290 CLASSES will be filled by the register class used to pass each word
6291 of the operand. The number of words is returned. In case the parameter
6292 should be passed in memory, 0 is returned. As a special case for zero
6293 sized containers, classes[0] will be NO_CLASS and 1 is returned.
6295 BIT_OFFSET is used internally for handling records and specifies offset
6296 of the offset in bits modulo 256 to avoid overflow cases.
6298 See the x86-64 PS ABI for details.
6299 */
6301 static int
6304 {
6305 HOST_WIDE_INT bytes =
6307 int words
6310 /* Variable sized entities are always passed/returned in memory. */
6319 {
6321 tree field;
6324 /* On x86-64 we pass structures larger than 32 bytes on the stack. */
6331 /* Zero sized arrays or structures are NO_CLASS. We return 0 to
6332 signalize memory class, so handle it as special case. */
6334 {
6337 }
6339 /* Classify each field of record and merge classes. */
6341 {
6343 /* And now merge the fields of structure. */
6345 {
6347 {
6353 /* Bitfields are always classified as integer. Handle them
6354 early, since later code would consider them to be
6355 misaligned integers. */
6357 {
6366 }
6367 else
6368 {
6373 /* Flexible array member is ignored. */
6380 {
6384 {
6387 "the ABI of passing struct with"
6388 " a flexible array member has"
6390 }
6392 }
6394 subclasses,
6404 }
6405 }
6406 }
6410 /* Arrays are handled as small records. */
6411 {
6418 /* The partial classes are now full classes. */
6429 }
6432 /* Unions are similar to RECORD_TYPE but offset is always 0.
6433 */
6435 {
6437 {
6445 bit_offset);
6450 }
6451 }
6456 }
6459 {
6460 /* When size > 16 bytes, if the first one isn't
6461 X86_64_SSE_CLASS or any other ones aren't
6462 X86_64_SSEUP_CLASS, everything should be passed in
6463 memory. */
6470 }
6472 /* Final merger cleanup. */
6474 {
6475 /* If one class is MEMORY, everything should be passed in
6476 memory. */
6480 /* The X86_64_SSEUP_CLASS should be always preceded by
6481 X86_64_SSE_CLASS or X86_64_SSEUP_CLASS. */
6485 {
6486 /* The first one should never be X86_64_SSEUP_CLASS. */
6489 }
6491 /* If X86_64_X87UP_CLASS isn't preceded by X86_64_X87_CLASS,
6492 everything should be passed in memory. */
6495 {
6498 /* The first one should never be X86_64_X87UP_CLASS. */
6501 {
6504 "the ABI of passing union with long double"
6506 }
6508 }
6509 }
6511 }
6513 /* Compute alignment needed. We align all types to natural boundaries with
6514 exception of XFmode that is aligned to 64bits. */
6516 {
6525 /* Misaligned fields are always returned in memory. */
6528 }
6530 /* for V1xx modes, just use the base mode */
6535 /* Classification of atomic types. */
6537 {
6553 {
6557 {
6560 }
6562 {
6565 }
6567 {
6571 }
6573 {
6576 }
6577 else
6579 }
6586 /* OImode shouldn't be used directly. */
6593 else
6611 else
6612 {
6616 {
6619 "the ABI of passing structure with complex float"
6621 }
6624 }
6633 /* This modes is larger than 16 bytes. */
6676 else
6680 }
6681 }
6683 /* Examine the argument and return set number of register required in each
6684 class. Return 0 iff parameter should be passed in memory. */
6685 static int
6688 {
6698 {
6720 }
6722 }
6724 /* Construct container for the argument used by GCC interface. See
6725 FUNCTION_ARG for the detailed description. */
6727 static rtx
6731 {
6732 /* The following variables hold the static issued_error state. */
6746 rtx ret;
6757 /* We allowed the user to turn off SSE for kernel mode. Don't crash if
6758 some less clueful developer tries to use floating-point anyway. */
6760 {
6762 {
6764 {
6767 }
6768 }
6770 {
6773 }
6775 }
6777 /* Likewise, error if the ABI requires us to return values in the
6778 x87 registers and the user specified -mno-80387. */
6784 {
6786 {
6789 }
6791 }
6793 /* First construct simple cases. Avoid SCmode, since we want to use
6794 single register to pass this type. */
6797 {
6812 /* Zero sized array, struct or class. */
6816 }
6843 /* Otherwise figure out the entries of the PARALLEL. */
6845 {
6849 {
6854 /* Merge TImodes on aligned occasions here too. */
6856 tmpmode
6860 else
6862 /* We've requested 24 bytes we
6863 don't have mode for. Use DImode. */
6870 intreg++;
6878 sse_regno++;
6886 sse_regno++;
6891 {
6897 {
6899 i++;
6900 }
6901 else
6914 }
6920 sse_regno++;
6924 }
6925 }
6927 /* Empty aligned struct, union or class. */
6935 }
6937 /* Update the data in CUM to advance over an argument of mode MODE
6938 and data type TYPE. (TYPE is null for libcalls where that information
6939 may not be available.) */
6941 static void
6944 HOST_WIDE_INT words)
6945 {
6947 {
6954 /* FALLTHRU */
6965 {
6968 }
6972 /* OImode shouldn't be used directly. */
6981 /* FALLTHRU */
6997 {
7002 {
7005 }
7006 }
7016 {
7021 {
7024 }
7025 }
7027 }
7028 }
7030 static void
7033 {
7036 /* Unnamed 256bit vector mode parameters are passed on stack. */
7042 {
7047 }
7048 else
7049 {
7053 }
7054 }
7056 static void
7058 HOST_WIDE_INT words)
7059 {
7060 /* Otherwise, this should be passed indirect. */
7065 {
7068 }
7069 }
7071 /* Update the data in CUM to advance over an argument of mode MODE and
7072 data type TYPE. (TYPE is null for libcalls where that information
7073 may not be available.) */
7075 static void
7078 {
7084 else
7095 else
7097 }
7099 /* Define where to put the arguments to a function.
7100 Value is zero to push the argument on the stack,
7101 or a hard register in which to store the argument.
7103 MODE is the argument's machine mode.
7104 TYPE is the data type of the argument (as a tree).
7105 This is null for libcalls where that information may
7106 not be available.
7107 CUM is a variable of type CUMULATIVE_ARGS which gives info about
7108 the preceding args and about the function being called.
7109 NAMED is nonzero if this argument is a named parameter
7110 (otherwise it is an extra parameter matching an ellipsis). */
7112 static rtx
7116 {
7119 /* Avoid the AL settings for the Unix64 ABI. */
7124 {
7131 /* FALLTHRU */
7137 {
7140 /* Fastcall allocates the first two DWORD (SImode) or
7141 smaller arguments to ECX and EDX if it isn't an
7142 aggregate type . */
7144 {
7150 /* ECX not EAX is the first allocated register. */
7153 }
7155 }
7164 /* FALLTHRU */
7166 /* In 32bit, we pass TImode in xmm registers. */
7174 {
7176 {
7180 }
7184 }
7188 /* OImode shouldn't be used directly. */
7198 {
7202 }
7212 {
7214 {
7218 }
7222 }
7224 }
7227 }
7229 static rtx
7232 {
7233 /* Handle a hidden AL argument containing number of registers
7234 for varargs x86-64 functions. */
7238 ? X86_64_SSE_REGPARM_MAX
7243 {
7253 /* Unnamed 256bit vector mode parameters are passed on stack. */
7257 }
7263 }
7265 static rtx
7268 HOST_WIDE_INT bytes)
7269 {
7272 /* We need to add clobber for MS_ABI->SYSV ABI calls in expand_call.
7273 We use value of -2 to specify that current function call is MSABI. */
7277 /* If we've run out of registers, it goes on the stack. */
7283 /* Only floating point modes are passed in anything but integer regs. */
7285 {
7288 else
7289 {
7292 /* Unnamed floating parameters are passed in both the
7293 SSE and integer registers. */
7299 }
7300 }
7301 /* Handle aggregated types passed in register. */
7303 {
7308 }
7311 }
7313 /* Return where to put the arguments to a function.
7314 Return zero to push the argument on the stack, or a hard register in which to store the argument.
7316 MODE is the argument's machine mode. TYPE is the data type of the
7317 argument. It is null for libcalls where that information may not be
7318 available. CUM gives information about the preceding args and about
7319 the function being called. NAMED is nonzero if this argument is a
7320 named parameter (otherwise it is an extra parameter matching an
7321 ellipsis). */
7323 static rtx
7326 {
7330 rtx arg;
7334 else
7338 /* To simplify the code below, represent vector types with a vector mode
7339 even if MMX/SSE are not active. */
7347 else
7351 }
7353 /* A C expression that indicates when an argument must be passed by
7354 reference. If nonzero for an argument, a copy of that argument is
7355 made in memory and a pointer to the argument is passed instead of
7356 the argument itself. The pointer is passed in whatever way is
7357 appropriate for passing a pointer to that type. */
7359 static bool
7362 {
7365 /* See Windows x64 Software Convention. */
7367 {
7370 {
7371 /* Arrays are passed by reference. */
7376 {
7377 /* Structs/unions of sizes other than 8, 16, 32, or 64 bits
7378 are passed by reference. */
7380 }
7381 }
7383 /* __m128 is passed by reference. */
7389 }
7390 }
7395 }
7397 /* Return true when TYPE should be 128bit aligned for 32bit argument
7398 passing ABI. XXX: This function is obsolete and is only used for
7399 checking psABI compatibility with previous versions of GCC. */
7401 static bool
7403 {
7415 {
7416 /* Walk the aggregates recursively. */
7418 {
7422 {
7423 tree field;
7425 /* Walk all the structure fields. */
7427 {
7431 }
7433 }
7436 /* Just for use if some languages passes arrays by value. */
7443 }
7444 }
7446 }
7448 /* Return the alignment boundary for MODE and TYPE with alignment ALIGN.
7449 XXX: This function is obsolete and is only used for checking psABI
7450 compatibility with previous versions of GCC. */
7452 static unsigned int
7455 {
7456 /* In 32bit, only _Decimal128 and __float128 are aligned to their
7457 natural boundaries. */
7459 {
7460 /* i386 ABI defines all arguments to be 4 byte aligned. We have to
7461 make an exception for SSE modes since these require 128bit
7462 alignment.
7464 The handling here differs from field_alignment. ICC aligns MMX
7465 arguments to 4 byte boundaries, while structure fields are aligned
7466 to 8 byte boundaries. */
7468 {
7471 }
7472 else
7473 {
7476 }
7477 }
7481 }
7483 /* Return true when TYPE should be 128bit aligned for 32bit argument
7484 passing ABI. */
7486 static bool
7488 {
7498 {
7499 /* Walk the aggregates recursively. */
7501 {
7505 {
7506 tree field;
7508 /* Walk all the structure fields. */
7510 field;
7512 {
7516 }
7518 }
7521 /* Just for use if some languages passes arrays by value. */
7528 }
7529 }
7530 else
7534 }
7536 /* Gives the alignment boundary, in bits, of an argument with the
7537 specified mode and type. */
7539 static unsigned int
7541 {
7544 {
7545 /* Since the main variant type is used for call, we convert it to
7546 the main variant type. */
7549 }
7550 else
7554 else
7555 {
7560 {
7561 /* i386 ABI defines XFmode arguments to be 4 byte aligned. */
7563 {
7566 }
7572 }
7575 && !warned
7577 saved_align))
7578 {
7581 "The ABI for passing parameters with %d-byte"
7584 }
7585 }
7588 }
7590 /* Return true if N is a possible register number of function value. */
7592 static bool
7594 {
7596 {
7604 /* Complex values are returned in %st(0)/%st(1) pair. */
7607 /* TODO: The function should depend on current function ABI but
7608 builtins.c would need updating then. Therefore we use the
7609 default ABI. */
7614 /* Complex values are returned in %xmm0/%xmm1 pair. */
7623 }
7626 }
7628 /* Define how to find the value returned by a function.
7629 VALTYPE is the data type of the value (as a tree).
7630 If the precise function being called is known, FUNC is its FUNCTION_DECL;
7631 otherwise, FUNC is 0. */
7633 static rtx
7636 {
7639 /* 8-byte vector modes in %mm0. See ix86_return_in_memory for where
7640 we normally prevent this case when mmx is not available. However
7641 some ABIs may require the result to be returned like DImode. */
7645 /* 16-byte vector modes in %xmm0. See ix86_return_in_memory for where
7646 we prevent this case when sse is not available. However some ABIs
7647 may require the result to be returned like integer TImode. */
7652 /* 32-byte vector modes in %ymm0. */
7656 /* Floating point return values in %st(0) (unless -mno-fp-ret-in-387). */
7659 else
7660 /* Most things go in %eax. */
7663 /* Override FP return register with %xmm0 for local functions when
7664 SSE math is enabled or for functions with sseregparm attribute. */
7666 {
7671 }
7673 /* OImode shouldn't be used directly. */
7677 }
7679 static rtx
7681 const_tree valtype)
7682 {
7683 rtx ret;
7685 /* Handle libcalls, which don't provide a type node. */
7687 {
7691 {
7710 }
7713 }
7715 {
7716 /* Pointers are always returned in word_mode. */
7718 }
7724 /* For zero sized structures, construct_container returns NULL, but we
7725 need to keep rest of compiler happy by returning meaningful value. */
7730 }
7732 static rtx
7734 const_tree valtype)
7735 {
7739 {
7741 {
7760 }
7761 }
7763 }
7765 static rtx
7768 {
7780 else
7782 }
7784 static rtx
7787 {
7793 }
7795 /* Pointer function arguments and return values are promoted to
7796 word_mode. */
7798 static enum machine_mode
7802 {
7804 {
7807 }
7809 for_return);
7810 }
7812 /* Return true if a structure, union or array with MODE containing FIELD
7813 should be accessed using BLKmode. */
7815 static bool
7817 {
7818 /* Union with XFmode must be in BLKmode. */
7822 }
7824 rtx
7826 {
7828 }
7830 /* Return true iff type is returned in memory. */
7832 static bool ATTRIBUTE_UNUSED
7834 {
7835 HOST_WIDE_INT size;
7846 {
7847 /* User-created vectors small enough to fit in EAX. */
7851 /* MMX/3dNow values are returned in MM0,
7852 except when it doesn't exits or the ABI prescribes otherwise. */
7856 /* SSE values are returned in XMM0, except when it doesn't exist. */
7860 /* AVX values are returned in YMM0, except when it doesn't exist. */
7863 }
7871 /* OImode shouldn't be used directly. */
7875 }
7877 static bool ATTRIBUTE_UNUSED
7879 {
7882 }
7884 static bool ATTRIBUTE_UNUSED
7886 {
7889 /* __m128 is returned in xmm0. */
7896 /* Otherwise, the size must be exactly in [1248]. */
7898 }
7900 static bool
7902 {
7903 #ifdef SUBTARGET_RETURN_IN_MEMORY
7905 #else
7909 {
7912 else
7914 }
7915 else
7917 #endif
7918 }
7920 /* When returning SSE vector types, we have a choice of either
7921 (1) being abi incompatible with a -march switch, or
7922 (2) generating an error.
7923 Given no good solution, I think the safest thing is one warning.
7924 The user won't be able to use -Werror, but....
7926 Choose the STRUCT_VALUE_RTX hook because that's (at present) only
7927 called in response to actually generating a caller or callee that
7928 uses such a type. As opposed to TARGET_RETURN_IN_MEMORY, which is called
7929 via aggregate_value_p for general type probing from tree-ssa. */
7931 static rtx
7933 {
7937 {
7938 /* Look at the return type of the function, not the function type. */
7942 {
7945 {
7949 }
7950 }
7953 {
7955 {
7959 }
7960 }
7961 }
7964 }
7966 \f
7967 /* Create the va_list data type. */
7969 /* Returns the calling convention specific va_list date type.
7970 The argument ABI can be DEFAULT_ABI, MS_ABI, or SYSV_ABI. */
7972 static tree
7974 {
7977 /* For i386 we use plain pointer to argument area. */
7987 unsigned_type_node);
7990 unsigned_type_node);
7993 ptr_type_node);
7996 ptr_type_node);
8015 /* The correct type is an array type of one element. */
8017 }
8019 /* Setup the builtin va_list data type and for 64-bit the additional
8020 calling convention specific va_list data types. */
8022 static tree
8024 {
8027 /* Initialize abi specific va_list builtin types. */
8029 {
8030 tree t;
8032 {
8037 }
8038 else
8039 {
8044 }
8046 {
8051 }
8052 else
8053 {
8058 }
8059 }
8062 }
8064 /* Worker function for TARGET_SETUP_INCOMING_VARARGS. */
8066 static void
8068 {
8070 alias_set_type set;
8073 /* GPR size of varargs save area. */
8076 else
8079 /* FPR size of varargs save area. We don't need it if we don't pass
8080 anything in SSE registers. */
8083 else
8097 {
8105 }
8108 {
8112 /* Now emit code to save SSE registers. The AX parameter contains number
8113 of SSE parameter registers used to call this function, though all we
8114 actually check here is the zero/non-zero status. */
8119 label));
8121 /* ??? If !TARGET_SSE_TYPELESS_STORES, would we perform better if
8122 we used movdqa (i.e. TImode) instead? Perhaps even better would
8123 be if we could determine the real mode of the data, via a hook
8124 into pass_stdarg. Ignore all that for now. */
8134 {
8143 }
8146 }
8147 }
8149 static void
8151 {
8155 /* Reset to zero, as there might be a sysv vaarg used
8156 before. */
8161 {
8172 }
8173 }
8175 static void
8179 {
8181 CUMULATIVE_ARGS next_cum;
8182 tree fntype;
8184 /* This argument doesn't appear to be used anymore. Which is good,
8185 because the old code here didn't suppress rtl generation. */
8193 /* For varargs, we do not want to skip the dummy va_dcl argument.
8194 For stdargs, we do want to skip the last named argument. */
8202 else
8204 }
8206 /* Checks if TYPE is of kind va_list char *. */
8208 static bool
8210 {
8211 tree canonic;
8213 /* For 32-bit it is always true. */
8219 }
8221 /* Implement va_start. */
8223 static void
8225 {
8229 tree type;
8230 rtx ovf_rtx;
8234 {
8237 /* When we are splitting the stack, we can't refer to the stack
8238 arguments using internal_arg_pointer, because they may be on
8239 the old stack. The split stack prologue will arrange to
8240 leave a pointer to the old stack arguments in a scratch
8241 register, which we here copy to a pseudo-register. The split
8242 stack prologue can't set the pseudo-register directly because
8243 it (the prologue) runs before any registers have been saved. */
8247 {
8261 }
8262 }
8264 /* Only 64bit target needs something special. */
8266 {
8269 else
8270 {
8279 }
8281 }
8290 /* The following should be folded into the MEM_REF offset. */
8300 /* Count number of gp and fp argument registers used. */
8306 {
8312 }
8315 {
8321 }
8323 /* Find the overflow area. */
8327 else
8337 {
8338 /* Find the register save area.
8339 Prologue of the function save it right above stack frame. */
8347 }
8348 }
8350 /* Implement va_arg. */
8352 static tree
8355 {
8362 rtx container;
8364 tree ptrtype;
8368 /* Only 64bit target needs something special. */
8392 {
8399 /* Unnamed 256bit vector mode parameters are passed on stack. */
8401 {
8404 }
8412 }
8414 /* Pull the value out of the saved registers. */
8419 {
8433 /* In case we are passing structure, verify that it is consecutive block
8434 on the register save area. If not we need to do moves. */
8436 {
8437 /* Verify that all registers are strictly consecutive */
8439 {
8443 {
8448 }
8449 }
8450 else
8451 {
8455 {
8460 }
8461 }
8462 }
8464 {
8467 }
8468 else
8469 {
8472 }
8474 /* First ensure that we fit completely in registers. */
8476 {
8483 }
8485 {
8493 }
8495 /* Compute index to start of area used for integer regs. */
8497 {
8498 /* int_addr = gpr + sav; */
8501 }
8503 {
8504 /* sse_addr = fpr + sav; */
8507 }
8509 {
8513 /* addr = &temp; */
8518 {
8522 tree piece_type;
8523 tree addr_type;
8524 tree daddr_type;
8533 {
8538 }
8542 else
8549 {
8552 }
8553 else
8554 {
8557 }
8564 {
8569 }
8570 else
8571 {
8572 tree copy
8577 }
8579 }
8580 }
8583 {
8587 }
8590 {
8594 }
8599 }
8601 /* ... otherwise out of the overflow area. */
8603 /* When we align parameter on stack for caller, if the parameter
8604 alignment is beyond MAX_SUPPORTED_STACK_ALIGNMENT, it will be
8605 aligned at MAX_SUPPORTED_STACK_ALIGNMENT. We will match callee
8606 here with caller. */
8611 /* Care for on-stack alignment if needed. */
8614 else
8615 {
8620 }
8637 }
8638 \f
8639 /* Return true if OPNUM's MEM should be matched
8640 in movabs* patterns. */
8642 bool
8644 {
8656 }
8657 \f
8658 /* Initialize the table of extra 80387 mathematical constants. */
8660 static void
8662 {
8664 {
8670 };
8674 {
8676 /* Ensure each constant is rounded to XFmode precision. */
8679 }
8682 }
8684 /* Return non-zero if the constant is something that
8685 can be loaded with a special instruction. */
8687 int
8689 {
8692 REAL_VALUE_TYPE r;
8704 /* For XFmode constants, try to find a special 80387 instruction when
8705 optimizing for size or on those CPUs that benefit from them. */
8708 {
8717 }
8719 /* Load of the constant -0.0 or -1.0 will be split as
8720 fldz;fchs or fld1;fchs sequence. */
8727 }
8729 /* Return the opcode of the special instruction to be used to load
8730 the constant X. */
8734 {
8736 {
8756 }
8757 }
8759 /* Return the CONST_DOUBLE representing the 80387 constant that is
8760 loaded by the specified special instruction. The argument IDX
8761 matches the return value from standard_80387_constant_p. */
8763 rtx
8765 {
8772 {
8783 }
8786 XFmode);
8787 }
8789 /* Return 1 if X is all 0s and 2 if x is all 1s
8790 in supported SSE/AVX vector mode. */
8792 int
8794 {
8801 {
8816 }
8819 }
8821 /* Return the opcode of the special instruction to be used to load
8822 the constant X. */
8826 {
8828 {
8831 {
8848 }
8857 else
8862 }
8864 }
8866 /* Returns true if OP contains a symbol reference */
8868 bool
8870 {
8879 {
8881 {
8887 }
8891 }
8894 }
8896 /* Return true if it is appropriate to emit `ret' instructions in the
8897 body of a function. Do this only if the epilogue is simple, needing a
8898 couple of insns. Prior to reloading, we can't tell how many registers
8899 must be saved, so return false then. Return false if there is no frame
8900 marker to de-allocate. */
8902 bool
8904 {
8910 /* Don't allow more than 32k pop, since that's all we can do
8911 with one instruction. */
8918 }
8919 \f
8920 /* Value should be nonzero if functions must have frame pointers.
8921 Zero means the frame pointer need not be set up (and parms may
8922 be accessed via the stack pointer) in functions that seem suitable. */
8924 static bool
8926 {
8927 /* If we accessed previous frames, then the generated code expects
8928 to be able to access the saved ebp value in our frame. */
8932 /* Several x86 os'es need a frame pointer for other reasons,
8933 usually pertaining to setjmp. */
8937 /* For older 32-bit runtimes setjmp requires valid frame-pointer. */
8941 /* Win64 SEH, very large frames need a frame-pointer as maximum stack
8942 allocation is 4GB. */
8946 /* In ix86_option_override_internal, TARGET_OMIT_LEAF_FRAME_POINTER
8947 turns off the frame pointer by default. Turn it back on now if
8948 we've not got a leaf function. */
8958 }
8960 /* Record that the current function accesses previous call frames. */
8962 void
8964 {
8966 }
8967 \f
8968 #ifndef USE_HIDDEN_LINKONCE
8969 # if defined(HAVE_GAS_HIDDEN) && (SUPPORTS_ONE_ONLY - 0)
8970 # define USE_HIDDEN_LINKONCE 1
8971 # else
8972 # define USE_HIDDEN_LINKONCE 0
8973 # endif
8974 #endif
8978 /* Fills in the label name that should be used for a pc thunk for
8979 the given register. */
8981 static void
8983 {
8988 else
8990 }
8993 /* This function generates code for -fpic that loads %ebx with
8994 the return address of the caller and then returns. */
8996 static void
8998 {
9003 {
9005 tree decl;
9021 #if TARGET_MACHO
9023 {
9032 }
9033 else
9034 #endif
9036 {
9047 }
9048 else
9049 {
9052 }
9058 /* Make sure unwind info is emitted for the thunk if needed. */
9061 /* Pad stack IP move with 4 instructions (two NOPs count
9062 as one instruction). */
9064 {
9069 }
9080 }
9084 }
9086 /* Emit code for the SET_GOT patterns. */
9090 {
9096 {
9097 /* Load (*VXWORKS_GOTT_BASE) into the PIC register. */
9102 /* Load (*VXWORKS_GOTT_BASE)[VXWORKS_GOTT_INDEX] into the PIC register.
9103 Use %P and a local symbol in order to print VXWORKS_GOTT_INDEX as
9104 an unadorned address. */
9109 }
9114 {
9116 /* We don't need a pic base, we're not producing pic. */
9123 }
9124 else
9125 {
9134 #if TARGET_MACHO
9135 /* Output the Mach-O "canonical" pic base label name ("Lxx$pb") here.
9136 This is what will be referenced by the Mach-O PIC subsystem. */
9140 /* When we are restoring the pic base at the site of a nonlocal label,
9141 and we decided to emit the pic base above, we will still output a
9142 local label used for calculating the correction offset (even though
9143 the offset will be 0 in that case). */
9147 #endif
9148 }
9154 }
9156 /* Generate an "push" pattern for input ARG. */
9158 static rtx
9160 {
9173 stack_pointer_rtx)),
9174 arg);
9175 }
9177 /* Generate an "pop" pattern for input ARG. */
9179 static rtx
9181 {
9186 arg,
9189 stack_pointer_rtx)));
9190 }
9192 /* Return >= 0 if there is an unused call-clobbered register available
9193 for the entire function. */
9195 static unsigned int
9197 {
9201 {
9203 /* Can't use the same register for both PIC and DRAP. */
9206 else
9211 }
9214 }
9216 /* Return TRUE if we need to save REGNO. */
9218 static bool
9220 {
9231 {
9234 {
9240 }
9241 }
9250 }
9252 /* Return number of saved general prupose registers. */
9254 static int
9256 {
9262 nregs ++;
9264 }
9266 /* Return number of saved SSE registrers. */
9268 static int
9270 {
9278 nregs ++;
9280 }
9282 /* Given FROM and TO register numbers, say whether this elimination is
9283 allowed. If stack alignment is needed, we can only replace argument
9284 pointer with hard frame pointer, or replace frame pointer with stack
9285 pointer. Otherwise, frame pointer elimination is automatically
9286 handled and all other eliminations are valid. */
9288 static bool
9290 {
9296 else
9298 }
9300 /* Return the offset between two registers, one to be eliminated, and the other
9301 its replacement, at the start of a routine. */
9303 HOST_WIDE_INT
9305 {
9314 else
9315 {
9323 }
9324 }
9326 /* In a dynamically-aligned function, we can't know the offset from
9327 stack pointer to frame pointer, so we must ensure that setjmp
9328 eliminates fp against the hard fp (%ebp) rather than trying to
9329 index from %esp up to the top of the frame across a gap that is
9330 of unknown (at compile-time) size. */
9331 static rtx
9333 {
9335 }
9337 /* When using -fsplit-stack, the allocation routines set a field in
9338 the TCB to the bottom of the stack plus this much space, measured
9339 in bytes. */
9341 #define SPLIT_STACK_AVAILABLE 256
9343 /* Fill structure ix86_frame about frame of currently computed function. */
9345 static void
9347 {
9349 HOST_WIDE_INT offset;
9352 HOST_WIDE_INT to_allocate;
9360 /* 64-bit MS ABI seem to require stack alignment to be always 16 except for
9361 function prologues and leaf. */
9365 {
9370 }
9376 /* For SEH we have to limit the amount of code movement into the prologue.
9377 At present we do this via a BLOCKAGE, at which point there's very little
9378 scheduling that can be done, which means that there's very little point
9379 in doing anything except PUSHs. */
9383 /* During reload iteration the amount of registers saved can change.
9384 Recompute the value as needed. Do not recompute when amount of registers
9385 didn't change as reload does multiple calls to the function and does not
9386 expect the decision to change within single iteration. */
9389 {
9395 /* The fast prologue uses move instead of push to save registers. This
9396 is significantly longer, but also executes faster as modern hardware
9397 can execute the moves in parallel, but can't do that for push/pop.
9399 Be careful about choosing what prologue to emit: When function takes
9400 many instructions to execute we may use slow version as well as in
9401 case function is known to be outside hot spot (this is known with
9402 feedback only). Weight the size of function by number of registers
9403 to save as it is cheap to use one or two push instructions but very
9404 slow to use many of them. */
9408 || (flag_branch_probabilities
9411 else
9414 }
9416 frame->save_regs_using_mov
9418 /* If static stack checking is enabled and done with probes,
9419 the registers need to be saved before allocating the frame. */
9422 /* Skip return address. */
9425 /* Skip pushed static chain. */
9429 /* Skip saved base pointer. */
9434 /* The traditional frame pointer location is at the top of the frame. */
9437 /* Register save area */
9441 /* On SEH target, registers are pushed just before the frame pointer
9442 location. */
9446 /* Align and set SSE register save area. */
9448 {
9449 /* The only ABI that has saved SSE registers (Win64) also has a
9450 16-byte aligned default stack, and thus we don't need to be
9451 within the re-aligned local stack frame to save them. */
9455 }
9458 /* The re-aligned stack starts here. Values before this point are not
9459 directly comparable with values below this point. In order to make
9460 sure that no value happens to be the same before and after, force
9461 the alignment computation below to add a non-zero value. */
9465 /* Va-arg area */
9469 /* Align start of frame for local function. */
9478 /* Frame pointer points here. */
9483 /* Add outgoing arguments area. Can be skipped if we eliminated
9484 all the function calls as dead code.
9485 Skipping is however impossible when function calls alloca. Alloca
9486 expander assumes that last crtl->outgoing_args_size
9487 of stack frame are unused. */
9491 {
9494 }
9495 else
9498 /* Align stack boundary. Only needed if we're calling another function
9499 or using alloca. */
9504 /* We've reached end of stack frame. */
9507 /* Size prologue needs to allocate. */
9518 {
9524 }
9525 else
9529 /* The SEH frame pointer location is near the bottom of the frame.
9530 This is enforced by the fact that the difference between the
9531 stack pointer and the frame pointer is limited to 240 bytes in
9532 the unwind data structure. */
9534 {
9535 HOST_WIDE_INT diff;
9537 /* If we can leave the frame pointer where it is, do so. Also, returns
9538 the establisher frame for __builtin_frame_address (0). */
9543 {
9544 /* Ideally we'd determine what portion of the local stack frame
9545 (within the constraint of the lowest 240) is most heavily used.
9546 But without that complication, simply bias the frame pointer
9547 by 128 bytes so as to maximize the amount of the local stack
9548 frame that is addressable with 8-bit offsets. */
9550 }
9551 }
9552 }
9554 /* This is semi-inlined memory_address_length, but simplified
9555 since we know that we're always dealing with reg+offset, and
9556 to avoid having to create and discard all that rtl. */
9560 {
9564 {
9565 /* EBP and R13 cannot be encoded without an offset. */
9567 }
9571 /* ESP and R12 must be encoded with a SIB byte. */
9573 len++;
9576 }
9578 /* Return an RTX that points to CFA_OFFSET within the stack frame.
9579 The valid base registers are taken from CFUN->MACHINE->FS. */
9581 static rtx
9583 {
9589 {
9590 /* Choose the base register most likely to allow the most scheduling
9591 opportunities. Generally FP is valid throughout the function,
9592 while DRAP must be reloaded within the epilogue. But choose either
9593 over the SP due to increased encoding size. */
9596 {
9599 }
9601 {
9604 }
9606 {
9609 }
9610 }
9611 else
9612 {
9613 HOST_WIDE_INT toffset;
9616 /* Choose the base register with the smallest address encoding.
9617 With a tie, choose FP > DRAP > SP. */
9619 {
9623 }
9625 {
9629 {
9633 }
9634 }
9636 {
9640 {
9644 }
9645 }
9646 }
9650 }
9652 /* Emit code to save registers in the prologue. */
9654 static void
9656 {
9658 rtx insn;
9662 {
9665 }
9666 }
9668 /* Emit a single register save at CFA - CFA_OFFSET. */
9670 static void
9672 HOST_WIDE_INT cfa_offset)
9673 {
9681 /* For SSE saves, we need to indicate the 128-bit alignment. */
9692 /* When saving registers into a re-aligned local stack frame, avoid
9693 any tricky guessing by dwarf2out. */
9695 {
9699 {
9700 /* A bit of a hack. We force the DRAP register to be saved in
9701 the re-aligned stack frame, which provides us with a copy
9702 of the CFA that will last past the prologue. Install it. */
9708 }
9709 else
9710 {
9711 /* The frame pointer is a stable reference within the
9712 aligned frame. Use it. */
9719 }
9720 }
9722 /* The memory may not be relative to the current CFA register,
9723 which means that we may need to generate a new pattern for
9724 use by the unwind info. */
9726 {
9731 }
9732 }
9734 /* Emit code to save registers using MOV insns.
9735 First register is stored at CFA - CFA_OFFSET. */
9736 static void
9738 {
9743 {
9746 }
9747 }
9749 /* Emit code to save SSE registers using MOV insns.
9750 First register is stored at CFA - CFA_OFFSET. */
9751 static void
9753 {
9758 {
9761 }
9762 }
9766 /* Add a REG_CFA_RESTORE REG note to INSN or queue them until next stack
9767 manipulation insn. The value is on the stack at CFA - CFA_OFFSET.
9768 Don't add the note if the previously saved value will be left untouched
9769 within stack red-zone till return, as unwinders can find the same value
9770 in the register and on the stack. */
9772 static void
9774 {
9780 {
9783 }
9784 else
9785 queued_cfa_restores
9787 }
9789 /* Add queued REG_CFA_RESTORE notes if any to INSN. */
9791 static void
9793 {
9794 rtx last;
9798 ;
9803 }
9805 /* Expand prologue or epilogue stack adjustment.
9806 The pattern exist to put a dependency on all ebp-based memory accesses.
9807 STYLE should be negative if instructions should be marked as frame related,
9808 zero if %r11 register is live and cannot be freely used and positive
9809 otherwise. */
9811 static void
9814 {
9816 rtx insn;
9823 else
9824 {
9825 rtx tmp;
9826 /* r11 is used by indirect sibcall return as well, set before the
9827 epilogue and used after the epilogue. */
9830 else
9831 {
9835 }
9841 }
9848 {
9849 rtx r;
9859 }
9861 {
9864 {
9868 }
9869 }
9872 {
9877 {
9880 }
9882 {
9885 }
9886 else
9887 {
9888 /* Else there are two possibilities: SP itself, which we set
9889 up as the default above. Or EH_RETURN_STACKADJ_RTX, which is
9890 taken care of this by hand along the eh_return path. */
9893 }
9897 }
9898 }
9900 /* Find an available register to be used as dynamic realign argument
9901 pointer regsiter. Such a register will be written in prologue and
9902 used in begin of body, so it must not be
9903 1. parameter passing register.
9904 2. GOT pointer.
9905 We reuse static-chain register if it is available. Otherwise, we
9906 use DI for i386 and R13 for x86-64. We chose R13 since it has
9907 shorter encoding.
9909 Return: the regno of chosen register. */
9911 static unsigned int
9913 {
9917 {
9918 /* Use R13 for nested function or function need static chain.
9919 Since function with tail call may use any caller-saved
9920 registers in epilogue, DRAP must not use caller-saved
9921 register in such case. */
9926 }
9927 else
9928 {
9929 /* Use DI for nested function or function need static chain.
9930 Since function with tail call may use any caller-saved
9931 registers in epilogue, DRAP must not use caller-saved
9932 register in such case. */
9936 /* Reuse static chain register if it isn't used for parameter
9937 passing. */
9939 {
9943 }
9945 }
9946 }
9948 /* Return minimum incoming stack alignment. */
9950 static unsigned int
9952 {
9955 /* Prefer the one specified at command line. */
9958 /* In 32bit, use MIN_STACK_BOUNDARY for incoming stack boundary
9959 if -mstackrealign is used, it isn't used for sibcall check and
9960 estimated stack alignment is 128bit. */
9962 && !TARGET_64BIT
9963 && ix86_force_align_arg_pointer
9966 else
9969 /* Incoming stack alignment can be changed on individual functions
9970 via force_align_arg_pointer attribute. We use the smallest
9971 incoming stack boundary. */
9977 /* The incoming stack frame has to be aligned at least at
9978 parm_stack_boundary. */
9982 /* Stack at entrance of main is aligned by runtime. We use the
9983 smallest incoming stack boundary. */
9991 }
9993 /* Update incoming stack boundary and estimated stack alignment. */
9995 static void
9997 {
9998 ix86_incoming_stack_boundary
10001 /* x86_64 vararg needs 16byte stack alignment for register save
10002 area. */
10007 }
10009 /* Handle the TARGET_GET_DRAP_RTX hook. Return NULL if no DRAP is
10010 needed or an rtx for DRAP otherwise. */
10012 static rtx
10014 {
10019 {
10020 /* Assign DRAP to vDRAP and returns vDRAP */
10022 rtx drap_vreg;
10023 rtx arg_ptr;
10036 {
10039 }
10041 }
10042 else
10044 }
10046 /* Handle the TARGET_INTERNAL_ARG_POINTER hook. */
10048 static rtx
10050 {
10052 }
10055 rtx reg;
10057 };
10059 /* Return a short-lived scratch register for use on function entry.
10060 In 32-bit mode, it is valid only after the registers are saved
10061 in the prologue. This register must be released by means of
10062 release_scratch_register_on_entry once it is dead. */
10064 static void
10066 {
10072 {
10073 /* We always use R11 in 64-bit mode. */
10075 }
10076 else
10077 {
10079 bool fastcall_p
10081 bool thiscall_p
10085 int drap_regno
10088 /* 'fastcall' sets regparm to 2, uses ecx/edx for arguments and eax
10089 for the static chain register. */
10093 /* 'thiscall' sets regparm to 1, uses ecx for arguments and edx
10094 for the static chain register. */
10099 /* ecx is the static chain register. */
10101 && !static_chain_p
10106 /* esi is the static chain register. */
10112 else
10113 {
10116 }
10117 }
10121 {
10124 }
10125 }
10127 /* Release a scratch register obtained from the preceding function. */
10129 static void
10131 {
10133 {
10137 /* The RTX_FRAME_RELATED_P mechanism doesn't know about pop. */
10143 }
10144 }
10146 #define PROBE_INTERVAL (1 << STACK_CHECK_PROBE_INTERVAL_EXP)
10148 /* Emit code to adjust the stack pointer by SIZE bytes while probing it. */
10150 static void
10152 {
10153 /* We skip the probe for the first interval + a small dope of 4 words and
10154 probe that many bytes past the specified size to maintain a protection
10155 area at the botton of the stack. */
10159 /* See if we have a constant small number of probes to generate. If so,
10160 that's the easy case. The run-time loop is made up of 11 insns in the
10161 generic case while the compile-time loop is made up of 3+2*(n-1) insns
10162 for n # of intervals. */
10164 {
10168 /* Adjust SP and probe at PROBE_INTERVAL + N * PROBE_INTERVAL for
10169 values of N from 1 until it exceeds SIZE. If only one probe is
10170 needed, this will not generate any code. Then adjust and probe
10171 to PROBE_INTERVAL + SIZE. */
10173 {
10175 {
10178 }
10179 else
10186 }
10190 else
10198 /* Adjust back to account for the additional first interval. */
10202 }
10204 /* Otherwise, do the same as above, but in a loop. Note that we must be
10205 extra careful with variables wrapping around because we might be at
10206 the very top (or the very bottom) of the address space and we have
10207 to be able to handle this case properly; in particular, we use an
10208 equality test for the loop condition. */
10209 else
10210 {
10211 HOST_WIDE_INT rounded_size;
10217 /* Step 1: round SIZE to the previous multiple of the interval. */
10222 /* Step 2: compute initial and final value of the loop counter. */
10224 /* SP = SP_0 + PROBE_INTERVAL. */
10229 /* LAST_ADDR = SP_0 + PROBE_INTERVAL + ROUNDED_SIZE. */
10233 stack_pointer_rtx)));
10236 /* Step 3: the loop
10238 while (SP != LAST_ADDR)
10239 {
10240 SP = SP + PROBE_INTERVAL
10241 probe at SP
10242 }
10244 adjusts SP and probes to PROBE_INTERVAL + N * PROBE_INTERVAL for
10245 values of N from 1 until it is equal to ROUNDED_SIZE. */
10250 /* Step 4: adjust SP and probe at PROBE_INTERVAL + SIZE if we cannot
10251 assert at compile-time that SIZE is equal to ROUNDED_SIZE. */
10254 {
10259 }
10261 /* Adjust back to account for the additional first interval. */
10267 }
10271 /* Even if the stack pointer isn't the CFA register, we need to correctly
10272 describe the adjustments made to it, in particular differentiate the
10273 frame-related ones from the frame-unrelated ones. */
10275 {
10288 }
10290 /* Make sure nothing is scheduled before we are done. */
10292 }
10294 /* Adjust the stack pointer up to REG while probing it. */
10298 {
10308 /* Jump to END_LAB if SP == LAST_ADDR. */
10316 /* SP = SP + PROBE_INTERVAL. */
10320 /* Probe at SP. */
10331 }
10333 /* Emit code to probe a range of stack addresses from FIRST to FIRST+SIZE,
10334 inclusive. These are offsets from the current stack pointer. */
10336 static void
10338 {
10339 /* See if we have a constant small number of probes to generate. If so,
10340 that's the easy case. The run-time loop is made up of 7 insns in the
10341 generic case while the compile-time loop is made up of n insns for n #
10342 of intervals. */
10344 {
10345 HOST_WIDE_INT i;
10347 /* Probe at FIRST + N * PROBE_INTERVAL for values of N from 1 until
10348 it exceeds SIZE. If only one probe is needed, this will not
10349 generate any code. Then probe at FIRST + SIZE. */
10356 }
10358 /* Otherwise, do the same as above, but in a loop. Note that we must be
10359 extra careful with variables wrapping around because we might be at
10360 the very top (or the very bottom) of the address space and we have
10361 to be able to handle this case properly; in particular, we use an
10362 equality test for the loop condition. */
10363 else
10364 {
10371 /* Step 1: round SIZE to the previous multiple of the interval. */
10376 /* Step 2: compute initial and final value of the loop counter. */
10378 /* TEST_OFFSET = FIRST. */
10381 /* LAST_OFFSET = FIRST + ROUNDED_SIZE. */
10385 /* Step 3: the loop
10387 while (TEST_ADDR != LAST_ADDR)
10388 {
10389 TEST_ADDR = TEST_ADDR + PROBE_INTERVAL
10390 probe at TEST_ADDR
10391 }
10393 probes at FIRST + N * PROBE_INTERVAL for values of N from 1
10394 until it is equal to ROUNDED_SIZE. */
10399 /* Step 4: probe at FIRST + SIZE if we cannot assert at compile-time
10400 that SIZE is equal to ROUNDED_SIZE. */
10405 stack_pointer_rtx,
10410 }
10412 /* Make sure nothing is scheduled before we are done. */
10414 }
10416 /* Probe a range of stack addresses from REG to END, inclusive. These are
10417 offsets from the current stack pointer. */
10421 {
10431 /* Jump to END_LAB if TEST_ADDR == LAST_ADDR. */
10439 /* TEST_ADDR = TEST_ADDR + PROBE_INTERVAL. */
10443 /* Probe at TEST_ADDR. */
10456 }
10458 /* Finalize stack_realign_needed flag, which will guide prologue/epilogue
10459 to be generated in correct form. */
10460 static void
10462 {
10463 /* Check if stack realign is really needed after reload, and
10464 stores result in cfun */
10465 unsigned int incoming_stack_boundary
10474 {
10475 /* After stack_realign_needed is finalized, we can't no longer
10476 change it. */
10479 }
10481 /* If the only reason for frame_pointer_needed is that we conservatively
10482 assumed stack realignment might be needed, but in the end nothing that
10483 needed the stack alignment had been spilled, clear frame_pointer_needed
10484 and say we don't need stack realignment. */
10487 && frame_pointer_needed
10489 && flag_omit_frame_pointer
10491 && !ix86_current_function_calls_tls_descriptor
10500 {
10502 basic_block bb;
10509 HARD_FRAME_POINTER_REGNUM);
10511 {
10512 rtx insn;
10516 set_up_by_prologue))
10517 {
10521 }
10522 }
10536 }
10540 }
10542 /* Expand the prologue into a bunch of separate insns. */
10544 void
10546 {
10551 HOST_WIDE_INT allocate;
10557 /* DRAP should not coexist with stack_realign_fp */
10562 /* Initialize CFA state for before the prologue. */
10566 /* Track SP offset to the CFA. We continue tracking this after we've
10567 swapped the CFA register away from SP. In the case of re-alignment
10568 this is fudged; we're interested to offsets within the local frame. */
10575 {
10576 /* We should have already generated an error for any use of
10577 ms_hook on a nested function. */
10580 /* Check if profiling is active and we shall use profiling before
10581 prologue variant. If so sorry. */
10586 /* In ix86_asm_output_function_label we emitted:
10587 8b ff movl.s %edi,%edi
10588 55 push %ebp
10589 8b ec movl.s %esp,%ebp
10591 This matches the hookable function prologue in Win32 API
10592 functions in Microsoft Windows XP Service Pack 2 and newer.
10593 Wine uses this to enable Windows apps to hook the Win32 API
10594 functions provided by Wine.
10596 What that means is that we've already set up the frame pointer. */
10600 {
10603 /* We've decided to use the frame pointer already set up.
10604 Describe this to the unwinder by pretending that both
10605 push and mov insns happen right here.
10607 Putting the unwind info here at the end of the ms_hook
10608 is done so that we can make absolutely certain we get
10609 the required byte sequence at the start of the function,
10610 rather than relying on an assembler that can produce
10611 the exact encoding required.
10613 However it does mean (in the unpatched case) that we have
10614 a 1 insn window where the asynchronous unwind info is
10615 incorrect. However, if we placed the unwind info at
10616 its correct location we would have incorrect unwind info
10617 in the patched case. Which is probably all moot since
10618 I don't expect Wine generates dwarf2 unwind info for the
10619 system libraries that use this feature. */
10625 stack_pointer_rtx);
10633 /* Note that gen_push incremented m->fs.cfa_offset, even
10634 though we didn't emit the push insn here. */
10638 }
10639 else
10640 {
10641 /* The frame pointer is not needed so pop %ebp again.
10642 This leaves us with a pristine state. */
10644 }
10645 }
10647 /* The first insn of a function that accepts its static chain on the
10648 stack is to push the register that would be filled in by a direct
10649 call. This insn will be skipped by the trampoline. */
10651 {
10655 /* We don't want to interpret this push insn as a register save,
10656 only as a stack adjustment. The real copy of the register as
10657 a save will be done later, if needed. */
10662 }
10664 /* Emit prologue code to adjust stack alignment and setup DRAP, in case
10665 of DRAP is needed and stack realignment is really needed after reload */
10667 {
10670 /* Only need to push parameter pointer reg if it is caller saved. */
10672 {
10673 /* Push arg pointer reg */
10676 }
10678 /* Grab the argument pointer. */
10685 /* Align the stack. */
10687 stack_pointer_rtx,
10691 /* Replicate the return address on the stack so that return
10692 address can be reached via (argp - 1) slot. This is needed
10693 to implement macro RETURN_ADDR_RTX and intrinsic function
10694 expand_builtin_return_addr etc. */
10700 /* For the purposes of frame and register save area addressing,
10701 we've started over with a new frame. */
10704 }
10710 {
10711 /* Note: AT&T enter does NOT have reversed args. Enter is probably
10712 slower on all targets. Also sdb doesn't like it. */
10716 /* Push registers now, before setting the frame pointer
10717 on SEH target. */
10719 && TARGET_SEH
10721 {
10725 }
10728 {
10736 }
10737 }
10740 {
10741 /* If saving registers via PUSH, do so now. */
10743 {
10747 }
10749 /* When using red zone we may start register saving before allocating
10750 the stack frame saving one cycle of the prologue. However, avoid
10751 doing this if we have to probe the stack; at least on x86_64 the
10752 stack probe can turn into a call that clobbers a red zone location. */
10754 && (! TARGET_STACK_PROBE
10756 {
10759 }
10760 }
10763 {
10767 /* The computation of the size of the re-aligned stack frame means
10768 that we must allocate the size of the register save area before
10769 performing the actual alignment. Otherwise we cannot guarantee
10770 that there's enough storage above the realignment point. */
10777 /* Align the stack. */
10779 stack_pointer_rtx,
10782 /* For the purposes of register save area addressing, the stack
10783 pointer is no longer valid. As for the value of sp_offset,
10784 see ix86_compute_frame_layout, which we need to match in order
10785 to pass verification of stack_pointer_offset at the end. */
10788 }
10793 {
10794 /* We start to count from ARG_POINTER. */
10797 /* If it was realigned, take into account the fake frame. */
10799 {
10806 /* This over-estimates by 1 minimal-stack-alignment-unit but
10807 mitigates that by counting in the new return address slot. */
10808 current_function_dynamic_stack_size
10810 }
10813 }
10815 /* On SEH target with very large frame size, allocate an area to save
10816 SSE registers (as the very large allocation won't be described). */
10820 {
10821 HOST_WIDE_INT sse_size =
10826 /* No need to do stack checking as the area will be immediately
10827 written. */
10834 }
10836 /* The stack has already been decremented by the instruction calling us
10837 so probe if the size is non-negative to preserve the protection area. */
10839 {
10840 /* We expect the registers to be saved when probes are used. */
10844 {
10847 {
10850 }
10851 }
10852 else
10853 {
10860 {
10862 {
10865 }
10866 else
10868 }
10869 else
10870 {
10872 {
10876 }
10877 else
10879 }
10880 }
10881 }
10884 ;
10887 {
10891 }
10892 else
10893 {
10906 /* Note that SEH directives need to continue tracking the stack
10907 pointer even after the frame pointer has been set up. */
10909 {
10913 {
10917 }
10918 }
10921 {
10926 {
10930 }
10931 }
10936 /* Use the fact that AX still contains ALLOCATE. */
10938 ? gen_pro_epilogue_adjust_stack_di_sub
10945 {
10953 }
10956 /* Use stack_pointer_rtx for relative addressing so that code
10957 works for realigned stack, too. */
10959 {
10967 }
10969 {
10974 }
10975 }
10978 /* If we havn't already set up the frame pointer, do so now. */
10980 {
10992 }
11000 /* We don't use pic-register for pe-coff target. */
11002 && !TARGET_PECOFF
11005 {
11012 }
11015 {
11017 {
11019 {
11029 label));
11033 }
11034 else
11036 }
11037 else
11038 {
11042 }
11043 }
11045 /* In the pic_reg_used case, make sure that the got load isn't deleted
11046 when mcount needs it. Blockage to avoid call movement across mcount
11047 call is emitted in generic code after the NOTE_INSN_PROLOGUE_END
11048 note. */
11053 {
11054 /* vDRAP is setup but after reload it turns out stack realign
11055 isn't necessary, here we will emit prologue to setup DRAP
11056 without stack realign adjustment */
11059 }
11061 /* Prevent instructions from being scheduled into register save push
11062 sequence when access to the redzone area is done through frame pointer.
11063 The offset between the frame pointer and the stack pointer is calculated
11064 relative to the value of the stack pointer at the end of the function
11065 prologue, and moving instructions that access redzone area via frame
11066 pointer inside push sequence violates this assumption. */
11070 /* Emit cld instruction if stringops are used in the function. */
11074 /* SEH requires that the prologue end within 256 bytes of the start of
11075 the function. Prevent instruction schedules that would extend that.
11076 Further, prevent alloca modifications to the stack pointer from being
11077 combined with prologue modifications. */
11080 }
11082 /* Emit code to restore REG using a POP insn. */
11084 static void
11086 {
11095 {
11096 /* Previously we'd represented the CFA as an expression
11097 like *(%ebp - 8). We've just popped that value from
11098 the stack, which means we need to reset the CFA to
11099 the drap register. This will remain until we restore
11100 the stack pointer. */
11104 /* This means that the DRAP register is valid for addressing too. */
11107 }
11110 {
11117 }
11119 /* When the frame pointer is the CFA, and we pop it, we are
11120 swapping back to the stack pointer as the CFA. This happens
11121 for stack frames that don't allocate other data, so we assume
11122 the stack pointer is now pointing at the return address, i.e.
11123 the function entry state, which makes the offset be 1 word. */
11125 {
11128 {
11136 }
11137 }
11138 }
11140 /* Emit code to restore saved registers using POP insns. */
11142 static void
11144 {
11150 }
11152 /* Emit code and notes for the LEAVE instruction. */
11154 static void
11156 {
11168 {
11176 }
11179 }
11181 /* Emit code to restore saved registers using MOV insns.
11182 First register is restored from CFA - CFA_OFFSET. */
11183 static void
11186 {
11192 {
11201 {
11202 /* Previously we'd represented the CFA as an expression
11203 like *(%ebp - 8). We've just popped that value from
11204 the stack, which means we need to reset the CFA to
11205 the drap register. This will remain until we restore
11206 the stack pointer. */
11210 /* This means that the DRAP register is valid for addressing. */
11212 }
11213 else
11217 }
11218 }
11220 /* Emit code to restore saved registers using MOV insns.
11221 First register is restored from CFA - CFA_OFFSET. */
11222 static void
11225 {
11230 {
11232 rtx mem;
11242 }
11243 }
11245 /* Restore function stack, frame, and registers. */
11247 void
11249 {
11265 /* The FP must be valid if the frame pointer is present. */
11270 /* We must have *some* valid pointer to the stack frame. */
11273 /* The DRAP is never valid at this point. */
11276 /* See the comment about red zone and frame
11277 pointer usage in ix86_expand_prologue. */
11284 /* Determine the CFA offset of the end of the red-zone. */
11287 {
11288 /* The red-zone begins below the return address. */
11291 /* When the register save area is in the aligned portion of
11292 the stack, determine the maximum runtime displacement that
11293 matches up with the aligned frame. */
11297 }
11299 /* Special care must be taken for the normal return case of a function
11300 using eh_return: the eax and edx registers are marked as saved, but
11301 not restored along this path. Adjust the save location to match. */
11305 /* EH_RETURN requires the use of moves to function properly. */
11308 /* SEH requires the use of pops to identify the epilogue. */
11311 /* If we're only restoring one register and sp is not valid then
11312 using a move instruction to restore the register since it's
11313 less work than reloading sp and popping the register. */
11326 && TARGET_USE_LEAVE
11330 else
11334 {
11335 /* Ensure that the entire register save area is addressable via
11336 the stack pointer, if we will restore via sp. */
11341 {
11345 style,
11347 }
11348 }
11350 /* If there are any SSE registers to restore, then we have to do it
11351 via moves, since there's obviously no pop for SSE regs. */
11357 {
11358 rtx t;
11363 /* eh_return epilogues need %ecx added to the stack pointer. */
11365 {
11368 /* Stack align doesn't work with eh_return. */
11370 /* Neither does regparm nested functions. */
11374 {
11382 /* Note that we use SA as a temporary CFA, as the return
11383 address is at the proper place relative to it. We
11384 pretend this happens at the FP restore insn because
11385 prior to this insn the FP would be stored at the wrong
11386 offset relative to SA, and after this insn we have no
11387 other reasonable register to use for the CFA. We don't
11388 bother resetting the CFA to the SP for the duration of
11389 the return insn. */
11402 }
11403 else
11404 {
11412 {
11416 UNITS_PER_WORD));
11418 }
11419 }
11422 }
11423 }
11424 else
11425 {
11426 /* SEH requires that the function end with (1) a stack adjustment
11427 if necessary, (2) a sequence of pops, and (3) a return or
11428 jump instruction. Prevent insns from the function body from
11429 being scheduled into this sequence. */
11431 {
11432 /* Prevent a catch region from being adjacent to the standard
11433 epilogue sequence. Unfortuantely crtl->uses_eh_lsda nor
11434 several other flags that would be interesting to test are
11435 not yet set up. */
11438 else
11440 }
11442 /* First step is to deallocate the stack frame so that we can
11443 pop the registers. Also do it on SEH target for very large
11444 frame as the emitted instructions aren't allowed by the ABI in
11445 epilogues. */
11447 || (TARGET_SEH
11450 {
11455 }
11457 {
11461 style,
11463 }
11466 }
11468 /* If we used a stack pointer and haven't already got rid of it,
11469 then do so now. */
11471 {
11472 /* If the stack pointer is valid and pointing at the frame
11473 pointer store address, then we only need a pop. */
11476 /* Leave results in shorter dependency chains on CPUs that are
11477 able to grok it fast. */
11482 else
11483 {
11485 hard_frame_pointer_rtx,
11488 }
11489 }
11492 {
11494 rtx insn;
11520 }
11522 /* At this point the stack pointer must be valid, and we must have
11523 restored all of the registers. We may not have deallocated the
11524 entire stack frame. We've delayed this until now because it may
11525 be possible to merge the local stack deallocation with the
11526 deallocation forced by ix86_static_chain_on_stack. */
11531 {
11535 }
11536 else
11539 /* Sibcall epilogues don't want a return instruction. */
11541 {
11544 }
11547 {
11550 /* i386 can only pop 64K bytes. If asked to pop more, pop return
11551 address, do explicit add, and jump indirectly to the caller. */
11554 {
11556 rtx insn;
11558 /* There is no "pascal" calling convention in any 64bit ABI. */
11574 }
11575 else
11577 }
11578 else
11581 /* Restore the state back to the state from the prologue,
11582 so that it's correct for the next epilogue. */
11584 }
11586 /* Reset from the function's potential modifications. */
11588 static void
11590 HOST_WIDE_INT size ATTRIBUTE_UNUSED)
11591 {
11594 #if TARGET_MACHO
11595 /* Mach-O doesn't support labels at the end of objects, so if
11596 it looks like we might want one, insert a NOP. */
11597 {
11603 {
11604 /* Don't insert a nop for NOTE_INSN_DELETED_DEBUG_LABEL
11605 notes only, instead set their CODE_LABEL_NUMBER to -1,
11606 otherwise there would be code generation differences
11607 in between -g and -g0. */
11611 }
11621 }
11622 #endif
11624 }
11626 /* Return a scratch register to use in the split stack prologue. The
11627 split stack prologue is used for -fsplit-stack. It is the first
11628 instructions in the function, even before the regular prologue.
11629 The scratch register can be any caller-saved register which is not
11630 used for parameters or for the static chain. */
11632 static unsigned int
11634 {
11637 else
11638 {
11651 {
11653 {
11657 }
11659 }
11661 {
11665 }
11667 {
11670 else
11671 {
11673 {
11677 }
11679 }
11680 }
11681 else
11682 {
11683 /* FIXME: We could make this work by pushing a register
11684 around the addition and comparison. */
11687 }
11688 }
11689 }
11691 /* A SYMBOL_REF for the function which allocates new stackspace for
11692 -fsplit-stack. */
11696 /* A SYMBOL_REF for the more stack function when using the large
11697 model. */
11701 /* Handle -fsplit-stack. These are the first instructions in the
11702 function, even before the regular prologue. */
11704 void
11706 {
11708 HOST_WIDE_INT allocate;
11713 rtx fn;
11721 /* This is the label we will branch to if we have enough stack
11722 space. We expect the basic block reordering pass to reverse this
11723 branch if optimizing, so that we branch in the unlikely case. */
11726 /* We need to compare the stack pointer minus the frame size with
11727 the stack boundary in the TCB. The stack boundary always gives
11728 us SPLIT_STACK_AVAILABLE bytes, so if we need less than that we
11729 can compare directly. Otherwise we need to do an addition. */
11732 UNSPEC_STACK_CHECK);
11737 else
11738 {
11740 rtx offset;
11742 /* We need a scratch register to hold the stack pointer minus
11743 the required frame size. Since this is the very start of the
11744 function, the scratch register can be any caller-saved
11745 register which is not used for parameters. */
11752 {
11753 /* We don't use ix86_gen_add3 in this case because it will
11754 want to split to lea, but when not optimizing the insn
11755 will not be split after this point. */
11758 offset)));
11759 }
11760 else
11761 {
11764 stack_pointer_rtx));
11765 }
11767 }
11773 /* Mark the jump as very likely to be taken. */
11781 /* Get more stack space. We pass in the desired stack space and the
11782 size of the arguments to copy to the new stack. In 32-bit mode
11783 we push the parameters; __morestack will return on a new stack
11784 anyhow. In 64-bit mode we pass the parameters in r10 and
11785 r11. */
11790 {
11796 /* If this function uses a static chain, it will be in %r10.
11797 Preserve it across the call to __morestack. */
11799 {
11800 rtx rax;
11805 }
11809 {
11810 HOST_WIDE_INT argval;
11813 /* When using the large model we need to load the address
11814 into a register, and we've run out of registers. So we
11815 switch to a different calling convention, and we call a
11816 different function: __morestack_large. We pass the
11817 argument size in the upper 32 bits of r10 and pass the
11818 frame size in the lower 32 bits. */
11823 split_stack_fn_large =
11827 {
11837 UNSPEC_GOT);
11843 }
11844 else
11851 }
11852 else
11853 {
11857 }
11860 }
11861 else
11862 {
11865 }
11871 /* In order to make call/return prediction work right, we now need
11872 to execute a return instruction. See
11873 libgcc/config/i386/morestack.S for the details on how this works.
11875 For flow purposes gcc must not see this as a return
11876 instruction--we need control flow to continue at the subsequent
11877 label. Therefore, we use an unspec. */
11881 /* If we are in 64-bit mode and this function uses a static chain,
11882 we saved %r10 in %rax before calling _morestack. */
11887 /* If this function calls va_start, we need to store a pointer to
11888 the arguments on the old stack, because they may not have been
11889 all copied to the new stack. At this point the old stack can be
11890 found at the frame pointer value used by __morestack, because
11891 __morestack has set that up before calling back to us. Here we
11892 store that pointer in a scratch register, and in
11893 ix86_expand_prologue we store the scratch register in a stack
11894 slot. */
11896 {
11898 rtx frame_reg;
11905 /* 64-bit:
11906 fp -> old fp value
11907 return address within this function
11908 return address of caller of this function
11909 stack arguments
11910 So we add three words to get to the stack arguments.
11912 32-bit:
11913 fp -> old fp value
11914 return address within this function
11915 first argument to __morestack
11916 second argument to __morestack
11917 return address of caller of this function
11918 stack arguments
11919 So we add five words to get to the stack arguments.
11920 */
11931 }
11936 /* If this function calls va_start, we now have to set the scratch
11937 register for the case where we do not call __morestack. In this
11938 case we need to set it based on the stack pointer. */
11940 {
11947 }
11948 }
11950 /* We may have to tell the dataflow pass that the split stack prologue
11951 is initializing a scratch register. */
11953 static void
11955 {
11957 {
11960 }
11961 }
11962 \f
11963 /* Extract the parts of an RTL expression that is a valid memory address
11964 for an instruction. Return 0 if the structure of the address is
11965 grossly off. Return -1 if the address contains ASHIFT, so it is not
11966 strictly valid, but still used for computing length of lea instruction. */
11968 int
11970 {
11975 rtx tmp;
11979 /* Allow zero-extended SImode addresses,
11980 they will be emitted with addr32 prefix. */
11982 {
11985 {
11989 }
11992 {
11999 }
12000 }
12002 /* Allow SImode subregs of DImode addresses,
12003 they will be emitted with addr32 prefix. */
12005 {
12008 {
12012 }
12013 }
12018 {
12021 else
12023 }
12025 {
12030 do
12031 {
12036 }
12043 {
12046 {
12071 /* FALLTHRU */
12075 && TARGET_TLS_DIRECT_SEG_REFS
12078 else
12085 /* FALLTHRU */
12092 else
12107 }
12108 }
12109 }
12111 {
12114 }
12116 {
12117 /* We're called for lea too, which implements ashift on occasion. */
12127 }
12128 else
12132 {
12134 ;
12137 ;
12138 else
12140 }
12142 /* Extract the integral value of scale. */
12144 {
12148 }
12153 /* Avoid useless 0 displacement. */
12157 /* Allow arg pointer and stack pointer as index if there is not scaling. */
12162 {
12163 rtx tmp;
12166 }
12168 /* Special case: %ebp cannot be encoded as a base without a displacement.
12169 Similarly %r13. */
12171 && base_reg
12180 /* Special case: on K6, [%esi] makes the instruction vector decoded.
12181 Avoid this by transforming to [%esi+0].
12182 Reload calls address legitimization without cfun defined, so we need
12183 to test cfun for being non-NULL. */
12189 /* Special case: encode reg+reg instead of reg*2. */
12193 /* Special case: scaling cannot be encoded without base or displacement. */
12204 }
12205 \f
12206 /* Return cost of the memory address x.
12207 For i386, it is better to use a complex address than let gcc copy
12208 the address into a reg and make a new pseudo. But not if the address
12209 requires to two regs - that would mean more pseudos with longer
12210 lifetimes. */
12211 static int
12213 addr_space_t as ATTRIBUTE_UNUSED,
12215 {
12227 /* Attempt to minimize number of registers in the address. */
12233 cost++;
12240 cost++;
12242 /* AMD-K6 don't like addresses with ModR/M set to 00_xxx_100b,
12243 since it's predecode logic can't detect the length of instructions
12244 and it degenerates to vector decoded. Increase cost of such
12245 addresses here. The penalty is minimally 2 cycles. It may be worthwhile
12246 to split such addresses or even refuse such addresses at all.
12248 Following addressing modes are affected:
12249 [base+scale*index]
12250 [scale*index+disp]
12251 [base+index]
12253 The first and last case may be avoidable by explicitly coding the zero in
12254 memory address, but I don't have AMD-K6 machine handy to check this
12255 theory. */
12264 }
12265 \f
12266 /* Allow {LABEL | SYMBOL}_REF - SYMBOL_REF-FOR-PICBASE for Mach-O as
12267 this is used for to form addresses to local data when -fPIC is in
12268 use. */
12270 static bool
12272 {
12275 }
12277 /* Determine if a given RTX is a valid constant. We already know this
12278 satisfies CONSTANT_P. */
12280 static bool
12282 {
12284 {
12289 {
12293 }
12298 /* Only some unspecs are valid as "constants". */
12301 {
12317 }
12319 /* We must have drilled down to a symbol. */
12324 /* FALLTHRU */
12327 /* TLS symbols are never valid. */
12331 /* DLLIMPORT symbols are never valid. */
12336 #if TARGET_MACHO
12337 /* mdynamic-no-pic */
12340 #endif
12356 }
12358 /* Otherwise we handle everything else in the move patterns. */
12360 }
12362 /* Determine if it's legal to put X into the constant pool. This
12363 is not possible for the address of thread-local symbols, which
12364 is checked above. */
12366 static bool
12368 {
12369 /* We can always put integral constants and vectors in memory. */
12371 {
12379 }
12381 }
12383 /* Nonzero if the symbol is marked as dllimport, or as stub-variable,
12384 otherwise zero. */
12386 static bool
12388 {
12394 }
12397 /* Nonzero if the constant value X is a legitimate general operand
12398 when generating PIC code. It is given that flag_pic is on and
12399 that X satisfies CONSTANT_P or is a CONST_DOUBLE. */
12401 bool
12403 {
12404 rtx inner;
12407 {
12414 /* Only some unspecs are valid as "constants". */
12417 {
12430 }
12431 /* FALLTHRU */
12439 }
12440 }
12442 /* Determine if a given CONST RTX is a valid memory displacement
12443 in PIC mode. */
12445 bool
12447 {
12450 /* In 64bit mode we can allow direct addresses of symbols and labels
12451 when they are not dynamic symbols. */
12453 {
12457 {
12481 /* FALLTHRU */
12484 /* TLS references should always be enclosed in UNSPEC.
12485 The dllimported symbol needs always to be resolved. */
12491 {
12499 /* Function-symbols need to be resolved only for
12500 large-model.
12501 For the small-model we don't need to resolve anything
12502 here. */
12507 /* Non-external symbols don't need to be resolved for
12508 large, and medium-model. */
12513 }
12522 }
12523 }
12529 {
12530 /* We are unsafe to allow PLUS expressions. This limit allowed distance
12531 of GOT tables. We should not need these anyway. */
12543 }
12547 {
12552 }
12561 {
12565 /* We need to check for both symbols and labels because VxWorks loads
12566 text labels with @GOT rather than @GOTOFF. See gotoff_operand for
12567 details. */
12571 /* Refuse GOTOFF in 64bit mode since it is always 64bit when used.
12572 While ABI specify also 32bit relocation but we don't produce it in
12573 small PIC model at all. */
12595 }
12598 }
12600 /* Our implementation of LEGITIMIZE_RELOAD_ADDRESS. Returns a value to
12601 replace the input X, or the original X if no replacement is called for.
12602 The output parameter *WIN is 1 if the calling macro should goto WIN,
12603 0 if it should not. */
12605 bool
12610 {
12611 /* Reload can generate:
12613 (plus:DI (plus:DI (unspec:DI [(const_int 0 [0])] UNSPEC_TP)
12614 (reg:DI 97))
12615 (reg:DI 2 cx))
12617 This RTX is rejected from ix86_legitimate_address_p due to
12618 non-strictness of base register 97. Following this rejection,
12619 reload pushes all three components into separate registers,
12620 creating invalid memory address RTX.
12622 Following code reloads only the invalid part of the
12623 memory address RTX. */
12629 {
12635 {
12640 }
12644 {
12649 }
12653 }
12656 }
12658 /* Determine if op is suitable RTX for an address register.
12659 Return naked register if a register or a register subreg is
12660 found, otherwise return NULL_RTX. */
12662 static rtx
12664 {
12667 /* Only SImode or DImode registers can form the address. */
12674 {
12682 /* Don't allow SUBREGs that span more than a word. It can
12683 lead to spill failures when the register is one word out
12684 of a two word structure. */
12688 /* Allow only SUBREGs of non-eliminable hard registers. */
12691 }
12693 /* Op is not a register. */
12695 }
12697 /* Recognizes RTL expressions that are valid memory addresses for an
12698 instruction. The MODE argument is the machine mode for the MEM
12699 expression that wants to use this address.
12701 It only recognizes address in canonical form. LEGITIMIZE_ADDRESS should
12702 convert common non-canonical forms to canonical form so that they will
12703 be recognized. */
12705 static bool
12708 {
12711 HOST_WIDE_INT scale;
12715 /* Decomposition failed. */
12724 /* Validate base register. */
12726 {
12734 /* Base is not valid. */
12736 }
12738 /* Validate index register. */
12740 {
12748 /* Index is not valid. */
12750 }
12752 /* Index and base should have the same mode. */
12757 /* Address override works only on the (%reg) part of %fs:(%reg). */
12763 /* Validate scale factor. */
12765 {
12767 /* Scale without index. */
12771 /* Scale is not a valid multiplier. */
12773 }
12775 /* Validate displacement. */
12777 {
12782 {
12783 /* Refuse GOTOFF and GOT in 64bit mode since it is always 64bit when
12784 used. While ABI specify also 32bit relocations, we don't produce
12785 them at all and use IP relative instead. */
12792 /* 64bit address unspec. */
12812 /* Invalid address unspec. */
12814 }
12817 && (flag_pic
12818 || (TARGET_MACHO
12819 #if TARGET_MACHO
12820 && MACHOPIC_INDIRECT
12822 #endif
12823 )))
12824 {
12826 is_legitimate_pic:
12828 {
12829 /* foo@dtpoff(%rX) is ok. */
12836 /* Non-constant pic memory reference. */
12838 }
12841 /* Displacement is an invalid pic construct. */
12843 #if TARGET_MACHO
12846 /* displacment must be referenced via non_lazy_pointer */
12848 #endif
12850 /* This code used to verify that a symbolic pic displacement
12851 includes the pic_offset_table_rtx register.
12853 While this is good idea, unfortunately these constructs may
12854 be created by "adds using lea" optimization for incorrect
12855 code like:
12857 int a;
12858 int foo(int i)
12859 {
12860 return *(&a+i);
12861 }
12863 This code is nonsensical, but results in addressing
12864 GOT table with pic_offset_table_rtx base. We can't
12865 just refuse it easily, since it gets matched by
12866 "addsi3" pattern, that later gets split to lea in the
12867 case output register differs from input. While this
12868 can be handled by separate addsi pattern for this case
12869 that never results in lea, this seems to be easier and
12870 correct fix for crash to disable this test. */
12871 }
12878 /* Displacement is not constant. */
12882 /* Displacement is out of range. */
12884 /* In x32 mode, constant addresses are sign extended to 64bit, so
12885 we have to prevent addresses from 0x80000000 to 0xffffffff. */
12890 }
12892 /* Everything looks valid. */
12894 }
12896 /* Determine if a given RTX is a valid constant address. */
12898 bool
12900 {
12902 }
12903 \f
12904 /* Return a unique alias set for the GOT. */
12906 static alias_set_type
12908 {
12913 }
12915 /* Return a legitimate reference for ORIG (an address) using the
12916 register REG. If REG is 0, a new pseudo is generated.
12918 There are two types of references that must be handled:
12920 1. Global data references must load the address from the GOT, via
12921 the PIC reg. An insn is emitted to do this load, and the reg is
12922 returned.
12924 2. Static data references, constant pool addresses, and code labels
12925 compute the address as an offset from the GOT, whose base is in
12926 the PIC reg. Static data objects have SYMBOL_FLAG_LOCAL set to
12927 differentiate them from global data objects. The returned
12928 address is the PIC reg + an unspec constant.
12930 TARGET_LEGITIMATE_ADDRESS_P rejects symbolic references unless the PIC
12931 reg also appears in the address. */
12933 static rtx
12935 {
12939 #if TARGET_MACHO
12941 {
12944 /* Use the generic Mach-O PIC machinery. */
12946 }
12947 #endif
12950 {
12954 }
12960 {
12961 rtx tmpreg;
12962 /* This symbol may be referenced via a displacement from the PIC
12963 base address (@GOTOFF). */
12970 {
12972 UNSPEC_GOTOFF);
12974 }
12975 else
12980 else
12985 {
12989 }
12990 else
12992 }
12994 {
12995 /* This symbol may be referenced via a displacement from the PIC
12996 base address (@GOTOFF). */
13003 {
13005 UNSPEC_GOTOFF);
13007 }
13008 else
13014 {
13017 }
13018 }
13020 /* We can't use @GOTOFF for text labels on VxWorks;
13021 see gotoff_operand. */
13023 {
13028 /* For x64 PE-COFF there is no GOT table. So we use address
13029 directly. */
13031 {
13039 }
13041 {
13049 /* Use directly gen_movsi, otherwise the address is loaded
13050 into register for CSE. We don't want to CSE this addresses,
13051 instead we CSE addresses from the GOT table, so skip this. */
13054 }
13055 else
13056 {
13057 /* This symbol must be referenced via a load from the
13058 Global Offset Table (@GOT). */
13074 }
13075 }
13076 else
13077 {
13080 {
13082 {
13085 }
13086 else
13088 }
13090 {
13093 /* We must match stuff we generate before. Assume the only
13094 unspecs that can get here are ours. Not that we could do
13095 anything with them anyway.... */
13101 }
13103 {
13106 /* Check first to see if this is a constant offset from a @GOTOFF
13107 symbol reference. */
13110 {
13112 {
13116 UNSPEC_GOTOFF);
13122 {
13125 }
13126 }
13127 else
13128 {
13131 {
13135 }
13136 }
13137 }
13138 else
13139 {
13142 new_rtx
13146 {
13149 {
13152 new_rtx
13154 }
13155 else
13157 }
13158 else
13159 {
13162 {
13165 }
13167 }
13168 }
13169 }
13170 }
13172 }
13173 \f
13174 /* Load the thread pointer. If TO_REG is true, force it into a register. */
13176 static rtx
13178 {
13182 {
13187 }
13193 }
13195 /* Construct the SYMBOL_REF for the tls_get_addr function. */
13199 static rtx
13201 {
13203 {
13209 }
13212 {
13214 UNSPEC_PLTOFF);
13217 }
13220 }
13222 /* Construct the SYMBOL_REF for the _TLS_MODULE_BASE_ symbol. */
13226 rtx
13228 {
13230 {
13231 ix86_tls_module_base_symbol
13236 }
13239 }
13241 /* A subroutine of ix86_legitimize_address and ix86_expand_move. FOR_MOV is
13242 false if we expect this to be used for a memory address and true if
13243 we expect to load the address into a register. */
13245 static rtx
13247 {
13254 {
13259 {
13262 else
13263 {
13266 }
13267 }
13270 {
13273 else
13283 }
13284 else
13285 {
13289 {
13291 rtx insns;
13294 emit_call_insn
13304 }
13305 else
13307 }
13314 {
13317 else
13318 {
13321 }
13322 }
13325 {
13330 else
13336 }
13337 else
13338 {
13342 {
13347 emit_call_insn
13352 /* Attach a unique REG_EQUAL, to allow the RTL optimizers to
13353 share the LD_BASE result with other LD model accesses. */
13355 UNSPEC_TLS_LD_BASE);
13359 }
13360 else
13362 }
13370 {
13377 }
13382 {
13384 {
13385 /* The Sun linker took the AMD64 TLS spec literally
13386 and can only handle %rax as destination of the
13387 initial executable code sequence. */
13392 }
13394 /* Generate DImode references to avoid %fs:(%reg32)
13395 problems and linker IE->LE relaxation bug. */
13399 }
13401 {
13406 }
13408 {
13412 }
13413 else
13414 {
13417 }
13427 {
13432 }
13433 else
13434 {
13438 }
13448 {
13452 }
13453 else
13454 {
13458 }
13463 }
13466 }
13468 /* Create or return the unique __imp_DECL dllimport symbol corresponding
13469 to symbol DECL if BEIMPORT is true. Otherwise create or return the
13470 unique refptr-DECL symbol corresponding to symbol DECL. */
13473 htab_t dllimport_map;
13475 static tree
13477 {
13484 tree to;
13485 rtx rtl;
13512 else
13525 {
13527 #ifdef SUB_TARGET_RECORD_STUB
13529 #endif
13530 }
13539 }
13541 /* Expand SYMBOL into its corresponding far-addresse symbol.
13542 WANT_REG is true if we require the result be a register. */
13544 static rtx
13546 {
13547 tree imp_decl;
13548 rtx x;
13557 }
13559 /* Expand SYMBOL into its corresponding dllimport symbol. WANT_REG is
13560 true if we require the result be a register. */
13562 static rtx
13564 {
13565 tree imp_decl;
13566 rtx x;
13575 }
13577 /* Expand SYMBOL into its corresponding dllimport or refptr symbol. WANT_REG
13578 is true if we require the result be a register. */
13580 static rtx
13582 {
13587 {
13594 {
13597 }
13598 }
13614 {
13617 }
13619 }
13621 /* Try machine-dependent ways of modifying an illegitimate address
13622 to be legitimate. If we find one, return the new, valid address.
13623 This macro is used in only one place: `memory_address' in explow.c.
13625 OLDX is the address as it was before break_out_memory_refs was called.
13626 In some cases it is useful to look at this to decide what needs to be done.
13628 It is always safe for this macro to do nothing. It exists to recognize
13629 opportunities to optimize the output.
13631 For the 80386, we handle X+REG by loading X into a register R and
13632 using R+REG. R will go in a general reg and indexing will be used.
13633 However, if REG is a broken-out memory address or multiplication,
13634 nothing needs to be done because REG can certainly go in a general reg.
13636 When -fpic is used, special handling is needed for symbolic references.
13637 See comments by legitimize_pic_address in i386.c for details. */
13639 static rtx
13642 {
13653 {
13657 }
13660 {
13664 }
13669 #if TARGET_MACHO
13672 #endif
13674 /* Canonicalize shifts by 0, 1, 2, 3 into multiply */
13678 {
13683 }
13686 {
13687 /* Canonicalize shifts by 0, 1, 2, 3 into multiply. */
13692 {
13698 }
13703 {
13709 }
13711 /* Put multiply first if it isn't already. */
13713 {
13718 }
13720 /* Canonicalize (plus (mult (reg) (const)) (plus (reg) (const)))
13721 into (plus (plus (mult (reg) (const)) (reg)) (const)). This can be
13722 created by virtual register instantiation, register elimination, and
13723 similar optimizations. */
13725 {
13731 }
13733 /* Canonicalize
13734 (plus (plus (mult (reg) (const)) (plus (reg) (const))) const)
13735 into (plus (plus (mult (reg) (const)) (reg)) (const)). */
13740 {
13741 rtx constant;
13745 {
13748 }
13750 {
13753 }
13754 else
13758 {
13765 }
13766 }
13772 {
13775 }
13778 {
13781 }
13789 {
13792 }
13798 {
13802 {
13805 }
13809 }
13812 {
13816 {
13819 }
13823 }
13824 }
13827 }
13828 \f
13829 /* Print an integer constant expression in assembler syntax. Addition
13830 and subtraction are the only arithmetic that may appear in these
13831 expressions. FILE is the stdio stream to write to, X is the rtx, and
13832 CODE is the operand print code from the output string. */
13834 static void
13836 {
13840 {
13849 else
13850 {
13853 /* Mark the decl as referenced so that cgraph will
13854 output the function. */
13858 #if TARGET_MACHO
13862 #endif
13864 }
13872 /* FALLTHRU */
13883 /* This used to output parentheses around the expression,
13884 but that does not work on the 386 (either ATT or BSD assembler). */
13890 {
13891 /* We can use %d if the number is <32 bits and positive. */
13896 else
13898 }
13899 else
13900 /* We can't handle floating point constants;
13901 TARGET_PRINT_OPERAND must handle them. */
13906 /* Some assemblers need integer constants to appear first. */
13908 {
13912 }
13913 else
13914 {
13919 }
13934 {
13938 }
13943 {
13962 /* FIXME: This might be @TPOFF in Sun ld too. */
13971 else
13981 else
13987 #if TARGET_MACHO
13992 #endif
13996 }
14001 }
14002 }
14004 /* This is called from dwarf2out.c via TARGET_ASM_OUTPUT_DWARF_DTPREL.
14005 We need to emit DTP-relative relocations. */
14007 static void ATTRIBUTE_UNUSED
14009 {
14014 {
14022 }
14023 }
14025 /* Return true if X is a representation of the PIC register. This copes
14026 with calls from ix86_find_base_term, where the register might have
14027 been replaced by a cselib value. */
14029 static bool
14031 {
14035 else
14037 }
14039 /* Helper function for ix86_delegitimize_address.
14040 Attempt to delegitimize TLS local-exec accesses. */
14042 static rtx
14044 {
14069 {
14074 }
14080 }
14082 /* In the name of slightly smaller debug output, and to cater to
14083 general assembler lossage, recognize PIC+GOTOFF and turn it back
14084 into a direct symbol reference.
14086 On Darwin, this is necessary to avoid a crash, because Darwin
14087 has a different PIC label for each routine but the DWARF debugging
14088 information is not associated with any particular routine, so it's
14089 necessary to remove references to the PIC label from RTL stored by
14090 the DWARF output code. */
14092 static rtx
14094 {
14096 /* addend is NULL or some rtx if x is something+GOTOFF where
14097 something doesn't include the PIC register. */
14099 /* reg_addend is NULL or a multiple of some register. */
14101 /* const_addend is NULL or a const_int. */
14103 /* This is the result, or NULL. */
14112 {
14119 {
14125 }
14132 {
14135 {
14140 }
14142 }
14147 /* Fall thru into the code shared with -m32 for -mcmodel=large -fpic
14148 and -mcmodel=medium -fpic. */
14149 }
14156 /* %ebx + GOT/GOTOFF */
14157 ;
14159 {
14160 /* %ebx + %reg * scale + GOT/GOTOFF */
14166 else
14167 {
14170 }
14171 }
14172 else
14178 {
14181 }
14202 {
14203 /* If the rest of original X doesn't involve the PIC register, add
14204 addend and subtract pic_offset_table_rtx. This can happen e.g.
14205 for code like:
14206 leal (%ebx, %ecx, 4), %ecx
14207 ...
14208 movl foo@GOTOFF(%ecx), %edx
14209 in which case we return (%ecx - %ebx) + foo. */
14212 pic_offset_table_rtx),
14213 result);
14214 else
14216 }
14218 {
14222 }
14224 }
14226 /* If X is a machine specific address (i.e. a symbol or label being
14227 referenced as a displacement from the GOT implemented using an
14228 UNSPEC), then return the base term. Otherwise return X. */
14230 rtx
14232 {
14233 rtx term;
14236 {
14250 }
14253 }
14254 \f
14255 static void
14258 {
14262 {
14265 }
14270 {
14273 {
14292 }
14296 {
14315 }
14322 /* ??? Use "nbe" instead of "a" for fcmov lossage on some assemblers.
14323 Those same assemblers have the same but opposite lossage on cmov. */
14326 else
14331 {
14344 }
14351 else
14356 {
14369 }
14376 else
14386 else
14397 }
14399 }
14401 /* Print the name of register X to FILE based on its machine mode and number.
14402 If CODE is 'w', pretend the mode is HImode.
14403 If CODE is 'b', pretend the mode is QImode.
14404 If CODE is 'k', pretend the mode is SImode.
14405 If CODE is 'q', pretend the mode is DImode.
14406 If CODE is 'x', pretend the mode is V4SFmode.
14407 If CODE is 't', pretend the mode is V8SFmode.
14408 If CODE is 'g', pretend the mode is V16SFmode.
14409 If CODE is 'h', pretend the reg is the 'high' byte register.
14410 If CODE is 'y', print "st(0)" instead of "st", if the reg is stack op.
14411 If CODE is 'd', duplicate the operand for AVX instruction.
14412 */
14414 void
14416 {
14425 {
14429 }
14456 else
14459 /* Irritatingly, AMD extended registers use different naming convention
14460 from the normal registers: "r%d[bwd]" */
14462 {
14467 {
14481 /* no suffix */
14486 }
14488 }
14492 {
14495 {
14498 }
14499 /* FALLTHRU */
14505 /* FALLTHRU */
14508 normal:
14523 {
14528 }
14531 {
14536 }
14540 }
14544 {
14547 else
14549 }
14550 }
14552 /* Locate some local-dynamic symbol still in use by this function
14553 so that we can print its name in some tls_local_dynamic_base
14554 pattern. */
14556 static int
14558 {
14563 {
14566 }
14569 }
14573 {
14574 rtx insn;
14585 }
14587 /* Meaning of CODE:
14588 L,W,B,Q,S,T -- print the opcode suffix for specified size of operand.
14589 C -- print opcode suffix for set/cmov insn.
14590 c -- like C, but print reversed condition
14591 F,f -- likewise, but for floating-point.
14592 O -- if HAVE_AS_IX86_CMOV_SUN_SYNTAX, expand to "w.", "l." or "q.",
14593 otherwise nothing
14594 R -- print the prefix for register names.
14595 z -- print the opcode suffix for the size of the current operand.
14596 Z -- likewise, with special suffixes for x87 instructions.
14597 * -- print a star (in certain assembler syntax)
14598 A -- print an absolute memory reference.
14599 E -- print address with DImode register names if TARGET_64BIT.
14600 w -- print the operand as if it's a "word" (HImode) even if it isn't.
14601 s -- print a shift double count, followed by the assemblers argument
14602 delimiter.
14603 b -- print the QImode name of the register for the indicated operand.
14604 %b0 would print %al if operands[0] is reg 0.
14605 w -- likewise, print the HImode name of the register.
14606 k -- likewise, print the SImode name of the register.
14607 q -- likewise, print the DImode name of the register.
14608 x -- likewise, print the V4SFmode name of the register.
14609 t -- likewise, print the V8SFmode name of the register.
14610 g -- likewise, print the V16SFmode name of the register.
14611 h -- print the QImode name for a "high" register, either ah, bh, ch or dh.
14612 y -- print "st(0)" instead of "st" as a register.
14613 d -- print duplicated register operand for AVX instruction.
14614 D -- print condition for SSE cmp instruction.
14615 P -- if PIC, print an @PLT suffix.
14616 p -- print raw symbol name.
14617 X -- don't print any sort of PIC '@' suffix for a symbol.
14618 & -- print some in-use local-dynamic symbol name.
14619 H -- print a memory address offset by 8; used for sse high-parts
14620 Y -- print condition for XOP pcom* instruction.
14621 + -- print a branch hint as 'cs' or 'ds' prefix
14622 ; -- print a semicolon (after prefixes due to bug in older gas).
14623 ~ -- print "i" if TARGET_AVX2, "f" otherwise.
14624 @ -- print a segment register of thread base pointer load
14625 ^ -- print addr32 prefix if TARGET_64BIT and Pmode != word_mode
14626 */
14628 void
14630 {
14632 {
14634 {
14637 {
14643 /* Intel syntax. For absolute addresses, registers should not
14644 be surrounded by braces. */
14646 {
14651 }
14656 }
14662 /* Wrap address in an UNSPEC to declare special handling. */
14700 #ifdef HAVE_AS_IX86_CMOV_SUN_SYNTAX
14705 {
14719 output_operand_lossage
14722 }
14725 #endif
14730 {
14731 /* Opcodes don't get size suffixes if using Intel opcodes. */
14736 {
14754 output_operand_lossage
14757 }
14758 }
14761 warning
14763 /* FALLTHRU */
14766 /* 387 opcodes don't get size suffixes if using Intel opcodes. */
14771 {
14773 {
14775 #ifdef HAVE_AS_IX86_FILDS
14777 #endif
14785 #ifdef HAVE_AS_IX86_FILDQ
14787 #else
14789 #endif
14794 }
14795 }
14797 {
14798 /* 387 opcodes don't get size suffixes
14799 if the operands are registers. */
14804 {
14820 }
14821 }
14822 else
14823 {
14824 output_operand_lossage
14827 }
14829 output_operand_lossage
14850 {
14853 }
14858 {
14909 }
14913 /* Little bit of braindamage here. The SSE compare instructions
14914 does use completely different names for the comparisons that the
14915 fp conditional moves. */
14917 {
14920 {
14923 }
14929 {
14932 }
14938 {
14941 }
14950 {
14953 }
14959 {
14962 }
14968 {
14971 }
14982 }
14987 #ifdef HAVE_AS_IX86_CMOV_SUN_SYNTAX
14990 #endif
14995 {
14999 }
15003 file);
15008 {
15012 }
15013 /* It doesn't actually matter what mode we use here, as we're
15014 only going to use this for printing. */
15016 /* Output 'qword ptr' for intel assembler dialect. */
15025 #ifdef HAVE_AS_IX86_HLE
15027 #else
15029 #endif
15031 #ifdef HAVE_AS_IX86_HLE
15033 #else
15035 #endif
15036 /* We do not want to print value of the operand. */
15050 {
15055 else
15058 }
15061 {
15062 rtx x;
15071 {
15076 {
15078 bool cputaken
15081 /* Emit hints only in the case default branch prediction
15082 heuristics would fail. */
15084 {
15085 /* We use 3e (DS) prefix for taken branches and
15086 2e (CS) prefix for not taken branches. */
15089 else
15091 }
15092 }
15093 }
15095 }
15098 #ifndef HAVE_AS_IX86_REP_LOCK_PREFIX
15100 #endif
15107 /* The kernel uses a different segment register for performance
15108 reasons; a system call would not have to trash the userspace
15109 segment register, which would be expensive. */
15112 else
15127 }
15128 }
15134 {
15135 /* No `byte ptr' prefix for call instructions or BLKmode operands. */
15138 {
15141 {
15150 else
15157 }
15159 /* Check for explicit size override (codes 'b', 'w', 'k',
15160 'q' and 'x') */
15174 }
15177 /* Avoid (%rip) for call operands. */
15183 else
15185 }
15188 {
15189 REAL_VALUE_TYPE r;
15197 /* Sign extend 32bit SFmode immediate to 8 bytes. */
15201 else
15203 }
15206 {
15207 REAL_VALUE_TYPE r;
15216 }
15218 /* These float cases don't actually occur as immediate operands. */
15220 {
15225 }
15227 else
15228 {
15229 /* We have patterns that allow zero sets of memory, for instance.
15230 In 64-bit mode, we should probably support all 8-byte vectors,
15231 since we can in fact encode that into an immediate. */
15233 {
15236 }
15239 {
15241 {
15244 }
15247 {
15250 else
15252 }
15253 }
15258 else
15260 }
15261 }
15263 static bool
15265 {
15268 }
15269 \f
15270 /* Print a memory operand whose address is ADDR. */
15272 static void
15274 {
15283 {
15290 }
15292 {
15296 }
15297 else
15308 {
15319 }
15321 /* Use one byte shorter RIP relative addressing for 64bit mode. */
15323 {
15335 }
15337 {
15338 /* Displacement only requires special attention. */
15341 {
15345 }
15348 else
15350 }
15351 else
15352 {
15353 /* Print SImode register names to force addr32 prefix. */
15355 {
15356 #ifdef ENABLE_CHECKING
15359 {
15370 }
15371 #endif
15374 }
15376 && TARGET_X32
15377 && disp
15380 {
15381 /* X32 runs in 64-bit mode, where displacement, DISP, in
15382 address DISP(%r64), is encoded as 32-bit immediate sign-
15383 extended from 32-bit to 64-bit. For -0x40000300(%r64),
15384 address is %r64 + 0xffffffffbffffd00. When %r64 <
15385 0x40000300, like 0x37ffe064, address is 0xfffffffff7ffdd64,
15386 which is invalid for x32. The correct address is %r64
15387 - 0x40000300 == 0xf7ffdd64. To properly encode
15388 -0x40000300(%r64) for x32, we zero-extend negative
15389 displacement by forcing addr32 prefix which truncates
15390 0xfffffffff7ffdd64 to 0xf7ffdd64. In theory, we should
15391 zero-extend all negative displacements, including -1(%rsp).
15392 However, for small negative displacements, sign-extension
15393 won't cause overflow. We only zero-extend negative
15394 displacements if they < -16*1024*1024, which is also used
15395 to check legitimate address displacements for PIC. */
15397 }
15400 {
15402 {
15407 else
15409 }
15415 {
15420 }
15422 }
15423 else
15424 {
15428 {
15429 /* Pull out the offset of a symbol; print any symbol itself. */
15433 {
15437 }
15445 else
15447 }
15451 {
15454 {
15458 }
15459 }
15462 else
15466 {
15471 }
15473 }
15474 }
15475 }
15477 /* Implementation of TARGET_ASM_OUTPUT_ADDR_CONST_EXTRA. */
15479 static bool
15481 {
15482 rtx op;
15489 {
15492 /* FIXME: This might be @TPOFF in Sun ld. */
15503 else
15515 else
15522 #if TARGET_MACHO
15528 #endif
15531 {
15536 #ifdef TARGET_THREAD_SPLIT_STACK_OFFSET
15538 #else
15540 #endif
15543 }
15548 }
15551 }
15552 \f
15553 /* Split one or more double-mode RTL references into pairs of half-mode
15554 references. The RTL can be REG, offsettable MEM, integer constant, or
15555 CONST_DOUBLE. "operands" is a pointer to an array of double-mode RTLs to
15556 split and "num" is its length. lo_half and hi_half are output arrays
15557 that parallel "operands". */
15559 void
15562 {
15567 {
15576 }
15581 {
15584 /* simplify_subreg refuse to split volatile memory addresses,
15585 but we still have to handle it. */
15587 {
15590 }
15591 else
15592 {
15599 }
15600 }
15601 }
15602 \f
15603 /* Output code to perform a 387 binary operation in INSN, one of PLUS,
15604 MINUS, MULT or DIV. OPERANDS are the insn operands, where operands[3]
15605 is the expression of the binary operation. The output may either be
15606 emitted here, or returned to the caller, like all output_* functions.
15608 There is no guarantee that the operands are the same mode, as they
15609 might be within FLOAT or FLOAT_EXTEND expressions. */
15611 #ifndef SYSV386_COMPAT
15612 /* Set to 1 for compatibility with brain-damaged assemblers. No-one
15613 wants to fix the assemblers because that causes incompatibility
15614 with gcc. No-one wants to fix gcc because that causes
15615 incompatibility with assemblers... You can use the option of
15616 -DSYSV386_COMPAT=0 if you recompile both gcc and gas this way. */
15617 #define SYSV386_COMPAT 1
15618 #endif
15622 {
15628 #ifdef ENABLE_CHECKING
15629 /* Even if we do not want to check the inputs, this documents input
15630 constraints. Which helps in understanding the following code. */
15640 else
15642 #endif
15645 {
15650 else
15659 else
15668 else
15677 else
15684 }
15687 {
15689 {
15693 else
15695 }
15696 else
15697 {
15701 else
15703 }
15705 }
15709 {
15713 {
15717 }
15719 /* know operands[0] == operands[1]. */
15722 {
15725 }
15728 {
15730 /* How is it that we are storing to a dead operand[2]?
15731 Well, presumably operands[1] is dead too. We can't
15732 store the result to st(0) as st(0) gets popped on this
15733 instruction. Instead store to operands[2] (which I
15734 think has to be st(1)). st(1) will be popped later.
15735 gcc <= 2.8.1 didn't have this check and generated
15736 assembly code that the Unixware assembler rejected. */
15738 else
15741 }
15745 else
15752 {
15755 }
15758 {
15761 }
15764 {
15765 #if SYSV386_COMPAT
15766 /* The SystemV/386 SVR3.2 assembler, and probably all AT&T
15767 derived assemblers, confusingly reverse the direction of
15768 the operation for fsub{r} and fdiv{r} when the
15769 destination register is not st(0). The Intel assembler
15770 doesn't have this brain damage. Read !SYSV386_COMPAT to
15771 figure out what the hardware really does. */
15774 else
15776 #else
15778 /* As above for fmul/fadd, we can't store to st(0). */
15780 else
15782 #endif
15784 }
15787 {
15788 #if SYSV386_COMPAT
15791 else
15793 #else
15796 else
15798 #endif
15800 }
15803 {
15806 else
15809 }
15811 {
15812 #if SYSV386_COMPAT
15814 #else
15816 #endif
15817 }
15818 else
15819 {
15820 #if SYSV386_COMPAT
15822 #else
15824 #endif
15825 }
15830 }
15834 }
15836 /* Check if a 256bit AVX register is referenced inside of EXP. */
15838 static int
15840 {
15851 }
15853 /* Return needed mode for entity in optimize_mode_switching pass. */
15855 static int
15857 {
15859 {
15860 rtx link;
15862 /* Needed mode is set to AVX_U128_CLEAN if there are
15863 no 256bit modes used in function arguments. */
15865 link;
15867 {
15869 {
15874 }
15875 }
15878 }
15880 /* Require DIRTY mode if a 256bit AVX register is referenced. Hardware
15881 changes state only when a 256bit register is written to, but we need
15882 to prevent the compiler from moving optimal insertion point above
15883 eventual read from 256bit register. */
15888 }
15890 /* Return mode that i387 must be switched into
15891 prior to the execution of insn. */
15893 static int
15895 {
15898 /* The mode UNINITIALIZED is used to store control word after a
15899 function call or ASM pattern. The mode ANY specify that function
15900 has no requirements on the control word and make no changes in the
15901 bits we are interested in. */
15915 {
15938 }
15941 }
15943 /* Return mode that entity must be switched into
15944 prior to the execution of insn. */
15946 int
15948 {
15950 {
15960 }
15962 }
15964 /* Check if a 256bit AVX register is referenced in stores. */
15966 static void
15968 {
15970 {
15973 }
15974 }
15976 /* Calculate mode of upper 128bit AVX registers after the insn. */
15978 static int
15980 {
15987 /* We know that state is clean after CALL insn if there are no
15988 256bit registers used in the function return register. */
15990 {
15995 }
15997 /* Otherwise, return current mode. Remember that if insn
15998 references AVX 256bit registers, the mode was already changed
15999 to DIRTY from MODE_NEEDED. */
16001 }
16003 /* Return the mode that an insn results in. */
16005 int
16007 {
16009 {
16019 }
16020 }
16022 static int
16024 {
16025 tree arg;
16027 /* Entry mode is set to AVX_U128_DIRTY if there are
16028 256bit modes used in function arguments. */
16031 {
16036 }
16039 }
16041 /* Return a mode that ENTITY is assumed to be
16042 switched to at function entry. */
16044 int
16046 {
16048 {
16058 }
16059 }
16061 static int
16063 {
16066 /* Exit mode is set to AVX_U128_DIRTY if there are
16067 256bit modes used in the function return register. */
16072 }
16074 /* Return a mode that ENTITY is assumed to be
16075 switched to at function exit. */
16077 int
16079 {
16081 {
16091 }
16092 }
16094 /* Output code to initialize control word copies used by trunc?f?i and
16095 rounding patterns. CURRENT_MODE is set to current control word,
16096 while NEW_MODE is set to new control word. */
16098 static void
16100 {
16102 rtx new_mode;
16113 {
16115 {
16117 /* round toward zero (truncate) */
16123 /* round down toward -oo */
16130 /* round up toward +oo */
16137 /* mask precision exception for nearbyint() */
16144 }
16145 }
16146 else
16147 {
16149 {
16151 /* round toward zero (truncate) */
16157 /* round down toward -oo */
16163 /* round up toward +oo */
16169 /* mask precision exception for nearbyint() */
16176 }
16177 }
16183 }
16185 /* Emit vzeroupper. */
16187 void
16189 {
16192 /* Cancel automatic vzeroupper insertion if there are
16193 live call-saved SSE registers at the insertion point. */
16205 }
16207 /* Generate one or more insns to set ENTITY to MODE. */
16209 void
16211 {
16213 {
16228 }
16229 }
16231 /* Output code for INSN to convert a float to a signed int. OPERANDS
16232 are the insn operands. The output may be [HSD]Imode and the input
16233 operand may be [SDX]Fmode. */
16237 {
16242 /* Jump through a hoop or two for DImode, since the hardware has no
16243 non-popping instruction. We used to do this a different way, but
16244 that was somewhat fragile and broke with post-reload splitters. */
16254 else
16255 {
16260 else
16264 }
16267 }
16269 /* Output code for x87 ffreep insn. The OPNO argument, which may only
16270 have the values zero or one, indicates the ffreep insn's operand
16271 from the OPERANDS array. */
16275 {
16277 #ifdef HAVE_AS_IX86_FFREEP
16279 #else
16280 {
16290 }
16291 #endif
16294 }
16297 /* Output code for INSN to compare OPERANDS. EFLAGS_P is 1 when fcomi
16298 should be used. UNORDERED_P is true when fucom should be used. */
16302 {
16308 {
16311 }
16312 else
16313 {
16316 }
16319 {
16323 else
16325 else
16328 else
16330 }
16337 {
16339 {
16342 }
16343 else
16345 }
16348 && stack_top_dies
16351 {
16352 /* If both the top of the 387 stack dies, and the other operand
16353 is also a stack register that dies, then this must be a
16354 `fcompp' float compare */
16357 {
16358 /* There is no double popping fcomi variant. Fortunately,
16359 eflags is immune from the fstp's cc clobbering. */
16362 else
16365 }
16366 else
16367 {
16370 else
16372 }
16373 }
16374 else
16375 {
16376 /* Encoded here as eflags_p | intmode | unordered_p | stack_top_dies. */
16379 {
16387 NULL,
16388 NULL,
16395 NULL,
16396 NULL,
16397 NULL,
16398 NULL
16399 };
16414 }
16415 }
16417 void
16419 {
16422 #ifdef ASM_QUAD
16425 #else
16427 #endif
16430 }
16432 void
16434 {
16437 #ifdef ASM_QUAD
16440 #else
16442 #endif
16443 /* We can't use @GOTOFF for text labels on VxWorks; see gotoff_operand. */
16449 #if TARGET_MACHO
16451 {
16455 }
16456 #endif
16457 else
16460 }
16461 \f
16462 /* Generate either "mov $0, reg" or "xor reg, reg", as appropriate
16463 for the target. */
16465 void
16467 {
16468 rtx tmp;
16470 /* We play register width games, which are only valid after reload. */
16473 /* Avoid HImode and its attendant prefix byte. */
16478 /* This predicate should match that for movsi_xor and movdi_xor_rex64. */
16480 {
16483 }
16486 }
16488 /* X is an unchanging MEM. If it is a constant pool reference, return
16489 the constant pool rtx, else NULL. */
16491 rtx
16493 {
16500 }
16502 void
16504 {
16512 {
16513 rtx tmp;
16517 {
16523 }
16526 }
16530 {
16533 rtx tmp;
16538 else
16542 {
16549 }
16550 }
16554 {
16556 {
16557 #if TARGET_MACHO
16558 /* dynamic-no-pic */
16560 {
16561 rtx temp = ((reload_in_progress
16569 }
16571 {
16575 }
16578 else
16579 {
16587 }
16588 /* dynamic-no-pic */
16589 #endif
16590 }
16591 else
16592 {
16596 {
16602 }
16603 }
16604 }
16605 else
16606 {
16617 /* Force large constants in 64bit compilation into register
16618 to get them CSEed. */
16630 {
16631 /* If we are loading a floating point constant to a register,
16632 force the value to memory now, since we'll get better code
16633 out the back end. */
16637 {
16642 }
16643 }
16644 }
16647 }
16649 void
16651 {
16655 /* Force constants other than zero into memory. We do not know how
16656 the instructions used to build constants modify the upper 64 bits
16657 of the register, once we have that information we may be able
16658 to handle some of them more efficiently. */
16667 /* We need to check memory alignment for SSE mode since attribute
16668 can make operands unaligned. */
16673 {
16676 /* ix86_expand_vector_move_misalign() does not like constants ... */
16682 /* ... nor both arguments in memory. */
16690 }
16692 /* Make operand1 a register if it isn't already. */
16696 {
16699 }
16702 }
16704 /* Split 32-byte AVX unaligned load and store if needed. */
16706 static void
16708 {
16709 rtx m;
16716 {
16737 }
16740 {
16742 {
16749 }
16750 /* Normal *mov<mode>_internal pattern will handle
16751 unaligned loads just fine if misaligned_operand
16752 is true, and without the UNSPEC it can be combined
16753 with arithmetic instructions. */
16756 else
16758 }
16760 {
16762 {
16767 }
16768 else
16770 }
16771 else
16773 }
16775 /* Implement the movmisalign patterns for SSE. Non-SSE modes go
16776 straight to ix86_expand_vector_move. */
16777 /* Code generation for scalar reg-reg moves of single and double precision data:
16778 if (x86_sse_partial_reg_dependency == true | x86_sse_split_regs == true)
16779 movaps reg, reg
16780 else
16781 movss reg, reg
16782 if (x86_sse_partial_reg_dependency == true)
16783 movapd reg, reg
16784 else
16785 movsd reg, reg
16787 Code generation for scalar loads of double precision data:
16788 if (x86_sse_split_regs == true)
16789 movlpd mem, reg (gas syntax)
16790 else
16791 movsd mem, reg
16793 Code generation for unaligned packed loads of single precision data
16794 (x86_sse_unaligned_move_optimal overrides x86_sse_partial_reg_dependency):
16795 if (x86_sse_unaligned_move_optimal)
16796 movups mem, reg
16798 if (x86_sse_partial_reg_dependency == true)
16799 {
16800 xorps reg, reg
16801 movlps mem, reg
16802 movhps mem+8, reg
16803 }
16804 else
16805 {
16806 movlps mem, reg
16807 movhps mem+8, reg
16808 }
16810 Code generation for unaligned packed loads of double precision data
16811 (x86_sse_unaligned_move_optimal overrides x86_sse_split_regs):
16812 if (x86_sse_unaligned_move_optimal)
16813 movupd mem, reg
16815 if (x86_sse_split_regs == true)
16816 {
16817 movlpd mem, reg
16818 movhpd mem+8, reg
16819 }
16820 else
16821 {
16822 movsd mem, reg
16823 movhpd mem+8, reg
16824 }
16825 */
16827 void
16829 {
16838 {
16840 {
16844 {
16846 {
16849 }
16850 else
16852 }
16854 /* FALLTHRU */
16858 {
16873 }
16879 else
16887 }
16890 }
16894 {
16896 {
16900 {
16902 {
16905 }
16906 else
16908 }
16910 /* FALLTHRU */
16920 }
16923 }
16926 {
16927 /* Normal *mov<mode>_internal pattern will handle
16928 unaligned loads just fine if misaligned_operand
16929 is true, and without the UNSPEC it can be combined
16930 with arithmetic instructions. */
16936 /* ??? If we have typed data, then it would appear that using
16937 movdqu is the only way to get unaligned data loaded with
16938 integer type. */
16940 {
16942 {
16945 }
16947 /* We will eventually emit movups based on insn attributes. */
16951 }
16953 {
16954 rtx zero;
16957 || TARGET_SSE_UNALIGNED_LOAD_OPTIMAL
16958 || TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL
16960 {
16961 /* We will eventually emit movups based on insn attributes. */
16964 }
16966 /* When SSE registers are split into halves, we can avoid
16967 writing to the top half twice. */
16969 {
16972 }
16973 else
16974 {
16975 /* ??? Not sure about the best option for the Intel chips.
16976 The following would seem to satisfy; the register is
16977 entirely cleared, breaking the dependency chain. We
16978 then store to the upper half, with a dependency depth
16979 of one. A rumor has it that Intel recommends two movsd
16980 followed by an unpacklpd, but this is unconfirmed. And
16981 given that the dependency depth of the unpacklpd would
16982 still be one, I'm not sure why this would be better. */
16984 }
16990 }
16991 else
16992 {
16993 rtx t;
16996 || TARGET_SSE_UNALIGNED_LOAD_OPTIMAL
16997 || TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL
16999 {
17001 {
17004 }
17011 }
17015 else
17020 else
17029 }
17030 }
17032 {
17034 {
17037 /* We will eventually emit movups based on insn attributes. */
17039 }
17041 {
17043 || TARGET_SSE_UNALIGNED_STORE_OPTIMAL
17044 || TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL
17046 /* We will eventually emit movups based on insn attributes. */
17048 else
17049 {
17054 }
17055 }
17056 else
17057 {
17062 || TARGET_SSE_UNALIGNED_STORE_OPTIMAL
17063 || TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL
17065 {
17068 }
17069 else
17070 {
17075 }
17076 }
17077 }
17078 else
17080 }
17082 /* Expand a push in MODE. This is some mode for which we do not support
17083 proper push instructions, at least from the registers that we expect
17084 the value to live in. */
17086 void
17088 {
17089 rtx tmp;
17099 /* When we push an operand onto stack, it has to be aligned at least
17100 at the function argument boundary. However since we don't have
17101 the argument type, we can't determine the actual argument
17102 boundary. */
17104 }
17106 /* Helper function of ix86_fixup_binary_operands to canonicalize
17107 operand order. Returns true if the operands should be swapped. */
17109 static bool
17111 rtx operands[])
17112 {
17117 /* If the operation is not commutative, we can't do anything. */
17121 /* Highest priority is that src1 should match dst. */
17127 /* Next highest priority is that immediate constants come second. */
17133 /* Lowest priority is that memory references should come second. */
17140 }
17143 /* Fix up OPERANDS to satisfy ix86_binary_operator_ok. Return the
17144 destination to use for the operation. If different from the true
17145 destination in operands[0], a copy operation will be required. */
17147 rtx
17149 rtx operands[])
17150 {
17155 /* Canonicalize operand order. */
17157 {
17158 rtx temp;
17160 /* It is invalid to swap operands of different modes. */
17166 }
17168 /* Both source operands cannot be in memory. */
17170 {
17171 /* Optimization: Only read from memory once. */
17173 {
17176 }
17179 else
17181 }
17183 /* If the destination is memory, and we do not have matching source
17184 operands, do things in registers. */
17188 /* Source 1 cannot be a constant. */
17192 /* Source 1 cannot be a non-matching memory. */
17196 /* Improve address combine. */
17205 }
17207 /* Similarly, but assume that the destination has already been
17208 set up properly. */
17210 void
17213 {
17216 }
17218 /* Attempt to expand a binary operator. Make the expansion closer to the
17219 actual machine, then just general_operand, which will allow 3 separate
17220 memory references (one output, two input) in a single insn. */
17222 void
17224 rtx operands[])
17225 {
17232 /* Emit the instruction. */
17236 {
17237 /* Reload doesn't know about the flags register, and doesn't know that
17238 it doesn't want to clobber it. We can only do this with PLUS. */
17241 }
17245 {
17246 /* This is going to be an LEA; avoid splitting it later. */
17248 }
17249 else
17250 {
17253 }
17255 /* Fix up the destination if needed. */
17258 }
17260 /* Expand vector logical operation CODE (AND, IOR, XOR) in MODE with
17261 the given OPERANDS. */
17263 void
17265 rtx operands[])
17266 {
17269 {
17272 }
17274 {
17277 }
17278 /* Optimize (__m128i) d | (__m128i) e and similar code
17279 when d and e are float vectors into float vector logical
17280 insn. In C/C++ without using intrinsics there is no other way
17281 to express vector logical operation on float vectors than
17282 to cast them temporarily to integer vectors. */
17284 && !TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL
17293 {
17294 rtx dst;
17296 {
17303 {
17306 }
17307 else
17308 {
17313 }
17324 }
17325 }
17334 }
17336 /* Return TRUE or FALSE depending on whether the binary operator meets the
17337 appropriate constraints. */
17339 bool
17342 {
17347 /* Both source operands cannot be in memory. */
17351 /* Canonicalize operand order for commutative operators. */
17353 {
17357 }
17359 /* If the destination is memory, we must have a matching source operand. */
17363 /* Source 1 cannot be a constant. */
17367 /* Source 1 cannot be a non-matching memory. */
17369 /* Support "andhi/andsi/anddi" as a zero-extending move. */
17377 }
17379 /* Attempt to expand a unary operator. Make the expansion closer to the
17380 actual machine, then just general_operand, which will allow 2 separate
17381 memory references (one output, one input) in a single insn. */
17383 void
17385 rtx operands[])
17386 {
17393 /* If the destination is memory, and we do not have matching source
17394 operands, do things in registers. */
17397 {
17400 else
17402 }
17404 /* When source operand is memory, destination must match. */
17408 /* Emit the instruction. */
17412 {
17413 /* Reload doesn't know about the flags register, and doesn't know that
17414 it doesn't want to clobber it. */
17417 }
17418 else
17419 {
17422 }
17424 /* Fix up the destination if needed. */
17427 }
17429 /* Split 32bit/64bit divmod with 8bit unsigned divmod if dividend and
17430 divisor are within the range [0-255]. */
17432 void
17435 {
17444 {
17457 }
17464 /* Use 8bit unsigned divimod if dividend and divisor are within
17465 the range [0-255]. */
17474 pc_rtx);
17479 /* Generate original signed/unsigned divimod. */
17484 /* Branch to the end. */
17488 /* Generate 8bit unsigned divide. */
17490 /* Don't use operands[0] for result of 8bit divide since not all
17491 registers support QImode ZERO_EXTRACT. */
17498 {
17501 }
17502 else
17503 {
17506 }
17508 /* Extract remainder from AH. */
17512 else
17513 {
17514 /* Need a new scratch register since the old one has result
17515 of 8bit divide. */
17519 }
17522 /* Zero extend quotient from AL. */
17528 }
17530 /* Whether it is OK to emit CFI directives when emitting asm code. */
17532 bool
17534 {
17536 }
17538 #define LEA_MAX_STALL (3)
17539 #define LEA_SEARCH_THRESHOLD (LEA_MAX_STALL << 1)
17541 /* Increase given DISTANCE in half-cycles according to
17542 dependencies between PREV and NEXT instructions.
17543 Add 1 half-cycle if there is no dependency and
17544 go to next cycle if there is some dependecy. */
17546 static unsigned int
17548 {
17565 }
17567 /* Function checks if instruction INSN defines register number
17568 REGNO1 or REGNO2. */
17570 static bool
17572 rtx insn)
17573 {
17581 {
17583 }
17586 }
17588 /* Function checks if instruction INSN uses register number
17589 REGNO as a part of address expression. */
17591 static bool
17593 {
17601 }
17603 /* Search backward for non-agu definition of register number REGNO1
17604 or register number REGNO2 in basic block starting from instruction
17605 START up to head of basic block or instruction INSN.
17607 Function puts true value into *FOUND var if definition was found
17608 and false otherwise.
17610 Distance in half-cycles between START and found instruction or head
17611 of BB is added to DISTANCE and returned. */
17613 static int
17617 {
17627 {
17629 {
17632 {
17635 {
17638 }
17639 }
17642 }
17647 }
17650 }
17652 /* Search backward for non-agu definition of register number REGNO1
17653 or register number REGNO2 in INSN's basic block until
17654 1. Pass LEA_SEARCH_THRESHOLD instructions, or
17655 2. Reach neighbour BBs boundary, or
17656 3. Reach agu definition.
17657 Returns the distance between the non-agu definition point and INSN.
17658 If no definition point, returns -1. */
17660 static int
17662 rtx insn)
17663 {
17674 {
17675 edge e;
17676 edge_iterator ei;
17681 {
17684 }
17690 else
17691 {
17696 {
17697 int bb_dist
17703 {
17710 }
17711 }
17714 }
17715 }
17717 /* get_attr_type may modify recog data. We want to make sure
17718 that recog data is valid for instruction INSN, on which
17719 distance_non_agu_define is called. INSN is unchanged here. */
17726 }
17728 /* Return the distance in half-cycles between INSN and the next
17729 insn that uses register number REGNO in memory address added
17730 to DISTANCE. Return -1 if REGNO0 is set.
17732 Put true value into *FOUND if register usage was found and
17733 false otherwise.
17734 Put true value into *REDEFINED if register redefinition was
17735 found and false otherwise. */
17737 static int
17741 {
17750 {
17753 /* If insn and start belong to the same bb, set prev to insn,
17754 so the call to increase_distance will increase the distance
17755 between insns by 1. */
17757 }
17762 {
17764 {
17767 {
17768 /* Return DISTANCE if OP0 is used in memory
17769 address in NEXT. */
17772 }
17775 {
17776 /* Return -1 if OP0 is set in NEXT. */
17779 }
17782 }
17788 }
17791 }
17793 /* Return the distance between INSN and the next insn that uses
17794 register number REGNO0 in memory address. Return -1 if no such
17795 a use is found within LEA_SEARCH_THRESHOLD or REGNO0 is set. */
17797 static int
17799 {
17811 {
17812 edge e;
17813 edge_iterator ei;
17818 {
17821 }
17827 else
17828 {
17834 {
17835 int bb_dist
17840 {
17847 }
17848 }
17851 }
17852 }
17858 }
17860 /* Define this macro to tune LEA priority vs ADD, it take effect when
17861 there is a dilemma of choicing LEA or ADD
17862 Negative value: ADD is more preferred than LEA
17863 Zero: Netrual
17864 Positive value: LEA is more preferred than ADD*/
17865 #define IX86_LEA_PRIORITY 0
17867 /* Return true if usage of lea INSN has performance advantage
17868 over a sequence of instructions. Instructions sequence has
17869 SPLIT_COST cycles higher latency than lea latency. */
17871 static bool
17874 {
17877 /* For Silvermont if using a 2-source or 3-source LEA for
17878 non-destructive destination purposes, or due to wanting
17879 ability to use SCALE, the use of LEA is justified. */
17881 {
17889 }
17895 {
17896 /* If there is no non AGU operand definition, no AGU
17897 operand usage and split cost is 0 then both lea
17898 and non lea variants have same priority. Currently
17899 we prefer lea for 64 bit code and non lea on 32 bit
17900 code. */
17903 else
17905 }
17907 /* With longer definitions distance lea is more preferable.
17908 Here we change it to take into account splitting cost and
17909 lea priority. */
17912 /* If there is no use in memory addess then we just check
17913 that split cost exceeds AGU stall. */
17917 /* If this insn has both backward non-agu dependence and forward
17918 agu dependence, the one with short distance takes effect. */
17920 }
17922 /* Return true if it is legal to clobber flags by INSN and
17923 false otherwise. */
17925 static bool
17927 {
17930 bitmap live;
17933 {
17935 {
17942 }
17948 }
17952 }
17954 /* Return true if we need to split op0 = op1 + op2 into a sequence of
17955 move and add to avoid AGU stalls. */
17957 bool
17959 {
17962 /* Check if we need to optimize. */
17966 /* Check it is correct to split here. */
17974 /* We need to split only adds with non destructive
17975 destination operand. */
17978 else
17980 }
17982 /* Return true if we should emit lea instruction instead of mov
17983 instruction. */
17985 bool
17987 {
17990 /* Check if we need to optimize. */
17994 /* Use lea for reg to reg moves only. */
18002 }
18004 /* Return true if we need to split lea into a sequence of
18005 instructions to avoid AGU stalls. */
18007 bool
18009 {
18015 /* Check we need to optimize. */
18019 /* Check it is correct to split here. */
18026 /* There should be at least two components in the address. */
18031 /* We should not split into add if non legitimate pic
18032 operand is used as displacement. */
18047 /* Compute how many cycles we will add to execution time
18048 if split lea into a sequence of instructions. */
18050 {
18051 /* Have to use mov instruction if non desctructive
18052 destination form is used. */
18056 /* Have to add index to base if both exist. */
18060 /* Have to use shift and adds if scale is 2 or greater. */
18062 {
18067 else
18069 }
18071 /* Have to use add instruction with immediate if
18072 disp is non zero. */
18076 /* Subtract the price of lea. */
18078 }
18082 }
18084 /* Emit x86 binary operand CODE in mode MODE, where the first operand
18085 matches destination. RTX includes clobber of FLAGS_REG. */
18087 static void
18090 {
18097 }
18099 /* Return true if regno1 def is nearest to the insn. */
18101 static bool
18103 {
18110 {
18112 {
18115 }
18121 }
18123 /* None of the regs is defined in the bb. */
18125 }
18127 /* Split lea instructions into a sequence of instructions
18128 which are executed on ALU to avoid AGU stalls.
18129 It is assumed that it is allowed to clobber flags register
18130 at lea position. */
18132 void
18134 {
18150 {
18153 }
18156 {
18159 }
18165 {
18166 /* Case r1 = r1 + ... */
18168 {
18169 /* If we have a case r1 = r1 + C * r1 then we
18170 should use multiplication which is very
18171 expensive. Assume cost model is wrong if we
18172 have such case here. */
18177 }
18178 else
18179 {
18180 /* r1 = r2 + r3 * C case. Need to move r3 into r1. */
18184 /* Use shift for scaling. */
18193 }
18194 }
18196 {
18199 }
18200 else
18201 {
18203 {
18206 }
18208 {
18211 }
18212 else
18213 {
18218 else
18219 {
18220 rtx tmp1;
18222 /* Find better operand for SET instruction, depending
18223 on which definition is farther from the insn. */
18226 else
18236 }
18239 }
18243 }
18244 }
18246 /* Return true if it is ok to optimize an ADD operation to LEA
18247 operation to avoid flag register consumation. For most processors,
18248 ADD is faster than LEA. For the processors like ATOM, if the
18249 destination register of LEA holds an actual address which will be
18250 used soon, LEA is better and otherwise ADD is better. */
18252 bool
18254 {
18259 /* If a = b + c, (a!=b && a!=c), must use lea form. */
18267 }
18269 /* Return true if destination reg of SET_BODY is shift count of
18270 USE_BODY. */
18272 static bool
18274 {
18275 rtx set_dest;
18276 rtx shift_rtx;
18279 /* Retrieve destination of SET_BODY. */
18281 {
18290 use_body))
18295 }
18297 /* Retrieve shift count of USE_BODY. */
18299 {
18311 }
18319 {
18322 /* Return true if shift count is dest of SET_BODY. */
18324 {
18325 /* Add check since it can be invoked before register
18326 allocation in pre-reload schedule. */
18332 }
18333 }
18336 }
18338 /* Return true if destination reg of SET_INSN is shift count of
18339 USE_INSN. */
18341 bool
18343 {
18346 }
18348 /* Return TRUE or FALSE depending on whether the unary operator meets the
18349 appropriate constraints. */
18351 bool
18355 {
18356 /* If one of operands is memory, source and destination must match. */
18362 }
18364 /* Return TRUE if the operands to a vec_interleave_{high,low}v2df
18365 are ok, keeping in mind the possible movddup alternative. */
18367 bool
18369 {
18375 }
18377 /* Post-reload splitter for converting an SF or DFmode value in an
18378 SSE register into an unsigned SImode. */
18380 void
18382 {
18393 /* Load up the value into the low element. We must ensure that the other
18394 elements are valid floats -- zero is the easiest such value. */
18396 {
18399 else
18401 }
18402 else
18403 {
18408 else
18410 }
18430 else
18435 }
18437 /* Convert an unsigned DImode value into a DFmode, using only SSE.
18438 Expects the 64-bit DImode to be supplied in a pair of integral
18439 registers. Requires SSE2; will use SSE3 if available. For x86_32,
18440 -mfpmath=sse, !optimize_size only. */
18442 void
18444 {
18448 rtx x;
18454 {
18457 }
18458 else
18459 {
18463 }
18471 /* int_xmm = {0x45300000UL, fp_xmm/hi, 0x43300000, fp_xmm/lo } */
18474 /* Concatenating (juxtaposing) (0x43300000UL ## fp_value_low_xmm)
18475 yields a valid DF value equal to (0x1.0p52 + double(fp_value_lo_xmm)).
18476 Similarly (0x45300000UL ## fp_value_hi_xmm) yields
18477 (0x1.0p84 + double(fp_value_hi_xmm)).
18478 Note these exponents differ by 32. */
18482 /* Subtract off those 0x1.0p52 and 0x1.0p84 biases, to produce values
18483 in [0,2**32-1] and [0]+[2**32,2**64-1] respectively. */
18492 /* Add the upper and lower DFmode values together. */
18495 else
18496 {
18500 }
18503 }
18505 /* Not used, but eases macroization of patterns. */
18506 void
18508 rtx input ATTRIBUTE_UNUSED)
18509 {
18511 }
18513 /* Convert an unsigned SImode value into a DFmode. Only currently used
18514 for SSE, but applicable anywhere. */
18516 void
18518 {
18519 REAL_VALUE_TYPE TWO31r;
18534 }
18536 /* Convert a signed DImode value into a DFmode. Only used for SSE in
18537 32-bit mode; otherwise we have a direct convert instruction. */
18539 void
18541 {
18542 REAL_VALUE_TYPE TWO32r;
18560 }
18562 /* Convert an unsigned SImode value into a SFmode, using only SSE.
18563 For x86_32, -mfpmath=sse, !optimize_size only. */
18564 void
18566 {
18567 REAL_VALUE_TYPE ONE16r;
18586 }
18588 /* floatunsv{4,8}siv{4,8}sf2 expander. Expand code to convert
18589 a vector of unsigned ints VAL to vector of floats TARGET. */
18591 void
18593 {
18595 REAL_VALUE_TYPE TWO16r;
18602 else
18607 OPTAB_DIRECT);
18618 OPTAB_DIRECT);
18620 OPTAB_DIRECT);
18623 }
18625 /* Adjust a V*SFmode/V*DFmode value VAL so that *sfix_trunc* resp. fix_trunc*
18626 pattern can be used on it instead of *ufix_trunc* resp. fixuns_trunc*.
18627 This is done by doing just signed conversion if < 0x1p31, and otherwise by
18628 subtracting 0x1p31 first and xoring in 0x80000000 from *XORP afterwards. */
18630 rtx
18632 {
18633 REAL_VALUE_TYPE TWO31r;
18648 {
18654 }
18663 OPTAB_DIRECT);
18664 else
18665 {
18671 OPTAB_DIRECT);
18672 }
18675 }
18677 /* A subroutine of ix86_build_signbit_mask. If VECT is true,
18678 then replicate the value for all elements of the vector
18679 register. */
18681 rtx
18683 {
18685 rtvec v;
18689 {
18716 }
18717 }
18719 /* A subroutine of ix86_expand_fp_absneg_operator, copysign expanders
18720 and ix86_expand_int_vcond. Create a mask for the sign bit in MODE
18721 for an SSE register. If VECT is true, then replicate the mask for
18722 all elements of the vector register. If INVERT is true, then create
18723 a mask excluding the sign bit. */
18725 rtx
18727 {
18731 rtx v;
18732 rtx mask;
18734 /* Find the sign bit, sign extended to 2*HWI. */
18736 {
18756 else
18764 {
18767 }
18768 else
18769 {
18770 rtvec vec;
18776 {
18779 }
18780 else
18790 }
18795 }
18800 /* Force this value into the low part of a fp vector constant. */
18809 }
18811 /* Generate code for floating point ABS or NEG. */
18813 void
18815 rtx operands[])
18816 {
18827 {
18833 }
18835 /* NEG and ABS performed with SSE use bitwise mask operations.
18836 Create the appropriate mask now. */
18839 else
18849 {
18851 rtvec par;
18856 else
18857 {
18860 }
18862 }
18863 else
18865 }
18867 /* Expand a copysign operation. Special case operand 0 being a constant. */
18869 void
18871 {
18885 else
18889 {
18896 {
18899 else
18900 {
18904 }
18905 }
18915 else
18919 }
18920 else
18921 {
18931 else
18935 }
18936 }
18938 /* Deconstruct a copysign operation into bit masks. Operand 0 is known to
18939 be a constant, and so has already been expanded into a vector constant. */
18941 void
18943 {
18959 {
18962 }
18963 }
18965 /* Deconstruct a copysign operation into bit masks. Operand 0 is variable,
18966 so we have to do two masks. */
18968 void
18970 {
18985 {
18986 /* Shouldn't happen often (it's useless, obviously), but when it does
18987 we'd generate incorrect code if we continue below. */
18990 }
18993 {
19004 }
19005 else
19006 {
19008 {
19010 }
19012 {
19016 }
19020 {
19023 }
19025 {
19030 }
19032 }
19036 }
19038 /* Return TRUE or FALSE depending on whether the first SET in INSN
19039 has source and destination with matching CC modes, and that the
19040 CC mode is at least as constrained as REQ_MODE. */
19042 bool
19044 {
19045 rtx set;
19056 {
19066 /* FALLTHRU */
19070 /* FALLTHRU */
19074 /* FALLTHRU */
19088 }
19091 }
19093 /* Generate insn patterns to do an integer compare of OPERANDS. */
19095 static rtx
19097 {
19104 /* This is very simple, but making the interface the same as in the
19105 FP case makes the rest of the code easier. */
19109 /* Return the test that should be put into the flags user, i.e.
19110 the bcc, scc, or cmov instruction. */
19112 }
19114 /* Figure out whether to use ordered or unordered fp comparisons.
19115 Return the appropriate mode to use. */
19117 enum machine_mode
19119 {
19120 /* ??? In order to make all comparisons reversible, we do all comparisons
19121 non-trapping when compiling for IEEE. Once gcc is able to distinguish
19122 all forms trapping and nontrapping comparisons, we can make inequality
19123 comparisons trapping again, since it results in better code when using
19124 FCOM based compares. */
19126 }
19128 enum machine_mode
19130 {
19134 {
19137 }
19140 {
19141 /* Only zero flag is needed. */
19145 /* Codes needing carry flag. */
19148 /* Detect overflow checks. They need just the carry flag. */
19152 else
19157 /* Codes possibly doable only with sign flag when
19158 comparing against zero. */
19163 else
19164 /* For other cases Carry flag is not required. */
19166 /* Codes doable only with sign flag when comparing
19167 against zero, but we miss jump instruction for it
19168 so we need to use relational tests against overflow
19169 that thus needs to be zero. */
19174 else
19176 /* strcmp pattern do (use flags) and combine may ask us for proper
19177 mode. */
19182 }
19183 }
19185 /* Return the fixed registers used for condition codes. */
19187 static bool
19189 {
19193 }
19195 /* If two condition code modes are compatible, return a condition code
19196 mode which is compatible with both. Otherwise, return
19197 VOIDmode. */
19199 static enum machine_mode
19201 {
19218 {
19232 {
19246 }
19250 /* These are only compatible with themselves, which we already
19251 checked above. */
19253 }
19254 }
19257 /* Return a comparison we can do and that it is equivalent to
19258 swap_condition (code) apart possibly from orderedness.
19259 But, never change orderedness if TARGET_IEEE_FP, returning
19260 UNKNOWN in that case if necessary. */
19262 static enum rtx_code
19264 {
19266 {
19277 }
19278 }
19280 /* Return cost of comparison CODE using the best strategy for performance.
19281 All following functions do use number of instructions as a cost metrics.
19282 In future this should be tweaked to compute bytes for optimize_size and
19283 take into account performance of various instructions on various CPUs. */
19285 static int
19287 {
19290 /* The cost of code using bit-twiddling on %ah. */
19292 {
19315 }
19318 {
19325 }
19326 }
19328 /* Return strategy to use for floating-point. We assume that fcomi is always
19329 preferrable where available, since that is also true when looking at size
19330 (2 bytes, vs. 3 for fnstsw+sahf and at least 5 for fnstsw+test). */
19332 enum ix86_fpcmp_strategy
19334 {
19335 /* Do fcomi/sahf based test when profitable. */
19344 }
19346 /* Swap, force into registers, or otherwise massage the two operands
19347 to a fp comparison. The operands are updated in place; the new
19348 comparison code is returned. */
19350 static enum rtx_code
19352 {
19358 /* All of the unordered compare instructions only work on registers.
19359 The same is true of the fcomi compare instructions. The XFmode
19360 compare instructions require registers except when comparing
19361 against zero or when converting operand 1 from fixed point to
19362 floating point. */
19371 {
19374 }
19375 else
19376 {
19377 /* %%% We only allow op1 in memory; op0 must be st(0). So swap
19378 things around if they appear profitable, otherwise force op0
19379 into a register. */
19385 {
19388 {
19389 rtx tmp;
19392 }
19393 }
19399 {
19404 {
19407 }
19408 else
19410 }
19411 }
19413 /* Try to rearrange the comparison to make it cheaper. */
19417 {
19418 rtx tmp;
19423 }
19428 }
19430 /* Convert comparison codes we use to represent FP comparison to integer
19431 code that will result in proper branch. Return UNKNOWN if no such code
19432 is available. */
19434 enum rtx_code
19436 {
19438 {
19461 }
19462 }
19464 /* Generate insn patterns to do a floating point compare of OPERANDS. */
19466 static rtx
19468 {
19475 /* Do fcomi/sahf based test when profitable. */
19477 {
19482 tmp);
19490 tmp);
19499 /* Sadness wrt reg-stack pops killing fpsr -- gotta get fnstsw first. */
19506 /* In the unordered case, we have to check C2 for NaN's, which
19507 doesn't happen to work out to anything nice combination-wise.
19508 So do some bit twiddling on the value we've got in AH to come
19509 up with an appropriate set of condition codes. */
19513 {
19517 {
19520 }
19521 else
19522 {
19528 }
19533 {
19538 }
19539 else
19540 {
19543 }
19548 {
19551 }
19552 else
19553 {
19557 }
19562 {
19568 }
19569 else
19570 {
19573 }
19578 {
19583 }
19584 else
19585 {
19588 }
19593 {
19598 }
19599 else
19600 {
19603 }
19617 }
19622 }
19624 /* Return the test that should be put into the flags user, i.e.
19625 the bcc, scc, or cmov instruction. */
19628 const0_rtx);
19629 }
19631 static rtx
19633 {
19634 rtx ret;
19640 {
19643 }
19644 else
19648 }
19650 void
19652 {
19654 rtx tmp;
19657 {
19664 simple:
19668 pc_rtx);
19676 /* Expand DImode branch into multiple compare+branch. */
19677 {
19683 {
19686 }
19693 /* When comparing for equality, we can use (hi0^hi1)|(lo0^lo1) to
19694 avoid two branches. This costs one extra insn, so disable when
19695 optimizing for size. */
19700 {
19718 }
19720 /* Otherwise, if we are doing less-than or greater-or-equal-than,
19721 op1 is a constant and the low word is zero, then we can just
19722 examine the high word. Similarly for low word -1 and
19723 less-or-equal-than or greater-than. */
19727 {
19730 {
19733 }
19737 {
19740 }
19744 }
19746 /* Otherwise, we need two or three jumps. */
19755 {
19769 }
19771 /*
19772 * a < b =>
19773 * if (hi(a) < hi(b)) goto true;
19774 * if (hi(a) > hi(b)) goto false;
19775 * if (lo(a) < lo(b)) goto true;
19776 * false:
19777 */
19789 }
19794 }
19795 }
19797 /* Split branch based on floating point condition. */
19798 void
19801 {
19802 rtx condition;
19803 rtx i;
19806 {
19811 }
19814 tmp);
19816 /* Remove pushed operand from stack. */
19826 }
19828 void
19830 {
19831 rtx ret;
19838 }
19840 /* Expand comparison setting or clearing carry flag. Return true when
19841 successful and set pop for the operation. */
19842 static bool
19844 {
19848 /* Do not handle double-mode compares that go through special path. */
19853 {
19858 /* Shortcut: following common codes never translate
19859 into carry flag compares. */
19864 /* These comparisons require zero flag; swap operands so they won't. */
19867 {
19872 }
19874 /* Try to expand the comparison and verify that we end up with
19875 carry flag based comparison. This fails to be true only when
19876 we decide to expand comparison using arithmetic that is not
19877 too common scenario. */
19886 else
19895 }
19901 {
19906 /* Convert a==0 into (unsigned)a<1. */
19915 /* Convert a>b into b<a or a>=b-1. */
19919 {
19921 /* Bail out on overflow. We still can swap operands but that
19922 would force loading of the constant into register. */
19927 }
19928 else
19929 {
19934 }
19937 /* Convert a>=0 into (unsigned)a<0x80000000. */
19955 }
19956 /* Swapping operands may cause constant to appear as first operand. */
19958 {
19962 }
19966 }
19968 bool
19970 {
19994 /* Don't attempt mode expansion here -- if we had to expand 5 or 6
19995 HImode insns, we'd be swallowed in word prefix ops. */
20001 {
20005 HOST_WIDE_INT diff;
20008 /* Sign bit compares are better done using shifts than we do by using
20009 sbb. */
20012 {
20013 /* Detect overlap between destination and compare sources. */
20017 {
20018 rtx flags;
20027 {
20029 compare_code
20031 }
20033 /* To simplify rest of code, restrict to the GEU case. */
20035 {
20041 }
20042 else
20043 {
20046 reverse_condition_maybe_unordered
20048 else
20051 }
20060 else
20063 }
20064 else
20065 {
20068 else
20069 {
20074 }
20076 }
20079 {
20080 /*
20081 * cmpl op0,op1
20082 * sbbl dest,dest
20083 * [addl dest, ct]
20084 *
20085 * Size 5 - 8.
20086 */
20091 }
20093 {
20094 /*
20095 * cmpl op0,op1
20096 * sbbl dest,dest
20097 * orl $ct, dest
20098 *
20099 * Size 8.
20100 */
20104 }
20106 {
20107 /*
20108 * cmpl op0,op1
20109 * sbbl dest,dest
20110 * notl dest
20111 * [addl dest, cf]
20112 *
20113 * Size 8 - 11.
20114 */
20120 }
20121 else
20122 {
20123 /*
20124 * cmpl op0,op1
20125 * sbbl dest,dest
20126 * [notl dest]
20127 * andl cf - ct, dest
20128 * [addl dest, ct]
20129 *
20130 * Size 8 - 11.
20131 */
20134 {
20138 }
20148 }
20154 }
20157 {
20160 HOST_WIDE_INT tmp;
20165 {
20168 /* We may be reversing unordered compare to normal compare, that
20169 is not valid in general (we may convert non-trapping condition
20170 to trapping one), however on i386 we currently emit all
20171 comparisons unordered. */
20174 }
20175 else
20176 {
20179 }
20180 }
20185 {
20190 {
20195 }
20196 }
20198 /* Optimize dest = (op0 < 0) ? -1 : cf. */
20202 {
20203 /* If lea code below could be used, only optimize
20204 if it results in a 2 insn sequence. */
20210 {
20211 /*
20212 * notl op1 (if necessary)
20213 * sarl $31, op1
20214 * orl cf, op1
20215 */
20217 {
20221 }
20232 }
20233 }
20241 {
20242 /*
20243 * xorl dest,dest
20244 * cmpl op1,op2
20245 * setcc dest
20246 * lea cf(dest*(ct-cf)),dest
20247 *
20248 * Size 14.
20249 *
20250 * This also catches the degenerate setcc-only case.
20251 */
20253 rtx tmp;
20259 /* On x86_64 the lea instruction operates on Pmode, so we need
20260 to get arithmetics done in proper mode to match. */
20263 else
20264 {
20265 rtx out1;
20268 nops++;
20270 {
20272 nops++;
20273 }
20274 }
20276 {
20278 nops++;
20279 }
20281 {
20284 else
20286 }
20291 }
20293 /*
20294 * General case: Jumpful:
20295 * xorl dest,dest cmpl op1, op2
20296 * cmpl op1, op2 movl ct, dest
20297 * setcc dest jcc 1f
20298 * decl dest movl cf, dest
20299 * andl (cf-ct),dest 1:
20300 * addl ct,dest
20301 *
20302 * Size 20. Size 14.
20303 *
20304 * This is reasonably steep, but branch mispredict costs are
20305 * high on modern cpus, so consider failing only if optimizing
20306 * for space.
20307 */
20312 {
20314 {
20321 {
20324 /* We may be reversing unordered compare to normal compare,
20325 that is not valid in general (we may convert non-trapping
20326 condition to trapping one), however on i386 we currently
20327 emit all comparisons unordered. */
20329 }
20330 else
20331 {
20335 }
20336 }
20339 {
20340 /* notl op1 (if needed)
20341 sarl $31, op1
20342 andl (cf-ct), op1
20343 addl ct, op1
20345 For x < 0 (resp. x <= -1) there will be no notl,
20346 so if possible swap the constants to get rid of the
20347 complement.
20348 True/false will be -1/0 while code below (store flag
20349 followed by decrement) is 0/-1, so the constants need
20350 to be exchanged once more. */
20353 {
20356 }
20357 else
20358 {
20362 }
20365 }
20366 else
20367 {
20371 constm1_rtx,
20373 }
20385 }
20386 }
20389 {
20390 /* Try a few things more with specific constants and a variable. */
20392 optab op;
20398 /* If one of the two operands is an interesting constant, load a
20399 constant with the above and mask it in with a logical operation. */
20402 {
20408 else
20410 }
20412 {
20418 else
20420 }
20421 else
20428 /* Recurse to get the constant loaded. */
20432 /* Mask in the interesting variable. */
20434 OPTAB_WIDEN);
20439 }
20441 /*
20442 * For comparison with above,
20443 *
20444 * movl cf,dest
20445 * movl ct,tmp
20446 * cmpl op1,op2
20447 * cmovcc tmp,dest
20448 *
20449 * Size 15.
20450 */
20472 }
20474 /* Swap, force into registers, or otherwise massage the two operands
20475 to an sse comparison with a mask result. Thus we differ a bit from
20476 ix86_prepare_fp_compare_args which expects to produce a flags result.
20478 The DEST operand exists to help determine whether to commute commutative
20479 operators. The POP0/POP1 operands are updated in place. The new
20480 comparison code is returned, or UNKNOWN if not implementable. */
20482 static enum rtx_code
20485 {
20486 rtx tmp;
20489 {
20492 /* AVX supports all the needed comparisons. */
20495 /* We have no LTGT as an operator. We could implement it with
20496 NE & ORDERED, but this requires an extra temporary. It's
20497 not clear that it's worth it. */
20504 /* These are supported directly. */
20511 /* AVX has 3 operand comparisons, no need to swap anything. */
20514 /* For commutative operators, try to canonicalize the destination
20515 operand to be first in the comparison - this helps reload to
20516 avoid extra moves. */
20519 /* FALLTHRU */
20525 /* These are not supported directly before AVX, and furthermore
20526 ix86_expand_sse_fp_minmax only optimizes LT/UNGE. Swap the
20527 comparison operands to transform into something that is
20528 supported. */
20537 }
20540 }
20542 /* Detect conditional moves that exactly match min/max operational
20543 semantics. Note that this is IEEE safe, as long as we don't
20544 interchange the operands.
20546 Returns FALSE if this conditional move doesn't match a MIN/MAX,
20547 and TRUE if the operation is successful and instructions are emitted. */
20549 static bool
20552 {
20555 rtx tmp;
20558 ;
20560 {
20564 }
20565 else
20572 else
20577 /* We want to check HONOR_NANS and HONOR_SIGNED_ZEROS here,
20578 but MODE may be a vector mode and thus not appropriate. */
20580 {
20582 rtvec v;
20587 }
20588 else
20589 {
20592 }
20596 }
20598 /* Expand an sse vector comparison. Return the register with the result. */
20600 static rtx
20603 {
20606 rtx x;
20619 {
20622 }
20623 else
20627 }
20629 /* Expand DEST = CMP ? OP_TRUE : OP_FALSE into a sequence of logical
20630 operations. This is used for both scalar and vector conditional moves. */
20632 static void
20634 {
20640 {
20642 }
20644 {
20648 }
20650 {
20655 }
20657 {
20661 }
20663 {
20671 op_true,
20672 op_false)));
20673 }
20674 else
20675 {
20685 {
20699 {
20706 }
20721 {
20728 }
20732 }
20735 {
20739 }
20740 else
20741 {
20747 else
20759 }
20760 }
20761 }
20763 /* Expand a floating-point conditional move. Return true if successful. */
20765 bool
20767 {
20775 {
20778 /* Since we've no cmove for sse registers, don't force bad register
20779 allocation just to gain access to it. Deny movcc when the
20780 comparison mode doesn't match the move mode. */
20799 }
20806 /* The floating point conditional move instructions don't directly
20807 support conditions resulting from a signed integer comparison. */
20811 {
20816 }
20823 }
20825 /* Expand a floating-point vector conditional move; a vcond operation
20826 rather than a movcc operation. */
20828 bool
20830 {
20832 rtx cmp;
20837 {
20838 rtx temp;
20840 {
20857 }
20859 OPTAB_DIRECT);
20862 }
20872 }
20874 /* Expand a signed/unsigned integral vector conditional move. */
20876 bool
20878 {
20888 /* Try to optimize x < 0 ? -1 : 0 into (signed) x >> 31
20889 and x < 0 ? 1 : 0 into (unsigned) x >> 31. */
20898 {
20902 {
20908 }
20911 {
20917 }
20918 }
20927 /* XOP supports all of the comparisons on all 128-bit vector int types. */
20931 ;
20932 else
20933 {
20934 /* Canonicalize the comparison to EQ, GT, GTU. */
20936 {
20953 /* FALLTHRU */
20963 }
20965 /* Only SSE4.1/SSE4.2 supports V2DImode. */
20967 {
20969 {
20971 /* SSE4.1 supports EQ. */
20978 /* SSE4.2 supports GT/GTU. */
20985 }
20986 }
20988 /* Unsigned parallel compare is not supported by the hardware.
20989 Play some tricks to turn this into a signed comparison
20990 against 0. */
20992 {
20996 {
21001 {
21006 {
21013 }
21014 /* Subtract (-(INT MAX) - 1) from both operands to make
21015 them signed. */
21026 }
21033 /* Perform a parallel unsigned saturating subtraction. */
21046 }
21047 }
21048 }
21050 /* Allow the comparison to be done in one mode, but the movcc to
21051 happen in another mode. */
21053 {
21056 }
21057 else
21058 {
21063 }
21068 }
21070 /* Expand a variable vector permutation. */
21072 void
21074 {
21085 /* Number of elements in the vector. */
21091 {
21093 {
21094 /* Unfortunately, the VPERMQ and VPERMPD instructions only support
21095 an constant shuffle operand. With a tiny bit of effort we can
21096 use VPERMD instead. A re-interpretation stall for V4DFmode is
21097 unfortunate but there's no avoiding it.
21098 Similarly for V16HImode we don't have instructions for variable
21099 shuffling, while for V32QImode we can use after preparing suitable
21100 masks vpshufb; vpshufb; vpermq; vpor. */
21103 {
21107 }
21108 else
21109 {
21113 }
21116 /* Replicate the low bits of the V4DImode mask into V8SImode:
21117 mask = { A B C D }
21118 t1 = { A A B B C C D D }. */
21126 else
21129 /* Multiply the shuffle indicies by two. */
21131 OPTAB_DIRECT);
21133 /* Add one to the odd shuffle indicies:
21134 t1 = { A*2, A*2+1, B*2, B*2+1, ... }. */
21136 {
21139 }
21143 OPTAB_DIRECT);
21145 /* Continue as if V8SImode (resp. V32QImode) was used initially. */
21150 }
21153 {
21155 /* The VPERMD and VPERMPS instructions already properly ignore
21156 the high bits of the shuffle elements. No need for us to
21157 perform an AND ourselves. */
21159 {
21164 }
21165 else
21166 {
21172 }
21179 else
21180 {
21186 }
21190 /* By combining the two 128-bit input vectors into one 256-bit
21191 input vector, we can use VPERMD and VPERMPS for the full
21192 two-operand shuffle. */
21224 /* From mask create two adjusted masks, which contain the same
21225 bits as mask in the low 7 bits of each vector element.
21226 The first mask will have the most significant bit clear
21227 if it requests element from the same 128-bit lane
21228 and MSB set if it requests element from the other 128-bit lane.
21229 The second mask will have the opposite values of the MSB,
21230 and additionally will have its 128-bit lanes swapped.
21231 E.g. { 07 12 1e 09 ... | 17 19 05 1f ... } mask vector will have
21232 t1 { 07 92 9e 09 ... | 17 19 85 1f ... } and
21233 t3 { 97 99 05 9f ... | 87 12 1e 89 ... } where each ...
21234 stands for other 12 bytes. */
21235 /* The bit whether element is from the same lane or the other
21236 lane is bit 4, so shift it up by 3 to the MSB position. */
21240 /* Clear MSB bits from the mask just in case it had them set. */
21242 /* After this t1 will have MSB set for elements from other lane. */
21244 /* Clear bits other than MSB. */
21246 /* Or in the lower bits from mask into t3. */
21248 /* And invert MSB bits in t1, so MSB is set for elements from the same
21249 lane. */
21251 /* Swap 128-bit lanes in t3. */
21256 /* And or in the lower bits from mask into t1. */
21259 {
21260 /* Each of these shuffles will put 0s in places where
21261 element from the other 128-bit lane is needed, otherwise
21262 will shuffle in the requested value. */
21266 /* For t3 the 128-bit lanes are swapped again. */
21271 /* And oring both together leads to the result. */
21278 }
21281 /* Similarly to the above one_operand_shuffle code,
21282 just for repeated twice for each operand. merge_two:
21283 code will merge the two results together. */
21307 }
21308 }
21311 {
21312 /* The XOP VPPERM insn supports three inputs. By ignoring the
21313 one_operand_shuffle special case, we avoid creating another
21314 set of constant vectors in memory. */
21317 /* mask = mask & {2*w-1, ...} */
21319 }
21320 else
21321 {
21322 /* mask = mask & {w-1, ...} */
21324 }
21332 /* For non-QImode operations, convert the word permutation control
21333 into a byte permutation control. */
21335 {
21340 /* Convert mask to vector of chars. */
21343 /* Replicate each of the input bytes into byte positions:
21344 (v2di) --> {0,0,0,0,0,0,0,0, 8,8,8,8,8,8,8,8}
21345 (v4si) --> {0,0,0,0, 4,4,4,4, 8,8,8,8, 12,12,12,12}
21346 (v8hi) --> {0,0, 2,2, 4,4, 6,6, ...}. */
21353 else
21356 /* Convert it into the byte positions by doing
21357 mask = mask + {0,1,..,16/w, 0,1,..,16/w, ...} */
21363 }
21365 /* The actual shuffle operations all operate on V16QImode. */
21370 {
21377 }
21379 {
21386 }
21387 else
21388 {
21392 /* Shuffle the two input vectors independently. */
21398 merge_two:
21399 /* Then merge them together. The key is whether any given control
21400 element contained a bit set that indicates the second word. */
21404 {
21405 /* Without SSE4.1, we don't have V2DImode EQ. Perform one
21406 more shuffle to convert the V2DI input mask into a V4SI
21407 input mask. At which point the masking that expand_int_vcond
21408 will work as desired. */
21416 }
21438 }
21439 }
21441 /* Unpack OP[1] into the next wider integer vector type. UNSIGNED_P is
21442 true if we should do zero extension, else sign extension. HIGH_P is
21443 true if we want the N/2 high elements, else the low elements. */
21445 void
21447 {
21449 rtx tmp;
21452 {
21458 {
21462 else
21465 extract
21471 else
21474 extract
21480 else
21483 extract
21489 else
21495 else
21501 else
21506 }
21509 {
21512 }
21514 {
21515 /* Shift higher 8 bytes to lower 8 bytes. */
21520 }
21521 else
21525 }
21526 else
21527 {
21531 {
21535 else
21541 else
21547 else
21552 }
21556 else
21563 }
21564 }
21566 /* Expand conditional increment or decrement using adb/sbb instructions.
21567 The default case using setcc followed by the conditional move can be
21568 done by generic code. */
21569 bool
21571 {
21573 rtx flags;
21575 rtx compare_op;
21593 {
21596 }
21599 {
21603 reverse_condition_maybe_unordered
21605 else
21607 }
21611 /* Construct either adc or sbb insn. */
21613 {
21615 {
21630 }
21631 }
21632 else
21633 {
21635 {
21650 }
21651 }
21655 }
21658 /* Split operands 0 and 1 into half-mode parts. Similar to split_double_mode,
21659 but works for floating pointer parameters and nonoffsetable memories.
21660 For pushes, it returns just stack offsets; the values will be saved
21661 in the right order. Maximally three parts are generated. */
21663 static int
21665 {
21670 else
21676 /* Optimize constant pool reference to immediates. This is used by fp
21677 moves, that force all constants to memory to allow combining. */
21679 {
21683 }
21686 {
21687 /* The only non-offsetable memories we handle are pushes. */
21696 }
21699 {
21701 /* Caution: if we looked through a constant pool memory above,
21702 the operand may actually have a different mode now. That's
21703 ok, since we want to pun this all the way back to an integer. */
21707 }
21710 {
21713 else
21714 {
21718 {
21722 }
21724 {
21729 }
21731 {
21732 REAL_VALUE_TYPE r;
21737 {
21744 /* We can't use REAL_VALUE_TO_TARGET_LONG_DOUBLE since
21745 long double may not be 80-bit. */
21754 }
21757 }
21758 else
21760 }
21761 }
21762 else
21763 {
21767 {
21770 {
21774 }
21776 {
21780 }
21782 {
21783 REAL_VALUE_TYPE r;
21789 /* Do not use shift by 32 to avoid warning on 32bit systems. */
21792 = gen_int_mode
21795 DImode);
21796 else
21803 = gen_int_mode
21806 DImode);
21807 else
21809 }
21810 else
21812 }
21813 }
21816 }
21818 /* Emit insns to perform a move or push of DI, DF, XF, and TF values.
21819 Return false when normal moves are needed; true when all required
21820 insns have been emitted. Operands 2-4 contain the input values
21821 int the correct order; operands 5-7 contain the output values. */
21823 void
21825 {
21833 /* The DFmode expanders may ask us to move double.
21834 For 64bit target this is single move. By hiding the fact
21835 here we simplify i386.md splitters. */
21837 {
21838 /* Optimize constant pool reference to immediates. This is used by
21839 fp moves, that force all constants to memory to allow combining. */
21846 {
21849 }
21850 else
21855 }
21857 /* The only non-offsettable memory we handle is push. */
21860 else
21867 /* When emitting push, take care for source operands on the stack. */
21870 {
21873 /* Compensate for the stack decrement by 4. */
21878 /* src_base refers to the stack pointer and is
21879 automatically decreased by emitted push. */
21883 }
21885 /* We need to do copy in the right order in case an address register
21886 of the source overlaps the destination. */
21888 {
21889 rtx tmp;
21892 {
21896 collisions++;
21897 }
21899 /* Collision in the middle part can be handled by reordering. */
21901 {
21904 }
21908 {
21910 {
21913 }
21914 else
21915 {
21918 }
21919 }
21921 /* If there are more collisions, we can't handle it by reordering.
21922 Do an lea to the last part and use only one colliding move. */
21924 {
21925 rtx base;
21931 /* Handle the case when the last part isn't valid for lea.
21932 Happens in 64-bit mode storing the 12-byte XFmode. */
21939 {
21942 }
21943 }
21944 }
21947 {
21949 {
21951 {
21956 }
21958 {
21961 }
21962 }
21963 else
21964 {
21965 /* In 64bit mode we don't have 32bit push available. In case this is
21966 register, it is OK - we will just use larger counterpart. We also
21967 retype memory - these comes from attempt to avoid REX prefix on
21968 moving of second half of TFmode value. */
21970 {
21972 {
21983 }
21987 }
21988 }
21992 }
21994 /* Choose correct order to not overwrite the source before it is copied. */
22004 {
22006 {
22009 }
22010 }
22011 else
22012 {
22014 {
22017 }
22018 }
22020 /* If optimizing for size, attempt to locally unCSE nonzero constants. */
22022 {
22031 }
22037 }
22039 /* Helper function of ix86_split_ashl used to generate an SImode/DImode
22040 left shift by a constant, either using a single shift or
22041 a sequence of add instructions. */
22043 static void
22045 {
22051 {
22055 }
22056 else
22057 {
22060 }
22061 }
22063 void
22065 {
22074 {
22079 {
22085 }
22086 else
22087 {
22095 }
22097 }
22104 {
22105 /* Assuming we've chosen a QImode capable registers, then 1 << N
22106 can be done with two 32/64-bit shifts, no branches, no cmoves. */
22108 {
22124 }
22126 /* Otherwise, we can get the same results by manually performing
22127 a bit extract operation on bit 5/6, and then performing the two
22128 shifts. The two methods of getting 0/1 into low/high are exactly
22129 the same size. Avoiding the shift in the bit extract case helps
22130 pentium4 a bit; no one else seems to care much either way. */
22131 else
22132 {
22137 HOST_WIDE_INT bits;
22138 rtx x;
22141 {
22147 }
22148 else
22149 {
22155 }
22159 else
22167 }
22172 }
22175 {
22176 /* For -1 << N, we can avoid the shld instruction, because we
22177 know that we're shifting 0...31/63 ones into a -1. */
22181 else
22183 }
22184 else
22185 {
22193 }
22198 {
22204 }
22205 else
22206 {
22211 }
22212 }
22214 void
22216 {
22226 {
22231 {
22237 }
22239 {
22248 }
22249 else
22250 {
22258 }
22259 }
22260 else
22261 {
22273 {
22281 scratch));
22282 }
22283 else
22284 {
22289 }
22290 }
22291 }
22293 void
22295 {
22305 {
22310 {
22317 }
22318 else
22319 {
22327 }
22328 }
22329 else
22330 {
22342 {
22348 scratch));
22349 }
22350 else
22351 {
22356 }
22357 }
22358 }
22360 /* Predict just emitted jump instruction to be taken with probability PROB. */
22361 static void
22363 {
22367 }
22369 /* Helper function for the string operations below. Dest VARIABLE whether
22370 it is aligned to VALUE bytes. If true, jump to the label. */
22371 static rtx
22373 {
22378 else
22384 else
22387 }
22389 /* Adjust COUNTER by the VALUE. */
22390 static void
22392 {
22397 }
22399 /* Zero extend possibly SImode EXP to Pmode register. */
22400 rtx
22402 {
22404 }
22406 /* Divide COUNTREG by SCALE. */
22407 static rtx
22409 {
22410 rtx sc;
22422 }
22424 /* Return mode for the memcpy/memset loop counter. Prefer SImode over
22425 DImode for constant loop counts. */
22427 static enum machine_mode
22429 {
22437 }
22439 /* Copy the address to a Pmode register. This is used for x32 to
22440 truncate DImode TLS address to a SImode register. */
22442 static rtx
22444 {
22447 else
22448 {
22451 }
22452 }
22454 /* When ISSETMEM is FALSE, output simple loop to move memory pointer to SRCPTR
22455 to DESTPTR via chunks of MODE unrolled UNROLL times, overall size is COUNT
22456 specified in bytes. When ISSETMEM is TRUE, output the equivalent loop to set
22457 memory by VALUE (supposed to be in MODE).
22459 The size is rounded down to whole number of chunk size moved at once.
22460 SRCMEM and DESTMEM provide MEMrtx to feed proper aliasing info. */
22463 static void
22468 {
22474 rtx size;
22483 /* Those two should combine. */
22485 {
22489 }
22496 /* This assert could be relaxed - in this case we'll need to compute
22497 smallest power of two, containing in PIECE_SIZE_N and pass it to
22498 offset_address. */
22504 {
22508 /* When unrolling for chips that reorder memory reads and writes,
22509 we can save registers by using single temporary.
22510 Also using 4 temporaries is overkill in 32bit mode. */
22512 {
22514 {
22516 {
22517 destmem =
22519 srcmem =
22521 }
22523 }
22524 }
22525 else
22526 {
22530 {
22533 {
22534 srcmem =
22536 }
22538 }
22540 {
22542 {
22543 destmem =
22545 }
22547 }
22548 }
22549 }
22550 else
22552 {
22554 destmem =
22557 }
22567 {
22573 else
22575 }
22576 else
22584 {
22589 }
22591 }
22593 /* Output "rep; mov" or "rep; stos" instruction depending on ISSETMEM argument.
22594 When ISSETMEM is true, arguments SRCMEM and SRCPTR are ignored.
22595 When ISSETMEM is false, arguments VALUE and ORIG_VALUE are ignored.
22596 For setmem case, VALUE is a promoted to a wider size ORIG_VALUE.
22597 ORIG_VALUE is the original value passed to memset to fill the memory with.
22598 Other arguments have same meaning as for previous function. */
22600 static void
22603 rtx count,
22605 {
22606 rtx destexp;
22607 rtx srcexp;
22608 rtx countreg;
22609 HOST_WIDE_INT rounded_count;
22611 /* If possible, it is shorter to use rep movs.
22612 TODO: Maybe it is better to move this logic to decide_alg. */
22623 {
22627 }
22628 else
22631 {
22636 }
22641 {
22644 }
22645 else
22646 {
22650 {
22654 }
22655 else
22658 {
22663 }
22664 else
22665 {
22668 }
22671 }
22672 }
22674 /* This function emits moves to copy SIZE_TO_MOVE bytes from SRCMEM to
22675 DESTMEM.
22676 SRC is passed by pointer to be updated on return.
22677 Return value is updated DST. */
22678 static rtx
22680 HOST_WIDE_INT size_to_move)
22681 {
22687 /* Find the widest mode in which we could perform moves.
22688 Start with the biggest power of 2 less than SIZE_TO_MOVE and half
22689 it until move of such size is supported. */
22694 {
22698 }
22700 /* Find the corresponding vector mode with the same size as MOVE_MODE.
22701 MOVE_MODE is an integer mode at the moment (SI, DI, TI, etc.). */
22703 {
22708 {
22712 }
22713 }
22719 /* Emit moves. We'll need SIZE_TO_MOVE/PIECE_SIZES moves. */
22723 {
22724 /* We move from memory to memory, so we'll need to do it via
22725 a temporary register. */
22736 piece_size);
22738 piece_size);
22739 }
22741 /* Update DST and SRC rtx. */
22744 }
22746 /* Output code to copy at most count & (max_size - 1) bytes from SRC to DEST. */
22747 static void
22750 {
22753 {
22758 /* For now MAX_SIZE should be a power of 2. This assert could be
22759 relaxed, but it'll require a bit more complicated epilogue
22760 expanding. */
22763 {
22766 }
22768 }
22770 {
22776 }
22778 /* When there are stringops, we can cheaply increase dest and src pointers.
22779 Otherwise we save code size by maintaining offset (zero is readily
22780 available from preceding rep operation) and using x86 addressing modes.
22781 */
22783 {
22785 {
22792 }
22794 {
22801 }
22803 {
22810 }
22811 }
22812 else
22813 {
22815 rtx tmp;
22818 {
22829 }
22831 {
22844 }
22846 {
22855 }
22856 }
22857 }
22859 /* This function emits moves to fill SIZE_TO_MOVE bytes starting from DESTMEM
22860 with value PROMOTED_VAL.
22861 SRC is passed by pointer to be updated on return.
22862 Return value is updated DST. */
22863 static rtx
22865 HOST_WIDE_INT size_to_move)
22866 {
22872 /* Find the widest mode in which we could perform moves.
22873 Start with the biggest power of 2 less than SIZE_TO_MOVE and half
22874 it until move of such size is supported. */
22879 {
22882 }
22889 /* Emit moves. We'll need SIZE_TO_MOVE/PIECE_SIZES moves. */
22893 {
22895 {
22898 piece_size);
22900 }
22908 piece_size);
22909 }
22911 /* Update DST rtx. */
22913 }
22914 /* Output code to set at most count & (max_size - 1) bytes starting by DEST. */
22915 static void
22918 {
22919 count =
22925 }
22927 /* Output code to set at most count & (max_size - 1) bytes starting by DEST. */
22928 static void
22931 {
22932 rtx dest;
22935 {
22940 /* For now MAX_SIZE should be a power of 2. This assert could be
22941 relaxed, but it'll require a bit more complicated epilogue
22942 expanding. */
22945 {
22947 {
22950 else
22952 }
22953 }
22955 }
22957 {
22960 }
22962 {
22965 {
22970 }
22971 else
22972 {
22981 }
22984 }
22986 {
22989 {
22992 }
22993 else
22994 {
22999 }
23002 }
23004 {
23010 }
23012 {
23018 }
23020 {
23026 }
23027 }
23029 /* Depending on ISSETMEM, copy enough from SRCMEM to DESTMEM or set enough to
23030 DESTMEM to align it to DESIRED_ALIGNMENT. Original alignment is ALIGN.
23031 Depending on ISSETMEM, either arguments SRCMEM/SRCPTR or VALUE/VEC_VALUE are
23032 ignored.
23033 Return value is updated DESTMEM. */
23034 static rtx
23039 {
23042 {
23044 {
23047 {
23050 else
23052 }
23053 else
23059 }
23060 }
23062 }
23064 /* Test if COUNT&SIZE is nonzero and if so, expand movme
23065 or setmem sequence that is valid for SIZE..2*SIZE-1 bytes
23066 and jump to DONE_LABEL. */
23067 static void
23073 {
23076 rtx modesize;
23079 /* If we do not have vector value to copy, we must reduce size. */
23081 {
23083 {
23088 }
23089 else
23091 }
23092 else
23093 {
23094 /* Choose appropriate vector mode. */
23100 }
23105 {
23108 else
23109 {
23112 }
23114 }
23121 else
23122 {
23127 }
23133 }
23135 /* Handle small memcpy (up to SIZE that is supposed to be small power of 2.
23136 and get ready for the main memcpy loop by copying iniital DESIRED_ALIGN-ALIGN
23137 bytes and last SIZE bytes adjusitng DESTPTR/SRCPTR/COUNT in a way we can
23138 proceed with an loop copying SIZE bytes at once. Do moves in MODE.
23139 DONE_LABEL is a label after the whole copying sequence. The label is created
23140 on demand if *DONE_LABEL is NULL.
23141 MIN_SIZE is minimal size of block copied. This value gets adjusted for new
23142 bounds after the initial copies.
23144 DESTMEM/SRCMEM are memory expressions pointing to the copies block,
23145 DESTPTR/SRCPTR are pointers to the block. DYNAMIC_CHECK indicate whether
23146 we will dispatch to a library call for large blocks.
23148 In pseudocode we do:
23150 if (COUNT < SIZE)
23151 {
23152 Assume that SIZE is 4. Bigger sizes are handled analogously
23153 if (COUNT & 4)
23154 {
23155 copy 4 bytes from SRCPTR to DESTPTR
23156 copy 4 bytes from SRCPTR + COUNT - 4 to DESTPTR + COUNT - 4
23157 goto done_label
23158 }
23159 if (!COUNT)
23160 goto done_label;
23161 copy 1 byte from SRCPTR to DESTPTR
23162 if (COUNT & 2)
23163 {
23164 copy 2 bytes from SRCPTR to DESTPTR
23165 copy 2 bytes from SRCPTR + COUNT - 2 to DESTPTR + COUNT - 2
23166 }
23167 }
23168 else
23169 {
23170 copy at least DESIRED_ALIGN-ALIGN bytes from SRCPTR to DESTPTR
23171 copy SIZE bytes from SRCPTR + COUNT - SIZE to DESTPTR + COUNT -SIZE
23173 OLD_DESPTR = DESTPTR;
23174 Align DESTPTR up to DESIRED_ALIGN
23175 SRCPTR += DESTPTR - OLD_DESTPTR
23176 COUNT -= DEST_PTR - OLD_DESTPTR
23177 if (DYNAMIC_CHECK)
23178 Round COUNT down to multiple of SIZE
23179 << optional caller supplied zero size guard is here >>
23180 << optional caller suppplied dynamic check is here >>
23181 << caller supplied main copy loop is here >>
23182 }
23183 done_label:
23184 */
23185 static void
23198 {
23201 rtx modesize;
23203 rtx mode_value;
23205 /* Chose proper value to copy. */
23208 else
23212 /* See if block is big or small, handle small blocks. */
23214 {
23225 /* Handle sizes > 3. */
23232 /* Nothing to copy? Jump to DONE_LABEL if so */
23236 /* Do a byte copy. */
23240 else
23241 {
23244 }
23246 /* Handle sizes 2 and 3. */
23253 else
23254 {
23259 }
23265 }
23266 else
23270 /* Start memcpy for COUNT >= SIZE. */
23272 {
23275 }
23277 /* Copy first desired_align bytes. */
23283 {
23286 else
23287 {
23290 }
23293 }
23296 /* Copy last SIZE bytes. */
23303 else
23304 {
23310 }
23312 {
23316 else
23317 {
23320 }
23321 }
23323 /* Align destination. */
23325 {
23329 /* Align destptr up, place it to new register. */
23336 /* See how many bytes we skipped. */
23340 /* Adjust srcptr and count. */
23346 /* We copied at most size + prolog_size. */
23349 else
23352 /* Our loops always round down the bock size, but for dispatch to library
23353 we need precise value. */
23357 }
23358 else
23359 {
23361 /* Decrease count, so we won't end up copying last word twice. */
23365 else
23369 }
23370 }
23373 /* This function is like the previous one, except here we know how many bytes
23374 need to be copied. That allows us to update alignment not only of DST, which
23375 is returned, but also of SRC, which is passed as a pointer for that
23376 reason. */
23377 static rtx
23382 {
23390 {
23394 }
23399 {
23401 {
23403 {
23406 else
23408 }
23409 else
23412 }
23413 }
23420 {
23426 {
23429 {
23433 }
23438 }
23442 }
23445 }
23447 /* Return true if ALG can be used in current context.
23448 Assume we expand memset if MEMSET is true. */
23449 static bool
23451 {
23456 /* Algorithms using the rep prefix want at least edi and ecx;
23457 additionally, memset wants eax and memcpy wants esi. Don't
23458 consider such algorithms if the user has appropriated those
23459 registers for their own purposes. */
23466 }
23468 /* Given COUNT and EXPECTED_SIZE, decide on codegen of string operation. */
23469 static enum stringop_alg
23473 {
23484 /* Even if the string operation call is cold, we still might spend a lot
23485 of time processing large blocks. */
23491 else
23497 else
23500 /* See maximal size for user defined algorithm. */
23502 {
23509 }
23511 /* If expected size is not known but max size is small enough
23512 so inline version is a win, set expected size into
23513 the range. */
23518 /* If user specified the algorithm, honnor it if possible. */
23522 /* rep; movq or rep; movl is the smallest variant. */
23524 {
23529 else
23532 }
23533 /* Very tiny blocks are best handled via the loop, REP is expensive to
23534 setup. */
23538 {
23542 {
23543 /* We get here if the algorithms that were not libcall-based
23544 were rep-prefix based and we are unable to use rep prefixes
23545 based on global register usage. Break out of the loop and
23546 use the heuristic below. */
23550 {
23554 {
23557 }
23558 /* Honor TARGET_INLINE_ALL_STRINGOPS by picking
23559 last non-libcall inline algorithm. */
23561 {
23562 /* When the current size is best to be copied by a libcall,
23563 but we are still forced to inline, run the heuristic below
23564 that will pick code for medium sized blocks. */
23566 {
23569 }
23571 }
23573 {
23576 }
23577 }
23578 }
23579 }
23580 /* When asked to inline the call anyway, try to pick meaningful choice.
23581 We look for maximal size of block that is faster to copy by hand and
23582 take blocks of at most of that size guessing that average size will
23583 be roughly half of the block.
23585 If this turns out to be bad, we might simply specify the preferred
23586 choice in ix86_costs. */
23590 {
23593 /* If there aren't any usable algorithms, then recursing on
23594 smaller sizes isn't going to find anything. Just return the
23595 simple byte-at-a-time copy loop. */
23597 {
23598 /* Pick something reasonable. */
23602 }
23612 }
23615 }
23617 /* Decide on alignment. We know that the operand is already aligned to ALIGN
23618 (ALIGN can be based on profile feedback and thus it is not 100% guaranteed). */
23619 static int
23624 {
23635 /* PentiumPro has special logic triggering for 8 byte aligned blocks.
23636 copying whole cacheline at once. */
23649 }
23652 /* Helper function for memcpy. For QImode value 0xXY produce
23653 0xXYXYXYXY of wide specified by MODE. This is essentially
23654 a * 0x10101010, but we can do slightly better than
23655 synth_mult by unwinding the sequence by hand on CPUs with
23656 slow multiply. */
23657 static rtx
23659 {
23661 rtx tmp;
23668 {
23676 }
23685 nops--;
23690 {
23694 OPTAB_DIRECT);
23695 }
23696 else
23697 {
23703 else
23705 else
23706 {
23709 reg =
23711 }
23721 }
23722 }
23724 /* Duplicate value VAL using promote_duplicated_reg into maximal size that will
23725 be needed by main loop copying SIZE_NEEDED chunks and prologue getting
23726 alignment from ALIGN to DESIRED_ALIGN. */
23727 static rtx
23730 {
23731 rtx promoted_val;
23740 else
23744 }
23746 /* Expand string move (memcpy) ot store (memset) operation. Use i386 string
23747 operations when profitable. The code depends upon architecture, block size
23748 and alignment, but always has one of the following overall structures:
23750 Aligned move sequence:
23752 1) Prologue guard: Conditional that jumps up to epilogues for small
23753 blocks that can be handled by epilogue alone. This is faster
23754 but also needed for correctness, since prologue assume the block
23755 is larger than the desired alignment.
23757 Optional dynamic check for size and libcall for large
23758 blocks is emitted here too, with -minline-stringops-dynamically.
23760 2) Prologue: copy first few bytes in order to get destination
23761 aligned to DESIRED_ALIGN. It is emitted only when ALIGN is less
23762 than DESIRED_ALIGN and up to DESIRED_ALIGN - ALIGN bytes can be
23763 copied. We emit either a jump tree on power of two sized
23764 blocks, or a byte loop.
23766 3) Main body: the copying loop itself, copying in SIZE_NEEDED chunks
23767 with specified algorithm.
23769 4) Epilogue: code copying tail of the block that is too small to be
23770 handled by main body (or up to size guarded by prologue guard).
23772 Misaligned move sequence
23774 1) missaligned move prologue/epilogue containing:
23775 a) Prologue handling small memory blocks and jumping to done_label
23776 (skipped if blocks are known to be large enough)
23777 b) Signle move copying first DESIRED_ALIGN-ALIGN bytes if alignment is
23778 needed by single possibly misaligned move
23779 (skipped if alignment is not needed)
23780 c) Copy of last SIZE_NEEDED bytes by possibly misaligned moves
23782 2) Zero size guard dispatching to done_label, if needed
23784 3) dispatch to library call, if needed,
23786 3) Main body: the copying loop itself, copying in SIZE_NEEDED chunks
23787 with specified algorithm. */
23788 bool
23794 {
23795 rtx destreg;
23798 rtx tmp;
23814 /* TODO: Once value ranges are available, fill in proper data. */
23822 /* i386 can do misaligned access on reasonably increased cost. */
23826 /* ALIGN is the minimum of destination and source alignment, but we care here
23827 just about destination alignment. */
23835 else
23836 {
23845 }
23847 /* Make sure we don't need to care about overflow later on. */
23851 /* Step 0: Decide on preferred algorithm, desired alignment and
23852 size of chunks to be copied by main loop. */
23854 issetmem,
23861 /* For now vector-version of memset is generated only for memory zeroing, as
23862 creating of promoted vector value is very cheap in this case. */
23875 {
23894 /* Find the widest supported mode. */
23900 /* Find the corresponding vector mode with the same size as MOVE_MODE.
23901 MOVE_MODE is an integer mode at the moment (SI, DI, TI, etc.). */
23903 {
23908 }
23920 }
23928 /* Step 1: Prologue guard. */
23930 /* Alignment code needs count to be in register. */
23932 {
23935 {
23936 align_bytes
23940 else
23942 }
23945 }
23948 /* Misaligned move sequences handle both prologue and epilogue at once.
23949 Default code generation results in a smaller code for large alignments
23950 and also avoids redundant job when sizes are known precisely. */
23951 misaligned_prologue_used
23952 = (TARGET_MISALIGNED_MOVE_STRING_PRO_EPILOGUES
23958 /* Do the cheap promotion to allow better CSE across the
23959 main loop and epilogue (ie one load of the big constant in the
23960 front of all code.
23961 For now the misaligned move sequences do not have fast path
23962 without broadcasting. */
23964 {
23966 {
23972 }
23973 else
23974 {
23977 }
23978 }
23979 /* Misaligned move sequences handles both prologues and epilogues at once.
23980 Default code generation results in smaller code for large alignments and
23981 also avoids redundant job when sizes are known precisely. */
23983 {
23984 /* Misaligned move prologue handled small blocks by itself. */
23985 expand_set_or_movmem_prologue_epilogue_by_misaligned_moves
23990 desired_align < align
23999 {
24000 /* It is possible that we copied enough so the main loop will not
24001 execute. */
24011 else
24013 }
24014 }
24015 /* Ensure that alignment prologue won't copy past end of block. */
24017 {
24019 /* Epilogue always copies COUNT_EXP & EPILOGUE_SIZE_NEEDED bytes.
24020 Make sure it is power of 2. */
24023 /* To improve performance of small blocks, we jump around the VAL
24024 promoting mode. This mean that if the promoted VAL is not constant,
24025 we might not use it in the epilogue and have to use byte
24026 loop variant. */
24031 {
24032 /* If main algorithm works on QImode, no epilogue is needed.
24033 For small sizes just don't align anything. */
24036 else
24038 }
24041 {
24048 else
24050 }
24051 }
24053 /* Emit code to decide on runtime whether library call or inline should be
24054 used. */
24056 {
24058 {
24060 {
24064 }
24065 }
24066 else
24067 {
24075 else
24079 }
24080 }
24082 /* Step 2: Alignment prologue. */
24083 /* Do the expensive promotion once we branched off the small blocks. */
24089 {
24091 {
24092 /* Except for the first move in prologue, we no longer know
24093 constant offset in aliasing info. It don't seems to worth
24094 the pain to maintain it for the first move, so throw away
24095 the info early. */
24102 issetmem);
24103 /* At most desired_align - align bytes are copied. */
24106 else
24108 }
24109 else
24110 {
24111 /* If we know how many bytes need to be stored before dst is
24112 sufficiently aligned, maintain aliasing info accurately. */
24114 srcreg,
24115 promoted_val,
24116 vec_promoted_val,
24117 desired_align,
24118 align_bytes,
24119 issetmem);
24126 }
24128 && !min_size
24133 {
24134 /* It is possible that we copied enough so the main loop will not
24135 execute. */
24145 else
24147 }
24148 }
24150 {
24157 }
24161 /* Step 3: Main loop. */
24164 {
24187 }
24188 /* Adjust properly the offset of src and dest memory for aliasing. */
24190 {
24196 }
24197 else
24198 {
24202 }
24204 /* Step 4: Epilogue to copy the remaining bytes. */
24205 epilogue:
24207 {
24208 /* When the main loop is done, COUNT_EXP might hold original count,
24209 while we want to copy only COUNT_EXP & SIZE_NEEDED bytes.
24210 Epilogue code will actually copy COUNT_EXP & EPILOGUE_SIZE_NEEDED
24211 bytes. Compensate if needed. */
24214 {
24215 tmp =
24218 OPTAB_DIRECT);
24221 }
24224 }
24227 {
24230 epilogue_size_needed);
24231 else
24232 {
24236 epilogue_size_needed);
24237 else
24239 epilogue_size_needed);
24240 }
24241 }
24245 }
24248 /* Expand the appropriate insns for doing strlen if not just doing
24249 repnz; scasb
24251 out = result, initialized with the start address
24252 align_rtx = alignment of the address.
24253 scratch = scratch register, initialized with the startaddress when
24254 not aligned, otherwise undefined
24256 This is just the body. It needs the initializations mentioned above and
24257 some address computing at the end. These things are done in i386.md. */
24259 static void
24261 {
24263 rtx tmp;
24268 rtx mem;
24271 rtx cmp;
24277 /* Loop to check 1..3 bytes for null to get an aligned pointer. */
24279 /* Is there a known alignment and is it less than 4? */
24281 {
24284 /* Is there a known alignment and is it not 2? */
24286 {
24290 /* Leave just the 3 lower bits. */
24300 }
24301 else
24302 {
24303 /* Since the alignment is 2, we have to check 2 or 0 bytes;
24304 check if is aligned to 4 - byte. */
24311 }
24315 /* Now compare the bytes. */
24317 /* Compare the first n unaligned byte on a byte per byte basis. */
24321 /* Increment the address. */
24324 /* Not needed with an alignment of 2 */
24326 {
24330 end_0_label);
24335 }
24338 end_0_label);
24341 }
24343 /* Generate loop to check 4 bytes at a time. It is not a good idea to
24344 align this loop. It gives only huge programs, but does not help to
24345 speed up. */
24352 /* This formula yields a nonzero result iff one of the bytes is zero.
24353 This saves three branches inside loop and many cycles. */
24361 align_4_label);
24364 {
24370 /* If zero is not in the first two bytes, move two bytes forward. */
24376 reg,
24377 tmpreg)));
24378 /* Emit lea manually to avoid clobbering of flags. */
24386 reg2,
24387 out)));
24388 }
24389 else
24390 {
24392 /* Is zero in the first two bytes? */
24399 pc_rtx);
24403 /* Not in the first two. Move two bytes forward. */
24409 }
24411 /* Avoid branch in fixing the byte. */
24419 }
24421 /* Expand strlen. */
24423 bool
24425 {
24428 /* The generic case of strlen expander is long. Avoid it's
24429 expanding unless TARGET_INLINE_ALL_STRINGOPS. */
24432 && !TARGET_INLINE_ALL_STRINGOPS
24442 {
24443 /* Well it seems that some optimizer does not combine a call like
24444 foo(strlen(bar), strlen(bar));
24445 when the move and the subtraction is done here. It does calculate
24446 the length just once when these instructions are done inside of
24447 output_strlen_unroll(). But I think since &bar[strlen(bar)] is
24448 often used and I use one fewer register for the lifetime of
24449 output_strlen_unroll() this is better. */
24455 /* strlensi_unroll_1 returns the address of the zero at the end of
24456 the string, like memchr(), so compute the length by subtracting
24457 the start address. */
24459 }
24460 else
24461 {
24462 rtx unspec;
24464 /* Can't use this if the user has appropriated eax, ecx, or edi. */
24477 /* If .md starts supporting :P, this can be done in .md. */
24483 }
24485 }
24487 /* For given symbol (function) construct code to compute address of it's PLT
24488 entry in large x86-64 PIC model. */
24489 static rtx
24491 {
24504 }
24506 rtx
24508 rtx callarg2,
24510 {
24511 unsigned int const cregs_size
24522 {
24523 #if TARGET_MACHO
24526 #endif
24527 }
24528 else
24529 {
24530 /* Static functions and indirect calls don't need the pic register. */
24532 && (!TARGET_64BIT
24538 }
24541 {
24545 }
24548 && !TARGET_PECOFF
24556 {
24559 }
24567 {
24571 }
24575 {
24579 UNSPEC_MS_TO_SYSV_CALL);
24582 {
24588 }
24589 }
24598 }
24600 /* Output the assembly for a call instruction. */
24604 {
24610 {
24613 /* SEH epilogue detection requires the indirect branch case
24614 to include REX.W. */
24617 else
24622 }
24624 /* SEH unwinding can require an extra nop to be emitted in several
24625 circumstances. Determine if we have one of those. */
24627 {
24628 rtx i;
24631 {
24632 /* If we get to another real insn, we don't need the nop. */
24636 /* If we get to the epilogue note, prevent a catch region from
24637 being adjacent to the standard epilogue sequence. If non-
24638 call-exceptions, we'll have done this during epilogue emission. */
24640 && !flag_non_call_exceptions
24642 {
24645 }
24646 }
24648 /* If we didn't find a real insn following the call, prevent the
24649 unwinder from looking into the next function. */
24652 }
24656 else
24665 }
24666 \f
24667 /* Clear stack slot assignments remembered from previous functions.
24668 This is called from INIT_EXPANDERS once before RTL is emitted for each
24669 function. */
24673 {
24681 }
24683 /* Return a MEM corresponding to a stack slot with mode MODE.
24684 Allocate a new slot if necessary.
24686 The RTL for a function can have several slots available: N is
24687 which slot to use. */
24689 rtx
24691 {
24708 }
24710 static void
24712 {
24718 }
24719 \f
24720 /* Check whether x86 address PARTS is a pc-relative address. */
24722 static bool
24724 {
24732 {
24734 {
24751 }
24752 }
24754 }
24756 /* Calculate the length of the memory address in the instruction encoding.
24757 Includes addr32 prefix, does not include the one-byte modrm, opcode,
24758 or other prefixes. We never generate addr32 prefix for LEA insn. */
24760 int
24762 {
24779 /* If this is not LEA instruction, add the length of addr32 prefix. */
24784 len++;
24798 /* Rule of thumb:
24799 - esp as the base always wants an index,
24800 - ebp as the base always wants a displacement,
24801 - r12 as the base always wants an index,
24802 - r13 as the base always wants a displacement. */
24804 /* Register Indirect. */
24806 {
24807 /* esp (for its index) and ebp (for its displacement) need
24808 the two-byte modrm form. Similarly for r12 and r13 in 64-bit
24809 code. */
24816 len++;
24817 }
24819 /* Direct Addressing. In 64-bit mode mod 00 r/m 5
24820 is not disp32, but disp32(%rip), so for disp32
24821 SIB byte is needed, unless print_operand_address
24822 optimizes it into disp32(%rip) or (%rip) is implied
24823 by UNSPEC. */
24825 {
24828 len++;
24829 }
24830 else
24831 {
24832 /* Find the length of the displacement constant. */
24834 {
24837 else
24839 }
24840 /* ebp always wants a displacement. Similarly r13. */
24842 len++;
24844 /* An index requires the two-byte modrm form.... */
24846 /* ...like esp (or r12), which always wants an index. */
24850 len++;
24851 }
24854 }
24856 /* Compute default value for "length_immediate" attribute. When SHORTFORM
24857 is set, expect that insn have 8bit immediate alternative. */
24858 int
24860 {
24866 {
24871 {
24874 {
24886 }
24888 {
24891 }
24892 }
24894 {
24904 /* Immediates for DImode instructions are encoded
24905 as 32bit sign extended values. */
24911 }
24912 }
24914 }
24916 /* Compute default value for "length_address" attribute. */
24917 int
24919 {
24923 {
24934 }
24939 {
24942 {
24947 constraints++;
24950 ;
24951 /* Skip ignored operands. */
24954 }
24956 }
24958 }
24960 /* Compute default value for "length_vex" attribute. It includes
24961 2 or 3 byte VEX prefix and 1 opcode byte. */
24963 int
24965 {
24968 /* Only 0f opcode can use 2 byte VEX prefix and VEX W bit uses 3
24969 byte VEX prefix. */
24973 /* We can always use 2 byte VEX prefix in 32bit. */
24981 {
24982 /* REX.W bit uses 3 byte VEX prefix. */
24986 }
24987 else
24988 {
24989 /* REX.X or REX.B bits use 3 byte VEX prefix. */
24993 }
24996 }
24997 \f
24998 /* Return the maximum number of instructions a cpu can issue. */
25000 static int
25002 {
25004 {
25034 }
25035 }
25037 /* A subroutine of ix86_adjust_cost -- return TRUE iff INSN reads flags set
25038 by DEP_INSN and nothing set by DEP_INSN. */
25040 static bool
25042 {
25045 /* Simplify the test for uninteresting insns. */
25053 {
25056 }
25061 {
25064 }
25065 else
25071 /* This test is true if the dependent insn reads the flags but
25072 not any other potentially set register. */
25080 }
25082 /* Return true iff USE_INSN has a memory address with operands set by
25083 SET_INSN. */
25085 bool
25087 {
25092 {
25095 }
25097 }
25099 /* Helper function for exact_store_load_dependency.
25100 Return true if addr is found in insn. */
25101 static bool
25103 {
25110 {
25116 CASE_CONST_ANY:
25125 }
25129 {
25131 {
25141 }
25142 }
25144 }
25146 /* Return true if there exists exact dependency for store & load, i.e.
25147 the same memory address is used in them. */
25148 static bool
25150 {
25164 }
25166 static int
25168 {
25174 /* Anti and output dependencies have zero cost on all CPUs. */
25180 /* If we can't recognize the insns, we can't really do anything. */
25188 {
25190 /* Address Generation Interlock adds a cycle of latency. */
25192 {
25203 }
25207 /* ??? Compares pair with jump/setcc. */
25211 /* Floating point stores require value to be ready one cycle earlier. */
25221 /* INT->FP conversion is expensive. */
25225 /* There is one cycle extra latency between an FP op and a store. */
25233 /* Show ability of reorder buffer to hide latency of load by executing
25234 in parallel with previous instruction in case
25235 previous instruction is not needed to compute the address. */
25238 {
25239 /* Claim moves to take one cycle, as core can issue one load
25240 at time and the next load can start cycle later. */
25245 cost--;
25246 }
25252 /* The esp dependency is resolved before the instruction is really
25253 finished. */
25258 /* INT->FP conversion is expensive. */
25262 /* Show ability of reorder buffer to hide latency of load by executing
25263 in parallel with previous instruction in case
25264 previous instruction is not needed to compute the address. */
25267 {
25268 /* Claim moves to take one cycle, as core can issue one load
25269 at time and the next load can start cycle later. */
25275 else
25277 }
25292 /* Stack engine allows to execute push&pop instructions in parall. */
25298 /* Show ability of reorder buffer to hide latency of load by executing
25299 in parallel with previous instruction in case
25300 previous instruction is not needed to compute the address. */
25303 {
25307 /* Because of the difference between the length of integer and
25308 floating unit pipeline preparation stages, the memory operands
25309 for floating point are cheaper.
25311 ??? For Athlon it the difference is most probably 2. */
25314 else
25319 else
25321 }
25330 /* Stack engine allows to execute push&pop instructions in parall. */
25335 /* Show ability of reorder buffer to hide latency of load by executing
25336 in parallel with previous instruction in case
25337 previous instruction is not needed to compute the address. */
25340 {
25343 else
25345 }
25352 /* Increase cost of integer loads. */
25355 {
25358 {
25361 else
25362 {
25363 /* Increase cost of ld/st for short int types only
25364 because of store forwarding issue. */
25368 {
25369 /* Increase cost of store/load insn if exact
25370 dependence exists and it is load insn. */
25375 }
25376 }
25377 }
25378 }
25382 }
25385 }
25387 /* How many alternative schedules to try. This should be as wide as the
25388 scheduling freedom in the DFA, but no wider. Making this value too
25389 large results extra work for the scheduler. */
25391 static int
25393 {
25395 {
25407 /* We use lookahead value 4 for BD both before and after reload
25408 schedules. Plan is to have value 8 included for O3. */
25417 /* Generally, we want haifa-sched:max_issue() to look ahead as far
25418 as many instructions can be executed on a cycle, i.e.,
25419 issue_rate. I wonder why tuning for many CPUs does not do this. */
25422 /* Don't use lookahead for pre-reload schedule to save compile time. */
25427 }
25428 }
25430 /* Return true if target platform supports macro-fusion. */
25432 static bool
25434 {
25436 }
25438 /* Check whether current microarchitecture support macro fusion
25439 for insn pair "CONDGEN + CONDJMP". Refer to
25440 "Intel Architectures Optimization Reference Manual". */
25442 static bool
25444 {
25463 else
25464 {
25469 {
25473 else
25475 }
25476 }
25483 /* Macro-fusion for cmp/test MEM-IMM + conditional jmp is not
25484 supported. */
25491 /* No fusion for RIP-relative address. */
25498 ix86_address parts;
25504 }
25509 /* Check whether conditional jump use Sign or Overflow Flags. */
25517 /* Return true for TYPE_TEST and TYPE_ICMP. */
25522 /* The following is the case that macro-fusion for alu + jmp. */
25526 /* No fusion for alu op with memory destination operand. */
25531 /* Macro-fusion for inc/dec + unsigned conditional jump is not
25532 supported. */
25541 }
25543 /* Try to reorder ready list to take advantage of Atom pipelined IMUL
25544 execution. It is applied if
25545 (1) IMUL instruction is on the top of list;
25546 (2) There exists the only producer of independent IMUL instruction in
25547 ready list.
25548 Return index of IMUL producer if it was found and -1 otherwise. */
25549 static int
25551 {
25553 sd_iterator_def sd_it;
25554 dep_t dep;
25561 /* Check that IMUL instruction is on the top of ready list. */
25570 /* Search for producer of independent IMUL instruction. */
25572 {
25576 /* Skip IMUL instruction. */
25586 {
25587 rtx con;
25598 {
25599 sd_iterator_def sd_it1;
25600 dep_t dep1;
25601 /* Check if there is no other dependee for IMUL. */
25604 {
25605 rtx pro;
25611 }
25614 }
25615 }
25618 }
25620 }
25622 /* Try to find the best candidate on the top of ready list if two insns
25623 have the same priority - candidate is best if its dependees were
25624 scheduled earlier. Applied for Silvermont only.
25625 Return true if top 2 insns must be interchanged. */
25626 static bool
25628 {
25631 rtx set;
25632 sd_iterator_def sd_it;
25633 dep_t dep;
25636 #define INSN_TICK(INSN) (HID (INSN)->tick)
25657 {
25660 /* Determine winner more precise. */
25662 {
25663 rtx pro;
25669 }
25671 {
25672 rtx pro;
25678 }
25681 {
25682 /* Determine winner - load must win. */
25688 }
25690 }
25692 #undef INSN_TICK
25693 }
25695 /* Perform possible reodering of ready list for Atom/Silvermont only.
25696 Return issue rate. */
25697 static int
25700 {
25704 rtx insn;
25707 /* Set up issue rate. */
25710 /* Do reodering for Atom/SLM only. */
25714 /* Nothing to do if ready list contains only 1 instruction. */
25718 /* Do reodering for post-reload scheduler only. */
25723 {
25728 /* Put IMUL producer (ready[index]) at the top of ready list. */
25734 }
25736 {
25740 /* Swap 2 top elements of ready list. */
25744 }
25746 }
25748 static bool
25751 /* Returns true if lhs of insn is HW function argument register and set up
25752 is_spilled to true if it is likely spilled HW register. */
25753 static bool
25755 {
25756 rtx dst;
25760 /* Call instructions are not movable, ignore it. */
25771 {
25772 /* Is it likely spilled HW register? */
25777 }
25779 }
25781 /* Add output dependencies for chain of function adjacent arguments if only
25782 there is a move to likely spilled HW register. Return first argument
25783 if at least one dependence was added or NULL otherwise. */
25784 static rtx
25786 {
25787 rtx insn;
25794 /* Find nearest to call argument passing instruction. */
25796 {
25805 }
25809 {
25816 {
25819 }
25821 {
25822 /* Add output depdendence between two function arguments if chain
25823 of output arguments contains likely spilled HW registers. */
25827 }
25828 else
25830 }
25834 }
25836 /* Add output or anti dependency from insn to first_arg to restrict its code
25837 motion. */
25838 static void
25840 {
25841 rtx set;
25842 rtx tmp;
25849 {
25850 /* Add output dependency to the first function argument. */
25853 }
25854 /* Add anti dependency. */
25856 }
25858 /* Avoid cross block motion of function argument through adding dependency
25859 from the first non-jump instruction in bb. */
25860 static void
25862 {
25866 {
25868 {
25871 {
25874 }
25875 }
25879 }
25880 }
25882 /* Hook for pre-reload schedule - avoid motion of function arguments
25883 passed in likely spilled HW registers. */
25884 static void
25886 {
25887 rtx insn;
25895 {
25898 {
25899 /* Add dependee for first argument to predecessors if only
25900 region contains more than one block. */
25904 /* Skip trivial regions and region head blocks that can have
25905 predecessors outside of region. */
25907 {
25908 edge e;
25909 edge_iterator ei;
25910 /* Assume that region is SCC, i.e. all immediate predecessors
25911 of non-head block are in the same region. */
25913 {
25914 /* Avoid creating of loop-carried dependencies through
25915 using topological odering in region. */
25918 }
25919 }
25923 }
25924 }
25927 }
25929 /* Hook for pre-reload schedule - set priority of moves from likely spilled
25930 HW registers to maximum, to schedule them at soon as possible. These are
25931 moves from function argument registers at the top of the function entry
25932 and moves from function return value registers after call. */
25933 static int
25935 {
25936 rtx set;
25946 {
25953 }
25956 }
25958 /* Model decoder of Core 2/i7.
25959 Below hooks for multipass scheduling (see haifa-sched.c:max_issue)
25960 track the instruction fetch block boundaries and make sure that long
25961 (9+ bytes) instructions are assigned to D0. */
25963 /* Maximum length of an insn that can be handled by
25964 a secondary decoder unit. '8' for Core 2/i7. */
25967 /* Ifetch block size, i.e., number of bytes decoder reads per cycle.
25968 '16' for Core 2/i7. */
25971 /* Maximum number of instructions decoder can handle per cycle.
25972 '6' for Core 2/i7. */
25976 ix86_first_cycle_multipass_data_t;
25978 const_ix86_first_cycle_multipass_data_t;
25980 /* A variable to store target state across calls to max_issue within
25981 one cycle. */
25985 /* Initialize DATA. */
25986 static void
25988 {
25989 ix86_first_cycle_multipass_data_t data
25996 }
25998 /* Advancing the cycle; reset ifetch block counts. */
25999 static void
26001 {
26008 }
26012 /* Filter out insns from ready_try that the core will not be able to issue
26013 on current cycle due to decoder. */
26014 static void
26015 core2i7_first_cycle_multipass_filter_ready_try
26018 {
26020 {
26021 rtx insn;
26031 (!first_cycle_insn_p
26033 /* ... or it would not fit into the ifetch block ... */
26035 /* ... or the decoder is full already ... */
26037 /* ... mask the insn out. */
26038 {
26043 }
26044 }
26045 }
26047 /* Prepare for a new round of multipass lookahead scheduling. */
26048 static void
26051 {
26052 ix86_first_cycle_multipass_data_t data
26054 const_ix86_first_cycle_multipass_data_t prev_data
26057 /* Restore the state from the end of the previous round. */
26061 /* Filter instructions that cannot be issued on current cycle due to
26062 decoder restrictions. */
26064 first_cycle_insn_p);
26065 }
26067 /* INSN is being issued in current solution. Account for its impact on
26068 the decoder model. */
26069 static void
26072 {
26073 ix86_first_cycle_multipass_data_t data
26075 const_ix86_first_cycle_multipass_data_t prev_data
26085 /* Allocate or resize the bitmap for storing INSN's effect on ready_try. */
26087 {
26090 }
26092 {
26096 }
26099 /* Filter out insns from ready_try that the core will not be able to issue
26100 on current cycle due to decoder. */
26103 }
26105 /* Revert the effect on ready_try. */
26106 static void
26110 {
26111 const_ix86_first_cycle_multipass_data_t data
26114 sbitmap_iterator sbi;
26118 {
26120 }
26121 }
26123 /* Save the result of multipass lookahead scheduling for the next round. */
26124 static void
26126 {
26127 const_ix86_first_cycle_multipass_data_t data
26129 ix86_first_cycle_multipass_data_t next_data
26133 {
26136 }
26137 }
26139 /* Deallocate target data. */
26140 static void
26142 {
26143 ix86_first_cycle_multipass_data_t data
26147 {
26151 }
26152 }
26154 /* Prepare for scheduling pass. */
26155 static void
26159 {
26160 /* Install scheduling hooks for current CPU. Some of these hooks are used
26161 in time-critical parts of the scheduler, so we only set them up when
26162 they are actually used. */
26164 {
26169 /* Do not perform multipass scheduling for pre-reload schedule
26170 to save compile time. */
26172 {
26188 /* Set decoder parameters. */
26193 }
26194 /* ... Fall through ... */
26204 }
26205 }
26207 \f
26208 /* Compute the alignment given to a constant that is being placed in memory.
26209 EXP is the constant and ALIGN is the alignment that the object would
26210 ordinarily have.
26211 The value of this function is used instead of that alignment to align
26212 the object. */
26214 int
26216 {
26219 {
26224 }
26230 }
26232 /* Compute the alignment for a static variable.
26233 TYPE is the data type, and ALIGN is the alignment that
26234 the object would ordinarily have. The value of this function is used
26235 instead of that alignment to align the object. */
26237 int
26239 {
26251 /* x86-64 ABI requires arrays greater than 16 bytes to be aligned
26252 to 16byte boundary. */
26254 {
26261 }
26267 {
26272 }
26274 {
26281 }
26286 {
26291 }
26294 {
26299 }
26302 }
26304 /* Compute the alignment for a local variable or a stack slot. EXP is
26305 the data type or decl itself, MODE is the widest mode available and
26306 ALIGN is the alignment that the object would ordinarily have. The
26307 value of this macro is used instead of that alignment to align the
26308 object. */
26310 unsigned int
26313 {
26317 {
26320 }
26321 else
26322 {
26325 }
26327 /* Don't do dynamic stack realignment for long long objects with
26328 -mpreferred-stack-boundary=2. */
26337 /* If TYPE is NULL, we are allocating a stack slot for caller-save
26338 register in MODE. We will return the largest alignment of XF
26339 and DF. */
26341 {
26345 }
26347 /* x86-64 ABI requires arrays greater than 16 bytes to be aligned
26348 to 16byte boundary. Exact wording is:
26350 An array uses the same alignment as its elements, except that a local or
26351 global array variable of length at least 16 bytes or
26352 a C99 variable-length array variable always has alignment of at least 16 bytes.
26354 This was added to allow use of aligned SSE instructions at arrays. This
26355 rule is meant for static storage (where compiler can not do the analysis
26356 by itself). We follow it for automatic variables only when convenient.
26357 We fully control everything in the function compiled and functions from
26358 other unit can not rely on the alignment.
26360 Exclude va_list type. It is the common case of local array where
26361 we can not benefit from the alignment.
26363 TODO: Probably one should optimize for size only when var is not escaping. */
26366 {
26376 }
26378 {
26383 }
26385 {
26391 }
26396 {
26401 }
26404 {
26410 }
26412 }
26414 /* Compute the minimum required alignment for dynamic stack realignment
26415 purposes for a local variable, parameter or a stack slot. EXP is
26416 the data type or decl itself, MODE is its mode and ALIGN is the
26417 alignment that the object would ordinarily have. */
26419 unsigned int
26422 {
26426 {
26429 }
26430 else
26431 {
26434 }
26439 /* Don't do dynamic stack realignment for long long objects with
26440 -mpreferred-stack-boundary=2. */
26447 }
26448 \f
26449 /* Find a location for the static chain incoming to a nested function.
26450 This is a register, unless all free registers are used by arguments. */
26452 static rtx
26454 {
26461 {
26462 /* We always use R10 in 64-bit mode. */
26464 }
26465 else
26466 {
26467 tree fntype;
26470 /* By default in 32-bit mode we use ECX to pass the static chain. */
26476 {
26477 /* Fastcall functions use ecx/edx for arguments, which leaves
26478 us with EAX for the static chain.
26479 Thiscall functions use ecx for arguments, which also
26480 leaves us with EAX for the static chain. */
26482 }
26484 {
26485 /* Thiscall functions use ecx for arguments, which leaves
26486 us with EAX and EDX for the static chain.
26487 We are using for abi-compatibility EAX. */
26489 }
26491 {
26492 /* For regparm 3, we have no free call-clobbered registers in
26493 which to store the static chain. In order to implement this,
26494 we have the trampoline push the static chain to the stack.
26495 However, we can't push a value below the return address when
26496 we call the nested function directly, so we have to use an
26497 alternate entry point. For this we use ESI, and have the
26498 alternate entry point push ESI, so that things appear the
26499 same once we're executing the nested function. */
26501 {
26507 }
26509 }
26510 }
26513 }
26515 /* Emit RTL insns to initialize the variable parts of a trampoline.
26516 FNDECL is the decl of the target address; M_TRAMP is a MEM for
26517 the trampoline, and CHAIN_VALUE is an RTX for the static chain
26518 to be passed to the target function. */
26520 static void
26522 {
26530 {
26533 /* Load the function address to r11. Try to load address using
26534 the shorter movl instead of movabs. We may want to support
26535 movq for kernel mode, but kernel does not use trampolines at
26536 the moment. FNADDR is a 32bit address and may not be in
26537 DImode when ptr_mode == SImode. Always use movl in this
26538 case. */
26541 {
26550 }
26551 else
26552 {
26559 }
26561 /* Load static chain using movabs to r10. Use the shorter movl
26562 instead of movabs when ptr_mode == SImode. */
26564 {
26567 }
26568 else
26569 {
26572 }
26581 /* Jump to r11; the last (unused) byte is a nop, only there to
26582 pad the write out to a single 32-bit store. */
26586 }
26587 else
26588 {
26591 /* Depending on the static chain location, either load a register
26592 with a constant, or push the constant to the stack. All of the
26593 instructions are the same size. */
26596 {
26598 {
26605 }
26606 }
26607 else
26622 /* Compute offset from the end of the jmp to the target function.
26623 In the case in which the trampoline stores the static chain on
26624 the stack, we need to skip the first insn which pushes the
26625 (call-saved) register static chain; this push is 1 byte. */
26632 }
26636 #ifdef HAVE_ENABLE_EXECUTE_STACK
26637 #ifdef CHECK_EXECUTE_STACK_ENABLED
26639 #endif
26642 #endif
26643 }
26644 \f
26645 /* The following file contains several enumerations and data structures
26646 built from the definitions in i386-builtin-types.def. */
26650 /* Table for the ix86 builtin non-function types. */
26653 /* Retrieve an element from the above table, building some of
26654 the types lazily. */
26656 static tree
26658 {
26670 {
26678 }
26679 else
26680 {
26686 else
26694 }
26698 }
26700 /* Table for the ix86 builtin function types. */
26703 /* Retrieve an element from the above table, building some of
26704 the types lazily. */
26706 static tree
26708 {
26709 tree type;
26718 {
26726 {
26729 }
26732 }
26733 else
26734 {
26740 }
26744 }
26747 /* Codes for all the SSE/MMX builtins. */
26748 enum ix86_builtins
26749 {
26750 IX86_BUILTIN_ADDPS,
26751 IX86_BUILTIN_ADDSS,
26752 IX86_BUILTIN_DIVPS,
26753 IX86_BUILTIN_DIVSS,
26754 IX86_BUILTIN_MULPS,
26755 IX86_BUILTIN_MULSS,
26756 IX86_BUILTIN_SUBPS,
26757 IX86_BUILTIN_SUBSS,
26759 IX86_BUILTIN_CMPEQPS,
26760 IX86_BUILTIN_CMPLTPS,
26761 IX86_BUILTIN_CMPLEPS,
26762 IX86_BUILTIN_CMPGTPS,
26763 IX86_BUILTIN_CMPGEPS,
26764 IX86_BUILTIN_CMPNEQPS,
26765 IX86_BUILTIN_CMPNLTPS,
26766 IX86_BUILTIN_CMPNLEPS,
26767 IX86_BUILTIN_CMPNGTPS,
26768 IX86_BUILTIN_CMPNGEPS,
26769 IX86_BUILTIN_CMPORDPS,
26770 IX86_BUILTIN_CMPUNORDPS,
26771 IX86_BUILTIN_CMPEQSS,
26772 IX86_BUILTIN_CMPLTSS,
26773 IX86_BUILTIN_CMPLESS,
26774 IX86_BUILTIN_CMPNEQSS,
26775 IX86_BUILTIN_CMPNLTSS,
26776 IX86_BUILTIN_CMPNLESS,
26777 IX86_BUILTIN_CMPORDSS,
26778 IX86_BUILTIN_CMPUNORDSS,
26780 IX86_BUILTIN_COMIEQSS,
26781 IX86_BUILTIN_COMILTSS,
26782 IX86_BUILTIN_COMILESS,
26783 IX86_BUILTIN_COMIGTSS,
26784 IX86_BUILTIN_COMIGESS,
26785 IX86_BUILTIN_COMINEQSS,
26786 IX86_BUILTIN_UCOMIEQSS,
26787 IX86_BUILTIN_UCOMILTSS,
26788 IX86_BUILTIN_UCOMILESS,
26789 IX86_BUILTIN_UCOMIGTSS,
26790 IX86_BUILTIN_UCOMIGESS,
26791 IX86_BUILTIN_UCOMINEQSS,
26793 IX86_BUILTIN_CVTPI2PS,
26794 IX86_BUILTIN_CVTPS2PI,
26795 IX86_BUILTIN_CVTSI2SS,
26796 IX86_BUILTIN_CVTSI642SS,
26797 IX86_BUILTIN_CVTSS2SI,
26798 IX86_BUILTIN_CVTSS2SI64,
26799 IX86_BUILTIN_CVTTPS2PI,
26800 IX86_BUILTIN_CVTTSS2SI,
26801 IX86_BUILTIN_CVTTSS2SI64,
26803 IX86_BUILTIN_MAXPS,
26804 IX86_BUILTIN_MAXSS,
26805 IX86_BUILTIN_MINPS,
26806 IX86_BUILTIN_MINSS,
26808 IX86_BUILTIN_LOADUPS,
26809 IX86_BUILTIN_STOREUPS,
26810 IX86_BUILTIN_MOVSS,
26812 IX86_BUILTIN_MOVHLPS,
26813 IX86_BUILTIN_MOVLHPS,
26814 IX86_BUILTIN_LOADHPS,
26815 IX86_BUILTIN_LOADLPS,
26816 IX86_BUILTIN_STOREHPS,
26817 IX86_BUILTIN_STORELPS,
26819 IX86_BUILTIN_MASKMOVQ,
26820 IX86_BUILTIN_MOVMSKPS,
26821 IX86_BUILTIN_PMOVMSKB,
26823 IX86_BUILTIN_MOVNTPS,
26824 IX86_BUILTIN_MOVNTQ,
26826 IX86_BUILTIN_LOADDQU,
26827 IX86_BUILTIN_STOREDQU,
26829 IX86_BUILTIN_PACKSSWB,
26830 IX86_BUILTIN_PACKSSDW,
26831 IX86_BUILTIN_PACKUSWB,
26833 IX86_BUILTIN_PADDB,
26834 IX86_BUILTIN_PADDW,
26835 IX86_BUILTIN_PADDD,
26836 IX86_BUILTIN_PADDQ,
26837 IX86_BUILTIN_PADDSB,
26838 IX86_BUILTIN_PADDSW,
26839 IX86_BUILTIN_PADDUSB,
26840 IX86_BUILTIN_PADDUSW,
26841 IX86_BUILTIN_PSUBB,
26842 IX86_BUILTIN_PSUBW,
26843 IX86_BUILTIN_PSUBD,
26844 IX86_BUILTIN_PSUBQ,
26845 IX86_BUILTIN_PSUBSB,
26846 IX86_BUILTIN_PSUBSW,
26847 IX86_BUILTIN_PSUBUSB,
26848 IX86_BUILTIN_PSUBUSW,
26850 IX86_BUILTIN_PAND,
26851 IX86_BUILTIN_PANDN,
26852 IX86_BUILTIN_POR,
26853 IX86_BUILTIN_PXOR,
26855 IX86_BUILTIN_PAVGB,
26856 IX86_BUILTIN_PAVGW,
26858 IX86_BUILTIN_PCMPEQB,
26859 IX86_BUILTIN_PCMPEQW,
26860 IX86_BUILTIN_PCMPEQD,
26861 IX86_BUILTIN_PCMPGTB,
26862 IX86_BUILTIN_PCMPGTW,
26863 IX86_BUILTIN_PCMPGTD,
26865 IX86_BUILTIN_PMADDWD,
26867 IX86_BUILTIN_PMAXSW,
26868 IX86_BUILTIN_PMAXUB,
26869 IX86_BUILTIN_PMINSW,
26870 IX86_BUILTIN_PMINUB,
26872 IX86_BUILTIN_PMULHUW,
26873 IX86_BUILTIN_PMULHW,
26874 IX86_BUILTIN_PMULLW,
26876 IX86_BUILTIN_PSADBW,
26877 IX86_BUILTIN_PSHUFW,
26879 IX86_BUILTIN_PSLLW,
26880 IX86_BUILTIN_PSLLD,
26881 IX86_BUILTIN_PSLLQ,
26882 IX86_BUILTIN_PSRAW,
26883 IX86_BUILTIN_PSRAD,
26884 IX86_BUILTIN_PSRLW,
26885 IX86_BUILTIN_PSRLD,
26886 IX86_BUILTIN_PSRLQ,
26887 IX86_BUILTIN_PSLLWI,
26888 IX86_BUILTIN_PSLLDI,
26889 IX86_BUILTIN_PSLLQI,
26890 IX86_BUILTIN_PSRAWI,
26891 IX86_BUILTIN_PSRADI,
26892 IX86_BUILTIN_PSRLWI,
26893 IX86_BUILTIN_PSRLDI,
26894 IX86_BUILTIN_PSRLQI,
26896 IX86_BUILTIN_PUNPCKHBW,
26897 IX86_BUILTIN_PUNPCKHWD,
26898 IX86_BUILTIN_PUNPCKHDQ,
26899 IX86_BUILTIN_PUNPCKLBW,
26900 IX86_BUILTIN_PUNPCKLWD,
26901 IX86_BUILTIN_PUNPCKLDQ,
26903 IX86_BUILTIN_SHUFPS,
26905 IX86_BUILTIN_RCPPS,
26906 IX86_BUILTIN_RCPSS,
26907 IX86_BUILTIN_RSQRTPS,
26908 IX86_BUILTIN_RSQRTPS_NR,
26909 IX86_BUILTIN_RSQRTSS,
26910 IX86_BUILTIN_RSQRTF,
26911 IX86_BUILTIN_SQRTPS,
26912 IX86_BUILTIN_SQRTPS_NR,
26913 IX86_BUILTIN_SQRTSS,
26915 IX86_BUILTIN_UNPCKHPS,
26916 IX86_BUILTIN_UNPCKLPS,
26918 IX86_BUILTIN_ANDPS,
26919 IX86_BUILTIN_ANDNPS,
26920 IX86_BUILTIN_ORPS,
26921 IX86_BUILTIN_XORPS,
26923 IX86_BUILTIN_EMMS,
26924 IX86_BUILTIN_LDMXCSR,
26925 IX86_BUILTIN_STMXCSR,
26926 IX86_BUILTIN_SFENCE,
26928 IX86_BUILTIN_FXSAVE,
26929 IX86_BUILTIN_FXRSTOR,
26930 IX86_BUILTIN_FXSAVE64,
26931 IX86_BUILTIN_FXRSTOR64,
26933 IX86_BUILTIN_XSAVE,
26934 IX86_BUILTIN_XRSTOR,
26935 IX86_BUILTIN_XSAVE64,
26936 IX86_BUILTIN_XRSTOR64,
26938 IX86_BUILTIN_XSAVEOPT,
26939 IX86_BUILTIN_XSAVEOPT64,
26941 /* 3DNow! Original */
26942 IX86_BUILTIN_FEMMS,
26943 IX86_BUILTIN_PAVGUSB,
26944 IX86_BUILTIN_PF2ID,
26945 IX86_BUILTIN_PFACC,
26946 IX86_BUILTIN_PFADD,
26947 IX86_BUILTIN_PFCMPEQ,
26948 IX86_BUILTIN_PFCMPGE,
26949 IX86_BUILTIN_PFCMPGT,
26950 IX86_BUILTIN_PFMAX,
26951 IX86_BUILTIN_PFMIN,
26952 IX86_BUILTIN_PFMUL,
26953 IX86_BUILTIN_PFRCP,
26954 IX86_BUILTIN_PFRCPIT1,
26955 IX86_BUILTIN_PFRCPIT2,
26956 IX86_BUILTIN_PFRSQIT1,
26957 IX86_BUILTIN_PFRSQRT,
26958 IX86_BUILTIN_PFSUB,
26959 IX86_BUILTIN_PFSUBR,
26960 IX86_BUILTIN_PI2FD,
26961 IX86_BUILTIN_PMULHRW,
26963 /* 3DNow! Athlon Extensions */
26964 IX86_BUILTIN_PF2IW,
26965 IX86_BUILTIN_PFNACC,
26966 IX86_BUILTIN_PFPNACC,
26967 IX86_BUILTIN_PI2FW,
26968 IX86_BUILTIN_PSWAPDSI,
26969 IX86_BUILTIN_PSWAPDSF,
26971 /* SSE2 */
26972 IX86_BUILTIN_ADDPD,
26973 IX86_BUILTIN_ADDSD,
26974 IX86_BUILTIN_DIVPD,
26975 IX86_BUILTIN_DIVSD,
26976 IX86_BUILTIN_MULPD,
26977 IX86_BUILTIN_MULSD,
26978 IX86_BUILTIN_SUBPD,
26979 IX86_BUILTIN_SUBSD,
26981 IX86_BUILTIN_CMPEQPD,
26982 IX86_BUILTIN_CMPLTPD,
26983 IX86_BUILTIN_CMPLEPD,
26984 IX86_BUILTIN_CMPGTPD,
26985 IX86_BUILTIN_CMPGEPD,
26986 IX86_BUILTIN_CMPNEQPD,
26987 IX86_BUILTIN_CMPNLTPD,
26988 IX86_BUILTIN_CMPNLEPD,
26989 IX86_BUILTIN_CMPNGTPD,
26990 IX86_BUILTIN_CMPNGEPD,
26991 IX86_BUILTIN_CMPORDPD,
26992 IX86_BUILTIN_CMPUNORDPD,
26993 IX86_BUILTIN_CMPEQSD,
26994 IX86_BUILTIN_CMPLTSD,
26995 IX86_BUILTIN_CMPLESD,
26996 IX86_BUILTIN_CMPNEQSD,
26997 IX86_BUILTIN_CMPNLTSD,
26998 IX86_BUILTIN_CMPNLESD,
26999 IX86_BUILTIN_CMPORDSD,
27000 IX86_BUILTIN_CMPUNORDSD,
27002 IX86_BUILTIN_COMIEQSD,
27003 IX86_BUILTIN_COMILTSD,
27004 IX86_BUILTIN_COMILESD,
27005 IX86_BUILTIN_COMIGTSD,
27006 IX86_BUILTIN_COMIGESD,
27007 IX86_BUILTIN_COMINEQSD,
27008 IX86_BUILTIN_UCOMIEQSD,
27009 IX86_BUILTIN_UCOMILTSD,
27010 IX86_BUILTIN_UCOMILESD,
27011 IX86_BUILTIN_UCOMIGTSD,
27012 IX86_BUILTIN_UCOMIGESD,
27013 IX86_BUILTIN_UCOMINEQSD,
27015 IX86_BUILTIN_MAXPD,
27016 IX86_BUILTIN_MAXSD,
27017 IX86_BUILTIN_MINPD,
27018 IX86_BUILTIN_MINSD,
27020 IX86_BUILTIN_ANDPD,
27021 IX86_BUILTIN_ANDNPD,
27022 IX86_BUILTIN_ORPD,
27023 IX86_BUILTIN_XORPD,
27025 IX86_BUILTIN_SQRTPD,
27026 IX86_BUILTIN_SQRTSD,
27028 IX86_BUILTIN_UNPCKHPD,
27029 IX86_BUILTIN_UNPCKLPD,
27031 IX86_BUILTIN_SHUFPD,
27033 IX86_BUILTIN_LOADUPD,
27034 IX86_BUILTIN_STOREUPD,
27035 IX86_BUILTIN_MOVSD,
27037 IX86_BUILTIN_LOADHPD,
27038 IX86_BUILTIN_LOADLPD,
27040 IX86_BUILTIN_CVTDQ2PD,
27041 IX86_BUILTIN_CVTDQ2PS,
27043 IX86_BUILTIN_CVTPD2DQ,
27044 IX86_BUILTIN_CVTPD2PI,
27045 IX86_BUILTIN_CVTPD2PS,
27046 IX86_BUILTIN_CVTTPD2DQ,
27047 IX86_BUILTIN_CVTTPD2PI,
27049 IX86_BUILTIN_CVTPI2PD,
27050 IX86_BUILTIN_CVTSI2SD,
27051 IX86_BUILTIN_CVTSI642SD,
27053 IX86_BUILTIN_CVTSD2SI,
27054 IX86_BUILTIN_CVTSD2SI64,
27055 IX86_BUILTIN_CVTSD2SS,
27056 IX86_BUILTIN_CVTSS2SD,
27057 IX86_BUILTIN_CVTTSD2SI,
27058 IX86_BUILTIN_CVTTSD2SI64,
27060 IX86_BUILTIN_CVTPS2DQ,
27061 IX86_BUILTIN_CVTPS2PD,
27062 IX86_BUILTIN_CVTTPS2DQ,
27064 IX86_BUILTIN_MOVNTI,
27065 IX86_BUILTIN_MOVNTI64,
27066 IX86_BUILTIN_MOVNTPD,
27067 IX86_BUILTIN_MOVNTDQ,
27069 IX86_BUILTIN_MOVQ128,
27071 /* SSE2 MMX */
27072 IX86_BUILTIN_MASKMOVDQU,
27073 IX86_BUILTIN_MOVMSKPD,
27074 IX86_BUILTIN_PMOVMSKB128,
27076 IX86_BUILTIN_PACKSSWB128,
27077 IX86_BUILTIN_PACKSSDW128,
27078 IX86_BUILTIN_PACKUSWB128,
27080 IX86_BUILTIN_PADDB128,
27081 IX86_BUILTIN_PADDW128,
27082 IX86_BUILTIN_PADDD128,
27083 IX86_BUILTIN_PADDQ128,
27084 IX86_BUILTIN_PADDSB128,
27085 IX86_BUILTIN_PADDSW128,
27086 IX86_BUILTIN_PADDUSB128,
27087 IX86_BUILTIN_PADDUSW128,
27088 IX86_BUILTIN_PSUBB128,
27089 IX86_BUILTIN_PSUBW128,
27090 IX86_BUILTIN_PSUBD128,
27091 IX86_BUILTIN_PSUBQ128,
27092 IX86_BUILTIN_PSUBSB128,
27093 IX86_BUILTIN_PSUBSW128,
27094 IX86_BUILTIN_PSUBUSB128,
27095 IX86_BUILTIN_PSUBUSW128,
27097 IX86_BUILTIN_PAND128,
27098 IX86_BUILTIN_PANDN128,
27099 IX86_BUILTIN_POR128,
27100 IX86_BUILTIN_PXOR128,
27102 IX86_BUILTIN_PAVGB128,
27103 IX86_BUILTIN_PAVGW128,
27105 IX86_BUILTIN_PCMPEQB128,
27106 IX86_BUILTIN_PCMPEQW128,
27107 IX86_BUILTIN_PCMPEQD128,
27108 IX86_BUILTIN_PCMPGTB128,
27109 IX86_BUILTIN_PCMPGTW128,
27110 IX86_BUILTIN_PCMPGTD128,
27112 IX86_BUILTIN_PMADDWD128,
27114 IX86_BUILTIN_PMAXSW128,
27115 IX86_BUILTIN_PMAXUB128,
27116 IX86_BUILTIN_PMINSW128,
27117 IX86_BUILTIN_PMINUB128,
27119 IX86_BUILTIN_PMULUDQ,
27120 IX86_BUILTIN_PMULUDQ128,
27121 IX86_BUILTIN_PMULHUW128,
27122 IX86_BUILTIN_PMULHW128,
27123 IX86_BUILTIN_PMULLW128,
27125 IX86_BUILTIN_PSADBW128,
27126 IX86_BUILTIN_PSHUFHW,
27127 IX86_BUILTIN_PSHUFLW,
27128 IX86_BUILTIN_PSHUFD,
27130 IX86_BUILTIN_PSLLDQI128,
27131 IX86_BUILTIN_PSLLWI128,
27132 IX86_BUILTIN_PSLLDI128,
27133 IX86_BUILTIN_PSLLQI128,
27134 IX86_BUILTIN_PSRAWI128,
27135 IX86_BUILTIN_PSRADI128,
27136 IX86_BUILTIN_PSRLDQI128,
27137 IX86_BUILTIN_PSRLWI128,
27138 IX86_BUILTIN_PSRLDI128,
27139 IX86_BUILTIN_PSRLQI128,
27141 IX86_BUILTIN_PSLLDQ128,
27142 IX86_BUILTIN_PSLLW128,
27143 IX86_BUILTIN_PSLLD128,
27144 IX86_BUILTIN_PSLLQ128,
27145 IX86_BUILTIN_PSRAW128,
27146 IX86_BUILTIN_PSRAD128,
27147 IX86_BUILTIN_PSRLW128,
27148 IX86_BUILTIN_PSRLD128,
27149 IX86_BUILTIN_PSRLQ128,
27151 IX86_BUILTIN_PUNPCKHBW128,
27152 IX86_BUILTIN_PUNPCKHWD128,
27153 IX86_BUILTIN_PUNPCKHDQ128,
27154 IX86_BUILTIN_PUNPCKHQDQ128,
27155 IX86_BUILTIN_PUNPCKLBW128,
27156 IX86_BUILTIN_PUNPCKLWD128,
27157 IX86_BUILTIN_PUNPCKLDQ128,
27158 IX86_BUILTIN_PUNPCKLQDQ128,
27160 IX86_BUILTIN_CLFLUSH,
27161 IX86_BUILTIN_MFENCE,
27162 IX86_BUILTIN_LFENCE,
27163 IX86_BUILTIN_PAUSE,
27165 IX86_BUILTIN_FNSTENV,
27166 IX86_BUILTIN_FLDENV,
27167 IX86_BUILTIN_FNSTSW,
27168 IX86_BUILTIN_FNCLEX,
27170 IX86_BUILTIN_BSRSI,
27171 IX86_BUILTIN_BSRDI,
27172 IX86_BUILTIN_RDPMC,
27173 IX86_BUILTIN_RDTSC,
27174 IX86_BUILTIN_RDTSCP,
27175 IX86_BUILTIN_ROLQI,
27176 IX86_BUILTIN_ROLHI,
27177 IX86_BUILTIN_RORQI,
27178 IX86_BUILTIN_RORHI,
27180 /* SSE3. */
27181 IX86_BUILTIN_ADDSUBPS,
27182 IX86_BUILTIN_HADDPS,
27183 IX86_BUILTIN_HSUBPS,
27184 IX86_BUILTIN_MOVSHDUP,
27185 IX86_BUILTIN_MOVSLDUP,
27186 IX86_BUILTIN_ADDSUBPD,
27187 IX86_BUILTIN_HADDPD,
27188 IX86_BUILTIN_HSUBPD,
27189 IX86_BUILTIN_LDDQU,
27191 IX86_BUILTIN_MONITOR,
27192 IX86_BUILTIN_MWAIT,
27194 /* SSSE3. */
27195 IX86_BUILTIN_PHADDW,
27196 IX86_BUILTIN_PHADDD,
27197 IX86_BUILTIN_PHADDSW,
27198 IX86_BUILTIN_PHSUBW,
27199 IX86_BUILTIN_PHSUBD,
27200 IX86_BUILTIN_PHSUBSW,
27201 IX86_BUILTIN_PMADDUBSW,
27202 IX86_BUILTIN_PMULHRSW,
27203 IX86_BUILTIN_PSHUFB,
27204 IX86_BUILTIN_PSIGNB,
27205 IX86_BUILTIN_PSIGNW,
27206 IX86_BUILTIN_PSIGND,
27207 IX86_BUILTIN_PALIGNR,
27208 IX86_BUILTIN_PABSB,
27209 IX86_BUILTIN_PABSW,
27210 IX86_BUILTIN_PABSD,
27212 IX86_BUILTIN_PHADDW128,
27213 IX86_BUILTIN_PHADDD128,
27214 IX86_BUILTIN_PHADDSW128,
27215 IX86_BUILTIN_PHSUBW128,
27216 IX86_BUILTIN_PHSUBD128,
27217 IX86_BUILTIN_PHSUBSW128,
27218 IX86_BUILTIN_PMADDUBSW128,
27219 IX86_BUILTIN_PMULHRSW128,
27220 IX86_BUILTIN_PSHUFB128,
27221 IX86_BUILTIN_PSIGNB128,
27222 IX86_BUILTIN_PSIGNW128,
27223 IX86_BUILTIN_PSIGND128,
27224 IX86_BUILTIN_PALIGNR128,
27225 IX86_BUILTIN_PABSB128,
27226 IX86_BUILTIN_PABSW128,
27227 IX86_BUILTIN_PABSD128,
27229 /* AMDFAM10 - SSE4A New Instructions. */
27230 IX86_BUILTIN_MOVNTSD,
27231 IX86_BUILTIN_MOVNTSS,
27232 IX86_BUILTIN_EXTRQI,
27233 IX86_BUILTIN_EXTRQ,
27234 IX86_BUILTIN_INSERTQI,
27235 IX86_BUILTIN_INSERTQ,
27237 /* SSE4.1. */
27238 IX86_BUILTIN_BLENDPD,
27239 IX86_BUILTIN_BLENDPS,
27240 IX86_BUILTIN_BLENDVPD,
27241 IX86_BUILTIN_BLENDVPS,
27242 IX86_BUILTIN_PBLENDVB128,
27243 IX86_BUILTIN_PBLENDW128,
27245 IX86_BUILTIN_DPPD,
27246 IX86_BUILTIN_DPPS,
27248 IX86_BUILTIN_INSERTPS128,
27250 IX86_BUILTIN_MOVNTDQA,
27251 IX86_BUILTIN_MPSADBW128,
27252 IX86_BUILTIN_PACKUSDW128,
27253 IX86_BUILTIN_PCMPEQQ,
27254 IX86_BUILTIN_PHMINPOSUW128,
27256 IX86_BUILTIN_PMAXSB128,
27257 IX86_BUILTIN_PMAXSD128,
27258 IX86_BUILTIN_PMAXUD128,
27259 IX86_BUILTIN_PMAXUW128,
27261 IX86_BUILTIN_PMINSB128,
27262 IX86_BUILTIN_PMINSD128,
27263 IX86_BUILTIN_PMINUD128,
27264 IX86_BUILTIN_PMINUW128,
27266 IX86_BUILTIN_PMOVSXBW128,
27267 IX86_BUILTIN_PMOVSXBD128,
27268 IX86_BUILTIN_PMOVSXBQ128,
27269 IX86_BUILTIN_PMOVSXWD128,
27270 IX86_BUILTIN_PMOVSXWQ128,
27271 IX86_BUILTIN_PMOVSXDQ128,
27273 IX86_BUILTIN_PMOVZXBW128,
27274 IX86_BUILTIN_PMOVZXBD128,
27275 IX86_BUILTIN_PMOVZXBQ128,
27276 IX86_BUILTIN_PMOVZXWD128,
27277 IX86_BUILTIN_PMOVZXWQ128,
27278 IX86_BUILTIN_PMOVZXDQ128,
27280 IX86_BUILTIN_PMULDQ128,
27281 IX86_BUILTIN_PMULLD128,
27283 IX86_BUILTIN_ROUNDSD,
27284 IX86_BUILTIN_ROUNDSS,
27286 IX86_BUILTIN_ROUNDPD,
27287 IX86_BUILTIN_ROUNDPS,
27289 IX86_BUILTIN_FLOORPD,
27290 IX86_BUILTIN_CEILPD,
27291 IX86_BUILTIN_TRUNCPD,
27292 IX86_BUILTIN_RINTPD,
27293 IX86_BUILTIN_ROUNDPD_AZ,
27295 IX86_BUILTIN_FLOORPD_VEC_PACK_SFIX,
27296 IX86_BUILTIN_CEILPD_VEC_PACK_SFIX,
27297 IX86_BUILTIN_ROUNDPD_AZ_VEC_PACK_SFIX,
27299 IX86_BUILTIN_FLOORPS,
27300 IX86_BUILTIN_CEILPS,
27301 IX86_BUILTIN_TRUNCPS,
27302 IX86_BUILTIN_RINTPS,
27303 IX86_BUILTIN_ROUNDPS_AZ,
27305 IX86_BUILTIN_FLOORPS_SFIX,
27306 IX86_BUILTIN_CEILPS_SFIX,
27307 IX86_BUILTIN_ROUNDPS_AZ_SFIX,
27309 IX86_BUILTIN_PTESTZ,
27310 IX86_BUILTIN_PTESTC,
27311 IX86_BUILTIN_PTESTNZC,
27313 IX86_BUILTIN_VEC_INIT_V2SI,
27314 IX86_BUILTIN_VEC_INIT_V4HI,
27315 IX86_BUILTIN_VEC_INIT_V8QI,
27316 IX86_BUILTIN_VEC_EXT_V2DF,
27317 IX86_BUILTIN_VEC_EXT_V2DI,
27318 IX86_BUILTIN_VEC_EXT_V4SF,
27319 IX86_BUILTIN_VEC_EXT_V4SI,
27320 IX86_BUILTIN_VEC_EXT_V8HI,
27321 IX86_BUILTIN_VEC_EXT_V2SI,
27322 IX86_BUILTIN_VEC_EXT_V4HI,
27323 IX86_BUILTIN_VEC_EXT_V16QI,
27324 IX86_BUILTIN_VEC_SET_V2DI,
27325 IX86_BUILTIN_VEC_SET_V4SF,
27326 IX86_BUILTIN_VEC_SET_V4SI,
27327 IX86_BUILTIN_VEC_SET_V8HI,
27328 IX86_BUILTIN_VEC_SET_V4HI,
27329 IX86_BUILTIN_VEC_SET_V16QI,
27331 IX86_BUILTIN_VEC_PACK_SFIX,
27332 IX86_BUILTIN_VEC_PACK_SFIX256,
27334 /* SSE4.2. */
27335 IX86_BUILTIN_CRC32QI,
27336 IX86_BUILTIN_CRC32HI,
27337 IX86_BUILTIN_CRC32SI,
27338 IX86_BUILTIN_CRC32DI,
27340 IX86_BUILTIN_PCMPESTRI128,
27341 IX86_BUILTIN_PCMPESTRM128,
27342 IX86_BUILTIN_PCMPESTRA128,
27343 IX86_BUILTIN_PCMPESTRC128,
27344 IX86_BUILTIN_PCMPESTRO128,
27345 IX86_BUILTIN_PCMPESTRS128,
27346 IX86_BUILTIN_PCMPESTRZ128,
27347 IX86_BUILTIN_PCMPISTRI128,
27348 IX86_BUILTIN_PCMPISTRM128,
27349 IX86_BUILTIN_PCMPISTRA128,
27350 IX86_BUILTIN_PCMPISTRC128,
27351 IX86_BUILTIN_PCMPISTRO128,
27352 IX86_BUILTIN_PCMPISTRS128,
27353 IX86_BUILTIN_PCMPISTRZ128,
27355 IX86_BUILTIN_PCMPGTQ,
27357 /* AES instructions */
27358 IX86_BUILTIN_AESENC128,
27359 IX86_BUILTIN_AESENCLAST128,
27360 IX86_BUILTIN_AESDEC128,
27361 IX86_BUILTIN_AESDECLAST128,
27362 IX86_BUILTIN_AESIMC128,
27363 IX86_BUILTIN_AESKEYGENASSIST128,
27365 /* PCLMUL instruction */
27366 IX86_BUILTIN_PCLMULQDQ128,
27368 /* AVX */
27369 IX86_BUILTIN_ADDPD256,
27370 IX86_BUILTIN_ADDPS256,
27371 IX86_BUILTIN_ADDSUBPD256,
27372 IX86_BUILTIN_ADDSUBPS256,
27373 IX86_BUILTIN_ANDPD256,
27374 IX86_BUILTIN_ANDPS256,
27375 IX86_BUILTIN_ANDNPD256,
27376 IX86_BUILTIN_ANDNPS256,
27377 IX86_BUILTIN_BLENDPD256,
27378 IX86_BUILTIN_BLENDPS256,
27379 IX86_BUILTIN_BLENDVPD256,
27380 IX86_BUILTIN_BLENDVPS256,
27381 IX86_BUILTIN_DIVPD256,
27382 IX86_BUILTIN_DIVPS256,
27383 IX86_BUILTIN_DPPS256,
27384 IX86_BUILTIN_HADDPD256,
27385 IX86_BUILTIN_HADDPS256,
27386 IX86_BUILTIN_HSUBPD256,
27387 IX86_BUILTIN_HSUBPS256,
27388 IX86_BUILTIN_MAXPD256,
27389 IX86_BUILTIN_MAXPS256,
27390 IX86_BUILTIN_MINPD256,
27391 IX86_BUILTIN_MINPS256,
27392 IX86_BUILTIN_MULPD256,
27393 IX86_BUILTIN_MULPS256,
27394 IX86_BUILTIN_ORPD256,
27395 IX86_BUILTIN_ORPS256,
27396 IX86_BUILTIN_SHUFPD256,
27397 IX86_BUILTIN_SHUFPS256,
27398 IX86_BUILTIN_SUBPD256,
27399 IX86_BUILTIN_SUBPS256,
27400 IX86_BUILTIN_XORPD256,
27401 IX86_BUILTIN_XORPS256,
27402 IX86_BUILTIN_CMPSD,
27403 IX86_BUILTIN_CMPSS,
27404 IX86_BUILTIN_CMPPD,
27405 IX86_BUILTIN_CMPPS,
27406 IX86_BUILTIN_CMPPD256,
27407 IX86_BUILTIN_CMPPS256,
27408 IX86_BUILTIN_CVTDQ2PD256,
27409 IX86_BUILTIN_CVTDQ2PS256,
27410 IX86_BUILTIN_CVTPD2PS256,
27411 IX86_BUILTIN_CVTPS2DQ256,
27412 IX86_BUILTIN_CVTPS2PD256,
27413 IX86_BUILTIN_CVTTPD2DQ256,
27414 IX86_BUILTIN_CVTPD2DQ256,
27415 IX86_BUILTIN_CVTTPS2DQ256,
27416 IX86_BUILTIN_EXTRACTF128PD256,
27417 IX86_BUILTIN_EXTRACTF128PS256,
27418 IX86_BUILTIN_EXTRACTF128SI256,
27419 IX86_BUILTIN_VZEROALL,
27420 IX86_BUILTIN_VZEROUPPER,
27421 IX86_BUILTIN_VPERMILVARPD,
27422 IX86_BUILTIN_VPERMILVARPS,
27423 IX86_BUILTIN_VPERMILVARPD256,
27424 IX86_BUILTIN_VPERMILVARPS256,
27425 IX86_BUILTIN_VPERMILPD,
27426 IX86_BUILTIN_VPERMILPS,
27427 IX86_BUILTIN_VPERMILPD256,
27428 IX86_BUILTIN_VPERMILPS256,
27429 IX86_BUILTIN_VPERMIL2PD,
27430 IX86_BUILTIN_VPERMIL2PS,
27431 IX86_BUILTIN_VPERMIL2PD256,
27432 IX86_BUILTIN_VPERMIL2PS256,
27433 IX86_BUILTIN_VPERM2F128PD256,
27434 IX86_BUILTIN_VPERM2F128PS256,
27435 IX86_BUILTIN_VPERM2F128SI256,
27436 IX86_BUILTIN_VBROADCASTSS,
27437 IX86_BUILTIN_VBROADCASTSD256,
27438 IX86_BUILTIN_VBROADCASTSS256,
27439 IX86_BUILTIN_VBROADCASTPD256,
27440 IX86_BUILTIN_VBROADCASTPS256,
27441 IX86_BUILTIN_VINSERTF128PD256,
27442 IX86_BUILTIN_VINSERTF128PS256,
27443 IX86_BUILTIN_VINSERTF128SI256,
27444 IX86_BUILTIN_LOADUPD256,
27445 IX86_BUILTIN_LOADUPS256,
27446 IX86_BUILTIN_STOREUPD256,
27447 IX86_BUILTIN_STOREUPS256,
27448 IX86_BUILTIN_LDDQU256,
27449 IX86_BUILTIN_MOVNTDQ256,
27450 IX86_BUILTIN_MOVNTPD256,
27451 IX86_BUILTIN_MOVNTPS256,
27452 IX86_BUILTIN_LOADDQU256,
27453 IX86_BUILTIN_STOREDQU256,
27454 IX86_BUILTIN_MASKLOADPD,
27455 IX86_BUILTIN_MASKLOADPS,
27456 IX86_BUILTIN_MASKSTOREPD,
27457 IX86_BUILTIN_MASKSTOREPS,
27458 IX86_BUILTIN_MASKLOADPD256,
27459 IX86_BUILTIN_MASKLOADPS256,
27460 IX86_BUILTIN_MASKSTOREPD256,
27461 IX86_BUILTIN_MASKSTOREPS256,
27462 IX86_BUILTIN_MOVSHDUP256,
27463 IX86_BUILTIN_MOVSLDUP256,
27464 IX86_BUILTIN_MOVDDUP256,
27466 IX86_BUILTIN_SQRTPD256,
27467 IX86_BUILTIN_SQRTPS256,
27468 IX86_BUILTIN_SQRTPS_NR256,
27469 IX86_BUILTIN_RSQRTPS256,
27470 IX86_BUILTIN_RSQRTPS_NR256,
27472 IX86_BUILTIN_RCPPS256,
27474 IX86_BUILTIN_ROUNDPD256,
27475 IX86_BUILTIN_ROUNDPS256,
27477 IX86_BUILTIN_FLOORPD256,
27478 IX86_BUILTIN_CEILPD256,
27479 IX86_BUILTIN_TRUNCPD256,
27480 IX86_BUILTIN_RINTPD256,
27481 IX86_BUILTIN_ROUNDPD_AZ256,
27483 IX86_BUILTIN_FLOORPD_VEC_PACK_SFIX256,
27484 IX86_BUILTIN_CEILPD_VEC_PACK_SFIX256,
27485 IX86_BUILTIN_ROUNDPD_AZ_VEC_PACK_SFIX256,
27487 IX86_BUILTIN_FLOORPS256,
27488 IX86_BUILTIN_CEILPS256,
27489 IX86_BUILTIN_TRUNCPS256,
27490 IX86_BUILTIN_RINTPS256,
27491 IX86_BUILTIN_ROUNDPS_AZ256,
27493 IX86_BUILTIN_FLOORPS_SFIX256,
27494 IX86_BUILTIN_CEILPS_SFIX256,
27495 IX86_BUILTIN_ROUNDPS_AZ_SFIX256,
27497 IX86_BUILTIN_UNPCKHPD256,
27498 IX86_BUILTIN_UNPCKLPD256,
27499 IX86_BUILTIN_UNPCKHPS256,
27500 IX86_BUILTIN_UNPCKLPS256,
27502 IX86_BUILTIN_SI256_SI,
27503 IX86_BUILTIN_PS256_PS,
27504 IX86_BUILTIN_PD256_PD,
27505 IX86_BUILTIN_SI_SI256,
27506 IX86_BUILTIN_PS_PS256,
27507 IX86_BUILTIN_PD_PD256,
27509 IX86_BUILTIN_VTESTZPD,
27510 IX86_BUILTIN_VTESTCPD,
27511 IX86_BUILTIN_VTESTNZCPD,
27512 IX86_BUILTIN_VTESTZPS,
27513 IX86_BUILTIN_VTESTCPS,
27514 IX86_BUILTIN_VTESTNZCPS,
27515 IX86_BUILTIN_VTESTZPD256,
27516 IX86_BUILTIN_VTESTCPD256,
27517 IX86_BUILTIN_VTESTNZCPD256,
27518 IX86_BUILTIN_VTESTZPS256,
27519 IX86_BUILTIN_VTESTCPS256,
27520 IX86_BUILTIN_VTESTNZCPS256,
27521 IX86_BUILTIN_PTESTZ256,
27522 IX86_BUILTIN_PTESTC256,
27523 IX86_BUILTIN_PTESTNZC256,
27525 IX86_BUILTIN_MOVMSKPD256,
27526 IX86_BUILTIN_MOVMSKPS256,
27528 /* AVX2 */
27529 IX86_BUILTIN_MPSADBW256,
27530 IX86_BUILTIN_PABSB256,
27531 IX86_BUILTIN_PABSW256,
27532 IX86_BUILTIN_PABSD256,
27533 IX86_BUILTIN_PACKSSDW256,
27534 IX86_BUILTIN_PACKSSWB256,
27535 IX86_BUILTIN_PACKUSDW256,
27536 IX86_BUILTIN_PACKUSWB256,
27537 IX86_BUILTIN_PADDB256,
27538 IX86_BUILTIN_PADDW256,
27539 IX86_BUILTIN_PADDD256,
27540 IX86_BUILTIN_PADDQ256,
27541 IX86_BUILTIN_PADDSB256,
27542 IX86_BUILTIN_PADDSW256,
27543 IX86_BUILTIN_PADDUSB256,
27544 IX86_BUILTIN_PADDUSW256,
27545 IX86_BUILTIN_PALIGNR256,
27546 IX86_BUILTIN_AND256I,
27547 IX86_BUILTIN_ANDNOT256I,
27548 IX86_BUILTIN_PAVGB256,
27549 IX86_BUILTIN_PAVGW256,
27550 IX86_BUILTIN_PBLENDVB256,
27551 IX86_BUILTIN_PBLENDVW256,
27552 IX86_BUILTIN_PCMPEQB256,
27553 IX86_BUILTIN_PCMPEQW256,
27554 IX86_BUILTIN_PCMPEQD256,
27555 IX86_BUILTIN_PCMPEQQ256,
27556 IX86_BUILTIN_PCMPGTB256,
27557 IX86_BUILTIN_PCMPGTW256,
27558 IX86_BUILTIN_PCMPGTD256,
27559 IX86_BUILTIN_PCMPGTQ256,
27560 IX86_BUILTIN_PHADDW256,
27561 IX86_BUILTIN_PHADDD256,
27562 IX86_BUILTIN_PHADDSW256,
27563 IX86_BUILTIN_PHSUBW256,
27564 IX86_BUILTIN_PHSUBD256,
27565 IX86_BUILTIN_PHSUBSW256,
27566 IX86_BUILTIN_PMADDUBSW256,
27567 IX86_BUILTIN_PMADDWD256,
27568 IX86_BUILTIN_PMAXSB256,
27569 IX86_BUILTIN_PMAXSW256,
27570 IX86_BUILTIN_PMAXSD256,
27571 IX86_BUILTIN_PMAXUB256,
27572 IX86_BUILTIN_PMAXUW256,
27573 IX86_BUILTIN_PMAXUD256,
27574 IX86_BUILTIN_PMINSB256,
27575 IX86_BUILTIN_PMINSW256,
27576 IX86_BUILTIN_PMINSD256,
27577 IX86_BUILTIN_PMINUB256,
27578 IX86_BUILTIN_PMINUW256,
27579 IX86_BUILTIN_PMINUD256,
27580 IX86_BUILTIN_PMOVMSKB256,
27581 IX86_BUILTIN_PMOVSXBW256,
27582 IX86_BUILTIN_PMOVSXBD256,
27583 IX86_BUILTIN_PMOVSXBQ256,
27584 IX86_BUILTIN_PMOVSXWD256,
27585 IX86_BUILTIN_PMOVSXWQ256,
27586 IX86_BUILTIN_PMOVSXDQ256,
27587 IX86_BUILTIN_PMOVZXBW256,
27588 IX86_BUILTIN_PMOVZXBD256,
27589 IX86_BUILTIN_PMOVZXBQ256,
27590 IX86_BUILTIN_PMOVZXWD256,
27591 IX86_BUILTIN_PMOVZXWQ256,
27592 IX86_BUILTIN_PMOVZXDQ256,
27593 IX86_BUILTIN_PMULDQ256,
27594 IX86_BUILTIN_PMULHRSW256,
27595 IX86_BUILTIN_PMULHUW256,
27596 IX86_BUILTIN_PMULHW256,
27597 IX86_BUILTIN_PMULLW256,
27598 IX86_BUILTIN_PMULLD256,
27599 IX86_BUILTIN_PMULUDQ256,
27600 IX86_BUILTIN_POR256,
27601 IX86_BUILTIN_PSADBW256,
27602 IX86_BUILTIN_PSHUFB256,
27603 IX86_BUILTIN_PSHUFD256,
27604 IX86_BUILTIN_PSHUFHW256,
27605 IX86_BUILTIN_PSHUFLW256,
27606 IX86_BUILTIN_PSIGNB256,
27607 IX86_BUILTIN_PSIGNW256,
27608 IX86_BUILTIN_PSIGND256,
27609 IX86_BUILTIN_PSLLDQI256,
27610 IX86_BUILTIN_PSLLWI256,
27611 IX86_BUILTIN_PSLLW256,
27612 IX86_BUILTIN_PSLLDI256,
27613 IX86_BUILTIN_PSLLD256,
27614 IX86_BUILTIN_PSLLQI256,
27615 IX86_BUILTIN_PSLLQ256,
27616 IX86_BUILTIN_PSRAWI256,
27617 IX86_BUILTIN_PSRAW256,
27618 IX86_BUILTIN_PSRADI256,
27619 IX86_BUILTIN_PSRAD256,
27620 IX86_BUILTIN_PSRLDQI256,
27621 IX86_BUILTIN_PSRLWI256,
27622 IX86_BUILTIN_PSRLW256,
27623 IX86_BUILTIN_PSRLDI256,
27624 IX86_BUILTIN_PSRLD256,
27625 IX86_BUILTIN_PSRLQI256,
27626 IX86_BUILTIN_PSRLQ256,
27627 IX86_BUILTIN_PSUBB256,
27628 IX86_BUILTIN_PSUBW256,
27629 IX86_BUILTIN_PSUBD256,
27630 IX86_BUILTIN_PSUBQ256,
27631 IX86_BUILTIN_PSUBSB256,
27632 IX86_BUILTIN_PSUBSW256,
27633 IX86_BUILTIN_PSUBUSB256,
27634 IX86_BUILTIN_PSUBUSW256,
27635 IX86_BUILTIN_PUNPCKHBW256,
27636 IX86_BUILTIN_PUNPCKHWD256,
27637 IX86_BUILTIN_PUNPCKHDQ256,
27638 IX86_BUILTIN_PUNPCKHQDQ256,
27639 IX86_BUILTIN_PUNPCKLBW256,
27640 IX86_BUILTIN_PUNPCKLWD256,
27641 IX86_BUILTIN_PUNPCKLDQ256,
27642 IX86_BUILTIN_PUNPCKLQDQ256,
27643 IX86_BUILTIN_PXOR256,
27644 IX86_BUILTIN_MOVNTDQA256,
27645 IX86_BUILTIN_VBROADCASTSS_PS,
27646 IX86_BUILTIN_VBROADCASTSS_PS256,
27647 IX86_BUILTIN_VBROADCASTSD_PD256,
27648 IX86_BUILTIN_VBROADCASTSI256,
27649 IX86_BUILTIN_PBLENDD256,
27650 IX86_BUILTIN_PBLENDD128,
27651 IX86_BUILTIN_PBROADCASTB256,
27652 IX86_BUILTIN_PBROADCASTW256,
27653 IX86_BUILTIN_PBROADCASTD256,
27654 IX86_BUILTIN_PBROADCASTQ256,
27655 IX86_BUILTIN_PBROADCASTB128,
27656 IX86_BUILTIN_PBROADCASTW128,
27657 IX86_BUILTIN_PBROADCASTD128,
27658 IX86_BUILTIN_PBROADCASTQ128,
27659 IX86_BUILTIN_VPERMVARSI256,
27660 IX86_BUILTIN_VPERMDF256,
27661 IX86_BUILTIN_VPERMVARSF256,
27662 IX86_BUILTIN_VPERMDI256,
27663 IX86_BUILTIN_VPERMTI256,
27664 IX86_BUILTIN_VEXTRACT128I256,
27665 IX86_BUILTIN_VINSERT128I256,
27666 IX86_BUILTIN_MASKLOADD,
27667 IX86_BUILTIN_MASKLOADQ,
27668 IX86_BUILTIN_MASKLOADD256,
27669 IX86_BUILTIN_MASKLOADQ256,
27670 IX86_BUILTIN_MASKSTORED,
27671 IX86_BUILTIN_MASKSTOREQ,
27672 IX86_BUILTIN_MASKSTORED256,
27673 IX86_BUILTIN_MASKSTOREQ256,
27674 IX86_BUILTIN_PSLLVV4DI,
27675 IX86_BUILTIN_PSLLVV2DI,
27676 IX86_BUILTIN_PSLLVV8SI,
27677 IX86_BUILTIN_PSLLVV4SI,
27678 IX86_BUILTIN_PSRAVV8SI,
27679 IX86_BUILTIN_PSRAVV4SI,
27680 IX86_BUILTIN_PSRLVV4DI,
27681 IX86_BUILTIN_PSRLVV2DI,
27682 IX86_BUILTIN_PSRLVV8SI,
27683 IX86_BUILTIN_PSRLVV4SI,
27685 IX86_BUILTIN_GATHERSIV2DF,
27686 IX86_BUILTIN_GATHERSIV4DF,
27687 IX86_BUILTIN_GATHERDIV2DF,
27688 IX86_BUILTIN_GATHERDIV4DF,
27689 IX86_BUILTIN_GATHERSIV4SF,
27690 IX86_BUILTIN_GATHERSIV8SF,
27691 IX86_BUILTIN_GATHERDIV4SF,
27692 IX86_BUILTIN_GATHERDIV8SF,
27693 IX86_BUILTIN_GATHERSIV2DI,
27694 IX86_BUILTIN_GATHERSIV4DI,
27695 IX86_BUILTIN_GATHERDIV2DI,
27696 IX86_BUILTIN_GATHERDIV4DI,
27697 IX86_BUILTIN_GATHERSIV4SI,
27698 IX86_BUILTIN_GATHERSIV8SI,
27699 IX86_BUILTIN_GATHERDIV4SI,
27700 IX86_BUILTIN_GATHERDIV8SI,
27702 /* Alternate 4 element gather for the vectorizer where
27703 all operands are 32-byte wide. */
27704 IX86_BUILTIN_GATHERALTSIV4DF,
27705 IX86_BUILTIN_GATHERALTDIV8SF,
27706 IX86_BUILTIN_GATHERALTSIV4DI,
27707 IX86_BUILTIN_GATHERALTDIV8SI,
27709 /* TFmode support builtins. */
27710 IX86_BUILTIN_INFQ,
27711 IX86_BUILTIN_HUGE_VALQ,
27712 IX86_BUILTIN_FABSQ,
27713 IX86_BUILTIN_COPYSIGNQ,
27715 /* Vectorizer support builtins. */
27716 IX86_BUILTIN_CPYSGNPS,
27717 IX86_BUILTIN_CPYSGNPD,
27718 IX86_BUILTIN_CPYSGNPS256,
27719 IX86_BUILTIN_CPYSGNPD256,
27721 /* FMA4 instructions. */
27722 IX86_BUILTIN_VFMADDSS,
27723 IX86_BUILTIN_VFMADDSD,
27724 IX86_BUILTIN_VFMADDPS,
27725 IX86_BUILTIN_VFMADDPD,
27726 IX86_BUILTIN_VFMADDPS256,
27727 IX86_BUILTIN_VFMADDPD256,
27728 IX86_BUILTIN_VFMADDSUBPS,
27729 IX86_BUILTIN_VFMADDSUBPD,
27730 IX86_BUILTIN_VFMADDSUBPS256,
27731 IX86_BUILTIN_VFMADDSUBPD256,
27733 /* FMA3 instructions. */
27734 IX86_BUILTIN_VFMADDSS3,
27735 IX86_BUILTIN_VFMADDSD3,
27737 /* XOP instructions. */
27738 IX86_BUILTIN_VPCMOV,
27739 IX86_BUILTIN_VPCMOV_V2DI,
27740 IX86_BUILTIN_VPCMOV_V4SI,
27741 IX86_BUILTIN_VPCMOV_V8HI,
27742 IX86_BUILTIN_VPCMOV_V16QI,
27743 IX86_BUILTIN_VPCMOV_V4SF,
27744 IX86_BUILTIN_VPCMOV_V2DF,
27745 IX86_BUILTIN_VPCMOV256,
27746 IX86_BUILTIN_VPCMOV_V4DI256,
27747 IX86_BUILTIN_VPCMOV_V8SI256,
27748 IX86_BUILTIN_VPCMOV_V16HI256,
27749 IX86_BUILTIN_VPCMOV_V32QI256,
27750 IX86_BUILTIN_VPCMOV_V8SF256,
27751 IX86_BUILTIN_VPCMOV_V4DF256,
27753 IX86_BUILTIN_VPPERM,
27755 IX86_BUILTIN_VPMACSSWW,
27756 IX86_BUILTIN_VPMACSWW,
27757 IX86_BUILTIN_VPMACSSWD,
27758 IX86_BUILTIN_VPMACSWD,
27759 IX86_BUILTIN_VPMACSSDD,
27760 IX86_BUILTIN_VPMACSDD,
27761 IX86_BUILTIN_VPMACSSDQL,
27762 IX86_BUILTIN_VPMACSSDQH,
27763 IX86_BUILTIN_VPMACSDQL,
27764 IX86_BUILTIN_VPMACSDQH,
27765 IX86_BUILTIN_VPMADCSSWD,
27766 IX86_BUILTIN_VPMADCSWD,
27768 IX86_BUILTIN_VPHADDBW,
27769 IX86_BUILTIN_VPHADDBD,
27770 IX86_BUILTIN_VPHADDBQ,
27771 IX86_BUILTIN_VPHADDWD,
27772 IX86_BUILTIN_VPHADDWQ,
27773 IX86_BUILTIN_VPHADDDQ,
27774 IX86_BUILTIN_VPHADDUBW,
27775 IX86_BUILTIN_VPHADDUBD,
27776 IX86_BUILTIN_VPHADDUBQ,
27777 IX86_BUILTIN_VPHADDUWD,
27778 IX86_BUILTIN_VPHADDUWQ,
27779 IX86_BUILTIN_VPHADDUDQ,
27780 IX86_BUILTIN_VPHSUBBW,
27781 IX86_BUILTIN_VPHSUBWD,
27782 IX86_BUILTIN_VPHSUBDQ,
27784 IX86_BUILTIN_VPROTB,
27785 IX86_BUILTIN_VPROTW,
27786 IX86_BUILTIN_VPROTD,
27787 IX86_BUILTIN_VPROTQ,
27788 IX86_BUILTIN_VPROTB_IMM,
27789 IX86_BUILTIN_VPROTW_IMM,
27790 IX86_BUILTIN_VPROTD_IMM,
27791 IX86_BUILTIN_VPROTQ_IMM,
27793 IX86_BUILTIN_VPSHLB,
27794 IX86_BUILTIN_VPSHLW,
27795 IX86_BUILTIN_VPSHLD,
27796 IX86_BUILTIN_VPSHLQ,
27797 IX86_BUILTIN_VPSHAB,
27798 IX86_BUILTIN_VPSHAW,
27799 IX86_BUILTIN_VPSHAD,
27800 IX86_BUILTIN_VPSHAQ,
27802 IX86_BUILTIN_VFRCZSS,
27803 IX86_BUILTIN_VFRCZSD,
27804 IX86_BUILTIN_VFRCZPS,
27805 IX86_BUILTIN_VFRCZPD,
27806 IX86_BUILTIN_VFRCZPS256,
27807 IX86_BUILTIN_VFRCZPD256,
27809 IX86_BUILTIN_VPCOMEQUB,
27810 IX86_BUILTIN_VPCOMNEUB,
27811 IX86_BUILTIN_VPCOMLTUB,
27812 IX86_BUILTIN_VPCOMLEUB,
27813 IX86_BUILTIN_VPCOMGTUB,
27814 IX86_BUILTIN_VPCOMGEUB,
27815 IX86_BUILTIN_VPCOMFALSEUB,
27816 IX86_BUILTIN_VPCOMTRUEUB,
27818 IX86_BUILTIN_VPCOMEQUW,
27819 IX86_BUILTIN_VPCOMNEUW,
27820 IX86_BUILTIN_VPCOMLTUW,
27821 IX86_BUILTIN_VPCOMLEUW,
27822 IX86_BUILTIN_VPCOMGTUW,
27823 IX86_BUILTIN_VPCOMGEUW,
27824 IX86_BUILTIN_VPCOMFALSEUW,
27825 IX86_BUILTIN_VPCOMTRUEUW,
27827 IX86_BUILTIN_VPCOMEQUD,
27828 IX86_BUILTIN_VPCOMNEUD,
27829 IX86_BUILTIN_VPCOMLTUD,
27830 IX86_BUILTIN_VPCOMLEUD,
27831 IX86_BUILTIN_VPCOMGTUD,
27832 IX86_BUILTIN_VPCOMGEUD,
27833 IX86_BUILTIN_VPCOMFALSEUD,
27834 IX86_BUILTIN_VPCOMTRUEUD,
27836 IX86_BUILTIN_VPCOMEQUQ,
27837 IX86_BUILTIN_VPCOMNEUQ,
27838 IX86_BUILTIN_VPCOMLTUQ,
27839 IX86_BUILTIN_VPCOMLEUQ,
27840 IX86_BUILTIN_VPCOMGTUQ,
27841 IX86_BUILTIN_VPCOMGEUQ,
27842 IX86_BUILTIN_VPCOMFALSEUQ,
27843 IX86_BUILTIN_VPCOMTRUEUQ,
27845 IX86_BUILTIN_VPCOMEQB,
27846 IX86_BUILTIN_VPCOMNEB,
27847 IX86_BUILTIN_VPCOMLTB,
27848 IX86_BUILTIN_VPCOMLEB,
27849 IX86_BUILTIN_VPCOMGTB,
27850 IX86_BUILTIN_VPCOMGEB,
27851 IX86_BUILTIN_VPCOMFALSEB,
27852 IX86_BUILTIN_VPCOMTRUEB,
27854 IX86_BUILTIN_VPCOMEQW,
27855 IX86_BUILTIN_VPCOMNEW,
27856 IX86_BUILTIN_VPCOMLTW,
27857 IX86_BUILTIN_VPCOMLEW,
27858 IX86_BUILTIN_VPCOMGTW,
27859 IX86_BUILTIN_VPCOMGEW,
27860 IX86_BUILTIN_VPCOMFALSEW,
27861 IX86_BUILTIN_VPCOMTRUEW,
27863 IX86_BUILTIN_VPCOMEQD,
27864 IX86_BUILTIN_VPCOMNED,
27865 IX86_BUILTIN_VPCOMLTD,
27866 IX86_BUILTIN_VPCOMLED,
27867 IX86_BUILTIN_VPCOMGTD,
27868 IX86_BUILTIN_VPCOMGED,
27869 IX86_BUILTIN_VPCOMFALSED,
27870 IX86_BUILTIN_VPCOMTRUED,
27872 IX86_BUILTIN_VPCOMEQQ,
27873 IX86_BUILTIN_VPCOMNEQ,
27874 IX86_BUILTIN_VPCOMLTQ,
27875 IX86_BUILTIN_VPCOMLEQ,
27876 IX86_BUILTIN_VPCOMGTQ,
27877 IX86_BUILTIN_VPCOMGEQ,
27878 IX86_BUILTIN_VPCOMFALSEQ,
27879 IX86_BUILTIN_VPCOMTRUEQ,
27881 /* LWP instructions. */
27882 IX86_BUILTIN_LLWPCB,
27883 IX86_BUILTIN_SLWPCB,
27884 IX86_BUILTIN_LWPVAL32,
27885 IX86_BUILTIN_LWPVAL64,
27886 IX86_BUILTIN_LWPINS32,
27887 IX86_BUILTIN_LWPINS64,
27889 IX86_BUILTIN_CLZS,
27891 /* RTM */
27892 IX86_BUILTIN_XBEGIN,
27893 IX86_BUILTIN_XEND,
27894 IX86_BUILTIN_XABORT,
27895 IX86_BUILTIN_XTEST,
27897 /* BMI instructions. */
27898 IX86_BUILTIN_BEXTR32,
27899 IX86_BUILTIN_BEXTR64,
27900 IX86_BUILTIN_CTZS,
27902 /* TBM instructions. */
27903 IX86_BUILTIN_BEXTRI32,
27904 IX86_BUILTIN_BEXTRI64,
27906 /* BMI2 instructions. */
27907 IX86_BUILTIN_BZHI32,
27908 IX86_BUILTIN_BZHI64,
27909 IX86_BUILTIN_PDEP32,
27910 IX86_BUILTIN_PDEP64,
27911 IX86_BUILTIN_PEXT32,
27912 IX86_BUILTIN_PEXT64,
27914 /* ADX instructions. */
27915 IX86_BUILTIN_ADDCARRYX32,
27916 IX86_BUILTIN_ADDCARRYX64,
27918 /* FSGSBASE instructions. */
27919 IX86_BUILTIN_RDFSBASE32,
27920 IX86_BUILTIN_RDFSBASE64,
27921 IX86_BUILTIN_RDGSBASE32,
27922 IX86_BUILTIN_RDGSBASE64,
27923 IX86_BUILTIN_WRFSBASE32,
27924 IX86_BUILTIN_WRFSBASE64,
27925 IX86_BUILTIN_WRGSBASE32,
27926 IX86_BUILTIN_WRGSBASE64,
27928 /* RDRND instructions. */
27929 IX86_BUILTIN_RDRAND16_STEP,
27930 IX86_BUILTIN_RDRAND32_STEP,
27931 IX86_BUILTIN_RDRAND64_STEP,
27933 /* RDSEED instructions. */
27934 IX86_BUILTIN_RDSEED16_STEP,
27935 IX86_BUILTIN_RDSEED32_STEP,
27936 IX86_BUILTIN_RDSEED64_STEP,
27938 /* F16C instructions. */
27939 IX86_BUILTIN_CVTPH2PS,
27940 IX86_BUILTIN_CVTPH2PS256,
27941 IX86_BUILTIN_CVTPS2PH,
27942 IX86_BUILTIN_CVTPS2PH256,
27944 /* CFString built-in for darwin */
27945 IX86_BUILTIN_CFSTRING,
27947 /* Builtins to get CPU type and supported features. */
27948 IX86_BUILTIN_CPU_INIT,
27949 IX86_BUILTIN_CPU_IS,
27950 IX86_BUILTIN_CPU_SUPPORTS,
27952 /* Read/write FLAGS register built-ins. */
27953 IX86_BUILTIN_READ_FLAGS,
27954 IX86_BUILTIN_WRITE_FLAGS,
27956 IX86_BUILTIN_MAX
27957 };
27959 /* Table for the ix86 builtin decls. */
27962 /* Table of all of the builtin functions that are possible with different ISA's
27963 but are waiting to be built until a function is declared to use that
27964 ISA. */
27971 };
27976 /* Add an ix86 target builtin function with CODE, NAME and TYPE. Save the MASK
27977 of which isa_flags to use in the ix86_builtins_isa array. Stores the
27978 function decl in the ix86_builtins array. Returns the function decl or
27979 NULL_TREE, if the builtin was not added.
27981 If the front end has a special hook for builtin functions, delay adding
27982 builtin functions that aren't in the current ISA until the ISA is changed
27983 with function specific optimization. Doing so, can save about 300K for the
27984 default compiler. When the builtin is expanded, check at that time whether
27985 it is valid.
27987 If the front end doesn't have a special hook, record all builtins, even if
27988 it isn't an instruction set in the current ISA in case the user uses
27989 function specific options for a different ISA, so that we don't get scope
27990 errors if a builtin is added in the middle of a function scope. */
27996 {
28000 {
28009 {
28015 }
28016 else
28017 {
28023 }
28024 }
28027 }
28029 /* Like def_builtin, but also marks the function decl "const". */
28034 {
28038 else
28042 }
28044 /* Add any new builtin functions for a given ISA that may not have been
28045 declared. This saves a bit of space compared to adding all of the
28046 declarations to the tree, even if we didn't use them. */
28048 static void
28050 {
28054 {
28057 {
28060 /* Don't define the builtin again. */
28066 NULL_TREE);
28071 }
28072 }
28073 }
28075 /* Bits for builtin_description.flag. */
28077 /* Set when we don't support the comparison natively, and should
28078 swap_comparison in order to support it. */
28079 #define BUILTIN_DESC_SWAP_OPERANDS 1
28081 struct builtin_description
28082 {
28089 };
28092 {
28093 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_comi, "__builtin_ia32_comieq", IX86_BUILTIN_COMIEQSS, UNEQ, 0 },
28094 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_comi, "__builtin_ia32_comilt", IX86_BUILTIN_COMILTSS, UNLT, 0 },
28095 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_comi, "__builtin_ia32_comile", IX86_BUILTIN_COMILESS, UNLE, 0 },
28096 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_comi, "__builtin_ia32_comigt", IX86_BUILTIN_COMIGTSS, GT, 0 },
28097 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_comi, "__builtin_ia32_comige", IX86_BUILTIN_COMIGESS, GE, 0 },
28098 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_comi, "__builtin_ia32_comineq", IX86_BUILTIN_COMINEQSS, LTGT, 0 },
28099 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_ucomi, "__builtin_ia32_ucomieq", IX86_BUILTIN_UCOMIEQSS, UNEQ, 0 },
28100 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_ucomi, "__builtin_ia32_ucomilt", IX86_BUILTIN_UCOMILTSS, UNLT, 0 },
28101 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_ucomi, "__builtin_ia32_ucomile", IX86_BUILTIN_UCOMILESS, UNLE, 0 },
28102 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_ucomi, "__builtin_ia32_ucomigt", IX86_BUILTIN_UCOMIGTSS, GT, 0 },
28103 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_ucomi, "__builtin_ia32_ucomige", IX86_BUILTIN_UCOMIGESS, GE, 0 },
28104 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_ucomi, "__builtin_ia32_ucomineq", IX86_BUILTIN_UCOMINEQSS, LTGT, 0 },
28105 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_comi, "__builtin_ia32_comisdeq", IX86_BUILTIN_COMIEQSD, UNEQ, 0 },
28106 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_comi, "__builtin_ia32_comisdlt", IX86_BUILTIN_COMILTSD, UNLT, 0 },
28107 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_comi, "__builtin_ia32_comisdle", IX86_BUILTIN_COMILESD, UNLE, 0 },
28108 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_comi, "__builtin_ia32_comisdgt", IX86_BUILTIN_COMIGTSD, GT, 0 },
28109 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_comi, "__builtin_ia32_comisdge", IX86_BUILTIN_COMIGESD, GE, 0 },
28110 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_comi, "__builtin_ia32_comisdneq", IX86_BUILTIN_COMINEQSD, LTGT, 0 },
28111 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ucomi, "__builtin_ia32_ucomisdeq", IX86_BUILTIN_UCOMIEQSD, UNEQ, 0 },
28112 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ucomi, "__builtin_ia32_ucomisdlt", IX86_BUILTIN_UCOMILTSD, UNLT, 0 },
28113 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ucomi, "__builtin_ia32_ucomisdle", IX86_BUILTIN_UCOMILESD, UNLE, 0 },
28114 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ucomi, "__builtin_ia32_ucomisdgt", IX86_BUILTIN_UCOMIGTSD, GT, 0 },
28115 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ucomi, "__builtin_ia32_ucomisdge", IX86_BUILTIN_UCOMIGESD, GE, 0 },
28116 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ucomi, "__builtin_ia32_ucomisdneq", IX86_BUILTIN_UCOMINEQSD, LTGT, 0 },
28117 };
28120 {
28121 /* SSE4.2 */
28122 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpestr, "__builtin_ia32_pcmpestri128", IX86_BUILTIN_PCMPESTRI128, UNKNOWN, 0 },
28123 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpestr, "__builtin_ia32_pcmpestrm128", IX86_BUILTIN_PCMPESTRM128, UNKNOWN, 0 },
28124 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpestr, "__builtin_ia32_pcmpestria128", IX86_BUILTIN_PCMPESTRA128, UNKNOWN, (int) CCAmode },
28125 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpestr, "__builtin_ia32_pcmpestric128", IX86_BUILTIN_PCMPESTRC128, UNKNOWN, (int) CCCmode },
28126 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpestr, "__builtin_ia32_pcmpestrio128", IX86_BUILTIN_PCMPESTRO128, UNKNOWN, (int) CCOmode },
28127 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpestr, "__builtin_ia32_pcmpestris128", IX86_BUILTIN_PCMPESTRS128, UNKNOWN, (int) CCSmode },
28128 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpestr, "__builtin_ia32_pcmpestriz128", IX86_BUILTIN_PCMPESTRZ128, UNKNOWN, (int) CCZmode },
28129 };
28132 {
28133 /* SSE4.2 */
28134 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpistr, "__builtin_ia32_pcmpistri128", IX86_BUILTIN_PCMPISTRI128, UNKNOWN, 0 },
28135 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpistr, "__builtin_ia32_pcmpistrm128", IX86_BUILTIN_PCMPISTRM128, UNKNOWN, 0 },
28136 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpistr, "__builtin_ia32_pcmpistria128", IX86_BUILTIN_PCMPISTRA128, UNKNOWN, (int) CCAmode },
28137 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpistr, "__builtin_ia32_pcmpistric128", IX86_BUILTIN_PCMPISTRC128, UNKNOWN, (int) CCCmode },
28138 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpistr, "__builtin_ia32_pcmpistrio128", IX86_BUILTIN_PCMPISTRO128, UNKNOWN, (int) CCOmode },
28139 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpistr, "__builtin_ia32_pcmpistris128", IX86_BUILTIN_PCMPISTRS128, UNKNOWN, (int) CCSmode },
28140 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpistr, "__builtin_ia32_pcmpistriz128", IX86_BUILTIN_PCMPISTRZ128, UNKNOWN, (int) CCZmode },
28141 };
28143 /* Special builtins with variable number of arguments. */
28145 {
28146 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_nothing, "__builtin_ia32_rdtsc", IX86_BUILTIN_RDTSC, UNKNOWN, (int) UINT64_FTYPE_VOID },
28147 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_nothing, "__builtin_ia32_rdtscp", IX86_BUILTIN_RDTSCP, UNKNOWN, (int) UINT64_FTYPE_PUNSIGNED },
28148 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_pause, "__builtin_ia32_pause", IX86_BUILTIN_PAUSE, UNKNOWN, (int) VOID_FTYPE_VOID },
28150 /* 80387 (for use internally for atomic compound assignment). */
28151 { 0, CODE_FOR_fnstenv, "__builtin_ia32_fnstenv", IX86_BUILTIN_FNSTENV, UNKNOWN, (int) VOID_FTYPE_PVOID },
28152 { 0, CODE_FOR_fldenv, "__builtin_ia32_fldenv", IX86_BUILTIN_FLDENV, UNKNOWN, (int) VOID_FTYPE_PCVOID },
28153 { 0, CODE_FOR_fnstsw, "__builtin_ia32_fnstsw", IX86_BUILTIN_FNSTSW, UNKNOWN, (int) VOID_FTYPE_PUSHORT },
28154 { 0, CODE_FOR_fnclex, "__builtin_ia32_fnclex", IX86_BUILTIN_FNCLEX, UNKNOWN, (int) VOID_FTYPE_VOID },
28156 /* MMX */
28157 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_emms, "__builtin_ia32_emms", IX86_BUILTIN_EMMS, UNKNOWN, (int) VOID_FTYPE_VOID },
28159 /* 3DNow! */
28160 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_femms, "__builtin_ia32_femms", IX86_BUILTIN_FEMMS, UNKNOWN, (int) VOID_FTYPE_VOID },
28162 /* FXSR, XSAVE and XSAVEOPT */
28163 { OPTION_MASK_ISA_FXSR, CODE_FOR_nothing, "__builtin_ia32_fxsave", IX86_BUILTIN_FXSAVE, UNKNOWN, (int) VOID_FTYPE_PVOID },
28164 { OPTION_MASK_ISA_FXSR, CODE_FOR_nothing, "__builtin_ia32_fxrstor", IX86_BUILTIN_FXRSTOR, UNKNOWN, (int) VOID_FTYPE_PVOID },
28165 { OPTION_MASK_ISA_XSAVE, CODE_FOR_nothing, "__builtin_ia32_xsave", IX86_BUILTIN_XSAVE, UNKNOWN, (int) VOID_FTYPE_PVOID_INT64 },
28166 { OPTION_MASK_ISA_XSAVE, CODE_FOR_nothing, "__builtin_ia32_xrstor", IX86_BUILTIN_XRSTOR, UNKNOWN, (int) VOID_FTYPE_PVOID_INT64 },
28167 { OPTION_MASK_ISA_XSAVEOPT, CODE_FOR_nothing, "__builtin_ia32_xsaveopt", IX86_BUILTIN_XSAVEOPT, UNKNOWN, (int) VOID_FTYPE_PVOID_INT64 },
28169 { OPTION_MASK_ISA_FXSR | OPTION_MASK_ISA_64BIT, CODE_FOR_nothing, "__builtin_ia32_fxsave64", IX86_BUILTIN_FXSAVE64, UNKNOWN, (int) VOID_FTYPE_PVOID },
28170 { OPTION_MASK_ISA_FXSR | OPTION_MASK_ISA_64BIT, CODE_FOR_nothing, "__builtin_ia32_fxrstor64", IX86_BUILTIN_FXRSTOR64, UNKNOWN, (int) VOID_FTYPE_PVOID },
28171 { OPTION_MASK_ISA_XSAVE | OPTION_MASK_ISA_64BIT, CODE_FOR_nothing, "__builtin_ia32_xsave64", IX86_BUILTIN_XSAVE64, UNKNOWN, (int) VOID_FTYPE_PVOID_INT64 },
28172 { OPTION_MASK_ISA_XSAVE | OPTION_MASK_ISA_64BIT, CODE_FOR_nothing, "__builtin_ia32_xrstor64", IX86_BUILTIN_XRSTOR64, UNKNOWN, (int) VOID_FTYPE_PVOID_INT64 },
28173 { OPTION_MASK_ISA_XSAVEOPT | OPTION_MASK_ISA_64BIT, CODE_FOR_nothing, "__builtin_ia32_xsaveopt64", IX86_BUILTIN_XSAVEOPT64, UNKNOWN, (int) VOID_FTYPE_PVOID_INT64 },
28175 /* SSE */
28176 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_storeups, "__builtin_ia32_storeups", IX86_BUILTIN_STOREUPS, UNKNOWN, (int) VOID_FTYPE_PFLOAT_V4SF },
28177 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_movntv4sf, "__builtin_ia32_movntps", IX86_BUILTIN_MOVNTPS, UNKNOWN, (int) VOID_FTYPE_PFLOAT_V4SF },
28178 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_loadups, "__builtin_ia32_loadups", IX86_BUILTIN_LOADUPS, UNKNOWN, (int) V4SF_FTYPE_PCFLOAT },
28180 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_loadhps_exp, "__builtin_ia32_loadhps", IX86_BUILTIN_LOADHPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_PCV2SF },
28181 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_loadlps_exp, "__builtin_ia32_loadlps", IX86_BUILTIN_LOADLPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_PCV2SF },
28182 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_storehps, "__builtin_ia32_storehps", IX86_BUILTIN_STOREHPS, UNKNOWN, (int) VOID_FTYPE_PV2SF_V4SF },
28183 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_storelps, "__builtin_ia32_storelps", IX86_BUILTIN_STORELPS, UNKNOWN, (int) VOID_FTYPE_PV2SF_V4SF },
28185 /* SSE or 3DNow!A */
28186 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_sse_sfence, "__builtin_ia32_sfence", IX86_BUILTIN_SFENCE, UNKNOWN, (int) VOID_FTYPE_VOID },
28187 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_sse_movntq, "__builtin_ia32_movntq", IX86_BUILTIN_MOVNTQ, UNKNOWN, (int) VOID_FTYPE_PULONGLONG_ULONGLONG },
28189 /* SSE2 */
28190 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_lfence, "__builtin_ia32_lfence", IX86_BUILTIN_LFENCE, UNKNOWN, (int) VOID_FTYPE_VOID },
28191 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_mfence, 0, IX86_BUILTIN_MFENCE, UNKNOWN, (int) VOID_FTYPE_VOID },
28192 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_storeupd, "__builtin_ia32_storeupd", IX86_BUILTIN_STOREUPD, UNKNOWN, (int) VOID_FTYPE_PDOUBLE_V2DF },
28193 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_storedquv16qi, "__builtin_ia32_storedqu", IX86_BUILTIN_STOREDQU, UNKNOWN, (int) VOID_FTYPE_PCHAR_V16QI },
28194 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_movntv2df, "__builtin_ia32_movntpd", IX86_BUILTIN_MOVNTPD, UNKNOWN, (int) VOID_FTYPE_PDOUBLE_V2DF },
28195 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_movntv2di, "__builtin_ia32_movntdq", IX86_BUILTIN_MOVNTDQ, UNKNOWN, (int) VOID_FTYPE_PV2DI_V2DI },
28196 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_movntisi, "__builtin_ia32_movnti", IX86_BUILTIN_MOVNTI, UNKNOWN, (int) VOID_FTYPE_PINT_INT },
28197 { OPTION_MASK_ISA_SSE2 | OPTION_MASK_ISA_64BIT, CODE_FOR_sse2_movntidi, "__builtin_ia32_movnti64", IX86_BUILTIN_MOVNTI64, UNKNOWN, (int) VOID_FTYPE_PLONGLONG_LONGLONG },
28198 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_loadupd, "__builtin_ia32_loadupd", IX86_BUILTIN_LOADUPD, UNKNOWN, (int) V2DF_FTYPE_PCDOUBLE },
28199 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_loaddquv16qi, "__builtin_ia32_loaddqu", IX86_BUILTIN_LOADDQU, UNKNOWN, (int) V16QI_FTYPE_PCCHAR },
28201 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_loadhpd_exp, "__builtin_ia32_loadhpd", IX86_BUILTIN_LOADHPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_PCDOUBLE },
28202 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_loadlpd_exp, "__builtin_ia32_loadlpd", IX86_BUILTIN_LOADLPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_PCDOUBLE },
28204 /* SSE3 */
28205 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_lddqu, "__builtin_ia32_lddqu", IX86_BUILTIN_LDDQU, UNKNOWN, (int) V16QI_FTYPE_PCCHAR },
28207 /* SSE4.1 */
28208 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_movntdqa, "__builtin_ia32_movntdqa", IX86_BUILTIN_MOVNTDQA, UNKNOWN, (int) V2DI_FTYPE_PV2DI },
28210 /* SSE4A */
28211 { OPTION_MASK_ISA_SSE4A, CODE_FOR_sse4a_vmmovntv2df, "__builtin_ia32_movntsd", IX86_BUILTIN_MOVNTSD, UNKNOWN, (int) VOID_FTYPE_PDOUBLE_V2DF },
28212 { OPTION_MASK_ISA_SSE4A, CODE_FOR_sse4a_vmmovntv4sf, "__builtin_ia32_movntss", IX86_BUILTIN_MOVNTSS, UNKNOWN, (int) VOID_FTYPE_PFLOAT_V4SF },
28214 /* AVX */
28215 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vzeroall, "__builtin_ia32_vzeroall", IX86_BUILTIN_VZEROALL, UNKNOWN, (int) VOID_FTYPE_VOID },
28216 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vzeroupper, "__builtin_ia32_vzeroupper", IX86_BUILTIN_VZEROUPPER, UNKNOWN, (int) VOID_FTYPE_VOID },
28218 { OPTION_MASK_ISA_AVX, CODE_FOR_vec_dupv4sf, "__builtin_ia32_vbroadcastss", IX86_BUILTIN_VBROADCASTSS, UNKNOWN, (int) V4SF_FTYPE_PCFLOAT },
28219 { OPTION_MASK_ISA_AVX, CODE_FOR_vec_dupv4df, "__builtin_ia32_vbroadcastsd256", IX86_BUILTIN_VBROADCASTSD256, UNKNOWN, (int) V4DF_FTYPE_PCDOUBLE },
28220 { OPTION_MASK_ISA_AVX, CODE_FOR_vec_dupv8sf, "__builtin_ia32_vbroadcastss256", IX86_BUILTIN_VBROADCASTSS256, UNKNOWN, (int) V8SF_FTYPE_PCFLOAT },
28221 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vbroadcastf128_v4df, "__builtin_ia32_vbroadcastf128_pd256", IX86_BUILTIN_VBROADCASTPD256, UNKNOWN, (int) V4DF_FTYPE_PCV2DF },
28222 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vbroadcastf128_v8sf, "__builtin_ia32_vbroadcastf128_ps256", IX86_BUILTIN_VBROADCASTPS256, UNKNOWN, (int) V8SF_FTYPE_PCV4SF },
28224 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_loadupd256, "__builtin_ia32_loadupd256", IX86_BUILTIN_LOADUPD256, UNKNOWN, (int) V4DF_FTYPE_PCDOUBLE },
28225 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_loadups256, "__builtin_ia32_loadups256", IX86_BUILTIN_LOADUPS256, UNKNOWN, (int) V8SF_FTYPE_PCFLOAT },
28226 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_storeupd256, "__builtin_ia32_storeupd256", IX86_BUILTIN_STOREUPD256, UNKNOWN, (int) VOID_FTYPE_PDOUBLE_V4DF },
28227 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_storeups256, "__builtin_ia32_storeups256", IX86_BUILTIN_STOREUPS256, UNKNOWN, (int) VOID_FTYPE_PFLOAT_V8SF },
28228 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_loaddquv32qi, "__builtin_ia32_loaddqu256", IX86_BUILTIN_LOADDQU256, UNKNOWN, (int) V32QI_FTYPE_PCCHAR },
28229 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_storedquv32qi, "__builtin_ia32_storedqu256", IX86_BUILTIN_STOREDQU256, UNKNOWN, (int) VOID_FTYPE_PCHAR_V32QI },
28230 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_lddqu256, "__builtin_ia32_lddqu256", IX86_BUILTIN_LDDQU256, UNKNOWN, (int) V32QI_FTYPE_PCCHAR },
28232 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movntv4di, "__builtin_ia32_movntdq256", IX86_BUILTIN_MOVNTDQ256, UNKNOWN, (int) VOID_FTYPE_PV4DI_V4DI },
28233 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movntv4df, "__builtin_ia32_movntpd256", IX86_BUILTIN_MOVNTPD256, UNKNOWN, (int) VOID_FTYPE_PDOUBLE_V4DF },
28234 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movntv8sf, "__builtin_ia32_movntps256", IX86_BUILTIN_MOVNTPS256, UNKNOWN, (int) VOID_FTYPE_PFLOAT_V8SF },
28236 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskloadpd, "__builtin_ia32_maskloadpd", IX86_BUILTIN_MASKLOADPD, UNKNOWN, (int) V2DF_FTYPE_PCV2DF_V2DI },
28237 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskloadps, "__builtin_ia32_maskloadps", IX86_BUILTIN_MASKLOADPS, UNKNOWN, (int) V4SF_FTYPE_PCV4SF_V4SI },
28238 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskloadpd256, "__builtin_ia32_maskloadpd256", IX86_BUILTIN_MASKLOADPD256, UNKNOWN, (int) V4DF_FTYPE_PCV4DF_V4DI },
28239 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskloadps256, "__builtin_ia32_maskloadps256", IX86_BUILTIN_MASKLOADPS256, UNKNOWN, (int) V8SF_FTYPE_PCV8SF_V8SI },
28240 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskstorepd, "__builtin_ia32_maskstorepd", IX86_BUILTIN_MASKSTOREPD, UNKNOWN, (int) VOID_FTYPE_PV2DF_V2DI_V2DF },
28241 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskstoreps, "__builtin_ia32_maskstoreps", IX86_BUILTIN_MASKSTOREPS, UNKNOWN, (int) VOID_FTYPE_PV4SF_V4SI_V4SF },
28242 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskstorepd256, "__builtin_ia32_maskstorepd256", IX86_BUILTIN_MASKSTOREPD256, UNKNOWN, (int) VOID_FTYPE_PV4DF_V4DI_V4DF },
28243 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskstoreps256, "__builtin_ia32_maskstoreps256", IX86_BUILTIN_MASKSTOREPS256, UNKNOWN, (int) VOID_FTYPE_PV8SF_V8SI_V8SF },
28245 /* AVX2 */
28246 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_movntdqa, "__builtin_ia32_movntdqa256", IX86_BUILTIN_MOVNTDQA256, UNKNOWN, (int) V4DI_FTYPE_PV4DI },
28247 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_maskloadd, "__builtin_ia32_maskloadd", IX86_BUILTIN_MASKLOADD, UNKNOWN, (int) V4SI_FTYPE_PCV4SI_V4SI },
28248 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_maskloadq, "__builtin_ia32_maskloadq", IX86_BUILTIN_MASKLOADQ, UNKNOWN, (int) V2DI_FTYPE_PCV2DI_V2DI },
28249 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_maskloadd256, "__builtin_ia32_maskloadd256", IX86_BUILTIN_MASKLOADD256, UNKNOWN, (int) V8SI_FTYPE_PCV8SI_V8SI },
28250 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_maskloadq256, "__builtin_ia32_maskloadq256", IX86_BUILTIN_MASKLOADQ256, UNKNOWN, (int) V4DI_FTYPE_PCV4DI_V4DI },
28251 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_maskstored, "__builtin_ia32_maskstored", IX86_BUILTIN_MASKSTORED, UNKNOWN, (int) VOID_FTYPE_PV4SI_V4SI_V4SI },
28252 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_maskstoreq, "__builtin_ia32_maskstoreq", IX86_BUILTIN_MASKSTOREQ, UNKNOWN, (int) VOID_FTYPE_PV2DI_V2DI_V2DI },
28253 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_maskstored256, "__builtin_ia32_maskstored256", IX86_BUILTIN_MASKSTORED256, UNKNOWN, (int) VOID_FTYPE_PV8SI_V8SI_V8SI },
28254 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_maskstoreq256, "__builtin_ia32_maskstoreq256", IX86_BUILTIN_MASKSTOREQ256, UNKNOWN, (int) VOID_FTYPE_PV4DI_V4DI_V4DI },
28256 { OPTION_MASK_ISA_LWP, CODE_FOR_lwp_llwpcb, "__builtin_ia32_llwpcb", IX86_BUILTIN_LLWPCB, UNKNOWN, (int) VOID_FTYPE_PVOID },
28257 { OPTION_MASK_ISA_LWP, CODE_FOR_lwp_slwpcb, "__builtin_ia32_slwpcb", IX86_BUILTIN_SLWPCB, UNKNOWN, (int) PVOID_FTYPE_VOID },
28258 { OPTION_MASK_ISA_LWP, CODE_FOR_lwp_lwpvalsi3, "__builtin_ia32_lwpval32", IX86_BUILTIN_LWPVAL32, UNKNOWN, (int) VOID_FTYPE_UINT_UINT_UINT },
28259 { OPTION_MASK_ISA_LWP, CODE_FOR_lwp_lwpvaldi3, "__builtin_ia32_lwpval64", IX86_BUILTIN_LWPVAL64, UNKNOWN, (int) VOID_FTYPE_UINT64_UINT_UINT },
28260 { OPTION_MASK_ISA_LWP, CODE_FOR_lwp_lwpinssi3, "__builtin_ia32_lwpins32", IX86_BUILTIN_LWPINS32, UNKNOWN, (int) UCHAR_FTYPE_UINT_UINT_UINT },
28261 { OPTION_MASK_ISA_LWP, CODE_FOR_lwp_lwpinsdi3, "__builtin_ia32_lwpins64", IX86_BUILTIN_LWPINS64, UNKNOWN, (int) UCHAR_FTYPE_UINT64_UINT_UINT },
28263 /* FSGSBASE */
28264 { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_rdfsbasesi, "__builtin_ia32_rdfsbase32", IX86_BUILTIN_RDFSBASE32, UNKNOWN, (int) UNSIGNED_FTYPE_VOID },
28265 { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_rdfsbasedi, "__builtin_ia32_rdfsbase64", IX86_BUILTIN_RDFSBASE64, UNKNOWN, (int) UINT64_FTYPE_VOID },
28266 { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_rdgsbasesi, "__builtin_ia32_rdgsbase32", IX86_BUILTIN_RDGSBASE32, UNKNOWN, (int) UNSIGNED_FTYPE_VOID },
28267 { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_rdgsbasedi, "__builtin_ia32_rdgsbase64", IX86_BUILTIN_RDGSBASE64, UNKNOWN, (int) UINT64_FTYPE_VOID },
28268 { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_wrfsbasesi, "__builtin_ia32_wrfsbase32", IX86_BUILTIN_WRFSBASE32, UNKNOWN, (int) VOID_FTYPE_UNSIGNED },
28269 { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_wrfsbasedi, "__builtin_ia32_wrfsbase64", IX86_BUILTIN_WRFSBASE64, UNKNOWN, (int) VOID_FTYPE_UINT64 },
28270 { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_wrgsbasesi, "__builtin_ia32_wrgsbase32", IX86_BUILTIN_WRGSBASE32, UNKNOWN, (int) VOID_FTYPE_UNSIGNED },
28271 { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_wrgsbasedi, "__builtin_ia32_wrgsbase64", IX86_BUILTIN_WRGSBASE64, UNKNOWN, (int) VOID_FTYPE_UINT64 },
28273 /* RTM */
28274 { OPTION_MASK_ISA_RTM, CODE_FOR_xbegin, "__builtin_ia32_xbegin", IX86_BUILTIN_XBEGIN, UNKNOWN, (int) UNSIGNED_FTYPE_VOID },
28275 { OPTION_MASK_ISA_RTM, CODE_FOR_xend, "__builtin_ia32_xend", IX86_BUILTIN_XEND, UNKNOWN, (int) VOID_FTYPE_VOID },
28276 { OPTION_MASK_ISA_RTM, CODE_FOR_xtest, "__builtin_ia32_xtest", IX86_BUILTIN_XTEST, UNKNOWN, (int) INT_FTYPE_VOID },
28277 };
28279 /* Builtins with variable number of arguments. */
28281 {
28282 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_bsr, "__builtin_ia32_bsrsi", IX86_BUILTIN_BSRSI, UNKNOWN, (int) INT_FTYPE_INT },
28283 { OPTION_MASK_ISA_64BIT, CODE_FOR_bsr_rex64, "__builtin_ia32_bsrdi", IX86_BUILTIN_BSRDI, UNKNOWN, (int) INT64_FTYPE_INT64 },
28284 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_nothing, "__builtin_ia32_rdpmc", IX86_BUILTIN_RDPMC, UNKNOWN, (int) UINT64_FTYPE_INT },
28285 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_rotlqi3, "__builtin_ia32_rolqi", IX86_BUILTIN_ROLQI, UNKNOWN, (int) UINT8_FTYPE_UINT8_INT },
28286 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_rotlhi3, "__builtin_ia32_rolhi", IX86_BUILTIN_ROLHI, UNKNOWN, (int) UINT16_FTYPE_UINT16_INT },
28287 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_rotrqi3, "__builtin_ia32_rorqi", IX86_BUILTIN_RORQI, UNKNOWN, (int) UINT8_FTYPE_UINT8_INT },
28288 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_rotrhi3, "__builtin_ia32_rorhi", IX86_BUILTIN_RORHI, UNKNOWN, (int) UINT16_FTYPE_UINT16_INT },
28290 /* MMX */
28291 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_addv8qi3, "__builtin_ia32_paddb", IX86_BUILTIN_PADDB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
28292 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_addv4hi3, "__builtin_ia32_paddw", IX86_BUILTIN_PADDW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
28293 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_addv2si3, "__builtin_ia32_paddd", IX86_BUILTIN_PADDD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
28294 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_subv8qi3, "__builtin_ia32_psubb", IX86_BUILTIN_PSUBB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
28295 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_subv4hi3, "__builtin_ia32_psubw", IX86_BUILTIN_PSUBW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
28296 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_subv2si3, "__builtin_ia32_psubd", IX86_BUILTIN_PSUBD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
28298 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ssaddv8qi3, "__builtin_ia32_paddsb", IX86_BUILTIN_PADDSB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
28299 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ssaddv4hi3, "__builtin_ia32_paddsw", IX86_BUILTIN_PADDSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
28300 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_sssubv8qi3, "__builtin_ia32_psubsb", IX86_BUILTIN_PSUBSB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
28301 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_sssubv4hi3, "__builtin_ia32_psubsw", IX86_BUILTIN_PSUBSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
28302 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_usaddv8qi3, "__builtin_ia32_paddusb", IX86_BUILTIN_PADDUSB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
28303 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_usaddv4hi3, "__builtin_ia32_paddusw", IX86_BUILTIN_PADDUSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
28304 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ussubv8qi3, "__builtin_ia32_psubusb", IX86_BUILTIN_PSUBUSB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
28305 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ussubv4hi3, "__builtin_ia32_psubusw", IX86_BUILTIN_PSUBUSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
28307 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_mulv4hi3, "__builtin_ia32_pmullw", IX86_BUILTIN_PMULLW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
28308 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_smulv4hi3_highpart, "__builtin_ia32_pmulhw", IX86_BUILTIN_PMULHW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
28310 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_andv2si3, "__builtin_ia32_pand", IX86_BUILTIN_PAND, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
28311 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_andnotv2si3, "__builtin_ia32_pandn", IX86_BUILTIN_PANDN, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
28312 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_iorv2si3, "__builtin_ia32_por", IX86_BUILTIN_POR, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
28313 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_xorv2si3, "__builtin_ia32_pxor", IX86_BUILTIN_PXOR, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
28315 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_eqv8qi3, "__builtin_ia32_pcmpeqb", IX86_BUILTIN_PCMPEQB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
28316 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_eqv4hi3, "__builtin_ia32_pcmpeqw", IX86_BUILTIN_PCMPEQW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
28317 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_eqv2si3, "__builtin_ia32_pcmpeqd", IX86_BUILTIN_PCMPEQD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
28318 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_gtv8qi3, "__builtin_ia32_pcmpgtb", IX86_BUILTIN_PCMPGTB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
28319 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_gtv4hi3, "__builtin_ia32_pcmpgtw", IX86_BUILTIN_PCMPGTW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
28320 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_gtv2si3, "__builtin_ia32_pcmpgtd", IX86_BUILTIN_PCMPGTD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
28322 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_punpckhbw, "__builtin_ia32_punpckhbw", IX86_BUILTIN_PUNPCKHBW, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
28323 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_punpckhwd, "__builtin_ia32_punpckhwd", IX86_BUILTIN_PUNPCKHWD, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
28324 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_punpckhdq, "__builtin_ia32_punpckhdq", IX86_BUILTIN_PUNPCKHDQ, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
28325 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_punpcklbw, "__builtin_ia32_punpcklbw", IX86_BUILTIN_PUNPCKLBW, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
28326 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_punpcklwd, "__builtin_ia32_punpcklwd", IX86_BUILTIN_PUNPCKLWD, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI},
28327 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_punpckldq, "__builtin_ia32_punpckldq", IX86_BUILTIN_PUNPCKLDQ, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI},
28329 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_packsswb, "__builtin_ia32_packsswb", IX86_BUILTIN_PACKSSWB, UNKNOWN, (int) V8QI_FTYPE_V4HI_V4HI },
28330 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_packssdw, "__builtin_ia32_packssdw", IX86_BUILTIN_PACKSSDW, UNKNOWN, (int) V4HI_FTYPE_V2SI_V2SI },
28331 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_packuswb, "__builtin_ia32_packuswb", IX86_BUILTIN_PACKUSWB, UNKNOWN, (int) V8QI_FTYPE_V4HI_V4HI },
28333 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_pmaddwd, "__builtin_ia32_pmaddwd", IX86_BUILTIN_PMADDWD, UNKNOWN, (int) V2SI_FTYPE_V4HI_V4HI },
28335 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashlv4hi3, "__builtin_ia32_psllwi", IX86_BUILTIN_PSLLWI, UNKNOWN, (int) V4HI_FTYPE_V4HI_SI_COUNT },
28336 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashlv2si3, "__builtin_ia32_pslldi", IX86_BUILTIN_PSLLDI, UNKNOWN, (int) V2SI_FTYPE_V2SI_SI_COUNT },
28337 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashlv1di3, "__builtin_ia32_psllqi", IX86_BUILTIN_PSLLQI, UNKNOWN, (int) V1DI_FTYPE_V1DI_SI_COUNT },
28338 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashlv4hi3, "__builtin_ia32_psllw", IX86_BUILTIN_PSLLW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI_COUNT },
28339 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashlv2si3, "__builtin_ia32_pslld", IX86_BUILTIN_PSLLD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI_COUNT },
28340 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashlv1di3, "__builtin_ia32_psllq", IX86_BUILTIN_PSLLQ, UNKNOWN, (int) V1DI_FTYPE_V1DI_V1DI_COUNT },
28342 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_lshrv4hi3, "__builtin_ia32_psrlwi", IX86_BUILTIN_PSRLWI, UNKNOWN, (int) V4HI_FTYPE_V4HI_SI_COUNT },
28343 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_lshrv2si3, "__builtin_ia32_psrldi", IX86_BUILTIN_PSRLDI, UNKNOWN, (int) V2SI_FTYPE_V2SI_SI_COUNT },
28344 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_lshrv1di3, "__builtin_ia32_psrlqi", IX86_BUILTIN_PSRLQI, UNKNOWN, (int) V1DI_FTYPE_V1DI_SI_COUNT },
28345 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_lshrv4hi3, "__builtin_ia32_psrlw", IX86_BUILTIN_PSRLW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI_COUNT },
28346 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_lshrv2si3, "__builtin_ia32_psrld", IX86_BUILTIN_PSRLD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI_COUNT },
28347 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_lshrv1di3, "__builtin_ia32_psrlq", IX86_BUILTIN_PSRLQ, UNKNOWN, (int) V1DI_FTYPE_V1DI_V1DI_COUNT },
28349 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashrv4hi3, "__builtin_ia32_psrawi", IX86_BUILTIN_PSRAWI, UNKNOWN, (int) V4HI_FTYPE_V4HI_SI_COUNT },
28350 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashrv2si3, "__builtin_ia32_psradi", IX86_BUILTIN_PSRADI, UNKNOWN, (int) V2SI_FTYPE_V2SI_SI_COUNT },
28351 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashrv4hi3, "__builtin_ia32_psraw", IX86_BUILTIN_PSRAW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI_COUNT },
28352 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashrv2si3, "__builtin_ia32_psrad", IX86_BUILTIN_PSRAD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI_COUNT },
28354 /* 3DNow! */
28355 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_pf2id, "__builtin_ia32_pf2id", IX86_BUILTIN_PF2ID, UNKNOWN, (int) V2SI_FTYPE_V2SF },
28356 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_floatv2si2, "__builtin_ia32_pi2fd", IX86_BUILTIN_PI2FD, UNKNOWN, (int) V2SF_FTYPE_V2SI },
28357 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_rcpv2sf2, "__builtin_ia32_pfrcp", IX86_BUILTIN_PFRCP, UNKNOWN, (int) V2SF_FTYPE_V2SF },
28358 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_rsqrtv2sf2, "__builtin_ia32_pfrsqrt", IX86_BUILTIN_PFRSQRT, UNKNOWN, (int) V2SF_FTYPE_V2SF },
28360 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_uavgv8qi3, "__builtin_ia32_pavgusb", IX86_BUILTIN_PAVGUSB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
28361 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_haddv2sf3, "__builtin_ia32_pfacc", IX86_BUILTIN_PFACC, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
28362 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_addv2sf3, "__builtin_ia32_pfadd", IX86_BUILTIN_PFADD, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
28363 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_eqv2sf3, "__builtin_ia32_pfcmpeq", IX86_BUILTIN_PFCMPEQ, UNKNOWN, (int) V2SI_FTYPE_V2SF_V2SF },
28364 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_gev2sf3, "__builtin_ia32_pfcmpge", IX86_BUILTIN_PFCMPGE, UNKNOWN, (int) V2SI_FTYPE_V2SF_V2SF },
28365 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_gtv2sf3, "__builtin_ia32_pfcmpgt", IX86_BUILTIN_PFCMPGT, UNKNOWN, (int) V2SI_FTYPE_V2SF_V2SF },
28366 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_smaxv2sf3, "__builtin_ia32_pfmax", IX86_BUILTIN_PFMAX, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
28367 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_sminv2sf3, "__builtin_ia32_pfmin", IX86_BUILTIN_PFMIN, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
28368 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_mulv2sf3, "__builtin_ia32_pfmul", IX86_BUILTIN_PFMUL, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
28369 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_rcpit1v2sf3, "__builtin_ia32_pfrcpit1", IX86_BUILTIN_PFRCPIT1, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
28370 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_rcpit2v2sf3, "__builtin_ia32_pfrcpit2", IX86_BUILTIN_PFRCPIT2, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
28371 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_rsqit1v2sf3, "__builtin_ia32_pfrsqit1", IX86_BUILTIN_PFRSQIT1, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
28372 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_subv2sf3, "__builtin_ia32_pfsub", IX86_BUILTIN_PFSUB, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
28373 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_subrv2sf3, "__builtin_ia32_pfsubr", IX86_BUILTIN_PFSUBR, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
28374 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_pmulhrwv4hi3, "__builtin_ia32_pmulhrw", IX86_BUILTIN_PMULHRW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
28376 /* 3DNow!A */
28377 { OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_pf2iw, "__builtin_ia32_pf2iw", IX86_BUILTIN_PF2IW, UNKNOWN, (int) V2SI_FTYPE_V2SF },
28378 { OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_pi2fw, "__builtin_ia32_pi2fw", IX86_BUILTIN_PI2FW, UNKNOWN, (int) V2SF_FTYPE_V2SI },
28379 { OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_pswapdv2si2, "__builtin_ia32_pswapdsi", IX86_BUILTIN_PSWAPDSI, UNKNOWN, (int) V2SI_FTYPE_V2SI },
28380 { OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_pswapdv2sf2, "__builtin_ia32_pswapdsf", IX86_BUILTIN_PSWAPDSF, UNKNOWN, (int) V2SF_FTYPE_V2SF },
28381 { OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_hsubv2sf3, "__builtin_ia32_pfnacc", IX86_BUILTIN_PFNACC, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
28382 { OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_addsubv2sf3, "__builtin_ia32_pfpnacc", IX86_BUILTIN_PFPNACC, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
28384 /* SSE */
28385 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_movmskps, "__builtin_ia32_movmskps", IX86_BUILTIN_MOVMSKPS, UNKNOWN, (int) INT_FTYPE_V4SF },
28386 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_sqrtv4sf2, "__builtin_ia32_sqrtps", IX86_BUILTIN_SQRTPS, UNKNOWN, (int) V4SF_FTYPE_V4SF },
28387 { OPTION_MASK_ISA_SSE, CODE_FOR_sqrtv4sf2, "__builtin_ia32_sqrtps_nr", IX86_BUILTIN_SQRTPS_NR, UNKNOWN, (int) V4SF_FTYPE_V4SF },
28388 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_rsqrtv4sf2, "__builtin_ia32_rsqrtps", IX86_BUILTIN_RSQRTPS, UNKNOWN, (int) V4SF_FTYPE_V4SF },
28389 { OPTION_MASK_ISA_SSE, CODE_FOR_rsqrtv4sf2, "__builtin_ia32_rsqrtps_nr", IX86_BUILTIN_RSQRTPS_NR, UNKNOWN, (int) V4SF_FTYPE_V4SF },
28390 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_rcpv4sf2, "__builtin_ia32_rcpps", IX86_BUILTIN_RCPPS, UNKNOWN, (int) V4SF_FTYPE_V4SF },
28391 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_cvtps2pi, "__builtin_ia32_cvtps2pi", IX86_BUILTIN_CVTPS2PI, UNKNOWN, (int) V2SI_FTYPE_V4SF },
28392 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_cvtss2si, "__builtin_ia32_cvtss2si", IX86_BUILTIN_CVTSS2SI, UNKNOWN, (int) INT_FTYPE_V4SF },
28393 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_64BIT, CODE_FOR_sse_cvtss2siq, "__builtin_ia32_cvtss2si64", IX86_BUILTIN_CVTSS2SI64, UNKNOWN, (int) INT64_FTYPE_V4SF },
28394 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_cvttps2pi, "__builtin_ia32_cvttps2pi", IX86_BUILTIN_CVTTPS2PI, UNKNOWN, (int) V2SI_FTYPE_V4SF },
28395 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_cvttss2si, "__builtin_ia32_cvttss2si", IX86_BUILTIN_CVTTSS2SI, UNKNOWN, (int) INT_FTYPE_V4SF },
28396 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_64BIT, CODE_FOR_sse_cvttss2siq, "__builtin_ia32_cvttss2si64", IX86_BUILTIN_CVTTSS2SI64, UNKNOWN, (int) INT64_FTYPE_V4SF },
28398 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_shufps, "__builtin_ia32_shufps", IX86_BUILTIN_SHUFPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
28400 { OPTION_MASK_ISA_SSE, CODE_FOR_addv4sf3, "__builtin_ia32_addps", IX86_BUILTIN_ADDPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28401 { OPTION_MASK_ISA_SSE, CODE_FOR_subv4sf3, "__builtin_ia32_subps", IX86_BUILTIN_SUBPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28402 { OPTION_MASK_ISA_SSE, CODE_FOR_mulv4sf3, "__builtin_ia32_mulps", IX86_BUILTIN_MULPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28403 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_divv4sf3, "__builtin_ia32_divps", IX86_BUILTIN_DIVPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28404 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmaddv4sf3, "__builtin_ia32_addss", IX86_BUILTIN_ADDSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28405 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmsubv4sf3, "__builtin_ia32_subss", IX86_BUILTIN_SUBSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28406 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmulv4sf3, "__builtin_ia32_mulss", IX86_BUILTIN_MULSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28407 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmdivv4sf3, "__builtin_ia32_divss", IX86_BUILTIN_DIVSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28409 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpeqps", IX86_BUILTIN_CMPEQPS, EQ, (int) V4SF_FTYPE_V4SF_V4SF },
28410 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpltps", IX86_BUILTIN_CMPLTPS, LT, (int) V4SF_FTYPE_V4SF_V4SF },
28411 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpleps", IX86_BUILTIN_CMPLEPS, LE, (int) V4SF_FTYPE_V4SF_V4SF },
28412 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpgtps", IX86_BUILTIN_CMPGTPS, LT, (int) V4SF_FTYPE_V4SF_V4SF_SWAP },
28413 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpgeps", IX86_BUILTIN_CMPGEPS, LE, (int) V4SF_FTYPE_V4SF_V4SF_SWAP },
28414 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpunordps", IX86_BUILTIN_CMPUNORDPS, UNORDERED, (int) V4SF_FTYPE_V4SF_V4SF },
28415 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpneqps", IX86_BUILTIN_CMPNEQPS, NE, (int) V4SF_FTYPE_V4SF_V4SF },
28416 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpnltps", IX86_BUILTIN_CMPNLTPS, UNGE, (int) V4SF_FTYPE_V4SF_V4SF },
28417 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpnleps", IX86_BUILTIN_CMPNLEPS, UNGT, (int) V4SF_FTYPE_V4SF_V4SF },
28418 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpngtps", IX86_BUILTIN_CMPNGTPS, UNGE, (int) V4SF_FTYPE_V4SF_V4SF_SWAP },
28419 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpngeps", IX86_BUILTIN_CMPNGEPS, UNGT, (int) V4SF_FTYPE_V4SF_V4SF_SWAP},
28420 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpordps", IX86_BUILTIN_CMPORDPS, ORDERED, (int) V4SF_FTYPE_V4SF_V4SF },
28421 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpeqss", IX86_BUILTIN_CMPEQSS, EQ, (int) V4SF_FTYPE_V4SF_V4SF },
28422 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpltss", IX86_BUILTIN_CMPLTSS, LT, (int) V4SF_FTYPE_V4SF_V4SF },
28423 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpless", IX86_BUILTIN_CMPLESS, LE, (int) V4SF_FTYPE_V4SF_V4SF },
28424 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpunordss", IX86_BUILTIN_CMPUNORDSS, UNORDERED, (int) V4SF_FTYPE_V4SF_V4SF },
28425 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpneqss", IX86_BUILTIN_CMPNEQSS, NE, (int) V4SF_FTYPE_V4SF_V4SF },
28426 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpnltss", IX86_BUILTIN_CMPNLTSS, UNGE, (int) V4SF_FTYPE_V4SF_V4SF },
28427 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpnless", IX86_BUILTIN_CMPNLESS, UNGT, (int) V4SF_FTYPE_V4SF_V4SF },
28428 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpordss", IX86_BUILTIN_CMPORDSS, ORDERED, (int) V4SF_FTYPE_V4SF_V4SF },
28430 { OPTION_MASK_ISA_SSE, CODE_FOR_sminv4sf3, "__builtin_ia32_minps", IX86_BUILTIN_MINPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28431 { OPTION_MASK_ISA_SSE, CODE_FOR_smaxv4sf3, "__builtin_ia32_maxps", IX86_BUILTIN_MAXPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28432 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmsminv4sf3, "__builtin_ia32_minss", IX86_BUILTIN_MINSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28433 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmsmaxv4sf3, "__builtin_ia32_maxss", IX86_BUILTIN_MAXSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28435 { OPTION_MASK_ISA_SSE, CODE_FOR_andv4sf3, "__builtin_ia32_andps", IX86_BUILTIN_ANDPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28436 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_andnotv4sf3, "__builtin_ia32_andnps", IX86_BUILTIN_ANDNPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28437 { OPTION_MASK_ISA_SSE, CODE_FOR_iorv4sf3, "__builtin_ia32_orps", IX86_BUILTIN_ORPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28438 { OPTION_MASK_ISA_SSE, CODE_FOR_xorv4sf3, "__builtin_ia32_xorps", IX86_BUILTIN_XORPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28440 { OPTION_MASK_ISA_SSE, CODE_FOR_copysignv4sf3, "__builtin_ia32_copysignps", IX86_BUILTIN_CPYSGNPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28442 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_movss, "__builtin_ia32_movss", IX86_BUILTIN_MOVSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28443 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_movhlps_exp, "__builtin_ia32_movhlps", IX86_BUILTIN_MOVHLPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28444 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_movlhps_exp, "__builtin_ia32_movlhps", IX86_BUILTIN_MOVLHPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28445 { OPTION_MASK_ISA_SSE, CODE_FOR_vec_interleave_highv4sf, "__builtin_ia32_unpckhps", IX86_BUILTIN_UNPCKHPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28446 { OPTION_MASK_ISA_SSE, CODE_FOR_vec_interleave_lowv4sf, "__builtin_ia32_unpcklps", IX86_BUILTIN_UNPCKLPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28448 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_cvtpi2ps, "__builtin_ia32_cvtpi2ps", IX86_BUILTIN_CVTPI2PS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V2SI },
28449 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_cvtsi2ss, "__builtin_ia32_cvtsi2ss", IX86_BUILTIN_CVTSI2SS, UNKNOWN, (int) V4SF_FTYPE_V4SF_SI },
28450 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_64BIT, CODE_FOR_sse_cvtsi2ssq, "__builtin_ia32_cvtsi642ss", IX86_BUILTIN_CVTSI642SS, UNKNOWN, V4SF_FTYPE_V4SF_DI },
28452 { OPTION_MASK_ISA_SSE, CODE_FOR_rsqrtsf2, "__builtin_ia32_rsqrtf", IX86_BUILTIN_RSQRTF, UNKNOWN, (int) FLOAT_FTYPE_FLOAT },
28454 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmsqrtv4sf2, "__builtin_ia32_sqrtss", IX86_BUILTIN_SQRTSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_VEC_MERGE },
28455 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmrsqrtv4sf2, "__builtin_ia32_rsqrtss", IX86_BUILTIN_RSQRTSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_VEC_MERGE },
28456 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmrcpv4sf2, "__builtin_ia32_rcpss", IX86_BUILTIN_RCPSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_VEC_MERGE },
28458 { OPTION_MASK_ISA_SSE, CODE_FOR_abstf2, 0, IX86_BUILTIN_FABSQ, UNKNOWN, (int) FLOAT128_FTYPE_FLOAT128 },
28459 { OPTION_MASK_ISA_SSE, CODE_FOR_copysigntf3, 0, IX86_BUILTIN_COPYSIGNQ, UNKNOWN, (int) FLOAT128_FTYPE_FLOAT128_FLOAT128 },
28461 /* SSE MMX or 3Dnow!A */
28462 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_uavgv8qi3, "__builtin_ia32_pavgb", IX86_BUILTIN_PAVGB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
28463 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_uavgv4hi3, "__builtin_ia32_pavgw", IX86_BUILTIN_PAVGW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
28464 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_umulv4hi3_highpart, "__builtin_ia32_pmulhuw", IX86_BUILTIN_PMULHUW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
28466 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_umaxv8qi3, "__builtin_ia32_pmaxub", IX86_BUILTIN_PMAXUB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
28467 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_smaxv4hi3, "__builtin_ia32_pmaxsw", IX86_BUILTIN_PMAXSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
28468 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_uminv8qi3, "__builtin_ia32_pminub", IX86_BUILTIN_PMINUB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
28469 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_sminv4hi3, "__builtin_ia32_pminsw", IX86_BUILTIN_PMINSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
28471 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_psadbw, "__builtin_ia32_psadbw", IX86_BUILTIN_PSADBW, UNKNOWN, (int) V1DI_FTYPE_V8QI_V8QI },
28472 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_pmovmskb, "__builtin_ia32_pmovmskb", IX86_BUILTIN_PMOVMSKB, UNKNOWN, (int) INT_FTYPE_V8QI },
28474 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_pshufw, "__builtin_ia32_pshufw", IX86_BUILTIN_PSHUFW, UNKNOWN, (int) V4HI_FTYPE_V4HI_INT },
28476 /* SSE2 */
28477 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_shufpd, "__builtin_ia32_shufpd", IX86_BUILTIN_SHUFPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
28479 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_movmskpd, "__builtin_ia32_movmskpd", IX86_BUILTIN_MOVMSKPD, UNKNOWN, (int) INT_FTYPE_V2DF },
28480 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_pmovmskb, "__builtin_ia32_pmovmskb128", IX86_BUILTIN_PMOVMSKB128, UNKNOWN, (int) INT_FTYPE_V16QI },
28481 { OPTION_MASK_ISA_SSE2, CODE_FOR_sqrtv2df2, "__builtin_ia32_sqrtpd", IX86_BUILTIN_SQRTPD, UNKNOWN, (int) V2DF_FTYPE_V2DF },
28482 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtdq2pd, "__builtin_ia32_cvtdq2pd", IX86_BUILTIN_CVTDQ2PD, UNKNOWN, (int) V2DF_FTYPE_V4SI },
28483 { OPTION_MASK_ISA_SSE2, CODE_FOR_floatv4siv4sf2, "__builtin_ia32_cvtdq2ps", IX86_BUILTIN_CVTDQ2PS, UNKNOWN, (int) V4SF_FTYPE_V4SI },
28485 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtpd2dq, "__builtin_ia32_cvtpd2dq", IX86_BUILTIN_CVTPD2DQ, UNKNOWN, (int) V4SI_FTYPE_V2DF },
28486 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtpd2pi, "__builtin_ia32_cvtpd2pi", IX86_BUILTIN_CVTPD2PI, UNKNOWN, (int) V2SI_FTYPE_V2DF },
28487 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtpd2ps, "__builtin_ia32_cvtpd2ps", IX86_BUILTIN_CVTPD2PS, UNKNOWN, (int) V4SF_FTYPE_V2DF },
28488 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvttpd2dq, "__builtin_ia32_cvttpd2dq", IX86_BUILTIN_CVTTPD2DQ, UNKNOWN, (int) V4SI_FTYPE_V2DF },
28489 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvttpd2pi, "__builtin_ia32_cvttpd2pi", IX86_BUILTIN_CVTTPD2PI, UNKNOWN, (int) V2SI_FTYPE_V2DF },
28491 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtpi2pd, "__builtin_ia32_cvtpi2pd", IX86_BUILTIN_CVTPI2PD, UNKNOWN, (int) V2DF_FTYPE_V2SI },
28493 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtsd2si, "__builtin_ia32_cvtsd2si", IX86_BUILTIN_CVTSD2SI, UNKNOWN, (int) INT_FTYPE_V2DF },
28494 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvttsd2si, "__builtin_ia32_cvttsd2si", IX86_BUILTIN_CVTTSD2SI, UNKNOWN, (int) INT_FTYPE_V2DF },
28495 { OPTION_MASK_ISA_SSE2 | OPTION_MASK_ISA_64BIT, CODE_FOR_sse2_cvtsd2siq, "__builtin_ia32_cvtsd2si64", IX86_BUILTIN_CVTSD2SI64, UNKNOWN, (int) INT64_FTYPE_V2DF },
28496 { OPTION_MASK_ISA_SSE2 | OPTION_MASK_ISA_64BIT, CODE_FOR_sse2_cvttsd2siq, "__builtin_ia32_cvttsd2si64", IX86_BUILTIN_CVTTSD2SI64, UNKNOWN, (int) INT64_FTYPE_V2DF },
28498 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_fix_notruncv4sfv4si, "__builtin_ia32_cvtps2dq", IX86_BUILTIN_CVTPS2DQ, UNKNOWN, (int) V4SI_FTYPE_V4SF },
28499 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtps2pd, "__builtin_ia32_cvtps2pd", IX86_BUILTIN_CVTPS2PD, UNKNOWN, (int) V2DF_FTYPE_V4SF },
28500 { OPTION_MASK_ISA_SSE2, CODE_FOR_fix_truncv4sfv4si2, "__builtin_ia32_cvttps2dq", IX86_BUILTIN_CVTTPS2DQ, UNKNOWN, (int) V4SI_FTYPE_V4SF },
28502 { OPTION_MASK_ISA_SSE2, CODE_FOR_addv2df3, "__builtin_ia32_addpd", IX86_BUILTIN_ADDPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28503 { OPTION_MASK_ISA_SSE2, CODE_FOR_subv2df3, "__builtin_ia32_subpd", IX86_BUILTIN_SUBPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28504 { OPTION_MASK_ISA_SSE2, CODE_FOR_mulv2df3, "__builtin_ia32_mulpd", IX86_BUILTIN_MULPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28505 { OPTION_MASK_ISA_SSE2, CODE_FOR_divv2df3, "__builtin_ia32_divpd", IX86_BUILTIN_DIVPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28506 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmaddv2df3, "__builtin_ia32_addsd", IX86_BUILTIN_ADDSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28507 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmsubv2df3, "__builtin_ia32_subsd", IX86_BUILTIN_SUBSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28508 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmulv2df3, "__builtin_ia32_mulsd", IX86_BUILTIN_MULSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28509 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmdivv2df3, "__builtin_ia32_divsd", IX86_BUILTIN_DIVSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28511 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpeqpd", IX86_BUILTIN_CMPEQPD, EQ, (int) V2DF_FTYPE_V2DF_V2DF },
28512 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpltpd", IX86_BUILTIN_CMPLTPD, LT, (int) V2DF_FTYPE_V2DF_V2DF },
28513 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmplepd", IX86_BUILTIN_CMPLEPD, LE, (int) V2DF_FTYPE_V2DF_V2DF },
28514 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpgtpd", IX86_BUILTIN_CMPGTPD, LT, (int) V2DF_FTYPE_V2DF_V2DF_SWAP },
28515 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpgepd", IX86_BUILTIN_CMPGEPD, LE, (int) V2DF_FTYPE_V2DF_V2DF_SWAP},
28516 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpunordpd", IX86_BUILTIN_CMPUNORDPD, UNORDERED, (int) V2DF_FTYPE_V2DF_V2DF },
28517 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpneqpd", IX86_BUILTIN_CMPNEQPD, NE, (int) V2DF_FTYPE_V2DF_V2DF },
28518 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpnltpd", IX86_BUILTIN_CMPNLTPD, UNGE, (int) V2DF_FTYPE_V2DF_V2DF },
28519 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpnlepd", IX86_BUILTIN_CMPNLEPD, UNGT, (int) V2DF_FTYPE_V2DF_V2DF },
28520 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpngtpd", IX86_BUILTIN_CMPNGTPD, UNGE, (int) V2DF_FTYPE_V2DF_V2DF_SWAP },
28521 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpngepd", IX86_BUILTIN_CMPNGEPD, UNGT, (int) V2DF_FTYPE_V2DF_V2DF_SWAP },
28522 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpordpd", IX86_BUILTIN_CMPORDPD, ORDERED, (int) V2DF_FTYPE_V2DF_V2DF },
28523 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmpeqsd", IX86_BUILTIN_CMPEQSD, EQ, (int) V2DF_FTYPE_V2DF_V2DF },
28524 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmpltsd", IX86_BUILTIN_CMPLTSD, LT, (int) V2DF_FTYPE_V2DF_V2DF },
28525 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmplesd", IX86_BUILTIN_CMPLESD, LE, (int) V2DF_FTYPE_V2DF_V2DF },
28526 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmpunordsd", IX86_BUILTIN_CMPUNORDSD, UNORDERED, (int) V2DF_FTYPE_V2DF_V2DF },
28527 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmpneqsd", IX86_BUILTIN_CMPNEQSD, NE, (int) V2DF_FTYPE_V2DF_V2DF },
28528 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmpnltsd", IX86_BUILTIN_CMPNLTSD, UNGE, (int) V2DF_FTYPE_V2DF_V2DF },
28529 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmpnlesd", IX86_BUILTIN_CMPNLESD, UNGT, (int) V2DF_FTYPE_V2DF_V2DF },
28530 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmpordsd", IX86_BUILTIN_CMPORDSD, ORDERED, (int) V2DF_FTYPE_V2DF_V2DF },
28532 { OPTION_MASK_ISA_SSE2, CODE_FOR_sminv2df3, "__builtin_ia32_minpd", IX86_BUILTIN_MINPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28533 { OPTION_MASK_ISA_SSE2, CODE_FOR_smaxv2df3, "__builtin_ia32_maxpd", IX86_BUILTIN_MAXPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28534 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmsminv2df3, "__builtin_ia32_minsd", IX86_BUILTIN_MINSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28535 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmsmaxv2df3, "__builtin_ia32_maxsd", IX86_BUILTIN_MAXSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28537 { OPTION_MASK_ISA_SSE2, CODE_FOR_andv2df3, "__builtin_ia32_andpd", IX86_BUILTIN_ANDPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28538 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_andnotv2df3, "__builtin_ia32_andnpd", IX86_BUILTIN_ANDNPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28539 { OPTION_MASK_ISA_SSE2, CODE_FOR_iorv2df3, "__builtin_ia32_orpd", IX86_BUILTIN_ORPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28540 { OPTION_MASK_ISA_SSE2, CODE_FOR_xorv2df3, "__builtin_ia32_xorpd", IX86_BUILTIN_XORPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28542 { OPTION_MASK_ISA_SSE2, CODE_FOR_copysignv2df3, "__builtin_ia32_copysignpd", IX86_BUILTIN_CPYSGNPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28544 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_movsd, "__builtin_ia32_movsd", IX86_BUILTIN_MOVSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28545 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_highv2df, "__builtin_ia32_unpckhpd", IX86_BUILTIN_UNPCKHPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28546 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_lowv2df, "__builtin_ia32_unpcklpd", IX86_BUILTIN_UNPCKLPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28548 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_pack_sfix_v2df, "__builtin_ia32_vec_pack_sfix", IX86_BUILTIN_VEC_PACK_SFIX, UNKNOWN, (int) V4SI_FTYPE_V2DF_V2DF },
28550 { OPTION_MASK_ISA_SSE2, CODE_FOR_addv16qi3, "__builtin_ia32_paddb128", IX86_BUILTIN_PADDB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
28551 { OPTION_MASK_ISA_SSE2, CODE_FOR_addv8hi3, "__builtin_ia32_paddw128", IX86_BUILTIN_PADDW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28552 { OPTION_MASK_ISA_SSE2, CODE_FOR_addv4si3, "__builtin_ia32_paddd128", IX86_BUILTIN_PADDD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
28553 { OPTION_MASK_ISA_SSE2, CODE_FOR_addv2di3, "__builtin_ia32_paddq128", IX86_BUILTIN_PADDQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
28554 { OPTION_MASK_ISA_SSE2, CODE_FOR_subv16qi3, "__builtin_ia32_psubb128", IX86_BUILTIN_PSUBB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
28555 { OPTION_MASK_ISA_SSE2, CODE_FOR_subv8hi3, "__builtin_ia32_psubw128", IX86_BUILTIN_PSUBW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28556 { OPTION_MASK_ISA_SSE2, CODE_FOR_subv4si3, "__builtin_ia32_psubd128", IX86_BUILTIN_PSUBD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
28557 { OPTION_MASK_ISA_SSE2, CODE_FOR_subv2di3, "__builtin_ia32_psubq128", IX86_BUILTIN_PSUBQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
28559 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ssaddv16qi3, "__builtin_ia32_paddsb128", IX86_BUILTIN_PADDSB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
28560 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ssaddv8hi3, "__builtin_ia32_paddsw128", IX86_BUILTIN_PADDSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28561 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_sssubv16qi3, "__builtin_ia32_psubsb128", IX86_BUILTIN_PSUBSB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
28562 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_sssubv8hi3, "__builtin_ia32_psubsw128", IX86_BUILTIN_PSUBSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28563 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_usaddv16qi3, "__builtin_ia32_paddusb128", IX86_BUILTIN_PADDUSB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
28564 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_usaddv8hi3, "__builtin_ia32_paddusw128", IX86_BUILTIN_PADDUSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28565 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ussubv16qi3, "__builtin_ia32_psubusb128", IX86_BUILTIN_PSUBUSB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
28566 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ussubv8hi3, "__builtin_ia32_psubusw128", IX86_BUILTIN_PSUBUSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28568 { OPTION_MASK_ISA_SSE2, CODE_FOR_mulv8hi3, "__builtin_ia32_pmullw128", IX86_BUILTIN_PMULLW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28569 { OPTION_MASK_ISA_SSE2, CODE_FOR_smulv8hi3_highpart, "__builtin_ia32_pmulhw128", IX86_BUILTIN_PMULHW128, UNKNOWN,(int) V8HI_FTYPE_V8HI_V8HI },
28571 { OPTION_MASK_ISA_SSE2, CODE_FOR_andv2di3, "__builtin_ia32_pand128", IX86_BUILTIN_PAND128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
28572 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_andnotv2di3, "__builtin_ia32_pandn128", IX86_BUILTIN_PANDN128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
28573 { OPTION_MASK_ISA_SSE2, CODE_FOR_iorv2di3, "__builtin_ia32_por128", IX86_BUILTIN_POR128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
28574 { OPTION_MASK_ISA_SSE2, CODE_FOR_xorv2di3, "__builtin_ia32_pxor128", IX86_BUILTIN_PXOR128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
28576 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_uavgv16qi3, "__builtin_ia32_pavgb128", IX86_BUILTIN_PAVGB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
28577 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_uavgv8hi3, "__builtin_ia32_pavgw128", IX86_BUILTIN_PAVGW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28579 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_eqv16qi3, "__builtin_ia32_pcmpeqb128", IX86_BUILTIN_PCMPEQB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
28580 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_eqv8hi3, "__builtin_ia32_pcmpeqw128", IX86_BUILTIN_PCMPEQW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28581 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_eqv4si3, "__builtin_ia32_pcmpeqd128", IX86_BUILTIN_PCMPEQD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
28582 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_gtv16qi3, "__builtin_ia32_pcmpgtb128", IX86_BUILTIN_PCMPGTB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
28583 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_gtv8hi3, "__builtin_ia32_pcmpgtw128", IX86_BUILTIN_PCMPGTW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28584 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_gtv4si3, "__builtin_ia32_pcmpgtd128", IX86_BUILTIN_PCMPGTD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
28586 { OPTION_MASK_ISA_SSE2, CODE_FOR_umaxv16qi3, "__builtin_ia32_pmaxub128", IX86_BUILTIN_PMAXUB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
28587 { OPTION_MASK_ISA_SSE2, CODE_FOR_smaxv8hi3, "__builtin_ia32_pmaxsw128", IX86_BUILTIN_PMAXSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28588 { OPTION_MASK_ISA_SSE2, CODE_FOR_uminv16qi3, "__builtin_ia32_pminub128", IX86_BUILTIN_PMINUB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
28589 { OPTION_MASK_ISA_SSE2, CODE_FOR_sminv8hi3, "__builtin_ia32_pminsw128", IX86_BUILTIN_PMINSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28591 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_highv16qi, "__builtin_ia32_punpckhbw128", IX86_BUILTIN_PUNPCKHBW128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
28592 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_highv8hi, "__builtin_ia32_punpckhwd128", IX86_BUILTIN_PUNPCKHWD128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28593 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_highv4si, "__builtin_ia32_punpckhdq128", IX86_BUILTIN_PUNPCKHDQ128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
28594 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_highv2di, "__builtin_ia32_punpckhqdq128", IX86_BUILTIN_PUNPCKHQDQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
28595 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_lowv16qi, "__builtin_ia32_punpcklbw128", IX86_BUILTIN_PUNPCKLBW128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
28596 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_lowv8hi, "__builtin_ia32_punpcklwd128", IX86_BUILTIN_PUNPCKLWD128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28597 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_lowv4si, "__builtin_ia32_punpckldq128", IX86_BUILTIN_PUNPCKLDQ128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
28598 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_lowv2di, "__builtin_ia32_punpcklqdq128", IX86_BUILTIN_PUNPCKLQDQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
28600 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_packsswb, "__builtin_ia32_packsswb128", IX86_BUILTIN_PACKSSWB128, UNKNOWN, (int) V16QI_FTYPE_V8HI_V8HI },
28601 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_packssdw, "__builtin_ia32_packssdw128", IX86_BUILTIN_PACKSSDW128, UNKNOWN, (int) V8HI_FTYPE_V4SI_V4SI },
28602 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_packuswb, "__builtin_ia32_packuswb128", IX86_BUILTIN_PACKUSWB128, UNKNOWN, (int) V16QI_FTYPE_V8HI_V8HI },
28604 { OPTION_MASK_ISA_SSE2, CODE_FOR_umulv8hi3_highpart, "__builtin_ia32_pmulhuw128", IX86_BUILTIN_PMULHUW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28605 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_psadbw, "__builtin_ia32_psadbw128", IX86_BUILTIN_PSADBW128, UNKNOWN, (int) V2DI_FTYPE_V16QI_V16QI },
28607 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_umulv1siv1di3, "__builtin_ia32_pmuludq", IX86_BUILTIN_PMULUDQ, UNKNOWN, (int) V1DI_FTYPE_V2SI_V2SI },
28608 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_widen_umult_even_v4si, "__builtin_ia32_pmuludq128", IX86_BUILTIN_PMULUDQ128, UNKNOWN, (int) V2DI_FTYPE_V4SI_V4SI },
28610 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_pmaddwd, "__builtin_ia32_pmaddwd128", IX86_BUILTIN_PMADDWD128, UNKNOWN, (int) V4SI_FTYPE_V8HI_V8HI },
28612 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtsi2sd, "__builtin_ia32_cvtsi2sd", IX86_BUILTIN_CVTSI2SD, UNKNOWN, (int) V2DF_FTYPE_V2DF_SI },
28613 { OPTION_MASK_ISA_SSE2 | OPTION_MASK_ISA_64BIT, CODE_FOR_sse2_cvtsi2sdq, "__builtin_ia32_cvtsi642sd", IX86_BUILTIN_CVTSI642SD, UNKNOWN, (int) V2DF_FTYPE_V2DF_DI },
28614 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtsd2ss, "__builtin_ia32_cvtsd2ss", IX86_BUILTIN_CVTSD2SS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V2DF },
28615 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtss2sd, "__builtin_ia32_cvtss2sd", IX86_BUILTIN_CVTSS2SD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V4SF },
28617 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ashlv1ti3, "__builtin_ia32_pslldqi128", IX86_BUILTIN_PSLLDQI128, UNKNOWN, (int) V2DI_FTYPE_V2DI_INT_CONVERT },
28618 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashlv8hi3, "__builtin_ia32_psllwi128", IX86_BUILTIN_PSLLWI128, UNKNOWN, (int) V8HI_FTYPE_V8HI_SI_COUNT },
28619 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashlv4si3, "__builtin_ia32_pslldi128", IX86_BUILTIN_PSLLDI128, UNKNOWN, (int) V4SI_FTYPE_V4SI_SI_COUNT },
28620 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashlv2di3, "__builtin_ia32_psllqi128", IX86_BUILTIN_PSLLQI128, UNKNOWN, (int) V2DI_FTYPE_V2DI_SI_COUNT },
28621 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashlv8hi3, "__builtin_ia32_psllw128", IX86_BUILTIN_PSLLW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_COUNT },
28622 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashlv4si3, "__builtin_ia32_pslld128", IX86_BUILTIN_PSLLD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_COUNT },
28623 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashlv2di3, "__builtin_ia32_psllq128", IX86_BUILTIN_PSLLQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_COUNT },
28625 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_lshrv1ti3, "__builtin_ia32_psrldqi128", IX86_BUILTIN_PSRLDQI128, UNKNOWN, (int) V2DI_FTYPE_V2DI_INT_CONVERT },
28626 { OPTION_MASK_ISA_SSE2, CODE_FOR_lshrv8hi3, "__builtin_ia32_psrlwi128", IX86_BUILTIN_PSRLWI128, UNKNOWN, (int) V8HI_FTYPE_V8HI_SI_COUNT },
28627 { OPTION_MASK_ISA_SSE2, CODE_FOR_lshrv4si3, "__builtin_ia32_psrldi128", IX86_BUILTIN_PSRLDI128, UNKNOWN, (int) V4SI_FTYPE_V4SI_SI_COUNT },
28628 { OPTION_MASK_ISA_SSE2, CODE_FOR_lshrv2di3, "__builtin_ia32_psrlqi128", IX86_BUILTIN_PSRLQI128, UNKNOWN, (int) V2DI_FTYPE_V2DI_SI_COUNT },
28629 { OPTION_MASK_ISA_SSE2, CODE_FOR_lshrv8hi3, "__builtin_ia32_psrlw128", IX86_BUILTIN_PSRLW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_COUNT },
28630 { OPTION_MASK_ISA_SSE2, CODE_FOR_lshrv4si3, "__builtin_ia32_psrld128", IX86_BUILTIN_PSRLD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_COUNT },
28631 { OPTION_MASK_ISA_SSE2, CODE_FOR_lshrv2di3, "__builtin_ia32_psrlq128", IX86_BUILTIN_PSRLQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_COUNT },
28633 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashrv8hi3, "__builtin_ia32_psrawi128", IX86_BUILTIN_PSRAWI128, UNKNOWN, (int) V8HI_FTYPE_V8HI_SI_COUNT },
28634 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashrv4si3, "__builtin_ia32_psradi128", IX86_BUILTIN_PSRADI128, UNKNOWN, (int) V4SI_FTYPE_V4SI_SI_COUNT },
28635 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashrv8hi3, "__builtin_ia32_psraw128", IX86_BUILTIN_PSRAW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_COUNT },
28636 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashrv4si3, "__builtin_ia32_psrad128", IX86_BUILTIN_PSRAD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_COUNT },
28638 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_pshufd, "__builtin_ia32_pshufd", IX86_BUILTIN_PSHUFD, UNKNOWN, (int) V4SI_FTYPE_V4SI_INT },
28639 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_pshuflw, "__builtin_ia32_pshuflw", IX86_BUILTIN_PSHUFLW, UNKNOWN, (int) V8HI_FTYPE_V8HI_INT },
28640 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_pshufhw, "__builtin_ia32_pshufhw", IX86_BUILTIN_PSHUFHW, UNKNOWN, (int) V8HI_FTYPE_V8HI_INT },
28642 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmsqrtv2df2, "__builtin_ia32_sqrtsd", IX86_BUILTIN_SQRTSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_VEC_MERGE },
28644 { OPTION_MASK_ISA_SSE, CODE_FOR_sse2_movq128, "__builtin_ia32_movq128", IX86_BUILTIN_MOVQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI },
28646 /* SSE2 MMX */
28647 { OPTION_MASK_ISA_SSE2, CODE_FOR_mmx_addv1di3, "__builtin_ia32_paddq", IX86_BUILTIN_PADDQ, UNKNOWN, (int) V1DI_FTYPE_V1DI_V1DI },
28648 { OPTION_MASK_ISA_SSE2, CODE_FOR_mmx_subv1di3, "__builtin_ia32_psubq", IX86_BUILTIN_PSUBQ, UNKNOWN, (int) V1DI_FTYPE_V1DI_V1DI },
28650 /* SSE3 */
28651 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_movshdup, "__builtin_ia32_movshdup", IX86_BUILTIN_MOVSHDUP, UNKNOWN, (int) V4SF_FTYPE_V4SF},
28652 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_movsldup, "__builtin_ia32_movsldup", IX86_BUILTIN_MOVSLDUP, UNKNOWN, (int) V4SF_FTYPE_V4SF },
28654 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_addsubv4sf3, "__builtin_ia32_addsubps", IX86_BUILTIN_ADDSUBPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28655 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_addsubv2df3, "__builtin_ia32_addsubpd", IX86_BUILTIN_ADDSUBPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28656 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_haddv4sf3, "__builtin_ia32_haddps", IX86_BUILTIN_HADDPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28657 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_haddv2df3, "__builtin_ia32_haddpd", IX86_BUILTIN_HADDPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28658 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_hsubv4sf3, "__builtin_ia32_hsubps", IX86_BUILTIN_HSUBPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28659 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_hsubv2df3, "__builtin_ia32_hsubpd", IX86_BUILTIN_HSUBPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28661 /* SSSE3 */
28662 { OPTION_MASK_ISA_SSSE3, CODE_FOR_absv16qi2, "__builtin_ia32_pabsb128", IX86_BUILTIN_PABSB128, UNKNOWN, (int) V16QI_FTYPE_V16QI },
28663 { OPTION_MASK_ISA_SSSE3, CODE_FOR_absv8qi2, "__builtin_ia32_pabsb", IX86_BUILTIN_PABSB, UNKNOWN, (int) V8QI_FTYPE_V8QI },
28664 { OPTION_MASK_ISA_SSSE3, CODE_FOR_absv8hi2, "__builtin_ia32_pabsw128", IX86_BUILTIN_PABSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI },
28665 { OPTION_MASK_ISA_SSSE3, CODE_FOR_absv4hi2, "__builtin_ia32_pabsw", IX86_BUILTIN_PABSW, UNKNOWN, (int) V4HI_FTYPE_V4HI },
28666 { OPTION_MASK_ISA_SSSE3, CODE_FOR_absv4si2, "__builtin_ia32_pabsd128", IX86_BUILTIN_PABSD128, UNKNOWN, (int) V4SI_FTYPE_V4SI },
28667 { OPTION_MASK_ISA_SSSE3, CODE_FOR_absv2si2, "__builtin_ia32_pabsd", IX86_BUILTIN_PABSD, UNKNOWN, (int) V2SI_FTYPE_V2SI },
28669 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phaddwv8hi3, "__builtin_ia32_phaddw128", IX86_BUILTIN_PHADDW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28670 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phaddwv4hi3, "__builtin_ia32_phaddw", IX86_BUILTIN_PHADDW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
28671 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phadddv4si3, "__builtin_ia32_phaddd128", IX86_BUILTIN_PHADDD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
28672 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phadddv2si3, "__builtin_ia32_phaddd", IX86_BUILTIN_PHADDD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
28673 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phaddswv8hi3, "__builtin_ia32_phaddsw128", IX86_BUILTIN_PHADDSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28674 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phaddswv4hi3, "__builtin_ia32_phaddsw", IX86_BUILTIN_PHADDSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
28675 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phsubwv8hi3, "__builtin_ia32_phsubw128", IX86_BUILTIN_PHSUBW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28676 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phsubwv4hi3, "__builtin_ia32_phsubw", IX86_BUILTIN_PHSUBW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
28677 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phsubdv4si3, "__builtin_ia32_phsubd128", IX86_BUILTIN_PHSUBD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
28678 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phsubdv2si3, "__builtin_ia32_phsubd", IX86_BUILTIN_PHSUBD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
28679 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phsubswv8hi3, "__builtin_ia32_phsubsw128", IX86_BUILTIN_PHSUBSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28680 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phsubswv4hi3, "__builtin_ia32_phsubsw", IX86_BUILTIN_PHSUBSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
28681 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_pmaddubsw128, "__builtin_ia32_pmaddubsw128", IX86_BUILTIN_PMADDUBSW128, UNKNOWN, (int) V8HI_FTYPE_V16QI_V16QI },
28682 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_pmaddubsw, "__builtin_ia32_pmaddubsw", IX86_BUILTIN_PMADDUBSW, UNKNOWN, (int) V4HI_FTYPE_V8QI_V8QI },
28683 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_pmulhrswv8hi3, "__builtin_ia32_pmulhrsw128", IX86_BUILTIN_PMULHRSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28684 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_pmulhrswv4hi3, "__builtin_ia32_pmulhrsw", IX86_BUILTIN_PMULHRSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
28685 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_pshufbv16qi3, "__builtin_ia32_pshufb128", IX86_BUILTIN_PSHUFB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
28686 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_pshufbv8qi3, "__builtin_ia32_pshufb", IX86_BUILTIN_PSHUFB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
28687 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_psignv16qi3, "__builtin_ia32_psignb128", IX86_BUILTIN_PSIGNB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
28688 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_psignv8qi3, "__builtin_ia32_psignb", IX86_BUILTIN_PSIGNB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
28689 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_psignv8hi3, "__builtin_ia32_psignw128", IX86_BUILTIN_PSIGNW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28690 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_psignv4hi3, "__builtin_ia32_psignw", IX86_BUILTIN_PSIGNW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
28691 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_psignv4si3, "__builtin_ia32_psignd128", IX86_BUILTIN_PSIGND128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
28692 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_psignv2si3, "__builtin_ia32_psignd", IX86_BUILTIN_PSIGND, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
28694 /* SSSE3. */
28695 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_palignrti, "__builtin_ia32_palignr128", IX86_BUILTIN_PALIGNR128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_INT_CONVERT },
28696 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_palignrdi, "__builtin_ia32_palignr", IX86_BUILTIN_PALIGNR, UNKNOWN, (int) V1DI_FTYPE_V1DI_V1DI_INT_CONVERT },
28698 /* SSE4.1 */
28699 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_blendpd, "__builtin_ia32_blendpd", IX86_BUILTIN_BLENDPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
28700 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_blendps, "__builtin_ia32_blendps", IX86_BUILTIN_BLENDPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
28701 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_blendvpd, "__builtin_ia32_blendvpd", IX86_BUILTIN_BLENDVPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_V2DF },
28702 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_blendvps, "__builtin_ia32_blendvps", IX86_BUILTIN_BLENDVPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_V4SF },
28703 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_dppd, "__builtin_ia32_dppd", IX86_BUILTIN_DPPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
28704 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_dpps, "__builtin_ia32_dpps", IX86_BUILTIN_DPPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
28705 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_insertps, "__builtin_ia32_insertps128", IX86_BUILTIN_INSERTPS128, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
28706 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_mpsadbw, "__builtin_ia32_mpsadbw128", IX86_BUILTIN_MPSADBW128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI_INT },
28707 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_pblendvb, "__builtin_ia32_pblendvb128", IX86_BUILTIN_PBLENDVB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI_V16QI },
28708 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_pblendw, "__builtin_ia32_pblendw128", IX86_BUILTIN_PBLENDW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_INT },
28710 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_sign_extendv8qiv8hi2, "__builtin_ia32_pmovsxbw128", IX86_BUILTIN_PMOVSXBW128, UNKNOWN, (int) V8HI_FTYPE_V16QI },
28711 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_sign_extendv4qiv4si2, "__builtin_ia32_pmovsxbd128", IX86_BUILTIN_PMOVSXBD128, UNKNOWN, (int) V4SI_FTYPE_V16QI },
28712 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_sign_extendv2qiv2di2, "__builtin_ia32_pmovsxbq128", IX86_BUILTIN_PMOVSXBQ128, UNKNOWN, (int) V2DI_FTYPE_V16QI },
28713 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_sign_extendv4hiv4si2, "__builtin_ia32_pmovsxwd128", IX86_BUILTIN_PMOVSXWD128, UNKNOWN, (int) V4SI_FTYPE_V8HI },
28714 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_sign_extendv2hiv2di2, "__builtin_ia32_pmovsxwq128", IX86_BUILTIN_PMOVSXWQ128, UNKNOWN, (int) V2DI_FTYPE_V8HI },
28715 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_sign_extendv2siv2di2, "__builtin_ia32_pmovsxdq128", IX86_BUILTIN_PMOVSXDQ128, UNKNOWN, (int) V2DI_FTYPE_V4SI },
28716 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_zero_extendv8qiv8hi2, "__builtin_ia32_pmovzxbw128", IX86_BUILTIN_PMOVZXBW128, UNKNOWN, (int) V8HI_FTYPE_V16QI },
28717 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_zero_extendv4qiv4si2, "__builtin_ia32_pmovzxbd128", IX86_BUILTIN_PMOVZXBD128, UNKNOWN, (int) V4SI_FTYPE_V16QI },
28718 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_zero_extendv2qiv2di2, "__builtin_ia32_pmovzxbq128", IX86_BUILTIN_PMOVZXBQ128, UNKNOWN, (int) V2DI_FTYPE_V16QI },
28719 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_zero_extendv4hiv4si2, "__builtin_ia32_pmovzxwd128", IX86_BUILTIN_PMOVZXWD128, UNKNOWN, (int) V4SI_FTYPE_V8HI },
28720 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_zero_extendv2hiv2di2, "__builtin_ia32_pmovzxwq128", IX86_BUILTIN_PMOVZXWQ128, UNKNOWN, (int) V2DI_FTYPE_V8HI },
28721 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_zero_extendv2siv2di2, "__builtin_ia32_pmovzxdq128", IX86_BUILTIN_PMOVZXDQ128, UNKNOWN, (int) V2DI_FTYPE_V4SI },
28722 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_phminposuw, "__builtin_ia32_phminposuw128", IX86_BUILTIN_PHMINPOSUW128, UNKNOWN, (int) V8HI_FTYPE_V8HI },
28724 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_packusdw, "__builtin_ia32_packusdw128", IX86_BUILTIN_PACKUSDW128, UNKNOWN, (int) V8HI_FTYPE_V4SI_V4SI },
28725 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_eqv2di3, "__builtin_ia32_pcmpeqq", IX86_BUILTIN_PCMPEQQ, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
28726 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_smaxv16qi3, "__builtin_ia32_pmaxsb128", IX86_BUILTIN_PMAXSB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
28727 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_smaxv4si3, "__builtin_ia32_pmaxsd128", IX86_BUILTIN_PMAXSD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
28728 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_umaxv4si3, "__builtin_ia32_pmaxud128", IX86_BUILTIN_PMAXUD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
28729 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_umaxv8hi3, "__builtin_ia32_pmaxuw128", IX86_BUILTIN_PMAXUW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28730 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sminv16qi3, "__builtin_ia32_pminsb128", IX86_BUILTIN_PMINSB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
28731 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sminv4si3, "__builtin_ia32_pminsd128", IX86_BUILTIN_PMINSD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
28732 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_uminv4si3, "__builtin_ia32_pminud128", IX86_BUILTIN_PMINUD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
28733 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_uminv8hi3, "__builtin_ia32_pminuw128", IX86_BUILTIN_PMINUW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28734 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_mulv2siv2di3, "__builtin_ia32_pmuldq128", IX86_BUILTIN_PMULDQ128, UNKNOWN, (int) V2DI_FTYPE_V4SI_V4SI },
28735 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_mulv4si3, "__builtin_ia32_pmulld128", IX86_BUILTIN_PMULLD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
28737 /* SSE4.1 */
28738 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundpd, "__builtin_ia32_roundpd", IX86_BUILTIN_ROUNDPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_INT },
28739 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundps, "__builtin_ia32_roundps", IX86_BUILTIN_ROUNDPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_INT },
28740 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundsd, "__builtin_ia32_roundsd", IX86_BUILTIN_ROUNDSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
28741 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundss, "__builtin_ia32_roundss", IX86_BUILTIN_ROUNDSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
28743 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundpd, "__builtin_ia32_floorpd", IX86_BUILTIN_FLOORPD, (enum rtx_code) ROUND_FLOOR, (int) V2DF_FTYPE_V2DF_ROUND },
28744 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundpd, "__builtin_ia32_ceilpd", IX86_BUILTIN_CEILPD, (enum rtx_code) ROUND_CEIL, (int) V2DF_FTYPE_V2DF_ROUND },
28745 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundpd, "__builtin_ia32_truncpd", IX86_BUILTIN_TRUNCPD, (enum rtx_code) ROUND_TRUNC, (int) V2DF_FTYPE_V2DF_ROUND },
28746 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundpd, "__builtin_ia32_rintpd", IX86_BUILTIN_RINTPD, (enum rtx_code) ROUND_MXCSR, (int) V2DF_FTYPE_V2DF_ROUND },
28748 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundpd_vec_pack_sfix, "__builtin_ia32_floorpd_vec_pack_sfix", IX86_BUILTIN_FLOORPD_VEC_PACK_SFIX, (enum rtx_code) ROUND_FLOOR, (int) V4SI_FTYPE_V2DF_V2DF_ROUND },
28749 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundpd_vec_pack_sfix, "__builtin_ia32_ceilpd_vec_pack_sfix", IX86_BUILTIN_CEILPD_VEC_PACK_SFIX, (enum rtx_code) ROUND_CEIL, (int) V4SI_FTYPE_V2DF_V2DF_ROUND },
28751 { OPTION_MASK_ISA_ROUND, CODE_FOR_roundv2df2, "__builtin_ia32_roundpd_az", IX86_BUILTIN_ROUNDPD_AZ, UNKNOWN, (int) V2DF_FTYPE_V2DF },
28752 { OPTION_MASK_ISA_ROUND, CODE_FOR_roundv2df2_vec_pack_sfix, "__builtin_ia32_roundpd_az_vec_pack_sfix", IX86_BUILTIN_ROUNDPD_AZ_VEC_PACK_SFIX, UNKNOWN, (int) V4SI_FTYPE_V2DF_V2DF },
28754 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundps, "__builtin_ia32_floorps", IX86_BUILTIN_FLOORPS, (enum rtx_code) ROUND_FLOOR, (int) V4SF_FTYPE_V4SF_ROUND },
28755 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundps, "__builtin_ia32_ceilps", IX86_BUILTIN_CEILPS, (enum rtx_code) ROUND_CEIL, (int) V4SF_FTYPE_V4SF_ROUND },
28756 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundps, "__builtin_ia32_truncps", IX86_BUILTIN_TRUNCPS, (enum rtx_code) ROUND_TRUNC, (int) V4SF_FTYPE_V4SF_ROUND },
28757 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundps, "__builtin_ia32_rintps", IX86_BUILTIN_RINTPS, (enum rtx_code) ROUND_MXCSR, (int) V4SF_FTYPE_V4SF_ROUND },
28759 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundps_sfix, "__builtin_ia32_floorps_sfix", IX86_BUILTIN_FLOORPS_SFIX, (enum rtx_code) ROUND_FLOOR, (int) V4SI_FTYPE_V4SF_ROUND },
28760 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundps_sfix, "__builtin_ia32_ceilps_sfix", IX86_BUILTIN_CEILPS_SFIX, (enum rtx_code) ROUND_CEIL, (int) V4SI_FTYPE_V4SF_ROUND },
28762 { OPTION_MASK_ISA_ROUND, CODE_FOR_roundv4sf2, "__builtin_ia32_roundps_az", IX86_BUILTIN_ROUNDPS_AZ, UNKNOWN, (int) V4SF_FTYPE_V4SF },
28763 { OPTION_MASK_ISA_ROUND, CODE_FOR_roundv4sf2_sfix, "__builtin_ia32_roundps_az_sfix", IX86_BUILTIN_ROUNDPS_AZ_SFIX, UNKNOWN, (int) V4SI_FTYPE_V4SF },
28765 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_ptest, "__builtin_ia32_ptestz128", IX86_BUILTIN_PTESTZ, EQ, (int) INT_FTYPE_V2DI_V2DI_PTEST },
28766 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_ptest, "__builtin_ia32_ptestc128", IX86_BUILTIN_PTESTC, LTU, (int) INT_FTYPE_V2DI_V2DI_PTEST },
28767 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_ptest, "__builtin_ia32_ptestnzc128", IX86_BUILTIN_PTESTNZC, GTU, (int) INT_FTYPE_V2DI_V2DI_PTEST },
28769 /* SSE4.2 */
28770 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_gtv2di3, "__builtin_ia32_pcmpgtq", IX86_BUILTIN_PCMPGTQ, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
28771 { OPTION_MASK_ISA_SSE4_2 | OPTION_MASK_ISA_CRC32, CODE_FOR_sse4_2_crc32qi, "__builtin_ia32_crc32qi", IX86_BUILTIN_CRC32QI, UNKNOWN, (int) UINT_FTYPE_UINT_UCHAR },
28772 { OPTION_MASK_ISA_SSE4_2 | OPTION_MASK_ISA_CRC32, CODE_FOR_sse4_2_crc32hi, "__builtin_ia32_crc32hi", IX86_BUILTIN_CRC32HI, UNKNOWN, (int) UINT_FTYPE_UINT_USHORT },
28773 { OPTION_MASK_ISA_SSE4_2 | OPTION_MASK_ISA_CRC32, CODE_FOR_sse4_2_crc32si, "__builtin_ia32_crc32si", IX86_BUILTIN_CRC32SI, UNKNOWN, (int) UINT_FTYPE_UINT_UINT },
28774 { OPTION_MASK_ISA_SSE4_2 | OPTION_MASK_ISA_CRC32 | OPTION_MASK_ISA_64BIT, CODE_FOR_sse4_2_crc32di, "__builtin_ia32_crc32di", IX86_BUILTIN_CRC32DI, UNKNOWN, (int) UINT64_FTYPE_UINT64_UINT64 },
28776 /* SSE4A */
28777 { OPTION_MASK_ISA_SSE4A, CODE_FOR_sse4a_extrqi, "__builtin_ia32_extrqi", IX86_BUILTIN_EXTRQI, UNKNOWN, (int) V2DI_FTYPE_V2DI_UINT_UINT },
28778 { OPTION_MASK_ISA_SSE4A, CODE_FOR_sse4a_extrq, "__builtin_ia32_extrq", IX86_BUILTIN_EXTRQ, UNKNOWN, (int) V2DI_FTYPE_V2DI_V16QI },
28779 { OPTION_MASK_ISA_SSE4A, CODE_FOR_sse4a_insertqi, "__builtin_ia32_insertqi", IX86_BUILTIN_INSERTQI, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_UINT_UINT },
28780 { OPTION_MASK_ISA_SSE4A, CODE_FOR_sse4a_insertq, "__builtin_ia32_insertq", IX86_BUILTIN_INSERTQ, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
28782 /* AES */
28783 { OPTION_MASK_ISA_SSE2, CODE_FOR_aeskeygenassist, 0, IX86_BUILTIN_AESKEYGENASSIST128, UNKNOWN, (int) V2DI_FTYPE_V2DI_INT },
28784 { OPTION_MASK_ISA_SSE2, CODE_FOR_aesimc, 0, IX86_BUILTIN_AESIMC128, UNKNOWN, (int) V2DI_FTYPE_V2DI },
28786 { OPTION_MASK_ISA_SSE2, CODE_FOR_aesenc, 0, IX86_BUILTIN_AESENC128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
28787 { OPTION_MASK_ISA_SSE2, CODE_FOR_aesenclast, 0, IX86_BUILTIN_AESENCLAST128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
28788 { OPTION_MASK_ISA_SSE2, CODE_FOR_aesdec, 0, IX86_BUILTIN_AESDEC128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
28789 { OPTION_MASK_ISA_SSE2, CODE_FOR_aesdeclast, 0, IX86_BUILTIN_AESDECLAST128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
28791 /* PCLMUL */
28792 { OPTION_MASK_ISA_SSE2, CODE_FOR_pclmulqdq, 0, IX86_BUILTIN_PCLMULQDQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_INT },
28794 /* AVX */
28795 { OPTION_MASK_ISA_AVX, CODE_FOR_addv4df3, "__builtin_ia32_addpd256", IX86_BUILTIN_ADDPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
28796 { OPTION_MASK_ISA_AVX, CODE_FOR_addv8sf3, "__builtin_ia32_addps256", IX86_BUILTIN_ADDPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
28797 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_addsubv4df3, "__builtin_ia32_addsubpd256", IX86_BUILTIN_ADDSUBPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
28798 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_addsubv8sf3, "__builtin_ia32_addsubps256", IX86_BUILTIN_ADDSUBPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
28799 { OPTION_MASK_ISA_AVX, CODE_FOR_andv4df3, "__builtin_ia32_andpd256", IX86_BUILTIN_ANDPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
28800 { OPTION_MASK_ISA_AVX, CODE_FOR_andv8sf3, "__builtin_ia32_andps256", IX86_BUILTIN_ANDPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
28801 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_andnotv4df3, "__builtin_ia32_andnpd256", IX86_BUILTIN_ANDNPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
28802 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_andnotv8sf3, "__builtin_ia32_andnps256", IX86_BUILTIN_ANDNPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
28803 { OPTION_MASK_ISA_AVX, CODE_FOR_divv4df3, "__builtin_ia32_divpd256", IX86_BUILTIN_DIVPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
28804 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_divv8sf3, "__builtin_ia32_divps256", IX86_BUILTIN_DIVPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
28805 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_haddv4df3, "__builtin_ia32_haddpd256", IX86_BUILTIN_HADDPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
28806 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_hsubv8sf3, "__builtin_ia32_hsubps256", IX86_BUILTIN_HSUBPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
28807 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_hsubv4df3, "__builtin_ia32_hsubpd256", IX86_BUILTIN_HSUBPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
28808 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_haddv8sf3, "__builtin_ia32_haddps256", IX86_BUILTIN_HADDPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
28809 { OPTION_MASK_ISA_AVX, CODE_FOR_smaxv4df3, "__builtin_ia32_maxpd256", IX86_BUILTIN_MAXPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
28810 { OPTION_MASK_ISA_AVX, CODE_FOR_smaxv8sf3, "__builtin_ia32_maxps256", IX86_BUILTIN_MAXPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
28811 { OPTION_MASK_ISA_AVX, CODE_FOR_sminv4df3, "__builtin_ia32_minpd256", IX86_BUILTIN_MINPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
28812 { OPTION_MASK_ISA_AVX, CODE_FOR_sminv8sf3, "__builtin_ia32_minps256", IX86_BUILTIN_MINPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
28813 { OPTION_MASK_ISA_AVX, CODE_FOR_mulv4df3, "__builtin_ia32_mulpd256", IX86_BUILTIN_MULPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
28814 { OPTION_MASK_ISA_AVX, CODE_FOR_mulv8sf3, "__builtin_ia32_mulps256", IX86_BUILTIN_MULPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
28815 { OPTION_MASK_ISA_AVX, CODE_FOR_iorv4df3, "__builtin_ia32_orpd256", IX86_BUILTIN_ORPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
28816 { OPTION_MASK_ISA_AVX, CODE_FOR_iorv8sf3, "__builtin_ia32_orps256", IX86_BUILTIN_ORPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
28817 { OPTION_MASK_ISA_AVX, CODE_FOR_subv4df3, "__builtin_ia32_subpd256", IX86_BUILTIN_SUBPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
28818 { OPTION_MASK_ISA_AVX, CODE_FOR_subv8sf3, "__builtin_ia32_subps256", IX86_BUILTIN_SUBPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
28819 { OPTION_MASK_ISA_AVX, CODE_FOR_xorv4df3, "__builtin_ia32_xorpd256", IX86_BUILTIN_XORPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
28820 { OPTION_MASK_ISA_AVX, CODE_FOR_xorv8sf3, "__builtin_ia32_xorps256", IX86_BUILTIN_XORPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
28822 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilvarv2df3, "__builtin_ia32_vpermilvarpd", IX86_BUILTIN_VPERMILVARPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DI },
28823 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilvarv4sf3, "__builtin_ia32_vpermilvarps", IX86_BUILTIN_VPERMILVARPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SI },
28824 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilvarv4df3, "__builtin_ia32_vpermilvarpd256", IX86_BUILTIN_VPERMILVARPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DI },
28825 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilvarv8sf3, "__builtin_ia32_vpermilvarps256", IX86_BUILTIN_VPERMILVARPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SI },
28827 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_blendpd256, "__builtin_ia32_blendpd256", IX86_BUILTIN_BLENDPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_INT },
28828 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_blendps256, "__builtin_ia32_blendps256", IX86_BUILTIN_BLENDPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_INT },
28829 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_blendvpd256, "__builtin_ia32_blendvpd256", IX86_BUILTIN_BLENDVPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_V4DF },
28830 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_blendvps256, "__builtin_ia32_blendvps256", IX86_BUILTIN_BLENDVPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_V8SF },
28831 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_dpps256, "__builtin_ia32_dpps256", IX86_BUILTIN_DPPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_INT },
28832 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_shufpd256, "__builtin_ia32_shufpd256", IX86_BUILTIN_SHUFPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_INT },
28833 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_shufps256, "__builtin_ia32_shufps256", IX86_BUILTIN_SHUFPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_INT },
28834 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vmcmpv2df3, "__builtin_ia32_cmpsd", IX86_BUILTIN_CMPSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
28835 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vmcmpv4sf3, "__builtin_ia32_cmpss", IX86_BUILTIN_CMPSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
28836 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cmpv2df3, "__builtin_ia32_cmppd", IX86_BUILTIN_CMPPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
28837 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cmpv4sf3, "__builtin_ia32_cmpps", IX86_BUILTIN_CMPPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
28838 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cmpv4df3, "__builtin_ia32_cmppd256", IX86_BUILTIN_CMPPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_INT },
28839 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cmpv8sf3, "__builtin_ia32_cmpps256", IX86_BUILTIN_CMPPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_INT },
28840 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vextractf128v4df, "__builtin_ia32_vextractf128_pd256", IX86_BUILTIN_EXTRACTF128PD256, UNKNOWN, (int) V2DF_FTYPE_V4DF_INT },
28841 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vextractf128v8sf, "__builtin_ia32_vextractf128_ps256", IX86_BUILTIN_EXTRACTF128PS256, UNKNOWN, (int) V4SF_FTYPE_V8SF_INT },
28842 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vextractf128v8si, "__builtin_ia32_vextractf128_si256", IX86_BUILTIN_EXTRACTF128SI256, UNKNOWN, (int) V4SI_FTYPE_V8SI_INT },
28843 { OPTION_MASK_ISA_AVX, CODE_FOR_floatv4siv4df2, "__builtin_ia32_cvtdq2pd256", IX86_BUILTIN_CVTDQ2PD256, UNKNOWN, (int) V4DF_FTYPE_V4SI },
28844 { OPTION_MASK_ISA_AVX, CODE_FOR_floatv8siv8sf2, "__builtin_ia32_cvtdq2ps256", IX86_BUILTIN_CVTDQ2PS256, UNKNOWN, (int) V8SF_FTYPE_V8SI },
28845 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cvtpd2ps256, "__builtin_ia32_cvtpd2ps256", IX86_BUILTIN_CVTPD2PS256, UNKNOWN, (int) V4SF_FTYPE_V4DF },
28846 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_fix_notruncv8sfv8si, "__builtin_ia32_cvtps2dq256", IX86_BUILTIN_CVTPS2DQ256, UNKNOWN, (int) V8SI_FTYPE_V8SF },
28847 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cvtps2pd256, "__builtin_ia32_cvtps2pd256", IX86_BUILTIN_CVTPS2PD256, UNKNOWN, (int) V4DF_FTYPE_V4SF },
28848 { OPTION_MASK_ISA_AVX, CODE_FOR_fix_truncv4dfv4si2, "__builtin_ia32_cvttpd2dq256", IX86_BUILTIN_CVTTPD2DQ256, UNKNOWN, (int) V4SI_FTYPE_V4DF },
28849 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cvtpd2dq256, "__builtin_ia32_cvtpd2dq256", IX86_BUILTIN_CVTPD2DQ256, UNKNOWN, (int) V4SI_FTYPE_V4DF },
28850 { OPTION_MASK_ISA_AVX, CODE_FOR_fix_truncv8sfv8si2, "__builtin_ia32_cvttps2dq256", IX86_BUILTIN_CVTTPS2DQ256, UNKNOWN, (int) V8SI_FTYPE_V8SF },
28851 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vperm2f128v4df3, "__builtin_ia32_vperm2f128_pd256", IX86_BUILTIN_VPERM2F128PD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_INT },
28852 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vperm2f128v8sf3, "__builtin_ia32_vperm2f128_ps256", IX86_BUILTIN_VPERM2F128PS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_INT },
28853 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vperm2f128v8si3, "__builtin_ia32_vperm2f128_si256", IX86_BUILTIN_VPERM2F128SI256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_INT },
28854 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilv2df, "__builtin_ia32_vpermilpd", IX86_BUILTIN_VPERMILPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_INT },
28855 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilv4sf, "__builtin_ia32_vpermilps", IX86_BUILTIN_VPERMILPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_INT },
28856 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilv4df, "__builtin_ia32_vpermilpd256", IX86_BUILTIN_VPERMILPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_INT },
28857 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilv8sf, "__builtin_ia32_vpermilps256", IX86_BUILTIN_VPERMILPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_INT },
28858 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vinsertf128v4df, "__builtin_ia32_vinsertf128_pd256", IX86_BUILTIN_VINSERTF128PD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V2DF_INT },
28859 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vinsertf128v8sf, "__builtin_ia32_vinsertf128_ps256", IX86_BUILTIN_VINSERTF128PS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V4SF_INT },
28860 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vinsertf128v8si, "__builtin_ia32_vinsertf128_si256", IX86_BUILTIN_VINSERTF128SI256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V4SI_INT },
28862 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movshdup256, "__builtin_ia32_movshdup256", IX86_BUILTIN_MOVSHDUP256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
28863 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movsldup256, "__builtin_ia32_movsldup256", IX86_BUILTIN_MOVSLDUP256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
28864 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movddup256, "__builtin_ia32_movddup256", IX86_BUILTIN_MOVDDUP256, UNKNOWN, (int) V4DF_FTYPE_V4DF },
28866 { OPTION_MASK_ISA_AVX, CODE_FOR_sqrtv4df2, "__builtin_ia32_sqrtpd256", IX86_BUILTIN_SQRTPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF },
28867 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_sqrtv8sf2, "__builtin_ia32_sqrtps256", IX86_BUILTIN_SQRTPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
28868 { OPTION_MASK_ISA_AVX, CODE_FOR_sqrtv8sf2, "__builtin_ia32_sqrtps_nr256", IX86_BUILTIN_SQRTPS_NR256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
28869 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_rsqrtv8sf2, "__builtin_ia32_rsqrtps256", IX86_BUILTIN_RSQRTPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
28870 { OPTION_MASK_ISA_AVX, CODE_FOR_rsqrtv8sf2, "__builtin_ia32_rsqrtps_nr256", IX86_BUILTIN_RSQRTPS_NR256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
28872 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_rcpv8sf2, "__builtin_ia32_rcpps256", IX86_BUILTIN_RCPPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
28874 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundpd256, "__builtin_ia32_roundpd256", IX86_BUILTIN_ROUNDPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_INT },
28875 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundps256, "__builtin_ia32_roundps256", IX86_BUILTIN_ROUNDPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_INT },
28877 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundpd256, "__builtin_ia32_floorpd256", IX86_BUILTIN_FLOORPD256, (enum rtx_code) ROUND_FLOOR, (int) V4DF_FTYPE_V4DF_ROUND },
28878 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundpd256, "__builtin_ia32_ceilpd256", IX86_BUILTIN_CEILPD256, (enum rtx_code) ROUND_CEIL, (int) V4DF_FTYPE_V4DF_ROUND },
28879 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundpd256, "__builtin_ia32_truncpd256", IX86_BUILTIN_TRUNCPD256, (enum rtx_code) ROUND_TRUNC, (int) V4DF_FTYPE_V4DF_ROUND },
28880 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundpd256, "__builtin_ia32_rintpd256", IX86_BUILTIN_RINTPD256, (enum rtx_code) ROUND_MXCSR, (int) V4DF_FTYPE_V4DF_ROUND },
28882 { OPTION_MASK_ISA_AVX, CODE_FOR_roundv4df2, "__builtin_ia32_roundpd_az256", IX86_BUILTIN_ROUNDPD_AZ256, UNKNOWN, (int) V4DF_FTYPE_V4DF },
28883 { OPTION_MASK_ISA_AVX, CODE_FOR_roundv4df2_vec_pack_sfix, "__builtin_ia32_roundpd_az_vec_pack_sfix256", IX86_BUILTIN_ROUNDPD_AZ_VEC_PACK_SFIX256, UNKNOWN, (int) V8SI_FTYPE_V4DF_V4DF },
28885 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundpd_vec_pack_sfix256, "__builtin_ia32_floorpd_vec_pack_sfix256", IX86_BUILTIN_FLOORPD_VEC_PACK_SFIX256, (enum rtx_code) ROUND_FLOOR, (int) V8SI_FTYPE_V4DF_V4DF_ROUND },
28886 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundpd_vec_pack_sfix256, "__builtin_ia32_ceilpd_vec_pack_sfix256", IX86_BUILTIN_CEILPD_VEC_PACK_SFIX256, (enum rtx_code) ROUND_CEIL, (int) V8SI_FTYPE_V4DF_V4DF_ROUND },
28888 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundps256, "__builtin_ia32_floorps256", IX86_BUILTIN_FLOORPS256, (enum rtx_code) ROUND_FLOOR, (int) V8SF_FTYPE_V8SF_ROUND },
28889 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundps256, "__builtin_ia32_ceilps256", IX86_BUILTIN_CEILPS256, (enum rtx_code) ROUND_CEIL, (int) V8SF_FTYPE_V8SF_ROUND },
28890 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundps256, "__builtin_ia32_truncps256", IX86_BUILTIN_TRUNCPS256, (enum rtx_code) ROUND_TRUNC, (int) V8SF_FTYPE_V8SF_ROUND },
28891 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundps256, "__builtin_ia32_rintps256", IX86_BUILTIN_RINTPS256, (enum rtx_code) ROUND_MXCSR, (int) V8SF_FTYPE_V8SF_ROUND },
28893 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundps_sfix256, "__builtin_ia32_floorps_sfix256", IX86_BUILTIN_FLOORPS_SFIX256, (enum rtx_code) ROUND_FLOOR, (int) V8SI_FTYPE_V8SF_ROUND },
28894 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundps_sfix256, "__builtin_ia32_ceilps_sfix256", IX86_BUILTIN_CEILPS_SFIX256, (enum rtx_code) ROUND_CEIL, (int) V8SI_FTYPE_V8SF_ROUND },
28896 { OPTION_MASK_ISA_AVX, CODE_FOR_roundv8sf2, "__builtin_ia32_roundps_az256", IX86_BUILTIN_ROUNDPS_AZ256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
28897 { OPTION_MASK_ISA_AVX, CODE_FOR_roundv8sf2_sfix, "__builtin_ia32_roundps_az_sfix256", IX86_BUILTIN_ROUNDPS_AZ_SFIX256, UNKNOWN, (int) V8SI_FTYPE_V8SF },
28899 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_unpckhpd256, "__builtin_ia32_unpckhpd256", IX86_BUILTIN_UNPCKHPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
28900 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_unpcklpd256, "__builtin_ia32_unpcklpd256", IX86_BUILTIN_UNPCKLPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
28901 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_unpckhps256, "__builtin_ia32_unpckhps256", IX86_BUILTIN_UNPCKHPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
28902 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_unpcklps256, "__builtin_ia32_unpcklps256", IX86_BUILTIN_UNPCKLPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
28904 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_si256_si, "__builtin_ia32_si256_si", IX86_BUILTIN_SI256_SI, UNKNOWN, (int) V8SI_FTYPE_V4SI },
28905 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_ps256_ps, "__builtin_ia32_ps256_ps", IX86_BUILTIN_PS256_PS, UNKNOWN, (int) V8SF_FTYPE_V4SF },
28906 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_pd256_pd, "__builtin_ia32_pd256_pd", IX86_BUILTIN_PD256_PD, UNKNOWN, (int) V4DF_FTYPE_V2DF },
28907 { OPTION_MASK_ISA_AVX, CODE_FOR_vec_extract_lo_v8si, "__builtin_ia32_si_si256", IX86_BUILTIN_SI_SI256, UNKNOWN, (int) V4SI_FTYPE_V8SI },
28908 { OPTION_MASK_ISA_AVX, CODE_FOR_vec_extract_lo_v8sf, "__builtin_ia32_ps_ps256", IX86_BUILTIN_PS_PS256, UNKNOWN, (int) V4SF_FTYPE_V8SF },
28909 { OPTION_MASK_ISA_AVX, CODE_FOR_vec_extract_lo_v4df, "__builtin_ia32_pd_pd256", IX86_BUILTIN_PD_PD256, UNKNOWN, (int) V2DF_FTYPE_V4DF },
28911 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestpd, "__builtin_ia32_vtestzpd", IX86_BUILTIN_VTESTZPD, EQ, (int) INT_FTYPE_V2DF_V2DF_PTEST },
28912 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestpd, "__builtin_ia32_vtestcpd", IX86_BUILTIN_VTESTCPD, LTU, (int) INT_FTYPE_V2DF_V2DF_PTEST },
28913 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestpd, "__builtin_ia32_vtestnzcpd", IX86_BUILTIN_VTESTNZCPD, GTU, (int) INT_FTYPE_V2DF_V2DF_PTEST },
28914 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestps, "__builtin_ia32_vtestzps", IX86_BUILTIN_VTESTZPS, EQ, (int) INT_FTYPE_V4SF_V4SF_PTEST },
28915 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestps, "__builtin_ia32_vtestcps", IX86_BUILTIN_VTESTCPS, LTU, (int) INT_FTYPE_V4SF_V4SF_PTEST },
28916 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestps, "__builtin_ia32_vtestnzcps", IX86_BUILTIN_VTESTNZCPS, GTU, (int) INT_FTYPE_V4SF_V4SF_PTEST },
28917 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestpd256, "__builtin_ia32_vtestzpd256", IX86_BUILTIN_VTESTZPD256, EQ, (int) INT_FTYPE_V4DF_V4DF_PTEST },
28918 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestpd256, "__builtin_ia32_vtestcpd256", IX86_BUILTIN_VTESTCPD256, LTU, (int) INT_FTYPE_V4DF_V4DF_PTEST },
28919 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestpd256, "__builtin_ia32_vtestnzcpd256", IX86_BUILTIN_VTESTNZCPD256, GTU, (int) INT_FTYPE_V4DF_V4DF_PTEST },
28920 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestps256, "__builtin_ia32_vtestzps256", IX86_BUILTIN_VTESTZPS256, EQ, (int) INT_FTYPE_V8SF_V8SF_PTEST },
28921 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestps256, "__builtin_ia32_vtestcps256", IX86_BUILTIN_VTESTCPS256, LTU, (int) INT_FTYPE_V8SF_V8SF_PTEST },
28922 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestps256, "__builtin_ia32_vtestnzcps256", IX86_BUILTIN_VTESTNZCPS256, GTU, (int) INT_FTYPE_V8SF_V8SF_PTEST },
28923 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_ptest256, "__builtin_ia32_ptestz256", IX86_BUILTIN_PTESTZ256, EQ, (int) INT_FTYPE_V4DI_V4DI_PTEST },
28924 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_ptest256, "__builtin_ia32_ptestc256", IX86_BUILTIN_PTESTC256, LTU, (int) INT_FTYPE_V4DI_V4DI_PTEST },
28925 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_ptest256, "__builtin_ia32_ptestnzc256", IX86_BUILTIN_PTESTNZC256, GTU, (int) INT_FTYPE_V4DI_V4DI_PTEST },
28927 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movmskpd256, "__builtin_ia32_movmskpd256", IX86_BUILTIN_MOVMSKPD256, UNKNOWN, (int) INT_FTYPE_V4DF },
28928 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movmskps256, "__builtin_ia32_movmskps256", IX86_BUILTIN_MOVMSKPS256, UNKNOWN, (int) INT_FTYPE_V8SF },
28930 { OPTION_MASK_ISA_AVX, CODE_FOR_copysignv8sf3, "__builtin_ia32_copysignps256", IX86_BUILTIN_CPYSGNPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
28931 { OPTION_MASK_ISA_AVX, CODE_FOR_copysignv4df3, "__builtin_ia32_copysignpd256", IX86_BUILTIN_CPYSGNPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
28933 { OPTION_MASK_ISA_AVX, CODE_FOR_vec_pack_sfix_v4df, "__builtin_ia32_vec_pack_sfix256 ", IX86_BUILTIN_VEC_PACK_SFIX256, UNKNOWN, (int) V8SI_FTYPE_V4DF_V4DF },
28935 /* AVX2 */
28936 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_mpsadbw, "__builtin_ia32_mpsadbw256", IX86_BUILTIN_MPSADBW256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI_INT },
28937 { OPTION_MASK_ISA_AVX2, CODE_FOR_absv32qi2, "__builtin_ia32_pabsb256", IX86_BUILTIN_PABSB256, UNKNOWN, (int) V32QI_FTYPE_V32QI },
28938 { OPTION_MASK_ISA_AVX2, CODE_FOR_absv16hi2, "__builtin_ia32_pabsw256", IX86_BUILTIN_PABSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI },
28939 { OPTION_MASK_ISA_AVX2, CODE_FOR_absv8si2, "__builtin_ia32_pabsd256", IX86_BUILTIN_PABSD256, UNKNOWN, (int) V8SI_FTYPE_V8SI },
28940 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_packssdw, "__builtin_ia32_packssdw256", IX86_BUILTIN_PACKSSDW256, UNKNOWN, (int) V16HI_FTYPE_V8SI_V8SI },
28941 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_packsswb, "__builtin_ia32_packsswb256", IX86_BUILTIN_PACKSSWB256, UNKNOWN, (int) V32QI_FTYPE_V16HI_V16HI },
28942 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_packusdw, "__builtin_ia32_packusdw256", IX86_BUILTIN_PACKUSDW256, UNKNOWN, (int) V16HI_FTYPE_V8SI_V8SI },
28943 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_packuswb, "__builtin_ia32_packuswb256", IX86_BUILTIN_PACKUSWB256, UNKNOWN, (int) V32QI_FTYPE_V16HI_V16HI },
28944 { OPTION_MASK_ISA_AVX2, CODE_FOR_addv32qi3, "__builtin_ia32_paddb256", IX86_BUILTIN_PADDB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
28945 { OPTION_MASK_ISA_AVX2, CODE_FOR_addv16hi3, "__builtin_ia32_paddw256", IX86_BUILTIN_PADDW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28946 { OPTION_MASK_ISA_AVX2, CODE_FOR_addv8si3, "__builtin_ia32_paddd256", IX86_BUILTIN_PADDD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
28947 { OPTION_MASK_ISA_AVX2, CODE_FOR_addv4di3, "__builtin_ia32_paddq256", IX86_BUILTIN_PADDQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
28948 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ssaddv32qi3, "__builtin_ia32_paddsb256", IX86_BUILTIN_PADDSB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
28949 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ssaddv16hi3, "__builtin_ia32_paddsw256", IX86_BUILTIN_PADDSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28950 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_usaddv32qi3, "__builtin_ia32_paddusb256", IX86_BUILTIN_PADDUSB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
28951 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_usaddv16hi3, "__builtin_ia32_paddusw256", IX86_BUILTIN_PADDUSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28952 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_palignrv2ti, "__builtin_ia32_palignr256", IX86_BUILTIN_PALIGNR256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_INT_CONVERT },
28953 { OPTION_MASK_ISA_AVX2, CODE_FOR_andv4di3, "__builtin_ia32_andsi256", IX86_BUILTIN_AND256I, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
28954 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_andnotv4di3, "__builtin_ia32_andnotsi256", IX86_BUILTIN_ANDNOT256I, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
28955 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_uavgv32qi3, "__builtin_ia32_pavgb256", IX86_BUILTIN_PAVGB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
28956 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_uavgv16hi3, "__builtin_ia32_pavgw256", IX86_BUILTIN_PAVGW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28957 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pblendvb, "__builtin_ia32_pblendvb256", IX86_BUILTIN_PBLENDVB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI_V32QI },
28958 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pblendw, "__builtin_ia32_pblendw256", IX86_BUILTIN_PBLENDVW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI_INT },
28959 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_eqv32qi3, "__builtin_ia32_pcmpeqb256", IX86_BUILTIN_PCMPEQB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
28960 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_eqv16hi3, "__builtin_ia32_pcmpeqw256", IX86_BUILTIN_PCMPEQW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28961 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_eqv8si3, "__builtin_ia32_pcmpeqd256", IX86_BUILTIN_PCMPEQD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
28962 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_eqv4di3, "__builtin_ia32_pcmpeqq256", IX86_BUILTIN_PCMPEQQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
28963 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_gtv32qi3, "__builtin_ia32_pcmpgtb256", IX86_BUILTIN_PCMPGTB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
28964 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_gtv16hi3, "__builtin_ia32_pcmpgtw256", IX86_BUILTIN_PCMPGTW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28965 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_gtv8si3, "__builtin_ia32_pcmpgtd256", IX86_BUILTIN_PCMPGTD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
28966 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_gtv4di3, "__builtin_ia32_pcmpgtq256", IX86_BUILTIN_PCMPGTQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
28967 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_phaddwv16hi3, "__builtin_ia32_phaddw256", IX86_BUILTIN_PHADDW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28968 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_phadddv8si3, "__builtin_ia32_phaddd256", IX86_BUILTIN_PHADDD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
28969 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_phaddswv16hi3, "__builtin_ia32_phaddsw256", IX86_BUILTIN_PHADDSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28970 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_phsubwv16hi3, "__builtin_ia32_phsubw256", IX86_BUILTIN_PHSUBW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28971 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_phsubdv8si3, "__builtin_ia32_phsubd256", IX86_BUILTIN_PHSUBD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
28972 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_phsubswv16hi3, "__builtin_ia32_phsubsw256", IX86_BUILTIN_PHSUBSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28973 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pmaddubsw256, "__builtin_ia32_pmaddubsw256", IX86_BUILTIN_PMADDUBSW256, UNKNOWN, (int) V16HI_FTYPE_V32QI_V32QI },
28974 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pmaddwd, "__builtin_ia32_pmaddwd256", IX86_BUILTIN_PMADDWD256, UNKNOWN, (int) V8SI_FTYPE_V16HI_V16HI },
28975 { OPTION_MASK_ISA_AVX2, CODE_FOR_smaxv32qi3, "__builtin_ia32_pmaxsb256", IX86_BUILTIN_PMAXSB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
28976 { OPTION_MASK_ISA_AVX2, CODE_FOR_smaxv16hi3, "__builtin_ia32_pmaxsw256", IX86_BUILTIN_PMAXSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28977 { OPTION_MASK_ISA_AVX2, CODE_FOR_smaxv8si3 , "__builtin_ia32_pmaxsd256", IX86_BUILTIN_PMAXSD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
28978 { OPTION_MASK_ISA_AVX2, CODE_FOR_umaxv32qi3, "__builtin_ia32_pmaxub256", IX86_BUILTIN_PMAXUB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
28979 { OPTION_MASK_ISA_AVX2, CODE_FOR_umaxv16hi3, "__builtin_ia32_pmaxuw256", IX86_BUILTIN_PMAXUW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28980 { OPTION_MASK_ISA_AVX2, CODE_FOR_umaxv8si3 , "__builtin_ia32_pmaxud256", IX86_BUILTIN_PMAXUD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
28981 { OPTION_MASK_ISA_AVX2, CODE_FOR_sminv32qi3, "__builtin_ia32_pminsb256", IX86_BUILTIN_PMINSB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
28982 { OPTION_MASK_ISA_AVX2, CODE_FOR_sminv16hi3, "__builtin_ia32_pminsw256", IX86_BUILTIN_PMINSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28983 { OPTION_MASK_ISA_AVX2, CODE_FOR_sminv8si3 , "__builtin_ia32_pminsd256", IX86_BUILTIN_PMINSD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
28984 { OPTION_MASK_ISA_AVX2, CODE_FOR_uminv32qi3, "__builtin_ia32_pminub256", IX86_BUILTIN_PMINUB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
28985 { OPTION_MASK_ISA_AVX2, CODE_FOR_uminv16hi3, "__builtin_ia32_pminuw256", IX86_BUILTIN_PMINUW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28986 { OPTION_MASK_ISA_AVX2, CODE_FOR_uminv8si3 , "__builtin_ia32_pminud256", IX86_BUILTIN_PMINUD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
28987 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pmovmskb, "__builtin_ia32_pmovmskb256", IX86_BUILTIN_PMOVMSKB256, UNKNOWN, (int) INT_FTYPE_V32QI },
28988 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_sign_extendv16qiv16hi2, "__builtin_ia32_pmovsxbw256", IX86_BUILTIN_PMOVSXBW256, UNKNOWN, (int) V16HI_FTYPE_V16QI },
28989 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_sign_extendv8qiv8si2 , "__builtin_ia32_pmovsxbd256", IX86_BUILTIN_PMOVSXBD256, UNKNOWN, (int) V8SI_FTYPE_V16QI },
28990 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_sign_extendv4qiv4di2 , "__builtin_ia32_pmovsxbq256", IX86_BUILTIN_PMOVSXBQ256, UNKNOWN, (int) V4DI_FTYPE_V16QI },
28991 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_sign_extendv8hiv8si2 , "__builtin_ia32_pmovsxwd256", IX86_BUILTIN_PMOVSXWD256, UNKNOWN, (int) V8SI_FTYPE_V8HI },
28992 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_sign_extendv4hiv4di2 , "__builtin_ia32_pmovsxwq256", IX86_BUILTIN_PMOVSXWQ256, UNKNOWN, (int) V4DI_FTYPE_V8HI },
28993 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_sign_extendv4siv4di2 , "__builtin_ia32_pmovsxdq256", IX86_BUILTIN_PMOVSXDQ256, UNKNOWN, (int) V4DI_FTYPE_V4SI },
28994 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_zero_extendv16qiv16hi2, "__builtin_ia32_pmovzxbw256", IX86_BUILTIN_PMOVZXBW256, UNKNOWN, (int) V16HI_FTYPE_V16QI },
28995 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_zero_extendv8qiv8si2 , "__builtin_ia32_pmovzxbd256", IX86_BUILTIN_PMOVZXBD256, UNKNOWN, (int) V8SI_FTYPE_V16QI },
28996 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_zero_extendv4qiv4di2 , "__builtin_ia32_pmovzxbq256", IX86_BUILTIN_PMOVZXBQ256, UNKNOWN, (int) V4DI_FTYPE_V16QI },
28997 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_zero_extendv8hiv8si2 , "__builtin_ia32_pmovzxwd256", IX86_BUILTIN_PMOVZXWD256, UNKNOWN, (int) V8SI_FTYPE_V8HI },
28998 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_zero_extendv4hiv4di2 , "__builtin_ia32_pmovzxwq256", IX86_BUILTIN_PMOVZXWQ256, UNKNOWN, (int) V4DI_FTYPE_V8HI },
28999 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_zero_extendv4siv4di2 , "__builtin_ia32_pmovzxdq256", IX86_BUILTIN_PMOVZXDQ256, UNKNOWN, (int) V4DI_FTYPE_V4SI },
29000 { OPTION_MASK_ISA_AVX2, CODE_FOR_vec_widen_smult_even_v8si, "__builtin_ia32_pmuldq256", IX86_BUILTIN_PMULDQ256, UNKNOWN, (int) V4DI_FTYPE_V8SI_V8SI },
29001 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pmulhrswv16hi3 , "__builtin_ia32_pmulhrsw256", IX86_BUILTIN_PMULHRSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
29002 { OPTION_MASK_ISA_AVX2, CODE_FOR_umulv16hi3_highpart, "__builtin_ia32_pmulhuw256" , IX86_BUILTIN_PMULHUW256 , UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
29003 { OPTION_MASK_ISA_AVX2, CODE_FOR_smulv16hi3_highpart, "__builtin_ia32_pmulhw256" , IX86_BUILTIN_PMULHW256 , UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
29004 { OPTION_MASK_ISA_AVX2, CODE_FOR_mulv16hi3, "__builtin_ia32_pmullw256" , IX86_BUILTIN_PMULLW256 , UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
29005 { OPTION_MASK_ISA_AVX2, CODE_FOR_mulv8si3, "__builtin_ia32_pmulld256" , IX86_BUILTIN_PMULLD256 , UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
29006 { OPTION_MASK_ISA_AVX2, CODE_FOR_vec_widen_umult_even_v8si, "__builtin_ia32_pmuludq256", IX86_BUILTIN_PMULUDQ256, UNKNOWN, (int) V4DI_FTYPE_V8SI_V8SI },
29007 { OPTION_MASK_ISA_AVX2, CODE_FOR_iorv4di3, "__builtin_ia32_por256", IX86_BUILTIN_POR256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
29008 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_psadbw, "__builtin_ia32_psadbw256", IX86_BUILTIN_PSADBW256, UNKNOWN, (int) V16HI_FTYPE_V32QI_V32QI },
29009 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pshufbv32qi3, "__builtin_ia32_pshufb256", IX86_BUILTIN_PSHUFB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
29010 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pshufdv3, "__builtin_ia32_pshufd256", IX86_BUILTIN_PSHUFD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_INT },
29011 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pshufhwv3, "__builtin_ia32_pshufhw256", IX86_BUILTIN_PSHUFHW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_INT },
29012 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pshuflwv3, "__builtin_ia32_pshuflw256", IX86_BUILTIN_PSHUFLW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_INT },
29013 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_psignv32qi3, "__builtin_ia32_psignb256", IX86_BUILTIN_PSIGNB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
29014 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_psignv16hi3, "__builtin_ia32_psignw256", IX86_BUILTIN_PSIGNW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
29015 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_psignv8si3 , "__builtin_ia32_psignd256", IX86_BUILTIN_PSIGND256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
29016 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ashlv2ti3, "__builtin_ia32_pslldqi256", IX86_BUILTIN_PSLLDQI256, UNKNOWN, (int) V4DI_FTYPE_V4DI_INT_CONVERT },
29017 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashlv16hi3, "__builtin_ia32_psllwi256", IX86_BUILTIN_PSLLWI256 , UNKNOWN, (int) V16HI_FTYPE_V16HI_SI_COUNT },
29018 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashlv16hi3, "__builtin_ia32_psllw256", IX86_BUILTIN_PSLLW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V8HI_COUNT },
29019 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashlv8si3, "__builtin_ia32_pslldi256", IX86_BUILTIN_PSLLDI256, UNKNOWN, (int) V8SI_FTYPE_V8SI_SI_COUNT },
29020 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashlv8si3, "__builtin_ia32_pslld256", IX86_BUILTIN_PSLLD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V4SI_COUNT },
29021 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashlv4di3, "__builtin_ia32_psllqi256", IX86_BUILTIN_PSLLQI256, UNKNOWN, (int) V4DI_FTYPE_V4DI_INT_COUNT },
29022 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashlv4di3, "__builtin_ia32_psllq256", IX86_BUILTIN_PSLLQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V2DI_COUNT },
29023 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashrv16hi3, "__builtin_ia32_psrawi256", IX86_BUILTIN_PSRAWI256, UNKNOWN, (int) V16HI_FTYPE_V16HI_SI_COUNT },
29024 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashrv16hi3, "__builtin_ia32_psraw256", IX86_BUILTIN_PSRAW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V8HI_COUNT },
29025 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashrv8si3, "__builtin_ia32_psradi256", IX86_BUILTIN_PSRADI256, UNKNOWN, (int) V8SI_FTYPE_V8SI_SI_COUNT },
29026 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashrv8si3, "__builtin_ia32_psrad256", IX86_BUILTIN_PSRAD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V4SI_COUNT },
29027 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_lshrv2ti3, "__builtin_ia32_psrldqi256", IX86_BUILTIN_PSRLDQI256, UNKNOWN, (int) V4DI_FTYPE_V4DI_INT_CONVERT },
29028 { OPTION_MASK_ISA_AVX2, CODE_FOR_lshrv16hi3, "__builtin_ia32_psrlwi256", IX86_BUILTIN_PSRLWI256 , UNKNOWN, (int) V16HI_FTYPE_V16HI_SI_COUNT },
29029 { OPTION_MASK_ISA_AVX2, CODE_FOR_lshrv16hi3, "__builtin_ia32_psrlw256", IX86_BUILTIN_PSRLW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V8HI_COUNT },
29030 { OPTION_MASK_ISA_AVX2, CODE_FOR_lshrv8si3, "__builtin_ia32_psrldi256", IX86_BUILTIN_PSRLDI256, UNKNOWN, (int) V8SI_FTYPE_V8SI_SI_COUNT },
29031 { OPTION_MASK_ISA_AVX2, CODE_FOR_lshrv8si3, "__builtin_ia32_psrld256", IX86_BUILTIN_PSRLD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V4SI_COUNT },
29032 { OPTION_MASK_ISA_AVX2, CODE_FOR_lshrv4di3, "__builtin_ia32_psrlqi256", IX86_BUILTIN_PSRLQI256, UNKNOWN, (int) V4DI_FTYPE_V4DI_INT_COUNT },
29033 { OPTION_MASK_ISA_AVX2, CODE_FOR_lshrv4di3, "__builtin_ia32_psrlq256", IX86_BUILTIN_PSRLQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V2DI_COUNT },
29034 { OPTION_MASK_ISA_AVX2, CODE_FOR_subv32qi3, "__builtin_ia32_psubb256", IX86_BUILTIN_PSUBB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
29035 { OPTION_MASK_ISA_AVX2, CODE_FOR_subv16hi3, "__builtin_ia32_psubw256", IX86_BUILTIN_PSUBW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
29036 { OPTION_MASK_ISA_AVX2, CODE_FOR_subv8si3, "__builtin_ia32_psubd256", IX86_BUILTIN_PSUBD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
29037 { OPTION_MASK_ISA_AVX2, CODE_FOR_subv4di3, "__builtin_ia32_psubq256", IX86_BUILTIN_PSUBQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
29038 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_sssubv32qi3, "__builtin_ia32_psubsb256", IX86_BUILTIN_PSUBSB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
29039 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_sssubv16hi3, "__builtin_ia32_psubsw256", IX86_BUILTIN_PSUBSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
29040 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ussubv32qi3, "__builtin_ia32_psubusb256", IX86_BUILTIN_PSUBUSB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
29041 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ussubv16hi3, "__builtin_ia32_psubusw256", IX86_BUILTIN_PSUBUSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
29042 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_interleave_highv32qi, "__builtin_ia32_punpckhbw256", IX86_BUILTIN_PUNPCKHBW256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
29043 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_interleave_highv16hi, "__builtin_ia32_punpckhwd256", IX86_BUILTIN_PUNPCKHWD256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
29044 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_interleave_highv8si, "__builtin_ia32_punpckhdq256", IX86_BUILTIN_PUNPCKHDQ256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
29045 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_interleave_highv4di, "__builtin_ia32_punpckhqdq256", IX86_BUILTIN_PUNPCKHQDQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
29046 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_interleave_lowv32qi, "__builtin_ia32_punpcklbw256", IX86_BUILTIN_PUNPCKLBW256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
29047 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_interleave_lowv16hi, "__builtin_ia32_punpcklwd256", IX86_BUILTIN_PUNPCKLWD256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
29048 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_interleave_lowv8si, "__builtin_ia32_punpckldq256", IX86_BUILTIN_PUNPCKLDQ256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
29049 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_interleave_lowv4di, "__builtin_ia32_punpcklqdq256", IX86_BUILTIN_PUNPCKLQDQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
29050 { OPTION_MASK_ISA_AVX2, CODE_FOR_xorv4di3, "__builtin_ia32_pxor256", IX86_BUILTIN_PXOR256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
29051 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_vec_dupv4sf, "__builtin_ia32_vbroadcastss_ps", IX86_BUILTIN_VBROADCASTSS_PS, UNKNOWN, (int) V4SF_FTYPE_V4SF },
29052 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_vec_dupv8sf, "__builtin_ia32_vbroadcastss_ps256", IX86_BUILTIN_VBROADCASTSS_PS256, UNKNOWN, (int) V8SF_FTYPE_V4SF },
29053 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_vec_dupv4df, "__builtin_ia32_vbroadcastsd_pd256", IX86_BUILTIN_VBROADCASTSD_PD256, UNKNOWN, (int) V4DF_FTYPE_V2DF },
29054 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_vbroadcasti128_v4di, "__builtin_ia32_vbroadcastsi256", IX86_BUILTIN_VBROADCASTSI256, UNKNOWN, (int) V4DI_FTYPE_V2DI },
29055 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pblenddv4si, "__builtin_ia32_pblendd128", IX86_BUILTIN_PBLENDD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_INT },
29056 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pblenddv8si, "__builtin_ia32_pblendd256", IX86_BUILTIN_PBLENDD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_INT },
29057 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pbroadcastv32qi, "__builtin_ia32_pbroadcastb256", IX86_BUILTIN_PBROADCASTB256, UNKNOWN, (int) V32QI_FTYPE_V16QI },
29058 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pbroadcastv16hi, "__builtin_ia32_pbroadcastw256", IX86_BUILTIN_PBROADCASTW256, UNKNOWN, (int) V16HI_FTYPE_V8HI },
29059 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pbroadcastv8si, "__builtin_ia32_pbroadcastd256", IX86_BUILTIN_PBROADCASTD256, UNKNOWN, (int) V8SI_FTYPE_V4SI },
29060 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pbroadcastv4di, "__builtin_ia32_pbroadcastq256", IX86_BUILTIN_PBROADCASTQ256, UNKNOWN, (int) V4DI_FTYPE_V2DI },
29061 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pbroadcastv16qi, "__builtin_ia32_pbroadcastb128", IX86_BUILTIN_PBROADCASTB128, UNKNOWN, (int) V16QI_FTYPE_V16QI },
29062 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pbroadcastv8hi, "__builtin_ia32_pbroadcastw128", IX86_BUILTIN_PBROADCASTW128, UNKNOWN, (int) V8HI_FTYPE_V8HI },
29063 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pbroadcastv4si, "__builtin_ia32_pbroadcastd128", IX86_BUILTIN_PBROADCASTD128, UNKNOWN, (int) V4SI_FTYPE_V4SI },
29064 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pbroadcastv2di, "__builtin_ia32_pbroadcastq128", IX86_BUILTIN_PBROADCASTQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI },
29065 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_permvarv8si, "__builtin_ia32_permvarsi256", IX86_BUILTIN_VPERMVARSI256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
29066 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_permvarv8sf, "__builtin_ia32_permvarsf256", IX86_BUILTIN_VPERMVARSF256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SI },
29067 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_permv4df, "__builtin_ia32_permdf256", IX86_BUILTIN_VPERMDF256, UNKNOWN, (int) V4DF_FTYPE_V4DF_INT },
29068 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_permv4di, "__builtin_ia32_permdi256", IX86_BUILTIN_VPERMDI256, UNKNOWN, (int) V4DI_FTYPE_V4DI_INT },
29069 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_permv2ti, "__builtin_ia32_permti256", IX86_BUILTIN_VPERMTI256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_INT },
29070 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_extracti128, "__builtin_ia32_extract128i256", IX86_BUILTIN_VEXTRACT128I256, UNKNOWN, (int) V2DI_FTYPE_V4DI_INT },
29071 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_inserti128, "__builtin_ia32_insert128i256", IX86_BUILTIN_VINSERT128I256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V2DI_INT },
29072 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ashlvv4di, "__builtin_ia32_psllv4di", IX86_BUILTIN_PSLLVV4DI, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
29073 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ashlvv2di, "__builtin_ia32_psllv2di", IX86_BUILTIN_PSLLVV2DI, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
29074 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ashlvv8si, "__builtin_ia32_psllv8si", IX86_BUILTIN_PSLLVV8SI, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
29075 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ashlvv4si, "__builtin_ia32_psllv4si", IX86_BUILTIN_PSLLVV4SI, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
29076 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ashrvv8si, "__builtin_ia32_psrav8si", IX86_BUILTIN_PSRAVV8SI, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
29077 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ashrvv4si, "__builtin_ia32_psrav4si", IX86_BUILTIN_PSRAVV4SI, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
29078 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_lshrvv4di, "__builtin_ia32_psrlv4di", IX86_BUILTIN_PSRLVV4DI, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
29079 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_lshrvv2di, "__builtin_ia32_psrlv2di", IX86_BUILTIN_PSRLVV2DI, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
29080 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_lshrvv8si, "__builtin_ia32_psrlv8si", IX86_BUILTIN_PSRLVV8SI, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
29081 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_lshrvv4si, "__builtin_ia32_psrlv4si", IX86_BUILTIN_PSRLVV4SI, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
29083 { OPTION_MASK_ISA_LZCNT, CODE_FOR_clzhi2_lzcnt, "__builtin_clzs", IX86_BUILTIN_CLZS, UNKNOWN, (int) UINT16_FTYPE_UINT16 },
29085 /* BMI */
29086 { OPTION_MASK_ISA_BMI, CODE_FOR_bmi_bextr_si, "__builtin_ia32_bextr_u32", IX86_BUILTIN_BEXTR32, UNKNOWN, (int) UINT_FTYPE_UINT_UINT },
29087 { OPTION_MASK_ISA_BMI, CODE_FOR_bmi_bextr_di, "__builtin_ia32_bextr_u64", IX86_BUILTIN_BEXTR64, UNKNOWN, (int) UINT64_FTYPE_UINT64_UINT64 },
29088 { OPTION_MASK_ISA_BMI, CODE_FOR_ctzhi2, "__builtin_ctzs", IX86_BUILTIN_CTZS, UNKNOWN, (int) UINT16_FTYPE_UINT16 },
29090 /* TBM */
29091 { OPTION_MASK_ISA_TBM, CODE_FOR_tbm_bextri_si, "__builtin_ia32_bextri_u32", IX86_BUILTIN_BEXTRI32, UNKNOWN, (int) UINT_FTYPE_UINT_UINT },
29092 { OPTION_MASK_ISA_TBM, CODE_FOR_tbm_bextri_di, "__builtin_ia32_bextri_u64", IX86_BUILTIN_BEXTRI64, UNKNOWN, (int) UINT64_FTYPE_UINT64_UINT64 },
29094 /* F16C */
29095 { OPTION_MASK_ISA_F16C, CODE_FOR_vcvtph2ps, "__builtin_ia32_vcvtph2ps", IX86_BUILTIN_CVTPH2PS, UNKNOWN, (int) V4SF_FTYPE_V8HI },
29096 { OPTION_MASK_ISA_F16C, CODE_FOR_vcvtph2ps256, "__builtin_ia32_vcvtph2ps256", IX86_BUILTIN_CVTPH2PS256, UNKNOWN, (int) V8SF_FTYPE_V8HI },
29097 { OPTION_MASK_ISA_F16C, CODE_FOR_vcvtps2ph, "__builtin_ia32_vcvtps2ph", IX86_BUILTIN_CVTPS2PH, UNKNOWN, (int) V8HI_FTYPE_V4SF_INT },
29098 { OPTION_MASK_ISA_F16C, CODE_FOR_vcvtps2ph256, "__builtin_ia32_vcvtps2ph256", IX86_BUILTIN_CVTPS2PH256, UNKNOWN, (int) V8HI_FTYPE_V8SF_INT },
29100 /* BMI2 */
29101 { OPTION_MASK_ISA_BMI2, CODE_FOR_bmi2_bzhi_si3, "__builtin_ia32_bzhi_si", IX86_BUILTIN_BZHI32, UNKNOWN, (int) UINT_FTYPE_UINT_UINT },
29102 { OPTION_MASK_ISA_BMI2, CODE_FOR_bmi2_bzhi_di3, "__builtin_ia32_bzhi_di", IX86_BUILTIN_BZHI64, UNKNOWN, (int) UINT64_FTYPE_UINT64_UINT64 },
29103 { OPTION_MASK_ISA_BMI2, CODE_FOR_bmi2_pdep_si3, "__builtin_ia32_pdep_si", IX86_BUILTIN_PDEP32, UNKNOWN, (int) UINT_FTYPE_UINT_UINT },
29104 { OPTION_MASK_ISA_BMI2, CODE_FOR_bmi2_pdep_di3, "__builtin_ia32_pdep_di", IX86_BUILTIN_PDEP64, UNKNOWN, (int) UINT64_FTYPE_UINT64_UINT64 },
29105 { OPTION_MASK_ISA_BMI2, CODE_FOR_bmi2_pext_si3, "__builtin_ia32_pext_si", IX86_BUILTIN_PEXT32, UNKNOWN, (int) UINT_FTYPE_UINT_UINT },
29106 { OPTION_MASK_ISA_BMI2, CODE_FOR_bmi2_pext_di3, "__builtin_ia32_pext_di", IX86_BUILTIN_PEXT64, UNKNOWN, (int) UINT64_FTYPE_UINT64_UINT64 },
29107 };
29109 /* FMA4 and XOP. */
29110 #define MULTI_ARG_4_DF2_DI_I V2DF_FTYPE_V2DF_V2DF_V2DI_INT
29111 #define MULTI_ARG_4_DF2_DI_I1 V4DF_FTYPE_V4DF_V4DF_V4DI_INT
29112 #define MULTI_ARG_4_SF2_SI_I V4SF_FTYPE_V4SF_V4SF_V4SI_INT
29113 #define MULTI_ARG_4_SF2_SI_I1 V8SF_FTYPE_V8SF_V8SF_V8SI_INT
29114 #define MULTI_ARG_3_SF V4SF_FTYPE_V4SF_V4SF_V4SF
29115 #define MULTI_ARG_3_DF V2DF_FTYPE_V2DF_V2DF_V2DF
29116 #define MULTI_ARG_3_SF2 V8SF_FTYPE_V8SF_V8SF_V8SF
29117 #define MULTI_ARG_3_DF2 V4DF_FTYPE_V4DF_V4DF_V4DF
29118 #define MULTI_ARG_3_DI V2DI_FTYPE_V2DI_V2DI_V2DI
29119 #define MULTI_ARG_3_SI V4SI_FTYPE_V4SI_V4SI_V4SI
29120 #define MULTI_ARG_3_SI_DI V4SI_FTYPE_V4SI_V4SI_V2DI
29121 #define MULTI_ARG_3_HI V8HI_FTYPE_V8HI_V8HI_V8HI
29122 #define MULTI_ARG_3_HI_SI V8HI_FTYPE_V8HI_V8HI_V4SI
29123 #define MULTI_ARG_3_QI V16QI_FTYPE_V16QI_V16QI_V16QI
29124 #define MULTI_ARG_3_DI2 V4DI_FTYPE_V4DI_V4DI_V4DI
29125 #define MULTI_ARG_3_SI2 V8SI_FTYPE_V8SI_V8SI_V8SI
29126 #define MULTI_ARG_3_HI2 V16HI_FTYPE_V16HI_V16HI_V16HI
29127 #define MULTI_ARG_3_QI2 V32QI_FTYPE_V32QI_V32QI_V32QI
29128 #define MULTI_ARG_2_SF V4SF_FTYPE_V4SF_V4SF
29129 #define MULTI_ARG_2_DF V2DF_FTYPE_V2DF_V2DF
29130 #define MULTI_ARG_2_DI V2DI_FTYPE_V2DI_V2DI
29131 #define MULTI_ARG_2_SI V4SI_FTYPE_V4SI_V4SI
29132 #define MULTI_ARG_2_HI V8HI_FTYPE_V8HI_V8HI
29133 #define MULTI_ARG_2_QI V16QI_FTYPE_V16QI_V16QI
29134 #define MULTI_ARG_2_DI_IMM V2DI_FTYPE_V2DI_SI
29135 #define MULTI_ARG_2_SI_IMM V4SI_FTYPE_V4SI_SI
29136 #define MULTI_ARG_2_HI_IMM V8HI_FTYPE_V8HI_SI
29137 #define MULTI_ARG_2_QI_IMM V16QI_FTYPE_V16QI_SI
29138 #define MULTI_ARG_2_DI_CMP V2DI_FTYPE_V2DI_V2DI_CMP
29139 #define MULTI_ARG_2_SI_CMP V4SI_FTYPE_V4SI_V4SI_CMP
29140 #define MULTI_ARG_2_HI_CMP V8HI_FTYPE_V8HI_V8HI_CMP
29141 #define MULTI_ARG_2_QI_CMP V16QI_FTYPE_V16QI_V16QI_CMP
29142 #define MULTI_ARG_2_SF_TF V4SF_FTYPE_V4SF_V4SF_TF
29143 #define MULTI_ARG_2_DF_TF V2DF_FTYPE_V2DF_V2DF_TF
29144 #define MULTI_ARG_2_DI_TF V2DI_FTYPE_V2DI_V2DI_TF
29145 #define MULTI_ARG_2_SI_TF V4SI_FTYPE_V4SI_V4SI_TF
29146 #define MULTI_ARG_2_HI_TF V8HI_FTYPE_V8HI_V8HI_TF
29147 #define MULTI_ARG_2_QI_TF V16QI_FTYPE_V16QI_V16QI_TF
29148 #define MULTI_ARG_1_SF V4SF_FTYPE_V4SF
29149 #define MULTI_ARG_1_DF V2DF_FTYPE_V2DF
29150 #define MULTI_ARG_1_SF2 V8SF_FTYPE_V8SF
29151 #define MULTI_ARG_1_DF2 V4DF_FTYPE_V4DF
29152 #define MULTI_ARG_1_DI V2DI_FTYPE_V2DI
29153 #define MULTI_ARG_1_SI V4SI_FTYPE_V4SI
29154 #define MULTI_ARG_1_HI V8HI_FTYPE_V8HI
29155 #define MULTI_ARG_1_QI V16QI_FTYPE_V16QI
29156 #define MULTI_ARG_1_SI_DI V2DI_FTYPE_V4SI
29157 #define MULTI_ARG_1_HI_DI V2DI_FTYPE_V8HI
29158 #define MULTI_ARG_1_HI_SI V4SI_FTYPE_V8HI
29159 #define MULTI_ARG_1_QI_DI V2DI_FTYPE_V16QI
29160 #define MULTI_ARG_1_QI_SI V4SI_FTYPE_V16QI
29161 #define MULTI_ARG_1_QI_HI V8HI_FTYPE_V16QI
29164 {
29205 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v2di, "__builtin_ia32_vpcmov", IX86_BUILTIN_VPCMOV, UNKNOWN, (int)MULTI_ARG_3_DI },
29206 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v2di, "__builtin_ia32_vpcmov_v2di", IX86_BUILTIN_VPCMOV_V2DI, UNKNOWN, (int)MULTI_ARG_3_DI },
29207 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v4si, "__builtin_ia32_vpcmov_v4si", IX86_BUILTIN_VPCMOV_V4SI, UNKNOWN, (int)MULTI_ARG_3_SI },
29208 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v8hi, "__builtin_ia32_vpcmov_v8hi", IX86_BUILTIN_VPCMOV_V8HI, UNKNOWN, (int)MULTI_ARG_3_HI },
29209 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v16qi, "__builtin_ia32_vpcmov_v16qi",IX86_BUILTIN_VPCMOV_V16QI,UNKNOWN, (int)MULTI_ARG_3_QI },
29210 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v2df, "__builtin_ia32_vpcmov_v2df", IX86_BUILTIN_VPCMOV_V2DF, UNKNOWN, (int)MULTI_ARG_3_DF },
29211 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v4sf, "__builtin_ia32_vpcmov_v4sf", IX86_BUILTIN_VPCMOV_V4SF, UNKNOWN, (int)MULTI_ARG_3_SF },
29213 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v4di256, "__builtin_ia32_vpcmov256", IX86_BUILTIN_VPCMOV256, UNKNOWN, (int)MULTI_ARG_3_DI2 },
29214 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v4di256, "__builtin_ia32_vpcmov_v4di256", IX86_BUILTIN_VPCMOV_V4DI256, UNKNOWN, (int)MULTI_ARG_3_DI2 },
29215 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v8si256, "__builtin_ia32_vpcmov_v8si256", IX86_BUILTIN_VPCMOV_V8SI256, UNKNOWN, (int)MULTI_ARG_3_SI2 },
29216 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v16hi256, "__builtin_ia32_vpcmov_v16hi256", IX86_BUILTIN_VPCMOV_V16HI256, UNKNOWN, (int)MULTI_ARG_3_HI2 },
29217 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v32qi256, "__builtin_ia32_vpcmov_v32qi256", IX86_BUILTIN_VPCMOV_V32QI256, UNKNOWN, (int)MULTI_ARG_3_QI2 },
29218 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v4df256, "__builtin_ia32_vpcmov_v4df256", IX86_BUILTIN_VPCMOV_V4DF256, UNKNOWN, (int)MULTI_ARG_3_DF2 },
29219 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v8sf256, "__builtin_ia32_vpcmov_v8sf256", IX86_BUILTIN_VPCMOV_V8SF256, UNKNOWN, (int)MULTI_ARG_3_SF2 },
29221 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pperm, "__builtin_ia32_vpperm", IX86_BUILTIN_VPPERM, UNKNOWN, (int)MULTI_ARG_3_QI },
29223 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacssww, "__builtin_ia32_vpmacssww", IX86_BUILTIN_VPMACSSWW, UNKNOWN, (int)MULTI_ARG_3_HI },
29224 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacsww, "__builtin_ia32_vpmacsww", IX86_BUILTIN_VPMACSWW, UNKNOWN, (int)MULTI_ARG_3_HI },
29225 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacsswd, "__builtin_ia32_vpmacsswd", IX86_BUILTIN_VPMACSSWD, UNKNOWN, (int)MULTI_ARG_3_HI_SI },
29226 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacswd, "__builtin_ia32_vpmacswd", IX86_BUILTIN_VPMACSWD, UNKNOWN, (int)MULTI_ARG_3_HI_SI },
29227 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacssdd, "__builtin_ia32_vpmacssdd", IX86_BUILTIN_VPMACSSDD, UNKNOWN, (int)MULTI_ARG_3_SI },
29228 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacsdd, "__builtin_ia32_vpmacsdd", IX86_BUILTIN_VPMACSDD, UNKNOWN, (int)MULTI_ARG_3_SI },
29229 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacssdql, "__builtin_ia32_vpmacssdql", IX86_BUILTIN_VPMACSSDQL, UNKNOWN, (int)MULTI_ARG_3_SI_DI },
29230 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacssdqh, "__builtin_ia32_vpmacssdqh", IX86_BUILTIN_VPMACSSDQH, UNKNOWN, (int)MULTI_ARG_3_SI_DI },
29231 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacsdql, "__builtin_ia32_vpmacsdql", IX86_BUILTIN_VPMACSDQL, UNKNOWN, (int)MULTI_ARG_3_SI_DI },
29232 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacsdqh, "__builtin_ia32_vpmacsdqh", IX86_BUILTIN_VPMACSDQH, UNKNOWN, (int)MULTI_ARG_3_SI_DI },
29233 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmadcsswd, "__builtin_ia32_vpmadcsswd", IX86_BUILTIN_VPMADCSSWD, UNKNOWN, (int)MULTI_ARG_3_HI_SI },
29234 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmadcswd, "__builtin_ia32_vpmadcswd", IX86_BUILTIN_VPMADCSWD, UNKNOWN, (int)MULTI_ARG_3_HI_SI },
29236 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vrotlv2di3, "__builtin_ia32_vprotq", IX86_BUILTIN_VPROTQ, UNKNOWN, (int)MULTI_ARG_2_DI },
29237 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vrotlv4si3, "__builtin_ia32_vprotd", IX86_BUILTIN_VPROTD, UNKNOWN, (int)MULTI_ARG_2_SI },
29238 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vrotlv8hi3, "__builtin_ia32_vprotw", IX86_BUILTIN_VPROTW, UNKNOWN, (int)MULTI_ARG_2_HI },
29239 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vrotlv16qi3, "__builtin_ia32_vprotb", IX86_BUILTIN_VPROTB, UNKNOWN, (int)MULTI_ARG_2_QI },
29240 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_rotlv2di3, "__builtin_ia32_vprotqi", IX86_BUILTIN_VPROTQ_IMM, UNKNOWN, (int)MULTI_ARG_2_DI_IMM },
29241 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_rotlv4si3, "__builtin_ia32_vprotdi", IX86_BUILTIN_VPROTD_IMM, UNKNOWN, (int)MULTI_ARG_2_SI_IMM },
29242 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_rotlv8hi3, "__builtin_ia32_vprotwi", IX86_BUILTIN_VPROTW_IMM, UNKNOWN, (int)MULTI_ARG_2_HI_IMM },
29243 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_rotlv16qi3, "__builtin_ia32_vprotbi", IX86_BUILTIN_VPROTB_IMM, UNKNOWN, (int)MULTI_ARG_2_QI_IMM },
29244 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shav2di3, "__builtin_ia32_vpshaq", IX86_BUILTIN_VPSHAQ, UNKNOWN, (int)MULTI_ARG_2_DI },
29245 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shav4si3, "__builtin_ia32_vpshad", IX86_BUILTIN_VPSHAD, UNKNOWN, (int)MULTI_ARG_2_SI },
29246 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shav8hi3, "__builtin_ia32_vpshaw", IX86_BUILTIN_VPSHAW, UNKNOWN, (int)MULTI_ARG_2_HI },
29247 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shav16qi3, "__builtin_ia32_vpshab", IX86_BUILTIN_VPSHAB, UNKNOWN, (int)MULTI_ARG_2_QI },
29248 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shlv2di3, "__builtin_ia32_vpshlq", IX86_BUILTIN_VPSHLQ, UNKNOWN, (int)MULTI_ARG_2_DI },
29249 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shlv4si3, "__builtin_ia32_vpshld", IX86_BUILTIN_VPSHLD, UNKNOWN, (int)MULTI_ARG_2_SI },
29250 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shlv8hi3, "__builtin_ia32_vpshlw", IX86_BUILTIN_VPSHLW, UNKNOWN, (int)MULTI_ARG_2_HI },
29251 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shlv16qi3, "__builtin_ia32_vpshlb", IX86_BUILTIN_VPSHLB, UNKNOWN, (int)MULTI_ARG_2_QI },
29253 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vmfrczv4sf2, "__builtin_ia32_vfrczss", IX86_BUILTIN_VFRCZSS, UNKNOWN, (int)MULTI_ARG_1_SF },
29254 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vmfrczv2df2, "__builtin_ia32_vfrczsd", IX86_BUILTIN_VFRCZSD, UNKNOWN, (int)MULTI_ARG_1_DF },
29255 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_frczv4sf2, "__builtin_ia32_vfrczps", IX86_BUILTIN_VFRCZPS, UNKNOWN, (int)MULTI_ARG_1_SF },
29256 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_frczv2df2, "__builtin_ia32_vfrczpd", IX86_BUILTIN_VFRCZPD, UNKNOWN, (int)MULTI_ARG_1_DF },
29257 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_frczv8sf2, "__builtin_ia32_vfrczps256", IX86_BUILTIN_VFRCZPS256, UNKNOWN, (int)MULTI_ARG_1_SF2 },
29258 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_frczv4df2, "__builtin_ia32_vfrczpd256", IX86_BUILTIN_VFRCZPD256, UNKNOWN, (int)MULTI_ARG_1_DF2 },
29260 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddbw, "__builtin_ia32_vphaddbw", IX86_BUILTIN_VPHADDBW, UNKNOWN, (int)MULTI_ARG_1_QI_HI },
29261 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddbd, "__builtin_ia32_vphaddbd", IX86_BUILTIN_VPHADDBD, UNKNOWN, (int)MULTI_ARG_1_QI_SI },
29262 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddbq, "__builtin_ia32_vphaddbq", IX86_BUILTIN_VPHADDBQ, UNKNOWN, (int)MULTI_ARG_1_QI_DI },
29263 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddwd, "__builtin_ia32_vphaddwd", IX86_BUILTIN_VPHADDWD, UNKNOWN, (int)MULTI_ARG_1_HI_SI },
29264 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddwq, "__builtin_ia32_vphaddwq", IX86_BUILTIN_VPHADDWQ, UNKNOWN, (int)MULTI_ARG_1_HI_DI },
29265 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phadddq, "__builtin_ia32_vphadddq", IX86_BUILTIN_VPHADDDQ, UNKNOWN, (int)MULTI_ARG_1_SI_DI },
29266 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddubw, "__builtin_ia32_vphaddubw", IX86_BUILTIN_VPHADDUBW, UNKNOWN, (int)MULTI_ARG_1_QI_HI },
29267 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddubd, "__builtin_ia32_vphaddubd", IX86_BUILTIN_VPHADDUBD, UNKNOWN, (int)MULTI_ARG_1_QI_SI },
29268 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddubq, "__builtin_ia32_vphaddubq", IX86_BUILTIN_VPHADDUBQ, UNKNOWN, (int)MULTI_ARG_1_QI_DI },
29269 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phadduwd, "__builtin_ia32_vphadduwd", IX86_BUILTIN_VPHADDUWD, UNKNOWN, (int)MULTI_ARG_1_HI_SI },
29270 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phadduwq, "__builtin_ia32_vphadduwq", IX86_BUILTIN_VPHADDUWQ, UNKNOWN, (int)MULTI_ARG_1_HI_DI },
29271 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddudq, "__builtin_ia32_vphaddudq", IX86_BUILTIN_VPHADDUDQ, UNKNOWN, (int)MULTI_ARG_1_SI_DI },
29272 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phsubbw, "__builtin_ia32_vphsubbw", IX86_BUILTIN_VPHSUBBW, UNKNOWN, (int)MULTI_ARG_1_QI_HI },
29273 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phsubwd, "__builtin_ia32_vphsubwd", IX86_BUILTIN_VPHSUBWD, UNKNOWN, (int)MULTI_ARG_1_HI_SI },
29274 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phsubdq, "__builtin_ia32_vphsubdq", IX86_BUILTIN_VPHSUBDQ, UNKNOWN, (int)MULTI_ARG_1_SI_DI },
29276 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv16qi3, "__builtin_ia32_vpcomeqb", IX86_BUILTIN_VPCOMEQB, EQ, (int)MULTI_ARG_2_QI_CMP },
29277 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv16qi3, "__builtin_ia32_vpcomneb", IX86_BUILTIN_VPCOMNEB, NE, (int)MULTI_ARG_2_QI_CMP },
29278 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv16qi3, "__builtin_ia32_vpcomneqb", IX86_BUILTIN_VPCOMNEB, NE, (int)MULTI_ARG_2_QI_CMP },
29279 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv16qi3, "__builtin_ia32_vpcomltb", IX86_BUILTIN_VPCOMLTB, LT, (int)MULTI_ARG_2_QI_CMP },
29280 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv16qi3, "__builtin_ia32_vpcomleb", IX86_BUILTIN_VPCOMLEB, LE, (int)MULTI_ARG_2_QI_CMP },
29281 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv16qi3, "__builtin_ia32_vpcomgtb", IX86_BUILTIN_VPCOMGTB, GT, (int)MULTI_ARG_2_QI_CMP },
29282 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv16qi3, "__builtin_ia32_vpcomgeb", IX86_BUILTIN_VPCOMGEB, GE, (int)MULTI_ARG_2_QI_CMP },
29284 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv8hi3, "__builtin_ia32_vpcomeqw", IX86_BUILTIN_VPCOMEQW, EQ, (int)MULTI_ARG_2_HI_CMP },
29285 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv8hi3, "__builtin_ia32_vpcomnew", IX86_BUILTIN_VPCOMNEW, NE, (int)MULTI_ARG_2_HI_CMP },
29286 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv8hi3, "__builtin_ia32_vpcomneqw", IX86_BUILTIN_VPCOMNEW, NE, (int)MULTI_ARG_2_HI_CMP },
29287 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv8hi3, "__builtin_ia32_vpcomltw", IX86_BUILTIN_VPCOMLTW, LT, (int)MULTI_ARG_2_HI_CMP },
29288 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv8hi3, "__builtin_ia32_vpcomlew", IX86_BUILTIN_VPCOMLEW, LE, (int)MULTI_ARG_2_HI_CMP },
29289 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv8hi3, "__builtin_ia32_vpcomgtw", IX86_BUILTIN_VPCOMGTW, GT, (int)MULTI_ARG_2_HI_CMP },
29290 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv8hi3, "__builtin_ia32_vpcomgew", IX86_BUILTIN_VPCOMGEW, GE, (int)MULTI_ARG_2_HI_CMP },
29292 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv4si3, "__builtin_ia32_vpcomeqd", IX86_BUILTIN_VPCOMEQD, EQ, (int)MULTI_ARG_2_SI_CMP },
29293 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv4si3, "__builtin_ia32_vpcomned", IX86_BUILTIN_VPCOMNED, NE, (int)MULTI_ARG_2_SI_CMP },
29294 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv4si3, "__builtin_ia32_vpcomneqd", IX86_BUILTIN_VPCOMNED, NE, (int)MULTI_ARG_2_SI_CMP },
29295 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv4si3, "__builtin_ia32_vpcomltd", IX86_BUILTIN_VPCOMLTD, LT, (int)MULTI_ARG_2_SI_CMP },
29296 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv4si3, "__builtin_ia32_vpcomled", IX86_BUILTIN_VPCOMLED, LE, (int)MULTI_ARG_2_SI_CMP },
29297 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv4si3, "__builtin_ia32_vpcomgtd", IX86_BUILTIN_VPCOMGTD, GT, (int)MULTI_ARG_2_SI_CMP },
29298 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv4si3, "__builtin_ia32_vpcomged", IX86_BUILTIN_VPCOMGED, GE, (int)MULTI_ARG_2_SI_CMP },
29300 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv2di3, "__builtin_ia32_vpcomeqq", IX86_BUILTIN_VPCOMEQQ, EQ, (int)MULTI_ARG_2_DI_CMP },
29301 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv2di3, "__builtin_ia32_vpcomneq", IX86_BUILTIN_VPCOMNEQ, NE, (int)MULTI_ARG_2_DI_CMP },
29302 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv2di3, "__builtin_ia32_vpcomneqq", IX86_BUILTIN_VPCOMNEQ, NE, (int)MULTI_ARG_2_DI_CMP },
29303 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv2di3, "__builtin_ia32_vpcomltq", IX86_BUILTIN_VPCOMLTQ, LT, (int)MULTI_ARG_2_DI_CMP },
29304 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv2di3, "__builtin_ia32_vpcomleq", IX86_BUILTIN_VPCOMLEQ, LE, (int)MULTI_ARG_2_DI_CMP },
29305 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv2di3, "__builtin_ia32_vpcomgtq", IX86_BUILTIN_VPCOMGTQ, GT, (int)MULTI_ARG_2_DI_CMP },
29306 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv2di3, "__builtin_ia32_vpcomgeq", IX86_BUILTIN_VPCOMGEQ, GE, (int)MULTI_ARG_2_DI_CMP },
29308 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v16qi3,"__builtin_ia32_vpcomequb", IX86_BUILTIN_VPCOMEQUB, EQ, (int)MULTI_ARG_2_QI_CMP },
29309 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v16qi3,"__builtin_ia32_vpcomneub", IX86_BUILTIN_VPCOMNEUB, NE, (int)MULTI_ARG_2_QI_CMP },
29310 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v16qi3,"__builtin_ia32_vpcomnequb", IX86_BUILTIN_VPCOMNEUB, NE, (int)MULTI_ARG_2_QI_CMP },
29311 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv16qi3, "__builtin_ia32_vpcomltub", IX86_BUILTIN_VPCOMLTUB, LTU, (int)MULTI_ARG_2_QI_CMP },
29312 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv16qi3, "__builtin_ia32_vpcomleub", IX86_BUILTIN_VPCOMLEUB, LEU, (int)MULTI_ARG_2_QI_CMP },
29313 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv16qi3, "__builtin_ia32_vpcomgtub", IX86_BUILTIN_VPCOMGTUB, GTU, (int)MULTI_ARG_2_QI_CMP },
29314 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv16qi3, "__builtin_ia32_vpcomgeub", IX86_BUILTIN_VPCOMGEUB, GEU, (int)MULTI_ARG_2_QI_CMP },
29316 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v8hi3, "__builtin_ia32_vpcomequw", IX86_BUILTIN_VPCOMEQUW, EQ, (int)MULTI_ARG_2_HI_CMP },
29317 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v8hi3, "__builtin_ia32_vpcomneuw", IX86_BUILTIN_VPCOMNEUW, NE, (int)MULTI_ARG_2_HI_CMP },
29318 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v8hi3, "__builtin_ia32_vpcomnequw", IX86_BUILTIN_VPCOMNEUW, NE, (int)MULTI_ARG_2_HI_CMP },
29319 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv8hi3, "__builtin_ia32_vpcomltuw", IX86_BUILTIN_VPCOMLTUW, LTU, (int)MULTI_ARG_2_HI_CMP },
29320 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv8hi3, "__builtin_ia32_vpcomleuw", IX86_BUILTIN_VPCOMLEUW, LEU, (int)MULTI_ARG_2_HI_CMP },
29321 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv8hi3, "__builtin_ia32_vpcomgtuw", IX86_BUILTIN_VPCOMGTUW, GTU, (int)MULTI_ARG_2_HI_CMP },
29322 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv8hi3, "__builtin_ia32_vpcomgeuw", IX86_BUILTIN_VPCOMGEUW, GEU, (int)MULTI_ARG_2_HI_CMP },
29324 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v4si3, "__builtin_ia32_vpcomequd", IX86_BUILTIN_VPCOMEQUD, EQ, (int)MULTI_ARG_2_SI_CMP },
29325 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v4si3, "__builtin_ia32_vpcomneud", IX86_BUILTIN_VPCOMNEUD, NE, (int)MULTI_ARG_2_SI_CMP },
29326 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v4si3, "__builtin_ia32_vpcomnequd", IX86_BUILTIN_VPCOMNEUD, NE, (int)MULTI_ARG_2_SI_CMP },
29327 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv4si3, "__builtin_ia32_vpcomltud", IX86_BUILTIN_VPCOMLTUD, LTU, (int)MULTI_ARG_2_SI_CMP },
29328 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv4si3, "__builtin_ia32_vpcomleud", IX86_BUILTIN_VPCOMLEUD, LEU, (int)MULTI_ARG_2_SI_CMP },
29329 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv4si3, "__builtin_ia32_vpcomgtud", IX86_BUILTIN_VPCOMGTUD, GTU, (int)MULTI_ARG_2_SI_CMP },
29330 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv4si3, "__builtin_ia32_vpcomgeud", IX86_BUILTIN_VPCOMGEUD, GEU, (int)MULTI_ARG_2_SI_CMP },
29332 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v2di3, "__builtin_ia32_vpcomequq", IX86_BUILTIN_VPCOMEQUQ, EQ, (int)MULTI_ARG_2_DI_CMP },
29333 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v2di3, "__builtin_ia32_vpcomneuq", IX86_BUILTIN_VPCOMNEUQ, NE, (int)MULTI_ARG_2_DI_CMP },
29334 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v2di3, "__builtin_ia32_vpcomnequq", IX86_BUILTIN_VPCOMNEUQ, NE, (int)MULTI_ARG_2_DI_CMP },
29335 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv2di3, "__builtin_ia32_vpcomltuq", IX86_BUILTIN_VPCOMLTUQ, LTU, (int)MULTI_ARG_2_DI_CMP },
29336 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv2di3, "__builtin_ia32_vpcomleuq", IX86_BUILTIN_VPCOMLEUQ, LEU, (int)MULTI_ARG_2_DI_CMP },
29337 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv2di3, "__builtin_ia32_vpcomgtuq", IX86_BUILTIN_VPCOMGTUQ, GTU, (int)MULTI_ARG_2_DI_CMP },
29338 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv2di3, "__builtin_ia32_vpcomgeuq", IX86_BUILTIN_VPCOMGEUQ, GEU, (int)MULTI_ARG_2_DI_CMP },
29340 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv16qi3, "__builtin_ia32_vpcomfalseb", IX86_BUILTIN_VPCOMFALSEB, (enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_QI_TF },
29341 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv8hi3, "__builtin_ia32_vpcomfalsew", IX86_BUILTIN_VPCOMFALSEW, (enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_HI_TF },
29342 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv4si3, "__builtin_ia32_vpcomfalsed", IX86_BUILTIN_VPCOMFALSED, (enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_SI_TF },
29343 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv2di3, "__builtin_ia32_vpcomfalseq", IX86_BUILTIN_VPCOMFALSEQ, (enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_DI_TF },
29344 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv16qi3, "__builtin_ia32_vpcomfalseub",IX86_BUILTIN_VPCOMFALSEUB,(enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_QI_TF },
29345 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv8hi3, "__builtin_ia32_vpcomfalseuw",IX86_BUILTIN_VPCOMFALSEUW,(enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_HI_TF },
29346 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv4si3, "__builtin_ia32_vpcomfalseud",IX86_BUILTIN_VPCOMFALSEUD,(enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_SI_TF },
29347 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv2di3, "__builtin_ia32_vpcomfalseuq",IX86_BUILTIN_VPCOMFALSEUQ,(enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_DI_TF },
29349 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv16qi3, "__builtin_ia32_vpcomtrueb", IX86_BUILTIN_VPCOMTRUEB, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_QI_TF },
29350 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv8hi3, "__builtin_ia32_vpcomtruew", IX86_BUILTIN_VPCOMTRUEW, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_HI_TF },
29351 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv4si3, "__builtin_ia32_vpcomtrued", IX86_BUILTIN_VPCOMTRUED, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_SI_TF },
29352 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv2di3, "__builtin_ia32_vpcomtrueq", IX86_BUILTIN_VPCOMTRUEQ, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_DI_TF },
29353 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv16qi3, "__builtin_ia32_vpcomtrueub", IX86_BUILTIN_VPCOMTRUEUB, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_QI_TF },
29354 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv8hi3, "__builtin_ia32_vpcomtrueuw", IX86_BUILTIN_VPCOMTRUEUW, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_HI_TF },
29355 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv4si3, "__builtin_ia32_vpcomtrueud", IX86_BUILTIN_VPCOMTRUEUD, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_SI_TF },
29356 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv2di3, "__builtin_ia32_vpcomtrueuq", IX86_BUILTIN_VPCOMTRUEUQ, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_DI_TF },
29358 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vpermil2v2df3, "__builtin_ia32_vpermil2pd", IX86_BUILTIN_VPERMIL2PD, UNKNOWN, (int)MULTI_ARG_4_DF2_DI_I },
29359 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vpermil2v4sf3, "__builtin_ia32_vpermil2ps", IX86_BUILTIN_VPERMIL2PS, UNKNOWN, (int)MULTI_ARG_4_SF2_SI_I },
29360 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vpermil2v4df3, "__builtin_ia32_vpermil2pd256", IX86_BUILTIN_VPERMIL2PD256, UNKNOWN, (int)MULTI_ARG_4_DF2_DI_I1 },
29361 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vpermil2v8sf3, "__builtin_ia32_vpermil2ps256", IX86_BUILTIN_VPERMIL2PS256, UNKNOWN, (int)MULTI_ARG_4_SF2_SI_I1 },
29363 };
29364 \f
29365 /* TM vector builtins. */
29367 /* Reuse the existing x86-specific `struct builtin_description' cause
29368 we're lazy. Add casts to make them fit. */
29370 {
29371 { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_WM64", (enum ix86_builtins) BUILT_IN_TM_STORE_M64, UNKNOWN, VOID_FTYPE_PV2SI_V2SI },
29372 { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_WaRM64", (enum ix86_builtins) BUILT_IN_TM_STORE_WAR_M64, UNKNOWN, VOID_FTYPE_PV2SI_V2SI },
29373 { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_WaWM64", (enum ix86_builtins) BUILT_IN_TM_STORE_WAW_M64, UNKNOWN, VOID_FTYPE_PV2SI_V2SI },
29374 { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_RM64", (enum ix86_builtins) BUILT_IN_TM_LOAD_M64, UNKNOWN, V2SI_FTYPE_PCV2SI },
29375 { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_RaRM64", (enum ix86_builtins) BUILT_IN_TM_LOAD_RAR_M64, UNKNOWN, V2SI_FTYPE_PCV2SI },
29376 { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_RaWM64", (enum ix86_builtins) BUILT_IN_TM_LOAD_RAW_M64, UNKNOWN, V2SI_FTYPE_PCV2SI },
29377 { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_RfWM64", (enum ix86_builtins) BUILT_IN_TM_LOAD_RFW_M64, UNKNOWN, V2SI_FTYPE_PCV2SI },
29379 { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_WM128", (enum ix86_builtins) BUILT_IN_TM_STORE_M128, UNKNOWN, VOID_FTYPE_PV4SF_V4SF },
29380 { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_WaRM128", (enum ix86_builtins) BUILT_IN_TM_STORE_WAR_M128, UNKNOWN, VOID_FTYPE_PV4SF_V4SF },
29381 { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_WaWM128", (enum ix86_builtins) BUILT_IN_TM_STORE_WAW_M128, UNKNOWN, VOID_FTYPE_PV4SF_V4SF },
29382 { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_RM128", (enum ix86_builtins) BUILT_IN_TM_LOAD_M128, UNKNOWN, V4SF_FTYPE_PCV4SF },
29383 { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_RaRM128", (enum ix86_builtins) BUILT_IN_TM_LOAD_RAR_M128, UNKNOWN, V4SF_FTYPE_PCV4SF },
29384 { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_RaWM128", (enum ix86_builtins) BUILT_IN_TM_LOAD_RAW_M128, UNKNOWN, V4SF_FTYPE_PCV4SF },
29385 { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_RfWM128", (enum ix86_builtins) BUILT_IN_TM_LOAD_RFW_M128, UNKNOWN, V4SF_FTYPE_PCV4SF },
29387 { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_WM256", (enum ix86_builtins) BUILT_IN_TM_STORE_M256, UNKNOWN, VOID_FTYPE_PV8SF_V8SF },
29388 { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_WaRM256", (enum ix86_builtins) BUILT_IN_TM_STORE_WAR_M256, UNKNOWN, VOID_FTYPE_PV8SF_V8SF },
29389 { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_WaWM256", (enum ix86_builtins) BUILT_IN_TM_STORE_WAW_M256, UNKNOWN, VOID_FTYPE_PV8SF_V8SF },
29390 { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_RM256", (enum ix86_builtins) BUILT_IN_TM_LOAD_M256, UNKNOWN, V8SF_FTYPE_PCV8SF },
29391 { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_RaRM256", (enum ix86_builtins) BUILT_IN_TM_LOAD_RAR_M256, UNKNOWN, V8SF_FTYPE_PCV8SF },
29392 { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_RaWM256", (enum ix86_builtins) BUILT_IN_TM_LOAD_RAW_M256, UNKNOWN, V8SF_FTYPE_PCV8SF },
29393 { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_RfWM256", (enum ix86_builtins) BUILT_IN_TM_LOAD_RFW_M256, UNKNOWN, V8SF_FTYPE_PCV8SF },
29395 { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_LM64", (enum ix86_builtins) BUILT_IN_TM_LOG_M64, UNKNOWN, VOID_FTYPE_PCVOID },
29396 { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_LM128", (enum ix86_builtins) BUILT_IN_TM_LOG_M128, UNKNOWN, VOID_FTYPE_PCVOID },
29397 { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_LM256", (enum ix86_builtins) BUILT_IN_TM_LOG_M256, UNKNOWN, VOID_FTYPE_PCVOID },
29398 };
29400 /* TM callbacks. */
29402 /* Return the builtin decl needed to load a vector of TYPE. */
29404 static tree
29406 {
29408 {
29410 {
29417 }
29418 }
29420 }
29422 /* Return the builtin decl needed to store a vector of TYPE. */
29424 static tree
29426 {
29428 {
29430 {
29437 }
29438 }
29440 }
29441 \f
29442 /* Initialize the transactional memory vector load/store builtins. */
29444 static void
29446 {
29450 tree decl;
29457 /* If there are no builtins defined, we must be compiling in a
29458 language without trans-mem support. */
29462 /* Use whatever attributes a normal TM load has. */
29466 /* Use whatever attributes a normal TM store has. */
29470 /* Use whatever attributes a normal TM log has. */
29478 {
29482 {
29490 {
29493 }
29495 {
29498 }
29499 else
29500 {
29503 }
29505 /* The builtin without the prefix for
29506 calling it directly. */
29508 attrs);
29509 /* add_builtin_function() will set the DECL_ATTRIBUTES, now
29510 set the TYPE_ATTRIBUTES. */
29514 }
29515 }
29516 }
29518 /* Set up all the MMX/SSE builtins, even builtins for instructions that are not
29519 in the current target ISA to allow the user to compile particular modules
29520 with different target specific options that differ from the command line
29521 options. */
29522 static void
29524 {
29529 /* Add all special builtins with variable number of operands. */
29533 {
29539 }
29541 /* Add all builtins with variable number of operands. */
29545 {
29551 }
29553 /* pcmpestr[im] insns. */
29557 {
29560 else
29563 }
29565 /* pcmpistr[im] insns. */
29569 {
29572 else
29575 }
29577 /* comi/ucomi insns. */
29579 {
29582 else
29585 }
29587 /* SSE */
29593 /* SSE or 3DNow!A */
29596 IX86_BUILTIN_MASKMOVQ);
29598 /* SSE2 */
29607 /* SSE3. */
29613 /* AES */
29627 /* PCLMUL */
29631 /* RDRND */
29638 IX86_BUILTIN_RDRAND64_STEP);
29640 /* AVX2 */
29642 V2DF_FTYPE_V2DF_PCDOUBLE_V4SI_V2DF_INT,
29643 IX86_BUILTIN_GATHERSIV2DF);
29646 V4DF_FTYPE_V4DF_PCDOUBLE_V4SI_V4DF_INT,
29647 IX86_BUILTIN_GATHERSIV4DF);
29650 V2DF_FTYPE_V2DF_PCDOUBLE_V2DI_V2DF_INT,
29651 IX86_BUILTIN_GATHERDIV2DF);
29654 V4DF_FTYPE_V4DF_PCDOUBLE_V4DI_V4DF_INT,
29655 IX86_BUILTIN_GATHERDIV4DF);
29658 V4SF_FTYPE_V4SF_PCFLOAT_V4SI_V4SF_INT,
29659 IX86_BUILTIN_GATHERSIV4SF);
29662 V8SF_FTYPE_V8SF_PCFLOAT_V8SI_V8SF_INT,
29663 IX86_BUILTIN_GATHERSIV8SF);
29666 V4SF_FTYPE_V4SF_PCFLOAT_V2DI_V4SF_INT,
29667 IX86_BUILTIN_GATHERDIV4SF);
29670 V4SF_FTYPE_V4SF_PCFLOAT_V4DI_V4SF_INT,
29671 IX86_BUILTIN_GATHERDIV8SF);
29674 V2DI_FTYPE_V2DI_PCINT64_V4SI_V2DI_INT,
29675 IX86_BUILTIN_GATHERSIV2DI);
29678 V4DI_FTYPE_V4DI_PCINT64_V4SI_V4DI_INT,
29679 IX86_BUILTIN_GATHERSIV4DI);
29682 V2DI_FTYPE_V2DI_PCINT64_V2DI_V2DI_INT,
29683 IX86_BUILTIN_GATHERDIV2DI);
29686 V4DI_FTYPE_V4DI_PCINT64_V4DI_V4DI_INT,
29687 IX86_BUILTIN_GATHERDIV4DI);
29690 V4SI_FTYPE_V4SI_PCINT_V4SI_V4SI_INT,
29691 IX86_BUILTIN_GATHERSIV4SI);
29694 V8SI_FTYPE_V8SI_PCINT_V8SI_V8SI_INT,
29695 IX86_BUILTIN_GATHERSIV8SI);
29698 V4SI_FTYPE_V4SI_PCINT_V2DI_V4SI_INT,
29699 IX86_BUILTIN_GATHERDIV4SI);
29702 V4SI_FTYPE_V4SI_PCINT_V4DI_V4SI_INT,
29703 IX86_BUILTIN_GATHERDIV8SI);
29706 V4DF_FTYPE_V4DF_PCDOUBLE_V8SI_V4DF_INT,
29707 IX86_BUILTIN_GATHERALTSIV4DF);
29710 V8SF_FTYPE_V8SF_PCFLOAT_V4DI_V8SF_INT,
29711 IX86_BUILTIN_GATHERALTDIV8SF);
29714 V4DI_FTYPE_V4DI_PCINT64_V8SI_V4DI_INT,
29715 IX86_BUILTIN_GATHERALTSIV4DI);
29718 V8SI_FTYPE_V8SI_PCINT_V4DI_V8SI_INT,
29719 IX86_BUILTIN_GATHERALTDIV8SI);
29721 /* RTM. */
29725 /* MMX access to the vec_init patterns. */
29730 V4HI_FTYPE_HI_HI_HI_HI,
29731 IX86_BUILTIN_VEC_INIT_V4HI);
29734 V8QI_FTYPE_QI_QI_QI_QI_QI_QI_QI_QI,
29735 IX86_BUILTIN_VEC_INIT_V8QI);
29737 /* Access to the vec_extract patterns. */
29759 /* Access to the vec_set patterns. */
29780 /* RDSEED */
29789 /* ADCX */
29794 UCHAR_FTYPE_UCHAR_ULONGLONG_ULONGLONG_PULONGLONG,
29795 IX86_BUILTIN_ADDCARRYX64);
29797 /* Read/write FLAGS. */
29808 /* Add FMA4 multi-arg argument instructions */
29810 {
29816 }
29817 }
29819 /* This adds a condition to the basic_block NEW_BB in function FUNCTION_DECL
29820 to return a pointer to VERSION_DECL if the outcome of the expression
29821 formed by PREDICATE_CHAIN is true. This function will be called during
29822 version dispatch to decide which function version to execute. It returns
29823 the basic block at the end, to which more conditions can be added. */
29825 static basic_block
29828 {
29829 gimple return_stmt;
29831 gimple convert_stmt;
29832 gimple call_cond_stmt;
29833 gimple if_else_stmt;
29839 gimple_seq gseq;
29856 {
29864 }
29867 {
29882 else
29883 {
29884 gimple assign_stmt;
29885 /* Use MIN_EXPR to check if any integer is zero?.
29886 and_expr_var = min_expr <cond_var, and_expr_var> */
29894 }
29895 }
29898 integer_zero_node,
29928 }
29930 /* This parses the attribute arguments to target in DECL and determines
29931 the right builtin to use to match the platform specification.
29932 It returns the priority value for this version decl. If PREDICATE_LIST
29933 is not NULL, it stores the list of cpu features that need to be checked
29934 before dispatching this function. */
29936 static unsigned int
29938 {
29939 tree attrs;
29941 tree target_node;
29948 /* Priority of i386 features, greater value is higher priority. This is
29949 used to decide the order in which function dispatch must happen. For
29950 instance, a version specialized for SSE4.2 should be checked for dispatch
29951 before a version for SSE3, as SSE4.2 implies SSE3. */
29952 enum feature_priority
29953 {
29955 P_MMX,
29956 P_SSE,
29957 P_SSE2,
29958 P_SSE3,
29959 P_SSSE3,
29960 P_PROC_SSSE3,
29961 P_SSE4_a,
29962 P_PROC_SSE4_a,
29963 P_SSE4_1,
29964 P_SSE4_2,
29965 P_PROC_SSE4_2,
29966 P_POPCNT,
29967 P_AVX,
29968 P_AVX2,
29969 P_FMA,
29970 P_PROC_FMA
29971 };
29975 /* These are the target attribute strings for which a dispatcher is
29976 available, from fold_builtin_cpu. */
29978 static struct _feature_list
29979 {
29982 }
29984 {
29995 };
29998 static unsigned int NUM_FEATURES
30014 /* Return priority zero for default function. */
30018 /* Handle arch= if specified. For priority, set it to be 1 more than
30019 the best instruction set the processor can handle. For instance, if
30020 there is a version for atom and a version for ssse3 (the highest ISA
30021 priority for atom), the atom version must be checked for dispatch
30022 before the ssse3 version. */
30024 {
30034 {
30036 {
30065 }
30066 }
30071 {
30075 }
30078 {
30080 /* For a C string literal the length includes the trailing NULL. */
30083 predicate_chain);
30084 }
30085 }
30087 /* Process feature name. */
30094 {
30095 /* Do not process "arch=" */
30097 {
30100 }
30102 {
30104 {
30106 {
30111 predicate_chain);
30112 }
30113 /* Find the maximum priority feature. */
30118 }
30119 }
30121 {
30125 }
30127 }
30131 {
30134 attrs_str);
30136 }
30138 {
30141 }
30144 }
30146 /* This compares the priority of target features in function DECL1
30147 and DECL2. It returns positive value if DECL1 is higher priority,
30148 negative value if DECL2 is higher priority and 0 if they are the
30149 same. */
30151 static int
30153 {
30158 }
30160 /* V1 and V2 point to function versions with different priorities
30161 based on the target ISA. This function compares their priorities. */
30163 static int
30165 {
30167 {
30168 tree version_decl;
30169 tree predicate_chain;
30176 }
30178 /* This function generates the dispatch function for
30179 multi-versioned functions. DISPATCH_DECL is the function which will
30180 contain the dispatch logic. FNDECLS are the function choices for
30181 dispatch, and is a tree chain. EMPTY_BB is the basic block pointer
30182 in DISPATCH_DECL in which the dispatch code is generated. */
30184 static int
30188 {
30189 tree default_decl;
30190 gimple ifunc_cpu_init_stmt;
30191 gimple_seq gseq;
30193 tree ele;
30199 struct _function_version_info
30200 {
30201 tree version_decl;
30202 tree predicate_chain;
30210 /*fndecls_p is actually a vector. */
30213 /* At least one more version other than the default. */
30220 /* The first version in the vector is the default decl. */
30226 /* Function version dispatch is via IFUNC. IFUNC resolvers fire before
30227 constructors, so explicity call __builtin_cpu_init here. */
30238 {
30242 /* Get attribute string, parse it and find the right predicate decl.
30243 The predicate function could be a lengthy combination of many
30244 features, like arch-type and various isa-variants. */
30255 actual_versions++;
30256 }
30258 /* Sort the versions according to descending order of dispatch priority. The
30259 priority is based on the ISA. This is not a perfect solution. There
30260 could still be ambiguity. If more than one function version is suitable
30261 to execute, which one should be dispatched? In future, allow the user
30262 to specify a dispatch priority next to the version. */
30272 /* dispatch default version at the end. */
30278 }
30280 /* Comparator function to be used in qsort routine to sort attribute
30281 specification strings to "target". */
30283 static int
30285 {
30289 }
30291 /* ARGLIST is the argument to target attribute. This function tokenizes
30292 the comma separated arguments, sorts them and returns a string which
30293 is a unique identifier for the comma separated arguments. It also
30294 replaces non-identifier characters "=,-" with "_". */
30298 {
30299 tree arg;
30308 {
30313 argnum++;
30316 argnum++;
30317 }
30322 {
30328 }
30330 /* Replace "=,-" with "_". */
30343 {
30345 i++;
30347 }
30354 {
30359 }
30364 }
30366 /* This function changes the assembler name for functions that are
30367 versions. If DECL is a function version and has a "target"
30368 attribute, it appends the attribute string to its assembler name. */
30370 static tree
30372 {
30373 tree version_attr;
30381 "Function versions cannot be marked as gnu_inline,"
30390 /* target attribute string cannot be NULL. */
30394 version_string
30405 /* Allow assembler name to be modified if already set. */
30413 }
30415 /* This function returns true if FN1 and FN2 are versions of the same function,
30416 that is, the target strings of the function decls are different. This assumes
30417 that FN1 and FN2 have the same signature. */
30419 static bool
30421 {
30433 /* At least one function decl should have the target attribute specified. */
30437 /* Diagnose missing target attribute if one of the decls is already
30438 multi-versioned. */
30440 {
30442 {
30444 {
30449 }
30452 fn2);
30455 /* Prevent diagnosing of the same error multiple times. */
30460 }
30462 }
30467 /* The sorted target strings must be different for fn1 and fn2
30468 to be versions. */
30471 else
30478 }
30480 static tree
30482 {
30483 /* For function version, add the target suffix to the assembler name. */
30487 #ifdef SUBTARGET_MANGLE_DECL_ASSEMBLER_NAME
30489 #endif
30492 }
30494 /* Return a new name by appending SUFFIX to the DECL name. If make_unique
30495 is true, append the full path name of the source file. */
30499 {
30507 /* Get a unique name that can be used globally without any chances
30508 of collision at link time. */
30518 /* Use '.' to concatenate names as it is demangler friendly. */
30521 suffix);
30522 else
30526 }
30528 #if defined (ASM_OUTPUT_TYPE_DIRECTIVE)
30530 /* Make a dispatcher declaration for the multi-versioned function DECL.
30531 Calls to DECL function will be replaced with calls to the dispatcher
30532 by the front-end. Return the decl created. */
30534 static tree
30536 {
30537 tree func_decl;
30557 /* Mark this func as external, the resolver will flip it again if
30558 it gets generated. */
30560 /* This will be of type IFUNCs have to be externally visible. */
30564 }
30566 #endif
30568 /* Returns true if decl is multi-versioned and DECL is the default function,
30569 that is it is not tagged with target specific optimization. */
30571 static bool
30573 {
30582 }
30584 /* Make a dispatcher declaration for the multi-versioned function DECL.
30585 Calls to DECL function will be replaced with calls to the dispatcher
30586 by the front-end. Returns the decl of the dispatcher function. */
30588 static tree
30590 {
30612 /* Find the default version and make it the first node. */
30614 /* Go to the beginning of the chain. */
30619 {
30624 }
30626 /* If there is no default node, just return NULL. */
30630 /* Make default info the first node. */
30632 {
30639 }
30643 #if defined (ASM_OUTPUT_TYPE_DIRECTIVE)
30645 {
30650 /* Right now, the dispatching is done via ifunc. */
30656 dispatcher_version_info
30661 /* Set the dispatcher for all the versions. */
30664 {
30667 }
30668 }
30669 else
30670 #endif
30671 {
30673 "multiversioning needs ifunc which is not supported "
30675 }
30678 }
30680 /* Makes a function attribute of the form NAME(ARG_NAME) and chains
30681 it to CHAIN. */
30683 static tree
30685 {
30686 tree attr_name;
30687 tree attr_arg_name;
30688 tree attr_args;
30689 tree attr;
30696 }
30698 /* Make the resolver function decl to dispatch the versions of
30699 a multi-versioned function, DEFAULT_DECL. Create an
30700 empty basic block in the resolver and store the pointer in
30701 EMPTY_BB. Return the decl of the resolver function. */
30703 static tree
30707 {
30712 /* IFUNC's have to be globally visible. So, if the default_decl is
30713 not, then the name of the IFUNC should be made unique. */
30717 /* Append the filename to the resolver function if the versions are
30718 not externally visible. This is because the resolver function has
30719 to be externally visible for the loader to find it. So, appending
30720 the filename will prevent conflicts with a resolver function from
30721 another module which is based on the same version name. */
30724 /* The resolver function should return a (void *). */
30735 /* IFUNC resolvers have to be externally visible. */
30739 /* Resolver is not external, body is generated. */
30749 {
30750 /* In this case, each translation unit with a call to this
30751 versioned function will put out a resolver. Ensure it
30752 is comdat to keep just one copy. */
30755 }
30756 /* Build result decl and add to function_decl. */
30772 /* Mark dispatch_decl as "ifunc" with resolver as resolver_name. */
30776 /* Create the alias for dispatch to resolver here. */
30777 /*cgraph_create_function_alias (dispatch_decl, decl);*/
30781 }
30783 /* Generate the dispatching code body to dispatch multi-versioned function
30784 DECL. The target hook is called to process the "target" attributes and
30785 provide the code to dispatch the right function at run-time. NODE points
30786 to the dispatcher decl whose body will be created. */
30788 static tree
30790 {
30791 tree resolver_decl;
30792 basic_block empty_bb;
30793 tree default_ver_decl;
30809 /* The first version in the chain corresponds to the default version. */
30812 /* node is going to be an alias, so remove the finalized bit. */
30826 {
30828 /* Check for virtual functions here again, as by this time it should
30829 have been determined if this function needs a vtable index or
30830 not. This happens for methods in derived classes that override
30831 virtual methods in base classes but are not explicitly marked as
30832 virtual. */
30837 }
30843 }
30844 /* This builds the processor_model struct type defined in
30845 libgcc/config/i386/cpuinfo.c */
30847 static tree
30849 {
30856 /* The first 3 fields are unsigned int. */
30858 {
30864 }
30866 /* The last field is an array of unsigned integers of size one. */
30877 }
30879 /* Returns a extern, comdat VAR_DECL of type TYPE and name NAME. */
30881 static tree
30883 {
30884 tree new_decl;
30887 VAR_DECL,
30889 type);
30902 }
30904 /* FNDECL is a __builtin_cpu_is or a __builtin_cpu_supports call that is folded
30905 into an integer defined in libgcc/config/i386/cpuinfo.c */
30907 static tree
30909 {
30915 /* This is the order of bit-fields in __processor_features in cpuinfo.c */
30916 enum processor_features
30917 {
30919 F_MMX,
30920 F_POPCNT,
30921 F_SSE,
30922 F_SSE2,
30923 F_SSE3,
30924 F_SSSE3,
30925 F_SSE4_1,
30926 F_SSE4_2,
30927 F_AVX,
30928 F_AVX2,
30929 F_MAX
30930 };
30932 /* These are the values for vendor types and cpu types and subtypes
30933 in cpuinfo.c. Cpu types and subtypes should be subtracted by
30934 the corresponding start value. */
30935 enum processor_model
30936 {
30938 M_AMD,
30939 M_CPU_TYPE_START,
30940 M_INTEL_ATOM,
30941 M_INTEL_CORE2,
30942 M_INTEL_COREI7,
30943 M_AMDFAM10H,
30944 M_AMDFAM15H,
30945 M_INTEL_SLM,
30946 M_CPU_SUBTYPE_START,
30947 M_INTEL_COREI7_NEHALEM,
30948 M_INTEL_COREI7_WESTMERE,
30949 M_INTEL_COREI7_SANDYBRIDGE,
30950 M_AMDFAM10H_BARCELONA,
30951 M_AMDFAM10H_SHANGHAI,
30952 M_AMDFAM10H_ISTANBUL,
30953 M_AMDFAM15H_BDVER1,
30954 M_AMDFAM15H_BDVER2,
30955 M_AMDFAM15H_BDVER3,
30956 M_AMDFAM15H_BDVER4
30957 };
30959 static struct _arch_names_table
30960 {
30963 }
30965 {
30984 };
30986 static struct _isa_names_table
30987 {
30990 }
30992 {
31004 };
31018 {
31019 /* *args must be a expr that can contain other EXPRS leading to a
31020 STRING_CST. */
31022 {
31025 }
31027 }
31032 {
31033 tree ref;
31034 tree field;
31035 tree final;
31038 unsigned int NUM_ARCH_NAMES
31047 {
31051 }
31056 /* CPU types are stored in the next field. */
31059 {
31062 }
31064 /* CPU subtypes are stored in the next field. */
31066 {
31069 }
31071 /* Get the appropriate field in __cpu_model. */
31075 /* Check the value. */
31079 }
31081 {
31082 tree ref;
31083 tree array_elt;
31084 tree field;
31085 tree final;
31088 unsigned int NUM_ISA_NAMES
31097 {
31101 }
31104 /* Get the last field, which is __cpu_features. */
31108 /* Get the appropriate field: __cpu_model.__cpu_features */
31112 /* Access the 0th element of __cpu_features array. */
31117 /* Return __cpu_model.__cpu_features[0] & field_val */
31121 }
31123 }
31125 static tree
31128 {
31130 {
31135 {
31138 }
31139 }
31141 #ifdef SUBTARGET_FOLD_BUILTIN
31143 #endif
31146 }
31148 /* Make builtins to detect cpu type and features supported. NAME is
31149 the builtin name, CODE is the builtin code, and FTYPE is the function
31150 type of the builtin. */
31152 static void
31155 {
31156 tree decl;
31157 tree type;
31165 }
31167 /* Make builtins to get CPU type and features supported. The created
31168 builtins are :
31170 __builtin_cpu_init (), to detect cpu type and features,
31171 __builtin_cpu_is ("<CPUNAME>"), to check if cpu is of type <CPUNAME>,
31172 __builtin_cpu_supports ("<FEATURE>"), to check if cpu supports <FEATURE>
31173 */
31175 static void
31177 {
31184 }
31186 /* Internal method for ix86_init_builtins. */
31188 static void
31190 {
31202 sysv_va_ref =
31205 fnvoid_va_end_ms =
31207 fnvoid_va_start_ms =
31209 fnvoid_va_end_sysv =
31211 fnvoid_va_start_sysv =
31213 NULL_TREE);
31214 fnvoid_va_copy_ms =
31216 NULL_TREE);
31217 fnvoid_va_copy_sysv =
31233 }
31235 static void
31237 {
31240 /* The __float80 type. */
31243 {
31244 /* The __float80 type. */
31249 }
31252 /* The __float128 type. */
31258 /* This macro is built by i386-builtin-types.awk. */
31259 DEFINE_BUILTIN_PRIMITIVE_TYPES;
31260 }
31262 static void
31264 {
31265 tree t;
31269 /* Builtins to get CPU type and features. */
31272 /* TFmode support builtins. */
31278 /* We will expand them to normal call if SSE isn't available since
31279 they are used by libgcc. */
31298 #ifdef SUBTARGET_INIT_BUILTINS
31299 SUBTARGET_INIT_BUILTINS;
31300 #endif
31301 }
31303 /* Return the ix86 builtin for CODE. */
31305 static tree
31307 {
31312 }
31314 /* Errors in the source file can cause expand_expr to return const0_rtx
31315 where we expect a vector. To avoid crashing, use one of the vector
31316 clear instructions. */
31317 static rtx
31319 {
31323 }
31325 /* Subroutine of ix86_expand_builtin to take care of binop insns. */
31327 static rtx
31329 {
31330 rtx pat;
31350 {
31354 }
31368 }
31370 /* Subroutine of ix86_expand_builtin to take care of 2-4 argument insns. */
31372 static rtx
31376 {
31377 rtx pat;
31385 rtx op;
31392 {
31472 }
31482 {
31489 {
31491 {
31494 {
31512 xop_rotl:
31514 {
31517 gcc_checking_assert
31519 }
31520 else
31521 {
31522 gcc_checking_assert
31533 }
31537 }
31538 }
31539 }
31540 else
31541 {
31542 non_constant:
31546 /* If we aren't optimizing, only allow one memory operand to be
31547 generated. */
31549 num_memory++;
31557 }
31561 }
31564 {
31575 else
31576 {
31582 }
31595 }
31602 }
31604 /* Subroutine of ix86_expand_args_builtin to take care of scalar unop
31605 insns with vec_merge. */
31607 static rtx
31609 rtx target)
31610 {
31611 rtx pat;
31638 }
31640 /* Subroutine of ix86_expand_builtin to take care of comparison insns. */
31642 static rtx
31645 {
31646 rtx pat;
31651 rtx op2;
31662 /* Swap operands if we have a comparison that isn't available in
31663 hardware. */
31665 {
31670 }
31690 }
31692 /* Subroutine of ix86_expand_builtin to take care of comi insns. */
31694 static rtx
31696 rtx target)
31697 {
31698 rtx pat;
31712 /* Swap operands if we have a comparison that isn't available in
31713 hardware. */
31715 {
31719 }
31740 const0_rtx)));
31743 }
31745 /* Subroutines of ix86_expand_args_builtin to take care of round insns. */
31747 static rtx
31749 rtx target)
31750 {
31751 rtx pat;
31776 }
31778 static rtx
31781 {
31782 rtx pat;
31787 rtx op2;
31814 }
31816 /* Subroutine of ix86_expand_builtin to take care of ptest insns. */
31818 static rtx
31820 rtx target)
31821 {
31822 rtx pat;
31855 const0_rtx)));
31858 }
31860 /* Subroutine of ix86_expand_builtin to take care of pcmpestr[im] insns. */
31862 static rtx
31865 {
31866 rtx pat;
31904 {
31907 }
31910 {
31919 }
31921 {
31930 }
31931 else
31932 {
31939 }
31947 {
31952 emit_insn
31956 FLAGS_REG),
31957 const0_rtx)));
31959 }
31960 else
31962 }
31965 /* Subroutine of ix86_expand_builtin to take care of pcmpistr[im] insns. */
31967 static rtx
31970 {
31971 rtx pat;
31999 {
32002 }
32005 {
32014 }
32016 {
32025 }
32026 else
32027 {
32034 }
32042 {
32047 emit_insn
32051 FLAGS_REG),
32052 const0_rtx)));
32054 }
32055 else
32057 }
32059 /* Subroutine of ix86_expand_builtin to take care of insns with
32060 variable number of operands. */
32062 static rtx
32065 {
32070 struct
32071 {
32072 rtx op;
32084 {
32363 }
32368 {
32371 }
32374 {
32381 }
32382 else
32383 {
32386 }
32389 {
32396 {
32397 /* SIMD shift insns take either an 8-bit immediate or
32398 register as count. But builtin functions take int as
32399 count. If count doesn't match, we put it in register. */
32401 {
32405 }
32406 }
32408 {
32411 {
32465 {
32468 {
32471 }
32477 }
32479 }
32480 }
32481 else
32482 {
32486 /* If we aren't optimizing, only allow one memory operand to
32487 be generated. */
32489 num_memory++;
32492 {
32495 }
32496 else
32497 {
32500 }
32501 }
32505 }
32508 {
32525 }
32532 }
32534 /* Subroutine of ix86_expand_builtin to take care of special insns
32535 with variable number of operands. */
32537 static rtx
32540 {
32541 tree arg;
32545 struct
32546 {
32547 rtx op;
32557 {
32591 {
32598 }
32614 /* Reserve memory operand for target. */
32617 {
32618 /* These builtins and instructions require the memory
32619 to be properly aligned. */
32635 }
32665 /* Reserve memory operand for target. */
32679 }
32684 {
32689 {
32692 /* target at this point has just BITS_PER_UNIT MEM_ALIGN
32693 on it. Try to improve it using get_pointer_alignment,
32694 and if the special builtin is one that requires strict
32695 mode alignment, also from it's GET_MODE_ALIGNMENT.
32696 Failure to do so could lead to ix86_legitimate_combined_insn
32697 rejecting all changes to such insns. */
32703 }
32704 else
32707 }
32708 else
32709 {
32716 }
32719 {
32728 {
32730 {
32736 else
32739 }
32740 }
32741 else
32742 {
32744 {
32745 /* This must be the memory operand. */
32748 /* op at this point has just BITS_PER_UNIT MEM_ALIGN
32749 on it. Try to improve it using get_pointer_alignment,
32750 and if the special builtin is one that requires strict
32751 mode alignment, also from it's GET_MODE_ALIGNMENT.
32752 Failure to do so could lead to ix86_legitimate_combined_insn
32753 rejecting all changes to such insns. */
32759 }
32760 else
32761 {
32762 /* This must be register. */
32769 }
32770 }
32774 }
32777 {
32792 }
32798 }
32800 /* Return the integer constant in ARG. Constrain it to be in the range
32801 of the subparts of VEC_TYPE; issue an error if not. */
32803 static int
32805 {
32810 {
32813 }
32816 }
32818 /* A subroutine of ix86_expand_builtin. These builtins are a wrapper around
32819 ix86_expand_vector_init. We DO have language-level syntax for this, in
32820 the form of (type){ init-list }. Except that since we can't place emms
32821 instructions from inside the compiler, we can't allow the use of MMX
32822 registers unless the user explicitly asks for it. So we do *not* define
32823 vec_set/vec_extract/vec_init patterns for MMX modes in mmx.md. Instead
32824 we have builtins invoked by mmintrin.h that gives us license to emit
32825 these sorts of instructions. */
32827 static rtx
32829 {
32839 {
32842 }
32849 }
32851 /* A subroutine of ix86_expand_builtin. These builtins are a wrapper around
32852 ix86_expand_vector_extract. They would be redundant (for non-MMX) if we
32853 had a language-level syntax for referencing vector elements. */
32855 static rtx
32857 {
32861 rtx op0;
32881 }
32883 /* A subroutine of ix86_expand_builtin. These builtins are a wrapper around
32884 ix86_expand_vector_set. They would be redundant (for non-MMX) if we had
32885 a language-level syntax for referencing vector elements. */
32887 static rtx
32889 {
32913 /* OP0 is the source of these builtin functions and shouldn't be
32914 modified. Create a copy, use it and return it as target. */
32920 }
32922 /* Expand an expression EXP that calls a built-in function,
32923 with result going to TARGET if that's convenient
32924 (and in mode MODE if that's convenient).
32925 SUBTARGET may be used as the target for computing one of EXP's operands.
32926 IGNORE is nonzero if the value is to be ignored. */
32928 static rtx
32931 {
32941 /* For CPU builtins that can be folded, fold first and expand the fold. */
32943 {
32945 {
32946 /* Make it call __cpu_indicator_init in libgcc. */
32952 }
32955 {
32960 }
32961 }
32963 /* Determine whether the builtin function is available under the current ISA.
32964 Originally the builtin was not created if it wasn't applicable to the
32965 current ISA based on the command line switches. With function specific
32966 options, we need to check in the context of the function making the call
32967 whether it is supported. */
32970 {
32976 else
32977 {
32981 }
32983 }
32986 {
32990 ? CODE_FOR_mmx_maskmovq
32992 /* Note the arg order is different from the operand order. */
33093 {
33094 REAL_VALUE_TYPE inf;
33095 rtx tmp;
33107 }
33117 {
33123 insn = (TARGET_64BIT
33127 }
33129 {
33130 insn = (TARGET_64BIT
33134 }
33135 else
33136 {
33139 insn = (TARGET_64BIT
33147 {
33150 }
33152 }
33155 {
33156 /* mode is VOIDmode if __builtin_rd* has been called
33157 without lhs. */
33161 }
33164 {
33169 }
33183 {
33208 }
33214 {
33217 }
33237 {
33240 }
33246 {
33250 {
33271 }
33276 }
33277 else
33278 {
33280 {
33292 }
33294 }
33324 ? CODE_FOR_tbm_bextri_si
33327 {
33330 }
33331 else
33332 {
33341 }
33357 rdrand_step:
33364 {
33367 }
33373 /* Emit SImode conditional move. */
33375 {
33378 }
33381 else
33388 const0_rtx);
33407 rdseed_step:
33414 {
33417 }
33423 const0_rtx);
33441 addcarryx:
33449 /* Generate CF from input operand. */
33454 /* Gen ADCX instruction to compute X+Y+CF. */
33469 /* Store the result. */
33472 {
33475 }
33478 /* Return current CF value. */
33570 gather_gen:
33581 /* Note the arg order is different from the operand order. */
33590 else
33595 {
33601 }
33604 {
33609 else
33619 }
33621 /* Force memory operand only with base register here. But we
33622 don't want to do it on memory operand for other builtin
33623 functions. */
33635 {
33638 }
33640 /* Optimize. If mask is known to have all high bits set,
33641 replace op0 with pc_rtx to signal that the instruction
33642 overwrites the whole destination and doesn't use its
33643 previous contents. */
33645 {
33647 {
33650 {
33654 negative++;
33657 negative++;
33658 }
33661 }
33663 {
33664 /* Recognize also when mask is like:
33665 __v2df src = _mm_setzero_pd ();
33666 __v2df mask = _mm_cmpeq_pd (src, src);
33667 or
33668 __v8sf src = _mm256_setzero_ps ();
33669 __v8sf mask = _mm256_cmp_ps (src, src, _CMP_EQ_OQ);
33670 as that is a cheaper way to load all ones into
33671 a register than having to load a constant from
33672 memory. */
33675 {
33680 {
33687 /* FALLTHRU */
33697 }
33698 }
33699 }
33700 }
33709 {
33716 else
33718 }
33719 else
33730 {
33733 }
33739 }
33752 {
33756 /* Emit a normal call if SSE isn't available. */
33760 }
33785 }
33787 /* This returns the target-specific builtin with code CODE if
33788 current_function_decl has visibility on this builtin, which is checked
33789 using isa flags. Returns NULL_TREE otherwise. */
33792 {
33796 /* Determine the isa flags of current_function_decl. */
33808 else
33810 }
33812 /* Returns a function decl for a vectorized version of the builtin function
33813 with builtin function code FN and the result vector type TYPE, or NULL_TREE
33814 if it is not available. */
33816 static tree
33818 tree type_in)
33819 {
33835 {
33838 {
33843 }
33848 {
33853 }
33859 /* The round insn does not trap on denormals. */
33864 {
33869 }
33875 /* The round insn does not trap on denormals. */
33880 {
33885 }
33891 /* The round insn does not trap on denormals. */
33896 {
33901 }
33907 /* The round insn does not trap on denormals. */
33912 {
33917 }
33924 {
33929 }
33936 {
33941 }
33947 /* The round insn does not trap on denormals. */
33952 {
33957 }
33963 /* The round insn does not trap on denormals. */
33968 {
33973 }
33978 {
33983 }
33988 {
33993 }
33997 /* The round insn does not trap on denormals. */
34002 {
34007 }
34011 /* The round insn does not trap on denormals. */
34016 {
34021 }
34025 /* The round insn does not trap on denormals. */
34030 {
34035 }
34039 /* The round insn does not trap on denormals. */
34044 {
34049 }
34053 /* The round insn does not trap on denormals. */
34058 {
34063 }
34067 /* The round insn does not trap on denormals. */
34072 {
34077 }
34081 /* The round insn does not trap on denormals. */
34086 {
34091 }
34095 /* The round insn does not trap on denormals. */
34100 {
34105 }
34109 /* The round insn does not trap on denormals. */
34114 {
34119 }
34123 /* The round insn does not trap on denormals. */
34128 {
34133 }
34138 {
34143 }
34148 {
34153 }
34158 }
34160 /* Dispatch to a handler for a vectorization library. */
34163 type_in);
34166 }
34168 /* Handler for an SVML-style interface to
34169 a library with vectorized intrinsics. */
34171 static tree
34173 {
34181 /* The SVML is suitable for unsafe math only. */
34194 {
34241 }
34250 {
34253 }
34254 else
34257 /* Convert to uppercase. */
34262 args;
34264 arity++;
34268 else
34271 /* Build a function declaration for the vectorized function. */
34280 }
34282 /* Handler for an ACML-style interface to
34283 a library with vectorized intrinsics. */
34285 static tree
34287 {
34295 /* The ACML is 64bits only and suitable for unsafe math only as
34296 it does not correctly support parts of IEEE with the required
34297 precision such as denormals. */
34311 {
34341 }
34348 args;
34350 arity++;
34354 else
34357 /* Build a function declaration for the vectorized function. */
34366 }
34368 /* Returns a decl of a function that implements gather load with
34369 memory type MEM_VECTYPE and index type INDEX_VECTYPE and SCALE.
34370 Return NULL_TREE if it is not available. */
34372 static tree
34375 {
34391 /* v*gather* insn sign extends index to pointer mode. */
34403 {
34430 }
34433 }
34435 /* Returns a code for a target-specific builtin that implements
34436 reciprocal of the function, or NULL_TREE if not available. */
34438 static tree
34441 {
34448 /* Machine dependent builtins. */
34450 {
34451 /* Vectorized version of sqrt to rsqrt conversion. */
34460 }
34461 else
34462 /* Normal builtins. */
34464 {
34465 /* Sqrt to rsqrt conversion. */
34471 }
34472 }
34473 \f
34474 /* Helper for avx_vpermilps256_operand et al. This is also used by
34475 the expansion functions to turn the parallel back into a mask.
34476 The return value is 0 for no match and the imm8+1 for a match. */
34478 int
34480 {
34488 /* Validate that all of the elements are constants, and not totally
34489 out of range. Copy the data into an integral array to make the
34490 subsequent checks easier. */
34492 {
34502 }
34505 {
34507 /* In the 256-bit DFmode case, we can only move elements within
34508 a 128-bit lane. */
34510 {
34514 }
34516 {
34520 }
34524 /* In the 256-bit SFmode case, we have full freedom of movement
34525 within the low 128-bit lane, but the high 128-bit lane must
34526 mirror the exact same pattern. */
34531 /* FALLTHRU */
34535 /* In the 128-bit case, we've full freedom in the placement of
34536 the elements from the source operand. */
34543 }
34545 /* Make sure success has a non-zero value by adding one. */
34547 }
34549 /* Helper for avx_vperm2f128_v4df_operand et al. This is also used by
34550 the expansion functions to turn the parallel back into a mask.
34551 The return value is 0 for no match and the imm8+1 for a match. */
34553 int
34555 {
34563 /* Validate that all of the elements are constants, and not totally
34564 out of range. Copy the data into an integral array to make the
34565 subsequent checks easier. */
34567 {
34577 }
34579 /* Validate that the halves of the permute are halves. */
34587 /* Reconstruct the mask. */
34589 {
34595 }
34597 /* Make sure success has a non-zero value by adding one. */
34599 }
34600 \f
34601 /* Store OPERAND to the memory after reload is completed. This means
34602 that we can't easily use assign_stack_local. */
34603 rtx
34605 {
34606 rtx result;
34610 {
34613 stack_pointer_rtx,
34616 }
34618 {
34620 {
34624 /* FALLTHRU */
34630 stack_pointer_rtx)),
34631 operand));
34635 }
34637 }
34638 else
34639 {
34641 {
34643 {
34650 stack_pointer_rtx)),
34656 stack_pointer_rtx)),
34658 }
34661 /* Store HImodes as SImodes. */
34663 /* FALLTHRU */
34669 stack_pointer_rtx)),
34670 operand));
34674 }
34676 }
34678 }
34680 /* Free operand from the memory. */
34681 void
34683 {
34685 {
34690 else
34692 /* Use LEA to deallocate stack space. In peephole2 it will be converted
34693 to pop or add instruction if registers are available. */
34697 }
34698 }
34700 /* Return a register priority for hard reg REGNO. */
34701 static int
34703 {
34704 /* ebp and r13 as the base always wants a displacement, r12 as the
34705 base always wants an index. So discourage their usage in an
34706 address. */
34711 /* New x86-64 int registers result in bigger code size. Discourage
34712 them. */
34715 /* New x86-64 SSE registers result in bigger code size. Discourage
34716 them. */
34719 /* Usage of AX register results in smaller code. Prefer it. */
34723 }
34725 /* Implement TARGET_PREFERRED_RELOAD_CLASS.
34727 Put float CONST_DOUBLE in the constant pool instead of fp regs.
34728 QImode must go into class Q_REGS.
34729 Narrow ALL_REGS to GENERAL_REGS. This supports allowing movsf and
34730 movdf to do mem-to-mem moves through integer regs. */
34732 static reg_class_t
34734 {
34737 /* We're only allowed to return a subclass of CLASS. Many of the
34738 following checks fail for NO_REGS, so eliminate that early. */
34742 /* All classes can load zeros. */
34746 /* Force constants into memory if we are loading a (nonzero) constant into
34747 an MMX, SSE or MASK register. This is because there are no MMX/SSE/MASK
34748 instructions to load from a constant. */
34755 /* Prefer SSE regs only, if we can use them for math. */
34759 /* Floating-point constants need more complex checks. */
34761 {
34762 /* General regs can load everything. */
34766 /* Floats can load 0 and 1 plus some others. Note that we eliminated
34767 zero above. We only want to wind up preferring 80387 registers if
34768 we plan on doing computation with them. */
34771 {
34772 /* Limit class to non-sse. */
34781 }
34784 }
34786 /* Generally when we see PLUS here, it's the function invariant
34787 (plus soft-fp const_int). Which can only be computed into general
34788 regs. */
34792 /* QImode constants are easy to load, but non-constant QImode data
34793 must go into Q_REGS. */
34795 {
34801 }
34804 }
34806 /* Discourage putting floating-point values in SSE registers unless
34807 SSE math is being used, and likewise for the 387 registers. */
34808 static reg_class_t
34810 {
34813 /* Restrict the output reload class to the register bank that we are doing
34814 math on. If we would like not to return a subset of CLASS, reject this
34815 alternative: if reload cannot do this, it will still use its choice. */
34821 {
34826 else
34828 }
34831 }
34833 static reg_class_t
34836 {
34837 /* Double-word spills from general registers to non-offsettable memory
34838 references (zero-extended addresses) require special handling. */
34844 {
34846 ? CODE_FOR_reload_noff_load
34848 /* Add the cost of moving address to a temporary. */
34852 }
34854 /* QImode spills from non-QI registers require
34855 intermediate register on 32bit targets. */
34861 {
34866 else
34872 /* Return Q_REGS if the operand is in memory. */
34875 }
34877 /* This condition handles corner case where an expression involving
34878 pointers gets vectorized. We're trying to use the address of a
34879 stack slot as a vector initializer.
34881 (set (reg:V2DI 74 [ vect_cst_.2 ])
34882 (vec_duplicate:V2DI (reg/f:DI 20 frame)))
34884 Eventually frame gets turned into sp+offset like this:
34886 (set (reg:V2DI 21 xmm0 [orig:74 vect_cst_.2 ] [74])
34887 (vec_duplicate:V2DI (plus:DI (reg/f:DI 7 sp)
34888 (const_int 392 [0x188]))))
34890 That later gets turned into:
34892 (set (reg:V2DI 21 xmm0 [orig:74 vect_cst_.2 ] [74])
34893 (vec_duplicate:V2DI (plus:DI (reg/f:DI 7 sp)
34894 (mem/u/c/i:DI (symbol_ref/u:DI ("*.LC0") [flags 0x2]) [0 S8 A64]))))
34896 We'll have the following reload recorded:
34898 Reload 0: reload_in (DI) =
34899 (plus:DI (reg/f:DI 7 sp)
34900 (mem/u/c/i:DI (symbol_ref/u:DI ("*.LC0") [flags 0x2]) [0 S8 A64]))
34901 reload_out (V2DI) = (reg:V2DI 21 xmm0 [orig:74 vect_cst_.2 ] [74])
34902 SSE_REGS, RELOAD_OTHER (opnum = 0), can't combine
34903 reload_in_reg: (plus:DI (reg/f:DI 7 sp) (const_int 392 [0x188]))
34904 reload_out_reg: (reg:V2DI 21 xmm0 [orig:74 vect_cst_.2 ] [74])
34905 reload_reg_rtx: (reg:V2DI 22 xmm1)
34907 Which isn't going to work since SSE instructions can't handle scalar
34908 additions. Returning GENERAL_REGS forces the addition into integer
34909 register and reload can handle subsequent reloads without problems. */
34917 }
34919 /* Implement TARGET_CLASS_LIKELY_SPILLED_P. */
34921 static bool
34923 {
34925 {
34940 }
34943 }
34945 /* If we are copying between general and FP registers, we need a memory
34946 location. The same is true for SSE and MMX registers.
34948 To optimize register_move_cost performance, allow inline variant.
34950 The macro can't work reliably when one of the CLASSES is class containing
34951 registers from multiple units (SSE, MMX, integer). We avoid this by never
34952 combining those units in single alternative in the machine description.
34953 Ensure that this constraint holds to avoid unexpected surprises.
34955 When STRICT is false, we are being called from REGISTER_MOVE_COST, so do not
34956 enforce these sanity checks. */
34961 {
34970 {
34973 }
34978 /* ??? This is a lie. We do have moves between mmx/general, and for
34979 mmx/sse2. But by saying we need secondary memory we discourage the
34980 register allocator from using the mmx registers unless needed. */
34985 {
34986 /* SSE1 doesn't have any direct moves from other classes. */
34990 /* If the target says that inter-unit moves are more expensive
34991 than moving through memory, then don't generate them. */
34996 /* Between SSE and general, we have moves no larger than word size. */
34999 }
35002 }
35004 bool
35007 {
35009 }
35011 /* Implement the TARGET_CLASS_MAX_NREGS hook.
35013 On the 80386, this is the size of MODE in words,
35014 except in the FP regs, where a single reg is always enough. */
35016 static unsigned char
35018 {
35020 {
35025 else
35027 }
35028 else
35029 {
35032 else
35034 }
35035 }
35037 /* Return true if the registers in CLASS cannot represent the change from
35038 modes FROM to TO. */
35040 bool
35043 {
35047 /* x87 registers can't do subreg at all, as all values are reformatted
35048 to extended precision. */
35053 {
35054 /* Vector registers do not support QI or HImode loads. If we don't
35055 disallow a change to these modes, reload will assume it's ok to
35056 drop the subreg from (subreg:SI (reg:HI 100) 0). This affects
35057 the vec_dupv4hi pattern. */
35061 /* Vector registers do not support subreg with nonzero offsets, which
35062 are otherwise valid for integer registers. Since we can't see
35063 whether we have a nonzero offset from here, prohibit all
35064 nonparadoxical subregs changing size. */
35067 }
35070 }
35072 /* Return the cost of moving data of mode M between a
35073 register and memory. A value of 2 is the default; this cost is
35074 relative to those in `REGISTER_MOVE_COST'.
35076 This function is used extensively by register_move_cost that is used to
35077 build tables at startup. Make it inline in this case.
35078 When IN is 2, return maximum of in and out move cost.
35080 If moving between registers and memory is more expensive than
35081 between two registers, you should define this macro to express the
35082 relative cost.
35084 Model also increased moving costs of QImode registers in non
35085 Q_REGS classes.
35086 */
35090 {
35093 {
35096 {
35108 }
35112 }
35114 {
35117 {
35129 }
35133 }
35135 {
35138 {
35147 }
35151 }
35153 {
35156 {
35162 else
35167 }
35168 else
35169 {
35174 else
35176 }
35183 /* Compute number of 32bit moves needed. TFmode is moved as XFmode. */
35190 else
35194 }
35195 }
35197 static int
35200 {
35202 }
35205 /* Return the cost of moving data from a register in class CLASS1 to
35206 one in class CLASS2.
35208 It is not required that the cost always equal 2 when FROM is the same as TO;
35209 on some machines it is expensive to move between registers if they are not
35210 general registers. */
35212 static int
35214 reg_class_t class2_i)
35215 {
35219 /* In case we require secondary memory, compute cost of the store followed
35220 by load. In order to avoid bad register allocation choices, we need
35221 for this to be *at least* as high as the symmetric MEMORY_MOVE_COST. */
35224 {
35230 /* In case of copying from general_purpose_register we may emit multiple
35231 stores followed by single load causing memory size mismatch stall.
35232 Count this as arbitrarily high cost of 20. */
35237 /* In the case of FP/MMX moves, the registers actually overlap, and we
35238 have to switch modes in order to treat them differently. */
35244 }
35246 /* Moves between SSE/MMX and integer unit are expensive. */
35250 /* ??? By keeping returned value relatively high, we limit the number
35251 of moves between integer and MMX/SSE registers for all targets.
35252 Additionally, high value prevents problem with x86_modes_tieable_p(),
35253 where integer modes in MMX/SSE registers are not tieable
35254 because of missing QImode and HImode moves to, from or between
35255 MMX/SSE registers. */
35265 }
35267 /* Return TRUE if hard register REGNO can hold a value of machine-mode
35268 MODE. */
35270 bool
35272 {
35273 /* Flags and only flags can only hold CCmode values. */
35285 {
35286 /* We implement the move patterns for all vector modes into and
35287 out of SSE registers, even when no operation instructions
35288 are available. */
35290 /* For AVX-512 we allow, regardless of regno:
35291 - XI mode
35292 - any of 512-bit wide vector mode
35293 - any scalar mode. */
35300 /* xmm16-xmm31 are only available for AVX-512. */
35304 /* OImode move is available only when AVX is enabled. */
35311 }
35313 {
35314 /* We implement the move patterns for 3DNOW modes even in MMX mode,
35315 so if the register is available at all, then we can move data of
35316 the given mode into or out of it. */
35319 }
35322 {
35323 /* Take care for QImode values - they can be in non-QI regs,
35324 but then they do cause partial register stalls. */
35329 /* LRA checks if the hard register is OK for the given mode.
35330 QImode values can live in non-QI regs, so we allow all
35331 registers here. */
35335 }
35336 /* We handle both integer and floats in the general purpose registers. */
35343 /* Lots of MMX code casts 8 byte vector modes to DImode. If we then go
35344 on to use that value in smaller contexts, this can easily force a
35345 pseudo to be allocated to GENERAL_REGS. Since this is no worse than
35346 supporting DImode, allow it. */
35351 }
35353 /* A subroutine of ix86_modes_tieable_p. Return true if MODE is a
35354 tieable integer mode. */
35356 static bool
35358 {
35360 {
35373 }
35374 }
35376 /* Return true if MODE1 is accessible in a register that can hold MODE2
35377 without copying. That is, all register classes that can hold MODE2
35378 can also hold MODE1. */
35380 bool
35382 {
35390 /* MODE2 being XFmode implies fp stack or general regs, which means we
35391 can tie any smaller floating point modes to it. Note that we do not
35392 tie this with TFmode. */
35396 /* MODE2 being DFmode implies fp stack, general or sse regs, which means
35397 that we can tie it with SFmode. */
35401 /* If MODE2 is only appropriate for an SSE register, then tie with
35402 any other mode acceptable to SSE registers. */
35412 /* If MODE2 is appropriate for an MMX register, then tie
35413 with any other mode acceptable to MMX registers. */
35420 }
35422 /* Return the cost of moving between two registers of mode MODE. */
35424 static int
35426 {
35430 {
35462 }
35464 /* Return the cost of moving between two registers of mode MODE,
35465 assuming that the move will be in pieces of at most UNITS bytes. */
35467 }
35469 /* Compute a (partial) cost for rtx X. Return true if the complete
35470 cost has been computed, and false if subexpressions should be
35471 scanned. In either case, *TOTAL contains the cost result. */
35473 static bool
35476 {
35483 {
35487 {
35490 }
35502 && (!TARGET_64BIT
35507 else
35513 {
35516 }
35518 {
35528 }
35530 {
35533 {
35542 }
35543 }
35544 /* Fall back to (MEM (SYMBOL_REF)), since that's where
35545 it'll probably end up. Add a penalty for size. */
35552 /* The zero extensions is often completely free on x86_64, so make
35553 it as cheap as possible. */
35559 else
35571 {
35574 {
35577 }
35580 {
35583 }
35584 }
35585 /* FALLTHRU */
35592 {
35593 /* ??? Should be SSE vector operation cost. */
35594 /* At least for published AMD latencies, this really is the same
35595 as the latency for a simple fpu operation like fabs. */
35596 /* V*QImode is emulated with 1-11 insns. */
35598 {
35601 {
35602 /* For XOP we use vpshab, which requires a broadcast of the
35603 value to the variable shift insn. For constants this
35604 means a V16Q const in mem; even when we can perform the
35605 shift with one insn set the cost to prefer paddb. */
35607 {
35612 }
35614 }
35618 }
35619 else
35621 }
35623 {
35625 {
35628 else
35630 }
35631 else
35632 {
35635 else
35637 }
35638 }
35639 else
35640 {
35645 {
35646 /* Return the cost after shift-and truncation. */
35649 }
35650 else
35652 }
35656 {
35657 rtx sub;
35662 /* ??? SSE scalar/vector cost should be used here. */
35663 /* ??? Bald assumption that fma has the same cost as fmul. */
35667 /* Negate in op0 or op2 is free: FMS, FNMA, FNMS. */
35678 }
35682 {
35683 /* ??? SSE scalar cost should be used here. */
35686 }
35688 {
35691 }
35693 {
35694 /* ??? SSE vector cost should be used here. */
35697 }
35699 {
35700 /* V*QImode is emulated with 7-13 insns. */
35702 {
35709 }
35710 /* V*DImode is emulated with 5-8 insns. */
35712 {
35715 else
35717 }
35718 /* Without sse4.1, we don't have PMULLD; it's emulated with 7
35719 insns, including two PMULUDQ. */
35722 else
35725 }
35726 else
35727 {
35732 {
35735 nbits++;
35736 }
35737 else
35738 /* This is arbitrary. */
35741 /* Compute costs correctly for widening multiplication. */
35745 {
35752 {
35756 else
35758 }
35762 }
35770 }
35777 /* ??? SSE cost should be used here. */
35782 /* ??? SSE vector cost should be used here. */
35784 else
35791 {
35796 {
35799 {
35807 }
35808 }
35811 {
35814 {
35820 }
35821 }
35823 {
35831 }
35832 }
35833 /* FALLTHRU */
35837 {
35838 /* ??? SSE cost should be used here. */
35841 }
35843 {
35846 }
35848 {
35849 /* ??? SSE vector cost should be used here. */
35852 }
35853 /* FALLTHRU */
35860 {
35867 }
35868 /* FALLTHRU */
35872 {
35873 /* ??? SSE cost should be used here. */
35876 }
35878 {
35881 }
35883 {
35884 /* ??? SSE vector cost should be used here. */
35887 }
35888 /* FALLTHRU */
35892 {
35893 /* ??? Should be SSE vector operation cost. */
35894 /* At least for published AMD latencies, this really is the same
35895 as the latency for a simple fpu operation like fabs. */
35897 }
35900 else
35909 {
35910 /* This kind of construct is implemented using test[bwl].
35911 Treat it as if we had an AND. */
35916 }
35926 /* ??? SSE cost should be used here. */
35931 /* ??? SSE vector cost should be used here. */
35937 /* ??? SSE cost should be used here. */
35942 /* ??? SSE vector cost should be used here. */
35955 /* ??? Assume all of these vector manipulation patterns are
35956 recognizable. In which case they all pretty much have the
35957 same cost. */
35963 }
35964 }
35966 #if TARGET_MACHO
35970 /* Given a symbol name and its associated stub, write out the
35971 definition of the stub. */
35973 void
35975 {
35980 /* For 64-bit we shouldn't get here. */
35983 /* Lose our funky encoding stuff so it doesn't contaminate the stub. */
36000 else
36007 {
36009 }
36011 {
36012 /* PIC stub. */
36013 /* 25-byte PIC stub using "CALL get_pc_thunk". */
36019 }
36020 else
36023 /* The AT&T-style ("self-modifying") stub is not lazily bound, thus
36024 it needs no stub-binding-helper. */
36031 {
36034 }
36035 else
36040 /* N.B. Keep the correspondence of these
36041 'symbol_ptr/symbol_ptr2/symbol_ptr3' sections consistent with the
36042 old-pic/new-pic/non-pic stubs; altering this will break
36043 compatibility with existing dylibs. */
36045 {
36046 /* 25-byte PIC stub using "CALL get_pc_thunk". */
36048 }
36049 else
36050 /* 16-byte -mdynamic-no-pic stub. */
36056 }
36059 /* Order the registers for register allocator. */
36061 void
36063 {
36067 /* First allocate the local general purpose registers. */
36072 /* Global general purpose registers. */
36077 /* x87 registers come first in case we are doing FP math
36078 using them. */
36083 /* SSE registers. */
36089 /* Extended REX SSE registers. */
36093 /* Mask register. */
36097 /* x87 registers. */
36105 /* Initialize the rest of array as we do not allocate some registers
36106 at all. */
36109 }
36111 /* Handle a "callee_pop_aggregate_return" attribute; arguments as
36112 in struct attribute_spec handler. */
36113 static tree
36115 tree args,
36118 {
36123 {
36125 name);
36128 }
36130 {
36132 name);
36135 }
36137 {
36138 tree cst;
36142 {
36145 name);
36147 }
36150 {
36153 name);
36155 }
36158 }
36161 }
36163 /* Handle a "ms_abi" or "sysv" attribute; arguments as in
36164 struct attribute_spec.handler. */
36165 static tree
36167 tree args ATTRIBUTE_UNUSED,
36169 {
36174 {
36176 name);
36179 }
36181 /* Can combine regparm with all attributes but fastcall. */
36183 {
36185 {
36187 }
36190 }
36192 {
36194 {
36196 }
36199 }
36202 }
36204 /* Handle a "ms_struct" or "gcc_struct" attribute; arguments as in
36205 struct attribute_spec.handler. */
36206 static tree
36208 tree args ATTRIBUTE_UNUSED,
36210 {
36213 {
36216 }
36217 else
36221 {
36223 name);
36225 }
36231 {
36233 name);
36235 }
36238 }
36240 static tree
36242 tree args ATTRIBUTE_UNUSED,
36244 {
36246 {
36248 name);
36250 }
36252 }
36254 static bool
36256 {
36260 }
36262 /* Returns an expression indicating where the this parameter is
36263 located on entry to the FUNCTION. */
36265 static rtx
36267 {
36273 {
36278 else
36281 }
36286 {
36293 {
36298 }
36299 else
36300 {
36303 {
36309 }
36310 }
36312 }
36316 }
36318 /* Determine whether x86_output_mi_thunk can succeed. */
36320 static bool
36322 HOST_WIDE_INT delta ATTRIBUTE_UNUSED,
36324 {
36325 /* 64-bit can handle anything. */
36329 /* For 32-bit, everything's fine if we have one free register. */
36333 /* Need a free register for vcall_offset. */
36337 /* Need a free register for GOT references. */
36341 /* Otherwise ok. */
36343 }
36345 /* Output the assembler code for a thunk function. THUNK_DECL is the
36346 declaration for the thunk function itself, FUNCTION is the decl for
36347 the target function. DELTA is an immediate constant offset to be
36348 added to THIS. If VCALL_OFFSET is nonzero, the word at
36349 *(*this + vcall_offset) should be added to THIS. */
36351 static void
36355 {
36362 else
36363 {
36369 else
36371 }
36375 /* If VCALL_OFFSET, we'll need THIS in a register. Might as well
36376 pull it in now and let DELTA benefit. */
36380 {
36381 /* Put the this parameter into %eax. */
36384 }
36385 else
36388 /* Adjust the this parameter by a fixed constant. */
36390 {
36395 {
36397 {
36401 }
36402 }
36405 }
36407 /* Adjust the this parameter by a value stored in the vtable. */
36409 {
36419 /* Adjust the this parameter. */
36423 {
36427 }
36434 vcall_mem));
36435 else
36437 }
36439 /* If necessary, drop THIS back to its stack slot. */
36445 {
36448 ;
36449 else
36450 {
36454 }
36455 }
36456 else
36457 {
36459 ;
36460 #if TARGET_MACHO
36462 {
36465 }
36467 else
36468 {
36475 }
36476 }
36478 /* Our sibling call patterns do not allow memories, because we have no
36479 predicate that can distinguish between frame and non-frame memory.
36480 For our purposes here, we can get away with (ab)using a jump pattern,
36481 because we're going to do no optimization. */
36484 else
36485 {
36491 {
36497 }
36503 }
36506 /* Emit just enough of rest_of_compilation to get the insns emitted.
36507 Note that use_thunk calls assemble_start_function et al. */
36513 }
36515 static void
36517 {
36519 #if TARGET_MACHO
36521 #endif
36528 }
36530 int
36532 {
36544 }
36546 /* Output assembler code to FILE to increment profiler label # LABELNO
36547 for profiling a function entry. */
36548 void
36550 {
36555 {
36556 #ifndef NO_PROFILE_COUNTERS
36558 #endif
36562 else
36564 }
36566 {
36567 #ifndef NO_PROFILE_COUNTERS
36570 #endif
36572 }
36573 else
36574 {
36575 #ifndef NO_PROFILE_COUNTERS
36578 #endif
36580 }
36581 }
36583 /* We don't have exact information about the insn sizes, but we may assume
36584 quite safely that we are informed about all 1 byte insns and memory
36585 address sizes. This is enough to eliminate unnecessary padding in
36586 99% of cases. */
36588 static int
36590 {
36596 /* Discard alignments we've emit and jump instructions. */
36601 /* Important case - calls are always 5 bytes.
36602 It is common to have many calls in the row. */
36611 /* For normal instructions we rely on get_attr_length being exact,
36612 with a few exceptions. */
36614 {
36618 {
36628 /* Otherwise trust get_attr_length. */
36630 }
36635 }
36638 else
36640 }
36642 #ifdef ASM_OUTPUT_MAX_SKIP_PAD
36644 /* AMD K8 core mispredicts jumps when there are more than 3 jumps in 16 byte
36645 window. */
36647 static void
36649 {
36654 /* Look for all minimal intervals of instructions containing 4 jumps.
36655 The intervals are bounded by START and INSN. NBYTES is the total
36656 size of instructions in the interval including INSN and not including
36657 START. When the NBYTES is smaller than 16 bytes, it is possible
36658 that the end of START and INSN ends up in the same 16byte page.
36660 The smallest offset in the page INSN can start is the case where START
36661 ends on the offset 0. Offset of INSN is then NBYTES - sizeof (INSN).
36662 We add p2align to 16byte window with maxskip 15 - NBYTES + sizeof (INSN).
36663 */
36665 {
36669 {
36675 /* If align > 3, only up to 16 - max_skip - 1 bytes can be
36676 already in the current 16 byte page, because otherwise
36677 ASM_OUTPUT_MAX_SKIP_ALIGN could skip max_skip or fewer
36678 bytes to reach 16 byte boundary. */
36686 {
36688 {
36692 else
36695 }
36696 }
36698 }
36706 njumps++;
36707 else
36711 {
36715 else
36718 }
36725 {
36732 }
36733 }
36734 }
36735 #endif
36737 /* AMD Athlon works faster
36738 when RET is not destination of conditional jump or directly preceded
36739 by other jump instruction. We avoid the penalty by inserting NOP just
36740 before the RET instructions in such cases. */
36741 static void
36743 {
36744 edge e;
36745 edge_iterator ei;
36748 {
36751 rtx prev;
36761 {
36762 edge e;
36763 edge_iterator ei;
36768 {
36771 }
36772 }
36774 {
36780 /* Empty functions get branch mispredict even when
36781 the jump destination is not visible to us. */
36784 }
36786 {
36789 }
36790 }
36791 }
36793 /* Count the minimum number of instructions in BB. Return 4 if the
36794 number of instructions >= 4. */
36796 static int
36798 {
36799 rtx insn;
36802 /* Count number of instructions in this block. Return 4 if the number
36803 of instructions >= 4. */
36805 {
36806 /* Only happen in exit blocks. */
36814 {
36815 insn_count++;
36818 }
36819 }
36822 }
36825 /* Count the minimum number of instructions in code path in BB.
36826 Return 4 if the number of instructions >= 4. */
36828 static int
36830 {
36831 edge e;
36832 edge_iterator ei;
36835 /* Only bother counting instructions along paths with no
36836 more than 2 basic blocks between entry and exit. Given
36837 that BB has an edge to exit, determine if a predecessor
36838 of BB has an edge from entry. If so, compute the number
36839 of instructions in the predecessor block. If there
36840 happen to be multiple such blocks, compute the minimum. */
36843 {
36844 edge prev_e;
36845 edge_iterator prev_ei;
36848 {
36851 }
36853 {
36855 {
36860 }
36861 }
36862 }
36868 }
36870 /* Pad short function to 4 instructions. */
36872 static void
36874 {
36875 edge e;
36876 edge_iterator ei;
36879 {
36882 {
36885 /* Pad short function. */
36887 {
36890 /* Find epilogue. */
36899 /* Two NOPs count as one instruction. */
36902 }
36903 }
36904 }
36905 }
36907 /* Fix up a Windows system unwinder issue. If an EH region falls through into
36908 the epilogue, the Windows system unwinder will apply epilogue logic and
36909 produce incorrect offsets. This can be avoided by adding a nop between
36910 the last insn that can throw and the first insn of the epilogue. */
36912 static void
36914 {
36915 edge e;
36916 edge_iterator ei;
36919 {
36922 /* Find the beginning of the epilogue. */
36929 /* We only care about preceding insns that can throw. */
36934 /* Do not separate calls from their debug information. */
36940 else
36944 }
36945 }
36947 /* Implement machine specific optimizations. We implement padding of returns
36948 for K8 CPUs and pass to avoid 4 jumps in the single 16 byte window. */
36949 static void
36951 {
36952 /* We are freeing block_for_insn in the toplev to keep compatibility
36953 with old MDEP_REORGS that are not CFG based. Recompute it now. */
36960 {
36965 #ifdef ASM_OUTPUT_MAX_SKIP_PAD
36968 #endif
36969 }
36970 }
36972 /* Return nonzero when QImode register that must be represented via REX prefix
36973 is used. */
36974 bool
36976 {
36984 }
36986 /* Return nonzero when P points to register encoded via REX prefix.
36987 Called via for_each_rtx. */
36988 static int
36990 {
36996 }
36998 /* Return true when INSN mentions register that must be encoded using REX
36999 prefix. */
37000 bool
37002 {
37005 }
37007 /* If profitable, negate (without causing overflow) integer constant
37008 of mode MODE at location LOC. Return true in this case. */
37009 bool
37011 {
37012 HOST_WIDE_INT val;
37018 {
37020 /* DImode x86_64 constants must fit in 32 bits. */
37033 }
37035 /* Avoid overflows. */
37041 /* Make things pretty and `subl $4,%eax' rather than `addl $-4,%eax'.
37042 Exceptions: -128 encodes smaller than 128, so swap sign and op. */
37045 {
37048 }
37051 }
37053 /* Generate an unsigned DImode/SImode to FP conversion. This is the same code
37054 optabs would emit if we didn't have TFmode patterns. */
37056 void
37058 {
37092 }
37093 \f
37094 /* AVX512F does support 64-byte integer vector operations,
37095 thus the longest vector we are faced with is V64QImode. */
37096 #define MAX_VECT_LEN 64
37098 struct expand_vec_perm_d
37099 {
37106 };
37112 /* Get a vector mode of the same size as the original but with elements
37113 twice as wide. This is only guaranteed to apply to integral vectors. */
37117 {
37118 /* ??? Rely on the ordering that genmodes.c gives to vectors. */
37123 }
37125 /* A subroutine of ix86_expand_vector_init. Store into TARGET a vector
37126 with all elements equal to VAR. Return true if successful. */
37128 static bool
37131 {
37135 {
37140 /* FALLTHRU */
37150 {
37153 /* First attempt to recognize VAL as-is. */
37157 {
37158 rtx seq;
37159 /* If that fails, force VAL into a register. */
37170 }
37171 }
37178 {
37179 rtx x;
37186 }
37196 {
37200 permute:
37208 /* Extend to SImode using a paradoxical SUBREG. */
37212 /* Insert the SImode value as low element of a V4SImode vector. */
37221 }
37229 widen:
37230 /* Replicate the value once into the next wider mode and recurse. */
37231 {
37233 rtx x;
37250 }
37254 {
37263 }
37268 }
37269 }
37271 /* A subroutine of ix86_expand_vector_init. Store into TARGET a vector
37272 whose ONE_VAR element is VAR, and other elements are zero. Return true
37273 if successful. */
37275 static bool
37278 {
37280 rtx new_target;
37285 {
37287 /* For SSE4.1, we normally use vector set. But if the second
37288 element is zero and inter-unit moves are OK, we use movq
37289 instead. */
37291 && !(TARGET_INTER_UNIT_MOVES_TO_VEC
37313 /* Use ix86_expand_vector_set in 64bit mode only. */
37318 }
37321 {
37326 }
37329 {
37334 /* FALLTHRU */
37349 else
37356 {
37357 /* We need to shuffle the value to the correct position, so
37358 create a new pseudo to store the intermediate result. */
37360 /* With SSE2, we can use the integer shuffle insns. */
37362 {
37364 const1_rtx,
37371 }
37373 /* Otherwise convert the intermediate result to V4SFmode and
37374 use the SSE1 shuffle instructions. */
37376 {
37379 }
37380 else
37384 const1_rtx,
37393 }
37408 widen:
37412 /* Zero extend the variable element to SImode and recurse. */
37425 }
37426 }
37428 /* A subroutine of ix86_expand_vector_init. Store into TARGET a vector
37429 consisting of the values in VALS. It is known that all elements
37430 except ONE_VAR are constants. Return true if successful. */
37432 static bool
37435 {
37445 {
37450 /* For the two element vectors, it's just as easy to use
37451 the general case. */
37455 /* Use ix86_expand_vector_set in 64bit mode only. */
37477 widen:
37478 /* There's no way to set one QImode entry easily. Combine
37479 the variable value with its adjacent constant value, and
37480 promote to an HImode set. */
37483 {
37488 }
37489 else
37490 {
37493 }
37507 }
37512 }
37514 /* A subroutine of ix86_expand_vector_init_general. Use vector
37515 concatenate to handle the most general case: all values variable,
37516 and none identical. */
37518 static void
37521 {
37524 rtvec v;
37528 {
37531 {
37564 }
37577 {
37592 }
37597 {
37608 }
37611 half:
37612 /* FIXME: We process inputs backward to help RA. PR 36222. */
37616 {
37621 }
37625 {
37628 {
37632 }
37635 }
37636 else
37642 }
37643 }
37645 /* A subroutine of ix86_expand_vector_init_general. Use vector
37646 interleave to handle the most general case: all values variable,
37647 and none identical. */
37649 static void
37652 {
37661 {
37682 }
37685 {
37686 /* Extend the odd elment to SImode using a paradoxical SUBREG. */
37690 /* Insert the SImode value as low element of V4SImode vector. */
37694 op0),
37696 const1_rtx);
37699 /* Cast the V4SImode vector back to a vector in orignal mode. */
37703 /* Load even elements into the second position. */
37707 const1_rtx));
37709 /* Cast vector to FIRST_IMODE vector. */
37712 }
37714 /* Interleave low FIRST_IMODE vectors. */
37716 {
37720 /* Cast FIRST_IMODE vector to SECOND_IMODE vector. */
37723 }
37725 /* Interleave low SECOND_IMODE vectors. */
37727 {
37730 {
37735 /* Cast the SECOND_IMODE vector to the THIRD_IMODE
37736 vector. */
37739 }
37742 /* FALLTHRU */
37749 /* Cast the SECOND_IMODE vector back to a vector on original
37750 mode. */
37757 }
37758 }
37760 /* A subroutine of ix86_expand_vector_init. Handle the most general case:
37761 all values variable, and none identical. */
37763 static void
37766 {
37772 {
37777 /* FALLTHRU */
37801 half:
37818 /* FALLTHRU */
37824 /* Don't use ix86_expand_vector_init_interleave if we can't
37825 move from GPR to SSE register directly. */
37841 }
37843 {
37855 {
37859 {
37865 else
37866 {
37871 }
37872 }
37875 }
37880 {
37886 }
37888 {
37894 }
37895 else
37897 }
37898 }
37900 /* Initialize vector TARGET via VALS. Suppress the use of MMX
37901 instructions unless MMX_OK is true. */
37903 void
37905 {
37912 rtx x;
37915 {
37925 }
37927 /* Constants are best loaded from the constant pool. */
37929 {
37932 }
37934 /* If all values are identical, broadcast the value. */
37940 /* Values where only one field is non-constant are best loaded from
37941 the pool and overwritten via move later. */
37943 {
37947 one_var))
37952 }
37955 }
37957 void
37959 {
37964 rtx tmp;
37966 = {
37973 };
37975 = {
37982 };
37986 {
37990 {
37995 else
37999 }
38011 else
38017 {
38020 /* For the two element vectors, we implement a VEC_CONCAT with
38021 the extraction of the other element. */
38028 else
38033 }
38042 {
38048 /* tmp = target = A B C D */
38050 /* target = A A B B */
38052 /* target = X A B B */
38054 /* target = A X C D */
38061 /* tmp = target = A B C D */
38063 /* tmp = X B C D */
38065 /* target = A B X D */
38072 /* tmp = target = A B C D */
38074 /* tmp = X B C D */
38076 /* target = A B X D */
38084 }
38092 /* Element 0 handled by vec_merge below. */
38094 {
38097 }
38100 {
38101 /* With SSE2, use integer shuffles to swap element 0 and ELT,
38102 store into element 0, then shuffle them back. */
38119 }
38120 else
38121 {
38122 /* For SSE1, we have to reuse the V4SF code. */
38126 }
38179 half:
38180 /* Compute offset. */
38186 /* Extract the half. */
38190 /* Put val in tmp at elt. */
38193 /* Put it back. */
38199 }
38202 {
38206 }
38207 else
38208 {
38217 }
38218 }
38220 void
38222 {
38226 rtx tmp;
38229 {
38234 /* FALLTHRU */
38247 {
38267 }
38279 {
38281 {
38301 }
38305 }
38306 else
38307 {
38308 /* For SSE1, we have to reuse the V4SF code. */
38312 }
38328 {
38332 else
38336 }
38341 {
38345 else
38349 }
38354 {
38358 else
38362 }
38367 {
38371 else
38375 }
38380 {
38384 else
38388 }
38393 {
38397 else
38401 }
38405 /* ??? Could extract the appropriate HImode element and shift. */
38408 }
38411 {
38415 /* Let the rtl optimizers know about the zero extension performed. */
38417 {
38420 }
38423 }
38424 else
38425 {
38432 }
38433 }
38435 /* Generate code to copy vector bits i / 2 ... i - 1 from vector SRC
38436 to bits 0 ... i / 2 - 1 of vector DEST, which has the same mode.
38437 The upper bits of DEST are undefined, though they shouldn't cause
38438 exceptions (some bits from src or all zeros are ok). */
38440 static void
38442 {
38445 {
38449 else
38467 else
38474 else
38482 {
38487 const1_rtx);
38488 }
38489 else
38490 {
38494 }
38498 }
38502 }
38504 /* Expand a vector reduction. FN is the binary pattern to reduce;
38505 DEST is the destination; IN is the input vector. */
38507 void
38509 {
38514 /* SSE4 has a special instruction for V8HImode UMIN reduction. */
38518 {
38521 }
38526 {
38531 else
38535 }
38536 }
38537 \f
38538 /* Target hook for scalar_mode_supported_p. */
38539 static bool
38541 {
38546 else
38548 }
38550 /* Implements target hook vector_mode_supported_p. */
38551 static bool
38553 {
38565 }
38567 /* Target hook for c_mode_for_suffix. */
38568 static enum machine_mode
38570 {
38577 }
38579 /* Worker function for TARGET_MD_ASM_CLOBBERS.
38581 We do this in the new i386 backend to maintain source compatibility
38582 with the old cc0-based compiler. */
38584 static tree
38586 tree inputs ATTRIBUTE_UNUSED,
38587 tree clobbers)
38588 {
38590 clobbers);
38592 clobbers);
38594 }
38596 /* Implements target vector targetm.asm.encode_section_info. */
38598 static void ATTRIBUTE_UNUSED
38600 {
38607 }
38609 /* Worker function for REVERSE_CONDITION. */
38611 enum rtx_code
38613 {
38617 }
38619 /* Output code to perform an x87 FP register move, from OPERANDS[1]
38620 to OPERANDS[0]. */
38624 {
38626 {
38629 {
38633 }
38637 }
38639 {
38643 else
38644 {
38645 /* There is no non-popping store to memory for XFmode.
38646 So if we need one, follow the store with a load. */
38649 else
38651 }
38652 }
38653 else
38655 }
38657 /* Output code to perform a conditional jump to LABEL, if C2 flag in
38658 FP status register is set. */
38660 void
38662 {
38664 rtx temp;
38669 {
38674 }
38675 else
38676 {
38681 }
38685 pc_rtx);
38690 }
38692 /* Output code to perform a log1p XFmode calculation. */
38695 {
38701 rtx test;
38707 XFmode));
38721 }
38723 /* Emit code for round calculation. */
38725 {
38732 rtx insn;
38737 {
38749 }
38752 {
38773 }
38781 /* round(a) = sgn(a) * floor(fabs(a) + 0.5) */
38783 /* scratch = fxam(op1) */
38786 UNSPEC_FXAM)));
38787 /* e1 = fabs(op1) */
38790 /* e2 = e1 + 0.5 */
38795 /* res = floor(e2) */
38797 {
38802 }
38803 else
38807 {
38810 {
38817 UNSPEC_TRUNC_NOOP)));
38818 }
38834 }
38836 /* flags = signbit(a) */
38839 /* if (flags) then res = -res */
38843 pc_rtx);
38854 }
38856 /* Output code to perform a Newton-Rhapson approximation of a single precision
38857 floating point divide [http://en.wikipedia.org/wiki/N-th_root_algorithm]. */
38860 {
38868 /* a / b = a * ((rcp(b) + rcp(b)) - (b * rcp(b) * rcp (b))) */
38872 /* x0 = rcp(b) estimate */
38875 UNSPEC_RCP)));
38876 /* e0 = x0 * b */
38880 /* e0 = x0 * e0 */
38884 /* e1 = x0 + x0 */
38888 /* x1 = e1 - e0 */
38892 /* res = a * x1 */
38895 }
38897 /* Output code to perform a Newton-Rhapson approximation of a
38898 single precision floating point [reciprocal] square root. */
38902 {
38904 REAL_VALUE_TYPE r;
38919 {
38922 }
38924 /* sqrt(a) = -0.5 * a * rsqrtss(a) * (a * rsqrtss(a) * rsqrtss(a) - 3.0)
38925 rsqrt(a) = -0.5 * rsqrtss(a) * (a * rsqrtss(a) * rsqrtss(a) - 3.0) */
38929 /* x0 = rsqrt(a) estimate */
38932 UNSPEC_RSQRT)));
38934 /* If (a == 0.0) Filter out infinity to prevent NaN for sqrt(0.0). */
38936 {
38948 }
38950 /* e0 = x0 * a */
38953 /* e1 = e0 * x0 */
38957 /* e2 = e1 - 3. */
38964 /* e3 = -.5 * x0 */
38967 else
38968 /* e3 = -.5 * e0 */
38971 /* ret = e2 * e3 */
38974 }
38976 #ifdef TARGET_SOLARIS
38977 /* Solaris implementation of TARGET_ASM_NAMED_SECTION. */
38979 static void
38981 tree decl)
38982 {
38983 /* With Binutils 2.15, the "@unwind" marker must be specified on
38984 every occurrence of the ".eh_frame" section, not just the first
38985 one. */
38988 {
38992 }
38994 #ifndef USE_GAS
38996 {
38999 }
39000 #endif
39003 }
39006 /* Return the mangling of TYPE if it is an extended fundamental type. */
39010 {
39018 {
39020 /* __float128 is "g". */
39023 /* "long double" or __float80 is "e". */
39027 }
39028 }
39030 /* For 32-bit code we can save PIC register setup by using
39031 __stack_chk_fail_local hidden function instead of calling
39032 __stack_chk_fail directly. 64-bit code doesn't need to setup any PIC
39033 register, so it is better to call __stack_chk_fail directly. */
39035 static tree ATTRIBUTE_UNUSED
39037 {
39038 return TARGET_64BIT
39041 }
39043 /* Select a format to encode pointers in exception handling data. CODE
39044 is 0 for data, 1 for code labels, 2 for function pointers. GLOBAL is
39045 true if the symbol may be affected by dynamic relocations.
39047 ??? All x86 object file formats are capable of representing this.
39048 After all, the relocation needed is the same as for the call insn.
39049 Whether or not a particular assembler allows us to enter such, I
39050 guess we'll have to see. */
39051 int
39053 {
39055 {
39062 }
39067 }
39068 \f
39069 /* Expand copysign from SIGN to the positive value ABS_VALUE
39070 storing in RESULT. If MASK is non-null, it shall be a mask to mask out
39071 the sign-bit. */
39072 static void
39074 {
39078 {
39085 else
39090 {
39091 /* We need to generate a scalar mode mask in this case. */
39096 }
39097 }
39098 else
39104 }
39106 /* Expand fabs (OP0) and return a new rtx that holds the result. The
39107 mask for masking out the sign-bit is stored in *SMASK, if that is
39108 non-null. */
39109 static rtx
39111 {
39120 else
39124 {
39125 /* We need to generate a scalar mode mask in this case. */
39130 }
39138 }
39140 /* Expands a comparison of OP0 with OP1 using comparison code CODE,
39141 swapping the operands if SWAP_OPERANDS is true. The expanded
39142 code is a forward jump to a newly created label in case the
39143 comparison is true. The generated label rtx is returned. */
39144 static rtx
39147 {
39152 {
39156 }
39169 }
39171 /* Expand a mask generating SSE comparison instruction comparing OP0 with OP1
39172 using comparison code CODE. Operands are swapped for the comparison if
39173 SWAP_OPERANDS is true. Returns a rtx for the generated mask. */
39174 static rtx
39177 {
39183 {
39187 }
39194 }
39196 /* Generate and return a rtx of mode MODE for 2**n where n is the number
39197 of bits of the mantissa of MODE, which must be one of DFmode or SFmode. */
39198 static rtx
39200 {
39201 REAL_VALUE_TYPE TWO52r;
39202 rtx TWO52;
39209 }
39211 /* Expand SSE sequence for computing lround from OP1 storing
39212 into OP0. */
39213 void
39215 {
39216 /* C code for the stuff we're doing below:
39217 tmp = op1 + copysign (nextafter (0.5, 0.0), op1)
39218 return (long)tmp;
39219 */
39223 rtx adj;
39225 /* load nextafter (0.5, 0.0) */
39230 /* adj = copysign (0.5, op1) */
39234 /* adj = op1 + adj */
39237 /* op0 = (imode)adj */
39239 }
39241 /* Expand SSE2 sequence for computing lround from OPERAND1 storing
39242 into OPERAND0. */
39243 void
39245 {
39246 /* C code for the stuff we're doing below (for do_floor):
39247 xi = (long)op1;
39248 xi -= (double)xi > op1 ? 1 : 0;
39249 return xi;
39250 */
39255 /* reg = (long)op1 */
39259 /* freg = (double)reg */
39263 /* ireg = (freg > op1) ? ireg - 1 : ireg */
39274 }
39276 /* Expand rint (IEEE round to nearest) rounding OPERAND1 and storing the
39277 result in OPERAND0. */
39278 void
39280 {
39281 /* C code for the stuff we're doing below:
39282 xa = fabs (operand1);
39283 if (!isless (xa, 2**52))
39284 return operand1;
39285 xa = xa + 2**52 - 2**52;
39286 return copysign (xa, operand1);
39287 */
39294 /* xa = abs (operand1) */
39297 /* if (!isless (xa, TWO52)) goto label; */
39310 }
39312 /* Expand SSE2 sequence for computing floor or ceil from OPERAND1 storing
39313 into OPERAND0. */
39314 void
39316 {
39317 /* C code for the stuff we expand below.
39318 double xa = fabs (x), x2;
39319 if (!isless (xa, TWO52))
39320 return x;
39321 xa = xa + TWO52 - TWO52;
39322 x2 = copysign (xa, x);
39323 Compensate. Floor:
39324 if (x2 > x)
39325 x2 -= 1;
39326 Compensate. Ceil:
39327 if (x2 < x)
39328 x2 -= -1;
39329 return x2;
39330 */
39336 /* Temporary for holding the result, initialized to the input
39337 operand to ease control flow. */
39341 /* xa = abs (operand1) */
39344 /* if (!isless (xa, TWO52)) goto label; */
39347 /* xa = xa + TWO52 - TWO52; */
39351 /* xa = copysign (xa, operand1) */
39354 /* generate 1.0 or -1.0 */
39359 /* Compensate: xa = xa - (xa > operand1 ? 1 : 0) */
39363 /* We always need to subtract here to preserve signed zero. */
39372 }
39374 /* Expand SSE2 sequence for computing floor or ceil from OPERAND1 storing
39375 into OPERAND0. */
39376 void
39378 {
39379 /* C code for the stuff we expand below.
39380 double xa = fabs (x), x2;
39381 if (!isless (xa, TWO52))
39382 return x;
39383 x2 = (double)(long)x;
39384 Compensate. Floor:
39385 if (x2 > x)
39386 x2 -= 1;
39387 Compensate. Ceil:
39388 if (x2 < x)
39389 x2 += 1;
39390 if (HONOR_SIGNED_ZEROS (mode))
39391 return copysign (x2, x);
39392 return x2;
39393 */
39399 /* Temporary for holding the result, initialized to the input
39400 operand to ease control flow. */
39404 /* xa = abs (operand1) */
39407 /* if (!isless (xa, TWO52)) goto label; */
39410 /* xa = (double)(long)x */
39415 /* generate 1.0 */
39418 /* Compensate: xa = xa - (xa > operand1 ? 1 : 0) */
39433 }
39435 /* Expand SSE sequence for computing round from OPERAND1 storing
39436 into OPERAND0. Sequence that works without relying on DImode truncation
39437 via cvttsd2siq that is only available on 64bit targets. */
39438 void
39440 {
39441 /* C code for the stuff we expand below.
39442 double xa = fabs (x), xa2, x2;
39443 if (!isless (xa, TWO52))
39444 return x;
39445 Using the absolute value and copying back sign makes
39446 -0.0 -> -0.0 correct.
39447 xa2 = xa + TWO52 - TWO52;
39448 Compensate.
39449 dxa = xa2 - xa;
39450 if (dxa <= -0.5)
39451 xa2 += 1;
39452 else if (dxa > 0.5)
39453 xa2 -= 1;
39454 x2 = copysign (xa2, x);
39455 return x2;
39456 */
39462 /* Temporary for holding the result, initialized to the input
39463 operand to ease control flow. */
39467 /* xa = abs (operand1) */
39470 /* if (!isless (xa, TWO52)) goto label; */
39473 /* xa2 = xa + TWO52 - TWO52; */
39477 /* dxa = xa2 - xa; */
39480 /* generate 0.5, 1.0 and -0.5 */
39486 /* Compensate. */
39488 /* xa2 = xa2 - (dxa > 0.5 ? 1 : 0) */
39493 /* xa2 = xa2 + (dxa <= -0.5 ? 1 : 0) */
39499 /* res = copysign (xa2, operand1) */
39506 }
39508 /* Expand SSE sequence for computing trunc from OPERAND1 storing
39509 into OPERAND0. */
39510 void
39512 {
39513 /* C code for SSE variant we expand below.
39514 double xa = fabs (x), x2;
39515 if (!isless (xa, TWO52))
39516 return x;
39517 x2 = (double)(long)x;
39518 if (HONOR_SIGNED_ZEROS (mode))
39519 return copysign (x2, x);
39520 return x2;
39521 */
39527 /* Temporary for holding the result, initialized to the input
39528 operand to ease control flow. */
39532 /* xa = abs (operand1) */
39535 /* if (!isless (xa, TWO52)) goto label; */
39538 /* x = (double)(long)x */
39550 }
39552 /* Expand SSE sequence for computing trunc from OPERAND1 storing
39553 into OPERAND0. */
39554 void
39556 {
39560 /* C code for SSE variant we expand below.
39561 double xa = fabs (x), x2;
39562 if (!isless (xa, TWO52))
39563 return x;
39564 xa2 = xa + TWO52 - TWO52;
39565 Compensate:
39566 if (xa2 > xa)
39567 xa2 -= 1.0;
39568 x2 = copysign (xa2, x);
39569 return x2;
39570 */
39574 /* Temporary for holding the result, initialized to the input
39575 operand to ease control flow. */
39579 /* xa = abs (operand1) */
39582 /* if (!isless (xa, TWO52)) goto label; */
39585 /* res = xa + TWO52 - TWO52; */
39590 /* generate 1.0 */
39593 /* Compensate: res = xa2 - (res > xa ? 1 : 0) */
39601 /* res = copysign (res, operand1) */
39608 }
39610 /* Expand SSE sequence for computing round from OPERAND1 storing
39611 into OPERAND0. */
39612 void
39614 {
39615 /* C code for the stuff we're doing below:
39616 double xa = fabs (x);
39617 if (!isless (xa, TWO52))
39618 return x;
39619 xa = (double)(long)(xa + nextafter (0.5, 0.0));
39620 return copysign (xa, x);
39621 */
39627 /* Temporary for holding the result, initialized to the input
39628 operand to ease control flow. */
39636 /* load nextafter (0.5, 0.0) */
39641 /* xa = xa + 0.5 */
39645 /* xa = (double)(int64_t)xa */
39650 /* res = copysign (xa, operand1) */
39657 }
39659 /* Expand SSE sequence for computing round
39660 from OP1 storing into OP0 using sse4 round insn. */
39661 void
39663 {
39672 {
39683 }
39685 /* round (a) = trunc (a + copysign (0.5, a)) */
39687 /* load nextafter (0.5, 0.0) */
39693 /* e1 = copysign (0.5, op1) */
39697 /* e2 = op1 + e1 */
39700 /* res = trunc (e2) */
39705 }
39706 \f
39708 /* Table of valid machine attributes. */
39710 {
39711 /* { name, min_len, max_len, decl_req, type_req, fn_type_req, handler,
39712 affects_type_identity } */
39713 /* Stdcall attribute says callee is responsible for popping arguments
39714 if they are not variable. */
39717 /* Fastcall attribute says callee is responsible for popping arguments
39718 if they are not variable. */
39721 /* Thiscall attribute says callee is responsible for popping arguments
39722 if they are not variable. */
39725 /* Cdecl attribute says the callee is a normal C declaration */
39728 /* Regparm attribute specifies how many integer arguments are to be
39729 passed in registers. */
39732 /* Sseregparm attribute says we are using x86_64 calling conventions
39733 for FP arguments. */
39736 /* The transactional memory builtins are implicitly regparm or fastcall
39737 depending on the ABI. Override the generic do-nothing attribute that
39738 these builtins were declared with. */
39741 /* force_align_arg_pointer says this function realigns the stack at entry. */
39744 #if TARGET_DLLIMPORT_DECL_ATTRIBUTES
39749 #endif
39754 #ifdef SUBTARGET_ATTRIBUTE_TABLE
39755 SUBTARGET_ATTRIBUTE_TABLE,
39756 #endif
39757 /* ms_abi and sysv_abi calling convention function attributes. */
39764 /* End element. */
39766 };
39768 /* Implement targetm.vectorize.builtin_vectorization_cost. */
39769 static int
39771 tree vectype,
39773 {
39777 {
39822 }
39823 }
39825 /* A cached (set (nil) (vselect (vconcat (nil) (nil)) (parallel [])))
39826 insn, so that expand_vselect{,_vconcat} doesn't have to create a fresh
39827 insn every time. */
39831 /* Initialize vselect_insn. */
39833 static void
39835 {
39837 rtx x;
39848 }
39850 /* Construct (set target (vec_select op0 (parallel perm))) and
39851 return true if that's a valid instruction in the active ISA. */
39853 static bool
39856 {
39882 }
39884 /* Similar, but generate a vec_concat from op0 and op1 as well. */
39886 static bool
39890 {
39892 rtx x;
39907 }
39909 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to implement D
39910 in terms of blendp[sd] / pblendw / pblendvb / vpblendd. */
39912 static bool
39914 {
39923 ;
39925 ;
39927 ;
39928 else
39931 /* This is a blend, not a permute. Elements must stay in their
39932 respective lanes. */
39934 {
39938 }
39943 /* ??? Without SSE4.1, we could implement this with and/andn/or. This
39944 decision should be extracted elsewhere, so that we only try that
39945 sequence once all budget==3 options have been tried. */
39952 {
39976 /* See if bytes move in pairs so we can use pblendw with
39977 an immediate argument, rather than pblendvb with a vector
39978 argument. */
39981 {
39982 use_pblendvb:
39986 finish_pblendvb:
39992 else
39997 }
40002 /* FALLTHRU */
40004 do_subreg:
40011 /* See if bytes move in pairs. If not, vpblendvb must be used. */
40015 /* See if bytes move in quadruplets. If yes, vpblendd
40016 with immediate can be used. */
40021 {
40022 /* See if bytes move the same in both lanes. If yes,
40023 vpblendw with immediate can be used. */
40028 /* Use vpblendw. */
40033 }
40035 /* Use vpblendd. */
40042 /* See if words move in pairs. If yes, vpblendd can be used. */
40047 {
40048 /* See if words move the same in both lanes. If not,
40049 vpblendvb must be used. */
40052 {
40053 /* Use vpblendvb. */
40063 }
40065 /* Use vpblendw. */
40069 }
40071 /* Use vpblendd. */
40078 /* Use vpblendd. */
40086 }
40088 /* This matches five different patterns with the different modes. */
40096 }
40098 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to implement D
40099 in terms of the variable form of vpermilps.
40101 Note that we will have already failed the immediate input vpermilps,
40102 which requires that the high and low part shuffle be identical; the
40103 variable form doesn't require that. */
40105 static bool
40107 {
40114 /* We can only permute within the 128-bit lane. */
40116 {
40120 }
40126 {
40129 /* Within each 128-bit lane, the elements of op0 are numbered
40130 from 0 and the elements of op1 are numbered from 4. */
40137 }
40144 }
40146 /* Return true if permutation D can be performed as VMODE permutation
40147 instead. */
40149 static bool
40151 {
40166 else
40172 }
40174 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to implement D
40175 in terms of pshufb, vpperm, vpermq, vpermd, vpermps or vperm2i128. */
40177 static bool
40179 {
40188 {
40190 {
40193 {
40197 /* Use vperm2i128 insn. The pattern uses
40198 V4DImode instead of V2TImode. */
40211 }
40213 }
40214 }
40215 else
40216 {
40218 {
40221 }
40223 {
40227 /* V4DImode should be already handled through
40228 expand_vselect by vpermq instruction. */
40235 {
40236 /* First see if vpermq can be used for
40237 V8SImode/V16HImode/V32QImode. */
40239 {
40247 {
40251 }
40253 }
40255 /* Next see if vpermd can be used. */
40258 }
40259 /* Or if vpermps can be used. */
40264 {
40265 /* vpshufb only works intra lanes, it is not
40266 possible to shuffle bytes in between the lanes. */
40270 }
40271 }
40272 else
40274 }
40282 else
40283 {
40289 else
40293 {
40297 }
40298 }
40309 {
40316 else
40318 }
40319 else
40320 {
40323 }
40328 }
40330 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to instantiate D
40331 in a single instruction. */
40333 static bool
40335 {
40339 /* Check plain VEC_SELECT first, because AVX has instructions that could
40340 match both SEL and SEL+CONCAT, but the plain SEL will allow a memory
40341 input where SEL+CONCAT may not. */
40343 {
40349 {
40355 }
40358 {
40362 }
40364 {
40365 /* Use vpbroadcast{b,w,d}. */
40368 {
40387 /* For other modes prefer other shuffles this function creates. */
40389 }
40391 {
40395 }
40396 }
40401 /* There are plenty of patterns in sse.md that are written for
40402 SEL+CONCAT and are not replicated for a single op. Perhaps
40403 that should be changed, to avoid the nastiness here. */
40405 /* Recognize interleave style patterns, which means incrementing
40406 every other permutation operand. */
40408 {
40411 }
40416 /* Recognize shufps, which means adding {0, 0, nelt, nelt}. */
40418 {
40420 {
40425 }
40430 }
40431 }
40433 /* Finally, try the fully general two operand permute. */
40438 /* Recognize interleave style patterns with reversed operands. */
40440 {
40442 {
40446 else
40449 }
40454 }
40456 /* Try the SSE4.1 blend variable merge instructions. */
40460 /* Try one of the AVX vpermil variable permutations. */
40464 /* Try the SSSE3 pshufb or XOP vpperm or AVX2 vperm2i128,
40465 vpshufb, vpermd, vpermps or vpermq variable permutation. */
40470 }
40472 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to implement D
40473 in terms of a pair of pshuflw + pshufhw instructions. */
40475 static bool
40477 {
40485 /* The two permutations only operate in 64-bit lanes. */
40496 /* Emit the pshuflw. */
40503 /* Emit the pshufhw. */
40511 }
40513 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to simplify
40514 the permutation using the SSSE3 palignr instruction. This succeeds
40515 when all of the elements in PERM fit within one vector and we merely
40516 need to shift them down so that a single vector permutation has a
40517 chance to succeed. */
40519 static bool
40521 {
40528 /* Even with AVX, palignr only operates on 128-bit vectors. */
40534 {
40540 }
40544 /* Given that we have SSSE3, we know we'll be able to implement the
40545 single operand permutation after the palignr with pshufb. */
40560 {
40565 }
40567 /* Test for the degenerate case where the alignment by itself
40568 produces the desired permutation. */
40570 {
40573 }
40579 }
40583 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to simplify
40584 a two vector permutation into a single vector permutation by using
40585 an interleave operation to merge the vectors. */
40587 static bool
40589 {
40594 rtx seq;
40598 {
40601 }
40603 {
40606 /* For 32-byte modes allow even d->one_operand_p.
40607 The lack of cross-lane shuffling in some instructions
40608 might prevent a single insn shuffle. */
40611 /* If expand_vec_perm_interleave3 can expand this into
40612 a 3 insn sequence, give up and let it be expanded as
40613 3 insn sequence. While that is one insn longer,
40614 it doesn't need a memory operand and in the common
40615 case that both interleave low and high permutations
40616 with the same operands are adjacent needs 4 insns
40617 for both after CSE. */
40620 }
40621 else
40624 /* Examine from whence the elements come. */
40633 {
40636 /* Split the two input vectors into 4 halves. */
40642 /* If the elements from the low halves use interleave low, and similarly
40643 for interleave high. If the elements are from mis-matched halves, we
40644 can use shufps for V4SF/V4SI or do a DImode shuffle. */
40646 {
40647 /* punpckl* */
40649 {
40654 }
40657 }
40659 {
40660 /* punpckh* */
40662 {
40667 }
40670 }
40672 {
40673 /* shufps */
40675 {
40680 }
40682 {
40683 /* shufpd */
40688 }
40689 }
40691 {
40692 /* shufps */
40694 {
40699 }
40701 {
40702 /* shufpd */
40707 }
40708 }
40709 else
40711 }
40712 else
40713 {
40718 /* Split the two input vectors into 8 quarters. */
40724 {
40727 }
40730 {
40734 }
40736 {
40739 }
40742 {
40744 {
40745 /* Attempt to increase the likelihood that dfinal
40746 shuffle will be intra-lane. */
40750 }
40752 /* vperm2f128 or vperm2i128. */
40754 {
40759 }
40764 {
40768 {
40771 }
40772 }
40773 }
40778 {
40779 /* vpunpckl* */
40781 {
40790 }
40791 }
40794 {
40795 /* vpunpckh* */
40797 {
40806 }
40807 }
40808 else
40810 }
40812 /* Use the remapping array set up above to move the elements from their
40813 swizzled locations into their final destinations. */
40816 {
40819 /* If same_halves is true, both halves of the remapped vector are the
40820 same. Avoid cross-lane accesses if possible. */
40822 {
40825 }
40826 else
40828 }
40834 /* Test if the final remap can be done with a single insn. For V4SFmode or
40835 V4SImode this *will* succeed. For V8HImode or V16QImode it may not. */
40848 {
40851 }
40858 }
40860 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to simplify
40861 a single vector cross-lane permutation into vpermq followed
40862 by any of the single insn permutations. */
40864 static bool
40866 {
40880 {
40883 }
40886 {
40891 }
40904 {
40911 }
40918 {
40923 ;
40926 else
40928 }
40937 }
40939 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to expand
40940 a vector permutation using two instructions, vperm2f128 resp.
40941 vperm2i128 followed by any single in-lane permutation. */
40943 static bool
40945 {
40959 /* ((perm << 2)|perm) & 0x33 is the vperm2[fi]128
40960 immediate. For perm < 16 the second permutation uses
40961 d->op0 as first operand, for perm >= 16 it uses d->op1
40962 as first operand. The second operand is the result of
40963 vperm2[fi]128. */
40965 {
40966 /* Ignore permutations which do not move anything cross-lane. */
40968 {
40969 /* The second shuffle for e.g. V4DFmode has
40970 0123 and ABCD operands.
40971 Ignore AB23, as 23 is already in the second lane
40972 of the first operand. */
40974 /* And 01CD, as 01 is in the first lane of the first
40975 operand. */
40977 /* And 4567, as then the vperm2[fi]128 doesn't change
40978 anything on the original 4567 second operand. */
40980 }
40981 else
40982 {
40983 /* The second shuffle for e.g. V4DFmode has
40984 4567 and ABCD operands.
40985 Ignore AB67, as 67 is already in the second lane
40986 of the first operand. */
40988 /* And 45CD, as 45 is in the first lane of the first
40989 operand. */
40991 /* And 0123, as then the vperm2[fi]128 doesn't change
40992 anything on the original 0123 first operand. */
40994 }
40997 {
41003 else
41005 }
41008 {
41012 }
41013 else
41017 {
41021 /* Found a usable second shuffle. dfirst will be
41022 vperm2f128 on d->op0 and d->op1. */
41033 /* And dsecond is some single insn shuffle, taking
41034 d->op0 and result of vperm2f128 (if perm < 16) or
41035 d->op1 and result of vperm2f128 (otherwise). */
41044 }
41046 /* For one operand, the only useful vperm2f128 permutation is 0x10. */
41049 }
41052 }
41054 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to simplify
41055 a two vector permutation using 2 intra-lane interleave insns
41056 and cross-lane shuffle for 32-byte vectors. */
41058 static bool
41060 {
41067 ;
41069 ;
41070 else
41085 {
41089 else
41095 else
41101 else
41107 else
41113 else
41119 else
41124 }
41128 }
41130 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to implement
41131 a single vector permutation using a single intra-lane vector
41132 permutation, vperm2f128 swapping the lanes and vblend* insn blending
41133 the non-swapped and swapped vectors together. */
41135 static bool
41137 {
41140 rtx seq;
41145 || TARGET_AVX2
41154 {
41161 }
41196 }
41198 /* A subroutine of ix86_expand_vec_perm_builtin_1. Implement a V4DF
41199 permutation using two vperm2f128, followed by a vshufpd insn blending
41200 the two vectors together. */
41202 static bool
41204 {
41247 }
41249 /* A subroutine of expand_vec_perm_even_odd_1. Implement the double-word
41250 permutation with two pshufb insns and an ior. We should have already
41251 failed all two instruction sequences. */
41253 static bool
41255 {
41266 /* Generate two permutation masks. If the required element is within
41267 the given vector it is shuffled into the proper lane. If the required
41268 element is in the other vector, force a zero into the lane by setting
41269 bit 7 in the permutation mask. */
41272 {
41279 {
41282 }
41283 }
41307 }
41309 /* Implement arbitrary permutation of one V32QImode and V16QImode operand
41310 with two vpshufb insns, vpermq and vpor. We should have already failed
41311 all two or three instruction sequences. */
41313 static bool
41315 {
41330 /* Generate two permutation masks. If the required element is within
41331 the same lane, it is shuffled in. If the required element from the
41332 other lane, force a zero by setting bit 7 in the permutation mask.
41333 In the other mask the mask has non-negative elements if element
41334 is requested from the other lane, but also moved to the other lane,
41335 so that the result of vpshufb can have the two V2TImode halves
41336 swapped. */
41339 {
41344 {
41347 }
41348 }
41357 /* Swap the 128-byte lanes of h into hp. */
41361 const1_rtx));
41378 }
41380 /* A subroutine of expand_vec_perm_even_odd_1. Implement extract-even
41381 and extract-odd permutations of two V32QImode and V16QImode operand
41382 with two vpshufb insns, vpor and vpermq. We should have already
41383 failed all two or three instruction sequences. */
41385 static bool
41387 {
41406 /* Generate two permutation masks. In the first permutation mask
41407 the first quarter will contain indexes for the first half
41408 of the op0, the second quarter will contain bit 7 set, third quarter
41409 will contain indexes for the second half of the op0 and the
41410 last quarter bit 7 set. In the second permutation mask
41411 the first quarter will contain bit 7 set, the second quarter
41412 indexes for the first half of the op1, the third quarter bit 7 set
41413 and last quarter indexes for the second half of the op1.
41414 I.e. the first mask e.g. for V32QImode extract even will be:
41415 0, 2, ..., 0xe, -128, ..., -128, 0, 2, ..., 0xe, -128, ..., -128
41416 (all values masked with 0xf except for -128) and second mask
41417 for extract even will be
41418 -128, ..., -128, 0, 2, ..., 0xe, -128, ..., -128, 0, 2, ..., 0xe. */
41421 {
41427 {
41430 }
41431 }
41450 /* Permute the V4DImode quarters using { 0, 2, 1, 3 } permutation. */
41458 }
41460 /* A subroutine of ix86_expand_vec_perm_builtin_1. Implement extract-even
41461 and extract-odd permutations. */
41463 static bool
41465 {
41469 {
41474 /* Shuffle the lanes around into { 0 1 4 5 } and { 2 3 6 7 }. */
41478 /* Now an unpck[lh]pd will produce the result required. */
41481 else
41487 {
41494 /* Shuffle within the 128-bit lanes to produce:
41495 { 0 2 8 a 4 6 c e } | { 1 3 9 b 5 7 d f }. */
41499 /* Shuffle the lanes around to produce:
41500 { 4 6 c e 0 2 8 a } and { 5 7 d f 1 3 9 b }. */
41504 /* Shuffle within the 128-bit lanes to produce:
41505 { 0 2 4 6 4 6 0 2 } | { 1 3 5 7 5 7 1 3 }. */
41508 /* Shuffle within the 128-bit lanes to produce:
41509 { 8 a c e c e 8 a } | { 9 b d f d f 9 b }. */
41512 /* Shuffle the lanes around to produce:
41513 { 0 2 4 6 8 a c e } | { 1 3 5 7 9 b d f }. */
41516 }
41523 /* These are always directly implementable by expand_vec_perm_1. */
41529 else
41530 {
41531 /* We need 2*log2(N)-1 operations to achieve odd/even
41532 with interleave. */
41541 else
41544 }
41550 else
41551 {
41563 else
41566 }
41575 {
41582 {
41587 }
41589 }
41594 /* Shuffle the lanes around into { 0 1 4 5 } and { 2 3 6 7 }. */
41598 /* Now an vpunpck[lh]qdq will produce the result required. */
41601 else
41608 {
41615 {
41620 }
41622 }
41630 /* Shuffle the lanes around into
41631 { 0 1 2 3 8 9 a b } and { 4 5 6 7 c d e f }. */
41639 /* Swap the 2nd and 3rd position in each lane into
41640 { 0 2 1 3 8 a 9 b } and { 4 6 5 7 c e d f }. */
41646 /* Now an vpunpck[lh]qdq will produce
41647 { 0 2 4 6 8 a c e } resp. { 1 3 5 7 9 b d f }. */
41651 else
41660 }
41663 }
41665 /* A subroutine of ix86_expand_vec_perm_builtin_1. Pattern match
41666 extract-even and extract-odd permutations. */
41668 static bool
41670 {
41682 }
41684 /* A subroutine of ix86_expand_vec_perm_builtin_1. Implement broadcast
41685 permutations. We assume that expand_vec_perm_1 has already failed. */
41687 static bool
41689 {
41697 {
41700 /* These are special-cased in sse.md so that we can optionally
41701 use the vbroadcast instruction. They expand to two insns
41702 if the input happens to be in a register. */
41709 /* These are always implementable using standard shuffle patterns. */
41714 /* These can be implemented via interleave. We save one insn by
41715 stopping once we have promoted to V4SImode and then use pshufd. */
41716 do
41717 {
41718 rtx dest;
41724 {
41728 }
41735 }
41750 /* For AVX2 broadcasts of the first element vpbroadcast* or
41751 vpermq should be used by expand_vec_perm_1. */
41757 }
41758 }
41760 /* A subroutine of ix86_expand_vec_perm_builtin_1. Pattern match
41761 broadcast permutations. */
41763 static bool
41765 {
41777 }
41779 /* Implement arbitrary permutation of two V32QImode and V16QImode operands
41780 with 4 vpshufb insns, 2 vpermq and 3 vpor. We should have already failed
41781 all the shorter instruction sequences. */
41783 static bool
41785 {
41801 /* Generate 4 permutation masks. If the required element is within
41802 the same lane, it is shuffled in. If the required element from the
41803 other lane, force a zero by setting bit 7 in the permutation mask.
41804 In the other mask the mask has non-negative elements if element
41805 is requested from the other lane, but also moved to the other lane,
41806 so that the result of vpshufb can have the two V2TImode halves
41807 swapped. */
41810 {
41815 }
41821 {
41829 }
41832 {
41834 {
41837 }
41844 }
41846 /* Swap the 128-byte lanes of h[X]. */
41848 {
41854 const1_rtx));
41856 }
41859 {
41861 {
41864 }
41870 }
41873 {
41875 {
41879 }
41882 }
41892 }
41894 /* The guts of ix86_expand_vec_perm_const, also used by the ok hook.
41895 With all of the interface bits taken care of, perform the expansion
41896 in D and return true on success. */
41898 static bool
41900 {
41901 /* Try a single instruction expansion. */
41905 /* Try sequences of two instructions. */
41925 /* Try sequences of three instructions. */
41939 /* Try sequences of four instructions. */
41947 /* ??? Look for narrow permutations whose element orderings would
41948 allow the promotion to a wider mode. */
41950 /* ??? Look for sequences of interleave or a wider permute that place
41951 the data into the correct lanes for a half-vector shuffle like
41952 pshuf[lh]w or vpermilps. */
41954 /* ??? Look for sequences of interleave that produce the desired results.
41955 The combinatorics of punpck[lh] get pretty ugly... */
41960 /* Even longer sequences. */
41965 }
41967 /* If a permutation only uses one operand, make it clear. Returns true
41968 if the permutation references both operands. */
41970 static bool
41972 {
41980 {
41986 {
41989 }
41990 /* The elements of PERM do not suggest that only the first operand
41991 is used, but both operands are identical. Allow easier matching
41992 of the permutation by folding the permutation into the single
41993 input vector. */
41994 /* FALLTHRU */
42005 }
42008 }
42010 bool
42012 {
42017 rtx sel;
42034 {
42039 }
42046 /* If the selector says both arguments are needed, but the operands are the
42047 same, the above tried to expand with one_operand_p and flattened selector.
42048 If that didn't work, retry without one_operand_p; we succeeded with that
42049 during testing. */
42051 {
42055 }
42058 }
42060 /* Implement targetm.vectorize.vec_perm_const_ok. */
42062 static bool
42065 {
42074 /* Given sufficient ISA support we can just return true here
42075 for selected vector modes. */
42077 {
42078 /* All implementable with a single vpperm insn. */
42081 /* All implementable with 2 pshufb + 1 ior. */
42084 /* All implementable with shufpd or unpck[lh]pd. */
42087 }
42089 /* Extract the values from the vector CST into the permutation
42090 array in D. */
42093 {
42097 }
42099 /* For all elements from second vector, fold the elements to first. */
42104 /* Check whether the mask can be applied to the vector type. */
42107 /* Implementable with shufps or pshufd. */
42111 /* Otherwise we have to go through the motions and see if we can
42112 figure out how to generate the requested permutation. */
42123 }
42125 void
42127 {
42142 /* We'll either be able to implement the permutation directly... */
42146 /* ... or we use the special-case patterns. */
42148 }
42150 static void
42152 {
42167 {
42170 }
42172 /* Note that for AVX this isn't one instruction. */
42175 }
42178 /* Expand a vector operation CODE for a V*QImode in terms of the
42179 same operation on V*HImode. */
42181 void
42183 {
42195 {
42208 }
42212 {
42214 /* Unpack data such that we've got a source byte in each low byte of
42215 each word. We don't care what goes into the high byte of each word.
42216 Rather than trying to get zero in there, most convenient is to let
42217 it be a copy of the low byte. */
42222 /* FALLTHRU */
42234 /* FALLTHRU */
42244 }
42246 /* Perform the operation. */
42253 /* Merge the data back into the right place. */
42263 {
42264 /* For SSE2, we used an full interleave, so the desired
42265 results are in the even elements. */
42268 }
42269 else
42270 {
42271 /* For AVX, the interleave used above was not cross-lane. So the
42272 extraction is evens but with the second and third quarter swapped.
42273 Happily, that is even one insn shorter than even extraction. */
42276 }
42283 }
42285 /* Helper function of ix86_expand_mul_widen_evenodd. Return true
42286 if op is CONST_VECTOR with all odd elements equal to their
42287 preceding element. */
42289 static bool
42291 {
42301 }
42303 void
42306 {
42309 rtx x;
42317 /* We only play even/odd games with vectors of SImode. */
42320 /* If we're looking for the odd results, shift those members down to
42321 the even slots. For some cpus this is faster than a PSHUFD. */
42323 {
42324 /* For XOP use vpmacsdqh, but only for smult, as it is only
42325 signed. */
42327 {
42331 }
42342 }
42345 {
42348 else
42350 }
42355 else
42356 {
42359 /* The easiest way to implement this without PMULDQ is to go through
42360 the motions as if we are performing a full 64-bit multiply. With
42361 the exception that we need to do less shuffling of the elements. */
42363 /* Compute the sign-extension, aka highparts, of the two operands. */
42369 /* Multiply LO(A) * HI(B), and vice-versa. */
42375 /* Multiply LO(A) * LO(B). */
42379 /* Combine and shift the highparts into place. */
42384 /* Combine high and low parts. */
42387 }
42389 }
42391 void
42394 {
42400 {
42405 {
42406 /* With XOP, we have pmacsdqh, aka mul_widen_odd. In this case,
42407 shuffle the elements once so that all elements are in the right
42408 place for immediate use: { A C B D }. */
42413 }
42414 else
42415 {
42416 /* Put the elements into place for the multiply. */
42420 }
42425 /* Shuffle the elements between the lanes. After this we
42426 have { A B E F | C D G H } for each operand. */
42436 /* Shuffle the elements within the lanes. After this we
42437 have { A A B B | C C D D } or { E E F F | G G H H }. */
42475 }
42476 }
42478 void
42480 {
42490 /* Move the results in element 2 down to element 1; we don't care
42491 what goes in elements 2 and 3. Then we can merge the parts
42492 back together with an interleave.
42494 Note that two other sequences were tried:
42495 (1) Use interleaves at the start instead of psrldq, which allows
42496 us to use a single shufps to merge things back at the end.
42497 (2) Use shufps here to combine the two vectors, then pshufd to
42498 put the elements in the correct order.
42499 In both cases the cost of the reformatting stall was too high
42500 and the overall sequence slower. */
42511 }
42513 void
42515 {
42520 {
42521 /* op1: A,B,C,D, op2: E,F,G,H */
42530 /* t1: B,A,D,C */
42537 /* t2: (B*E),(A*F),(D*G),(C*H) */
42540 /* t3: (B*E)+(A*F), (D*G)+(C*H) */
42543 /* t4: ((B*E)+(A*F))<<32, ((D*G)+(C*H))<<32 */
42546 /* op0: (((B*E)+(A*F))<<32)+(B*F), (((D*G)+(C*H))<<32)+(D*H) */
42548 }
42549 else
42550 {
42555 {
42558 }
42560 {
42563 }
42564 else
42568 /* Multiply low parts. */
42572 /* Shift input vectors right 32 bits so we can multiply high parts. */
42577 /* Multiply high parts by low parts. */
42583 /* Combine and shift the highparts back. */
42587 /* Combine high and low parts. */
42589 }
42593 }
42595 /* Calculate integer abs() using only SSE2 instructions. */
42597 void
42599 {
42604 {
42605 /* For 32-bit signed integer X, the best way to calculate the absolute
42606 value of X is (((signed) X >> (W-1)) ^ X) - ((signed) X >> (W-1)). */
42618 /* For 16-bit signed integer X, the best way to calculate the absolute
42619 value of X is max (X, -X), as SSE2 provides the PMAXSW insn. */
42627 /* For 8-bit signed integer X, the best way to calculate the absolute
42628 value of X is min ((unsigned char) X, (unsigned char) (-X)),
42629 as SSE2 provides the PMINUB insn. */
42639 }
42643 }
42645 /* Expand an insert into a vector register through pinsr insn.
42646 Return true if successful. */
42648 bool
42650 {
42658 {
42661 }
42667 {
42672 {
42679 {
42711 }
42725 }
42729 }
42730 }
42731 \f
42732 /* This function returns the calling abi specific va_list type node.
42733 It returns the FNDECL specific va_list type. */
42735 static tree
42737 {
42744 else
42746 }
42748 /* Returns the canonical va_list type specified by TYPE. If there
42749 is no valid TYPE provided, it return NULL_TREE. */
42751 static tree
42753 {
42756 /* Resolve references and pointers to va_list type. */
42765 {
42770 {
42771 /* If va_list is an array type, the argument may have decayed
42772 to a pointer type, e.g. by being passed to another function.
42773 In that case, unwrap both types so that we can compare the
42774 underlying records. */
42777 {
42780 }
42781 }
42788 {
42789 /* If va_list is an array type, the argument may have decayed
42790 to a pointer type, e.g. by being passed to another function.
42791 In that case, unwrap both types so that we can compare the
42792 underlying records. */
42795 {
42798 }
42799 }
42806 {
42807 /* If va_list is an array type, the argument may have decayed
42808 to a pointer type, e.g. by being passed to another function.
42809 In that case, unwrap both types so that we can compare the
42810 underlying records. */
42813 {
42816 }
42817 }
42821 }
42823 }
42825 /* Iterate through the target-specific builtin types for va_list.
42826 IDX denotes the iterator, *PTREE is set to the result type of
42827 the va_list builtin, and *PNAME to its internal type.
42828 Returns zero if there is no element for this index, otherwise
42829 IDX should be increased upon the next call.
42830 Note, do not iterate a base builtin's name like __builtin_va_list.
42831 Used from c_common_nodes_and_builtins. */
42833 static int
42835 {
42837 {
42839 {
42852 }
42853 }
42856 }
42858 #undef TARGET_SCHED_DISPATCH
42859 #define TARGET_SCHED_DISPATCH has_dispatch
42860 #undef TARGET_SCHED_DISPATCH_DO
42861 #define TARGET_SCHED_DISPATCH_DO do_dispatch
42862 #undef TARGET_SCHED_REASSOCIATION_WIDTH
42863 #define TARGET_SCHED_REASSOCIATION_WIDTH ix86_reassociation_width
42864 #undef TARGET_SCHED_REORDER
42865 #define TARGET_SCHED_REORDER ix86_sched_reorder
42866 #undef TARGET_SCHED_ADJUST_PRIORITY
42867 #define TARGET_SCHED_ADJUST_PRIORITY ix86_adjust_priority
42868 #undef TARGET_SCHED_DEPENDENCIES_EVALUATION_HOOK
42869 #define TARGET_SCHED_DEPENDENCIES_EVALUATION_HOOK \
42870 ix86_dependencies_evaluation_hook
42872 /* The size of the dispatch window is the total number of bytes of
42873 object code allowed in a window. */
42874 #define DISPATCH_WINDOW_SIZE 16
42876 /* Number of dispatch windows considered for scheduling. */
42877 #define MAX_DISPATCH_WINDOWS 3
42879 /* Maximum number of instructions in a window. */
42880 #define MAX_INSN 4
42882 /* Maximum number of immediate operands in a window. */
42883 #define MAX_IMM 4
42885 /* Maximum number of immediate bits allowed in a window. */
42886 #define MAX_IMM_SIZE 128
42888 /* Maximum number of 32 bit immediates allowed in a window. */
42889 #define MAX_IMM_32 4
42891 /* Maximum number of 64 bit immediates allowed in a window. */
42892 #define MAX_IMM_64 2
42894 /* Maximum total of loads or prefetches allowed in a window. */
42895 #define MAX_LOAD 2
42897 /* Maximum total of stores allowed in a window. */
42898 #define MAX_STORE 1
42900 #undef BIG
42901 #define BIG 100
42904 /* Dispatch groups. Istructions that affect the mix in a dispatch window. */
42907 disp_load,
42908 disp_store,
42909 disp_load_store,
42910 disp_prefetch,
42911 disp_imm,
42912 disp_imm_32,
42913 disp_imm_64,
42914 disp_branch,
42915 disp_cmp,
42916 disp_jcc,
42917 disp_last
42918 };
42920 /* Number of allowable groups in a dispatch window. It is an array
42921 indexed by dispatch_group enum. 100 is used as a big number,
42922 because the number of these kind of operations does not have any
42923 effect in dispatch window, but we need them for other reasons in
42924 the table. */
42927 };
42933 };
42935 /* Instruction path. */
42941 last_path
42942 };
42944 /* sched_insn_info defines a window to the instructions scheduled in
42945 the basic block. It contains a pointer to the insn_info table and
42946 the instruction scheduled.
42948 Windows are allocated for each basic block and are linked
42949 together. */
42951 rtx insn;
42958 /* Linked list of dispatch windows. This is a two way list of
42959 dispatch windows of a basic block. It contains information about
42960 the number of uops in the window and the total number of
42961 instructions and of bytes in the object code for this dispatch
42962 window. */
42980 /* Immediate valuse used in an insn. */
42982 {
42991 /* Get dispatch group of insn. */
42993 static enum dispatch_group
42995 {
43011 }
43013 /* Return true if insn is a compare instruction. */
43015 static bool
43017 {
43025 }
43027 /* Return true if a dispatch violation encountered. */
43029 static bool
43031 {
43035 }
43037 /* Return true if insn is a branch instruction. */
43039 static bool
43041 {
43043 }
43045 /* Return true if insn is a prefetch instruction. */
43047 static bool
43049 {
43051 }
43053 /* This function initializes a dispatch window and the list container holding a
43054 pointer to the window. */
43056 static void
43058 {
43064 else
43082 {
43088 }
43090 }
43092 /* This function allocates and initializes a dispatch window and the
43093 list container holding a pointer to the window. */
43097 {
43102 }
43104 /* This routine initializes the dispatch scheduling information. It
43105 initiates building dispatch scheduler tables and constructs the
43106 first dispatch window. */
43108 static void
43110 {
43111 /* Allocate a dispatch list and a window. */
43116 }
43118 /* This function returns true if a branch is detected. End of a basic block
43119 does not have to be a branch, but here we assume only branches end a
43120 window. */
43122 static bool
43124 {
43126 }
43128 /* This function is called when the end of a window processing is reached. */
43130 static void
43132 {
43135 {
43140 }
43142 }
43144 /* Allocates a new dispatch window and adds it to WINDOW_LIST.
43145 WINDOW_NUM is either 0 or 1. A maximum of two windows are generated
43146 for 48 bytes of instructions. Note that these windows are not dispatch
43147 windows that their sizes are DISPATCH_WINDOW_SIZE. */
43151 {
43153 {
43158 }
43164 }
43166 /* Increment the number of immediate operands of an instruction. */
43168 static int
43170 {
43175 {
43182 else
43193 {
43196 }
43201 }
43204 }
43206 /* Compute number of immediate operands of an instruction. */
43208 static void
43210 {
43213 }
43215 /* Return total size of immediate operands of an instruction along with number
43216 of corresponding immediate-operands. It initializes its parameters to zero
43217 befor calling FIND_CONSTANT.
43218 INSN is the input instruction. IMM is the total of immediates.
43219 IMM32 is the number of 32 bit immediates. IMM64 is the number of 64
43220 bit immediates. */
43222 static int
43224 {
43232 }
43234 /* This function indicates if an operand of an instruction is an
43235 immediate. */
43237 static bool
43239 {
43248 }
43250 /* Return single or double path for instructions. */
43252 static enum insn_path
43254 {
43264 }
43266 /* Return insn dispatch group. */
43268 static enum dispatch_group
43270 {
43288 }
43290 /* Count number of GROUP restricted instructions in a dispatch
43291 window WINDOW_LIST. */
43293 static int
43295 {
43306 {
43325 }
43338 }
43340 /* This function returns true if insn satisfies dispatch rules on the
43341 last window scheduled. */
43343 static bool
43345 {
43353 /* Make disp_cmp and disp_jcc get scheduled at the latest. These
43354 instructions should be given the lowest priority in the
43355 scheduling process in Haifa scheduler to make sure they will be
43356 scheduled in the same dispatch window as the reference to them. */
43360 /* Check nonrestricted. */
43364 /* Get last dispatch window. */
43369 {
43374 /* Window 1 is full. Go for next window. */
43376 }
43383 /* See if it fits in the first window. */
43385 {
43386 /* The first widow should have only single and double path
43387 uops. */
43393 }
43395 }
43397 /* Add an instruction INSN with NUM_UOPS micro-operations to the
43398 dispatch window WINDOW_LIST. */
43400 static void
43402 {
43420 /* Initialize window with new instruction. */
43441 {
43444 }
43445 }
43447 /* Adds a scheduled instruction, INSN, to the current dispatch window.
43448 If the total bytes of instructions or the number of instructions in
43449 the window exceed allowable, it allocates a new window. */
43451 static void
43453 {
43476 /* Get the last dispatch window. */
43484 else
43487 /* If current window is full, get a new window.
43488 Window number zero is full, if MAX_INSN uops are scheduled in it.
43489 Window number one is full, if window zero's bytes plus window
43490 one's bytes is 32, or if the bytes of the new instruction added
43491 to the total makes it greater than 48, or it has already MAX_INSN
43492 instructions in it. */
43501 {
43504 }
43507 {
43511 {
43514 }
43515 }
43517 {
43522 {
43525 }
43528 }
43529 else
43533 {
43534 /* End of basic block is reached do end-basic-block process. */
43537 }
43538 }
43540 /* Print the dispatch window, WINDOW_NUM, to FILE. */
43542 DEBUG_FUNCTION static void
43544 {
43550 else
43564 {
43567 fprintf (file, " group[%d] = %s, insn[%d] = %p, path[%d] = %d byte_len[%d] = %d, imm_bytes[%d] = %d\n",
43573 }
43574 }
43576 /* Print to stdout a dispatch window. */
43578 DEBUG_FUNCTION void
43580 {
43582 }
43584 /* Print INSN dispatch information to FILE. */
43586 DEBUG_FUNCTION static void
43588 {
43611 }
43613 /* Print to STDERR the status of the ready list with respect to
43614 dispatch windows. */
43616 DEBUG_FUNCTION void
43618 {
43626 }
43628 /* This routine is the driver of the dispatch scheduler. */
43630 static void
43632 {
43637 }
43639 /* Return TRUE if Dispatch Scheduling is supported. */
43641 static bool
43643 {
43647 {
43663 }
43666 }
43668 /* Implementation of reassociation_width target hook used by
43669 reassoc phase to identify parallelism level in reassociated
43670 tree. Statements tree_code is passed in OPC. Arguments type
43671 is passed in MODE.
43673 Currently parallel reassociation is enabled for Atom
43674 processors only and we set reassociation width to be 2
43675 because Atom may issue up to 2 instructions per cycle.
43677 Return value should be fixed if parallel reassociation is
43678 enabled for other processors. */
43680 static int
43683 {
43692 }
43694 /* ??? No autovectorization into MMX or 3DNOW until we can reliably
43695 place emms and femms instructions. */
43697 static enum machine_mode
43699 {
43704 {
43717 else
43727 /* FALLTHRU */
43731 }
43732 }
43734 /* If AVX is enabled then try vectorizing with both 256bit and 128bit
43735 vectors. */
43737 static unsigned int
43739 {
43741 }
43743 \f
43745 /* Return class of registers which could be used for pseudo of MODE
43746 and of class RCLASS for spilling instead of memory. Return NO_REGS
43747 if it is not possible or non-profitable. */
43748 static reg_class_t
43750 {
43756 }
43758 /* Implement targetm.vectorize.init_cost. */
43762 {
43766 }
43768 /* Implement targetm.vectorize.add_stmt_cost. */
43770 static unsigned
43774 {
43781 /* Statements in an inner loop relative to the loop being
43782 vectorized are weighted more heavily. The value here is
43783 arbitrary and could potentially be improved with analysis. */
43791 }
43793 /* Implement targetm.vectorize.finish_cost. */
43795 static void
43798 {
43803 }
43805 /* Implement targetm.vectorize.destroy_cost_data. */
43807 static void
43809 {
43811 }
43813 /* Validate target specific memory model bits in VAL. */
43815 static unsigned HOST_WIDE_INT
43817 {
43824 {
43828 }
43831 {
43835 }
43837 {
43841 }
43843 }
43845 /* Set CLONEI->vecsize_mangle, CLONEI->vecsize_int,
43846 CLONEI->vecsize_float and if CLONEI->simdlen is 0, also
43847 CLONEI->simdlen. Return 0 if SIMD clones shouldn't be emitted,
43848 or number of vecsize_mangle variants that should be emitted. */
43850 static int
43854 {
43861 {
43865 }
43870 {
43877 /* case SCmode: */
43878 /* case DCmode: */
43884 }
43886 tree t;
43890 /* FIXME: Shouldn't we allow such arguments if they are uniform? */
43892 {
43899 /* case SCmode: */
43900 /* case DCmode: */
43906 }
43909 {
43910 /* Parse here processor clause. If not present, default to 'b'. */
43912 }
43914 {
43915 /* If the function isn't exported, we can pick up just one ISA
43916 for the clones. */
43921 else
43924 }
43925 else
43926 {
43929 }
43931 {
43944 }
43946 {
43949 else
43954 }
43956 }
43958 /* Add target attribute to SIMD clone NODE if needed. */
43960 static void
43962 {
43966 {
43981 }
43991 }
43993 /* If SIMD clone NODE can't be used in a vectorized loop
43994 in current function, return -1, otherwise return a badness of using it
43995 (0 if it is most desirable from vecsize_mangle point of view, 1
43996 slightly less desirable, etc.). */
43998 static int
44000 {
44002 {
44020 }
44021 }
44023 /* Implement TARGET_FLOAT_EXCEPTIONS_ROUNDING_SUPPORTED_P. */
44025 static bool
44027 {
44028 /* For x87 floating point with standard excess precision handling,
44029 there is no adddf3 pattern (since x87 floating point only has
44030 XFmode operations) so the default hook implementation gets this
44031 wrong. */
44033 }
44035 /* Implement TARGET_ATOMIC_ASSIGN_EXPAND_FENV. */
44037 static void
44039 {
44044 {
44059 hold_fnclex);
44073 }
44075 {
44084 mxcsr_orig_var,
44094 ldmxcsr_hold_call);
44097 else
44102 ldmxcsr_clear_call);
44103 else
44107 stxmcsr_update_call);
44109 {
44115 exceptions_assign);
44116 }
44117 else
44122 ldmxcsr_update_call);
44123 }
44124 tree atomic_feraiseexcept
44129 atomic_feraiseexcept_call);
44130 }
44132 /* Initialize the GCC target structure. */
44133 #undef TARGET_RETURN_IN_MEMORY
44134 #define TARGET_RETURN_IN_MEMORY ix86_return_in_memory
44136 #undef TARGET_LEGITIMIZE_ADDRESS
44137 #define TARGET_LEGITIMIZE_ADDRESS ix86_legitimize_address
44139 #undef TARGET_ATTRIBUTE_TABLE
44140 #define TARGET_ATTRIBUTE_TABLE ix86_attribute_table
44141 #undef TARGET_FUNCTION_ATTRIBUTE_INLINABLE_P
44142 #define TARGET_FUNCTION_ATTRIBUTE_INLINABLE_P hook_bool_const_tree_true
44143 #if TARGET_DLLIMPORT_DECL_ATTRIBUTES
44144 # undef TARGET_MERGE_DECL_ATTRIBUTES
44145 # define TARGET_MERGE_DECL_ATTRIBUTES merge_dllimport_decl_attributes
44146 #endif
44148 #undef TARGET_COMP_TYPE_ATTRIBUTES
44149 #define TARGET_COMP_TYPE_ATTRIBUTES ix86_comp_type_attributes
44151 #undef TARGET_INIT_BUILTINS
44152 #define TARGET_INIT_BUILTINS ix86_init_builtins
44153 #undef TARGET_BUILTIN_DECL
44154 #define TARGET_BUILTIN_DECL ix86_builtin_decl
44155 #undef TARGET_EXPAND_BUILTIN
44156 #define TARGET_EXPAND_BUILTIN ix86_expand_builtin
44158 #undef TARGET_VECTORIZE_BUILTIN_VECTORIZED_FUNCTION
44159 #define TARGET_VECTORIZE_BUILTIN_VECTORIZED_FUNCTION \
44160 ix86_builtin_vectorized_function
44162 #undef TARGET_VECTORIZE_BUILTIN_TM_LOAD
44163 #define TARGET_VECTORIZE_BUILTIN_TM_LOAD ix86_builtin_tm_load
44165 #undef TARGET_VECTORIZE_BUILTIN_TM_STORE
44166 #define TARGET_VECTORIZE_BUILTIN_TM_STORE ix86_builtin_tm_store
44168 #undef TARGET_VECTORIZE_BUILTIN_GATHER
44169 #define TARGET_VECTORIZE_BUILTIN_GATHER ix86_vectorize_builtin_gather
44171 #undef TARGET_BUILTIN_RECIPROCAL
44172 #define TARGET_BUILTIN_RECIPROCAL ix86_builtin_reciprocal
44174 #undef TARGET_ASM_FUNCTION_EPILOGUE
44175 #define TARGET_ASM_FUNCTION_EPILOGUE ix86_output_function_epilogue
44177 #undef TARGET_ENCODE_SECTION_INFO
44178 #ifndef SUBTARGET_ENCODE_SECTION_INFO
44179 #define TARGET_ENCODE_SECTION_INFO ix86_encode_section_info
44180 #else
44181 #define TARGET_ENCODE_SECTION_INFO SUBTARGET_ENCODE_SECTION_INFO
44182 #endif
44184 #undef TARGET_ASM_OPEN_PAREN
44186 #undef TARGET_ASM_CLOSE_PAREN
44189 #undef TARGET_ASM_BYTE_OP
44190 #define TARGET_ASM_BYTE_OP ASM_BYTE
44192 #undef TARGET_ASM_ALIGNED_HI_OP
44193 #define TARGET_ASM_ALIGNED_HI_OP ASM_SHORT
44194 #undef TARGET_ASM_ALIGNED_SI_OP
44195 #define TARGET_ASM_ALIGNED_SI_OP ASM_LONG
44196 #ifdef ASM_QUAD
44197 #undef TARGET_ASM_ALIGNED_DI_OP
44198 #define TARGET_ASM_ALIGNED_DI_OP ASM_QUAD
44199 #endif
44201 #undef TARGET_PROFILE_BEFORE_PROLOGUE
44202 #define TARGET_PROFILE_BEFORE_PROLOGUE ix86_profile_before_prologue
44204 #undef TARGET_MANGLE_DECL_ASSEMBLER_NAME
44205 #define TARGET_MANGLE_DECL_ASSEMBLER_NAME ix86_mangle_decl_assembler_name
44207 #undef TARGET_ASM_UNALIGNED_HI_OP
44208 #define TARGET_ASM_UNALIGNED_HI_OP TARGET_ASM_ALIGNED_HI_OP
44209 #undef TARGET_ASM_UNALIGNED_SI_OP
44210 #define TARGET_ASM_UNALIGNED_SI_OP TARGET_ASM_ALIGNED_SI_OP
44211 #undef TARGET_ASM_UNALIGNED_DI_OP
44212 #define TARGET_ASM_UNALIGNED_DI_OP TARGET_ASM_ALIGNED_DI_OP
44214 #undef TARGET_PRINT_OPERAND
44215 #define TARGET_PRINT_OPERAND ix86_print_operand
44216 #undef TARGET_PRINT_OPERAND_ADDRESS
44217 #define TARGET_PRINT_OPERAND_ADDRESS ix86_print_operand_address
44218 #undef TARGET_PRINT_OPERAND_PUNCT_VALID_P
44219 #define TARGET_PRINT_OPERAND_PUNCT_VALID_P ix86_print_operand_punct_valid_p
44220 #undef TARGET_ASM_OUTPUT_ADDR_CONST_EXTRA
44221 #define TARGET_ASM_OUTPUT_ADDR_CONST_EXTRA i386_asm_output_addr_const_extra
44223 #undef TARGET_SCHED_INIT_GLOBAL
44224 #define TARGET_SCHED_INIT_GLOBAL ix86_sched_init_global
44225 #undef TARGET_SCHED_ADJUST_COST
44226 #define TARGET_SCHED_ADJUST_COST ix86_adjust_cost
44227 #undef TARGET_SCHED_ISSUE_RATE
44228 #define TARGET_SCHED_ISSUE_RATE ix86_issue_rate
44229 #undef TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD
44230 #define TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD \
44231 ia32_multipass_dfa_lookahead
44232 #undef TARGET_SCHED_MACRO_FUSION_P
44233 #define TARGET_SCHED_MACRO_FUSION_P ix86_macro_fusion_p
44234 #undef TARGET_SCHED_MACRO_FUSION_PAIR_P
44235 #define TARGET_SCHED_MACRO_FUSION_PAIR_P ix86_macro_fusion_pair_p
44237 #undef TARGET_FUNCTION_OK_FOR_SIBCALL
44238 #define TARGET_FUNCTION_OK_FOR_SIBCALL ix86_function_ok_for_sibcall
44240 #undef TARGET_MEMMODEL_CHECK
44241 #define TARGET_MEMMODEL_CHECK ix86_memmodel_check
44243 #undef TARGET_ATOMIC_ASSIGN_EXPAND_FENV
44244 #define TARGET_ATOMIC_ASSIGN_EXPAND_FENV ix86_atomic_assign_expand_fenv
44246 #ifdef HAVE_AS_TLS
44247 #undef TARGET_HAVE_TLS
44248 #define TARGET_HAVE_TLS true
44249 #endif
44250 #undef TARGET_CANNOT_FORCE_CONST_MEM
44251 #define TARGET_CANNOT_FORCE_CONST_MEM ix86_cannot_force_const_mem
44252 #undef TARGET_USE_BLOCKS_FOR_CONSTANT_P
44253 #define TARGET_USE_BLOCKS_FOR_CONSTANT_P hook_bool_mode_const_rtx_true
44255 #undef TARGET_DELEGITIMIZE_ADDRESS
44256 #define TARGET_DELEGITIMIZE_ADDRESS ix86_delegitimize_address
44258 #undef TARGET_MS_BITFIELD_LAYOUT_P
44259 #define TARGET_MS_BITFIELD_LAYOUT_P ix86_ms_bitfield_layout_p
44261 #if TARGET_MACHO
44262 #undef TARGET_BINDS_LOCAL_P
44263 #define TARGET_BINDS_LOCAL_P darwin_binds_local_p
44264 #endif
44265 #if TARGET_DLLIMPORT_DECL_ATTRIBUTES
44266 #undef TARGET_BINDS_LOCAL_P
44267 #define TARGET_BINDS_LOCAL_P i386_pe_binds_local_p
44268 #endif
44270 #undef TARGET_ASM_OUTPUT_MI_THUNK
44271 #define TARGET_ASM_OUTPUT_MI_THUNK x86_output_mi_thunk
44272 #undef TARGET_ASM_CAN_OUTPUT_MI_THUNK
44273 #define TARGET_ASM_CAN_OUTPUT_MI_THUNK x86_can_output_mi_thunk
44275 #undef TARGET_ASM_FILE_START
44276 #define TARGET_ASM_FILE_START x86_file_start
44278 #undef TARGET_OPTION_OVERRIDE
44279 #define TARGET_OPTION_OVERRIDE ix86_option_override
44281 #undef TARGET_REGISTER_MOVE_COST
44282 #define TARGET_REGISTER_MOVE_COST ix86_register_move_cost
44283 #undef TARGET_MEMORY_MOVE_COST
44284 #define TARGET_MEMORY_MOVE_COST ix86_memory_move_cost
44285 #undef TARGET_RTX_COSTS
44286 #define TARGET_RTX_COSTS ix86_rtx_costs
44287 #undef TARGET_ADDRESS_COST
44288 #define TARGET_ADDRESS_COST ix86_address_cost
44290 #undef TARGET_FIXED_CONDITION_CODE_REGS
44291 #define TARGET_FIXED_CONDITION_CODE_REGS ix86_fixed_condition_code_regs
44292 #undef TARGET_CC_MODES_COMPATIBLE
44293 #define TARGET_CC_MODES_COMPATIBLE ix86_cc_modes_compatible
44295 #undef TARGET_MACHINE_DEPENDENT_REORG
44296 #define TARGET_MACHINE_DEPENDENT_REORG ix86_reorg
44298 #undef TARGET_BUILTIN_SETJMP_FRAME_VALUE
44299 #define TARGET_BUILTIN_SETJMP_FRAME_VALUE ix86_builtin_setjmp_frame_value
44301 #undef TARGET_BUILD_BUILTIN_VA_LIST
44302 #define TARGET_BUILD_BUILTIN_VA_LIST ix86_build_builtin_va_list
44304 #undef TARGET_FOLD_BUILTIN
44305 #define TARGET_FOLD_BUILTIN ix86_fold_builtin
44307 #undef TARGET_COMPARE_VERSION_PRIORITY
44308 #define TARGET_COMPARE_VERSION_PRIORITY ix86_compare_version_priority
44310 #undef TARGET_GENERATE_VERSION_DISPATCHER_BODY
44311 #define TARGET_GENERATE_VERSION_DISPATCHER_BODY \
44312 ix86_generate_version_dispatcher_body
44314 #undef TARGET_GET_FUNCTION_VERSIONS_DISPATCHER
44315 #define TARGET_GET_FUNCTION_VERSIONS_DISPATCHER \
44316 ix86_get_function_versions_dispatcher
44318 #undef TARGET_ENUM_VA_LIST_P
44319 #define TARGET_ENUM_VA_LIST_P ix86_enum_va_list
44321 #undef TARGET_FN_ABI_VA_LIST
44322 #define TARGET_FN_ABI_VA_LIST ix86_fn_abi_va_list
44324 #undef TARGET_CANONICAL_VA_LIST_TYPE
44325 #define TARGET_CANONICAL_VA_LIST_TYPE ix86_canonical_va_list_type
44327 #undef TARGET_EXPAND_BUILTIN_VA_START
44328 #define TARGET_EXPAND_BUILTIN_VA_START ix86_va_start
44330 #undef TARGET_MD_ASM_CLOBBERS
44331 #define TARGET_MD_ASM_CLOBBERS ix86_md_asm_clobbers
44333 #undef TARGET_PROMOTE_PROTOTYPES
44334 #define TARGET_PROMOTE_PROTOTYPES hook_bool_const_tree_true
44335 #undef TARGET_STRUCT_VALUE_RTX
44336 #define TARGET_STRUCT_VALUE_RTX ix86_struct_value_rtx
44337 #undef TARGET_SETUP_INCOMING_VARARGS
44338 #define TARGET_SETUP_INCOMING_VARARGS ix86_setup_incoming_varargs
44339 #undef TARGET_MUST_PASS_IN_STACK
44340 #define TARGET_MUST_PASS_IN_STACK ix86_must_pass_in_stack
44341 #undef TARGET_FUNCTION_ARG_ADVANCE
44342 #define TARGET_FUNCTION_ARG_ADVANCE ix86_function_arg_advance
44343 #undef TARGET_FUNCTION_ARG
44344 #define TARGET_FUNCTION_ARG ix86_function_arg
44345 #undef TARGET_FUNCTION_ARG_BOUNDARY
44346 #define TARGET_FUNCTION_ARG_BOUNDARY ix86_function_arg_boundary
44347 #undef TARGET_PASS_BY_REFERENCE
44348 #define TARGET_PASS_BY_REFERENCE ix86_pass_by_reference
44349 #undef TARGET_INTERNAL_ARG_POINTER
44350 #define TARGET_INTERNAL_ARG_POINTER ix86_internal_arg_pointer
44351 #undef TARGET_UPDATE_STACK_BOUNDARY
44352 #define TARGET_UPDATE_STACK_BOUNDARY ix86_update_stack_boundary
44353 #undef TARGET_GET_DRAP_RTX
44354 #define TARGET_GET_DRAP_RTX ix86_get_drap_rtx
44355 #undef TARGET_STRICT_ARGUMENT_NAMING
44356 #define TARGET_STRICT_ARGUMENT_NAMING hook_bool_CUMULATIVE_ARGS_true
44357 #undef TARGET_STATIC_CHAIN
44358 #define TARGET_STATIC_CHAIN ix86_static_chain
44359 #undef TARGET_TRAMPOLINE_INIT
44360 #define TARGET_TRAMPOLINE_INIT ix86_trampoline_init
44361 #undef TARGET_RETURN_POPS_ARGS
44362 #define TARGET_RETURN_POPS_ARGS ix86_return_pops_args
44364 #undef TARGET_LEGITIMATE_COMBINED_INSN
44365 #define TARGET_LEGITIMATE_COMBINED_INSN ix86_legitimate_combined_insn
44367 #undef TARGET_ASAN_SHADOW_OFFSET
44368 #define TARGET_ASAN_SHADOW_OFFSET ix86_asan_shadow_offset
44370 #undef TARGET_GIMPLIFY_VA_ARG_EXPR
44371 #define TARGET_GIMPLIFY_VA_ARG_EXPR ix86_gimplify_va_arg
44373 #undef TARGET_SCALAR_MODE_SUPPORTED_P
44374 #define TARGET_SCALAR_MODE_SUPPORTED_P ix86_scalar_mode_supported_p
44376 #undef TARGET_VECTOR_MODE_SUPPORTED_P
44377 #define TARGET_VECTOR_MODE_SUPPORTED_P ix86_vector_mode_supported_p
44379 #undef TARGET_C_MODE_FOR_SUFFIX
44380 #define TARGET_C_MODE_FOR_SUFFIX ix86_c_mode_for_suffix
44382 #ifdef HAVE_AS_TLS
44383 #undef TARGET_ASM_OUTPUT_DWARF_DTPREL
44384 #define TARGET_ASM_OUTPUT_DWARF_DTPREL i386_output_dwarf_dtprel
44385 #endif
44387 #ifdef SUBTARGET_INSERT_ATTRIBUTES
44388 #undef TARGET_INSERT_ATTRIBUTES
44389 #define TARGET_INSERT_ATTRIBUTES SUBTARGET_INSERT_ATTRIBUTES
44390 #endif
44392 #undef TARGET_MANGLE_TYPE
44393 #define TARGET_MANGLE_TYPE ix86_mangle_type
44395 #if !TARGET_MACHO
44396 #undef TARGET_STACK_PROTECT_FAIL
44397 #define TARGET_STACK_PROTECT_FAIL ix86_stack_protect_fail
44398 #endif
44400 #undef TARGET_FUNCTION_VALUE
44401 #define TARGET_FUNCTION_VALUE ix86_function_value
44403 #undef TARGET_FUNCTION_VALUE_REGNO_P
44404 #define TARGET_FUNCTION_VALUE_REGNO_P ix86_function_value_regno_p
44406 #undef TARGET_PROMOTE_FUNCTION_MODE
44407 #define TARGET_PROMOTE_FUNCTION_MODE ix86_promote_function_mode
44409 #undef TARGET_MEMBER_TYPE_FORCES_BLK
44410 #define TARGET_MEMBER_TYPE_FORCES_BLK ix86_member_type_forces_blk
44412 #undef TARGET_INSTANTIATE_DECLS
44413 #define TARGET_INSTANTIATE_DECLS ix86_instantiate_decls
44415 #undef TARGET_SECONDARY_RELOAD
44416 #define TARGET_SECONDARY_RELOAD ix86_secondary_reload
44418 #undef TARGET_CLASS_MAX_NREGS
44419 #define TARGET_CLASS_MAX_NREGS ix86_class_max_nregs
44421 #undef TARGET_PREFERRED_RELOAD_CLASS
44422 #define TARGET_PREFERRED_RELOAD_CLASS ix86_preferred_reload_class
44423 #undef TARGET_PREFERRED_OUTPUT_RELOAD_CLASS
44424 #define TARGET_PREFERRED_OUTPUT_RELOAD_CLASS ix86_preferred_output_reload_class
44425 #undef TARGET_CLASS_LIKELY_SPILLED_P
44426 #define TARGET_CLASS_LIKELY_SPILLED_P ix86_class_likely_spilled_p
44428 #undef TARGET_VECTORIZE_BUILTIN_VECTORIZATION_COST
44429 #define TARGET_VECTORIZE_BUILTIN_VECTORIZATION_COST \
44430 ix86_builtin_vectorization_cost
44431 #undef TARGET_VECTORIZE_VEC_PERM_CONST_OK
44432 #define TARGET_VECTORIZE_VEC_PERM_CONST_OK \
44433 ix86_vectorize_vec_perm_const_ok
44434 #undef TARGET_VECTORIZE_PREFERRED_SIMD_MODE
44435 #define TARGET_VECTORIZE_PREFERRED_SIMD_MODE \
44436 ix86_preferred_simd_mode
44437 #undef TARGET_VECTORIZE_AUTOVECTORIZE_VECTOR_SIZES
44438 #define TARGET_VECTORIZE_AUTOVECTORIZE_VECTOR_SIZES \
44439 ix86_autovectorize_vector_sizes
44440 #undef TARGET_VECTORIZE_INIT_COST
44441 #define TARGET_VECTORIZE_INIT_COST ix86_init_cost
44442 #undef TARGET_VECTORIZE_ADD_STMT_COST
44443 #define TARGET_VECTORIZE_ADD_STMT_COST ix86_add_stmt_cost
44444 #undef TARGET_VECTORIZE_FINISH_COST
44445 #define TARGET_VECTORIZE_FINISH_COST ix86_finish_cost
44446 #undef TARGET_VECTORIZE_DESTROY_COST_DATA
44447 #define TARGET_VECTORIZE_DESTROY_COST_DATA ix86_destroy_cost_data
44449 #undef TARGET_SET_CURRENT_FUNCTION
44450 #define TARGET_SET_CURRENT_FUNCTION ix86_set_current_function
44452 #undef TARGET_OPTION_VALID_ATTRIBUTE_P
44453 #define TARGET_OPTION_VALID_ATTRIBUTE_P ix86_valid_target_attribute_p
44455 #undef TARGET_OPTION_SAVE
44456 #define TARGET_OPTION_SAVE ix86_function_specific_save
44458 #undef TARGET_OPTION_RESTORE
44459 #define TARGET_OPTION_RESTORE ix86_function_specific_restore
44461 #undef TARGET_OPTION_PRINT
44462 #define TARGET_OPTION_PRINT ix86_function_specific_print
44464 #undef TARGET_OPTION_FUNCTION_VERSIONS
44465 #define TARGET_OPTION_FUNCTION_VERSIONS ix86_function_versions
44467 #undef TARGET_CAN_INLINE_P
44468 #define TARGET_CAN_INLINE_P ix86_can_inline_p
44470 #undef TARGET_EXPAND_TO_RTL_HOOK
44471 #define TARGET_EXPAND_TO_RTL_HOOK ix86_maybe_switch_abi
44473 #undef TARGET_LEGITIMATE_ADDRESS_P
44474 #define TARGET_LEGITIMATE_ADDRESS_P ix86_legitimate_address_p
44476 #undef TARGET_LRA_P
44477 #define TARGET_LRA_P hook_bool_void_true
44479 #undef TARGET_REGISTER_PRIORITY
44480 #define TARGET_REGISTER_PRIORITY ix86_register_priority
44482 #undef TARGET_REGISTER_USAGE_LEVELING_P
44483 #define TARGET_REGISTER_USAGE_LEVELING_P hook_bool_void_true
44485 #undef TARGET_LEGITIMATE_CONSTANT_P
44486 #define TARGET_LEGITIMATE_CONSTANT_P ix86_legitimate_constant_p
44488 #undef TARGET_FRAME_POINTER_REQUIRED
44489 #define TARGET_FRAME_POINTER_REQUIRED ix86_frame_pointer_required
44491 #undef TARGET_CAN_ELIMINATE
44492 #define TARGET_CAN_ELIMINATE ix86_can_eliminate
44494 #undef TARGET_EXTRA_LIVE_ON_ENTRY
44495 #define TARGET_EXTRA_LIVE_ON_ENTRY ix86_live_on_entry
44497 #undef TARGET_ASM_CODE_END
44498 #define TARGET_ASM_CODE_END ix86_code_end
44500 #undef TARGET_CONDITIONAL_REGISTER_USAGE
44501 #define TARGET_CONDITIONAL_REGISTER_USAGE ix86_conditional_register_usage
44503 #if TARGET_MACHO
44504 #undef TARGET_INIT_LIBFUNCS
44505 #define TARGET_INIT_LIBFUNCS darwin_rename_builtins
44506 #endif
44508 #undef TARGET_SPILL_CLASS
44509 #define TARGET_SPILL_CLASS ix86_spill_class
44511 #undef TARGET_SIMD_CLONE_COMPUTE_VECSIZE_AND_SIMDLEN
44512 #define TARGET_SIMD_CLONE_COMPUTE_VECSIZE_AND_SIMDLEN \
44513 ix86_simd_clone_compute_vecsize_and_simdlen
44515 #undef TARGET_SIMD_CLONE_ADJUST
44516 #define TARGET_SIMD_CLONE_ADJUST \
44517 ix86_simd_clone_adjust
44519 #undef TARGET_SIMD_CLONE_USABLE
44520 #define TARGET_SIMD_CLONE_USABLE \
44521 ix86_simd_clone_usable
44523 #undef TARGET_FLOAT_EXCEPTIONS_ROUNDING_SUPPORTED_P
44524 #define TARGET_FLOAT_EXCEPTIONS_ROUNDING_SUPPORTED_P \
44525 ix86_float_exceptions_rounding_supported_p
44528 \f