| blob | bb6d15a2b2acf9231defc58b8623331c9340274b |
1 /* Subroutines used for code generation on IA-32.
2 Copyright (C) 1988-2013 Free Software Foundation, Inc.
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 3, or (at your option)
9 any later version.
11 GCC is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
72 #ifndef CHECK_STACK_LIMIT
73 #define CHECK_STACK_LIMIT (-1)
74 #endif
76 /* Return index of given mode in mult and division cost tables. */
77 #define MODE_INDEX(mode) \
78 ((mode) == QImode ? 0 \
79 : (mode) == HImode ? 1 \
80 : (mode) == SImode ? 2 \
81 : (mode) == DImode ? 3 \
82 : 4)
84 /* Processor costs (relative to an add) */
85 /* We assume COSTS_N_INSNS is defined as (N)*4 and an addition is 2 bytes. */
86 #define COSTS_N_BYTES(N) ((N) * 2)
88 #define DUMMY_STRINGOP_ALGS {libcall, {{-1, libcall, false}}}
97 const
120 in QImode, HImode and SImode.
121 Relative to reg-reg move (2). */
125 in SFmode, DFmode and XFmode */
127 in SFmode, DFmode and XFmode */
130 in SImode and DImode */
132 in SImode and DImode */
135 in SImode, DImode and TImode */
137 in SImode, DImode and TImode */
150 ix86_size_memcpy,
151 ix86_size_memset,
163 };
165 /* Processor costs (relative to an add) */
168 DUMMY_STRINGOP_ALGS};
171 DUMMY_STRINGOP_ALGS};
173 static const
196 in QImode, HImode and SImode.
197 Relative to reg-reg move (2). */
201 in SFmode, DFmode and XFmode */
203 in SFmode, DFmode and XFmode */
206 in SImode and DImode */
208 in SImode and DImode */
211 in SImode, DImode and TImode */
213 in SImode, DImode and TImode */
226 i386_memcpy,
227 i386_memset,
239 };
243 DUMMY_STRINGOP_ALGS};
246 DUMMY_STRINGOP_ALGS};
248 static const
271 in QImode, HImode and SImode.
272 Relative to reg-reg move (2). */
276 in SFmode, DFmode and XFmode */
278 in SFmode, DFmode and XFmode */
281 in SImode and DImode */
283 in SImode and DImode */
286 in SImode, DImode and TImode */
288 in SImode, DImode and TImode */
291 shared for code and data, so 4kB is
292 not really precise. */
303 i486_memcpy,
304 i486_memset,
316 };
320 DUMMY_STRINGOP_ALGS};
323 DUMMY_STRINGOP_ALGS};
325 static const
348 in QImode, HImode and SImode.
349 Relative to reg-reg move (2). */
353 in SFmode, DFmode and XFmode */
355 in SFmode, DFmode and XFmode */
358 in SImode and DImode */
360 in SImode and DImode */
363 in SImode, DImode and TImode */
365 in SImode, DImode and TImode */
378 pentium_memcpy,
379 pentium_memset,
391 };
393 /* PentiumPro has optimized rep instructions for blocks aligned by 8 bytes
394 (we ensure the alignment). For small blocks inline loop is still a
395 noticeable win, for bigger blocks either rep movsl or rep movsb is
396 way to go. Rep movsb has apparently more expensive startup time in CPU,
397 but after 4K the difference is down in the noise. */
402 DUMMY_STRINGOP_ALGS};
407 DUMMY_STRINGOP_ALGS};
408 static const
431 in QImode, HImode and SImode.
432 Relative to reg-reg move (2). */
436 in SFmode, DFmode and XFmode */
438 in SFmode, DFmode and XFmode */
441 in SImode and DImode */
443 in SImode and DImode */
446 in SImode, DImode and TImode */
448 in SImode, DImode and TImode */
461 pentiumpro_memcpy,
462 pentiumpro_memset,
474 };
478 DUMMY_STRINGOP_ALGS};
481 DUMMY_STRINGOP_ALGS};
482 static const
505 in QImode, HImode and SImode.
506 Relative to reg-reg move (2). */
510 in SFmode, DFmode and XFmode */
512 in SFmode, DFmode and XFmode */
516 in SImode and DImode */
518 in SImode and DImode */
521 in SImode, DImode and TImode */
523 in SImode, DImode and TImode */
536 geode_memcpy,
537 geode_memset,
549 };
553 DUMMY_STRINGOP_ALGS};
556 DUMMY_STRINGOP_ALGS};
557 static const
580 in QImode, HImode and SImode.
581 Relative to reg-reg move (2). */
585 in SFmode, DFmode and XFmode */
587 in SFmode, DFmode and XFmode */
590 in SImode and DImode */
592 in SImode and DImode */
595 in SImode, DImode and TImode */
597 in SImode, DImode and TImode */
601 have integrated l2 cache, but
602 optimizing for k6 is not important
603 enough to worry about that. */
613 k6_memcpy,
614 k6_memset,
626 };
628 /* For some reason, Athlon deals better with REP prefix (relative to loops)
629 compared to K8. Alignment becomes important after 8 bytes for memcpy and
630 128 bytes for memset. */
633 DUMMY_STRINGOP_ALGS};
636 DUMMY_STRINGOP_ALGS};
637 static const
660 in QImode, HImode and SImode.
661 Relative to reg-reg move (2). */
665 in SFmode, DFmode and XFmode */
667 in SFmode, DFmode and XFmode */
670 in SImode and DImode */
672 in SImode and DImode */
675 in SImode, DImode and TImode */
677 in SImode, DImode and TImode */
690 athlon_memcpy,
691 athlon_memset,
703 };
705 /* K8 has optimized REP instruction for medium sized blocks, but for very
706 small blocks it is better to use loop. For large blocks, libcall can
707 do nontemporary accesses and beat inline considerably. */
718 static const
741 in QImode, HImode and SImode.
742 Relative to reg-reg move (2). */
746 in SFmode, DFmode and XFmode */
748 in SFmode, DFmode and XFmode */
751 in SImode and DImode */
753 in SImode and DImode */
756 in SImode, DImode and TImode */
758 in SImode, DImode and TImode */
763 /* New AMD processors never drop prefetches; if they cannot be performed
764 immediately, they are queued. We set number of simultaneous prefetches
765 to a large constant to reflect this (it probably is not a good idea not
766 to limit number of prefetches at all, as their execution also takes some
767 time). */
777 k8_memcpy,
778 k8_memset,
790 };
792 /* AMDFAM10 has optimized REP instruction for medium sized blocks, but for
793 very small blocks it is better to use loop. For large blocks, libcall can
794 do nontemporary accesses and beat inline considerably. */
827 in QImode, HImode and SImode.
828 Relative to reg-reg move (2). */
832 in SFmode, DFmode and XFmode */
834 in SFmode, DFmode and XFmode */
837 in SImode and DImode */
839 in SImode and DImode */
842 in SImode, DImode and TImode */
844 in SImode, DImode and TImode */
846 /* On K8:
847 MOVD reg64, xmmreg Double FSTORE 4
848 MOVD reg32, xmmreg Double FSTORE 4
849 On AMDFAM10:
850 MOVD reg64, xmmreg Double FADD 3
851 1/1 1/1
852 MOVD reg32, xmmreg Double FADD 3
853 1/1 1/1 */
857 /* New AMD processors never drop prefetches; if they cannot be performed
858 immediately, they are queued. We set number of simultaneous prefetches
859 to a large constant to reflect this (it probably is not a good idea not
860 to limit number of prefetches at all, as their execution also takes some
861 time). */
871 amdfam10_memcpy,
872 amdfam10_memset,
884 };
886 /* BDVER1 has optimized REP instruction for medium sized blocks, but for
887 very small blocks it is better to use loop. For large blocks, libcall
888 can do nontemporary accesses and beat inline considerably. */
922 in QImode, HImode and SImode.
923 Relative to reg-reg move (2). */
927 in SFmode, DFmode and XFmode */
929 in SFmode, DFmode and XFmode */
932 in SImode and DImode */
934 in SImode and DImode */
937 in SImode, DImode and TImode */
939 in SImode, DImode and TImode */
941 /* On K8:
942 MOVD reg64, xmmreg Double FSTORE 4
943 MOVD reg32, xmmreg Double FSTORE 4
944 On AMDFAM10:
945 MOVD reg64, xmmreg Double FADD 3
946 1/1 1/1
947 MOVD reg32, xmmreg Double FADD 3
948 1/1 1/1 */
952 /* New AMD processors never drop prefetches; if they cannot be performed
953 immediately, they are queued. We set number of simultaneous prefetches
954 to a large constant to reflect this (it probably is not a good idea not
955 to limit number of prefetches at all, as their execution also takes some
956 time). */
966 bdver1_memcpy,
967 bdver1_memset,
979 };
981 /* BDVER2 has optimized REP instruction for medium sized blocks, but for
982 very small blocks it is better to use loop. For large blocks, libcall
983 can do nontemporary accesses and beat inline considerably. */
1018 in QImode, HImode and SImode.
1019 Relative to reg-reg move (2). */
1023 in SFmode, DFmode and XFmode */
1025 in SFmode, DFmode and XFmode */
1028 in SImode and DImode */
1030 in SImode and DImode */
1033 in SImode, DImode and TImode */
1035 in SImode, DImode and TImode */
1037 /* On K8:
1038 MOVD reg64, xmmreg Double FSTORE 4
1039 MOVD reg32, xmmreg Double FSTORE 4
1040 On AMDFAM10:
1041 MOVD reg64, xmmreg Double FADD 3
1042 1/1 1/1
1043 MOVD reg32, xmmreg Double FADD 3
1044 1/1 1/1 */
1048 /* New AMD processors never drop prefetches; if they cannot be performed
1049 immediately, they are queued. We set number of simultaneous prefetches
1050 to a large constant to reflect this (it probably is not a good idea not
1051 to limit number of prefetches at all, as their execution also takes some
1052 time). */
1062 bdver2_memcpy,
1063 bdver2_memset,
1075 };
1078 /* BDVER3 has optimized REP instruction for medium sized blocks, but for
1079 very small blocks it is better to use loop. For large blocks, libcall
1080 can do nontemporary accesses and beat inline considerably. */
1113 in QImode, HImode and SImode.
1114 Relative to reg-reg move (2). */
1118 in SFmode, DFmode and XFmode */
1120 in SFmode, DFmode and XFmode */
1123 in SImode and DImode */
1125 in SImode and DImode */
1128 in SImode, DImode and TImode */
1130 in SImode, DImode and TImode */
1135 /* New AMD processors never drop prefetches; if they cannot be performed
1136 immediately, they are queued. We set number of simultaneous prefetches
1137 to a large constant to reflect this (it probably is not a good idea not
1138 to limit number of prefetches at all, as their execution also takes some
1139 time). */
1149 bdver3_memcpy,
1150 bdver3_memset,
1162 };
1164 /* BDVER4 has optimized REP instruction for medium sized blocks, but for
1165 very small blocks it is better to use loop. For large blocks, libcall
1166 can do nontemporary accesses and beat inline considerably. */
1199 in QImode, HImode and SImode.
1200 Relative to reg-reg move (2). */
1204 in SFmode, DFmode and XFmode */
1206 in SFmode, DFmode and XFmode */
1209 in SImode and DImode */
1211 in SImode and DImode */
1214 in SImode, DImode and TImode */
1216 in SImode, DImode and TImode */
1221 /* New AMD processors never drop prefetches; if they cannot be performed
1222 immediately, they are queued. We set number of simultaneous prefetches
1223 to a large constant to reflect this (it probably is not a good idea not
1224 to limit number of prefetches at all, as their execution also takes some
1225 time). */
1235 bdver4_memcpy,
1236 bdver4_memset,
1248 };
1250 /* BTVER1 has optimized REP instruction for medium sized blocks, but for
1251 very small blocks it is better to use loop. For large blocks, libcall can
1252 do nontemporary accesses and beat inline considerably. */
1285 in QImode, HImode and SImode.
1286 Relative to reg-reg move (2). */
1290 in SFmode, DFmode and XFmode */
1292 in SFmode, DFmode and XFmode */
1295 in SImode and DImode */
1297 in SImode and DImode */
1300 in SImode, DImode and TImode */
1302 in SImode, DImode and TImode */
1304 /* On K8:
1305 MOVD reg64, xmmreg Double FSTORE 4
1306 MOVD reg32, xmmreg Double FSTORE 4
1307 On AMDFAM10:
1308 MOVD reg64, xmmreg Double FADD 3
1309 1/1 1/1
1310 MOVD reg32, xmmreg Double FADD 3
1311 1/1 1/1 */
1324 btver1_memcpy,
1325 btver1_memset,
1337 };
1371 in QImode, HImode and SImode.
1372 Relative to reg-reg move (2). */
1376 in SFmode, DFmode and XFmode */
1378 in SFmode, DFmode and XFmode */
1381 in SImode and DImode */
1383 in SImode and DImode */
1386 in SImode, DImode and TImode */
1388 in SImode, DImode and TImode */
1390 /* On K8:
1391 MOVD reg64, xmmreg Double FSTORE 4
1392 MOVD reg32, xmmreg Double FSTORE 4
1393 On AMDFAM10:
1394 MOVD reg64, xmmreg Double FADD 3
1395 1/1 1/1
1396 MOVD reg32, xmmreg Double FADD 3
1397 1/1 1/1 */
1409 btver2_memcpy,
1410 btver2_memset,
1422 };
1426 DUMMY_STRINGOP_ALGS};
1430 DUMMY_STRINGOP_ALGS};
1432 static const
1455 in QImode, HImode and SImode.
1456 Relative to reg-reg move (2). */
1460 in SFmode, DFmode and XFmode */
1462 in SFmode, DFmode and XFmode */
1465 in SImode and DImode */
1467 in SImode and DImode */
1470 in SImode, DImode and TImode */
1472 in SImode, DImode and TImode */
1485 pentium4_memcpy,
1486 pentium4_memset,
1498 };
1511 static const
1534 in QImode, HImode and SImode.
1535 Relative to reg-reg move (2). */
1539 in SFmode, DFmode and XFmode */
1541 in SFmode, DFmode and XFmode */
1544 in SImode and DImode */
1546 in SImode and DImode */
1549 in SImode, DImode and TImode */
1551 in SImode, DImode and TImode */
1564 nocona_memcpy,
1565 nocona_memset,
1577 };
1588 static const
1611 in QImode, HImode and SImode.
1612 Relative to reg-reg move (2). */
1616 in SFmode, DFmode and XFmode */
1618 in SFmode, DFmode and XFmode */
1621 in SImode and DImode */
1623 in SImode and DImode */
1626 in SImode, DImode and TImode */
1628 in SImode, DImode and TImode */
1641 atom_memcpy,
1642 atom_memset,
1654 };
1665 static const
1688 in QImode, HImode and SImode.
1689 Relative to reg-reg move (2). */
1693 in SFmode, DFmode and XFmode */
1695 in SFmode, DFmode and XFmode */
1698 in SImode and DImode */
1700 in SImode and DImode */
1703 in SImode, DImode and TImode */
1705 in SImode, DImode and TImode */
1718 slm_memcpy,
1719 slm_memset,
1731 };
1733 /* Generic should produce code tuned for Core-i7 (and newer chips)
1734 and btver1 (and newer chips). */
1746 static const
1749 /* On all chips taken into consideration lea is 2 cycles and more. With
1750 this cost however our current implementation of synth_mult results in
1751 use of unnecessary temporary registers causing regression on several
1752 SPECfp benchmarks. */
1773 in QImode, HImode and SImode.
1774 Relative to reg-reg move (2). */
1778 in SFmode, DFmode and XFmode */
1780 in SFmode, DFmode and XFmode */
1783 in SImode and DImode */
1785 in SImode and DImode */
1788 in SImode, DImode and TImode */
1790 in SImode, DImode and TImode */
1796 /* Benchmarks shows large regressions on K8 sixtrack benchmark when this
1797 value is increased to perhaps more appropriate value of 5. */
1805 generic_memcpy,
1806 generic_memset,
1818 };
1820 /* core_cost should produce code tuned for Core familly of CPUs. */
1833 static const
1836 /* On all chips taken into consideration lea is 2 cycles and more. With
1837 this cost however our current implementation of synth_mult results in
1838 use of unnecessary temporary registers causing regression on several
1839 SPECfp benchmarks. */
1860 in QImode, HImode and SImode.
1861 Relative to reg-reg move (2). */
1865 in SFmode, DFmode and XFmode */
1867 in SFmode, DFmode and XFmode */
1870 in SImode and DImode */
1872 in SImode and DImode */
1875 in SImode, DImode and TImode */
1877 in SImode, DImode and TImode */
1883 /* FIXME perhaps more appropriate value is 5. */
1891 core_memcpy,
1892 core_memset,
1904 };
1907 /* Set by -mtune. */
1910 /* Set by -mtune or -Os. */
1913 /* Processor feature/optimization bitmasks. */
1914 #define m_386 (1<<PROCESSOR_I386)
1915 #define m_486 (1<<PROCESSOR_I486)
1916 #define m_PENT (1<<PROCESSOR_PENTIUM)
1917 #define m_PPRO (1<<PROCESSOR_PENTIUMPRO)
1918 #define m_PENT4 (1<<PROCESSOR_PENTIUM4)
1919 #define m_NOCONA (1<<PROCESSOR_NOCONA)
1920 #define m_P4_NOCONA (m_PENT4 | m_NOCONA)
1921 #define m_CORE2 (1<<PROCESSOR_CORE2)
1922 #define m_COREI7 (1<<PROCESSOR_COREI7)
1923 #define m_COREI7_AVX (1<<PROCESSOR_COREI7_AVX)
1924 #define m_HASWELL (1<<PROCESSOR_HASWELL)
1925 #define m_CORE_ALL (m_CORE2 | m_COREI7 | m_COREI7_AVX | m_HASWELL)
1926 #define m_ATOM (1<<PROCESSOR_ATOM)
1927 #define m_SLM (1<<PROCESSOR_SLM)
1929 #define m_GEODE (1<<PROCESSOR_GEODE)
1930 #define m_K6 (1<<PROCESSOR_K6)
1931 #define m_K6_GEODE (m_K6 | m_GEODE)
1932 #define m_K8 (1<<PROCESSOR_K8)
1933 #define m_ATHLON (1<<PROCESSOR_ATHLON)
1934 #define m_ATHLON_K8 (m_K8 | m_ATHLON)
1935 #define m_AMDFAM10 (1<<PROCESSOR_AMDFAM10)
1936 #define m_BDVER1 (1<<PROCESSOR_BDVER1)
1937 #define m_BDVER2 (1<<PROCESSOR_BDVER2)
1938 #define m_BDVER3 (1<<PROCESSOR_BDVER3)
1939 #define m_BDVER4 (1<<PROCESSOR_BDVER4)
1940 #define m_BTVER1 (1<<PROCESSOR_BTVER1)
1941 #define m_BTVER2 (1<<PROCESSOR_BTVER2)
1942 #define m_BDVER (m_BDVER1 | m_BDVER2 | m_BDVER3 | m_BDVER4)
1943 #define m_BTVER (m_BTVER1 | m_BTVER2)
1944 #define m_AMD_MULTIPLE (m_ATHLON_K8 | m_AMDFAM10 | m_BDVER | m_BTVER)
1946 #define m_GENERIC (1<<PROCESSOR_GENERIC)
1949 #undef DEF_TUNE
1950 #define DEF_TUNE(tune, name, selector) name,
1952 #undef DEF_TUNE
1953 };
1955 /* Feature tests against the various tunings. */
1958 /* Feature tests against the various tunings used to create ix86_tune_features
1959 based on the processor mask. */
1961 #undef DEF_TUNE
1962 #define DEF_TUNE(tune, name, selector) selector,
1964 #undef DEF_TUNE
1965 };
1967 /* Feature tests against the various architecture variations. */
1970 /* Feature tests against the various architecture variations, used to create
1971 ix86_arch_features based on the processor mask. */
1973 /* X86_ARCH_CMOV: Conditional move was added for pentiumpro. */
1976 /* X86_ARCH_CMPXCHG: Compare and exchange was added for 80486. */
1979 /* X86_ARCH_CMPXCHG8B: Compare and exchange 8 bytes was added for pentium. */
1982 /* X86_ARCH_XADD: Exchange and add was added for 80486. */
1985 /* X86_ARCH_BSWAP: Byteswap was added for 80486. */
1987 };
1989 /* In case the average insn count for single function invocation is
1990 lower than this constant, emit fast (but longer) prologue and
1991 epilogue code. */
1992 #define FAST_PROLOGUE_INSN_COUNT 20
1994 /* Names for 8 (low), 8 (high), and 16-bit registers, respectively. */
1999 /* Array of the smallest class containing reg number REGNO, indexed by
2000 REGNO. Used by REGNO_REG_CLASS in i386.h. */
2003 {
2004 /* ax, dx, cx, bx */
2006 /* si, di, bp, sp */
2008 /* FP registers */
2011 /* arg pointer */
2012 NON_Q_REGS,
2013 /* flags, fpsr, fpcr, frame */
2015 /* SSE registers */
2018 /* MMX registers */
2021 /* REX registers */
2024 /* SSE REX registers */
2027 /* AVX-512 SSE registers */
2032 /* Mask registers. */
2035 /* MPX bound registers */
2037 };
2039 /* The "default" register map used in 32bit mode. */
2042 {
2054 };
2056 /* The "default" register map used in 64bit mode. */
2059 {
2071 };
2073 /* Define the register numbers to be used in Dwarf debugging information.
2074 The SVR4 reference port C compiler uses the following register numbers
2075 in its Dwarf output code:
2076 0 for %eax (gcc regno = 0)
2077 1 for %ecx (gcc regno = 2)
2078 2 for %edx (gcc regno = 1)
2079 3 for %ebx (gcc regno = 3)
2080 4 for %esp (gcc regno = 7)
2081 5 for %ebp (gcc regno = 6)
2082 6 for %esi (gcc regno = 4)
2083 7 for %edi (gcc regno = 5)
2084 The following three DWARF register numbers are never generated by
2085 the SVR4 C compiler or by the GNU compilers, but SDB on x86/svr4
2086 believes these numbers have these meanings.
2087 8 for %eip (no gcc equivalent)
2088 9 for %eflags (gcc regno = 17)
2089 10 for %trapno (no gcc equivalent)
2090 It is not at all clear how we should number the FP stack registers
2091 for the x86 architecture. If the version of SDB on x86/svr4 were
2092 a bit less brain dead with respect to floating-point then we would
2093 have a precedent to follow with respect to DWARF register numbers
2094 for x86 FP registers, but the SDB on x86/svr4 is so completely
2095 broken with respect to FP registers that it is hardly worth thinking
2096 of it as something to strive for compatibility with.
2097 The version of x86/svr4 SDB I have at the moment does (partially)
2098 seem to believe that DWARF register number 11 is associated with
2099 the x86 register %st(0), but that's about all. Higher DWARF
2100 register numbers don't seem to be associated with anything in
2101 particular, and even for DWARF regno 11, SDB only seems to under-
2102 stand that it should say that a variable lives in %st(0) (when
2103 asked via an `=' command) if we said it was in DWARF regno 11,
2104 but SDB still prints garbage when asked for the value of the
2105 variable in question (via a `/' command).
2106 (Also note that the labels SDB prints for various FP stack regs
2107 when doing an `x' command are all wrong.)
2108 Note that these problems generally don't affect the native SVR4
2109 C compiler because it doesn't allow the use of -O with -g and
2110 because when it is *not* optimizing, it allocates a memory
2111 location for each floating-point variable, and the memory
2112 location is what gets described in the DWARF AT_location
2113 attribute for the variable in question.
2114 Regardless of the severe mental illness of the x86/svr4 SDB, we
2115 do something sensible here and we use the following DWARF
2116 register numbers. Note that these are all stack-top-relative
2117 numbers.
2118 11 for %st(0) (gcc regno = 8)
2119 12 for %st(1) (gcc regno = 9)
2120 13 for %st(2) (gcc regno = 10)
2121 14 for %st(3) (gcc regno = 11)
2122 15 for %st(4) (gcc regno = 12)
2123 16 for %st(5) (gcc regno = 13)
2124 17 for %st(6) (gcc regno = 14)
2125 18 for %st(7) (gcc regno = 15)
2126 */
2128 {
2140 };
2142 /* Define parameter passing and return registers. */
2145 {
2147 };
2150 {
2152 };
2155 {
2157 };
2159 /* Additional registers that are clobbered by SYSV calls. */
2162 {
2167 };
2169 /* Define the structure for the machine field in struct function. */
2174 rtx rtl;
2176 };
2178 /* Structure describing stack frame layout.
2179 Stack grows downward:
2181 [arguments]
2182 <- ARG_POINTER
2183 saved pc
2185 saved static chain if ix86_static_chain_on_stack
2187 saved frame pointer if frame_pointer_needed
2188 <- HARD_FRAME_POINTER
2189 [saved regs]
2190 <- regs_save_offset
2191 [padding0]
2193 [saved SSE regs]
2194 <- sse_regs_save_offset
2195 [padding1] |
2196 | <- FRAME_POINTER
2197 [va_arg registers] |
2198 |
2199 [frame] |
2200 |
2201 [padding2] | = to_allocate
2202 <- STACK_POINTER
2203 */
2204 struct ix86_frame
2205 {
2212 /* The offsets relative to ARG_POINTER. */
2213 HOST_WIDE_INT frame_pointer_offset;
2214 HOST_WIDE_INT hard_frame_pointer_offset;
2215 HOST_WIDE_INT stack_pointer_offset;
2216 HOST_WIDE_INT hfp_save_offset;
2217 HOST_WIDE_INT reg_save_offset;
2218 HOST_WIDE_INT sse_reg_save_offset;
2220 /* When save_regs_using_mov is set, emit prologue using
2221 move instead of push instructions. */
2223 };
2225 /* Which cpu are we scheduling for. */
2228 /* Which cpu are we optimizing for. */
2231 /* Which instruction set architecture to use. */
2234 /* True if processor has SSE prefetch instruction. */
2237 /* -mstackrealign option */
2254 /* Preferred alignment for stack boundary in bits. */
2257 /* Alignment for incoming stack boundary in bits specified at
2258 command line. */
2261 /* Default alignment for incoming stack boundary in bits. */
2264 /* Alignment for incoming stack boundary in bits. */
2267 /* Calling abi specific va_list type nodes. */
2271 /* Prefix built by ASM_GENERATE_INTERNAL_LABEL. */
2275 /* Fence to use after loop using movnt. */
2276 tree x86_mfence;
2278 /* Register class used for passing given 64bit part of the argument.
2279 These represent classes as documented by the PS ABI, with the exception
2280 of SSESF, SSEDF classes, that are basically SSE class, just gcc will
2281 use SF or DFmode move instead of DImode to avoid reformatting penalties.
2283 Similarly we play games with INTEGERSI_CLASS to use cheaper SImode moves
2284 whenever possible (upper half does contain padding). */
2285 enum x86_64_reg_class
2286 {
2287 X86_64_NO_CLASS,
2288 X86_64_INTEGER_CLASS,
2289 X86_64_INTEGERSI_CLASS,
2290 X86_64_SSE_CLASS,
2291 X86_64_SSESF_CLASS,
2292 X86_64_SSEDF_CLASS,
2293 X86_64_SSEUP_CLASS,
2294 X86_64_X87_CLASS,
2295 X86_64_X87UP_CLASS,
2296 X86_64_COMPLEX_X87_CLASS,
2297 X86_64_MEMORY_CLASS
2298 };
2300 #define MAX_CLASSES 4
2302 /* Table of constants used by fldpi, fldln2, etc.... */
2306 \f
2311 const_tree);
2323 enum ix86_function_specific_strings
2324 {
2325 IX86_FUNCTION_SPECIFIC_ARCH,
2326 IX86_FUNCTION_SPECIFIC_TUNE,
2327 IX86_FUNCTION_SPECIFIC_MAX
2328 };
2349 \f
2350 #ifndef SUBTARGET32_DEFAULT_CPU
2352 #endif
2354 /* Whether -mtune= or -march= were specified */
2358 /* Vectorization library interface and handlers. */
2364 /* Processor target table, indexed by processor number */
2365 struct ptt
2366 {
2373 };
2376 {
2387 /* Core 2 */
2389 /* Core i7 */
2391 /* Core i7 avx */
2393 /* Core avx2 */
2405 };
2408 {
2440 "btver2"
2441 };
2442 \f
2443 static bool
2445 {
2447 }
2449 static unsigned int
2451 {
2454 /* vzeroupper instructions are inserted immediately after reload to
2455 account for possible spills from 256bit registers. The pass
2456 reuses mode switching infrastructure by re-running mode insertion
2457 pass, so disable entities that have already been processed. */
2463 /* Call optimize_mode_switching. */
2466 }
2471 {
2483 };
2486 {
2490 {}
2492 /* opt_pass methods: */
2500 rtl_opt_pass *
2502 {
2504 }
2506 /* Return true if a red-zone is in use. */
2510 {
2512 }
2513 \f
2514 /* Return a string that documents the current -m options. The caller is
2515 responsible for freeing the string. */
2521 {
2522 struct ix86_target_opts
2523 {
2526 };
2528 /* This table is ordered so that options like -msse4.2 that imply
2529 preceding options while match those first. */
2531 {
2572 };
2574 /* Flag options. */
2576 {
2604 };
2621 /* Add -march= option. */
2623 {
2626 }
2628 /* Add -mtune= option. */
2630 {
2633 }
2635 /* Add -m32/-m64/-mx32. */
2637 {
2640 else
2642 isa &= ~ (OPTION_MASK_ISA_64BIT
2643 | OPTION_MASK_ABI_64
2645 }
2646 else
2650 /* Pick out the options in isa options. */
2652 {
2654 {
2657 }
2658 }
2661 {
2664 isa);
2665 }
2667 /* Add flag options. */
2669 {
2671 {
2674 }
2675 }
2678 {
2681 }
2683 /* Add -fpmath= option. */
2685 {
2688 {
2703 }
2704 }
2706 /* Any options? */
2712 /* Size the string. */
2716 {
2721 }
2723 /* Build the string. */
2728 {
2735 {
2737 line_len++;
2740 {
2744 }
2745 }
2749 {
2753 }
2754 }
2760 }
2762 /* Return true, if profiling code should be emitted before
2763 prologue. Otherwise it returns false.
2764 Note: For x86 with "hotfix" it is sorried. */
2765 static bool
2767 {
2769 }
2771 /* Function that is callable from the debugger to print the current
2772 options. */
2773 void ATTRIBUTE_UNUSED
2775 {
2781 {
2784 }
2785 else
2789 }
2792 #define DEF_ENUM
2793 #define DEF_ALG(alg, name) #name,
2795 #undef DEF_ENUM
2796 #undef DEF_ALG
2797 };
2799 /* Parse parameter string passed to -mmemcpy-strategy= or -mmemset-strategy=.
2800 The string is of the following form (or comma separated list of it):
2802 strategy_alg:max_size:[align|noalign]
2804 where the full size range for the strategy is either [0, max_size] or
2805 [min_size, max_size], in which min_size is the max_size + 1 of the
2806 preceding range. The last size range must have max_size == -1.
2808 Examples:
2810 1.
2811 -mmemcpy-strategy=libcall:-1:noalign
2813 this is equivalent to (for known size memcpy) -mstringop-strategy=libcall
2816 2.
2817 -mmemset-strategy=rep_8byte:16:noalign,vector_loop:2048:align,libcall:-1:noalign
2819 This is to tell the compiler to use the following strategy for memset
2820 1) when the expected size is between [1, 16], use rep_8byte strategy;
2821 2) when the size is between [17, 2048], use vector_loop;
2822 3) when the size is > 2048, use libcall. */
2824 struct stringop_size_range
2825 {
2827 stringop_alg alg;
2829 };
2831 static void
2833 {
2841 else
2846 do
2847 {
2849 stringop_alg alg;
2858 {
2862 }
2865 {
2869 }
2872 {
2874 {
2877 }
2878 }
2881 {
2883 alg_name,
2886 }
2894 else
2895 {
2899 }
2900 n++;
2902 }
2906 {
2911 }
2914 {
2918 }
2920 /* Now override the default algs array. */
2922 {
2928 }
2929 }
2931 \f
2932 /* parse -mtune-ctrl= option. When DUMP is true,
2933 print the features that are explicitly set. */
2935 static void
2937 {
2945 do
2946 {
2953 {
2954 curr_feature_string++;
2956 }
2958 {
2960 {
2966 }
2967 }
2972 }
2975 }
2977 /* Helper function to set ix86_tune_features. IX86_TUNE is the
2978 processor type. */
2980 static void
2982 {
2987 {
2990 else
2992 }
2995 {
3000 }
3003 }
3006 /* Override various settings based on options. If MAIN_ARGS_P, the
3007 options are from the command line, otherwise they are from
3008 attributes. */
3010 static void
3014 {
3022 #define PTA_3DNOW (HOST_WIDE_INT_1 << 0)
3023 #define PTA_3DNOW_A (HOST_WIDE_INT_1 << 1)
3024 #define PTA_64BIT (HOST_WIDE_INT_1 << 2)
3025 #define PTA_ABM (HOST_WIDE_INT_1 << 3)
3026 #define PTA_AES (HOST_WIDE_INT_1 << 4)
3027 #define PTA_AVX (HOST_WIDE_INT_1 << 5)
3028 #define PTA_BMI (HOST_WIDE_INT_1 << 6)
3029 #define PTA_CX16 (HOST_WIDE_INT_1 << 7)
3030 #define PTA_F16C (HOST_WIDE_INT_1 << 8)
3031 #define PTA_FMA (HOST_WIDE_INT_1 << 9)
3032 #define PTA_FMA4 (HOST_WIDE_INT_1 << 10)
3033 #define PTA_FSGSBASE (HOST_WIDE_INT_1 << 11)
3034 #define PTA_LWP (HOST_WIDE_INT_1 << 12)
3035 #define PTA_LZCNT (HOST_WIDE_INT_1 << 13)
3036 #define PTA_MMX (HOST_WIDE_INT_1 << 14)
3037 #define PTA_MOVBE (HOST_WIDE_INT_1 << 15)
3038 #define PTA_NO_SAHF (HOST_WIDE_INT_1 << 16)
3039 #define PTA_PCLMUL (HOST_WIDE_INT_1 << 17)
3040 #define PTA_POPCNT (HOST_WIDE_INT_1 << 18)
3041 #define PTA_PREFETCH_SSE (HOST_WIDE_INT_1 << 19)
3042 #define PTA_RDRND (HOST_WIDE_INT_1 << 20)
3043 #define PTA_SSE (HOST_WIDE_INT_1 << 21)
3044 #define PTA_SSE2 (HOST_WIDE_INT_1 << 22)
3045 #define PTA_SSE3 (HOST_WIDE_INT_1 << 23)
3046 #define PTA_SSE4_1 (HOST_WIDE_INT_1 << 24)
3047 #define PTA_SSE4_2 (HOST_WIDE_INT_1 << 25)
3048 #define PTA_SSE4A (HOST_WIDE_INT_1 << 26)
3049 #define PTA_SSSE3 (HOST_WIDE_INT_1 << 27)
3050 #define PTA_TBM (HOST_WIDE_INT_1 << 28)
3051 #define PTA_XOP (HOST_WIDE_INT_1 << 29)
3052 #define PTA_AVX2 (HOST_WIDE_INT_1 << 30)
3053 #define PTA_BMI2 (HOST_WIDE_INT_1 << 31)
3054 #define PTA_RTM (HOST_WIDE_INT_1 << 32)
3055 #define PTA_HLE (HOST_WIDE_INT_1 << 33)
3056 #define PTA_PRFCHW (HOST_WIDE_INT_1 << 34)
3057 #define PTA_RDSEED (HOST_WIDE_INT_1 << 35)
3058 #define PTA_ADX (HOST_WIDE_INT_1 << 36)
3059 #define PTA_FXSR (HOST_WIDE_INT_1 << 37)
3060 #define PTA_XSAVE (HOST_WIDE_INT_1 << 38)
3061 #define PTA_XSAVEOPT (HOST_WIDE_INT_1 << 39)
3062 #define PTA_AVX512F (HOST_WIDE_INT_1 << 40)
3063 #define PTA_AVX512ER (HOST_WIDE_INT_1 << 41)
3064 #define PTA_AVX512PF (HOST_WIDE_INT_1 << 42)
3065 #define PTA_AVX512CD (HOST_WIDE_INT_1 << 43)
3066 #define PTA_MPX (HOST_WIDE_INT_1 << 44)
3068 /* if this reaches 64, need to widen struct pta flags below */
3070 static struct pta
3071 {
3076 }
3078 {
3218 PTA_64BIT
3220 };
3222 /* -mrecip options. */
3223 static struct
3224 {
3227 }
3229 {
3236 };
3240 /* Set up prefix/suffix so the error messages refer to either the command
3241 line argument, or the attribute(target). */
3243 {
3247 }
3248 else
3249 {
3253 }
3255 /* Turn off both OPTION_MASK_ABI_64 and OPTION_MASK_ABI_X32 if
3256 TARGET_64BIT_DEFAULT is true and TARGET_64BIT is false. */
3259 #ifdef TARGET_BI_ARCH
3260 else
3261 {
3262 #if TARGET_BI_ARCH == 1
3263 /* When TARGET_BI_ARCH == 1, by default, OPTION_MASK_ABI_64
3264 is on and OPTION_MASK_ABI_X32 is off. We turn off
3265 OPTION_MASK_ABI_64 if OPTION_MASK_ABI_X32 is turned on by
3266 -mx32. */
3269 #else
3270 /* When TARGET_BI_ARCH == 2, by default, OPTION_MASK_ABI_X32 is
3271 on and OPTION_MASK_ABI_64 is off. We turn off
3272 OPTION_MASK_ABI_X32 if OPTION_MASK_ABI_64 is turned on by
3273 -m64. */
3276 #endif
3277 }
3278 #endif
3281 {
3282 /* Always turn on OPTION_MASK_ISA_64BIT and turn off
3283 OPTION_MASK_ABI_64 for TARGET_X32. */
3286 }
3288 {
3289 /* Always turn on OPTION_MASK_ISA_64BIT and turn off
3290 OPTION_MASK_ABI_X32 for TARGET_LP64. */
3293 }
3295 #ifdef SUBTARGET_OVERRIDE_OPTIONS
3296 SUBTARGET_OVERRIDE_OPTIONS;
3297 #endif
3299 #ifdef SUBSUBTARGET_OVERRIDE_OPTIONS
3300 SUBSUBTARGET_OVERRIDE_OPTIONS;
3301 #endif
3303 /* -fPIC is the default for x86_64. */
3307 /* Need to check -mtune=generic first. */
3309 {
3312 /* As special support for cross compilers we read -mtune=native
3313 as -mtune=generic. With native compilers we won't see the
3314 -mtune=native, as it was changed by the driver. */
3316 {
3318 }
3319 /* If this call is for setting the option attribute, allow the
3320 generic that was previously set. */
3323 ;
3331 }
3332 else
3333 {
3337 {
3340 }
3342 /* opts->x_ix86_tune_string is set to opts->x_ix86_arch_string
3343 or defaulted. We need to use a sensible tune option. */
3347 {
3349 }
3350 }
3354 {
3355 /* rep; movq isn't available in 32-bit code. */
3358 }
3361 opts->x_ix86_arch_string
3364 else
3368 {
3376 }
3377 else
3384 /* For targets using ms ABI enable ms-extensions, if not
3385 explicit turned off. For non-ms ABI we turn off this
3386 option. */
3391 {
3393 {
3437 {
3440 }
3448 }
3449 }
3450 else
3451 {
3452 /* For TARGET_64BIT and MS_ABI, force pic on, in order to enable the
3453 use of rip-relative addressing. This eliminates fixups that
3454 would otherwise be needed if this object is to be placed in a
3455 DLL, and is essentially just as efficient as direct addressing. */
3461 else
3463 }
3465 {
3468 }
3476 {
3479 /* Default cpu tuning to the architecture. */
3618 }
3633 {
3637 {
3639 {
3641 {
3649 }
3650 else
3653 }
3654 }
3655 /* Intel CPUs have always interpreted SSE prefetch instructions as
3656 NOPs; so, we can enable SSE prefetch instructions even when
3657 -mtune (rather than -march) points us to a processor that has them.
3658 However, the VIA C3 gives a SIGILL, so we only do that for i686 and
3659 higher processors. */
3664 }
3672 #ifndef USE_IX86_FRAME_POINTER
3673 #define USE_IX86_FRAME_POINTER 0
3674 #endif
3676 #ifndef USE_X86_64_FRAME_POINTER
3677 #define USE_X86_64_FRAME_POINTER 0
3678 #endif
3680 /* Set the default values for switches whose default depends on TARGET_64BIT
3681 in case they weren't overwritten by command line options. */
3683 {
3687 opts->x_flag_unwind_tables
3691 }
3692 else
3693 {
3695 opts->x_flag_omit_frame_pointer
3701 }
3706 else
3709 /* Arrange to set up i386_stack_locals for all functions. */
3712 /* Validate -mregparm= value. */
3714 {
3718 {
3722 }
3723 }
3727 /* Default align_* from the processor table. */
3729 {
3732 }
3734 {
3737 }
3739 {
3741 }
3743 /* Provide default for -mbranch-cost= value. */
3748 {
3749 opts->x_target_flags
3752 /* Enable by default the SSE and MMX builtins. Do allow the user to
3753 explicitly disable any of these. In particular, disabling SSE and
3754 MMX for kernel code is extremely useful. */
3756 opts->x_ix86_isa_flags
3763 }
3764 else
3765 {
3766 opts->x_target_flags
3770 opts->x_ix86_isa_flags
3773 /* i386 ABI does not specify red zone. It still makes sense to use it
3774 when programmer takes care to stack from being destroyed. */
3777 }
3779 /* Keep nonleaf frame pointers. */
3785 /* If we're doing fast math, we don't care about comparison order
3786 wrt NaNs. This lets us use a shorter comparison sequence. */
3790 /* If the architecture always has an FPU, turn off NO_FANCY_MATH_387,
3791 since the insns won't need emulation. */
3795 /* Likewise, if the target doesn't have a 387, or we've specified
3796 software floating point, don't use 387 inline intrinsics. */
3800 /* Turn on MMX builtins for -msse. */
3802 opts->x_ix86_isa_flags
3805 /* Enable SSE prefetch. */
3810 /* Enable prefetch{,w} instructions for -m3dnow. */
3812 opts->x_ix86_isa_flags
3815 /* Enable popcnt instruction for -msse4.2 or -mabm. */
3818 opts->x_ix86_isa_flags
3821 /* Enable lzcnt instruction for -mabm. */
3823 opts->x_ix86_isa_flags
3826 /* Validate -mpreferred-stack-boundary= value or default it to
3827 PREFERRED_STACK_BOUNDARY_DEFAULT. */
3830 {
3837 {
3841 else
3844 }
3845 else
3846 ix86_preferred_stack_boundary
3848 }
3850 /* Set the default value for -mstackrealign. */
3856 /* Validate -mincoming-stack-boundary= value or default it to
3857 MIN_STACK_BOUNDARY/PREFERRED_STACK_BOUNDARY. */
3860 {
3865 ix86_incoming_stack_boundary_arg,
3867 else
3868 {
3869 ix86_user_incoming_stack_boundary
3871 ix86_incoming_stack_boundary
3873 }
3874 }
3876 /* Accept -msseregparm only if at least SSE support is enabled. */
3882 {
3884 {
3886 {
3889 }
3892 {
3895 }
3896 }
3897 }
3898 /* For all chips supporting SSE2, -mfpmath=sse performs better than
3899 fpmath=387. The second is however default at many targets since the
3900 extra 80bit precision of temporaries is considered to be part of ABI.
3901 Overwrite the default at least for -ffast-math.
3902 TODO: -mfpmath=both seems to produce same performing code with bit
3903 smaller binaries. It is however not clear if register allocation is
3904 ready for this setting.
3905 Also -mfpmath=387 is overall a lot more compact (bout 4-5%) than SSE
3906 codegen. We may switch to 387 with -ffast-math for size optimized
3907 functions. */
3911 else
3914 /* If the i387 is disabled, then do not return values in it. */
3918 /* Use external vectorized library in vectorizing intrinsics. */
3921 {
3932 }
3939 /* If stack probes are required, the space used for large function
3940 arguments on the stack must also be probed, so enable
3941 -maccumulate-outgoing-args so this happens in the prologue. */
3944 {
3949 }
3951 /* Figure out what ASM_GENERATE_INTERNAL_LABEL builds as a prefix. */
3952 {
3958 }
3960 /* When scheduling description is not available, disable scheduler pass
3961 so it won't slow down the compilation and make x87 code slower. */
3982 /* Enable sw prefetching at -O3 for CPUS that prefetching is helpful. */
3984 && HAVE_prefetch
3989 /* If using typedef char *va_list, signal that __builtin_va_start (&ap, 0)
3990 can be opts->x_optimized to ap = __builtin_next_arg (0). */
3995 {
3998 {
4000 ix86_gen_tls_local_dynamic_base_64
4002 }
4003 else
4004 {
4006 ix86_gen_tls_local_dynamic_base_64
4008 }
4009 }
4010 else
4014 {
4024 }
4025 else
4026 {
4036 }
4038 #ifdef USE_IX86_CLD
4039 /* Use -mcld by default for 32-bit code if configured with --enable-cld. */
4042 #endif
4045 {
4050 }
4052 {
4056 }
4058 {
4059 #if defined(PROFILE_BEFORE_PROLOGUE)
4061 #else
4063 #endif
4064 }
4066 /* When not opts->x_optimize for size, enable vzeroupper optimization for
4067 TARGET_AVX with -fexpensive-optimizations and split 32-byte
4068 AVX unaligned load/store. */
4070 {
4080 /* Enable 128-bit AVX instruction generation
4081 for the auto-vectorizer. */
4085 }
4088 {
4095 {
4098 {
4100 q++;
4101 }
4102 else
4107 else
4108 {
4111 {
4114 }
4117 {
4121 }
4122 }
4127 else
4129 }
4130 }
4137 /* Default long double to 64-bit for Bionic. */
4142 /* Save the initial options in case the user does function specific
4143 options. */
4145 target_option_default_node = target_option_current_node
4148 /* Handle stack protector */
4150 opts->x_ix86_stack_protector_guard
4153 /* Handle -mmemcpy-strategy= and -mmemset-strategy= */
4155 {
4159 }
4162 {
4166 }
4167 }
4169 /* Implement the TARGET_OPTION_OVERRIDE hook. */
4171 static void
4173 {
4175 static struct register_pass_info insert_vzeroupper_info
4178 };
4183 /* This needs to be done at start up. It's convenient to do it here. */
4185 }
4187 /* Update register usage after having seen the compiler flags. */
4189 static void
4191 {
4195 /* The PIC register, if it exists, is fixed. */
4200 /* For 32-bit targets, squash the REX registers. */
4202 {
4209 }
4211 /* See the definition of CALL_USED_REGISTERS in i386.h. */
4219 {
4220 /* Set/reset conditionally defined registers from
4221 CALL_USED_REGISTERS initializer. */
4225 /* Calculate registers of CLOBBERED_REGS register set
4226 as call used registers from GENERAL_REGS register set. */
4230 }
4232 /* If MMX is disabled, squash the registers. */
4238 /* If SSE is disabled, squash the registers. */
4244 /* If the FPU is disabled, squash the registers. */
4250 /* If AVX512F is disabled, squash the registers. */
4252 {
4258 }
4260 /* If MPX is disabled, squash the registers. */
4264 }
4266 \f
4267 /* Save the current options */
4269 static void
4272 {
4283 /* The fields are char but the variables are not; make sure the
4284 values fit in the fields. */
4289 }
4291 /* Restore the current options */
4293 static void
4296 {
4312 /* Recreate the arch feature tests if the arch changed */
4314 {
4319 }
4321 /* Recreate the tune optimization tests */
4324 }
4326 /* Print the current options */
4328 static void
4331 {
4353 {
4356 }
4357 }
4359 \f
4360 /* Inner function to process the attribute((target(...))), take an argument and
4361 set the current options from the argument. If we have a list, recursively go
4362 over the list. */
4364 static bool
4369 {
4373 #define IX86_ATTR_ISA(S,O) { S, sizeof (S)-1, ix86_opt_isa, O, 0 }
4374 #define IX86_ATTR_STR(S,O) { S, sizeof (S)-1, ix86_opt_str, O, 0 }
4375 #define IX86_ATTR_ENUM(S,O) { S, sizeof (S)-1, ix86_opt_enum, O, 0 }
4376 #define IX86_ATTR_YES(S,O,M) { S, sizeof (S)-1, ix86_opt_yes, O, M }
4377 #define IX86_ATTR_NO(S,O,M) { S, sizeof (S)-1, ix86_opt_no, O, M }
4379 enum ix86_opt_type
4380 {
4381 ix86_opt_unknown,
4382 ix86_opt_yes,
4383 ix86_opt_no,
4384 ix86_opt_str,
4385 ix86_opt_enum,
4386 ix86_opt_isa
4387 };
4389 static const struct
4390 {
4397 /* isa options */
4438 /* enum options */
4441 /* string options */
4445 /* flag options */
4447 OPT_mcld,
4448 MASK_CLD),
4451 OPT_mfancy_math_387,
4452 MASK_NO_FANCY_MATH_387),
4455 OPT_mieee_fp,
4456 MASK_IEEE_FP),
4459 OPT_minline_all_stringops,
4460 MASK_INLINE_ALL_STRINGOPS),
4463 OPT_minline_stringops_dynamically,
4464 MASK_INLINE_STRINGOPS_DYNAMICALLY),
4467 OPT_mno_align_stringops,
4468 MASK_NO_ALIGN_STRINGOPS),
4471 OPT_mrecip,
4472 MASK_RECIP),
4474 };
4476 /* If this is a list, recurse to get the options. */
4478 {
4485 enum_opts_set))
4489 }
4492 {
4495 }
4497 /* Handle multiple arguments separated by commas. */
4501 {
4515 {
4519 }
4520 else
4521 {
4524 }
4526 /* Recognize no-xxx. */
4528 {
4532 }
4533 else
4536 /* Find the option. */
4540 {
4548 {
4553 }
4554 }
4556 /* Process the option. */
4558 {
4561 }
4564 {
4570 }
4573 {
4579 else
4581 }
4584 {
4586 {
4589 }
4590 else
4592 }
4595 {
4603 global_dc);
4604 else
4605 {
4608 }
4609 }
4611 else
4613 }
4616 }
4618 /* Return a TARGET_OPTION_NODE tree of the target options listed or NULL. */
4620 tree
4624 {
4639 /* Process each of the options on the chain. */
4644 /* If the changed options are different from the default, rerun
4645 ix86_option_override_internal, and then save the options away.
4646 The string options are are attribute options, and will be undone
4647 when we copy the save structure. */
4653 {
4654 /* If we are using the default tune= or arch=, undo the string assigned,
4655 and use the default. */
4666 /* If fpmath= is not set, and we now have sse2 on 32-bit, use it. */
4671 {
4674 }
4676 /* Do any overrides, such as arch=xxx, or tune=xxx support. */
4679 /* Add any builtin functions with the new isa if any. */
4682 /* Save the current options unless we are validating options for
4683 #pragma. */
4690 /* Free up memory allocated to hold the strings */
4693 }
4696 }
4698 /* Hook to validate attribute((target("string"))). */
4700 static bool
4703 tree args,
4705 {
4710 /* attribute((target("default"))) does nothing, beyond
4711 affecting multi-versioning. */
4720 /* Get the optimization options of the current function. */
4726 /* Init func_options. */
4734 /* Initialize func_options to the default before its target options can
4735 be set. */
4748 {
4753 }
4756 }
4758 \f
4759 /* Hook to determine if one function can safely inline another. */
4761 static bool
4763 {
4768 /* If callee has no option attributes, then it is ok to inline. */
4772 /* If caller has no option attributes, but callee does then it is not ok to
4773 inline. */
4777 else
4778 {
4782 /* Callee's isa options should a subset of the caller's, i.e. a SSE4 function
4783 can inline a SSE2 function but a SSE2 function can't inline a SSE4
4784 function. */
4789 /* See if we have the same non-isa options. */
4793 /* See if arch, tune, etc. are the same. */
4806 else
4808 }
4811 }
4813 \f
4814 /* Remember the last target of ix86_set_current_function. */
4817 /* Invalidate ix86_previous_fndecl cache. */
4818 void
4820 {
4822 }
4824 /* Establish appropriate back-end context for processing the function
4825 FNDECL. The argument might be NULL to indicate processing at top
4826 level, outside of any function scope. */
4827 static void
4829 {
4830 /* Only change the context if the function changes. This hook is called
4831 several times in the course of compiling a function, and we don't want to
4832 slow things down too much or call target_reinit when it isn't safe. */
4834 {
4835 tree old_tree = (ix86_previous_fndecl
4839 tree new_tree = (fndecl
4845 ;
4848 {
4852 }
4855 {
4861 }
4862 }
4863 }
4865 \f
4866 /* Return true if this goes in large data/bss. */
4868 static bool
4870 {
4874 /* Functions are never large data. */
4879 {
4885 }
4886 else
4887 {
4890 /* If this is an incomplete type with size 0, then we can't put it
4891 in data because it might be too big when completed. */
4894 }
4897 }
4899 /* Switch to the appropriate section for output of DECL.
4900 DECL is either a `VAR_DECL' node or a constant of some sort.
4901 RELOC indicates whether forming the initial value of DECL requires
4902 link-time relocations. */
4907 {
4910 {
4914 {
4948 /* We don't split these for medium model. Place them into
4949 default sections and hope for best. */
4951 }
4953 {
4954 /* We might get called with string constants, but get_named_section
4955 doesn't like them as they are not DECLs. Also, we need to set
4956 flags in that case. */
4960 }
4961 }
4963 }
4965 /* Select a set of attributes for section NAME based on the properties
4966 of DECL and whether or not RELOC indicates that DECL's initializer
4967 might contain runtime relocations. */
4969 static unsigned int ATTRIBUTE_UNUSED
4971 {
4985 }
4987 /* Build up a unique section name, expressed as a
4988 STRING_CST node, and assign it to DECL_SECTION_NAME (decl).
4989 RELOC indicates whether the initial value of EXP requires
4990 link-time relocations. */
4992 static void ATTRIBUTE_UNUSED
4994 {
4997 {
4999 /* We only need to use .gnu.linkonce if we don't have COMDAT groups. */
5003 {
5027 /* We don't split these for medium model. Place them into
5028 default sections and hope for best. */
5030 }
5032 {
5039 /* If we're using one_only, then there needs to be a .gnu.linkonce
5040 prefix to the section name. */
5047 }
5048 }
5050 }
5052 #ifdef COMMON_ASM_OP
5053 /* This says how to output assembler code to declare an
5054 uninitialized external linkage data object.
5056 For medium model x86-64 we need to use .largecomm opcode for
5057 large objects. */
5058 void
5062 {
5066 else
5071 }
5072 #endif
5074 /* Utility function for targets to use in implementing
5075 ASM_OUTPUT_ALIGNED_BSS. */
5077 void
5081 {
5085 else
5088 #ifdef ASM_DECLARE_OBJECT_NAME
5091 #else
5092 /* Standard thing is just output label for the object. */
5096 }
5097 \f
5098 /* Decide whether we must probe the stack before any space allocation
5099 on this target. It's essentially TARGET_STACK_PROBE except when
5100 -fstack-check causes the stack to be already probed differently. */
5102 bool
5104 {
5105 /* Do not probe the stack twice if static stack checking is enabled. */
5110 }
5111 \f
5112 /* Decide whether we can make a sibling call to a function. DECL is the
5113 declaration of the function being targeted by the call and EXP is the
5114 CALL_EXPR representing the call. */
5116 static bool
5118 {
5122 /* If we are generating position-independent code, we cannot sibcall
5123 optimize any indirect call, or a direct call to a global function,
5124 as the PLT requires %ebx be live. (Darwin does not have a PLT.) */
5126 && !TARGET_64BIT
5127 && flag_pic
5131 /* If we need to align the outgoing stack, then sibcalling would
5132 unalign the stack, which may break the called function. */
5138 {
5141 }
5142 else
5143 {
5144 /* We're looking at the CALL_EXPR, we need the type of the function. */
5149 }
5151 /* Check that the return value locations are the same. Like
5152 if we are returning floats on the 80387 register stack, we cannot
5153 make a sibcall from a function that doesn't return a float to a
5154 function that does or, conversely, from a function that does return
5155 a float to a function that doesn't; the necessary stack adjustment
5156 would not be executed. This is also the place we notice
5157 differences in the return value ABI. Note that it is ok for one
5158 of the functions to have void return type as long as the return
5159 value of the other is passed in a register. */
5164 {
5167 }
5169 ;
5174 {
5175 /* The SYSV ABI has more call-clobbered registers;
5176 disallow sibcalls from MS to SYSV. */
5180 }
5181 else
5182 {
5183 /* If this call is indirect, we'll need to be able to use a
5184 call-clobbered register for the address of the target function.
5185 Make sure that all such registers are not used for passing
5186 parameters. Note that DLLIMPORT functions are indirect. */
5189 {
5191 {
5192 /* ??? Need to count the actual number of registers to be used,
5193 not the possible number of registers. Fix later. */
5195 }
5196 }
5197 }
5199 /* Otherwise okay. That also includes certain types of indirect calls. */
5201 }
5203 /* Handle "cdecl", "stdcall", "fastcall", "regparm", "thiscall",
5204 and "sseregparm" calling convention attributes;
5205 arguments as in struct attribute_spec.handler. */
5207 static tree
5209 tree args,
5212 {
5217 {
5219 name);
5222 }
5224 /* Can combine regparm with all attributes but fastcall, and thiscall. */
5226 {
5227 tree cst;
5230 {
5232 }
5235 {
5237 }
5241 {
5244 name);
5246 }
5248 {
5252 }
5255 }
5258 {
5259 /* Do not warn when emulating the MS ABI. */
5264 name);
5267 }
5269 /* Can combine fastcall with stdcall (redundant) and sseregparm. */
5271 {
5273 {
5275 }
5277 {
5279 }
5281 {
5283 }
5285 {
5287 }
5288 }
5290 /* Can combine stdcall with fastcall (redundant), regparm and
5291 sseregparm. */
5293 {
5295 {
5297 }
5299 {
5301 }
5303 {
5305 }
5306 }
5308 /* Can combine cdecl with regparm and sseregparm. */
5310 {
5312 {
5314 }
5316 {
5318 }
5320 {
5322 }
5323 }
5325 {
5328 name);
5330 {
5332 }
5334 {
5336 }
5338 {
5340 }
5341 }
5343 /* Can combine sseregparm with all attributes. */
5346 }
5348 /* The transactional memory builtins are implicitly regparm or fastcall
5349 depending on the ABI. Override the generic do-nothing attribute that
5350 these builtins were declared with, and replace it with one of the two
5351 attributes that we expect elsewhere. */
5353 static tree
5355 tree args ATTRIBUTE_UNUSED,
5357 {
5358 tree alt;
5360 /* In no case do we want to add the placeholder attribute. */
5363 /* The 64-bit ABI is unchanged for transactional memory. */
5367 /* ??? Is there a better way to validate 32-bit windows? We have
5368 cfun->machine->call_abi, but that seems to be set only for 64-bit. */
5371 else
5372 {
5375 }
5379 }
5381 /* This function determines from TYPE the calling-convention. */
5383 unsigned int
5385 {
5388 tree attrs;
5395 {
5405 /* Regparam isn't allowed for thiscall and fastcall. */
5407 {
5412 }
5416 }
5423 || is_stdarg
5429 }
5431 /* Return 0 if the attributes for two types are incompatible, 1 if they
5432 are compatible, and 2 if they are nearly compatible (which causes a
5433 warning to be generated). */
5435 static int
5437 {
5453 }
5454 \f
5455 /* Return the regparm value for a function with the indicated TYPE and DECL.
5456 DECL may be NULL when calling function indirectly
5457 or considering a libcall. */
5459 static int
5461 {
5462 tree attr;
5473 {
5476 {
5479 }
5480 }
5486 /* Use register calling convention for local functions when possible. */
5489 && optimize
5491 {
5492 /* FIXME: remove this CONST_CAST when cgraph.[ch] is constified. */
5495 {
5498 /* Make sure no regparm register is taken by a
5499 fixed register variable. */
5504 /* We don't want to use regparm(3) for nested functions as
5505 these use a static chain pointer in the third argument. */
5509 /* In 32-bit mode save a register for the split stack. */
5513 /* Each fixed register usage increases register pressure,
5514 so less registers should be used for argument passing.
5515 This functionality can be overriden by an explicit
5516 regparm value. */
5519 globals++;
5521 local_regparm
5526 }
5527 }
5530 }
5532 /* Return 1 or 2, if we can pass up to SSE_REGPARM_MAX SFmode (1) and
5533 DFmode (2) arguments in SSE registers for a function with the
5534 indicated TYPE and DECL. DECL may be NULL when calling function
5535 indirectly or considering a libcall. Otherwise return 0. */
5537 static int
5539 {
5542 /* Use SSE registers to pass SFmode and DFmode arguments if requested
5543 by the sseregparm attribute. */
5546 {
5548 {
5550 {
5554 else
5557 }
5559 }
5562 }
5564 /* For local functions, pass up to SSE_REGPARM_MAX SFmode
5565 (and DFmode for SSE2) arguments in SSE registers. */
5568 {
5569 /* FIXME: remove this CONST_CAST when cgraph.[ch] is constified. */
5573 }
5576 }
5578 /* Return true if EAX is live at the start of the function. Used by
5579 ix86_expand_prologue to determine if we need special help before
5580 calling allocate_stack_worker. */
5582 static bool
5584 {
5585 /* Cheat. Don't bother working forward from ix86_function_regparm
5586 to the function type to whether an actual argument is located in
5587 eax. Instead just look at cfg info, which is still close enough
5588 to correct at this point. This gives false positives for broken
5589 functions that might use uninitialized data that happens to be
5590 allocated in eax, but who cares? */
5592 }
5594 static bool
5596 {
5597 tree attr;
5600 {
5606 /* For 32-bit MS-ABI the default is to keep aggregate
5607 return pointer. */
5610 }
5612 }
5614 /* Value is the number of bytes of arguments automatically
5615 popped when returning from a subroutine call.
5616 FUNDECL is the declaration node of the function (as a tree),
5617 FUNTYPE is the data type of the function (as a tree),
5618 or for a library call it is an identifier node for the subroutine name.
5619 SIZE is the number of bytes of arguments passed on the stack.
5621 On the 80386, the RTD insn may be used to pop them if the number
5622 of args is fixed, but if the number is variable then the caller
5623 must pop them all. RTD can't be used for library calls now
5624 because the library is compiled with the Unix compiler.
5625 Use of RTD is a selectable option, since it is incompatible with
5626 standard Unix calling sequences. If the option is not selected,
5627 the caller must always pop the args.
5629 The attribute stdcall is equivalent to RTD on a per module basis. */
5631 static int
5633 {
5636 /* None of the 64-bit ABIs pop arguments. */
5647 /* Lose any fake structure return argument if it is passed on the stack. */
5650 {
5654 }
5657 }
5659 /* Implement the TARGET_LEGITIMATE_COMBINED_INSN hook. */
5661 static bool
5663 {
5664 /* Check operand constraints in case hard registers were propagated
5665 into insn pattern. This check prevents combine pass from
5666 generating insn patterns with invalid hard register operands.
5667 These invalid insns can eventually confuse reload to error out
5668 with a spill failure. See also PRs 46829 and 46843. */
5670 {
5677 {
5685 /* A unary operator may be accepted by the predicate, but it
5686 is irrelevant for matching constraints. */
5691 {
5699 }
5706 /* Operand has no constraints, anything is OK. */
5710 {
5713 && operands_match_p
5717 {
5720 }
5721 }
5725 }
5726 }
5729 }
5730 \f
5731 /* Implement the TARGET_ASAN_SHADOW_OFFSET hook. */
5733 static unsigned HOST_WIDE_INT
5735 {
5739 }
5740 \f
5741 /* Argument support functions. */
5743 /* Return true when register may be used to pass function parameters. */
5744 bool
5746 {
5751 {
5755 else
5761 }
5767 /* TODO: The function should depend on current function ABI but
5768 builtins.c would need updating then. Therefore we use the
5769 default ABI. */
5771 /* RAX is used as hidden argument to va_arg functions. */
5777 else
5784 }
5786 /* Return if we do not know how to pass TYPE solely in registers. */
5788 static bool
5790 {
5794 /* For 32-bit, we want TImode aggregates to go on the stack. But watch out!
5795 The layout_type routine is crafty and tries to trick us into passing
5796 currently unsupported vector types on the stack by using TImode. */
5799 }
5801 /* It returns the size, in bytes, of the area reserved for arguments passed
5802 in registers for the function represented by fndecl dependent to the used
5803 abi format. */
5804 int
5806 {
5810 else
5815 }
5817 /* Returns value SYSV_ABI, MS_ABI dependent on fntype, specifying the
5818 call abi used. */
5819 enum calling_abi
5821 {
5823 {
5826 {
5829 }
5833 }
5835 }
5837 /* We add this as a workaround in order to use libc_has_function
5838 hook in i386.md. */
5839 bool
5841 {
5843 }
5845 static bool
5847 {
5849 {
5853 else
5855 }
5857 }
5859 static enum calling_abi
5861 {
5865 }
5867 /* Returns value SYSV_ABI, MS_ABI dependent on cfun, specifying the
5868 call abi used. */
5869 enum calling_abi
5871 {
5875 }
5877 /* Write the extra assembler code needed to declare a function properly. */
5879 void
5881 tree decl)
5882 {
5886 {
5892 }
5894 #ifdef SUBTARGET_ASM_UNWIND_INIT
5896 #endif
5900 /* Output magic byte marker, if hot-patch attribute is set. */
5902 {
5904 {
5905 /* leaq [%rsp + 0], %rsp */
5908 }
5909 else
5910 {
5911 /* movl.s %edi, %edi
5912 push %ebp
5913 movl.s %esp, %ebp */
5916 }
5917 }
5918 }
5920 /* regclass.c */
5923 /* Implementation of call abi switching target hook. Specific to FNDECL
5924 the specific call register sets are set. See also
5925 ix86_conditional_register_usage for more details. */
5926 void
5928 {
5931 else
5933 }
5935 /* 64-bit MS and SYSV ABI have different set of call used registers. Avoid
5936 expensive re-initialization of init_regs each time we switch function context
5937 since this is needed only during RTL expansion. */
5938 static void
5940 {
5944 }
5946 /* Initialize a variable CUM of type CUMULATIVE_ARGS
5947 for a call to a function whose data type is FNTYPE.
5948 For a library call, FNTYPE is 0. */
5950 void
5954 tree fndecl,
5956 {
5962 {
5965 }
5966 else
5967 {
5970 }
5974 /* Set up the number of registers to use for passing arguments. */
5981 {
5983 ? X86_64_REGPARM_MAX
5985 }
5987 {
5990 {
5992 ? X86_64_SSE_REGPARM_MAX
5994 }
5995 }
6002 /* Because type might mismatch in between caller and callee, we need to
6003 use actual type of function for local calls.
6004 FIXME: cgraph_analyze can be told to actually record if function uses
6005 va_start so for local functions maybe_vaarg can be made aggressive
6006 helping K&R code.
6007 FIXME: once typesytem is fixed, we won't need this code anymore. */
6015 {
6016 /* If there are variable arguments, then we won't pass anything
6017 in registers in 32-bit mode. */
6019 {
6027 }
6029 /* Use ecx and edx registers if function has fastcall attribute,
6030 else look for regparm information. */
6032 {
6035 {
6038 }
6040 {
6043 }
6044 else
6046 }
6048 /* Set up the number of SSE registers used for passing SFmode
6049 and DFmode arguments. Warn for mismatching ABI. */
6051 }
6052 }
6054 /* Return the "natural" mode for TYPE. In most cases, this is just TYPE_MODE.
6055 But in the case of vector types, it is some vector mode.
6057 When we have only some of our vector isa extensions enabled, then there
6058 are some modes for which vector_mode_supported_p is false. For these
6059 modes, the generic vector support in gcc will choose some non-vector mode
6060 in order to implement the type. By computing the natural mode, we'll
6061 select the proper ABI location for the operand and not depend on whatever
6062 the middle-end decides to do with these vector types.
6064 The midde-end can't deal with the vector types > 16 bytes. In this
6065 case, we return the original mode and warn ABI change if CUM isn't
6066 NULL. */
6068 static enum machine_mode
6070 {
6074 {
6077 /* ??? Generic code allows us to create width 1 vectors. Ignore. */
6079 {
6084 else
6087 /* Get the mode which has this inner mode and number of units. */
6091 {
6093 {
6097 && !warnedavx
6099 {
6103 }
6105 }
6107 {
6111 && !warnedsse
6113 {
6117 }
6119 }
6120 else
6122 }
6125 }
6126 }
6129 }
6131 /* We want to pass a value in REGNO whose "natural" mode is MODE. However,
6132 this may not agree with the mode that the type system has chosen for the
6133 register, which is ORIG_MODE. If ORIG_MODE is not BLKmode, then we can
6134 go ahead and use it. Otherwise we have to build a PARALLEL instead. */
6136 static rtx
6139 {
6140 rtx tmp;
6144 else
6145 {
6149 }
6152 }
6154 /* x86-64 register passing implementation. See x86-64 ABI for details. Goal
6155 of this code is to classify each 8bytes of incoming argument by the register
6156 class and assign registers accordingly. */
6158 /* Return the union class of CLASS1 and CLASS2.
6159 See the x86-64 PS ABI for details. */
6161 static enum x86_64_reg_class
6163 {
6164 /* Rule #1: If both classes are equal, this is the resulting class. */
6168 /* Rule #2: If one of the classes is NO_CLASS, the resulting class is
6169 the other class. */
6175 /* Rule #3: If one of the classes is MEMORY, the result is MEMORY. */
6179 /* Rule #4: If one of the classes is INTEGER, the result is INTEGER. */
6187 /* Rule #5: If one of the classes is X87, X87UP, or COMPLEX_X87 class,
6188 MEMORY is used. */
6197 /* Rule #6: Otherwise class SSE is used. */
6199 }
6201 /* Classify the argument of type TYPE and mode MODE.
6202 CLASSES will be filled by the register class used to pass each word
6203 of the operand. The number of words is returned. In case the parameter
6204 should be passed in memory, 0 is returned. As a special case for zero
6205 sized containers, classes[0] will be NO_CLASS and 1 is returned.
6207 BIT_OFFSET is used internally for handling records and specifies offset
6208 of the offset in bits modulo 256 to avoid overflow cases.
6210 See the x86-64 PS ABI for details.
6211 */
6213 static int
6216 {
6217 HOST_WIDE_INT bytes =
6219 int words
6222 /* Variable sized entities are always passed/returned in memory. */
6231 {
6233 tree field;
6236 /* On x86-64 we pass structures larger than 32 bytes on the stack. */
6243 /* Zero sized arrays or structures are NO_CLASS. We return 0 to
6244 signalize memory class, so handle it as special case. */
6246 {
6249 }
6251 /* Classify each field of record and merge classes. */
6253 {
6255 /* And now merge the fields of structure. */
6257 {
6259 {
6265 /* Bitfields are always classified as integer. Handle them
6266 early, since later code would consider them to be
6267 misaligned integers. */
6269 {
6278 }
6279 else
6280 {
6285 /* Flexible array member is ignored. */
6292 {
6296 {
6299 "the ABI of passing struct with"
6300 " a flexible array member has"
6302 }
6304 }
6306 subclasses,
6316 }
6317 }
6318 }
6322 /* Arrays are handled as small records. */
6323 {
6330 /* The partial classes are now full classes. */
6341 }
6344 /* Unions are similar to RECORD_TYPE but offset is always 0.
6345 */
6347 {
6349 {
6357 bit_offset);
6362 }
6363 }
6368 }
6371 {
6372 /* When size > 16 bytes, if the first one isn't
6373 X86_64_SSE_CLASS or any other ones aren't
6374 X86_64_SSEUP_CLASS, everything should be passed in
6375 memory. */
6382 }
6384 /* Final merger cleanup. */
6386 {
6387 /* If one class is MEMORY, everything should be passed in
6388 memory. */
6392 /* The X86_64_SSEUP_CLASS should be always preceded by
6393 X86_64_SSE_CLASS or X86_64_SSEUP_CLASS. */
6397 {
6398 /* The first one should never be X86_64_SSEUP_CLASS. */
6401 }
6403 /* If X86_64_X87UP_CLASS isn't preceded by X86_64_X87_CLASS,
6404 everything should be passed in memory. */
6407 {
6410 /* The first one should never be X86_64_X87UP_CLASS. */
6413 {
6416 "the ABI of passing union with long double"
6418 }
6420 }
6421 }
6423 }
6425 /* Compute alignment needed. We align all types to natural boundaries with
6426 exception of XFmode that is aligned to 64bits. */
6428 {
6437 /* Misaligned fields are always returned in memory. */
6440 }
6442 /* for V1xx modes, just use the base mode */
6447 /* Classification of atomic types. */
6449 {
6465 {
6469 {
6472 }
6474 {
6477 }
6479 {
6483 }
6485 {
6488 }
6489 else
6491 }
6498 /* OImode shouldn't be used directly. */
6505 else
6523 else
6524 {
6528 {
6531 "the ABI of passing structure with complex float"
6533 }
6536 }
6545 /* This modes is larger than 16 bytes. */
6588 else
6592 }
6593 }
6595 /* Examine the argument and return set number of register required in each
6596 class. Return 0 iff parameter should be passed in memory. */
6597 static int
6600 {
6610 {
6632 }
6634 }
6636 /* Construct container for the argument used by GCC interface. See
6637 FUNCTION_ARG for the detailed description. */
6639 static rtx
6643 {
6644 /* The following variables hold the static issued_error state. */
6658 rtx ret;
6669 /* We allowed the user to turn off SSE for kernel mode. Don't crash if
6670 some less clueful developer tries to use floating-point anyway. */
6672 {
6674 {
6676 {
6679 }
6680 }
6682 {
6685 }
6687 }
6689 /* Likewise, error if the ABI requires us to return values in the
6690 x87 registers and the user specified -mno-80387. */
6696 {
6698 {
6701 }
6703 }
6705 /* First construct simple cases. Avoid SCmode, since we want to use
6706 single register to pass this type. */
6709 {
6724 /* Zero sized array, struct or class. */
6728 }
6755 /* Otherwise figure out the entries of the PARALLEL. */
6757 {
6761 {
6766 /* Merge TImodes on aligned occasions here too. */
6768 tmpmode
6772 else
6774 /* We've requested 24 bytes we
6775 don't have mode for. Use DImode. */
6782 intreg++;
6790 sse_regno++;
6798 sse_regno++;
6803 {
6809 {
6811 i++;
6812 }
6813 else
6826 }
6832 sse_regno++;
6836 }
6837 }
6839 /* Empty aligned struct, union or class. */
6847 }
6849 /* Update the data in CUM to advance over an argument of mode MODE
6850 and data type TYPE. (TYPE is null for libcalls where that information
6851 may not be available.) */
6853 static void
6856 HOST_WIDE_INT words)
6857 {
6859 {
6866 /* FALLTHRU */
6877 {
6880 }
6884 /* OImode shouldn't be used directly. */
6893 /* FALLTHRU */
6909 {
6914 {
6917 }
6918 }
6928 {
6933 {
6936 }
6937 }
6939 }
6940 }
6942 static void
6945 {
6948 /* Unnamed 256bit vector mode parameters are passed on stack. */
6954 {
6959 }
6960 else
6961 {
6965 }
6966 }
6968 static void
6970 HOST_WIDE_INT words)
6971 {
6972 /* Otherwise, this should be passed indirect. */
6977 {
6980 }
6981 }
6983 /* Update the data in CUM to advance over an argument of mode MODE and
6984 data type TYPE. (TYPE is null for libcalls where that information
6985 may not be available.) */
6987 static void
6990 {
6996 else
7007 else
7009 }
7011 /* Define where to put the arguments to a function.
7012 Value is zero to push the argument on the stack,
7013 or a hard register in which to store the argument.
7015 MODE is the argument's machine mode.
7016 TYPE is the data type of the argument (as a tree).
7017 This is null for libcalls where that information may
7018 not be available.
7019 CUM is a variable of type CUMULATIVE_ARGS which gives info about
7020 the preceding args and about the function being called.
7021 NAMED is nonzero if this argument is a named parameter
7022 (otherwise it is an extra parameter matching an ellipsis). */
7024 static rtx
7028 {
7031 /* Avoid the AL settings for the Unix64 ABI. */
7036 {
7043 /* FALLTHRU */
7049 {
7052 /* Fastcall allocates the first two DWORD (SImode) or
7053 smaller arguments to ECX and EDX if it isn't an
7054 aggregate type . */
7056 {
7062 /* ECX not EAX is the first allocated register. */
7065 }
7067 }
7076 /* FALLTHRU */
7078 /* In 32bit, we pass TImode in xmm registers. */
7086 {
7088 {
7092 }
7096 }
7100 /* OImode shouldn't be used directly. */
7110 {
7114 }
7124 {
7126 {
7130 }
7134 }
7136 }
7139 }
7141 static rtx
7144 {
7145 /* Handle a hidden AL argument containing number of registers
7146 for varargs x86-64 functions. */
7150 ? X86_64_SSE_REGPARM_MAX
7155 {
7165 /* Unnamed 256bit vector mode parameters are passed on stack. */
7169 }
7175 }
7177 static rtx
7180 HOST_WIDE_INT bytes)
7181 {
7184 /* We need to add clobber for MS_ABI->SYSV ABI calls in expand_call.
7185 We use value of -2 to specify that current function call is MSABI. */
7189 /* If we've run out of registers, it goes on the stack. */
7195 /* Only floating point modes are passed in anything but integer regs. */
7197 {
7200 else
7201 {
7204 /* Unnamed floating parameters are passed in both the
7205 SSE and integer registers. */
7211 }
7212 }
7213 /* Handle aggregated types passed in register. */
7215 {
7220 }
7223 }
7225 /* Return where to put the arguments to a function.
7226 Return zero to push the argument on the stack, or a hard register in which to store the argument.
7228 MODE is the argument's machine mode. TYPE is the data type of the
7229 argument. It is null for libcalls where that information may not be
7230 available. CUM gives information about the preceding args and about
7231 the function being called. NAMED is nonzero if this argument is a
7232 named parameter (otherwise it is an extra parameter matching an
7233 ellipsis). */
7235 static rtx
7238 {
7242 rtx arg;
7246 else
7250 /* To simplify the code below, represent vector types with a vector mode
7251 even if MMX/SSE are not active. */
7259 else
7263 }
7265 /* A C expression that indicates when an argument must be passed by
7266 reference. If nonzero for an argument, a copy of that argument is
7267 made in memory and a pointer to the argument is passed instead of
7268 the argument itself. The pointer is passed in whatever way is
7269 appropriate for passing a pointer to that type. */
7271 static bool
7274 {
7277 /* See Windows x64 Software Convention. */
7279 {
7282 {
7283 /* Arrays are passed by reference. */
7288 {
7289 /* Structs/unions of sizes other than 8, 16, 32, or 64 bits
7290 are passed by reference. */
7292 }
7293 }
7295 /* __m128 is passed by reference. */
7301 }
7302 }
7307 }
7309 /* Return true when TYPE should be 128bit aligned for 32bit argument
7310 passing ABI. XXX: This function is obsolete and is only used for
7311 checking psABI compatibility with previous versions of GCC. */
7313 static bool
7315 {
7327 {
7328 /* Walk the aggregates recursively. */
7330 {
7334 {
7335 tree field;
7337 /* Walk all the structure fields. */
7339 {
7343 }
7345 }
7348 /* Just for use if some languages passes arrays by value. */
7355 }
7356 }
7358 }
7360 /* Return the alignment boundary for MODE and TYPE with alignment ALIGN.
7361 XXX: This function is obsolete and is only used for checking psABI
7362 compatibility with previous versions of GCC. */
7364 static unsigned int
7367 {
7368 /* In 32bit, only _Decimal128 and __float128 are aligned to their
7369 natural boundaries. */
7371 {
7372 /* i386 ABI defines all arguments to be 4 byte aligned. We have to
7373 make an exception for SSE modes since these require 128bit
7374 alignment.
7376 The handling here differs from field_alignment. ICC aligns MMX
7377 arguments to 4 byte boundaries, while structure fields are aligned
7378 to 8 byte boundaries. */
7380 {
7383 }
7384 else
7385 {
7388 }
7389 }
7393 }
7395 /* Return true when TYPE should be 128bit aligned for 32bit argument
7396 passing ABI. */
7398 static bool
7400 {
7410 {
7411 /* Walk the aggregates recursively. */
7413 {
7417 {
7418 tree field;
7420 /* Walk all the structure fields. */
7422 field;
7424 {
7428 }
7430 }
7433 /* Just for use if some languages passes arrays by value. */
7440 }
7441 }
7442 else
7446 }
7448 /* Gives the alignment boundary, in bits, of an argument with the
7449 specified mode and type. */
7451 static unsigned int
7453 {
7456 {
7457 /* Since the main variant type is used for call, we convert it to
7458 the main variant type. */
7461 }
7462 else
7466 else
7467 {
7472 {
7473 /* i386 ABI defines XFmode arguments to be 4 byte aligned. */
7475 {
7478 }
7484 }
7487 && !warned
7489 saved_align))
7490 {
7493 "The ABI for passing parameters with %d-byte"
7496 }
7497 }
7500 }
7502 /* Return true if N is a possible register number of function value. */
7504 static bool
7506 {
7508 {
7516 /* Complex values are returned in %st(0)/%st(1) pair. */
7519 /* TODO: The function should depend on current function ABI but
7520 builtins.c would need updating then. Therefore we use the
7521 default ABI. */
7526 /* Complex values are returned in %xmm0/%xmm1 pair. */
7535 }
7538 }
7540 /* Define how to find the value returned by a function.
7541 VALTYPE is the data type of the value (as a tree).
7542 If the precise function being called is known, FUNC is its FUNCTION_DECL;
7543 otherwise, FUNC is 0. */
7545 static rtx
7548 {
7551 /* 8-byte vector modes in %mm0. See ix86_return_in_memory for where
7552 we normally prevent this case when mmx is not available. However
7553 some ABIs may require the result to be returned like DImode. */
7557 /* 16-byte vector modes in %xmm0. See ix86_return_in_memory for where
7558 we prevent this case when sse is not available. However some ABIs
7559 may require the result to be returned like integer TImode. */
7564 /* 32-byte vector modes in %ymm0. */
7568 /* Floating point return values in %st(0) (unless -mno-fp-ret-in-387). */
7571 else
7572 /* Most things go in %eax. */
7575 /* Override FP return register with %xmm0 for local functions when
7576 SSE math is enabled or for functions with sseregparm attribute. */
7578 {
7583 }
7585 /* OImode shouldn't be used directly. */
7589 }
7591 static rtx
7593 const_tree valtype)
7594 {
7595 rtx ret;
7597 /* Handle libcalls, which don't provide a type node. */
7599 {
7603 {
7622 }
7625 }
7627 {
7628 /* Pointers are always returned in word_mode. */
7630 }
7636 /* For zero sized structures, construct_container returns NULL, but we
7637 need to keep rest of compiler happy by returning meaningful value. */
7642 }
7644 static rtx
7646 const_tree valtype)
7647 {
7651 {
7653 {
7672 }
7673 }
7675 }
7677 static rtx
7680 {
7692 else
7694 }
7696 static rtx
7699 {
7705 }
7707 /* Pointer function arguments and return values are promoted to
7708 word_mode. */
7710 static enum machine_mode
7714 {
7716 {
7719 }
7721 for_return);
7722 }
7724 /* Return true if a structure, union or array with MODE containing FIELD
7725 should be accessed using BLKmode. */
7727 static bool
7729 {
7730 /* Union with XFmode must be in BLKmode. */
7734 }
7736 rtx
7738 {
7740 }
7742 /* Return true iff type is returned in memory. */
7744 static bool ATTRIBUTE_UNUSED
7746 {
7747 HOST_WIDE_INT size;
7758 {
7759 /* User-created vectors small enough to fit in EAX. */
7763 /* MMX/3dNow values are returned in MM0,
7764 except when it doesn't exits or the ABI prescribes otherwise. */
7768 /* SSE values are returned in XMM0, except when it doesn't exist. */
7772 /* AVX values are returned in YMM0, except when it doesn't exist. */
7775 }
7783 /* OImode shouldn't be used directly. */
7787 }
7789 static bool ATTRIBUTE_UNUSED
7791 {
7794 }
7796 static bool ATTRIBUTE_UNUSED
7798 {
7801 /* __m128 is returned in xmm0. */
7808 /* Otherwise, the size must be exactly in [1248]. */
7810 }
7812 static bool
7814 {
7815 #ifdef SUBTARGET_RETURN_IN_MEMORY
7817 #else
7821 {
7824 else
7826 }
7827 else
7829 #endif
7830 }
7832 /* When returning SSE vector types, we have a choice of either
7833 (1) being abi incompatible with a -march switch, or
7834 (2) generating an error.
7835 Given no good solution, I think the safest thing is one warning.
7836 The user won't be able to use -Werror, but....
7838 Choose the STRUCT_VALUE_RTX hook because that's (at present) only
7839 called in response to actually generating a caller or callee that
7840 uses such a type. As opposed to TARGET_RETURN_IN_MEMORY, which is called
7841 via aggregate_value_p for general type probing from tree-ssa. */
7843 static rtx
7845 {
7849 {
7850 /* Look at the return type of the function, not the function type. */
7854 {
7857 {
7861 }
7862 }
7865 {
7867 {
7871 }
7872 }
7873 }
7876 }
7878 \f
7879 /* Create the va_list data type. */
7881 /* Returns the calling convention specific va_list date type.
7882 The argument ABI can be DEFAULT_ABI, MS_ABI, or SYSV_ABI. */
7884 static tree
7886 {
7889 /* For i386 we use plain pointer to argument area. */
7899 unsigned_type_node);
7902 unsigned_type_node);
7905 ptr_type_node);
7908 ptr_type_node);
7927 /* The correct type is an array type of one element. */
7929 }
7931 /* Setup the builtin va_list data type and for 64-bit the additional
7932 calling convention specific va_list data types. */
7934 static tree
7936 {
7939 /* Initialize abi specific va_list builtin types. */
7941 {
7942 tree t;
7944 {
7949 }
7950 else
7951 {
7956 }
7958 {
7963 }
7964 else
7965 {
7970 }
7971 }
7974 }
7976 /* Worker function for TARGET_SETUP_INCOMING_VARARGS. */
7978 static void
7980 {
7982 alias_set_type set;
7985 /* GPR size of varargs save area. */
7988 else
7991 /* FPR size of varargs save area. We don't need it if we don't pass
7992 anything in SSE registers. */
7995 else
8009 {
8017 }
8020 {
8024 /* Now emit code to save SSE registers. The AX parameter contains number
8025 of SSE parameter registers used to call this function, though all we
8026 actually check here is the zero/non-zero status. */
8031 label));
8033 /* ??? If !TARGET_SSE_TYPELESS_STORES, would we perform better if
8034 we used movdqa (i.e. TImode) instead? Perhaps even better would
8035 be if we could determine the real mode of the data, via a hook
8036 into pass_stdarg. Ignore all that for now. */
8046 {
8055 }
8058 }
8059 }
8061 static void
8063 {
8067 /* Reset to zero, as there might be a sysv vaarg used
8068 before. */
8073 {
8084 }
8085 }
8087 static void
8091 {
8093 CUMULATIVE_ARGS next_cum;
8094 tree fntype;
8096 /* This argument doesn't appear to be used anymore. Which is good,
8097 because the old code here didn't suppress rtl generation. */
8105 /* For varargs, we do not want to skip the dummy va_dcl argument.
8106 For stdargs, we do want to skip the last named argument. */
8114 else
8116 }
8118 /* Checks if TYPE is of kind va_list char *. */
8120 static bool
8122 {
8123 tree canonic;
8125 /* For 32-bit it is always true. */
8131 }
8133 /* Implement va_start. */
8135 static void
8137 {
8141 tree type;
8142 rtx ovf_rtx;
8146 {
8149 /* When we are splitting the stack, we can't refer to the stack
8150 arguments using internal_arg_pointer, because they may be on
8151 the old stack. The split stack prologue will arrange to
8152 leave a pointer to the old stack arguments in a scratch
8153 register, which we here copy to a pseudo-register. The split
8154 stack prologue can't set the pseudo-register directly because
8155 it (the prologue) runs before any registers have been saved. */
8159 {
8173 }
8174 }
8176 /* Only 64bit target needs something special. */
8178 {
8181 else
8182 {
8191 }
8193 }
8202 /* The following should be folded into the MEM_REF offset. */
8212 /* Count number of gp and fp argument registers used. */
8218 {
8224 }
8227 {
8233 }
8235 /* Find the overflow area. */
8239 else
8249 {
8250 /* Find the register save area.
8251 Prologue of the function save it right above stack frame. */
8259 }
8260 }
8262 /* Implement va_arg. */
8264 static tree
8267 {
8274 rtx container;
8276 tree ptrtype;
8280 /* Only 64bit target needs something special. */
8304 {
8311 /* Unnamed 256bit vector mode parameters are passed on stack. */
8313 {
8316 }
8324 }
8326 /* Pull the value out of the saved registers. */
8331 {
8345 /* In case we are passing structure, verify that it is consecutive block
8346 on the register save area. If not we need to do moves. */
8348 {
8349 /* Verify that all registers are strictly consecutive */
8351 {
8355 {
8360 }
8361 }
8362 else
8363 {
8367 {
8372 }
8373 }
8374 }
8376 {
8379 }
8380 else
8381 {
8384 }
8386 /* First ensure that we fit completely in registers. */
8388 {
8395 }
8397 {
8405 }
8407 /* Compute index to start of area used for integer regs. */
8409 {
8410 /* int_addr = gpr + sav; */
8413 }
8415 {
8416 /* sse_addr = fpr + sav; */
8419 }
8421 {
8425 /* addr = &temp; */
8430 {
8434 tree piece_type;
8435 tree addr_type;
8436 tree daddr_type;
8445 {
8450 }
8454 else
8461 {
8464 }
8465 else
8466 {
8469 }
8476 {
8481 }
8482 else
8483 {
8484 tree copy
8489 }
8491 }
8492 }
8495 {
8499 }
8502 {
8506 }
8511 }
8513 /* ... otherwise out of the overflow area. */
8515 /* When we align parameter on stack for caller, if the parameter
8516 alignment is beyond MAX_SUPPORTED_STACK_ALIGNMENT, it will be
8517 aligned at MAX_SUPPORTED_STACK_ALIGNMENT. We will match callee
8518 here with caller. */
8523 /* Care for on-stack alignment if needed. */
8526 else
8527 {
8532 }
8549 }
8550 \f
8551 /* Return true if OPNUM's MEM should be matched
8552 in movabs* patterns. */
8554 bool
8556 {
8568 }
8569 \f
8570 /* Initialize the table of extra 80387 mathematical constants. */
8572 static void
8574 {
8576 {
8582 };
8586 {
8588 /* Ensure each constant is rounded to XFmode precision. */
8591 }
8594 }
8596 /* Return non-zero if the constant is something that
8597 can be loaded with a special instruction. */
8599 int
8601 {
8604 REAL_VALUE_TYPE r;
8616 /* For XFmode constants, try to find a special 80387 instruction when
8617 optimizing for size or on those CPUs that benefit from them. */
8620 {
8629 }
8631 /* Load of the constant -0.0 or -1.0 will be split as
8632 fldz;fchs or fld1;fchs sequence. */
8639 }
8641 /* Return the opcode of the special instruction to be used to load
8642 the constant X. */
8646 {
8648 {
8668 }
8669 }
8671 /* Return the CONST_DOUBLE representing the 80387 constant that is
8672 loaded by the specified special instruction. The argument IDX
8673 matches the return value from standard_80387_constant_p. */
8675 rtx
8677 {
8684 {
8695 }
8698 XFmode);
8699 }
8701 /* Return 1 if X is all 0s and 2 if x is all 1s
8702 in supported SSE/AVX vector mode. */
8704 int
8706 {
8713 {
8728 }
8731 }
8733 /* Return the opcode of the special instruction to be used to load
8734 the constant X. */
8738 {
8740 {
8743 {
8760 }
8769 else
8774 }
8776 }
8778 /* Returns true if OP contains a symbol reference */
8780 bool
8782 {
8791 {
8793 {
8799 }
8803 }
8806 }
8808 /* Return true if it is appropriate to emit `ret' instructions in the
8809 body of a function. Do this only if the epilogue is simple, needing a
8810 couple of insns. Prior to reloading, we can't tell how many registers
8811 must be saved, so return false then. Return false if there is no frame
8812 marker to de-allocate. */
8814 bool
8816 {
8822 /* Don't allow more than 32k pop, since that's all we can do
8823 with one instruction. */
8830 }
8831 \f
8832 /* Value should be nonzero if functions must have frame pointers.
8833 Zero means the frame pointer need not be set up (and parms may
8834 be accessed via the stack pointer) in functions that seem suitable. */
8836 static bool
8838 {
8839 /* If we accessed previous frames, then the generated code expects
8840 to be able to access the saved ebp value in our frame. */
8844 /* Several x86 os'es need a frame pointer for other reasons,
8845 usually pertaining to setjmp. */
8849 /* For older 32-bit runtimes setjmp requires valid frame-pointer. */
8853 /* Win64 SEH, very large frames need a frame-pointer as maximum stack
8854 allocation is 4GB. */
8858 /* In ix86_option_override_internal, TARGET_OMIT_LEAF_FRAME_POINTER
8859 turns off the frame pointer by default. Turn it back on now if
8860 we've not got a leaf function. */
8870 }
8872 /* Record that the current function accesses previous call frames. */
8874 void
8876 {
8878 }
8879 \f
8880 #ifndef USE_HIDDEN_LINKONCE
8881 # if defined(HAVE_GAS_HIDDEN) && (SUPPORTS_ONE_ONLY - 0)
8882 # define USE_HIDDEN_LINKONCE 1
8883 # else
8884 # define USE_HIDDEN_LINKONCE 0
8885 # endif
8886 #endif
8890 /* Fills in the label name that should be used for a pc thunk for
8891 the given register. */
8893 static void
8895 {
8900 else
8902 }
8905 /* This function generates code for -fpic that loads %ebx with
8906 the return address of the caller and then returns. */
8908 static void
8910 {
8915 {
8917 tree decl;
8933 #if TARGET_MACHO
8935 {
8944 }
8945 else
8946 #endif
8948 {
8959 }
8960 else
8961 {
8964 }
8970 /* Make sure unwind info is emitted for the thunk if needed. */
8973 /* Pad stack IP move with 4 instructions (two NOPs count
8974 as one instruction). */
8976 {
8981 }
8992 }
8996 }
8998 /* Emit code for the SET_GOT patterns. */
9002 {
9008 {
9009 /* Load (*VXWORKS_GOTT_BASE) into the PIC register. */
9014 /* Load (*VXWORKS_GOTT_BASE)[VXWORKS_GOTT_INDEX] into the PIC register.
9015 Use %P and a local symbol in order to print VXWORKS_GOTT_INDEX as
9016 an unadorned address. */
9021 }
9026 {
9028 /* We don't need a pic base, we're not producing pic. */
9035 }
9036 else
9037 {
9046 #if TARGET_MACHO
9047 /* Output the Mach-O "canonical" pic base label name ("Lxx$pb") here.
9048 This is what will be referenced by the Mach-O PIC subsystem. */
9052 /* When we are restoring the pic base at the site of a nonlocal label,
9053 and we decided to emit the pic base above, we will still output a
9054 local label used for calculating the correction offset (even though
9055 the offset will be 0 in that case). */
9059 #endif
9060 }
9066 }
9068 /* Generate an "push" pattern for input ARG. */
9070 static rtx
9072 {
9085 stack_pointer_rtx)),
9086 arg);
9087 }
9089 /* Generate an "pop" pattern for input ARG. */
9091 static rtx
9093 {
9098 arg,
9101 stack_pointer_rtx)));
9102 }
9104 /* Return >= 0 if there is an unused call-clobbered register available
9105 for the entire function. */
9107 static unsigned int
9109 {
9113 {
9115 /* Can't use the same register for both PIC and DRAP. */
9118 else
9123 }
9126 }
9128 /* Return TRUE if we need to save REGNO. */
9130 static bool
9132 {
9143 {
9146 {
9152 }
9153 }
9162 }
9164 /* Return number of saved general prupose registers. */
9166 static int
9168 {
9174 nregs ++;
9176 }
9178 /* Return number of saved SSE registrers. */
9180 static int
9182 {
9190 nregs ++;
9192 }
9194 /* Given FROM and TO register numbers, say whether this elimination is
9195 allowed. If stack alignment is needed, we can only replace argument
9196 pointer with hard frame pointer, or replace frame pointer with stack
9197 pointer. Otherwise, frame pointer elimination is automatically
9198 handled and all other eliminations are valid. */
9200 static bool
9202 {
9208 else
9210 }
9212 /* Return the offset between two registers, one to be eliminated, and the other
9213 its replacement, at the start of a routine. */
9215 HOST_WIDE_INT
9217 {
9226 else
9227 {
9235 }
9236 }
9238 /* In a dynamically-aligned function, we can't know the offset from
9239 stack pointer to frame pointer, so we must ensure that setjmp
9240 eliminates fp against the hard fp (%ebp) rather than trying to
9241 index from %esp up to the top of the frame across a gap that is
9242 of unknown (at compile-time) size. */
9243 static rtx
9245 {
9247 }
9249 /* When using -fsplit-stack, the allocation routines set a field in
9250 the TCB to the bottom of the stack plus this much space, measured
9251 in bytes. */
9253 #define SPLIT_STACK_AVAILABLE 256
9255 /* Fill structure ix86_frame about frame of currently computed function. */
9257 static void
9259 {
9261 HOST_WIDE_INT offset;
9264 HOST_WIDE_INT to_allocate;
9272 /* 64-bit MS ABI seem to require stack alignment to be always 16 except for
9273 function prologues and leaf. */
9277 {
9282 }
9288 /* For SEH we have to limit the amount of code movement into the prologue.
9289 At present we do this via a BLOCKAGE, at which point there's very little
9290 scheduling that can be done, which means that there's very little point
9291 in doing anything except PUSHs. */
9295 /* During reload iteration the amount of registers saved can change.
9296 Recompute the value as needed. Do not recompute when amount of registers
9297 didn't change as reload does multiple calls to the function and does not
9298 expect the decision to change within single iteration. */
9301 {
9307 /* The fast prologue uses move instead of push to save registers. This
9308 is significantly longer, but also executes faster as modern hardware
9309 can execute the moves in parallel, but can't do that for push/pop.
9311 Be careful about choosing what prologue to emit: When function takes
9312 many instructions to execute we may use slow version as well as in
9313 case function is known to be outside hot spot (this is known with
9314 feedback only). Weight the size of function by number of registers
9315 to save as it is cheap to use one or two push instructions but very
9316 slow to use many of them. */
9320 || (flag_branch_probabilities
9323 else
9326 }
9328 frame->save_regs_using_mov
9330 /* If static stack checking is enabled and done with probes,
9331 the registers need to be saved before allocating the frame. */
9334 /* Skip return address. */
9337 /* Skip pushed static chain. */
9341 /* Skip saved base pointer. */
9346 /* The traditional frame pointer location is at the top of the frame. */
9349 /* Register save area */
9353 /* On SEH target, registers are pushed just before the frame pointer
9354 location. */
9358 /* Align and set SSE register save area. */
9360 {
9361 /* The only ABI that has saved SSE registers (Win64) also has a
9362 16-byte aligned default stack, and thus we don't need to be
9363 within the re-aligned local stack frame to save them. */
9367 }
9370 /* The re-aligned stack starts here. Values before this point are not
9371 directly comparable with values below this point. In order to make
9372 sure that no value happens to be the same before and after, force
9373 the alignment computation below to add a non-zero value. */
9377 /* Va-arg area */
9381 /* Align start of frame for local function. */
9390 /* Frame pointer points here. */
9395 /* Add outgoing arguments area. Can be skipped if we eliminated
9396 all the function calls as dead code.
9397 Skipping is however impossible when function calls alloca. Alloca
9398 expander assumes that last crtl->outgoing_args_size
9399 of stack frame are unused. */
9403 {
9406 }
9407 else
9410 /* Align stack boundary. Only needed if we're calling another function
9411 or using alloca. */
9416 /* We've reached end of stack frame. */
9419 /* Size prologue needs to allocate. */
9430 {
9436 }
9437 else
9441 /* The SEH frame pointer location is near the bottom of the frame.
9442 This is enforced by the fact that the difference between the
9443 stack pointer and the frame pointer is limited to 240 bytes in
9444 the unwind data structure. */
9446 {
9447 HOST_WIDE_INT diff;
9449 /* If we can leave the frame pointer where it is, do so. Also, returns
9450 the establisher frame for __builtin_frame_address (0). */
9455 {
9456 /* Ideally we'd determine what portion of the local stack frame
9457 (within the constraint of the lowest 240) is most heavily used.
9458 But without that complication, simply bias the frame pointer
9459 by 128 bytes so as to maximize the amount of the local stack
9460 frame that is addressable with 8-bit offsets. */
9462 }
9463 }
9464 }
9466 /* This is semi-inlined memory_address_length, but simplified
9467 since we know that we're always dealing with reg+offset, and
9468 to avoid having to create and discard all that rtl. */
9472 {
9476 {
9477 /* EBP and R13 cannot be encoded without an offset. */
9479 }
9483 /* ESP and R12 must be encoded with a SIB byte. */
9485 len++;
9488 }
9490 /* Return an RTX that points to CFA_OFFSET within the stack frame.
9491 The valid base registers are taken from CFUN->MACHINE->FS. */
9493 static rtx
9495 {
9501 {
9502 /* Choose the base register most likely to allow the most scheduling
9503 opportunities. Generally FP is valid throughout the function,
9504 while DRAP must be reloaded within the epilogue. But choose either
9505 over the SP due to increased encoding size. */
9508 {
9511 }
9513 {
9516 }
9518 {
9521 }
9522 }
9523 else
9524 {
9525 HOST_WIDE_INT toffset;
9528 /* Choose the base register with the smallest address encoding.
9529 With a tie, choose FP > DRAP > SP. */
9531 {
9535 }
9537 {
9541 {
9545 }
9546 }
9548 {
9552 {
9556 }
9557 }
9558 }
9562 }
9564 /* Emit code to save registers in the prologue. */
9566 static void
9568 {
9570 rtx insn;
9574 {
9577 }
9578 }
9580 /* Emit a single register save at CFA - CFA_OFFSET. */
9582 static void
9584 HOST_WIDE_INT cfa_offset)
9585 {
9593 /* For SSE saves, we need to indicate the 128-bit alignment. */
9604 /* When saving registers into a re-aligned local stack frame, avoid
9605 any tricky guessing by dwarf2out. */
9607 {
9611 {
9612 /* A bit of a hack. We force the DRAP register to be saved in
9613 the re-aligned stack frame, which provides us with a copy
9614 of the CFA that will last past the prologue. Install it. */
9620 }
9621 else
9622 {
9623 /* The frame pointer is a stable reference within the
9624 aligned frame. Use it. */
9631 }
9632 }
9634 /* The memory may not be relative to the current CFA register,
9635 which means that we may need to generate a new pattern for
9636 use by the unwind info. */
9638 {
9643 }
9644 }
9646 /* Emit code to save registers using MOV insns.
9647 First register is stored at CFA - CFA_OFFSET. */
9648 static void
9650 {
9655 {
9658 }
9659 }
9661 /* Emit code to save SSE registers using MOV insns.
9662 First register is stored at CFA - CFA_OFFSET. */
9663 static void
9665 {
9670 {
9673 }
9674 }
9678 /* Add a REG_CFA_RESTORE REG note to INSN or queue them until next stack
9679 manipulation insn. The value is on the stack at CFA - CFA_OFFSET.
9680 Don't add the note if the previously saved value will be left untouched
9681 within stack red-zone till return, as unwinders can find the same value
9682 in the register and on the stack. */
9684 static void
9686 {
9692 {
9695 }
9696 else
9697 queued_cfa_restores
9699 }
9701 /* Add queued REG_CFA_RESTORE notes if any to INSN. */
9703 static void
9705 {
9706 rtx last;
9710 ;
9715 }
9717 /* Expand prologue or epilogue stack adjustment.
9718 The pattern exist to put a dependency on all ebp-based memory accesses.
9719 STYLE should be negative if instructions should be marked as frame related,
9720 zero if %r11 register is live and cannot be freely used and positive
9721 otherwise. */
9723 static void
9726 {
9728 rtx insn;
9735 else
9736 {
9737 rtx tmp;
9738 /* r11 is used by indirect sibcall return as well, set before the
9739 epilogue and used after the epilogue. */
9742 else
9743 {
9747 }
9753 }
9760 {
9761 rtx r;
9771 }
9773 {
9776 {
9780 }
9781 }
9784 {
9789 {
9792 }
9794 {
9797 }
9798 else
9799 {
9800 /* Else there are two possibilities: SP itself, which we set
9801 up as the default above. Or EH_RETURN_STACKADJ_RTX, which is
9802 taken care of this by hand along the eh_return path. */
9805 }
9809 }
9810 }
9812 /* Find an available register to be used as dynamic realign argument
9813 pointer regsiter. Such a register will be written in prologue and
9814 used in begin of body, so it must not be
9815 1. parameter passing register.
9816 2. GOT pointer.
9817 We reuse static-chain register if it is available. Otherwise, we
9818 use DI for i386 and R13 for x86-64. We chose R13 since it has
9819 shorter encoding.
9821 Return: the regno of chosen register. */
9823 static unsigned int
9825 {
9829 {
9830 /* Use R13 for nested function or function need static chain.
9831 Since function with tail call may use any caller-saved
9832 registers in epilogue, DRAP must not use caller-saved
9833 register in such case. */
9838 }
9839 else
9840 {
9841 /* Use DI for nested function or function need static chain.
9842 Since function with tail call may use any caller-saved
9843 registers in epilogue, DRAP must not use caller-saved
9844 register in such case. */
9848 /* Reuse static chain register if it isn't used for parameter
9849 passing. */
9851 {
9855 }
9857 }
9858 }
9860 /* Return minimum incoming stack alignment. */
9862 static unsigned int
9864 {
9867 /* Prefer the one specified at command line. */
9870 /* In 32bit, use MIN_STACK_BOUNDARY for incoming stack boundary
9871 if -mstackrealign is used, it isn't used for sibcall check and
9872 estimated stack alignment is 128bit. */
9874 && !TARGET_64BIT
9875 && ix86_force_align_arg_pointer
9878 else
9881 /* Incoming stack alignment can be changed on individual functions
9882 via force_align_arg_pointer attribute. We use the smallest
9883 incoming stack boundary. */
9889 /* The incoming stack frame has to be aligned at least at
9890 parm_stack_boundary. */
9894 /* Stack at entrance of main is aligned by runtime. We use the
9895 smallest incoming stack boundary. */
9903 }
9905 /* Update incoming stack boundary and estimated stack alignment. */
9907 static void
9909 {
9910 ix86_incoming_stack_boundary
9913 /* x86_64 vararg needs 16byte stack alignment for register save
9914 area. */
9919 }
9921 /* Handle the TARGET_GET_DRAP_RTX hook. Return NULL if no DRAP is
9922 needed or an rtx for DRAP otherwise. */
9924 static rtx
9926 {
9931 {
9932 /* Assign DRAP to vDRAP and returns vDRAP */
9934 rtx drap_vreg;
9935 rtx arg_ptr;
9948 {
9951 }
9953 }
9954 else
9956 }
9958 /* Handle the TARGET_INTERNAL_ARG_POINTER hook. */
9960 static rtx
9962 {
9964 }
9967 rtx reg;
9969 };
9971 /* Return a short-lived scratch register for use on function entry.
9972 In 32-bit mode, it is valid only after the registers are saved
9973 in the prologue. This register must be released by means of
9974 release_scratch_register_on_entry once it is dead. */
9976 static void
9978 {
9984 {
9985 /* We always use R11 in 64-bit mode. */
9987 }
9988 else
9989 {
9991 bool fastcall_p
9993 bool thiscall_p
9997 int drap_regno
10000 /* 'fastcall' sets regparm to 2, uses ecx/edx for arguments and eax
10001 for the static chain register. */
10005 /* 'thiscall' sets regparm to 1, uses ecx for arguments and edx
10006 for the static chain register. */
10011 /* ecx is the static chain register. */
10013 && !static_chain_p
10018 /* esi is the static chain register. */
10024 else
10025 {
10028 }
10029 }
10033 {
10036 }
10037 }
10039 /* Release a scratch register obtained from the preceding function. */
10041 static void
10043 {
10045 {
10049 /* The RTX_FRAME_RELATED_P mechanism doesn't know about pop. */
10055 }
10056 }
10058 #define PROBE_INTERVAL (1 << STACK_CHECK_PROBE_INTERVAL_EXP)
10060 /* Emit code to adjust the stack pointer by SIZE bytes while probing it. */
10062 static void
10064 {
10065 /* We skip the probe for the first interval + a small dope of 4 words and
10066 probe that many bytes past the specified size to maintain a protection
10067 area at the botton of the stack. */
10071 /* See if we have a constant small number of probes to generate. If so,
10072 that's the easy case. The run-time loop is made up of 11 insns in the
10073 generic case while the compile-time loop is made up of 3+2*(n-1) insns
10074 for n # of intervals. */
10076 {
10080 /* Adjust SP and probe at PROBE_INTERVAL + N * PROBE_INTERVAL for
10081 values of N from 1 until it exceeds SIZE. If only one probe is
10082 needed, this will not generate any code. Then adjust and probe
10083 to PROBE_INTERVAL + SIZE. */
10085 {
10087 {
10090 }
10091 else
10098 }
10102 else
10110 /* Adjust back to account for the additional first interval. */
10114 }
10116 /* Otherwise, do the same as above, but in a loop. Note that we must be
10117 extra careful with variables wrapping around because we might be at
10118 the very top (or the very bottom) of the address space and we have
10119 to be able to handle this case properly; in particular, we use an
10120 equality test for the loop condition. */
10121 else
10122 {
10123 HOST_WIDE_INT rounded_size;
10129 /* Step 1: round SIZE to the previous multiple of the interval. */
10134 /* Step 2: compute initial and final value of the loop counter. */
10136 /* SP = SP_0 + PROBE_INTERVAL. */
10141 /* LAST_ADDR = SP_0 + PROBE_INTERVAL + ROUNDED_SIZE. */
10145 stack_pointer_rtx)));
10148 /* Step 3: the loop
10150 while (SP != LAST_ADDR)
10151 {
10152 SP = SP + PROBE_INTERVAL
10153 probe at SP
10154 }
10156 adjusts SP and probes to PROBE_INTERVAL + N * PROBE_INTERVAL for
10157 values of N from 1 until it is equal to ROUNDED_SIZE. */
10162 /* Step 4: adjust SP and probe at PROBE_INTERVAL + SIZE if we cannot
10163 assert at compile-time that SIZE is equal to ROUNDED_SIZE. */
10166 {
10171 }
10173 /* Adjust back to account for the additional first interval. */
10179 }
10183 /* Even if the stack pointer isn't the CFA register, we need to correctly
10184 describe the adjustments made to it, in particular differentiate the
10185 frame-related ones from the frame-unrelated ones. */
10187 {
10200 }
10202 /* Make sure nothing is scheduled before we are done. */
10204 }
10206 /* Adjust the stack pointer up to REG while probing it. */
10210 {
10220 /* Jump to END_LAB if SP == LAST_ADDR. */
10228 /* SP = SP + PROBE_INTERVAL. */
10232 /* Probe at SP. */
10243 }
10245 /* Emit code to probe a range of stack addresses from FIRST to FIRST+SIZE,
10246 inclusive. These are offsets from the current stack pointer. */
10248 static void
10250 {
10251 /* See if we have a constant small number of probes to generate. If so,
10252 that's the easy case. The run-time loop is made up of 7 insns in the
10253 generic case while the compile-time loop is made up of n insns for n #
10254 of intervals. */
10256 {
10257 HOST_WIDE_INT i;
10259 /* Probe at FIRST + N * PROBE_INTERVAL for values of N from 1 until
10260 it exceeds SIZE. If only one probe is needed, this will not
10261 generate any code. Then probe at FIRST + SIZE. */
10268 }
10270 /* Otherwise, do the same as above, but in a loop. Note that we must be
10271 extra careful with variables wrapping around because we might be at
10272 the very top (or the very bottom) of the address space and we have
10273 to be able to handle this case properly; in particular, we use an
10274 equality test for the loop condition. */
10275 else
10276 {
10283 /* Step 1: round SIZE to the previous multiple of the interval. */
10288 /* Step 2: compute initial and final value of the loop counter. */
10290 /* TEST_OFFSET = FIRST. */
10293 /* LAST_OFFSET = FIRST + ROUNDED_SIZE. */
10297 /* Step 3: the loop
10299 while (TEST_ADDR != LAST_ADDR)
10300 {
10301 TEST_ADDR = TEST_ADDR + PROBE_INTERVAL
10302 probe at TEST_ADDR
10303 }
10305 probes at FIRST + N * PROBE_INTERVAL for values of N from 1
10306 until it is equal to ROUNDED_SIZE. */
10311 /* Step 4: probe at FIRST + SIZE if we cannot assert at compile-time
10312 that SIZE is equal to ROUNDED_SIZE. */
10317 stack_pointer_rtx,
10322 }
10324 /* Make sure nothing is scheduled before we are done. */
10326 }
10328 /* Probe a range of stack addresses from REG to END, inclusive. These are
10329 offsets from the current stack pointer. */
10333 {
10343 /* Jump to END_LAB if TEST_ADDR == LAST_ADDR. */
10351 /* TEST_ADDR = TEST_ADDR + PROBE_INTERVAL. */
10355 /* Probe at TEST_ADDR. */
10368 }
10370 /* Finalize stack_realign_needed flag, which will guide prologue/epilogue
10371 to be generated in correct form. */
10372 static void
10374 {
10375 /* Check if stack realign is really needed after reload, and
10376 stores result in cfun */
10377 unsigned int incoming_stack_boundary
10386 {
10387 /* After stack_realign_needed is finalized, we can't no longer
10388 change it. */
10391 }
10393 /* If the only reason for frame_pointer_needed is that we conservatively
10394 assumed stack realignment might be needed, but in the end nothing that
10395 needed the stack alignment had been spilled, clear frame_pointer_needed
10396 and say we don't need stack realignment. */
10399 && frame_pointer_needed
10401 && flag_omit_frame_pointer
10403 && !ix86_current_function_calls_tls_descriptor
10412 {
10414 basic_block bb;
10421 HARD_FRAME_POINTER_REGNUM);
10423 {
10424 rtx insn;
10428 set_up_by_prologue))
10429 {
10433 }
10434 }
10448 }
10452 }
10454 /* Expand the prologue into a bunch of separate insns. */
10456 void
10458 {
10463 HOST_WIDE_INT allocate;
10469 /* DRAP should not coexist with stack_realign_fp */
10474 /* Initialize CFA state for before the prologue. */
10478 /* Track SP offset to the CFA. We continue tracking this after we've
10479 swapped the CFA register away from SP. In the case of re-alignment
10480 this is fudged; we're interested to offsets within the local frame. */
10487 {
10488 /* We should have already generated an error for any use of
10489 ms_hook on a nested function. */
10492 /* Check if profiling is active and we shall use profiling before
10493 prologue variant. If so sorry. */
10498 /* In ix86_asm_output_function_label we emitted:
10499 8b ff movl.s %edi,%edi
10500 55 push %ebp
10501 8b ec movl.s %esp,%ebp
10503 This matches the hookable function prologue in Win32 API
10504 functions in Microsoft Windows XP Service Pack 2 and newer.
10505 Wine uses this to enable Windows apps to hook the Win32 API
10506 functions provided by Wine.
10508 What that means is that we've already set up the frame pointer. */
10512 {
10515 /* We've decided to use the frame pointer already set up.
10516 Describe this to the unwinder by pretending that both
10517 push and mov insns happen right here.
10519 Putting the unwind info here at the end of the ms_hook
10520 is done so that we can make absolutely certain we get
10521 the required byte sequence at the start of the function,
10522 rather than relying on an assembler that can produce
10523 the exact encoding required.
10525 However it does mean (in the unpatched case) that we have
10526 a 1 insn window where the asynchronous unwind info is
10527 incorrect. However, if we placed the unwind info at
10528 its correct location we would have incorrect unwind info
10529 in the patched case. Which is probably all moot since
10530 I don't expect Wine generates dwarf2 unwind info for the
10531 system libraries that use this feature. */
10537 stack_pointer_rtx);
10545 /* Note that gen_push incremented m->fs.cfa_offset, even
10546 though we didn't emit the push insn here. */
10550 }
10551 else
10552 {
10553 /* The frame pointer is not needed so pop %ebp again.
10554 This leaves us with a pristine state. */
10556 }
10557 }
10559 /* The first insn of a function that accepts its static chain on the
10560 stack is to push the register that would be filled in by a direct
10561 call. This insn will be skipped by the trampoline. */
10563 {
10567 /* We don't want to interpret this push insn as a register save,
10568 only as a stack adjustment. The real copy of the register as
10569 a save will be done later, if needed. */
10574 }
10576 /* Emit prologue code to adjust stack alignment and setup DRAP, in case
10577 of DRAP is needed and stack realignment is really needed after reload */
10579 {
10582 /* Only need to push parameter pointer reg if it is caller saved. */
10584 {
10585 /* Push arg pointer reg */
10588 }
10590 /* Grab the argument pointer. */
10597 /* Align the stack. */
10599 stack_pointer_rtx,
10603 /* Replicate the return address on the stack so that return
10604 address can be reached via (argp - 1) slot. This is needed
10605 to implement macro RETURN_ADDR_RTX and intrinsic function
10606 expand_builtin_return_addr etc. */
10612 /* For the purposes of frame and register save area addressing,
10613 we've started over with a new frame. */
10616 }
10622 {
10623 /* Note: AT&T enter does NOT have reversed args. Enter is probably
10624 slower on all targets. Also sdb doesn't like it. */
10628 /* Push registers now, before setting the frame pointer
10629 on SEH target. */
10631 && TARGET_SEH
10633 {
10637 }
10640 {
10648 }
10649 }
10652 {
10653 /* If saving registers via PUSH, do so now. */
10655 {
10659 }
10661 /* When using red zone we may start register saving before allocating
10662 the stack frame saving one cycle of the prologue. However, avoid
10663 doing this if we have to probe the stack; at least on x86_64 the
10664 stack probe can turn into a call that clobbers a red zone location. */
10666 && (! TARGET_STACK_PROBE
10668 {
10671 }
10672 }
10675 {
10679 /* The computation of the size of the re-aligned stack frame means
10680 that we must allocate the size of the register save area before
10681 performing the actual alignment. Otherwise we cannot guarantee
10682 that there's enough storage above the realignment point. */
10689 /* Align the stack. */
10691 stack_pointer_rtx,
10694 /* For the purposes of register save area addressing, the stack
10695 pointer is no longer valid. As for the value of sp_offset,
10696 see ix86_compute_frame_layout, which we need to match in order
10697 to pass verification of stack_pointer_offset at the end. */
10700 }
10705 {
10706 /* We start to count from ARG_POINTER. */
10709 /* If it was realigned, take into account the fake frame. */
10711 {
10718 /* This over-estimates by 1 minimal-stack-alignment-unit but
10719 mitigates that by counting in the new return address slot. */
10720 current_function_dynamic_stack_size
10722 }
10725 }
10727 /* On SEH target with very large frame size, allocate an area to save
10728 SSE registers (as the very large allocation won't be described). */
10732 {
10733 HOST_WIDE_INT sse_size =
10738 /* No need to do stack checking as the area will be immediately
10739 written. */
10746 }
10748 /* The stack has already been decremented by the instruction calling us
10749 so probe if the size is non-negative to preserve the protection area. */
10751 {
10752 /* We expect the registers to be saved when probes are used. */
10756 {
10759 {
10762 }
10763 }
10764 else
10765 {
10772 {
10774 {
10777 }
10778 else
10780 }
10781 else
10782 {
10784 {
10788 }
10789 else
10791 }
10792 }
10793 }
10796 ;
10799 {
10803 }
10804 else
10805 {
10818 /* Note that SEH directives need to continue tracking the stack
10819 pointer even after the frame pointer has been set up. */
10821 {
10825 {
10829 }
10830 }
10833 {
10838 {
10842 }
10843 }
10848 /* Use the fact that AX still contains ALLOCATE. */
10850 ? gen_pro_epilogue_adjust_stack_di_sub
10857 {
10865 }
10869 {
10876 }
10878 {
10883 }
10884 }
10887 /* If we havn't already set up the frame pointer, do so now. */
10889 {
10901 }
10909 /* We don't use pic-register for pe-coff target. */
10911 && !TARGET_PECOFF
10914 {
10921 }
10924 {
10926 {
10928 {
10938 label));
10942 }
10943 else
10945 }
10946 else
10947 {
10951 }
10952 }
10954 /* In the pic_reg_used case, make sure that the got load isn't deleted
10955 when mcount needs it. Blockage to avoid call movement across mcount
10956 call is emitted in generic code after the NOTE_INSN_PROLOGUE_END
10957 note. */
10962 {
10963 /* vDRAP is setup but after reload it turns out stack realign
10964 isn't necessary, here we will emit prologue to setup DRAP
10965 without stack realign adjustment */
10968 }
10970 /* Prevent instructions from being scheduled into register save push
10971 sequence when access to the redzone area is done through frame pointer.
10972 The offset between the frame pointer and the stack pointer is calculated
10973 relative to the value of the stack pointer at the end of the function
10974 prologue, and moving instructions that access redzone area via frame
10975 pointer inside push sequence violates this assumption. */
10979 /* Emit cld instruction if stringops are used in the function. */
10983 /* SEH requires that the prologue end within 256 bytes of the start of
10984 the function. Prevent instruction schedules that would extend that.
10985 Further, prevent alloca modifications to the stack pointer from being
10986 combined with prologue modifications. */
10989 }
10991 /* Emit code to restore REG using a POP insn. */
10993 static void
10995 {
11004 {
11005 /* Previously we'd represented the CFA as an expression
11006 like *(%ebp - 8). We've just popped that value from
11007 the stack, which means we need to reset the CFA to
11008 the drap register. This will remain until we restore
11009 the stack pointer. */
11013 /* This means that the DRAP register is valid for addressing too. */
11016 }
11019 {
11026 }
11028 /* When the frame pointer is the CFA, and we pop it, we are
11029 swapping back to the stack pointer as the CFA. This happens
11030 for stack frames that don't allocate other data, so we assume
11031 the stack pointer is now pointing at the return address, i.e.
11032 the function entry state, which makes the offset be 1 word. */
11034 {
11037 {
11045 }
11046 }
11047 }
11049 /* Emit code to restore saved registers using POP insns. */
11051 static void
11053 {
11059 }
11061 /* Emit code and notes for the LEAVE instruction. */
11063 static void
11065 {
11077 {
11085 }
11088 }
11090 /* Emit code to restore saved registers using MOV insns.
11091 First register is restored from CFA - CFA_OFFSET. */
11092 static void
11095 {
11101 {
11110 {
11111 /* Previously we'd represented the CFA as an expression
11112 like *(%ebp - 8). We've just popped that value from
11113 the stack, which means we need to reset the CFA to
11114 the drap register. This will remain until we restore
11115 the stack pointer. */
11119 /* This means that the DRAP register is valid for addressing. */
11121 }
11122 else
11126 }
11127 }
11129 /* Emit code to restore saved registers using MOV insns.
11130 First register is restored from CFA - CFA_OFFSET. */
11131 static void
11134 {
11139 {
11141 rtx mem;
11151 }
11152 }
11154 /* Restore function stack, frame, and registers. */
11156 void
11158 {
11174 /* The FP must be valid if the frame pointer is present. */
11179 /* We must have *some* valid pointer to the stack frame. */
11182 /* The DRAP is never valid at this point. */
11185 /* See the comment about red zone and frame
11186 pointer usage in ix86_expand_prologue. */
11193 /* Determine the CFA offset of the end of the red-zone. */
11196 {
11197 /* The red-zone begins below the return address. */
11200 /* When the register save area is in the aligned portion of
11201 the stack, determine the maximum runtime displacement that
11202 matches up with the aligned frame. */
11206 }
11208 /* Special care must be taken for the normal return case of a function
11209 using eh_return: the eax and edx registers are marked as saved, but
11210 not restored along this path. Adjust the save location to match. */
11214 /* EH_RETURN requires the use of moves to function properly. */
11217 /* SEH requires the use of pops to identify the epilogue. */
11220 /* If we're only restoring one register and sp is not valid then
11221 using a move instruction to restore the register since it's
11222 less work than reloading sp and popping the register. */
11235 && TARGET_USE_LEAVE
11239 else
11243 {
11244 /* Ensure that the entire register save area is addressable via
11245 the stack pointer, if we will restore via sp. */
11250 {
11254 style,
11256 }
11257 }
11259 /* If there are any SSE registers to restore, then we have to do it
11260 via moves, since there's obviously no pop for SSE regs. */
11266 {
11267 rtx t;
11272 /* eh_return epilogues need %ecx added to the stack pointer. */
11274 {
11277 /* Stack align doesn't work with eh_return. */
11279 /* Neither does regparm nested functions. */
11283 {
11291 /* Note that we use SA as a temporary CFA, as the return
11292 address is at the proper place relative to it. We
11293 pretend this happens at the FP restore insn because
11294 prior to this insn the FP would be stored at the wrong
11295 offset relative to SA, and after this insn we have no
11296 other reasonable register to use for the CFA. We don't
11297 bother resetting the CFA to the SP for the duration of
11298 the return insn. */
11311 }
11312 else
11313 {
11321 {
11325 UNITS_PER_WORD));
11327 }
11328 }
11331 }
11332 }
11333 else
11334 {
11335 /* SEH requires that the function end with (1) a stack adjustment
11336 if necessary, (2) a sequence of pops, and (3) a return or
11337 jump instruction. Prevent insns from the function body from
11338 being scheduled into this sequence. */
11340 {
11341 /* Prevent a catch region from being adjacent to the standard
11342 epilogue sequence. Unfortuantely crtl->uses_eh_lsda nor
11343 several other flags that would be interesting to test are
11344 not yet set up. */
11347 else
11349 }
11351 /* First step is to deallocate the stack frame so that we can
11352 pop the registers. Also do it on SEH target for very large
11353 frame as the emitted instructions aren't allowed by the ABI in
11354 epilogues. */
11356 || (TARGET_SEH
11359 {
11364 }
11366 {
11370 style,
11372 }
11375 }
11377 /* If we used a stack pointer and haven't already got rid of it,
11378 then do so now. */
11380 {
11381 /* If the stack pointer is valid and pointing at the frame
11382 pointer store address, then we only need a pop. */
11385 /* Leave results in shorter dependency chains on CPUs that are
11386 able to grok it fast. */
11391 else
11392 {
11394 hard_frame_pointer_rtx,
11397 }
11398 }
11401 {
11403 rtx insn;
11429 }
11431 /* At this point the stack pointer must be valid, and we must have
11432 restored all of the registers. We may not have deallocated the
11433 entire stack frame. We've delayed this until now because it may
11434 be possible to merge the local stack deallocation with the
11435 deallocation forced by ix86_static_chain_on_stack. */
11440 {
11444 }
11445 else
11448 /* Sibcall epilogues don't want a return instruction. */
11450 {
11453 }
11456 {
11459 /* i386 can only pop 64K bytes. If asked to pop more, pop return
11460 address, do explicit add, and jump indirectly to the caller. */
11463 {
11465 rtx insn;
11467 /* There is no "pascal" calling convention in any 64bit ABI. */
11483 }
11484 else
11486 }
11487 else
11490 /* Restore the state back to the state from the prologue,
11491 so that it's correct for the next epilogue. */
11493 }
11495 /* Reset from the function's potential modifications. */
11497 static void
11499 HOST_WIDE_INT size ATTRIBUTE_UNUSED)
11500 {
11503 #if TARGET_MACHO
11504 /* Mach-O doesn't support labels at the end of objects, so if
11505 it looks like we might want one, insert a NOP. */
11506 {
11512 {
11513 /* Don't insert a nop for NOTE_INSN_DELETED_DEBUG_LABEL
11514 notes only, instead set their CODE_LABEL_NUMBER to -1,
11515 otherwise there would be code generation differences
11516 in between -g and -g0. */
11520 }
11530 }
11531 #endif
11533 }
11535 /* Return a scratch register to use in the split stack prologue. The
11536 split stack prologue is used for -fsplit-stack. It is the first
11537 instructions in the function, even before the regular prologue.
11538 The scratch register can be any caller-saved register which is not
11539 used for parameters or for the static chain. */
11541 static unsigned int
11543 {
11546 else
11547 {
11560 {
11562 {
11566 }
11568 }
11570 {
11574 }
11576 {
11579 else
11580 {
11582 {
11586 }
11588 }
11589 }
11590 else
11591 {
11592 /* FIXME: We could make this work by pushing a register
11593 around the addition and comparison. */
11596 }
11597 }
11598 }
11600 /* A SYMBOL_REF for the function which allocates new stackspace for
11601 -fsplit-stack. */
11605 /* A SYMBOL_REF for the more stack function when using the large
11606 model. */
11610 /* Handle -fsplit-stack. These are the first instructions in the
11611 function, even before the regular prologue. */
11613 void
11615 {
11617 HOST_WIDE_INT allocate;
11622 rtx fn;
11630 /* This is the label we will branch to if we have enough stack
11631 space. We expect the basic block reordering pass to reverse this
11632 branch if optimizing, so that we branch in the unlikely case. */
11635 /* We need to compare the stack pointer minus the frame size with
11636 the stack boundary in the TCB. The stack boundary always gives
11637 us SPLIT_STACK_AVAILABLE bytes, so if we need less than that we
11638 can compare directly. Otherwise we need to do an addition. */
11641 UNSPEC_STACK_CHECK);
11646 else
11647 {
11649 rtx offset;
11651 /* We need a scratch register to hold the stack pointer minus
11652 the required frame size. Since this is the very start of the
11653 function, the scratch register can be any caller-saved
11654 register which is not used for parameters. */
11661 {
11662 /* We don't use ix86_gen_add3 in this case because it will
11663 want to split to lea, but when not optimizing the insn
11664 will not be split after this point. */
11667 offset)));
11668 }
11669 else
11670 {
11673 stack_pointer_rtx));
11674 }
11676 }
11682 /* Mark the jump as very likely to be taken. */
11690 /* Get more stack space. We pass in the desired stack space and the
11691 size of the arguments to copy to the new stack. In 32-bit mode
11692 we push the parameters; __morestack will return on a new stack
11693 anyhow. In 64-bit mode we pass the parameters in r10 and
11694 r11. */
11699 {
11705 /* If this function uses a static chain, it will be in %r10.
11706 Preserve it across the call to __morestack. */
11708 {
11709 rtx rax;
11714 }
11718 {
11719 HOST_WIDE_INT argval;
11722 /* When using the large model we need to load the address
11723 into a register, and we've run out of registers. So we
11724 switch to a different calling convention, and we call a
11725 different function: __morestack_large. We pass the
11726 argument size in the upper 32 bits of r10 and pass the
11727 frame size in the lower 32 bits. */
11732 split_stack_fn_large =
11736 {
11746 UNSPEC_GOT);
11752 }
11753 else
11760 }
11761 else
11762 {
11766 }
11769 }
11770 else
11771 {
11774 }
11780 /* In order to make call/return prediction work right, we now need
11781 to execute a return instruction. See
11782 libgcc/config/i386/morestack.S for the details on how this works.
11784 For flow purposes gcc must not see this as a return
11785 instruction--we need control flow to continue at the subsequent
11786 label. Therefore, we use an unspec. */
11790 /* If we are in 64-bit mode and this function uses a static chain,
11791 we saved %r10 in %rax before calling _morestack. */
11796 /* If this function calls va_start, we need to store a pointer to
11797 the arguments on the old stack, because they may not have been
11798 all copied to the new stack. At this point the old stack can be
11799 found at the frame pointer value used by __morestack, because
11800 __morestack has set that up before calling back to us. Here we
11801 store that pointer in a scratch register, and in
11802 ix86_expand_prologue we store the scratch register in a stack
11803 slot. */
11805 {
11807 rtx frame_reg;
11814 /* 64-bit:
11815 fp -> old fp value
11816 return address within this function
11817 return address of caller of this function
11818 stack arguments
11819 So we add three words to get to the stack arguments.
11821 32-bit:
11822 fp -> old fp value
11823 return address within this function
11824 first argument to __morestack
11825 second argument to __morestack
11826 return address of caller of this function
11827 stack arguments
11828 So we add five words to get to the stack arguments.
11829 */
11840 }
11845 /* If this function calls va_start, we now have to set the scratch
11846 register for the case where we do not call __morestack. In this
11847 case we need to set it based on the stack pointer. */
11849 {
11856 }
11857 }
11859 /* We may have to tell the dataflow pass that the split stack prologue
11860 is initializing a scratch register. */
11862 static void
11864 {
11866 {
11869 }
11870 }
11871 \f
11872 /* Determine if op is suitable SUBREG RTX for address. */
11874 static bool
11876 {
11884 /* Don't allow SUBREGs that span more than a word. It can lead to spill
11885 failures when the register is one word out of a two word structure. */
11889 /* Allow only SUBREGs of non-eliminable hard registers. */
11891 }
11893 /* Extract the parts of an RTL expression that is a valid memory address
11894 for an instruction. Return 0 if the structure of the address is
11895 grossly off. Return -1 if the address contains ASHIFT, so it is not
11896 strictly valid, but still used for computing length of lea instruction. */
11898 int
11900 {
11905 rtx tmp;
11909 /* Allow zero-extended SImode addresses,
11910 they will be emitted with addr32 prefix. */
11912 {
11915 {
11919 }
11922 {
11929 }
11930 }
11932 /* Allow SImode subregs of DImode addresses,
11933 they will be emitted with addr32 prefix. */
11935 {
11938 {
11942 }
11943 }
11948 {
11951 else
11953 }
11955 {
11960 do
11961 {
11966 }
11973 {
11976 {
12001 /* FALLTHRU */
12005 && TARGET_TLS_DIRECT_SEG_REFS
12008 else
12015 /* FALLTHRU */
12022 else
12037 }
12038 }
12039 }
12041 {
12044 }
12046 {
12047 /* We're called for lea too, which implements ashift on occasion. */
12057 }
12058 else
12062 {
12064 ;
12067 ;
12068 else
12070 }
12072 /* Address override works only on the (%reg) part of %fs:(%reg). */
12078 /* Extract the integral value of scale. */
12080 {
12084 }
12089 /* Avoid useless 0 displacement. */
12093 /* Allow arg pointer and stack pointer as index if there is not scaling. */
12098 {
12099 rtx tmp;
12102 }
12104 /* Special case: %ebp cannot be encoded as a base without a displacement.
12105 Similarly %r13. */
12107 && base_reg
12116 /* Special case: on K6, [%esi] makes the instruction vector decoded.
12117 Avoid this by transforming to [%esi+0].
12118 Reload calls address legitimization without cfun defined, so we need
12119 to test cfun for being non-NULL. */
12125 /* Special case: encode reg+reg instead of reg*2. */
12129 /* Special case: scaling cannot be encoded without base or displacement. */
12140 }
12141 \f
12142 /* Return cost of the memory address x.
12143 For i386, it is better to use a complex address than let gcc copy
12144 the address into a reg and make a new pseudo. But not if the address
12145 requires to two regs - that would mean more pseudos with longer
12146 lifetimes. */
12147 static int
12149 addr_space_t as ATTRIBUTE_UNUSED,
12151 {
12163 /* Attempt to minimize number of registers in the address. */
12169 cost++;
12176 cost++;
12178 /* AMD-K6 don't like addresses with ModR/M set to 00_xxx_100b,
12179 since it's predecode logic can't detect the length of instructions
12180 and it degenerates to vector decoded. Increase cost of such
12181 addresses here. The penalty is minimally 2 cycles. It may be worthwhile
12182 to split such addresses or even refuse such addresses at all.
12184 Following addressing modes are affected:
12185 [base+scale*index]
12186 [scale*index+disp]
12187 [base+index]
12189 The first and last case may be avoidable by explicitly coding the zero in
12190 memory address, but I don't have AMD-K6 machine handy to check this
12191 theory. */
12200 }
12201 \f
12202 /* Allow {LABEL | SYMBOL}_REF - SYMBOL_REF-FOR-PICBASE for Mach-O as
12203 this is used for to form addresses to local data when -fPIC is in
12204 use. */
12206 static bool
12208 {
12211 }
12213 /* Determine if a given RTX is a valid constant. We already know this
12214 satisfies CONSTANT_P. */
12216 static bool
12218 {
12220 {
12225 {
12229 }
12234 /* Only some unspecs are valid as "constants". */
12237 {
12253 }
12255 /* We must have drilled down to a symbol. */
12260 /* FALLTHRU */
12263 /* TLS symbols are never valid. */
12267 /* DLLIMPORT symbols are never valid. */
12272 #if TARGET_MACHO
12273 /* mdynamic-no-pic */
12276 #endif
12292 }
12294 /* Otherwise we handle everything else in the move patterns. */
12296 }
12298 /* Determine if it's legal to put X into the constant pool. This
12299 is not possible for the address of thread-local symbols, which
12300 is checked above. */
12302 static bool
12304 {
12305 /* We can always put integral constants and vectors in memory. */
12307 {
12315 }
12317 }
12319 /* Nonzero if the symbol is marked as dllimport, or as stub-variable,
12320 otherwise zero. */
12322 static bool
12324 {
12330 }
12333 /* Nonzero if the constant value X is a legitimate general operand
12334 when generating PIC code. It is given that flag_pic is on and
12335 that X satisfies CONSTANT_P or is a CONST_DOUBLE. */
12337 bool
12339 {
12340 rtx inner;
12343 {
12350 /* Only some unspecs are valid as "constants". */
12353 {
12366 }
12367 /* FALLTHRU */
12375 }
12376 }
12378 /* Determine if a given CONST RTX is a valid memory displacement
12379 in PIC mode. */
12381 bool
12383 {
12386 /* In 64bit mode we can allow direct addresses of symbols and labels
12387 when they are not dynamic symbols. */
12389 {
12393 {
12417 /* FALLTHRU */
12420 /* TLS references should always be enclosed in UNSPEC.
12421 The dllimported symbol needs always to be resolved. */
12427 {
12435 /* Function-symbols need to be resolved only for
12436 large-model.
12437 For the small-model we don't need to resolve anything
12438 here. */
12443 /* Non-external symbols don't need to be resolved for
12444 large, and medium-model. */
12449 }
12458 }
12459 }
12465 {
12466 /* We are unsafe to allow PLUS expressions. This limit allowed distance
12467 of GOT tables. We should not need these anyway. */
12479 }
12483 {
12488 }
12497 {
12501 /* We need to check for both symbols and labels because VxWorks loads
12502 text labels with @GOT rather than @GOTOFF. See gotoff_operand for
12503 details. */
12507 /* Refuse GOTOFF in 64bit mode since it is always 64bit when used.
12508 While ABI specify also 32bit relocation but we don't produce it in
12509 small PIC model at all. */
12531 }
12534 }
12536 /* Our implementation of LEGITIMIZE_RELOAD_ADDRESS. Returns a value to
12537 replace the input X, or the original X if no replacement is called for.
12538 The output parameter *WIN is 1 if the calling macro should goto WIN,
12539 0 if it should not. */
12541 bool
12546 {
12547 /* Reload can generate:
12549 (plus:DI (plus:DI (unspec:DI [(const_int 0 [0])] UNSPEC_TP)
12550 (reg:DI 97))
12551 (reg:DI 2 cx))
12553 This RTX is rejected from ix86_legitimate_address_p due to
12554 non-strictness of base register 97. Following this rejection,
12555 reload pushes all three components into separate registers,
12556 creating invalid memory address RTX.
12558 Following code reloads only the invalid part of the
12559 memory address RTX. */
12565 {
12571 {
12576 }
12580 {
12585 }
12589 }
12592 }
12594 /* Recognizes RTL expressions that are valid memory addresses for an
12595 instruction. The MODE argument is the machine mode for the MEM
12596 expression that wants to use this address.
12598 It only recognizes address in canonical form. LEGITIMIZE_ADDRESS should
12599 convert common non-canonical forms to canonical form so that they will
12600 be recognized. */
12602 static bool
12605 {
12608 HOST_WIDE_INT scale;
12611 /* Decomposition failed. */
12619 /* Validate base register. */
12621 {
12622 rtx reg;
12628 else
12629 /* Base is not a register. */
12637 /* Base is not valid. */
12639 }
12641 /* Validate index register. */
12643 {
12644 rtx reg;
12650 else
12651 /* Index is not a register. */
12659 /* Index is not valid. */
12661 }
12663 /* Index and base should have the same mode. */
12668 /* Validate scale factor. */
12670 {
12672 /* Scale without index. */
12676 /* Scale is not a valid multiplier. */
12678 }
12680 /* Validate displacement. */
12682 {
12687 {
12688 /* Refuse GOTOFF and GOT in 64bit mode since it is always 64bit when
12689 used. While ABI specify also 32bit relocations, we don't produce
12690 them at all and use IP relative instead. */
12697 /* 64bit address unspec. */
12717 /* Invalid address unspec. */
12719 }
12722 && (flag_pic
12723 || (TARGET_MACHO
12724 #if TARGET_MACHO
12725 && MACHOPIC_INDIRECT
12727 #endif
12728 )))
12729 {
12731 is_legitimate_pic:
12733 {
12734 /* foo@dtpoff(%rX) is ok. */
12741 /* Non-constant pic memory reference. */
12743 }
12746 /* Displacement is an invalid pic construct. */
12748 #if TARGET_MACHO
12751 /* displacment must be referenced via non_lazy_pointer */
12753 #endif
12755 /* This code used to verify that a symbolic pic displacement
12756 includes the pic_offset_table_rtx register.
12758 While this is good idea, unfortunately these constructs may
12759 be created by "adds using lea" optimization for incorrect
12760 code like:
12762 int a;
12763 int foo(int i)
12764 {
12765 return *(&a+i);
12766 }
12768 This code is nonsensical, but results in addressing
12769 GOT table with pic_offset_table_rtx base. We can't
12770 just refuse it easily, since it gets matched by
12771 "addsi3" pattern, that later gets split to lea in the
12772 case output register differs from input. While this
12773 can be handled by separate addsi pattern for this case
12774 that never results in lea, this seems to be easier and
12775 correct fix for crash to disable this test. */
12776 }
12783 /* Displacement is not constant. */
12787 /* Displacement is out of range. */
12789 /* In x32 mode, constant addresses are sign extended to 64bit, so
12790 we have to prevent addresses from 0x80000000 to 0xffffffff. */
12795 }
12797 /* Everything looks valid. */
12799 }
12801 /* Determine if a given RTX is a valid constant address. */
12803 bool
12805 {
12807 }
12808 \f
12809 /* Return a unique alias set for the GOT. */
12811 static alias_set_type
12813 {
12818 }
12820 /* Return a legitimate reference for ORIG (an address) using the
12821 register REG. If REG is 0, a new pseudo is generated.
12823 There are two types of references that must be handled:
12825 1. Global data references must load the address from the GOT, via
12826 the PIC reg. An insn is emitted to do this load, and the reg is
12827 returned.
12829 2. Static data references, constant pool addresses, and code labels
12830 compute the address as an offset from the GOT, whose base is in
12831 the PIC reg. Static data objects have SYMBOL_FLAG_LOCAL set to
12832 differentiate them from global data objects. The returned
12833 address is the PIC reg + an unspec constant.
12835 TARGET_LEGITIMATE_ADDRESS_P rejects symbolic references unless the PIC
12836 reg also appears in the address. */
12838 static rtx
12840 {
12844 #if TARGET_MACHO
12846 {
12849 /* Use the generic Mach-O PIC machinery. */
12851 }
12852 #endif
12855 {
12859 }
12865 {
12866 rtx tmpreg;
12867 /* This symbol may be referenced via a displacement from the PIC
12868 base address (@GOTOFF). */
12875 {
12877 UNSPEC_GOTOFF);
12879 }
12880 else
12885 else
12890 {
12894 }
12895 else
12897 }
12899 {
12900 /* This symbol may be referenced via a displacement from the PIC
12901 base address (@GOTOFF). */
12908 {
12910 UNSPEC_GOTOFF);
12912 }
12913 else
12919 {
12922 }
12923 }
12925 /* We can't use @GOTOFF for text labels on VxWorks;
12926 see gotoff_operand. */
12928 {
12933 /* For x64 PE-COFF there is no GOT table. So we use address
12934 directly. */
12936 {
12944 }
12946 {
12954 /* Use directly gen_movsi, otherwise the address is loaded
12955 into register for CSE. We don't want to CSE this addresses,
12956 instead we CSE addresses from the GOT table, so skip this. */
12959 }
12960 else
12961 {
12962 /* This symbol must be referenced via a load from the
12963 Global Offset Table (@GOT). */
12979 }
12980 }
12981 else
12982 {
12985 {
12987 {
12990 }
12991 else
12993 }
12995 {
12998 /* We must match stuff we generate before. Assume the only
12999 unspecs that can get here are ours. Not that we could do
13000 anything with them anyway.... */
13006 }
13008 {
13011 /* Check first to see if this is a constant offset from a @GOTOFF
13012 symbol reference. */
13015 {
13017 {
13021 UNSPEC_GOTOFF);
13027 {
13030 }
13031 }
13032 else
13033 {
13036 {
13040 }
13041 }
13042 }
13043 else
13044 {
13047 new_rtx
13051 {
13054 {
13057 new_rtx
13059 }
13060 else
13062 }
13063 else
13064 {
13067 {
13070 }
13072 }
13073 }
13074 }
13075 }
13077 }
13078 \f
13079 /* Load the thread pointer. If TO_REG is true, force it into a register. */
13081 static rtx
13083 {
13087 {
13092 }
13098 }
13100 /* Construct the SYMBOL_REF for the tls_get_addr function. */
13104 static rtx
13106 {
13108 {
13114 }
13117 {
13119 UNSPEC_PLTOFF);
13122 }
13125 }
13127 /* Construct the SYMBOL_REF for the _TLS_MODULE_BASE_ symbol. */
13131 rtx
13133 {
13135 {
13136 ix86_tls_module_base_symbol
13141 }
13144 }
13146 /* A subroutine of ix86_legitimize_address and ix86_expand_move. FOR_MOV is
13147 false if we expect this to be used for a memory address and true if
13148 we expect to load the address into a register. */
13150 static rtx
13152 {
13159 {
13164 {
13167 else
13168 {
13171 }
13172 }
13175 {
13178 else
13188 }
13189 else
13190 {
13194 {
13196 rtx insns;
13199 emit_call_insn
13209 }
13210 else
13212 }
13219 {
13222 else
13223 {
13226 }
13227 }
13230 {
13235 else
13241 }
13242 else
13243 {
13247 {
13252 emit_call_insn
13257 /* Attach a unique REG_EQUAL, to allow the RTL optimizers to
13258 share the LD_BASE result with other LD model accesses. */
13260 UNSPEC_TLS_LD_BASE);
13264 }
13265 else
13267 }
13275 {
13282 }
13287 {
13289 {
13290 /* The Sun linker took the AMD64 TLS spec literally
13291 and can only handle %rax as destination of the
13292 initial executable code sequence. */
13297 }
13299 /* Generate DImode references to avoid %fs:(%reg32)
13300 problems and linker IE->LE relaxation bug. */
13304 }
13306 {
13311 }
13313 {
13317 }
13318 else
13319 {
13322 }
13332 {
13337 }
13338 else
13339 {
13343 }
13353 {
13357 }
13358 else
13359 {
13363 }
13368 }
13371 }
13373 /* Create or return the unique __imp_DECL dllimport symbol corresponding
13374 to symbol DECL if BEIMPORT is true. Otherwise create or return the
13375 unique refptr-DECL symbol corresponding to symbol DECL. */
13378 htab_t dllimport_map;
13380 static tree
13382 {
13389 tree to;
13390 rtx rtl;
13417 else
13430 {
13432 #ifdef SUB_TARGET_RECORD_STUB
13434 #endif
13435 }
13444 }
13446 /* Expand SYMBOL into its corresponding far-addresse symbol.
13447 WANT_REG is true if we require the result be a register. */
13449 static rtx
13451 {
13452 tree imp_decl;
13453 rtx x;
13462 }
13464 /* Expand SYMBOL into its corresponding dllimport symbol. WANT_REG is
13465 true if we require the result be a register. */
13467 static rtx
13469 {
13470 tree imp_decl;
13471 rtx x;
13480 }
13482 /* Expand SYMBOL into its corresponding dllimport or refptr symbol. WANT_REG
13483 is true if we require the result be a register. */
13485 static rtx
13487 {
13492 {
13499 {
13502 }
13503 }
13519 {
13522 }
13524 }
13526 /* Try machine-dependent ways of modifying an illegitimate address
13527 to be legitimate. If we find one, return the new, valid address.
13528 This macro is used in only one place: `memory_address' in explow.c.
13530 OLDX is the address as it was before break_out_memory_refs was called.
13531 In some cases it is useful to look at this to decide what needs to be done.
13533 It is always safe for this macro to do nothing. It exists to recognize
13534 opportunities to optimize the output.
13536 For the 80386, we handle X+REG by loading X into a register R and
13537 using R+REG. R will go in a general reg and indexing will be used.
13538 However, if REG is a broken-out memory address or multiplication,
13539 nothing needs to be done because REG can certainly go in a general reg.
13541 When -fpic is used, special handling is needed for symbolic references.
13542 See comments by legitimize_pic_address in i386.c for details. */
13544 static rtx
13547 {
13558 {
13562 }
13565 {
13569 }
13574 #if TARGET_MACHO
13577 #endif
13579 /* Canonicalize shifts by 0, 1, 2, 3 into multiply */
13583 {
13588 }
13591 {
13592 /* Canonicalize shifts by 0, 1, 2, 3 into multiply. */
13597 {
13603 }
13608 {
13614 }
13616 /* Put multiply first if it isn't already. */
13618 {
13623 }
13625 /* Canonicalize (plus (mult (reg) (const)) (plus (reg) (const)))
13626 into (plus (plus (mult (reg) (const)) (reg)) (const)). This can be
13627 created by virtual register instantiation, register elimination, and
13628 similar optimizations. */
13630 {
13636 }
13638 /* Canonicalize
13639 (plus (plus (mult (reg) (const)) (plus (reg) (const))) const)
13640 into (plus (plus (mult (reg) (const)) (reg)) (const)). */
13645 {
13646 rtx constant;
13650 {
13653 }
13655 {
13658 }
13659 else
13663 {
13670 }
13671 }
13677 {
13680 }
13683 {
13686 }
13694 {
13697 }
13703 {
13707 {
13710 }
13714 }
13717 {
13721 {
13724 }
13728 }
13729 }
13732 }
13733 \f
13734 /* Print an integer constant expression in assembler syntax. Addition
13735 and subtraction are the only arithmetic that may appear in these
13736 expressions. FILE is the stdio stream to write to, X is the rtx, and
13737 CODE is the operand print code from the output string. */
13739 static void
13741 {
13745 {
13754 else
13755 {
13758 /* Mark the decl as referenced so that cgraph will
13759 output the function. */
13763 #if TARGET_MACHO
13767 #endif
13769 }
13777 /* FALLTHRU */
13788 /* This used to output parentheses around the expression,
13789 but that does not work on the 386 (either ATT or BSD assembler). */
13795 {
13796 /* We can use %d if the number is <32 bits and positive. */
13801 else
13803 }
13804 else
13805 /* We can't handle floating point constants;
13806 TARGET_PRINT_OPERAND must handle them. */
13811 /* Some assemblers need integer constants to appear first. */
13813 {
13817 }
13818 else
13819 {
13824 }
13839 {
13843 }
13848 {
13867 /* FIXME: This might be @TPOFF in Sun ld too. */
13876 else
13886 else
13892 #if TARGET_MACHO
13897 #endif
13901 }
13906 }
13907 }
13909 /* This is called from dwarf2out.c via TARGET_ASM_OUTPUT_DWARF_DTPREL.
13910 We need to emit DTP-relative relocations. */
13912 static void ATTRIBUTE_UNUSED
13914 {
13919 {
13927 }
13928 }
13930 /* Return true if X is a representation of the PIC register. This copes
13931 with calls from ix86_find_base_term, where the register might have
13932 been replaced by a cselib value. */
13934 static bool
13936 {
13940 else
13942 }
13944 /* Helper function for ix86_delegitimize_address.
13945 Attempt to delegitimize TLS local-exec accesses. */
13947 static rtx
13949 {
13974 {
13979 }
13985 }
13987 /* In the name of slightly smaller debug output, and to cater to
13988 general assembler lossage, recognize PIC+GOTOFF and turn it back
13989 into a direct symbol reference.
13991 On Darwin, this is necessary to avoid a crash, because Darwin
13992 has a different PIC label for each routine but the DWARF debugging
13993 information is not associated with any particular routine, so it's
13994 necessary to remove references to the PIC label from RTL stored by
13995 the DWARF output code. */
13997 static rtx
13999 {
14001 /* addend is NULL or some rtx if x is something+GOTOFF where
14002 something doesn't include the PIC register. */
14004 /* reg_addend is NULL or a multiple of some register. */
14006 /* const_addend is NULL or a const_int. */
14008 /* This is the result, or NULL. */
14017 {
14024 {
14030 }
14037 {
14040 {
14045 }
14047 }
14052 /* Fall thru into the code shared with -m32 for -mcmodel=large -fpic
14053 and -mcmodel=medium -fpic. */
14054 }
14061 /* %ebx + GOT/GOTOFF */
14062 ;
14064 {
14065 /* %ebx + %reg * scale + GOT/GOTOFF */
14071 else
14072 {
14075 }
14076 }
14077 else
14083 {
14086 }
14107 {
14108 /* If the rest of original X doesn't involve the PIC register, add
14109 addend and subtract pic_offset_table_rtx. This can happen e.g.
14110 for code like:
14111 leal (%ebx, %ecx, 4), %ecx
14112 ...
14113 movl foo@GOTOFF(%ecx), %edx
14114 in which case we return (%ecx - %ebx) + foo. */
14117 pic_offset_table_rtx),
14118 result);
14119 else
14121 }
14123 {
14127 }
14129 }
14131 /* If X is a machine specific address (i.e. a symbol or label being
14132 referenced as a displacement from the GOT implemented using an
14133 UNSPEC), then return the base term. Otherwise return X. */
14135 rtx
14137 {
14138 rtx term;
14141 {
14155 }
14158 }
14159 \f
14160 static void
14163 {
14167 {
14170 }
14175 {
14178 {
14197 }
14201 {
14220 }
14227 /* ??? Use "nbe" instead of "a" for fcmov lossage on some assemblers.
14228 Those same assemblers have the same but opposite lossage on cmov. */
14231 else
14236 {
14249 }
14256 else
14261 {
14274 }
14281 else
14291 else
14302 }
14304 }
14306 /* Print the name of register X to FILE based on its machine mode and number.
14307 If CODE is 'w', pretend the mode is HImode.
14308 If CODE is 'b', pretend the mode is QImode.
14309 If CODE is 'k', pretend the mode is SImode.
14310 If CODE is 'q', pretend the mode is DImode.
14311 If CODE is 'x', pretend the mode is V4SFmode.
14312 If CODE is 't', pretend the mode is V8SFmode.
14313 If CODE is 'g', pretend the mode is V16SFmode.
14314 If CODE is 'h', pretend the reg is the 'high' byte register.
14315 If CODE is 'y', print "st(0)" instead of "st", if the reg is stack op.
14316 If CODE is 'd', duplicate the operand for AVX instruction.
14317 */
14319 void
14321 {
14330 {
14334 }
14361 else
14364 /* Irritatingly, AMD extended registers use different naming convention
14365 from the normal registers: "r%d[bwd]" */
14367 {
14372 {
14386 /* no suffix */
14391 }
14393 }
14397 {
14400 {
14403 }
14404 /* FALLTHRU */
14410 /* FALLTHRU */
14413 normal:
14428 {
14433 }
14436 {
14441 }
14445 }
14449 {
14452 else
14454 }
14455 }
14457 /* Locate some local-dynamic symbol still in use by this function
14458 so that we can print its name in some tls_local_dynamic_base
14459 pattern. */
14461 static int
14463 {
14468 {
14471 }
14474 }
14478 {
14479 rtx insn;
14490 }
14492 /* Meaning of CODE:
14493 L,W,B,Q,S,T -- print the opcode suffix for specified size of operand.
14494 C -- print opcode suffix for set/cmov insn.
14495 c -- like C, but print reversed condition
14496 F,f -- likewise, but for floating-point.
14497 O -- if HAVE_AS_IX86_CMOV_SUN_SYNTAX, expand to "w.", "l." or "q.",
14498 otherwise nothing
14499 R -- print the prefix for register names.
14500 z -- print the opcode suffix for the size of the current operand.
14501 Z -- likewise, with special suffixes for x87 instructions.
14502 * -- print a star (in certain assembler syntax)
14503 A -- print an absolute memory reference.
14504 E -- print address with DImode register names if TARGET_64BIT.
14505 w -- print the operand as if it's a "word" (HImode) even if it isn't.
14506 s -- print a shift double count, followed by the assemblers argument
14507 delimiter.
14508 b -- print the QImode name of the register for the indicated operand.
14509 %b0 would print %al if operands[0] is reg 0.
14510 w -- likewise, print the HImode name of the register.
14511 k -- likewise, print the SImode name of the register.
14512 q -- likewise, print the DImode name of the register.
14513 x -- likewise, print the V4SFmode name of the register.
14514 t -- likewise, print the V8SFmode name of the register.
14515 g -- likewise, print the V16SFmode name of the register.
14516 h -- print the QImode name for a "high" register, either ah, bh, ch or dh.
14517 y -- print "st(0)" instead of "st" as a register.
14518 d -- print duplicated register operand for AVX instruction.
14519 D -- print condition for SSE cmp instruction.
14520 P -- if PIC, print an @PLT suffix.
14521 p -- print raw symbol name.
14522 X -- don't print any sort of PIC '@' suffix for a symbol.
14523 & -- print some in-use local-dynamic symbol name.
14524 H -- print a memory address offset by 8; used for sse high-parts
14525 Y -- print condition for XOP pcom* instruction.
14526 + -- print a branch hint as 'cs' or 'ds' prefix
14527 ; -- print a semicolon (after prefixes due to bug in older gas).
14528 ~ -- print "i" if TARGET_AVX2, "f" otherwise.
14529 @ -- print a segment register of thread base pointer load
14530 ^ -- print addr32 prefix if TARGET_64BIT and Pmode != word_mode
14531 ! -- print MPX prefix for jxx/call/ret instructions if required.
14532 */
14534 void
14536 {
14538 {
14540 {
14543 {
14549 /* Intel syntax. For absolute addresses, registers should not
14550 be surrounded by braces. */
14552 {
14557 }
14562 }
14568 /* Wrap address in an UNSPEC to declare special handling. */
14606 #ifdef HAVE_AS_IX86_CMOV_SUN_SYNTAX
14611 {
14625 output_operand_lossage
14628 }
14631 #endif
14636 {
14637 /* Opcodes don't get size suffixes if using Intel opcodes. */
14642 {
14660 output_operand_lossage
14663 }
14664 }
14667 warning
14669 /* FALLTHRU */
14672 /* 387 opcodes don't get size suffixes if using Intel opcodes. */
14677 {
14679 {
14681 #ifdef HAVE_AS_IX86_FILDS
14683 #endif
14691 #ifdef HAVE_AS_IX86_FILDQ
14693 #else
14695 #endif
14700 }
14701 }
14703 {
14704 /* 387 opcodes don't get size suffixes
14705 if the operands are registers. */
14710 {
14726 }
14727 }
14728 else
14729 {
14730 output_operand_lossage
14733 }
14735 output_operand_lossage
14756 {
14759 }
14764 {
14815 }
14819 /* Little bit of braindamage here. The SSE compare instructions
14820 does use completely different names for the comparisons that the
14821 fp conditional moves. */
14823 {
14826 {
14829 }
14835 {
14838 }
14844 {
14847 }
14856 {
14859 }
14865 {
14868 }
14874 {
14877 }
14888 }
14893 #ifdef HAVE_AS_IX86_CMOV_SUN_SYNTAX
14896 #endif
14901 {
14905 }
14909 file);
14914 {
14918 }
14919 /* It doesn't actually matter what mode we use here, as we're
14920 only going to use this for printing. */
14922 /* Output 'qword ptr' for intel assembler dialect. */
14931 #ifdef HAVE_AS_IX86_HLE
14933 #else
14935 #endif
14937 #ifdef HAVE_AS_IX86_HLE
14939 #else
14941 #endif
14942 /* We do not want to print value of the operand. */
14956 {
14961 else
14964 }
14967 {
14968 rtx x;
14977 {
14982 {
14984 bool cputaken
14987 /* Emit hints only in the case default branch prediction
14988 heuristics would fail. */
14990 {
14991 /* We use 3e (DS) prefix for taken branches and
14992 2e (CS) prefix for not taken branches. */
14995 else
14997 }
14998 }
14999 }
15001 }
15004 #ifndef HAVE_AS_IX86_REP_LOCK_PREFIX
15006 #endif
15013 /* The kernel uses a different segment register for performance
15014 reasons; a system call would not have to trash the userspace
15015 segment register, which would be expensive. */
15018 else
15038 }
15039 }
15045 {
15046 /* No `byte ptr' prefix for call instructions or BLKmode operands. */
15049 {
15052 {
15061 else
15068 }
15070 /* Check for explicit size override (codes 'b', 'w', 'k',
15071 'q' and 'x') */
15085 }
15088 /* Avoid (%rip) for call operands. */
15094 else
15096 }
15099 {
15100 REAL_VALUE_TYPE r;
15108 /* Sign extend 32bit SFmode immediate to 8 bytes. */
15112 else
15114 }
15117 {
15118 REAL_VALUE_TYPE r;
15127 }
15129 /* These float cases don't actually occur as immediate operands. */
15131 {
15136 }
15138 else
15139 {
15140 /* We have patterns that allow zero sets of memory, for instance.
15141 In 64-bit mode, we should probably support all 8-byte vectors,
15142 since we can in fact encode that into an immediate. */
15144 {
15147 }
15150 {
15152 {
15155 }
15158 {
15161 else
15163 }
15164 }
15169 else
15171 }
15172 }
15174 static bool
15176 {
15179 }
15180 \f
15181 /* Print a memory operand whose address is ADDR. */
15183 static void
15185 {
15194 {
15201 }
15203 {
15207 }
15209 {
15213 {
15216 }
15219 }
15221 {
15226 }
15227 else
15238 {
15249 }
15251 /* Use one byte shorter RIP relative addressing for 64bit mode. */
15253 {
15265 }
15267 {
15268 /* Displacement only requires special attention. */
15271 {
15275 }
15278 else
15280 }
15281 else
15282 {
15283 /* Print SImode register names to force addr32 prefix. */
15285 {
15286 #ifdef ENABLE_CHECKING
15289 {
15300 }
15301 #endif
15304 }
15306 && TARGET_X32
15307 && disp
15310 {
15311 /* X32 runs in 64-bit mode, where displacement, DISP, in
15312 address DISP(%r64), is encoded as 32-bit immediate sign-
15313 extended from 32-bit to 64-bit. For -0x40000300(%r64),
15314 address is %r64 + 0xffffffffbffffd00. When %r64 <
15315 0x40000300, like 0x37ffe064, address is 0xfffffffff7ffdd64,
15316 which is invalid for x32. The correct address is %r64
15317 - 0x40000300 == 0xf7ffdd64. To properly encode
15318 -0x40000300(%r64) for x32, we zero-extend negative
15319 displacement by forcing addr32 prefix which truncates
15320 0xfffffffff7ffdd64 to 0xf7ffdd64. In theory, we should
15321 zero-extend all negative displacements, including -1(%rsp).
15322 However, for small negative displacements, sign-extension
15323 won't cause overflow. We only zero-extend negative
15324 displacements if they < -16*1024*1024, which is also used
15325 to check legitimate address displacements for PIC. */
15327 }
15330 {
15332 {
15337 else
15339 }
15345 {
15350 }
15352 }
15353 else
15354 {
15358 {
15359 /* Pull out the offset of a symbol; print any symbol itself. */
15363 {
15367 }
15375 else
15377 }
15381 {
15384 {
15388 }
15389 }
15392 else
15396 {
15401 }
15403 }
15404 }
15405 }
15407 /* Implementation of TARGET_ASM_OUTPUT_ADDR_CONST_EXTRA. */
15409 static bool
15411 {
15412 rtx op;
15419 {
15422 /* FIXME: This might be @TPOFF in Sun ld. */
15433 else
15445 else
15452 #if TARGET_MACHO
15458 #endif
15461 {
15466 #ifdef TARGET_THREAD_SPLIT_STACK_OFFSET
15468 #else
15470 #endif
15473 }
15478 }
15481 }
15482 \f
15483 /* Split one or more double-mode RTL references into pairs of half-mode
15484 references. The RTL can be REG, offsettable MEM, integer constant, or
15485 CONST_DOUBLE. "operands" is a pointer to an array of double-mode RTLs to
15486 split and "num" is its length. lo_half and hi_half are output arrays
15487 that parallel "operands". */
15489 void
15492 {
15497 {
15506 }
15511 {
15514 /* simplify_subreg refuse to split volatile memory addresses,
15515 but we still have to handle it. */
15517 {
15520 }
15521 else
15522 {
15529 }
15530 }
15531 }
15532 \f
15533 /* Output code to perform a 387 binary operation in INSN, one of PLUS,
15534 MINUS, MULT or DIV. OPERANDS are the insn operands, where operands[3]
15535 is the expression of the binary operation. The output may either be
15536 emitted here, or returned to the caller, like all output_* functions.
15538 There is no guarantee that the operands are the same mode, as they
15539 might be within FLOAT or FLOAT_EXTEND expressions. */
15541 #ifndef SYSV386_COMPAT
15542 /* Set to 1 for compatibility with brain-damaged assemblers. No-one
15543 wants to fix the assemblers because that causes incompatibility
15544 with gcc. No-one wants to fix gcc because that causes
15545 incompatibility with assemblers... You can use the option of
15546 -DSYSV386_COMPAT=0 if you recompile both gcc and gas this way. */
15547 #define SYSV386_COMPAT 1
15548 #endif
15552 {
15558 #ifdef ENABLE_CHECKING
15559 /* Even if we do not want to check the inputs, this documents input
15560 constraints. Which helps in understanding the following code. */
15570 else
15572 #endif
15575 {
15580 else
15589 else
15598 else
15607 else
15614 }
15617 {
15619 {
15623 else
15625 }
15626 else
15627 {
15631 else
15633 }
15635 }
15639 {
15643 {
15647 }
15649 /* know operands[0] == operands[1]. */
15652 {
15655 }
15658 {
15660 /* How is it that we are storing to a dead operand[2]?
15661 Well, presumably operands[1] is dead too. We can't
15662 store the result to st(0) as st(0) gets popped on this
15663 instruction. Instead store to operands[2] (which I
15664 think has to be st(1)). st(1) will be popped later.
15665 gcc <= 2.8.1 didn't have this check and generated
15666 assembly code that the Unixware assembler rejected. */
15668 else
15671 }
15675 else
15682 {
15685 }
15688 {
15691 }
15694 {
15695 #if SYSV386_COMPAT
15696 /* The SystemV/386 SVR3.2 assembler, and probably all AT&T
15697 derived assemblers, confusingly reverse the direction of
15698 the operation for fsub{r} and fdiv{r} when the
15699 destination register is not st(0). The Intel assembler
15700 doesn't have this brain damage. Read !SYSV386_COMPAT to
15701 figure out what the hardware really does. */
15704 else
15706 #else
15708 /* As above for fmul/fadd, we can't store to st(0). */
15710 else
15712 #endif
15714 }
15717 {
15718 #if SYSV386_COMPAT
15721 else
15723 #else
15726 else
15728 #endif
15730 }
15733 {
15736 else
15739 }
15741 {
15742 #if SYSV386_COMPAT
15744 #else
15746 #endif
15747 }
15748 else
15749 {
15750 #if SYSV386_COMPAT
15752 #else
15754 #endif
15755 }
15760 }
15764 }
15766 /* Check if a 256bit AVX register is referenced inside of EXP. */
15768 static int
15770 {
15781 }
15783 /* Return needed mode for entity in optimize_mode_switching pass. */
15785 static int
15787 {
15789 {
15790 rtx link;
15792 /* Needed mode is set to AVX_U128_CLEAN if there are
15793 no 256bit modes used in function arguments. */
15795 link;
15797 {
15799 {
15804 }
15805 }
15808 }
15810 /* Require DIRTY mode if a 256bit AVX register is referenced. Hardware
15811 changes state only when a 256bit register is written to, but we need
15812 to prevent the compiler from moving optimal insertion point above
15813 eventual read from 256bit register. */
15818 }
15820 /* Return mode that i387 must be switched into
15821 prior to the execution of insn. */
15823 static int
15825 {
15828 /* The mode UNINITIALIZED is used to store control word after a
15829 function call or ASM pattern. The mode ANY specify that function
15830 has no requirements on the control word and make no changes in the
15831 bits we are interested in. */
15845 {
15868 }
15871 }
15873 /* Return mode that entity must be switched into
15874 prior to the execution of insn. */
15876 int
15878 {
15880 {
15890 }
15892 }
15894 /* Check if a 256bit AVX register is referenced in stores. */
15896 static void
15898 {
15900 {
15903 }
15904 }
15906 /* Calculate mode of upper 128bit AVX registers after the insn. */
15908 static int
15910 {
15917 /* We know that state is clean after CALL insn if there are no
15918 256bit registers used in the function return register. */
15920 {
15925 }
15927 /* Otherwise, return current mode. Remember that if insn
15928 references AVX 256bit registers, the mode was already changed
15929 to DIRTY from MODE_NEEDED. */
15931 }
15933 /* Return the mode that an insn results in. */
15935 int
15937 {
15939 {
15949 }
15950 }
15952 static int
15954 {
15955 tree arg;
15957 /* Entry mode is set to AVX_U128_DIRTY if there are
15958 256bit modes used in function arguments. */
15961 {
15966 }
15969 }
15971 /* Return a mode that ENTITY is assumed to be
15972 switched to at function entry. */
15974 int
15976 {
15978 {
15988 }
15989 }
15991 static int
15993 {
15996 /* Exit mode is set to AVX_U128_DIRTY if there are
15997 256bit modes used in the function return register. */
16002 }
16004 /* Return a mode that ENTITY is assumed to be
16005 switched to at function exit. */
16007 int
16009 {
16011 {
16021 }
16022 }
16024 /* Output code to initialize control word copies used by trunc?f?i and
16025 rounding patterns. CURRENT_MODE is set to current control word,
16026 while NEW_MODE is set to new control word. */
16028 static void
16030 {
16032 rtx new_mode;
16043 {
16045 {
16047 /* round toward zero (truncate) */
16053 /* round down toward -oo */
16060 /* round up toward +oo */
16067 /* mask precision exception for nearbyint() */
16074 }
16075 }
16076 else
16077 {
16079 {
16081 /* round toward zero (truncate) */
16087 /* round down toward -oo */
16093 /* round up toward +oo */
16099 /* mask precision exception for nearbyint() */
16106 }
16107 }
16113 }
16115 /* Emit vzeroupper. */
16117 void
16119 {
16122 /* Cancel automatic vzeroupper insertion if there are
16123 live call-saved SSE registers at the insertion point. */
16135 }
16137 /* Generate one or more insns to set ENTITY to MODE. */
16139 void
16141 {
16143 {
16158 }
16159 }
16161 /* Output code for INSN to convert a float to a signed int. OPERANDS
16162 are the insn operands. The output may be [HSD]Imode and the input
16163 operand may be [SDX]Fmode. */
16167 {
16172 /* Jump through a hoop or two for DImode, since the hardware has no
16173 non-popping instruction. We used to do this a different way, but
16174 that was somewhat fragile and broke with post-reload splitters. */
16184 else
16185 {
16190 else
16194 }
16197 }
16199 /* Output code for x87 ffreep insn. The OPNO argument, which may only
16200 have the values zero or one, indicates the ffreep insn's operand
16201 from the OPERANDS array. */
16205 {
16207 #ifdef HAVE_AS_IX86_FFREEP
16209 #else
16210 {
16220 }
16221 #endif
16224 }
16227 /* Output code for INSN to compare OPERANDS. EFLAGS_P is 1 when fcomi
16228 should be used. UNORDERED_P is true when fucom should be used. */
16232 {
16238 {
16241 }
16242 else
16243 {
16246 }
16249 {
16253 else
16255 else
16258 else
16260 }
16267 {
16269 {
16272 }
16273 else
16275 }
16278 && stack_top_dies
16281 {
16282 /* If both the top of the 387 stack dies, and the other operand
16283 is also a stack register that dies, then this must be a
16284 `fcompp' float compare */
16287 {
16288 /* There is no double popping fcomi variant. Fortunately,
16289 eflags is immune from the fstp's cc clobbering. */
16292 else
16295 }
16296 else
16297 {
16300 else
16302 }
16303 }
16304 else
16305 {
16306 /* Encoded here as eflags_p | intmode | unordered_p | stack_top_dies. */
16309 {
16317 NULL,
16318 NULL,
16325 NULL,
16326 NULL,
16327 NULL,
16328 NULL
16329 };
16344 }
16345 }
16347 void
16349 {
16352 #ifdef ASM_QUAD
16355 #else
16357 #endif
16360 }
16362 void
16364 {
16367 #ifdef ASM_QUAD
16370 #else
16372 #endif
16373 /* We can't use @GOTOFF for text labels on VxWorks; see gotoff_operand. */
16379 #if TARGET_MACHO
16381 {
16385 }
16386 #endif
16387 else
16390 }
16391 \f
16392 /* Generate either "mov $0, reg" or "xor reg, reg", as appropriate
16393 for the target. */
16395 void
16397 {
16398 rtx tmp;
16400 /* We play register width games, which are only valid after reload. */
16403 /* Avoid HImode and its attendant prefix byte. */
16408 /* This predicate should match that for movsi_xor and movdi_xor_rex64. */
16410 {
16413 }
16416 }
16418 /* X is an unchanging MEM. If it is a constant pool reference, return
16419 the constant pool rtx, else NULL. */
16421 rtx
16423 {
16430 }
16432 void
16434 {
16442 {
16443 rtx tmp;
16447 {
16453 }
16456 }
16460 {
16463 rtx tmp;
16468 else
16472 {
16479 }
16480 }
16484 {
16486 {
16487 #if TARGET_MACHO
16488 /* dynamic-no-pic */
16490 {
16491 rtx temp = ((reload_in_progress
16499 }
16501 {
16505 }
16508 else
16509 {
16517 }
16518 /* dynamic-no-pic */
16519 #endif
16520 }
16521 else
16522 {
16526 {
16532 }
16533 }
16534 }
16535 else
16536 {
16547 /* Force large constants in 64bit compilation into register
16548 to get them CSEed. */
16560 {
16561 /* If we are loading a floating point constant to a register,
16562 force the value to memory now, since we'll get better code
16563 out the back end. */
16567 {
16572 }
16573 }
16574 }
16577 }
16579 void
16581 {
16585 /* Force constants other than zero into memory. We do not know how
16586 the instructions used to build constants modify the upper 64 bits
16587 of the register, once we have that information we may be able
16588 to handle some of them more efficiently. */
16597 /* We need to check memory alignment for SSE mode since attribute
16598 can make operands unaligned. */
16603 {
16606 /* ix86_expand_vector_move_misalign() does not like constants ... */
16612 /* ... nor both arguments in memory. */
16620 }
16622 /* Make operand1 a register if it isn't already. */
16626 {
16629 }
16632 }
16634 /* Split 32-byte AVX unaligned load and store if needed. */
16636 static void
16638 {
16639 rtx m;
16646 {
16667 }
16670 {
16672 {
16679 }
16680 /* Normal *mov<mode>_internal pattern will handle
16681 unaligned loads just fine if misaligned_operand
16682 is true, and without the UNSPEC it can be combined
16683 with arithmetic instructions. */
16686 else
16688 }
16690 {
16692 {
16697 }
16698 else
16700 }
16701 else
16703 }
16705 /* Implement the movmisalign patterns for SSE. Non-SSE modes go
16706 straight to ix86_expand_vector_move. */
16707 /* Code generation for scalar reg-reg moves of single and double precision data:
16708 if (x86_sse_partial_reg_dependency == true | x86_sse_split_regs == true)
16709 movaps reg, reg
16710 else
16711 movss reg, reg
16712 if (x86_sse_partial_reg_dependency == true)
16713 movapd reg, reg
16714 else
16715 movsd reg, reg
16717 Code generation for scalar loads of double precision data:
16718 if (x86_sse_split_regs == true)
16719 movlpd mem, reg (gas syntax)
16720 else
16721 movsd mem, reg
16723 Code generation for unaligned packed loads of single precision data
16724 (x86_sse_unaligned_move_optimal overrides x86_sse_partial_reg_dependency):
16725 if (x86_sse_unaligned_move_optimal)
16726 movups mem, reg
16728 if (x86_sse_partial_reg_dependency == true)
16729 {
16730 xorps reg, reg
16731 movlps mem, reg
16732 movhps mem+8, reg
16733 }
16734 else
16735 {
16736 movlps mem, reg
16737 movhps mem+8, reg
16738 }
16740 Code generation for unaligned packed loads of double precision data
16741 (x86_sse_unaligned_move_optimal overrides x86_sse_split_regs):
16742 if (x86_sse_unaligned_move_optimal)
16743 movupd mem, reg
16745 if (x86_sse_split_regs == true)
16746 {
16747 movlpd mem, reg
16748 movhpd mem+8, reg
16749 }
16750 else
16751 {
16752 movsd mem, reg
16753 movhpd mem+8, reg
16754 }
16755 */
16757 void
16759 {
16768 {
16770 {
16774 {
16776 {
16779 }
16780 else
16782 }
16784 /* FALLTHRU */
16788 {
16803 }
16809 else
16817 }
16820 }
16824 {
16826 {
16830 {
16832 {
16835 }
16836 else
16838 }
16840 /* FALLTHRU */
16850 }
16853 }
16856 {
16857 /* Normal *mov<mode>_internal pattern will handle
16858 unaligned loads just fine if misaligned_operand
16859 is true, and without the UNSPEC it can be combined
16860 with arithmetic instructions. */
16866 /* ??? If we have typed data, then it would appear that using
16867 movdqu is the only way to get unaligned data loaded with
16868 integer type. */
16870 {
16872 {
16875 }
16877 /* We will eventually emit movups based on insn attributes. */
16881 }
16883 {
16884 rtx zero;
16887 || TARGET_SSE_UNALIGNED_LOAD_OPTIMAL
16888 || TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL
16890 {
16891 /* We will eventually emit movups based on insn attributes. */
16894 }
16896 /* When SSE registers are split into halves, we can avoid
16897 writing to the top half twice. */
16899 {
16902 }
16903 else
16904 {
16905 /* ??? Not sure about the best option for the Intel chips.
16906 The following would seem to satisfy; the register is
16907 entirely cleared, breaking the dependency chain. We
16908 then store to the upper half, with a dependency depth
16909 of one. A rumor has it that Intel recommends two movsd
16910 followed by an unpacklpd, but this is unconfirmed. And
16911 given that the dependency depth of the unpacklpd would
16912 still be one, I'm not sure why this would be better. */
16914 }
16920 }
16921 else
16922 {
16923 rtx t;
16926 || TARGET_SSE_UNALIGNED_LOAD_OPTIMAL
16927 || TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL
16929 {
16931 {
16934 }
16941 }
16945 else
16950 else
16959 }
16960 }
16962 {
16964 {
16967 /* We will eventually emit movups based on insn attributes. */
16969 }
16971 {
16973 || TARGET_SSE_UNALIGNED_STORE_OPTIMAL
16974 || TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL
16976 /* We will eventually emit movups based on insn attributes. */
16978 else
16979 {
16984 }
16985 }
16986 else
16987 {
16992 || TARGET_SSE_UNALIGNED_STORE_OPTIMAL
16993 || TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL
16995 {
16998 }
16999 else
17000 {
17005 }
17006 }
17007 }
17008 else
17010 }
17012 /* Expand a push in MODE. This is some mode for which we do not support
17013 proper push instructions, at least from the registers that we expect
17014 the value to live in. */
17016 void
17018 {
17019 rtx tmp;
17029 /* When we push an operand onto stack, it has to be aligned at least
17030 at the function argument boundary. However since we don't have
17031 the argument type, we can't determine the actual argument
17032 boundary. */
17034 }
17036 /* Helper function of ix86_fixup_binary_operands to canonicalize
17037 operand order. Returns true if the operands should be swapped. */
17039 static bool
17041 rtx operands[])
17042 {
17047 /* If the operation is not commutative, we can't do anything. */
17051 /* Highest priority is that src1 should match dst. */
17057 /* Next highest priority is that immediate constants come second. */
17063 /* Lowest priority is that memory references should come second. */
17070 }
17073 /* Fix up OPERANDS to satisfy ix86_binary_operator_ok. Return the
17074 destination to use for the operation. If different from the true
17075 destination in operands[0], a copy operation will be required. */
17077 rtx
17079 rtx operands[])
17080 {
17085 /* Canonicalize operand order. */
17087 {
17088 rtx temp;
17090 /* It is invalid to swap operands of different modes. */
17096 }
17098 /* Both source operands cannot be in memory. */
17100 {
17101 /* Optimization: Only read from memory once. */
17103 {
17106 }
17109 else
17111 }
17113 /* If the destination is memory, and we do not have matching source
17114 operands, do things in registers. */
17118 /* Source 1 cannot be a constant. */
17122 /* Source 1 cannot be a non-matching memory. */
17126 /* Improve address combine. */
17135 }
17137 /* Similarly, but assume that the destination has already been
17138 set up properly. */
17140 void
17143 {
17146 }
17148 /* Attempt to expand a binary operator. Make the expansion closer to the
17149 actual machine, then just general_operand, which will allow 3 separate
17150 memory references (one output, two input) in a single insn. */
17152 void
17154 rtx operands[])
17155 {
17162 /* Emit the instruction. */
17166 {
17167 /* Reload doesn't know about the flags register, and doesn't know that
17168 it doesn't want to clobber it. We can only do this with PLUS. */
17171 }
17175 {
17176 /* This is going to be an LEA; avoid splitting it later. */
17178 }
17179 else
17180 {
17183 }
17185 /* Fix up the destination if needed. */
17188 }
17190 /* Expand vector logical operation CODE (AND, IOR, XOR) in MODE with
17191 the given OPERANDS. */
17193 void
17195 rtx operands[])
17196 {
17199 {
17202 }
17204 {
17207 }
17208 /* Optimize (__m128i) d | (__m128i) e and similar code
17209 when d and e are float vectors into float vector logical
17210 insn. In C/C++ without using intrinsics there is no other way
17211 to express vector logical operation on float vectors than
17212 to cast them temporarily to integer vectors. */
17214 && !TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL
17223 {
17224 rtx dst;
17226 {
17233 {
17236 }
17237 else
17238 {
17243 }
17254 }
17255 }
17264 }
17266 /* Return TRUE or FALSE depending on whether the binary operator meets the
17267 appropriate constraints. */
17269 bool
17272 {
17277 /* Both source operands cannot be in memory. */
17281 /* Canonicalize operand order for commutative operators. */
17283 {
17287 }
17289 /* If the destination is memory, we must have a matching source operand. */
17293 /* Source 1 cannot be a constant. */
17297 /* Source 1 cannot be a non-matching memory. */
17299 /* Support "andhi/andsi/anddi" as a zero-extending move. */
17307 }
17309 /* Attempt to expand a unary operator. Make the expansion closer to the
17310 actual machine, then just general_operand, which will allow 2 separate
17311 memory references (one output, one input) in a single insn. */
17313 void
17315 rtx operands[])
17316 {
17323 /* If the destination is memory, and we do not have matching source
17324 operands, do things in registers. */
17327 {
17330 else
17332 }
17334 /* When source operand is memory, destination must match. */
17338 /* Emit the instruction. */
17342 {
17343 /* Reload doesn't know about the flags register, and doesn't know that
17344 it doesn't want to clobber it. */
17347 }
17348 else
17349 {
17352 }
17354 /* Fix up the destination if needed. */
17357 }
17359 /* Split 32bit/64bit divmod with 8bit unsigned divmod if dividend and
17360 divisor are within the range [0-255]. */
17362 void
17365 {
17374 {
17387 }
17394 /* Use 8bit unsigned divimod if dividend and divisor are within
17395 the range [0-255]. */
17404 pc_rtx);
17409 /* Generate original signed/unsigned divimod. */
17414 /* Branch to the end. */
17418 /* Generate 8bit unsigned divide. */
17420 /* Don't use operands[0] for result of 8bit divide since not all
17421 registers support QImode ZERO_EXTRACT. */
17428 {
17431 }
17432 else
17433 {
17436 }
17438 /* Extract remainder from AH. */
17442 else
17443 {
17444 /* Need a new scratch register since the old one has result
17445 of 8bit divide. */
17449 }
17452 /* Zero extend quotient from AL. */
17458 }
17460 /* Whether it is OK to emit CFI directives when emitting asm code. */
17462 bool
17464 {
17466 }
17468 #define LEA_MAX_STALL (3)
17469 #define LEA_SEARCH_THRESHOLD (LEA_MAX_STALL << 1)
17471 /* Increase given DISTANCE in half-cycles according to
17472 dependencies between PREV and NEXT instructions.
17473 Add 1 half-cycle if there is no dependency and
17474 go to next cycle if there is some dependecy. */
17476 static unsigned int
17478 {
17495 }
17497 /* Function checks if instruction INSN defines register number
17498 REGNO1 or REGNO2. */
17500 static bool
17502 rtx insn)
17503 {
17511 {
17513 }
17516 }
17518 /* Function checks if instruction INSN uses register number
17519 REGNO as a part of address expression. */
17521 static bool
17523 {
17531 }
17533 /* Search backward for non-agu definition of register number REGNO1
17534 or register number REGNO2 in basic block starting from instruction
17535 START up to head of basic block or instruction INSN.
17537 Function puts true value into *FOUND var if definition was found
17538 and false otherwise.
17540 Distance in half-cycles between START and found instruction or head
17541 of BB is added to DISTANCE and returned. */
17543 static int
17547 {
17557 {
17559 {
17562 {
17565 {
17568 }
17569 }
17572 }
17577 }
17580 }
17582 /* Search backward for non-agu definition of register number REGNO1
17583 or register number REGNO2 in INSN's basic block until
17584 1. Pass LEA_SEARCH_THRESHOLD instructions, or
17585 2. Reach neighbour BBs boundary, or
17586 3. Reach agu definition.
17587 Returns the distance between the non-agu definition point and INSN.
17588 If no definition point, returns -1. */
17590 static int
17592 rtx insn)
17593 {
17604 {
17605 edge e;
17606 edge_iterator ei;
17611 {
17614 }
17620 else
17621 {
17626 {
17627 int bb_dist
17633 {
17640 }
17641 }
17644 }
17645 }
17647 /* get_attr_type may modify recog data. We want to make sure
17648 that recog data is valid for instruction INSN, on which
17649 distance_non_agu_define is called. INSN is unchanged here. */
17656 }
17658 /* Return the distance in half-cycles between INSN and the next
17659 insn that uses register number REGNO in memory address added
17660 to DISTANCE. Return -1 if REGNO0 is set.
17662 Put true value into *FOUND if register usage was found and
17663 false otherwise.
17664 Put true value into *REDEFINED if register redefinition was
17665 found and false otherwise. */
17667 static int
17671 {
17680 {
17683 /* If insn and start belong to the same bb, set prev to insn,
17684 so the call to increase_distance will increase the distance
17685 between insns by 1. */
17687 }
17692 {
17694 {
17697 {
17698 /* Return DISTANCE if OP0 is used in memory
17699 address in NEXT. */
17702 }
17705 {
17706 /* Return -1 if OP0 is set in NEXT. */
17709 }
17712 }
17718 }
17721 }
17723 /* Return the distance between INSN and the next insn that uses
17724 register number REGNO0 in memory address. Return -1 if no such
17725 a use is found within LEA_SEARCH_THRESHOLD or REGNO0 is set. */
17727 static int
17729 {
17741 {
17742 edge e;
17743 edge_iterator ei;
17748 {
17751 }
17757 else
17758 {
17764 {
17765 int bb_dist
17770 {
17777 }
17778 }
17781 }
17782 }
17788 }
17790 /* Define this macro to tune LEA priority vs ADD, it take effect when
17791 there is a dilemma of choicing LEA or ADD
17792 Negative value: ADD is more preferred than LEA
17793 Zero: Netrual
17794 Positive value: LEA is more preferred than ADD*/
17795 #define IX86_LEA_PRIORITY 0
17797 /* Return true if usage of lea INSN has performance advantage
17798 over a sequence of instructions. Instructions sequence has
17799 SPLIT_COST cycles higher latency than lea latency. */
17801 static bool
17804 {
17807 /* For Silvermont if using a 2-source or 3-source LEA for
17808 non-destructive destination purposes, or due to wanting
17809 ability to use SCALE, the use of LEA is justified. */
17811 {
17819 }
17825 {
17826 /* If there is no non AGU operand definition, no AGU
17827 operand usage and split cost is 0 then both lea
17828 and non lea variants have same priority. Currently
17829 we prefer lea for 64 bit code and non lea on 32 bit
17830 code. */
17833 else
17835 }
17837 /* With longer definitions distance lea is more preferable.
17838 Here we change it to take into account splitting cost and
17839 lea priority. */
17842 /* If there is no use in memory addess then we just check
17843 that split cost exceeds AGU stall. */
17847 /* If this insn has both backward non-agu dependence and forward
17848 agu dependence, the one with short distance takes effect. */
17850 }
17852 /* Return true if it is legal to clobber flags by INSN and
17853 false otherwise. */
17855 static bool
17857 {
17860 bitmap live;
17863 {
17865 {
17872 }
17878 }
17882 }
17884 /* Return true if we need to split op0 = op1 + op2 into a sequence of
17885 move and add to avoid AGU stalls. */
17887 bool
17889 {
17892 /* Check if we need to optimize. */
17896 /* Check it is correct to split here. */
17904 /* We need to split only adds with non destructive
17905 destination operand. */
17908 else
17910 }
17912 /* Return true if we should emit lea instruction instead of mov
17913 instruction. */
17915 bool
17917 {
17920 /* Check if we need to optimize. */
17924 /* Use lea for reg to reg moves only. */
17932 }
17934 /* Return true if we need to split lea into a sequence of
17935 instructions to avoid AGU stalls. */
17937 bool
17939 {
17945 /* Check we need to optimize. */
17949 /* Check it is correct to split here. */
17956 /* There should be at least two components in the address. */
17961 /* We should not split into add if non legitimate pic
17962 operand is used as displacement. */
17977 /* Compute how many cycles we will add to execution time
17978 if split lea into a sequence of instructions. */
17980 {
17981 /* Have to use mov instruction if non desctructive
17982 destination form is used. */
17986 /* Have to add index to base if both exist. */
17990 /* Have to use shift and adds if scale is 2 or greater. */
17992 {
17997 else
17999 }
18001 /* Have to use add instruction with immediate if
18002 disp is non zero. */
18006 /* Subtract the price of lea. */
18008 }
18012 }
18014 /* Emit x86 binary operand CODE in mode MODE, where the first operand
18015 matches destination. RTX includes clobber of FLAGS_REG. */
18017 static void
18020 {
18027 }
18029 /* Return true if regno1 def is nearest to the insn. */
18031 static bool
18033 {
18040 {
18042 {
18045 }
18051 }
18053 /* None of the regs is defined in the bb. */
18055 }
18057 /* Split lea instructions into a sequence of instructions
18058 which are executed on ALU to avoid AGU stalls.
18059 It is assumed that it is allowed to clobber flags register
18060 at lea position. */
18062 void
18064 {
18080 {
18083 }
18086 {
18089 }
18095 {
18096 /* Case r1 = r1 + ... */
18098 {
18099 /* If we have a case r1 = r1 + C * r1 then we
18100 should use multiplication which is very
18101 expensive. Assume cost model is wrong if we
18102 have such case here. */
18107 }
18108 else
18109 {
18110 /* r1 = r2 + r3 * C case. Need to move r3 into r1. */
18114 /* Use shift for scaling. */
18123 }
18124 }
18126 {
18129 }
18130 else
18131 {
18133 {
18136 }
18138 {
18141 }
18142 else
18143 {
18148 else
18149 {
18150 rtx tmp1;
18152 /* Find better operand for SET instruction, depending
18153 on which definition is farther from the insn. */
18156 else
18166 }
18169 }
18173 }
18174 }
18176 /* Return true if it is ok to optimize an ADD operation to LEA
18177 operation to avoid flag register consumation. For most processors,
18178 ADD is faster than LEA. For the processors like ATOM, if the
18179 destination register of LEA holds an actual address which will be
18180 used soon, LEA is better and otherwise ADD is better. */
18182 bool
18184 {
18189 /* If a = b + c, (a!=b && a!=c), must use lea form. */
18197 }
18199 /* Return true if destination reg of SET_BODY is shift count of
18200 USE_BODY. */
18202 static bool
18204 {
18205 rtx set_dest;
18206 rtx shift_rtx;
18209 /* Retrieve destination of SET_BODY. */
18211 {
18220 use_body))
18225 }
18227 /* Retrieve shift count of USE_BODY. */
18229 {
18241 }
18249 {
18252 /* Return true if shift count is dest of SET_BODY. */
18254 {
18255 /* Add check since it can be invoked before register
18256 allocation in pre-reload schedule. */
18262 }
18263 }
18266 }
18268 /* Return true if destination reg of SET_INSN is shift count of
18269 USE_INSN. */
18271 bool
18273 {
18276 }
18278 /* Return TRUE or FALSE depending on whether the unary operator meets the
18279 appropriate constraints. */
18281 bool
18285 {
18286 /* If one of operands is memory, source and destination must match. */
18292 }
18294 /* Return TRUE if the operands to a vec_interleave_{high,low}v2df
18295 are ok, keeping in mind the possible movddup alternative. */
18297 bool
18299 {
18305 }
18307 /* Post-reload splitter for converting an SF or DFmode value in an
18308 SSE register into an unsigned SImode. */
18310 void
18312 {
18323 /* Load up the value into the low element. We must ensure that the other
18324 elements are valid floats -- zero is the easiest such value. */
18326 {
18329 else
18331 }
18332 else
18333 {
18338 else
18340 }
18360 else
18365 }
18367 /* Convert an unsigned DImode value into a DFmode, using only SSE.
18368 Expects the 64-bit DImode to be supplied in a pair of integral
18369 registers. Requires SSE2; will use SSE3 if available. For x86_32,
18370 -mfpmath=sse, !optimize_size only. */
18372 void
18374 {
18378 rtx x;
18384 {
18387 }
18388 else
18389 {
18393 }
18401 /* int_xmm = {0x45300000UL, fp_xmm/hi, 0x43300000, fp_xmm/lo } */
18404 /* Concatenating (juxtaposing) (0x43300000UL ## fp_value_low_xmm)
18405 yields a valid DF value equal to (0x1.0p52 + double(fp_value_lo_xmm)).
18406 Similarly (0x45300000UL ## fp_value_hi_xmm) yields
18407 (0x1.0p84 + double(fp_value_hi_xmm)).
18408 Note these exponents differ by 32. */
18412 /* Subtract off those 0x1.0p52 and 0x1.0p84 biases, to produce values
18413 in [0,2**32-1] and [0]+[2**32,2**64-1] respectively. */
18422 /* Add the upper and lower DFmode values together. */
18425 else
18426 {
18430 }
18433 }
18435 /* Not used, but eases macroization of patterns. */
18436 void
18438 rtx input ATTRIBUTE_UNUSED)
18439 {
18441 }
18443 /* Convert an unsigned SImode value into a DFmode. Only currently used
18444 for SSE, but applicable anywhere. */
18446 void
18448 {
18449 REAL_VALUE_TYPE TWO31r;
18464 }
18466 /* Convert a signed DImode value into a DFmode. Only used for SSE in
18467 32-bit mode; otherwise we have a direct convert instruction. */
18469 void
18471 {
18472 REAL_VALUE_TYPE TWO32r;
18490 }
18492 /* Convert an unsigned SImode value into a SFmode, using only SSE.
18493 For x86_32, -mfpmath=sse, !optimize_size only. */
18494 void
18496 {
18497 REAL_VALUE_TYPE ONE16r;
18516 }
18518 /* floatunsv{4,8}siv{4,8}sf2 expander. Expand code to convert
18519 a vector of unsigned ints VAL to vector of floats TARGET. */
18521 void
18523 {
18525 REAL_VALUE_TYPE TWO16r;
18532 else
18537 OPTAB_DIRECT);
18548 OPTAB_DIRECT);
18550 OPTAB_DIRECT);
18553 }
18555 /* Adjust a V*SFmode/V*DFmode value VAL so that *sfix_trunc* resp. fix_trunc*
18556 pattern can be used on it instead of *ufix_trunc* resp. fixuns_trunc*.
18557 This is done by doing just signed conversion if < 0x1p31, and otherwise by
18558 subtracting 0x1p31 first and xoring in 0x80000000 from *XORP afterwards. */
18560 rtx
18562 {
18563 REAL_VALUE_TYPE TWO31r;
18578 {
18584 }
18593 OPTAB_DIRECT);
18594 else
18595 {
18601 OPTAB_DIRECT);
18602 }
18605 }
18607 /* A subroutine of ix86_build_signbit_mask. If VECT is true,
18608 then replicate the value for all elements of the vector
18609 register. */
18611 rtx
18613 {
18615 rtvec v;
18619 {
18646 }
18647 }
18649 /* A subroutine of ix86_expand_fp_absneg_operator, copysign expanders
18650 and ix86_expand_int_vcond. Create a mask for the sign bit in MODE
18651 for an SSE register. If VECT is true, then replicate the mask for
18652 all elements of the vector register. If INVERT is true, then create
18653 a mask excluding the sign bit. */
18655 rtx
18657 {
18661 rtx v;
18662 rtx mask;
18664 /* Find the sign bit, sign extended to 2*HWI. */
18666 {
18686 else
18694 {
18697 }
18698 else
18699 {
18700 rtvec vec;
18706 {
18709 }
18710 else
18720 }
18725 }
18730 /* Force this value into the low part of a fp vector constant. */
18739 }
18741 /* Generate code for floating point ABS or NEG. */
18743 void
18745 rtx operands[])
18746 {
18757 {
18763 }
18765 /* NEG and ABS performed with SSE use bitwise mask operations.
18766 Create the appropriate mask now. */
18769 else
18779 {
18781 rtvec par;
18786 else
18787 {
18790 }
18792 }
18793 else
18795 }
18797 /* Expand a copysign operation. Special case operand 0 being a constant. */
18799 void
18801 {
18815 else
18819 {
18826 {
18829 else
18830 {
18834 }
18835 }
18845 else
18849 }
18850 else
18851 {
18861 else
18865 }
18866 }
18868 /* Deconstruct a copysign operation into bit masks. Operand 0 is known to
18869 be a constant, and so has already been expanded into a vector constant. */
18871 void
18873 {
18889 {
18892 }
18893 }
18895 /* Deconstruct a copysign operation into bit masks. Operand 0 is variable,
18896 so we have to do two masks. */
18898 void
18900 {
18915 {
18916 /* Shouldn't happen often (it's useless, obviously), but when it does
18917 we'd generate incorrect code if we continue below. */
18920 }
18923 {
18934 }
18935 else
18936 {
18938 {
18940 }
18942 {
18946 }
18950 {
18953 }
18955 {
18960 }
18962 }
18966 }
18968 /* Return TRUE or FALSE depending on whether the first SET in INSN
18969 has source and destination with matching CC modes, and that the
18970 CC mode is at least as constrained as REQ_MODE. */
18972 bool
18974 {
18975 rtx set;
18986 {
18996 /* FALLTHRU */
19000 /* FALLTHRU */
19004 /* FALLTHRU */
19018 }
19021 }
19023 /* Generate insn patterns to do an integer compare of OPERANDS. */
19025 static rtx
19027 {
19034 /* This is very simple, but making the interface the same as in the
19035 FP case makes the rest of the code easier. */
19039 /* Return the test that should be put into the flags user, i.e.
19040 the bcc, scc, or cmov instruction. */
19042 }
19044 /* Figure out whether to use ordered or unordered fp comparisons.
19045 Return the appropriate mode to use. */
19047 enum machine_mode
19049 {
19050 /* ??? In order to make all comparisons reversible, we do all comparisons
19051 non-trapping when compiling for IEEE. Once gcc is able to distinguish
19052 all forms trapping and nontrapping comparisons, we can make inequality
19053 comparisons trapping again, since it results in better code when using
19054 FCOM based compares. */
19056 }
19058 enum machine_mode
19060 {
19064 {
19067 }
19070 {
19071 /* Only zero flag is needed. */
19075 /* Codes needing carry flag. */
19078 /* Detect overflow checks. They need just the carry flag. */
19082 else
19087 /* Codes possibly doable only with sign flag when
19088 comparing against zero. */
19093 else
19094 /* For other cases Carry flag is not required. */
19096 /* Codes doable only with sign flag when comparing
19097 against zero, but we miss jump instruction for it
19098 so we need to use relational tests against overflow
19099 that thus needs to be zero. */
19104 else
19106 /* strcmp pattern do (use flags) and combine may ask us for proper
19107 mode. */
19112 }
19113 }
19115 /* Return the fixed registers used for condition codes. */
19117 static bool
19119 {
19123 }
19125 /* If two condition code modes are compatible, return a condition code
19126 mode which is compatible with both. Otherwise, return
19127 VOIDmode. */
19129 static enum machine_mode
19131 {
19148 {
19162 {
19176 }
19180 /* These are only compatible with themselves, which we already
19181 checked above. */
19183 }
19184 }
19187 /* Return a comparison we can do and that it is equivalent to
19188 swap_condition (code) apart possibly from orderedness.
19189 But, never change orderedness if TARGET_IEEE_FP, returning
19190 UNKNOWN in that case if necessary. */
19192 static enum rtx_code
19194 {
19196 {
19207 }
19208 }
19210 /* Return cost of comparison CODE using the best strategy for performance.
19211 All following functions do use number of instructions as a cost metrics.
19212 In future this should be tweaked to compute bytes for optimize_size and
19213 take into account performance of various instructions on various CPUs. */
19215 static int
19217 {
19220 /* The cost of code using bit-twiddling on %ah. */
19222 {
19245 }
19248 {
19255 }
19256 }
19258 /* Return strategy to use for floating-point. We assume that fcomi is always
19259 preferrable where available, since that is also true when looking at size
19260 (2 bytes, vs. 3 for fnstsw+sahf and at least 5 for fnstsw+test). */
19262 enum ix86_fpcmp_strategy
19264 {
19265 /* Do fcomi/sahf based test when profitable. */
19274 }
19276 /* Swap, force into registers, or otherwise massage the two operands
19277 to a fp comparison. The operands are updated in place; the new
19278 comparison code is returned. */
19280 static enum rtx_code
19282 {
19288 /* All of the unordered compare instructions only work on registers.
19289 The same is true of the fcomi compare instructions. The XFmode
19290 compare instructions require registers except when comparing
19291 against zero or when converting operand 1 from fixed point to
19292 floating point. */
19301 {
19304 }
19305 else
19306 {
19307 /* %%% We only allow op1 in memory; op0 must be st(0). So swap
19308 things around if they appear profitable, otherwise force op0
19309 into a register. */
19315 {
19318 {
19319 rtx tmp;
19322 }
19323 }
19329 {
19334 {
19337 }
19338 else
19340 }
19341 }
19343 /* Try to rearrange the comparison to make it cheaper. */
19347 {
19348 rtx tmp;
19353 }
19358 }
19360 /* Convert comparison codes we use to represent FP comparison to integer
19361 code that will result in proper branch. Return UNKNOWN if no such code
19362 is available. */
19364 enum rtx_code
19366 {
19368 {
19391 }
19392 }
19394 /* Generate insn patterns to do a floating point compare of OPERANDS. */
19396 static rtx
19398 {
19405 /* Do fcomi/sahf based test when profitable. */
19407 {
19412 tmp);
19420 tmp);
19429 /* Sadness wrt reg-stack pops killing fpsr -- gotta get fnstsw first. */
19436 /* In the unordered case, we have to check C2 for NaN's, which
19437 doesn't happen to work out to anything nice combination-wise.
19438 So do some bit twiddling on the value we've got in AH to come
19439 up with an appropriate set of condition codes. */
19443 {
19447 {
19450 }
19451 else
19452 {
19458 }
19463 {
19468 }
19469 else
19470 {
19473 }
19478 {
19481 }
19482 else
19483 {
19487 }
19492 {
19498 }
19499 else
19500 {
19503 }
19508 {
19513 }
19514 else
19515 {
19518 }
19523 {
19528 }
19529 else
19530 {
19533 }
19547 }
19552 }
19554 /* Return the test that should be put into the flags user, i.e.
19555 the bcc, scc, or cmov instruction. */
19558 const0_rtx);
19559 }
19561 static rtx
19563 {
19564 rtx ret;
19570 {
19573 }
19574 else
19578 }
19580 void
19582 {
19584 rtx tmp;
19587 {
19594 simple:
19598 pc_rtx);
19606 /* Expand DImode branch into multiple compare+branch. */
19607 {
19613 {
19616 }
19623 /* When comparing for equality, we can use (hi0^hi1)|(lo0^lo1) to
19624 avoid two branches. This costs one extra insn, so disable when
19625 optimizing for size. */
19630 {
19648 }
19650 /* Otherwise, if we are doing less-than or greater-or-equal-than,
19651 op1 is a constant and the low word is zero, then we can just
19652 examine the high word. Similarly for low word -1 and
19653 less-or-equal-than or greater-than. */
19657 {
19660 {
19663 }
19667 {
19670 }
19674 }
19676 /* Otherwise, we need two or three jumps. */
19685 {
19699 }
19701 /*
19702 * a < b =>
19703 * if (hi(a) < hi(b)) goto true;
19704 * if (hi(a) > hi(b)) goto false;
19705 * if (lo(a) < lo(b)) goto true;
19706 * false:
19707 */
19719 }
19724 }
19725 }
19727 /* Split branch based on floating point condition. */
19728 void
19731 {
19732 rtx condition;
19733 rtx i;
19736 {
19741 }
19744 tmp);
19746 /* Remove pushed operand from stack. */
19756 }
19758 void
19760 {
19761 rtx ret;
19768 }
19770 /* Expand comparison setting or clearing carry flag. Return true when
19771 successful and set pop for the operation. */
19772 static bool
19774 {
19778 /* Do not handle double-mode compares that go through special path. */
19783 {
19788 /* Shortcut: following common codes never translate
19789 into carry flag compares. */
19794 /* These comparisons require zero flag; swap operands so they won't. */
19797 {
19802 }
19804 /* Try to expand the comparison and verify that we end up with
19805 carry flag based comparison. This fails to be true only when
19806 we decide to expand comparison using arithmetic that is not
19807 too common scenario. */
19816 else
19825 }
19831 {
19836 /* Convert a==0 into (unsigned)a<1. */
19845 /* Convert a>b into b<a or a>=b-1. */
19849 {
19851 /* Bail out on overflow. We still can swap operands but that
19852 would force loading of the constant into register. */
19857 }
19858 else
19859 {
19864 }
19867 /* Convert a>=0 into (unsigned)a<0x80000000. */
19885 }
19886 /* Swapping operands may cause constant to appear as first operand. */
19888 {
19892 }
19896 }
19898 bool
19900 {
19924 /* Don't attempt mode expansion here -- if we had to expand 5 or 6
19925 HImode insns, we'd be swallowed in word prefix ops. */
19931 {
19935 HOST_WIDE_INT diff;
19938 /* Sign bit compares are better done using shifts than we do by using
19939 sbb. */
19942 {
19943 /* Detect overlap between destination and compare sources. */
19947 {
19948 rtx flags;
19957 {
19959 compare_code
19961 }
19963 /* To simplify rest of code, restrict to the GEU case. */
19965 {
19971 }
19972 else
19973 {
19976 reverse_condition_maybe_unordered
19978 else
19981 }
19990 else
19993 }
19994 else
19995 {
19998 else
19999 {
20004 }
20006 }
20009 {
20010 /*
20011 * cmpl op0,op1
20012 * sbbl dest,dest
20013 * [addl dest, ct]
20014 *
20015 * Size 5 - 8.
20016 */
20021 }
20023 {
20024 /*
20025 * cmpl op0,op1
20026 * sbbl dest,dest
20027 * orl $ct, dest
20028 *
20029 * Size 8.
20030 */
20034 }
20036 {
20037 /*
20038 * cmpl op0,op1
20039 * sbbl dest,dest
20040 * notl dest
20041 * [addl dest, cf]
20042 *
20043 * Size 8 - 11.
20044 */
20050 }
20051 else
20052 {
20053 /*
20054 * cmpl op0,op1
20055 * sbbl dest,dest
20056 * [notl dest]
20057 * andl cf - ct, dest
20058 * [addl dest, ct]
20059 *
20060 * Size 8 - 11.
20061 */
20064 {
20068 }
20078 }
20084 }
20087 {
20090 HOST_WIDE_INT tmp;
20095 {
20098 /* We may be reversing unordered compare to normal compare, that
20099 is not valid in general (we may convert non-trapping condition
20100 to trapping one), however on i386 we currently emit all
20101 comparisons unordered. */
20104 }
20105 else
20106 {
20109 }
20110 }
20115 {
20120 {
20125 }
20126 }
20128 /* Optimize dest = (op0 < 0) ? -1 : cf. */
20132 {
20133 /* If lea code below could be used, only optimize
20134 if it results in a 2 insn sequence. */
20140 {
20141 /*
20142 * notl op1 (if necessary)
20143 * sarl $31, op1
20144 * orl cf, op1
20145 */
20147 {
20151 }
20162 }
20163 }
20171 {
20172 /*
20173 * xorl dest,dest
20174 * cmpl op1,op2
20175 * setcc dest
20176 * lea cf(dest*(ct-cf)),dest
20177 *
20178 * Size 14.
20179 *
20180 * This also catches the degenerate setcc-only case.
20181 */
20183 rtx tmp;
20189 /* On x86_64 the lea instruction operates on Pmode, so we need
20190 to get arithmetics done in proper mode to match. */
20193 else
20194 {
20195 rtx out1;
20198 nops++;
20200 {
20202 nops++;
20203 }
20204 }
20206 {
20208 nops++;
20209 }
20211 {
20214 else
20216 }
20221 }
20223 /*
20224 * General case: Jumpful:
20225 * xorl dest,dest cmpl op1, op2
20226 * cmpl op1, op2 movl ct, dest
20227 * setcc dest jcc 1f
20228 * decl dest movl cf, dest
20229 * andl (cf-ct),dest 1:
20230 * addl ct,dest
20231 *
20232 * Size 20. Size 14.
20233 *
20234 * This is reasonably steep, but branch mispredict costs are
20235 * high on modern cpus, so consider failing only if optimizing
20236 * for space.
20237 */
20242 {
20244 {
20251 {
20254 /* We may be reversing unordered compare to normal compare,
20255 that is not valid in general (we may convert non-trapping
20256 condition to trapping one), however on i386 we currently
20257 emit all comparisons unordered. */
20259 }
20260 else
20261 {
20265 }
20266 }
20269 {
20270 /* notl op1 (if needed)
20271 sarl $31, op1
20272 andl (cf-ct), op1
20273 addl ct, op1
20275 For x < 0 (resp. x <= -1) there will be no notl,
20276 so if possible swap the constants to get rid of the
20277 complement.
20278 True/false will be -1/0 while code below (store flag
20279 followed by decrement) is 0/-1, so the constants need
20280 to be exchanged once more. */
20283 {
20286 }
20287 else
20288 {
20292 }
20295 }
20296 else
20297 {
20301 constm1_rtx,
20303 }
20315 }
20316 }
20319 {
20320 /* Try a few things more with specific constants and a variable. */
20322 optab op;
20328 /* If one of the two operands is an interesting constant, load a
20329 constant with the above and mask it in with a logical operation. */
20332 {
20338 else
20340 }
20342 {
20348 else
20350 }
20351 else
20358 /* Recurse to get the constant loaded. */
20362 /* Mask in the interesting variable. */
20364 OPTAB_WIDEN);
20369 }
20371 /*
20372 * For comparison with above,
20373 *
20374 * movl cf,dest
20375 * movl ct,tmp
20376 * cmpl op1,op2
20377 * cmovcc tmp,dest
20378 *
20379 * Size 15.
20380 */
20402 }
20404 /* Swap, force into registers, or otherwise massage the two operands
20405 to an sse comparison with a mask result. Thus we differ a bit from
20406 ix86_prepare_fp_compare_args which expects to produce a flags result.
20408 The DEST operand exists to help determine whether to commute commutative
20409 operators. The POP0/POP1 operands are updated in place. The new
20410 comparison code is returned, or UNKNOWN if not implementable. */
20412 static enum rtx_code
20415 {
20416 rtx tmp;
20419 {
20422 /* AVX supports all the needed comparisons. */
20425 /* We have no LTGT as an operator. We could implement it with
20426 NE & ORDERED, but this requires an extra temporary. It's
20427 not clear that it's worth it. */
20434 /* These are supported directly. */
20441 /* AVX has 3 operand comparisons, no need to swap anything. */
20444 /* For commutative operators, try to canonicalize the destination
20445 operand to be first in the comparison - this helps reload to
20446 avoid extra moves. */
20449 /* FALLTHRU */
20455 /* These are not supported directly before AVX, and furthermore
20456 ix86_expand_sse_fp_minmax only optimizes LT/UNGE. Swap the
20457 comparison operands to transform into something that is
20458 supported. */
20467 }
20470 }
20472 /* Detect conditional moves that exactly match min/max operational
20473 semantics. Note that this is IEEE safe, as long as we don't
20474 interchange the operands.
20476 Returns FALSE if this conditional move doesn't match a MIN/MAX,
20477 and TRUE if the operation is successful and instructions are emitted. */
20479 static bool
20482 {
20485 rtx tmp;
20488 ;
20490 {
20494 }
20495 else
20502 else
20507 /* We want to check HONOR_NANS and HONOR_SIGNED_ZEROS here,
20508 but MODE may be a vector mode and thus not appropriate. */
20510 {
20512 rtvec v;
20517 }
20518 else
20519 {
20522 }
20526 }
20528 /* Expand an sse vector comparison. Return the register with the result. */
20530 static rtx
20533 {
20536 rtx x;
20549 {
20552 }
20553 else
20557 }
20559 /* Expand DEST = CMP ? OP_TRUE : OP_FALSE into a sequence of logical
20560 operations. This is used for both scalar and vector conditional moves. */
20562 static void
20564 {
20570 {
20572 }
20574 {
20578 }
20580 {
20585 }
20587 {
20591 }
20593 {
20601 op_true,
20602 op_false)));
20603 }
20604 else
20605 {
20615 {
20629 {
20636 }
20651 {
20658 }
20662 }
20665 {
20669 }
20670 else
20671 {
20677 else
20689 }
20690 }
20691 }
20693 /* Expand a floating-point conditional move. Return true if successful. */
20695 bool
20697 {
20705 {
20708 /* Since we've no cmove for sse registers, don't force bad register
20709 allocation just to gain access to it. Deny movcc when the
20710 comparison mode doesn't match the move mode. */
20729 }
20736 /* The floating point conditional move instructions don't directly
20737 support conditions resulting from a signed integer comparison. */
20741 {
20746 }
20753 }
20755 /* Expand a floating-point vector conditional move; a vcond operation
20756 rather than a movcc operation. */
20758 bool
20760 {
20762 rtx cmp;
20767 {
20768 rtx temp;
20770 {
20787 }
20789 OPTAB_DIRECT);
20792 }
20802 }
20804 /* Expand a signed/unsigned integral vector conditional move. */
20806 bool
20808 {
20818 /* Try to optimize x < 0 ? -1 : 0 into (signed) x >> 31
20819 and x < 0 ? 1 : 0 into (unsigned) x >> 31. */
20828 {
20832 {
20838 }
20841 {
20847 }
20848 }
20857 /* XOP supports all of the comparisons on all 128-bit vector int types. */
20861 ;
20862 else
20863 {
20864 /* Canonicalize the comparison to EQ, GT, GTU. */
20866 {
20883 /* FALLTHRU */
20893 }
20895 /* Only SSE4.1/SSE4.2 supports V2DImode. */
20897 {
20899 {
20901 /* SSE4.1 supports EQ. */
20908 /* SSE4.2 supports GT/GTU. */
20915 }
20916 }
20918 /* Unsigned parallel compare is not supported by the hardware.
20919 Play some tricks to turn this into a signed comparison
20920 against 0. */
20922 {
20926 {
20931 {
20936 {
20943 }
20944 /* Subtract (-(INT MAX) - 1) from both operands to make
20945 them signed. */
20956 }
20963 /* Perform a parallel unsigned saturating subtraction. */
20976 }
20977 }
20978 }
20980 /* Allow the comparison to be done in one mode, but the movcc to
20981 happen in another mode. */
20983 {
20986 }
20987 else
20988 {
20993 }
20998 }
21000 /* Expand a variable vector permutation. */
21002 void
21004 {
21015 /* Number of elements in the vector. */
21021 {
21023 {
21024 /* Unfortunately, the VPERMQ and VPERMPD instructions only support
21025 an constant shuffle operand. With a tiny bit of effort we can
21026 use VPERMD instead. A re-interpretation stall for V4DFmode is
21027 unfortunate but there's no avoiding it.
21028 Similarly for V16HImode we don't have instructions for variable
21029 shuffling, while for V32QImode we can use after preparing suitable
21030 masks vpshufb; vpshufb; vpermq; vpor. */
21033 {
21037 }
21038 else
21039 {
21043 }
21046 /* Replicate the low bits of the V4DImode mask into V8SImode:
21047 mask = { A B C D }
21048 t1 = { A A B B C C D D }. */
21056 else
21059 /* Multiply the shuffle indicies by two. */
21061 OPTAB_DIRECT);
21063 /* Add one to the odd shuffle indicies:
21064 t1 = { A*2, A*2+1, B*2, B*2+1, ... }. */
21066 {
21069 }
21073 OPTAB_DIRECT);
21075 /* Continue as if V8SImode (resp. V32QImode) was used initially. */
21080 }
21083 {
21085 /* The VPERMD and VPERMPS instructions already properly ignore
21086 the high bits of the shuffle elements. No need for us to
21087 perform an AND ourselves. */
21089 {
21094 }
21095 else
21096 {
21102 }
21109 else
21110 {
21116 }
21120 /* By combining the two 128-bit input vectors into one 256-bit
21121 input vector, we can use VPERMD and VPERMPS for the full
21122 two-operand shuffle. */
21154 /* From mask create two adjusted masks, which contain the same
21155 bits as mask in the low 7 bits of each vector element.
21156 The first mask will have the most significant bit clear
21157 if it requests element from the same 128-bit lane
21158 and MSB set if it requests element from the other 128-bit lane.
21159 The second mask will have the opposite values of the MSB,
21160 and additionally will have its 128-bit lanes swapped.
21161 E.g. { 07 12 1e 09 ... | 17 19 05 1f ... } mask vector will have
21162 t1 { 07 92 9e 09 ... | 17 19 85 1f ... } and
21163 t3 { 97 99 05 9f ... | 87 12 1e 89 ... } where each ...
21164 stands for other 12 bytes. */
21165 /* The bit whether element is from the same lane or the other
21166 lane is bit 4, so shift it up by 3 to the MSB position. */
21170 /* Clear MSB bits from the mask just in case it had them set. */
21172 /* After this t1 will have MSB set for elements from other lane. */
21174 /* Clear bits other than MSB. */
21176 /* Or in the lower bits from mask into t3. */
21178 /* And invert MSB bits in t1, so MSB is set for elements from the same
21179 lane. */
21181 /* Swap 128-bit lanes in t3. */
21186 /* And or in the lower bits from mask into t1. */
21189 {
21190 /* Each of these shuffles will put 0s in places where
21191 element from the other 128-bit lane is needed, otherwise
21192 will shuffle in the requested value. */
21196 /* For t3 the 128-bit lanes are swapped again. */
21201 /* And oring both together leads to the result. */
21208 }
21211 /* Similarly to the above one_operand_shuffle code,
21212 just for repeated twice for each operand. merge_two:
21213 code will merge the two results together. */
21237 }
21238 }
21241 {
21242 /* The XOP VPPERM insn supports three inputs. By ignoring the
21243 one_operand_shuffle special case, we avoid creating another
21244 set of constant vectors in memory. */
21247 /* mask = mask & {2*w-1, ...} */
21249 }
21250 else
21251 {
21252 /* mask = mask & {w-1, ...} */
21254 }
21262 /* For non-QImode operations, convert the word permutation control
21263 into a byte permutation control. */
21265 {
21270 /* Convert mask to vector of chars. */
21273 /* Replicate each of the input bytes into byte positions:
21274 (v2di) --> {0,0,0,0,0,0,0,0, 8,8,8,8,8,8,8,8}
21275 (v4si) --> {0,0,0,0, 4,4,4,4, 8,8,8,8, 12,12,12,12}
21276 (v8hi) --> {0,0, 2,2, 4,4, 6,6, ...}. */
21283 else
21286 /* Convert it into the byte positions by doing
21287 mask = mask + {0,1,..,16/w, 0,1,..,16/w, ...} */
21293 }
21295 /* The actual shuffle operations all operate on V16QImode. */
21300 {
21307 }
21309 {
21316 }
21317 else
21318 {
21322 /* Shuffle the two input vectors independently. */
21328 merge_two:
21329 /* Then merge them together. The key is whether any given control
21330 element contained a bit set that indicates the second word. */
21334 {
21335 /* Without SSE4.1, we don't have V2DImode EQ. Perform one
21336 more shuffle to convert the V2DI input mask into a V4SI
21337 input mask. At which point the masking that expand_int_vcond
21338 will work as desired. */
21346 }
21368 }
21369 }
21371 /* Unpack OP[1] into the next wider integer vector type. UNSIGNED_P is
21372 true if we should do zero extension, else sign extension. HIGH_P is
21373 true if we want the N/2 high elements, else the low elements. */
21375 void
21377 {
21379 rtx tmp;
21382 {
21388 {
21392 else
21395 extract
21401 else
21404 extract
21410 else
21413 extract
21419 else
21425 else
21431 else
21436 }
21439 {
21442 }
21444 {
21445 /* Shift higher 8 bytes to lower 8 bytes. */
21450 }
21451 else
21455 }
21456 else
21457 {
21461 {
21465 else
21471 else
21477 else
21482 }
21486 else
21493 }
21494 }
21496 /* Expand conditional increment or decrement using adb/sbb instructions.
21497 The default case using setcc followed by the conditional move can be
21498 done by generic code. */
21499 bool
21501 {
21503 rtx flags;
21505 rtx compare_op;
21523 {
21526 }
21529 {
21533 reverse_condition_maybe_unordered
21535 else
21537 }
21541 /* Construct either adc or sbb insn. */
21543 {
21545 {
21560 }
21561 }
21562 else
21563 {
21565 {
21580 }
21581 }
21585 }
21588 /* Split operands 0 and 1 into half-mode parts. Similar to split_double_mode,
21589 but works for floating pointer parameters and nonoffsetable memories.
21590 For pushes, it returns just stack offsets; the values will be saved
21591 in the right order. Maximally three parts are generated. */
21593 static int
21595 {
21600 else
21606 /* Optimize constant pool reference to immediates. This is used by fp
21607 moves, that force all constants to memory to allow combining. */
21609 {
21613 }
21616 {
21617 /* The only non-offsetable memories we handle are pushes. */
21626 }
21629 {
21631 /* Caution: if we looked through a constant pool memory above,
21632 the operand may actually have a different mode now. That's
21633 ok, since we want to pun this all the way back to an integer. */
21637 }
21640 {
21643 else
21644 {
21648 {
21652 }
21654 {
21659 }
21661 {
21662 REAL_VALUE_TYPE r;
21667 {
21674 /* We can't use REAL_VALUE_TO_TARGET_LONG_DOUBLE since
21675 long double may not be 80-bit. */
21684 }
21687 }
21688 else
21690 }
21691 }
21692 else
21693 {
21697 {
21700 {
21704 }
21706 {
21710 }
21712 {
21713 REAL_VALUE_TYPE r;
21719 /* Do not use shift by 32 to avoid warning on 32bit systems. */
21722 = gen_int_mode
21725 DImode);
21726 else
21733 = gen_int_mode
21736 DImode);
21737 else
21739 }
21740 else
21742 }
21743 }
21746 }
21748 /* Emit insns to perform a move or push of DI, DF, XF, and TF values.
21749 Return false when normal moves are needed; true when all required
21750 insns have been emitted. Operands 2-4 contain the input values
21751 int the correct order; operands 5-7 contain the output values. */
21753 void
21755 {
21763 /* The DFmode expanders may ask us to move double.
21764 For 64bit target this is single move. By hiding the fact
21765 here we simplify i386.md splitters. */
21767 {
21768 /* Optimize constant pool reference to immediates. This is used by
21769 fp moves, that force all constants to memory to allow combining. */
21776 {
21779 }
21780 else
21785 }
21787 /* The only non-offsettable memory we handle is push. */
21790 else
21797 /* When emitting push, take care for source operands on the stack. */
21800 {
21803 /* Compensate for the stack decrement by 4. */
21808 /* src_base refers to the stack pointer and is
21809 automatically decreased by emitted push. */
21813 }
21815 /* We need to do copy in the right order in case an address register
21816 of the source overlaps the destination. */
21818 {
21819 rtx tmp;
21822 {
21826 collisions++;
21827 }
21829 /* Collision in the middle part can be handled by reordering. */
21831 {
21834 }
21838 {
21840 {
21843 }
21844 else
21845 {
21848 }
21849 }
21851 /* If there are more collisions, we can't handle it by reordering.
21852 Do an lea to the last part and use only one colliding move. */
21854 {
21855 rtx base;
21861 /* Handle the case when the last part isn't valid for lea.
21862 Happens in 64-bit mode storing the 12-byte XFmode. */
21869 {
21872 }
21873 }
21874 }
21877 {
21879 {
21881 {
21886 }
21888 {
21891 }
21892 }
21893 else
21894 {
21895 /* In 64bit mode we don't have 32bit push available. In case this is
21896 register, it is OK - we will just use larger counterpart. We also
21897 retype memory - these comes from attempt to avoid REX prefix on
21898 moving of second half of TFmode value. */
21900 {
21902 {
21913 }
21917 }
21918 }
21922 }
21924 /* Choose correct order to not overwrite the source before it is copied. */
21934 {
21936 {
21939 }
21940 }
21941 else
21942 {
21944 {
21947 }
21948 }
21950 /* If optimizing for size, attempt to locally unCSE nonzero constants. */
21952 {
21961 }
21967 }
21969 /* Helper function of ix86_split_ashl used to generate an SImode/DImode
21970 left shift by a constant, either using a single shift or
21971 a sequence of add instructions. */
21973 static void
21975 {
21981 {
21985 }
21986 else
21987 {
21990 }
21991 }
21993 void
21995 {
22004 {
22009 {
22015 }
22016 else
22017 {
22025 }
22027 }
22034 {
22035 /* Assuming we've chosen a QImode capable registers, then 1 << N
22036 can be done with two 32/64-bit shifts, no branches, no cmoves. */
22038 {
22054 }
22056 /* Otherwise, we can get the same results by manually performing
22057 a bit extract operation on bit 5/6, and then performing the two
22058 shifts. The two methods of getting 0/1 into low/high are exactly
22059 the same size. Avoiding the shift in the bit extract case helps
22060 pentium4 a bit; no one else seems to care much either way. */
22061 else
22062 {
22067 HOST_WIDE_INT bits;
22068 rtx x;
22071 {
22077 }
22078 else
22079 {
22085 }
22089 else
22097 }
22102 }
22105 {
22106 /* For -1 << N, we can avoid the shld instruction, because we
22107 know that we're shifting 0...31/63 ones into a -1. */
22111 else
22113 }
22114 else
22115 {
22123 }
22128 {
22134 }
22135 else
22136 {
22141 }
22142 }
22144 void
22146 {
22156 {
22161 {
22167 }
22169 {
22178 }
22179 else
22180 {
22188 }
22189 }
22190 else
22191 {
22203 {
22211 scratch));
22212 }
22213 else
22214 {
22219 }
22220 }
22221 }
22223 void
22225 {
22235 {
22240 {
22247 }
22248 else
22249 {
22257 }
22258 }
22259 else
22260 {
22272 {
22278 scratch));
22279 }
22280 else
22281 {
22286 }
22287 }
22288 }
22290 /* Predict just emitted jump instruction to be taken with probability PROB. */
22291 static void
22293 {
22297 }
22299 /* Helper function for the string operations below. Dest VARIABLE whether
22300 it is aligned to VALUE bytes. If true, jump to the label. */
22301 static rtx
22303 {
22308 else
22314 else
22317 }
22319 /* Adjust COUNTER by the VALUE. */
22320 static void
22322 {
22327 }
22329 /* Zero extend possibly SImode EXP to Pmode register. */
22330 rtx
22332 {
22334 }
22336 /* Divide COUNTREG by SCALE. */
22337 static rtx
22339 {
22340 rtx sc;
22352 }
22354 /* Return mode for the memcpy/memset loop counter. Prefer SImode over
22355 DImode for constant loop counts. */
22357 static enum machine_mode
22359 {
22367 }
22369 /* Copy the address to a Pmode register. This is used for x32 to
22370 truncate DImode TLS address to a SImode register. */
22372 static rtx
22374 {
22377 else
22378 {
22381 }
22382 }
22384 /* When ISSETMEM is FALSE, output simple loop to move memory pointer to SRCPTR
22385 to DESTPTR via chunks of MODE unrolled UNROLL times, overall size is COUNT
22386 specified in bytes. When ISSETMEM is TRUE, output the equivalent loop to set
22387 memory by VALUE (supposed to be in MODE).
22389 The size is rounded down to whole number of chunk size moved at once.
22390 SRCMEM and DESTMEM provide MEMrtx to feed proper aliasing info. */
22393 static void
22398 {
22404 rtx size;
22413 /* Those two should combine. */
22415 {
22419 }
22426 /* This assert could be relaxed - in this case we'll need to compute
22427 smallest power of two, containing in PIECE_SIZE_N and pass it to
22428 offset_address. */
22434 {
22438 /* When unrolling for chips that reorder memory reads and writes,
22439 we can save registers by using single temporary.
22440 Also using 4 temporaries is overkill in 32bit mode. */
22442 {
22444 {
22446 {
22447 destmem =
22449 srcmem =
22451 }
22453 }
22454 }
22455 else
22456 {
22460 {
22463 {
22464 srcmem =
22466 }
22468 }
22470 {
22472 {
22473 destmem =
22475 }
22477 }
22478 }
22479 }
22480 else
22482 {
22484 destmem =
22487 }
22497 {
22503 else
22505 }
22506 else
22514 {
22519 }
22521 }
22523 /* Output "rep; mov" or "rep; stos" instruction depending on ISSETMEM argument.
22524 When ISSETMEM is true, arguments SRCMEM and SRCPTR are ignored.
22525 When ISSETMEM is false, arguments VALUE and ORIG_VALUE are ignored.
22526 For setmem case, VALUE is a promoted to a wider size ORIG_VALUE.
22527 ORIG_VALUE is the original value passed to memset to fill the memory with.
22528 Other arguments have same meaning as for previous function. */
22530 static void
22533 rtx count,
22535 {
22536 rtx destexp;
22537 rtx srcexp;
22538 rtx countreg;
22539 HOST_WIDE_INT rounded_count;
22541 /* If possible, it is shorter to use rep movs.
22542 TODO: Maybe it is better to move this logic to decide_alg. */
22553 {
22557 }
22558 else
22561 {
22566 }
22571 {
22574 }
22575 else
22576 {
22580 {
22584 }
22585 else
22588 {
22593 }
22594 else
22595 {
22598 }
22601 }
22602 }
22604 /* This function emits moves to copy SIZE_TO_MOVE bytes from SRCMEM to
22605 DESTMEM.
22606 SRC is passed by pointer to be updated on return.
22607 Return value is updated DST. */
22608 static rtx
22610 HOST_WIDE_INT size_to_move)
22611 {
22617 /* Find the widest mode in which we could perform moves.
22618 Start with the biggest power of 2 less than SIZE_TO_MOVE and half
22619 it until move of such size is supported. */
22624 {
22628 }
22630 /* Find the corresponding vector mode with the same size as MOVE_MODE.
22631 MOVE_MODE is an integer mode at the moment (SI, DI, TI, etc.). */
22633 {
22638 {
22642 }
22643 }
22649 /* Emit moves. We'll need SIZE_TO_MOVE/PIECE_SIZES moves. */
22653 {
22654 /* We move from memory to memory, so we'll need to do it via
22655 a temporary register. */
22666 piece_size);
22668 piece_size);
22669 }
22671 /* Update DST and SRC rtx. */
22674 }
22676 /* Output code to copy at most count & (max_size - 1) bytes from SRC to DEST. */
22677 static void
22680 {
22683 {
22688 /* For now MAX_SIZE should be a power of 2. This assert could be
22689 relaxed, but it'll require a bit more complicated epilogue
22690 expanding. */
22693 {
22696 }
22698 }
22700 {
22706 }
22708 /* When there are stringops, we can cheaply increase dest and src pointers.
22709 Otherwise we save code size by maintaining offset (zero is readily
22710 available from preceding rep operation) and using x86 addressing modes.
22711 */
22713 {
22715 {
22722 }
22724 {
22731 }
22733 {
22740 }
22741 }
22742 else
22743 {
22745 rtx tmp;
22748 {
22759 }
22761 {
22774 }
22776 {
22785 }
22786 }
22787 }
22789 /* This function emits moves to fill SIZE_TO_MOVE bytes starting from DESTMEM
22790 with value PROMOTED_VAL.
22791 SRC is passed by pointer to be updated on return.
22792 Return value is updated DST. */
22793 static rtx
22795 HOST_WIDE_INT size_to_move)
22796 {
22802 /* Find the widest mode in which we could perform moves.
22803 Start with the biggest power of 2 less than SIZE_TO_MOVE and half
22804 it until move of such size is supported. */
22809 {
22812 }
22819 /* Emit moves. We'll need SIZE_TO_MOVE/PIECE_SIZES moves. */
22823 {
22825 {
22828 }
22836 piece_size);
22837 }
22839 /* Update DST rtx. */
22841 }
22842 /* Output code to set at most count & (max_size - 1) bytes starting by DEST. */
22843 static void
22846 {
22847 count =
22853 }
22855 /* Output code to set at most count & (max_size - 1) bytes starting by DEST. */
22856 static void
22859 {
22860 rtx dest;
22863 {
22868 /* For now MAX_SIZE should be a power of 2. This assert could be
22869 relaxed, but it'll require a bit more complicated epilogue
22870 expanding. */
22873 {
22875 {
22878 else
22880 }
22881 }
22883 }
22885 {
22888 }
22890 {
22893 {
22897 }
22898 else
22899 {
22905 }
22908 }
22910 {
22913 {
22916 }
22917 else
22918 {
22922 }
22925 }
22927 {
22933 }
22935 {
22941 }
22943 {
22949 }
22950 }
22952 /* Depending on ISSETMEM, copy enough from SRCMEM to DESTMEM or set enough to
22953 DESTMEM to align it to DESIRED_ALIGNMENT. Original alignment is ALIGN.
22954 Depending on ISSETMEM, either arguments SRCMEM/SRCPTR or VALUE/VEC_VALUE are
22955 ignored.
22956 Return value is updated DESTMEM. */
22957 static rtx
22962 {
22965 {
22967 {
22970 {
22973 else
22975 }
22976 else
22982 }
22983 }
22985 }
22987 /* Test if COUNT&SIZE is nonzero and if so, expand movme
22988 or setmem sequence that is valid for SIZE..2*SIZE-1 bytes
22989 and jump to DONE_LABEL. */
22990 static void
22996 {
22999 rtx modesize;
23002 /* If we do not have vector value to copy, we must reduce size. */
23004 {
23006 {
23011 }
23012 else
23014 }
23015 else
23016 {
23017 /* Choose appropriate vector mode. */
23023 }
23028 {
23031 else
23032 {
23035 }
23037 }
23044 else
23045 {
23050 }
23056 }
23058 /* Handle small memcpy (up to SIZE that is supposed to be small power of 2.
23059 and get ready for the main memcpy loop by copying iniital DESIRED_ALIGN-ALIGN
23060 bytes and last SIZE bytes adjusitng DESTPTR/SRCPTR/COUNT in a way we can
23061 proceed with an loop copying SIZE bytes at once. Do moves in MODE.
23062 DONE_LABEL is a label after the whole copying sequence. The label is created
23063 on demand if *DONE_LABEL is NULL.
23064 MIN_SIZE is minimal size of block copied. This value gets adjusted for new
23065 bounds after the initial copies.
23067 DESTMEM/SRCMEM are memory expressions pointing to the copies block,
23068 DESTPTR/SRCPTR are pointers to the block. DYNAMIC_CHECK indicate whether
23069 we will dispatch to a library call for large blocks.
23071 In pseudocode we do:
23073 if (COUNT < SIZE)
23074 {
23075 Assume that SIZE is 4. Bigger sizes are handled analogously
23076 if (COUNT & 4)
23077 {
23078 copy 4 bytes from SRCPTR to DESTPTR
23079 copy 4 bytes from SRCPTR + COUNT - 4 to DESTPTR + COUNT - 4
23080 goto done_label
23081 }
23082 if (!COUNT)
23083 goto done_label;
23084 copy 1 byte from SRCPTR to DESTPTR
23085 if (COUNT & 2)
23086 {
23087 copy 2 bytes from SRCPTR to DESTPTR
23088 copy 2 bytes from SRCPTR + COUNT - 2 to DESTPTR + COUNT - 2
23089 }
23090 }
23091 else
23092 {
23093 copy at least DESIRED_ALIGN-ALIGN bytes from SRCPTR to DESTPTR
23094 copy SIZE bytes from SRCPTR + COUNT - SIZE to DESTPTR + COUNT -SIZE
23096 OLD_DESPTR = DESTPTR;
23097 Align DESTPTR up to DESIRED_ALIGN
23098 SRCPTR += DESTPTR - OLD_DESTPTR
23099 COUNT -= DEST_PTR - OLD_DESTPTR
23100 if (DYNAMIC_CHECK)
23101 Round COUNT down to multiple of SIZE
23102 << optional caller supplied zero size guard is here >>
23103 << optional caller suppplied dynamic check is here >>
23104 << caller supplied main copy loop is here >>
23105 }
23106 done_label:
23107 */
23108 static void
23121 {
23124 rtx modesize;
23126 rtx mode_value;
23128 /* Chose proper value to copy. */
23131 else
23135 /* See if block is big or small, handle small blocks. */
23137 {
23148 /* Handle sizes > 3. */
23155 /* Nothing to copy? Jump to DONE_LABEL if so */
23159 /* Do a byte copy. */
23163 else
23164 {
23167 }
23169 /* Handle sizes 2 and 3. */
23176 else
23177 {
23182 }
23188 }
23189 else
23193 /* Start memcpy for COUNT >= SIZE. */
23195 {
23198 }
23200 /* Copy first desired_align bytes. */
23206 {
23209 else
23210 {
23213 }
23216 }
23219 /* Copy last SIZE bytes. */
23226 else
23227 {
23233 }
23235 {
23239 else
23240 {
23243 }
23244 }
23246 /* Align destination. */
23248 {
23252 /* Align destptr up, place it to new register. */
23259 /* See how many bytes we skipped. */
23263 /* Adjust srcptr and count. */
23269 /* We copied at most size + prolog_size. */
23272 else
23275 /* Our loops always round down the bock size, but for dispatch to library
23276 we need precise value. */
23280 }
23281 else
23282 {
23284 /* Decrease count, so we won't end up copying last word twice. */
23288 else
23292 }
23293 }
23296 /* This function is like the previous one, except here we know how many bytes
23297 need to be copied. That allows us to update alignment not only of DST, which
23298 is returned, but also of SRC, which is passed as a pointer for that
23299 reason. */
23300 static rtx
23305 {
23313 {
23317 }
23322 {
23324 {
23326 {
23329 else
23331 }
23332 else
23335 }
23336 }
23343 {
23349 {
23352 {
23356 }
23361 }
23365 }
23368 }
23370 /* Return true if ALG can be used in current context.
23371 Assume we expand memset if MEMSET is true. */
23372 static bool
23374 {
23379 /* Algorithms using the rep prefix want at least edi and ecx;
23380 additionally, memset wants eax and memcpy wants esi. Don't
23381 consider such algorithms if the user has appropriated those
23382 registers for their own purposes. */
23389 }
23391 /* Given COUNT and EXPECTED_SIZE, decide on codegen of string operation. */
23392 static enum stringop_alg
23396 {
23407 /* Even if the string operation call is cold, we still might spend a lot
23408 of time processing large blocks. */
23414 else
23420 else
23423 /* See maximal size for user defined algorithm. */
23425 {
23432 }
23434 /* If expected size is not known but max size is small enough
23435 so inline version is a win, set expected size into
23436 the range. */
23440 /* If user specified the algorithm, honnor it if possible. */
23444 /* rep; movq or rep; movl is the smallest variant. */
23446 {
23451 else
23454 }
23455 /* Very tiny blocks are best handled via the loop, REP is expensive to
23456 setup. */
23460 {
23464 {
23465 /* We get here if the algorithms that were not libcall-based
23466 were rep-prefix based and we are unable to use rep prefixes
23467 based on global register usage. Break out of the loop and
23468 use the heuristic below. */
23472 {
23476 {
23479 }
23480 /* Honor TARGET_INLINE_ALL_STRINGOPS by picking
23481 last non-libcall inline algorithm. */
23483 {
23484 /* When the current size is best to be copied by a libcall,
23485 but we are still forced to inline, run the heuristic below
23486 that will pick code for medium sized blocks. */
23488 {
23491 }
23493 }
23495 {
23498 }
23499 }
23500 }
23501 }
23502 /* When asked to inline the call anyway, try to pick meaningful choice.
23503 We look for maximal size of block that is faster to copy by hand and
23504 take blocks of at most of that size guessing that average size will
23505 be roughly half of the block.
23507 If this turns out to be bad, we might simply specify the preferred
23508 choice in ix86_costs. */
23512 {
23515 /* If there aren't any usable algorithms, then recursing on
23516 smaller sizes isn't going to find anything. Just return the
23517 simple byte-at-a-time copy loop. */
23519 {
23520 /* Pick something reasonable. */
23524 }
23534 }
23537 }
23539 /* Decide on alignment. We know that the operand is already aligned to ALIGN
23540 (ALIGN can be based on profile feedback and thus it is not 100% guaranteed). */
23541 static int
23546 {
23557 /* PentiumPro has special logic triggering for 8 byte aligned blocks.
23558 copying whole cacheline at once. */
23571 }
23574 /* Helper function for memcpy. For QImode value 0xXY produce
23575 0xXYXYXYXY of wide specified by MODE. This is essentially
23576 a * 0x10101010, but we can do slightly better than
23577 synth_mult by unwinding the sequence by hand on CPUs with
23578 slow multiply. */
23579 static rtx
23581 {
23583 rtx tmp;
23590 {
23598 }
23607 nops--;
23612 {
23616 OPTAB_DIRECT);
23617 }
23618 else
23619 {
23625 else
23627 else
23628 {
23631 reg =
23633 }
23643 }
23644 }
23646 /* Duplicate value VAL using promote_duplicated_reg into maximal size that will
23647 be needed by main loop copying SIZE_NEEDED chunks and prologue getting
23648 alignment from ALIGN to DESIRED_ALIGN. */
23649 static rtx
23652 {
23653 rtx promoted_val;
23662 else
23666 }
23668 /* Expand string move (memcpy) ot store (memset) operation. Use i386 string
23669 operations when profitable. The code depends upon architecture, block size
23670 and alignment, but always has one of the following overall structures:
23672 Aligned move sequence:
23674 1) Prologue guard: Conditional that jumps up to epilogues for small
23675 blocks that can be handled by epilogue alone. This is faster
23676 but also needed for correctness, since prologue assume the block
23677 is larger than the desired alignment.
23679 Optional dynamic check for size and libcall for large
23680 blocks is emitted here too, with -minline-stringops-dynamically.
23682 2) Prologue: copy first few bytes in order to get destination
23683 aligned to DESIRED_ALIGN. It is emitted only when ALIGN is less
23684 than DESIRED_ALIGN and up to DESIRED_ALIGN - ALIGN bytes can be
23685 copied. We emit either a jump tree on power of two sized
23686 blocks, or a byte loop.
23688 3) Main body: the copying loop itself, copying in SIZE_NEEDED chunks
23689 with specified algorithm.
23691 4) Epilogue: code copying tail of the block that is too small to be
23692 handled by main body (or up to size guarded by prologue guard).
23694 Misaligned move sequence
23696 1) missaligned move prologue/epilogue containing:
23697 a) Prologue handling small memory blocks and jumping to done_label
23698 (skipped if blocks are known to be large enough)
23699 b) Signle move copying first DESIRED_ALIGN-ALIGN bytes if alignment is
23700 needed by single possibly misaligned move
23701 (skipped if alignment is not needed)
23702 c) Copy of last SIZE_NEEDED bytes by possibly misaligned moves
23704 2) Zero size guard dispatching to done_label, if needed
23706 3) dispatch to library call, if needed,
23708 3) Main body: the copying loop itself, copying in SIZE_NEEDED chunks
23709 with specified algorithm. */
23710 bool
23715 {
23716 rtx destreg;
23719 rtx tmp;
23735 /* TODO: Once vlaue ranges are available, fill in proper data. */
23742 /* i386 can do misaligned access on reasonably increased cost. */
23746 /* ALIGN is the minimum of destination and source alignment, but we care here
23747 just about destination alignment. */
23761 /* Make sure we don't need to care about overflow later on. */
23765 /* Step 0: Decide on preferred algorithm, desired alignment and
23766 size of chunks to be copied by main loop. */
23774 /* For now vector-version of memset is generated only for memory zeroing, as
23775 creating of promoted vector value is very cheap in this case. */
23788 {
23807 /* Find the widest supported mode. */
23813 /* Find the corresponding vector mode with the same size as MOVE_MODE.
23814 MOVE_MODE is an integer mode at the moment (SI, DI, TI, etc.). */
23816 {
23821 }
23833 }
23841 /* Step 1: Prologue guard. */
23843 /* Alignment code needs count to be in register. */
23845 {
23848 {
23849 align_bytes
23853 else
23855 }
23858 }
23861 /* Misaligned move sequences handle both prologue and epilogue at once.
23862 Default code generation results in a smaller code for large alignments
23863 and also avoids redundant job when sizes are known precisely. */
23864 misaligned_prologue_used
23865 = (TARGET_MISALIGNED_MOVE_STRING_PRO_EPILOGUES
23871 /* Do the cheap promotion to allow better CSE across the
23872 main loop and epilogue (ie one load of the big constant in the
23873 front of all code.
23874 For now the misaligned move sequences do not have fast path
23875 without broadcasting. */
23877 {
23879 {
23885 }
23886 else
23887 {
23890 }
23891 }
23892 /* Misaligned move sequences handles both prologues and epilogues at once.
23893 Default code generation results in smaller code for large alignments and
23894 also avoids redundant job when sizes are known precisely. */
23896 {
23897 /* Misaligned move prologue handled small blocks by itself. */
23898 expand_set_or_movmem_prologue_epilogue_by_misaligned_moves
23903 desired_align < align
23912 {
23913 /* It is possible that we copied enough so the main loop will not
23914 execute. */
23924 else
23926 }
23927 }
23928 /* Ensure that alignment prologue won't copy past end of block. */
23930 {
23932 /* Epilogue always copies COUNT_EXP & EPILOGUE_SIZE_NEEDED bytes.
23933 Make sure it is power of 2. */
23936 /* To improve performance of small blocks, we jump around the VAL
23937 promoting mode. This mean that if the promoted VAL is not constant,
23938 we might not use it in the epilogue and have to use byte
23939 loop variant. */
23943 {
23945 {
23946 /* If main algorithm works on QImode, no epilogue is needed.
23947 For small sizes just don't align anything. */
23950 else
23952 }
23953 }
23955 {
23963 else
23965 }
23966 }
23968 /* Emit code to decide on runtime whether library call or inline should be
23969 used. */
23971 {
23973 {
23975 {
23979 }
23980 }
23981 else
23982 {
23990 else
23994 }
23995 }
23997 /* Step 2: Alignment prologue. */
23998 /* Do the expensive promotion once we branched off the small blocks. */
24004 {
24006 {
24007 /* Except for the first move in prologue, we no longer know
24008 constant offset in aliasing info. It don't seems to worth
24009 the pain to maintain it for the first move, so throw away
24010 the info early. */
24017 issetmem);
24018 /* At most desired_align - align bytes are copied. */
24021 else
24023 }
24024 else
24025 {
24026 /* If we know how many bytes need to be stored before dst is
24027 sufficiently aligned, maintain aliasing info accurately. */
24029 srcreg,
24030 promoted_val,
24031 vec_promoted_val,
24032 desired_align,
24033 align_bytes,
24034 issetmem);
24041 }
24043 && !min_size
24048 {
24049 /* It is possible that we copied enough so the main loop will not
24050 execute. */
24060 else
24062 }
24063 }
24065 {
24072 }
24076 /* Step 3: Main loop. */
24079 {
24102 }
24103 /* Adjust properly the offset of src and dest memory for aliasing. */
24105 {
24111 }
24112 else
24113 {
24117 }
24119 /* Step 4: Epilogue to copy the remaining bytes. */
24120 epilogue:
24122 {
24123 /* When the main loop is done, COUNT_EXP might hold original count,
24124 while we want to copy only COUNT_EXP & SIZE_NEEDED bytes.
24125 Epilogue code will actually copy COUNT_EXP & EPILOGUE_SIZE_NEEDED
24126 bytes. Compensate if needed. */
24129 {
24130 tmp =
24133 OPTAB_DIRECT);
24136 }
24139 }
24142 {
24145 epilogue_size_needed);
24146 else
24147 {
24151 epilogue_size_needed);
24152 else
24154 epilogue_size_needed);
24155 }
24156 }
24160 }
24163 /* Expand the appropriate insns for doing strlen if not just doing
24164 repnz; scasb
24166 out = result, initialized with the start address
24167 align_rtx = alignment of the address.
24168 scratch = scratch register, initialized with the startaddress when
24169 not aligned, otherwise undefined
24171 This is just the body. It needs the initializations mentioned above and
24172 some address computing at the end. These things are done in i386.md. */
24174 static void
24176 {
24178 rtx tmp;
24183 rtx mem;
24186 rtx cmp;
24192 /* Loop to check 1..3 bytes for null to get an aligned pointer. */
24194 /* Is there a known alignment and is it less than 4? */
24196 {
24199 /* Is there a known alignment and is it not 2? */
24201 {
24205 /* Leave just the 3 lower bits. */
24215 }
24216 else
24217 {
24218 /* Since the alignment is 2, we have to check 2 or 0 bytes;
24219 check if is aligned to 4 - byte. */
24226 }
24230 /* Now compare the bytes. */
24232 /* Compare the first n unaligned byte on a byte per byte basis. */
24236 /* Increment the address. */
24239 /* Not needed with an alignment of 2 */
24241 {
24245 end_0_label);
24250 }
24253 end_0_label);
24256 }
24258 /* Generate loop to check 4 bytes at a time. It is not a good idea to
24259 align this loop. It gives only huge programs, but does not help to
24260 speed up. */
24267 /* This formula yields a nonzero result iff one of the bytes is zero.
24268 This saves three branches inside loop and many cycles. */
24276 align_4_label);
24279 {
24285 /* If zero is not in the first two bytes, move two bytes forward. */
24291 reg,
24292 tmpreg)));
24293 /* Emit lea manually to avoid clobbering of flags. */
24301 reg2,
24302 out)));
24303 }
24304 else
24305 {
24307 /* Is zero in the first two bytes? */
24314 pc_rtx);
24318 /* Not in the first two. Move two bytes forward. */
24324 }
24326 /* Avoid branch in fixing the byte. */
24334 }
24336 /* Expand strlen. */
24338 bool
24340 {
24343 /* The generic case of strlen expander is long. Avoid it's
24344 expanding unless TARGET_INLINE_ALL_STRINGOPS. */
24347 && !TARGET_INLINE_ALL_STRINGOPS
24357 {
24358 /* Well it seems that some optimizer does not combine a call like
24359 foo(strlen(bar), strlen(bar));
24360 when the move and the subtraction is done here. It does calculate
24361 the length just once when these instructions are done inside of
24362 output_strlen_unroll(). But I think since &bar[strlen(bar)] is
24363 often used and I use one fewer register for the lifetime of
24364 output_strlen_unroll() this is better. */
24370 /* strlensi_unroll_1 returns the address of the zero at the end of
24371 the string, like memchr(), so compute the length by subtracting
24372 the start address. */
24374 }
24375 else
24376 {
24377 rtx unspec;
24379 /* Can't use this if the user has appropriated eax, ecx, or edi. */
24392 /* If .md starts supporting :P, this can be done in .md. */
24398 }
24400 }
24402 /* For given symbol (function) construct code to compute address of it's PLT
24403 entry in large x86-64 PIC model. */
24404 static rtx
24406 {
24419 }
24421 rtx
24423 rtx callarg2,
24425 {
24426 unsigned int const cregs_size
24437 {
24438 #if TARGET_MACHO
24441 #endif
24442 }
24443 else
24444 {
24445 /* Static functions and indirect calls don't need the pic register. */
24447 && (!TARGET_64BIT
24453 }
24456 {
24460 }
24463 && !TARGET_PECOFF
24471 {
24474 }
24482 {
24486 }
24490 {
24494 UNSPEC_MS_TO_SYSV_CALL);
24497 {
24503 }
24504 }
24513 }
24515 /* Output the assembly for a call instruction. */
24519 {
24525 {
24528 /* SEH epilogue detection requires the indirect branch case
24529 to include REX.W. */
24532 else
24537 }
24539 /* SEH unwinding can require an extra nop to be emitted in several
24540 circumstances. Determine if we have one of those. */
24542 {
24543 rtx i;
24546 {
24547 /* If we get to another real insn, we don't need the nop. */
24551 /* If we get to the epilogue note, prevent a catch region from
24552 being adjacent to the standard epilogue sequence. If non-
24553 call-exceptions, we'll have done this during epilogue emission. */
24555 && !flag_non_call_exceptions
24557 {
24560 }
24561 }
24563 /* If we didn't find a real insn following the call, prevent the
24564 unwinder from looking into the next function. */
24567 }
24571 else
24580 }
24581 \f
24582 /* Clear stack slot assignments remembered from previous functions.
24583 This is called from INIT_EXPANDERS once before RTL is emitted for each
24584 function. */
24588 {
24596 }
24598 /* Return a MEM corresponding to a stack slot with mode MODE.
24599 Allocate a new slot if necessary.
24601 The RTL for a function can have several slots available: N is
24602 which slot to use. */
24604 rtx
24606 {
24623 }
24625 static void
24627 {
24633 }
24634 \f
24635 /* Check whether x86 address PARTS is a pc-relative address. */
24637 static bool
24639 {
24647 {
24649 {
24666 }
24667 }
24669 }
24671 /* Calculate the length of the memory address in the instruction encoding.
24672 Includes addr32 prefix, does not include the one-byte modrm, opcode,
24673 or other prefixes. We never generate addr32 prefix for LEA insn. */
24675 int
24677 {
24694 /* If this is not LEA instruction, add the length of addr32 prefix. */
24699 len++;
24713 /* Rule of thumb:
24714 - esp as the base always wants an index,
24715 - ebp as the base always wants a displacement,
24716 - r12 as the base always wants an index,
24717 - r13 as the base always wants a displacement. */
24719 /* Register Indirect. */
24721 {
24722 /* esp (for its index) and ebp (for its displacement) need
24723 the two-byte modrm form. Similarly for r12 and r13 in 64-bit
24724 code. */
24731 len++;
24732 }
24734 /* Direct Addressing. In 64-bit mode mod 00 r/m 5
24735 is not disp32, but disp32(%rip), so for disp32
24736 SIB byte is needed, unless print_operand_address
24737 optimizes it into disp32(%rip) or (%rip) is implied
24738 by UNSPEC. */
24740 {
24743 len++;
24744 }
24745 else
24746 {
24747 /* Find the length of the displacement constant. */
24749 {
24752 else
24754 }
24755 /* ebp always wants a displacement. Similarly r13. */
24757 len++;
24759 /* An index requires the two-byte modrm form.... */
24761 /* ...like esp (or r12), which always wants an index. */
24765 len++;
24766 }
24769 }
24771 /* Compute default value for "length_immediate" attribute. When SHORTFORM
24772 is set, expect that insn have 8bit immediate alternative. */
24773 int
24775 {
24781 {
24786 {
24789 {
24801 }
24803 {
24806 }
24807 }
24809 {
24819 /* Immediates for DImode instructions are encoded
24820 as 32bit sign extended values. */
24826 }
24827 }
24829 }
24831 /* Compute default value for "length_address" attribute. */
24832 int
24834 {
24838 {
24849 }
24854 {
24857 {
24862 constraints++;
24865 ;
24866 /* Skip ignored operands. */
24869 }
24871 }
24873 }
24875 /* Compute default value for "length_vex" attribute. It includes
24876 2 or 3 byte VEX prefix and 1 opcode byte. */
24878 int
24880 {
24883 /* Only 0f opcode can use 2 byte VEX prefix and VEX W bit uses 3
24884 byte VEX prefix. */
24888 /* We can always use 2 byte VEX prefix in 32bit. */
24896 {
24897 /* REX.W bit uses 3 byte VEX prefix. */
24901 }
24902 else
24903 {
24904 /* REX.X or REX.B bits use 3 byte VEX prefix. */
24908 }
24911 }
24912 \f
24913 /* Return the maximum number of instructions a cpu can issue. */
24915 static int
24917 {
24919 {
24949 }
24950 }
24952 /* A subroutine of ix86_adjust_cost -- return TRUE iff INSN reads flags set
24953 by DEP_INSN and nothing set by DEP_INSN. */
24955 static bool
24957 {
24960 /* Simplify the test for uninteresting insns. */
24968 {
24971 }
24976 {
24979 }
24980 else
24986 /* This test is true if the dependent insn reads the flags but
24987 not any other potentially set register. */
24995 }
24997 /* Return true iff USE_INSN has a memory address with operands set by
24998 SET_INSN. */
25000 bool
25002 {
25007 {
25010 }
25012 }
25014 /* Helper function for exact_store_load_dependency.
25015 Return true if addr is found in insn. */
25016 static bool
25018 {
25025 {
25031 CASE_CONST_ANY:
25040 }
25044 {
25046 {
25056 }
25057 }
25059 }
25061 /* Return true if there exists exact dependency for store & load, i.e.
25062 the same memory address is used in them. */
25063 static bool
25065 {
25079 }
25081 static int
25083 {
25089 /* Anti and output dependencies have zero cost on all CPUs. */
25095 /* If we can't recognize the insns, we can't really do anything. */
25103 {
25105 /* Address Generation Interlock adds a cycle of latency. */
25107 {
25118 }
25122 /* ??? Compares pair with jump/setcc. */
25126 /* Floating point stores require value to be ready one cycle earlier. */
25136 /* INT->FP conversion is expensive. */
25140 /* There is one cycle extra latency between an FP op and a store. */
25148 /* Show ability of reorder buffer to hide latency of load by executing
25149 in parallel with previous instruction in case
25150 previous instruction is not needed to compute the address. */
25153 {
25154 /* Claim moves to take one cycle, as core can issue one load
25155 at time and the next load can start cycle later. */
25160 cost--;
25161 }
25167 /* The esp dependency is resolved before the instruction is really
25168 finished. */
25173 /* INT->FP conversion is expensive. */
25177 /* Show ability of reorder buffer to hide latency of load by executing
25178 in parallel with previous instruction in case
25179 previous instruction is not needed to compute the address. */
25182 {
25183 /* Claim moves to take one cycle, as core can issue one load
25184 at time and the next load can start cycle later. */
25190 else
25192 }
25207 /* Stack engine allows to execute push&pop instructions in parall. */
25213 /* Show ability of reorder buffer to hide latency of load by executing
25214 in parallel with previous instruction in case
25215 previous instruction is not needed to compute the address. */
25218 {
25222 /* Because of the difference between the length of integer and
25223 floating unit pipeline preparation stages, the memory operands
25224 for floating point are cheaper.
25226 ??? For Athlon it the difference is most probably 2. */
25229 else
25234 else
25236 }
25245 /* Stack engine allows to execute push&pop instructions in parall. */
25250 /* Show ability of reorder buffer to hide latency of load by executing
25251 in parallel with previous instruction in case
25252 previous instruction is not needed to compute the address. */
25255 {
25258 else
25260 }
25267 /* Increase cost of integer loads. */
25270 {
25273 {
25276 else
25277 {
25278 /* Increase cost of ld/st for short int types only
25279 because of store forwarding issue. */
25283 {
25284 /* Increase cost of store/load insn if exact
25285 dependence exists and it is load insn. */
25290 }
25291 }
25292 }
25293 }
25297 }
25300 }
25302 /* How many alternative schedules to try. This should be as wide as the
25303 scheduling freedom in the DFA, but no wider. Making this value too
25304 large results extra work for the scheduler. */
25306 static int
25308 {
25310 {
25322 /* We use lookahead value 4 for BD both before and after reload
25323 schedules. Plan is to have value 8 included for O3. */
25332 /* Generally, we want haifa-sched:max_issue() to look ahead as far
25333 as many instructions can be executed on a cycle, i.e.,
25334 issue_rate. I wonder why tuning for many CPUs does not do this. */
25337 /* Don't use lookahead for pre-reload schedule to save compile time. */
25342 }
25343 }
25345 /* Return true if target platform supports macro-fusion. */
25347 static bool
25349 {
25351 }
25353 /* Check whether current microarchitecture support macro fusion
25354 for insn pair "CONDGEN + CONDJMP". Refer to
25355 "Intel Architectures Optimization Reference Manual". */
25357 static bool
25359 {
25378 else
25379 {
25384 {
25388 else
25390 }
25391 }
25398 /* Macro-fusion for cmp/test MEM-IMM + conditional jmp is not
25399 supported. */
25406 /* No fusion for RIP-relative address. */
25413 ix86_address parts;
25419 }
25424 /* Check whether conditional jump use Sign or Overflow Flags. */
25432 /* Return true for TYPE_TEST and TYPE_ICMP. */
25437 /* The following is the case that macro-fusion for alu + jmp. */
25441 /* No fusion for alu op with memory destination operand. */
25446 /* Macro-fusion for inc/dec + unsigned conditional jump is not
25447 supported. */
25456 }
25458 /* Try to reorder ready list to take advantage of Atom pipelined IMUL
25459 execution. It is applied if
25460 (1) IMUL instruction is on the top of list;
25461 (2) There exists the only producer of independent IMUL instruction in
25462 ready list.
25463 Return index of IMUL producer if it was found and -1 otherwise. */
25464 static int
25466 {
25468 sd_iterator_def sd_it;
25469 dep_t dep;
25476 /* Check that IMUL instruction is on the top of ready list. */
25485 /* Search for producer of independent IMUL instruction. */
25487 {
25491 /* Skip IMUL instruction. */
25501 {
25502 rtx con;
25513 {
25514 sd_iterator_def sd_it1;
25515 dep_t dep1;
25516 /* Check if there is no other dependee for IMUL. */
25519 {
25520 rtx pro;
25526 }
25529 }
25530 }
25533 }
25535 }
25537 /* Try to find the best candidate on the top of ready list if two insns
25538 have the same priority - candidate is best if its dependees were
25539 scheduled earlier. Applied for Silvermont only.
25540 Return true if top 2 insns must be interchanged. */
25541 static bool
25543 {
25546 rtx set;
25547 sd_iterator_def sd_it;
25548 dep_t dep;
25551 #define INSN_TICK(INSN) (HID (INSN)->tick)
25572 {
25575 /* Determine winner more precise. */
25577 {
25578 rtx pro;
25584 }
25586 {
25587 rtx pro;
25593 }
25596 {
25597 /* Determine winner - load must win. */
25603 }
25605 }
25607 #undef INSN_TICK
25608 }
25610 /* Perform possible reodering of ready list for Atom/Silvermont only.
25611 Return issue rate. */
25612 static int
25615 {
25619 rtx insn;
25622 /* Set up issue rate. */
25625 /* Do reodering for Atom/SLM only. */
25629 /* Nothing to do if ready list contains only 1 instruction. */
25633 /* Do reodering for post-reload scheduler only. */
25638 {
25643 /* Put IMUL producer (ready[index]) at the top of ready list. */
25649 }
25651 {
25655 /* Swap 2 top elements of ready list. */
25659 }
25661 }
25663 static bool
25666 /* Returns true if lhs of insn is HW function argument register and set up
25667 is_spilled to true if it is likely spilled HW register. */
25668 static bool
25670 {
25671 rtx dst;
25675 /* Call instructions are not movable, ignore it. */
25686 {
25687 /* Is it likely spilled HW register? */
25692 }
25694 }
25696 /* Add output dependencies for chain of function adjacent arguments if only
25697 there is a move to likely spilled HW register. Return first argument
25698 if at least one dependence was added or NULL otherwise. */
25699 static rtx
25701 {
25702 rtx insn;
25709 /* Find nearest to call argument passing instruction. */
25711 {
25720 }
25724 {
25731 {
25734 }
25736 {
25737 /* Add output depdendence between two function arguments if chain
25738 of output arguments contains likely spilled HW registers. */
25742 }
25743 else
25745 }
25749 }
25751 /* Add output or anti dependency from insn to first_arg to restrict its code
25752 motion. */
25753 static void
25755 {
25756 rtx set;
25757 rtx tmp;
25764 {
25765 /* Add output dependency to the first function argument. */
25768 }
25769 /* Add anti dependency. */
25771 }
25773 /* Avoid cross block motion of function argument through adding dependency
25774 from the first non-jump instruction in bb. */
25775 static void
25777 {
25781 {
25783 {
25786 {
25789 }
25790 }
25794 }
25795 }
25797 /* Hook for pre-reload schedule - avoid motion of function arguments
25798 passed in likely spilled HW registers. */
25799 static void
25801 {
25802 rtx insn;
25810 {
25813 {
25814 /* Add dependee for first argument to predecessors if only
25815 region contains more than one block. */
25819 /* Skip trivial regions and region head blocks that can have
25820 predecessors outside of region. */
25822 {
25823 edge e;
25824 edge_iterator ei;
25825 /* Assume that region is SCC, i.e. all immediate predecessors
25826 of non-head block are in the same region. */
25828 {
25829 /* Avoid creating of loop-carried dependencies through
25830 using topological odering in region. */
25833 }
25834 }
25838 }
25839 }
25842 }
25844 /* Hook for pre-reload schedule - set priority of moves from likely spilled
25845 HW registers to maximum, to schedule them at soon as possible. These are
25846 moves from function argument registers at the top of the function entry
25847 and moves from function return value registers after call. */
25848 static int
25850 {
25851 rtx set;
25861 {
25868 }
25871 }
25873 /* Model decoder of Core 2/i7.
25874 Below hooks for multipass scheduling (see haifa-sched.c:max_issue)
25875 track the instruction fetch block boundaries and make sure that long
25876 (9+ bytes) instructions are assigned to D0. */
25878 /* Maximum length of an insn that can be handled by
25879 a secondary decoder unit. '8' for Core 2/i7. */
25882 /* Ifetch block size, i.e., number of bytes decoder reads per cycle.
25883 '16' for Core 2/i7. */
25886 /* Maximum number of instructions decoder can handle per cycle.
25887 '6' for Core 2/i7. */
25891 ix86_first_cycle_multipass_data_t;
25893 const_ix86_first_cycle_multipass_data_t;
25895 /* A variable to store target state across calls to max_issue within
25896 one cycle. */
25900 /* Initialize DATA. */
25901 static void
25903 {
25904 ix86_first_cycle_multipass_data_t data
25911 }
25913 /* Advancing the cycle; reset ifetch block counts. */
25914 static void
25916 {
25923 }
25927 /* Filter out insns from ready_try that the core will not be able to issue
25928 on current cycle due to decoder. */
25929 static void
25930 core2i7_first_cycle_multipass_filter_ready_try
25933 {
25935 {
25936 rtx insn;
25946 (!first_cycle_insn_p
25948 /* ... or it would not fit into the ifetch block ... */
25950 /* ... or the decoder is full already ... */
25952 /* ... mask the insn out. */
25953 {
25958 }
25959 }
25960 }
25962 /* Prepare for a new round of multipass lookahead scheduling. */
25963 static void
25966 {
25967 ix86_first_cycle_multipass_data_t data
25969 const_ix86_first_cycle_multipass_data_t prev_data
25972 /* Restore the state from the end of the previous round. */
25976 /* Filter instructions that cannot be issued on current cycle due to
25977 decoder restrictions. */
25979 first_cycle_insn_p);
25980 }
25982 /* INSN is being issued in current solution. Account for its impact on
25983 the decoder model. */
25984 static void
25987 {
25988 ix86_first_cycle_multipass_data_t data
25990 const_ix86_first_cycle_multipass_data_t prev_data
26000 /* Allocate or resize the bitmap for storing INSN's effect on ready_try. */
26002 {
26005 }
26007 {
26011 }
26014 /* Filter out insns from ready_try that the core will not be able to issue
26015 on current cycle due to decoder. */
26018 }
26020 /* Revert the effect on ready_try. */
26021 static void
26025 {
26026 const_ix86_first_cycle_multipass_data_t data
26029 sbitmap_iterator sbi;
26033 {
26035 }
26036 }
26038 /* Save the result of multipass lookahead scheduling for the next round. */
26039 static void
26041 {
26042 const_ix86_first_cycle_multipass_data_t data
26044 ix86_first_cycle_multipass_data_t next_data
26048 {
26051 }
26052 }
26054 /* Deallocate target data. */
26055 static void
26057 {
26058 ix86_first_cycle_multipass_data_t data
26062 {
26066 }
26067 }
26069 /* Prepare for scheduling pass. */
26070 static void
26074 {
26075 /* Install scheduling hooks for current CPU. Some of these hooks are used
26076 in time-critical parts of the scheduler, so we only set them up when
26077 they are actually used. */
26079 {
26084 /* Do not perform multipass scheduling for pre-reload schedule
26085 to save compile time. */
26087 {
26103 /* Set decoder parameters. */
26108 }
26109 /* ... Fall through ... */
26119 }
26120 }
26122 \f
26123 /* Compute the alignment given to a constant that is being placed in memory.
26124 EXP is the constant and ALIGN is the alignment that the object would
26125 ordinarily have.
26126 The value of this function is used instead of that alignment to align
26127 the object. */
26129 int
26131 {
26134 {
26139 }
26145 }
26147 /* Compute the alignment for a static variable.
26148 TYPE is the data type, and ALIGN is the alignment that
26149 the object would ordinarily have. The value of this function is used
26150 instead of that alignment to align the object. */
26152 int
26154 {
26166 /* x86-64 ABI requires arrays greater than 16 bytes to be aligned
26167 to 16byte boundary. */
26169 {
26176 }
26182 {
26187 }
26189 {
26196 }
26201 {
26206 }
26209 {
26214 }
26217 }
26219 /* Compute the alignment for a local variable or a stack slot. EXP is
26220 the data type or decl itself, MODE is the widest mode available and
26221 ALIGN is the alignment that the object would ordinarily have. The
26222 value of this macro is used instead of that alignment to align the
26223 object. */
26225 unsigned int
26228 {
26232 {
26235 }
26236 else
26237 {
26240 }
26242 /* Don't do dynamic stack realignment for long long objects with
26243 -mpreferred-stack-boundary=2. */
26252 /* If TYPE is NULL, we are allocating a stack slot for caller-save
26253 register in MODE. We will return the largest alignment of XF
26254 and DF. */
26256 {
26260 }
26262 /* x86-64 ABI requires arrays greater than 16 bytes to be aligned
26263 to 16byte boundary. Exact wording is:
26265 An array uses the same alignment as its elements, except that a local or
26266 global array variable of length at least 16 bytes or
26267 a C99 variable-length array variable always has alignment of at least 16 bytes.
26269 This was added to allow use of aligned SSE instructions at arrays. This
26270 rule is meant for static storage (where compiler can not do the analysis
26271 by itself). We follow it for automatic variables only when convenient.
26272 We fully control everything in the function compiled and functions from
26273 other unit can not rely on the alignment.
26275 Exclude va_list type. It is the common case of local array where
26276 we can not benefit from the alignment.
26278 TODO: Probably one should optimize for size only when var is not escaping. */
26281 {
26291 }
26293 {
26298 }
26300 {
26306 }
26311 {
26316 }
26319 {
26325 }
26327 }
26329 /* Compute the minimum required alignment for dynamic stack realignment
26330 purposes for a local variable, parameter or a stack slot. EXP is
26331 the data type or decl itself, MODE is its mode and ALIGN is the
26332 alignment that the object would ordinarily have. */
26334 unsigned int
26337 {
26341 {
26344 }
26345 else
26346 {
26349 }
26354 /* Don't do dynamic stack realignment for long long objects with
26355 -mpreferred-stack-boundary=2. */
26362 }
26363 \f
26364 /* Find a location for the static chain incoming to a nested function.
26365 This is a register, unless all free registers are used by arguments. */
26367 static rtx
26369 {
26376 {
26377 /* We always use R10 in 64-bit mode. */
26379 }
26380 else
26381 {
26382 tree fntype;
26385 /* By default in 32-bit mode we use ECX to pass the static chain. */
26391 {
26392 /* Fastcall functions use ecx/edx for arguments, which leaves
26393 us with EAX for the static chain.
26394 Thiscall functions use ecx for arguments, which also
26395 leaves us with EAX for the static chain. */
26397 }
26399 {
26400 /* Thiscall functions use ecx for arguments, which leaves
26401 us with EAX and EDX for the static chain.
26402 We are using for abi-compatibility EAX. */
26404 }
26406 {
26407 /* For regparm 3, we have no free call-clobbered registers in
26408 which to store the static chain. In order to implement this,
26409 we have the trampoline push the static chain to the stack.
26410 However, we can't push a value below the return address when
26411 we call the nested function directly, so we have to use an
26412 alternate entry point. For this we use ESI, and have the
26413 alternate entry point push ESI, so that things appear the
26414 same once we're executing the nested function. */
26416 {
26422 }
26424 }
26425 }
26428 }
26430 /* Emit RTL insns to initialize the variable parts of a trampoline.
26431 FNDECL is the decl of the target address; M_TRAMP is a MEM for
26432 the trampoline, and CHAIN_VALUE is an RTX for the static chain
26433 to be passed to the target function. */
26435 static void
26437 {
26445 {
26448 /* Load the function address to r11. Try to load address using
26449 the shorter movl instead of movabs. We may want to support
26450 movq for kernel mode, but kernel does not use trampolines at
26451 the moment. FNADDR is a 32bit address and may not be in
26452 DImode when ptr_mode == SImode. Always use movl in this
26453 case. */
26456 {
26465 }
26466 else
26467 {
26474 }
26476 /* Load static chain using movabs to r10. Use the shorter movl
26477 instead of movabs when ptr_mode == SImode. */
26479 {
26482 }
26483 else
26484 {
26487 }
26496 /* Jump to r11; the last (unused) byte is a nop, only there to
26497 pad the write out to a single 32-bit store. */
26501 }
26502 else
26503 {
26506 /* Depending on the static chain location, either load a register
26507 with a constant, or push the constant to the stack. All of the
26508 instructions are the same size. */
26511 {
26513 {
26520 }
26521 }
26522 else
26537 /* Compute offset from the end of the jmp to the target function.
26538 In the case in which the trampoline stores the static chain on
26539 the stack, we need to skip the first insn which pushes the
26540 (call-saved) register static chain; this push is 1 byte. */
26547 }
26551 #ifdef HAVE_ENABLE_EXECUTE_STACK
26552 #ifdef CHECK_EXECUTE_STACK_ENABLED
26554 #endif
26557 #endif
26558 }
26559 \f
26560 /* The following file contains several enumerations and data structures
26561 built from the definitions in i386-builtin-types.def. */
26565 /* Table for the ix86 builtin non-function types. */
26568 /* Retrieve an element from the above table, building some of
26569 the types lazily. */
26571 static tree
26573 {
26585 {
26593 }
26594 else
26595 {
26601 else
26609 }
26613 }
26615 /* Table for the ix86 builtin function types. */
26618 /* Retrieve an element from the above table, building some of
26619 the types lazily. */
26621 static tree
26623 {
26624 tree type;
26633 {
26641 {
26644 }
26647 }
26648 else
26649 {
26655 }
26659 }
26662 /* Codes for all the SSE/MMX builtins. */
26663 enum ix86_builtins
26664 {
26665 IX86_BUILTIN_ADDPS,
26666 IX86_BUILTIN_ADDSS,
26667 IX86_BUILTIN_DIVPS,
26668 IX86_BUILTIN_DIVSS,
26669 IX86_BUILTIN_MULPS,
26670 IX86_BUILTIN_MULSS,
26671 IX86_BUILTIN_SUBPS,
26672 IX86_BUILTIN_SUBSS,
26674 IX86_BUILTIN_CMPEQPS,
26675 IX86_BUILTIN_CMPLTPS,
26676 IX86_BUILTIN_CMPLEPS,
26677 IX86_BUILTIN_CMPGTPS,
26678 IX86_BUILTIN_CMPGEPS,
26679 IX86_BUILTIN_CMPNEQPS,
26680 IX86_BUILTIN_CMPNLTPS,
26681 IX86_BUILTIN_CMPNLEPS,
26682 IX86_BUILTIN_CMPNGTPS,
26683 IX86_BUILTIN_CMPNGEPS,
26684 IX86_BUILTIN_CMPORDPS,
26685 IX86_BUILTIN_CMPUNORDPS,
26686 IX86_BUILTIN_CMPEQSS,
26687 IX86_BUILTIN_CMPLTSS,
26688 IX86_BUILTIN_CMPLESS,
26689 IX86_BUILTIN_CMPNEQSS,
26690 IX86_BUILTIN_CMPNLTSS,
26691 IX86_BUILTIN_CMPNLESS,
26692 IX86_BUILTIN_CMPORDSS,
26693 IX86_BUILTIN_CMPUNORDSS,
26695 IX86_BUILTIN_COMIEQSS,
26696 IX86_BUILTIN_COMILTSS,
26697 IX86_BUILTIN_COMILESS,
26698 IX86_BUILTIN_COMIGTSS,
26699 IX86_BUILTIN_COMIGESS,
26700 IX86_BUILTIN_COMINEQSS,
26701 IX86_BUILTIN_UCOMIEQSS,
26702 IX86_BUILTIN_UCOMILTSS,
26703 IX86_BUILTIN_UCOMILESS,
26704 IX86_BUILTIN_UCOMIGTSS,
26705 IX86_BUILTIN_UCOMIGESS,
26706 IX86_BUILTIN_UCOMINEQSS,
26708 IX86_BUILTIN_CVTPI2PS,
26709 IX86_BUILTIN_CVTPS2PI,
26710 IX86_BUILTIN_CVTSI2SS,
26711 IX86_BUILTIN_CVTSI642SS,
26712 IX86_BUILTIN_CVTSS2SI,
26713 IX86_BUILTIN_CVTSS2SI64,
26714 IX86_BUILTIN_CVTTPS2PI,
26715 IX86_BUILTIN_CVTTSS2SI,
26716 IX86_BUILTIN_CVTTSS2SI64,
26718 IX86_BUILTIN_MAXPS,
26719 IX86_BUILTIN_MAXSS,
26720 IX86_BUILTIN_MINPS,
26721 IX86_BUILTIN_MINSS,
26723 IX86_BUILTIN_LOADUPS,
26724 IX86_BUILTIN_STOREUPS,
26725 IX86_BUILTIN_MOVSS,
26727 IX86_BUILTIN_MOVHLPS,
26728 IX86_BUILTIN_MOVLHPS,
26729 IX86_BUILTIN_LOADHPS,
26730 IX86_BUILTIN_LOADLPS,
26731 IX86_BUILTIN_STOREHPS,
26732 IX86_BUILTIN_STORELPS,
26734 IX86_BUILTIN_MASKMOVQ,
26735 IX86_BUILTIN_MOVMSKPS,
26736 IX86_BUILTIN_PMOVMSKB,
26738 IX86_BUILTIN_MOVNTPS,
26739 IX86_BUILTIN_MOVNTQ,
26741 IX86_BUILTIN_LOADDQU,
26742 IX86_BUILTIN_STOREDQU,
26744 IX86_BUILTIN_PACKSSWB,
26745 IX86_BUILTIN_PACKSSDW,
26746 IX86_BUILTIN_PACKUSWB,
26748 IX86_BUILTIN_PADDB,
26749 IX86_BUILTIN_PADDW,
26750 IX86_BUILTIN_PADDD,
26751 IX86_BUILTIN_PADDQ,
26752 IX86_BUILTIN_PADDSB,
26753 IX86_BUILTIN_PADDSW,
26754 IX86_BUILTIN_PADDUSB,
26755 IX86_BUILTIN_PADDUSW,
26756 IX86_BUILTIN_PSUBB,
26757 IX86_BUILTIN_PSUBW,
26758 IX86_BUILTIN_PSUBD,
26759 IX86_BUILTIN_PSUBQ,
26760 IX86_BUILTIN_PSUBSB,
26761 IX86_BUILTIN_PSUBSW,
26762 IX86_BUILTIN_PSUBUSB,
26763 IX86_BUILTIN_PSUBUSW,
26765 IX86_BUILTIN_PAND,
26766 IX86_BUILTIN_PANDN,
26767 IX86_BUILTIN_POR,
26768 IX86_BUILTIN_PXOR,
26770 IX86_BUILTIN_PAVGB,
26771 IX86_BUILTIN_PAVGW,
26773 IX86_BUILTIN_PCMPEQB,
26774 IX86_BUILTIN_PCMPEQW,
26775 IX86_BUILTIN_PCMPEQD,
26776 IX86_BUILTIN_PCMPGTB,
26777 IX86_BUILTIN_PCMPGTW,
26778 IX86_BUILTIN_PCMPGTD,
26780 IX86_BUILTIN_PMADDWD,
26782 IX86_BUILTIN_PMAXSW,
26783 IX86_BUILTIN_PMAXUB,
26784 IX86_BUILTIN_PMINSW,
26785 IX86_BUILTIN_PMINUB,
26787 IX86_BUILTIN_PMULHUW,
26788 IX86_BUILTIN_PMULHW,
26789 IX86_BUILTIN_PMULLW,
26791 IX86_BUILTIN_PSADBW,
26792 IX86_BUILTIN_PSHUFW,
26794 IX86_BUILTIN_PSLLW,
26795 IX86_BUILTIN_PSLLD,
26796 IX86_BUILTIN_PSLLQ,
26797 IX86_BUILTIN_PSRAW,
26798 IX86_BUILTIN_PSRAD,
26799 IX86_BUILTIN_PSRLW,
26800 IX86_BUILTIN_PSRLD,
26801 IX86_BUILTIN_PSRLQ,
26802 IX86_BUILTIN_PSLLWI,
26803 IX86_BUILTIN_PSLLDI,
26804 IX86_BUILTIN_PSLLQI,
26805 IX86_BUILTIN_PSRAWI,
26806 IX86_BUILTIN_PSRADI,
26807 IX86_BUILTIN_PSRLWI,
26808 IX86_BUILTIN_PSRLDI,
26809 IX86_BUILTIN_PSRLQI,
26811 IX86_BUILTIN_PUNPCKHBW,
26812 IX86_BUILTIN_PUNPCKHWD,
26813 IX86_BUILTIN_PUNPCKHDQ,
26814 IX86_BUILTIN_PUNPCKLBW,
26815 IX86_BUILTIN_PUNPCKLWD,
26816 IX86_BUILTIN_PUNPCKLDQ,
26818 IX86_BUILTIN_SHUFPS,
26820 IX86_BUILTIN_RCPPS,
26821 IX86_BUILTIN_RCPSS,
26822 IX86_BUILTIN_RSQRTPS,
26823 IX86_BUILTIN_RSQRTPS_NR,
26824 IX86_BUILTIN_RSQRTSS,
26825 IX86_BUILTIN_RSQRTF,
26826 IX86_BUILTIN_SQRTPS,
26827 IX86_BUILTIN_SQRTPS_NR,
26828 IX86_BUILTIN_SQRTSS,
26830 IX86_BUILTIN_UNPCKHPS,
26831 IX86_BUILTIN_UNPCKLPS,
26833 IX86_BUILTIN_ANDPS,
26834 IX86_BUILTIN_ANDNPS,
26835 IX86_BUILTIN_ORPS,
26836 IX86_BUILTIN_XORPS,
26838 IX86_BUILTIN_EMMS,
26839 IX86_BUILTIN_LDMXCSR,
26840 IX86_BUILTIN_STMXCSR,
26841 IX86_BUILTIN_SFENCE,
26843 IX86_BUILTIN_FXSAVE,
26844 IX86_BUILTIN_FXRSTOR,
26845 IX86_BUILTIN_FXSAVE64,
26846 IX86_BUILTIN_FXRSTOR64,
26848 IX86_BUILTIN_XSAVE,
26849 IX86_BUILTIN_XRSTOR,
26850 IX86_BUILTIN_XSAVE64,
26851 IX86_BUILTIN_XRSTOR64,
26853 IX86_BUILTIN_XSAVEOPT,
26854 IX86_BUILTIN_XSAVEOPT64,
26856 /* 3DNow! Original */
26857 IX86_BUILTIN_FEMMS,
26858 IX86_BUILTIN_PAVGUSB,
26859 IX86_BUILTIN_PF2ID,
26860 IX86_BUILTIN_PFACC,
26861 IX86_BUILTIN_PFADD,
26862 IX86_BUILTIN_PFCMPEQ,
26863 IX86_BUILTIN_PFCMPGE,
26864 IX86_BUILTIN_PFCMPGT,
26865 IX86_BUILTIN_PFMAX,
26866 IX86_BUILTIN_PFMIN,
26867 IX86_BUILTIN_PFMUL,
26868 IX86_BUILTIN_PFRCP,
26869 IX86_BUILTIN_PFRCPIT1,
26870 IX86_BUILTIN_PFRCPIT2,
26871 IX86_BUILTIN_PFRSQIT1,
26872 IX86_BUILTIN_PFRSQRT,
26873 IX86_BUILTIN_PFSUB,
26874 IX86_BUILTIN_PFSUBR,
26875 IX86_BUILTIN_PI2FD,
26876 IX86_BUILTIN_PMULHRW,
26878 /* 3DNow! Athlon Extensions */
26879 IX86_BUILTIN_PF2IW,
26880 IX86_BUILTIN_PFNACC,
26881 IX86_BUILTIN_PFPNACC,
26882 IX86_BUILTIN_PI2FW,
26883 IX86_BUILTIN_PSWAPDSI,
26884 IX86_BUILTIN_PSWAPDSF,
26886 /* SSE2 */
26887 IX86_BUILTIN_ADDPD,
26888 IX86_BUILTIN_ADDSD,
26889 IX86_BUILTIN_DIVPD,
26890 IX86_BUILTIN_DIVSD,
26891 IX86_BUILTIN_MULPD,
26892 IX86_BUILTIN_MULSD,
26893 IX86_BUILTIN_SUBPD,
26894 IX86_BUILTIN_SUBSD,
26896 IX86_BUILTIN_CMPEQPD,
26897 IX86_BUILTIN_CMPLTPD,
26898 IX86_BUILTIN_CMPLEPD,
26899 IX86_BUILTIN_CMPGTPD,
26900 IX86_BUILTIN_CMPGEPD,
26901 IX86_BUILTIN_CMPNEQPD,
26902 IX86_BUILTIN_CMPNLTPD,
26903 IX86_BUILTIN_CMPNLEPD,
26904 IX86_BUILTIN_CMPNGTPD,
26905 IX86_BUILTIN_CMPNGEPD,
26906 IX86_BUILTIN_CMPORDPD,
26907 IX86_BUILTIN_CMPUNORDPD,
26908 IX86_BUILTIN_CMPEQSD,
26909 IX86_BUILTIN_CMPLTSD,
26910 IX86_BUILTIN_CMPLESD,
26911 IX86_BUILTIN_CMPNEQSD,
26912 IX86_BUILTIN_CMPNLTSD,
26913 IX86_BUILTIN_CMPNLESD,
26914 IX86_BUILTIN_CMPORDSD,
26915 IX86_BUILTIN_CMPUNORDSD,
26917 IX86_BUILTIN_COMIEQSD,
26918 IX86_BUILTIN_COMILTSD,
26919 IX86_BUILTIN_COMILESD,
26920 IX86_BUILTIN_COMIGTSD,
26921 IX86_BUILTIN_COMIGESD,
26922 IX86_BUILTIN_COMINEQSD,
26923 IX86_BUILTIN_UCOMIEQSD,
26924 IX86_BUILTIN_UCOMILTSD,
26925 IX86_BUILTIN_UCOMILESD,
26926 IX86_BUILTIN_UCOMIGTSD,
26927 IX86_BUILTIN_UCOMIGESD,
26928 IX86_BUILTIN_UCOMINEQSD,
26930 IX86_BUILTIN_MAXPD,
26931 IX86_BUILTIN_MAXSD,
26932 IX86_BUILTIN_MINPD,
26933 IX86_BUILTIN_MINSD,
26935 IX86_BUILTIN_ANDPD,
26936 IX86_BUILTIN_ANDNPD,
26937 IX86_BUILTIN_ORPD,
26938 IX86_BUILTIN_XORPD,
26940 IX86_BUILTIN_SQRTPD,
26941 IX86_BUILTIN_SQRTSD,
26943 IX86_BUILTIN_UNPCKHPD,
26944 IX86_BUILTIN_UNPCKLPD,
26946 IX86_BUILTIN_SHUFPD,
26948 IX86_BUILTIN_LOADUPD,
26949 IX86_BUILTIN_STOREUPD,
26950 IX86_BUILTIN_MOVSD,
26952 IX86_BUILTIN_LOADHPD,
26953 IX86_BUILTIN_LOADLPD,
26955 IX86_BUILTIN_CVTDQ2PD,
26956 IX86_BUILTIN_CVTDQ2PS,
26958 IX86_BUILTIN_CVTPD2DQ,
26959 IX86_BUILTIN_CVTPD2PI,
26960 IX86_BUILTIN_CVTPD2PS,
26961 IX86_BUILTIN_CVTTPD2DQ,
26962 IX86_BUILTIN_CVTTPD2PI,
26964 IX86_BUILTIN_CVTPI2PD,
26965 IX86_BUILTIN_CVTSI2SD,
26966 IX86_BUILTIN_CVTSI642SD,
26968 IX86_BUILTIN_CVTSD2SI,
26969 IX86_BUILTIN_CVTSD2SI64,
26970 IX86_BUILTIN_CVTSD2SS,
26971 IX86_BUILTIN_CVTSS2SD,
26972 IX86_BUILTIN_CVTTSD2SI,
26973 IX86_BUILTIN_CVTTSD2SI64,
26975 IX86_BUILTIN_CVTPS2DQ,
26976 IX86_BUILTIN_CVTPS2PD,
26977 IX86_BUILTIN_CVTTPS2DQ,
26979 IX86_BUILTIN_MOVNTI,
26980 IX86_BUILTIN_MOVNTI64,
26981 IX86_BUILTIN_MOVNTPD,
26982 IX86_BUILTIN_MOVNTDQ,
26984 IX86_BUILTIN_MOVQ128,
26986 /* SSE2 MMX */
26987 IX86_BUILTIN_MASKMOVDQU,
26988 IX86_BUILTIN_MOVMSKPD,
26989 IX86_BUILTIN_PMOVMSKB128,
26991 IX86_BUILTIN_PACKSSWB128,
26992 IX86_BUILTIN_PACKSSDW128,
26993 IX86_BUILTIN_PACKUSWB128,
26995 IX86_BUILTIN_PADDB128,
26996 IX86_BUILTIN_PADDW128,
26997 IX86_BUILTIN_PADDD128,
26998 IX86_BUILTIN_PADDQ128,
26999 IX86_BUILTIN_PADDSB128,
27000 IX86_BUILTIN_PADDSW128,
27001 IX86_BUILTIN_PADDUSB128,
27002 IX86_BUILTIN_PADDUSW128,
27003 IX86_BUILTIN_PSUBB128,
27004 IX86_BUILTIN_PSUBW128,
27005 IX86_BUILTIN_PSUBD128,
27006 IX86_BUILTIN_PSUBQ128,
27007 IX86_BUILTIN_PSUBSB128,
27008 IX86_BUILTIN_PSUBSW128,
27009 IX86_BUILTIN_PSUBUSB128,
27010 IX86_BUILTIN_PSUBUSW128,
27012 IX86_BUILTIN_PAND128,
27013 IX86_BUILTIN_PANDN128,
27014 IX86_BUILTIN_POR128,
27015 IX86_BUILTIN_PXOR128,
27017 IX86_BUILTIN_PAVGB128,
27018 IX86_BUILTIN_PAVGW128,
27020 IX86_BUILTIN_PCMPEQB128,
27021 IX86_BUILTIN_PCMPEQW128,
27022 IX86_BUILTIN_PCMPEQD128,
27023 IX86_BUILTIN_PCMPGTB128,
27024 IX86_BUILTIN_PCMPGTW128,
27025 IX86_BUILTIN_PCMPGTD128,
27027 IX86_BUILTIN_PMADDWD128,
27029 IX86_BUILTIN_PMAXSW128,
27030 IX86_BUILTIN_PMAXUB128,
27031 IX86_BUILTIN_PMINSW128,
27032 IX86_BUILTIN_PMINUB128,
27034 IX86_BUILTIN_PMULUDQ,
27035 IX86_BUILTIN_PMULUDQ128,
27036 IX86_BUILTIN_PMULHUW128,
27037 IX86_BUILTIN_PMULHW128,
27038 IX86_BUILTIN_PMULLW128,
27040 IX86_BUILTIN_PSADBW128,
27041 IX86_BUILTIN_PSHUFHW,
27042 IX86_BUILTIN_PSHUFLW,
27043 IX86_BUILTIN_PSHUFD,
27045 IX86_BUILTIN_PSLLDQI128,
27046 IX86_BUILTIN_PSLLWI128,
27047 IX86_BUILTIN_PSLLDI128,
27048 IX86_BUILTIN_PSLLQI128,
27049 IX86_BUILTIN_PSRAWI128,
27050 IX86_BUILTIN_PSRADI128,
27051 IX86_BUILTIN_PSRLDQI128,
27052 IX86_BUILTIN_PSRLWI128,
27053 IX86_BUILTIN_PSRLDI128,
27054 IX86_BUILTIN_PSRLQI128,
27056 IX86_BUILTIN_PSLLDQ128,
27057 IX86_BUILTIN_PSLLW128,
27058 IX86_BUILTIN_PSLLD128,
27059 IX86_BUILTIN_PSLLQ128,
27060 IX86_BUILTIN_PSRAW128,
27061 IX86_BUILTIN_PSRAD128,
27062 IX86_BUILTIN_PSRLW128,
27063 IX86_BUILTIN_PSRLD128,
27064 IX86_BUILTIN_PSRLQ128,
27066 IX86_BUILTIN_PUNPCKHBW128,
27067 IX86_BUILTIN_PUNPCKHWD128,
27068 IX86_BUILTIN_PUNPCKHDQ128,
27069 IX86_BUILTIN_PUNPCKHQDQ128,
27070 IX86_BUILTIN_PUNPCKLBW128,
27071 IX86_BUILTIN_PUNPCKLWD128,
27072 IX86_BUILTIN_PUNPCKLDQ128,
27073 IX86_BUILTIN_PUNPCKLQDQ128,
27075 IX86_BUILTIN_CLFLUSH,
27076 IX86_BUILTIN_MFENCE,
27077 IX86_BUILTIN_LFENCE,
27078 IX86_BUILTIN_PAUSE,
27080 IX86_BUILTIN_FNSTENV,
27081 IX86_BUILTIN_FLDENV,
27082 IX86_BUILTIN_FNSTSW,
27083 IX86_BUILTIN_FNCLEX,
27085 IX86_BUILTIN_BSRSI,
27086 IX86_BUILTIN_BSRDI,
27087 IX86_BUILTIN_RDPMC,
27088 IX86_BUILTIN_RDTSC,
27089 IX86_BUILTIN_RDTSCP,
27090 IX86_BUILTIN_ROLQI,
27091 IX86_BUILTIN_ROLHI,
27092 IX86_BUILTIN_RORQI,
27093 IX86_BUILTIN_RORHI,
27095 /* SSE3. */
27096 IX86_BUILTIN_ADDSUBPS,
27097 IX86_BUILTIN_HADDPS,
27098 IX86_BUILTIN_HSUBPS,
27099 IX86_BUILTIN_MOVSHDUP,
27100 IX86_BUILTIN_MOVSLDUP,
27101 IX86_BUILTIN_ADDSUBPD,
27102 IX86_BUILTIN_HADDPD,
27103 IX86_BUILTIN_HSUBPD,
27104 IX86_BUILTIN_LDDQU,
27106 IX86_BUILTIN_MONITOR,
27107 IX86_BUILTIN_MWAIT,
27109 /* SSSE3. */
27110 IX86_BUILTIN_PHADDW,
27111 IX86_BUILTIN_PHADDD,
27112 IX86_BUILTIN_PHADDSW,
27113 IX86_BUILTIN_PHSUBW,
27114 IX86_BUILTIN_PHSUBD,
27115 IX86_BUILTIN_PHSUBSW,
27116 IX86_BUILTIN_PMADDUBSW,
27117 IX86_BUILTIN_PMULHRSW,
27118 IX86_BUILTIN_PSHUFB,
27119 IX86_BUILTIN_PSIGNB,
27120 IX86_BUILTIN_PSIGNW,
27121 IX86_BUILTIN_PSIGND,
27122 IX86_BUILTIN_PALIGNR,
27123 IX86_BUILTIN_PABSB,
27124 IX86_BUILTIN_PABSW,
27125 IX86_BUILTIN_PABSD,
27127 IX86_BUILTIN_PHADDW128,
27128 IX86_BUILTIN_PHADDD128,
27129 IX86_BUILTIN_PHADDSW128,
27130 IX86_BUILTIN_PHSUBW128,
27131 IX86_BUILTIN_PHSUBD128,
27132 IX86_BUILTIN_PHSUBSW128,
27133 IX86_BUILTIN_PMADDUBSW128,
27134 IX86_BUILTIN_PMULHRSW128,
27135 IX86_BUILTIN_PSHUFB128,
27136 IX86_BUILTIN_PSIGNB128,
27137 IX86_BUILTIN_PSIGNW128,
27138 IX86_BUILTIN_PSIGND128,
27139 IX86_BUILTIN_PALIGNR128,
27140 IX86_BUILTIN_PABSB128,
27141 IX86_BUILTIN_PABSW128,
27142 IX86_BUILTIN_PABSD128,
27144 /* AMDFAM10 - SSE4A New Instructions. */
27145 IX86_BUILTIN_MOVNTSD,
27146 IX86_BUILTIN_MOVNTSS,
27147 IX86_BUILTIN_EXTRQI,
27148 IX86_BUILTIN_EXTRQ,
27149 IX86_BUILTIN_INSERTQI,
27150 IX86_BUILTIN_INSERTQ,
27152 /* SSE4.1. */
27153 IX86_BUILTIN_BLENDPD,
27154 IX86_BUILTIN_BLENDPS,
27155 IX86_BUILTIN_BLENDVPD,
27156 IX86_BUILTIN_BLENDVPS,
27157 IX86_BUILTIN_PBLENDVB128,
27158 IX86_BUILTIN_PBLENDW128,
27160 IX86_BUILTIN_DPPD,
27161 IX86_BUILTIN_DPPS,
27163 IX86_BUILTIN_INSERTPS128,
27165 IX86_BUILTIN_MOVNTDQA,
27166 IX86_BUILTIN_MPSADBW128,
27167 IX86_BUILTIN_PACKUSDW128,
27168 IX86_BUILTIN_PCMPEQQ,
27169 IX86_BUILTIN_PHMINPOSUW128,
27171 IX86_BUILTIN_PMAXSB128,
27172 IX86_BUILTIN_PMAXSD128,
27173 IX86_BUILTIN_PMAXUD128,
27174 IX86_BUILTIN_PMAXUW128,
27176 IX86_BUILTIN_PMINSB128,
27177 IX86_BUILTIN_PMINSD128,
27178 IX86_BUILTIN_PMINUD128,
27179 IX86_BUILTIN_PMINUW128,
27181 IX86_BUILTIN_PMOVSXBW128,
27182 IX86_BUILTIN_PMOVSXBD128,
27183 IX86_BUILTIN_PMOVSXBQ128,
27184 IX86_BUILTIN_PMOVSXWD128,
27185 IX86_BUILTIN_PMOVSXWQ128,
27186 IX86_BUILTIN_PMOVSXDQ128,
27188 IX86_BUILTIN_PMOVZXBW128,
27189 IX86_BUILTIN_PMOVZXBD128,
27190 IX86_BUILTIN_PMOVZXBQ128,
27191 IX86_BUILTIN_PMOVZXWD128,
27192 IX86_BUILTIN_PMOVZXWQ128,
27193 IX86_BUILTIN_PMOVZXDQ128,
27195 IX86_BUILTIN_PMULDQ128,
27196 IX86_BUILTIN_PMULLD128,
27198 IX86_BUILTIN_ROUNDSD,
27199 IX86_BUILTIN_ROUNDSS,
27201 IX86_BUILTIN_ROUNDPD,
27202 IX86_BUILTIN_ROUNDPS,
27204 IX86_BUILTIN_FLOORPD,
27205 IX86_BUILTIN_CEILPD,
27206 IX86_BUILTIN_TRUNCPD,
27207 IX86_BUILTIN_RINTPD,
27208 IX86_BUILTIN_ROUNDPD_AZ,
27210 IX86_BUILTIN_FLOORPD_VEC_PACK_SFIX,
27211 IX86_BUILTIN_CEILPD_VEC_PACK_SFIX,
27212 IX86_BUILTIN_ROUNDPD_AZ_VEC_PACK_SFIX,
27214 IX86_BUILTIN_FLOORPS,
27215 IX86_BUILTIN_CEILPS,
27216 IX86_BUILTIN_TRUNCPS,
27217 IX86_BUILTIN_RINTPS,
27218 IX86_BUILTIN_ROUNDPS_AZ,
27220 IX86_BUILTIN_FLOORPS_SFIX,
27221 IX86_BUILTIN_CEILPS_SFIX,
27222 IX86_BUILTIN_ROUNDPS_AZ_SFIX,
27224 IX86_BUILTIN_PTESTZ,
27225 IX86_BUILTIN_PTESTC,
27226 IX86_BUILTIN_PTESTNZC,
27228 IX86_BUILTIN_VEC_INIT_V2SI,
27229 IX86_BUILTIN_VEC_INIT_V4HI,
27230 IX86_BUILTIN_VEC_INIT_V8QI,
27231 IX86_BUILTIN_VEC_EXT_V2DF,
27232 IX86_BUILTIN_VEC_EXT_V2DI,
27233 IX86_BUILTIN_VEC_EXT_V4SF,
27234 IX86_BUILTIN_VEC_EXT_V4SI,
27235 IX86_BUILTIN_VEC_EXT_V8HI,
27236 IX86_BUILTIN_VEC_EXT_V2SI,
27237 IX86_BUILTIN_VEC_EXT_V4HI,
27238 IX86_BUILTIN_VEC_EXT_V16QI,
27239 IX86_BUILTIN_VEC_SET_V2DI,
27240 IX86_BUILTIN_VEC_SET_V4SF,
27241 IX86_BUILTIN_VEC_SET_V4SI,
27242 IX86_BUILTIN_VEC_SET_V8HI,
27243 IX86_BUILTIN_VEC_SET_V4HI,
27244 IX86_BUILTIN_VEC_SET_V16QI,
27246 IX86_BUILTIN_VEC_PACK_SFIX,
27247 IX86_BUILTIN_VEC_PACK_SFIX256,
27249 /* SSE4.2. */
27250 IX86_BUILTIN_CRC32QI,
27251 IX86_BUILTIN_CRC32HI,
27252 IX86_BUILTIN_CRC32SI,
27253 IX86_BUILTIN_CRC32DI,
27255 IX86_BUILTIN_PCMPESTRI128,
27256 IX86_BUILTIN_PCMPESTRM128,
27257 IX86_BUILTIN_PCMPESTRA128,
27258 IX86_BUILTIN_PCMPESTRC128,
27259 IX86_BUILTIN_PCMPESTRO128,
27260 IX86_BUILTIN_PCMPESTRS128,
27261 IX86_BUILTIN_PCMPESTRZ128,
27262 IX86_BUILTIN_PCMPISTRI128,
27263 IX86_BUILTIN_PCMPISTRM128,
27264 IX86_BUILTIN_PCMPISTRA128,
27265 IX86_BUILTIN_PCMPISTRC128,
27266 IX86_BUILTIN_PCMPISTRO128,
27267 IX86_BUILTIN_PCMPISTRS128,
27268 IX86_BUILTIN_PCMPISTRZ128,
27270 IX86_BUILTIN_PCMPGTQ,
27272 /* AES instructions */
27273 IX86_BUILTIN_AESENC128,
27274 IX86_BUILTIN_AESENCLAST128,
27275 IX86_BUILTIN_AESDEC128,
27276 IX86_BUILTIN_AESDECLAST128,
27277 IX86_BUILTIN_AESIMC128,
27278 IX86_BUILTIN_AESKEYGENASSIST128,
27280 /* PCLMUL instruction */
27281 IX86_BUILTIN_PCLMULQDQ128,
27283 /* AVX */
27284 IX86_BUILTIN_ADDPD256,
27285 IX86_BUILTIN_ADDPS256,
27286 IX86_BUILTIN_ADDSUBPD256,
27287 IX86_BUILTIN_ADDSUBPS256,
27288 IX86_BUILTIN_ANDPD256,
27289 IX86_BUILTIN_ANDPS256,
27290 IX86_BUILTIN_ANDNPD256,
27291 IX86_BUILTIN_ANDNPS256,
27292 IX86_BUILTIN_BLENDPD256,
27293 IX86_BUILTIN_BLENDPS256,
27294 IX86_BUILTIN_BLENDVPD256,
27295 IX86_BUILTIN_BLENDVPS256,
27296 IX86_BUILTIN_DIVPD256,
27297 IX86_BUILTIN_DIVPS256,
27298 IX86_BUILTIN_DPPS256,
27299 IX86_BUILTIN_HADDPD256,
27300 IX86_BUILTIN_HADDPS256,
27301 IX86_BUILTIN_HSUBPD256,
27302 IX86_BUILTIN_HSUBPS256,
27303 IX86_BUILTIN_MAXPD256,
27304 IX86_BUILTIN_MAXPS256,
27305 IX86_BUILTIN_MINPD256,
27306 IX86_BUILTIN_MINPS256,
27307 IX86_BUILTIN_MULPD256,
27308 IX86_BUILTIN_MULPS256,
27309 IX86_BUILTIN_ORPD256,
27310 IX86_BUILTIN_ORPS256,
27311 IX86_BUILTIN_SHUFPD256,
27312 IX86_BUILTIN_SHUFPS256,
27313 IX86_BUILTIN_SUBPD256,
27314 IX86_BUILTIN_SUBPS256,
27315 IX86_BUILTIN_XORPD256,
27316 IX86_BUILTIN_XORPS256,
27317 IX86_BUILTIN_CMPSD,
27318 IX86_BUILTIN_CMPSS,
27319 IX86_BUILTIN_CMPPD,
27320 IX86_BUILTIN_CMPPS,
27321 IX86_BUILTIN_CMPPD256,
27322 IX86_BUILTIN_CMPPS256,
27323 IX86_BUILTIN_CVTDQ2PD256,
27324 IX86_BUILTIN_CVTDQ2PS256,
27325 IX86_BUILTIN_CVTPD2PS256,
27326 IX86_BUILTIN_CVTPS2DQ256,
27327 IX86_BUILTIN_CVTPS2PD256,
27328 IX86_BUILTIN_CVTTPD2DQ256,
27329 IX86_BUILTIN_CVTPD2DQ256,
27330 IX86_BUILTIN_CVTTPS2DQ256,
27331 IX86_BUILTIN_EXTRACTF128PD256,
27332 IX86_BUILTIN_EXTRACTF128PS256,
27333 IX86_BUILTIN_EXTRACTF128SI256,
27334 IX86_BUILTIN_VZEROALL,
27335 IX86_BUILTIN_VZEROUPPER,
27336 IX86_BUILTIN_VPERMILVARPD,
27337 IX86_BUILTIN_VPERMILVARPS,
27338 IX86_BUILTIN_VPERMILVARPD256,
27339 IX86_BUILTIN_VPERMILVARPS256,
27340 IX86_BUILTIN_VPERMILPD,
27341 IX86_BUILTIN_VPERMILPS,
27342 IX86_BUILTIN_VPERMILPD256,
27343 IX86_BUILTIN_VPERMILPS256,
27344 IX86_BUILTIN_VPERMIL2PD,
27345 IX86_BUILTIN_VPERMIL2PS,
27346 IX86_BUILTIN_VPERMIL2PD256,
27347 IX86_BUILTIN_VPERMIL2PS256,
27348 IX86_BUILTIN_VPERM2F128PD256,
27349 IX86_BUILTIN_VPERM2F128PS256,
27350 IX86_BUILTIN_VPERM2F128SI256,
27351 IX86_BUILTIN_VBROADCASTSS,
27352 IX86_BUILTIN_VBROADCASTSD256,
27353 IX86_BUILTIN_VBROADCASTSS256,
27354 IX86_BUILTIN_VBROADCASTPD256,
27355 IX86_BUILTIN_VBROADCASTPS256,
27356 IX86_BUILTIN_VINSERTF128PD256,
27357 IX86_BUILTIN_VINSERTF128PS256,
27358 IX86_BUILTIN_VINSERTF128SI256,
27359 IX86_BUILTIN_LOADUPD256,
27360 IX86_BUILTIN_LOADUPS256,
27361 IX86_BUILTIN_STOREUPD256,
27362 IX86_BUILTIN_STOREUPS256,
27363 IX86_BUILTIN_LDDQU256,
27364 IX86_BUILTIN_MOVNTDQ256,
27365 IX86_BUILTIN_MOVNTPD256,
27366 IX86_BUILTIN_MOVNTPS256,
27367 IX86_BUILTIN_LOADDQU256,
27368 IX86_BUILTIN_STOREDQU256,
27369 IX86_BUILTIN_MASKLOADPD,
27370 IX86_BUILTIN_MASKLOADPS,
27371 IX86_BUILTIN_MASKSTOREPD,
27372 IX86_BUILTIN_MASKSTOREPS,
27373 IX86_BUILTIN_MASKLOADPD256,
27374 IX86_BUILTIN_MASKLOADPS256,
27375 IX86_BUILTIN_MASKSTOREPD256,
27376 IX86_BUILTIN_MASKSTOREPS256,
27377 IX86_BUILTIN_MOVSHDUP256,
27378 IX86_BUILTIN_MOVSLDUP256,
27379 IX86_BUILTIN_MOVDDUP256,
27381 IX86_BUILTIN_SQRTPD256,
27382 IX86_BUILTIN_SQRTPS256,
27383 IX86_BUILTIN_SQRTPS_NR256,
27384 IX86_BUILTIN_RSQRTPS256,
27385 IX86_BUILTIN_RSQRTPS_NR256,
27387 IX86_BUILTIN_RCPPS256,
27389 IX86_BUILTIN_ROUNDPD256,
27390 IX86_BUILTIN_ROUNDPS256,
27392 IX86_BUILTIN_FLOORPD256,
27393 IX86_BUILTIN_CEILPD256,
27394 IX86_BUILTIN_TRUNCPD256,
27395 IX86_BUILTIN_RINTPD256,
27396 IX86_BUILTIN_ROUNDPD_AZ256,
27398 IX86_BUILTIN_FLOORPD_VEC_PACK_SFIX256,
27399 IX86_BUILTIN_CEILPD_VEC_PACK_SFIX256,
27400 IX86_BUILTIN_ROUNDPD_AZ_VEC_PACK_SFIX256,
27402 IX86_BUILTIN_FLOORPS256,
27403 IX86_BUILTIN_CEILPS256,
27404 IX86_BUILTIN_TRUNCPS256,
27405 IX86_BUILTIN_RINTPS256,
27406 IX86_BUILTIN_ROUNDPS_AZ256,
27408 IX86_BUILTIN_FLOORPS_SFIX256,
27409 IX86_BUILTIN_CEILPS_SFIX256,
27410 IX86_BUILTIN_ROUNDPS_AZ_SFIX256,
27412 IX86_BUILTIN_UNPCKHPD256,
27413 IX86_BUILTIN_UNPCKLPD256,
27414 IX86_BUILTIN_UNPCKHPS256,
27415 IX86_BUILTIN_UNPCKLPS256,
27417 IX86_BUILTIN_SI256_SI,
27418 IX86_BUILTIN_PS256_PS,
27419 IX86_BUILTIN_PD256_PD,
27420 IX86_BUILTIN_SI_SI256,
27421 IX86_BUILTIN_PS_PS256,
27422 IX86_BUILTIN_PD_PD256,
27424 IX86_BUILTIN_VTESTZPD,
27425 IX86_BUILTIN_VTESTCPD,
27426 IX86_BUILTIN_VTESTNZCPD,
27427 IX86_BUILTIN_VTESTZPS,
27428 IX86_BUILTIN_VTESTCPS,
27429 IX86_BUILTIN_VTESTNZCPS,
27430 IX86_BUILTIN_VTESTZPD256,
27431 IX86_BUILTIN_VTESTCPD256,
27432 IX86_BUILTIN_VTESTNZCPD256,
27433 IX86_BUILTIN_VTESTZPS256,
27434 IX86_BUILTIN_VTESTCPS256,
27435 IX86_BUILTIN_VTESTNZCPS256,
27436 IX86_BUILTIN_PTESTZ256,
27437 IX86_BUILTIN_PTESTC256,
27438 IX86_BUILTIN_PTESTNZC256,
27440 IX86_BUILTIN_MOVMSKPD256,
27441 IX86_BUILTIN_MOVMSKPS256,
27443 /* AVX2 */
27444 IX86_BUILTIN_MPSADBW256,
27445 IX86_BUILTIN_PABSB256,
27446 IX86_BUILTIN_PABSW256,
27447 IX86_BUILTIN_PABSD256,
27448 IX86_BUILTIN_PACKSSDW256,
27449 IX86_BUILTIN_PACKSSWB256,
27450 IX86_BUILTIN_PACKUSDW256,
27451 IX86_BUILTIN_PACKUSWB256,
27452 IX86_BUILTIN_PADDB256,
27453 IX86_BUILTIN_PADDW256,
27454 IX86_BUILTIN_PADDD256,
27455 IX86_BUILTIN_PADDQ256,
27456 IX86_BUILTIN_PADDSB256,
27457 IX86_BUILTIN_PADDSW256,
27458 IX86_BUILTIN_PADDUSB256,
27459 IX86_BUILTIN_PADDUSW256,
27460 IX86_BUILTIN_PALIGNR256,
27461 IX86_BUILTIN_AND256I,
27462 IX86_BUILTIN_ANDNOT256I,
27463 IX86_BUILTIN_PAVGB256,
27464 IX86_BUILTIN_PAVGW256,
27465 IX86_BUILTIN_PBLENDVB256,
27466 IX86_BUILTIN_PBLENDVW256,
27467 IX86_BUILTIN_PCMPEQB256,
27468 IX86_BUILTIN_PCMPEQW256,
27469 IX86_BUILTIN_PCMPEQD256,
27470 IX86_BUILTIN_PCMPEQQ256,
27471 IX86_BUILTIN_PCMPGTB256,
27472 IX86_BUILTIN_PCMPGTW256,
27473 IX86_BUILTIN_PCMPGTD256,
27474 IX86_BUILTIN_PCMPGTQ256,
27475 IX86_BUILTIN_PHADDW256,
27476 IX86_BUILTIN_PHADDD256,
27477 IX86_BUILTIN_PHADDSW256,
27478 IX86_BUILTIN_PHSUBW256,
27479 IX86_BUILTIN_PHSUBD256,
27480 IX86_BUILTIN_PHSUBSW256,
27481 IX86_BUILTIN_PMADDUBSW256,
27482 IX86_BUILTIN_PMADDWD256,
27483 IX86_BUILTIN_PMAXSB256,
27484 IX86_BUILTIN_PMAXSW256,
27485 IX86_BUILTIN_PMAXSD256,
27486 IX86_BUILTIN_PMAXUB256,
27487 IX86_BUILTIN_PMAXUW256,
27488 IX86_BUILTIN_PMAXUD256,
27489 IX86_BUILTIN_PMINSB256,
27490 IX86_BUILTIN_PMINSW256,
27491 IX86_BUILTIN_PMINSD256,
27492 IX86_BUILTIN_PMINUB256,
27493 IX86_BUILTIN_PMINUW256,
27494 IX86_BUILTIN_PMINUD256,
27495 IX86_BUILTIN_PMOVMSKB256,
27496 IX86_BUILTIN_PMOVSXBW256,
27497 IX86_BUILTIN_PMOVSXBD256,
27498 IX86_BUILTIN_PMOVSXBQ256,
27499 IX86_BUILTIN_PMOVSXWD256,
27500 IX86_BUILTIN_PMOVSXWQ256,
27501 IX86_BUILTIN_PMOVSXDQ256,
27502 IX86_BUILTIN_PMOVZXBW256,
27503 IX86_BUILTIN_PMOVZXBD256,
27504 IX86_BUILTIN_PMOVZXBQ256,
27505 IX86_BUILTIN_PMOVZXWD256,
27506 IX86_BUILTIN_PMOVZXWQ256,
27507 IX86_BUILTIN_PMOVZXDQ256,
27508 IX86_BUILTIN_PMULDQ256,
27509 IX86_BUILTIN_PMULHRSW256,
27510 IX86_BUILTIN_PMULHUW256,
27511 IX86_BUILTIN_PMULHW256,
27512 IX86_BUILTIN_PMULLW256,
27513 IX86_BUILTIN_PMULLD256,
27514 IX86_BUILTIN_PMULUDQ256,
27515 IX86_BUILTIN_POR256,
27516 IX86_BUILTIN_PSADBW256,
27517 IX86_BUILTIN_PSHUFB256,
27518 IX86_BUILTIN_PSHUFD256,
27519 IX86_BUILTIN_PSHUFHW256,
27520 IX86_BUILTIN_PSHUFLW256,
27521 IX86_BUILTIN_PSIGNB256,
27522 IX86_BUILTIN_PSIGNW256,
27523 IX86_BUILTIN_PSIGND256,
27524 IX86_BUILTIN_PSLLDQI256,
27525 IX86_BUILTIN_PSLLWI256,
27526 IX86_BUILTIN_PSLLW256,
27527 IX86_BUILTIN_PSLLDI256,
27528 IX86_BUILTIN_PSLLD256,
27529 IX86_BUILTIN_PSLLQI256,
27530 IX86_BUILTIN_PSLLQ256,
27531 IX86_BUILTIN_PSRAWI256,
27532 IX86_BUILTIN_PSRAW256,
27533 IX86_BUILTIN_PSRADI256,
27534 IX86_BUILTIN_PSRAD256,
27535 IX86_BUILTIN_PSRLDQI256,
27536 IX86_BUILTIN_PSRLWI256,
27537 IX86_BUILTIN_PSRLW256,
27538 IX86_BUILTIN_PSRLDI256,
27539 IX86_BUILTIN_PSRLD256,
27540 IX86_BUILTIN_PSRLQI256,
27541 IX86_BUILTIN_PSRLQ256,
27542 IX86_BUILTIN_PSUBB256,
27543 IX86_BUILTIN_PSUBW256,
27544 IX86_BUILTIN_PSUBD256,
27545 IX86_BUILTIN_PSUBQ256,
27546 IX86_BUILTIN_PSUBSB256,
27547 IX86_BUILTIN_PSUBSW256,
27548 IX86_BUILTIN_PSUBUSB256,
27549 IX86_BUILTIN_PSUBUSW256,
27550 IX86_BUILTIN_PUNPCKHBW256,
27551 IX86_BUILTIN_PUNPCKHWD256,
27552 IX86_BUILTIN_PUNPCKHDQ256,
27553 IX86_BUILTIN_PUNPCKHQDQ256,
27554 IX86_BUILTIN_PUNPCKLBW256,
27555 IX86_BUILTIN_PUNPCKLWD256,
27556 IX86_BUILTIN_PUNPCKLDQ256,
27557 IX86_BUILTIN_PUNPCKLQDQ256,
27558 IX86_BUILTIN_PXOR256,
27559 IX86_BUILTIN_MOVNTDQA256,
27560 IX86_BUILTIN_VBROADCASTSS_PS,
27561 IX86_BUILTIN_VBROADCASTSS_PS256,
27562 IX86_BUILTIN_VBROADCASTSD_PD256,
27563 IX86_BUILTIN_VBROADCASTSI256,
27564 IX86_BUILTIN_PBLENDD256,
27565 IX86_BUILTIN_PBLENDD128,
27566 IX86_BUILTIN_PBROADCASTB256,
27567 IX86_BUILTIN_PBROADCASTW256,
27568 IX86_BUILTIN_PBROADCASTD256,
27569 IX86_BUILTIN_PBROADCASTQ256,
27570 IX86_BUILTIN_PBROADCASTB128,
27571 IX86_BUILTIN_PBROADCASTW128,
27572 IX86_BUILTIN_PBROADCASTD128,
27573 IX86_BUILTIN_PBROADCASTQ128,
27574 IX86_BUILTIN_VPERMVARSI256,
27575 IX86_BUILTIN_VPERMDF256,
27576 IX86_BUILTIN_VPERMVARSF256,
27577 IX86_BUILTIN_VPERMDI256,
27578 IX86_BUILTIN_VPERMTI256,
27579 IX86_BUILTIN_VEXTRACT128I256,
27580 IX86_BUILTIN_VINSERT128I256,
27581 IX86_BUILTIN_MASKLOADD,
27582 IX86_BUILTIN_MASKLOADQ,
27583 IX86_BUILTIN_MASKLOADD256,
27584 IX86_BUILTIN_MASKLOADQ256,
27585 IX86_BUILTIN_MASKSTORED,
27586 IX86_BUILTIN_MASKSTOREQ,
27587 IX86_BUILTIN_MASKSTORED256,
27588 IX86_BUILTIN_MASKSTOREQ256,
27589 IX86_BUILTIN_PSLLVV4DI,
27590 IX86_BUILTIN_PSLLVV2DI,
27591 IX86_BUILTIN_PSLLVV8SI,
27592 IX86_BUILTIN_PSLLVV4SI,
27593 IX86_BUILTIN_PSRAVV8SI,
27594 IX86_BUILTIN_PSRAVV4SI,
27595 IX86_BUILTIN_PSRLVV4DI,
27596 IX86_BUILTIN_PSRLVV2DI,
27597 IX86_BUILTIN_PSRLVV8SI,
27598 IX86_BUILTIN_PSRLVV4SI,
27600 IX86_BUILTIN_GATHERSIV2DF,
27601 IX86_BUILTIN_GATHERSIV4DF,
27602 IX86_BUILTIN_GATHERDIV2DF,
27603 IX86_BUILTIN_GATHERDIV4DF,
27604 IX86_BUILTIN_GATHERSIV4SF,
27605 IX86_BUILTIN_GATHERSIV8SF,
27606 IX86_BUILTIN_GATHERDIV4SF,
27607 IX86_BUILTIN_GATHERDIV8SF,
27608 IX86_BUILTIN_GATHERSIV2DI,
27609 IX86_BUILTIN_GATHERSIV4DI,
27610 IX86_BUILTIN_GATHERDIV2DI,
27611 IX86_BUILTIN_GATHERDIV4DI,
27612 IX86_BUILTIN_GATHERSIV4SI,
27613 IX86_BUILTIN_GATHERSIV8SI,
27614 IX86_BUILTIN_GATHERDIV4SI,
27615 IX86_BUILTIN_GATHERDIV8SI,
27617 /* Alternate 4 element gather for the vectorizer where
27618 all operands are 32-byte wide. */
27619 IX86_BUILTIN_GATHERALTSIV4DF,
27620 IX86_BUILTIN_GATHERALTDIV8SF,
27621 IX86_BUILTIN_GATHERALTSIV4DI,
27622 IX86_BUILTIN_GATHERALTDIV8SI,
27624 /* TFmode support builtins. */
27625 IX86_BUILTIN_INFQ,
27626 IX86_BUILTIN_HUGE_VALQ,
27627 IX86_BUILTIN_FABSQ,
27628 IX86_BUILTIN_COPYSIGNQ,
27630 /* Vectorizer support builtins. */
27631 IX86_BUILTIN_CPYSGNPS,
27632 IX86_BUILTIN_CPYSGNPD,
27633 IX86_BUILTIN_CPYSGNPS256,
27634 IX86_BUILTIN_CPYSGNPD256,
27636 /* FMA4 instructions. */
27637 IX86_BUILTIN_VFMADDSS,
27638 IX86_BUILTIN_VFMADDSD,
27639 IX86_BUILTIN_VFMADDPS,
27640 IX86_BUILTIN_VFMADDPD,
27641 IX86_BUILTIN_VFMADDPS256,
27642 IX86_BUILTIN_VFMADDPD256,
27643 IX86_BUILTIN_VFMADDSUBPS,
27644 IX86_BUILTIN_VFMADDSUBPD,
27645 IX86_BUILTIN_VFMADDSUBPS256,
27646 IX86_BUILTIN_VFMADDSUBPD256,
27648 /* FMA3 instructions. */
27649 IX86_BUILTIN_VFMADDSS3,
27650 IX86_BUILTIN_VFMADDSD3,
27652 /* XOP instructions. */
27653 IX86_BUILTIN_VPCMOV,
27654 IX86_BUILTIN_VPCMOV_V2DI,
27655 IX86_BUILTIN_VPCMOV_V4SI,
27656 IX86_BUILTIN_VPCMOV_V8HI,
27657 IX86_BUILTIN_VPCMOV_V16QI,
27658 IX86_BUILTIN_VPCMOV_V4SF,
27659 IX86_BUILTIN_VPCMOV_V2DF,
27660 IX86_BUILTIN_VPCMOV256,
27661 IX86_BUILTIN_VPCMOV_V4DI256,
27662 IX86_BUILTIN_VPCMOV_V8SI256,
27663 IX86_BUILTIN_VPCMOV_V16HI256,
27664 IX86_BUILTIN_VPCMOV_V32QI256,
27665 IX86_BUILTIN_VPCMOV_V8SF256,
27666 IX86_BUILTIN_VPCMOV_V4DF256,
27668 IX86_BUILTIN_VPPERM,
27670 IX86_BUILTIN_VPMACSSWW,
27671 IX86_BUILTIN_VPMACSWW,
27672 IX86_BUILTIN_VPMACSSWD,
27673 IX86_BUILTIN_VPMACSWD,
27674 IX86_BUILTIN_VPMACSSDD,
27675 IX86_BUILTIN_VPMACSDD,
27676 IX86_BUILTIN_VPMACSSDQL,
27677 IX86_BUILTIN_VPMACSSDQH,
27678 IX86_BUILTIN_VPMACSDQL,
27679 IX86_BUILTIN_VPMACSDQH,
27680 IX86_BUILTIN_VPMADCSSWD,
27681 IX86_BUILTIN_VPMADCSWD,
27683 IX86_BUILTIN_VPHADDBW,
27684 IX86_BUILTIN_VPHADDBD,
27685 IX86_BUILTIN_VPHADDBQ,
27686 IX86_BUILTIN_VPHADDWD,
27687 IX86_BUILTIN_VPHADDWQ,
27688 IX86_BUILTIN_VPHADDDQ,
27689 IX86_BUILTIN_VPHADDUBW,
27690 IX86_BUILTIN_VPHADDUBD,
27691 IX86_BUILTIN_VPHADDUBQ,
27692 IX86_BUILTIN_VPHADDUWD,
27693 IX86_BUILTIN_VPHADDUWQ,
27694 IX86_BUILTIN_VPHADDUDQ,
27695 IX86_BUILTIN_VPHSUBBW,
27696 IX86_BUILTIN_VPHSUBWD,
27697 IX86_BUILTIN_VPHSUBDQ,
27699 IX86_BUILTIN_VPROTB,
27700 IX86_BUILTIN_VPROTW,
27701 IX86_BUILTIN_VPROTD,
27702 IX86_BUILTIN_VPROTQ,
27703 IX86_BUILTIN_VPROTB_IMM,
27704 IX86_BUILTIN_VPROTW_IMM,
27705 IX86_BUILTIN_VPROTD_IMM,
27706 IX86_BUILTIN_VPROTQ_IMM,
27708 IX86_BUILTIN_VPSHLB,
27709 IX86_BUILTIN_VPSHLW,
27710 IX86_BUILTIN_VPSHLD,
27711 IX86_BUILTIN_VPSHLQ,
27712 IX86_BUILTIN_VPSHAB,
27713 IX86_BUILTIN_VPSHAW,
27714 IX86_BUILTIN_VPSHAD,
27715 IX86_BUILTIN_VPSHAQ,
27717 IX86_BUILTIN_VFRCZSS,
27718 IX86_BUILTIN_VFRCZSD,
27719 IX86_BUILTIN_VFRCZPS,
27720 IX86_BUILTIN_VFRCZPD,
27721 IX86_BUILTIN_VFRCZPS256,
27722 IX86_BUILTIN_VFRCZPD256,
27724 IX86_BUILTIN_VPCOMEQUB,
27725 IX86_BUILTIN_VPCOMNEUB,
27726 IX86_BUILTIN_VPCOMLTUB,
27727 IX86_BUILTIN_VPCOMLEUB,
27728 IX86_BUILTIN_VPCOMGTUB,
27729 IX86_BUILTIN_VPCOMGEUB,
27730 IX86_BUILTIN_VPCOMFALSEUB,
27731 IX86_BUILTIN_VPCOMTRUEUB,
27733 IX86_BUILTIN_VPCOMEQUW,
27734 IX86_BUILTIN_VPCOMNEUW,
27735 IX86_BUILTIN_VPCOMLTUW,
27736 IX86_BUILTIN_VPCOMLEUW,
27737 IX86_BUILTIN_VPCOMGTUW,
27738 IX86_BUILTIN_VPCOMGEUW,
27739 IX86_BUILTIN_VPCOMFALSEUW,
27740 IX86_BUILTIN_VPCOMTRUEUW,
27742 IX86_BUILTIN_VPCOMEQUD,
27743 IX86_BUILTIN_VPCOMNEUD,
27744 IX86_BUILTIN_VPCOMLTUD,
27745 IX86_BUILTIN_VPCOMLEUD,
27746 IX86_BUILTIN_VPCOMGTUD,
27747 IX86_BUILTIN_VPCOMGEUD,
27748 IX86_BUILTIN_VPCOMFALSEUD,
27749 IX86_BUILTIN_VPCOMTRUEUD,
27751 IX86_BUILTIN_VPCOMEQUQ,
27752 IX86_BUILTIN_VPCOMNEUQ,
27753 IX86_BUILTIN_VPCOMLTUQ,
27754 IX86_BUILTIN_VPCOMLEUQ,
27755 IX86_BUILTIN_VPCOMGTUQ,
27756 IX86_BUILTIN_VPCOMGEUQ,
27757 IX86_BUILTIN_VPCOMFALSEUQ,
27758 IX86_BUILTIN_VPCOMTRUEUQ,
27760 IX86_BUILTIN_VPCOMEQB,
27761 IX86_BUILTIN_VPCOMNEB,
27762 IX86_BUILTIN_VPCOMLTB,
27763 IX86_BUILTIN_VPCOMLEB,
27764 IX86_BUILTIN_VPCOMGTB,
27765 IX86_BUILTIN_VPCOMGEB,
27766 IX86_BUILTIN_VPCOMFALSEB,
27767 IX86_BUILTIN_VPCOMTRUEB,
27769 IX86_BUILTIN_VPCOMEQW,
27770 IX86_BUILTIN_VPCOMNEW,
27771 IX86_BUILTIN_VPCOMLTW,
27772 IX86_BUILTIN_VPCOMLEW,
27773 IX86_BUILTIN_VPCOMGTW,
27774 IX86_BUILTIN_VPCOMGEW,
27775 IX86_BUILTIN_VPCOMFALSEW,
27776 IX86_BUILTIN_VPCOMTRUEW,
27778 IX86_BUILTIN_VPCOMEQD,
27779 IX86_BUILTIN_VPCOMNED,
27780 IX86_BUILTIN_VPCOMLTD,
27781 IX86_BUILTIN_VPCOMLED,
27782 IX86_BUILTIN_VPCOMGTD,
27783 IX86_BUILTIN_VPCOMGED,
27784 IX86_BUILTIN_VPCOMFALSED,
27785 IX86_BUILTIN_VPCOMTRUED,
27787 IX86_BUILTIN_VPCOMEQQ,
27788 IX86_BUILTIN_VPCOMNEQ,
27789 IX86_BUILTIN_VPCOMLTQ,
27790 IX86_BUILTIN_VPCOMLEQ,
27791 IX86_BUILTIN_VPCOMGTQ,
27792 IX86_BUILTIN_VPCOMGEQ,
27793 IX86_BUILTIN_VPCOMFALSEQ,
27794 IX86_BUILTIN_VPCOMTRUEQ,
27796 /* LWP instructions. */
27797 IX86_BUILTIN_LLWPCB,
27798 IX86_BUILTIN_SLWPCB,
27799 IX86_BUILTIN_LWPVAL32,
27800 IX86_BUILTIN_LWPVAL64,
27801 IX86_BUILTIN_LWPINS32,
27802 IX86_BUILTIN_LWPINS64,
27804 IX86_BUILTIN_CLZS,
27806 /* RTM */
27807 IX86_BUILTIN_XBEGIN,
27808 IX86_BUILTIN_XEND,
27809 IX86_BUILTIN_XABORT,
27810 IX86_BUILTIN_XTEST,
27812 /* BMI instructions. */
27813 IX86_BUILTIN_BEXTR32,
27814 IX86_BUILTIN_BEXTR64,
27815 IX86_BUILTIN_CTZS,
27817 /* TBM instructions. */
27818 IX86_BUILTIN_BEXTRI32,
27819 IX86_BUILTIN_BEXTRI64,
27821 /* BMI2 instructions. */
27822 IX86_BUILTIN_BZHI32,
27823 IX86_BUILTIN_BZHI64,
27824 IX86_BUILTIN_PDEP32,
27825 IX86_BUILTIN_PDEP64,
27826 IX86_BUILTIN_PEXT32,
27827 IX86_BUILTIN_PEXT64,
27829 /* ADX instructions. */
27830 IX86_BUILTIN_ADDCARRYX32,
27831 IX86_BUILTIN_ADDCARRYX64,
27833 /* FSGSBASE instructions. */
27834 IX86_BUILTIN_RDFSBASE32,
27835 IX86_BUILTIN_RDFSBASE64,
27836 IX86_BUILTIN_RDGSBASE32,
27837 IX86_BUILTIN_RDGSBASE64,
27838 IX86_BUILTIN_WRFSBASE32,
27839 IX86_BUILTIN_WRFSBASE64,
27840 IX86_BUILTIN_WRGSBASE32,
27841 IX86_BUILTIN_WRGSBASE64,
27843 /* RDRND instructions. */
27844 IX86_BUILTIN_RDRAND16_STEP,
27845 IX86_BUILTIN_RDRAND32_STEP,
27846 IX86_BUILTIN_RDRAND64_STEP,
27848 /* RDSEED instructions. */
27849 IX86_BUILTIN_RDSEED16_STEP,
27850 IX86_BUILTIN_RDSEED32_STEP,
27851 IX86_BUILTIN_RDSEED64_STEP,
27853 /* F16C instructions. */
27854 IX86_BUILTIN_CVTPH2PS,
27855 IX86_BUILTIN_CVTPH2PS256,
27856 IX86_BUILTIN_CVTPS2PH,
27857 IX86_BUILTIN_CVTPS2PH256,
27859 /* CFString built-in for darwin */
27860 IX86_BUILTIN_CFSTRING,
27862 /* Builtins to get CPU type and supported features. */
27863 IX86_BUILTIN_CPU_INIT,
27864 IX86_BUILTIN_CPU_IS,
27865 IX86_BUILTIN_CPU_SUPPORTS,
27867 IX86_BUILTIN_MAX
27868 };
27870 /* Table for the ix86 builtin decls. */
27873 /* Table of all of the builtin functions that are possible with different ISA's
27874 but are waiting to be built until a function is declared to use that
27875 ISA. */
27882 };
27887 /* Add an ix86 target builtin function with CODE, NAME and TYPE. Save the MASK
27888 of which isa_flags to use in the ix86_builtins_isa array. Stores the
27889 function decl in the ix86_builtins array. Returns the function decl or
27890 NULL_TREE, if the builtin was not added.
27892 If the front end has a special hook for builtin functions, delay adding
27893 builtin functions that aren't in the current ISA until the ISA is changed
27894 with function specific optimization. Doing so, can save about 300K for the
27895 default compiler. When the builtin is expanded, check at that time whether
27896 it is valid.
27898 If the front end doesn't have a special hook, record all builtins, even if
27899 it isn't an instruction set in the current ISA in case the user uses
27900 function specific options for a different ISA, so that we don't get scope
27901 errors if a builtin is added in the middle of a function scope. */
27907 {
27911 {
27920 {
27926 }
27927 else
27928 {
27934 }
27935 }
27938 }
27940 /* Like def_builtin, but also marks the function decl "const". */
27945 {
27949 else
27953 }
27955 /* Add any new builtin functions for a given ISA that may not have been
27956 declared. This saves a bit of space compared to adding all of the
27957 declarations to the tree, even if we didn't use them. */
27959 static void
27961 {
27965 {
27968 {
27971 /* Don't define the builtin again. */
27977 NULL_TREE);
27982 }
27983 }
27984 }
27986 /* Bits for builtin_description.flag. */
27988 /* Set when we don't support the comparison natively, and should
27989 swap_comparison in order to support it. */
27990 #define BUILTIN_DESC_SWAP_OPERANDS 1
27992 struct builtin_description
27993 {
28000 };
28003 {
28004 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_comi, "__builtin_ia32_comieq", IX86_BUILTIN_COMIEQSS, UNEQ, 0 },
28005 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_comi, "__builtin_ia32_comilt", IX86_BUILTIN_COMILTSS, UNLT, 0 },
28006 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_comi, "__builtin_ia32_comile", IX86_BUILTIN_COMILESS, UNLE, 0 },
28007 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_comi, "__builtin_ia32_comigt", IX86_BUILTIN_COMIGTSS, GT, 0 },
28008 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_comi, "__builtin_ia32_comige", IX86_BUILTIN_COMIGESS, GE, 0 },
28009 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_comi, "__builtin_ia32_comineq", IX86_BUILTIN_COMINEQSS, LTGT, 0 },
28010 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_ucomi, "__builtin_ia32_ucomieq", IX86_BUILTIN_UCOMIEQSS, UNEQ, 0 },
28011 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_ucomi, "__builtin_ia32_ucomilt", IX86_BUILTIN_UCOMILTSS, UNLT, 0 },
28012 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_ucomi, "__builtin_ia32_ucomile", IX86_BUILTIN_UCOMILESS, UNLE, 0 },
28013 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_ucomi, "__builtin_ia32_ucomigt", IX86_BUILTIN_UCOMIGTSS, GT, 0 },
28014 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_ucomi, "__builtin_ia32_ucomige", IX86_BUILTIN_UCOMIGESS, GE, 0 },
28015 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_ucomi, "__builtin_ia32_ucomineq", IX86_BUILTIN_UCOMINEQSS, LTGT, 0 },
28016 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_comi, "__builtin_ia32_comisdeq", IX86_BUILTIN_COMIEQSD, UNEQ, 0 },
28017 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_comi, "__builtin_ia32_comisdlt", IX86_BUILTIN_COMILTSD, UNLT, 0 },
28018 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_comi, "__builtin_ia32_comisdle", IX86_BUILTIN_COMILESD, UNLE, 0 },
28019 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_comi, "__builtin_ia32_comisdgt", IX86_BUILTIN_COMIGTSD, GT, 0 },
28020 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_comi, "__builtin_ia32_comisdge", IX86_BUILTIN_COMIGESD, GE, 0 },
28021 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_comi, "__builtin_ia32_comisdneq", IX86_BUILTIN_COMINEQSD, LTGT, 0 },
28022 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ucomi, "__builtin_ia32_ucomisdeq", IX86_BUILTIN_UCOMIEQSD, UNEQ, 0 },
28023 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ucomi, "__builtin_ia32_ucomisdlt", IX86_BUILTIN_UCOMILTSD, UNLT, 0 },
28024 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ucomi, "__builtin_ia32_ucomisdle", IX86_BUILTIN_UCOMILESD, UNLE, 0 },
28025 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ucomi, "__builtin_ia32_ucomisdgt", IX86_BUILTIN_UCOMIGTSD, GT, 0 },
28026 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ucomi, "__builtin_ia32_ucomisdge", IX86_BUILTIN_UCOMIGESD, GE, 0 },
28027 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ucomi, "__builtin_ia32_ucomisdneq", IX86_BUILTIN_UCOMINEQSD, LTGT, 0 },
28028 };
28031 {
28032 /* SSE4.2 */
28033 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpestr, "__builtin_ia32_pcmpestri128", IX86_BUILTIN_PCMPESTRI128, UNKNOWN, 0 },
28034 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpestr, "__builtin_ia32_pcmpestrm128", IX86_BUILTIN_PCMPESTRM128, UNKNOWN, 0 },
28035 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpestr, "__builtin_ia32_pcmpestria128", IX86_BUILTIN_PCMPESTRA128, UNKNOWN, (int) CCAmode },
28036 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpestr, "__builtin_ia32_pcmpestric128", IX86_BUILTIN_PCMPESTRC128, UNKNOWN, (int) CCCmode },
28037 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpestr, "__builtin_ia32_pcmpestrio128", IX86_BUILTIN_PCMPESTRO128, UNKNOWN, (int) CCOmode },
28038 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpestr, "__builtin_ia32_pcmpestris128", IX86_BUILTIN_PCMPESTRS128, UNKNOWN, (int) CCSmode },
28039 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpestr, "__builtin_ia32_pcmpestriz128", IX86_BUILTIN_PCMPESTRZ128, UNKNOWN, (int) CCZmode },
28040 };
28043 {
28044 /* SSE4.2 */
28045 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpistr, "__builtin_ia32_pcmpistri128", IX86_BUILTIN_PCMPISTRI128, UNKNOWN, 0 },
28046 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpistr, "__builtin_ia32_pcmpistrm128", IX86_BUILTIN_PCMPISTRM128, UNKNOWN, 0 },
28047 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpistr, "__builtin_ia32_pcmpistria128", IX86_BUILTIN_PCMPISTRA128, UNKNOWN, (int) CCAmode },
28048 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpistr, "__builtin_ia32_pcmpistric128", IX86_BUILTIN_PCMPISTRC128, UNKNOWN, (int) CCCmode },
28049 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpistr, "__builtin_ia32_pcmpistrio128", IX86_BUILTIN_PCMPISTRO128, UNKNOWN, (int) CCOmode },
28050 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpistr, "__builtin_ia32_pcmpistris128", IX86_BUILTIN_PCMPISTRS128, UNKNOWN, (int) CCSmode },
28051 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpistr, "__builtin_ia32_pcmpistriz128", IX86_BUILTIN_PCMPISTRZ128, UNKNOWN, (int) CCZmode },
28052 };
28054 /* Special builtins with variable number of arguments. */
28056 {
28057 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_nothing, "__builtin_ia32_rdtsc", IX86_BUILTIN_RDTSC, UNKNOWN, (int) UINT64_FTYPE_VOID },
28058 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_nothing, "__builtin_ia32_rdtscp", IX86_BUILTIN_RDTSCP, UNKNOWN, (int) UINT64_FTYPE_PUNSIGNED },
28059 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_pause, "__builtin_ia32_pause", IX86_BUILTIN_PAUSE, UNKNOWN, (int) VOID_FTYPE_VOID },
28061 /* 80387 (for use internally for atomic compound assignment). */
28062 { 0, CODE_FOR_fnstenv, "__builtin_ia32_fnstenv", IX86_BUILTIN_FNSTENV, UNKNOWN, (int) VOID_FTYPE_PVOID },
28063 { 0, CODE_FOR_fldenv, "__builtin_ia32_fldenv", IX86_BUILTIN_FLDENV, UNKNOWN, (int) VOID_FTYPE_PCVOID },
28064 { 0, CODE_FOR_fnstsw, "__builtin_ia32_fnstsw", IX86_BUILTIN_FNSTSW, UNKNOWN, (int) VOID_FTYPE_PUSHORT },
28065 { 0, CODE_FOR_fnclex, "__builtin_ia32_fnclex", IX86_BUILTIN_FNCLEX, UNKNOWN, (int) VOID_FTYPE_VOID },
28067 /* MMX */
28068 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_emms, "__builtin_ia32_emms", IX86_BUILTIN_EMMS, UNKNOWN, (int) VOID_FTYPE_VOID },
28070 /* 3DNow! */
28071 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_femms, "__builtin_ia32_femms", IX86_BUILTIN_FEMMS, UNKNOWN, (int) VOID_FTYPE_VOID },
28073 /* FXSR, XSAVE and XSAVEOPT */
28074 { OPTION_MASK_ISA_FXSR, CODE_FOR_nothing, "__builtin_ia32_fxsave", IX86_BUILTIN_FXSAVE, UNKNOWN, (int) VOID_FTYPE_PVOID },
28075 { OPTION_MASK_ISA_FXSR, CODE_FOR_nothing, "__builtin_ia32_fxrstor", IX86_BUILTIN_FXRSTOR, UNKNOWN, (int) VOID_FTYPE_PVOID },
28076 { OPTION_MASK_ISA_XSAVE, CODE_FOR_nothing, "__builtin_ia32_xsave", IX86_BUILTIN_XSAVE, UNKNOWN, (int) VOID_FTYPE_PVOID_INT64 },
28077 { OPTION_MASK_ISA_XSAVE, CODE_FOR_nothing, "__builtin_ia32_xrstor", IX86_BUILTIN_XRSTOR, UNKNOWN, (int) VOID_FTYPE_PVOID_INT64 },
28078 { OPTION_MASK_ISA_XSAVEOPT, CODE_FOR_nothing, "__builtin_ia32_xsaveopt", IX86_BUILTIN_XSAVEOPT, UNKNOWN, (int) VOID_FTYPE_PVOID_INT64 },
28080 { OPTION_MASK_ISA_FXSR | OPTION_MASK_ISA_64BIT, CODE_FOR_nothing, "__builtin_ia32_fxsave64", IX86_BUILTIN_FXSAVE64, UNKNOWN, (int) VOID_FTYPE_PVOID },
28081 { OPTION_MASK_ISA_FXSR | OPTION_MASK_ISA_64BIT, CODE_FOR_nothing, "__builtin_ia32_fxrstor64", IX86_BUILTIN_FXRSTOR64, UNKNOWN, (int) VOID_FTYPE_PVOID },
28082 { OPTION_MASK_ISA_XSAVE | OPTION_MASK_ISA_64BIT, CODE_FOR_nothing, "__builtin_ia32_xsave64", IX86_BUILTIN_XSAVE64, UNKNOWN, (int) VOID_FTYPE_PVOID_INT64 },
28083 { OPTION_MASK_ISA_XSAVE | OPTION_MASK_ISA_64BIT, CODE_FOR_nothing, "__builtin_ia32_xrstor64", IX86_BUILTIN_XRSTOR64, UNKNOWN, (int) VOID_FTYPE_PVOID_INT64 },
28084 { OPTION_MASK_ISA_XSAVEOPT | OPTION_MASK_ISA_64BIT, CODE_FOR_nothing, "__builtin_ia32_xsaveopt64", IX86_BUILTIN_XSAVEOPT64, UNKNOWN, (int) VOID_FTYPE_PVOID_INT64 },
28086 /* SSE */
28087 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_storeups, "__builtin_ia32_storeups", IX86_BUILTIN_STOREUPS, UNKNOWN, (int) VOID_FTYPE_PFLOAT_V4SF },
28088 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_movntv4sf, "__builtin_ia32_movntps", IX86_BUILTIN_MOVNTPS, UNKNOWN, (int) VOID_FTYPE_PFLOAT_V4SF },
28089 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_loadups, "__builtin_ia32_loadups", IX86_BUILTIN_LOADUPS, UNKNOWN, (int) V4SF_FTYPE_PCFLOAT },
28091 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_loadhps_exp, "__builtin_ia32_loadhps", IX86_BUILTIN_LOADHPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_PCV2SF },
28092 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_loadlps_exp, "__builtin_ia32_loadlps", IX86_BUILTIN_LOADLPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_PCV2SF },
28093 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_storehps, "__builtin_ia32_storehps", IX86_BUILTIN_STOREHPS, UNKNOWN, (int) VOID_FTYPE_PV2SF_V4SF },
28094 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_storelps, "__builtin_ia32_storelps", IX86_BUILTIN_STORELPS, UNKNOWN, (int) VOID_FTYPE_PV2SF_V4SF },
28096 /* SSE or 3DNow!A */
28097 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_sse_sfence, "__builtin_ia32_sfence", IX86_BUILTIN_SFENCE, UNKNOWN, (int) VOID_FTYPE_VOID },
28098 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_sse_movntq, "__builtin_ia32_movntq", IX86_BUILTIN_MOVNTQ, UNKNOWN, (int) VOID_FTYPE_PULONGLONG_ULONGLONG },
28100 /* SSE2 */
28101 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_lfence, "__builtin_ia32_lfence", IX86_BUILTIN_LFENCE, UNKNOWN, (int) VOID_FTYPE_VOID },
28102 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_mfence, 0, IX86_BUILTIN_MFENCE, UNKNOWN, (int) VOID_FTYPE_VOID },
28103 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_storeupd, "__builtin_ia32_storeupd", IX86_BUILTIN_STOREUPD, UNKNOWN, (int) VOID_FTYPE_PDOUBLE_V2DF },
28104 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_storedquv16qi, "__builtin_ia32_storedqu", IX86_BUILTIN_STOREDQU, UNKNOWN, (int) VOID_FTYPE_PCHAR_V16QI },
28105 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_movntv2df, "__builtin_ia32_movntpd", IX86_BUILTIN_MOVNTPD, UNKNOWN, (int) VOID_FTYPE_PDOUBLE_V2DF },
28106 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_movntv2di, "__builtin_ia32_movntdq", IX86_BUILTIN_MOVNTDQ, UNKNOWN, (int) VOID_FTYPE_PV2DI_V2DI },
28107 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_movntisi, "__builtin_ia32_movnti", IX86_BUILTIN_MOVNTI, UNKNOWN, (int) VOID_FTYPE_PINT_INT },
28108 { OPTION_MASK_ISA_SSE2 | OPTION_MASK_ISA_64BIT, CODE_FOR_sse2_movntidi, "__builtin_ia32_movnti64", IX86_BUILTIN_MOVNTI64, UNKNOWN, (int) VOID_FTYPE_PLONGLONG_LONGLONG },
28109 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_loadupd, "__builtin_ia32_loadupd", IX86_BUILTIN_LOADUPD, UNKNOWN, (int) V2DF_FTYPE_PCDOUBLE },
28110 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_loaddquv16qi, "__builtin_ia32_loaddqu", IX86_BUILTIN_LOADDQU, UNKNOWN, (int) V16QI_FTYPE_PCCHAR },
28112 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_loadhpd_exp, "__builtin_ia32_loadhpd", IX86_BUILTIN_LOADHPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_PCDOUBLE },
28113 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_loadlpd_exp, "__builtin_ia32_loadlpd", IX86_BUILTIN_LOADLPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_PCDOUBLE },
28115 /* SSE3 */
28116 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_lddqu, "__builtin_ia32_lddqu", IX86_BUILTIN_LDDQU, UNKNOWN, (int) V16QI_FTYPE_PCCHAR },
28118 /* SSE4.1 */
28119 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_movntdqa, "__builtin_ia32_movntdqa", IX86_BUILTIN_MOVNTDQA, UNKNOWN, (int) V2DI_FTYPE_PV2DI },
28121 /* SSE4A */
28122 { OPTION_MASK_ISA_SSE4A, CODE_FOR_sse4a_vmmovntv2df, "__builtin_ia32_movntsd", IX86_BUILTIN_MOVNTSD, UNKNOWN, (int) VOID_FTYPE_PDOUBLE_V2DF },
28123 { OPTION_MASK_ISA_SSE4A, CODE_FOR_sse4a_vmmovntv4sf, "__builtin_ia32_movntss", IX86_BUILTIN_MOVNTSS, UNKNOWN, (int) VOID_FTYPE_PFLOAT_V4SF },
28125 /* AVX */
28126 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vzeroall, "__builtin_ia32_vzeroall", IX86_BUILTIN_VZEROALL, UNKNOWN, (int) VOID_FTYPE_VOID },
28127 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vzeroupper, "__builtin_ia32_vzeroupper", IX86_BUILTIN_VZEROUPPER, UNKNOWN, (int) VOID_FTYPE_VOID },
28129 { OPTION_MASK_ISA_AVX, CODE_FOR_vec_dupv4sf, "__builtin_ia32_vbroadcastss", IX86_BUILTIN_VBROADCASTSS, UNKNOWN, (int) V4SF_FTYPE_PCFLOAT },
28130 { OPTION_MASK_ISA_AVX, CODE_FOR_vec_dupv4df, "__builtin_ia32_vbroadcastsd256", IX86_BUILTIN_VBROADCASTSD256, UNKNOWN, (int) V4DF_FTYPE_PCDOUBLE },
28131 { OPTION_MASK_ISA_AVX, CODE_FOR_vec_dupv8sf, "__builtin_ia32_vbroadcastss256", IX86_BUILTIN_VBROADCASTSS256, UNKNOWN, (int) V8SF_FTYPE_PCFLOAT },
28132 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vbroadcastf128_v4df, "__builtin_ia32_vbroadcastf128_pd256", IX86_BUILTIN_VBROADCASTPD256, UNKNOWN, (int) V4DF_FTYPE_PCV2DF },
28133 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vbroadcastf128_v8sf, "__builtin_ia32_vbroadcastf128_ps256", IX86_BUILTIN_VBROADCASTPS256, UNKNOWN, (int) V8SF_FTYPE_PCV4SF },
28135 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_loadupd256, "__builtin_ia32_loadupd256", IX86_BUILTIN_LOADUPD256, UNKNOWN, (int) V4DF_FTYPE_PCDOUBLE },
28136 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_loadups256, "__builtin_ia32_loadups256", IX86_BUILTIN_LOADUPS256, UNKNOWN, (int) V8SF_FTYPE_PCFLOAT },
28137 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_storeupd256, "__builtin_ia32_storeupd256", IX86_BUILTIN_STOREUPD256, UNKNOWN, (int) VOID_FTYPE_PDOUBLE_V4DF },
28138 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_storeups256, "__builtin_ia32_storeups256", IX86_BUILTIN_STOREUPS256, UNKNOWN, (int) VOID_FTYPE_PFLOAT_V8SF },
28139 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_loaddquv32qi, "__builtin_ia32_loaddqu256", IX86_BUILTIN_LOADDQU256, UNKNOWN, (int) V32QI_FTYPE_PCCHAR },
28140 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_storedquv32qi, "__builtin_ia32_storedqu256", IX86_BUILTIN_STOREDQU256, UNKNOWN, (int) VOID_FTYPE_PCHAR_V32QI },
28141 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_lddqu256, "__builtin_ia32_lddqu256", IX86_BUILTIN_LDDQU256, UNKNOWN, (int) V32QI_FTYPE_PCCHAR },
28143 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movntv4di, "__builtin_ia32_movntdq256", IX86_BUILTIN_MOVNTDQ256, UNKNOWN, (int) VOID_FTYPE_PV4DI_V4DI },
28144 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movntv4df, "__builtin_ia32_movntpd256", IX86_BUILTIN_MOVNTPD256, UNKNOWN, (int) VOID_FTYPE_PDOUBLE_V4DF },
28145 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movntv8sf, "__builtin_ia32_movntps256", IX86_BUILTIN_MOVNTPS256, UNKNOWN, (int) VOID_FTYPE_PFLOAT_V8SF },
28147 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskloadpd, "__builtin_ia32_maskloadpd", IX86_BUILTIN_MASKLOADPD, UNKNOWN, (int) V2DF_FTYPE_PCV2DF_V2DI },
28148 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskloadps, "__builtin_ia32_maskloadps", IX86_BUILTIN_MASKLOADPS, UNKNOWN, (int) V4SF_FTYPE_PCV4SF_V4SI },
28149 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskloadpd256, "__builtin_ia32_maskloadpd256", IX86_BUILTIN_MASKLOADPD256, UNKNOWN, (int) V4DF_FTYPE_PCV4DF_V4DI },
28150 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskloadps256, "__builtin_ia32_maskloadps256", IX86_BUILTIN_MASKLOADPS256, UNKNOWN, (int) V8SF_FTYPE_PCV8SF_V8SI },
28151 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskstorepd, "__builtin_ia32_maskstorepd", IX86_BUILTIN_MASKSTOREPD, UNKNOWN, (int) VOID_FTYPE_PV2DF_V2DI_V2DF },
28152 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskstoreps, "__builtin_ia32_maskstoreps", IX86_BUILTIN_MASKSTOREPS, UNKNOWN, (int) VOID_FTYPE_PV4SF_V4SI_V4SF },
28153 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskstorepd256, "__builtin_ia32_maskstorepd256", IX86_BUILTIN_MASKSTOREPD256, UNKNOWN, (int) VOID_FTYPE_PV4DF_V4DI_V4DF },
28154 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskstoreps256, "__builtin_ia32_maskstoreps256", IX86_BUILTIN_MASKSTOREPS256, UNKNOWN, (int) VOID_FTYPE_PV8SF_V8SI_V8SF },
28156 /* AVX2 */
28157 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_movntdqa, "__builtin_ia32_movntdqa256", IX86_BUILTIN_MOVNTDQA256, UNKNOWN, (int) V4DI_FTYPE_PV4DI },
28158 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_maskloadd, "__builtin_ia32_maskloadd", IX86_BUILTIN_MASKLOADD, UNKNOWN, (int) V4SI_FTYPE_PCV4SI_V4SI },
28159 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_maskloadq, "__builtin_ia32_maskloadq", IX86_BUILTIN_MASKLOADQ, UNKNOWN, (int) V2DI_FTYPE_PCV2DI_V2DI },
28160 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_maskloadd256, "__builtin_ia32_maskloadd256", IX86_BUILTIN_MASKLOADD256, UNKNOWN, (int) V8SI_FTYPE_PCV8SI_V8SI },
28161 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_maskloadq256, "__builtin_ia32_maskloadq256", IX86_BUILTIN_MASKLOADQ256, UNKNOWN, (int) V4DI_FTYPE_PCV4DI_V4DI },
28162 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_maskstored, "__builtin_ia32_maskstored", IX86_BUILTIN_MASKSTORED, UNKNOWN, (int) VOID_FTYPE_PV4SI_V4SI_V4SI },
28163 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_maskstoreq, "__builtin_ia32_maskstoreq", IX86_BUILTIN_MASKSTOREQ, UNKNOWN, (int) VOID_FTYPE_PV2DI_V2DI_V2DI },
28164 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_maskstored256, "__builtin_ia32_maskstored256", IX86_BUILTIN_MASKSTORED256, UNKNOWN, (int) VOID_FTYPE_PV8SI_V8SI_V8SI },
28165 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_maskstoreq256, "__builtin_ia32_maskstoreq256", IX86_BUILTIN_MASKSTOREQ256, UNKNOWN, (int) VOID_FTYPE_PV4DI_V4DI_V4DI },
28167 { OPTION_MASK_ISA_LWP, CODE_FOR_lwp_llwpcb, "__builtin_ia32_llwpcb", IX86_BUILTIN_LLWPCB, UNKNOWN, (int) VOID_FTYPE_PVOID },
28168 { OPTION_MASK_ISA_LWP, CODE_FOR_lwp_slwpcb, "__builtin_ia32_slwpcb", IX86_BUILTIN_SLWPCB, UNKNOWN, (int) PVOID_FTYPE_VOID },
28169 { OPTION_MASK_ISA_LWP, CODE_FOR_lwp_lwpvalsi3, "__builtin_ia32_lwpval32", IX86_BUILTIN_LWPVAL32, UNKNOWN, (int) VOID_FTYPE_UINT_UINT_UINT },
28170 { OPTION_MASK_ISA_LWP, CODE_FOR_lwp_lwpvaldi3, "__builtin_ia32_lwpval64", IX86_BUILTIN_LWPVAL64, UNKNOWN, (int) VOID_FTYPE_UINT64_UINT_UINT },
28171 { OPTION_MASK_ISA_LWP, CODE_FOR_lwp_lwpinssi3, "__builtin_ia32_lwpins32", IX86_BUILTIN_LWPINS32, UNKNOWN, (int) UCHAR_FTYPE_UINT_UINT_UINT },
28172 { OPTION_MASK_ISA_LWP, CODE_FOR_lwp_lwpinsdi3, "__builtin_ia32_lwpins64", IX86_BUILTIN_LWPINS64, UNKNOWN, (int) UCHAR_FTYPE_UINT64_UINT_UINT },
28174 /* FSGSBASE */
28175 { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_rdfsbasesi, "__builtin_ia32_rdfsbase32", IX86_BUILTIN_RDFSBASE32, UNKNOWN, (int) UNSIGNED_FTYPE_VOID },
28176 { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_rdfsbasedi, "__builtin_ia32_rdfsbase64", IX86_BUILTIN_RDFSBASE64, UNKNOWN, (int) UINT64_FTYPE_VOID },
28177 { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_rdgsbasesi, "__builtin_ia32_rdgsbase32", IX86_BUILTIN_RDGSBASE32, UNKNOWN, (int) UNSIGNED_FTYPE_VOID },
28178 { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_rdgsbasedi, "__builtin_ia32_rdgsbase64", IX86_BUILTIN_RDGSBASE64, UNKNOWN, (int) UINT64_FTYPE_VOID },
28179 { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_wrfsbasesi, "__builtin_ia32_wrfsbase32", IX86_BUILTIN_WRFSBASE32, UNKNOWN, (int) VOID_FTYPE_UNSIGNED },
28180 { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_wrfsbasedi, "__builtin_ia32_wrfsbase64", IX86_BUILTIN_WRFSBASE64, UNKNOWN, (int) VOID_FTYPE_UINT64 },
28181 { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_wrgsbasesi, "__builtin_ia32_wrgsbase32", IX86_BUILTIN_WRGSBASE32, UNKNOWN, (int) VOID_FTYPE_UNSIGNED },
28182 { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_wrgsbasedi, "__builtin_ia32_wrgsbase64", IX86_BUILTIN_WRGSBASE64, UNKNOWN, (int) VOID_FTYPE_UINT64 },
28184 /* RTM */
28185 { OPTION_MASK_ISA_RTM, CODE_FOR_xbegin, "__builtin_ia32_xbegin", IX86_BUILTIN_XBEGIN, UNKNOWN, (int) UNSIGNED_FTYPE_VOID },
28186 { OPTION_MASK_ISA_RTM, CODE_FOR_xend, "__builtin_ia32_xend", IX86_BUILTIN_XEND, UNKNOWN, (int) VOID_FTYPE_VOID },
28187 { OPTION_MASK_ISA_RTM, CODE_FOR_xtest, "__builtin_ia32_xtest", IX86_BUILTIN_XTEST, UNKNOWN, (int) INT_FTYPE_VOID },
28188 };
28190 /* Builtins with variable number of arguments. */
28192 {
28193 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_bsr, "__builtin_ia32_bsrsi", IX86_BUILTIN_BSRSI, UNKNOWN, (int) INT_FTYPE_INT },
28194 { OPTION_MASK_ISA_64BIT, CODE_FOR_bsr_rex64, "__builtin_ia32_bsrdi", IX86_BUILTIN_BSRDI, UNKNOWN, (int) INT64_FTYPE_INT64 },
28195 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_nothing, "__builtin_ia32_rdpmc", IX86_BUILTIN_RDPMC, UNKNOWN, (int) UINT64_FTYPE_INT },
28196 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_rotlqi3, "__builtin_ia32_rolqi", IX86_BUILTIN_ROLQI, UNKNOWN, (int) UINT8_FTYPE_UINT8_INT },
28197 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_rotlhi3, "__builtin_ia32_rolhi", IX86_BUILTIN_ROLHI, UNKNOWN, (int) UINT16_FTYPE_UINT16_INT },
28198 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_rotrqi3, "__builtin_ia32_rorqi", IX86_BUILTIN_RORQI, UNKNOWN, (int) UINT8_FTYPE_UINT8_INT },
28199 { ~OPTION_MASK_ISA_64BIT, CODE_FOR_rotrhi3, "__builtin_ia32_rorhi", IX86_BUILTIN_RORHI, UNKNOWN, (int) UINT16_FTYPE_UINT16_INT },
28201 /* MMX */
28202 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_addv8qi3, "__builtin_ia32_paddb", IX86_BUILTIN_PADDB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
28203 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_addv4hi3, "__builtin_ia32_paddw", IX86_BUILTIN_PADDW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
28204 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_addv2si3, "__builtin_ia32_paddd", IX86_BUILTIN_PADDD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
28205 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_subv8qi3, "__builtin_ia32_psubb", IX86_BUILTIN_PSUBB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
28206 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_subv4hi3, "__builtin_ia32_psubw", IX86_BUILTIN_PSUBW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
28207 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_subv2si3, "__builtin_ia32_psubd", IX86_BUILTIN_PSUBD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
28209 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ssaddv8qi3, "__builtin_ia32_paddsb", IX86_BUILTIN_PADDSB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
28210 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ssaddv4hi3, "__builtin_ia32_paddsw", IX86_BUILTIN_PADDSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
28211 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_sssubv8qi3, "__builtin_ia32_psubsb", IX86_BUILTIN_PSUBSB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
28212 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_sssubv4hi3, "__builtin_ia32_psubsw", IX86_BUILTIN_PSUBSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
28213 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_usaddv8qi3, "__builtin_ia32_paddusb", IX86_BUILTIN_PADDUSB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
28214 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_usaddv4hi3, "__builtin_ia32_paddusw", IX86_BUILTIN_PADDUSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
28215 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ussubv8qi3, "__builtin_ia32_psubusb", IX86_BUILTIN_PSUBUSB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
28216 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ussubv4hi3, "__builtin_ia32_psubusw", IX86_BUILTIN_PSUBUSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
28218 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_mulv4hi3, "__builtin_ia32_pmullw", IX86_BUILTIN_PMULLW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
28219 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_smulv4hi3_highpart, "__builtin_ia32_pmulhw", IX86_BUILTIN_PMULHW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
28221 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_andv2si3, "__builtin_ia32_pand", IX86_BUILTIN_PAND, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
28222 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_andnotv2si3, "__builtin_ia32_pandn", IX86_BUILTIN_PANDN, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
28223 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_iorv2si3, "__builtin_ia32_por", IX86_BUILTIN_POR, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
28224 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_xorv2si3, "__builtin_ia32_pxor", IX86_BUILTIN_PXOR, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
28226 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_eqv8qi3, "__builtin_ia32_pcmpeqb", IX86_BUILTIN_PCMPEQB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
28227 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_eqv4hi3, "__builtin_ia32_pcmpeqw", IX86_BUILTIN_PCMPEQW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
28228 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_eqv2si3, "__builtin_ia32_pcmpeqd", IX86_BUILTIN_PCMPEQD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
28229 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_gtv8qi3, "__builtin_ia32_pcmpgtb", IX86_BUILTIN_PCMPGTB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
28230 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_gtv4hi3, "__builtin_ia32_pcmpgtw", IX86_BUILTIN_PCMPGTW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
28231 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_gtv2si3, "__builtin_ia32_pcmpgtd", IX86_BUILTIN_PCMPGTD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
28233 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_punpckhbw, "__builtin_ia32_punpckhbw", IX86_BUILTIN_PUNPCKHBW, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
28234 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_punpckhwd, "__builtin_ia32_punpckhwd", IX86_BUILTIN_PUNPCKHWD, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
28235 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_punpckhdq, "__builtin_ia32_punpckhdq", IX86_BUILTIN_PUNPCKHDQ, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
28236 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_punpcklbw, "__builtin_ia32_punpcklbw", IX86_BUILTIN_PUNPCKLBW, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
28237 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_punpcklwd, "__builtin_ia32_punpcklwd", IX86_BUILTIN_PUNPCKLWD, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI},
28238 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_punpckldq, "__builtin_ia32_punpckldq", IX86_BUILTIN_PUNPCKLDQ, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI},
28240 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_packsswb, "__builtin_ia32_packsswb", IX86_BUILTIN_PACKSSWB, UNKNOWN, (int) V8QI_FTYPE_V4HI_V4HI },
28241 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_packssdw, "__builtin_ia32_packssdw", IX86_BUILTIN_PACKSSDW, UNKNOWN, (int) V4HI_FTYPE_V2SI_V2SI },
28242 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_packuswb, "__builtin_ia32_packuswb", IX86_BUILTIN_PACKUSWB, UNKNOWN, (int) V8QI_FTYPE_V4HI_V4HI },
28244 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_pmaddwd, "__builtin_ia32_pmaddwd", IX86_BUILTIN_PMADDWD, UNKNOWN, (int) V2SI_FTYPE_V4HI_V4HI },
28246 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashlv4hi3, "__builtin_ia32_psllwi", IX86_BUILTIN_PSLLWI, UNKNOWN, (int) V4HI_FTYPE_V4HI_SI_COUNT },
28247 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashlv2si3, "__builtin_ia32_pslldi", IX86_BUILTIN_PSLLDI, UNKNOWN, (int) V2SI_FTYPE_V2SI_SI_COUNT },
28248 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashlv1di3, "__builtin_ia32_psllqi", IX86_BUILTIN_PSLLQI, UNKNOWN, (int) V1DI_FTYPE_V1DI_SI_COUNT },
28249 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashlv4hi3, "__builtin_ia32_psllw", IX86_BUILTIN_PSLLW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI_COUNT },
28250 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashlv2si3, "__builtin_ia32_pslld", IX86_BUILTIN_PSLLD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI_COUNT },
28251 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashlv1di3, "__builtin_ia32_psllq", IX86_BUILTIN_PSLLQ, UNKNOWN, (int) V1DI_FTYPE_V1DI_V1DI_COUNT },
28253 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_lshrv4hi3, "__builtin_ia32_psrlwi", IX86_BUILTIN_PSRLWI, UNKNOWN, (int) V4HI_FTYPE_V4HI_SI_COUNT },
28254 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_lshrv2si3, "__builtin_ia32_psrldi", IX86_BUILTIN_PSRLDI, UNKNOWN, (int) V2SI_FTYPE_V2SI_SI_COUNT },
28255 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_lshrv1di3, "__builtin_ia32_psrlqi", IX86_BUILTIN_PSRLQI, UNKNOWN, (int) V1DI_FTYPE_V1DI_SI_COUNT },
28256 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_lshrv4hi3, "__builtin_ia32_psrlw", IX86_BUILTIN_PSRLW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI_COUNT },
28257 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_lshrv2si3, "__builtin_ia32_psrld", IX86_BUILTIN_PSRLD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI_COUNT },
28258 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_lshrv1di3, "__builtin_ia32_psrlq", IX86_BUILTIN_PSRLQ, UNKNOWN, (int) V1DI_FTYPE_V1DI_V1DI_COUNT },
28260 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashrv4hi3, "__builtin_ia32_psrawi", IX86_BUILTIN_PSRAWI, UNKNOWN, (int) V4HI_FTYPE_V4HI_SI_COUNT },
28261 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashrv2si3, "__builtin_ia32_psradi", IX86_BUILTIN_PSRADI, UNKNOWN, (int) V2SI_FTYPE_V2SI_SI_COUNT },
28262 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashrv4hi3, "__builtin_ia32_psraw", IX86_BUILTIN_PSRAW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI_COUNT },
28263 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashrv2si3, "__builtin_ia32_psrad", IX86_BUILTIN_PSRAD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI_COUNT },
28265 /* 3DNow! */
28266 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_pf2id, "__builtin_ia32_pf2id", IX86_BUILTIN_PF2ID, UNKNOWN, (int) V2SI_FTYPE_V2SF },
28267 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_floatv2si2, "__builtin_ia32_pi2fd", IX86_BUILTIN_PI2FD, UNKNOWN, (int) V2SF_FTYPE_V2SI },
28268 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_rcpv2sf2, "__builtin_ia32_pfrcp", IX86_BUILTIN_PFRCP, UNKNOWN, (int) V2SF_FTYPE_V2SF },
28269 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_rsqrtv2sf2, "__builtin_ia32_pfrsqrt", IX86_BUILTIN_PFRSQRT, UNKNOWN, (int) V2SF_FTYPE_V2SF },
28271 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_uavgv8qi3, "__builtin_ia32_pavgusb", IX86_BUILTIN_PAVGUSB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
28272 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_haddv2sf3, "__builtin_ia32_pfacc", IX86_BUILTIN_PFACC, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
28273 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_addv2sf3, "__builtin_ia32_pfadd", IX86_BUILTIN_PFADD, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
28274 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_eqv2sf3, "__builtin_ia32_pfcmpeq", IX86_BUILTIN_PFCMPEQ, UNKNOWN, (int) V2SI_FTYPE_V2SF_V2SF },
28275 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_gev2sf3, "__builtin_ia32_pfcmpge", IX86_BUILTIN_PFCMPGE, UNKNOWN, (int) V2SI_FTYPE_V2SF_V2SF },
28276 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_gtv2sf3, "__builtin_ia32_pfcmpgt", IX86_BUILTIN_PFCMPGT, UNKNOWN, (int) V2SI_FTYPE_V2SF_V2SF },
28277 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_smaxv2sf3, "__builtin_ia32_pfmax", IX86_BUILTIN_PFMAX, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
28278 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_sminv2sf3, "__builtin_ia32_pfmin", IX86_BUILTIN_PFMIN, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
28279 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_mulv2sf3, "__builtin_ia32_pfmul", IX86_BUILTIN_PFMUL, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
28280 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_rcpit1v2sf3, "__builtin_ia32_pfrcpit1", IX86_BUILTIN_PFRCPIT1, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
28281 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_rcpit2v2sf3, "__builtin_ia32_pfrcpit2", IX86_BUILTIN_PFRCPIT2, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
28282 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_rsqit1v2sf3, "__builtin_ia32_pfrsqit1", IX86_BUILTIN_PFRSQIT1, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
28283 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_subv2sf3, "__builtin_ia32_pfsub", IX86_BUILTIN_PFSUB, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
28284 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_subrv2sf3, "__builtin_ia32_pfsubr", IX86_BUILTIN_PFSUBR, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
28285 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_pmulhrwv4hi3, "__builtin_ia32_pmulhrw", IX86_BUILTIN_PMULHRW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
28287 /* 3DNow!A */
28288 { OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_pf2iw, "__builtin_ia32_pf2iw", IX86_BUILTIN_PF2IW, UNKNOWN, (int) V2SI_FTYPE_V2SF },
28289 { OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_pi2fw, "__builtin_ia32_pi2fw", IX86_BUILTIN_PI2FW, UNKNOWN, (int) V2SF_FTYPE_V2SI },
28290 { OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_pswapdv2si2, "__builtin_ia32_pswapdsi", IX86_BUILTIN_PSWAPDSI, UNKNOWN, (int) V2SI_FTYPE_V2SI },
28291 { OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_pswapdv2sf2, "__builtin_ia32_pswapdsf", IX86_BUILTIN_PSWAPDSF, UNKNOWN, (int) V2SF_FTYPE_V2SF },
28292 { OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_hsubv2sf3, "__builtin_ia32_pfnacc", IX86_BUILTIN_PFNACC, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
28293 { OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_addsubv2sf3, "__builtin_ia32_pfpnacc", IX86_BUILTIN_PFPNACC, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
28295 /* SSE */
28296 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_movmskps, "__builtin_ia32_movmskps", IX86_BUILTIN_MOVMSKPS, UNKNOWN, (int) INT_FTYPE_V4SF },
28297 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_sqrtv4sf2, "__builtin_ia32_sqrtps", IX86_BUILTIN_SQRTPS, UNKNOWN, (int) V4SF_FTYPE_V4SF },
28298 { OPTION_MASK_ISA_SSE, CODE_FOR_sqrtv4sf2, "__builtin_ia32_sqrtps_nr", IX86_BUILTIN_SQRTPS_NR, UNKNOWN, (int) V4SF_FTYPE_V4SF },
28299 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_rsqrtv4sf2, "__builtin_ia32_rsqrtps", IX86_BUILTIN_RSQRTPS, UNKNOWN, (int) V4SF_FTYPE_V4SF },
28300 { OPTION_MASK_ISA_SSE, CODE_FOR_rsqrtv4sf2, "__builtin_ia32_rsqrtps_nr", IX86_BUILTIN_RSQRTPS_NR, UNKNOWN, (int) V4SF_FTYPE_V4SF },
28301 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_rcpv4sf2, "__builtin_ia32_rcpps", IX86_BUILTIN_RCPPS, UNKNOWN, (int) V4SF_FTYPE_V4SF },
28302 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_cvtps2pi, "__builtin_ia32_cvtps2pi", IX86_BUILTIN_CVTPS2PI, UNKNOWN, (int) V2SI_FTYPE_V4SF },
28303 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_cvtss2si, "__builtin_ia32_cvtss2si", IX86_BUILTIN_CVTSS2SI, UNKNOWN, (int) INT_FTYPE_V4SF },
28304 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_64BIT, CODE_FOR_sse_cvtss2siq, "__builtin_ia32_cvtss2si64", IX86_BUILTIN_CVTSS2SI64, UNKNOWN, (int) INT64_FTYPE_V4SF },
28305 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_cvttps2pi, "__builtin_ia32_cvttps2pi", IX86_BUILTIN_CVTTPS2PI, UNKNOWN, (int) V2SI_FTYPE_V4SF },
28306 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_cvttss2si, "__builtin_ia32_cvttss2si", IX86_BUILTIN_CVTTSS2SI, UNKNOWN, (int) INT_FTYPE_V4SF },
28307 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_64BIT, CODE_FOR_sse_cvttss2siq, "__builtin_ia32_cvttss2si64", IX86_BUILTIN_CVTTSS2SI64, UNKNOWN, (int) INT64_FTYPE_V4SF },
28309 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_shufps, "__builtin_ia32_shufps", IX86_BUILTIN_SHUFPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
28311 { OPTION_MASK_ISA_SSE, CODE_FOR_addv4sf3, "__builtin_ia32_addps", IX86_BUILTIN_ADDPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28312 { OPTION_MASK_ISA_SSE, CODE_FOR_subv4sf3, "__builtin_ia32_subps", IX86_BUILTIN_SUBPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28313 { OPTION_MASK_ISA_SSE, CODE_FOR_mulv4sf3, "__builtin_ia32_mulps", IX86_BUILTIN_MULPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28314 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_divv4sf3, "__builtin_ia32_divps", IX86_BUILTIN_DIVPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28315 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmaddv4sf3, "__builtin_ia32_addss", IX86_BUILTIN_ADDSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28316 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmsubv4sf3, "__builtin_ia32_subss", IX86_BUILTIN_SUBSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28317 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmulv4sf3, "__builtin_ia32_mulss", IX86_BUILTIN_MULSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28318 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmdivv4sf3, "__builtin_ia32_divss", IX86_BUILTIN_DIVSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28320 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpeqps", IX86_BUILTIN_CMPEQPS, EQ, (int) V4SF_FTYPE_V4SF_V4SF },
28321 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpltps", IX86_BUILTIN_CMPLTPS, LT, (int) V4SF_FTYPE_V4SF_V4SF },
28322 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpleps", IX86_BUILTIN_CMPLEPS, LE, (int) V4SF_FTYPE_V4SF_V4SF },
28323 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpgtps", IX86_BUILTIN_CMPGTPS, LT, (int) V4SF_FTYPE_V4SF_V4SF_SWAP },
28324 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpgeps", IX86_BUILTIN_CMPGEPS, LE, (int) V4SF_FTYPE_V4SF_V4SF_SWAP },
28325 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpunordps", IX86_BUILTIN_CMPUNORDPS, UNORDERED, (int) V4SF_FTYPE_V4SF_V4SF },
28326 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpneqps", IX86_BUILTIN_CMPNEQPS, NE, (int) V4SF_FTYPE_V4SF_V4SF },
28327 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpnltps", IX86_BUILTIN_CMPNLTPS, UNGE, (int) V4SF_FTYPE_V4SF_V4SF },
28328 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpnleps", IX86_BUILTIN_CMPNLEPS, UNGT, (int) V4SF_FTYPE_V4SF_V4SF },
28329 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpngtps", IX86_BUILTIN_CMPNGTPS, UNGE, (int) V4SF_FTYPE_V4SF_V4SF_SWAP },
28330 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpngeps", IX86_BUILTIN_CMPNGEPS, UNGT, (int) V4SF_FTYPE_V4SF_V4SF_SWAP},
28331 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpordps", IX86_BUILTIN_CMPORDPS, ORDERED, (int) V4SF_FTYPE_V4SF_V4SF },
28332 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpeqss", IX86_BUILTIN_CMPEQSS, EQ, (int) V4SF_FTYPE_V4SF_V4SF },
28333 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpltss", IX86_BUILTIN_CMPLTSS, LT, (int) V4SF_FTYPE_V4SF_V4SF },
28334 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpless", IX86_BUILTIN_CMPLESS, LE, (int) V4SF_FTYPE_V4SF_V4SF },
28335 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpunordss", IX86_BUILTIN_CMPUNORDSS, UNORDERED, (int) V4SF_FTYPE_V4SF_V4SF },
28336 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpneqss", IX86_BUILTIN_CMPNEQSS, NE, (int) V4SF_FTYPE_V4SF_V4SF },
28337 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpnltss", IX86_BUILTIN_CMPNLTSS, UNGE, (int) V4SF_FTYPE_V4SF_V4SF },
28338 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpnless", IX86_BUILTIN_CMPNLESS, UNGT, (int) V4SF_FTYPE_V4SF_V4SF },
28339 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpordss", IX86_BUILTIN_CMPORDSS, ORDERED, (int) V4SF_FTYPE_V4SF_V4SF },
28341 { OPTION_MASK_ISA_SSE, CODE_FOR_sminv4sf3, "__builtin_ia32_minps", IX86_BUILTIN_MINPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28342 { OPTION_MASK_ISA_SSE, CODE_FOR_smaxv4sf3, "__builtin_ia32_maxps", IX86_BUILTIN_MAXPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28343 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmsminv4sf3, "__builtin_ia32_minss", IX86_BUILTIN_MINSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28344 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmsmaxv4sf3, "__builtin_ia32_maxss", IX86_BUILTIN_MAXSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28346 { OPTION_MASK_ISA_SSE, CODE_FOR_andv4sf3, "__builtin_ia32_andps", IX86_BUILTIN_ANDPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28347 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_andnotv4sf3, "__builtin_ia32_andnps", IX86_BUILTIN_ANDNPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28348 { OPTION_MASK_ISA_SSE, CODE_FOR_iorv4sf3, "__builtin_ia32_orps", IX86_BUILTIN_ORPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28349 { OPTION_MASK_ISA_SSE, CODE_FOR_xorv4sf3, "__builtin_ia32_xorps", IX86_BUILTIN_XORPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28351 { OPTION_MASK_ISA_SSE, CODE_FOR_copysignv4sf3, "__builtin_ia32_copysignps", IX86_BUILTIN_CPYSGNPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28353 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_movss, "__builtin_ia32_movss", IX86_BUILTIN_MOVSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28354 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_movhlps_exp, "__builtin_ia32_movhlps", IX86_BUILTIN_MOVHLPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28355 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_movlhps_exp, "__builtin_ia32_movlhps", IX86_BUILTIN_MOVLHPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28356 { OPTION_MASK_ISA_SSE, CODE_FOR_vec_interleave_highv4sf, "__builtin_ia32_unpckhps", IX86_BUILTIN_UNPCKHPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28357 { OPTION_MASK_ISA_SSE, CODE_FOR_vec_interleave_lowv4sf, "__builtin_ia32_unpcklps", IX86_BUILTIN_UNPCKLPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28359 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_cvtpi2ps, "__builtin_ia32_cvtpi2ps", IX86_BUILTIN_CVTPI2PS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V2SI },
28360 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_cvtsi2ss, "__builtin_ia32_cvtsi2ss", IX86_BUILTIN_CVTSI2SS, UNKNOWN, (int) V4SF_FTYPE_V4SF_SI },
28361 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_64BIT, CODE_FOR_sse_cvtsi2ssq, "__builtin_ia32_cvtsi642ss", IX86_BUILTIN_CVTSI642SS, UNKNOWN, V4SF_FTYPE_V4SF_DI },
28363 { OPTION_MASK_ISA_SSE, CODE_FOR_rsqrtsf2, "__builtin_ia32_rsqrtf", IX86_BUILTIN_RSQRTF, UNKNOWN, (int) FLOAT_FTYPE_FLOAT },
28365 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmsqrtv4sf2, "__builtin_ia32_sqrtss", IX86_BUILTIN_SQRTSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_VEC_MERGE },
28366 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmrsqrtv4sf2, "__builtin_ia32_rsqrtss", IX86_BUILTIN_RSQRTSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_VEC_MERGE },
28367 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmrcpv4sf2, "__builtin_ia32_rcpss", IX86_BUILTIN_RCPSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_VEC_MERGE },
28369 { OPTION_MASK_ISA_SSE, CODE_FOR_abstf2, 0, IX86_BUILTIN_FABSQ, UNKNOWN, (int) FLOAT128_FTYPE_FLOAT128 },
28370 { OPTION_MASK_ISA_SSE, CODE_FOR_copysigntf3, 0, IX86_BUILTIN_COPYSIGNQ, UNKNOWN, (int) FLOAT128_FTYPE_FLOAT128_FLOAT128 },
28372 /* SSE MMX or 3Dnow!A */
28373 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_uavgv8qi3, "__builtin_ia32_pavgb", IX86_BUILTIN_PAVGB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
28374 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_uavgv4hi3, "__builtin_ia32_pavgw", IX86_BUILTIN_PAVGW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
28375 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_umulv4hi3_highpart, "__builtin_ia32_pmulhuw", IX86_BUILTIN_PMULHUW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
28377 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_umaxv8qi3, "__builtin_ia32_pmaxub", IX86_BUILTIN_PMAXUB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
28378 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_smaxv4hi3, "__builtin_ia32_pmaxsw", IX86_BUILTIN_PMAXSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
28379 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_uminv8qi3, "__builtin_ia32_pminub", IX86_BUILTIN_PMINUB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
28380 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_sminv4hi3, "__builtin_ia32_pminsw", IX86_BUILTIN_PMINSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
28382 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_psadbw, "__builtin_ia32_psadbw", IX86_BUILTIN_PSADBW, UNKNOWN, (int) V1DI_FTYPE_V8QI_V8QI },
28383 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_pmovmskb, "__builtin_ia32_pmovmskb", IX86_BUILTIN_PMOVMSKB, UNKNOWN, (int) INT_FTYPE_V8QI },
28385 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_pshufw, "__builtin_ia32_pshufw", IX86_BUILTIN_PSHUFW, UNKNOWN, (int) V4HI_FTYPE_V4HI_INT },
28387 /* SSE2 */
28388 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_shufpd, "__builtin_ia32_shufpd", IX86_BUILTIN_SHUFPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
28390 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_movmskpd, "__builtin_ia32_movmskpd", IX86_BUILTIN_MOVMSKPD, UNKNOWN, (int) INT_FTYPE_V2DF },
28391 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_pmovmskb, "__builtin_ia32_pmovmskb128", IX86_BUILTIN_PMOVMSKB128, UNKNOWN, (int) INT_FTYPE_V16QI },
28392 { OPTION_MASK_ISA_SSE2, CODE_FOR_sqrtv2df2, "__builtin_ia32_sqrtpd", IX86_BUILTIN_SQRTPD, UNKNOWN, (int) V2DF_FTYPE_V2DF },
28393 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtdq2pd, "__builtin_ia32_cvtdq2pd", IX86_BUILTIN_CVTDQ2PD, UNKNOWN, (int) V2DF_FTYPE_V4SI },
28394 { OPTION_MASK_ISA_SSE2, CODE_FOR_floatv4siv4sf2, "__builtin_ia32_cvtdq2ps", IX86_BUILTIN_CVTDQ2PS, UNKNOWN, (int) V4SF_FTYPE_V4SI },
28396 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtpd2dq, "__builtin_ia32_cvtpd2dq", IX86_BUILTIN_CVTPD2DQ, UNKNOWN, (int) V4SI_FTYPE_V2DF },
28397 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtpd2pi, "__builtin_ia32_cvtpd2pi", IX86_BUILTIN_CVTPD2PI, UNKNOWN, (int) V2SI_FTYPE_V2DF },
28398 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtpd2ps, "__builtin_ia32_cvtpd2ps", IX86_BUILTIN_CVTPD2PS, UNKNOWN, (int) V4SF_FTYPE_V2DF },
28399 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvttpd2dq, "__builtin_ia32_cvttpd2dq", IX86_BUILTIN_CVTTPD2DQ, UNKNOWN, (int) V4SI_FTYPE_V2DF },
28400 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvttpd2pi, "__builtin_ia32_cvttpd2pi", IX86_BUILTIN_CVTTPD2PI, UNKNOWN, (int) V2SI_FTYPE_V2DF },
28402 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtpi2pd, "__builtin_ia32_cvtpi2pd", IX86_BUILTIN_CVTPI2PD, UNKNOWN, (int) V2DF_FTYPE_V2SI },
28404 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtsd2si, "__builtin_ia32_cvtsd2si", IX86_BUILTIN_CVTSD2SI, UNKNOWN, (int) INT_FTYPE_V2DF },
28405 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvttsd2si, "__builtin_ia32_cvttsd2si", IX86_BUILTIN_CVTTSD2SI, UNKNOWN, (int) INT_FTYPE_V2DF },
28406 { OPTION_MASK_ISA_SSE2 | OPTION_MASK_ISA_64BIT, CODE_FOR_sse2_cvtsd2siq, "__builtin_ia32_cvtsd2si64", IX86_BUILTIN_CVTSD2SI64, UNKNOWN, (int) INT64_FTYPE_V2DF },
28407 { OPTION_MASK_ISA_SSE2 | OPTION_MASK_ISA_64BIT, CODE_FOR_sse2_cvttsd2siq, "__builtin_ia32_cvttsd2si64", IX86_BUILTIN_CVTTSD2SI64, UNKNOWN, (int) INT64_FTYPE_V2DF },
28409 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_fix_notruncv4sfv4si, "__builtin_ia32_cvtps2dq", IX86_BUILTIN_CVTPS2DQ, UNKNOWN, (int) V4SI_FTYPE_V4SF },
28410 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtps2pd, "__builtin_ia32_cvtps2pd", IX86_BUILTIN_CVTPS2PD, UNKNOWN, (int) V2DF_FTYPE_V4SF },
28411 { OPTION_MASK_ISA_SSE2, CODE_FOR_fix_truncv4sfv4si2, "__builtin_ia32_cvttps2dq", IX86_BUILTIN_CVTTPS2DQ, UNKNOWN, (int) V4SI_FTYPE_V4SF },
28413 { OPTION_MASK_ISA_SSE2, CODE_FOR_addv2df3, "__builtin_ia32_addpd", IX86_BUILTIN_ADDPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28414 { OPTION_MASK_ISA_SSE2, CODE_FOR_subv2df3, "__builtin_ia32_subpd", IX86_BUILTIN_SUBPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28415 { OPTION_MASK_ISA_SSE2, CODE_FOR_mulv2df3, "__builtin_ia32_mulpd", IX86_BUILTIN_MULPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28416 { OPTION_MASK_ISA_SSE2, CODE_FOR_divv2df3, "__builtin_ia32_divpd", IX86_BUILTIN_DIVPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28417 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmaddv2df3, "__builtin_ia32_addsd", IX86_BUILTIN_ADDSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28418 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmsubv2df3, "__builtin_ia32_subsd", IX86_BUILTIN_SUBSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28419 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmulv2df3, "__builtin_ia32_mulsd", IX86_BUILTIN_MULSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28420 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmdivv2df3, "__builtin_ia32_divsd", IX86_BUILTIN_DIVSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28422 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpeqpd", IX86_BUILTIN_CMPEQPD, EQ, (int) V2DF_FTYPE_V2DF_V2DF },
28423 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpltpd", IX86_BUILTIN_CMPLTPD, LT, (int) V2DF_FTYPE_V2DF_V2DF },
28424 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmplepd", IX86_BUILTIN_CMPLEPD, LE, (int) V2DF_FTYPE_V2DF_V2DF },
28425 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpgtpd", IX86_BUILTIN_CMPGTPD, LT, (int) V2DF_FTYPE_V2DF_V2DF_SWAP },
28426 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpgepd", IX86_BUILTIN_CMPGEPD, LE, (int) V2DF_FTYPE_V2DF_V2DF_SWAP},
28427 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpunordpd", IX86_BUILTIN_CMPUNORDPD, UNORDERED, (int) V2DF_FTYPE_V2DF_V2DF },
28428 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpneqpd", IX86_BUILTIN_CMPNEQPD, NE, (int) V2DF_FTYPE_V2DF_V2DF },
28429 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpnltpd", IX86_BUILTIN_CMPNLTPD, UNGE, (int) V2DF_FTYPE_V2DF_V2DF },
28430 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpnlepd", IX86_BUILTIN_CMPNLEPD, UNGT, (int) V2DF_FTYPE_V2DF_V2DF },
28431 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpngtpd", IX86_BUILTIN_CMPNGTPD, UNGE, (int) V2DF_FTYPE_V2DF_V2DF_SWAP },
28432 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpngepd", IX86_BUILTIN_CMPNGEPD, UNGT, (int) V2DF_FTYPE_V2DF_V2DF_SWAP },
28433 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpordpd", IX86_BUILTIN_CMPORDPD, ORDERED, (int) V2DF_FTYPE_V2DF_V2DF },
28434 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmpeqsd", IX86_BUILTIN_CMPEQSD, EQ, (int) V2DF_FTYPE_V2DF_V2DF },
28435 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmpltsd", IX86_BUILTIN_CMPLTSD, LT, (int) V2DF_FTYPE_V2DF_V2DF },
28436 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmplesd", IX86_BUILTIN_CMPLESD, LE, (int) V2DF_FTYPE_V2DF_V2DF },
28437 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmpunordsd", IX86_BUILTIN_CMPUNORDSD, UNORDERED, (int) V2DF_FTYPE_V2DF_V2DF },
28438 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmpneqsd", IX86_BUILTIN_CMPNEQSD, NE, (int) V2DF_FTYPE_V2DF_V2DF },
28439 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmpnltsd", IX86_BUILTIN_CMPNLTSD, UNGE, (int) V2DF_FTYPE_V2DF_V2DF },
28440 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmpnlesd", IX86_BUILTIN_CMPNLESD, UNGT, (int) V2DF_FTYPE_V2DF_V2DF },
28441 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmpordsd", IX86_BUILTIN_CMPORDSD, ORDERED, (int) V2DF_FTYPE_V2DF_V2DF },
28443 { OPTION_MASK_ISA_SSE2, CODE_FOR_sminv2df3, "__builtin_ia32_minpd", IX86_BUILTIN_MINPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28444 { OPTION_MASK_ISA_SSE2, CODE_FOR_smaxv2df3, "__builtin_ia32_maxpd", IX86_BUILTIN_MAXPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28445 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmsminv2df3, "__builtin_ia32_minsd", IX86_BUILTIN_MINSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28446 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmsmaxv2df3, "__builtin_ia32_maxsd", IX86_BUILTIN_MAXSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28448 { OPTION_MASK_ISA_SSE2, CODE_FOR_andv2df3, "__builtin_ia32_andpd", IX86_BUILTIN_ANDPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28449 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_andnotv2df3, "__builtin_ia32_andnpd", IX86_BUILTIN_ANDNPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28450 { OPTION_MASK_ISA_SSE2, CODE_FOR_iorv2df3, "__builtin_ia32_orpd", IX86_BUILTIN_ORPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28451 { OPTION_MASK_ISA_SSE2, CODE_FOR_xorv2df3, "__builtin_ia32_xorpd", IX86_BUILTIN_XORPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28453 { OPTION_MASK_ISA_SSE2, CODE_FOR_copysignv2df3, "__builtin_ia32_copysignpd", IX86_BUILTIN_CPYSGNPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28455 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_movsd, "__builtin_ia32_movsd", IX86_BUILTIN_MOVSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28456 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_highv2df, "__builtin_ia32_unpckhpd", IX86_BUILTIN_UNPCKHPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28457 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_lowv2df, "__builtin_ia32_unpcklpd", IX86_BUILTIN_UNPCKLPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28459 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_pack_sfix_v2df, "__builtin_ia32_vec_pack_sfix", IX86_BUILTIN_VEC_PACK_SFIX, UNKNOWN, (int) V4SI_FTYPE_V2DF_V2DF },
28461 { OPTION_MASK_ISA_SSE2, CODE_FOR_addv16qi3, "__builtin_ia32_paddb128", IX86_BUILTIN_PADDB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
28462 { OPTION_MASK_ISA_SSE2, CODE_FOR_addv8hi3, "__builtin_ia32_paddw128", IX86_BUILTIN_PADDW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28463 { OPTION_MASK_ISA_SSE2, CODE_FOR_addv4si3, "__builtin_ia32_paddd128", IX86_BUILTIN_PADDD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
28464 { OPTION_MASK_ISA_SSE2, CODE_FOR_addv2di3, "__builtin_ia32_paddq128", IX86_BUILTIN_PADDQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
28465 { OPTION_MASK_ISA_SSE2, CODE_FOR_subv16qi3, "__builtin_ia32_psubb128", IX86_BUILTIN_PSUBB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
28466 { OPTION_MASK_ISA_SSE2, CODE_FOR_subv8hi3, "__builtin_ia32_psubw128", IX86_BUILTIN_PSUBW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28467 { OPTION_MASK_ISA_SSE2, CODE_FOR_subv4si3, "__builtin_ia32_psubd128", IX86_BUILTIN_PSUBD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
28468 { OPTION_MASK_ISA_SSE2, CODE_FOR_subv2di3, "__builtin_ia32_psubq128", IX86_BUILTIN_PSUBQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
28470 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ssaddv16qi3, "__builtin_ia32_paddsb128", IX86_BUILTIN_PADDSB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
28471 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ssaddv8hi3, "__builtin_ia32_paddsw128", IX86_BUILTIN_PADDSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28472 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_sssubv16qi3, "__builtin_ia32_psubsb128", IX86_BUILTIN_PSUBSB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
28473 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_sssubv8hi3, "__builtin_ia32_psubsw128", IX86_BUILTIN_PSUBSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28474 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_usaddv16qi3, "__builtin_ia32_paddusb128", IX86_BUILTIN_PADDUSB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
28475 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_usaddv8hi3, "__builtin_ia32_paddusw128", IX86_BUILTIN_PADDUSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28476 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ussubv16qi3, "__builtin_ia32_psubusb128", IX86_BUILTIN_PSUBUSB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
28477 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ussubv8hi3, "__builtin_ia32_psubusw128", IX86_BUILTIN_PSUBUSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28479 { OPTION_MASK_ISA_SSE2, CODE_FOR_mulv8hi3, "__builtin_ia32_pmullw128", IX86_BUILTIN_PMULLW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28480 { OPTION_MASK_ISA_SSE2, CODE_FOR_smulv8hi3_highpart, "__builtin_ia32_pmulhw128", IX86_BUILTIN_PMULHW128, UNKNOWN,(int) V8HI_FTYPE_V8HI_V8HI },
28482 { OPTION_MASK_ISA_SSE2, CODE_FOR_andv2di3, "__builtin_ia32_pand128", IX86_BUILTIN_PAND128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
28483 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_andnotv2di3, "__builtin_ia32_pandn128", IX86_BUILTIN_PANDN128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
28484 { OPTION_MASK_ISA_SSE2, CODE_FOR_iorv2di3, "__builtin_ia32_por128", IX86_BUILTIN_POR128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
28485 { OPTION_MASK_ISA_SSE2, CODE_FOR_xorv2di3, "__builtin_ia32_pxor128", IX86_BUILTIN_PXOR128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
28487 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_uavgv16qi3, "__builtin_ia32_pavgb128", IX86_BUILTIN_PAVGB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
28488 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_uavgv8hi3, "__builtin_ia32_pavgw128", IX86_BUILTIN_PAVGW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28490 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_eqv16qi3, "__builtin_ia32_pcmpeqb128", IX86_BUILTIN_PCMPEQB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
28491 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_eqv8hi3, "__builtin_ia32_pcmpeqw128", IX86_BUILTIN_PCMPEQW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28492 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_eqv4si3, "__builtin_ia32_pcmpeqd128", IX86_BUILTIN_PCMPEQD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
28493 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_gtv16qi3, "__builtin_ia32_pcmpgtb128", IX86_BUILTIN_PCMPGTB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
28494 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_gtv8hi3, "__builtin_ia32_pcmpgtw128", IX86_BUILTIN_PCMPGTW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28495 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_gtv4si3, "__builtin_ia32_pcmpgtd128", IX86_BUILTIN_PCMPGTD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
28497 { OPTION_MASK_ISA_SSE2, CODE_FOR_umaxv16qi3, "__builtin_ia32_pmaxub128", IX86_BUILTIN_PMAXUB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
28498 { OPTION_MASK_ISA_SSE2, CODE_FOR_smaxv8hi3, "__builtin_ia32_pmaxsw128", IX86_BUILTIN_PMAXSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28499 { OPTION_MASK_ISA_SSE2, CODE_FOR_uminv16qi3, "__builtin_ia32_pminub128", IX86_BUILTIN_PMINUB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
28500 { OPTION_MASK_ISA_SSE2, CODE_FOR_sminv8hi3, "__builtin_ia32_pminsw128", IX86_BUILTIN_PMINSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28502 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_highv16qi, "__builtin_ia32_punpckhbw128", IX86_BUILTIN_PUNPCKHBW128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
28503 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_highv8hi, "__builtin_ia32_punpckhwd128", IX86_BUILTIN_PUNPCKHWD128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28504 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_highv4si, "__builtin_ia32_punpckhdq128", IX86_BUILTIN_PUNPCKHDQ128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
28505 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_highv2di, "__builtin_ia32_punpckhqdq128", IX86_BUILTIN_PUNPCKHQDQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
28506 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_lowv16qi, "__builtin_ia32_punpcklbw128", IX86_BUILTIN_PUNPCKLBW128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
28507 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_lowv8hi, "__builtin_ia32_punpcklwd128", IX86_BUILTIN_PUNPCKLWD128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28508 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_lowv4si, "__builtin_ia32_punpckldq128", IX86_BUILTIN_PUNPCKLDQ128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
28509 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_interleave_lowv2di, "__builtin_ia32_punpcklqdq128", IX86_BUILTIN_PUNPCKLQDQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
28511 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_packsswb, "__builtin_ia32_packsswb128", IX86_BUILTIN_PACKSSWB128, UNKNOWN, (int) V16QI_FTYPE_V8HI_V8HI },
28512 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_packssdw, "__builtin_ia32_packssdw128", IX86_BUILTIN_PACKSSDW128, UNKNOWN, (int) V8HI_FTYPE_V4SI_V4SI },
28513 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_packuswb, "__builtin_ia32_packuswb128", IX86_BUILTIN_PACKUSWB128, UNKNOWN, (int) V16QI_FTYPE_V8HI_V8HI },
28515 { OPTION_MASK_ISA_SSE2, CODE_FOR_umulv8hi3_highpart, "__builtin_ia32_pmulhuw128", IX86_BUILTIN_PMULHUW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28516 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_psadbw, "__builtin_ia32_psadbw128", IX86_BUILTIN_PSADBW128, UNKNOWN, (int) V2DI_FTYPE_V16QI_V16QI },
28518 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_umulv1siv1di3, "__builtin_ia32_pmuludq", IX86_BUILTIN_PMULUDQ, UNKNOWN, (int) V1DI_FTYPE_V2SI_V2SI },
28519 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_widen_umult_even_v4si, "__builtin_ia32_pmuludq128", IX86_BUILTIN_PMULUDQ128, UNKNOWN, (int) V2DI_FTYPE_V4SI_V4SI },
28521 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_pmaddwd, "__builtin_ia32_pmaddwd128", IX86_BUILTIN_PMADDWD128, UNKNOWN, (int) V4SI_FTYPE_V8HI_V8HI },
28523 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtsi2sd, "__builtin_ia32_cvtsi2sd", IX86_BUILTIN_CVTSI2SD, UNKNOWN, (int) V2DF_FTYPE_V2DF_SI },
28524 { OPTION_MASK_ISA_SSE2 | OPTION_MASK_ISA_64BIT, CODE_FOR_sse2_cvtsi2sdq, "__builtin_ia32_cvtsi642sd", IX86_BUILTIN_CVTSI642SD, UNKNOWN, (int) V2DF_FTYPE_V2DF_DI },
28525 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtsd2ss, "__builtin_ia32_cvtsd2ss", IX86_BUILTIN_CVTSD2SS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V2DF },
28526 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtss2sd, "__builtin_ia32_cvtss2sd", IX86_BUILTIN_CVTSS2SD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V4SF },
28528 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ashlv1ti3, "__builtin_ia32_pslldqi128", IX86_BUILTIN_PSLLDQI128, UNKNOWN, (int) V2DI_FTYPE_V2DI_INT_CONVERT },
28529 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashlv8hi3, "__builtin_ia32_psllwi128", IX86_BUILTIN_PSLLWI128, UNKNOWN, (int) V8HI_FTYPE_V8HI_SI_COUNT },
28530 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashlv4si3, "__builtin_ia32_pslldi128", IX86_BUILTIN_PSLLDI128, UNKNOWN, (int) V4SI_FTYPE_V4SI_SI_COUNT },
28531 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashlv2di3, "__builtin_ia32_psllqi128", IX86_BUILTIN_PSLLQI128, UNKNOWN, (int) V2DI_FTYPE_V2DI_SI_COUNT },
28532 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashlv8hi3, "__builtin_ia32_psllw128", IX86_BUILTIN_PSLLW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_COUNT },
28533 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashlv4si3, "__builtin_ia32_pslld128", IX86_BUILTIN_PSLLD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_COUNT },
28534 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashlv2di3, "__builtin_ia32_psllq128", IX86_BUILTIN_PSLLQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_COUNT },
28536 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_lshrv1ti3, "__builtin_ia32_psrldqi128", IX86_BUILTIN_PSRLDQI128, UNKNOWN, (int) V2DI_FTYPE_V2DI_INT_CONVERT },
28537 { OPTION_MASK_ISA_SSE2, CODE_FOR_lshrv8hi3, "__builtin_ia32_psrlwi128", IX86_BUILTIN_PSRLWI128, UNKNOWN, (int) V8HI_FTYPE_V8HI_SI_COUNT },
28538 { OPTION_MASK_ISA_SSE2, CODE_FOR_lshrv4si3, "__builtin_ia32_psrldi128", IX86_BUILTIN_PSRLDI128, UNKNOWN, (int) V4SI_FTYPE_V4SI_SI_COUNT },
28539 { OPTION_MASK_ISA_SSE2, CODE_FOR_lshrv2di3, "__builtin_ia32_psrlqi128", IX86_BUILTIN_PSRLQI128, UNKNOWN, (int) V2DI_FTYPE_V2DI_SI_COUNT },
28540 { OPTION_MASK_ISA_SSE2, CODE_FOR_lshrv8hi3, "__builtin_ia32_psrlw128", IX86_BUILTIN_PSRLW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_COUNT },
28541 { OPTION_MASK_ISA_SSE2, CODE_FOR_lshrv4si3, "__builtin_ia32_psrld128", IX86_BUILTIN_PSRLD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_COUNT },
28542 { OPTION_MASK_ISA_SSE2, CODE_FOR_lshrv2di3, "__builtin_ia32_psrlq128", IX86_BUILTIN_PSRLQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_COUNT },
28544 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashrv8hi3, "__builtin_ia32_psrawi128", IX86_BUILTIN_PSRAWI128, UNKNOWN, (int) V8HI_FTYPE_V8HI_SI_COUNT },
28545 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashrv4si3, "__builtin_ia32_psradi128", IX86_BUILTIN_PSRADI128, UNKNOWN, (int) V4SI_FTYPE_V4SI_SI_COUNT },
28546 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashrv8hi3, "__builtin_ia32_psraw128", IX86_BUILTIN_PSRAW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_COUNT },
28547 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashrv4si3, "__builtin_ia32_psrad128", IX86_BUILTIN_PSRAD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_COUNT },
28549 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_pshufd, "__builtin_ia32_pshufd", IX86_BUILTIN_PSHUFD, UNKNOWN, (int) V4SI_FTYPE_V4SI_INT },
28550 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_pshuflw, "__builtin_ia32_pshuflw", IX86_BUILTIN_PSHUFLW, UNKNOWN, (int) V8HI_FTYPE_V8HI_INT },
28551 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_pshufhw, "__builtin_ia32_pshufhw", IX86_BUILTIN_PSHUFHW, UNKNOWN, (int) V8HI_FTYPE_V8HI_INT },
28553 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmsqrtv2df2, "__builtin_ia32_sqrtsd", IX86_BUILTIN_SQRTSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_VEC_MERGE },
28555 { OPTION_MASK_ISA_SSE, CODE_FOR_sse2_movq128, "__builtin_ia32_movq128", IX86_BUILTIN_MOVQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI },
28557 /* SSE2 MMX */
28558 { OPTION_MASK_ISA_SSE2, CODE_FOR_mmx_addv1di3, "__builtin_ia32_paddq", IX86_BUILTIN_PADDQ, UNKNOWN, (int) V1DI_FTYPE_V1DI_V1DI },
28559 { OPTION_MASK_ISA_SSE2, CODE_FOR_mmx_subv1di3, "__builtin_ia32_psubq", IX86_BUILTIN_PSUBQ, UNKNOWN, (int) V1DI_FTYPE_V1DI_V1DI },
28561 /* SSE3 */
28562 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_movshdup, "__builtin_ia32_movshdup", IX86_BUILTIN_MOVSHDUP, UNKNOWN, (int) V4SF_FTYPE_V4SF},
28563 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_movsldup, "__builtin_ia32_movsldup", IX86_BUILTIN_MOVSLDUP, UNKNOWN, (int) V4SF_FTYPE_V4SF },
28565 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_addsubv4sf3, "__builtin_ia32_addsubps", IX86_BUILTIN_ADDSUBPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28566 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_addsubv2df3, "__builtin_ia32_addsubpd", IX86_BUILTIN_ADDSUBPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28567 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_haddv4sf3, "__builtin_ia32_haddps", IX86_BUILTIN_HADDPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28568 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_haddv2df3, "__builtin_ia32_haddpd", IX86_BUILTIN_HADDPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28569 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_hsubv4sf3, "__builtin_ia32_hsubps", IX86_BUILTIN_HSUBPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
28570 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_hsubv2df3, "__builtin_ia32_hsubpd", IX86_BUILTIN_HSUBPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
28572 /* SSSE3 */
28573 { OPTION_MASK_ISA_SSSE3, CODE_FOR_absv16qi2, "__builtin_ia32_pabsb128", IX86_BUILTIN_PABSB128, UNKNOWN, (int) V16QI_FTYPE_V16QI },
28574 { OPTION_MASK_ISA_SSSE3, CODE_FOR_absv8qi2, "__builtin_ia32_pabsb", IX86_BUILTIN_PABSB, UNKNOWN, (int) V8QI_FTYPE_V8QI },
28575 { OPTION_MASK_ISA_SSSE3, CODE_FOR_absv8hi2, "__builtin_ia32_pabsw128", IX86_BUILTIN_PABSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI },
28576 { OPTION_MASK_ISA_SSSE3, CODE_FOR_absv4hi2, "__builtin_ia32_pabsw", IX86_BUILTIN_PABSW, UNKNOWN, (int) V4HI_FTYPE_V4HI },
28577 { OPTION_MASK_ISA_SSSE3, CODE_FOR_absv4si2, "__builtin_ia32_pabsd128", IX86_BUILTIN_PABSD128, UNKNOWN, (int) V4SI_FTYPE_V4SI },
28578 { OPTION_MASK_ISA_SSSE3, CODE_FOR_absv2si2, "__builtin_ia32_pabsd", IX86_BUILTIN_PABSD, UNKNOWN, (int) V2SI_FTYPE_V2SI },
28580 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phaddwv8hi3, "__builtin_ia32_phaddw128", IX86_BUILTIN_PHADDW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28581 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phaddwv4hi3, "__builtin_ia32_phaddw", IX86_BUILTIN_PHADDW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
28582 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phadddv4si3, "__builtin_ia32_phaddd128", IX86_BUILTIN_PHADDD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
28583 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phadddv2si3, "__builtin_ia32_phaddd", IX86_BUILTIN_PHADDD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
28584 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phaddswv8hi3, "__builtin_ia32_phaddsw128", IX86_BUILTIN_PHADDSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28585 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phaddswv4hi3, "__builtin_ia32_phaddsw", IX86_BUILTIN_PHADDSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
28586 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phsubwv8hi3, "__builtin_ia32_phsubw128", IX86_BUILTIN_PHSUBW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28587 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phsubwv4hi3, "__builtin_ia32_phsubw", IX86_BUILTIN_PHSUBW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
28588 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phsubdv4si3, "__builtin_ia32_phsubd128", IX86_BUILTIN_PHSUBD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
28589 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phsubdv2si3, "__builtin_ia32_phsubd", IX86_BUILTIN_PHSUBD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
28590 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phsubswv8hi3, "__builtin_ia32_phsubsw128", IX86_BUILTIN_PHSUBSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28591 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phsubswv4hi3, "__builtin_ia32_phsubsw", IX86_BUILTIN_PHSUBSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
28592 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_pmaddubsw128, "__builtin_ia32_pmaddubsw128", IX86_BUILTIN_PMADDUBSW128, UNKNOWN, (int) V8HI_FTYPE_V16QI_V16QI },
28593 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_pmaddubsw, "__builtin_ia32_pmaddubsw", IX86_BUILTIN_PMADDUBSW, UNKNOWN, (int) V4HI_FTYPE_V8QI_V8QI },
28594 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_pmulhrswv8hi3, "__builtin_ia32_pmulhrsw128", IX86_BUILTIN_PMULHRSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28595 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_pmulhrswv4hi3, "__builtin_ia32_pmulhrsw", IX86_BUILTIN_PMULHRSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
28596 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_pshufbv16qi3, "__builtin_ia32_pshufb128", IX86_BUILTIN_PSHUFB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
28597 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_pshufbv8qi3, "__builtin_ia32_pshufb", IX86_BUILTIN_PSHUFB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
28598 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_psignv16qi3, "__builtin_ia32_psignb128", IX86_BUILTIN_PSIGNB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
28599 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_psignv8qi3, "__builtin_ia32_psignb", IX86_BUILTIN_PSIGNB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
28600 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_psignv8hi3, "__builtin_ia32_psignw128", IX86_BUILTIN_PSIGNW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28601 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_psignv4hi3, "__builtin_ia32_psignw", IX86_BUILTIN_PSIGNW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
28602 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_psignv4si3, "__builtin_ia32_psignd128", IX86_BUILTIN_PSIGND128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
28603 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_psignv2si3, "__builtin_ia32_psignd", IX86_BUILTIN_PSIGND, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
28605 /* SSSE3. */
28606 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_palignrti, "__builtin_ia32_palignr128", IX86_BUILTIN_PALIGNR128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_INT_CONVERT },
28607 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_palignrdi, "__builtin_ia32_palignr", IX86_BUILTIN_PALIGNR, UNKNOWN, (int) V1DI_FTYPE_V1DI_V1DI_INT_CONVERT },
28609 /* SSE4.1 */
28610 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_blendpd, "__builtin_ia32_blendpd", IX86_BUILTIN_BLENDPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
28611 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_blendps, "__builtin_ia32_blendps", IX86_BUILTIN_BLENDPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
28612 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_blendvpd, "__builtin_ia32_blendvpd", IX86_BUILTIN_BLENDVPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_V2DF },
28613 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_blendvps, "__builtin_ia32_blendvps", IX86_BUILTIN_BLENDVPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_V4SF },
28614 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_dppd, "__builtin_ia32_dppd", IX86_BUILTIN_DPPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
28615 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_dpps, "__builtin_ia32_dpps", IX86_BUILTIN_DPPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
28616 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_insertps, "__builtin_ia32_insertps128", IX86_BUILTIN_INSERTPS128, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
28617 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_mpsadbw, "__builtin_ia32_mpsadbw128", IX86_BUILTIN_MPSADBW128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI_INT },
28618 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_pblendvb, "__builtin_ia32_pblendvb128", IX86_BUILTIN_PBLENDVB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI_V16QI },
28619 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_pblendw, "__builtin_ia32_pblendw128", IX86_BUILTIN_PBLENDW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_INT },
28621 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_sign_extendv8qiv8hi2, "__builtin_ia32_pmovsxbw128", IX86_BUILTIN_PMOVSXBW128, UNKNOWN, (int) V8HI_FTYPE_V16QI },
28622 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_sign_extendv4qiv4si2, "__builtin_ia32_pmovsxbd128", IX86_BUILTIN_PMOVSXBD128, UNKNOWN, (int) V4SI_FTYPE_V16QI },
28623 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_sign_extendv2qiv2di2, "__builtin_ia32_pmovsxbq128", IX86_BUILTIN_PMOVSXBQ128, UNKNOWN, (int) V2DI_FTYPE_V16QI },
28624 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_sign_extendv4hiv4si2, "__builtin_ia32_pmovsxwd128", IX86_BUILTIN_PMOVSXWD128, UNKNOWN, (int) V4SI_FTYPE_V8HI },
28625 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_sign_extendv2hiv2di2, "__builtin_ia32_pmovsxwq128", IX86_BUILTIN_PMOVSXWQ128, UNKNOWN, (int) V2DI_FTYPE_V8HI },
28626 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_sign_extendv2siv2di2, "__builtin_ia32_pmovsxdq128", IX86_BUILTIN_PMOVSXDQ128, UNKNOWN, (int) V2DI_FTYPE_V4SI },
28627 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_zero_extendv8qiv8hi2, "__builtin_ia32_pmovzxbw128", IX86_BUILTIN_PMOVZXBW128, UNKNOWN, (int) V8HI_FTYPE_V16QI },
28628 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_zero_extendv4qiv4si2, "__builtin_ia32_pmovzxbd128", IX86_BUILTIN_PMOVZXBD128, UNKNOWN, (int) V4SI_FTYPE_V16QI },
28629 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_zero_extendv2qiv2di2, "__builtin_ia32_pmovzxbq128", IX86_BUILTIN_PMOVZXBQ128, UNKNOWN, (int) V2DI_FTYPE_V16QI },
28630 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_zero_extendv4hiv4si2, "__builtin_ia32_pmovzxwd128", IX86_BUILTIN_PMOVZXWD128, UNKNOWN, (int) V4SI_FTYPE_V8HI },
28631 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_zero_extendv2hiv2di2, "__builtin_ia32_pmovzxwq128", IX86_BUILTIN_PMOVZXWQ128, UNKNOWN, (int) V2DI_FTYPE_V8HI },
28632 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_zero_extendv2siv2di2, "__builtin_ia32_pmovzxdq128", IX86_BUILTIN_PMOVZXDQ128, UNKNOWN, (int) V2DI_FTYPE_V4SI },
28633 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_phminposuw, "__builtin_ia32_phminposuw128", IX86_BUILTIN_PHMINPOSUW128, UNKNOWN, (int) V8HI_FTYPE_V8HI },
28635 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_packusdw, "__builtin_ia32_packusdw128", IX86_BUILTIN_PACKUSDW128, UNKNOWN, (int) V8HI_FTYPE_V4SI_V4SI },
28636 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_eqv2di3, "__builtin_ia32_pcmpeqq", IX86_BUILTIN_PCMPEQQ, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
28637 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_smaxv16qi3, "__builtin_ia32_pmaxsb128", IX86_BUILTIN_PMAXSB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
28638 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_smaxv4si3, "__builtin_ia32_pmaxsd128", IX86_BUILTIN_PMAXSD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
28639 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_umaxv4si3, "__builtin_ia32_pmaxud128", IX86_BUILTIN_PMAXUD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
28640 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_umaxv8hi3, "__builtin_ia32_pmaxuw128", IX86_BUILTIN_PMAXUW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28641 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sminv16qi3, "__builtin_ia32_pminsb128", IX86_BUILTIN_PMINSB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
28642 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sminv4si3, "__builtin_ia32_pminsd128", IX86_BUILTIN_PMINSD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
28643 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_uminv4si3, "__builtin_ia32_pminud128", IX86_BUILTIN_PMINUD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
28644 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_uminv8hi3, "__builtin_ia32_pminuw128", IX86_BUILTIN_PMINUW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
28645 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_mulv2siv2di3, "__builtin_ia32_pmuldq128", IX86_BUILTIN_PMULDQ128, UNKNOWN, (int) V2DI_FTYPE_V4SI_V4SI },
28646 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_mulv4si3, "__builtin_ia32_pmulld128", IX86_BUILTIN_PMULLD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
28648 /* SSE4.1 */
28649 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundpd, "__builtin_ia32_roundpd", IX86_BUILTIN_ROUNDPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_INT },
28650 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundps, "__builtin_ia32_roundps", IX86_BUILTIN_ROUNDPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_INT },
28651 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundsd, "__builtin_ia32_roundsd", IX86_BUILTIN_ROUNDSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
28652 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundss, "__builtin_ia32_roundss", IX86_BUILTIN_ROUNDSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
28654 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundpd, "__builtin_ia32_floorpd", IX86_BUILTIN_FLOORPD, (enum rtx_code) ROUND_FLOOR, (int) V2DF_FTYPE_V2DF_ROUND },
28655 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundpd, "__builtin_ia32_ceilpd", IX86_BUILTIN_CEILPD, (enum rtx_code) ROUND_CEIL, (int) V2DF_FTYPE_V2DF_ROUND },
28656 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundpd, "__builtin_ia32_truncpd", IX86_BUILTIN_TRUNCPD, (enum rtx_code) ROUND_TRUNC, (int) V2DF_FTYPE_V2DF_ROUND },
28657 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundpd, "__builtin_ia32_rintpd", IX86_BUILTIN_RINTPD, (enum rtx_code) ROUND_MXCSR, (int) V2DF_FTYPE_V2DF_ROUND },
28659 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundpd_vec_pack_sfix, "__builtin_ia32_floorpd_vec_pack_sfix", IX86_BUILTIN_FLOORPD_VEC_PACK_SFIX, (enum rtx_code) ROUND_FLOOR, (int) V4SI_FTYPE_V2DF_V2DF_ROUND },
28660 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundpd_vec_pack_sfix, "__builtin_ia32_ceilpd_vec_pack_sfix", IX86_BUILTIN_CEILPD_VEC_PACK_SFIX, (enum rtx_code) ROUND_CEIL, (int) V4SI_FTYPE_V2DF_V2DF_ROUND },
28662 { OPTION_MASK_ISA_ROUND, CODE_FOR_roundv2df2, "__builtin_ia32_roundpd_az", IX86_BUILTIN_ROUNDPD_AZ, UNKNOWN, (int) V2DF_FTYPE_V2DF },
28663 { OPTION_MASK_ISA_ROUND, CODE_FOR_roundv2df2_vec_pack_sfix, "__builtin_ia32_roundpd_az_vec_pack_sfix", IX86_BUILTIN_ROUNDPD_AZ_VEC_PACK_SFIX, UNKNOWN, (int) V4SI_FTYPE_V2DF_V2DF },
28665 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundps, "__builtin_ia32_floorps", IX86_BUILTIN_FLOORPS, (enum rtx_code) ROUND_FLOOR, (int) V4SF_FTYPE_V4SF_ROUND },
28666 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundps, "__builtin_ia32_ceilps", IX86_BUILTIN_CEILPS, (enum rtx_code) ROUND_CEIL, (int) V4SF_FTYPE_V4SF_ROUND },
28667 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundps, "__builtin_ia32_truncps", IX86_BUILTIN_TRUNCPS, (enum rtx_code) ROUND_TRUNC, (int) V4SF_FTYPE_V4SF_ROUND },
28668 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundps, "__builtin_ia32_rintps", IX86_BUILTIN_RINTPS, (enum rtx_code) ROUND_MXCSR, (int) V4SF_FTYPE_V4SF_ROUND },
28670 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundps_sfix, "__builtin_ia32_floorps_sfix", IX86_BUILTIN_FLOORPS_SFIX, (enum rtx_code) ROUND_FLOOR, (int) V4SI_FTYPE_V4SF_ROUND },
28671 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundps_sfix, "__builtin_ia32_ceilps_sfix", IX86_BUILTIN_CEILPS_SFIX, (enum rtx_code) ROUND_CEIL, (int) V4SI_FTYPE_V4SF_ROUND },
28673 { OPTION_MASK_ISA_ROUND, CODE_FOR_roundv4sf2, "__builtin_ia32_roundps_az", IX86_BUILTIN_ROUNDPS_AZ, UNKNOWN, (int) V4SF_FTYPE_V4SF },
28674 { OPTION_MASK_ISA_ROUND, CODE_FOR_roundv4sf2_sfix, "__builtin_ia32_roundps_az_sfix", IX86_BUILTIN_ROUNDPS_AZ_SFIX, UNKNOWN, (int) V4SI_FTYPE_V4SF },
28676 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_ptest, "__builtin_ia32_ptestz128", IX86_BUILTIN_PTESTZ, EQ, (int) INT_FTYPE_V2DI_V2DI_PTEST },
28677 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_ptest, "__builtin_ia32_ptestc128", IX86_BUILTIN_PTESTC, LTU, (int) INT_FTYPE_V2DI_V2DI_PTEST },
28678 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_ptest, "__builtin_ia32_ptestnzc128", IX86_BUILTIN_PTESTNZC, GTU, (int) INT_FTYPE_V2DI_V2DI_PTEST },
28680 /* SSE4.2 */
28681 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_gtv2di3, "__builtin_ia32_pcmpgtq", IX86_BUILTIN_PCMPGTQ, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
28682 { OPTION_MASK_ISA_SSE4_2 | OPTION_MASK_ISA_CRC32, CODE_FOR_sse4_2_crc32qi, "__builtin_ia32_crc32qi", IX86_BUILTIN_CRC32QI, UNKNOWN, (int) UINT_FTYPE_UINT_UCHAR },
28683 { OPTION_MASK_ISA_SSE4_2 | OPTION_MASK_ISA_CRC32, CODE_FOR_sse4_2_crc32hi, "__builtin_ia32_crc32hi", IX86_BUILTIN_CRC32HI, UNKNOWN, (int) UINT_FTYPE_UINT_USHORT },
28684 { OPTION_MASK_ISA_SSE4_2 | OPTION_MASK_ISA_CRC32, CODE_FOR_sse4_2_crc32si, "__builtin_ia32_crc32si", IX86_BUILTIN_CRC32SI, UNKNOWN, (int) UINT_FTYPE_UINT_UINT },
28685 { OPTION_MASK_ISA_SSE4_2 | OPTION_MASK_ISA_CRC32 | OPTION_MASK_ISA_64BIT, CODE_FOR_sse4_2_crc32di, "__builtin_ia32_crc32di", IX86_BUILTIN_CRC32DI, UNKNOWN, (int) UINT64_FTYPE_UINT64_UINT64 },
28687 /* SSE4A */
28688 { OPTION_MASK_ISA_SSE4A, CODE_FOR_sse4a_extrqi, "__builtin_ia32_extrqi", IX86_BUILTIN_EXTRQI, UNKNOWN, (int) V2DI_FTYPE_V2DI_UINT_UINT },
28689 { OPTION_MASK_ISA_SSE4A, CODE_FOR_sse4a_extrq, "__builtin_ia32_extrq", IX86_BUILTIN_EXTRQ, UNKNOWN, (int) V2DI_FTYPE_V2DI_V16QI },
28690 { OPTION_MASK_ISA_SSE4A, CODE_FOR_sse4a_insertqi, "__builtin_ia32_insertqi", IX86_BUILTIN_INSERTQI, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_UINT_UINT },
28691 { OPTION_MASK_ISA_SSE4A, CODE_FOR_sse4a_insertq, "__builtin_ia32_insertq", IX86_BUILTIN_INSERTQ, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
28693 /* AES */
28694 { OPTION_MASK_ISA_SSE2, CODE_FOR_aeskeygenassist, 0, IX86_BUILTIN_AESKEYGENASSIST128, UNKNOWN, (int) V2DI_FTYPE_V2DI_INT },
28695 { OPTION_MASK_ISA_SSE2, CODE_FOR_aesimc, 0, IX86_BUILTIN_AESIMC128, UNKNOWN, (int) V2DI_FTYPE_V2DI },
28697 { OPTION_MASK_ISA_SSE2, CODE_FOR_aesenc, 0, IX86_BUILTIN_AESENC128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
28698 { OPTION_MASK_ISA_SSE2, CODE_FOR_aesenclast, 0, IX86_BUILTIN_AESENCLAST128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
28699 { OPTION_MASK_ISA_SSE2, CODE_FOR_aesdec, 0, IX86_BUILTIN_AESDEC128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
28700 { OPTION_MASK_ISA_SSE2, CODE_FOR_aesdeclast, 0, IX86_BUILTIN_AESDECLAST128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
28702 /* PCLMUL */
28703 { OPTION_MASK_ISA_SSE2, CODE_FOR_pclmulqdq, 0, IX86_BUILTIN_PCLMULQDQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_INT },
28705 /* AVX */
28706 { OPTION_MASK_ISA_AVX, CODE_FOR_addv4df3, "__builtin_ia32_addpd256", IX86_BUILTIN_ADDPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
28707 { OPTION_MASK_ISA_AVX, CODE_FOR_addv8sf3, "__builtin_ia32_addps256", IX86_BUILTIN_ADDPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
28708 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_addsubv4df3, "__builtin_ia32_addsubpd256", IX86_BUILTIN_ADDSUBPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
28709 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_addsubv8sf3, "__builtin_ia32_addsubps256", IX86_BUILTIN_ADDSUBPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
28710 { OPTION_MASK_ISA_AVX, CODE_FOR_andv4df3, "__builtin_ia32_andpd256", IX86_BUILTIN_ANDPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
28711 { OPTION_MASK_ISA_AVX, CODE_FOR_andv8sf3, "__builtin_ia32_andps256", IX86_BUILTIN_ANDPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
28712 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_andnotv4df3, "__builtin_ia32_andnpd256", IX86_BUILTIN_ANDNPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
28713 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_andnotv8sf3, "__builtin_ia32_andnps256", IX86_BUILTIN_ANDNPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
28714 { OPTION_MASK_ISA_AVX, CODE_FOR_divv4df3, "__builtin_ia32_divpd256", IX86_BUILTIN_DIVPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
28715 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_divv8sf3, "__builtin_ia32_divps256", IX86_BUILTIN_DIVPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
28716 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_haddv4df3, "__builtin_ia32_haddpd256", IX86_BUILTIN_HADDPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
28717 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_hsubv8sf3, "__builtin_ia32_hsubps256", IX86_BUILTIN_HSUBPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
28718 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_hsubv4df3, "__builtin_ia32_hsubpd256", IX86_BUILTIN_HSUBPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
28719 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_haddv8sf3, "__builtin_ia32_haddps256", IX86_BUILTIN_HADDPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
28720 { OPTION_MASK_ISA_AVX, CODE_FOR_smaxv4df3, "__builtin_ia32_maxpd256", IX86_BUILTIN_MAXPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
28721 { OPTION_MASK_ISA_AVX, CODE_FOR_smaxv8sf3, "__builtin_ia32_maxps256", IX86_BUILTIN_MAXPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
28722 { OPTION_MASK_ISA_AVX, CODE_FOR_sminv4df3, "__builtin_ia32_minpd256", IX86_BUILTIN_MINPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
28723 { OPTION_MASK_ISA_AVX, CODE_FOR_sminv8sf3, "__builtin_ia32_minps256", IX86_BUILTIN_MINPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
28724 { OPTION_MASK_ISA_AVX, CODE_FOR_mulv4df3, "__builtin_ia32_mulpd256", IX86_BUILTIN_MULPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
28725 { OPTION_MASK_ISA_AVX, CODE_FOR_mulv8sf3, "__builtin_ia32_mulps256", IX86_BUILTIN_MULPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
28726 { OPTION_MASK_ISA_AVX, CODE_FOR_iorv4df3, "__builtin_ia32_orpd256", IX86_BUILTIN_ORPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
28727 { OPTION_MASK_ISA_AVX, CODE_FOR_iorv8sf3, "__builtin_ia32_orps256", IX86_BUILTIN_ORPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
28728 { OPTION_MASK_ISA_AVX, CODE_FOR_subv4df3, "__builtin_ia32_subpd256", IX86_BUILTIN_SUBPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
28729 { OPTION_MASK_ISA_AVX, CODE_FOR_subv8sf3, "__builtin_ia32_subps256", IX86_BUILTIN_SUBPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
28730 { OPTION_MASK_ISA_AVX, CODE_FOR_xorv4df3, "__builtin_ia32_xorpd256", IX86_BUILTIN_XORPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
28731 { OPTION_MASK_ISA_AVX, CODE_FOR_xorv8sf3, "__builtin_ia32_xorps256", IX86_BUILTIN_XORPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
28733 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilvarv2df3, "__builtin_ia32_vpermilvarpd", IX86_BUILTIN_VPERMILVARPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DI },
28734 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilvarv4sf3, "__builtin_ia32_vpermilvarps", IX86_BUILTIN_VPERMILVARPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SI },
28735 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilvarv4df3, "__builtin_ia32_vpermilvarpd256", IX86_BUILTIN_VPERMILVARPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DI },
28736 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilvarv8sf3, "__builtin_ia32_vpermilvarps256", IX86_BUILTIN_VPERMILVARPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SI },
28738 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_blendpd256, "__builtin_ia32_blendpd256", IX86_BUILTIN_BLENDPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_INT },
28739 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_blendps256, "__builtin_ia32_blendps256", IX86_BUILTIN_BLENDPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_INT },
28740 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_blendvpd256, "__builtin_ia32_blendvpd256", IX86_BUILTIN_BLENDVPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_V4DF },
28741 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_blendvps256, "__builtin_ia32_blendvps256", IX86_BUILTIN_BLENDVPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_V8SF },
28742 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_dpps256, "__builtin_ia32_dpps256", IX86_BUILTIN_DPPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_INT },
28743 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_shufpd256, "__builtin_ia32_shufpd256", IX86_BUILTIN_SHUFPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_INT },
28744 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_shufps256, "__builtin_ia32_shufps256", IX86_BUILTIN_SHUFPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_INT },
28745 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vmcmpv2df3, "__builtin_ia32_cmpsd", IX86_BUILTIN_CMPSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
28746 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vmcmpv4sf3, "__builtin_ia32_cmpss", IX86_BUILTIN_CMPSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
28747 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cmpv2df3, "__builtin_ia32_cmppd", IX86_BUILTIN_CMPPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
28748 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cmpv4sf3, "__builtin_ia32_cmpps", IX86_BUILTIN_CMPPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
28749 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cmpv4df3, "__builtin_ia32_cmppd256", IX86_BUILTIN_CMPPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_INT },
28750 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cmpv8sf3, "__builtin_ia32_cmpps256", IX86_BUILTIN_CMPPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_INT },
28751 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vextractf128v4df, "__builtin_ia32_vextractf128_pd256", IX86_BUILTIN_EXTRACTF128PD256, UNKNOWN, (int) V2DF_FTYPE_V4DF_INT },
28752 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vextractf128v8sf, "__builtin_ia32_vextractf128_ps256", IX86_BUILTIN_EXTRACTF128PS256, UNKNOWN, (int) V4SF_FTYPE_V8SF_INT },
28753 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vextractf128v8si, "__builtin_ia32_vextractf128_si256", IX86_BUILTIN_EXTRACTF128SI256, UNKNOWN, (int) V4SI_FTYPE_V8SI_INT },
28754 { OPTION_MASK_ISA_AVX, CODE_FOR_floatv4siv4df2, "__builtin_ia32_cvtdq2pd256", IX86_BUILTIN_CVTDQ2PD256, UNKNOWN, (int) V4DF_FTYPE_V4SI },
28755 { OPTION_MASK_ISA_AVX, CODE_FOR_floatv8siv8sf2, "__builtin_ia32_cvtdq2ps256", IX86_BUILTIN_CVTDQ2PS256, UNKNOWN, (int) V8SF_FTYPE_V8SI },
28756 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cvtpd2ps256, "__builtin_ia32_cvtpd2ps256", IX86_BUILTIN_CVTPD2PS256, UNKNOWN, (int) V4SF_FTYPE_V4DF },
28757 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_fix_notruncv8sfv8si, "__builtin_ia32_cvtps2dq256", IX86_BUILTIN_CVTPS2DQ256, UNKNOWN, (int) V8SI_FTYPE_V8SF },
28758 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cvtps2pd256, "__builtin_ia32_cvtps2pd256", IX86_BUILTIN_CVTPS2PD256, UNKNOWN, (int) V4DF_FTYPE_V4SF },
28759 { OPTION_MASK_ISA_AVX, CODE_FOR_fix_truncv4dfv4si2, "__builtin_ia32_cvttpd2dq256", IX86_BUILTIN_CVTTPD2DQ256, UNKNOWN, (int) V4SI_FTYPE_V4DF },
28760 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cvtpd2dq256, "__builtin_ia32_cvtpd2dq256", IX86_BUILTIN_CVTPD2DQ256, UNKNOWN, (int) V4SI_FTYPE_V4DF },
28761 { OPTION_MASK_ISA_AVX, CODE_FOR_fix_truncv8sfv8si2, "__builtin_ia32_cvttps2dq256", IX86_BUILTIN_CVTTPS2DQ256, UNKNOWN, (int) V8SI_FTYPE_V8SF },
28762 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vperm2f128v4df3, "__builtin_ia32_vperm2f128_pd256", IX86_BUILTIN_VPERM2F128PD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_INT },
28763 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vperm2f128v8sf3, "__builtin_ia32_vperm2f128_ps256", IX86_BUILTIN_VPERM2F128PS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_INT },
28764 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vperm2f128v8si3, "__builtin_ia32_vperm2f128_si256", IX86_BUILTIN_VPERM2F128SI256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_INT },
28765 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilv2df, "__builtin_ia32_vpermilpd", IX86_BUILTIN_VPERMILPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_INT },
28766 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilv4sf, "__builtin_ia32_vpermilps", IX86_BUILTIN_VPERMILPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_INT },
28767 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilv4df, "__builtin_ia32_vpermilpd256", IX86_BUILTIN_VPERMILPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_INT },
28768 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilv8sf, "__builtin_ia32_vpermilps256", IX86_BUILTIN_VPERMILPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_INT },
28769 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vinsertf128v4df, "__builtin_ia32_vinsertf128_pd256", IX86_BUILTIN_VINSERTF128PD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V2DF_INT },
28770 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vinsertf128v8sf, "__builtin_ia32_vinsertf128_ps256", IX86_BUILTIN_VINSERTF128PS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V4SF_INT },
28771 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vinsertf128v8si, "__builtin_ia32_vinsertf128_si256", IX86_BUILTIN_VINSERTF128SI256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V4SI_INT },
28773 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movshdup256, "__builtin_ia32_movshdup256", IX86_BUILTIN_MOVSHDUP256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
28774 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movsldup256, "__builtin_ia32_movsldup256", IX86_BUILTIN_MOVSLDUP256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
28775 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movddup256, "__builtin_ia32_movddup256", IX86_BUILTIN_MOVDDUP256, UNKNOWN, (int) V4DF_FTYPE_V4DF },
28777 { OPTION_MASK_ISA_AVX, CODE_FOR_sqrtv4df2, "__builtin_ia32_sqrtpd256", IX86_BUILTIN_SQRTPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF },
28778 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_sqrtv8sf2, "__builtin_ia32_sqrtps256", IX86_BUILTIN_SQRTPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
28779 { OPTION_MASK_ISA_AVX, CODE_FOR_sqrtv8sf2, "__builtin_ia32_sqrtps_nr256", IX86_BUILTIN_SQRTPS_NR256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
28780 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_rsqrtv8sf2, "__builtin_ia32_rsqrtps256", IX86_BUILTIN_RSQRTPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
28781 { OPTION_MASK_ISA_AVX, CODE_FOR_rsqrtv8sf2, "__builtin_ia32_rsqrtps_nr256", IX86_BUILTIN_RSQRTPS_NR256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
28783 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_rcpv8sf2, "__builtin_ia32_rcpps256", IX86_BUILTIN_RCPPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
28785 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundpd256, "__builtin_ia32_roundpd256", IX86_BUILTIN_ROUNDPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_INT },
28786 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundps256, "__builtin_ia32_roundps256", IX86_BUILTIN_ROUNDPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_INT },
28788 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundpd256, "__builtin_ia32_floorpd256", IX86_BUILTIN_FLOORPD256, (enum rtx_code) ROUND_FLOOR, (int) V4DF_FTYPE_V4DF_ROUND },
28789 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundpd256, "__builtin_ia32_ceilpd256", IX86_BUILTIN_CEILPD256, (enum rtx_code) ROUND_CEIL, (int) V4DF_FTYPE_V4DF_ROUND },
28790 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundpd256, "__builtin_ia32_truncpd256", IX86_BUILTIN_TRUNCPD256, (enum rtx_code) ROUND_TRUNC, (int) V4DF_FTYPE_V4DF_ROUND },
28791 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundpd256, "__builtin_ia32_rintpd256", IX86_BUILTIN_RINTPD256, (enum rtx_code) ROUND_MXCSR, (int) V4DF_FTYPE_V4DF_ROUND },
28793 { OPTION_MASK_ISA_AVX, CODE_FOR_roundv4df2, "__builtin_ia32_roundpd_az256", IX86_BUILTIN_ROUNDPD_AZ256, UNKNOWN, (int) V4DF_FTYPE_V4DF },
28794 { OPTION_MASK_ISA_AVX, CODE_FOR_roundv4df2_vec_pack_sfix, "__builtin_ia32_roundpd_az_vec_pack_sfix256", IX86_BUILTIN_ROUNDPD_AZ_VEC_PACK_SFIX256, UNKNOWN, (int) V8SI_FTYPE_V4DF_V4DF },
28796 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundpd_vec_pack_sfix256, "__builtin_ia32_floorpd_vec_pack_sfix256", IX86_BUILTIN_FLOORPD_VEC_PACK_SFIX256, (enum rtx_code) ROUND_FLOOR, (int) V8SI_FTYPE_V4DF_V4DF_ROUND },
28797 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundpd_vec_pack_sfix256, "__builtin_ia32_ceilpd_vec_pack_sfix256", IX86_BUILTIN_CEILPD_VEC_PACK_SFIX256, (enum rtx_code) ROUND_CEIL, (int) V8SI_FTYPE_V4DF_V4DF_ROUND },
28799 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundps256, "__builtin_ia32_floorps256", IX86_BUILTIN_FLOORPS256, (enum rtx_code) ROUND_FLOOR, (int) V8SF_FTYPE_V8SF_ROUND },
28800 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundps256, "__builtin_ia32_ceilps256", IX86_BUILTIN_CEILPS256, (enum rtx_code) ROUND_CEIL, (int) V8SF_FTYPE_V8SF_ROUND },
28801 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundps256, "__builtin_ia32_truncps256", IX86_BUILTIN_TRUNCPS256, (enum rtx_code) ROUND_TRUNC, (int) V8SF_FTYPE_V8SF_ROUND },
28802 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundps256, "__builtin_ia32_rintps256", IX86_BUILTIN_RINTPS256, (enum rtx_code) ROUND_MXCSR, (int) V8SF_FTYPE_V8SF_ROUND },
28804 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundps_sfix256, "__builtin_ia32_floorps_sfix256", IX86_BUILTIN_FLOORPS_SFIX256, (enum rtx_code) ROUND_FLOOR, (int) V8SI_FTYPE_V8SF_ROUND },
28805 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundps_sfix256, "__builtin_ia32_ceilps_sfix256", IX86_BUILTIN_CEILPS_SFIX256, (enum rtx_code) ROUND_CEIL, (int) V8SI_FTYPE_V8SF_ROUND },
28807 { OPTION_MASK_ISA_AVX, CODE_FOR_roundv8sf2, "__builtin_ia32_roundps_az256", IX86_BUILTIN_ROUNDPS_AZ256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
28808 { OPTION_MASK_ISA_AVX, CODE_FOR_roundv8sf2_sfix, "__builtin_ia32_roundps_az_sfix256", IX86_BUILTIN_ROUNDPS_AZ_SFIX256, UNKNOWN, (int) V8SI_FTYPE_V8SF },
28810 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_unpckhpd256, "__builtin_ia32_unpckhpd256", IX86_BUILTIN_UNPCKHPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
28811 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_unpcklpd256, "__builtin_ia32_unpcklpd256", IX86_BUILTIN_UNPCKLPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
28812 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_unpckhps256, "__builtin_ia32_unpckhps256", IX86_BUILTIN_UNPCKHPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
28813 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_unpcklps256, "__builtin_ia32_unpcklps256", IX86_BUILTIN_UNPCKLPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
28815 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_si256_si, "__builtin_ia32_si256_si", IX86_BUILTIN_SI256_SI, UNKNOWN, (int) V8SI_FTYPE_V4SI },
28816 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_ps256_ps, "__builtin_ia32_ps256_ps", IX86_BUILTIN_PS256_PS, UNKNOWN, (int) V8SF_FTYPE_V4SF },
28817 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_pd256_pd, "__builtin_ia32_pd256_pd", IX86_BUILTIN_PD256_PD, UNKNOWN, (int) V4DF_FTYPE_V2DF },
28818 { OPTION_MASK_ISA_AVX, CODE_FOR_vec_extract_lo_v8si, "__builtin_ia32_si_si256", IX86_BUILTIN_SI_SI256, UNKNOWN, (int) V4SI_FTYPE_V8SI },
28819 { OPTION_MASK_ISA_AVX, CODE_FOR_vec_extract_lo_v8sf, "__builtin_ia32_ps_ps256", IX86_BUILTIN_PS_PS256, UNKNOWN, (int) V4SF_FTYPE_V8SF },
28820 { OPTION_MASK_ISA_AVX, CODE_FOR_vec_extract_lo_v4df, "__builtin_ia32_pd_pd256", IX86_BUILTIN_PD_PD256, UNKNOWN, (int) V2DF_FTYPE_V4DF },
28822 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestpd, "__builtin_ia32_vtestzpd", IX86_BUILTIN_VTESTZPD, EQ, (int) INT_FTYPE_V2DF_V2DF_PTEST },
28823 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestpd, "__builtin_ia32_vtestcpd", IX86_BUILTIN_VTESTCPD, LTU, (int) INT_FTYPE_V2DF_V2DF_PTEST },
28824 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestpd, "__builtin_ia32_vtestnzcpd", IX86_BUILTIN_VTESTNZCPD, GTU, (int) INT_FTYPE_V2DF_V2DF_PTEST },
28825 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestps, "__builtin_ia32_vtestzps", IX86_BUILTIN_VTESTZPS, EQ, (int) INT_FTYPE_V4SF_V4SF_PTEST },
28826 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestps, "__builtin_ia32_vtestcps", IX86_BUILTIN_VTESTCPS, LTU, (int) INT_FTYPE_V4SF_V4SF_PTEST },
28827 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestps, "__builtin_ia32_vtestnzcps", IX86_BUILTIN_VTESTNZCPS, GTU, (int) INT_FTYPE_V4SF_V4SF_PTEST },
28828 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestpd256, "__builtin_ia32_vtestzpd256", IX86_BUILTIN_VTESTZPD256, EQ, (int) INT_FTYPE_V4DF_V4DF_PTEST },
28829 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestpd256, "__builtin_ia32_vtestcpd256", IX86_BUILTIN_VTESTCPD256, LTU, (int) INT_FTYPE_V4DF_V4DF_PTEST },
28830 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestpd256, "__builtin_ia32_vtestnzcpd256", IX86_BUILTIN_VTESTNZCPD256, GTU, (int) INT_FTYPE_V4DF_V4DF_PTEST },
28831 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestps256, "__builtin_ia32_vtestzps256", IX86_BUILTIN_VTESTZPS256, EQ, (int) INT_FTYPE_V8SF_V8SF_PTEST },
28832 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestps256, "__builtin_ia32_vtestcps256", IX86_BUILTIN_VTESTCPS256, LTU, (int) INT_FTYPE_V8SF_V8SF_PTEST },
28833 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestps256, "__builtin_ia32_vtestnzcps256", IX86_BUILTIN_VTESTNZCPS256, GTU, (int) INT_FTYPE_V8SF_V8SF_PTEST },
28834 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_ptest256, "__builtin_ia32_ptestz256", IX86_BUILTIN_PTESTZ256, EQ, (int) INT_FTYPE_V4DI_V4DI_PTEST },
28835 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_ptest256, "__builtin_ia32_ptestc256", IX86_BUILTIN_PTESTC256, LTU, (int) INT_FTYPE_V4DI_V4DI_PTEST },
28836 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_ptest256, "__builtin_ia32_ptestnzc256", IX86_BUILTIN_PTESTNZC256, GTU, (int) INT_FTYPE_V4DI_V4DI_PTEST },
28838 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movmskpd256, "__builtin_ia32_movmskpd256", IX86_BUILTIN_MOVMSKPD256, UNKNOWN, (int) INT_FTYPE_V4DF },
28839 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movmskps256, "__builtin_ia32_movmskps256", IX86_BUILTIN_MOVMSKPS256, UNKNOWN, (int) INT_FTYPE_V8SF },
28841 { OPTION_MASK_ISA_AVX, CODE_FOR_copysignv8sf3, "__builtin_ia32_copysignps256", IX86_BUILTIN_CPYSGNPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
28842 { OPTION_MASK_ISA_AVX, CODE_FOR_copysignv4df3, "__builtin_ia32_copysignpd256", IX86_BUILTIN_CPYSGNPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
28844 { OPTION_MASK_ISA_AVX, CODE_FOR_vec_pack_sfix_v4df, "__builtin_ia32_vec_pack_sfix256 ", IX86_BUILTIN_VEC_PACK_SFIX256, UNKNOWN, (int) V8SI_FTYPE_V4DF_V4DF },
28846 /* AVX2 */
28847 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_mpsadbw, "__builtin_ia32_mpsadbw256", IX86_BUILTIN_MPSADBW256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI_INT },
28848 { OPTION_MASK_ISA_AVX2, CODE_FOR_absv32qi2, "__builtin_ia32_pabsb256", IX86_BUILTIN_PABSB256, UNKNOWN, (int) V32QI_FTYPE_V32QI },
28849 { OPTION_MASK_ISA_AVX2, CODE_FOR_absv16hi2, "__builtin_ia32_pabsw256", IX86_BUILTIN_PABSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI },
28850 { OPTION_MASK_ISA_AVX2, CODE_FOR_absv8si2, "__builtin_ia32_pabsd256", IX86_BUILTIN_PABSD256, UNKNOWN, (int) V8SI_FTYPE_V8SI },
28851 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_packssdw, "__builtin_ia32_packssdw256", IX86_BUILTIN_PACKSSDW256, UNKNOWN, (int) V16HI_FTYPE_V8SI_V8SI },
28852 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_packsswb, "__builtin_ia32_packsswb256", IX86_BUILTIN_PACKSSWB256, UNKNOWN, (int) V32QI_FTYPE_V16HI_V16HI },
28853 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_packusdw, "__builtin_ia32_packusdw256", IX86_BUILTIN_PACKUSDW256, UNKNOWN, (int) V16HI_FTYPE_V8SI_V8SI },
28854 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_packuswb, "__builtin_ia32_packuswb256", IX86_BUILTIN_PACKUSWB256, UNKNOWN, (int) V32QI_FTYPE_V16HI_V16HI },
28855 { OPTION_MASK_ISA_AVX2, CODE_FOR_addv32qi3, "__builtin_ia32_paddb256", IX86_BUILTIN_PADDB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
28856 { OPTION_MASK_ISA_AVX2, CODE_FOR_addv16hi3, "__builtin_ia32_paddw256", IX86_BUILTIN_PADDW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28857 { OPTION_MASK_ISA_AVX2, CODE_FOR_addv8si3, "__builtin_ia32_paddd256", IX86_BUILTIN_PADDD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
28858 { OPTION_MASK_ISA_AVX2, CODE_FOR_addv4di3, "__builtin_ia32_paddq256", IX86_BUILTIN_PADDQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
28859 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ssaddv32qi3, "__builtin_ia32_paddsb256", IX86_BUILTIN_PADDSB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
28860 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ssaddv16hi3, "__builtin_ia32_paddsw256", IX86_BUILTIN_PADDSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28861 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_usaddv32qi3, "__builtin_ia32_paddusb256", IX86_BUILTIN_PADDUSB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
28862 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_usaddv16hi3, "__builtin_ia32_paddusw256", IX86_BUILTIN_PADDUSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28863 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_palignrv2ti, "__builtin_ia32_palignr256", IX86_BUILTIN_PALIGNR256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_INT_CONVERT },
28864 { OPTION_MASK_ISA_AVX2, CODE_FOR_andv4di3, "__builtin_ia32_andsi256", IX86_BUILTIN_AND256I, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
28865 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_andnotv4di3, "__builtin_ia32_andnotsi256", IX86_BUILTIN_ANDNOT256I, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
28866 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_uavgv32qi3, "__builtin_ia32_pavgb256", IX86_BUILTIN_PAVGB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
28867 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_uavgv16hi3, "__builtin_ia32_pavgw256", IX86_BUILTIN_PAVGW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28868 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pblendvb, "__builtin_ia32_pblendvb256", IX86_BUILTIN_PBLENDVB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI_V32QI },
28869 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pblendw, "__builtin_ia32_pblendw256", IX86_BUILTIN_PBLENDVW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI_INT },
28870 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_eqv32qi3, "__builtin_ia32_pcmpeqb256", IX86_BUILTIN_PCMPEQB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
28871 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_eqv16hi3, "__builtin_ia32_pcmpeqw256", IX86_BUILTIN_PCMPEQW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28872 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_eqv8si3, "__builtin_ia32_pcmpeqd256", IX86_BUILTIN_PCMPEQD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
28873 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_eqv4di3, "__builtin_ia32_pcmpeqq256", IX86_BUILTIN_PCMPEQQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
28874 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_gtv32qi3, "__builtin_ia32_pcmpgtb256", IX86_BUILTIN_PCMPGTB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
28875 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_gtv16hi3, "__builtin_ia32_pcmpgtw256", IX86_BUILTIN_PCMPGTW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28876 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_gtv8si3, "__builtin_ia32_pcmpgtd256", IX86_BUILTIN_PCMPGTD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
28877 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_gtv4di3, "__builtin_ia32_pcmpgtq256", IX86_BUILTIN_PCMPGTQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
28878 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_phaddwv16hi3, "__builtin_ia32_phaddw256", IX86_BUILTIN_PHADDW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28879 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_phadddv8si3, "__builtin_ia32_phaddd256", IX86_BUILTIN_PHADDD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
28880 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_phaddswv16hi3, "__builtin_ia32_phaddsw256", IX86_BUILTIN_PHADDSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28881 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_phsubwv16hi3, "__builtin_ia32_phsubw256", IX86_BUILTIN_PHSUBW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28882 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_phsubdv8si3, "__builtin_ia32_phsubd256", IX86_BUILTIN_PHSUBD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
28883 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_phsubswv16hi3, "__builtin_ia32_phsubsw256", IX86_BUILTIN_PHSUBSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28884 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pmaddubsw256, "__builtin_ia32_pmaddubsw256", IX86_BUILTIN_PMADDUBSW256, UNKNOWN, (int) V16HI_FTYPE_V32QI_V32QI },
28885 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pmaddwd, "__builtin_ia32_pmaddwd256", IX86_BUILTIN_PMADDWD256, UNKNOWN, (int) V8SI_FTYPE_V16HI_V16HI },
28886 { OPTION_MASK_ISA_AVX2, CODE_FOR_smaxv32qi3, "__builtin_ia32_pmaxsb256", IX86_BUILTIN_PMAXSB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
28887 { OPTION_MASK_ISA_AVX2, CODE_FOR_smaxv16hi3, "__builtin_ia32_pmaxsw256", IX86_BUILTIN_PMAXSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28888 { OPTION_MASK_ISA_AVX2, CODE_FOR_smaxv8si3 , "__builtin_ia32_pmaxsd256", IX86_BUILTIN_PMAXSD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
28889 { OPTION_MASK_ISA_AVX2, CODE_FOR_umaxv32qi3, "__builtin_ia32_pmaxub256", IX86_BUILTIN_PMAXUB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
28890 { OPTION_MASK_ISA_AVX2, CODE_FOR_umaxv16hi3, "__builtin_ia32_pmaxuw256", IX86_BUILTIN_PMAXUW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28891 { OPTION_MASK_ISA_AVX2, CODE_FOR_umaxv8si3 , "__builtin_ia32_pmaxud256", IX86_BUILTIN_PMAXUD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
28892 { OPTION_MASK_ISA_AVX2, CODE_FOR_sminv32qi3, "__builtin_ia32_pminsb256", IX86_BUILTIN_PMINSB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
28893 { OPTION_MASK_ISA_AVX2, CODE_FOR_sminv16hi3, "__builtin_ia32_pminsw256", IX86_BUILTIN_PMINSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28894 { OPTION_MASK_ISA_AVX2, CODE_FOR_sminv8si3 , "__builtin_ia32_pminsd256", IX86_BUILTIN_PMINSD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
28895 { OPTION_MASK_ISA_AVX2, CODE_FOR_uminv32qi3, "__builtin_ia32_pminub256", IX86_BUILTIN_PMINUB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
28896 { OPTION_MASK_ISA_AVX2, CODE_FOR_uminv16hi3, "__builtin_ia32_pminuw256", IX86_BUILTIN_PMINUW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28897 { OPTION_MASK_ISA_AVX2, CODE_FOR_uminv8si3 , "__builtin_ia32_pminud256", IX86_BUILTIN_PMINUD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
28898 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pmovmskb, "__builtin_ia32_pmovmskb256", IX86_BUILTIN_PMOVMSKB256, UNKNOWN, (int) INT_FTYPE_V32QI },
28899 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_sign_extendv16qiv16hi2, "__builtin_ia32_pmovsxbw256", IX86_BUILTIN_PMOVSXBW256, UNKNOWN, (int) V16HI_FTYPE_V16QI },
28900 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_sign_extendv8qiv8si2 , "__builtin_ia32_pmovsxbd256", IX86_BUILTIN_PMOVSXBD256, UNKNOWN, (int) V8SI_FTYPE_V16QI },
28901 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_sign_extendv4qiv4di2 , "__builtin_ia32_pmovsxbq256", IX86_BUILTIN_PMOVSXBQ256, UNKNOWN, (int) V4DI_FTYPE_V16QI },
28902 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_sign_extendv8hiv8si2 , "__builtin_ia32_pmovsxwd256", IX86_BUILTIN_PMOVSXWD256, UNKNOWN, (int) V8SI_FTYPE_V8HI },
28903 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_sign_extendv4hiv4di2 , "__builtin_ia32_pmovsxwq256", IX86_BUILTIN_PMOVSXWQ256, UNKNOWN, (int) V4DI_FTYPE_V8HI },
28904 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_sign_extendv4siv4di2 , "__builtin_ia32_pmovsxdq256", IX86_BUILTIN_PMOVSXDQ256, UNKNOWN, (int) V4DI_FTYPE_V4SI },
28905 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_zero_extendv16qiv16hi2, "__builtin_ia32_pmovzxbw256", IX86_BUILTIN_PMOVZXBW256, UNKNOWN, (int) V16HI_FTYPE_V16QI },
28906 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_zero_extendv8qiv8si2 , "__builtin_ia32_pmovzxbd256", IX86_BUILTIN_PMOVZXBD256, UNKNOWN, (int) V8SI_FTYPE_V16QI },
28907 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_zero_extendv4qiv4di2 , "__builtin_ia32_pmovzxbq256", IX86_BUILTIN_PMOVZXBQ256, UNKNOWN, (int) V4DI_FTYPE_V16QI },
28908 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_zero_extendv8hiv8si2 , "__builtin_ia32_pmovzxwd256", IX86_BUILTIN_PMOVZXWD256, UNKNOWN, (int) V8SI_FTYPE_V8HI },
28909 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_zero_extendv4hiv4di2 , "__builtin_ia32_pmovzxwq256", IX86_BUILTIN_PMOVZXWQ256, UNKNOWN, (int) V4DI_FTYPE_V8HI },
28910 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_zero_extendv4siv4di2 , "__builtin_ia32_pmovzxdq256", IX86_BUILTIN_PMOVZXDQ256, UNKNOWN, (int) V4DI_FTYPE_V4SI },
28911 { OPTION_MASK_ISA_AVX2, CODE_FOR_vec_widen_smult_even_v8si, "__builtin_ia32_pmuldq256", IX86_BUILTIN_PMULDQ256, UNKNOWN, (int) V4DI_FTYPE_V8SI_V8SI },
28912 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pmulhrswv16hi3 , "__builtin_ia32_pmulhrsw256", IX86_BUILTIN_PMULHRSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28913 { OPTION_MASK_ISA_AVX2, CODE_FOR_umulv16hi3_highpart, "__builtin_ia32_pmulhuw256" , IX86_BUILTIN_PMULHUW256 , UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28914 { OPTION_MASK_ISA_AVX2, CODE_FOR_smulv16hi3_highpart, "__builtin_ia32_pmulhw256" , IX86_BUILTIN_PMULHW256 , UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28915 { OPTION_MASK_ISA_AVX2, CODE_FOR_mulv16hi3, "__builtin_ia32_pmullw256" , IX86_BUILTIN_PMULLW256 , UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28916 { OPTION_MASK_ISA_AVX2, CODE_FOR_mulv8si3, "__builtin_ia32_pmulld256" , IX86_BUILTIN_PMULLD256 , UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
28917 { OPTION_MASK_ISA_AVX2, CODE_FOR_vec_widen_umult_even_v8si, "__builtin_ia32_pmuludq256", IX86_BUILTIN_PMULUDQ256, UNKNOWN, (int) V4DI_FTYPE_V8SI_V8SI },
28918 { OPTION_MASK_ISA_AVX2, CODE_FOR_iorv4di3, "__builtin_ia32_por256", IX86_BUILTIN_POR256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
28919 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_psadbw, "__builtin_ia32_psadbw256", IX86_BUILTIN_PSADBW256, UNKNOWN, (int) V16HI_FTYPE_V32QI_V32QI },
28920 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pshufbv32qi3, "__builtin_ia32_pshufb256", IX86_BUILTIN_PSHUFB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
28921 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pshufdv3, "__builtin_ia32_pshufd256", IX86_BUILTIN_PSHUFD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_INT },
28922 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pshufhwv3, "__builtin_ia32_pshufhw256", IX86_BUILTIN_PSHUFHW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_INT },
28923 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pshuflwv3, "__builtin_ia32_pshuflw256", IX86_BUILTIN_PSHUFLW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_INT },
28924 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_psignv32qi3, "__builtin_ia32_psignb256", IX86_BUILTIN_PSIGNB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
28925 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_psignv16hi3, "__builtin_ia32_psignw256", IX86_BUILTIN_PSIGNW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28926 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_psignv8si3 , "__builtin_ia32_psignd256", IX86_BUILTIN_PSIGND256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
28927 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ashlv2ti3, "__builtin_ia32_pslldqi256", IX86_BUILTIN_PSLLDQI256, UNKNOWN, (int) V4DI_FTYPE_V4DI_INT_CONVERT },
28928 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashlv16hi3, "__builtin_ia32_psllwi256", IX86_BUILTIN_PSLLWI256 , UNKNOWN, (int) V16HI_FTYPE_V16HI_SI_COUNT },
28929 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashlv16hi3, "__builtin_ia32_psllw256", IX86_BUILTIN_PSLLW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V8HI_COUNT },
28930 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashlv8si3, "__builtin_ia32_pslldi256", IX86_BUILTIN_PSLLDI256, UNKNOWN, (int) V8SI_FTYPE_V8SI_SI_COUNT },
28931 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashlv8si3, "__builtin_ia32_pslld256", IX86_BUILTIN_PSLLD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V4SI_COUNT },
28932 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashlv4di3, "__builtin_ia32_psllqi256", IX86_BUILTIN_PSLLQI256, UNKNOWN, (int) V4DI_FTYPE_V4DI_INT_COUNT },
28933 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashlv4di3, "__builtin_ia32_psllq256", IX86_BUILTIN_PSLLQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V2DI_COUNT },
28934 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashrv16hi3, "__builtin_ia32_psrawi256", IX86_BUILTIN_PSRAWI256, UNKNOWN, (int) V16HI_FTYPE_V16HI_SI_COUNT },
28935 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashrv16hi3, "__builtin_ia32_psraw256", IX86_BUILTIN_PSRAW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V8HI_COUNT },
28936 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashrv8si3, "__builtin_ia32_psradi256", IX86_BUILTIN_PSRADI256, UNKNOWN, (int) V8SI_FTYPE_V8SI_SI_COUNT },
28937 { OPTION_MASK_ISA_AVX2, CODE_FOR_ashrv8si3, "__builtin_ia32_psrad256", IX86_BUILTIN_PSRAD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V4SI_COUNT },
28938 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_lshrv2ti3, "__builtin_ia32_psrldqi256", IX86_BUILTIN_PSRLDQI256, UNKNOWN, (int) V4DI_FTYPE_V4DI_INT_CONVERT },
28939 { OPTION_MASK_ISA_AVX2, CODE_FOR_lshrv16hi3, "__builtin_ia32_psrlwi256", IX86_BUILTIN_PSRLWI256 , UNKNOWN, (int) V16HI_FTYPE_V16HI_SI_COUNT },
28940 { OPTION_MASK_ISA_AVX2, CODE_FOR_lshrv16hi3, "__builtin_ia32_psrlw256", IX86_BUILTIN_PSRLW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V8HI_COUNT },
28941 { OPTION_MASK_ISA_AVX2, CODE_FOR_lshrv8si3, "__builtin_ia32_psrldi256", IX86_BUILTIN_PSRLDI256, UNKNOWN, (int) V8SI_FTYPE_V8SI_SI_COUNT },
28942 { OPTION_MASK_ISA_AVX2, CODE_FOR_lshrv8si3, "__builtin_ia32_psrld256", IX86_BUILTIN_PSRLD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V4SI_COUNT },
28943 { OPTION_MASK_ISA_AVX2, CODE_FOR_lshrv4di3, "__builtin_ia32_psrlqi256", IX86_BUILTIN_PSRLQI256, UNKNOWN, (int) V4DI_FTYPE_V4DI_INT_COUNT },
28944 { OPTION_MASK_ISA_AVX2, CODE_FOR_lshrv4di3, "__builtin_ia32_psrlq256", IX86_BUILTIN_PSRLQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V2DI_COUNT },
28945 { OPTION_MASK_ISA_AVX2, CODE_FOR_subv32qi3, "__builtin_ia32_psubb256", IX86_BUILTIN_PSUBB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
28946 { OPTION_MASK_ISA_AVX2, CODE_FOR_subv16hi3, "__builtin_ia32_psubw256", IX86_BUILTIN_PSUBW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28947 { OPTION_MASK_ISA_AVX2, CODE_FOR_subv8si3, "__builtin_ia32_psubd256", IX86_BUILTIN_PSUBD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
28948 { OPTION_MASK_ISA_AVX2, CODE_FOR_subv4di3, "__builtin_ia32_psubq256", IX86_BUILTIN_PSUBQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
28949 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_sssubv32qi3, "__builtin_ia32_psubsb256", IX86_BUILTIN_PSUBSB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
28950 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_sssubv16hi3, "__builtin_ia32_psubsw256", IX86_BUILTIN_PSUBSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28951 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ussubv32qi3, "__builtin_ia32_psubusb256", IX86_BUILTIN_PSUBUSB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
28952 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ussubv16hi3, "__builtin_ia32_psubusw256", IX86_BUILTIN_PSUBUSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28953 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_interleave_highv32qi, "__builtin_ia32_punpckhbw256", IX86_BUILTIN_PUNPCKHBW256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
28954 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_interleave_highv16hi, "__builtin_ia32_punpckhwd256", IX86_BUILTIN_PUNPCKHWD256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28955 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_interleave_highv8si, "__builtin_ia32_punpckhdq256", IX86_BUILTIN_PUNPCKHDQ256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
28956 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_interleave_highv4di, "__builtin_ia32_punpckhqdq256", IX86_BUILTIN_PUNPCKHQDQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
28957 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_interleave_lowv32qi, "__builtin_ia32_punpcklbw256", IX86_BUILTIN_PUNPCKLBW256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
28958 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_interleave_lowv16hi, "__builtin_ia32_punpcklwd256", IX86_BUILTIN_PUNPCKLWD256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
28959 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_interleave_lowv8si, "__builtin_ia32_punpckldq256", IX86_BUILTIN_PUNPCKLDQ256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
28960 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_interleave_lowv4di, "__builtin_ia32_punpcklqdq256", IX86_BUILTIN_PUNPCKLQDQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
28961 { OPTION_MASK_ISA_AVX2, CODE_FOR_xorv4di3, "__builtin_ia32_pxor256", IX86_BUILTIN_PXOR256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
28962 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_vec_dupv4sf, "__builtin_ia32_vbroadcastss_ps", IX86_BUILTIN_VBROADCASTSS_PS, UNKNOWN, (int) V4SF_FTYPE_V4SF },
28963 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_vec_dupv8sf, "__builtin_ia32_vbroadcastss_ps256", IX86_BUILTIN_VBROADCASTSS_PS256, UNKNOWN, (int) V8SF_FTYPE_V4SF },
28964 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_vec_dupv4df, "__builtin_ia32_vbroadcastsd_pd256", IX86_BUILTIN_VBROADCASTSD_PD256, UNKNOWN, (int) V4DF_FTYPE_V2DF },
28965 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_vbroadcasti128_v4di, "__builtin_ia32_vbroadcastsi256", IX86_BUILTIN_VBROADCASTSI256, UNKNOWN, (int) V4DI_FTYPE_V2DI },
28966 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pblenddv4si, "__builtin_ia32_pblendd128", IX86_BUILTIN_PBLENDD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_INT },
28967 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pblenddv8si, "__builtin_ia32_pblendd256", IX86_BUILTIN_PBLENDD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_INT },
28968 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pbroadcastv32qi, "__builtin_ia32_pbroadcastb256", IX86_BUILTIN_PBROADCASTB256, UNKNOWN, (int) V32QI_FTYPE_V16QI },
28969 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pbroadcastv16hi, "__builtin_ia32_pbroadcastw256", IX86_BUILTIN_PBROADCASTW256, UNKNOWN, (int) V16HI_FTYPE_V8HI },
28970 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pbroadcastv8si, "__builtin_ia32_pbroadcastd256", IX86_BUILTIN_PBROADCASTD256, UNKNOWN, (int) V8SI_FTYPE_V4SI },
28971 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pbroadcastv4di, "__builtin_ia32_pbroadcastq256", IX86_BUILTIN_PBROADCASTQ256, UNKNOWN, (int) V4DI_FTYPE_V2DI },
28972 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pbroadcastv16qi, "__builtin_ia32_pbroadcastb128", IX86_BUILTIN_PBROADCASTB128, UNKNOWN, (int) V16QI_FTYPE_V16QI },
28973 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pbroadcastv8hi, "__builtin_ia32_pbroadcastw128", IX86_BUILTIN_PBROADCASTW128, UNKNOWN, (int) V8HI_FTYPE_V8HI },
28974 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pbroadcastv4si, "__builtin_ia32_pbroadcastd128", IX86_BUILTIN_PBROADCASTD128, UNKNOWN, (int) V4SI_FTYPE_V4SI },
28975 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_pbroadcastv2di, "__builtin_ia32_pbroadcastq128", IX86_BUILTIN_PBROADCASTQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI },
28976 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_permvarv8si, "__builtin_ia32_permvarsi256", IX86_BUILTIN_VPERMVARSI256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
28977 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_permvarv8sf, "__builtin_ia32_permvarsf256", IX86_BUILTIN_VPERMVARSF256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SI },
28978 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_permv4df, "__builtin_ia32_permdf256", IX86_BUILTIN_VPERMDF256, UNKNOWN, (int) V4DF_FTYPE_V4DF_INT },
28979 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_permv4di, "__builtin_ia32_permdi256", IX86_BUILTIN_VPERMDI256, UNKNOWN, (int) V4DI_FTYPE_V4DI_INT },
28980 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_permv2ti, "__builtin_ia32_permti256", IX86_BUILTIN_VPERMTI256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_INT },
28981 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_extracti128, "__builtin_ia32_extract128i256", IX86_BUILTIN_VEXTRACT128I256, UNKNOWN, (int) V2DI_FTYPE_V4DI_INT },
28982 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_inserti128, "__builtin_ia32_insert128i256", IX86_BUILTIN_VINSERT128I256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V2DI_INT },
28983 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ashlvv4di, "__builtin_ia32_psllv4di", IX86_BUILTIN_PSLLVV4DI, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
28984 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ashlvv2di, "__builtin_ia32_psllv2di", IX86_BUILTIN_PSLLVV2DI, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
28985 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ashlvv8si, "__builtin_ia32_psllv8si", IX86_BUILTIN_PSLLVV8SI, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
28986 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ashlvv4si, "__builtin_ia32_psllv4si", IX86_BUILTIN_PSLLVV4SI, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
28987 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ashrvv8si, "__builtin_ia32_psrav8si", IX86_BUILTIN_PSRAVV8SI, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
28988 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ashrvv4si, "__builtin_ia32_psrav4si", IX86_BUILTIN_PSRAVV4SI, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
28989 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_lshrvv4di, "__builtin_ia32_psrlv4di", IX86_BUILTIN_PSRLVV4DI, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
28990 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_lshrvv2di, "__builtin_ia32_psrlv2di", IX86_BUILTIN_PSRLVV2DI, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
28991 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_lshrvv8si, "__builtin_ia32_psrlv8si", IX86_BUILTIN_PSRLVV8SI, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
28992 { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_lshrvv4si, "__builtin_ia32_psrlv4si", IX86_BUILTIN_PSRLVV4SI, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
28994 { OPTION_MASK_ISA_LZCNT, CODE_FOR_clzhi2_lzcnt, "__builtin_clzs", IX86_BUILTIN_CLZS, UNKNOWN, (int) UINT16_FTYPE_UINT16 },
28996 /* BMI */
28997 { OPTION_MASK_ISA_BMI, CODE_FOR_bmi_bextr_si, "__builtin_ia32_bextr_u32", IX86_BUILTIN_BEXTR32, UNKNOWN, (int) UINT_FTYPE_UINT_UINT },
28998 { OPTION_MASK_ISA_BMI, CODE_FOR_bmi_bextr_di, "__builtin_ia32_bextr_u64", IX86_BUILTIN_BEXTR64, UNKNOWN, (int) UINT64_FTYPE_UINT64_UINT64 },
28999 { OPTION_MASK_ISA_BMI, CODE_FOR_ctzhi2, "__builtin_ctzs", IX86_BUILTIN_CTZS, UNKNOWN, (int) UINT16_FTYPE_UINT16 },
29001 /* TBM */
29002 { OPTION_MASK_ISA_TBM, CODE_FOR_tbm_bextri_si, "__builtin_ia32_bextri_u32", IX86_BUILTIN_BEXTRI32, UNKNOWN, (int) UINT_FTYPE_UINT_UINT },
29003 { OPTION_MASK_ISA_TBM, CODE_FOR_tbm_bextri_di, "__builtin_ia32_bextri_u64", IX86_BUILTIN_BEXTRI64, UNKNOWN, (int) UINT64_FTYPE_UINT64_UINT64 },
29005 /* F16C */
29006 { OPTION_MASK_ISA_F16C, CODE_FOR_vcvtph2ps, "__builtin_ia32_vcvtph2ps", IX86_BUILTIN_CVTPH2PS, UNKNOWN, (int) V4SF_FTYPE_V8HI },
29007 { OPTION_MASK_ISA_F16C, CODE_FOR_vcvtph2ps256, "__builtin_ia32_vcvtph2ps256", IX86_BUILTIN_CVTPH2PS256, UNKNOWN, (int) V8SF_FTYPE_V8HI },
29008 { OPTION_MASK_ISA_F16C, CODE_FOR_vcvtps2ph, "__builtin_ia32_vcvtps2ph", IX86_BUILTIN_CVTPS2PH, UNKNOWN, (int) V8HI_FTYPE_V4SF_INT },
29009 { OPTION_MASK_ISA_F16C, CODE_FOR_vcvtps2ph256, "__builtin_ia32_vcvtps2ph256", IX86_BUILTIN_CVTPS2PH256, UNKNOWN, (int) V8HI_FTYPE_V8SF_INT },
29011 /* BMI2 */
29012 { OPTION_MASK_ISA_BMI2, CODE_FOR_bmi2_bzhi_si3, "__builtin_ia32_bzhi_si", IX86_BUILTIN_BZHI32, UNKNOWN, (int) UINT_FTYPE_UINT_UINT },
29013 { OPTION_MASK_ISA_BMI2, CODE_FOR_bmi2_bzhi_di3, "__builtin_ia32_bzhi_di", IX86_BUILTIN_BZHI64, UNKNOWN, (int) UINT64_FTYPE_UINT64_UINT64 },
29014 { OPTION_MASK_ISA_BMI2, CODE_FOR_bmi2_pdep_si3, "__builtin_ia32_pdep_si", IX86_BUILTIN_PDEP32, UNKNOWN, (int) UINT_FTYPE_UINT_UINT },
29015 { OPTION_MASK_ISA_BMI2, CODE_FOR_bmi2_pdep_di3, "__builtin_ia32_pdep_di", IX86_BUILTIN_PDEP64, UNKNOWN, (int) UINT64_FTYPE_UINT64_UINT64 },
29016 { OPTION_MASK_ISA_BMI2, CODE_FOR_bmi2_pext_si3, "__builtin_ia32_pext_si", IX86_BUILTIN_PEXT32, UNKNOWN, (int) UINT_FTYPE_UINT_UINT },
29017 { OPTION_MASK_ISA_BMI2, CODE_FOR_bmi2_pext_di3, "__builtin_ia32_pext_di", IX86_BUILTIN_PEXT64, UNKNOWN, (int) UINT64_FTYPE_UINT64_UINT64 },
29018 };
29020 /* FMA4 and XOP. */
29021 #define MULTI_ARG_4_DF2_DI_I V2DF_FTYPE_V2DF_V2DF_V2DI_INT
29022 #define MULTI_ARG_4_DF2_DI_I1 V4DF_FTYPE_V4DF_V4DF_V4DI_INT
29023 #define MULTI_ARG_4_SF2_SI_I V4SF_FTYPE_V4SF_V4SF_V4SI_INT
29024 #define MULTI_ARG_4_SF2_SI_I1 V8SF_FTYPE_V8SF_V8SF_V8SI_INT
29025 #define MULTI_ARG_3_SF V4SF_FTYPE_V4SF_V4SF_V4SF
29026 #define MULTI_ARG_3_DF V2DF_FTYPE_V2DF_V2DF_V2DF
29027 #define MULTI_ARG_3_SF2 V8SF_FTYPE_V8SF_V8SF_V8SF
29028 #define MULTI_ARG_3_DF2 V4DF_FTYPE_V4DF_V4DF_V4DF
29029 #define MULTI_ARG_3_DI V2DI_FTYPE_V2DI_V2DI_V2DI
29030 #define MULTI_ARG_3_SI V4SI_FTYPE_V4SI_V4SI_V4SI
29031 #define MULTI_ARG_3_SI_DI V4SI_FTYPE_V4SI_V4SI_V2DI
29032 #define MULTI_ARG_3_HI V8HI_FTYPE_V8HI_V8HI_V8HI
29033 #define MULTI_ARG_3_HI_SI V8HI_FTYPE_V8HI_V8HI_V4SI
29034 #define MULTI_ARG_3_QI V16QI_FTYPE_V16QI_V16QI_V16QI
29035 #define MULTI_ARG_3_DI2 V4DI_FTYPE_V4DI_V4DI_V4DI
29036 #define MULTI_ARG_3_SI2 V8SI_FTYPE_V8SI_V8SI_V8SI
29037 #define MULTI_ARG_3_HI2 V16HI_FTYPE_V16HI_V16HI_V16HI
29038 #define MULTI_ARG_3_QI2 V32QI_FTYPE_V32QI_V32QI_V32QI
29039 #define MULTI_ARG_2_SF V4SF_FTYPE_V4SF_V4SF
29040 #define MULTI_ARG_2_DF V2DF_FTYPE_V2DF_V2DF
29041 #define MULTI_ARG_2_DI V2DI_FTYPE_V2DI_V2DI
29042 #define MULTI_ARG_2_SI V4SI_FTYPE_V4SI_V4SI
29043 #define MULTI_ARG_2_HI V8HI_FTYPE_V8HI_V8HI
29044 #define MULTI_ARG_2_QI V16QI_FTYPE_V16QI_V16QI
29045 #define MULTI_ARG_2_DI_IMM V2DI_FTYPE_V2DI_SI
29046 #define MULTI_ARG_2_SI_IMM V4SI_FTYPE_V4SI_SI
29047 #define MULTI_ARG_2_HI_IMM V8HI_FTYPE_V8HI_SI
29048 #define MULTI_ARG_2_QI_IMM V16QI_FTYPE_V16QI_SI
29049 #define MULTI_ARG_2_DI_CMP V2DI_FTYPE_V2DI_V2DI_CMP
29050 #define MULTI_ARG_2_SI_CMP V4SI_FTYPE_V4SI_V4SI_CMP
29051 #define MULTI_ARG_2_HI_CMP V8HI_FTYPE_V8HI_V8HI_CMP
29052 #define MULTI_ARG_2_QI_CMP V16QI_FTYPE_V16QI_V16QI_CMP
29053 #define MULTI_ARG_2_SF_TF V4SF_FTYPE_V4SF_V4SF_TF
29054 #define MULTI_ARG_2_DF_TF V2DF_FTYPE_V2DF_V2DF_TF
29055 #define MULTI_ARG_2_DI_TF V2DI_FTYPE_V2DI_V2DI_TF
29056 #define MULTI_ARG_2_SI_TF V4SI_FTYPE_V4SI_V4SI_TF
29057 #define MULTI_ARG_2_HI_TF V8HI_FTYPE_V8HI_V8HI_TF
29058 #define MULTI_ARG_2_QI_TF V16QI_FTYPE_V16QI_V16QI_TF
29059 #define MULTI_ARG_1_SF V4SF_FTYPE_V4SF
29060 #define MULTI_ARG_1_DF V2DF_FTYPE_V2DF
29061 #define MULTI_ARG_1_SF2 V8SF_FTYPE_V8SF
29062 #define MULTI_ARG_1_DF2 V4DF_FTYPE_V4DF
29063 #define MULTI_ARG_1_DI V2DI_FTYPE_V2DI
29064 #define MULTI_ARG_1_SI V4SI_FTYPE_V4SI
29065 #define MULTI_ARG_1_HI V8HI_FTYPE_V8HI
29066 #define MULTI_ARG_1_QI V16QI_FTYPE_V16QI
29067 #define MULTI_ARG_1_SI_DI V2DI_FTYPE_V4SI
29068 #define MULTI_ARG_1_HI_DI V2DI_FTYPE_V8HI
29069 #define MULTI_ARG_1_HI_SI V4SI_FTYPE_V8HI
29070 #define MULTI_ARG_1_QI_DI V2DI_FTYPE_V16QI
29071 #define MULTI_ARG_1_QI_SI V4SI_FTYPE_V16QI
29072 #define MULTI_ARG_1_QI_HI V8HI_FTYPE_V16QI
29075 {
29116 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v2di, "__builtin_ia32_vpcmov", IX86_BUILTIN_VPCMOV, UNKNOWN, (int)MULTI_ARG_3_DI },
29117 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v2di, "__builtin_ia32_vpcmov_v2di", IX86_BUILTIN_VPCMOV_V2DI, UNKNOWN, (int)MULTI_ARG_3_DI },
29118 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v4si, "__builtin_ia32_vpcmov_v4si", IX86_BUILTIN_VPCMOV_V4SI, UNKNOWN, (int)MULTI_ARG_3_SI },
29119 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v8hi, "__builtin_ia32_vpcmov_v8hi", IX86_BUILTIN_VPCMOV_V8HI, UNKNOWN, (int)MULTI_ARG_3_HI },
29120 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v16qi, "__builtin_ia32_vpcmov_v16qi",IX86_BUILTIN_VPCMOV_V16QI,UNKNOWN, (int)MULTI_ARG_3_QI },
29121 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v2df, "__builtin_ia32_vpcmov_v2df", IX86_BUILTIN_VPCMOV_V2DF, UNKNOWN, (int)MULTI_ARG_3_DF },
29122 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v4sf, "__builtin_ia32_vpcmov_v4sf", IX86_BUILTIN_VPCMOV_V4SF, UNKNOWN, (int)MULTI_ARG_3_SF },
29124 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v4di256, "__builtin_ia32_vpcmov256", IX86_BUILTIN_VPCMOV256, UNKNOWN, (int)MULTI_ARG_3_DI2 },
29125 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v4di256, "__builtin_ia32_vpcmov_v4di256", IX86_BUILTIN_VPCMOV_V4DI256, UNKNOWN, (int)MULTI_ARG_3_DI2 },
29126 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v8si256, "__builtin_ia32_vpcmov_v8si256", IX86_BUILTIN_VPCMOV_V8SI256, UNKNOWN, (int)MULTI_ARG_3_SI2 },
29127 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v16hi256, "__builtin_ia32_vpcmov_v16hi256", IX86_BUILTIN_VPCMOV_V16HI256, UNKNOWN, (int)MULTI_ARG_3_HI2 },
29128 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v32qi256, "__builtin_ia32_vpcmov_v32qi256", IX86_BUILTIN_VPCMOV_V32QI256, UNKNOWN, (int)MULTI_ARG_3_QI2 },
29129 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v4df256, "__builtin_ia32_vpcmov_v4df256", IX86_BUILTIN_VPCMOV_V4DF256, UNKNOWN, (int)MULTI_ARG_3_DF2 },
29130 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v8sf256, "__builtin_ia32_vpcmov_v8sf256", IX86_BUILTIN_VPCMOV_V8SF256, UNKNOWN, (int)MULTI_ARG_3_SF2 },
29132 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pperm, "__builtin_ia32_vpperm", IX86_BUILTIN_VPPERM, UNKNOWN, (int)MULTI_ARG_3_QI },
29134 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacssww, "__builtin_ia32_vpmacssww", IX86_BUILTIN_VPMACSSWW, UNKNOWN, (int)MULTI_ARG_3_HI },
29135 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacsww, "__builtin_ia32_vpmacsww", IX86_BUILTIN_VPMACSWW, UNKNOWN, (int)MULTI_ARG_3_HI },
29136 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacsswd, "__builtin_ia32_vpmacsswd", IX86_BUILTIN_VPMACSSWD, UNKNOWN, (int)MULTI_ARG_3_HI_SI },
29137 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacswd, "__builtin_ia32_vpmacswd", IX86_BUILTIN_VPMACSWD, UNKNOWN, (int)MULTI_ARG_3_HI_SI },
29138 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacssdd, "__builtin_ia32_vpmacssdd", IX86_BUILTIN_VPMACSSDD, UNKNOWN, (int)MULTI_ARG_3_SI },
29139 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacsdd, "__builtin_ia32_vpmacsdd", IX86_BUILTIN_VPMACSDD, UNKNOWN, (int)MULTI_ARG_3_SI },
29140 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacssdql, "__builtin_ia32_vpmacssdql", IX86_BUILTIN_VPMACSSDQL, UNKNOWN, (int)MULTI_ARG_3_SI_DI },
29141 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacssdqh, "__builtin_ia32_vpmacssdqh", IX86_BUILTIN_VPMACSSDQH, UNKNOWN, (int)MULTI_ARG_3_SI_DI },
29142 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacsdql, "__builtin_ia32_vpmacsdql", IX86_BUILTIN_VPMACSDQL, UNKNOWN, (int)MULTI_ARG_3_SI_DI },
29143 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmacsdqh, "__builtin_ia32_vpmacsdqh", IX86_BUILTIN_VPMACSDQH, UNKNOWN, (int)MULTI_ARG_3_SI_DI },
29144 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmadcsswd, "__builtin_ia32_vpmadcsswd", IX86_BUILTIN_VPMADCSSWD, UNKNOWN, (int)MULTI_ARG_3_HI_SI },
29145 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pmadcswd, "__builtin_ia32_vpmadcswd", IX86_BUILTIN_VPMADCSWD, UNKNOWN, (int)MULTI_ARG_3_HI_SI },
29147 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vrotlv2di3, "__builtin_ia32_vprotq", IX86_BUILTIN_VPROTQ, UNKNOWN, (int)MULTI_ARG_2_DI },
29148 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vrotlv4si3, "__builtin_ia32_vprotd", IX86_BUILTIN_VPROTD, UNKNOWN, (int)MULTI_ARG_2_SI },
29149 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vrotlv8hi3, "__builtin_ia32_vprotw", IX86_BUILTIN_VPROTW, UNKNOWN, (int)MULTI_ARG_2_HI },
29150 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vrotlv16qi3, "__builtin_ia32_vprotb", IX86_BUILTIN_VPROTB, UNKNOWN, (int)MULTI_ARG_2_QI },
29151 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_rotlv2di3, "__builtin_ia32_vprotqi", IX86_BUILTIN_VPROTQ_IMM, UNKNOWN, (int)MULTI_ARG_2_DI_IMM },
29152 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_rotlv4si3, "__builtin_ia32_vprotdi", IX86_BUILTIN_VPROTD_IMM, UNKNOWN, (int)MULTI_ARG_2_SI_IMM },
29153 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_rotlv8hi3, "__builtin_ia32_vprotwi", IX86_BUILTIN_VPROTW_IMM, UNKNOWN, (int)MULTI_ARG_2_HI_IMM },
29154 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_rotlv16qi3, "__builtin_ia32_vprotbi", IX86_BUILTIN_VPROTB_IMM, UNKNOWN, (int)MULTI_ARG_2_QI_IMM },
29155 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shav2di3, "__builtin_ia32_vpshaq", IX86_BUILTIN_VPSHAQ, UNKNOWN, (int)MULTI_ARG_2_DI },
29156 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shav4si3, "__builtin_ia32_vpshad", IX86_BUILTIN_VPSHAD, UNKNOWN, (int)MULTI_ARG_2_SI },
29157 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shav8hi3, "__builtin_ia32_vpshaw", IX86_BUILTIN_VPSHAW, UNKNOWN, (int)MULTI_ARG_2_HI },
29158 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shav16qi3, "__builtin_ia32_vpshab", IX86_BUILTIN_VPSHAB, UNKNOWN, (int)MULTI_ARG_2_QI },
29159 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shlv2di3, "__builtin_ia32_vpshlq", IX86_BUILTIN_VPSHLQ, UNKNOWN, (int)MULTI_ARG_2_DI },
29160 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shlv4si3, "__builtin_ia32_vpshld", IX86_BUILTIN_VPSHLD, UNKNOWN, (int)MULTI_ARG_2_SI },
29161 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shlv8hi3, "__builtin_ia32_vpshlw", IX86_BUILTIN_VPSHLW, UNKNOWN, (int)MULTI_ARG_2_HI },
29162 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shlv16qi3, "__builtin_ia32_vpshlb", IX86_BUILTIN_VPSHLB, UNKNOWN, (int)MULTI_ARG_2_QI },
29164 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vmfrczv4sf2, "__builtin_ia32_vfrczss", IX86_BUILTIN_VFRCZSS, UNKNOWN, (int)MULTI_ARG_2_SF },
29165 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vmfrczv2df2, "__builtin_ia32_vfrczsd", IX86_BUILTIN_VFRCZSD, UNKNOWN, (int)MULTI_ARG_2_DF },
29166 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_frczv4sf2, "__builtin_ia32_vfrczps", IX86_BUILTIN_VFRCZPS, UNKNOWN, (int)MULTI_ARG_1_SF },
29167 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_frczv2df2, "__builtin_ia32_vfrczpd", IX86_BUILTIN_VFRCZPD, UNKNOWN, (int)MULTI_ARG_1_DF },
29168 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_frczv8sf2, "__builtin_ia32_vfrczps256", IX86_BUILTIN_VFRCZPS256, UNKNOWN, (int)MULTI_ARG_1_SF2 },
29169 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_frczv4df2, "__builtin_ia32_vfrczpd256", IX86_BUILTIN_VFRCZPD256, UNKNOWN, (int)MULTI_ARG_1_DF2 },
29171 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddbw, "__builtin_ia32_vphaddbw", IX86_BUILTIN_VPHADDBW, UNKNOWN, (int)MULTI_ARG_1_QI_HI },
29172 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddbd, "__builtin_ia32_vphaddbd", IX86_BUILTIN_VPHADDBD, UNKNOWN, (int)MULTI_ARG_1_QI_SI },
29173 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddbq, "__builtin_ia32_vphaddbq", IX86_BUILTIN_VPHADDBQ, UNKNOWN, (int)MULTI_ARG_1_QI_DI },
29174 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddwd, "__builtin_ia32_vphaddwd", IX86_BUILTIN_VPHADDWD, UNKNOWN, (int)MULTI_ARG_1_HI_SI },
29175 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddwq, "__builtin_ia32_vphaddwq", IX86_BUILTIN_VPHADDWQ, UNKNOWN, (int)MULTI_ARG_1_HI_DI },
29176 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phadddq, "__builtin_ia32_vphadddq", IX86_BUILTIN_VPHADDDQ, UNKNOWN, (int)MULTI_ARG_1_SI_DI },
29177 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddubw, "__builtin_ia32_vphaddubw", IX86_BUILTIN_VPHADDUBW, UNKNOWN, (int)MULTI_ARG_1_QI_HI },
29178 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddubd, "__builtin_ia32_vphaddubd", IX86_BUILTIN_VPHADDUBD, UNKNOWN, (int)MULTI_ARG_1_QI_SI },
29179 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddubq, "__builtin_ia32_vphaddubq", IX86_BUILTIN_VPHADDUBQ, UNKNOWN, (int)MULTI_ARG_1_QI_DI },
29180 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phadduwd, "__builtin_ia32_vphadduwd", IX86_BUILTIN_VPHADDUWD, UNKNOWN, (int)MULTI_ARG_1_HI_SI },
29181 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phadduwq, "__builtin_ia32_vphadduwq", IX86_BUILTIN_VPHADDUWQ, UNKNOWN, (int)MULTI_ARG_1_HI_DI },
29182 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddudq, "__builtin_ia32_vphaddudq", IX86_BUILTIN_VPHADDUDQ, UNKNOWN, (int)MULTI_ARG_1_SI_DI },
29183 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phsubbw, "__builtin_ia32_vphsubbw", IX86_BUILTIN_VPHSUBBW, UNKNOWN, (int)MULTI_ARG_1_QI_HI },
29184 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phsubwd, "__builtin_ia32_vphsubwd", IX86_BUILTIN_VPHSUBWD, UNKNOWN, (int)MULTI_ARG_1_HI_SI },
29185 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phsubdq, "__builtin_ia32_vphsubdq", IX86_BUILTIN_VPHSUBDQ, UNKNOWN, (int)MULTI_ARG_1_SI_DI },
29187 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv16qi3, "__builtin_ia32_vpcomeqb", IX86_BUILTIN_VPCOMEQB, EQ, (int)MULTI_ARG_2_QI_CMP },
29188 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv16qi3, "__builtin_ia32_vpcomneb", IX86_BUILTIN_VPCOMNEB, NE, (int)MULTI_ARG_2_QI_CMP },
29189 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv16qi3, "__builtin_ia32_vpcomneqb", IX86_BUILTIN_VPCOMNEB, NE, (int)MULTI_ARG_2_QI_CMP },
29190 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv16qi3, "__builtin_ia32_vpcomltb", IX86_BUILTIN_VPCOMLTB, LT, (int)MULTI_ARG_2_QI_CMP },
29191 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv16qi3, "__builtin_ia32_vpcomleb", IX86_BUILTIN_VPCOMLEB, LE, (int)MULTI_ARG_2_QI_CMP },
29192 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv16qi3, "__builtin_ia32_vpcomgtb", IX86_BUILTIN_VPCOMGTB, GT, (int)MULTI_ARG_2_QI_CMP },
29193 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv16qi3, "__builtin_ia32_vpcomgeb", IX86_BUILTIN_VPCOMGEB, GE, (int)MULTI_ARG_2_QI_CMP },
29195 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv8hi3, "__builtin_ia32_vpcomeqw", IX86_BUILTIN_VPCOMEQW, EQ, (int)MULTI_ARG_2_HI_CMP },
29196 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv8hi3, "__builtin_ia32_vpcomnew", IX86_BUILTIN_VPCOMNEW, NE, (int)MULTI_ARG_2_HI_CMP },
29197 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv8hi3, "__builtin_ia32_vpcomneqw", IX86_BUILTIN_VPCOMNEW, NE, (int)MULTI_ARG_2_HI_CMP },
29198 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv8hi3, "__builtin_ia32_vpcomltw", IX86_BUILTIN_VPCOMLTW, LT, (int)MULTI_ARG_2_HI_CMP },
29199 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv8hi3, "__builtin_ia32_vpcomlew", IX86_BUILTIN_VPCOMLEW, LE, (int)MULTI_ARG_2_HI_CMP },
29200 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv8hi3, "__builtin_ia32_vpcomgtw", IX86_BUILTIN_VPCOMGTW, GT, (int)MULTI_ARG_2_HI_CMP },
29201 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv8hi3, "__builtin_ia32_vpcomgew", IX86_BUILTIN_VPCOMGEW, GE, (int)MULTI_ARG_2_HI_CMP },
29203 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv4si3, "__builtin_ia32_vpcomeqd", IX86_BUILTIN_VPCOMEQD, EQ, (int)MULTI_ARG_2_SI_CMP },
29204 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv4si3, "__builtin_ia32_vpcomned", IX86_BUILTIN_VPCOMNED, NE, (int)MULTI_ARG_2_SI_CMP },
29205 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv4si3, "__builtin_ia32_vpcomneqd", IX86_BUILTIN_VPCOMNED, NE, (int)MULTI_ARG_2_SI_CMP },
29206 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv4si3, "__builtin_ia32_vpcomltd", IX86_BUILTIN_VPCOMLTD, LT, (int)MULTI_ARG_2_SI_CMP },
29207 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv4si3, "__builtin_ia32_vpcomled", IX86_BUILTIN_VPCOMLED, LE, (int)MULTI_ARG_2_SI_CMP },
29208 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv4si3, "__builtin_ia32_vpcomgtd", IX86_BUILTIN_VPCOMGTD, GT, (int)MULTI_ARG_2_SI_CMP },
29209 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv4si3, "__builtin_ia32_vpcomged", IX86_BUILTIN_VPCOMGED, GE, (int)MULTI_ARG_2_SI_CMP },
29211 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv2di3, "__builtin_ia32_vpcomeqq", IX86_BUILTIN_VPCOMEQQ, EQ, (int)MULTI_ARG_2_DI_CMP },
29212 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv2di3, "__builtin_ia32_vpcomneq", IX86_BUILTIN_VPCOMNEQ, NE, (int)MULTI_ARG_2_DI_CMP },
29213 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv2di3, "__builtin_ia32_vpcomneqq", IX86_BUILTIN_VPCOMNEQ, NE, (int)MULTI_ARG_2_DI_CMP },
29214 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv2di3, "__builtin_ia32_vpcomltq", IX86_BUILTIN_VPCOMLTQ, LT, (int)MULTI_ARG_2_DI_CMP },
29215 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv2di3, "__builtin_ia32_vpcomleq", IX86_BUILTIN_VPCOMLEQ, LE, (int)MULTI_ARG_2_DI_CMP },
29216 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv2di3, "__builtin_ia32_vpcomgtq", IX86_BUILTIN_VPCOMGTQ, GT, (int)MULTI_ARG_2_DI_CMP },
29217 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmpv2di3, "__builtin_ia32_vpcomgeq", IX86_BUILTIN_VPCOMGEQ, GE, (int)MULTI_ARG_2_DI_CMP },
29219 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v16qi3,"__builtin_ia32_vpcomequb", IX86_BUILTIN_VPCOMEQUB, EQ, (int)MULTI_ARG_2_QI_CMP },
29220 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v16qi3,"__builtin_ia32_vpcomneub", IX86_BUILTIN_VPCOMNEUB, NE, (int)MULTI_ARG_2_QI_CMP },
29221 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v16qi3,"__builtin_ia32_vpcomnequb", IX86_BUILTIN_VPCOMNEUB, NE, (int)MULTI_ARG_2_QI_CMP },
29222 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv16qi3, "__builtin_ia32_vpcomltub", IX86_BUILTIN_VPCOMLTUB, LTU, (int)MULTI_ARG_2_QI_CMP },
29223 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv16qi3, "__builtin_ia32_vpcomleub", IX86_BUILTIN_VPCOMLEUB, LEU, (int)MULTI_ARG_2_QI_CMP },
29224 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv16qi3, "__builtin_ia32_vpcomgtub", IX86_BUILTIN_VPCOMGTUB, GTU, (int)MULTI_ARG_2_QI_CMP },
29225 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv16qi3, "__builtin_ia32_vpcomgeub", IX86_BUILTIN_VPCOMGEUB, GEU, (int)MULTI_ARG_2_QI_CMP },
29227 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v8hi3, "__builtin_ia32_vpcomequw", IX86_BUILTIN_VPCOMEQUW, EQ, (int)MULTI_ARG_2_HI_CMP },
29228 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v8hi3, "__builtin_ia32_vpcomneuw", IX86_BUILTIN_VPCOMNEUW, NE, (int)MULTI_ARG_2_HI_CMP },
29229 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v8hi3, "__builtin_ia32_vpcomnequw", IX86_BUILTIN_VPCOMNEUW, NE, (int)MULTI_ARG_2_HI_CMP },
29230 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv8hi3, "__builtin_ia32_vpcomltuw", IX86_BUILTIN_VPCOMLTUW, LTU, (int)MULTI_ARG_2_HI_CMP },
29231 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv8hi3, "__builtin_ia32_vpcomleuw", IX86_BUILTIN_VPCOMLEUW, LEU, (int)MULTI_ARG_2_HI_CMP },
29232 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv8hi3, "__builtin_ia32_vpcomgtuw", IX86_BUILTIN_VPCOMGTUW, GTU, (int)MULTI_ARG_2_HI_CMP },
29233 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv8hi3, "__builtin_ia32_vpcomgeuw", IX86_BUILTIN_VPCOMGEUW, GEU, (int)MULTI_ARG_2_HI_CMP },
29235 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v4si3, "__builtin_ia32_vpcomequd", IX86_BUILTIN_VPCOMEQUD, EQ, (int)MULTI_ARG_2_SI_CMP },
29236 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v4si3, "__builtin_ia32_vpcomneud", IX86_BUILTIN_VPCOMNEUD, NE, (int)MULTI_ARG_2_SI_CMP },
29237 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v4si3, "__builtin_ia32_vpcomnequd", IX86_BUILTIN_VPCOMNEUD, NE, (int)MULTI_ARG_2_SI_CMP },
29238 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv4si3, "__builtin_ia32_vpcomltud", IX86_BUILTIN_VPCOMLTUD, LTU, (int)MULTI_ARG_2_SI_CMP },
29239 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv4si3, "__builtin_ia32_vpcomleud", IX86_BUILTIN_VPCOMLEUD, LEU, (int)MULTI_ARG_2_SI_CMP },
29240 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv4si3, "__builtin_ia32_vpcomgtud", IX86_BUILTIN_VPCOMGTUD, GTU, (int)MULTI_ARG_2_SI_CMP },
29241 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv4si3, "__builtin_ia32_vpcomgeud", IX86_BUILTIN_VPCOMGEUD, GEU, (int)MULTI_ARG_2_SI_CMP },
29243 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v2di3, "__builtin_ia32_vpcomequq", IX86_BUILTIN_VPCOMEQUQ, EQ, (int)MULTI_ARG_2_DI_CMP },
29244 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v2di3, "__builtin_ia32_vpcomneuq", IX86_BUILTIN_VPCOMNEUQ, NE, (int)MULTI_ARG_2_DI_CMP },
29245 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_uns2v2di3, "__builtin_ia32_vpcomnequq", IX86_BUILTIN_VPCOMNEUQ, NE, (int)MULTI_ARG_2_DI_CMP },
29246 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv2di3, "__builtin_ia32_vpcomltuq", IX86_BUILTIN_VPCOMLTUQ, LTU, (int)MULTI_ARG_2_DI_CMP },
29247 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv2di3, "__builtin_ia32_vpcomleuq", IX86_BUILTIN_VPCOMLEUQ, LEU, (int)MULTI_ARG_2_DI_CMP },
29248 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv2di3, "__builtin_ia32_vpcomgtuq", IX86_BUILTIN_VPCOMGTUQ, GTU, (int)MULTI_ARG_2_DI_CMP },
29249 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_maskcmp_unsv2di3, "__builtin_ia32_vpcomgeuq", IX86_BUILTIN_VPCOMGEUQ, GEU, (int)MULTI_ARG_2_DI_CMP },
29251 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv16qi3, "__builtin_ia32_vpcomfalseb", IX86_BUILTIN_VPCOMFALSEB, (enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_QI_TF },
29252 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv8hi3, "__builtin_ia32_vpcomfalsew", IX86_BUILTIN_VPCOMFALSEW, (enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_HI_TF },
29253 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv4si3, "__builtin_ia32_vpcomfalsed", IX86_BUILTIN_VPCOMFALSED, (enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_SI_TF },
29254 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv2di3, "__builtin_ia32_vpcomfalseq", IX86_BUILTIN_VPCOMFALSEQ, (enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_DI_TF },
29255 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv16qi3, "__builtin_ia32_vpcomfalseub",IX86_BUILTIN_VPCOMFALSEUB,(enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_QI_TF },
29256 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv8hi3, "__builtin_ia32_vpcomfalseuw",IX86_BUILTIN_VPCOMFALSEUW,(enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_HI_TF },
29257 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv4si3, "__builtin_ia32_vpcomfalseud",IX86_BUILTIN_VPCOMFALSEUD,(enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_SI_TF },
29258 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv2di3, "__builtin_ia32_vpcomfalseuq",IX86_BUILTIN_VPCOMFALSEUQ,(enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_DI_TF },
29260 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv16qi3, "__builtin_ia32_vpcomtrueb", IX86_BUILTIN_VPCOMTRUEB, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_QI_TF },
29261 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv8hi3, "__builtin_ia32_vpcomtruew", IX86_BUILTIN_VPCOMTRUEW, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_HI_TF },
29262 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv4si3, "__builtin_ia32_vpcomtrued", IX86_BUILTIN_VPCOMTRUED, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_SI_TF },
29263 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv2di3, "__builtin_ia32_vpcomtrueq", IX86_BUILTIN_VPCOMTRUEQ, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_DI_TF },
29264 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv16qi3, "__builtin_ia32_vpcomtrueub", IX86_BUILTIN_VPCOMTRUEUB, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_QI_TF },
29265 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv8hi3, "__builtin_ia32_vpcomtrueuw", IX86_BUILTIN_VPCOMTRUEUW, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_HI_TF },
29266 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv4si3, "__builtin_ia32_vpcomtrueud", IX86_BUILTIN_VPCOMTRUEUD, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_SI_TF },
29267 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv2di3, "__builtin_ia32_vpcomtrueuq", IX86_BUILTIN_VPCOMTRUEUQ, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_DI_TF },
29269 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vpermil2v2df3, "__builtin_ia32_vpermil2pd", IX86_BUILTIN_VPERMIL2PD, UNKNOWN, (int)MULTI_ARG_4_DF2_DI_I },
29270 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vpermil2v4sf3, "__builtin_ia32_vpermil2ps", IX86_BUILTIN_VPERMIL2PS, UNKNOWN, (int)MULTI_ARG_4_SF2_SI_I },
29271 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vpermil2v4df3, "__builtin_ia32_vpermil2pd256", IX86_BUILTIN_VPERMIL2PD256, UNKNOWN, (int)MULTI_ARG_4_DF2_DI_I1 },
29272 { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vpermil2v8sf3, "__builtin_ia32_vpermil2ps256", IX86_BUILTIN_VPERMIL2PS256, UNKNOWN, (int)MULTI_ARG_4_SF2_SI_I1 },
29274 };
29275 \f
29276 /* TM vector builtins. */
29278 /* Reuse the existing x86-specific `struct builtin_description' cause
29279 we're lazy. Add casts to make them fit. */
29281 {
29282 { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_WM64", (enum ix86_builtins) BUILT_IN_TM_STORE_M64, UNKNOWN, VOID_FTYPE_PV2SI_V2SI },
29283 { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_WaRM64", (enum ix86_builtins) BUILT_IN_TM_STORE_WAR_M64, UNKNOWN, VOID_FTYPE_PV2SI_V2SI },
29284 { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_WaWM64", (enum ix86_builtins) BUILT_IN_TM_STORE_WAW_M64, UNKNOWN, VOID_FTYPE_PV2SI_V2SI },
29285 { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_RM64", (enum ix86_builtins) BUILT_IN_TM_LOAD_M64, UNKNOWN, V2SI_FTYPE_PCV2SI },
29286 { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_RaRM64", (enum ix86_builtins) BUILT_IN_TM_LOAD_RAR_M64, UNKNOWN, V2SI_FTYPE_PCV2SI },
29287 { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_RaWM64", (enum ix86_builtins) BUILT_IN_TM_LOAD_RAW_M64, UNKNOWN, V2SI_FTYPE_PCV2SI },
29288 { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_RfWM64", (enum ix86_builtins) BUILT_IN_TM_LOAD_RFW_M64, UNKNOWN, V2SI_FTYPE_PCV2SI },
29290 { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_WM128", (enum ix86_builtins) BUILT_IN_TM_STORE_M128, UNKNOWN, VOID_FTYPE_PV4SF_V4SF },
29291 { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_WaRM128", (enum ix86_builtins) BUILT_IN_TM_STORE_WAR_M128, UNKNOWN, VOID_FTYPE_PV4SF_V4SF },
29292 { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_WaWM128", (enum ix86_builtins) BUILT_IN_TM_STORE_WAW_M128, UNKNOWN, VOID_FTYPE_PV4SF_V4SF },
29293 { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_RM128", (enum ix86_builtins) BUILT_IN_TM_LOAD_M128, UNKNOWN, V4SF_FTYPE_PCV4SF },
29294 { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_RaRM128", (enum ix86_builtins) BUILT_IN_TM_LOAD_RAR_M128, UNKNOWN, V4SF_FTYPE_PCV4SF },
29295 { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_RaWM128", (enum ix86_builtins) BUILT_IN_TM_LOAD_RAW_M128, UNKNOWN, V4SF_FTYPE_PCV4SF },
29296 { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_RfWM128", (enum ix86_builtins) BUILT_IN_TM_LOAD_RFW_M128, UNKNOWN, V4SF_FTYPE_PCV4SF },
29298 { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_WM256", (enum ix86_builtins) BUILT_IN_TM_STORE_M256, UNKNOWN, VOID_FTYPE_PV8SF_V8SF },
29299 { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_WaRM256", (enum ix86_builtins) BUILT_IN_TM_STORE_WAR_M256, UNKNOWN, VOID_FTYPE_PV8SF_V8SF },
29300 { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_WaWM256", (enum ix86_builtins) BUILT_IN_TM_STORE_WAW_M256, UNKNOWN, VOID_FTYPE_PV8SF_V8SF },
29301 { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_RM256", (enum ix86_builtins) BUILT_IN_TM_LOAD_M256, UNKNOWN, V8SF_FTYPE_PCV8SF },
29302 { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_RaRM256", (enum ix86_builtins) BUILT_IN_TM_LOAD_RAR_M256, UNKNOWN, V8SF_FTYPE_PCV8SF },
29303 { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_RaWM256", (enum ix86_builtins) BUILT_IN_TM_LOAD_RAW_M256, UNKNOWN, V8SF_FTYPE_PCV8SF },
29304 { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_RfWM256", (enum ix86_builtins) BUILT_IN_TM_LOAD_RFW_M256, UNKNOWN, V8SF_FTYPE_PCV8SF },
29306 { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_LM64", (enum ix86_builtins) BUILT_IN_TM_LOG_M64, UNKNOWN, VOID_FTYPE_PCVOID },
29307 { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_LM128", (enum ix86_builtins) BUILT_IN_TM_LOG_M128, UNKNOWN, VOID_FTYPE_PCVOID },
29308 { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_LM256", (enum ix86_builtins) BUILT_IN_TM_LOG_M256, UNKNOWN, VOID_FTYPE_PCVOID },
29309 };
29311 /* TM callbacks. */
29313 /* Return the builtin decl needed to load a vector of TYPE. */
29315 static tree
29317 {
29319 {
29321 {
29328 }
29329 }
29331 }
29333 /* Return the builtin decl needed to store a vector of TYPE. */
29335 static tree
29337 {
29339 {
29341 {
29348 }
29349 }
29351 }
29352 \f
29353 /* Initialize the transactional memory vector load/store builtins. */
29355 static void
29357 {
29361 tree decl;
29368 /* If there are no builtins defined, we must be compiling in a
29369 language without trans-mem support. */
29373 /* Use whatever attributes a normal TM load has. */
29377 /* Use whatever attributes a normal TM store has. */
29381 /* Use whatever attributes a normal TM log has. */
29389 {
29393 {
29401 {
29404 }
29406 {
29409 }
29410 else
29411 {
29414 }
29416 /* The builtin without the prefix for
29417 calling it directly. */
29419 attrs);
29420 /* add_builtin_function() will set the DECL_ATTRIBUTES, now
29421 set the TYPE_ATTRIBUTES. */
29425 }
29426 }
29427 }
29429 /* Set up all the MMX/SSE builtins, even builtins for instructions that are not
29430 in the current target ISA to allow the user to compile particular modules
29431 with different target specific options that differ from the command line
29432 options. */
29433 static void
29435 {
29440 /* Add all special builtins with variable number of operands. */
29444 {
29450 }
29452 /* Add all builtins with variable number of operands. */
29456 {
29462 }
29464 /* pcmpestr[im] insns. */
29468 {
29471 else
29474 }
29476 /* pcmpistr[im] insns. */
29480 {
29483 else
29486 }
29488 /* comi/ucomi insns. */
29490 {
29493 else
29496 }
29498 /* SSE */
29504 /* SSE or 3DNow!A */
29507 IX86_BUILTIN_MASKMOVQ);
29509 /* SSE2 */
29518 /* SSE3. */
29524 /* AES */
29538 /* PCLMUL */
29542 /* RDRND */
29549 IX86_BUILTIN_RDRAND64_STEP);
29551 /* AVX2 */
29553 V2DF_FTYPE_V2DF_PCDOUBLE_V4SI_V2DF_INT,
29554 IX86_BUILTIN_GATHERSIV2DF);
29557 V4DF_FTYPE_V4DF_PCDOUBLE_V4SI_V4DF_INT,
29558 IX86_BUILTIN_GATHERSIV4DF);
29561 V2DF_FTYPE_V2DF_PCDOUBLE_V2DI_V2DF_INT,
29562 IX86_BUILTIN_GATHERDIV2DF);
29565 V4DF_FTYPE_V4DF_PCDOUBLE_V4DI_V4DF_INT,
29566 IX86_BUILTIN_GATHERDIV4DF);
29569 V4SF_FTYPE_V4SF_PCFLOAT_V4SI_V4SF_INT,
29570 IX86_BUILTIN_GATHERSIV4SF);
29573 V8SF_FTYPE_V8SF_PCFLOAT_V8SI_V8SF_INT,
29574 IX86_BUILTIN_GATHERSIV8SF);
29577 V4SF_FTYPE_V4SF_PCFLOAT_V2DI_V4SF_INT,
29578 IX86_BUILTIN_GATHERDIV4SF);
29581 V4SF_FTYPE_V4SF_PCFLOAT_V4DI_V4SF_INT,
29582 IX86_BUILTIN_GATHERDIV8SF);
29585 V2DI_FTYPE_V2DI_PCINT64_V4SI_V2DI_INT,
29586 IX86_BUILTIN_GATHERSIV2DI);
29589 V4DI_FTYPE_V4DI_PCINT64_V4SI_V4DI_INT,
29590 IX86_BUILTIN_GATHERSIV4DI);
29593 V2DI_FTYPE_V2DI_PCINT64_V2DI_V2DI_INT,
29594 IX86_BUILTIN_GATHERDIV2DI);
29597 V4DI_FTYPE_V4DI_PCINT64_V4DI_V4DI_INT,
29598 IX86_BUILTIN_GATHERDIV4DI);
29601 V4SI_FTYPE_V4SI_PCINT_V4SI_V4SI_INT,
29602 IX86_BUILTIN_GATHERSIV4SI);
29605 V8SI_FTYPE_V8SI_PCINT_V8SI_V8SI_INT,
29606 IX86_BUILTIN_GATHERSIV8SI);
29609 V4SI_FTYPE_V4SI_PCINT_V2DI_V4SI_INT,
29610 IX86_BUILTIN_GATHERDIV4SI);
29613 V4SI_FTYPE_V4SI_PCINT_V4DI_V4SI_INT,
29614 IX86_BUILTIN_GATHERDIV8SI);
29617 V4DF_FTYPE_V4DF_PCDOUBLE_V8SI_V4DF_INT,
29618 IX86_BUILTIN_GATHERALTSIV4DF);
29621 V8SF_FTYPE_V8SF_PCFLOAT_V4DI_V8SF_INT,
29622 IX86_BUILTIN_GATHERALTDIV8SF);
29625 V4DI_FTYPE_V4DI_PCINT64_V8SI_V4DI_INT,
29626 IX86_BUILTIN_GATHERALTSIV4DI);
29629 V8SI_FTYPE_V8SI_PCINT_V4DI_V8SI_INT,
29630 IX86_BUILTIN_GATHERALTDIV8SI);
29632 /* RTM. */
29636 /* MMX access to the vec_init patterns. */
29641 V4HI_FTYPE_HI_HI_HI_HI,
29642 IX86_BUILTIN_VEC_INIT_V4HI);
29645 V8QI_FTYPE_QI_QI_QI_QI_QI_QI_QI_QI,
29646 IX86_BUILTIN_VEC_INIT_V8QI);
29648 /* Access to the vec_extract patterns. */
29670 /* Access to the vec_set patterns. */
29691 /* RDSEED */
29700 /* ADCX */
29705 UCHAR_FTYPE_UCHAR_ULONGLONG_ULONGLONG_PULONGLONG,
29706 IX86_BUILTIN_ADDCARRYX64);
29708 /* Add FMA4 multi-arg argument instructions */
29710 {
29716 }
29717 }
29719 /* This adds a condition to the basic_block NEW_BB in function FUNCTION_DECL
29720 to return a pointer to VERSION_DECL if the outcome of the expression
29721 formed by PREDICATE_CHAIN is true. This function will be called during
29722 version dispatch to decide which function version to execute. It returns
29723 the basic block at the end, to which more conditions can be added. */
29725 static basic_block
29728 {
29729 gimple return_stmt;
29731 gimple convert_stmt;
29732 gimple call_cond_stmt;
29733 gimple if_else_stmt;
29739 gimple_seq gseq;
29756 {
29764 }
29767 {
29782 else
29783 {
29784 gimple assign_stmt;
29785 /* Use MIN_EXPR to check if any integer is zero?.
29786 and_expr_var = min_expr <cond_var, and_expr_var> */
29794 }
29795 }
29798 integer_zero_node,
29828 }
29830 /* This parses the attribute arguments to target in DECL and determines
29831 the right builtin to use to match the platform specification.
29832 It returns the priority value for this version decl. If PREDICATE_LIST
29833 is not NULL, it stores the list of cpu features that need to be checked
29834 before dispatching this function. */
29836 static unsigned int
29838 {
29839 tree attrs;
29841 tree target_node;
29848 /* Priority of i386 features, greater value is higher priority. This is
29849 used to decide the order in which function dispatch must happen. For
29850 instance, a version specialized for SSE4.2 should be checked for dispatch
29851 before a version for SSE3, as SSE4.2 implies SSE3. */
29852 enum feature_priority
29853 {
29855 P_MMX,
29856 P_SSE,
29857 P_SSE2,
29858 P_SSE3,
29859 P_SSSE3,
29860 P_PROC_SSSE3,
29861 P_SSE4_a,
29862 P_PROC_SSE4_a,
29863 P_SSE4_1,
29864 P_SSE4_2,
29865 P_PROC_SSE4_2,
29866 P_POPCNT,
29867 P_AVX,
29868 P_AVX2,
29869 P_FMA,
29870 P_PROC_FMA
29871 };
29875 /* These are the target attribute strings for which a dispatcher is
29876 available, from fold_builtin_cpu. */
29878 static struct _feature_list
29879 {
29882 }
29884 {
29895 };
29898 static unsigned int NUM_FEATURES
29914 /* Return priority zero for default function. */
29918 /* Handle arch= if specified. For priority, set it to be 1 more than
29919 the best instruction set the processor can handle. For instance, if
29920 there is a version for atom and a version for ssse3 (the highest ISA
29921 priority for atom), the atom version must be checked for dispatch
29922 before the ssse3 version. */
29924 {
29934 {
29936 {
29965 }
29966 }
29971 {
29975 }
29978 {
29980 /* For a C string literal the length includes the trailing NULL. */
29983 predicate_chain);
29984 }
29985 }
29987 /* Process feature name. */
29994 {
29995 /* Do not process "arch=" */
29997 {
30000 }
30002 {
30004 {
30006 {
30011 predicate_chain);
30012 }
30013 /* Find the maximum priority feature. */
30018 }
30019 }
30021 {
30025 }
30027 }
30031 {
30034 attrs_str);
30036 }
30038 {
30041 }
30044 }
30046 /* This compares the priority of target features in function DECL1
30047 and DECL2. It returns positive value if DECL1 is higher priority,
30048 negative value if DECL2 is higher priority and 0 if they are the
30049 same. */
30051 static int
30053 {
30058 }
30060 /* V1 and V2 point to function versions with different priorities
30061 based on the target ISA. This function compares their priorities. */
30063 static int
30065 {
30067 {
30068 tree version_decl;
30069 tree predicate_chain;
30076 }
30078 /* This function generates the dispatch function for
30079 multi-versioned functions. DISPATCH_DECL is the function which will
30080 contain the dispatch logic. FNDECLS are the function choices for
30081 dispatch, and is a tree chain. EMPTY_BB is the basic block pointer
30082 in DISPATCH_DECL in which the dispatch code is generated. */
30084 static int
30088 {
30089 tree default_decl;
30090 gimple ifunc_cpu_init_stmt;
30091 gimple_seq gseq;
30093 tree ele;
30099 struct _function_version_info
30100 {
30101 tree version_decl;
30102 tree predicate_chain;
30110 /*fndecls_p is actually a vector. */
30113 /* At least one more version other than the default. */
30120 /* The first version in the vector is the default decl. */
30126 /* Function version dispatch is via IFUNC. IFUNC resolvers fire before
30127 constructors, so explicity call __builtin_cpu_init here. */
30138 {
30142 /* Get attribute string, parse it and find the right predicate decl.
30143 The predicate function could be a lengthy combination of many
30144 features, like arch-type and various isa-variants. */
30155 actual_versions++;
30156 }
30158 /* Sort the versions according to descending order of dispatch priority. The
30159 priority is based on the ISA. This is not a perfect solution. There
30160 could still be ambiguity. If more than one function version is suitable
30161 to execute, which one should be dispatched? In future, allow the user
30162 to specify a dispatch priority next to the version. */
30172 /* dispatch default version at the end. */
30178 }
30180 /* Comparator function to be used in qsort routine to sort attribute
30181 specification strings to "target". */
30183 static int
30185 {
30189 }
30191 /* ARGLIST is the argument to target attribute. This function tokenizes
30192 the comma separated arguments, sorts them and returns a string which
30193 is a unique identifier for the comma separated arguments. It also
30194 replaces non-identifier characters "=,-" with "_". */
30198 {
30199 tree arg;
30208 {
30213 argnum++;
30216 argnum++;
30217 }
30222 {
30228 }
30230 /* Replace "=,-" with "_". */
30243 {
30245 i++;
30247 }
30254 {
30259 }
30264 }
30266 /* This function changes the assembler name for functions that are
30267 versions. If DECL is a function version and has a "target"
30268 attribute, it appends the attribute string to its assembler name. */
30270 static tree
30272 {
30273 tree version_attr;
30281 "Function versions cannot be marked as gnu_inline,"
30290 /* target attribute string cannot be NULL. */
30294 version_string
30305 /* Allow assembler name to be modified if already set. */
30313 }
30315 /* This function returns true if FN1 and FN2 are versions of the same function,
30316 that is, the target strings of the function decls are different. This assumes
30317 that FN1 and FN2 have the same signature. */
30319 static bool
30321 {
30333 /* At least one function decl should have the target attribute specified. */
30337 /* Diagnose missing target attribute if one of the decls is already
30338 multi-versioned. */
30340 {
30342 {
30344 {
30349 }
30352 fn2);
30355 /* Prevent diagnosing of the same error multiple times. */
30360 }
30362 }
30367 /* The sorted target strings must be different for fn1 and fn2
30368 to be versions. */
30371 else
30378 }
30380 static tree
30382 {
30383 /* For function version, add the target suffix to the assembler name. */
30387 #ifdef SUBTARGET_MANGLE_DECL_ASSEMBLER_NAME
30389 #endif
30392 }
30394 /* Return a new name by appending SUFFIX to the DECL name. If make_unique
30395 is true, append the full path name of the source file. */
30399 {
30407 /* Get a unique name that can be used globally without any chances
30408 of collision at link time. */
30418 /* Use '.' to concatenate names as it is demangler friendly. */
30421 suffix);
30422 else
30426 }
30428 #if defined (ASM_OUTPUT_TYPE_DIRECTIVE)
30430 /* Make a dispatcher declaration for the multi-versioned function DECL.
30431 Calls to DECL function will be replaced with calls to the dispatcher
30432 by the front-end. Return the decl created. */
30434 static tree
30436 {
30437 tree func_decl;
30457 /* Mark this func as external, the resolver will flip it again if
30458 it gets generated. */
30460 /* This will be of type IFUNCs have to be externally visible. */
30464 }
30466 #endif
30468 /* Returns true if decl is multi-versioned and DECL is the default function,
30469 that is it is not tagged with target specific optimization. */
30471 static bool
30473 {
30482 }
30484 /* Make a dispatcher declaration for the multi-versioned function DECL.
30485 Calls to DECL function will be replaced with calls to the dispatcher
30486 by the front-end. Returns the decl of the dispatcher function. */
30488 static tree
30490 {
30512 /* Find the default version and make it the first node. */
30514 /* Go to the beginning of the chain. */
30519 {
30524 }
30526 /* If there is no default node, just return NULL. */
30530 /* Make default info the first node. */
30532 {
30539 }
30543 #if defined (ASM_OUTPUT_TYPE_DIRECTIVE)
30545 {
30550 /* Right now, the dispatching is done via ifunc. */
30556 dispatcher_version_info
30561 /* Set the dispatcher for all the versions. */
30564 {
30567 }
30568 }
30569 else
30570 #endif
30571 {
30573 "multiversioning needs ifunc which is not supported "
30575 }
30578 }
30580 /* Makes a function attribute of the form NAME(ARG_NAME) and chains
30581 it to CHAIN. */
30583 static tree
30585 {
30586 tree attr_name;
30587 tree attr_arg_name;
30588 tree attr_args;
30589 tree attr;
30596 }
30598 /* Make the resolver function decl to dispatch the versions of
30599 a multi-versioned function, DEFAULT_DECL. Create an
30600 empty basic block in the resolver and store the pointer in
30601 EMPTY_BB. Return the decl of the resolver function. */
30603 static tree
30607 {
30612 /* IFUNC's have to be globally visible. So, if the default_decl is
30613 not, then the name of the IFUNC should be made unique. */
30617 /* Append the filename to the resolver function if the versions are
30618 not externally visible. This is because the resolver function has
30619 to be externally visible for the loader to find it. So, appending
30620 the filename will prevent conflicts with a resolver function from
30621 another module which is based on the same version name. */
30624 /* The resolver function should return a (void *). */
30635 /* IFUNC resolvers have to be externally visible. */
30639 /* Resolver is not external, body is generated. */
30649 {
30650 /* In this case, each translation unit with a call to this
30651 versioned function will put out a resolver. Ensure it
30652 is comdat to keep just one copy. */
30655 }
30656 /* Build result decl and add to function_decl. */
30672 /* Mark dispatch_decl as "ifunc" with resolver as resolver_name. */
30676 /* Create the alias for dispatch to resolver here. */
30677 /*cgraph_create_function_alias (dispatch_decl, decl);*/
30681 }
30683 /* Generate the dispatching code body to dispatch multi-versioned function
30684 DECL. The target hook is called to process the "target" attributes and
30685 provide the code to dispatch the right function at run-time. NODE points
30686 to the dispatcher decl whose body will be created. */
30688 static tree
30690 {
30691 tree resolver_decl;
30692 basic_block empty_bb;
30694 tree default_ver_decl;
30710 /* The first version in the chain corresponds to the default version. */
30713 /* node is going to be an alias, so remove the finalized bit. */
30727 {
30729 /* Check for virtual functions here again, as by this time it should
30730 have been determined if this function needs a vtable index or
30731 not. This happens for methods in derived classes that override
30732 virtual methods in base classes but are not explicitly marked as
30733 virtual. */
30738 }
30745 }
30746 /* This builds the processor_model struct type defined in
30747 libgcc/config/i386/cpuinfo.c */
30749 static tree
30751 {
30758 /* The first 3 fields are unsigned int. */
30760 {
30766 }
30768 /* The last field is an array of unsigned integers of size one. */
30779 }
30781 /* Returns a extern, comdat VAR_DECL of type TYPE and name NAME. */
30783 static tree
30785 {
30786 tree new_decl;
30789 VAR_DECL,
30791 type);
30804 }
30806 /* FNDECL is a __builtin_cpu_is or a __builtin_cpu_supports call that is folded
30807 into an integer defined in libgcc/config/i386/cpuinfo.c */
30809 static tree
30811 {
30817 /* This is the order of bit-fields in __processor_features in cpuinfo.c */
30818 enum processor_features
30819 {
30821 F_MMX,
30822 F_POPCNT,
30823 F_SSE,
30824 F_SSE2,
30825 F_SSE3,
30826 F_SSSE3,
30827 F_SSE4_1,
30828 F_SSE4_2,
30829 F_AVX,
30830 F_AVX2,
30831 F_MAX
30832 };
30834 /* These are the values for vendor types and cpu types and subtypes
30835 in cpuinfo.c. Cpu types and subtypes should be subtracted by
30836 the corresponding start value. */
30837 enum processor_model
30838 {
30840 M_AMD,
30841 M_CPU_TYPE_START,
30842 M_INTEL_ATOM,
30843 M_INTEL_CORE2,
30844 M_INTEL_COREI7,
30845 M_AMDFAM10H,
30846 M_AMDFAM15H,
30847 M_INTEL_SLM,
30848 M_CPU_SUBTYPE_START,
30849 M_INTEL_COREI7_NEHALEM,
30850 M_INTEL_COREI7_WESTMERE,
30851 M_INTEL_COREI7_SANDYBRIDGE,
30852 M_AMDFAM10H_BARCELONA,
30853 M_AMDFAM10H_SHANGHAI,
30854 M_AMDFAM10H_ISTANBUL,
30855 M_AMDFAM15H_BDVER1,
30856 M_AMDFAM15H_BDVER2,
30857 M_AMDFAM15H_BDVER3,
30858 M_AMDFAM15H_BDVER4
30859 };
30861 static struct _arch_names_table
30862 {
30865 }
30867 {
30886 };
30888 static struct _isa_names_table
30889 {
30892 }
30894 {
30906 };
30920 {
30921 /* *args must be a expr that can contain other EXPRS leading to a
30922 STRING_CST. */
30924 {
30927 }
30929 }
30934 {
30935 tree ref;
30936 tree field;
30937 tree final;
30940 unsigned int NUM_ARCH_NAMES
30949 {
30953 }
30958 /* CPU types are stored in the next field. */
30961 {
30964 }
30966 /* CPU subtypes are stored in the next field. */
30968 {
30971 }
30973 /* Get the appropriate field in __cpu_model. */
30977 /* Check the value. */
30981 }
30983 {
30984 tree ref;
30985 tree array_elt;
30986 tree field;
30987 tree final;
30990 unsigned int NUM_ISA_NAMES
30999 {
31003 }
31006 /* Get the last field, which is __cpu_features. */
31010 /* Get the appropriate field: __cpu_model.__cpu_features */
31014 /* Access the 0th element of __cpu_features array. */
31019 /* Return __cpu_model.__cpu_features[0] & field_val */
31023 }
31025 }
31027 static tree
31030 {
31032 {
31037 {
31040 }
31041 }
31043 #ifdef SUBTARGET_FOLD_BUILTIN
31045 #endif
31048 }
31050 /* Make builtins to detect cpu type and features supported. NAME is
31051 the builtin name, CODE is the builtin code, and FTYPE is the function
31052 type of the builtin. */
31054 static void
31057 {
31058 tree decl;
31059 tree type;
31067 }
31069 /* Make builtins to get CPU type and features supported. The created
31070 builtins are :
31072 __builtin_cpu_init (), to detect cpu type and features,
31073 __builtin_cpu_is ("<CPUNAME>"), to check if cpu is of type <CPUNAME>,
31074 __builtin_cpu_supports ("<FEATURE>"), to check if cpu supports <FEATURE>
31075 */
31077 static void
31079 {
31086 }
31088 /* Internal method for ix86_init_builtins. */
31090 static void
31092 {
31104 sysv_va_ref =
31107 fnvoid_va_end_ms =
31109 fnvoid_va_start_ms =
31111 fnvoid_va_end_sysv =
31113 fnvoid_va_start_sysv =
31115 NULL_TREE);
31116 fnvoid_va_copy_ms =
31118 NULL_TREE);
31119 fnvoid_va_copy_sysv =
31135 }
31137 static void
31139 {
31142 /* The __float80 type. */
31145 {
31146 /* The __float80 type. */
31151 }
31154 /* The __float128 type. */
31160 /* This macro is built by i386-builtin-types.awk. */
31161 DEFINE_BUILTIN_PRIMITIVE_TYPES;
31162 }
31164 static void
31166 {
31167 tree t;
31171 /* Builtins to get CPU type and features. */
31174 /* TFmode support builtins. */
31180 /* We will expand them to normal call if SSE isn't available since
31181 they are used by libgcc. */
31200 #ifdef SUBTARGET_INIT_BUILTINS
31201 SUBTARGET_INIT_BUILTINS;
31202 #endif
31203 }
31205 /* Return the ix86 builtin for CODE. */
31207 static tree
31209 {
31214 }
31216 /* Errors in the source file can cause expand_expr to return const0_rtx
31217 where we expect a vector. To avoid crashing, use one of the vector
31218 clear instructions. */
31219 static rtx
31221 {
31225 }
31227 /* Subroutine of ix86_expand_builtin to take care of binop insns. */
31229 static rtx
31231 {
31232 rtx pat;
31252 {
31256 }
31270 }
31272 /* Subroutine of ix86_expand_builtin to take care of 2-4 argument insns. */
31274 static rtx
31278 {
31279 rtx pat;
31287 rtx op;
31294 {
31374 }
31384 {
31391 {
31393 {
31396 {
31414 xop_rotl:
31416 {
31419 gcc_checking_assert
31421 }
31422 else
31423 {
31424 gcc_checking_assert
31435 }
31439 }
31440 }
31441 }
31442 else
31443 {
31444 non_constant:
31448 /* If we aren't optimizing, only allow one memory operand to be
31449 generated. */
31451 num_memory++;
31459 }
31463 }
31466 {
31477 else
31478 {
31484 }
31497 }
31504 }
31506 /* Subroutine of ix86_expand_args_builtin to take care of scalar unop
31507 insns with vec_merge. */
31509 static rtx
31511 rtx target)
31512 {
31513 rtx pat;
31540 }
31542 /* Subroutine of ix86_expand_builtin to take care of comparison insns. */
31544 static rtx
31547 {
31548 rtx pat;
31553 rtx op2;
31564 /* Swap operands if we have a comparison that isn't available in
31565 hardware. */
31567 {
31572 }
31592 }
31594 /* Subroutine of ix86_expand_builtin to take care of comi insns. */
31596 static rtx
31598 rtx target)
31599 {
31600 rtx pat;
31614 /* Swap operands if we have a comparison that isn't available in
31615 hardware. */
31617 {
31621 }
31642 const0_rtx)));
31645 }
31647 /* Subroutines of ix86_expand_args_builtin to take care of round insns. */
31649 static rtx
31651 rtx target)
31652 {
31653 rtx pat;
31678 }
31680 static rtx
31683 {
31684 rtx pat;
31689 rtx op2;
31716 }
31718 /* Subroutine of ix86_expand_builtin to take care of ptest insns. */
31720 static rtx
31722 rtx target)
31723 {
31724 rtx pat;
31757 const0_rtx)));
31760 }
31762 /* Subroutine of ix86_expand_builtin to take care of pcmpestr[im] insns. */
31764 static rtx
31767 {
31768 rtx pat;
31806 {
31809 }
31812 {
31821 }
31823 {
31832 }
31833 else
31834 {
31841 }
31849 {
31854 emit_insn
31858 FLAGS_REG),
31859 const0_rtx)));
31861 }
31862 else
31864 }
31867 /* Subroutine of ix86_expand_builtin to take care of pcmpistr[im] insns. */
31869 static rtx
31872 {
31873 rtx pat;
31901 {
31904 }
31907 {
31916 }
31918 {
31927 }
31928 else
31929 {
31936 }
31944 {
31949 emit_insn
31953 FLAGS_REG),
31954 const0_rtx)));
31956 }
31957 else
31959 }
31961 /* Subroutine of ix86_expand_builtin to take care of insns with
31962 variable number of operands. */
31964 static rtx
31967 {
31972 struct
31973 {
31974 rtx op;
31986 {
32265 }
32270 {
32273 }
32276 {
32283 }
32284 else
32285 {
32288 }
32291 {
32298 {
32299 /* SIMD shift insns take either an 8-bit immediate or
32300 register as count. But builtin functions take int as
32301 count. If count doesn't match, we put it in register. */
32303 {
32307 }
32308 }
32310 {
32313 {
32367 {
32370 {
32373 }
32379 }
32381 }
32382 }
32383 else
32384 {
32388 /* If we aren't optimizing, only allow one memory operand to
32389 be generated. */
32391 num_memory++;
32394 {
32397 }
32398 else
32399 {
32402 }
32403 }
32407 }
32410 {
32427 }
32434 }
32436 /* Subroutine of ix86_expand_builtin to take care of special insns
32437 with variable number of operands. */
32439 static rtx
32442 {
32443 tree arg;
32446 struct
32447 {
32448 rtx op;
32458 {
32506 /* Reserve memory operand for target. */
32537 /* Reserve memory operand for target. */
32551 }
32556 {
32561 {
32564 }
32565 else
32568 }
32569 else
32570 {
32577 }
32580 {
32589 {
32591 {
32597 else
32600 }
32601 }
32602 else
32603 {
32605 {
32606 /* This must be the memory operand. */
32611 }
32612 else
32613 {
32614 /* This must be register. */
32621 }
32622 }
32626 }
32629 {
32644 }
32650 }
32652 /* Return the integer constant in ARG. Constrain it to be in the range
32653 of the subparts of VEC_TYPE; issue an error if not. */
32655 static int
32657 {
32662 {
32665 }
32668 }
32670 /* A subroutine of ix86_expand_builtin. These builtins are a wrapper around
32671 ix86_expand_vector_init. We DO have language-level syntax for this, in
32672 the form of (type){ init-list }. Except that since we can't place emms
32673 instructions from inside the compiler, we can't allow the use of MMX
32674 registers unless the user explicitly asks for it. So we do *not* define
32675 vec_set/vec_extract/vec_init patterns for MMX modes in mmx.md. Instead
32676 we have builtins invoked by mmintrin.h that gives us license to emit
32677 these sorts of instructions. */
32679 static rtx
32681 {
32691 {
32694 }
32701 }
32703 /* A subroutine of ix86_expand_builtin. These builtins are a wrapper around
32704 ix86_expand_vector_extract. They would be redundant (for non-MMX) if we
32705 had a language-level syntax for referencing vector elements. */
32707 static rtx
32709 {
32713 rtx op0;
32733 }
32735 /* A subroutine of ix86_expand_builtin. These builtins are a wrapper around
32736 ix86_expand_vector_set. They would be redundant (for non-MMX) if we had
32737 a language-level syntax for referencing vector elements. */
32739 static rtx
32741 {
32765 /* OP0 is the source of these builtin functions and shouldn't be
32766 modified. Create a copy, use it and return it as target. */
32772 }
32774 /* Expand an expression EXP that calls a built-in function,
32775 with result going to TARGET if that's convenient
32776 (and in mode MODE if that's convenient).
32777 SUBTARGET may be used as the target for computing one of EXP's operands.
32778 IGNORE is nonzero if the value is to be ignored. */
32780 static rtx
32783 {
32793 /* For CPU builtins that can be folded, fold first and expand the fold. */
32795 {
32797 {
32798 /* Make it call __cpu_indicator_init in libgcc. */
32804 }
32807 {
32812 }
32813 }
32815 /* Determine whether the builtin function is available under the current ISA.
32816 Originally the builtin was not created if it wasn't applicable to the
32817 current ISA based on the command line switches. With function specific
32818 options, we need to check in the context of the function making the call
32819 whether it is supported. */
32822 {
32828 else
32829 {
32833 }
32835 }
32838 {
32842 ? CODE_FOR_mmx_maskmovq
32844 /* Note the arg order is different from the operand order. */
32945 {
32946 REAL_VALUE_TYPE inf;
32947 rtx tmp;
32959 }
32969 {
32975 insn = (TARGET_64BIT
32979 }
32981 {
32982 insn = (TARGET_64BIT
32986 }
32987 else
32988 {
32991 insn = (TARGET_64BIT
32999 {
33002 }
33004 }
33007 {
33008 /* mode is VOIDmode if __builtin_rd* has been called
33009 without lhs. */
33013 }
33016 {
33021 }
33035 {
33060 }
33066 {
33069 }
33089 {
33092 }
33098 {
33102 {
33123 }
33128 }
33129 else
33130 {
33132 {
33144 }
33146 }
33176 ? CODE_FOR_tbm_bextri_si
33179 {
33182 }
33183 else
33184 {
33193 }
33209 rdrand_step:
33216 {
33219 }
33225 /* Emit SImode conditional move. */
33227 {
33230 }
33233 else
33240 const0_rtx);
33259 rdseed_step:
33266 {
33269 }
33275 const0_rtx);
33293 addcarryx:
33301 /* Generate CF from input operand. */
33306 /* Gen ADCX instruction to compute X+Y+CF. */
33321 /* Store the result. */
33324 {
33327 }
33330 /* Return current CF value. */
33399 gather_gen:
33410 /* Note the arg order is different from the operand order. */
33419 else
33424 {
33430 }
33433 {
33438 else
33448 }
33450 /* Force memory operand only with base register here. But we
33451 don't want to do it on memory operand for other builtin
33452 functions. */
33464 {
33467 }
33469 /* Optimize. If mask is known to have all high bits set,
33470 replace op0 with pc_rtx to signal that the instruction
33471 overwrites the whole destination and doesn't use its
33472 previous contents. */
33474 {
33476 {
33479 {
33483 negative++;
33486 negative++;
33487 }
33490 }
33492 {
33493 /* Recognize also when mask is like:
33494 __v2df src = _mm_setzero_pd ();
33495 __v2df mask = _mm_cmpeq_pd (src, src);
33496 or
33497 __v8sf src = _mm256_setzero_ps ();
33498 __v8sf mask = _mm256_cmp_ps (src, src, _CMP_EQ_OQ);
33499 as that is a cheaper way to load all ones into
33500 a register than having to load a constant from
33501 memory. */
33504 {
33509 {
33516 /* FALLTHRU */
33526 }
33527 }
33528 }
33529 }
33538 {
33545 else
33547 }
33548 else
33559 {
33562 }
33568 }
33581 {
33585 /* Emit a normal call if SSE isn't available. */
33589 }
33614 }
33616 /* Returns a function decl for a vectorized version of the builtin function
33617 with builtin function code FN and the result vector type TYPE, or NULL_TREE
33618 if it is not available. */
33620 static tree
33622 tree type_in)
33623 {
33639 {
33642 {
33647 }
33652 {
33657 }
33663 /* The round insn does not trap on denormals. */
33668 {
33673 }
33679 /* The round insn does not trap on denormals. */
33684 {
33689 }
33695 /* The round insn does not trap on denormals. */
33700 {
33705 }
33711 /* The round insn does not trap on denormals. */
33716 {
33721 }
33728 {
33733 }
33740 {
33745 }
33751 /* The round insn does not trap on denormals. */
33756 {
33761 }
33767 /* The round insn does not trap on denormals. */
33772 {
33777 }
33782 {
33787 }
33792 {
33797 }
33801 /* The round insn does not trap on denormals. */
33806 {
33811 }
33815 /* The round insn does not trap on denormals. */
33820 {
33825 }
33829 /* The round insn does not trap on denormals. */
33834 {
33839 }
33843 /* The round insn does not trap on denormals. */
33848 {
33853 }
33857 /* The round insn does not trap on denormals. */
33862 {
33867 }
33871 /* The round insn does not trap on denormals. */
33876 {
33881 }
33885 /* The round insn does not trap on denormals. */
33890 {
33895 }
33899 /* The round insn does not trap on denormals. */
33904 {
33909 }
33913 /* The round insn does not trap on denormals. */
33918 {
33923 }
33927 /* The round insn does not trap on denormals. */
33932 {
33937 }
33942 {
33947 }
33952 {
33957 }
33962 }
33964 /* Dispatch to a handler for a vectorization library. */
33967 type_in);
33970 }
33972 /* Handler for an SVML-style interface to
33973 a library with vectorized intrinsics. */
33975 static tree
33977 {
33985 /* The SVML is suitable for unsafe math only. */
33998 {
34045 }
34054 {
34057 }
34058 else
34061 /* Convert to uppercase. */
34066 args;
34068 arity++;
34072 else
34075 /* Build a function declaration for the vectorized function. */
34084 }
34086 /* Handler for an ACML-style interface to
34087 a library with vectorized intrinsics. */
34089 static tree
34091 {
34099 /* The ACML is 64bits only and suitable for unsafe math only as
34100 it does not correctly support parts of IEEE with the required
34101 precision such as denormals. */
34115 {
34145 }
34152 args;
34154 arity++;
34158 else
34161 /* Build a function declaration for the vectorized function. */
34170 }
34172 /* Returns a decl of a function that implements gather load with
34173 memory type MEM_VECTYPE and index type INDEX_VECTYPE and SCALE.
34174 Return NULL_TREE if it is not available. */
34176 static tree
34179 {
34195 /* v*gather* insn sign extends index to pointer mode. */
34207 {
34234 }
34237 }
34239 /* Returns a code for a target-specific builtin that implements
34240 reciprocal of the function, or NULL_TREE if not available. */
34242 static tree
34245 {
34252 /* Machine dependent builtins. */
34254 {
34255 /* Vectorized version of sqrt to rsqrt conversion. */
34264 }
34265 else
34266 /* Normal builtins. */
34268 {
34269 /* Sqrt to rsqrt conversion. */
34275 }
34276 }
34277 \f
34278 /* Helper for avx_vpermilps256_operand et al. This is also used by
34279 the expansion functions to turn the parallel back into a mask.
34280 The return value is 0 for no match and the imm8+1 for a match. */
34282 int
34284 {
34292 /* Validate that all of the elements are constants, and not totally
34293 out of range. Copy the data into an integral array to make the
34294 subsequent checks easier. */
34296 {
34306 }
34309 {
34311 /* In the 256-bit DFmode case, we can only move elements within
34312 a 128-bit lane. */
34314 {
34318 }
34320 {
34324 }
34328 /* In the 256-bit SFmode case, we have full freedom of movement
34329 within the low 128-bit lane, but the high 128-bit lane must
34330 mirror the exact same pattern. */
34335 /* FALLTHRU */
34339 /* In the 128-bit case, we've full freedom in the placement of
34340 the elements from the source operand. */
34347 }
34349 /* Make sure success has a non-zero value by adding one. */
34351 }
34353 /* Helper for avx_vperm2f128_v4df_operand et al. This is also used by
34354 the expansion functions to turn the parallel back into a mask.
34355 The return value is 0 for no match and the imm8+1 for a match. */
34357 int
34359 {
34367 /* Validate that all of the elements are constants, and not totally
34368 out of range. Copy the data into an integral array to make the
34369 subsequent checks easier. */
34371 {
34381 }
34383 /* Validate that the halves of the permute are halves. */
34391 /* Reconstruct the mask. */
34393 {
34399 }
34401 /* Make sure success has a non-zero value by adding one. */
34403 }
34404 \f
34405 /* Store OPERAND to the memory after reload is completed. This means
34406 that we can't easily use assign_stack_local. */
34407 rtx
34409 {
34410 rtx result;
34414 {
34417 stack_pointer_rtx,
34420 }
34422 {
34424 {
34428 /* FALLTHRU */
34434 stack_pointer_rtx)),
34435 operand));
34439 }
34441 }
34442 else
34443 {
34445 {
34447 {
34454 stack_pointer_rtx)),
34460 stack_pointer_rtx)),
34462 }
34465 /* Store HImodes as SImodes. */
34467 /* FALLTHRU */
34473 stack_pointer_rtx)),
34474 operand));
34478 }
34480 }
34482 }
34484 /* Free operand from the memory. */
34485 void
34487 {
34489 {
34494 else
34496 /* Use LEA to deallocate stack space. In peephole2 it will be converted
34497 to pop or add instruction if registers are available. */
34501 }
34502 }
34504 /* Return a register priority for hard reg REGNO. */
34505 static int
34507 {
34508 /* ebp and r13 as the base always wants a displacement, r12 as the
34509 base always wants an index. So discourage their usage in an
34510 address. */
34515 /* New x86-64 int registers result in bigger code size. Discourage
34516 them. */
34519 /* New x86-64 SSE registers result in bigger code size. Discourage
34520 them. */
34523 /* Usage of AX register results in smaller code. Prefer it. */
34527 }
34529 /* Implement TARGET_PREFERRED_RELOAD_CLASS.
34531 Put float CONST_DOUBLE in the constant pool instead of fp regs.
34532 QImode must go into class Q_REGS.
34533 Narrow ALL_REGS to GENERAL_REGS. This supports allowing movsf and
34534 movdf to do mem-to-mem moves through integer regs. */
34536 static reg_class_t
34538 {
34541 /* We're only allowed to return a subclass of CLASS. Many of the
34542 following checks fail for NO_REGS, so eliminate that early. */
34546 /* All classes can load zeros. */
34550 /* Force constants into memory if we are loading a (nonzero) constant into
34551 an MMX, SSE or MASK register. This is because there are no MMX/SSE/MASK
34552 instructions to load from a constant. */
34559 /* Prefer SSE regs only, if we can use them for math. */
34563 /* Floating-point constants need more complex checks. */
34565 {
34566 /* General regs can load everything. */
34570 /* Floats can load 0 and 1 plus some others. Note that we eliminated
34571 zero above. We only want to wind up preferring 80387 registers if
34572 we plan on doing computation with them. */
34575 {
34576 /* Limit class to non-sse. */
34585 }
34588 }
34590 /* Generally when we see PLUS here, it's the function invariant
34591 (plus soft-fp const_int). Which can only be computed into general
34592 regs. */
34596 /* QImode constants are easy to load, but non-constant QImode data
34597 must go into Q_REGS. */
34599 {
34605 }
34608 }
34610 /* Discourage putting floating-point values in SSE registers unless
34611 SSE math is being used, and likewise for the 387 registers. */
34612 static reg_class_t
34614 {
34617 /* Restrict the output reload class to the register bank that we are doing
34618 math on. If we would like not to return a subset of CLASS, reject this
34619 alternative: if reload cannot do this, it will still use its choice. */
34625 {
34630 else
34632 }
34635 }
34637 static reg_class_t
34640 {
34641 /* Double-word spills from general registers to non-offsettable memory
34642 references (zero-extended addresses) require special handling. */
34648 {
34650 ? CODE_FOR_reload_noff_load
34652 /* Add the cost of moving address to a temporary. */
34656 }
34658 /* QImode spills from non-QI registers require
34659 intermediate register on 32bit targets. */
34665 {
34670 else
34676 /* Return Q_REGS if the operand is in memory. */
34679 }
34681 /* This condition handles corner case where an expression involving
34682 pointers gets vectorized. We're trying to use the address of a
34683 stack slot as a vector initializer.
34685 (set (reg:V2DI 74 [ vect_cst_.2 ])
34686 (vec_duplicate:V2DI (reg/f:DI 20 frame)))
34688 Eventually frame gets turned into sp+offset like this:
34690 (set (reg:V2DI 21 xmm0 [orig:74 vect_cst_.2 ] [74])
34691 (vec_duplicate:V2DI (plus:DI (reg/f:DI 7 sp)
34692 (const_int 392 [0x188]))))
34694 That later gets turned into:
34696 (set (reg:V2DI 21 xmm0 [orig:74 vect_cst_.2 ] [74])
34697 (vec_duplicate:V2DI (plus:DI (reg/f:DI 7 sp)
34698 (mem/u/c/i:DI (symbol_ref/u:DI ("*.LC0") [flags 0x2]) [0 S8 A64]))))
34700 We'll have the following reload recorded:
34702 Reload 0: reload_in (DI) =
34703 (plus:DI (reg/f:DI 7 sp)
34704 (mem/u/c/i:DI (symbol_ref/u:DI ("*.LC0") [flags 0x2]) [0 S8 A64]))
34705 reload_out (V2DI) = (reg:V2DI 21 xmm0 [orig:74 vect_cst_.2 ] [74])
34706 SSE_REGS, RELOAD_OTHER (opnum = 0), can't combine
34707 reload_in_reg: (plus:DI (reg/f:DI 7 sp) (const_int 392 [0x188]))
34708 reload_out_reg: (reg:V2DI 21 xmm0 [orig:74 vect_cst_.2 ] [74])
34709 reload_reg_rtx: (reg:V2DI 22 xmm1)
34711 Which isn't going to work since SSE instructions can't handle scalar
34712 additions. Returning GENERAL_REGS forces the addition into integer
34713 register and reload can handle subsequent reloads without problems. */
34721 }
34723 /* Implement TARGET_CLASS_LIKELY_SPILLED_P. */
34725 static bool
34727 {
34729 {
34745 }
34748 }
34750 /* If we are copying between general and FP registers, we need a memory
34751 location. The same is true for SSE and MMX registers.
34753 To optimize register_move_cost performance, allow inline variant.
34755 The macro can't work reliably when one of the CLASSES is class containing
34756 registers from multiple units (SSE, MMX, integer). We avoid this by never
34757 combining those units in single alternative in the machine description.
34758 Ensure that this constraint holds to avoid unexpected surprises.
34760 When STRICT is false, we are being called from REGISTER_MOVE_COST, so do not
34761 enforce these sanity checks. */
34766 {
34775 {
34778 }
34783 /* ??? This is a lie. We do have moves between mmx/general, and for
34784 mmx/sse2. But by saying we need secondary memory we discourage the
34785 register allocator from using the mmx registers unless needed. */
34790 {
34791 /* SSE1 doesn't have any direct moves from other classes. */
34795 /* If the target says that inter-unit moves are more expensive
34796 than moving through memory, then don't generate them. */
34801 /* Between SSE and general, we have moves no larger than word size. */
34804 }
34807 }
34809 bool
34812 {
34814 }
34816 /* Implement the TARGET_CLASS_MAX_NREGS hook.
34818 On the 80386, this is the size of MODE in words,
34819 except in the FP regs, where a single reg is always enough. */
34821 static unsigned char
34823 {
34825 {
34830 else
34832 }
34833 else
34834 {
34837 else
34839 }
34840 }
34842 /* Return true if the registers in CLASS cannot represent the change from
34843 modes FROM to TO. */
34845 bool
34848 {
34852 /* x87 registers can't do subreg at all, as all values are reformatted
34853 to extended precision. */
34858 {
34859 /* Vector registers do not support QI or HImode loads. If we don't
34860 disallow a change to these modes, reload will assume it's ok to
34861 drop the subreg from (subreg:SI (reg:HI 100) 0). This affects
34862 the vec_dupv4hi pattern. */
34866 /* Vector registers do not support subreg with nonzero offsets, which
34867 are otherwise valid for integer registers. Since we can't see
34868 whether we have a nonzero offset from here, prohibit all
34869 nonparadoxical subregs changing size. */
34872 }
34875 }
34877 /* Return the cost of moving data of mode M between a
34878 register and memory. A value of 2 is the default; this cost is
34879 relative to those in `REGISTER_MOVE_COST'.
34881 This function is used extensively by register_move_cost that is used to
34882 build tables at startup. Make it inline in this case.
34883 When IN is 2, return maximum of in and out move cost.
34885 If moving between registers and memory is more expensive than
34886 between two registers, you should define this macro to express the
34887 relative cost.
34889 Model also increased moving costs of QImode registers in non
34890 Q_REGS classes.
34891 */
34895 {
34898 {
34901 {
34913 }
34917 }
34919 {
34922 {
34934 }
34938 }
34940 {
34943 {
34952 }
34956 }
34958 {
34961 {
34967 else
34972 }
34973 else
34974 {
34979 else
34981 }
34988 /* Compute number of 32bit moves needed. TFmode is moved as XFmode. */
34995 else
34999 }
35000 }
35002 static int
35005 {
35007 }
35010 /* Return the cost of moving data from a register in class CLASS1 to
35011 one in class CLASS2.
35013 It is not required that the cost always equal 2 when FROM is the same as TO;
35014 on some machines it is expensive to move between registers if they are not
35015 general registers. */
35017 static int
35019 reg_class_t class2_i)
35020 {
35024 /* In case we require secondary memory, compute cost of the store followed
35025 by load. In order to avoid bad register allocation choices, we need
35026 for this to be *at least* as high as the symmetric MEMORY_MOVE_COST. */
35029 {
35035 /* In case of copying from general_purpose_register we may emit multiple
35036 stores followed by single load causing memory size mismatch stall.
35037 Count this as arbitrarily high cost of 20. */
35042 /* In the case of FP/MMX moves, the registers actually overlap, and we
35043 have to switch modes in order to treat them differently. */
35049 }
35051 /* Moves between SSE/MMX and integer unit are expensive. */
35055 /* ??? By keeping returned value relatively high, we limit the number
35056 of moves between integer and MMX/SSE registers for all targets.
35057 Additionally, high value prevents problem with x86_modes_tieable_p(),
35058 where integer modes in MMX/SSE registers are not tieable
35059 because of missing QImode and HImode moves to, from or between
35060 MMX/SSE registers. */
35070 }
35072 /* Return TRUE if hard register REGNO can hold a value of machine-mode
35073 MODE. */
35075 bool
35077 {
35078 /* Flags and only flags can only hold CCmode values. */
35092 {
35093 /* We implement the move patterns for all vector modes into and
35094 out of SSE registers, even when no operation instructions
35095 are available. */
35097 /* For AVX-512 we allow, regardless of regno:
35098 - XI mode
35099 - any of 512-bit wide vector mode
35100 - any scalar mode. */
35107 /* xmm16-xmm31 are only available for AVX-512. */
35111 /* OImode move is available only when AVX is enabled. */
35118 }
35120 {
35121 /* We implement the move patterns for 3DNOW modes even in MMX mode,
35122 so if the register is available at all, then we can move data of
35123 the given mode into or out of it. */
35126 }
35129 {
35130 /* Take care for QImode values - they can be in non-QI regs,
35131 but then they do cause partial register stalls. */
35136 /* LRA checks if the hard register is OK for the given mode.
35137 QImode values can live in non-QI regs, so we allow all
35138 registers here. */
35142 }
35143 /* We handle both integer and floats in the general purpose registers. */
35150 /* Lots of MMX code casts 8 byte vector modes to DImode. If we then go
35151 on to use that value in smaller contexts, this can easily force a
35152 pseudo to be allocated to GENERAL_REGS. Since this is no worse than
35153 supporting DImode, allow it. */
35158 }
35160 /* A subroutine of ix86_modes_tieable_p. Return true if MODE is a
35161 tieable integer mode. */
35163 static bool
35165 {
35167 {
35180 }
35181 }
35183 /* Return true if MODE1 is accessible in a register that can hold MODE2
35184 without copying. That is, all register classes that can hold MODE2
35185 can also hold MODE1. */
35187 bool
35189 {
35197 /* MODE2 being XFmode implies fp stack or general regs, which means we
35198 can tie any smaller floating point modes to it. Note that we do not
35199 tie this with TFmode. */
35203 /* MODE2 being DFmode implies fp stack, general or sse regs, which means
35204 that we can tie it with SFmode. */
35208 /* If MODE2 is only appropriate for an SSE register, then tie with
35209 any other mode acceptable to SSE registers. */
35219 /* If MODE2 is appropriate for an MMX register, then tie
35220 with any other mode acceptable to MMX registers. */
35227 }
35229 /* Return the cost of moving between two registers of mode MODE. */
35231 static int
35233 {
35237 {
35269 }
35271 /* Return the cost of moving between two registers of mode MODE,
35272 assuming that the move will be in pieces of at most UNITS bytes. */
35274 }
35276 /* Compute a (partial) cost for rtx X. Return true if the complete
35277 cost has been computed, and false if subexpressions should be
35278 scanned. In either case, *TOTAL contains the cost result. */
35280 static bool
35283 {
35290 {
35294 {
35297 }
35309 && (!TARGET_64BIT
35314 else
35320 {
35323 }
35325 {
35335 }
35337 {
35340 {
35349 }
35350 }
35351 /* Fall back to (MEM (SYMBOL_REF)), since that's where
35352 it'll probably end up. Add a penalty for size. */
35359 /* The zero extensions is often completely free on x86_64, so make
35360 it as cheap as possible. */
35366 else
35378 {
35381 {
35384 }
35387 {
35390 }
35391 }
35392 /* FALLTHRU */
35399 {
35400 /* ??? Should be SSE vector operation cost. */
35401 /* At least for published AMD latencies, this really is the same
35402 as the latency for a simple fpu operation like fabs. */
35403 /* V*QImode is emulated with 1-11 insns. */
35405 {
35408 {
35409 /* For XOP we use vpshab, which requires a broadcast of the
35410 value to the variable shift insn. For constants this
35411 means a V16Q const in mem; even when we can perform the
35412 shift with one insn set the cost to prefer paddb. */
35414 {
35419 }
35421 }
35425 }
35426 else
35428 }
35430 {
35432 {
35435 else
35437 }
35438 else
35439 {
35442 else
35444 }
35445 }
35446 else
35447 {
35452 {
35453 /* Return the cost after shift-and truncation. */
35456 }
35457 else
35459 }
35463 {
35464 rtx sub;
35469 /* ??? SSE scalar/vector cost should be used here. */
35470 /* ??? Bald assumption that fma has the same cost as fmul. */
35474 /* Negate in op0 or op2 is free: FMS, FNMA, FNMS. */
35485 }
35489 {
35490 /* ??? SSE scalar cost should be used here. */
35493 }
35495 {
35498 }
35500 {
35501 /* ??? SSE vector cost should be used here. */
35504 }
35506 {
35507 /* V*QImode is emulated with 7-13 insns. */
35509 {
35516 }
35517 /* V*DImode is emulated with 5-8 insns. */
35519 {
35522 else
35524 }
35525 /* Without sse4.1, we don't have PMULLD; it's emulated with 7
35526 insns, including two PMULUDQ. */
35529 else
35532 }
35533 else
35534 {
35539 {
35542 nbits++;
35543 }
35544 else
35545 /* This is arbitrary. */
35548 /* Compute costs correctly for widening multiplication. */
35552 {
35559 {
35563 else
35565 }
35569 }
35577 }
35584 /* ??? SSE cost should be used here. */
35589 /* ??? SSE vector cost should be used here. */
35591 else
35598 {
35603 {
35606 {
35614 }
35615 }
35618 {
35621 {
35627 }
35628 }
35630 {
35638 }
35639 }
35640 /* FALLTHRU */
35644 {
35645 /* ??? SSE cost should be used here. */
35648 }
35650 {
35653 }
35655 {
35656 /* ??? SSE vector cost should be used here. */
35659 }
35660 /* FALLTHRU */
35667 {
35674 }
35675 /* FALLTHRU */
35679 {
35680 /* ??? SSE cost should be used here. */
35683 }
35685 {
35688 }
35690 {
35691 /* ??? SSE vector cost should be used here. */
35694 }
35695 /* FALLTHRU */
35699 {
35700 /* ??? Should be SSE vector operation cost. */
35701 /* At least for published AMD latencies, this really is the same
35702 as the latency for a simple fpu operation like fabs. */
35704 }
35707 else
35716 {
35717 /* This kind of construct is implemented using test[bwl].
35718 Treat it as if we had an AND. */
35723 }
35733 /* ??? SSE cost should be used here. */
35738 /* ??? SSE vector cost should be used here. */
35744 /* ??? SSE cost should be used here. */
35749 /* ??? SSE vector cost should be used here. */
35762 /* ??? Assume all of these vector manipulation patterns are
35763 recognizable. In which case they all pretty much have the
35764 same cost. */
35770 }
35771 }
35773 #if TARGET_MACHO
35777 /* Given a symbol name and its associated stub, write out the
35778 definition of the stub. */
35780 void
35782 {
35787 /* For 64-bit we shouldn't get here. */
35790 /* Lose our funky encoding stuff so it doesn't contaminate the stub. */
35807 else
35814 {
35816 }
35818 {
35819 /* PIC stub. */
35820 /* 25-byte PIC stub using "CALL get_pc_thunk". */
35826 }
35827 else
35830 /* The AT&T-style ("self-modifying") stub is not lazily bound, thus
35831 it needs no stub-binding-helper. */
35838 {
35841 }
35842 else
35847 /* N.B. Keep the correspondence of these
35848 'symbol_ptr/symbol_ptr2/symbol_ptr3' sections consistent with the
35849 old-pic/new-pic/non-pic stubs; altering this will break
35850 compatibility with existing dylibs. */
35852 {
35853 /* 25-byte PIC stub using "CALL get_pc_thunk". */
35855 }
35856 else
35857 /* 16-byte -mdynamic-no-pic stub. */
35863 }
35866 /* Order the registers for register allocator. */
35868 void
35870 {
35874 /* First allocate the local general purpose registers. */
35879 /* Global general purpose registers. */
35884 /* x87 registers come first in case we are doing FP math
35885 using them. */
35890 /* SSE registers. */
35896 /* Extended REX SSE registers. */
35900 /* Mask register. */
35904 /* MPX bound registers. */
35908 /* x87 registers. */
35916 /* Initialize the rest of array as we do not allocate some registers
35917 at all. */
35920 }
35922 /* Handle a "callee_pop_aggregate_return" attribute; arguments as
35923 in struct attribute_spec handler. */
35924 static tree
35926 tree args,
35929 {
35934 {
35936 name);
35939 }
35941 {
35943 name);
35946 }
35948 {
35949 tree cst;
35953 {
35956 name);
35958 }
35961 {
35964 name);
35966 }
35969 }
35972 }
35974 /* Handle a "ms_abi" or "sysv" attribute; arguments as in
35975 struct attribute_spec.handler. */
35976 static tree
35978 tree args ATTRIBUTE_UNUSED,
35980 {
35985 {
35987 name);
35990 }
35992 /* Can combine regparm with all attributes but fastcall. */
35994 {
35996 {
35998 }
36001 }
36003 {
36005 {
36007 }
36010 }
36013 }
36015 /* Handle a "ms_struct" or "gcc_struct" attribute; arguments as in
36016 struct attribute_spec.handler. */
36017 static tree
36019 tree args ATTRIBUTE_UNUSED,
36021 {
36024 {
36027 }
36028 else
36032 {
36034 name);
36036 }
36042 {
36044 name);
36046 }
36049 }
36051 static tree
36053 tree args ATTRIBUTE_UNUSED,
36055 {
36057 {
36059 name);
36061 }
36063 }
36065 static bool
36067 {
36071 }
36073 /* Returns an expression indicating where the this parameter is
36074 located on entry to the FUNCTION. */
36076 static rtx
36078 {
36084 {
36089 else
36092 }
36097 {
36104 {
36109 }
36110 else
36111 {
36114 {
36120 }
36121 }
36123 }
36127 }
36129 /* Determine whether x86_output_mi_thunk can succeed. */
36131 static bool
36133 HOST_WIDE_INT delta ATTRIBUTE_UNUSED,
36135 {
36136 /* 64-bit can handle anything. */
36140 /* For 32-bit, everything's fine if we have one free register. */
36144 /* Need a free register for vcall_offset. */
36148 /* Need a free register for GOT references. */
36152 /* Otherwise ok. */
36154 }
36156 /* Output the assembler code for a thunk function. THUNK_DECL is the
36157 declaration for the thunk function itself, FUNCTION is the decl for
36158 the target function. DELTA is an immediate constant offset to be
36159 added to THIS. If VCALL_OFFSET is nonzero, the word at
36160 *(*this + vcall_offset) should be added to THIS. */
36162 static void
36166 {
36173 else
36174 {
36180 else
36182 }
36186 /* If VCALL_OFFSET, we'll need THIS in a register. Might as well
36187 pull it in now and let DELTA benefit. */
36191 {
36192 /* Put the this parameter into %eax. */
36195 }
36196 else
36199 /* Adjust the this parameter by a fixed constant. */
36201 {
36206 {
36208 {
36212 }
36213 }
36216 }
36218 /* Adjust the this parameter by a value stored in the vtable. */
36220 {
36230 /* Adjust the this parameter. */
36234 {
36238 }
36245 vcall_mem));
36246 else
36248 }
36250 /* If necessary, drop THIS back to its stack slot. */
36256 {
36259 ;
36260 else
36261 {
36265 }
36266 }
36267 else
36268 {
36270 ;
36271 #if TARGET_MACHO
36273 {
36276 }
36278 else
36279 {
36286 }
36287 }
36289 /* Our sibling call patterns do not allow memories, because we have no
36290 predicate that can distinguish between frame and non-frame memory.
36291 For our purposes here, we can get away with (ab)using a jump pattern,
36292 because we're going to do no optimization. */
36295 else
36296 {
36302 {
36308 }
36314 }
36317 /* Emit just enough of rest_of_compilation to get the insns emitted.
36318 Note that use_thunk calls assemble_start_function et al. */
36324 }
36326 static void
36328 {
36330 #if TARGET_MACHO
36332 #endif
36339 }
36341 int
36343 {
36355 }
36357 /* Output assembler code to FILE to increment profiler label # LABELNO
36358 for profiling a function entry. */
36359 void
36361 {
36366 {
36367 #ifndef NO_PROFILE_COUNTERS
36369 #endif
36373 else
36375 }
36377 {
36378 #ifndef NO_PROFILE_COUNTERS
36381 #endif
36383 }
36384 else
36385 {
36386 #ifndef NO_PROFILE_COUNTERS
36389 #endif
36391 }
36392 }
36394 /* We don't have exact information about the insn sizes, but we may assume
36395 quite safely that we are informed about all 1 byte insns and memory
36396 address sizes. This is enough to eliminate unnecessary padding in
36397 99% of cases. */
36399 static int
36401 {
36407 /* Discard alignments we've emit and jump instructions. */
36412 /* Important case - calls are always 5 bytes.
36413 It is common to have many calls in the row. */
36422 /* For normal instructions we rely on get_attr_length being exact,
36423 with a few exceptions. */
36425 {
36429 {
36439 /* Otherwise trust get_attr_length. */
36441 }
36446 }
36449 else
36451 }
36453 #ifdef ASM_OUTPUT_MAX_SKIP_PAD
36455 /* AMD K8 core mispredicts jumps when there are more than 3 jumps in 16 byte
36456 window. */
36458 static void
36460 {
36465 /* Look for all minimal intervals of instructions containing 4 jumps.
36466 The intervals are bounded by START and INSN. NBYTES is the total
36467 size of instructions in the interval including INSN and not including
36468 START. When the NBYTES is smaller than 16 bytes, it is possible
36469 that the end of START and INSN ends up in the same 16byte page.
36471 The smallest offset in the page INSN can start is the case where START
36472 ends on the offset 0. Offset of INSN is then NBYTES - sizeof (INSN).
36473 We add p2align to 16byte window with maxskip 15 - NBYTES + sizeof (INSN).
36474 */
36476 {
36480 {
36486 /* If align > 3, only up to 16 - max_skip - 1 bytes can be
36487 already in the current 16 byte page, because otherwise
36488 ASM_OUTPUT_MAX_SKIP_ALIGN could skip max_skip or fewer
36489 bytes to reach 16 byte boundary. */
36497 {
36499 {
36503 else
36506 }
36507 }
36509 }
36517 njumps++;
36518 else
36522 {
36526 else
36529 }
36536 {
36543 }
36544 }
36545 }
36546 #endif
36548 /* AMD Athlon works faster
36549 when RET is not destination of conditional jump or directly preceded
36550 by other jump instruction. We avoid the penalty by inserting NOP just
36551 before the RET instructions in such cases. */
36552 static void
36554 {
36555 edge e;
36556 edge_iterator ei;
36559 {
36562 rtx prev;
36572 {
36573 edge e;
36574 edge_iterator ei;
36579 {
36582 }
36583 }
36585 {
36591 /* Empty functions get branch mispredict even when
36592 the jump destination is not visible to us. */
36595 }
36597 {
36600 }
36601 }
36602 }
36604 /* Count the minimum number of instructions in BB. Return 4 if the
36605 number of instructions >= 4. */
36607 static int
36609 {
36610 rtx insn;
36613 /* Count number of instructions in this block. Return 4 if the number
36614 of instructions >= 4. */
36616 {
36617 /* Only happen in exit blocks. */
36625 {
36626 insn_count++;
36629 }
36630 }
36633 }
36636 /* Count the minimum number of instructions in code path in BB.
36637 Return 4 if the number of instructions >= 4. */
36639 static int
36641 {
36642 edge e;
36643 edge_iterator ei;
36646 /* Only bother counting instructions along paths with no
36647 more than 2 basic blocks between entry and exit. Given
36648 that BB has an edge to exit, determine if a predecessor
36649 of BB has an edge from entry. If so, compute the number
36650 of instructions in the predecessor block. If there
36651 happen to be multiple such blocks, compute the minimum. */
36654 {
36655 edge prev_e;
36656 edge_iterator prev_ei;
36659 {
36662 }
36664 {
36666 {
36671 }
36672 }
36673 }
36679 }
36681 /* Pad short function to 4 instructions. */
36683 static void
36685 {
36686 edge e;
36687 edge_iterator ei;
36690 {
36693 {
36696 /* Pad short function. */
36698 {
36701 /* Find epilogue. */
36710 /* Two NOPs count as one instruction. */
36713 }
36714 }
36715 }
36716 }
36718 /* Fix up a Windows system unwinder issue. If an EH region falls through into
36719 the epilogue, the Windows system unwinder will apply epilogue logic and
36720 produce incorrect offsets. This can be avoided by adding a nop between
36721 the last insn that can throw and the first insn of the epilogue. */
36723 static void
36725 {
36726 edge e;
36727 edge_iterator ei;
36730 {
36733 /* Find the beginning of the epilogue. */
36740 /* We only care about preceding insns that can throw. */
36745 /* Do not separate calls from their debug information. */
36751 else
36755 }
36756 }
36758 /* Implement machine specific optimizations. We implement padding of returns
36759 for K8 CPUs and pass to avoid 4 jumps in the single 16 byte window. */
36760 static void
36762 {
36763 /* We are freeing block_for_insn in the toplev to keep compatibility
36764 with old MDEP_REORGS that are not CFG based. Recompute it now. */
36771 {
36776 #ifdef ASM_OUTPUT_MAX_SKIP_PAD
36779 #endif
36780 }
36781 }
36783 /* Return nonzero when QImode register that must be represented via REX prefix
36784 is used. */
36785 bool
36787 {
36795 }
36797 /* Return nonzero when P points to register encoded via REX prefix.
36798 Called via for_each_rtx. */
36799 static int
36801 {
36807 }
36809 /* Return true when INSN mentions register that must be encoded using REX
36810 prefix. */
36811 bool
36813 {
36816 }
36818 /* If profitable, negate (without causing overflow) integer constant
36819 of mode MODE at location LOC. Return true in this case. */
36820 bool
36822 {
36823 HOST_WIDE_INT val;
36829 {
36831 /* DImode x86_64 constants must fit in 32 bits. */
36844 }
36846 /* Avoid overflows. */
36852 /* Make things pretty and `subl $4,%eax' rather than `addl $-4,%eax'.
36853 Exceptions: -128 encodes smaller than 128, so swap sign and op. */
36856 {
36859 }
36862 }
36864 /* Generate an unsigned DImode/SImode to FP conversion. This is the same code
36865 optabs would emit if we didn't have TFmode patterns. */
36867 void
36869 {
36903 }
36904 \f
36905 /* AVX512F does support 64-byte integer vector operations,
36906 thus the longest vector we are faced with is V64QImode. */
36907 #define MAX_VECT_LEN 64
36909 struct expand_vec_perm_d
36910 {
36917 };
36923 /* Get a vector mode of the same size as the original but with elements
36924 twice as wide. This is only guaranteed to apply to integral vectors. */
36928 {
36929 /* ??? Rely on the ordering that genmodes.c gives to vectors. */
36934 }
36936 /* A subroutine of ix86_expand_vector_init. Store into TARGET a vector
36937 with all elements equal to VAR. Return true if successful. */
36939 static bool
36942 {
36946 {
36951 /* FALLTHRU */
36961 {
36964 /* First attempt to recognize VAL as-is. */
36968 {
36969 rtx seq;
36970 /* If that fails, force VAL into a register. */
36981 }
36982 }
36989 {
36990 rtx x;
36997 }
37007 {
37011 permute:
37019 /* Extend to SImode using a paradoxical SUBREG. */
37023 /* Insert the SImode value as low element of a V4SImode vector. */
37032 }
37040 widen:
37041 /* Replicate the value once into the next wider mode and recurse. */
37042 {
37044 rtx x;
37061 }
37065 {
37074 }
37079 }
37080 }
37082 /* A subroutine of ix86_expand_vector_init. Store into TARGET a vector
37083 whose ONE_VAR element is VAR, and other elements are zero. Return true
37084 if successful. */
37086 static bool
37089 {
37091 rtx new_target;
37096 {
37098 /* For SSE4.1, we normally use vector set. But if the second
37099 element is zero and inter-unit moves are OK, we use movq
37100 instead. */
37102 && !(TARGET_INTER_UNIT_MOVES_TO_VEC
37124 /* Use ix86_expand_vector_set in 64bit mode only. */
37129 }
37132 {
37137 }
37140 {
37145 /* FALLTHRU */
37160 else
37167 {
37168 /* We need to shuffle the value to the correct position, so
37169 create a new pseudo to store the intermediate result. */
37171 /* With SSE2, we can use the integer shuffle insns. */
37173 {
37175 const1_rtx,
37182 }
37184 /* Otherwise convert the intermediate result to V4SFmode and
37185 use the SSE1 shuffle instructions. */
37187 {
37190 }
37191 else
37195 const1_rtx,
37204 }
37219 widen:
37223 /* Zero extend the variable element to SImode and recurse. */
37236 }
37237 }
37239 /* A subroutine of ix86_expand_vector_init. Store into TARGET a vector
37240 consisting of the values in VALS. It is known that all elements
37241 except ONE_VAR are constants. Return true if successful. */
37243 static bool
37246 {
37256 {
37261 /* For the two element vectors, it's just as easy to use
37262 the general case. */
37266 /* Use ix86_expand_vector_set in 64bit mode only. */
37288 widen:
37289 /* There's no way to set one QImode entry easily. Combine
37290 the variable value with its adjacent constant value, and
37291 promote to an HImode set. */
37294 {
37299 }
37300 else
37301 {
37304 }
37318 }
37323 }
37325 /* A subroutine of ix86_expand_vector_init_general. Use vector
37326 concatenate to handle the most general case: all values variable,
37327 and none identical. */
37329 static void
37332 {
37335 rtvec v;
37339 {
37342 {
37375 }
37388 {
37403 }
37408 {
37419 }
37422 half:
37423 /* FIXME: We process inputs backward to help RA. PR 36222. */
37427 {
37432 }
37436 {
37439 {
37443 }
37446 }
37447 else
37453 }
37454 }
37456 /* A subroutine of ix86_expand_vector_init_general. Use vector
37457 interleave to handle the most general case: all values variable,
37458 and none identical. */
37460 static void
37463 {
37472 {
37493 }
37496 {
37497 /* Extend the odd elment to SImode using a paradoxical SUBREG. */
37501 /* Insert the SImode value as low element of V4SImode vector. */
37505 op0),
37507 const1_rtx);
37510 /* Cast the V4SImode vector back to a vector in orignal mode. */
37514 /* Load even elements into the second position. */
37518 const1_rtx));
37520 /* Cast vector to FIRST_IMODE vector. */
37523 }
37525 /* Interleave low FIRST_IMODE vectors. */
37527 {
37531 /* Cast FIRST_IMODE vector to SECOND_IMODE vector. */
37534 }
37536 /* Interleave low SECOND_IMODE vectors. */
37538 {
37541 {
37546 /* Cast the SECOND_IMODE vector to the THIRD_IMODE
37547 vector. */
37550 }
37553 /* FALLTHRU */
37560 /* Cast the SECOND_IMODE vector back to a vector on original
37561 mode. */
37568 }
37569 }
37571 /* A subroutine of ix86_expand_vector_init. Handle the most general case:
37572 all values variable, and none identical. */
37574 static void
37577 {
37583 {
37588 /* FALLTHRU */
37612 half:
37629 /* FALLTHRU */
37635 /* Don't use ix86_expand_vector_init_interleave if we can't
37636 move from GPR to SSE register directly. */
37652 }
37654 {
37666 {
37670 {
37676 else
37677 {
37682 }
37683 }
37686 }
37691 {
37697 }
37699 {
37705 }
37706 else
37708 }
37709 }
37711 /* Initialize vector TARGET via VALS. Suppress the use of MMX
37712 instructions unless MMX_OK is true. */
37714 void
37716 {
37723 rtx x;
37726 {
37736 }
37738 /* Constants are best loaded from the constant pool. */
37740 {
37743 }
37745 /* If all values are identical, broadcast the value. */
37751 /* Values where only one field is non-constant are best loaded from
37752 the pool and overwritten via move later. */
37754 {
37758 one_var))
37763 }
37766 }
37768 void
37770 {
37775 rtx tmp;
37777 = {
37784 };
37786 = {
37793 };
37797 {
37801 {
37806 else
37810 }
37822 else
37828 {
37831 /* For the two element vectors, we implement a VEC_CONCAT with
37832 the extraction of the other element. */
37839 else
37844 }
37853 {
37859 /* tmp = target = A B C D */
37861 /* target = A A B B */
37863 /* target = X A B B */
37865 /* target = A X C D */
37872 /* tmp = target = A B C D */
37874 /* tmp = X B C D */
37876 /* target = A B X D */
37883 /* tmp = target = A B C D */
37885 /* tmp = X B C D */
37887 /* target = A B X D */
37895 }
37903 /* Element 0 handled by vec_merge below. */
37905 {
37908 }
37911 {
37912 /* With SSE2, use integer shuffles to swap element 0 and ELT,
37913 store into element 0, then shuffle them back. */
37930 }
37931 else
37932 {
37933 /* For SSE1, we have to reuse the V4SF code. */
37937 }
37990 half:
37991 /* Compute offset. */
37997 /* Extract the half. */
38001 /* Put val in tmp at elt. */
38004 /* Put it back. */
38010 }
38013 {
38017 }
38018 else
38019 {
38028 }
38029 }
38031 void
38033 {
38037 rtx tmp;
38040 {
38045 /* FALLTHRU */
38058 {
38078 }
38090 {
38092 {
38112 }
38116 }
38117 else
38118 {
38119 /* For SSE1, we have to reuse the V4SF code. */
38123 }
38139 {
38143 else
38147 }
38152 {
38156 else
38160 }
38165 {
38169 else
38173 }
38178 {
38182 else
38186 }
38191 {
38195 else
38199 }
38204 {
38208 else
38212 }
38216 /* ??? Could extract the appropriate HImode element and shift. */
38219 }
38222 {
38226 /* Let the rtl optimizers know about the zero extension performed. */
38228 {
38231 }
38234 }
38235 else
38236 {
38243 }
38244 }
38246 /* Generate code to copy vector bits i / 2 ... i - 1 from vector SRC
38247 to bits 0 ... i / 2 - 1 of vector DEST, which has the same mode.
38248 The upper bits of DEST are undefined, though they shouldn't cause
38249 exceptions (some bits from src or all zeros are ok). */
38251 static void
38253 {
38256 {
38260 else
38278 else
38285 else
38293 {
38298 const1_rtx);
38299 }
38300 else
38301 {
38305 }
38309 }
38313 }
38315 /* Expand a vector reduction. FN is the binary pattern to reduce;
38316 DEST is the destination; IN is the input vector. */
38318 void
38320 {
38325 /* SSE4 has a special instruction for V8HImode UMIN reduction. */
38329 {
38332 }
38337 {
38342 else
38346 }
38347 }
38348 \f
38349 /* Target hook for scalar_mode_supported_p. */
38350 static bool
38352 {
38357 else
38359 }
38361 /* Implements target hook vector_mode_supported_p. */
38362 static bool
38364 {
38376 }
38378 /* Target hook for c_mode_for_suffix. */
38379 static enum machine_mode
38381 {
38388 }
38390 /* Worker function for TARGET_MD_ASM_CLOBBERS.
38392 We do this in the new i386 backend to maintain source compatibility
38393 with the old cc0-based compiler. */
38395 static tree
38397 tree inputs ATTRIBUTE_UNUSED,
38398 tree clobbers)
38399 {
38401 clobbers);
38403 clobbers);
38405 }
38407 /* Implements target vector targetm.asm.encode_section_info. */
38409 static void ATTRIBUTE_UNUSED
38411 {
38418 }
38420 /* Worker function for REVERSE_CONDITION. */
38422 enum rtx_code
38424 {
38428 }
38430 /* Output code to perform an x87 FP register move, from OPERANDS[1]
38431 to OPERANDS[0]. */
38435 {
38437 {
38440 {
38444 }
38448 }
38450 {
38454 else
38455 {
38456 /* There is no non-popping store to memory for XFmode.
38457 So if we need one, follow the store with a load. */
38460 else
38462 }
38463 }
38464 else
38466 }
38468 /* Output code to perform a conditional jump to LABEL, if C2 flag in
38469 FP status register is set. */
38471 void
38473 {
38475 rtx temp;
38480 {
38485 }
38486 else
38487 {
38492 }
38496 pc_rtx);
38501 }
38503 /* Output code to perform a log1p XFmode calculation. */
38506 {
38512 rtx test;
38518 XFmode));
38532 }
38534 /* Emit code for round calculation. */
38536 {
38543 rtx insn;
38548 {
38560 }
38563 {
38584 }
38592 /* round(a) = sgn(a) * floor(fabs(a) + 0.5) */
38594 /* scratch = fxam(op1) */
38597 UNSPEC_FXAM)));
38598 /* e1 = fabs(op1) */
38601 /* e2 = e1 + 0.5 */
38606 /* res = floor(e2) */
38608 {
38613 }
38614 else
38618 {
38621 {
38628 UNSPEC_TRUNC_NOOP)));
38629 }
38645 }
38647 /* flags = signbit(a) */
38650 /* if (flags) then res = -res */
38654 pc_rtx);
38665 }
38667 /* Output code to perform a Newton-Rhapson approximation of a single precision
38668 floating point divide [http://en.wikipedia.org/wiki/N-th_root_algorithm]. */
38671 {
38679 /* a / b = a * ((rcp(b) + rcp(b)) - (b * rcp(b) * rcp (b))) */
38683 /* x0 = rcp(b) estimate */
38686 UNSPEC_RCP)));
38687 /* e0 = x0 * b */
38691 /* e0 = x0 * e0 */
38695 /* e1 = x0 + x0 */
38699 /* x1 = e1 - e0 */
38703 /* res = a * x1 */
38706 }
38708 /* Output code to perform a Newton-Rhapson approximation of a
38709 single precision floating point [reciprocal] square root. */
38713 {
38715 REAL_VALUE_TYPE r;
38730 {
38733 }
38735 /* sqrt(a) = -0.5 * a * rsqrtss(a) * (a * rsqrtss(a) * rsqrtss(a) - 3.0)
38736 rsqrt(a) = -0.5 * rsqrtss(a) * (a * rsqrtss(a) * rsqrtss(a) - 3.0) */
38740 /* x0 = rsqrt(a) estimate */
38743 UNSPEC_RSQRT)));
38745 /* If (a == 0.0) Filter out infinity to prevent NaN for sqrt(0.0). */
38747 {
38759 }
38761 /* e0 = x0 * a */
38764 /* e1 = e0 * x0 */
38768 /* e2 = e1 - 3. */
38775 /* e3 = -.5 * x0 */
38778 else
38779 /* e3 = -.5 * e0 */
38782 /* ret = e2 * e3 */
38785 }
38787 #ifdef TARGET_SOLARIS
38788 /* Solaris implementation of TARGET_ASM_NAMED_SECTION. */
38790 static void
38792 tree decl)
38793 {
38794 /* With Binutils 2.15, the "@unwind" marker must be specified on
38795 every occurrence of the ".eh_frame" section, not just the first
38796 one. */
38799 {
38803 }
38805 #ifndef USE_GAS
38807 {
38810 }
38811 #endif
38814 }
38817 /* Return the mangling of TYPE if it is an extended fundamental type. */
38821 {
38829 {
38831 /* __float128 is "g". */
38834 /* "long double" or __float80 is "e". */
38838 }
38839 }
38841 /* For 32-bit code we can save PIC register setup by using
38842 __stack_chk_fail_local hidden function instead of calling
38843 __stack_chk_fail directly. 64-bit code doesn't need to setup any PIC
38844 register, so it is better to call __stack_chk_fail directly. */
38846 static tree ATTRIBUTE_UNUSED
38848 {
38849 return TARGET_64BIT
38852 }
38854 /* Select a format to encode pointers in exception handling data. CODE
38855 is 0 for data, 1 for code labels, 2 for function pointers. GLOBAL is
38856 true if the symbol may be affected by dynamic relocations.
38858 ??? All x86 object file formats are capable of representing this.
38859 After all, the relocation needed is the same as for the call insn.
38860 Whether or not a particular assembler allows us to enter such, I
38861 guess we'll have to see. */
38862 int
38864 {
38866 {
38873 }
38878 }
38879 \f
38880 /* Expand copysign from SIGN to the positive value ABS_VALUE
38881 storing in RESULT. If MASK is non-null, it shall be a mask to mask out
38882 the sign-bit. */
38883 static void
38885 {
38889 {
38896 else
38901 {
38902 /* We need to generate a scalar mode mask in this case. */
38907 }
38908 }
38909 else
38915 }
38917 /* Expand fabs (OP0) and return a new rtx that holds the result. The
38918 mask for masking out the sign-bit is stored in *SMASK, if that is
38919 non-null. */
38920 static rtx
38922 {
38931 else
38935 {
38936 /* We need to generate a scalar mode mask in this case. */
38941 }
38949 }
38951 /* Expands a comparison of OP0 with OP1 using comparison code CODE,
38952 swapping the operands if SWAP_OPERANDS is true. The expanded
38953 code is a forward jump to a newly created label in case the
38954 comparison is true. The generated label rtx is returned. */
38955 static rtx
38958 {
38963 {
38967 }
38980 }
38982 /* Expand a mask generating SSE comparison instruction comparing OP0 with OP1
38983 using comparison code CODE. Operands are swapped for the comparison if
38984 SWAP_OPERANDS is true. Returns a rtx for the generated mask. */
38985 static rtx
38988 {
38994 {
38998 }
39005 }
39007 /* Generate and return a rtx of mode MODE for 2**n where n is the number
39008 of bits of the mantissa of MODE, which must be one of DFmode or SFmode. */
39009 static rtx
39011 {
39012 REAL_VALUE_TYPE TWO52r;
39013 rtx TWO52;
39020 }
39022 /* Expand SSE sequence for computing lround from OP1 storing
39023 into OP0. */
39024 void
39026 {
39027 /* C code for the stuff we're doing below:
39028 tmp = op1 + copysign (nextafter (0.5, 0.0), op1)
39029 return (long)tmp;
39030 */
39034 rtx adj;
39036 /* load nextafter (0.5, 0.0) */
39041 /* adj = copysign (0.5, op1) */
39045 /* adj = op1 + adj */
39048 /* op0 = (imode)adj */
39050 }
39052 /* Expand SSE2 sequence for computing lround from OPERAND1 storing
39053 into OPERAND0. */
39054 void
39056 {
39057 /* C code for the stuff we're doing below (for do_floor):
39058 xi = (long)op1;
39059 xi -= (double)xi > op1 ? 1 : 0;
39060 return xi;
39061 */
39066 /* reg = (long)op1 */
39070 /* freg = (double)reg */
39074 /* ireg = (freg > op1) ? ireg - 1 : ireg */
39085 }
39087 /* Expand rint (IEEE round to nearest) rounding OPERAND1 and storing the
39088 result in OPERAND0. */
39089 void
39091 {
39092 /* C code for the stuff we're doing below:
39093 xa = fabs (operand1);
39094 if (!isless (xa, 2**52))
39095 return operand1;
39096 xa = xa + 2**52 - 2**52;
39097 return copysign (xa, operand1);
39098 */
39105 /* xa = abs (operand1) */
39108 /* if (!isless (xa, TWO52)) goto label; */
39121 }
39123 /* Expand SSE2 sequence for computing floor or ceil from OPERAND1 storing
39124 into OPERAND0. */
39125 void
39127 {
39128 /* C code for the stuff we expand below.
39129 double xa = fabs (x), x2;
39130 if (!isless (xa, TWO52))
39131 return x;
39132 xa = xa + TWO52 - TWO52;
39133 x2 = copysign (xa, x);
39134 Compensate. Floor:
39135 if (x2 > x)
39136 x2 -= 1;
39137 Compensate. Ceil:
39138 if (x2 < x)
39139 x2 -= -1;
39140 return x2;
39141 */
39147 /* Temporary for holding the result, initialized to the input
39148 operand to ease control flow. */
39152 /* xa = abs (operand1) */
39155 /* if (!isless (xa, TWO52)) goto label; */
39158 /* xa = xa + TWO52 - TWO52; */
39162 /* xa = copysign (xa, operand1) */
39165 /* generate 1.0 or -1.0 */
39170 /* Compensate: xa = xa - (xa > operand1 ? 1 : 0) */
39174 /* We always need to subtract here to preserve signed zero. */
39183 }
39185 /* Expand SSE2 sequence for computing floor or ceil from OPERAND1 storing
39186 into OPERAND0. */
39187 void
39189 {
39190 /* C code for the stuff we expand below.
39191 double xa = fabs (x), x2;
39192 if (!isless (xa, TWO52))
39193 return x;
39194 x2 = (double)(long)x;
39195 Compensate. Floor:
39196 if (x2 > x)
39197 x2 -= 1;
39198 Compensate. Ceil:
39199 if (x2 < x)
39200 x2 += 1;
39201 if (HONOR_SIGNED_ZEROS (mode))
39202 return copysign (x2, x);
39203 return x2;
39204 */
39210 /* Temporary for holding the result, initialized to the input
39211 operand to ease control flow. */
39215 /* xa = abs (operand1) */
39218 /* if (!isless (xa, TWO52)) goto label; */
39221 /* xa = (double)(long)x */
39226 /* generate 1.0 */
39229 /* Compensate: xa = xa - (xa > operand1 ? 1 : 0) */
39244 }
39246 /* Expand SSE sequence for computing round from OPERAND1 storing
39247 into OPERAND0. Sequence that works without relying on DImode truncation
39248 via cvttsd2siq that is only available on 64bit targets. */
39249 void
39251 {
39252 /* C code for the stuff we expand below.
39253 double xa = fabs (x), xa2, x2;
39254 if (!isless (xa, TWO52))
39255 return x;
39256 Using the absolute value and copying back sign makes
39257 -0.0 -> -0.0 correct.
39258 xa2 = xa + TWO52 - TWO52;
39259 Compensate.
39260 dxa = xa2 - xa;
39261 if (dxa <= -0.5)
39262 xa2 += 1;
39263 else if (dxa > 0.5)
39264 xa2 -= 1;
39265 x2 = copysign (xa2, x);
39266 return x2;
39267 */
39273 /* Temporary for holding the result, initialized to the input
39274 operand to ease control flow. */
39278 /* xa = abs (operand1) */
39281 /* if (!isless (xa, TWO52)) goto label; */
39284 /* xa2 = xa + TWO52 - TWO52; */
39288 /* dxa = xa2 - xa; */
39291 /* generate 0.5, 1.0 and -0.5 */
39297 /* Compensate. */
39299 /* xa2 = xa2 - (dxa > 0.5 ? 1 : 0) */
39304 /* xa2 = xa2 + (dxa <= -0.5 ? 1 : 0) */
39310 /* res = copysign (xa2, operand1) */
39317 }
39319 /* Expand SSE sequence for computing trunc from OPERAND1 storing
39320 into OPERAND0. */
39321 void
39323 {
39324 /* C code for SSE variant we expand below.
39325 double xa = fabs (x), x2;
39326 if (!isless (xa, TWO52))
39327 return x;
39328 x2 = (double)(long)x;
39329 if (HONOR_SIGNED_ZEROS (mode))
39330 return copysign (x2, x);
39331 return x2;
39332 */
39338 /* Temporary for holding the result, initialized to the input
39339 operand to ease control flow. */
39343 /* xa = abs (operand1) */
39346 /* if (!isless (xa, TWO52)) goto label; */
39349 /* x = (double)(long)x */
39361 }
39363 /* Expand SSE sequence for computing trunc from OPERAND1 storing
39364 into OPERAND0. */
39365 void
39367 {
39371 /* C code for SSE variant we expand below.
39372 double xa = fabs (x), x2;
39373 if (!isless (xa, TWO52))
39374 return x;
39375 xa2 = xa + TWO52 - TWO52;
39376 Compensate:
39377 if (xa2 > xa)
39378 xa2 -= 1.0;
39379 x2 = copysign (xa2, x);
39380 return x2;
39381 */
39385 /* Temporary for holding the result, initialized to the input
39386 operand to ease control flow. */
39390 /* xa = abs (operand1) */
39393 /* if (!isless (xa, TWO52)) goto label; */
39396 /* res = xa + TWO52 - TWO52; */
39401 /* generate 1.0 */
39404 /* Compensate: res = xa2 - (res > xa ? 1 : 0) */
39412 /* res = copysign (res, operand1) */
39419 }
39421 /* Expand SSE sequence for computing round from OPERAND1 storing
39422 into OPERAND0. */
39423 void
39425 {
39426 /* C code for the stuff we're doing below:
39427 double xa = fabs (x);
39428 if (!isless (xa, TWO52))
39429 return x;
39430 xa = (double)(long)(xa + nextafter (0.5, 0.0));
39431 return copysign (xa, x);
39432 */
39438 /* Temporary for holding the result, initialized to the input
39439 operand to ease control flow. */
39447 /* load nextafter (0.5, 0.0) */
39452 /* xa = xa + 0.5 */
39456 /* xa = (double)(int64_t)xa */
39461 /* res = copysign (xa, operand1) */
39468 }
39470 /* Expand SSE sequence for computing round
39471 from OP1 storing into OP0 using sse4 round insn. */
39472 void
39474 {
39483 {
39494 }
39496 /* round (a) = trunc (a + copysign (0.5, a)) */
39498 /* load nextafter (0.5, 0.0) */
39504 /* e1 = copysign (0.5, op1) */
39508 /* e2 = op1 + e1 */
39511 /* res = trunc (e2) */
39516 }
39517 \f
39519 /* Table of valid machine attributes. */
39521 {
39522 /* { name, min_len, max_len, decl_req, type_req, fn_type_req, handler,
39523 affects_type_identity } */
39524 /* Stdcall attribute says callee is responsible for popping arguments
39525 if they are not variable. */
39528 /* Fastcall attribute says callee is responsible for popping arguments
39529 if they are not variable. */
39532 /* Thiscall attribute says callee is responsible for popping arguments
39533 if they are not variable. */
39536 /* Cdecl attribute says the callee is a normal C declaration */
39539 /* Regparm attribute specifies how many integer arguments are to be
39540 passed in registers. */
39543 /* Sseregparm attribute says we are using x86_64 calling conventions
39544 for FP arguments. */
39547 /* The transactional memory builtins are implicitly regparm or fastcall
39548 depending on the ABI. Override the generic do-nothing attribute that
39549 these builtins were declared with. */
39552 /* force_align_arg_pointer says this function realigns the stack at entry. */
39555 #if TARGET_DLLIMPORT_DECL_ATTRIBUTES
39560 #endif
39565 #ifdef SUBTARGET_ATTRIBUTE_TABLE
39566 SUBTARGET_ATTRIBUTE_TABLE,
39567 #endif
39568 /* ms_abi and sysv_abi calling convention function attributes. */
39575 /* End element. */
39577 };
39579 /* Implement targetm.vectorize.builtin_vectorization_cost. */
39580 static int
39582 tree vectype,
39584 {
39588 {
39633 }
39634 }
39636 /* A cached (set (nil) (vselect (vconcat (nil) (nil)) (parallel [])))
39637 insn, so that expand_vselect{,_vconcat} doesn't have to create a fresh
39638 insn every time. */
39642 /* Initialize vselect_insn. */
39644 static void
39646 {
39648 rtx x;
39659 }
39661 /* Construct (set target (vec_select op0 (parallel perm))) and
39662 return true if that's a valid instruction in the active ISA. */
39664 static bool
39667 {
39693 }
39695 /* Similar, but generate a vec_concat from op0 and op1 as well. */
39697 static bool
39701 {
39703 rtx x;
39718 }
39720 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to implement D
39721 in terms of blendp[sd] / pblendw / pblendvb / vpblendd. */
39723 static bool
39725 {
39734 ;
39736 ;
39738 ;
39739 else
39742 /* This is a blend, not a permute. Elements must stay in their
39743 respective lanes. */
39745 {
39749 }
39754 /* ??? Without SSE4.1, we could implement this with and/andn/or. This
39755 decision should be extracted elsewhere, so that we only try that
39756 sequence once all budget==3 options have been tried. */
39763 {
39787 /* See if bytes move in pairs so we can use pblendw with
39788 an immediate argument, rather than pblendvb with a vector
39789 argument. */
39792 {
39793 use_pblendvb:
39797 finish_pblendvb:
39803 else
39808 }
39813 /* FALLTHRU */
39815 do_subreg:
39822 /* See if bytes move in pairs. If not, vpblendvb must be used. */
39826 /* See if bytes move in quadruplets. If yes, vpblendd
39827 with immediate can be used. */
39832 {
39833 /* See if bytes move the same in both lanes. If yes,
39834 vpblendw with immediate can be used. */
39839 /* Use vpblendw. */
39844 }
39846 /* Use vpblendd. */
39853 /* See if words move in pairs. If yes, vpblendd can be used. */
39858 {
39859 /* See if words move the same in both lanes. If not,
39860 vpblendvb must be used. */
39863 {
39864 /* Use vpblendvb. */
39874 }
39876 /* Use vpblendw. */
39880 }
39882 /* Use vpblendd. */
39889 /* Use vpblendd. */
39897 }
39899 /* This matches five different patterns with the different modes. */
39907 }
39909 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to implement D
39910 in terms of the variable form of vpermilps.
39912 Note that we will have already failed the immediate input vpermilps,
39913 which requires that the high and low part shuffle be identical; the
39914 variable form doesn't require that. */
39916 static bool
39918 {
39925 /* We can only permute within the 128-bit lane. */
39927 {
39931 }
39937 {
39940 /* Within each 128-bit lane, the elements of op0 are numbered
39941 from 0 and the elements of op1 are numbered from 4. */
39948 }
39955 }
39957 /* Return true if permutation D can be performed as VMODE permutation
39958 instead. */
39960 static bool
39962 {
39977 else
39983 }
39985 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to implement D
39986 in terms of pshufb, vpperm, vpermq, vpermd, vpermps or vperm2i128. */
39988 static bool
39990 {
39999 {
40001 {
40004 {
40008 /* Use vperm2i128 insn. The pattern uses
40009 V4DImode instead of V2TImode. */
40022 }
40024 }
40025 }
40026 else
40027 {
40029 {
40032 }
40034 {
40038 /* V4DImode should be already handled through
40039 expand_vselect by vpermq instruction. */
40046 {
40047 /* First see if vpermq can be used for
40048 V8SImode/V16HImode/V32QImode. */
40050 {
40058 {
40062 }
40064 }
40066 /* Next see if vpermd can be used. */
40069 }
40070 /* Or if vpermps can be used. */
40075 {
40076 /* vpshufb only works intra lanes, it is not
40077 possible to shuffle bytes in between the lanes. */
40081 }
40082 }
40083 else
40085 }
40093 else
40094 {
40100 else
40104 {
40108 }
40109 }
40120 {
40127 else
40129 }
40130 else
40131 {
40134 }
40139 }
40141 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to instantiate D
40142 in a single instruction. */
40144 static bool
40146 {
40150 /* Check plain VEC_SELECT first, because AVX has instructions that could
40151 match both SEL and SEL+CONCAT, but the plain SEL will allow a memory
40152 input where SEL+CONCAT may not. */
40154 {
40160 {
40166 }
40169 {
40173 }
40175 {
40176 /* Use vpbroadcast{b,w,d}. */
40179 {
40198 /* For other modes prefer other shuffles this function creates. */
40200 }
40202 {
40206 }
40207 }
40212 /* There are plenty of patterns in sse.md that are written for
40213 SEL+CONCAT and are not replicated for a single op. Perhaps
40214 that should be changed, to avoid the nastiness here. */
40216 /* Recognize interleave style patterns, which means incrementing
40217 every other permutation operand. */
40219 {
40222 }
40227 /* Recognize shufps, which means adding {0, 0, nelt, nelt}. */
40229 {
40231 {
40236 }
40241 }
40242 }
40244 /* Finally, try the fully general two operand permute. */
40249 /* Recognize interleave style patterns with reversed operands. */
40251 {
40253 {
40257 else
40260 }
40265 }
40267 /* Try the SSE4.1 blend variable merge instructions. */
40271 /* Try one of the AVX vpermil variable permutations. */
40275 /* Try the SSSE3 pshufb or XOP vpperm or AVX2 vperm2i128,
40276 vpshufb, vpermd, vpermps or vpermq variable permutation. */
40281 }
40283 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to implement D
40284 in terms of a pair of pshuflw + pshufhw instructions. */
40286 static bool
40288 {
40296 /* The two permutations only operate in 64-bit lanes. */
40307 /* Emit the pshuflw. */
40314 /* Emit the pshufhw. */
40322 }
40324 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to simplify
40325 the permutation using the SSSE3 palignr instruction. This succeeds
40326 when all of the elements in PERM fit within one vector and we merely
40327 need to shift them down so that a single vector permutation has a
40328 chance to succeed. */
40330 static bool
40332 {
40339 /* Even with AVX, palignr only operates on 128-bit vectors. */
40345 {
40351 }
40355 /* Given that we have SSSE3, we know we'll be able to implement the
40356 single operand permutation after the palignr with pshufb. */
40371 {
40376 }
40378 /* Test for the degenerate case where the alignment by itself
40379 produces the desired permutation. */
40381 {
40384 }
40390 }
40394 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to simplify
40395 a two vector permutation into a single vector permutation by using
40396 an interleave operation to merge the vectors. */
40398 static bool
40400 {
40405 rtx seq;
40409 {
40412 }
40414 {
40417 /* For 32-byte modes allow even d->one_operand_p.
40418 The lack of cross-lane shuffling in some instructions
40419 might prevent a single insn shuffle. */
40422 /* If expand_vec_perm_interleave3 can expand this into
40423 a 3 insn sequence, give up and let it be expanded as
40424 3 insn sequence. While that is one insn longer,
40425 it doesn't need a memory operand and in the common
40426 case that both interleave low and high permutations
40427 with the same operands are adjacent needs 4 insns
40428 for both after CSE. */
40431 }
40432 else
40435 /* Examine from whence the elements come. */
40444 {
40447 /* Split the two input vectors into 4 halves. */
40453 /* If the elements from the low halves use interleave low, and similarly
40454 for interleave high. If the elements are from mis-matched halves, we
40455 can use shufps for V4SF/V4SI or do a DImode shuffle. */
40457 {
40458 /* punpckl* */
40460 {
40465 }
40468 }
40470 {
40471 /* punpckh* */
40473 {
40478 }
40481 }
40483 {
40484 /* shufps */
40486 {
40491 }
40493 {
40494 /* shufpd */
40499 }
40500 }
40502 {
40503 /* shufps */
40505 {
40510 }
40512 {
40513 /* shufpd */
40518 }
40519 }
40520 else
40522 }
40523 else
40524 {
40529 /* Split the two input vectors into 8 quarters. */
40535 {
40538 }
40541 {
40545 }
40547 {
40550 }
40553 {
40555 {
40556 /* Attempt to increase the likelihood that dfinal
40557 shuffle will be intra-lane. */
40561 }
40563 /* vperm2f128 or vperm2i128. */
40565 {
40570 }
40575 {
40579 {
40582 }
40583 }
40584 }
40589 {
40590 /* vpunpckl* */
40592 {
40601 }
40602 }
40605 {
40606 /* vpunpckh* */
40608 {
40617 }
40618 }
40619 else
40621 }
40623 /* Use the remapping array set up above to move the elements from their
40624 swizzled locations into their final destinations. */
40627 {
40630 /* If same_halves is true, both halves of the remapped vector are the
40631 same. Avoid cross-lane accesses if possible. */
40633 {
40636 }
40637 else
40639 }
40645 /* Test if the final remap can be done with a single insn. For V4SFmode or
40646 V4SImode this *will* succeed. For V8HImode or V16QImode it may not. */
40659 {
40662 }
40669 }
40671 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to simplify
40672 a single vector cross-lane permutation into vpermq followed
40673 by any of the single insn permutations. */
40675 static bool
40677 {
40691 {
40694 }
40697 {
40702 }
40715 {
40722 }
40729 {
40734 ;
40737 else
40739 }
40748 }
40750 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to expand
40751 a vector permutation using two instructions, vperm2f128 resp.
40752 vperm2i128 followed by any single in-lane permutation. */
40754 static bool
40756 {
40770 /* ((perm << 2)|perm) & 0x33 is the vperm2[fi]128
40771 immediate. For perm < 16 the second permutation uses
40772 d->op0 as first operand, for perm >= 16 it uses d->op1
40773 as first operand. The second operand is the result of
40774 vperm2[fi]128. */
40776 {
40777 /* Ignore permutations which do not move anything cross-lane. */
40779 {
40780 /* The second shuffle for e.g. V4DFmode has
40781 0123 and ABCD operands.
40782 Ignore AB23, as 23 is already in the second lane
40783 of the first operand. */
40785 /* And 01CD, as 01 is in the first lane of the first
40786 operand. */
40788 /* And 4567, as then the vperm2[fi]128 doesn't change
40789 anything on the original 4567 second operand. */
40791 }
40792 else
40793 {
40794 /* The second shuffle for e.g. V4DFmode has
40795 4567 and ABCD operands.
40796 Ignore AB67, as 67 is already in the second lane
40797 of the first operand. */
40799 /* And 45CD, as 45 is in the first lane of the first
40800 operand. */
40802 /* And 0123, as then the vperm2[fi]128 doesn't change
40803 anything on the original 0123 first operand. */
40805 }
40808 {
40814 else
40816 }
40819 {
40823 }
40824 else
40828 {
40832 /* Found a usable second shuffle. dfirst will be
40833 vperm2f128 on d->op0 and d->op1. */
40844 /* And dsecond is some single insn shuffle, taking
40845 d->op0 and result of vperm2f128 (if perm < 16) or
40846 d->op1 and result of vperm2f128 (otherwise). */
40855 }
40857 /* For one operand, the only useful vperm2f128 permutation is 0x10. */
40860 }
40863 }
40865 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to simplify
40866 a two vector permutation using 2 intra-lane interleave insns
40867 and cross-lane shuffle for 32-byte vectors. */
40869 static bool
40871 {
40878 ;
40880 ;
40881 else
40896 {
40900 else
40906 else
40912 else
40918 else
40924 else
40930 else
40935 }
40939 }
40941 /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to implement
40942 a single vector permutation using a single intra-lane vector
40943 permutation, vperm2f128 swapping the lanes and vblend* insn blending
40944 the non-swapped and swapped vectors together. */
40946 static bool
40948 {
40951 rtx seq;
40956 || TARGET_AVX2
40965 {
40972 }
41007 }
41009 /* A subroutine of ix86_expand_vec_perm_builtin_1. Implement a V4DF
41010 permutation using two vperm2f128, followed by a vshufpd insn blending
41011 the two vectors together. */
41013 static bool
41015 {
41058 }
41060 /* A subroutine of expand_vec_perm_even_odd_1. Implement the double-word
41061 permutation with two pshufb insns and an ior. We should have already
41062 failed all two instruction sequences. */
41064 static bool
41066 {
41077 /* Generate two permutation masks. If the required element is within
41078 the given vector it is shuffled into the proper lane. If the required
41079 element is in the other vector, force a zero into the lane by setting
41080 bit 7 in the permutation mask. */
41083 {
41090 {
41093 }
41094 }
41118 }
41120 /* Implement arbitrary permutation of one V32QImode and V16QImode operand
41121 with two vpshufb insns, vpermq and vpor. We should have already failed
41122 all two or three instruction sequences. */
41124 static bool
41126 {
41141 /* Generate two permutation masks. If the required element is within
41142 the same lane, it is shuffled in. If the required element from the
41143 other lane, force a zero by setting bit 7 in the permutation mask.
41144 In the other mask the mask has non-negative elements if element
41145 is requested from the other lane, but also moved to the other lane,
41146 so that the result of vpshufb can have the two V2TImode halves
41147 swapped. */
41150 {
41155 {
41158 }
41159 }
41168 /* Swap the 128-byte lanes of h into hp. */
41172 const1_rtx));
41189 }
41191 /* A subroutine of expand_vec_perm_even_odd_1. Implement extract-even
41192 and extract-odd permutations of two V32QImode and V16QImode operand
41193 with two vpshufb insns, vpor and vpermq. We should have already
41194 failed all two or three instruction sequences. */
41196 static bool
41198 {
41217 /* Generate two permutation masks. In the first permutation mask
41218 the first quarter will contain indexes for the first half
41219 of the op0, the second quarter will contain bit 7 set, third quarter
41220 will contain indexes for the second half of the op0 and the
41221 last quarter bit 7 set. In the second permutation mask
41222 the first quarter will contain bit 7 set, the second quarter
41223 indexes for the first half of the op1, the third quarter bit 7 set
41224 and last quarter indexes for the second half of the op1.
41225 I.e. the first mask e.g. for V32QImode extract even will be:
41226 0, 2, ..., 0xe, -128, ..., -128, 0, 2, ..., 0xe, -128, ..., -128
41227 (all values masked with 0xf except for -128) and second mask
41228 for extract even will be
41229 -128, ..., -128, 0, 2, ..., 0xe, -128, ..., -128, 0, 2, ..., 0xe. */
41232 {
41238 {
41241 }
41242 }
41261 /* Permute the V4DImode quarters using { 0, 2, 1, 3 } permutation. */
41269 }
41271 /* A subroutine of ix86_expand_vec_perm_builtin_1. Implement extract-even
41272 and extract-odd permutations. */
41274 static bool
41276 {
41280 {
41285 /* Shuffle the lanes around into { 0 1 4 5 } and { 2 3 6 7 }. */
41289 /* Now an unpck[lh]pd will produce the result required. */
41292 else
41298 {
41305 /* Shuffle within the 128-bit lanes to produce:
41306 { 0 2 8 a 4 6 c e } | { 1 3 9 b 5 7 d f }. */
41310 /* Shuffle the lanes around to produce:
41311 { 4 6 c e 0 2 8 a } and { 5 7 d f 1 3 9 b }. */
41315 /* Shuffle within the 128-bit lanes to produce:
41316 { 0 2 4 6 4 6 0 2 } | { 1 3 5 7 5 7 1 3 }. */
41319 /* Shuffle within the 128-bit lanes to produce:
41320 { 8 a c e c e 8 a } | { 9 b d f d f 9 b }. */
41323 /* Shuffle the lanes around to produce:
41324 { 0 2 4 6 8 a c e } | { 1 3 5 7 9 b d f }. */
41327 }
41334 /* These are always directly implementable by expand_vec_perm_1. */
41340 else
41341 {
41342 /* We need 2*log2(N)-1 operations to achieve odd/even
41343 with interleave. */
41352 else
41355 }
41361 else
41362 {
41374 else
41377 }
41386 {
41393 {
41398 }
41400 }
41405 /* Shuffle the lanes around into { 0 1 4 5 } and { 2 3 6 7 }. */
41409 /* Now an vpunpck[lh]qdq will produce the result required. */
41412 else
41419 {
41426 {
41431 }
41433 }
41441 /* Shuffle the lanes around into
41442 { 0 1 2 3 8 9 a b } and { 4 5 6 7 c d e f }. */
41450 /* Swap the 2nd and 3rd position in each lane into
41451 { 0 2 1 3 8 a 9 b } and { 4 6 5 7 c e d f }. */
41457 /* Now an vpunpck[lh]qdq will produce
41458 { 0 2 4 6 8 a c e } resp. { 1 3 5 7 9 b d f }. */
41462 else
41471 }
41474 }
41476 /* A subroutine of ix86_expand_vec_perm_builtin_1. Pattern match
41477 extract-even and extract-odd permutations. */
41479 static bool
41481 {
41493 }
41495 /* A subroutine of ix86_expand_vec_perm_builtin_1. Implement broadcast
41496 permutations. We assume that expand_vec_perm_1 has already failed. */
41498 static bool
41500 {
41508 {
41511 /* These are special-cased in sse.md so that we can optionally
41512 use the vbroadcast instruction. They expand to two insns
41513 if the input happens to be in a register. */
41520 /* These are always implementable using standard shuffle patterns. */
41525 /* These can be implemented via interleave. We save one insn by
41526 stopping once we have promoted to V4SImode and then use pshufd. */
41527 do
41528 {
41529 rtx dest;
41535 {
41539 }
41546 }
41561 /* For AVX2 broadcasts of the first element vpbroadcast* or
41562 vpermq should be used by expand_vec_perm_1. */
41568 }
41569 }
41571 /* A subroutine of ix86_expand_vec_perm_builtin_1. Pattern match
41572 broadcast permutations. */
41574 static bool
41576 {
41588 }
41590 /* Implement arbitrary permutation of two V32QImode and V16QImode operands
41591 with 4 vpshufb insns, 2 vpermq and 3 vpor. We should have already failed
41592 all the shorter instruction sequences. */
41594 static bool
41596 {
41612 /* Generate 4 permutation masks. If the required element is within
41613 the same lane, it is shuffled in. If the required element from the
41614 other lane, force a zero by setting bit 7 in the permutation mask.
41615 In the other mask the mask has non-negative elements if element
41616 is requested from the other lane, but also moved to the other lane,
41617 so that the result of vpshufb can have the two V2TImode halves
41618 swapped. */
41621 {
41626 }
41632 {
41640 }
41643 {
41645 {
41648 }
41655 }
41657 /* Swap the 128-byte lanes of h[X]. */
41659 {
41665 const1_rtx));
41667 }
41670 {
41672 {
41675 }
41681 }
41684 {
41686 {
41690 }
41693 }
41703 }
41705 /* The guts of ix86_expand_vec_perm_const, also used by the ok hook.
41706 With all of the interface bits taken care of, perform the expansion
41707 in D and return true on success. */
41709 static bool
41711 {
41712 /* Try a single instruction expansion. */
41716 /* Try sequences of two instructions. */
41736 /* Try sequences of three instructions. */
41750 /* Try sequences of four instructions. */
41758 /* ??? Look for narrow permutations whose element orderings would
41759 allow the promotion to a wider mode. */
41761 /* ??? Look for sequences of interleave or a wider permute that place
41762 the data into the correct lanes for a half-vector shuffle like
41763 pshuf[lh]w or vpermilps. */
41765 /* ??? Look for sequences of interleave that produce the desired results.
41766 The combinatorics of punpck[lh] get pretty ugly... */
41771 /* Even longer sequences. */
41776 }
41778 /* If a permutation only uses one operand, make it clear. Returns true
41779 if the permutation references both operands. */
41781 static bool
41783 {
41791 {
41797 {
41800 }
41801 /* The elements of PERM do not suggest that only the first operand
41802 is used, but both operands are identical. Allow easier matching
41803 of the permutation by folding the permutation into the single
41804 input vector. */
41805 /* FALLTHRU */
41816 }
41819 }
41821 bool
41823 {
41828 rtx sel;
41845 {
41850 }
41857 /* If the selector says both arguments are needed, but the operands are the
41858 same, the above tried to expand with one_operand_p and flattened selector.
41859 If that didn't work, retry without one_operand_p; we succeeded with that
41860 during testing. */
41862 {
41866 }
41869 }
41871 /* Implement targetm.vectorize.vec_perm_const_ok. */
41873 static bool
41876 {
41885 /* Given sufficient ISA support we can just return true here
41886 for selected vector modes. */
41888 {
41889 /* All implementable with a single vpperm insn. */
41892 /* All implementable with 2 pshufb + 1 ior. */
41895 /* All implementable with shufpd or unpck[lh]pd. */
41898 }
41900 /* Extract the values from the vector CST into the permutation
41901 array in D. */
41904 {
41908 }
41910 /* For all elements from second vector, fold the elements to first. */
41915 /* Check whether the mask can be applied to the vector type. */
41918 /* Implementable with shufps or pshufd. */
41922 /* Otherwise we have to go through the motions and see if we can
41923 figure out how to generate the requested permutation. */
41934 }
41936 void
41938 {
41953 /* We'll either be able to implement the permutation directly... */
41957 /* ... or we use the special-case patterns. */
41959 }
41961 static void
41963 {
41978 {
41981 }
41983 /* Note that for AVX this isn't one instruction. */
41986 }
41989 /* Expand a vector operation CODE for a V*QImode in terms of the
41990 same operation on V*HImode. */
41992 void
41994 {
42006 {
42019 }
42023 {
42025 /* Unpack data such that we've got a source byte in each low byte of
42026 each word. We don't care what goes into the high byte of each word.
42027 Rather than trying to get zero in there, most convenient is to let
42028 it be a copy of the low byte. */
42033 /* FALLTHRU */
42045 /* FALLTHRU */
42055 }
42057 /* Perform the operation. */
42064 /* Merge the data back into the right place. */
42074 {
42075 /* For SSE2, we used an full interleave, so the desired
42076 results are in the even elements. */
42079 }
42080 else
42081 {
42082 /* For AVX, the interleave used above was not cross-lane. So the
42083 extraction is evens but with the second and third quarter swapped.
42084 Happily, that is even one insn shorter than even extraction. */
42087 }
42094 }
42096 /* Helper function of ix86_expand_mul_widen_evenodd. Return true
42097 if op is CONST_VECTOR with all odd elements equal to their
42098 preceding element. */
42100 static bool
42102 {
42112 }
42114 void
42117 {
42120 rtx x;
42128 /* We only play even/odd games with vectors of SImode. */
42131 /* If we're looking for the odd results, shift those members down to
42132 the even slots. For some cpus this is faster than a PSHUFD. */
42134 {
42135 /* For XOP use vpmacsdqh, but only for smult, as it is only
42136 signed. */
42138 {
42142 }
42153 }
42156 {
42159 else
42161 }
42166 else
42167 {
42170 /* The easiest way to implement this without PMULDQ is to go through
42171 the motions as if we are performing a full 64-bit multiply. With
42172 the exception that we need to do less shuffling of the elements. */
42174 /* Compute the sign-extension, aka highparts, of the two operands. */
42180 /* Multiply LO(A) * HI(B), and vice-versa. */
42186 /* Multiply LO(A) * LO(B). */
42190 /* Combine and shift the highparts into place. */
42195 /* Combine high and low parts. */
42198 }
42200 }
42202 void
42205 {
42211 {
42216 {
42217 /* With XOP, we have pmacsdqh, aka mul_widen_odd. In this case,
42218 shuffle the elements once so that all elements are in the right
42219 place for immediate use: { A C B D }. */
42224 }
42225 else
42226 {
42227 /* Put the elements into place for the multiply. */
42231 }
42236 /* Shuffle the elements between the lanes. After this we
42237 have { A B E F | C D G H } for each operand. */
42247 /* Shuffle the elements within the lanes. After this we
42248 have { A A B B | C C D D } or { E E F F | G G H H }. */
42286 }
42287 }
42289 void
42291 {
42301 /* Move the results in element 2 down to element 1; we don't care
42302 what goes in elements 2 and 3. Then we can merge the parts
42303 back together with an interleave.
42305 Note that two other sequences were tried:
42306 (1) Use interleaves at the start instead of psrldq, which allows
42307 us to use a single shufps to merge things back at the end.
42308 (2) Use shufps here to combine the two vectors, then pshufd to
42309 put the elements in the correct order.
42310 In both cases the cost of the reformatting stall was too high
42311 and the overall sequence slower. */
42322 }
42324 void
42326 {
42331 {
42332 /* op1: A,B,C,D, op2: E,F,G,H */
42341 /* t1: B,A,D,C */
42348 /* t2: (B*E),(A*F),(D*G),(C*H) */
42351 /* t3: (B*E)+(A*F), (D*G)+(C*H) */
42354 /* t4: ((B*E)+(A*F))<<32, ((D*G)+(C*H))<<32 */
42357 /* op0: (((B*E)+(A*F))<<32)+(B*F), (((D*G)+(C*H))<<32)+(D*H) */
42359 }
42360 else
42361 {
42366 {
42369 }
42371 {
42374 }
42375 else
42379 /* Multiply low parts. */
42383 /* Shift input vectors right 32 bits so we can multiply high parts. */
42388 /* Multiply high parts by low parts. */
42394 /* Combine and shift the highparts back. */
42398 /* Combine high and low parts. */
42400 }
42404 }
42406 /* Return 1 if control tansfer instruction INSN
42407 should be encoded with bnd prefix.
42408 If insn is NULL then return 1 when control
42409 transfer instructions should be prefixed with
42410 bnd by default for current function. */
42412 bool
42414 {
42416 }
42418 /* Calculate integer abs() using only SSE2 instructions. */
42420 void
42422 {
42427 {
42428 /* For 32-bit signed integer X, the best way to calculate the absolute
42429 value of X is (((signed) X >> (W-1)) ^ X) - ((signed) X >> (W-1)). */
42441 /* For 16-bit signed integer X, the best way to calculate the absolute
42442 value of X is max (X, -X), as SSE2 provides the PMAXSW insn. */
42450 /* For 8-bit signed integer X, the best way to calculate the absolute
42451 value of X is min ((unsigned char) X, (unsigned char) (-X)),
42452 as SSE2 provides the PMINUB insn. */
42462 }
42466 }
42468 /* Expand an insert into a vector register through pinsr insn.
42469 Return true if successful. */
42471 bool
42473 {
42481 {
42484 }
42490 {
42495 {
42502 {
42534 }
42548 }
42552 }
42553 }
42554 \f
42555 /* This function returns the calling abi specific va_list type node.
42556 It returns the FNDECL specific va_list type. */
42558 static tree
42560 {
42567 else
42569 }
42571 /* Returns the canonical va_list type specified by TYPE. If there
42572 is no valid TYPE provided, it return NULL_TREE. */
42574 static tree
42576 {
42579 /* Resolve references and pointers to va_list type. */
42588 {
42593 {
42594 /* If va_list is an array type, the argument may have decayed
42595 to a pointer type, e.g. by being passed to another function.
42596 In that case, unwrap both types so that we can compare the
42597 underlying records. */
42600 {
42603 }
42604 }
42611 {
42612 /* If va_list is an array type, the argument may have decayed
42613 to a pointer type, e.g. by being passed to another function.
42614 In that case, unwrap both types so that we can compare the
42615 underlying records. */
42618 {
42621 }
42622 }
42629 {
42630 /* If va_list is an array type, the argument may have decayed
42631 to a pointer type, e.g. by being passed to another function.
42632 In that case, unwrap both types so that we can compare the
42633 underlying records. */
42636 {
42639 }
42640 }
42644 }
42646 }
42648 /* Iterate through the target-specific builtin types for va_list.
42649 IDX denotes the iterator, *PTREE is set to the result type of
42650 the va_list builtin, and *PNAME to its internal type.
42651 Returns zero if there is no element for this index, otherwise
42652 IDX should be increased upon the next call.
42653 Note, do not iterate a base builtin's name like __builtin_va_list.
42654 Used from c_common_nodes_and_builtins. */
42656 static int
42658 {
42660 {
42662 {
42675 }
42676 }
42679 }
42681 #undef TARGET_SCHED_DISPATCH
42682 #define TARGET_SCHED_DISPATCH has_dispatch
42683 #undef TARGET_SCHED_DISPATCH_DO
42684 #define TARGET_SCHED_DISPATCH_DO do_dispatch
42685 #undef TARGET_SCHED_REASSOCIATION_WIDTH
42686 #define TARGET_SCHED_REASSOCIATION_WIDTH ix86_reassociation_width
42687 #undef TARGET_SCHED_REORDER
42688 #define TARGET_SCHED_REORDER ix86_sched_reorder
42689 #undef TARGET_SCHED_ADJUST_PRIORITY
42690 #define TARGET_SCHED_ADJUST_PRIORITY ix86_adjust_priority
42691 #undef TARGET_SCHED_DEPENDENCIES_EVALUATION_HOOK
42692 #define TARGET_SCHED_DEPENDENCIES_EVALUATION_HOOK \
42693 ix86_dependencies_evaluation_hook
42695 /* The size of the dispatch window is the total number of bytes of
42696 object code allowed in a window. */
42697 #define DISPATCH_WINDOW_SIZE 16
42699 /* Number of dispatch windows considered for scheduling. */
42700 #define MAX_DISPATCH_WINDOWS 3
42702 /* Maximum number of instructions in a window. */
42703 #define MAX_INSN 4
42705 /* Maximum number of immediate operands in a window. */
42706 #define MAX_IMM 4
42708 /* Maximum number of immediate bits allowed in a window. */
42709 #define MAX_IMM_SIZE 128
42711 /* Maximum number of 32 bit immediates allowed in a window. */
42712 #define MAX_IMM_32 4
42714 /* Maximum number of 64 bit immediates allowed in a window. */
42715 #define MAX_IMM_64 2
42717 /* Maximum total of loads or prefetches allowed in a window. */
42718 #define MAX_LOAD 2
42720 /* Maximum total of stores allowed in a window. */
42721 #define MAX_STORE 1
42723 #undef BIG
42724 #define BIG 100
42727 /* Dispatch groups. Istructions that affect the mix in a dispatch window. */
42730 disp_load,
42731 disp_store,
42732 disp_load_store,
42733 disp_prefetch,
42734 disp_imm,
42735 disp_imm_32,
42736 disp_imm_64,
42737 disp_branch,
42738 disp_cmp,
42739 disp_jcc,
42740 disp_last
42741 };
42743 /* Number of allowable groups in a dispatch window. It is an array
42744 indexed by dispatch_group enum. 100 is used as a big number,
42745 because the number of these kind of operations does not have any
42746 effect in dispatch window, but we need them for other reasons in
42747 the table. */
42750 };
42756 };
42758 /* Instruction path. */
42764 last_path
42765 };
42767 /* sched_insn_info defines a window to the instructions scheduled in
42768 the basic block. It contains a pointer to the insn_info table and
42769 the instruction scheduled.
42771 Windows are allocated for each basic block and are linked
42772 together. */
42774 rtx insn;
42781 /* Linked list of dispatch windows. This is a two way list of
42782 dispatch windows of a basic block. It contains information about
42783 the number of uops in the window and the total number of
42784 instructions and of bytes in the object code for this dispatch
42785 window. */
42803 /* Immediate valuse used in an insn. */
42805 {
42814 /* Get dispatch group of insn. */
42816 static enum dispatch_group
42818 {
42834 }
42836 /* Return true if insn is a compare instruction. */
42838 static bool
42840 {
42848 }
42850 /* Return true if a dispatch violation encountered. */
42852 static bool
42854 {
42858 }
42860 /* Return true if insn is a branch instruction. */
42862 static bool
42864 {
42866 }
42868 /* Return true if insn is a prefetch instruction. */
42870 static bool
42872 {
42874 }
42876 /* This function initializes a dispatch window and the list container holding a
42877 pointer to the window. */
42879 static void
42881 {
42887 else
42905 {
42911 }
42913 }
42915 /* This function allocates and initializes a dispatch window and the
42916 list container holding a pointer to the window. */
42920 {
42925 }
42927 /* This routine initializes the dispatch scheduling information. It
42928 initiates building dispatch scheduler tables and constructs the
42929 first dispatch window. */
42931 static void
42933 {
42934 /* Allocate a dispatch list and a window. */
42939 }
42941 /* This function returns true if a branch is detected. End of a basic block
42942 does not have to be a branch, but here we assume only branches end a
42943 window. */
42945 static bool
42947 {
42949 }
42951 /* This function is called when the end of a window processing is reached. */
42953 static void
42955 {
42958 {
42963 }
42965 }
42967 /* Allocates a new dispatch window and adds it to WINDOW_LIST.
42968 WINDOW_NUM is either 0 or 1. A maximum of two windows are generated
42969 for 48 bytes of instructions. Note that these windows are not dispatch
42970 windows that their sizes are DISPATCH_WINDOW_SIZE. */
42974 {
42976 {
42981 }
42987 }
42989 /* Increment the number of immediate operands of an instruction. */
42991 static int
42993 {
42998 {
43005 else
43016 {
43019 }
43024 }
43027 }
43029 /* Compute number of immediate operands of an instruction. */
43031 static void
43033 {
43036 }
43038 /* Return total size of immediate operands of an instruction along with number
43039 of corresponding immediate-operands. It initializes its parameters to zero
43040 befor calling FIND_CONSTANT.
43041 INSN is the input instruction. IMM is the total of immediates.
43042 IMM32 is the number of 32 bit immediates. IMM64 is the number of 64
43043 bit immediates. */
43045 static int
43047 {
43055 }
43057 /* This function indicates if an operand of an instruction is an
43058 immediate. */
43060 static bool
43062 {
43071 }
43073 /* Return single or double path for instructions. */
43075 static enum insn_path
43077 {
43087 }
43089 /* Return insn dispatch group. */
43091 static enum dispatch_group
43093 {
43111 }
43113 /* Count number of GROUP restricted instructions in a dispatch
43114 window WINDOW_LIST. */
43116 static int
43118 {
43129 {
43148 }
43161 }
43163 /* This function returns true if insn satisfies dispatch rules on the
43164 last window scheduled. */
43166 static bool
43168 {
43176 /* Make disp_cmp and disp_jcc get scheduled at the latest. These
43177 instructions should be given the lowest priority in the
43178 scheduling process in Haifa scheduler to make sure they will be
43179 scheduled in the same dispatch window as the reference to them. */
43183 /* Check nonrestricted. */
43187 /* Get last dispatch window. */
43192 {
43197 /* Window 1 is full. Go for next window. */
43199 }
43206 /* See if it fits in the first window. */
43208 {
43209 /* The first widow should have only single and double path
43210 uops. */
43216 }
43218 }
43220 /* Add an instruction INSN with NUM_UOPS micro-operations to the
43221 dispatch window WINDOW_LIST. */
43223 static void
43225 {
43243 /* Initialize window with new instruction. */
43264 {
43267 }
43268 }
43270 /* Adds a scheduled instruction, INSN, to the current dispatch window.
43271 If the total bytes of instructions or the number of instructions in
43272 the window exceed allowable, it allocates a new window. */
43274 static void
43276 {
43299 /* Get the last dispatch window. */
43307 else
43310 /* If current window is full, get a new window.
43311 Window number zero is full, if MAX_INSN uops are scheduled in it.
43312 Window number one is full, if window zero's bytes plus window
43313 one's bytes is 32, or if the bytes of the new instruction added
43314 to the total makes it greater than 48, or it has already MAX_INSN
43315 instructions in it. */
43324 {
43327 }
43330 {
43334 {
43337 }
43338 }
43340 {
43345 {
43348 }
43351 }
43352 else
43356 {
43357 /* End of basic block is reached do end-basic-block process. */
43360 }
43361 }
43363 /* Print the dispatch window, WINDOW_NUM, to FILE. */
43365 DEBUG_FUNCTION static void
43367 {
43373 else
43387 {
43390 fprintf (file, " group[%d] = %s, insn[%d] = %p, path[%d] = %d byte_len[%d] = %d, imm_bytes[%d] = %d\n",
43396 }
43397 }
43399 /* Print to stdout a dispatch window. */
43401 DEBUG_FUNCTION void
43403 {
43405 }
43407 /* Print INSN dispatch information to FILE. */
43409 DEBUG_FUNCTION static void
43411 {
43434 }
43436 /* Print to STDERR the status of the ready list with respect to
43437 dispatch windows. */
43439 DEBUG_FUNCTION void
43441 {
43449 }
43451 /* This routine is the driver of the dispatch scheduler. */
43453 static void
43455 {
43460 }
43462 /* Return TRUE if Dispatch Scheduling is supported. */
43464 static bool
43466 {
43470 {
43486 }
43489 }
43491 /* Implementation of reassociation_width target hook used by
43492 reassoc phase to identify parallelism level in reassociated
43493 tree. Statements tree_code is passed in OPC. Arguments type
43494 is passed in MODE.
43496 Currently parallel reassociation is enabled for Atom
43497 processors only and we set reassociation width to be 2
43498 because Atom may issue up to 2 instructions per cycle.
43500 Return value should be fixed if parallel reassociation is
43501 enabled for other processors. */
43503 static int
43506 {
43515 }
43517 /* ??? No autovectorization into MMX or 3DNOW until we can reliably
43518 place emms and femms instructions. */
43520 static enum machine_mode
43522 {
43527 {
43540 else
43550 /* FALLTHRU */
43554 }
43555 }
43557 /* If AVX is enabled then try vectorizing with both 256bit and 128bit
43558 vectors. */
43560 static unsigned int
43562 {
43564 }
43566 \f
43568 /* Return class of registers which could be used for pseudo of MODE
43569 and of class RCLASS for spilling instead of memory. Return NO_REGS
43570 if it is not possible or non-profitable. */
43571 static reg_class_t
43573 {
43579 }
43581 /* Implement targetm.vectorize.init_cost. */
43585 {
43589 }
43591 /* Implement targetm.vectorize.add_stmt_cost. */
43593 static unsigned
43597 {
43604 /* Statements in an inner loop relative to the loop being
43605 vectorized are weighted more heavily. The value here is
43606 arbitrary and could potentially be improved with analysis. */
43614 }
43616 /* Implement targetm.vectorize.finish_cost. */
43618 static void
43621 {
43626 }
43628 /* Implement targetm.vectorize.destroy_cost_data. */
43630 static void
43632 {
43634 }
43636 /* Validate target specific memory model bits in VAL. */
43638 static unsigned HOST_WIDE_INT
43640 {
43647 {
43651 }
43654 {
43658 }
43660 {
43664 }
43666 }
43668 /* Implement TARGET_FLOAT_EXCEPTIONS_ROUNDING_SUPPORTED_P. */
43670 static bool
43672 {
43673 /* For x87 floating point with standard excess precision handling,
43674 there is no adddf3 pattern (since x87 floating point only has
43675 XFmode operations) so the default hook implementation gets this
43676 wrong. */
43678 }
43680 /* Implement TARGET_ATOMIC_ASSIGN_EXPAND_FENV. */
43682 static void
43684 {
43689 {
43704 hold_fnclex);
43718 }
43720 {
43729 mxcsr_orig_var,
43739 ldmxcsr_hold_call);
43742 else
43747 ldmxcsr_clear_call);
43748 else
43752 stxmcsr_update_call);
43754 {
43760 exceptions_assign);
43761 }
43762 else
43767 ldmxcsr_update_call);
43768 }
43769 tree atomic_feraiseexcept
43774 atomic_feraiseexcept_call);
43775 }
43777 /* Initialize the GCC target structure. */
43778 #undef TARGET_RETURN_IN_MEMORY
43779 #define TARGET_RETURN_IN_MEMORY ix86_return_in_memory
43781 #undef TARGET_LEGITIMIZE_ADDRESS
43782 #define TARGET_LEGITIMIZE_ADDRESS ix86_legitimize_address
43784 #undef TARGET_ATTRIBUTE_TABLE
43785 #define TARGET_ATTRIBUTE_TABLE ix86_attribute_table
43786 #undef TARGET_FUNCTION_ATTRIBUTE_INLINABLE_P
43787 #define TARGET_FUNCTION_ATTRIBUTE_INLINABLE_P hook_bool_const_tree_true
43788 #if TARGET_DLLIMPORT_DECL_ATTRIBUTES
43789 # undef TARGET_MERGE_DECL_ATTRIBUTES
43790 # define TARGET_MERGE_DECL_ATTRIBUTES merge_dllimport_decl_attributes
43791 #endif
43793 #undef TARGET_COMP_TYPE_ATTRIBUTES
43794 #define TARGET_COMP_TYPE_ATTRIBUTES ix86_comp_type_attributes
43796 #undef TARGET_INIT_BUILTINS
43797 #define TARGET_INIT_BUILTINS ix86_init_builtins
43798 #undef TARGET_BUILTIN_DECL
43799 #define TARGET_BUILTIN_DECL ix86_builtin_decl
43800 #undef TARGET_EXPAND_BUILTIN
43801 #define TARGET_EXPAND_BUILTIN ix86_expand_builtin
43803 #undef TARGET_VECTORIZE_BUILTIN_VECTORIZED_FUNCTION
43804 #define TARGET_VECTORIZE_BUILTIN_VECTORIZED_FUNCTION \
43805 ix86_builtin_vectorized_function
43807 #undef TARGET_VECTORIZE_BUILTIN_TM_LOAD
43808 #define TARGET_VECTORIZE_BUILTIN_TM_LOAD ix86_builtin_tm_load
43810 #undef TARGET_VECTORIZE_BUILTIN_TM_STORE
43811 #define TARGET_VECTORIZE_BUILTIN_TM_STORE ix86_builtin_tm_store
43813 #undef TARGET_VECTORIZE_BUILTIN_GATHER
43814 #define TARGET_VECTORIZE_BUILTIN_GATHER ix86_vectorize_builtin_gather
43816 #undef TARGET_BUILTIN_RECIPROCAL
43817 #define TARGET_BUILTIN_RECIPROCAL ix86_builtin_reciprocal
43819 #undef TARGET_ASM_FUNCTION_EPILOGUE
43820 #define TARGET_ASM_FUNCTION_EPILOGUE ix86_output_function_epilogue
43822 #undef TARGET_ENCODE_SECTION_INFO
43823 #ifndef SUBTARGET_ENCODE_SECTION_INFO
43824 #define TARGET_ENCODE_SECTION_INFO ix86_encode_section_info
43825 #else
43826 #define TARGET_ENCODE_SECTION_INFO SUBTARGET_ENCODE_SECTION_INFO
43827 #endif
43829 #undef TARGET_ASM_OPEN_PAREN
43831 #undef TARGET_ASM_CLOSE_PAREN
43834 #undef TARGET_ASM_BYTE_OP
43835 #define TARGET_ASM_BYTE_OP ASM_BYTE
43837 #undef TARGET_ASM_ALIGNED_HI_OP
43838 #define TARGET_ASM_ALIGNED_HI_OP ASM_SHORT
43839 #undef TARGET_ASM_ALIGNED_SI_OP
43840 #define TARGET_ASM_ALIGNED_SI_OP ASM_LONG
43841 #ifdef ASM_QUAD
43842 #undef TARGET_ASM_ALIGNED_DI_OP
43843 #define TARGET_ASM_ALIGNED_DI_OP ASM_QUAD
43844 #endif
43846 #undef TARGET_PROFILE_BEFORE_PROLOGUE
43847 #define TARGET_PROFILE_BEFORE_PROLOGUE ix86_profile_before_prologue
43849 #undef TARGET_MANGLE_DECL_ASSEMBLER_NAME
43850 #define TARGET_MANGLE_DECL_ASSEMBLER_NAME ix86_mangle_decl_assembler_name
43852 #undef TARGET_ASM_UNALIGNED_HI_OP
43853 #define TARGET_ASM_UNALIGNED_HI_OP TARGET_ASM_ALIGNED_HI_OP
43854 #undef TARGET_ASM_UNALIGNED_SI_OP
43855 #define TARGET_ASM_UNALIGNED_SI_OP TARGET_ASM_ALIGNED_SI_OP
43856 #undef TARGET_ASM_UNALIGNED_DI_OP
43857 #define TARGET_ASM_UNALIGNED_DI_OP TARGET_ASM_ALIGNED_DI_OP
43859 #undef TARGET_PRINT_OPERAND
43860 #define TARGET_PRINT_OPERAND ix86_print_operand
43861 #undef TARGET_PRINT_OPERAND_ADDRESS
43862 #define TARGET_PRINT_OPERAND_ADDRESS ix86_print_operand_address
43863 #undef TARGET_PRINT_OPERAND_PUNCT_VALID_P
43864 #define TARGET_PRINT_OPERAND_PUNCT_VALID_P ix86_print_operand_punct_valid_p
43865 #undef TARGET_ASM_OUTPUT_ADDR_CONST_EXTRA
43866 #define TARGET_ASM_OUTPUT_ADDR_CONST_EXTRA i386_asm_output_addr_const_extra
43868 #undef TARGET_SCHED_INIT_GLOBAL
43869 #define TARGET_SCHED_INIT_GLOBAL ix86_sched_init_global
43870 #undef TARGET_SCHED_ADJUST_COST
43871 #define TARGET_SCHED_ADJUST_COST ix86_adjust_cost
43872 #undef TARGET_SCHED_ISSUE_RATE
43873 #define TARGET_SCHED_ISSUE_RATE ix86_issue_rate
43874 #undef TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD
43875 #define TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD \
43876 ia32_multipass_dfa_lookahead
43877 #undef TARGET_SCHED_MACRO_FUSION_P
43878 #define TARGET_SCHED_MACRO_FUSION_P ix86_macro_fusion_p
43879 #undef TARGET_SCHED_MACRO_FUSION_PAIR_P
43880 #define TARGET_SCHED_MACRO_FUSION_PAIR_P ix86_macro_fusion_pair_p
43882 #undef TARGET_FUNCTION_OK_FOR_SIBCALL
43883 #define TARGET_FUNCTION_OK_FOR_SIBCALL ix86_function_ok_for_sibcall
43885 #undef TARGET_MEMMODEL_CHECK
43886 #define TARGET_MEMMODEL_CHECK ix86_memmodel_check
43888 #undef TARGET_ATOMIC_ASSIGN_EXPAND_FENV
43889 #define TARGET_ATOMIC_ASSIGN_EXPAND_FENV ix86_atomic_assign_expand_fenv
43891 #ifdef HAVE_AS_TLS
43892 #undef TARGET_HAVE_TLS
43893 #define TARGET_HAVE_TLS true
43894 #endif
43895 #undef TARGET_CANNOT_FORCE_CONST_MEM
43896 #define TARGET_CANNOT_FORCE_CONST_MEM ix86_cannot_force_const_mem
43897 #undef TARGET_USE_BLOCKS_FOR_CONSTANT_P
43898 #define TARGET_USE_BLOCKS_FOR_CONSTANT_P hook_bool_mode_const_rtx_true
43900 #undef TARGET_DELEGITIMIZE_ADDRESS
43901 #define TARGET_DELEGITIMIZE_ADDRESS ix86_delegitimize_address
43903 #undef TARGET_MS_BITFIELD_LAYOUT_P
43904 #define TARGET_MS_BITFIELD_LAYOUT_P ix86_ms_bitfield_layout_p
43906 #if TARGET_MACHO
43907 #undef TARGET_BINDS_LOCAL_P
43908 #define TARGET_BINDS_LOCAL_P darwin_binds_local_p
43909 #endif
43910 #if TARGET_DLLIMPORT_DECL_ATTRIBUTES
43911 #undef TARGET_BINDS_LOCAL_P
43912 #define TARGET_BINDS_LOCAL_P i386_pe_binds_local_p
43913 #endif
43915 #undef TARGET_ASM_OUTPUT_MI_THUNK
43916 #define TARGET_ASM_OUTPUT_MI_THUNK x86_output_mi_thunk
43917 #undef TARGET_ASM_CAN_OUTPUT_MI_THUNK
43918 #define TARGET_ASM_CAN_OUTPUT_MI_THUNK x86_can_output_mi_thunk
43920 #undef TARGET_ASM_FILE_START
43921 #define TARGET_ASM_FILE_START x86_file_start
43923 #undef TARGET_OPTION_OVERRIDE
43924 #define TARGET_OPTION_OVERRIDE ix86_option_override
43926 #undef TARGET_REGISTER_MOVE_COST
43927 #define TARGET_REGISTER_MOVE_COST ix86_register_move_cost
43928 #undef TARGET_MEMORY_MOVE_COST
43929 #define TARGET_MEMORY_MOVE_COST ix86_memory_move_cost
43930 #undef TARGET_RTX_COSTS
43931 #define TARGET_RTX_COSTS ix86_rtx_costs
43932 #undef TARGET_ADDRESS_COST
43933 #define TARGET_ADDRESS_COST ix86_address_cost
43935 #undef TARGET_FIXED_CONDITION_CODE_REGS
43936 #define TARGET_FIXED_CONDITION_CODE_REGS ix86_fixed_condition_code_regs
43937 #undef TARGET_CC_MODES_COMPATIBLE
43938 #define TARGET_CC_MODES_COMPATIBLE ix86_cc_modes_compatible
43940 #undef TARGET_MACHINE_DEPENDENT_REORG
43941 #define TARGET_MACHINE_DEPENDENT_REORG ix86_reorg
43943 #undef TARGET_BUILTIN_SETJMP_FRAME_VALUE
43944 #define TARGET_BUILTIN_SETJMP_FRAME_VALUE ix86_builtin_setjmp_frame_value
43946 #undef TARGET_BUILD_BUILTIN_VA_LIST
43947 #define TARGET_BUILD_BUILTIN_VA_LIST ix86_build_builtin_va_list
43949 #undef TARGET_FOLD_BUILTIN
43950 #define TARGET_FOLD_BUILTIN ix86_fold_builtin
43952 #undef TARGET_COMPARE_VERSION_PRIORITY
43953 #define TARGET_COMPARE_VERSION_PRIORITY ix86_compare_version_priority
43955 #undef TARGET_GENERATE_VERSION_DISPATCHER_BODY
43956 #define TARGET_GENERATE_VERSION_DISPATCHER_BODY \
43957 ix86_generate_version_dispatcher_body
43959 #undef TARGET_GET_FUNCTION_VERSIONS_DISPATCHER
43960 #define TARGET_GET_FUNCTION_VERSIONS_DISPATCHER \
43961 ix86_get_function_versions_dispatcher
43963 #undef TARGET_ENUM_VA_LIST_P
43964 #define TARGET_ENUM_VA_LIST_P ix86_enum_va_list
43966 #undef TARGET_FN_ABI_VA_LIST
43967 #define TARGET_FN_ABI_VA_LIST ix86_fn_abi_va_list
43969 #undef TARGET_CANONICAL_VA_LIST_TYPE
43970 #define TARGET_CANONICAL_VA_LIST_TYPE ix86_canonical_va_list_type
43972 #undef TARGET_EXPAND_BUILTIN_VA_START
43973 #define TARGET_EXPAND_BUILTIN_VA_START ix86_va_start
43975 #undef TARGET_MD_ASM_CLOBBERS
43976 #define TARGET_MD_ASM_CLOBBERS ix86_md_asm_clobbers
43978 #undef TARGET_PROMOTE_PROTOTYPES
43979 #define TARGET_PROMOTE_PROTOTYPES hook_bool_const_tree_true
43980 #undef TARGET_STRUCT_VALUE_RTX
43981 #define TARGET_STRUCT_VALUE_RTX ix86_struct_value_rtx
43982 #undef TARGET_SETUP_INCOMING_VARARGS
43983 #define TARGET_SETUP_INCOMING_VARARGS ix86_setup_incoming_varargs
43984 #undef TARGET_MUST_PASS_IN_STACK
43985 #define TARGET_MUST_PASS_IN_STACK ix86_must_pass_in_stack
43986 #undef TARGET_FUNCTION_ARG_ADVANCE
43987 #define TARGET_FUNCTION_ARG_ADVANCE ix86_function_arg_advance
43988 #undef TARGET_FUNCTION_ARG
43989 #define TARGET_FUNCTION_ARG ix86_function_arg
43990 #undef TARGET_FUNCTION_ARG_BOUNDARY
43991 #define TARGET_FUNCTION_ARG_BOUNDARY ix86_function_arg_boundary
43992 #undef TARGET_PASS_BY_REFERENCE
43993 #define TARGET_PASS_BY_REFERENCE ix86_pass_by_reference
43994 #undef TARGET_INTERNAL_ARG_POINTER
43995 #define TARGET_INTERNAL_ARG_POINTER ix86_internal_arg_pointer
43996 #undef TARGET_UPDATE_STACK_BOUNDARY
43997 #define TARGET_UPDATE_STACK_BOUNDARY ix86_update_stack_boundary
43998 #undef TARGET_GET_DRAP_RTX
43999 #define TARGET_GET_DRAP_RTX ix86_get_drap_rtx
44000 #undef TARGET_STRICT_ARGUMENT_NAMING
44001 #define TARGET_STRICT_ARGUMENT_NAMING hook_bool_CUMULATIVE_ARGS_true
44002 #undef TARGET_STATIC_CHAIN
44003 #define TARGET_STATIC_CHAIN ix86_static_chain
44004 #undef TARGET_TRAMPOLINE_INIT
44005 #define TARGET_TRAMPOLINE_INIT ix86_trampoline_init
44006 #undef TARGET_RETURN_POPS_ARGS
44007 #define TARGET_RETURN_POPS_ARGS ix86_return_pops_args
44009 #undef TARGET_LEGITIMATE_COMBINED_INSN
44010 #define TARGET_LEGITIMATE_COMBINED_INSN ix86_legitimate_combined_insn
44012 #undef TARGET_ASAN_SHADOW_OFFSET
44013 #define TARGET_ASAN_SHADOW_OFFSET ix86_asan_shadow_offset
44015 #undef TARGET_GIMPLIFY_VA_ARG_EXPR
44016 #define TARGET_GIMPLIFY_VA_ARG_EXPR ix86_gimplify_va_arg
44018 #undef TARGET_SCALAR_MODE_SUPPORTED_P
44019 #define TARGET_SCALAR_MODE_SUPPORTED_P ix86_scalar_mode_supported_p
44021 #undef TARGET_VECTOR_MODE_SUPPORTED_P
44022 #define TARGET_VECTOR_MODE_SUPPORTED_P ix86_vector_mode_supported_p
44024 #undef TARGET_C_MODE_FOR_SUFFIX
44025 #define TARGET_C_MODE_FOR_SUFFIX ix86_c_mode_for_suffix
44027 #ifdef HAVE_AS_TLS
44028 #undef TARGET_ASM_OUTPUT_DWARF_DTPREL
44029 #define TARGET_ASM_OUTPUT_DWARF_DTPREL i386_output_dwarf_dtprel
44030 #endif
44032 #ifdef SUBTARGET_INSERT_ATTRIBUTES
44033 #undef TARGET_INSERT_ATTRIBUTES
44034 #define TARGET_INSERT_ATTRIBUTES SUBTARGET_INSERT_ATTRIBUTES
44035 #endif
44037 #undef TARGET_MANGLE_TYPE
44038 #define TARGET_MANGLE_TYPE ix86_mangle_type
44040 #if !TARGET_MACHO
44041 #undef TARGET_STACK_PROTECT_FAIL
44042 #define TARGET_STACK_PROTECT_FAIL ix86_stack_protect_fail
44043 #endif
44045 #undef TARGET_FUNCTION_VALUE
44046 #define TARGET_FUNCTION_VALUE ix86_function_value
44048 #undef TARGET_FUNCTION_VALUE_REGNO_P
44049 #define TARGET_FUNCTION_VALUE_REGNO_P ix86_function_value_regno_p
44051 #undef TARGET_PROMOTE_FUNCTION_MODE
44052 #define TARGET_PROMOTE_FUNCTION_MODE ix86_promote_function_mode
44054 #undef TARGET_MEMBER_TYPE_FORCES_BLK
44055 #define TARGET_MEMBER_TYPE_FORCES_BLK ix86_member_type_forces_blk
44057 #undef TARGET_INSTANTIATE_DECLS
44058 #define TARGET_INSTANTIATE_DECLS ix86_instantiate_decls
44060 #undef TARGET_SECONDARY_RELOAD
44061 #define TARGET_SECONDARY_RELOAD ix86_secondary_reload
44063 #undef TARGET_CLASS_MAX_NREGS
44064 #define TARGET_CLASS_MAX_NREGS ix86_class_max_nregs
44066 #undef TARGET_PREFERRED_RELOAD_CLASS
44067 #define TARGET_PREFERRED_RELOAD_CLASS ix86_preferred_reload_class
44068 #undef TARGET_PREFERRED_OUTPUT_RELOAD_CLASS
44069 #define TARGET_PREFERRED_OUTPUT_RELOAD_CLASS ix86_preferred_output_reload_class
44070 #undef TARGET_CLASS_LIKELY_SPILLED_P
44071 #define TARGET_CLASS_LIKELY_SPILLED_P ix86_class_likely_spilled_p
44073 #undef TARGET_VECTORIZE_BUILTIN_VECTORIZATION_COST
44074 #define TARGET_VECTORIZE_BUILTIN_VECTORIZATION_COST \
44075 ix86_builtin_vectorization_cost
44076 #undef TARGET_VECTORIZE_VEC_PERM_CONST_OK
44077 #define TARGET_VECTORIZE_VEC_PERM_CONST_OK \
44078 ix86_vectorize_vec_perm_const_ok
44079 #undef TARGET_VECTORIZE_PREFERRED_SIMD_MODE
44080 #define TARGET_VECTORIZE_PREFERRED_SIMD_MODE \
44081 ix86_preferred_simd_mode
44082 #undef TARGET_VECTORIZE_AUTOVECTORIZE_VECTOR_SIZES
44083 #define TARGET_VECTORIZE_AUTOVECTORIZE_VECTOR_SIZES \
44084 ix86_autovectorize_vector_sizes
44085 #undef TARGET_VECTORIZE_INIT_COST
44086 #define TARGET_VECTORIZE_INIT_COST ix86_init_cost
44087 #undef TARGET_VECTORIZE_ADD_STMT_COST
44088 #define TARGET_VECTORIZE_ADD_STMT_COST ix86_add_stmt_cost
44089 #undef TARGET_VECTORIZE_FINISH_COST
44090 #define TARGET_VECTORIZE_FINISH_COST ix86_finish_cost
44091 #undef TARGET_VECTORIZE_DESTROY_COST_DATA
44092 #define TARGET_VECTORIZE_DESTROY_COST_DATA ix86_destroy_cost_data
44094 #undef TARGET_SET_CURRENT_FUNCTION
44095 #define TARGET_SET_CURRENT_FUNCTION ix86_set_current_function
44097 #undef TARGET_OPTION_VALID_ATTRIBUTE_P
44098 #define TARGET_OPTION_VALID_ATTRIBUTE_P ix86_valid_target_attribute_p
44100 #undef TARGET_OPTION_SAVE
44101 #define TARGET_OPTION_SAVE ix86_function_specific_save
44103 #undef TARGET_OPTION_RESTORE
44104 #define TARGET_OPTION_RESTORE ix86_function_specific_restore
44106 #undef TARGET_OPTION_PRINT
44107 #define TARGET_OPTION_PRINT ix86_function_specific_print
44109 #undef TARGET_OPTION_FUNCTION_VERSIONS
44110 #define TARGET_OPTION_FUNCTION_VERSIONS ix86_function_versions
44112 #undef TARGET_CAN_INLINE_P
44113 #define TARGET_CAN_INLINE_P ix86_can_inline_p
44115 #undef TARGET_EXPAND_TO_RTL_HOOK
44116 #define TARGET_EXPAND_TO_RTL_HOOK ix86_maybe_switch_abi
44118 #undef TARGET_LEGITIMATE_ADDRESS_P
44119 #define TARGET_LEGITIMATE_ADDRESS_P ix86_legitimate_address_p
44121 #undef TARGET_LRA_P
44122 #define TARGET_LRA_P hook_bool_void_true
44124 #undef TARGET_REGISTER_PRIORITY
44125 #define TARGET_REGISTER_PRIORITY ix86_register_priority
44127 #undef TARGET_REGISTER_USAGE_LEVELING_P
44128 #define TARGET_REGISTER_USAGE_LEVELING_P hook_bool_void_true
44130 #undef TARGET_LEGITIMATE_CONSTANT_P
44131 #define TARGET_LEGITIMATE_CONSTANT_P ix86_legitimate_constant_p
44133 #undef TARGET_FRAME_POINTER_REQUIRED
44134 #define TARGET_FRAME_POINTER_REQUIRED ix86_frame_pointer_required
44136 #undef TARGET_CAN_ELIMINATE
44137 #define TARGET_CAN_ELIMINATE ix86_can_eliminate
44139 #undef TARGET_EXTRA_LIVE_ON_ENTRY
44140 #define TARGET_EXTRA_LIVE_ON_ENTRY ix86_live_on_entry
44142 #undef TARGET_ASM_CODE_END
44143 #define TARGET_ASM_CODE_END ix86_code_end
44145 #undef TARGET_CONDITIONAL_REGISTER_USAGE
44146 #define TARGET_CONDITIONAL_REGISTER_USAGE ix86_conditional_register_usage
44148 #if TARGET_MACHO
44149 #undef TARGET_INIT_LIBFUNCS
44150 #define TARGET_INIT_LIBFUNCS darwin_rename_builtins
44151 #endif
44153 #undef TARGET_SPILL_CLASS
44154 #define TARGET_SPILL_CLASS ix86_spill_class
44156 #undef TARGET_FLOAT_EXCEPTIONS_ROUNDING_SUPPORTED_P
44157 #define TARGET_FLOAT_EXCEPTIONS_ROUNDING_SUPPORTED_P \
44158 ix86_float_exceptions_rounding_supported_p
44161 \f